diff --git a/dlomix/reports/RetentionTimeReportModelComparisonWandb.py b/dlomix/reports/RetentionTimeReportModelComparisonWandb.py
index 0aa736342da85c45f61cae37fd4dd12102919b80..d6ee86e61bbc095e31476909c452b1636cb5ffbd 100644
--- a/dlomix/reports/RetentionTimeReportModelComparisonWandb.py
+++ b/dlomix/reports/RetentionTimeReportModelComparisonWandb.py
@@ -46,7 +46,7 @@ class RetentionTimeReportModelComparisonWandb:
add_r2_section=True,
add_density_section=True,
):
- """Creates the report in wandb.
+ """Creates the report in wandb_run.
Args:
add_data_section (bool, optional): Add a section for input data to the report. Defaults to True.
@@ -215,18 +215,18 @@ class RetentionTimeReportModelComparisonWandb:
"residuals": self.calculate_residuals(targets, predictions),
}
)
- # log df as table to wandb
+ # log df as table to wandb_run
table = wandb.Table(dataframe=results_df)
wandb.log({f"results_table_{current_model}": table})
- # log r2 to wandb
+ # log r2 to wandb_run
r2 = self.calculate_r2(targets, predictions)
wandb.log({"r2": r2})
# finish run
wandb.finish()
- # function to log sequence length table to wandb
+ # function to log sequence length table to wandb_run
def log_sequence_length_table(
self, data: pd.DataFrame, seq_col: str = "modified_sequence"
):
@@ -235,7 +235,7 @@ class RetentionTimeReportModelComparisonWandb:
# convert to df for easier handling
counts_df = counts.to_frame()
table = wandb.Table(dataframe=counts_df)
- # log to wandb
+ # log to wandb_run
hist = wandb.plot_table(
vega_spec_name=f"{RetentionTimeReportModelComparisonWandb.VEGA_LITE_PRESETS_ID}/histogram_peptide_length",
data_table=table,
@@ -251,14 +251,14 @@ class RetentionTimeReportModelComparisonWandb:
data[seq_col].replace(pattern, "", inplace=True)
return data[seq_col].str.len()
- # function to log retention time table to wandb
+ # function to log retention time table to wandb_run
def log_rt_table(self, data: pd.DataFrame, rt_col: str = "indexed_retention_time"):
name_hist = "rt_hist"
rt = data.loc[:, rt_col]
# convert to df for easier handling
rt_df = rt.to_frame()
table = wandb.Table(dataframe=rt_df)
- # log to wandb
+ # log to wandb_run
hist = wandb.plot_table(
vega_spec_name=f"{RetentionTimeReportModelComparisonWandb.VEGA_LITE_PRESETS_ID}/histogram_irt",
data_table=table,
diff --git a/dlomix/reports/RetentionTimeReportRunComparisonWandb.py b/dlomix/reports/RetentionTimeReportRunComparisonWandb.py
index 3921f553cde3cc6fdcce76d12a76d5f90bf688f7..00e8a330d11cdaa4d63e03ffb0eaafddaa9deb77 100644
--- a/dlomix/reports/RetentionTimeReportRunComparisonWandb.py
+++ b/dlomix/reports/RetentionTimeReportRunComparisonWandb.py
@@ -34,9 +34,9 @@ class RetentionTimeReportRunComparisonWandb:
"""Create WandB report for comparing runs.
Args:
- project (str): Name of the project to be used in wandb.
- title (str): Title of the report in wandb.
- description (str): Description of the report in wandb.
+ project (str): Name of the project to be used in wandb_run.
+ title (str): Title of the report in wandb_run.
+ description (str): Description of the report in wandb_run.
dataset (RetentionTimeDataset, optional): The retention time dataset if logging the data is desired. Defaults to None, no logging of input data.
"""
self.project = project
@@ -58,7 +58,7 @@ class RetentionTimeReportRunComparisonWandb:
add_train_val_section=True,
add_model_section=True,
):
- """Create a report in wandb.
+ """Create a report in wandb_run.
Args:
add_config_section (bool, optional): Add a section for config parameters and the run to the report. Defaults to True.
@@ -292,21 +292,21 @@ class RetentionTimeReportRunComparisonWandb:
def log_sequence_length_table(
self, data: pd.DataFrame, seq_col: str = "modified_sequence"
):
- """Log sequence length table to wandb
+ """Log sequence length table to wandb_run
Args:
data (pd.DataFrame): input data
seq_col (str, optional): Name of the column containing the sequences in the data frame. Defaults to "modified_sequence".
Returns:
- str: Name of the histogram created by wandb after logging the data.
+ str: Name of the histogram created by wandb_run after logging the data.
"""
name_hist = "counts_hist"
counts = self.count_seq_length(data, seq_col)
# convert to df for easier handling
counts_df = counts.to_frame()
table = wandb.Table(dataframe=counts_df)
- # log to wandb
+ # log to wandb_run
hist = wandb.plot_table(
vega_spec_name=f"{RetentionTimeReportRunComparisonWandb.VEGA_LITE_PRESETS_ID}/histogram_peptide_length",
data_table=table,
@@ -322,14 +322,14 @@ class RetentionTimeReportRunComparisonWandb:
data[seq_col].replace(pattern, "", inplace=True)
return data[seq_col].str.len()
- # function to log retention time table to wandb
+ # function to log retention time table to wandb_run
def log_rt_table(self, data: pd.DataFrame, rt_col: str = "indexed_retention_time"):
name_hist = "rt_hist"
rt = data.loc[:, rt_col]
# convert to df for easier handling
rt_df = rt.to_frame()
table = wandb.Table(dataframe=rt_df)
- # log to wandb
+ # log to wandb_run
hist = wandb.plot_table(
vega_spec_name=f"{RetentionTimeReportRunComparisonWandb.VEGA_LITE_PRESETS_ID}/histogram_irt",
data_table=table,
@@ -422,7 +422,7 @@ class RetentionTimeReportRunComparisonWandb:
column_names = layer_info[0]
layer_info_df = pd.DataFrame(layer_info[1:], columns=column_names)
- # log layer_table to wandb
+ # log layer_table to wandb_run
layer_table = wandb.Table(dataframe=layer_info_df)
wandb.log({"layer_table": layer_table})
diff --git a/seaborn/__init__.py b/seaborn/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..d1ca9754d31183f7bbad2dad059edf8e3a46aa4a
--- /dev/null
+++ b/seaborn/__init__.py
@@ -0,0 +1,21 @@
+# Import seaborn objects
+from .rcmod import * # noqa: F401,F403
+from .utils import * # noqa: F401,F403
+from .palettes import * # noqa: F401,F403
+from .relational import * # noqa: F401,F403
+from .regression import * # noqa: F401,F403
+from .categorical import * # noqa: F401,F403
+from .distributions import * # noqa: F401,F403
+from .matrix import * # noqa: F401,F403
+from .miscplot import * # noqa: F401,F403
+from .axisgrid import * # noqa: F401,F403
+from .widgets import * # noqa: F401,F403
+from .colors import xkcd_rgb, crayons # noqa: F401
+from . import cm # noqa: F401
+
+# Capture the original matplotlib rcParams
+import matplotlib as mpl
+_orig_rc_params = mpl.rcParams.copy()
+
+# Define the seaborn version
+__version__ = "0.14.0.dev0"
diff --git a/seaborn/_base.py b/seaborn/_base.py
new file mode 100644
index 0000000000000000000000000000000000000000..0b43523193782ec89ca1f2f573da01683fdaae6f
--- /dev/null
+++ b/seaborn/_base.py
@@ -0,0 +1,1777 @@
+from __future__ import annotations
+import warnings
+import itertools
+from copy import copy
+from collections import UserString
+from collections.abc import Iterable, Sequence, Mapping
+from numbers import Number
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+import matplotlib as mpl
+
+from seaborn._core.data import PlotData
+from seaborn.palettes import (
+ QUAL_PALETTES,
+ color_palette,
+)
+from seaborn.utils import (
+ _check_argument,
+ _version_predates,
+ desaturate,
+ locator_to_legend_entries,
+ get_color_cycle,
+ remove_na,
+)
+
+
+class SemanticMapping:
+ """Base class for mapping data values to plot attributes."""
+
+ # -- Default attributes that all SemanticMapping subclasses must set
+
+ # Whether the mapping is numeric, categorical, or datetime
+ map_type: str | None = None
+
+ # Ordered list of unique values in the input data
+ levels = None
+
+ # A mapping from the data values to corresponding plot attributes
+ lookup_table = None
+
+ def __init__(self, plotter):
+
+ # TODO Putting this here so we can continue to use a lot of the
+ # logic that's built into the library, but the idea of this class
+ # is to move towards semantic mappings that are agnostic about the
+ # kind of plot they're going to be used to draw.
+ # Fully achieving that is going to take some thinking.
+ self.plotter = plotter
+
+ def _check_list_length(self, levels, values, variable):
+ """Input check when values are provided as a list."""
+ # Copied from _core/properties; eventually will be replaced for that.
+ message = ""
+ if len(levels) > len(values):
+ message = " ".join([
+ f"\nThe {variable} list has fewer values ({len(values)})",
+ f"than needed ({len(levels)}) and will cycle, which may",
+ "produce an uninterpretable plot."
+ ])
+ values = [x for _, x in zip(levels, itertools.cycle(values))]
+
+ elif len(values) > len(levels):
+ message = " ".join([
+ f"The {variable} list has more values ({len(values)})",
+ f"than needed ({len(levels)}), which may not be intended.",
+ ])
+ values = values[:len(levels)]
+
+ if message:
+ warnings.warn(message, UserWarning, stacklevel=6)
+
+ return values
+
+ def _lookup_single(self, key):
+ """Apply the mapping to a single data value."""
+ return self.lookup_table[key]
+
+ def __call__(self, key, *args, **kwargs):
+ """Get the attribute(s) values for the data key."""
+ if isinstance(key, (list, np.ndarray, pd.Series)):
+ return [self._lookup_single(k, *args, **kwargs) for k in key]
+ else:
+ return self._lookup_single(key, *args, **kwargs)
+
+
+class HueMapping(SemanticMapping):
+ """Mapping that sets artist colors according to data values."""
+ # A specification of the colors that should appear in the plot
+ palette = None
+
+ # An object that normalizes data values to [0, 1] range for color mapping
+ norm = None
+
+ # A continuous colormap object for interpolating in a numeric context
+ cmap = None
+
+ def __init__(
+ self, plotter, palette=None, order=None, norm=None, saturation=1,
+ ):
+ """Map the levels of the `hue` variable to distinct colors.
+
+ Parameters
+ ----------
+ # TODO add generic parameters
+
+ """
+ super().__init__(plotter)
+
+ data = plotter.plot_data.get("hue", pd.Series(dtype=float))
+
+ if isinstance(palette, np.ndarray):
+ msg = (
+ "Numpy array is not a supported type for `palette`. "
+ "Please convert your palette to a list. "
+ "This will become an error in v0.14"
+ )
+ warnings.warn(msg, stacklevel=4)
+ palette = palette.tolist()
+
+ if data.isna().all():
+ if palette is not None:
+ msg = "Ignoring `palette` because no `hue` variable has been assigned."
+ warnings.warn(msg, stacklevel=4)
+ else:
+
+ map_type = self.infer_map_type(
+ palette, norm, plotter.input_format, plotter.var_types["hue"]
+ )
+
+ # Our goal is to end up with a dictionary mapping every unique
+ # value in `data` to a color. We will also keep track of the
+ # metadata about this mapping we will need for, e.g., a legend
+
+ # --- Option 1: numeric mapping with a matplotlib colormap
+
+ if map_type == "numeric":
+
+ data = pd.to_numeric(data)
+ levels, lookup_table, norm, cmap = self.numeric_mapping(
+ data, palette, norm,
+ )
+
+ # --- Option 2: categorical mapping using seaborn palette
+
+ elif map_type == "categorical":
+
+ cmap = norm = None
+ levels, lookup_table = self.categorical_mapping(
+ data, palette, order,
+ )
+
+ # --- Option 3: datetime mapping
+
+ else:
+ # TODO this needs actual implementation
+ cmap = norm = None
+ levels, lookup_table = self.categorical_mapping(
+ # Casting data to list to handle differences in the way
+ # pandas and numpy represent datetime64 data
+ list(data), palette, order,
+ )
+
+ self.saturation = saturation
+ self.map_type = map_type
+ self.lookup_table = lookup_table
+ self.palette = palette
+ self.levels = levels
+ self.norm = norm
+ self.cmap = cmap
+
+ def _lookup_single(self, key):
+ """Get the color for a single value, using colormap to interpolate."""
+ try:
+ # Use a value that's in the original data vector
+ value = self.lookup_table[key]
+ except KeyError:
+
+ if self.norm is None:
+ # Currently we only get here in scatterplot with hue_order,
+ # because scatterplot does not consider hue a grouping variable
+ # So unused hue levels are in the data, but not the lookup table
+ return (0, 0, 0, 0)
+
+ # Use the colormap to interpolate between existing datapoints
+ # (e.g. in the context of making a continuous legend)
+ try:
+ normed = self.norm(key)
+ except TypeError as err:
+ if np.isnan(key):
+ value = (0, 0, 0, 0)
+ else:
+ raise err
+ else:
+ if np.ma.is_masked(normed):
+ normed = np.nan
+ value = self.cmap(normed)
+
+ if self.saturation < 1:
+ value = desaturate(value, self.saturation)
+
+ return value
+
+ def infer_map_type(self, palette, norm, input_format, var_type):
+ """Determine how to implement the mapping."""
+ if palette in QUAL_PALETTES:
+ map_type = "categorical"
+ elif norm is not None:
+ map_type = "numeric"
+ elif isinstance(palette, (dict, list)):
+ map_type = "categorical"
+ elif input_format == "wide":
+ map_type = "categorical"
+ else:
+ map_type = var_type
+
+ return map_type
+
+ def categorical_mapping(self, data, palette, order):
+ """Determine colors when the hue mapping is categorical."""
+ # -- Identify the order and name of the levels
+
+ levels = categorical_order(data, order)
+ n_colors = len(levels)
+
+ # -- Identify the set of colors to use
+
+ if isinstance(palette, dict):
+
+ missing = set(levels) - set(palette)
+ if any(missing):
+ err = "The palette dictionary is missing keys: {}"
+ raise ValueError(err.format(missing))
+
+ lookup_table = palette
+
+ else:
+
+ if palette is None:
+ if n_colors <= len(get_color_cycle()):
+ colors = color_palette(None, n_colors)
+ else:
+ colors = color_palette("husl", n_colors)
+ elif isinstance(palette, list):
+ colors = self._check_list_length(levels, palette, "palette")
+ else:
+ colors = color_palette(palette, n_colors)
+
+ lookup_table = dict(zip(levels, colors))
+
+ return levels, lookup_table
+
+ def numeric_mapping(self, data, palette, norm):
+ """Determine colors when the hue variable is quantitative."""
+ if isinstance(palette, dict):
+
+ # The presence of a norm object overrides a dictionary of hues
+ # in specifying a numeric mapping, so we need to process it here.
+ levels = list(sorted(palette))
+ colors = [palette[k] for k in sorted(palette)]
+ cmap = mpl.colors.ListedColormap(colors)
+ lookup_table = palette.copy()
+
+ else:
+
+ # The levels are the sorted unique values in the data
+ levels = list(np.sort(remove_na(data.unique())))
+
+ # --- Sort out the colormap to use from the palette argument
+
+ # Default numeric palette is our default cubehelix palette
+ # TODO do we want to do something complicated to ensure contrast?
+ palette = "ch:" if palette is None else palette
+
+ if isinstance(palette, mpl.colors.Colormap):
+ cmap = palette
+ else:
+ cmap = color_palette(palette, as_cmap=True)
+
+ # Now sort out the data normalization
+ if norm is None:
+ norm = mpl.colors.Normalize()
+ elif isinstance(norm, tuple):
+ norm = mpl.colors.Normalize(*norm)
+ elif not isinstance(norm, mpl.colors.Normalize):
+ err = "``hue_norm`` must be None, tuple, or Normalize object."
+ raise ValueError(err)
+
+ if not norm.scaled():
+ norm(np.asarray(data.dropna()))
+
+ lookup_table = dict(zip(levels, cmap(norm(levels))))
+
+ return levels, lookup_table, norm, cmap
+
+
+class SizeMapping(SemanticMapping):
+ """Mapping that sets artist sizes according to data values."""
+ # An object that normalizes data values to [0, 1] range
+ norm = None
+
+ def __init__(
+ self, plotter, sizes=None, order=None, norm=None,
+ ):
+ """Map the levels of the `size` variable to distinct values.
+
+ Parameters
+ ----------
+ # TODO add generic parameters
+
+ """
+ super().__init__(plotter)
+
+ data = plotter.plot_data.get("size", pd.Series(dtype=float))
+
+ if data.notna().any():
+
+ map_type = self.infer_map_type(
+ norm, sizes, plotter.var_types["size"]
+ )
+
+ # --- Option 1: numeric mapping
+
+ if map_type == "numeric":
+
+ levels, lookup_table, norm, size_range = self.numeric_mapping(
+ data, sizes, norm,
+ )
+
+ # --- Option 2: categorical mapping
+
+ elif map_type == "categorical":
+
+ levels, lookup_table = self.categorical_mapping(
+ data, sizes, order,
+ )
+ size_range = None
+
+ # --- Option 3: datetime mapping
+
+ # TODO this needs an actual implementation
+ else:
+
+ levels, lookup_table = self.categorical_mapping(
+ # Casting data to list to handle differences in the way
+ # pandas and numpy represent datetime64 data
+ list(data), sizes, order,
+ )
+ size_range = None
+
+ self.map_type = map_type
+ self.levels = levels
+ self.norm = norm
+ self.sizes = sizes
+ self.size_range = size_range
+ self.lookup_table = lookup_table
+
+ def infer_map_type(self, norm, sizes, var_type):
+
+ if norm is not None:
+ map_type = "numeric"
+ elif isinstance(sizes, (dict, list)):
+ map_type = "categorical"
+ else:
+ map_type = var_type
+
+ return map_type
+
+ def _lookup_single(self, key):
+
+ try:
+ value = self.lookup_table[key]
+ except KeyError:
+ normed = self.norm(key)
+ if np.ma.is_masked(normed):
+ normed = np.nan
+ value = self.size_range[0] + normed * np.ptp(self.size_range)
+ return value
+
+ def categorical_mapping(self, data, sizes, order):
+
+ levels = categorical_order(data, order)
+
+ if isinstance(sizes, dict):
+
+ # Dict inputs map existing data values to the size attribute
+ missing = set(levels) - set(sizes)
+ if any(missing):
+ err = f"Missing sizes for the following levels: {missing}"
+ raise ValueError(err)
+ lookup_table = sizes.copy()
+
+ elif isinstance(sizes, list):
+
+ # List inputs give size values in the same order as the levels
+ sizes = self._check_list_length(levels, sizes, "sizes")
+ lookup_table = dict(zip(levels, sizes))
+
+ else:
+
+ if isinstance(sizes, tuple):
+
+ # Tuple input sets the min, max size values
+ if len(sizes) != 2:
+ err = "A `sizes` tuple must have only 2 values"
+ raise ValueError(err)
+
+ elif sizes is not None:
+
+ err = f"Value for `sizes` not understood: {sizes}"
+ raise ValueError(err)
+
+ else:
+
+ # Otherwise, we need to get the min, max size values from
+ # the plotter object we are attached to.
+
+ # TODO this is going to cause us trouble later, because we
+ # want to restructure things so that the plotter is generic
+ # across the visual representation of the data. But at this
+ # point, we don't know the visual representation. Likely we
+ # want to change the logic of this Mapping so that it gives
+ # points on a normalized range that then gets un-normalized
+ # when we know what we're drawing. But given the way the
+ # package works now, this way is cleanest.
+ sizes = self.plotter._default_size_range
+
+ # For categorical sizes, use regularly-spaced linear steps
+ # between the minimum and maximum sizes. Then reverse the
+ # ramp so that the largest value is used for the first entry
+ # in size_order, etc. This is because "ordered" categories
+ # are often though to go in decreasing priority.
+ sizes = np.linspace(*sizes, len(levels))[::-1]
+ lookup_table = dict(zip(levels, sizes))
+
+ return levels, lookup_table
+
+ def numeric_mapping(self, data, sizes, norm):
+
+ if isinstance(sizes, dict):
+ # The presence of a norm object overrides a dictionary of sizes
+ # in specifying a numeric mapping, so we need to process it
+ # dictionary here
+ levels = list(np.sort(list(sizes)))
+ size_values = sizes.values()
+ size_range = min(size_values), max(size_values)
+
+ else:
+
+ # The levels here will be the unique values in the data
+ levels = list(np.sort(remove_na(data.unique())))
+
+ if isinstance(sizes, tuple):
+
+ # For numeric inputs, the size can be parametrized by
+ # the minimum and maximum artist values to map to. The
+ # norm object that gets set up next specifies how to
+ # do the mapping.
+
+ if len(sizes) != 2:
+ err = "A `sizes` tuple must have only 2 values"
+ raise ValueError(err)
+
+ size_range = sizes
+
+ elif sizes is not None:
+
+ err = f"Value for `sizes` not understood: {sizes}"
+ raise ValueError(err)
+
+ else:
+
+ # When not provided, we get the size range from the plotter
+ # object we are attached to. See the note in the categorical
+ # method about how this is suboptimal for future development.
+ size_range = self.plotter._default_size_range
+
+ # Now that we know the minimum and maximum sizes that will get drawn,
+ # we need to map the data values that we have into that range. We will
+ # use a matplotlib Normalize class, which is typically used for numeric
+ # color mapping but works fine here too. It takes data values and maps
+ # them into a [0, 1] interval, potentially nonlinear-ly.
+
+ if norm is None:
+ # Default is a linear function between the min and max data values
+ norm = mpl.colors.Normalize()
+ elif isinstance(norm, tuple):
+ # It is also possible to give different limits in data space
+ norm = mpl.colors.Normalize(*norm)
+ elif not isinstance(norm, mpl.colors.Normalize):
+ err = f"Value for size `norm` parameter not understood: {norm}"
+ raise ValueError(err)
+ else:
+ # If provided with Normalize object, copy it so we can modify
+ norm = copy(norm)
+
+ # Set the mapping so all output values are in [0, 1]
+ norm.clip = True
+
+ # If the input range is not set, use the full range of the data
+ if not norm.scaled():
+ norm(levels)
+
+ # Map from data values to [0, 1] range
+ sizes_scaled = norm(levels)
+
+ # Now map from the scaled range into the artist units
+ if isinstance(sizes, dict):
+ lookup_table = sizes
+ else:
+ lo, hi = size_range
+ sizes = lo + sizes_scaled * (hi - lo)
+ lookup_table = dict(zip(levels, sizes))
+
+ return levels, lookup_table, norm, size_range
+
+
+class StyleMapping(SemanticMapping):
+ """Mapping that sets artist style according to data values."""
+
+ # Style mapping is always treated as categorical
+ map_type = "categorical"
+
+ def __init__(self, plotter, markers=None, dashes=None, order=None):
+ """Map the levels of the `style` variable to distinct values.
+
+ Parameters
+ ----------
+ # TODO add generic parameters
+
+ """
+ super().__init__(plotter)
+
+ data = plotter.plot_data.get("style", pd.Series(dtype=float))
+
+ if data.notna().any():
+
+ # Cast to list to handle numpy/pandas datetime quirks
+ if variable_type(data) == "datetime":
+ data = list(data)
+
+ # Find ordered unique values
+ levels = categorical_order(data, order)
+
+ markers = self._map_attributes(
+ markers, levels, unique_markers(len(levels)), "markers",
+ )
+ dashes = self._map_attributes(
+ dashes, levels, unique_dashes(len(levels)), "dashes",
+ )
+
+ # Build the paths matplotlib will use to draw the markers
+ paths = {}
+ filled_markers = []
+ for k, m in markers.items():
+ if not isinstance(m, mpl.markers.MarkerStyle):
+ m = mpl.markers.MarkerStyle(m)
+ paths[k] = m.get_path().transformed(m.get_transform())
+ filled_markers.append(m.is_filled())
+
+ # Mixture of filled and unfilled markers will show line art markers
+ # in the edge color, which defaults to white. This can be handled,
+ # but there would be additional complexity with specifying the
+ # weight of the line art markers without overwhelming the filled
+ # ones with the edges. So for now, we will disallow mixtures.
+ if any(filled_markers) and not all(filled_markers):
+ err = "Filled and line art markers cannot be mixed"
+ raise ValueError(err)
+
+ lookup_table = {}
+ for key in levels:
+ lookup_table[key] = {}
+ if markers:
+ lookup_table[key]["marker"] = markers[key]
+ lookup_table[key]["path"] = paths[key]
+ if dashes:
+ lookup_table[key]["dashes"] = dashes[key]
+
+ self.levels = levels
+ self.lookup_table = lookup_table
+
+ def _lookup_single(self, key, attr=None):
+ """Get attribute(s) for a given data point."""
+ if attr is None:
+ value = self.lookup_table[key]
+ else:
+ value = self.lookup_table[key][attr]
+ return value
+
+ def _map_attributes(self, arg, levels, defaults, attr):
+ """Handle the specification for a given style attribute."""
+ if arg is True:
+ lookup_table = dict(zip(levels, defaults))
+ elif isinstance(arg, dict):
+ missing = set(levels) - set(arg)
+ if missing:
+ err = f"These `{attr}` levels are missing values: {missing}"
+ raise ValueError(err)
+ lookup_table = arg
+ elif isinstance(arg, Sequence):
+ arg = self._check_list_length(levels, arg, attr)
+ lookup_table = dict(zip(levels, arg))
+ elif arg:
+ err = f"This `{attr}` argument was not understood: {arg}"
+ raise ValueError(err)
+ else:
+ lookup_table = {}
+
+ return lookup_table
+
+
+# =========================================================================== #
+
+
+class VectorPlotter:
+ """Base class for objects underlying *plot functions."""
+
+ wide_structure = {
+ "x": "@index", "y": "@values", "hue": "@columns", "style": "@columns",
+ }
+ flat_structure = {"x": "@index", "y": "@values"}
+
+ _default_size_range = 1, 2 # Unused but needed in tests, ugh
+
+ def __init__(self, data=None, variables={}):
+
+ self._var_levels = {}
+ # var_ordered is relevant only for categorical axis variables, and may
+ # be better handled by an internal axis information object that tracks
+ # such information and is set up by the scale_* methods. The analogous
+ # information for numeric axes would be information about log scales.
+ self._var_ordered = {"x": False, "y": False} # alt., used DefaultDict
+ self.assign_variables(data, variables)
+
+ # TODO Lots of tests assume that these are called to initialize the
+ # mappings to default values on class initialization. I'd prefer to
+ # move away from that and only have a mapping when explicitly called.
+ for var in ["hue", "size", "style"]:
+ if var in variables:
+ getattr(self, f"map_{var}")()
+
+ @property
+ def has_xy_data(self):
+ """Return True at least one of x or y is defined."""
+ return bool({"x", "y"} & set(self.variables))
+
+ @property
+ def var_levels(self):
+ """Property interface to ordered list of variables levels.
+
+ Each time it's accessed, it updates the var_levels dictionary with the
+ list of levels in the current semantic mappers. But it also allows the
+ dictionary to persist, so it can be used to set levels by a key. This is
+ used to track the list of col/row levels using an attached FacetGrid
+ object, but it's kind of messy and ideally fixed by improving the
+ faceting logic so it interfaces better with the modern approach to
+ tracking plot variables.
+
+ """
+ for var in self.variables:
+ if (map_obj := getattr(self, f"_{var}_map", None)) is not None:
+ self._var_levels[var] = map_obj.levels
+ return self._var_levels
+
+ def assign_variables(self, data=None, variables={}):
+ """Define plot variables, optionally using lookup from `data`."""
+ x = variables.get("x", None)
+ y = variables.get("y", None)
+
+ if x is None and y is None:
+ self.input_format = "wide"
+ frame, names = self._assign_variables_wideform(data, **variables)
+ else:
+ # When dealing with long-form input, use the newer PlotData
+ # object (internal but introduced for the objects interface)
+ # to centralize / standardize data consumption logic.
+ self.input_format = "long"
+ plot_data = PlotData(data, variables)
+ frame = plot_data.frame
+ names = plot_data.names
+
+ self.plot_data = frame
+ self.variables = names
+ self.var_types = {
+ v: variable_type(
+ frame[v],
+ boolean_type="numeric" if v in "xy" else "categorical"
+ )
+ for v in names
+ }
+
+ return self
+
+ def _assign_variables_wideform(self, data=None, **kwargs):
+ """Define plot variables given wide-form data.
+
+ Parameters
+ ----------
+ data : flat vector or collection of vectors
+ Data can be a vector or mapping that is coerceable to a Series
+ or a sequence- or mapping-based collection of such vectors, or a
+ rectangular numpy array, or a Pandas DataFrame.
+ kwargs : variable -> data mappings
+ Behavior with keyword arguments is currently undefined.
+
+ Returns
+ -------
+ plot_data : :class:`pandas.DataFrame`
+ Long-form data object mapping seaborn variables (x, y, hue, ...)
+ to data vectors.
+ variables : dict
+ Keys are defined seaborn variables; values are names inferred from
+ the inputs (or None when no name can be determined).
+
+ """
+ # Raise if semantic or other variables are assigned in wide-form mode
+ assigned = [k for k, v in kwargs.items() if v is not None]
+ if any(assigned):
+ s = "s" if len(assigned) > 1 else ""
+ err = f"The following variable{s} cannot be assigned with wide-form data: "
+ err += ", ".join(f"`{v}`" for v in assigned)
+ raise ValueError(err)
+
+ # Determine if the data object actually has any data in it
+ empty = data is None or not len(data)
+
+ # Then, determine if we have "flat" data (a single vector)
+ if isinstance(data, dict):
+ values = data.values()
+ else:
+ values = np.atleast_1d(np.asarray(data, dtype=object))
+ flat = not any(
+ isinstance(v, Iterable) and not isinstance(v, (str, bytes))
+ for v in values
+ )
+
+ if empty:
+
+ # Make an object with the structure of plot_data, but empty
+ plot_data = pd.DataFrame()
+ variables = {}
+
+ elif flat:
+
+ # Handle flat data by converting to pandas Series and using the
+ # index and/or values to define x and/or y
+ # (Could be accomplished with a more general to_series() interface)
+ flat_data = pd.Series(data).copy()
+ names = {
+ "@values": flat_data.name,
+ "@index": flat_data.index.name
+ }
+
+ plot_data = {}
+ variables = {}
+
+ for var in ["x", "y"]:
+ if var in self.flat_structure:
+ attr = self.flat_structure[var]
+ plot_data[var] = getattr(flat_data, attr[1:])
+ variables[var] = names[self.flat_structure[var]]
+
+ plot_data = pd.DataFrame(plot_data)
+
+ else:
+
+ # Otherwise assume we have some collection of vectors.
+
+ # Handle Python sequences such that entries end up in the columns,
+ # not in the rows, of the intermediate wide DataFrame.
+ # One way to accomplish this is to convert to a dict of Series.
+ if isinstance(data, Sequence):
+ data_dict = {}
+ for i, var in enumerate(data):
+ key = getattr(var, "name", i)
+ # TODO is there a safer/more generic way to ensure Series?
+ # sort of like np.asarray, but for pandas?
+ data_dict[key] = pd.Series(var)
+
+ data = data_dict
+
+ # Pandas requires that dict values either be Series objects
+ # or all have the same length, but we want to allow "ragged" inputs
+ if isinstance(data, Mapping):
+ data = {key: pd.Series(val) for key, val in data.items()}
+
+ # Otherwise, delegate to the pandas DataFrame constructor
+ # This is where we'd prefer to use a general interface that says
+ # "give me this data as a pandas DataFrame", so we can accept
+ # DataFrame objects from other libraries
+ wide_data = pd.DataFrame(data, copy=True)
+
+ # At this point we should reduce the dataframe to numeric cols
+ numeric_cols = [
+ k for k, v in wide_data.items() if variable_type(v) == "numeric"
+ ]
+ wide_data = wide_data[numeric_cols]
+
+ # Now melt the data to long form
+ melt_kws = {"var_name": "@columns", "value_name": "@values"}
+ use_index = "@index" in self.wide_structure.values()
+ if use_index:
+ melt_kws["id_vars"] = "@index"
+ try:
+ orig_categories = wide_data.columns.categories
+ orig_ordered = wide_data.columns.ordered
+ wide_data.columns = wide_data.columns.add_categories("@index")
+ except AttributeError:
+ category_columns = False
+ else:
+ category_columns = True
+ wide_data["@index"] = wide_data.index.to_series()
+
+ plot_data = wide_data.melt(**melt_kws)
+
+ if use_index and category_columns:
+ plot_data["@columns"] = pd.Categorical(plot_data["@columns"],
+ orig_categories,
+ orig_ordered)
+
+ # Assign names corresponding to plot semantics
+ for var, attr in self.wide_structure.items():
+ plot_data[var] = plot_data[attr]
+
+ # Define the variable names
+ variables = {}
+ for var, attr in self.wide_structure.items():
+ obj = getattr(wide_data, attr[1:])
+ variables[var] = getattr(obj, "name", None)
+
+ # Remove redundant columns from plot_data
+ plot_data = plot_data[list(variables)]
+
+ return plot_data, variables
+
+ def map_hue(self, palette=None, order=None, norm=None, saturation=1):
+ mapping = HueMapping(self, palette, order, norm, saturation)
+ self._hue_map = mapping
+
+ def map_size(self, sizes=None, order=None, norm=None):
+ mapping = SizeMapping(self, sizes, order, norm)
+ self._size_map = mapping
+
+ def map_style(self, markers=None, dashes=None, order=None):
+ mapping = StyleMapping(self, markers, dashes, order)
+ self._style_map = mapping
+
+ def iter_data(
+ self, grouping_vars=None, *,
+ reverse=False, from_comp_data=False,
+ by_facet=True, allow_empty=False, dropna=True,
+ ):
+ """Generator for getting subsets of data defined by semantic variables.
+
+ Also injects "col" and "row" into grouping semantics.
+
+ Parameters
+ ----------
+ grouping_vars : string or list of strings
+ Semantic variables that define the subsets of data.
+ reverse : bool
+ If True, reverse the order of iteration.
+ from_comp_data : bool
+ If True, use self.comp_data rather than self.plot_data
+ by_facet : bool
+ If True, add faceting variables to the set of grouping variables.
+ allow_empty : bool
+ If True, yield an empty dataframe when no observations exist for
+ combinations of grouping variables.
+ dropna : bool
+ If True, remove rows with missing data.
+
+ Yields
+ ------
+ sub_vars : dict
+ Keys are semantic names, values are the level of that semantic.
+ sub_data : :class:`pandas.DataFrame`
+ Subset of ``plot_data`` for this combination of semantic values.
+
+ """
+ # TODO should this default to using all (non x/y?) semantics?
+ # or define grouping vars somewhere?
+ if grouping_vars is None:
+ grouping_vars = []
+ elif isinstance(grouping_vars, str):
+ grouping_vars = [grouping_vars]
+ elif isinstance(grouping_vars, tuple):
+ grouping_vars = list(grouping_vars)
+
+ # Always insert faceting variables
+ if by_facet:
+ facet_vars = {"col", "row"}
+ grouping_vars.extend(
+ facet_vars & set(self.variables) - set(grouping_vars)
+ )
+
+ # Reduce to the semantics used in this plot
+ grouping_vars = [var for var in grouping_vars if var in self.variables]
+
+ if from_comp_data:
+ data = self.comp_data
+ else:
+ data = self.plot_data
+
+ if dropna:
+ data = data.dropna()
+
+ levels = self.var_levels.copy()
+ if from_comp_data:
+ for axis in {"x", "y"} & set(grouping_vars):
+ converter = self.converters[axis].iloc[0]
+ if self.var_types[axis] == "categorical":
+ if self._var_ordered[axis]:
+ # If the axis is ordered, then the axes in a possible
+ # facet grid are by definition "shared", or there is a
+ # single axis with a unique cat -> idx mapping.
+ # So we can just take the first converter object.
+ levels[axis] = converter.convert_units(levels[axis])
+ else:
+ # Otherwise, the mappings may not be unique, but we can
+ # use the unique set of index values in comp_data.
+ levels[axis] = np.sort(data[axis].unique())
+ else:
+ transform = converter.get_transform().transform
+ levels[axis] = transform(converter.convert_units(levels[axis]))
+
+ if grouping_vars:
+
+ grouped_data = data.groupby(
+ grouping_vars, sort=False, as_index=False, observed=False,
+ )
+
+ grouping_keys = []
+ for var in grouping_vars:
+ key = levels.get(var)
+ grouping_keys.append([] if key is None else key)
+
+ iter_keys = itertools.product(*grouping_keys)
+ if reverse:
+ iter_keys = reversed(list(iter_keys))
+
+ for key in iter_keys:
+
+ pd_key = (
+ key[0] if len(key) == 1 and _version_predates(pd, "2.2.0") else key
+ )
+ try:
+ data_subset = grouped_data.get_group(pd_key)
+ except KeyError:
+ # XXX we are adding this to allow backwards compatibility
+ # with the empty artists that old categorical plots would
+ # add (before 0.12), which we may decide to break, in which
+ # case this option could be removed
+ data_subset = data.loc[[]]
+
+ if data_subset.empty and not allow_empty:
+ continue
+
+ sub_vars = dict(zip(grouping_vars, key))
+
+ yield sub_vars, data_subset.copy()
+
+ else:
+
+ yield {}, data.copy()
+
+ @property
+ def comp_data(self):
+ """Dataframe with numeric x and y, after unit conversion and log scaling."""
+ if not hasattr(self, "ax"):
+ # Probably a good idea, but will need a bunch of tests updated
+ # Most of these tests should just use the external interface
+ # Then this can be re-enabled.
+ # raise AttributeError("No Axes attached to plotter")
+ return self.plot_data
+
+ if not hasattr(self, "_comp_data"):
+
+ comp_data = (
+ self.plot_data
+ .copy(deep=False)
+ .drop(["x", "y"], axis=1, errors="ignore")
+ )
+
+ for var in "yx":
+ if var not in self.variables:
+ continue
+
+ parts = []
+ grouped = self.plot_data[var].groupby(self.converters[var], sort=False)
+ for converter, orig in grouped:
+ orig = orig.mask(orig.isin([np.inf, -np.inf]), np.nan)
+ orig = orig.dropna()
+ if var in self.var_levels:
+ # TODO this should happen in some centralized location
+ # it is similar to GH2419, but more complicated because
+ # supporting `order` in categorical plots is tricky
+ orig = orig[orig.isin(self.var_levels[var])]
+ comp = pd.to_numeric(converter.convert_units(orig)).astype(float)
+ transform = converter.get_transform().transform
+ parts.append(pd.Series(transform(comp), orig.index, name=orig.name))
+ if parts:
+ comp_col = pd.concat(parts)
+ else:
+ comp_col = pd.Series(dtype=float, name=var)
+ comp_data.insert(0, var, comp_col)
+
+ self._comp_data = comp_data
+
+ return self._comp_data
+
+ def _get_axes(self, sub_vars):
+ """Return an Axes object based on existence of row/col variables."""
+ row = sub_vars.get("row", None)
+ col = sub_vars.get("col", None)
+ if row is not None and col is not None:
+ return self.facets.axes_dict[(row, col)]
+ elif row is not None:
+ return self.facets.axes_dict[row]
+ elif col is not None:
+ return self.facets.axes_dict[col]
+ elif self.ax is None:
+ return self.facets.ax
+ else:
+ return self.ax
+
+ def _attach(
+ self,
+ obj,
+ allowed_types=None,
+ log_scale=None,
+ ):
+ """Associate the plotter with an Axes manager and initialize its units.
+
+ Parameters
+ ----------
+ obj : :class:`matplotlib.axes.Axes` or :class:'FacetGrid`
+ Structural object that we will eventually plot onto.
+ allowed_types : str or list of str
+ If provided, raise when either the x or y variable does not have
+ one of the declared seaborn types.
+ log_scale : bool, number, or pair of bools or numbers
+ If not False, set the axes to use log scaling, with the given
+ base or defaulting to 10. If a tuple, interpreted as separate
+ arguments for the x and y axes.
+
+ """
+ from .axisgrid import FacetGrid
+ if isinstance(obj, FacetGrid):
+ self.ax = None
+ self.facets = obj
+ ax_list = obj.axes.flatten()
+ if obj.col_names is not None:
+ self.var_levels["col"] = obj.col_names
+ if obj.row_names is not None:
+ self.var_levels["row"] = obj.row_names
+ else:
+ self.ax = obj
+ self.facets = None
+ ax_list = [obj]
+
+ # Identify which "axis" variables we have defined
+ axis_variables = set("xy").intersection(self.variables)
+
+ # -- Verify the types of our x and y variables here.
+ # This doesn't really make complete sense being here here, but it's a fine
+ # place for it, given the current system.
+ # (Note that for some plots, there might be more complicated restrictions)
+ # e.g. the categorical plots have their own check that as specific to the
+ # non-categorical axis.
+ if allowed_types is None:
+ allowed_types = ["numeric", "datetime", "categorical"]
+ elif isinstance(allowed_types, str):
+ allowed_types = [allowed_types]
+
+ for var in axis_variables:
+ var_type = self.var_types[var]
+ if var_type not in allowed_types:
+ err = (
+ f"The {var} variable is {var_type}, but one of "
+ f"{allowed_types} is required"
+ )
+ raise TypeError(err)
+
+ # -- Get axis objects for each row in plot_data for type conversions and scaling
+
+ facet_dim = {"x": "col", "y": "row"}
+
+ self.converters = {}
+ for var in axis_variables:
+ other_var = {"x": "y", "y": "x"}[var]
+
+ converter = pd.Series(index=self.plot_data.index, name=var, dtype=object)
+ share_state = getattr(self.facets, f"_share{var}", True)
+
+ # Simplest cases are that we have a single axes, all axes are shared,
+ # or sharing is only on the orthogonal facet dimension. In these cases,
+ # all datapoints get converted the same way, so use the first axis
+ if share_state is True or share_state == facet_dim[other_var]:
+ converter.loc[:] = getattr(ax_list[0], f"{var}axis")
+
+ else:
+
+ # Next simplest case is when no axes are shared, and we can
+ # use the axis objects within each facet
+ if share_state is False:
+ for axes_vars, axes_data in self.iter_data():
+ ax = self._get_axes(axes_vars)
+ converter.loc[axes_data.index] = getattr(ax, f"{var}axis")
+
+ # In the more complicated case, the axes are shared within each
+ # "file" of the facetgrid. In that case, we need to subset the data
+ # for that file and assign it the first axis in the slice of the grid
+ else:
+
+ names = getattr(self.facets, f"{share_state}_names")
+ for i, level in enumerate(names):
+ idx = (i, 0) if share_state == "row" else (0, i)
+ axis = getattr(self.facets.axes[idx], f"{var}axis")
+ converter.loc[self.plot_data[share_state] == level] = axis
+
+ # Store the converter vector, which we use elsewhere (e.g comp_data)
+ self.converters[var] = converter
+
+ # Now actually update the matplotlib objects to do the conversion we want
+ grouped = self.plot_data[var].groupby(self.converters[var], sort=False)
+ for converter, seed_data in grouped:
+ if self.var_types[var] == "categorical":
+ if self._var_ordered[var]:
+ order = self.var_levels[var]
+ else:
+ order = None
+ seed_data = categorical_order(seed_data, order)
+ converter.update_units(seed_data)
+
+ # -- Set numerical axis scales
+
+ # First unpack the log_scale argument
+ if log_scale is None:
+ scalex = scaley = False
+ else:
+ # Allow single value or x, y tuple
+ try:
+ scalex, scaley = log_scale
+ except TypeError:
+ scalex = log_scale if self.var_types.get("x") == "numeric" else False
+ scaley = log_scale if self.var_types.get("y") == "numeric" else False
+
+ # Now use it
+ for axis, scale in zip("xy", (scalex, scaley)):
+ if scale:
+ for ax in ax_list:
+ set_scale = getattr(ax, f"set_{axis}scale")
+ if scale is True:
+ set_scale("log", nonpositive="mask")
+ else:
+ set_scale("log", base=scale, nonpositive="mask")
+
+ # For categorical y, we want the "first" level to be at the top of the axis
+ if self.var_types.get("y", None) == "categorical":
+ for ax in ax_list:
+ ax.yaxis.set_inverted(True)
+
+ # TODO -- Add axes labels
+
+ def _get_scale_transforms(self, axis):
+ """Return a function implementing the scale transform (or its inverse)."""
+ if self.ax is None:
+ axis_list = [getattr(ax, f"{axis}axis") for ax in self.facets.axes.flat]
+ scales = {axis.get_scale() for axis in axis_list}
+ if len(scales) > 1:
+ # It is a simplifying assumption that faceted axes will always have
+ # the same scale (even if they are unshared and have distinct limits).
+ # Nothing in the seaborn API allows you to create a FacetGrid with
+ # a mixture of scales, although it's possible via matplotlib.
+ # This is constraining, but no more so than previous behavior that
+ # only (properly) handled log scales, and there are some places where
+ # it would be much too complicated to use axes-specific transforms.
+ err = "Cannot determine transform with mixed scales on faceted axes."
+ raise RuntimeError(err)
+ transform_obj = axis_list[0].get_transform()
+ else:
+ # This case is more straightforward
+ transform_obj = getattr(self.ax, f"{axis}axis").get_transform()
+
+ return transform_obj.transform, transform_obj.inverted().transform
+
+ def _add_axis_labels(self, ax, default_x="", default_y=""):
+ """Add axis labels if not present, set visibility to match ticklabels."""
+ # TODO ax could default to None and use attached axes if present
+ # but what to do about the case of facets? Currently using FacetGrid's
+ # set_axis_labels method, which doesn't add labels to the interior even
+ # when the axes are not shared. Maybe that makes sense?
+ if not ax.get_xlabel():
+ x_visible = any(t.get_visible() for t in ax.get_xticklabels())
+ ax.set_xlabel(self.variables.get("x", default_x), visible=x_visible)
+ if not ax.get_ylabel():
+ y_visible = any(t.get_visible() for t in ax.get_yticklabels())
+ ax.set_ylabel(self.variables.get("y", default_y), visible=y_visible)
+
+ def add_legend_data(
+ self, ax, func, common_kws=None, attrs=None, semantic_kws=None,
+ ):
+ """Add labeled artists to represent the different plot semantics."""
+ verbosity = self.legend
+ if isinstance(verbosity, str) and verbosity not in ["auto", "brief", "full"]:
+ err = "`legend` must be 'auto', 'brief', 'full', or a boolean."
+ raise ValueError(err)
+ elif verbosity is True:
+ verbosity = "auto"
+
+ keys = []
+ legend_kws = {}
+ common_kws = {} if common_kws is None else common_kws.copy()
+ semantic_kws = {} if semantic_kws is None else semantic_kws.copy()
+
+ # Assign a legend title if there is only going to be one sub-legend,
+ # otherwise, subtitles will be inserted into the texts list with an
+ # invisible handle (which is a hack)
+ titles = {
+ title for title in
+ (self.variables.get(v, None) for v in ["hue", "size", "style"])
+ if title is not None
+ }
+ title = "" if len(titles) != 1 else titles.pop()
+ title_kws = dict(
+ visible=False, color="w", s=0, linewidth=0, marker="", dashes=""
+ )
+
+ def update(var_name, val_name, **kws):
+
+ key = var_name, val_name
+ if key in legend_kws:
+ legend_kws[key].update(**kws)
+ else:
+ keys.append(key)
+ legend_kws[key] = dict(**kws)
+
+ if attrs is None:
+ attrs = {"hue": "color", "size": ["linewidth", "s"], "style": None}
+ for var, names in attrs.items():
+ self._update_legend_data(
+ update, var, verbosity, title, title_kws, names, semantic_kws.get(var),
+ )
+
+ legend_data = {}
+ legend_order = []
+
+ # Don't allow color=None so we can set a neutral color for size/style legends
+ if common_kws.get("color", False) is None:
+ common_kws.pop("color")
+
+ for key in keys:
+
+ _, label = key
+ kws = legend_kws[key]
+ level_kws = {}
+ use_attrs = [
+ *self._legend_attributes,
+ *common_kws,
+ *[attr for var_attrs in semantic_kws.values() for attr in var_attrs],
+ ]
+ for attr in use_attrs:
+ if attr in kws:
+ level_kws[attr] = kws[attr]
+ artist = func(label=label, **{"color": ".2", **common_kws, **level_kws})
+ if _version_predates(mpl, "3.5.0"):
+ if isinstance(artist, mpl.lines.Line2D):
+ ax.add_line(artist)
+ elif isinstance(artist, mpl.patches.Patch):
+ ax.add_patch(artist)
+ elif isinstance(artist, mpl.collections.Collection):
+ ax.add_collection(artist)
+ else:
+ ax.add_artist(artist)
+ legend_data[key] = artist
+ legend_order.append(key)
+
+ self.legend_title = title
+ self.legend_data = legend_data
+ self.legend_order = legend_order
+
+ def _update_legend_data(
+ self,
+ update,
+ var,
+ verbosity,
+ title,
+ title_kws,
+ attr_names,
+ other_props,
+ ):
+ """Generate legend tick values and formatted labels."""
+ brief_ticks = 6
+ mapper = getattr(self, f"_{var}_map", None)
+ if mapper is None:
+ return
+
+ brief = mapper.map_type == "numeric" and (
+ verbosity == "brief"
+ or (verbosity == "auto" and len(mapper.levels) > brief_ticks)
+ )
+ if brief:
+ if isinstance(mapper.norm, mpl.colors.LogNorm):
+ locator = mpl.ticker.LogLocator(numticks=brief_ticks)
+ else:
+ locator = mpl.ticker.MaxNLocator(nbins=brief_ticks)
+ limits = min(mapper.levels), max(mapper.levels)
+ levels, formatted_levels = locator_to_legend_entries(
+ locator, limits, self.plot_data[var].infer_objects().dtype
+ )
+ elif mapper.levels is None:
+ levels = formatted_levels = []
+ else:
+ levels = formatted_levels = mapper.levels
+
+ if not title and self.variables.get(var, None) is not None:
+ update((self.variables[var], "title"), self.variables[var], **title_kws)
+
+ other_props = {} if other_props is None else other_props
+
+ for level, formatted_level in zip(levels, formatted_levels):
+ if level is not None:
+ attr = mapper(level)
+ if isinstance(attr_names, list):
+ attr = {name: attr for name in attr_names}
+ elif attr_names is not None:
+ attr = {attr_names: attr}
+ attr.update({k: v[level] for k, v in other_props.items() if level in v})
+ update(self.variables[var], formatted_level, **attr)
+
+ # XXX If the scale_* methods are going to modify the plot_data structure, they
+ # can't be called twice. That means that if they are called twice, they should
+ # raise. Alternatively, we could store an original version of plot_data and each
+ # time they are called they operate on the store, not the current state.
+
+ def scale_native(self, axis, *args, **kwargs):
+
+ # Default, defer to matplotlib
+
+ raise NotImplementedError
+
+ def scale_numeric(self, axis, *args, **kwargs):
+
+ # Feels needed to completeness, what should it do?
+ # Perhaps handle log scaling? Set the ticker/formatter/limits?
+
+ raise NotImplementedError
+
+ def scale_datetime(self, axis, *args, **kwargs):
+
+ # Use pd.to_datetime to convert strings or numbers to datetime objects
+ # Note, use day-resolution for numeric->datetime to match matplotlib
+
+ raise NotImplementedError
+
+ def scale_categorical(self, axis, order=None, formatter=None):
+ """
+ Enforce categorical (fixed-scale) rules for the data on given axis.
+
+ Parameters
+ ----------
+ axis : "x" or "y"
+ Axis of the plot to operate on.
+ order : list
+ Order that unique values should appear in.
+ formatter : callable
+ Function mapping values to a string representation.
+
+ Returns
+ -------
+ self
+
+ """
+ # This method both modifies the internal representation of the data
+ # (converting it to string) and sets some attributes on self. It might be
+ # a good idea to have a separate object attached to self that contains the
+ # information in those attributes (i.e. whether to enforce variable order
+ # across facets, the order to use) similar to the SemanticMapping objects
+ # we have for semantic variables. That object could also hold the converter
+ # objects that get used, if we can decouple those from an existing axis
+ # (cf. https://github.com/matplotlib/matplotlib/issues/19229).
+ # There are some interactions with faceting information that would need
+ # to be thought through, since the converts to use depend on facets.
+ # If we go that route, these methods could become "borrowed" methods similar
+ # to what happens with the alternate semantic mapper constructors, although
+ # that approach is kind of fussy and confusing.
+
+ # TODO this method could also set the grid state? Since we like to have no
+ # grid on the categorical axis by default. Again, a case where we'll need to
+ # store information until we use it, so best to have a way to collect the
+ # attributes that this method sets.
+
+ # TODO if we are going to set visual properties of the axes with these methods,
+ # then we could do the steps currently in CategoricalPlotter._adjust_cat_axis
+
+ # TODO another, and distinct idea, is to expose a cut= param here
+
+ _check_argument("axis", ["x", "y"], axis)
+
+ # Categorical plots can be "univariate" in which case they get an anonymous
+ # category label on the opposite axis.
+ if axis not in self.variables:
+ self.variables[axis] = None
+ self.var_types[axis] = "categorical"
+ self.plot_data[axis] = ""
+
+ # If the "categorical" variable has a numeric type, sort the rows so that
+ # the default result from categorical_order has those values sorted after
+ # they have been coerced to strings. The reason for this is so that later
+ # we can get facet-wise orders that are correct.
+ # XXX Should this also sort datetimes?
+ # It feels more consistent, but technically will be a default change
+ # If so, should also change categorical_order to behave that way
+ if self.var_types[axis] == "numeric":
+ self.plot_data = self.plot_data.sort_values(axis, kind="mergesort")
+
+ # Now get a reference to the categorical data vector and remove na values
+ cat_data = self.plot_data[axis].dropna()
+
+ # Get the initial categorical order, which we do before string
+ # conversion to respect the original types of the order list.
+ # Track whether the order is given explicitly so that we can know
+ # whether or not to use the order constructed here downstream
+ self._var_ordered[axis] = order is not None or cat_data.dtype.name == "category"
+ order = pd.Index(categorical_order(cat_data, order), name=axis)
+
+ # Then convert data to strings. This is because in matplotlib,
+ # "categorical" data really mean "string" data, so doing this artists
+ # will be drawn on the categorical axis with a fixed scale.
+ # TODO implement formatter here; check that it returns strings?
+ if formatter is not None:
+ cat_data = cat_data.map(formatter)
+ order = order.map(formatter)
+ else:
+ cat_data = cat_data.astype(str)
+ order = order.astype(str)
+
+ # Update the levels list with the type-converted order variable
+ self.var_levels[axis] = order
+
+ # Now ensure that seaborn will use categorical rules internally
+ self.var_types[axis] = "categorical"
+
+ # Put the string-typed categorical vector back into the plot_data structure
+ self.plot_data[axis] = cat_data
+
+ return self
+
+
+class VariableType(UserString):
+ """
+ Prevent comparisons elsewhere in the library from using the wrong name.
+
+ Errors are simple assertions because users should not be able to trigger
+ them. If that changes, they should be more verbose.
+
+ """
+ # TODO we can replace this with typing.Literal on Python 3.8+
+ allowed = "numeric", "datetime", "categorical"
+
+ def __init__(self, data):
+ assert data in self.allowed, data
+ super().__init__(data)
+
+ def __eq__(self, other):
+ assert other in self.allowed, other
+ return self.data == other
+
+
+def variable_type(vector, boolean_type="numeric"):
+ """
+ Determine whether a vector contains numeric, categorical, or datetime data.
+
+ This function differs from the pandas typing API in two ways:
+
+ - Python sequences or object-typed PyData objects are considered numeric if
+ all of their entries are numeric.
+ - String or mixed-type data are considered categorical even if not
+ explicitly represented as a :class:`pandas.api.types.CategoricalDtype`.
+
+ Parameters
+ ----------
+ vector : :func:`pandas.Series`, :func:`numpy.ndarray`, or Python sequence
+ Input data to test.
+ boolean_type : 'numeric' or 'categorical'
+ Type to use for vectors containing only 0s and 1s (and NAs).
+
+ Returns
+ -------
+ var_type : 'numeric', 'categorical', or 'datetime'
+ Name identifying the type of data in the vector.
+ """
+ vector = pd.Series(vector)
+
+ # If a categorical dtype is set, infer categorical
+ if isinstance(vector.dtype, pd.CategoricalDtype):
+ return VariableType("categorical")
+
+ # Special-case all-na data, which is always "numeric"
+ if pd.isna(vector).all():
+ return VariableType("numeric")
+
+ # At this point, drop nans to simplify further type inference
+ vector = vector.dropna()
+
+ # Special-case binary/boolean data, allow caller to determine
+ # This triggers a numpy warning when vector has strings/objects
+ # https://github.com/numpy/numpy/issues/6784
+ # Because we reduce with .all(), we are agnostic about whether the
+ # comparison returns a scalar or vector, so we will ignore the warning.
+ # It triggers a separate DeprecationWarning when the vector has datetimes:
+ # https://github.com/numpy/numpy/issues/13548
+ # This is considered a bug by numpy and will likely go away.
+ with warnings.catch_warnings():
+ warnings.simplefilter(
+ action='ignore', category=(FutureWarning, DeprecationWarning)
+ )
+ try:
+ if np.isin(vector, [0, 1]).all():
+ return VariableType(boolean_type)
+ except TypeError:
+ # .isin comparison is not guaranteed to be possible under NumPy
+ # casting rules, depending on the (unknown) dtype of 'vector'
+ pass
+
+ # Defer to positive pandas tests
+ if pd.api.types.is_numeric_dtype(vector):
+ return VariableType("numeric")
+
+ if pd.api.types.is_datetime64_dtype(vector):
+ return VariableType("datetime")
+
+ # --- If we get to here, we need to check the entries
+
+ # Check for a collection where everything is a number
+
+ def all_numeric(x):
+ for x_i in x:
+ if not isinstance(x_i, Number):
+ return False
+ return True
+
+ if all_numeric(vector):
+ return VariableType("numeric")
+
+ # Check for a collection where everything is a datetime
+
+ def all_datetime(x):
+ for x_i in x:
+ if not isinstance(x_i, (datetime, np.datetime64)):
+ return False
+ return True
+
+ if all_datetime(vector):
+ return VariableType("datetime")
+
+ # Otherwise, our final fallback is to consider things categorical
+
+ return VariableType("categorical")
+
+
+def infer_orient(x=None, y=None, orient=None, require_numeric=True):
+ """Determine how the plot should be oriented based on the data.
+
+ For historical reasons, the convention is to call a plot "horizontally"
+ or "vertically" oriented based on the axis representing its dependent
+ variable. Practically, this is used when determining the axis for
+ numerical aggregation.
+
+ Parameters
+ ----------
+ x, y : Vector data or None
+ Positional data vectors for the plot.
+ orient : string or None
+ Specified orientation. If not None, can be "x" or "y", or otherwise
+ must start with "v" or "h".
+ require_numeric : bool
+ If set, raise when the implied dependent variable is not numeric.
+
+ Returns
+ -------
+ orient : "x" or "y"
+
+ Raises
+ ------
+ ValueError: When `orient` is an unknown string.
+ TypeError: When dependent variable is not numeric, with `require_numeric`
+
+ """
+
+ x_type = None if x is None else variable_type(x)
+ y_type = None if y is None else variable_type(y)
+
+ nonnumeric_dv_error = "{} orientation requires numeric `{}` variable."
+ single_var_warning = "{} orientation ignored with only `{}` specified."
+
+ if x is None:
+ if str(orient).startswith("h"):
+ warnings.warn(single_var_warning.format("Horizontal", "y"))
+ if require_numeric and y_type != "numeric":
+ raise TypeError(nonnumeric_dv_error.format("Vertical", "y"))
+ return "x"
+
+ elif y is None:
+ if str(orient).startswith("v"):
+ warnings.warn(single_var_warning.format("Vertical", "x"))
+ if require_numeric and x_type != "numeric":
+ raise TypeError(nonnumeric_dv_error.format("Horizontal", "x"))
+ return "y"
+
+ elif str(orient).startswith("v") or orient == "x":
+ if require_numeric and y_type != "numeric":
+ raise TypeError(nonnumeric_dv_error.format("Vertical", "y"))
+ return "x"
+
+ elif str(orient).startswith("h") or orient == "y":
+ if require_numeric and x_type != "numeric":
+ raise TypeError(nonnumeric_dv_error.format("Horizontal", "x"))
+ return "y"
+
+ elif orient is not None:
+ err = (
+ "`orient` must start with 'v' or 'h' or be None, "
+ f"but `{repr(orient)}` was passed."
+ )
+ raise ValueError(err)
+
+ elif x_type != "categorical" and y_type == "categorical":
+ return "y"
+
+ elif x_type != "numeric" and y_type == "numeric":
+ return "x"
+
+ elif x_type == "numeric" and y_type != "numeric":
+ return "y"
+
+ elif require_numeric and "numeric" not in (x_type, y_type):
+ err = "Neither the `x` nor `y` variable appears to be numeric."
+ raise TypeError(err)
+
+ else:
+ return "x"
+
+
+def unique_dashes(n):
+ """Build an arbitrarily long list of unique dash styles for lines.
+
+ Parameters
+ ----------
+ n : int
+ Number of unique dash specs to generate.
+
+ Returns
+ -------
+ dashes : list of strings or tuples
+ Valid arguments for the ``dashes`` parameter on
+ :class:`matplotlib.lines.Line2D`. The first spec is a solid
+ line (``""``), the remainder are sequences of long and short
+ dashes.
+
+ """
+ # Start with dash specs that are well distinguishable
+ dashes = [
+ "",
+ (4, 1.5),
+ (1, 1),
+ (3, 1.25, 1.5, 1.25),
+ (5, 1, 1, 1),
+ ]
+
+ # Now programmatically build as many as we need
+ p = 3
+ while len(dashes) < n:
+
+ # Take combinations of long and short dashes
+ a = itertools.combinations_with_replacement([3, 1.25], p)
+ b = itertools.combinations_with_replacement([4, 1], p)
+
+ # Interleave the combinations, reversing one of the streams
+ segment_list = itertools.chain(*zip(
+ list(a)[1:-1][::-1],
+ list(b)[1:-1]
+ ))
+
+ # Now insert the gaps
+ for segments in segment_list:
+ gap = min(segments)
+ spec = tuple(itertools.chain(*((seg, gap) for seg in segments)))
+ dashes.append(spec)
+
+ p += 1
+
+ return dashes[:n]
+
+
+def unique_markers(n):
+ """Build an arbitrarily long list of unique marker styles for points.
+
+ Parameters
+ ----------
+ n : int
+ Number of unique marker specs to generate.
+
+ Returns
+ -------
+ markers : list of string or tuples
+ Values for defining :class:`matplotlib.markers.MarkerStyle` objects.
+ All markers will be filled.
+
+ """
+ # Start with marker specs that are well distinguishable
+ markers = [
+ "o",
+ "X",
+ (4, 0, 45),
+ "P",
+ (4, 0, 0),
+ (4, 1, 0),
+ "^",
+ (4, 1, 45),
+ "v",
+ ]
+
+ # Now generate more from regular polygons of increasing order
+ s = 5
+ while len(markers) < n:
+ a = 360 / (s + 1) / 2
+ markers.extend([
+ (s + 1, 1, a),
+ (s + 1, 0, a),
+ (s, 1, 0),
+ (s, 0, 0),
+ ])
+ s += 1
+
+ # Convert to MarkerStyle object, using only exactly what we need
+ # markers = [mpl.markers.MarkerStyle(m) for m in markers[:n]]
+
+ return markers[:n]
+
+
+def categorical_order(vector, order=None):
+ """Return a list of unique data values.
+
+ Determine an ordered list of levels in ``values``.
+
+ Parameters
+ ----------
+ vector : list, array, Categorical, or Series
+ Vector of "categorical" values
+ order : list-like, optional
+ Desired order of category levels to override the order determined
+ from the ``values`` object.
+
+ Returns
+ -------
+ order : list
+ Ordered list of category levels not including null values.
+
+ """
+ if order is None:
+ if hasattr(vector, "categories"):
+ order = vector.categories
+ else:
+ try:
+ order = vector.cat.categories
+ except (TypeError, AttributeError):
+
+ order = pd.Series(vector).unique()
+
+ if variable_type(vector) == "numeric":
+ order = np.sort(order)
+
+ order = filter(pd.notnull, order)
+ return list(order)
diff --git a/seaborn/_compat.py b/seaborn/_compat.py
new file mode 100644
index 0000000000000000000000000000000000000000..bd2f0c12d3a254f09d2cf85a300d9ee177c75f00
--- /dev/null
+++ b/seaborn/_compat.py
@@ -0,0 +1,123 @@
+from __future__ import annotations
+from typing import Literal
+
+import numpy as np
+import pandas as pd
+import matplotlib as mpl
+from matplotlib.figure import Figure
+from seaborn.utils import _version_predates
+
+
+def norm_from_scale(scale, norm):
+ """Produce a Normalize object given a Scale and min/max domain limits."""
+ # This is an internal maplotlib function that simplifies things to access
+ # It is likely to become part of the matplotlib API at some point:
+ # https://github.com/matplotlib/matplotlib/issues/20329
+ if isinstance(norm, mpl.colors.Normalize):
+ return norm
+
+ if scale is None:
+ return None
+
+ if norm is None:
+ vmin = vmax = None
+ else:
+ vmin, vmax = norm # TODO more helpful error if this fails?
+
+ class ScaledNorm(mpl.colors.Normalize):
+
+ def __call__(self, value, clip=None):
+ # From github.com/matplotlib/matplotlib/blob/v3.4.2/lib/matplotlib/colors.py
+ # See github.com/matplotlib/matplotlib/tree/v3.4.2/LICENSE
+ value, is_scalar = self.process_value(value)
+ self.autoscale_None(value)
+ if self.vmin > self.vmax:
+ raise ValueError("vmin must be less or equal to vmax")
+ if self.vmin == self.vmax:
+ return np.full_like(value, 0)
+ if clip is None:
+ clip = self.clip
+ if clip:
+ value = np.clip(value, self.vmin, self.vmax)
+ # ***** Seaborn changes start ****
+ t_value = self.transform(value).reshape(np.shape(value))
+ t_vmin, t_vmax = self.transform([self.vmin, self.vmax])
+ # ***** Seaborn changes end *****
+ if not np.isfinite([t_vmin, t_vmax]).all():
+ raise ValueError("Invalid vmin or vmax")
+ t_value -= t_vmin
+ t_value /= (t_vmax - t_vmin)
+ t_value = np.ma.masked_invalid(t_value, copy=False)
+ return t_value[0] if is_scalar else t_value
+
+ new_norm = ScaledNorm(vmin, vmax)
+ new_norm.transform = scale.get_transform().transform
+
+ return new_norm
+
+
+def get_colormap(name):
+ """Handle changes to matplotlib colormap interface in 3.6."""
+ try:
+ return mpl.colormaps[name]
+ except AttributeError:
+ return mpl.cm.get_cmap(name)
+
+
+def register_colormap(name, cmap):
+ """Handle changes to matplotlib colormap interface in 3.6."""
+ try:
+ if name not in mpl.colormaps:
+ mpl.colormaps.register(cmap, name=name)
+ except AttributeError:
+ mpl.cm.register_cmap(name, cmap)
+
+
+def set_layout_engine(
+ fig: Figure,
+ engine: Literal["constrained", "compressed", "tight", "none"],
+) -> None:
+ """Handle changes to auto layout engine interface in 3.6"""
+ if hasattr(fig, "set_layout_engine"):
+ fig.set_layout_engine(engine)
+ else:
+ # _version_predates(mpl, 3.6)
+ if engine == "tight":
+ fig.set_tight_layout(True) # type: ignore # predates typing
+ elif engine == "constrained":
+ fig.set_constrained_layout(True) # type: ignore
+ elif engine == "none":
+ fig.set_tight_layout(False) # type: ignore
+ fig.set_constrained_layout(False) # type: ignore
+
+
+def get_layout_engine(fig: Figure) -> mpl.layout_engine.LayoutEngine | None:
+ """Handle changes to auto layout engine interface in 3.6"""
+ if hasattr(fig, "get_layout_engine"):
+ return fig.get_layout_engine()
+ else:
+ # _version_predates(mpl, 3.6)
+ return None
+
+
+def share_axis(ax0, ax1, which):
+ """Handle changes to post-hoc axis sharing."""
+ if _version_predates(mpl, "3.5"):
+ group = getattr(ax0, f"get_shared_{which}_axes")()
+ group.join(ax1, ax0)
+ else:
+ getattr(ax1, f"share{which}")(ax0)
+
+
+def get_legend_handles(legend):
+ """Handle legendHandles attribute rename."""
+ if _version_predates(mpl, "3.7"):
+ return legend.legendHandles
+ else:
+ return legend.legend_handles
+
+
+def groupby_apply_include_groups(val):
+ if _version_predates(pd, "2.2.0"):
+ return {}
+ return {"include_groups": val}
diff --git a/seaborn/_core/__init__.py b/seaborn/_core/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/seaborn/_core/data.py b/seaborn/_core/data.py
new file mode 100644
index 0000000000000000000000000000000000000000..c17bfe95c55d738b4d4ef69bfb945b0647343163
--- /dev/null
+++ b/seaborn/_core/data.py
@@ -0,0 +1,319 @@
+"""
+Components for parsing variable assignments and internally representing plot data.
+"""
+from __future__ import annotations
+
+from collections.abc import Mapping, Sized
+from typing import cast
+import warnings
+
+import pandas as pd
+from pandas import DataFrame
+
+from seaborn._core.typing import DataSource, VariableSpec, ColumnName
+from seaborn.utils import _version_predates
+
+
+class PlotData:
+ """
+ Data table with plot variable schema and mapping to original names.
+
+ Contains logic for parsing variable specification arguments and updating
+ the table with layer-specific data and/or mappings.
+
+ Parameters
+ ----------
+ data
+ Input data where variable names map to vector values.
+ variables
+ Keys are names of plot variables (x, y, ...) each value is one of:
+
+ - name of a column (or index level, or dictionary entry) in `data`
+ - vector in any format that can construct a :class:`pandas.DataFrame`
+
+ Attributes
+ ----------
+ frame
+ Data table with column names having defined plot variables.
+ names
+ Dictionary mapping plot variable names to names in source data structure(s).
+ ids
+ Dictionary mapping plot variable names to unique data source identifiers.
+
+ """
+ frame: DataFrame
+ frames: dict[tuple, DataFrame]
+ names: dict[str, str | None]
+ ids: dict[str, str | int]
+ source_data: DataSource
+ source_vars: dict[str, VariableSpec]
+
+ def __init__(
+ self,
+ data: DataSource,
+ variables: dict[str, VariableSpec],
+ ):
+
+ data = handle_data_source(data)
+ frame, names, ids = self._assign_variables(data, variables)
+
+ self.frame = frame
+ self.names = names
+ self.ids = ids
+
+ # The reason we possibly have a dictionary of frames is to support the
+ # Plot.pair operation, post scaling, where each x/y variable needs its
+ # own frame. This feels pretty clumsy and there are a bunch of places in
+ # the client code with awkard if frame / elif frames constructions.
+ # It would be great to have a cleaner abstraction here.
+ self.frames = {}
+
+ self.source_data = data
+ self.source_vars = variables
+
+ def __contains__(self, key: str) -> bool:
+ """Boolean check on whether a variable is defined in this dataset."""
+ if self.frame is None:
+ return any(key in df for df in self.frames.values())
+ return key in self.frame
+
+ def join(
+ self,
+ data: DataSource,
+ variables: dict[str, VariableSpec] | None,
+ ) -> PlotData:
+ """Add, replace, or drop variables and return as a new dataset."""
+ # Inherit the original source of the upstream data by default
+ if data is None:
+ data = self.source_data
+
+ # TODO allow `data` to be a function (that is called on the source data?)
+
+ if not variables:
+ variables = self.source_vars
+
+ # Passing var=None implies that we do not want that variable in this layer
+ disinherit = [k for k, v in variables.items() if v is None]
+
+ # Create a new dataset with just the info passed here
+ new = PlotData(data, variables)
+
+ # -- Update the inherited DataSource with this new information
+
+ drop_cols = [k for k in self.frame if k in new.frame or k in disinherit]
+ parts = [self.frame.drop(columns=drop_cols), new.frame]
+
+ # Because we are combining distinct columns, this is perhaps more
+ # naturally thought of as a "merge"/"join". But using concat because
+ # some simple testing suggests that it is marginally faster.
+ frame = pd.concat(parts, axis=1, sort=False, copy=False)
+
+ names = {k: v for k, v in self.names.items() if k not in disinherit}
+ names.update(new.names)
+
+ ids = {k: v for k, v in self.ids.items() if k not in disinherit}
+ ids.update(new.ids)
+
+ new.frame = frame
+ new.names = names
+ new.ids = ids
+
+ # Multiple chained operations should always inherit from the original object
+ new.source_data = self.source_data
+ new.source_vars = self.source_vars
+
+ return new
+
+ def _assign_variables(
+ self,
+ data: DataFrame | Mapping | None,
+ variables: dict[str, VariableSpec],
+ ) -> tuple[DataFrame, dict[str, str | None], dict[str, str | int]]:
+ """
+ Assign values for plot variables given long-form data and/or vector inputs.
+
+ Parameters
+ ----------
+ data
+ Input data where variable names map to vector values.
+ variables
+ Keys are names of plot variables (x, y, ...) each value is one of:
+
+ - name of a column (or index level, or dictionary entry) in `data`
+ - vector in any format that can construct a :class:`pandas.DataFrame`
+
+ Returns
+ -------
+ frame
+ Table mapping seaborn variables (x, y, color, ...) to data vectors.
+ names
+ Keys are defined seaborn variables; values are names inferred from
+ the inputs (or None when no name can be determined).
+ ids
+ Like the `names` dict, but `None` values are replaced by the `id()`
+ of the data object that defined the variable.
+
+ Raises
+ ------
+ TypeError
+ When data source is not a DataFrame or Mapping.
+ ValueError
+ When variables are strings that don't appear in `data`, or when they are
+ non-indexed vector datatypes that have a different length from `data`.
+
+ """
+ source_data: Mapping | DataFrame
+ frame: DataFrame
+ names: dict[str, str | None]
+ ids: dict[str, str | int]
+
+ plot_data = {}
+ names = {}
+ ids = {}
+
+ given_data = data is not None
+ if data is None:
+ # Data is optional; all variables can be defined as vectors
+ # But simplify downstream code by always having a usable source data object
+ source_data = {}
+ else:
+ source_data = data
+
+ # Variables can also be extracted from the index of a DataFrame
+ if isinstance(source_data, pd.DataFrame):
+ index = source_data.index.to_frame().to_dict("series")
+ else:
+ index = {}
+
+ for key, val in variables.items():
+
+ # Simply ignore variables with no specification
+ if val is None:
+ continue
+
+ # Try to treat the argument as a key for the data collection.
+ # But be flexible about what can be used as a key.
+ # Usually it will be a string, but allow other hashables when
+ # taking from the main data object. Allow only strings to reference
+ # fields in the index, because otherwise there is too much ambiguity.
+
+ # TODO this will be rendered unnecessary by the following pandas fix:
+ # https://github.com/pandas-dev/pandas/pull/41283
+ try:
+ hash(val)
+ val_is_hashable = True
+ except TypeError:
+ val_is_hashable = False
+
+ val_as_data_key = (
+ # See https://github.com/pandas-dev/pandas/pull/41283
+ # (isinstance(val, abc.Hashable) and val in source_data)
+ (val_is_hashable and val in source_data)
+ or (isinstance(val, str) and val in index)
+ )
+
+ if val_as_data_key:
+ val = cast(ColumnName, val)
+ if val in source_data:
+ plot_data[key] = source_data[val]
+ elif val in index:
+ plot_data[key] = index[val]
+ names[key] = ids[key] = str(val)
+
+ elif isinstance(val, str):
+
+ # This looks like a column name but, lookup failed.
+
+ err = f"Could not interpret value `{val}` for `{key}`. "
+ if not given_data:
+ err += "Value is a string, but `data` was not passed."
+ else:
+ err += "An entry with this name does not appear in `data`."
+ raise ValueError(err)
+
+ else:
+
+ # Otherwise, assume the value somehow represents data
+
+ # Ignore empty data structures
+ if isinstance(val, Sized) and len(val) == 0:
+ continue
+
+ # If vector has no index, it must match length of data table
+ if isinstance(data, pd.DataFrame) and not isinstance(val, pd.Series):
+ if isinstance(val, Sized) and len(data) != len(val):
+ val_cls = val.__class__.__name__
+ err = (
+ f"Length of {val_cls} vectors must match length of `data`"
+ f" when both are used, but `data` has length {len(data)}"
+ f" and the vector passed to `{key}` has length {len(val)}."
+ )
+ raise ValueError(err)
+
+ plot_data[key] = val
+
+ # Try to infer the original name using pandas-like metadata
+ if hasattr(val, "name"):
+ names[key] = ids[key] = str(val.name) # type: ignore # mypy/1424
+ else:
+ names[key] = None
+ ids[key] = id(val)
+
+ # Construct a tidy plot DataFrame. This will convert a number of
+ # types automatically, aligning on index in case of pandas objects
+ # TODO Note: this fails when variable specs *only* have scalars!
+ frame = pd.DataFrame(plot_data)
+
+ return frame, names, ids
+
+
+def handle_data_source(data: object) -> pd.DataFrame | Mapping | None:
+ """Convert the data source object to a common union representation."""
+ if isinstance(data, pd.DataFrame) or hasattr(data, "__dataframe__"):
+ # Check for pd.DataFrame inheritance could be removed once
+ # minimal pandas version supports dataframe interchange (1.5.0).
+ data = convert_dataframe_to_pandas(data)
+ elif data is not None and not isinstance(data, Mapping):
+ err = f"Data source must be a DataFrame or Mapping, not {type(data)!r}."
+ raise TypeError(err)
+
+ return data
+
+
+def convert_dataframe_to_pandas(data: object) -> pd.DataFrame:
+ """Use the DataFrame exchange protocol, or fail gracefully."""
+ if isinstance(data, pd.DataFrame):
+ return data
+
+ if not hasattr(pd.api, "interchange"):
+ msg = (
+ "Support for non-pandas DataFrame objects requires a version of pandas "
+ "that implements the DataFrame interchange protocol. Please upgrade "
+ "your pandas version or coerce your data to pandas before passing "
+ "it to seaborn."
+ )
+ raise TypeError(msg)
+
+ if _version_predates(pd, "2.0.2"):
+ msg = (
+ "DataFrame interchange with pandas<2.0.2 has some known issues. "
+ f"You are using pandas {pd.__version__}. "
+ "Continuing, but it is recommended to carefully inspect the results and to "
+ "consider upgrading."
+ )
+ warnings.warn(msg, stacklevel=2)
+
+ try:
+ # This is going to convert all columns in the input dataframe, even though
+ # we may only need one or two of them. It would be more efficient to select
+ # the columns that are going to be used in the plot prior to interchange.
+ # Solving that in general is a hard problem, especially with the objects
+ # interface where variables passed in Plot() may only be referenced later
+ # in Plot.add(). But noting here in case this seems to be a bottleneck.
+ return pd.api.interchange.from_dataframe(data)
+ except Exception as err:
+ msg = (
+ "Encountered an exception when converting data source "
+ "to a pandas DataFrame. See traceback above for details."
+ )
+ raise RuntimeError(msg) from err
diff --git a/seaborn/_core/exceptions.py b/seaborn/_core/exceptions.py
new file mode 100644
index 0000000000000000000000000000000000000000..048443b0f8639e2e90a635c74e6202ae62e3ca8b
--- /dev/null
+++ b/seaborn/_core/exceptions.py
@@ -0,0 +1,32 @@
+"""
+Custom exceptions for the seaborn.objects interface.
+
+This is very lightweight, but it's a separate module to avoid circular imports.
+
+"""
+from __future__ import annotations
+
+
+class PlotSpecError(RuntimeError):
+ """
+ Error class raised from seaborn.objects.Plot for compile-time failures.
+
+ In the declarative Plot interface, exceptions may not be triggered immediately
+ by bad user input (and validation at input time may not be possible). This class
+ is used to signal that indirect dependency. It should be raised in an exception
+ chain when compile-time operations fail with an error message providing useful
+ context (e.g., scaling errors could specify the variable that failed.)
+
+ """
+ @classmethod
+ def _during(cls, step: str, var: str = "") -> PlotSpecError:
+ """
+ Initialize the class to report the failure of a specific operation.
+ """
+ message = []
+ if var:
+ message.append(f"{step} failed for the `{var}` variable.")
+ else:
+ message.append(f"{step} failed.")
+ message.append("See the traceback above for more information.")
+ return cls(" ".join(message))
diff --git a/seaborn/_core/groupby.py b/seaborn/_core/groupby.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb63c670d29e8be63514d08ec4cbbfdfd0d79c46
--- /dev/null
+++ b/seaborn/_core/groupby.py
@@ -0,0 +1,129 @@
+"""Simplified split-apply-combine paradigm on dataframes for internal use."""
+from __future__ import annotations
+
+from typing import cast, Iterable
+
+import pandas as pd
+
+from seaborn._core.rules import categorical_order
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+ from typing import Callable
+ from pandas import DataFrame, MultiIndex, Index
+
+
+class GroupBy:
+ """
+ Interface for Pandas GroupBy operations allowing specified group order.
+
+ Writing our own class to do this has a few advantages:
+ - It constrains the interface between Plot and Stat/Move objects
+ - It allows control over the row order of the GroupBy result, which is
+ important when using in the context of some Move operations (dodge, stack, ...)
+ - It simplifies some complexities regarding the return type and Index contents
+ one encounters with Pandas, especially for DataFrame -> DataFrame applies
+ - It increases future flexibility regarding alternate DataFrame libraries
+
+ """
+ def __init__(self, order: list[str] | dict[str, list | None]):
+ """
+ Initialize the GroupBy from grouping variables and optional level orders.
+
+ Parameters
+ ----------
+ order
+ List of variable names or dict mapping names to desired level orders.
+ Level order values can be None to use default ordering rules. The
+ variables can include names that are not expected to appear in the
+ data; these will be dropped before the groups are defined.
+
+ """
+ if not order:
+ raise ValueError("GroupBy requires at least one grouping variable")
+
+ if isinstance(order, list):
+ order = {k: None for k in order}
+ self.order = order
+
+ def _get_groups(
+ self, data: DataFrame
+ ) -> tuple[str | list[str], Index | MultiIndex]:
+ """Return index with Cartesian product of ordered grouping variable levels."""
+ levels = {}
+ for var, order in self.order.items():
+ if var in data:
+ if order is None:
+ order = categorical_order(data[var])
+ levels[var] = order
+
+ grouper: str | list[str]
+ groups: Index | MultiIndex
+ if not levels:
+ grouper = []
+ groups = pd.Index([])
+ elif len(levels) > 1:
+ grouper = list(levels)
+ groups = pd.MultiIndex.from_product(levels.values(), names=grouper)
+ else:
+ grouper, = list(levels)
+ groups = pd.Index(levels[grouper], name=grouper)
+ return grouper, groups
+
+ def _reorder_columns(self, res, data):
+ """Reorder result columns to match original order with new columns appended."""
+ cols = [c for c in data if c in res]
+ cols += [c for c in res if c not in data]
+ return res.reindex(columns=pd.Index(cols))
+
+ def agg(self, data: DataFrame, *args, **kwargs) -> DataFrame:
+ """
+ Reduce each group to a single row in the output.
+
+ The output will have a row for each unique combination of the grouping
+ variable levels with null values for the aggregated variable(s) where
+ those combinations do not appear in the dataset.
+
+ """
+ grouper, groups = self._get_groups(data)
+
+ if not grouper:
+ # We will need to see whether there are valid usecases that end up here
+ raise ValueError("No grouping variables are present in dataframe")
+
+ res = (
+ data
+ .groupby(grouper, sort=False, observed=False)
+ .agg(*args, **kwargs)
+ .reindex(groups)
+ .reset_index()
+ .pipe(self._reorder_columns, data)
+ )
+
+ return res
+
+ def apply(
+ self, data: DataFrame, func: Callable[..., DataFrame],
+ *args, **kwargs,
+ ) -> DataFrame:
+ """Apply a DataFrame -> DataFrame mapping to each group."""
+ grouper, groups = self._get_groups(data)
+
+ if not grouper:
+ return self._reorder_columns(func(data, *args, **kwargs), data)
+
+ parts = {}
+ for key, part_df in data.groupby(grouper, sort=False, observed=False):
+ parts[key] = func(part_df, *args, **kwargs)
+ stack = []
+ for key in groups:
+ if key in parts:
+ if isinstance(grouper, list):
+ # Implies that we had a MultiIndex so key is iterable
+ group_ids = dict(zip(grouper, cast(Iterable, key)))
+ else:
+ group_ids = {grouper: key}
+ stack.append(parts[key].assign(**group_ids))
+
+ res = pd.concat(stack, ignore_index=True)
+ return self._reorder_columns(res, data)
diff --git a/seaborn/_core/moves.py b/seaborn/_core/moves.py
new file mode 100644
index 0000000000000000000000000000000000000000..179926e71789bb6a6891aa21d80ee38696f89236
--- /dev/null
+++ b/seaborn/_core/moves.py
@@ -0,0 +1,274 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import ClassVar, Callable, Optional, Union, cast
+
+import numpy as np
+from pandas import DataFrame
+
+from seaborn._core.groupby import GroupBy
+from seaborn._core.scales import Scale
+from seaborn._core.typing import Default
+
+default = Default()
+
+
+@dataclass
+class Move:
+ """Base class for objects that apply simple positional transforms."""
+
+ group_by_orient: ClassVar[bool] = True
+
+ def __call__(
+ self, data: DataFrame, groupby: GroupBy, orient: str, scales: dict[str, Scale],
+ ) -> DataFrame:
+ raise NotImplementedError
+
+
+@dataclass
+class Jitter(Move):
+ """
+ Random displacement along one or both axes to reduce overplotting.
+
+ Parameters
+ ----------
+ width : float
+ Magnitude of jitter, relative to mark width, along the orientation axis.
+ If not provided, the default value will be 0 when `x` or `y` are set, otherwise
+ there will be a small amount of jitter applied by default.
+ x : float
+ Magnitude of jitter, in data units, along the x axis.
+ y : float
+ Magnitude of jitter, in data units, along the y axis.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Jitter.rst
+
+ """
+ width: float | Default = default
+ x: float = 0
+ y: float = 0
+ seed: int | None = None
+
+ def __call__(
+ self, data: DataFrame, groupby: GroupBy, orient: str, scales: dict[str, Scale],
+ ) -> DataFrame:
+
+ data = data.copy()
+ rng = np.random.default_rng(self.seed)
+
+ def jitter(data, col, scale):
+ noise = rng.uniform(-.5, +.5, len(data))
+ offsets = noise * scale
+ return data[col] + offsets
+
+ if self.width is default:
+ width = 0.0 if self.x or self.y else 0.2
+ else:
+ width = cast(float, self.width)
+
+ if self.width:
+ data[orient] = jitter(data, orient, width * data["width"])
+ if self.x:
+ data["x"] = jitter(data, "x", self.x)
+ if self.y:
+ data["y"] = jitter(data, "y", self.y)
+
+ return data
+
+
+@dataclass
+class Dodge(Move):
+ """
+ Displacement and narrowing of overlapping marks along orientation axis.
+
+ Parameters
+ ----------
+ empty : {'keep', 'drop', 'fill'}
+ gap : float
+ Size of gap between dodged marks.
+ by : list of variable names
+ Variables to apply the movement to, otherwise use all.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Dodge.rst
+
+ """
+ empty: str = "keep" # Options: keep, drop, fill
+ gap: float = 0
+
+ # TODO accept just a str here?
+ # TODO should this always be present?
+ # TODO should the default be an "all" singleton?
+ by: Optional[list[str]] = None
+
+ def __call__(
+ self, data: DataFrame, groupby: GroupBy, orient: str, scales: dict[str, Scale],
+ ) -> DataFrame:
+
+ grouping_vars = [v for v in groupby.order if v in data]
+ groups = groupby.agg(data, {"width": "max"})
+ if self.empty == "fill":
+ groups = groups.dropna()
+
+ def groupby_pos(s):
+ grouper = [groups[v] for v in [orient, "col", "row"] if v in data]
+ return s.groupby(grouper, sort=False, observed=True)
+
+ def scale_widths(w):
+ # TODO what value to fill missing widths??? Hard problem...
+ # TODO short circuit this if outer widths has no variance?
+ empty = 0 if self.empty == "fill" else w.mean()
+ filled = w.fillna(empty)
+ scale = filled.max()
+ norm = filled.sum()
+ if self.empty == "keep":
+ w = filled
+ return w / norm * scale
+
+ def widths_to_offsets(w):
+ return w.shift(1).fillna(0).cumsum() + (w - w.sum()) / 2
+
+ new_widths = groupby_pos(groups["width"]).transform(scale_widths)
+ offsets = groupby_pos(new_widths).transform(widths_to_offsets)
+
+ if self.gap:
+ new_widths *= 1 - self.gap
+
+ groups["_dodged"] = groups[orient] + offsets
+ groups["width"] = new_widths
+
+ out = (
+ data
+ .drop("width", axis=1)
+ .merge(groups, on=grouping_vars, how="left")
+ .drop(orient, axis=1)
+ .rename(columns={"_dodged": orient})
+ )
+
+ return out
+
+
+@dataclass
+class Stack(Move):
+ """
+ Displacement of overlapping bar or area marks along the value axis.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Stack.rst
+
+ """
+ # TODO center? (or should this be a different move, eg. Stream())
+
+ def _stack(self, df, orient):
+
+ # TODO should stack do something with ymin/ymax style marks?
+ # Should there be an upstream conversion to baseline/height parameterization?
+
+ if df["baseline"].nunique() > 1:
+ err = "Stack move cannot be used when baselines are already heterogeneous"
+ raise RuntimeError(err)
+
+ other = {"x": "y", "y": "x"}[orient]
+ stacked_lengths = (df[other] - df["baseline"]).dropna().cumsum()
+ offsets = stacked_lengths.shift(1).fillna(0)
+
+ df[other] = stacked_lengths
+ df["baseline"] = df["baseline"] + offsets
+
+ return df
+
+ def __call__(
+ self, data: DataFrame, groupby: GroupBy, orient: str, scales: dict[str, Scale],
+ ) -> DataFrame:
+
+ # TODO where to ensure that other semantic variables are sorted properly?
+ # TODO why are we not using the passed in groupby here?
+ groupers = ["col", "row", orient]
+ return GroupBy(groupers).apply(data, self._stack, orient)
+
+
+@dataclass
+class Shift(Move):
+ """
+ Displacement of all marks with the same magnitude / direction.
+
+ Parameters
+ ----------
+ x, y : float
+ Magnitude of shift, in data units, along each axis.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Shift.rst
+
+ """
+ x: float = 0
+ y: float = 0
+
+ def __call__(
+ self, data: DataFrame, groupby: GroupBy, orient: str, scales: dict[str, Scale],
+ ) -> DataFrame:
+
+ data = data.copy(deep=False)
+ data["x"] = data["x"] + self.x
+ data["y"] = data["y"] + self.y
+ return data
+
+
+@dataclass
+class Norm(Move):
+ """
+ Divisive scaling on the value axis after aggregating within groups.
+
+ Parameters
+ ----------
+ func : str or callable
+ Function called on each group to define the comparison value.
+ where : str
+ Query string defining the subset used to define the comparison values.
+ by : list of variables
+ Variables used to define aggregation groups.
+ percent : bool
+ If True, multiply the result by 100.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Norm.rst
+
+ """
+
+ func: Union[Callable, str] = "max"
+ where: Optional[str] = None
+ by: Optional[list[str]] = None
+ percent: bool = False
+
+ group_by_orient: ClassVar[bool] = False
+
+ def _norm(self, df, var):
+
+ if self.where is None:
+ denom_data = df[var]
+ else:
+ denom_data = df.query(self.where)[var]
+ df[var] = df[var] / denom_data.agg(self.func)
+
+ if self.percent:
+ df[var] = df[var] * 100
+
+ return df
+
+ def __call__(
+ self, data: DataFrame, groupby: GroupBy, orient: str, scales: dict[str, Scale],
+ ) -> DataFrame:
+
+ other = {"x": "y", "y": "x"}[orient]
+ return groupby.apply(data, self._norm, other)
+
+
+# TODO
+# @dataclass
+# class Ridge(Move):
+# ...
diff --git a/seaborn/_core/plot.py b/seaborn/_core/plot.py
new file mode 100644
index 0000000000000000000000000000000000000000..14348e357fed9242c314530c88daf360cd9a69bd
--- /dev/null
+++ b/seaborn/_core/plot.py
@@ -0,0 +1,1830 @@
+"""The classes for specifying and compiling a declarative visualization."""
+from __future__ import annotations
+
+import io
+import os
+import re
+import inspect
+import itertools
+import textwrap
+from contextlib import contextmanager
+from collections import abc
+from collections.abc import Callable, Generator
+from typing import Any, List, Literal, Optional, cast
+from xml.etree import ElementTree
+
+from cycler import cycler
+import pandas as pd
+from pandas import DataFrame, Series, Index
+import matplotlib as mpl
+from matplotlib.axes import Axes
+from matplotlib.artist import Artist
+from matplotlib.figure import Figure
+import numpy as np
+from PIL import Image
+
+from seaborn._marks.base import Mark
+from seaborn._stats.base import Stat
+from seaborn._core.data import PlotData
+from seaborn._core.moves import Move
+from seaborn._core.scales import Scale
+from seaborn._core.subplots import Subplots
+from seaborn._core.groupby import GroupBy
+from seaborn._core.properties import PROPERTIES, Property
+from seaborn._core.typing import (
+ DataSource,
+ VariableSpec,
+ VariableSpecList,
+ OrderSpec,
+ Default,
+)
+from seaborn._core.exceptions import PlotSpecError
+from seaborn._core.rules import categorical_order
+from seaborn._compat import get_layout_engine, set_layout_engine
+from seaborn.utils import _version_predates
+from seaborn.rcmod import axes_style, plotting_context
+from seaborn.palettes import color_palette
+
+from typing import TYPE_CHECKING, TypedDict
+if TYPE_CHECKING:
+ from matplotlib.figure import SubFigure
+
+
+default = Default()
+
+
+# ---- Definitions for internal specs ---------------------------------------------- #
+
+
+class Layer(TypedDict, total=False):
+
+ mark: Mark # TODO allow list?
+ stat: Stat | None # TODO allow list?
+ move: Move | list[Move] | None
+ data: PlotData
+ source: DataSource
+ vars: dict[str, VariableSpec]
+ orient: str
+ legend: bool
+ label: str | None
+
+
+class FacetSpec(TypedDict, total=False):
+
+ variables: dict[str, VariableSpec]
+ structure: dict[str, list[str]]
+ wrap: int | None
+
+
+class PairSpec(TypedDict, total=False):
+
+ variables: dict[str, VariableSpec]
+ structure: dict[str, list[str]]
+ cross: bool
+ wrap: int | None
+
+
+# --- Local helpers ---------------------------------------------------------------- #
+
+
+@contextmanager
+def theme_context(params: dict[str, Any]) -> Generator:
+ """Temporarily modify specifc matplotlib rcParams."""
+ orig_params = {k: mpl.rcParams[k] for k in params}
+ color_codes = "bgrmyck"
+ nice_colors = [*color_palette("deep6"), (.15, .15, .15)]
+ orig_colors = [mpl.colors.colorConverter.colors[x] for x in color_codes]
+ # TODO how to allow this to reflect the color cycle when relevant?
+ try:
+ mpl.rcParams.update(params)
+ for (code, color) in zip(color_codes, nice_colors):
+ mpl.colors.colorConverter.colors[code] = color
+ yield
+ finally:
+ mpl.rcParams.update(orig_params)
+ for (code, color) in zip(color_codes, orig_colors):
+ mpl.colors.colorConverter.colors[code] = color
+
+
+def build_plot_signature(cls):
+ """
+ Decorator function for giving Plot a useful signature.
+
+ Currently this mostly saves us some duplicated typing, but we would
+ like eventually to have a way of registering new semantic properties,
+ at which point dynamic signature generation would become more important.
+
+ """
+ sig = inspect.signature(cls)
+ params = [
+ inspect.Parameter("args", inspect.Parameter.VAR_POSITIONAL),
+ inspect.Parameter("data", inspect.Parameter.KEYWORD_ONLY, default=None)
+ ]
+ params.extend([
+ inspect.Parameter(name, inspect.Parameter.KEYWORD_ONLY, default=None)
+ for name in PROPERTIES
+ ])
+ new_sig = sig.replace(parameters=params)
+ cls.__signature__ = new_sig
+
+ known_properties = textwrap.fill(
+ ", ".join([f"|{p}|" for p in PROPERTIES]),
+ width=78, subsequent_indent=" " * 8,
+ )
+
+ if cls.__doc__ is not None: # support python -OO mode
+ cls.__doc__ = cls.__doc__.format(known_properties=known_properties)
+
+ return cls
+
+
+# ---- Plot configuration ---------------------------------------------------------- #
+
+
+class ThemeConfig(mpl.RcParams):
+ """
+ Configuration object for the Plot.theme, using matplotlib rc parameters.
+ """
+ THEME_GROUPS = [
+ "axes", "figure", "font", "grid", "hatch", "legend", "lines",
+ "mathtext", "markers", "patch", "savefig", "scatter",
+ "xaxis", "xtick", "yaxis", "ytick",
+ ]
+
+ def __init__(self):
+ super().__init__()
+ self.reset()
+
+ @property
+ def _default(self) -> dict[str, Any]:
+
+ return {
+ **self._filter_params(mpl.rcParamsDefault),
+ **axes_style("darkgrid"),
+ **plotting_context("notebook"),
+ "axes.prop_cycle": cycler("color", color_palette("deep")),
+ }
+
+ def reset(self) -> None:
+ """Update the theme dictionary with seaborn's default values."""
+ self.update(self._default)
+
+ def update(self, other: dict[str, Any] | None = None, /, **kwds):
+ """Update the theme with a dictionary or keyword arguments of rc parameters."""
+ if other is not None:
+ theme = self._filter_params(other)
+ else:
+ theme = {}
+ theme.update(kwds)
+ super().update(theme)
+
+ def _filter_params(self, params: dict[str, Any]) -> dict[str, Any]:
+ """Restruct to thematic rc params."""
+ return {
+ k: v for k, v in params.items()
+ if any(k.startswith(p) for p in self.THEME_GROUPS)
+ }
+
+ def _html_table(self, params: dict[str, Any]) -> list[str]:
+
+ lines = ["<table>"]
+ for k, v in params.items():
+ row = f"<tr><td>{k}:</td><td style='text-align:left'>{v!r}</td></tr>"
+ lines.append(row)
+ lines.append("</table>")
+ return lines
+
+ def _repr_html_(self) -> str:
+
+ repr = [
+ "<div style='height: 300px'>",
+ "<div style='border-style: inset; border-width: 2px'>",
+ *self._html_table(self),
+ "</div>",
+ "</div>",
+ ]
+ return "\n".join(repr)
+
+
+class DisplayConfig(TypedDict):
+ """Configuration for IPython's rich display hooks."""
+ format: Literal["png", "svg"]
+ scaling: float
+ hidpi: bool
+
+
+class PlotConfig:
+ """Configuration for default behavior / appearance of class:`Plot` instances."""
+ def __init__(self):
+
+ self._theme = ThemeConfig()
+ self._display = {"format": "png", "scaling": .85, "hidpi": True}
+
+ @property
+ def theme(self) -> dict[str, Any]:
+ """
+ Dictionary of base theme parameters for :class:`Plot`.
+
+ Keys and values correspond to matplotlib rc params, as documented here:
+ https://matplotlib.org/stable/tutorials/introductory/customizing.html
+
+ """
+ return self._theme
+
+ @property
+ def display(self) -> DisplayConfig:
+ """
+ Dictionary of parameters for rich display in Jupyter notebook.
+
+ Valid parameters:
+
+ - format ("png" or "svg"): Image format to produce
+ - scaling (float): Relative scaling of embedded image
+ - hidpi (bool): When True, double the DPI while preserving the size
+
+ """
+ return self._display
+
+
+# ---- The main interface for declarative plotting --------------------------------- #
+
+
+@build_plot_signature
+class Plot:
+ """
+ An interface for declaratively specifying statistical graphics.
+
+ Plots are constructed by initializing this class and adding one or more
+ layers, comprising a `Mark` and optional `Stat` or `Move`. Additionally,
+ faceting variables or variable pairings may be defined to divide the space
+ into multiple subplots. The mappings from data values to visual properties
+ can be parametrized using scales, although the plot will try to infer good
+ defaults when scales are not explicitly defined.
+
+ The constructor accepts a data source (a :class:`pandas.DataFrame` or
+ dictionary with columnar values) and variable assignments. Variables can be
+ passed as keys to the data source or directly as data vectors. If multiple
+ data-containing objects are provided, they will be index-aligned.
+
+ The data source and variables defined in the constructor will be used for
+ all layers in the plot, unless overridden or disabled when adding a layer.
+
+ The following variables can be defined in the constructor:
+ {known_properties}
+
+ The `data`, `x`, and `y` variables can be passed as positional arguments or
+ using keywords. Whether the first positional argument is interpreted as a
+ data source or `x` variable depends on its type.
+
+ The methods of this class return a copy of the instance; use chaining to
+ build up a plot through multiple calls. Methods can be called in any order.
+
+ Most methods only add information to the plot spec; no actual processing
+ happens until the plot is shown or saved. It is also possible to compile
+ the plot without rendering it to access the lower-level representation.
+
+ """
+ config = PlotConfig()
+
+ _data: PlotData
+ _layers: list[Layer]
+
+ _scales: dict[str, Scale]
+ _shares: dict[str, bool | str]
+ _limits: dict[str, tuple[Any, Any]]
+ _labels: dict[str, str | Callable[[str], str]]
+ _theme: dict[str, Any]
+
+ _facet_spec: FacetSpec
+ _pair_spec: PairSpec
+
+ _figure_spec: dict[str, Any]
+ _subplot_spec: dict[str, Any]
+ _layout_spec: dict[str, Any]
+
+ def __init__(
+ self,
+ *args: DataSource | VariableSpec,
+ data: DataSource = None,
+ **variables: VariableSpec,
+ ):
+
+ if args:
+ data, variables = self._resolve_positionals(args, data, variables)
+
+ unknown = [x for x in variables if x not in PROPERTIES]
+ if unknown:
+ err = f"Plot() got unexpected keyword argument(s): {', '.join(unknown)}"
+ raise TypeError(err)
+
+ self._data = PlotData(data, variables)
+
+ self._layers = []
+
+ self._scales = {}
+ self._shares = {}
+ self._limits = {}
+ self._labels = {}
+ self._theme = {}
+
+ self._facet_spec = {}
+ self._pair_spec = {}
+
+ self._figure_spec = {}
+ self._subplot_spec = {}
+ self._layout_spec = {}
+
+ self._target = None
+
+ def _resolve_positionals(
+ self,
+ args: tuple[DataSource | VariableSpec, ...],
+ data: DataSource,
+ variables: dict[str, VariableSpec],
+ ) -> tuple[DataSource, dict[str, VariableSpec]]:
+ """Handle positional arguments, which may contain data / x / y."""
+ if len(args) > 3:
+ err = "Plot() accepts no more than 3 positional arguments (data, x, y)."
+ raise TypeError(err)
+
+ if (
+ isinstance(args[0], (abc.Mapping, pd.DataFrame))
+ or hasattr(args[0], "__dataframe__")
+ ):
+ if data is not None:
+ raise TypeError("`data` given by both name and position.")
+ data, args = args[0], args[1:]
+
+ if len(args) == 2:
+ x, y = args
+ elif len(args) == 1:
+ x, y = *args, None
+ else:
+ x = y = None
+
+ for name, var in zip("yx", (y, x)):
+ if var is not None:
+ if name in variables:
+ raise TypeError(f"`{name}` given by both name and position.")
+ # Keep coordinates at the front of the variables dict
+ # Cast type because we know this isn't a DataSource at this point
+ variables = {name: cast(VariableSpec, var), **variables}
+
+ return data, variables
+
+ def __add__(self, other):
+
+ if isinstance(other, Mark) or isinstance(other, Stat):
+ raise TypeError("Sorry, this isn't ggplot! Perhaps try Plot.add?")
+
+ other_type = other.__class__.__name__
+ raise TypeError(f"Unsupported operand type(s) for +: 'Plot' and '{other_type}")
+
+ def _repr_png_(self) -> tuple[bytes, dict[str, float]] | None:
+
+ if Plot.config.display["format"] != "png":
+ return None
+ return self.plot()._repr_png_()
+
+ def _repr_svg_(self) -> str | None:
+
+ if Plot.config.display["format"] != "svg":
+ return None
+ return self.plot()._repr_svg_()
+
+ def _clone(self) -> Plot:
+ """Generate a new object with the same information as the current spec."""
+ new = Plot()
+
+ # TODO any way to enforce that data does not get mutated?
+ new._data = self._data
+
+ new._layers.extend(self._layers)
+
+ new._scales.update(self._scales)
+ new._shares.update(self._shares)
+ new._limits.update(self._limits)
+ new._labels.update(self._labels)
+ new._theme.update(self._theme)
+
+ new._facet_spec.update(self._facet_spec)
+ new._pair_spec.update(self._pair_spec)
+
+ new._figure_spec.update(self._figure_spec)
+ new._subplot_spec.update(self._subplot_spec)
+ new._layout_spec.update(self._layout_spec)
+
+ new._target = self._target
+
+ return new
+
+ def _theme_with_defaults(self) -> dict[str, Any]:
+
+ theme = self.config.theme.copy()
+ theme.update(self._theme)
+ return theme
+
+ @property
+ def _variables(self) -> list[str]:
+
+ variables = (
+ list(self._data.frame)
+ + list(self._pair_spec.get("variables", []))
+ + list(self._facet_spec.get("variables", []))
+ )
+ for layer in self._layers:
+ variables.extend(v for v in layer["vars"] if v not in variables)
+
+ # Coerce to str in return to appease mypy; we know these will only
+ # ever be strings but I don't think we can type a DataFrame that way yet
+ return [str(v) for v in variables]
+
+ def on(self, target: Axes | SubFigure | Figure) -> Plot:
+ """
+ Provide existing Matplotlib figure or axes for drawing the plot.
+
+ When using this method, you will also need to explicitly call a method that
+ triggers compilation, such as :meth:`Plot.show` or :meth:`Plot.save`. If you
+ want to postprocess using matplotlib, you'd need to call :meth:`Plot.plot`
+ first to compile the plot without rendering it.
+
+ Parameters
+ ----------
+ target : Axes, SubFigure, or Figure
+ Matplotlib object to use. Passing :class:`matplotlib.axes.Axes` will add
+ artists without otherwise modifying the figure. Otherwise, subplots will be
+ created within the space of the given :class:`matplotlib.figure.Figure` or
+ :class:`matplotlib.figure.SubFigure`.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Plot.on.rst
+
+ """
+ accepted_types: tuple # Allow tuple of various length
+ accepted_types = (
+ mpl.axes.Axes, mpl.figure.SubFigure, mpl.figure.Figure
+ )
+ accepted_types_str = (
+ f"{mpl.axes.Axes}, {mpl.figure.SubFigure}, or {mpl.figure.Figure}"
+ )
+
+ if not isinstance(target, accepted_types):
+ err = (
+ f"The `Plot.on` target must be an instance of {accepted_types_str}. "
+ f"You passed an instance of {target.__class__} instead."
+ )
+ raise TypeError(err)
+
+ new = self._clone()
+ new._target = target
+
+ return new
+
+ def add(
+ self,
+ mark: Mark,
+ *transforms: Stat | Move,
+ orient: str | None = None,
+ legend: bool = True,
+ label: str | None = None,
+ data: DataSource = None,
+ **variables: VariableSpec,
+ ) -> Plot:
+ """
+ Specify a layer of the visualization in terms of mark and data transform(s).
+
+ This is the main method for specifying how the data should be visualized.
+ It can be called multiple times with different arguments to define
+ a plot with multiple layers.
+
+ Parameters
+ ----------
+ mark : :class:`Mark`
+ The visual representation of the data to use in this layer.
+ transforms : :class:`Stat` or :class:`Move`
+ Objects representing transforms to be applied before plotting the data.
+ Currently, at most one :class:`Stat` can be used, and it
+ must be passed first. This constraint will be relaxed in the future.
+ orient : "x", "y", "v", or "h"
+ The orientation of the mark, which also affects how transforms are computed.
+ Typically corresponds to the axis that defines groups for aggregation.
+ The "v" (vertical) and "h" (horizontal) options are synonyms for "x" / "y",
+ but may be more intuitive with some marks. When not provided, an
+ orientation will be inferred from characteristics of the data and scales.
+ legend : bool
+ Option to suppress the mark/mappings for this layer from the legend.
+ label : str
+ A label to use for the layer in the legend, independent of any mappings.
+ data : DataFrame or dict
+ Data source to override the global source provided in the constructor.
+ variables : data vectors or identifiers
+ Additional layer-specific variables, including variables that will be
+ passed directly to the transforms without scaling.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Plot.add.rst
+
+ """
+ if not isinstance(mark, Mark):
+ msg = f"mark must be a Mark instance, not {type(mark)!r}."
+ raise TypeError(msg)
+
+ # TODO This API for transforms was a late decision, and previously Plot.add
+ # accepted 0 or 1 Stat instances and 0, 1, or a list of Move instances.
+ # It will take some work to refactor the internals so that Stat and Move are
+ # treated identically, and until then well need to "unpack" the transforms
+ # here and enforce limitations on the order / types.
+
+ stat: Optional[Stat]
+ move: Optional[List[Move]]
+ error = False
+ if not transforms:
+ stat, move = None, None
+ elif isinstance(transforms[0], Stat):
+ stat = transforms[0]
+ move = [m for m in transforms[1:] if isinstance(m, Move)]
+ error = len(move) != len(transforms) - 1
+ else:
+ stat = None
+ move = [m for m in transforms if isinstance(m, Move)]
+ error = len(move) != len(transforms)
+
+ if error:
+ msg = " ".join([
+ "Transforms must have at most one Stat type (in the first position),",
+ "and all others must be a Move type. Given transform type(s):",
+ ", ".join(str(type(t).__name__) for t in transforms) + "."
+ ])
+ raise TypeError(msg)
+
+ new = self._clone()
+ new._layers.append({
+ "mark": mark,
+ "stat": stat,
+ "move": move,
+ # TODO it doesn't work to supply scalars to variables, but it should
+ "vars": variables,
+ "source": data,
+ "legend": legend,
+ "label": label,
+ "orient": {"v": "x", "h": "y"}.get(orient, orient), # type: ignore
+ })
+
+ return new
+
+ def pair(
+ self,
+ x: VariableSpecList = None,
+ y: VariableSpecList = None,
+ wrap: int | None = None,
+ cross: bool = True,
+ ) -> Plot:
+ """
+ Produce subplots by pairing multiple `x` and/or `y` variables.
+
+ Parameters
+ ----------
+ x, y : sequence(s) of data vectors or identifiers
+ Variables that will define the grid of subplots.
+ wrap : int
+ When using only `x` or `y`, "wrap" subplots across a two-dimensional grid
+ with this many columns (when using `x`) or rows (when using `y`).
+ cross : bool
+ When False, zip the `x` and `y` lists such that the first subplot gets the
+ first pair, the second gets the second pair, etc. Otherwise, create a
+ two-dimensional grid from the cartesian product of the lists.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Plot.pair.rst
+
+ """
+ # TODO Add transpose= arg, which would then draw pair(y=[...]) across rows
+ # This may also be possible by setting `wrap=1`, but is that too unobvious?
+ # TODO PairGrid features not currently implemented: diagonals, corner
+
+ pair_spec: PairSpec = {}
+
+ axes = {"x": [] if x is None else x, "y": [] if y is None else y}
+ for axis, arg in axes.items():
+ if isinstance(arg, (str, int)):
+ err = f"You must pass a sequence of variable keys to `{axis}`"
+ raise TypeError(err)
+
+ pair_spec["variables"] = {}
+ pair_spec["structure"] = {}
+
+ for axis in "xy":
+ keys = []
+ for i, col in enumerate(axes[axis]):
+ key = f"{axis}{i}"
+ keys.append(key)
+ pair_spec["variables"][key] = col
+
+ if keys:
+ pair_spec["structure"][axis] = keys
+
+ if not cross and len(axes["x"]) != len(axes["y"]):
+ err = "Lengths of the `x` and `y` lists must match with cross=False"
+ raise ValueError(err)
+
+ pair_spec["cross"] = cross
+ pair_spec["wrap"] = wrap
+
+ new = self._clone()
+ new._pair_spec.update(pair_spec)
+ return new
+
+ def facet(
+ self,
+ col: VariableSpec = None,
+ row: VariableSpec = None,
+ order: OrderSpec | dict[str, OrderSpec] = None,
+ wrap: int | None = None,
+ ) -> Plot:
+ """
+ Produce subplots with conditional subsets of the data.
+
+ Parameters
+ ----------
+ col, row : data vectors or identifiers
+ Variables used to define subsets along the columns and/or rows of the grid.
+ Can be references to the global data source passed in the constructor.
+ order : list of strings, or dict with dimensional keys
+ Define the order of the faceting variables.
+ wrap : int
+ When using only `col` or `row`, wrap subplots across a two-dimensional
+ grid with this many subplots on the faceting dimension.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Plot.facet.rst
+
+ """
+ variables: dict[str, VariableSpec] = {}
+ if col is not None:
+ variables["col"] = col
+ if row is not None:
+ variables["row"] = row
+
+ structure = {}
+ if isinstance(order, dict):
+ for dim in ["col", "row"]:
+ dim_order = order.get(dim)
+ if dim_order is not None:
+ structure[dim] = list(dim_order)
+ elif order is not None:
+ if col is not None and row is not None:
+ err = " ".join([
+ "When faceting on both col= and row=, passing `order` as a list"
+ "is ambiguous. Use a dict with 'col' and/or 'row' keys instead."
+ ])
+ raise RuntimeError(err)
+ elif col is not None:
+ structure["col"] = list(order)
+ elif row is not None:
+ structure["row"] = list(order)
+
+ spec: FacetSpec = {
+ "variables": variables,
+ "structure": structure,
+ "wrap": wrap,
+ }
+
+ new = self._clone()
+ new._facet_spec.update(spec)
+
+ return new
+
+ # TODO def twin()?
+
+ def scale(self, **scales: Scale) -> Plot:
+ """
+ Specify mappings from data units to visual properties.
+
+ Keywords correspond to variables defined in the plot, including coordinate
+ variables (`x`, `y`) and semantic variables (`color`, `pointsize`, etc.).
+
+ A number of "magic" arguments are accepted, including:
+ - The name of a transform (e.g., `"log"`, `"sqrt"`)
+ - The name of a palette (e.g., `"viridis"`, `"muted"`)
+ - A tuple of values, defining the output range (e.g. `(1, 5)`)
+ - A dict, implying a :class:`Nominal` scale (e.g. `{"a": .2, "b": .5}`)
+ - A list of values, implying a :class:`Nominal` scale (e.g. `["b", "r"]`)
+
+ For more explicit control, pass a scale spec object such as :class:`Continuous`
+ or :class:`Nominal`. Or pass `None` to use an "identity" scale, which treats
+ data values as literally encoding visual properties.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Plot.scale.rst
+
+ """
+ new = self._clone()
+ new._scales.update(scales)
+ return new
+
+ def share(self, **shares: bool | str) -> Plot:
+ """
+ Control sharing of axis limits and ticks across subplots.
+
+ Keywords correspond to variables defined in the plot, and values can be
+ boolean (to share across all subplots), or one of "row" or "col" (to share
+ more selectively across one dimension of a grid).
+
+ Behavior for non-coordinate variables is currently undefined.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Plot.share.rst
+
+ """
+ new = self._clone()
+ new._shares.update(shares)
+ return new
+
+ def limit(self, **limits: tuple[Any, Any]) -> Plot:
+ """
+ Control the range of visible data.
+
+ Keywords correspond to variables defined in the plot, and values are a
+ `(min, max)` tuple (where either can be `None` to leave unset).
+
+ Limits apply only to the axis; data outside the visible range are
+ still used for any stat transforms and added to the plot.
+
+ Behavior for non-coordinate variables is currently undefined.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Plot.limit.rst
+
+ """
+ new = self._clone()
+ new._limits.update(limits)
+ return new
+
+ def label(
+ self, *,
+ title: str | None = None,
+ legend: str | None = None,
+ **variables: str | Callable[[str], str]
+ ) -> Plot:
+ """
+ Control the labels and titles for axes, legends, and subplots.
+
+ Additional keywords correspond to variables defined in the plot.
+ Values can be one of the following types:
+
+ - string (used literally; pass "" to clear the default label)
+ - function (called on the default label)
+
+ For coordinate variables, the value sets the axis label.
+ For semantic variables, the value sets the legend title.
+ For faceting variables, `title=` modifies the subplot-specific label,
+ while `col=` and/or `row=` add a label for the faceting variable.
+
+ When using a single subplot, `title=` sets its title.
+
+ The `legend=` parameter sets the title for the "layer" legend
+ (i.e., when using `label` in :meth:`Plot.add`).
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Plot.label.rst
+
+
+ """
+ new = self._clone()
+ if title is not None:
+ new._labels["title"] = title
+ if legend is not None:
+ new._labels["legend"] = legend
+ new._labels.update(variables)
+ return new
+
+ def layout(
+ self,
+ *,
+ size: tuple[float, float] | Default = default,
+ engine: str | None | Default = default,
+ extent: tuple[float, float, float, float] | Default = default,
+ ) -> Plot:
+ """
+ Control the figure size and layout.
+
+ .. note::
+
+ Default figure sizes and the API for specifying the figure size are subject
+ to change in future "experimental" releases of the objects API. The default
+ layout engine may also change.
+
+ Parameters
+ ----------
+ size : (width, height)
+ Size of the resulting figure, in inches. Size is inclusive of legend when
+ using pyplot, but not otherwise.
+ engine : {{"tight", "constrained", "none"}}
+ Name of method for automatically adjusting the layout to remove overlap.
+ The default depends on whether :meth:`Plot.on` is used.
+ extent : (left, bottom, right, top)
+ Boundaries of the plot layout, in fractions of the figure size. Takes
+ effect through the layout engine; exact results will vary across engines.
+ Note: the extent includes axis decorations when using a layout engine,
+ but it is exclusive of them when `engine="none"`.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Plot.layout.rst
+
+ """
+ # TODO add an "auto" mode for figsize that roughly scales with the rcParams
+ # figsize (so that works), but expands to prevent subplots from being squished
+ # Also should we have height=, aspect=, exclusive with figsize? Or working
+ # with figsize when only one is defined?
+
+ new = self._clone()
+
+ if size is not default:
+ new._figure_spec["figsize"] = size
+ if engine is not default:
+ new._layout_spec["engine"] = engine
+ if extent is not default:
+ new._layout_spec["extent"] = extent
+
+ return new
+
+ # TODO def legend (ugh)
+
+ def theme(self, config: dict[str, Any], /) -> Plot:
+ """
+ Control the appearance of elements in the plot.
+
+ .. note::
+
+ The API for customizing plot appearance is not yet finalized.
+ Currently, the only valid argument is a dict of matplotlib rc parameters.
+ (This dict must be passed as a positional argument.)
+
+ It is likely that this method will be enhanced in future releases.
+
+ Matplotlib rc parameters are documented on the following page:
+ https://matplotlib.org/stable/tutorials/introductory/customizing.html
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Plot.theme.rst
+
+ """
+ new = self._clone()
+
+ rc = mpl.RcParams(config)
+ new._theme.update(rc)
+
+ return new
+
+ def save(self, loc, **kwargs) -> Plot:
+ """
+ Compile the plot and write it to a buffer or file on disk.
+
+ Parameters
+ ----------
+ loc : str, path, or buffer
+ Location on disk to save the figure, or a buffer to write into.
+ kwargs
+ Other keyword arguments are passed through to
+ :meth:`matplotlib.figure.Figure.savefig`.
+
+ """
+ # TODO expose important keyword arguments in our signature?
+ with theme_context(self._theme_with_defaults()):
+ self._plot().save(loc, **kwargs)
+ return self
+
+ def show(self, **kwargs) -> None:
+ """
+ Compile the plot and display it by hooking into pyplot.
+
+ Calling this method is not necessary to render a plot in notebook context,
+ but it may be in other environments (e.g., in a terminal). After compiling the
+ plot, it calls :func:`matplotlib.pyplot.show` (passing any keyword parameters).
+
+ Unlike other :class:`Plot` methods, there is no return value. This should be
+ the last method you call when specifying a plot.
+
+ """
+ # TODO make pyplot configurable at the class level, and when not using,
+ # import IPython.display and call on self to populate cell output?
+
+ # Keep an eye on whether matplotlib implements "attaching" an existing
+ # figure to pyplot: https://github.com/matplotlib/matplotlib/pull/14024
+
+ self.plot(pyplot=True).show(**kwargs)
+
+ def plot(self, pyplot: bool = False) -> Plotter:
+ """
+ Compile the plot spec and return the Plotter object.
+ """
+ with theme_context(self._theme_with_defaults()):
+ return self._plot(pyplot)
+
+ def _plot(self, pyplot: bool = False) -> Plotter:
+
+ # TODO if we have _target object, pyplot should be determined by whether it
+ # is hooked into the pyplot state machine (how do we check?)
+
+ plotter = Plotter(pyplot=pyplot, theme=self._theme_with_defaults())
+
+ # Process the variable assignments and initialize the figure
+ common, layers = plotter._extract_data(self)
+ plotter._setup_figure(self, common, layers)
+
+ # Process the scale spec for coordinate variables and transform their data
+ coord_vars = [v for v in self._variables if re.match(r"^x|y", v)]
+ plotter._setup_scales(self, common, layers, coord_vars)
+
+ # Apply statistical transform(s)
+ plotter._compute_stats(self, layers)
+
+ # Process scale spec for semantic variables and coordinates computed by stat
+ plotter._setup_scales(self, common, layers)
+
+ # TODO Remove these after updating other methods
+ # ---- Maybe have debug= param that attaches these when True?
+ plotter._data = common
+ plotter._layers = layers
+
+ # Process the data for each layer and add matplotlib artists
+ for layer in layers:
+ plotter._plot_layer(self, layer)
+
+ # Add various figure decorations
+ plotter._make_legend(self)
+ plotter._finalize_figure(self)
+
+ return plotter
+
+
+# ---- The plot compilation engine ---------------------------------------------- #
+
+
+class Plotter:
+ """
+ Engine for compiling a :class:`Plot` spec into a Matplotlib figure.
+
+ This class is not intended to be instantiated directly by users.
+
+ """
+ # TODO decide if we ever want these (Plot.plot(debug=True))?
+ _data: PlotData
+ _layers: list[Layer]
+ _figure: Figure
+
+ def __init__(self, pyplot: bool, theme: dict[str, Any]):
+
+ self._pyplot = pyplot
+ self._theme = theme
+ self._legend_contents: list[tuple[
+ tuple[str, str | int], list[Artist], list[str],
+ ]] = []
+ self._scales: dict[str, Scale] = {}
+
+ def save(self, loc, **kwargs) -> Plotter: # TODO type args
+ kwargs.setdefault("dpi", 96)
+ try:
+ loc = os.path.expanduser(loc)
+ except TypeError:
+ # loc may be a buffer in which case that would not work
+ pass
+ self._figure.savefig(loc, **kwargs)
+ return self
+
+ def show(self, **kwargs) -> None:
+ """
+ Display the plot by hooking into pyplot.
+
+ This method calls :func:`matplotlib.pyplot.show` with any keyword parameters.
+
+ """
+ # TODO if we did not create the Plotter with pyplot, is it possible to do this?
+ # If not we should clearly raise.
+ import matplotlib.pyplot as plt
+ with theme_context(self._theme):
+ plt.show(**kwargs)
+
+ # TODO API for accessing the underlying matplotlib objects
+ # TODO what else is useful in the public API for this class?
+
+ def _repr_png_(self) -> tuple[bytes, dict[str, float]] | None:
+
+ # TODO use matplotlib backend directly instead of going through savefig?
+
+ # TODO perhaps have self.show() flip a switch to disable this, so that
+ # user does not end up with two versions of the figure in the output
+
+ # TODO use bbox_inches="tight" like the inline backend?
+ # pro: better results, con: (sometimes) confusing results
+ # Better solution would be to default (with option to change)
+ # to using constrained/tight layout.
+
+ if Plot.config.display["format"] != "png":
+ return None
+
+ buffer = io.BytesIO()
+
+ factor = 2 if Plot.config.display["hidpi"] else 1
+ scaling = Plot.config.display["scaling"] / factor
+ dpi = 96 * factor # TODO put dpi in Plot.config?
+
+ with theme_context(self._theme): # TODO _theme_with_defaults?
+ self._figure.savefig(buffer, dpi=dpi, format="png", bbox_inches="tight")
+ data = buffer.getvalue()
+
+ w, h = Image.open(buffer).size
+ metadata = {"width": w * scaling, "height": h * scaling}
+ return data, metadata
+
+ def _repr_svg_(self) -> str | None:
+
+ if Plot.config.display["format"] != "svg":
+ return None
+
+ # TODO DPI for rasterized artists?
+
+ scaling = Plot.config.display["scaling"]
+
+ buffer = io.StringIO()
+ with theme_context(self._theme): # TODO _theme_with_defaults?
+ self._figure.savefig(buffer, format="svg", bbox_inches="tight")
+
+ root = ElementTree.fromstring(buffer.getvalue())
+ w = scaling * float(root.attrib["width"][:-2])
+ h = scaling * float(root.attrib["height"][:-2])
+ root.attrib.update(width=f"{w}pt", height=f"{h}pt", viewbox=f"0 0 {w} {h}")
+ ElementTree.ElementTree(root).write(out := io.BytesIO())
+
+ return out.getvalue().decode()
+
+ def _extract_data(self, p: Plot) -> tuple[PlotData, list[Layer]]:
+
+ common_data = (
+ p._data
+ .join(None, p._facet_spec.get("variables"))
+ .join(None, p._pair_spec.get("variables"))
+ )
+
+ layers: list[Layer] = []
+ for layer in p._layers:
+ spec = layer.copy()
+ spec["data"] = common_data.join(layer.get("source"), layer.get("vars"))
+ layers.append(spec)
+
+ return common_data, layers
+
+ def _resolve_label(self, p: Plot, var: str, auto_label: str | None) -> str:
+
+ if re.match(r"[xy]\d+", var):
+ key = var if var in p._labels else var[0]
+ else:
+ key = var
+
+ label: str
+ if key in p._labels:
+ manual_label = p._labels[key]
+ if callable(manual_label) and auto_label is not None:
+ label = manual_label(auto_label)
+ else:
+ label = cast(str, manual_label)
+ elif auto_label is None:
+ label = ""
+ else:
+ label = auto_label
+ return label
+
+ def _setup_figure(self, p: Plot, common: PlotData, layers: list[Layer]) -> None:
+
+ # --- Parsing the faceting/pairing parameterization to specify figure grid
+
+ subplot_spec = p._subplot_spec.copy()
+ facet_spec = p._facet_spec.copy()
+ pair_spec = p._pair_spec.copy()
+
+ for axis in "xy":
+ if axis in p._shares:
+ subplot_spec[f"share{axis}"] = p._shares[axis]
+
+ for dim in ["col", "row"]:
+ if dim in common.frame and dim not in facet_spec["structure"]:
+ order = categorical_order(common.frame[dim])
+ facet_spec["structure"][dim] = order
+
+ self._subplots = subplots = Subplots(subplot_spec, facet_spec, pair_spec)
+
+ # --- Figure initialization
+ self._figure = subplots.init_figure(
+ pair_spec, self._pyplot, p._figure_spec, p._target,
+ )
+
+ # --- Figure annotation
+ for sub in subplots:
+ ax = sub["ax"]
+ for axis in "xy":
+ axis_key = sub[axis]
+
+ # ~~ Axis labels
+
+ # TODO Should we make it possible to use only one x/y label for
+ # all rows/columns in a faceted plot? Maybe using sub{axis}label,
+ # although the alignments of the labels from that method leaves
+ # something to be desired (in terms of how it defines 'centered').
+ names = [
+ common.names.get(axis_key),
+ *(layer["data"].names.get(axis_key) for layer in layers)
+ ]
+ auto_label = next((name for name in names if name is not None), None)
+ label = self._resolve_label(p, axis_key, auto_label)
+ ax.set(**{f"{axis}label": label})
+
+ # ~~ Decoration visibility
+
+ # TODO there should be some override (in Plot.layout?) so that
+ # axis / tick labels can be shown on interior shared axes if desired
+
+ axis_obj = getattr(ax, f"{axis}axis")
+ visible_side = {"x": "bottom", "y": "left"}.get(axis)
+ show_axis_label = (
+ sub[visible_side]
+ or not p._pair_spec.get("cross", True)
+ or (
+ axis in p._pair_spec.get("structure", {})
+ and bool(p._pair_spec.get("wrap"))
+ )
+ )
+ axis_obj.get_label().set_visible(show_axis_label)
+
+ show_tick_labels = (
+ show_axis_label
+ or subplot_spec.get(f"share{axis}") not in (
+ True, "all", {"x": "col", "y": "row"}[axis]
+ )
+ )
+ for group in ("major", "minor"):
+ side = {"x": "bottom", "y": "left"}[axis]
+ axis_obj.set_tick_params(**{f"label{side}": show_tick_labels})
+ for t in getattr(axis_obj, f"get_{group}ticklabels")():
+ t.set_visible(show_tick_labels)
+
+ # TODO we want right-side titles for row facets in most cases?
+ # Let's have what we currently call "margin titles" but properly using the
+ # ax.set_title interface (see my gist)
+ title_parts = []
+ for dim in ["col", "row"]:
+ if sub[dim] is not None:
+ val = self._resolve_label(p, "title", f"{sub[dim]}")
+ if dim in p._labels:
+ key = self._resolve_label(p, dim, common.names.get(dim))
+ val = f"{key} {val}"
+ title_parts.append(val)
+
+ has_col = sub["col"] is not None
+ has_row = sub["row"] is not None
+ show_title = (
+ has_col and has_row
+ or (has_col or has_row) and p._facet_spec.get("wrap")
+ or (has_col and sub["top"])
+ # TODO or has_row and sub["right"] and <right titles>
+ or has_row # TODO and not <right titles>
+ )
+ if title_parts:
+ title = " | ".join(title_parts)
+ title_text = ax.set_title(title)
+ title_text.set_visible(show_title)
+ elif not (has_col or has_row):
+ title = self._resolve_label(p, "title", None)
+ title_text = ax.set_title(title)
+
+ def _compute_stats(self, spec: Plot, layers: list[Layer]) -> None:
+
+ grouping_vars = [v for v in PROPERTIES if v not in "xy"]
+ grouping_vars += ["col", "row", "group"]
+
+ pair_vars = spec._pair_spec.get("structure", {})
+
+ for layer in layers:
+
+ data = layer["data"]
+ mark = layer["mark"]
+ stat = layer["stat"]
+
+ if stat is None:
+ continue
+
+ iter_axes = itertools.product(*[
+ pair_vars.get(axis, [axis]) for axis in "xy"
+ ])
+
+ old = data.frame
+
+ if pair_vars:
+ data.frames = {}
+ data.frame = data.frame.iloc[:0] # TODO to simplify typing
+
+ for coord_vars in iter_axes:
+
+ pairings = "xy", coord_vars
+
+ df = old.copy()
+ scales = self._scales.copy()
+
+ for axis, var in zip(*pairings):
+ if axis != var:
+ df = df.rename(columns={var: axis})
+ drop_cols = [x for x in df if re.match(rf"{axis}\d+", str(x))]
+ df = df.drop(drop_cols, axis=1)
+ scales[axis] = scales[var]
+
+ orient = layer["orient"] or mark._infer_orient(scales)
+
+ if stat.group_by_orient:
+ grouper = [orient, *grouping_vars]
+ else:
+ grouper = grouping_vars
+ groupby = GroupBy(grouper)
+ res = stat(df, groupby, orient, scales)
+
+ if pair_vars:
+ data.frames[coord_vars] = res
+ else:
+ data.frame = res
+
+ def _get_scale(
+ self, p: Plot, var: str, prop: Property, values: Series
+ ) -> Scale:
+
+ if re.match(r"[xy]\d+", var):
+ key = var if var in p._scales else var[0]
+ else:
+ key = var
+
+ if key in p._scales:
+ arg = p._scales[key]
+ if arg is None or isinstance(arg, Scale):
+ scale = arg
+ else:
+ scale = prop.infer_scale(arg, values)
+ else:
+ scale = prop.default_scale(values)
+
+ return scale
+
+ def _get_subplot_data(self, df, var, view, share_state):
+
+ if share_state in [True, "all"]:
+ # The all-shared case is easiest, every subplot sees all the data
+ seed_values = df[var]
+ else:
+ # Otherwise, we need to setup separate scales for different subplots
+ if share_state in [False, "none"]:
+ # Fully independent axes are also easy: use each subplot's data
+ idx = self._get_subplot_index(df, view)
+ elif share_state in df:
+ # Sharing within row/col is more complicated
+ use_rows = df[share_state] == view[share_state]
+ idx = df.index[use_rows]
+ else:
+ # This configuration doesn't make much sense, but it's fine
+ idx = df.index
+
+ seed_values = df.loc[idx, var]
+
+ return seed_values
+
+ def _setup_scales(
+ self,
+ p: Plot,
+ common: PlotData,
+ layers: list[Layer],
+ variables: list[str] | None = None,
+ ) -> None:
+
+ if variables is None:
+ # Add variables that have data but not a scale, which happens
+ # because this method can be called multiple time, to handle
+ # variables added during the Stat transform.
+ variables = []
+ for layer in layers:
+ variables.extend(layer["data"].frame.columns)
+ for df in layer["data"].frames.values():
+ variables.extend(str(v) for v in df if v not in variables)
+ variables = [v for v in variables if v not in self._scales]
+
+ for var in variables:
+
+ # Determine whether this is a coordinate variable
+ # (i.e., x/y, paired x/y, or derivative such as xmax)
+ m = re.match(r"^(?P<coord>(?P<axis>x|y)\d*).*", var)
+ if m is None:
+ coord = axis = None
+ else:
+ coord = m["coord"]
+ axis = m["axis"]
+
+ # Get keys that handle things like x0, xmax, properly where relevant
+ prop_key = var if axis is None else axis
+ scale_key = var if coord is None else coord
+
+ if prop_key not in PROPERTIES:
+ continue
+
+ # Concatenate layers, using only the relevant coordinate and faceting vars,
+ # This is unnecessarily wasteful, as layer data will often be redundant.
+ # But figuring out the minimal amount we need is more complicated.
+ cols = [var, "col", "row"]
+ parts = [common.frame.filter(cols)]
+ for layer in layers:
+ parts.append(layer["data"].frame.filter(cols))
+ for df in layer["data"].frames.values():
+ parts.append(df.filter(cols))
+ var_df = pd.concat(parts, ignore_index=True)
+
+ prop = PROPERTIES[prop_key]
+ scale = self._get_scale(p, scale_key, prop, var_df[var])
+
+ if scale_key not in p._variables:
+ # TODO this implies that the variable was added by the stat
+ # It allows downstream orientation inference to work properly.
+ # But it feels rather hacky, so ideally revisit.
+ scale._priority = 0 # type: ignore
+
+ if axis is None:
+ # We could think about having a broader concept of (un)shared properties
+ # In general, not something you want to do (different scales in facets)
+ # But could make sense e.g. with paired plots. Build later.
+ share_state = None
+ subplots = []
+ else:
+ share_state = self._subplots.subplot_spec[f"share{axis}"]
+ subplots = [view for view in self._subplots if view[axis] == coord]
+
+ if scale is None:
+ self._scales[var] = Scale._identity()
+ else:
+ try:
+ self._scales[var] = scale._setup(var_df[var], prop)
+ except Exception as err:
+ raise PlotSpecError._during("Scale setup", var) from err
+
+ if axis is None or (var != coord and coord in p._variables):
+ # Everything below here applies only to coordinate variables
+ continue
+
+ # Set up an empty series to receive the transformed values.
+ # We need this to handle piecemeal transforms of categories -> floats.
+ transformed_data = []
+ for layer in layers:
+ index = layer["data"].frame.index
+ empty_series = pd.Series(dtype=float, index=index, name=var)
+ transformed_data.append(empty_series)
+
+ for view in subplots:
+
+ axis_obj = getattr(view["ax"], f"{axis}axis")
+ seed_values = self._get_subplot_data(var_df, var, view, share_state)
+ view_scale = scale._setup(seed_values, prop, axis=axis_obj)
+ view["ax"].set(**{f"{axis}scale": view_scale._matplotlib_scale})
+
+ for layer, new_series in zip(layers, transformed_data):
+ layer_df = layer["data"].frame
+ if var not in layer_df:
+ continue
+
+ idx = self._get_subplot_index(layer_df, view)
+ try:
+ new_series.loc[idx] = view_scale(layer_df.loc[idx, var])
+ except Exception as err:
+ spec_error = PlotSpecError._during("Scaling operation", var)
+ raise spec_error from err
+
+ # Now the transformed data series are complete, update the layer data
+ for layer, new_series in zip(layers, transformed_data):
+ layer_df = layer["data"].frame
+ if var in layer_df:
+ layer_df[var] = pd.to_numeric(new_series)
+
+ def _plot_layer(self, p: Plot, layer: Layer) -> None:
+
+ data = layer["data"]
+ mark = layer["mark"]
+ move = layer["move"]
+
+ default_grouping_vars = ["col", "row", "group"] # TODO where best to define?
+ grouping_properties = [v for v in PROPERTIES if v[0] not in "xy"]
+
+ pair_variables = p._pair_spec.get("structure", {})
+
+ for subplots, df, scales in self._generate_pairings(data, pair_variables):
+
+ orient = layer["orient"] or mark._infer_orient(scales)
+
+ def get_order(var):
+ # Ignore order for x/y: they have been scaled to numeric indices,
+ # so any original order is no longer valid. Default ordering rules
+ # sorted unique numbers will correctly reconstruct intended order
+ # TODO This is tricky, make sure we add some tests for this
+ if var not in "xy" and var in scales:
+ return getattr(scales[var], "order", None)
+
+ if orient in df:
+ width = pd.Series(index=df.index, dtype=float)
+ for view in subplots:
+ view_idx = self._get_subplot_data(
+ df, orient, view, p._shares.get(orient)
+ ).index
+ view_df = df.loc[view_idx]
+ if "width" in mark._mappable_props:
+ view_width = mark._resolve(view_df, "width", None)
+ elif "width" in df:
+ view_width = view_df["width"]
+ else:
+ view_width = 0.8 # TODO what default?
+ spacing = scales[orient]._spacing(view_df.loc[view_idx, orient])
+ width.loc[view_idx] = view_width * spacing
+ df["width"] = width
+
+ if "baseline" in mark._mappable_props:
+ # TODO what marks should have this?
+ # If we can set baseline with, e.g., Bar(), then the
+ # "other" (e.g. y for x oriented bars) parameterization
+ # is somewhat ambiguous.
+ baseline = mark._resolve(df, "baseline", None)
+ else:
+ # TODO unlike width, we might not want to add baseline to data
+ # if the mark doesn't use it. Practically, there is a concern about
+ # Mark abstraction like Area / Ribbon
+ baseline = 0 if "baseline" not in df else df["baseline"]
+ df["baseline"] = baseline
+
+ if move is not None:
+ moves = move if isinstance(move, list) else [move]
+ for move_step in moves:
+ move_by = getattr(move_step, "by", None)
+ if move_by is None:
+ move_by = grouping_properties
+ move_groupers = [*move_by, *default_grouping_vars]
+ if move_step.group_by_orient:
+ move_groupers.insert(0, orient)
+ order = {var: get_order(var) for var in move_groupers}
+ groupby = GroupBy(order)
+ df = move_step(df, groupby, orient, scales)
+
+ df = self._unscale_coords(subplots, df, orient)
+
+ grouping_vars = mark._grouping_props + default_grouping_vars
+ split_generator = self._setup_split_generator(grouping_vars, df, subplots)
+
+ mark._plot(split_generator, scales, orient)
+
+ # TODO is this the right place for this?
+ for view in self._subplots:
+ view["ax"].autoscale_view()
+
+ if layer["legend"]:
+ self._update_legend_contents(p, mark, data, scales, layer["label"])
+
+ def _unscale_coords(
+ self, subplots: list[dict], df: DataFrame, orient: str,
+ ) -> DataFrame:
+ # TODO do we still have numbers in the variable name at this point?
+ coord_cols = [c for c in df if re.match(r"^[xy]\D*$", str(c))]
+ out_df = (
+ df
+ .drop(coord_cols, axis=1)
+ .reindex(df.columns, axis=1) # So unscaled columns retain their place
+ .copy(deep=False)
+ )
+
+ for view in subplots:
+ view_df = self._filter_subplot_data(df, view)
+ axes_df = view_df[coord_cols]
+ for var, values in axes_df.items():
+
+ axis = getattr(view["ax"], f"{str(var)[0]}axis")
+ # TODO see https://github.com/matplotlib/matplotlib/issues/22713
+ transform = axis.get_transform().inverted().transform
+ inverted = transform(values)
+ out_df.loc[values.index, str(var)] = inverted
+
+ return out_df
+
+ def _generate_pairings(
+ self, data: PlotData, pair_variables: dict,
+ ) -> Generator[
+ tuple[list[dict], DataFrame, dict[str, Scale]], None, None
+ ]:
+ # TODO retype return with subplot_spec or similar
+
+ iter_axes = itertools.product(*[
+ pair_variables.get(axis, [axis]) for axis in "xy"
+ ])
+
+ for x, y in iter_axes:
+
+ subplots = []
+ for view in self._subplots:
+ if (view["x"] == x) and (view["y"] == y):
+ subplots.append(view)
+
+ if data.frame.empty and data.frames:
+ out_df = data.frames[(x, y)].copy()
+ elif not pair_variables:
+ out_df = data.frame.copy()
+ else:
+ if data.frame.empty and data.frames:
+ out_df = data.frames[(x, y)].copy()
+ else:
+ out_df = data.frame.copy()
+
+ scales = self._scales.copy()
+ if x in out_df:
+ scales["x"] = self._scales[x]
+ if y in out_df:
+ scales["y"] = self._scales[y]
+
+ for axis, var in zip("xy", (x, y)):
+ if axis != var:
+ out_df = out_df.rename(columns={var: axis})
+ cols = [col for col in out_df if re.match(rf"{axis}\d+", str(col))]
+ out_df = out_df.drop(cols, axis=1)
+
+ yield subplots, out_df, scales
+
+ def _get_subplot_index(self, df: DataFrame, subplot: dict) -> Index:
+
+ dims = df.columns.intersection(["col", "row"])
+ if dims.empty:
+ return df.index
+
+ keep_rows = pd.Series(True, df.index, dtype=bool)
+ for dim in dims:
+ keep_rows &= df[dim] == subplot[dim]
+ return df.index[keep_rows]
+
+ def _filter_subplot_data(self, df: DataFrame, subplot: dict) -> DataFrame:
+ # TODO note redundancies with preceding function ... needs refactoring
+ dims = df.columns.intersection(["col", "row"])
+ if dims.empty:
+ return df
+
+ keep_rows = pd.Series(True, df.index, dtype=bool)
+ for dim in dims:
+ keep_rows &= df[dim] == subplot[dim]
+ return df[keep_rows]
+
+ def _setup_split_generator(
+ self, grouping_vars: list[str], df: DataFrame, subplots: list[dict[str, Any]],
+ ) -> Callable[[], Generator]:
+
+ grouping_keys = []
+ grouping_vars = [
+ v for v in grouping_vars if v in df and v not in ["col", "row"]
+ ]
+ for var in grouping_vars:
+ order = getattr(self._scales[var], "order", None)
+ if order is None:
+ order = categorical_order(df[var])
+ grouping_keys.append(order)
+
+ def split_generator(keep_na=False) -> Generator:
+
+ for view in subplots:
+
+ axes_df = self._filter_subplot_data(df, view)
+
+ axes_df_inf_as_nan = axes_df.copy()
+ axes_df_inf_as_nan = axes_df_inf_as_nan.mask(
+ axes_df_inf_as_nan.isin([np.inf, -np.inf]), np.nan
+ )
+ if keep_na:
+ # The simpler thing to do would be x.dropna().reindex(x.index).
+ # But that doesn't work with the way that the subset iteration
+ # is written below, which assumes data for grouping vars.
+ # Matplotlib (usually?) masks nan data, so this should "work".
+ # Downstream code can also drop these rows, at some speed cost.
+ present = axes_df_inf_as_nan.notna().all(axis=1)
+ nulled = {}
+ for axis in "xy":
+ if axis in axes_df:
+ nulled[axis] = axes_df[axis].where(present)
+ axes_df = axes_df_inf_as_nan.assign(**nulled)
+ else:
+ axes_df = axes_df_inf_as_nan.dropna()
+
+ subplot_keys = {}
+ for dim in ["col", "row"]:
+ if view[dim] is not None:
+ subplot_keys[dim] = view[dim]
+
+ if not grouping_vars or not any(grouping_keys):
+ if not axes_df.empty:
+ yield subplot_keys, axes_df.copy(), view["ax"]
+ continue
+
+ grouped_df = axes_df.groupby(
+ grouping_vars, sort=False, as_index=False, observed=False,
+ )
+
+ for key in itertools.product(*grouping_keys):
+
+ pd_key = (
+ key[0] if len(key) == 1 and _version_predates(pd, "2.2.0")
+ else key
+ )
+ try:
+ df_subset = grouped_df.get_group(pd_key)
+ except KeyError:
+ # TODO (from initial work on categorical plots refactor)
+ # We are adding this to allow backwards compatability
+ # with the empty artists that old categorical plots would
+ # add (before 0.12), which we may decide to break, in which
+ # case this option could be removed
+ df_subset = axes_df.loc[[]]
+
+ if df_subset.empty:
+ continue
+
+ sub_vars = dict(zip(grouping_vars, key))
+ sub_vars.update(subplot_keys)
+
+ # TODO need copy(deep=...) policy (here, above, anywhere else?)
+ yield sub_vars, df_subset.copy(), view["ax"]
+
+ return split_generator
+
+ def _update_legend_contents(
+ self,
+ p: Plot,
+ mark: Mark,
+ data: PlotData,
+ scales: dict[str, Scale],
+ layer_label: str | None,
+ ) -> None:
+ """Add legend artists / labels for one layer in the plot."""
+ if data.frame.empty and data.frames:
+ legend_vars: list[str] = []
+ for frame in data.frames.values():
+ frame_vars = frame.columns.intersection(list(scales))
+ legend_vars.extend(v for v in frame_vars if v not in legend_vars)
+ else:
+ legend_vars = list(data.frame.columns.intersection(list(scales)))
+
+ # First handle layer legends, which occupy a single entry in legend_contents.
+ if layer_label is not None:
+ legend_title = str(p._labels.get("legend", ""))
+ layer_key = (legend_title, -1)
+ artist = mark._legend_artist([], None, {})
+ if artist is not None:
+ for content in self._legend_contents:
+ if content[0] == layer_key:
+ content[1].append(artist)
+ content[2].append(layer_label)
+ break
+ else:
+ self._legend_contents.append((layer_key, [artist], [layer_label]))
+
+ # Then handle the scale legends
+ # First pass: Identify the values that will be shown for each variable
+ schema: list[tuple[
+ tuple[str, str | int], list[str], tuple[list[Any], list[str]]
+ ]] = []
+ schema = []
+ for var in legend_vars:
+ var_legend = scales[var]._legend
+ if var_legend is not None:
+ values, labels = var_legend
+ for (_, part_id), part_vars, _ in schema:
+ if data.ids[var] == part_id:
+ # Allow multiple plot semantics to represent same data variable
+ part_vars.append(var)
+ break
+ else:
+ title = self._resolve_label(p, var, data.names[var])
+ entry = (title, data.ids[var]), [var], (values, labels)
+ schema.append(entry)
+
+ # Second pass, generate an artist corresponding to each value
+ contents: list[tuple[tuple[str, str | int], Any, list[str]]] = []
+ for key, variables, (values, labels) in schema:
+ artists = []
+ for val in values:
+ artist = mark._legend_artist(variables, val, scales)
+ if artist is not None:
+ artists.append(artist)
+ if artists:
+ contents.append((key, artists, labels))
+
+ self._legend_contents.extend(contents)
+
+ def _make_legend(self, p: Plot) -> None:
+ """Create the legend artist(s) and add onto the figure."""
+ # Combine artists representing same information across layers
+ # Input list has an entry for each distinct variable in each layer
+ # Output dict has an entry for each distinct variable
+ merged_contents: dict[
+ tuple[str, str | int], tuple[list[tuple[Artist, ...]], list[str]],
+ ] = {}
+ for key, new_artists, labels in self._legend_contents:
+ # Key is (name, id); we need the id to resolve variable uniqueness,
+ # but will need the name in the next step to title the legend
+ if key not in merged_contents:
+ # Matplotlib accepts a tuple of artists and will overlay them
+ new_artist_tuples = [tuple([a]) for a in new_artists]
+ merged_contents[key] = new_artist_tuples, labels
+ else:
+ existing_artists = merged_contents[key][0]
+ for i, new_artist in enumerate(new_artists):
+ existing_artists[i] += tuple([new_artist])
+
+ # When using pyplot, an "external" legend won't be shown, so this
+ # keeps it inside the axes (though still attached to the figure)
+ # This is necessary because matplotlib layout engines currently don't
+ # support figure legends — ideally this will change.
+ loc = "center right" if self._pyplot else "center left"
+
+ base_legend = None
+ for (name, _), (handles, labels) in merged_contents.items():
+
+ legend = mpl.legend.Legend(
+ self._figure,
+ handles, # type: ignore # matplotlib/issues/26639
+ labels,
+ title=name,
+ loc=loc,
+ bbox_to_anchor=(.98, .55),
+ )
+
+ if base_legend:
+ # Matplotlib has no public API for this so it is a bit of a hack.
+ # Ideally we'd define our own legend class with more flexibility,
+ # but that is a lot of work!
+ base_legend_box = base_legend.get_children()[0]
+ this_legend_box = legend.get_children()[0]
+ base_legend_box.get_children().extend(this_legend_box.get_children())
+ else:
+ base_legend = legend
+ self._figure.legends.append(legend)
+
+ def _finalize_figure(self, p: Plot) -> None:
+
+ for sub in self._subplots:
+ ax = sub["ax"]
+ for axis in "xy":
+ axis_key = sub[axis]
+ axis_obj = getattr(ax, f"{axis}axis")
+
+ # Axis limits
+ if axis_key in p._limits or axis in p._limits:
+ convert_units = getattr(ax, f"{axis}axis").convert_units
+ a, b = p._limits.get(axis_key) or p._limits[axis]
+ lo = a if a is None else convert_units(a)
+ hi = b if b is None else convert_units(b)
+ if isinstance(a, str):
+ lo = cast(float, lo) - 0.5
+ if isinstance(b, str):
+ hi = cast(float, hi) + 0.5
+ ax.set(**{f"{axis}lim": (lo, hi)})
+
+ if axis_key in self._scales: # TODO when would it not be?
+ self._scales[axis_key]._finalize(p, axis_obj)
+
+ if (engine_name := p._layout_spec.get("engine", default)) is not default:
+ # None is a valid arg for Figure.set_layout_engine, hence `default`
+ set_layout_engine(self._figure, engine_name)
+ elif p._target is None:
+ # Don't modify the layout engine if the user supplied their own
+ # matplotlib figure and didn't specify an engine through Plot
+ # TODO switch default to "constrained"?
+ # TODO either way, make configurable
+ set_layout_engine(self._figure, "tight")
+
+ if (extent := p._layout_spec.get("extent")) is not None:
+ engine = get_layout_engine(self._figure)
+ if engine is None:
+ self._figure.subplots_adjust(*extent)
+ else:
+ # Note the different parameterization for the layout engine rect...
+ left, bottom, right, top = extent
+ width, height = right - left, top - bottom
+ try:
+ # The base LayoutEngine.set method doesn't have rect= so we need
+ # to avoid typechecking this statement. We also catch a TypeError
+ # as a plugin LayoutEngine may not support it either.
+ # Alternatively we could guard this with a check on the engine type,
+ # but that would make later-developed engines would un-useable.
+ engine.set(rect=[left, bottom, width, height]) # type: ignore
+ except TypeError:
+ # Should we warn / raise? Note that we don't expect to get here
+ # under any normal circumstances.
+ pass
diff --git a/seaborn/_core/properties.py b/seaborn/_core/properties.py
new file mode 100644
index 0000000000000000000000000000000000000000..4e2df91b49faf267b6af020a4ef4078ea3a28566
--- /dev/null
+++ b/seaborn/_core/properties.py
@@ -0,0 +1,834 @@
+from __future__ import annotations
+import itertools
+import warnings
+
+import numpy as np
+from numpy.typing import ArrayLike
+from pandas import Series
+import matplotlib as mpl
+from matplotlib.colors import to_rgb, to_rgba, to_rgba_array
+from matplotlib.markers import MarkerStyle
+from matplotlib.path import Path
+
+from seaborn._core.scales import Scale, Boolean, Continuous, Nominal, Temporal
+from seaborn._core.rules import categorical_order, variable_type
+from seaborn.palettes import QUAL_PALETTES, color_palette, blend_palette
+from seaborn.utils import get_color_cycle
+
+from typing import Any, Callable, Tuple, List, Union, Optional
+
+RGBTuple = Tuple[float, float, float]
+RGBATuple = Tuple[float, float, float, float]
+ColorSpec = Union[RGBTuple, RGBATuple, str]
+
+DashPattern = Tuple[float, ...]
+DashPatternWithOffset = Tuple[float, Optional[DashPattern]]
+
+MarkerPattern = Union[
+ float,
+ str,
+ Tuple[int, int, float],
+ List[Tuple[float, float]],
+ Path,
+ MarkerStyle,
+]
+
+Mapping = Callable[[ArrayLike], ArrayLike]
+
+
+# =================================================================================== #
+# Base classes
+# =================================================================================== #
+
+
+class Property:
+ """Base class for visual properties that can be set directly or be data scaling."""
+
+ # When True, scales for this property will populate the legend by default
+ legend = False
+
+ # When True, scales for this property normalize data to [0, 1] before mapping
+ normed = False
+
+ def __init__(self, variable: str | None = None):
+ """Initialize the property with the name of the corresponding plot variable."""
+ if not variable:
+ variable = self.__class__.__name__.lower()
+ self.variable = variable
+
+ def default_scale(self, data: Series) -> Scale:
+ """Given data, initialize appropriate scale class."""
+
+ var_type = variable_type(data, boolean_type="boolean", strict_boolean=True)
+ if var_type == "numeric":
+ return Continuous()
+ elif var_type == "datetime":
+ return Temporal()
+ elif var_type == "boolean":
+ return Boolean()
+ else:
+ return Nominal()
+
+ def infer_scale(self, arg: Any, data: Series) -> Scale:
+ """Given data and a scaling argument, initialize appropriate scale class."""
+ # TODO put these somewhere external for validation
+ # TODO putting this here won't pick it up if subclasses define infer_scale
+ # (e.g. color). How best to handle that? One option is to call super after
+ # handling property-specific possibilities (e.g. for color check that the
+ # arg is not a valid palette name) but that could get tricky.
+ trans_args = ["log", "symlog", "logit", "pow", "sqrt"]
+ if isinstance(arg, str):
+ if any(arg.startswith(k) for k in trans_args):
+ # TODO validate numeric type? That should happen centrally somewhere
+ return Continuous(trans=arg)
+ else:
+ msg = f"Unknown magic arg for {self.variable} scale: '{arg}'."
+ raise ValueError(msg)
+ else:
+ arg_type = type(arg).__name__
+ msg = f"Magic arg for {self.variable} scale must be str, not {arg_type}."
+ raise TypeError(msg)
+
+ def get_mapping(self, scale: Scale, data: Series) -> Mapping:
+ """Return a function that maps from data domain to property range."""
+ def identity(x):
+ return x
+ return identity
+
+ def standardize(self, val: Any) -> Any:
+ """Coerce flexible property value to standardized representation."""
+ return val
+
+ def _check_dict_entries(self, levels: list, values: dict) -> None:
+ """Input check when values are provided as a dictionary."""
+ missing = set(levels) - set(values)
+ if missing:
+ formatted = ", ".join(map(repr, sorted(missing, key=str)))
+ err = f"No entry in {self.variable} dictionary for {formatted}"
+ raise ValueError(err)
+
+ def _check_list_length(self, levels: list, values: list) -> list:
+ """Input check when values are provided as a list."""
+ message = ""
+ if len(levels) > len(values):
+ message = " ".join([
+ f"\nThe {self.variable} list has fewer values ({len(values)})",
+ f"than needed ({len(levels)}) and will cycle, which may",
+ "produce an uninterpretable plot."
+ ])
+ values = [x for _, x in zip(levels, itertools.cycle(values))]
+
+ elif len(values) > len(levels):
+ message = " ".join([
+ f"The {self.variable} list has more values ({len(values)})",
+ f"than needed ({len(levels)}), which may not be intended.",
+ ])
+ values = values[:len(levels)]
+
+ # TODO look into custom PlotSpecWarning with better formatting
+ if message:
+ warnings.warn(message, UserWarning)
+
+ return values
+
+
+# =================================================================================== #
+# Properties relating to spatial position of marks on the plotting axes
+# =================================================================================== #
+
+
+class Coordinate(Property):
+ """The position of visual marks with respect to the axes of the plot."""
+ legend = False
+ normed = False
+
+
+# =================================================================================== #
+# Properties with numeric values where scale range can be defined as an interval
+# =================================================================================== #
+
+
+class IntervalProperty(Property):
+ """A numeric property where scale range can be defined as an interval."""
+ legend = True
+ normed = True
+
+ _default_range: tuple[float, float] = (0, 1)
+
+ @property
+ def default_range(self) -> tuple[float, float]:
+ """Min and max values used by default for semantic mapping."""
+ return self._default_range
+
+ def _forward(self, values: ArrayLike) -> ArrayLike:
+ """Transform applied to native values before linear mapping into interval."""
+ return values
+
+ def _inverse(self, values: ArrayLike) -> ArrayLike:
+ """Transform applied to results of mapping that returns to native values."""
+ return values
+
+ def infer_scale(self, arg: Any, data: Series) -> Scale:
+ """Given data and a scaling argument, initialize appropriate scale class."""
+
+ # TODO infer continuous based on log/sqrt etc?
+
+ var_type = variable_type(data, boolean_type="boolean", strict_boolean=True)
+
+ if var_type == "boolean":
+ return Boolean(arg)
+ elif isinstance(arg, (list, dict)):
+ return Nominal(arg)
+ elif var_type == "categorical":
+ return Nominal(arg)
+ elif var_type == "datetime":
+ return Temporal(arg)
+ # TODO other variable types
+ else:
+ return Continuous(arg)
+
+ def get_mapping(self, scale: Scale, data: Series) -> Mapping:
+ """Return a function that maps from data domain to property range."""
+ if isinstance(scale, Nominal):
+ return self._get_nominal_mapping(scale, data)
+ elif isinstance(scale, Boolean):
+ return self._get_boolean_mapping(scale, data)
+
+ if scale.values is None:
+ vmin, vmax = self._forward(self.default_range)
+ elif isinstance(scale.values, tuple) and len(scale.values) == 2:
+ vmin, vmax = self._forward(scale.values)
+ else:
+ if isinstance(scale.values, tuple):
+ actual = f"{len(scale.values)}-tuple"
+ else:
+ actual = str(type(scale.values))
+ scale_class = scale.__class__.__name__
+ err = " ".join([
+ f"Values for {self.variable} variables with {scale_class} scale",
+ f"must be 2-tuple; not {actual}.",
+ ])
+ raise TypeError(err)
+
+ def mapping(x):
+ return self._inverse(np.multiply(x, vmax - vmin) + vmin)
+
+ return mapping
+
+ def _get_nominal_mapping(self, scale: Nominal, data: Series) -> Mapping:
+ """Identify evenly-spaced values using interval or explicit mapping."""
+ levels = categorical_order(data, scale.order)
+ values = self._get_values(scale, levels)
+
+ def mapping(x):
+ ixs = np.asarray(x, np.intp)
+ out = np.full(len(x), np.nan)
+ use = np.isfinite(x)
+ out[use] = np.take(values, ixs[use])
+ return out
+
+ return mapping
+
+ def _get_boolean_mapping(self, scale: Boolean, data: Series) -> Mapping:
+ """Identify evenly-spaced values using interval or explicit mapping."""
+ values = self._get_values(scale, [True, False])
+
+ def mapping(x):
+ out = np.full(len(x), np.nan)
+ use = np.isfinite(x)
+ out[use] = np.where(x[use], *values)
+ return out
+
+ return mapping
+
+ def _get_values(self, scale: Scale, levels: list) -> list:
+ """Validate scale.values and identify a value for each level."""
+ if isinstance(scale.values, dict):
+ self._check_dict_entries(levels, scale.values)
+ values = [scale.values[x] for x in levels]
+ elif isinstance(scale.values, list):
+ values = self._check_list_length(levels, scale.values)
+ else:
+ if scale.values is None:
+ vmin, vmax = self.default_range
+ elif isinstance(scale.values, tuple):
+ vmin, vmax = scale.values
+ else:
+ scale_class = scale.__class__.__name__
+ err = " ".join([
+ f"Values for {self.variable} variables with {scale_class} scale",
+ f"must be a dict, list or tuple; not {type(scale.values)}",
+ ])
+ raise TypeError(err)
+
+ vmin, vmax = self._forward([vmin, vmax])
+ values = list(self._inverse(np.linspace(vmax, vmin, len(levels))))
+
+ return values
+
+
+class PointSize(IntervalProperty):
+ """Size (diameter) of a point mark, in points, with scaling by area."""
+ _default_range = 2, 8 # TODO use rcparams?
+
+ def _forward(self, values):
+ """Square native values to implement linear scaling of point area."""
+ return np.square(values)
+
+ def _inverse(self, values):
+ """Invert areal values back to point diameter."""
+ return np.sqrt(values)
+
+
+class LineWidth(IntervalProperty):
+ """Thickness of a line mark, in points."""
+ @property
+ def default_range(self) -> tuple[float, float]:
+ """Min and max values used by default for semantic mapping."""
+ base = mpl.rcParams["lines.linewidth"]
+ return base * .5, base * 2
+
+
+class EdgeWidth(IntervalProperty):
+ """Thickness of the edges on a patch mark, in points."""
+ @property
+ def default_range(self) -> tuple[float, float]:
+ """Min and max values used by default for semantic mapping."""
+ base = mpl.rcParams["patch.linewidth"]
+ return base * .5, base * 2
+
+
+class Stroke(IntervalProperty):
+ """Thickness of lines that define point glyphs."""
+ _default_range = .25, 2.5
+
+
+class Alpha(IntervalProperty):
+ """Opacity of the color values for an arbitrary mark."""
+ _default_range = .3, .95
+ # TODO validate / enforce that output is in [0, 1]
+
+
+class Offset(IntervalProperty):
+ """Offset for edge-aligned text, in point units."""
+ _default_range = 0, 5
+ _legend = False
+
+
+class FontSize(IntervalProperty):
+ """Font size for textual marks, in points."""
+ _legend = False
+
+ @property
+ def default_range(self) -> tuple[float, float]:
+ """Min and max values used by default for semantic mapping."""
+ base = mpl.rcParams["font.size"]
+ return base * .5, base * 2
+
+
+# =================================================================================== #
+# Properties defined by arbitrary objects with inherently nominal scaling
+# =================================================================================== #
+
+
+class ObjectProperty(Property):
+ """A property defined by arbitrary an object, with inherently nominal scaling."""
+ legend = True
+ normed = False
+
+ # Object representing null data, should appear invisible when drawn by matplotlib
+ # Note that we now drop nulls in Plot._plot_layer and thus may not need this
+ null_value: Any = None
+
+ def _default_values(self, n: int) -> list:
+ raise NotImplementedError()
+
+ def default_scale(self, data: Series) -> Scale:
+ var_type = variable_type(data, boolean_type="boolean", strict_boolean=True)
+ return Boolean() if var_type == "boolean" else Nominal()
+
+ def infer_scale(self, arg: Any, data: Series) -> Scale:
+ var_type = variable_type(data, boolean_type="boolean", strict_boolean=True)
+ return Boolean(arg) if var_type == "boolean" else Nominal(arg)
+
+ def get_mapping(self, scale: Scale, data: Series) -> Mapping:
+ """Define mapping as lookup into list of object values."""
+ boolean_scale = isinstance(scale, Boolean)
+ order = getattr(scale, "order", [True, False] if boolean_scale else None)
+ levels = categorical_order(data, order)
+ values = self._get_values(scale, levels)
+
+ if boolean_scale:
+ values = values[::-1]
+
+ def mapping(x):
+ ixs = np.asarray(np.nan_to_num(x), np.intp)
+ return [
+ values[ix] if np.isfinite(x_i) else self.null_value
+ for x_i, ix in zip(x, ixs)
+ ]
+
+ return mapping
+
+ def _get_values(self, scale: Scale, levels: list) -> list:
+ """Validate scale.values and identify a value for each level."""
+ n = len(levels)
+ if isinstance(scale.values, dict):
+ self._check_dict_entries(levels, scale.values)
+ values = [scale.values[x] for x in levels]
+ elif isinstance(scale.values, list):
+ values = self._check_list_length(levels, scale.values)
+ elif scale.values is None:
+ values = self._default_values(n)
+ else:
+ msg = " ".join([
+ f"Scale values for a {self.variable} variable must be provided",
+ f"in a dict or list; not {type(scale.values)}."
+ ])
+ raise TypeError(msg)
+
+ values = [self.standardize(x) for x in values]
+ return values
+
+
+class Marker(ObjectProperty):
+ """Shape of points in scatter-type marks or lines with data points marked."""
+ null_value = MarkerStyle("")
+
+ # TODO should we have named marker "palettes"? (e.g. see d3 options)
+
+ # TODO need some sort of "require_scale" functionality
+ # to raise when we get the wrong kind explicitly specified
+
+ def standardize(self, val: MarkerPattern) -> MarkerStyle:
+ return MarkerStyle(val)
+
+ def _default_values(self, n: int) -> list[MarkerStyle]:
+ """Build an arbitrarily long list of unique marker styles.
+
+ Parameters
+ ----------
+ n : int
+ Number of unique marker specs to generate.
+
+ Returns
+ -------
+ markers : list of string or tuples
+ Values for defining :class:`matplotlib.markers.MarkerStyle` objects.
+ All markers will be filled.
+
+ """
+ # Start with marker specs that are well distinguishable
+ markers = [
+ "o", "X", (4, 0, 45), "P", (4, 0, 0), (4, 1, 0), "^", (4, 1, 45), "v",
+ ]
+
+ # Now generate more from regular polygons of increasing order
+ s = 5
+ while len(markers) < n:
+ a = 360 / (s + 1) / 2
+ markers.extend([(s + 1, 1, a), (s + 1, 0, a), (s, 1, 0), (s, 0, 0)])
+ s += 1
+
+ markers = [MarkerStyle(m) for m in markers[:n]]
+
+ return markers
+
+
+class LineStyle(ObjectProperty):
+ """Dash pattern for line-type marks."""
+ null_value = ""
+
+ def standardize(self, val: str | DashPattern) -> DashPatternWithOffset:
+ return self._get_dash_pattern(val)
+
+ def _default_values(self, n: int) -> list[DashPatternWithOffset]:
+ """Build an arbitrarily long list of unique dash styles for lines.
+
+ Parameters
+ ----------
+ n : int
+ Number of unique dash specs to generate.
+
+ Returns
+ -------
+ dashes : list of strings or tuples
+ Valid arguments for the ``dashes`` parameter on
+ :class:`matplotlib.lines.Line2D`. The first spec is a solid
+ line (``""``), the remainder are sequences of long and short
+ dashes.
+
+ """
+ # Start with dash specs that are well distinguishable
+ dashes: list[str | DashPattern] = [
+ "-", (4, 1.5), (1, 1), (3, 1.25, 1.5, 1.25), (5, 1, 1, 1),
+ ]
+
+ # Now programmatically build as many as we need
+ p = 3
+ while len(dashes) < n:
+
+ # Take combinations of long and short dashes
+ a = itertools.combinations_with_replacement([3, 1.25], p)
+ b = itertools.combinations_with_replacement([4, 1], p)
+
+ # Interleave the combinations, reversing one of the streams
+ segment_list = itertools.chain(*zip(list(a)[1:-1][::-1], list(b)[1:-1]))
+
+ # Now insert the gaps
+ for segments in segment_list:
+ gap = min(segments)
+ spec = tuple(itertools.chain(*((seg, gap) for seg in segments)))
+ dashes.append(spec)
+
+ p += 1
+
+ return [self._get_dash_pattern(x) for x in dashes]
+
+ @staticmethod
+ def _get_dash_pattern(style: str | DashPattern) -> DashPatternWithOffset:
+ """Convert linestyle arguments to dash pattern with offset."""
+ # Copied and modified from Matplotlib 3.4
+ # go from short hand -> full strings
+ ls_mapper = {"-": "solid", "--": "dashed", "-.": "dashdot", ":": "dotted"}
+ if isinstance(style, str):
+ style = ls_mapper.get(style, style)
+ # un-dashed styles
+ if style in ["solid", "none", "None"]:
+ offset = 0
+ dashes = None
+ # dashed styles
+ elif style in ["dashed", "dashdot", "dotted"]:
+ offset = 0
+ dashes = tuple(mpl.rcParams[f"lines.{style}_pattern"])
+ else:
+ options = [*ls_mapper.values(), *ls_mapper.keys()]
+ msg = f"Linestyle string must be one of {options}, not {repr(style)}."
+ raise ValueError(msg)
+
+ elif isinstance(style, tuple):
+ if len(style) > 1 and isinstance(style[1], tuple):
+ offset, dashes = style
+ elif len(style) > 1 and style[1] is None:
+ offset, dashes = style
+ else:
+ offset = 0
+ dashes = style
+ else:
+ val_type = type(style).__name__
+ msg = f"Linestyle must be str or tuple, not {val_type}."
+ raise TypeError(msg)
+
+ # Normalize offset to be positive and shorter than the dash cycle
+ if dashes is not None:
+ try:
+ dsum = sum(dashes)
+ except TypeError as err:
+ msg = f"Invalid dash pattern: {dashes}"
+ raise TypeError(msg) from err
+ if dsum:
+ offset %= dsum
+
+ return offset, dashes
+
+
+class TextAlignment(ObjectProperty):
+ legend = False
+
+
+class HorizontalAlignment(TextAlignment):
+
+ def _default_values(self, n: int) -> list:
+ vals = itertools.cycle(["left", "right"])
+ return [next(vals) for _ in range(n)]
+
+
+class VerticalAlignment(TextAlignment):
+
+ def _default_values(self, n: int) -> list:
+ vals = itertools.cycle(["top", "bottom"])
+ return [next(vals) for _ in range(n)]
+
+
+# =================================================================================== #
+# Properties with RGB(A) color values
+# =================================================================================== #
+
+
+class Color(Property):
+ """Color, as RGB(A), scalable with nominal palettes or continuous gradients."""
+ legend = True
+ normed = True
+
+ def standardize(self, val: ColorSpec) -> RGBTuple | RGBATuple:
+ # Return color with alpha channel only if the input spec has it
+ # This is so that RGBA colors can override the Alpha property
+ if to_rgba(val) != to_rgba(val, 1):
+ return to_rgba(val)
+ else:
+ return to_rgb(val)
+
+ def _standardize_color_sequence(self, colors: ArrayLike) -> ArrayLike:
+ """Convert color sequence to RGB(A) array, preserving but not adding alpha."""
+ def has_alpha(x):
+ return to_rgba(x) != to_rgba(x, 1)
+
+ if isinstance(colors, np.ndarray):
+ needs_alpha = colors.shape[1] == 4
+ else:
+ needs_alpha = any(has_alpha(x) for x in colors)
+
+ if needs_alpha:
+ return to_rgba_array(colors)
+ else:
+ return to_rgba_array(colors)[:, :3]
+
+ def infer_scale(self, arg: Any, data: Series) -> Scale:
+ # TODO when inferring Continuous without data, verify type
+
+ # TODO need to rethink the variable type system
+ # (e.g. boolean, ordered categories as Ordinal, etc)..
+ var_type = variable_type(data, boolean_type="boolean", strict_boolean=True)
+
+ if var_type == "boolean":
+ return Boolean(arg)
+
+ if isinstance(arg, (dict, list)):
+ return Nominal(arg)
+
+ if isinstance(arg, tuple):
+ if var_type == "categorical":
+ # TODO It seems reasonable to allow a gradient mapping for nominal
+ # scale but it also feels "technically" wrong. Should this infer
+ # Ordinal with categorical data and, if so, verify orderedness?
+ return Nominal(arg)
+ return Continuous(arg)
+
+ if callable(arg):
+ return Continuous(arg)
+
+ # TODO Do we accept str like "log", "pow", etc. for semantics?
+
+ if not isinstance(arg, str):
+ msg = " ".join([
+ f"A single scale argument for {self.variable} variables must be",
+ f"a string, dict, tuple, list, or callable, not {type(arg)}."
+ ])
+ raise TypeError(msg)
+
+ if arg in QUAL_PALETTES:
+ return Nominal(arg)
+ elif var_type == "numeric":
+ return Continuous(arg)
+ # TODO implement scales for date variables and any others.
+ else:
+ return Nominal(arg)
+
+ def get_mapping(self, scale: Scale, data: Series) -> Mapping:
+ """Return a function that maps from data domain to color values."""
+ # TODO what is best way to do this conditional?
+ # Should it be class-based or should classes have behavioral attributes?
+ if isinstance(scale, Nominal):
+ return self._get_nominal_mapping(scale, data)
+ elif isinstance(scale, Boolean):
+ return self._get_boolean_mapping(scale, data)
+
+ if scale.values is None:
+ # TODO Rethink best default continuous color gradient
+ mapping = color_palette("ch:", as_cmap=True)
+ elif isinstance(scale.values, tuple):
+ # TODO blend_palette will strip alpha, but we should support
+ # interpolation on all four channels
+ mapping = blend_palette(scale.values, as_cmap=True)
+ elif isinstance(scale.values, str):
+ # TODO for matplotlib colormaps this will clip extremes, which is
+ # different from what using the named colormap directly would do
+ # This may or may not be desireable.
+ mapping = color_palette(scale.values, as_cmap=True)
+ elif callable(scale.values):
+ mapping = scale.values
+ else:
+ scale_class = scale.__class__.__name__
+ msg = " ".join([
+ f"Scale values for {self.variable} with a {scale_class} mapping",
+ f"must be string, tuple, or callable; not {type(scale.values)}."
+ ])
+ raise TypeError(msg)
+
+ def _mapping(x):
+ # Remove alpha channel so it does not override alpha property downstream
+ # TODO this will need to be more flexible to support RGBA tuples (see above)
+ invalid = ~np.isfinite(x)
+ out = mapping(x)[:, :3]
+ out[invalid] = np.nan
+ return out
+
+ return _mapping
+
+ def _get_nominal_mapping(self, scale: Nominal, data: Series) -> Mapping:
+
+ levels = categorical_order(data, scale.order)
+ colors = self._get_values(scale, levels)
+
+ def mapping(x):
+ ixs = np.asarray(np.nan_to_num(x), np.intp)
+ use = np.isfinite(x)
+ out = np.full((len(ixs), colors.shape[1]), np.nan)
+ out[use] = np.take(colors, ixs[use], axis=0)
+ return out
+
+ return mapping
+
+ def _get_boolean_mapping(self, scale: Boolean, data: Series) -> Mapping:
+
+ colors = self._get_values(scale, [True, False])
+
+ def mapping(x):
+
+ use = np.isfinite(x)
+ x = np.asarray(np.nan_to_num(x)).astype(bool)
+ out = np.full((len(x), colors.shape[1]), np.nan)
+ out[x & use] = colors[0]
+ out[~x & use] = colors[1]
+ return out
+
+ return mapping
+
+ def _get_values(self, scale: Scale, levels: list) -> ArrayLike:
+ """Validate scale.values and identify a value for each level."""
+ n = len(levels)
+ values = scale.values
+ if isinstance(values, dict):
+ self._check_dict_entries(levels, values)
+ colors = [values[x] for x in levels]
+ elif isinstance(values, list):
+ colors = self._check_list_length(levels, values)
+ elif isinstance(values, tuple):
+ colors = blend_palette(values, n)
+ elif isinstance(values, str):
+ colors = color_palette(values, n)
+ elif values is None:
+ if n <= len(get_color_cycle()):
+ # Use current (global) default palette
+ colors = color_palette(n_colors=n)
+ else:
+ colors = color_palette("husl", n)
+ else:
+ scale_class = scale.__class__.__name__
+ msg = " ".join([
+ f"Scale values for {self.variable} with a {scale_class} mapping",
+ f"must be string, list, tuple, or dict; not {type(scale.values)}."
+ ])
+ raise TypeError(msg)
+
+ return self._standardize_color_sequence(colors)
+
+
+# =================================================================================== #
+# Properties that can take only two states
+# =================================================================================== #
+
+
+class Fill(Property):
+ """Boolean property of points/bars/patches that can be solid or outlined."""
+ legend = True
+ normed = False
+
+ def default_scale(self, data: Series) -> Scale:
+ var_type = variable_type(data, boolean_type="boolean", strict_boolean=True)
+ return Boolean() if var_type == "boolean" else Nominal()
+
+ def infer_scale(self, arg: Any, data: Series) -> Scale:
+ var_type = variable_type(data, boolean_type="boolean", strict_boolean=True)
+ return Boolean(arg) if var_type == "boolean" else Nominal(arg)
+
+ def standardize(self, val: Any) -> bool:
+ return bool(val)
+
+ def _default_values(self, n: int) -> list:
+ """Return a list of n values, alternating True and False."""
+ if n > 2:
+ msg = " ".join([
+ f"The variable assigned to {self.variable} has more than two levels,",
+ f"so {self.variable} values will cycle and may be uninterpretable",
+ ])
+ # TODO fire in a "nice" way (see above)
+ warnings.warn(msg, UserWarning)
+ return [x for x, _ in zip(itertools.cycle([True, False]), range(n))]
+
+ def get_mapping(self, scale: Scale, data: Series) -> Mapping:
+ """Return a function that maps each data value to True or False."""
+ boolean_scale = isinstance(scale, Boolean)
+ order = getattr(scale, "order", [True, False] if boolean_scale else None)
+ levels = categorical_order(data, order)
+ values = self._get_values(scale, levels)
+
+ if boolean_scale:
+ values = values[::-1]
+
+ def mapping(x):
+ ixs = np.asarray(np.nan_to_num(x), np.intp)
+ return [
+ values[ix] if np.isfinite(x_i) else False
+ for x_i, ix in zip(x, ixs)
+ ]
+
+ return mapping
+
+ def _get_values(self, scale: Scale, levels: list) -> list:
+ """Validate scale.values and identify a value for each level."""
+ if isinstance(scale.values, list):
+ values = [bool(x) for x in scale.values]
+ elif isinstance(scale.values, dict):
+ values = [bool(scale.values[x]) for x in levels]
+ elif scale.values is None:
+ values = self._default_values(len(levels))
+ else:
+ msg = " ".join([
+ f"Scale values for {self.variable} must be passed in",
+ f"a list or dict; not {type(scale.values)}."
+ ])
+ raise TypeError(msg)
+
+ return values
+
+
+# =================================================================================== #
+# Enumeration of properties for use by Plot and Mark classes
+# =================================================================================== #
+# TODO turn this into a property registry with hooks, etc.
+# TODO Users do not interact directly with properties, so how to document them?
+
+
+PROPERTY_CLASSES = {
+ "x": Coordinate,
+ "y": Coordinate,
+ "color": Color,
+ "alpha": Alpha,
+ "fill": Fill,
+ "marker": Marker,
+ "pointsize": PointSize,
+ "stroke": Stroke,
+ "linewidth": LineWidth,
+ "linestyle": LineStyle,
+ "fillcolor": Color,
+ "fillalpha": Alpha,
+ "edgewidth": EdgeWidth,
+ "edgestyle": LineStyle,
+ "edgecolor": Color,
+ "edgealpha": Alpha,
+ "text": Property,
+ "halign": HorizontalAlignment,
+ "valign": VerticalAlignment,
+ "offset": Offset,
+ "fontsize": FontSize,
+ "xmin": Coordinate,
+ "xmax": Coordinate,
+ "ymin": Coordinate,
+ "ymax": Coordinate,
+ "group": Property,
+ # TODO pattern?
+ # TODO gradient?
+}
+
+PROPERTIES = {var: cls(var) for var, cls in PROPERTY_CLASSES.items()}
diff --git a/seaborn/_core/rules.py b/seaborn/_core/rules.py
new file mode 100644
index 0000000000000000000000000000000000000000..de6c651d97b6657bfb1bee2de370376958c750ae
--- /dev/null
+++ b/seaborn/_core/rules.py
@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+import warnings
+from collections import UserString
+from numbers import Number
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+ from typing import Literal
+ from pandas import Series
+
+
+class VarType(UserString):
+ """
+ Prevent comparisons elsewhere in the library from using the wrong name.
+
+ Errors are simple assertions because users should not be able to trigger
+ them. If that changes, they should be more verbose.
+
+ """
+ # TODO VarType is an awfully overloaded name, but so is DataType ...
+ # TODO adding unknown because we are using this in for scales, is that right?
+ allowed = "numeric", "datetime", "categorical", "boolean", "unknown"
+
+ def __init__(self, data):
+ assert data in self.allowed, data
+ super().__init__(data)
+
+ def __eq__(self, other):
+ assert other in self.allowed, other
+ return self.data == other
+
+
+def variable_type(
+ vector: Series,
+ boolean_type: Literal["numeric", "categorical", "boolean"] = "numeric",
+ strict_boolean: bool = False,
+) -> VarType:
+ """
+ Determine whether a vector contains numeric, categorical, or datetime data.
+
+ This function differs from the pandas typing API in a few ways:
+
+ - Python sequences or object-typed PyData objects are considered numeric if
+ all of their entries are numeric.
+ - String or mixed-type data are considered categorical even if not
+ explicitly represented as a :class:`pandas.api.types.CategoricalDtype`.
+ - There is some flexibility about how to treat binary / boolean data.
+
+ Parameters
+ ----------
+ vector : :func:`pandas.Series`, :func:`numpy.ndarray`, or Python sequence
+ Input data to test.
+ boolean_type : 'numeric', 'categorical', or 'boolean'
+ Type to use for vectors containing only 0s and 1s (and NAs).
+ strict_boolean : bool
+ If True, only consider data to be boolean when the dtype is bool or Boolean.
+
+ Returns
+ -------
+ var_type : 'numeric', 'categorical', or 'datetime'
+ Name identifying the type of data in the vector.
+ """
+
+ # If a categorical dtype is set, infer categorical
+ if isinstance(getattr(vector, 'dtype', None), pd.CategoricalDtype):
+ return VarType("categorical")
+
+ # Special-case all-na data, which is always "numeric"
+ if pd.isna(vector).all():
+ return VarType("numeric")
+
+ # Now drop nulls to simplify further type inference
+ vector = vector.dropna()
+
+ # Special-case binary/boolean data, allow caller to determine
+ # This triggers a numpy warning when vector has strings/objects
+ # https://github.com/numpy/numpy/issues/6784
+ # Because we reduce with .all(), we are agnostic about whether the
+ # comparison returns a scalar or vector, so we will ignore the warning.
+ # It triggers a separate DeprecationWarning when the vector has datetimes:
+ # https://github.com/numpy/numpy/issues/13548
+ # This is considered a bug by numpy and will likely go away.
+ with warnings.catch_warnings():
+ warnings.simplefilter(
+ action='ignore',
+ category=(FutureWarning, DeprecationWarning) # type: ignore # mypy bug?
+ )
+ if strict_boolean:
+ if isinstance(vector.dtype, pd.core.dtypes.base.ExtensionDtype):
+ boolean_dtypes = ["bool", "boolean"]
+ else:
+ boolean_dtypes = ["bool"]
+ boolean_vector = vector.dtype in boolean_dtypes
+ else:
+ try:
+ boolean_vector = bool(np.isin(vector, [0, 1]).all())
+ except TypeError:
+ # .isin comparison is not guaranteed to be possible under NumPy
+ # casting rules, depending on the (unknown) dtype of 'vector'
+ boolean_vector = False
+ if boolean_vector:
+ return VarType(boolean_type)
+
+ # Defer to positive pandas tests
+ if pd.api.types.is_numeric_dtype(vector):
+ return VarType("numeric")
+
+ if pd.api.types.is_datetime64_dtype(vector):
+ return VarType("datetime")
+
+ # --- If we get to here, we need to check the entries
+
+ # Check for a collection where everything is a number
+
+ def all_numeric(x):
+ for x_i in x:
+ if not isinstance(x_i, Number):
+ return False
+ return True
+
+ if all_numeric(vector):
+ return VarType("numeric")
+
+ # Check for a collection where everything is a datetime
+
+ def all_datetime(x):
+ for x_i in x:
+ if not isinstance(x_i, (datetime, np.datetime64)):
+ return False
+ return True
+
+ if all_datetime(vector):
+ return VarType("datetime")
+
+ # Otherwise, our final fallback is to consider things categorical
+
+ return VarType("categorical")
+
+
+def categorical_order(vector: Series, order: list | None = None) -> list:
+ """
+ Return a list of unique data values using seaborn's ordering rules.
+
+ Parameters
+ ----------
+ vector : Series
+ Vector of "categorical" values
+ order : list
+ Desired order of category levels to override the order determined
+ from the `data` object.
+
+ Returns
+ -------
+ order : list
+ Ordered list of category levels not including null values.
+
+ """
+ if order is not None:
+ return order
+
+ if vector.dtype.name == "category":
+ order = list(vector.cat.categories)
+ else:
+ order = list(filter(pd.notnull, vector.unique()))
+ if variable_type(pd.Series(order)) == "numeric":
+ order.sort()
+
+ return order
diff --git a/seaborn/_core/scales.py b/seaborn/_core/scales.py
new file mode 100644
index 0000000000000000000000000000000000000000..1e7bef8a5d29934f5afa18b1a474b103f605e611
--- /dev/null
+++ b/seaborn/_core/scales.py
@@ -0,0 +1,1090 @@
+from __future__ import annotations
+import re
+from copy import copy
+from collections.abc import Sequence
+from dataclasses import dataclass
+from functools import partial
+from typing import Any, Callable, Tuple, Optional, ClassVar
+
+import numpy as np
+import matplotlib as mpl
+from matplotlib.ticker import (
+ Locator,
+ Formatter,
+ AutoLocator,
+ AutoMinorLocator,
+ FixedLocator,
+ LinearLocator,
+ LogLocator,
+ SymmetricalLogLocator,
+ MaxNLocator,
+ MultipleLocator,
+ EngFormatter,
+ FuncFormatter,
+ LogFormatterSciNotation,
+ ScalarFormatter,
+ StrMethodFormatter,
+)
+from matplotlib.dates import (
+ AutoDateLocator,
+ AutoDateFormatter,
+ ConciseDateFormatter,
+)
+from matplotlib.axis import Axis
+from matplotlib.scale import ScaleBase
+from pandas import Series
+
+from seaborn._core.rules import categorical_order
+from seaborn._core.typing import Default, default
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+ from seaborn._core.plot import Plot
+ from seaborn._core.properties import Property
+ from numpy.typing import ArrayLike, NDArray
+
+ TransFuncs = Tuple[
+ Callable[[ArrayLike], ArrayLike], Callable[[ArrayLike], ArrayLike]
+ ]
+
+ # TODO Reverting typing to Any as it was proving too complicated to
+ # work out the right way to communicate the types to mypy. Revisit!
+ Pipeline = Sequence[Optional[Callable[[Any], Any]]]
+
+
+class Scale:
+ """Base class for objects that map data values to visual properties."""
+
+ values: tuple | str | list | dict | None
+
+ _priority: ClassVar[int]
+ _pipeline: Pipeline
+ _matplotlib_scale: ScaleBase
+ _spacer: staticmethod
+ _legend: tuple[list[Any], list[str]] | None
+
+ def __post_init__(self):
+
+ self._tick_params = None
+ self._label_params = None
+ self._legend = None
+
+ def tick(self):
+ raise NotImplementedError()
+
+ def label(self):
+ raise NotImplementedError()
+
+ def _get_locators(self):
+ raise NotImplementedError()
+
+ def _get_formatter(self, locator: Locator | None = None):
+ raise NotImplementedError()
+
+ def _get_scale(self, name: str, forward: Callable, inverse: Callable):
+
+ major_locator, minor_locator = self._get_locators(**self._tick_params)
+ major_formatter = self._get_formatter(major_locator, **self._label_params)
+
+ class InternalScale(mpl.scale.FuncScale):
+ def set_default_locators_and_formatters(self, axis):
+ axis.set_major_locator(major_locator)
+ if minor_locator is not None:
+ axis.set_minor_locator(minor_locator)
+ axis.set_major_formatter(major_formatter)
+
+ return InternalScale(name, (forward, inverse))
+
+ def _spacing(self, x: Series) -> float:
+ space = self._spacer(x)
+ if np.isnan(space):
+ # This happens when there is no variance in the orient coordinate data
+ # Not exactly clear what the right default is, but 1 seems reasonable?
+ return 1
+ return space
+
+ def _setup(
+ self, data: Series, prop: Property, axis: Axis | None = None,
+ ) -> Scale:
+ raise NotImplementedError()
+
+ def _finalize(self, p: Plot, axis: Axis) -> None:
+ """Perform scale-specific axis tweaks after adding artists."""
+ pass
+
+ def __call__(self, data: Series) -> ArrayLike:
+
+ trans_data: Series | NDArray | list
+
+ # TODO sometimes we need to handle scalars (e.g. for Line)
+ # but what is the best way to do that?
+ scalar_data = np.isscalar(data)
+ if scalar_data:
+ trans_data = np.array([data])
+ else:
+ trans_data = data
+
+ for func in self._pipeline:
+ if func is not None:
+ trans_data = func(trans_data)
+
+ if scalar_data:
+ return trans_data[0]
+ else:
+ return trans_data
+
+ @staticmethod
+ def _identity():
+
+ class Identity(Scale):
+ _pipeline = []
+ _spacer = None
+ _legend = None
+ _matplotlib_scale = None
+
+ return Identity()
+
+
+@dataclass
+class Boolean(Scale):
+ """
+ A scale with a discrete domain of True and False values.
+
+ The behavior is similar to the :class:`Nominal` scale, but property
+ mappings and legends will use a [True, False] ordering rather than
+ a sort using numeric rules. Coordinate variables accomplish this by
+ inverting axis limits so as to maintain underlying numeric positioning.
+ Input data are cast to boolean values, respecting missing data.
+
+ """
+ values: tuple | list | dict | None = None
+
+ _priority: ClassVar[int] = 3
+
+ def _setup(
+ self, data: Series, prop: Property, axis: Axis | None = None,
+ ) -> Scale:
+
+ new = copy(self)
+ if new._tick_params is None:
+ new = new.tick()
+ if new._label_params is None:
+ new = new.label()
+
+ def na_safe_cast(x):
+ # TODO this doesn't actually need to be a closure
+ if np.isscalar(x):
+ return float(bool(x))
+ else:
+ if hasattr(x, "notna"):
+ # Handle pd.NA; np<>pd interop with NA is tricky
+ use = x.notna().to_numpy()
+ else:
+ use = np.isfinite(x)
+ out = np.full(len(x), np.nan, dtype=float)
+ out[use] = x[use].astype(bool).astype(float)
+ return out
+
+ new._pipeline = [na_safe_cast, prop.get_mapping(new, data)]
+ new._spacer = _default_spacer
+ if prop.legend:
+ new._legend = [True, False], ["True", "False"]
+
+ forward, inverse = _make_identity_transforms()
+ mpl_scale = new._get_scale(str(data.name), forward, inverse)
+
+ axis = PseudoAxis(mpl_scale) if axis is None else axis
+ mpl_scale.set_default_locators_and_formatters(axis)
+ new._matplotlib_scale = mpl_scale
+
+ return new
+
+ def _finalize(self, p: Plot, axis: Axis) -> None:
+
+ # We want values to appear in a True, False order but also want
+ # True/False to be drawn at 1/0 positions respectively to avoid nasty
+ # surprises if additional artists are added through the matplotlib API.
+ # We accomplish this using axis inversion akin to what we do in Nominal.
+
+ ax = axis.axes
+ name = axis.axis_name
+ axis.grid(False, which="both")
+ if name not in p._limits:
+ nticks = len(axis.get_major_ticks())
+ lo, hi = -.5, nticks - .5
+ if name == "x":
+ lo, hi = hi, lo
+ set_lim = getattr(ax, f"set_{name}lim")
+ set_lim(lo, hi, auto=None)
+
+ def tick(self, locator: Locator | None = None):
+ new = copy(self)
+ new._tick_params = {"locator": locator}
+ return new
+
+ def label(self, formatter: Formatter | None = None):
+ new = copy(self)
+ new._label_params = {"formatter": formatter}
+ return new
+
+ def _get_locators(self, locator):
+ if locator is not None:
+ return locator
+ return FixedLocator([0, 1]), None
+
+ def _get_formatter(self, locator, formatter):
+ if formatter is not None:
+ return formatter
+ return FuncFormatter(lambda x, _: str(bool(x)))
+
+
+@dataclass
+class Nominal(Scale):
+ """
+ A categorical scale without relative importance / magnitude.
+ """
+ # Categorical (convert to strings), un-sortable
+
+ values: tuple | str | list | dict | None = None
+ order: list | None = None
+
+ _priority: ClassVar[int] = 4
+
+ def _setup(
+ self, data: Series, prop: Property, axis: Axis | None = None,
+ ) -> Scale:
+
+ new = copy(self)
+ if new._tick_params is None:
+ new = new.tick()
+ if new._label_params is None:
+ new = new.label()
+
+ # TODO flexibility over format() which isn't great for numbers / dates
+ stringify = np.vectorize(format, otypes=["object"])
+
+ units_seed = categorical_order(data, new.order)
+
+ # TODO move to Nominal._get_scale?
+ # TODO this needs some more complicated rethinking about how to pass
+ # a unit dictionary down to these methods, along with how much we want
+ # to invest in their API. What is it useful for tick() to do here?
+ # (Ordinal may be different if we draw that contrast).
+ # Any customization we do to allow, e.g., label wrapping will probably
+ # require defining our own Formatter subclass.
+ # We could also potentially implement auto-wrapping in an Axis subclass
+ # (see Axis.draw ... it already is computing the bboxes).
+ # major_locator, minor_locator = new._get_locators(**new._tick_params)
+ # major_formatter = new._get_formatter(major_locator, **new._label_params)
+
+ class CatScale(mpl.scale.LinearScale):
+ def set_default_locators_and_formatters(self, axis):
+ ...
+ # axis.set_major_locator(major_locator)
+ # if minor_locator is not None:
+ # axis.set_minor_locator(minor_locator)
+ # axis.set_major_formatter(major_formatter)
+
+ mpl_scale = CatScale(data.name)
+ if axis is None:
+ axis = PseudoAxis(mpl_scale)
+
+ # TODO Currently just used in non-Coordinate contexts, but should
+ # we use this to (A) set the padding we want for categorial plots
+ # and (B) allow the values parameter for a Coordinate to set xlim/ylim
+ axis.set_view_interval(0, len(units_seed) - 1)
+
+ new._matplotlib_scale = mpl_scale
+
+ # TODO array cast necessary to handle float/int mixture, which we need
+ # to solve in a more systematic way probably
+ # (i.e. if we have [1, 2.5], do we want [1.0, 2.5]? Unclear)
+ axis.update_units(stringify(np.array(units_seed)))
+
+ # TODO define this more centrally
+ def convert_units(x):
+ # TODO only do this with explicit order?
+ # (But also category dtype?)
+ # TODO isin fails when units_seed mixes numbers and strings (numpy error?)
+ # but np.isin also does not seem any faster? (Maybe not broadcasting in C)
+ # keep = x.isin(units_seed)
+ keep = np.array([x_ in units_seed for x_ in x], bool)
+ out = np.full(len(x), np.nan)
+ out[keep] = axis.convert_units(stringify(x[keep]))
+ return out
+
+ new._pipeline = [convert_units, prop.get_mapping(new, data)]
+ new._spacer = _default_spacer
+
+ if prop.legend:
+ new._legend = units_seed, list(stringify(units_seed))
+
+ return new
+
+ def _finalize(self, p: Plot, axis: Axis) -> None:
+
+ ax = axis.axes
+ name = axis.axis_name
+ axis.grid(False, which="both")
+ if name not in p._limits:
+ nticks = len(axis.get_major_ticks())
+ lo, hi = -.5, nticks - .5
+ if name == "y":
+ lo, hi = hi, lo
+ set_lim = getattr(ax, f"set_{name}lim")
+ set_lim(lo, hi, auto=None)
+
+ def tick(self, locator: Locator | None = None) -> Nominal:
+ """
+ Configure the selection of ticks for the scale's axis or legend.
+
+ .. note::
+ This API is under construction and will be enhanced over time.
+ At the moment, it is probably not very useful.
+
+ Parameters
+ ----------
+ locator : :class:`matplotlib.ticker.Locator` subclass
+ Pre-configured matplotlib locator; other parameters will not be used.
+
+ Returns
+ -------
+ Copy of self with new tick configuration.
+
+ """
+ new = copy(self)
+ new._tick_params = {"locator": locator}
+ return new
+
+ def label(self, formatter: Formatter | None = None) -> Nominal:
+ """
+ Configure the selection of labels for the scale's axis or legend.
+
+ .. note::
+ This API is under construction and will be enhanced over time.
+ At the moment, it is probably not very useful.
+
+ Parameters
+ ----------
+ formatter : :class:`matplotlib.ticker.Formatter` subclass
+ Pre-configured matplotlib formatter; other parameters will not be used.
+
+ Returns
+ -------
+ scale
+ Copy of self with new tick configuration.
+
+ """
+ new = copy(self)
+ new._label_params = {"formatter": formatter}
+ return new
+
+ def _get_locators(self, locator):
+
+ if locator is not None:
+ return locator, None
+
+ locator = mpl.category.StrCategoryLocator({})
+
+ return locator, None
+
+ def _get_formatter(self, locator, formatter):
+
+ if formatter is not None:
+ return formatter
+
+ formatter = mpl.category.StrCategoryFormatter({})
+
+ return formatter
+
+
+@dataclass
+class Ordinal(Scale):
+ # Categorical (convert to strings), sortable, can skip ticklabels
+ ...
+
+
+@dataclass
+class Discrete(Scale):
+ # Numeric, integral, can skip ticks/ticklabels
+ ...
+
+
+@dataclass
+class ContinuousBase(Scale):
+
+ values: tuple | str | None = None
+ norm: tuple | None = None
+
+ def _setup(
+ self, data: Series, prop: Property, axis: Axis | None = None,
+ ) -> Scale:
+
+ new = copy(self)
+ if new._tick_params is None:
+ new = new.tick()
+ if new._label_params is None:
+ new = new.label()
+
+ forward, inverse = new._get_transform()
+
+ mpl_scale = new._get_scale(str(data.name), forward, inverse)
+
+ if axis is None:
+ axis = PseudoAxis(mpl_scale)
+ axis.update_units(data)
+
+ mpl_scale.set_default_locators_and_formatters(axis)
+ new._matplotlib_scale = mpl_scale
+
+ normalize: Optional[Callable[[ArrayLike], ArrayLike]]
+ if prop.normed:
+ if new.norm is None:
+ vmin, vmax = data.min(), data.max()
+ else:
+ vmin, vmax = new.norm
+ vmin, vmax = map(float, axis.convert_units((vmin, vmax)))
+ a = forward(vmin)
+ b = forward(vmax) - forward(vmin)
+
+ def normalize(x):
+ return (x - a) / b
+
+ else:
+ normalize = vmin = vmax = None
+
+ new._pipeline = [
+ axis.convert_units,
+ forward,
+ normalize,
+ prop.get_mapping(new, data)
+ ]
+
+ def spacer(x):
+ x = x.dropna().unique()
+ if len(x) < 2:
+ return np.nan
+ return np.min(np.diff(np.sort(x)))
+ new._spacer = spacer
+
+ # TODO How to allow disabling of legend for all uses of property?
+ # Could add a Scale parameter, or perhaps Scale.suppress()?
+ # Are there other useful parameters that would be in Scale.legend()
+ # besides allowing Scale.legend(False)?
+ if prop.legend:
+ axis.set_view_interval(vmin, vmax)
+ locs = axis.major.locator()
+ locs = locs[(vmin <= locs) & (locs <= vmax)]
+ # Avoid having an offset / scientific notation in a legend
+ # as we don't represent that anywhere so it ends up incorrect.
+ # This could become an option (e.g. Continuous.label(offset=True))
+ # in which case we would need to figure out how to show it.
+ if hasattr(axis.major.formatter, "set_useOffset"):
+ axis.major.formatter.set_useOffset(False)
+ if hasattr(axis.major.formatter, "set_scientific"):
+ axis.major.formatter.set_scientific(False)
+ labels = axis.major.formatter.format_ticks(locs)
+ new._legend = list(locs), list(labels)
+
+ return new
+
+ def _get_transform(self):
+
+ arg = self.trans
+
+ def get_param(method, default):
+ if arg == method:
+ return default
+ return float(arg[len(method):])
+
+ if arg is None:
+ return _make_identity_transforms()
+ elif isinstance(arg, tuple):
+ return arg
+ elif isinstance(arg, str):
+ if arg == "ln":
+ return _make_log_transforms()
+ elif arg == "logit":
+ base = get_param("logit", 10)
+ return _make_logit_transforms(base)
+ elif arg.startswith("log"):
+ base = get_param("log", 10)
+ return _make_log_transforms(base)
+ elif arg.startswith("symlog"):
+ c = get_param("symlog", 1)
+ return _make_symlog_transforms(c)
+ elif arg.startswith("pow"):
+ exp = get_param("pow", 2)
+ return _make_power_transforms(exp)
+ elif arg == "sqrt":
+ return _make_sqrt_transforms()
+ else:
+ raise ValueError(f"Unknown value provided for trans: {arg!r}")
+
+
+@dataclass
+class Continuous(ContinuousBase):
+ """
+ A numeric scale supporting norms and functional transforms.
+ """
+ values: tuple | str | None = None
+ trans: str | TransFuncs | None = None
+
+ # TODO Add this to deal with outliers?
+ # outside: Literal["keep", "drop", "clip"] = "keep"
+
+ _priority: ClassVar[int] = 1
+
+ def tick(
+ self,
+ locator: Locator | None = None, *,
+ at: Sequence[float] | None = None,
+ upto: int | None = None,
+ count: int | None = None,
+ every: float | None = None,
+ between: tuple[float, float] | None = None,
+ minor: int | None = None,
+ ) -> Continuous:
+ """
+ Configure the selection of ticks for the scale's axis or legend.
+
+ Parameters
+ ----------
+ locator : :class:`matplotlib.ticker.Locator` subclass
+ Pre-configured matplotlib locator; other parameters will not be used.
+ at : sequence of floats
+ Place ticks at these specific locations (in data units).
+ upto : int
+ Choose "nice" locations for ticks, but do not exceed this number.
+ count : int
+ Choose exactly this number of ticks, bounded by `between` or axis limits.
+ every : float
+ Choose locations at this interval of separation (in data units).
+ between : pair of floats
+ Bound upper / lower ticks when using `every` or `count`.
+ minor : int
+ Number of unlabeled ticks to draw between labeled "major" ticks.
+
+ Returns
+ -------
+ scale
+ Copy of self with new tick configuration.
+
+ """
+ # Input checks
+ if locator is not None and not isinstance(locator, Locator):
+ raise TypeError(
+ f"Tick locator must be an instance of {Locator!r}, "
+ f"not {type(locator)!r}."
+ )
+ log_base, symlog_thresh = self._parse_for_log_params(self.trans)
+ if log_base or symlog_thresh:
+ if count is not None and between is None:
+ raise RuntimeError("`count` requires `between` with log transform.")
+ if every is not None:
+ raise RuntimeError("`every` not supported with log transform.")
+
+ new = copy(self)
+ new._tick_params = {
+ "locator": locator,
+ "at": at,
+ "upto": upto,
+ "count": count,
+ "every": every,
+ "between": between,
+ "minor": minor,
+ }
+ return new
+
+ def label(
+ self,
+ formatter: Formatter | None = None, *,
+ like: str | Callable | None = None,
+ base: int | None | Default = default,
+ unit: str | None = None,
+ ) -> Continuous:
+ """
+ Configure the appearance of tick labels for the scale's axis or legend.
+
+ Parameters
+ ----------
+ formatter : :class:`matplotlib.ticker.Formatter` subclass
+ Pre-configured formatter to use; other parameters will be ignored.
+ like : str or callable
+ Either a format pattern (e.g., `".2f"`), a format string with fields named
+ `x` and/or `pos` (e.g., `"${x:.2f}"`), or a callable with a signature like
+ `f(x: float, pos: int) -> str`. In the latter variants, `x` is passed as the
+ tick value and `pos` is passed as the tick index.
+ base : number
+ Use log formatter (with scientific notation) having this value as the base.
+ Set to `None` to override the default formatter with a log transform.
+ unit : str or (str, str) tuple
+ Use SI prefixes with these units (e.g., with `unit="g"`, a tick value
+ of 5000 will appear as `5 kg`). When a tuple, the first element gives the
+ separator between the number and unit.
+
+ Returns
+ -------
+ scale
+ Copy of self with new label configuration.
+
+ """
+ # Input checks
+ if formatter is not None and not isinstance(formatter, Formatter):
+ raise TypeError(
+ f"Label formatter must be an instance of {Formatter!r}, "
+ f"not {type(formatter)!r}"
+ )
+ if like is not None and not (isinstance(like, str) or callable(like)):
+ msg = f"`like` must be a string or callable, not {type(like).__name__}."
+ raise TypeError(msg)
+
+ new = copy(self)
+ new._label_params = {
+ "formatter": formatter,
+ "like": like,
+ "base": base,
+ "unit": unit,
+ }
+ return new
+
+ def _parse_for_log_params(
+ self, trans: str | TransFuncs | None
+ ) -> tuple[float | None, float | None]:
+
+ log_base = symlog_thresh = None
+ if isinstance(trans, str):
+ m = re.match(r"^log(\d*)", trans)
+ if m is not None:
+ log_base = float(m[1] or 10)
+ m = re.match(r"symlog(\d*)", trans)
+ if m is not None:
+ symlog_thresh = float(m[1] or 1)
+ return log_base, symlog_thresh
+
+ def _get_locators(self, locator, at, upto, count, every, between, minor):
+
+ log_base, symlog_thresh = self._parse_for_log_params(self.trans)
+
+ if locator is not None:
+ major_locator = locator
+
+ elif upto is not None:
+ if log_base:
+ major_locator = LogLocator(base=log_base, numticks=upto)
+ else:
+ major_locator = MaxNLocator(upto, steps=[1, 1.5, 2, 2.5, 3, 5, 10])
+
+ elif count is not None:
+ if between is None:
+ # This is rarely useful (unless you are setting limits)
+ major_locator = LinearLocator(count)
+ else:
+ if log_base or symlog_thresh:
+ forward, inverse = self._get_transform()
+ lo, hi = forward(between)
+ ticks = inverse(np.linspace(lo, hi, num=count))
+ else:
+ ticks = np.linspace(*between, num=count)
+ major_locator = FixedLocator(ticks)
+
+ elif every is not None:
+ if between is None:
+ major_locator = MultipleLocator(every)
+ else:
+ lo, hi = between
+ ticks = np.arange(lo, hi + every, every)
+ major_locator = FixedLocator(ticks)
+
+ elif at is not None:
+ major_locator = FixedLocator(at)
+
+ else:
+ if log_base:
+ major_locator = LogLocator(log_base)
+ elif symlog_thresh:
+ major_locator = SymmetricalLogLocator(linthresh=symlog_thresh, base=10)
+ else:
+ major_locator = AutoLocator()
+
+ if minor is None:
+ minor_locator = LogLocator(log_base, subs=None) if log_base else None
+ else:
+ if log_base:
+ subs = np.linspace(0, log_base, minor + 2)[1:-1]
+ minor_locator = LogLocator(log_base, subs=subs)
+ else:
+ minor_locator = AutoMinorLocator(minor + 1)
+
+ return major_locator, minor_locator
+
+ def _get_formatter(self, locator, formatter, like, base, unit):
+
+ log_base, symlog_thresh = self._parse_for_log_params(self.trans)
+ if base is default:
+ if symlog_thresh:
+ log_base = 10
+ base = log_base
+
+ if formatter is not None:
+ return formatter
+
+ if like is not None:
+ if isinstance(like, str):
+ if "{x" in like or "{pos" in like:
+ fmt = like
+ else:
+ fmt = f"{{x:{like}}}"
+ formatter = StrMethodFormatter(fmt)
+ else:
+ formatter = FuncFormatter(like)
+
+ elif base is not None:
+ # We could add other log options if necessary
+ formatter = LogFormatterSciNotation(base)
+
+ elif unit is not None:
+ if isinstance(unit, tuple):
+ sep, unit = unit
+ elif not unit:
+ sep = ""
+ else:
+ sep = " "
+ formatter = EngFormatter(unit, sep=sep)
+
+ else:
+ formatter = ScalarFormatter()
+
+ return formatter
+
+
+@dataclass
+class Temporal(ContinuousBase):
+ """
+ A scale for date/time data.
+ """
+ # TODO date: bool?
+ # For when we only care about the time component, would affect
+ # default formatter and norm conversion. Should also happen in
+ # Property.default_scale. The alternative was having distinct
+ # Calendric / Temporal scales, but that feels a bit fussy, and it
+ # would get in the way of using first-letter shorthands because
+ # Calendric and Continuous would collide. Still, we haven't implemented
+ # those yet, and having a clear distinction betewen date(time) / time
+ # may be more useful.
+
+ trans = None
+
+ _priority: ClassVar[int] = 2
+
+ def tick(
+ self, locator: Locator | None = None, *,
+ upto: int | None = None,
+ ) -> Temporal:
+ """
+ Configure the selection of ticks for the scale's axis or legend.
+
+ .. note::
+ This API is under construction and will be enhanced over time.
+
+ Parameters
+ ----------
+ locator : :class:`matplotlib.ticker.Locator` subclass
+ Pre-configured matplotlib locator; other parameters will not be used.
+ upto : int
+ Choose "nice" locations for ticks, but do not exceed this number.
+
+ Returns
+ -------
+ scale
+ Copy of self with new tick configuration.
+
+ """
+ if locator is not None and not isinstance(locator, Locator):
+ err = (
+ f"Tick locator must be an instance of {Locator!r}, "
+ f"not {type(locator)!r}."
+ )
+ raise TypeError(err)
+
+ new = copy(self)
+ new._tick_params = {"locator": locator, "upto": upto}
+ return new
+
+ def label(
+ self,
+ formatter: Formatter | None = None, *,
+ concise: bool = False,
+ ) -> Temporal:
+ """
+ Configure the appearance of tick labels for the scale's axis or legend.
+
+ .. note::
+ This API is under construction and will be enhanced over time.
+
+ Parameters
+ ----------
+ formatter : :class:`matplotlib.ticker.Formatter` subclass
+ Pre-configured formatter to use; other parameters will be ignored.
+ concise : bool
+ If True, use :class:`matplotlib.dates.ConciseDateFormatter` to make
+ the tick labels as compact as possible.
+
+ Returns
+ -------
+ scale
+ Copy of self with new label configuration.
+
+ """
+ new = copy(self)
+ new._label_params = {"formatter": formatter, "concise": concise}
+ return new
+
+ def _get_locators(self, locator, upto):
+
+ if locator is not None:
+ major_locator = locator
+ elif upto is not None:
+ major_locator = AutoDateLocator(minticks=2, maxticks=upto)
+
+ else:
+ major_locator = AutoDateLocator(minticks=2, maxticks=6)
+ minor_locator = None
+
+ return major_locator, minor_locator
+
+ def _get_formatter(self, locator, formatter, concise):
+
+ if formatter is not None:
+ return formatter
+
+ if concise:
+ # TODO ideally we would have concise coordinate ticks,
+ # but full semantic ticks. Is that possible?
+ formatter = ConciseDateFormatter(locator)
+ else:
+ formatter = AutoDateFormatter(locator)
+
+ return formatter
+
+
+# ----------------------------------------------------------------------------------- #
+
+
+# TODO Have this separate from Temporal or have Temporal(date=True) or similar?
+# class Calendric(Scale):
+
+# TODO Needed? Or handle this at layer (in stat or as param, eg binning=)
+# class Binned(Scale):
+
+# TODO any need for color-specific scales?
+# class Sequential(Continuous):
+# class Diverging(Continuous):
+# class Qualitative(Nominal):
+
+
+# ----------------------------------------------------------------------------------- #
+
+
+class PseudoAxis:
+ """
+ Internal class implementing minimal interface equivalent to matplotlib Axis.
+
+ Coordinate variables are typically scaled by attaching the Axis object from
+ the figure where the plot will end up. Matplotlib has no similar concept of
+ and axis for the other mappable variables (color, etc.), but to simplify the
+ code, this object acts like an Axis and can be used to scale other variables.
+
+ """
+ axis_name = "" # Matplotlib requirement but not actually used
+
+ def __init__(self, scale):
+
+ self.converter = None
+ self.units = None
+ self.scale = scale
+ self.major = mpl.axis.Ticker()
+ self.minor = mpl.axis.Ticker()
+
+ # It appears that this needs to be initialized this way on matplotlib 3.1,
+ # but not later versions. It is unclear whether there are any issues with it.
+ self._data_interval = None, None
+
+ scale.set_default_locators_and_formatters(self)
+ # self.set_default_intervals() Is this ever needed?
+
+ def set_view_interval(self, vmin, vmax):
+ self._view_interval = vmin, vmax
+
+ def get_view_interval(self):
+ return self._view_interval
+
+ # TODO do we want to distinguish view/data intervals? e.g. for a legend
+ # we probably want to represent the full range of the data values, but
+ # still norm the colormap. If so, we'll need to track data range separately
+ # from the norm, which we currently don't do.
+
+ def set_data_interval(self, vmin, vmax):
+ self._data_interval = vmin, vmax
+
+ def get_data_interval(self):
+ return self._data_interval
+
+ def get_tick_space(self):
+ # TODO how to do this in a configurable / auto way?
+ # Would be cool to have legend density adapt to figure size, etc.
+ return 5
+
+ def set_major_locator(self, locator):
+ self.major.locator = locator
+ locator.set_axis(self)
+
+ def set_major_formatter(self, formatter):
+ self.major.formatter = formatter
+ formatter.set_axis(self)
+
+ def set_minor_locator(self, locator):
+ self.minor.locator = locator
+ locator.set_axis(self)
+
+ def set_minor_formatter(self, formatter):
+ self.minor.formatter = formatter
+ formatter.set_axis(self)
+
+ def set_units(self, units):
+ self.units = units
+
+ def update_units(self, x):
+ """Pass units to the internal converter, potentially updating its mapping."""
+ self.converter = mpl.units.registry.get_converter(x)
+ if self.converter is not None:
+ self.converter.default_units(x, self)
+
+ info = self.converter.axisinfo(self.units, self)
+
+ if info is None:
+ return
+ if info.majloc is not None:
+ self.set_major_locator(info.majloc)
+ if info.majfmt is not None:
+ self.set_major_formatter(info.majfmt)
+
+ # This is in matplotlib method; do we need this?
+ # self.set_default_intervals()
+
+ def convert_units(self, x):
+ """Return a numeric representation of the input data."""
+ if np.issubdtype(np.asarray(x).dtype, np.number):
+ return x
+ elif self.converter is None:
+ return x
+ return self.converter.convert(x, self.units, self)
+
+ def get_scale(self):
+ # Note that matplotlib actually returns a string here!
+ # (e.g., with a log scale, axis.get_scale() returns "log")
+ # Currently we just hit it with minor ticks where it checks for
+ # scale == "log". I'm not sure how you'd actually use log-scale
+ # minor "ticks" in a legend context, so this is fine....
+ return self.scale
+
+ def get_majorticklocs(self):
+ return self.major.locator()
+
+
+# ------------------------------------------------------------------------------------ #
+# Transform function creation
+
+
+def _make_identity_transforms() -> TransFuncs:
+
+ def identity(x):
+ return x
+
+ return identity, identity
+
+
+def _make_logit_transforms(base: float | None = None) -> TransFuncs:
+
+ log, exp = _make_log_transforms(base)
+
+ def logit(x):
+ with np.errstate(invalid="ignore", divide="ignore"):
+ return log(x) - log(1 - x)
+
+ def expit(x):
+ with np.errstate(invalid="ignore", divide="ignore"):
+ return exp(x) / (1 + exp(x))
+
+ return logit, expit
+
+
+def _make_log_transforms(base: float | None = None) -> TransFuncs:
+
+ fs: TransFuncs
+ if base is None:
+ fs = np.log, np.exp
+ elif base == 2:
+ fs = np.log2, partial(np.power, 2)
+ elif base == 10:
+ fs = np.log10, partial(np.power, 10)
+ else:
+ def forward(x):
+ return np.log(x) / np.log(base)
+ fs = forward, partial(np.power, base)
+
+ def log(x: ArrayLike) -> ArrayLike:
+ with np.errstate(invalid="ignore", divide="ignore"):
+ return fs[0](x)
+
+ def exp(x: ArrayLike) -> ArrayLike:
+ with np.errstate(invalid="ignore", divide="ignore"):
+ return fs[1](x)
+
+ return log, exp
+
+
+def _make_symlog_transforms(c: float = 1, base: float = 10) -> TransFuncs:
+
+ # From https://iopscience.iop.org/article/10.1088/0957-0233/24/2/027001
+
+ # Note: currently not using base because we only get
+ # one parameter from the string, and are using c (this is consistent with d3)
+
+ log, exp = _make_log_transforms(base)
+
+ def symlog(x):
+ with np.errstate(invalid="ignore", divide="ignore"):
+ return np.sign(x) * log(1 + np.abs(np.divide(x, c)))
+
+ def symexp(x):
+ with np.errstate(invalid="ignore", divide="ignore"):
+ return np.sign(x) * c * (exp(np.abs(x)) - 1)
+
+ return symlog, symexp
+
+
+def _make_sqrt_transforms() -> TransFuncs:
+
+ def sqrt(x):
+ return np.sign(x) * np.sqrt(np.abs(x))
+
+ def square(x):
+ return np.sign(x) * np.square(x)
+
+ return sqrt, square
+
+
+def _make_power_transforms(exp: float) -> TransFuncs:
+
+ def forward(x):
+ return np.sign(x) * np.power(np.abs(x), exp)
+
+ def inverse(x):
+ return np.sign(x) * np.power(np.abs(x), 1 / exp)
+
+ return forward, inverse
+
+
+def _default_spacer(x: Series) -> float:
+ return 1
diff --git a/seaborn/_core/subplots.py b/seaborn/_core/subplots.py
new file mode 100644
index 0000000000000000000000000000000000000000..287f441670881f0967f674cdec6d607af58029ef
--- /dev/null
+++ b/seaborn/_core/subplots.py
@@ -0,0 +1,263 @@
+from __future__ import annotations
+from collections.abc import Generator
+
+import numpy as np
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+
+from matplotlib.axes import Axes
+from matplotlib.figure import Figure
+from typing import TYPE_CHECKING
+if TYPE_CHECKING: # TODO move to seaborn._core.typing?
+ from seaborn._core.plot import FacetSpec, PairSpec
+ from matplotlib.figure import SubFigure
+
+
+class Subplots:
+ """
+ Interface for creating and using matplotlib subplots based on seaborn parameters.
+
+ Parameters
+ ----------
+ subplot_spec : dict
+ Keyword args for :meth:`matplotlib.figure.Figure.subplots`.
+ facet_spec : dict
+ Parameters that control subplot faceting.
+ pair_spec : dict
+ Parameters that control subplot pairing.
+ data : PlotData
+ Data used to define figure setup.
+
+ """
+ def __init__(
+ self,
+ subplot_spec: dict, # TODO define as TypedDict
+ facet_spec: FacetSpec,
+ pair_spec: PairSpec,
+ ):
+
+ self.subplot_spec = subplot_spec
+
+ self._check_dimension_uniqueness(facet_spec, pair_spec)
+ self._determine_grid_dimensions(facet_spec, pair_spec)
+ self._handle_wrapping(facet_spec, pair_spec)
+ self._determine_axis_sharing(pair_spec)
+
+ def _check_dimension_uniqueness(
+ self, facet_spec: FacetSpec, pair_spec: PairSpec
+ ) -> None:
+ """Reject specs that pair and facet on (or wrap to) same figure dimension."""
+ err = None
+
+ facet_vars = facet_spec.get("variables", {})
+
+ if facet_spec.get("wrap") and {"col", "row"} <= set(facet_vars):
+ err = "Cannot wrap facets when specifying both `col` and `row`."
+ elif (
+ pair_spec.get("wrap")
+ and pair_spec.get("cross", True)
+ and len(pair_spec.get("structure", {}).get("x", [])) > 1
+ and len(pair_spec.get("structure", {}).get("y", [])) > 1
+ ):
+ err = "Cannot wrap subplots when pairing on both `x` and `y`."
+
+ collisions = {"x": ["columns", "rows"], "y": ["rows", "columns"]}
+ for pair_axis, (multi_dim, wrap_dim) in collisions.items():
+ if pair_axis not in pair_spec.get("structure", {}):
+ continue
+ elif multi_dim[:3] in facet_vars:
+ err = f"Cannot facet the {multi_dim} while pairing on `{pair_axis}``."
+ elif wrap_dim[:3] in facet_vars and facet_spec.get("wrap"):
+ err = f"Cannot wrap the {wrap_dim} while pairing on `{pair_axis}``."
+ elif wrap_dim[:3] in facet_vars and pair_spec.get("wrap"):
+ err = f"Cannot wrap the {multi_dim} while faceting the {wrap_dim}."
+
+ if err is not None:
+ raise RuntimeError(err) # TODO what err class? Define PlotSpecError?
+
+ def _determine_grid_dimensions(
+ self, facet_spec: FacetSpec, pair_spec: PairSpec
+ ) -> None:
+ """Parse faceting and pairing information to define figure structure."""
+ self.grid_dimensions: dict[str, list] = {}
+ for dim, axis in zip(["col", "row"], ["x", "y"]):
+
+ facet_vars = facet_spec.get("variables", {})
+ if dim in facet_vars:
+ self.grid_dimensions[dim] = facet_spec["structure"][dim]
+ elif axis in pair_spec.get("structure", {}):
+ self.grid_dimensions[dim] = [
+ None for _ in pair_spec.get("structure", {})[axis]
+ ]
+ else:
+ self.grid_dimensions[dim] = [None]
+
+ self.subplot_spec[f"n{dim}s"] = len(self.grid_dimensions[dim])
+
+ if not pair_spec.get("cross", True):
+ self.subplot_spec["nrows"] = 1
+
+ self.n_subplots = self.subplot_spec["ncols"] * self.subplot_spec["nrows"]
+
+ def _handle_wrapping(
+ self, facet_spec: FacetSpec, pair_spec: PairSpec
+ ) -> None:
+ """Update figure structure parameters based on facet/pair wrapping."""
+ self.wrap = wrap = facet_spec.get("wrap") or pair_spec.get("wrap")
+ if not wrap:
+ return
+
+ wrap_dim = "row" if self.subplot_spec["nrows"] > 1 else "col"
+ flow_dim = {"row": "col", "col": "row"}[wrap_dim]
+ n_subplots = self.subplot_spec[f"n{wrap_dim}s"]
+ flow = int(np.ceil(n_subplots / wrap))
+
+ if wrap < self.subplot_spec[f"n{wrap_dim}s"]:
+ self.subplot_spec[f"n{wrap_dim}s"] = wrap
+ self.subplot_spec[f"n{flow_dim}s"] = flow
+ self.n_subplots = n_subplots
+ self.wrap_dim = wrap_dim
+
+ def _determine_axis_sharing(self, pair_spec: PairSpec) -> None:
+ """Update subplot spec with default or specified axis sharing parameters."""
+ axis_to_dim = {"x": "col", "y": "row"}
+ key: str
+ val: str | bool
+ for axis in "xy":
+ key = f"share{axis}"
+ # Always use user-specified value, if present
+ if key not in self.subplot_spec:
+ if axis in pair_spec.get("structure", {}):
+ # Paired axes are shared along one dimension by default
+ if self.wrap is None and pair_spec.get("cross", True):
+ val = axis_to_dim[axis]
+ else:
+ val = False
+ else:
+ # This will pick up faceted plots, as well as single subplot
+ # figures, where the value doesn't really matter
+ val = True
+ self.subplot_spec[key] = val
+
+ def init_figure(
+ self,
+ pair_spec: PairSpec,
+ pyplot: bool = False,
+ figure_kws: dict | None = None,
+ target: Axes | Figure | SubFigure | None = None,
+ ) -> Figure:
+ """Initialize matplotlib objects and add seaborn-relevant metadata."""
+ # TODO reduce need to pass pair_spec here?
+
+ if figure_kws is None:
+ figure_kws = {}
+
+ if isinstance(target, mpl.axes.Axes):
+
+ if max(self.subplot_spec["nrows"], self.subplot_spec["ncols"]) > 1:
+ err = " ".join([
+ "Cannot create multiple subplots after calling `Plot.on` with",
+ f"a {mpl.axes.Axes} object.",
+ f" You may want to use a {mpl.figure.SubFigure} instead.",
+ ])
+ raise RuntimeError(err)
+
+ self._subplot_list = [{
+ "ax": target,
+ "left": True,
+ "right": True,
+ "top": True,
+ "bottom": True,
+ "col": None,
+ "row": None,
+ "x": "x",
+ "y": "y",
+ }]
+ self._figure = target.figure
+ return self._figure
+
+ elif isinstance(target, mpl.figure.SubFigure):
+ figure = target.figure
+ elif isinstance(target, mpl.figure.Figure):
+ figure = target
+ else:
+ if pyplot:
+ figure = plt.figure(**figure_kws)
+ else:
+ figure = mpl.figure.Figure(**figure_kws)
+ target = figure
+ self._figure = figure
+
+ axs = target.subplots(**self.subplot_spec, squeeze=False)
+
+ if self.wrap:
+ # Remove unused Axes and flatten the rest into a (2D) vector
+ axs_flat = axs.ravel({"col": "C", "row": "F"}[self.wrap_dim])
+ axs, extra = np.split(axs_flat, [self.n_subplots])
+ for ax in extra:
+ ax.remove()
+ if self.wrap_dim == "col":
+ axs = axs[np.newaxis, :]
+ else:
+ axs = axs[:, np.newaxis]
+
+ # Get i, j coordinates for each Axes object
+ # Note that i, j are with respect to faceting/pairing,
+ # not the subplot grid itself, (which only matters in the case of wrapping).
+ iter_axs: np.ndenumerate | zip
+ if not pair_spec.get("cross", True):
+ indices = np.arange(self.n_subplots)
+ iter_axs = zip(zip(indices, indices), axs.flat)
+ else:
+ iter_axs = np.ndenumerate(axs)
+
+ self._subplot_list = []
+ for (i, j), ax in iter_axs:
+
+ info = {"ax": ax}
+
+ nrows, ncols = self.subplot_spec["nrows"], self.subplot_spec["ncols"]
+ if not self.wrap:
+ info["left"] = j % ncols == 0
+ info["right"] = (j + 1) % ncols == 0
+ info["top"] = i == 0
+ info["bottom"] = i == nrows - 1
+ elif self.wrap_dim == "col":
+ info["left"] = j % ncols == 0
+ info["right"] = ((j + 1) % ncols == 0) or ((j + 1) == self.n_subplots)
+ info["top"] = j < ncols
+ info["bottom"] = j >= (self.n_subplots - ncols)
+ elif self.wrap_dim == "row":
+ info["left"] = i < nrows
+ info["right"] = i >= self.n_subplots - nrows
+ info["top"] = i % nrows == 0
+ info["bottom"] = ((i + 1) % nrows == 0) or ((i + 1) == self.n_subplots)
+
+ if not pair_spec.get("cross", True):
+ info["top"] = j < ncols
+ info["bottom"] = j >= self.n_subplots - ncols
+
+ for dim in ["row", "col"]:
+ idx = {"row": i, "col": j}[dim]
+ info[dim] = self.grid_dimensions[dim][idx]
+
+ for axis in "xy":
+
+ idx = {"x": j, "y": i}[axis]
+ if axis in pair_spec.get("structure", {}):
+ key = f"{axis}{idx}"
+ else:
+ key = axis
+ info[axis] = key
+
+ self._subplot_list.append(info)
+
+ return figure
+
+ def __iter__(self) -> Generator[dict, None, None]: # TODO TypedDict?
+ """Yield each subplot dictionary with Axes object and metadata."""
+ yield from self._subplot_list
+
+ def __len__(self) -> int:
+ """Return the number of subplots in this figure."""
+ return len(self._subplot_list)
diff --git a/seaborn/_core/typing.py b/seaborn/_core/typing.py
new file mode 100644
index 0000000000000000000000000000000000000000..9bdf8a6ef88dd112fa842f3aff026413ab906dbb
--- /dev/null
+++ b/seaborn/_core/typing.py
@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from datetime import date, datetime, timedelta
+from typing import Any, Optional, Union, Tuple, List, Dict
+
+from numpy import ndarray # TODO use ArrayLike?
+from pandas import Series, Index, Timestamp, Timedelta
+from matplotlib.colors import Colormap, Normalize
+
+
+ColumnName = Union[
+ str, bytes, date, datetime, timedelta, bool, complex, Timestamp, Timedelta
+]
+Vector = Union[Series, Index, ndarray]
+
+VariableSpec = Union[ColumnName, Vector, None]
+VariableSpecList = Union[List[VariableSpec], Index, None]
+
+# A DataSource can be an object implementing __dataframe__, or a Mapping
+# (and is optional in all contexts where it is used).
+# I don't think there's an abc for "has __dataframe__", so we type as object
+# but keep the (slightly odd) Union alias for better user-facing annotations.
+DataSource = Union[object, Mapping, None]
+
+OrderSpec = Union[Iterable, None] # TODO technically str is iterable
+NormSpec = Union[Tuple[Optional[float], Optional[float]], Normalize, None]
+
+# TODO for discrete mappings, it would be ideal to use a parameterized type
+# as the dict values / list entries should be of specific type(s) for each method
+PaletteSpec = Union[str, list, dict, Colormap, None]
+DiscreteValueSpec = Union[dict, list, None]
+ContinuousValueSpec = Union[
+ Tuple[float, float], List[float], Dict[Any, float], None,
+]
+
+
+class Default:
+ def __repr__(self):
+ return "<default>"
+
+
+class Deprecated:
+ def __repr__(self):
+ return "<deprecated>"
+
+
+default = Default()
+deprecated = Deprecated()
diff --git a/seaborn/_docstrings.py b/seaborn/_docstrings.py
new file mode 100644
index 0000000000000000000000000000000000000000..2ab210b6ffbf63f21ebee9a4a3d59dcbc94fcb57
--- /dev/null
+++ b/seaborn/_docstrings.py
@@ -0,0 +1,198 @@
+import re
+import pydoc
+from .external.docscrape import NumpyDocString
+
+
+class DocstringComponents:
+
+ regexp = re.compile(r"\n((\n|.)+)\n\s*", re.MULTILINE)
+
+ def __init__(self, comp_dict, strip_whitespace=True):
+ """Read entries from a dict, optionally stripping outer whitespace."""
+ if strip_whitespace:
+ entries = {}
+ for key, val in comp_dict.items():
+ m = re.match(self.regexp, val)
+ if m is None:
+ entries[key] = val
+ else:
+ entries[key] = m.group(1)
+ else:
+ entries = comp_dict.copy()
+
+ self.entries = entries
+
+ def __getattr__(self, attr):
+ """Provide dot access to entries for clean raw docstrings."""
+ if attr in self.entries:
+ return self.entries[attr]
+ else:
+ try:
+ return self.__getattribute__(attr)
+ except AttributeError as err:
+ # If Python is run with -OO, it will strip docstrings and our lookup
+ # from self.entries will fail. We check for __debug__, which is actually
+ # set to False by -O (it is True for normal execution).
+ # But we only want to see an error when building the docs;
+ # not something users should see, so this slight inconsistency is fine.
+ if __debug__:
+ raise err
+ else:
+ pass
+
+ @classmethod
+ def from_nested_components(cls, **kwargs):
+ """Add multiple sub-sets of components."""
+ return cls(kwargs, strip_whitespace=False)
+
+ @classmethod
+ def from_function_params(cls, func):
+ """Use the numpydoc parser to extract components from existing func."""
+ params = NumpyDocString(pydoc.getdoc(func))["Parameters"]
+ comp_dict = {}
+ for p in params:
+ name = p.name
+ type = p.type
+ desc = "\n ".join(p.desc)
+ comp_dict[name] = f"{name} : {type}\n {desc}"
+
+ return cls(comp_dict)
+
+
+# TODO is "vector" the best term here? We mean to imply 1D data with a variety
+# of types?
+
+# TODO now that we can parse numpydoc style strings, do we need to define dicts
+# of docstring components, or just write out a docstring?
+
+
+_core_params = dict(
+ data="""
+data : :class:`pandas.DataFrame`, :class:`numpy.ndarray`, mapping, or sequence
+ Input data structure. Either a long-form collection of vectors that can be
+ assigned to named variables or a wide-form dataset that will be internally
+ reshaped.
+ """, # TODO add link to user guide narrative when exists
+ xy="""
+x, y : vectors or keys in ``data``
+ Variables that specify positions on the x and y axes.
+ """,
+ hue="""
+hue : vector or key in ``data``
+ Semantic variable that is mapped to determine the color of plot elements.
+ """,
+ palette="""
+palette : string, list, dict, or :class:`matplotlib.colors.Colormap`
+ Method for choosing the colors to use when mapping the ``hue`` semantic.
+ String values are passed to :func:`color_palette`. List or dict values
+ imply categorical mapping, while a colormap object implies numeric mapping.
+ """, # noqa: E501
+ hue_order="""
+hue_order : vector of strings
+ Specify the order of processing and plotting for categorical levels of the
+ ``hue`` semantic.
+ """,
+ hue_norm="""
+hue_norm : tuple or :class:`matplotlib.colors.Normalize`
+ Either a pair of values that set the normalization range in data units
+ or an object that will map from data units into a [0, 1] interval. Usage
+ implies numeric mapping.
+ """,
+ color="""
+color : :mod:`matplotlib color <matplotlib.colors>`
+ Single color specification for when hue mapping is not used. Otherwise, the
+ plot will try to hook into the matplotlib property cycle.
+ """,
+ ax="""
+ax : :class:`matplotlib.axes.Axes`
+ Pre-existing axes for the plot. Otherwise, call :func:`matplotlib.pyplot.gca`
+ internally.
+ """, # noqa: E501
+)
+
+
+_core_returns = dict(
+ ax="""
+:class:`matplotlib.axes.Axes`
+ The matplotlib axes containing the plot.
+ """,
+ facetgrid="""
+:class:`FacetGrid`
+ An object managing one or more subplots that correspond to conditional data
+ subsets with convenient methods for batch-setting of axes attributes.
+ """,
+ jointgrid="""
+:class:`JointGrid`
+ An object managing multiple subplots that correspond to joint and marginal axes
+ for plotting a bivariate relationship or distribution.
+ """,
+ pairgrid="""
+:class:`PairGrid`
+ An object managing multiple subplots that correspond to joint and marginal axes
+ for pairwise combinations of multiple variables in a dataset.
+ """,
+)
+
+
+_seealso_blurbs = dict(
+
+ # Relational plots
+ scatterplot="""
+scatterplot : Plot data using points.
+ """,
+ lineplot="""
+lineplot : Plot data using lines.
+ """,
+
+ # Distribution plots
+ displot="""
+displot : Figure-level interface to distribution plot functions.
+ """,
+ histplot="""
+histplot : Plot a histogram of binned counts with optional normalization or smoothing.
+ """,
+ kdeplot="""
+kdeplot : Plot univariate or bivariate distributions using kernel density estimation.
+ """,
+ ecdfplot="""
+ecdfplot : Plot empirical cumulative distribution functions.
+ """,
+ rugplot="""
+rugplot : Plot a tick at each observation value along the x and/or y axes.
+ """,
+
+ # Categorical plots
+ stripplot="""
+stripplot : Plot a categorical scatter with jitter.
+ """,
+ swarmplot="""
+swarmplot : Plot a categorical scatter with non-overlapping points.
+ """,
+ violinplot="""
+violinplot : Draw an enhanced boxplot using kernel density estimation.
+ """,
+ pointplot="""
+pointplot : Plot point estimates and CIs using markers and lines.
+ """,
+
+ # Multiples
+ jointplot="""
+jointplot : Draw a bivariate plot with univariate marginal distributions.
+ """,
+ pairplot="""
+jointplot : Draw multiple bivariate plots with univariate marginal distributions.
+ """,
+ jointgrid="""
+JointGrid : Set up a figure with joint and marginal views on bivariate data.
+ """,
+ pairgrid="""
+PairGrid : Set up a figure with joint and marginal views on multiple variables.
+ """,
+)
+
+
+_core_docs = dict(
+ params=DocstringComponents(_core_params),
+ returns=DocstringComponents(_core_returns),
+ seealso=DocstringComponents(_seealso_blurbs),
+)
diff --git a/seaborn/_marks/__init__.py b/seaborn/_marks/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/seaborn/_marks/area.py b/seaborn/_marks/area.py
new file mode 100644
index 0000000000000000000000000000000000000000..7514a6d13b7a373ff3c89ccbe06abec77442c0f2
--- /dev/null
+++ b/seaborn/_marks/area.py
@@ -0,0 +1,170 @@
+from __future__ import annotations
+from collections import defaultdict
+from dataclasses import dataclass
+
+import numpy as np
+import matplotlib as mpl
+
+from seaborn._marks.base import (
+ Mark,
+ Mappable,
+ MappableBool,
+ MappableFloat,
+ MappableColor,
+ MappableStyle,
+ resolve_properties,
+ resolve_color,
+ document_properties,
+)
+
+
+class AreaBase:
+
+ def _plot(self, split_gen, scales, orient):
+
+ patches = defaultdict(list)
+
+ for keys, data, ax in split_gen():
+
+ kws = {}
+ data = self._standardize_coordinate_parameters(data, orient)
+ resolved = resolve_properties(self, keys, scales)
+ verts = self._get_verts(data, orient)
+ ax.update_datalim(verts)
+
+ # TODO should really move this logic into resolve_color
+ fc = resolve_color(self, keys, "", scales)
+ if not resolved["fill"]:
+ fc = mpl.colors.to_rgba(fc, 0)
+
+ kws["facecolor"] = fc
+ kws["edgecolor"] = resolve_color(self, keys, "edge", scales)
+ kws["linewidth"] = resolved["edgewidth"]
+ kws["linestyle"] = resolved["edgestyle"]
+
+ patches[ax].append(mpl.patches.Polygon(verts, **kws))
+
+ for ax, ax_patches in patches.items():
+
+ for patch in ax_patches:
+ self._postprocess_artist(patch, ax, orient)
+ ax.add_patch(patch)
+
+ def _standardize_coordinate_parameters(self, data, orient):
+ return data
+
+ def _postprocess_artist(self, artist, ax, orient):
+ pass
+
+ def _get_verts(self, data, orient):
+
+ dv = {"x": "y", "y": "x"}[orient]
+ data = data.sort_values(orient, kind="mergesort")
+ verts = np.concatenate([
+ data[[orient, f"{dv}min"]].to_numpy(),
+ data[[orient, f"{dv}max"]].to_numpy()[::-1],
+ ])
+ if orient == "y":
+ verts = verts[:, ::-1]
+ return verts
+
+ def _legend_artist(self, variables, value, scales):
+
+ keys = {v: value for v in variables}
+ resolved = resolve_properties(self, keys, scales)
+
+ fc = resolve_color(self, keys, "", scales)
+ if not resolved["fill"]:
+ fc = mpl.colors.to_rgba(fc, 0)
+
+ return mpl.patches.Patch(
+ facecolor=fc,
+ edgecolor=resolve_color(self, keys, "edge", scales),
+ linewidth=resolved["edgewidth"],
+ linestyle=resolved["edgestyle"],
+ **self.artist_kws,
+ )
+
+
+@document_properties
+@dataclass
+class Area(AreaBase, Mark):
+ """
+ A fill mark drawn from a baseline to data values.
+
+ See also
+ --------
+ Band : A fill mark representing an interval between values.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Area.rst
+
+ """
+ color: MappableColor = Mappable("C0", )
+ alpha: MappableFloat = Mappable(.2, )
+ fill: MappableBool = Mappable(True, )
+ edgecolor: MappableColor = Mappable(depend="color")
+ edgealpha: MappableFloat = Mappable(1, )
+ edgewidth: MappableFloat = Mappable(rc="patch.linewidth", )
+ edgestyle: MappableStyle = Mappable("-", )
+
+ # TODO should this be settable / mappable?
+ baseline: MappableFloat = Mappable(0, grouping=False)
+
+ def _standardize_coordinate_parameters(self, data, orient):
+ dv = {"x": "y", "y": "x"}[orient]
+ return data.rename(columns={"baseline": f"{dv}min", dv: f"{dv}max"})
+
+ def _postprocess_artist(self, artist, ax, orient):
+
+ # TODO copying a lot of code from Bar, let's abstract this
+ # See comments there, I am not going to repeat them too
+
+ artist.set_linewidth(artist.get_linewidth() * 2)
+
+ linestyle = artist.get_linestyle()
+ if linestyle[1]:
+ linestyle = (linestyle[0], tuple(x / 2 for x in linestyle[1]))
+ artist.set_linestyle(linestyle)
+
+ artist.set_clip_path(artist.get_path(), artist.get_transform() + ax.transData)
+ if self.artist_kws.get("clip_on", True):
+ artist.set_clip_box(ax.bbox)
+
+ val_idx = ["y", "x"].index(orient)
+ artist.sticky_edges[val_idx][:] = (0, np.inf)
+
+
+@document_properties
+@dataclass
+class Band(AreaBase, Mark):
+ """
+ A fill mark representing an interval between values.
+
+ See also
+ --------
+ Area : A fill mark drawn from a baseline to data values.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Band.rst
+
+ """
+ color: MappableColor = Mappable("C0", )
+ alpha: MappableFloat = Mappable(.2, )
+ fill: MappableBool = Mappable(True, )
+ edgecolor: MappableColor = Mappable(depend="color", )
+ edgealpha: MappableFloat = Mappable(1, )
+ edgewidth: MappableFloat = Mappable(0, )
+ edgestyle: MappableFloat = Mappable("-", )
+
+ def _standardize_coordinate_parameters(self, data, orient):
+ # dv = {"x": "y", "y": "x"}[orient]
+ # TODO assert that all(ymax >= ymin)?
+ # TODO what if only one exist?
+ other = {"x": "y", "y": "x"}[orient]
+ if not set(data.columns) & {f"{other}min", f"{other}max"}:
+ agg = {f"{other}min": (other, "min"), f"{other}max": (other, "max")}
+ data = data.groupby(orient).agg(**agg).reset_index()
+ return data
diff --git a/seaborn/_marks/bar.py b/seaborn/_marks/bar.py
new file mode 100644
index 0000000000000000000000000000000000000000..2aed6830a6becc256352a7adcac1f10238d84b98
--- /dev/null
+++ b/seaborn/_marks/bar.py
@@ -0,0 +1,252 @@
+from __future__ import annotations
+from collections import defaultdict
+from dataclasses import dataclass
+
+import numpy as np
+import matplotlib as mpl
+
+from seaborn._marks.base import (
+ Mark,
+ Mappable,
+ MappableBool,
+ MappableColor,
+ MappableFloat,
+ MappableStyle,
+ resolve_properties,
+ resolve_color,
+ document_properties
+)
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+ from typing import Any
+ from matplotlib.artist import Artist
+ from seaborn._core.scales import Scale
+
+
+class BarBase(Mark):
+
+ def _make_patches(self, data, scales, orient):
+
+ transform = scales[orient]._matplotlib_scale.get_transform()
+ forward = transform.transform
+ reverse = transform.inverted().transform
+
+ other = {"x": "y", "y": "x"}[orient]
+
+ pos = reverse(forward(data[orient]) - data["width"] / 2)
+ width = reverse(forward(data[orient]) + data["width"] / 2) - pos
+
+ val = (data[other] - data["baseline"]).to_numpy()
+ base = data["baseline"].to_numpy()
+
+ kws = self._resolve_properties(data, scales)
+ if orient == "x":
+ kws.update(x=pos, y=base, w=width, h=val)
+ else:
+ kws.update(x=base, y=pos, w=val, h=width)
+
+ kws.pop("width", None)
+ kws.pop("baseline", None)
+
+ val_dim = {"x": "h", "y": "w"}[orient]
+ bars, vals = [], []
+
+ for i in range(len(data)):
+
+ row = {k: v[i] for k, v in kws.items()}
+
+ # Skip bars with no value. It's possible we'll want to make this
+ # an option (i.e so you have an artist for animating or annotating),
+ # but let's keep things simple for now.
+ if not np.nan_to_num(row[val_dim]):
+ continue
+
+ bar = mpl.patches.Rectangle(
+ xy=(row["x"], row["y"]),
+ width=row["w"],
+ height=row["h"],
+ facecolor=row["facecolor"],
+ edgecolor=row["edgecolor"],
+ linestyle=row["edgestyle"],
+ linewidth=row["edgewidth"],
+ **self.artist_kws,
+ )
+ bars.append(bar)
+ vals.append(row[val_dim])
+
+ return bars, vals
+
+ def _resolve_properties(self, data, scales):
+
+ resolved = resolve_properties(self, data, scales)
+
+ resolved["facecolor"] = resolve_color(self, data, "", scales)
+ resolved["edgecolor"] = resolve_color(self, data, "edge", scales)
+
+ fc = resolved["facecolor"]
+ if isinstance(fc, tuple):
+ resolved["facecolor"] = fc[0], fc[1], fc[2], fc[3] * resolved["fill"]
+ else:
+ fc[:, 3] = fc[:, 3] * resolved["fill"] # TODO Is inplace mod a problem?
+ resolved["facecolor"] = fc
+
+ return resolved
+
+ def _legend_artist(
+ self, variables: list[str], value: Any, scales: dict[str, Scale],
+ ) -> Artist:
+ # TODO return some sensible default?
+ key = {v: value for v in variables}
+ key = self._resolve_properties(key, scales)
+ artist = mpl.patches.Patch(
+ facecolor=key["facecolor"],
+ edgecolor=key["edgecolor"],
+ linewidth=key["edgewidth"],
+ linestyle=key["edgestyle"],
+ )
+ return artist
+
+
+@document_properties
+@dataclass
+class Bar(BarBase):
+ """
+ A bar mark drawn between baseline and data values.
+
+ See also
+ --------
+ Bars : A faster bar mark with defaults more suitable for histograms.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Bar.rst
+
+ """
+ color: MappableColor = Mappable("C0", grouping=False)
+ alpha: MappableFloat = Mappable(.7, grouping=False)
+ fill: MappableBool = Mappable(True, grouping=False)
+ edgecolor: MappableColor = Mappable(depend="color", grouping=False)
+ edgealpha: MappableFloat = Mappable(1, grouping=False)
+ edgewidth: MappableFloat = Mappable(rc="patch.linewidth", grouping=False)
+ edgestyle: MappableStyle = Mappable("-", grouping=False)
+ # pattern: MappableString = Mappable(None) # TODO no Property yet
+
+ width: MappableFloat = Mappable(.8, grouping=False)
+ baseline: MappableFloat = Mappable(0, grouping=False) # TODO *is* this mappable?
+
+ def _plot(self, split_gen, scales, orient):
+
+ val_idx = ["y", "x"].index(orient)
+
+ for _, data, ax in split_gen():
+
+ bars, vals = self._make_patches(data, scales, orient)
+
+ for bar in bars:
+
+ # Because we are clipping the artist (see below), the edges end up
+ # looking half as wide as they actually are. I don't love this clumsy
+ # workaround, which is going to cause surprises if you work with the
+ # artists directly. We may need to revisit after feedback.
+ bar.set_linewidth(bar.get_linewidth() * 2)
+ linestyle = bar.get_linestyle()
+ if linestyle[1]:
+ linestyle = (linestyle[0], tuple(x / 2 for x in linestyle[1]))
+ bar.set_linestyle(linestyle)
+
+ # This is a bit of a hack to handle the fact that the edge lines are
+ # centered on the actual extents of the bar, and overlap when bars are
+ # stacked or dodged. We may discover that this causes problems and needs
+ # to be revisited at some point. Also it should be faster to clip with
+ # a bbox than a path, but I cant't work out how to get the intersection
+ # with the axes bbox.
+ bar.set_clip_path(bar.get_path(), bar.get_transform() + ax.transData)
+ if self.artist_kws.get("clip_on", True):
+ # It seems the above hack undoes the default axes clipping
+ bar.set_clip_box(ax.bbox)
+ bar.sticky_edges[val_idx][:] = (0, np.inf)
+ ax.add_patch(bar)
+
+ # Add a container which is useful for, e.g. Axes.bar_label
+ orientation = {"x": "vertical", "y": "horizontal"}[orient]
+ container_kws = dict(datavalues=vals, orientation=orientation)
+ container = mpl.container.BarContainer(bars, **container_kws)
+ ax.add_container(container)
+
+
+@document_properties
+@dataclass
+class Bars(BarBase):
+ """
+ A faster bar mark with defaults more suitable for histograms.
+
+ See also
+ --------
+ Bar : A bar mark drawn between baseline and data values.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Bars.rst
+
+ """
+ color: MappableColor = Mappable("C0", grouping=False)
+ alpha: MappableFloat = Mappable(.7, grouping=False)
+ fill: MappableBool = Mappable(True, grouping=False)
+ edgecolor: MappableColor = Mappable(rc="patch.edgecolor", grouping=False)
+ edgealpha: MappableFloat = Mappable(1, grouping=False)
+ edgewidth: MappableFloat = Mappable(auto=True, grouping=False)
+ edgestyle: MappableStyle = Mappable("-", grouping=False)
+ # pattern: MappableString = Mappable(None) # TODO no Property yet
+
+ width: MappableFloat = Mappable(1, grouping=False)
+ baseline: MappableFloat = Mappable(0, grouping=False) # TODO *is* this mappable?
+
+ def _plot(self, split_gen, scales, orient):
+
+ ori_idx = ["x", "y"].index(orient)
+ val_idx = ["y", "x"].index(orient)
+
+ patches = defaultdict(list)
+ for _, data, ax in split_gen():
+ bars, _ = self._make_patches(data, scales, orient)
+ patches[ax].extend(bars)
+
+ collections = {}
+ for ax, ax_patches in patches.items():
+
+ col = mpl.collections.PatchCollection(ax_patches, match_original=True)
+ col.sticky_edges[val_idx][:] = (0, np.inf)
+ ax.add_collection(col, autolim=False)
+ collections[ax] = col
+
+ # Workaround for matplotlib autoscaling bug
+ # https://github.com/matplotlib/matplotlib/issues/11898
+ # https://github.com/matplotlib/matplotlib/issues/23129
+ xys = np.vstack([path.vertices for path in col.get_paths()])
+ ax.update_datalim(xys)
+
+ if "edgewidth" not in scales and isinstance(self.edgewidth, Mappable):
+
+ for ax in collections:
+ ax.autoscale_view()
+
+ def get_dimensions(collection):
+ edges, widths = [], []
+ for verts in (path.vertices for path in collection.get_paths()):
+ edges.append(min(verts[:, ori_idx]))
+ widths.append(np.ptp(verts[:, ori_idx]))
+ return np.array(edges), np.array(widths)
+
+ min_width = np.inf
+ for ax, col in collections.items():
+ edges, widths = get_dimensions(col)
+ points = 72 / ax.figure.dpi * abs(
+ ax.transData.transform([edges + widths] * 2)
+ - ax.transData.transform([edges] * 2)
+ )
+ min_width = min(min_width, min(points[:, ori_idx]))
+
+ linewidth = min(.1 * min_width, mpl.rcParams["patch.linewidth"])
+ for _, col in collections.items():
+ col.set_linewidth(linewidth)
diff --git a/seaborn/_marks/base.py b/seaborn/_marks/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..ac8fdf4aa547b5aa2fd1dcad424f4f78548dc7b8
--- /dev/null
+++ b/seaborn/_marks/base.py
@@ -0,0 +1,317 @@
+from __future__ import annotations
+from dataclasses import dataclass, fields, field
+import textwrap
+from typing import Any, Callable, Union
+from collections.abc import Generator
+
+import numpy as np
+import pandas as pd
+import matplotlib as mpl
+
+from numpy import ndarray
+from pandas import DataFrame
+from matplotlib.artist import Artist
+
+from seaborn._core.scales import Scale
+from seaborn._core.properties import (
+ PROPERTIES,
+ Property,
+ RGBATuple,
+ DashPattern,
+ DashPatternWithOffset,
+)
+from seaborn._core.exceptions import PlotSpecError
+
+
+class Mappable:
+ def __init__(
+ self,
+ val: Any = None,
+ depend: str | None = None,
+ rc: str | None = None,
+ auto: bool = False,
+ grouping: bool = True,
+ ):
+ """
+ Property that can be mapped from data or set directly, with flexible defaults.
+
+ Parameters
+ ----------
+ val : Any
+ Use this value as the default.
+ depend : str
+ Use the value of this feature as the default.
+ rc : str
+ Use the value of this rcParam as the default.
+ auto : bool
+ The default value will depend on other parameters at compile time.
+ grouping : bool
+ If True, use the mapped variable to define groups.
+
+ """
+ if depend is not None:
+ assert depend in PROPERTIES
+ if rc is not None:
+ assert rc in mpl.rcParams
+
+ self._val = val
+ self._rc = rc
+ self._depend = depend
+ self._auto = auto
+ self._grouping = grouping
+
+ def __repr__(self):
+ """Nice formatting for when object appears in Mark init signature."""
+ if self._val is not None:
+ s = f"<{repr(self._val)}>"
+ elif self._depend is not None:
+ s = f"<depend:{self._depend}>"
+ elif self._rc is not None:
+ s = f"<rc:{self._rc}>"
+ elif self._auto:
+ s = "<auto>"
+ else:
+ s = "<undefined>"
+ return s
+
+ @property
+ def depend(self) -> Any:
+ """Return the name of the feature to source a default value from."""
+ return self._depend
+
+ @property
+ def grouping(self) -> bool:
+ return self._grouping
+
+ @property
+ def default(self) -> Any:
+ """Get the default value for this feature, or access the relevant rcParam."""
+ if self._val is not None:
+ return self._val
+ elif self._rc is not None:
+ return mpl.rcParams.get(self._rc)
+
+
+# TODO where is the right place to put this kind of type aliasing?
+
+MappableBool = Union[bool, Mappable]
+MappableString = Union[str, Mappable]
+MappableFloat = Union[float, Mappable]
+MappableColor = Union[str, tuple, Mappable]
+MappableStyle = Union[str, DashPattern, DashPatternWithOffset, Mappable]
+
+
+@dataclass
+class Mark:
+ """Base class for objects that visually represent data."""
+
+ artist_kws: dict = field(default_factory=dict)
+
+ @property
+ def _mappable_props(self):
+ return {
+ f.name: getattr(self, f.name) for f in fields(self)
+ if isinstance(f.default, Mappable)
+ }
+
+ @property
+ def _grouping_props(self):
+ # TODO does it make sense to have variation within a Mark's
+ # properties about whether they are grouping?
+ return [
+ f.name for f in fields(self)
+ if isinstance(f.default, Mappable) and f.default.grouping
+ ]
+
+ # TODO make this method private? Would extender every need to call directly?
+ def _resolve(
+ self,
+ data: DataFrame | dict[str, Any],
+ name: str,
+ scales: dict[str, Scale] | None = None,
+ ) -> Any:
+ """Obtain default, specified, or mapped value for a named feature.
+
+ Parameters
+ ----------
+ data : DataFrame or dict with scalar values
+ Container with data values for features that will be semantically mapped.
+ name : string
+ Identity of the feature / semantic.
+ scales: dict
+ Mapping from variable to corresponding scale object.
+
+ Returns
+ -------
+ value or array of values
+ Outer return type depends on whether `data` is a dict (implying that
+ we want a single value) or DataFrame (implying that we want an array
+ of values with matching length).
+
+ """
+ feature = self._mappable_props[name]
+ prop = PROPERTIES.get(name, Property(name))
+ directly_specified = not isinstance(feature, Mappable)
+ return_multiple = isinstance(data, pd.DataFrame)
+ return_array = return_multiple and not name.endswith("style")
+
+ # Special case width because it needs to be resolved and added to the dataframe
+ # during layer prep (so the Move operations use it properly).
+ # TODO how does width *scaling* work, e.g. for violin width by count?
+ if name == "width":
+ directly_specified = directly_specified and name not in data
+
+ if directly_specified:
+ feature = prop.standardize(feature)
+ if return_multiple:
+ feature = [feature] * len(data)
+ if return_array:
+ feature = np.array(feature)
+ return feature
+
+ if name in data:
+ if scales is None or name not in scales:
+ # TODO Might this obviate the identity scale? Just don't add a scale?
+ feature = data[name]
+ else:
+ scale = scales[name]
+ value = data[name]
+ try:
+ feature = scale(value)
+ except Exception as err:
+ raise PlotSpecError._during("Scaling operation", name) from err
+
+ if return_array:
+ feature = np.asarray(feature)
+ return feature
+
+ if feature.depend is not None:
+ # TODO add source_func or similar to transform the source value?
+ # e.g. set linewidth as a proportion of pointsize?
+ return self._resolve(data, feature.depend, scales)
+
+ default = prop.standardize(feature.default)
+ if return_multiple:
+ default = [default] * len(data)
+ if return_array:
+ default = np.array(default)
+ return default
+
+ def _infer_orient(self, scales: dict) -> str: # TODO type scales
+
+ # TODO The original version of this (in seaborn._base) did more checking.
+ # Paring that down here for the prototype to see what restrictions make sense.
+
+ # TODO rethink this to map from scale type to "DV priority" and use that?
+ # e.g. Nominal > Discrete > Continuous
+
+ x = 0 if "x" not in scales else scales["x"]._priority
+ y = 0 if "y" not in scales else scales["y"]._priority
+
+ if y > x:
+ return "y"
+ else:
+ return "x"
+
+ def _plot(
+ self,
+ split_generator: Callable[[], Generator],
+ scales: dict[str, Scale],
+ orient: str,
+ ) -> None:
+ """Main interface for creating a plot."""
+ raise NotImplementedError()
+
+ def _legend_artist(
+ self, variables: list[str], value: Any, scales: dict[str, Scale],
+ ) -> Artist | None:
+
+ return None
+
+
+def resolve_properties(
+ mark: Mark, data: DataFrame, scales: dict[str, Scale]
+) -> dict[str, Any]:
+
+ props = {
+ name: mark._resolve(data, name, scales) for name in mark._mappable_props
+ }
+ return props
+
+
+def resolve_color(
+ mark: Mark,
+ data: DataFrame | dict,
+ prefix: str = "",
+ scales: dict[str, Scale] | None = None,
+) -> RGBATuple | ndarray:
+ """
+ Obtain a default, specified, or mapped value for a color feature.
+
+ This method exists separately to support the relationship between a
+ color and its corresponding alpha. We want to respect alpha values that
+ are passed in specified (or mapped) color values but also make use of a
+ separate `alpha` variable, which can be mapped. This approach may also
+ be extended to support mapping of specific color channels (i.e.
+ luminance, chroma) in the future.
+
+ Parameters
+ ----------
+ mark :
+ Mark with the color property.
+ data :
+ Container with data values for features that will be semantically mapped.
+ prefix :
+ Support "color", "fillcolor", etc.
+
+ """
+ color = mark._resolve(data, f"{prefix}color", scales)
+
+ if f"{prefix}alpha" in mark._mappable_props:
+ alpha = mark._resolve(data, f"{prefix}alpha", scales)
+ else:
+ alpha = mark._resolve(data, "alpha", scales)
+
+ def visible(x, axis=None):
+ """Detect "invisible" colors to set alpha appropriately."""
+ # TODO First clause only needed to handle non-rgba arrays,
+ # which we are trying to handle upstream
+ return np.array(x).dtype.kind != "f" or np.isfinite(x).all(axis)
+
+ # Second check here catches vectors of strings with identity scale
+ # It could probably be handled better upstream. This is a tricky problem
+ if np.ndim(color) < 2 and all(isinstance(x, float) for x in color):
+ if len(color) == 4:
+ return mpl.colors.to_rgba(color)
+ alpha = alpha if visible(color) else np.nan
+ return mpl.colors.to_rgba(color, alpha)
+ else:
+ if np.ndim(color) == 2 and color.shape[1] == 4:
+ return mpl.colors.to_rgba_array(color)
+ alpha = np.where(visible(color, axis=1), alpha, np.nan)
+ return mpl.colors.to_rgba_array(color, alpha)
+
+ # TODO should we be implementing fill here too?
+ # (i.e. set fillalpha to 0 when fill=False)
+
+
+def document_properties(mark):
+
+ properties = [f.name for f in fields(mark) if isinstance(f.default, Mappable)]
+ text = [
+ "",
+ " This mark defines the following properties:",
+ textwrap.fill(
+ ", ".join([f"|{p}|" for p in properties]),
+ width=78, initial_indent=" " * 8, subsequent_indent=" " * 8,
+ ),
+ ]
+
+ docstring_lines = mark.__doc__.split("\n")
+ new_docstring = "\n".join([
+ *docstring_lines[:2],
+ *text,
+ *docstring_lines[2:],
+ ])
+ mark.__doc__ = new_docstring
+ return mark
diff --git a/seaborn/_marks/dot.py b/seaborn/_marks/dot.py
new file mode 100644
index 0000000000000000000000000000000000000000..beef412dec2030d791b986aeb0261f5c0ba69766
--- /dev/null
+++ b/seaborn/_marks/dot.py
@@ -0,0 +1,200 @@
+from __future__ import annotations
+from dataclasses import dataclass
+
+import numpy as np
+import matplotlib as mpl
+
+from seaborn._marks.base import (
+ Mark,
+ Mappable,
+ MappableBool,
+ MappableFloat,
+ MappableString,
+ MappableColor,
+ MappableStyle,
+ resolve_properties,
+ resolve_color,
+ document_properties,
+)
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+ from typing import Any
+ from matplotlib.artist import Artist
+ from seaborn._core.scales import Scale
+
+
+class DotBase(Mark):
+
+ def _resolve_paths(self, data):
+
+ paths = []
+ path_cache = {}
+ marker = data["marker"]
+
+ def get_transformed_path(m):
+ return m.get_path().transformed(m.get_transform())
+
+ if isinstance(marker, mpl.markers.MarkerStyle):
+ return get_transformed_path(marker)
+
+ for m in marker:
+ if m not in path_cache:
+ path_cache[m] = get_transformed_path(m)
+ paths.append(path_cache[m])
+ return paths
+
+ def _resolve_properties(self, data, scales):
+
+ resolved = resolve_properties(self, data, scales)
+ resolved["path"] = self._resolve_paths(resolved)
+ resolved["size"] = resolved["pointsize"] ** 2
+
+ if isinstance(data, dict): # Properties for single dot
+ filled_marker = resolved["marker"].is_filled()
+ else:
+ filled_marker = [m.is_filled() for m in resolved["marker"]]
+
+ resolved["fill"] = resolved["fill"] * filled_marker
+
+ return resolved
+
+ def _plot(self, split_gen, scales, orient):
+
+ # TODO Not backcompat with allowed (but nonfunctional) univariate plots
+ # (That should be solved upstream by defaulting to "" for unset x/y?)
+ # (Be mindful of xmin/xmax, etc!)
+
+ for _, data, ax in split_gen():
+
+ offsets = np.column_stack([data["x"], data["y"]])
+ data = self._resolve_properties(data, scales)
+
+ points = mpl.collections.PathCollection(
+ offsets=offsets,
+ paths=data["path"],
+ sizes=data["size"],
+ facecolors=data["facecolor"],
+ edgecolors=data["edgecolor"],
+ linewidths=data["linewidth"],
+ linestyles=data["edgestyle"],
+ transOffset=ax.transData,
+ transform=mpl.transforms.IdentityTransform(),
+ **self.artist_kws,
+ )
+ ax.add_collection(points)
+
+ def _legend_artist(
+ self, variables: list[str], value: Any, scales: dict[str, Scale],
+ ) -> Artist:
+
+ key = {v: value for v in variables}
+ res = self._resolve_properties(key, scales)
+
+ return mpl.collections.PathCollection(
+ paths=[res["path"]],
+ sizes=[res["size"]],
+ facecolors=[res["facecolor"]],
+ edgecolors=[res["edgecolor"]],
+ linewidths=[res["linewidth"]],
+ linestyles=[res["edgestyle"]],
+ transform=mpl.transforms.IdentityTransform(),
+ **self.artist_kws,
+ )
+
+
+@document_properties
+@dataclass
+class Dot(DotBase):
+ """
+ A mark suitable for dot plots or less-dense scatterplots.
+
+ See also
+ --------
+ Dots : A dot mark defined by strokes to better handle overplotting.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Dot.rst
+
+ """
+ marker: MappableString = Mappable("o", grouping=False)
+ pointsize: MappableFloat = Mappable(6, grouping=False) # TODO rcParam?
+ stroke: MappableFloat = Mappable(.75, grouping=False) # TODO rcParam?
+ color: MappableColor = Mappable("C0", grouping=False)
+ alpha: MappableFloat = Mappable(1, grouping=False)
+ fill: MappableBool = Mappable(True, grouping=False)
+ edgecolor: MappableColor = Mappable(depend="color", grouping=False)
+ edgealpha: MappableFloat = Mappable(depend="alpha", grouping=False)
+ edgewidth: MappableFloat = Mappable(.5, grouping=False) # TODO rcParam?
+ edgestyle: MappableStyle = Mappable("-", grouping=False)
+
+ def _resolve_properties(self, data, scales):
+
+ resolved = super()._resolve_properties(data, scales)
+ filled = resolved["fill"]
+
+ main_stroke = resolved["stroke"]
+ edge_stroke = resolved["edgewidth"]
+ resolved["linewidth"] = np.where(filled, edge_stroke, main_stroke)
+
+ main_color = resolve_color(self, data, "", scales)
+ edge_color = resolve_color(self, data, "edge", scales)
+
+ if not np.isscalar(filled):
+ # Expand dims to use in np.where with rgba arrays
+ filled = filled[:, None]
+ resolved["edgecolor"] = np.where(filled, edge_color, main_color)
+
+ filled = np.squeeze(filled)
+ if isinstance(main_color, tuple):
+ # TODO handle this in resolve_color
+ main_color = tuple([*main_color[:3], main_color[3] * filled])
+ else:
+ main_color = np.c_[main_color[:, :3], main_color[:, 3] * filled]
+ resolved["facecolor"] = main_color
+
+ return resolved
+
+
+@document_properties
+@dataclass
+class Dots(DotBase):
+ """
+ A dot mark defined by strokes to better handle overplotting.
+
+ See also
+ --------
+ Dot : A mark suitable for dot plots or less-dense scatterplots.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Dots.rst
+
+ """
+ # TODO retype marker as MappableMarker
+ marker: MappableString = Mappable(rc="scatter.marker", grouping=False)
+ pointsize: MappableFloat = Mappable(4, grouping=False) # TODO rcParam?
+ stroke: MappableFloat = Mappable(.75, grouping=False) # TODO rcParam?
+ color: MappableColor = Mappable("C0", grouping=False)
+ alpha: MappableFloat = Mappable(1, grouping=False) # TODO auto alpha?
+ fill: MappableBool = Mappable(True, grouping=False)
+ fillcolor: MappableColor = Mappable(depend="color", grouping=False)
+ fillalpha: MappableFloat = Mappable(.2, grouping=False)
+
+ def _resolve_properties(self, data, scales):
+
+ resolved = super()._resolve_properties(data, scales)
+ resolved["linewidth"] = resolved.pop("stroke")
+ resolved["facecolor"] = resolve_color(self, data, "fill", scales)
+ resolved["edgecolor"] = resolve_color(self, data, "", scales)
+ resolved.setdefault("edgestyle", (0, None))
+
+ fc = resolved["facecolor"]
+ if isinstance(fc, tuple):
+ resolved["facecolor"] = fc[0], fc[1], fc[2], fc[3] * resolved["fill"]
+ else:
+ fc[:, 3] = fc[:, 3] * resolved["fill"] # TODO Is inplace mod a problem?
+ resolved["facecolor"] = fc
+
+ return resolved
diff --git a/seaborn/_marks/line.py b/seaborn/_marks/line.py
new file mode 100644
index 0000000000000000000000000000000000000000..a517f1b8b79483c5bc0374322d73d6affe2bdbda
--- /dev/null
+++ b/seaborn/_marks/line.py
@@ -0,0 +1,285 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import ClassVar
+
+import numpy as np
+import matplotlib as mpl
+
+from seaborn._marks.base import (
+ Mark,
+ Mappable,
+ MappableFloat,
+ MappableString,
+ MappableColor,
+ resolve_properties,
+ resolve_color,
+ document_properties,
+)
+
+
+@document_properties
+@dataclass
+class Path(Mark):
+ """
+ A mark connecting data points in the order they appear.
+
+ See also
+ --------
+ Line : A mark connecting data points with sorting along the orientation axis.
+ Paths : A faster but less-flexible mark for drawing many paths.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Path.rst
+
+ """
+ color: MappableColor = Mappable("C0")
+ alpha: MappableFloat = Mappable(1)
+ linewidth: MappableFloat = Mappable(rc="lines.linewidth")
+ linestyle: MappableString = Mappable(rc="lines.linestyle")
+ marker: MappableString = Mappable(rc="lines.marker")
+ pointsize: MappableFloat = Mappable(rc="lines.markersize")
+ fillcolor: MappableColor = Mappable(depend="color")
+ edgecolor: MappableColor = Mappable(depend="color")
+ edgewidth: MappableFloat = Mappable(rc="lines.markeredgewidth")
+
+ _sort: ClassVar[bool] = False
+
+ def _plot(self, split_gen, scales, orient):
+
+ for keys, data, ax in split_gen(keep_na=not self._sort):
+
+ vals = resolve_properties(self, keys, scales)
+ vals["color"] = resolve_color(self, keys, scales=scales)
+ vals["fillcolor"] = resolve_color(self, keys, prefix="fill", scales=scales)
+ vals["edgecolor"] = resolve_color(self, keys, prefix="edge", scales=scales)
+
+ if self._sort:
+ data = data.sort_values(orient, kind="mergesort")
+
+ artist_kws = self.artist_kws.copy()
+ self._handle_capstyle(artist_kws, vals)
+
+ line = mpl.lines.Line2D(
+ data["x"].to_numpy(),
+ data["y"].to_numpy(),
+ color=vals["color"],
+ linewidth=vals["linewidth"],
+ linestyle=vals["linestyle"],
+ marker=vals["marker"],
+ markersize=vals["pointsize"],
+ markerfacecolor=vals["fillcolor"],
+ markeredgecolor=vals["edgecolor"],
+ markeredgewidth=vals["edgewidth"],
+ **artist_kws,
+ )
+ ax.add_line(line)
+
+ def _legend_artist(self, variables, value, scales):
+
+ keys = {v: value for v in variables}
+ vals = resolve_properties(self, keys, scales)
+ vals["color"] = resolve_color(self, keys, scales=scales)
+ vals["fillcolor"] = resolve_color(self, keys, prefix="fill", scales=scales)
+ vals["edgecolor"] = resolve_color(self, keys, prefix="edge", scales=scales)
+
+ artist_kws = self.artist_kws.copy()
+ self._handle_capstyle(artist_kws, vals)
+
+ return mpl.lines.Line2D(
+ [], [],
+ color=vals["color"],
+ linewidth=vals["linewidth"],
+ linestyle=vals["linestyle"],
+ marker=vals["marker"],
+ markersize=vals["pointsize"],
+ markerfacecolor=vals["fillcolor"],
+ markeredgecolor=vals["edgecolor"],
+ markeredgewidth=vals["edgewidth"],
+ **artist_kws,
+ )
+
+ def _handle_capstyle(self, kws, vals):
+
+ # Work around for this matplotlib issue:
+ # https://github.com/matplotlib/matplotlib/issues/23437
+ if vals["linestyle"][1] is None:
+ capstyle = kws.get("solid_capstyle", mpl.rcParams["lines.solid_capstyle"])
+ kws["dash_capstyle"] = capstyle
+
+
+@document_properties
+@dataclass
+class Line(Path):
+ """
+ A mark connecting data points with sorting along the orientation axis.
+
+ See also
+ --------
+ Path : A mark connecting data points in the order they appear.
+ Lines : A faster but less-flexible mark for drawing many lines.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Line.rst
+
+ """
+ _sort: ClassVar[bool] = True
+
+
+@document_properties
+@dataclass
+class Paths(Mark):
+ """
+ A faster but less-flexible mark for drawing many paths.
+
+ See also
+ --------
+ Path : A mark connecting data points in the order they appear.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Paths.rst
+
+ """
+ color: MappableColor = Mappable("C0")
+ alpha: MappableFloat = Mappable(1)
+ linewidth: MappableFloat = Mappable(rc="lines.linewidth")
+ linestyle: MappableString = Mappable(rc="lines.linestyle")
+
+ _sort: ClassVar[bool] = False
+
+ def __post_init__(self):
+
+ # LineCollection artists have a capstyle property but don't source its value
+ # from the rc, so we do that manually here. Unfortunately, because we add
+ # only one LineCollection, we have the use the same capstyle for all lines
+ # even when they are dashed. It's a slight inconsistency, but looks fine IMO.
+ self.artist_kws.setdefault("capstyle", mpl.rcParams["lines.solid_capstyle"])
+
+ def _plot(self, split_gen, scales, orient):
+
+ line_data = {}
+ for keys, data, ax in split_gen(keep_na=not self._sort):
+
+ if ax not in line_data:
+ line_data[ax] = {
+ "segments": [],
+ "colors": [],
+ "linewidths": [],
+ "linestyles": [],
+ }
+
+ segments = self._setup_segments(data, orient)
+ line_data[ax]["segments"].extend(segments)
+ n = len(segments)
+
+ vals = resolve_properties(self, keys, scales)
+ vals["color"] = resolve_color(self, keys, scales=scales)
+
+ line_data[ax]["colors"].extend([vals["color"]] * n)
+ line_data[ax]["linewidths"].extend([vals["linewidth"]] * n)
+ line_data[ax]["linestyles"].extend([vals["linestyle"]] * n)
+
+ for ax, ax_data in line_data.items():
+ lines = mpl.collections.LineCollection(**ax_data, **self.artist_kws)
+ # Handle datalim update manually
+ # https://github.com/matplotlib/matplotlib/issues/23129
+ ax.add_collection(lines, autolim=False)
+ if ax_data["segments"]:
+ xy = np.concatenate(ax_data["segments"])
+ ax.update_datalim(xy)
+
+ def _legend_artist(self, variables, value, scales):
+
+ key = resolve_properties(self, {v: value for v in variables}, scales)
+
+ artist_kws = self.artist_kws.copy()
+ capstyle = artist_kws.pop("capstyle")
+ artist_kws["solid_capstyle"] = capstyle
+ artist_kws["dash_capstyle"] = capstyle
+
+ return mpl.lines.Line2D(
+ [], [],
+ color=key["color"],
+ linewidth=key["linewidth"],
+ linestyle=key["linestyle"],
+ **artist_kws,
+ )
+
+ def _setup_segments(self, data, orient):
+
+ if self._sort:
+ data = data.sort_values(orient, kind="mergesort")
+
+ # Column stack to avoid block consolidation
+ xy = np.column_stack([data["x"], data["y"]])
+
+ return [xy]
+
+
+@document_properties
+@dataclass
+class Lines(Paths):
+ """
+ A faster but less-flexible mark for drawing many lines.
+
+ See also
+ --------
+ Line : A mark connecting data points with sorting along the orientation axis.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Lines.rst
+
+ """
+ _sort: ClassVar[bool] = True
+
+
+@document_properties
+@dataclass
+class Range(Paths):
+ """
+ An oriented line mark drawn between min/max values.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Range.rst
+
+ """
+ def _setup_segments(self, data, orient):
+
+ # TODO better checks on what variables we have
+ # TODO what if only one exist?
+ val = {"x": "y", "y": "x"}[orient]
+ if not set(data.columns) & {f"{val}min", f"{val}max"}:
+ agg = {f"{val}min": (val, "min"), f"{val}max": (val, "max")}
+ data = data.groupby(orient).agg(**agg).reset_index()
+
+ cols = [orient, f"{val}min", f"{val}max"]
+ data = data[cols].melt(orient, value_name=val)[["x", "y"]]
+ segments = [d.to_numpy() for _, d in data.groupby(orient)]
+ return segments
+
+
+@document_properties
+@dataclass
+class Dash(Paths):
+ """
+ A line mark drawn as an oriented segment for each datapoint.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Dash.rst
+
+ """
+ width: MappableFloat = Mappable(.8, grouping=False)
+
+ def _setup_segments(self, data, orient):
+
+ ori = ["x", "y"].index(orient)
+ xys = data[["x", "y"]].to_numpy().astype(float)
+ segments = np.stack([xys, xys], axis=1)
+ segments[:, 0, ori] -= data["width"] / 2
+ segments[:, 1, ori] += data["width"] / 2
+ return segments
diff --git a/seaborn/_marks/text.py b/seaborn/_marks/text.py
new file mode 100644
index 0000000000000000000000000000000000000000..58d757c1acefc3c2ef6ecb8bec01c152ea08729d
--- /dev/null
+++ b/seaborn/_marks/text.py
@@ -0,0 +1,76 @@
+from __future__ import annotations
+from collections import defaultdict
+from dataclasses import dataclass
+
+import numpy as np
+import matplotlib as mpl
+from matplotlib.transforms import ScaledTranslation
+
+from seaborn._marks.base import (
+ Mark,
+ Mappable,
+ MappableFloat,
+ MappableString,
+ MappableColor,
+ resolve_properties,
+ resolve_color,
+ document_properties,
+)
+
+
+@document_properties
+@dataclass
+class Text(Mark):
+ """
+ A textual mark to annotate or represent data values.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Text.rst
+
+ """
+ text: MappableString = Mappable("")
+ color: MappableColor = Mappable("k")
+ alpha: MappableFloat = Mappable(1)
+ fontsize: MappableFloat = Mappable(rc="font.size")
+ halign: MappableString = Mappable("center")
+ valign: MappableString = Mappable("center_baseline")
+ offset: MappableFloat = Mappable(4)
+
+ def _plot(self, split_gen, scales, orient):
+
+ ax_data = defaultdict(list)
+
+ for keys, data, ax in split_gen():
+
+ vals = resolve_properties(self, keys, scales)
+ color = resolve_color(self, keys, "", scales)
+
+ halign = vals["halign"]
+ valign = vals["valign"]
+ fontsize = vals["fontsize"]
+ offset = vals["offset"] / 72
+
+ offset_trans = ScaledTranslation(
+ {"right": -offset, "left": +offset}.get(halign, 0),
+ {"top": -offset, "bottom": +offset, "baseline": +offset}.get(valign, 0),
+ ax.figure.dpi_scale_trans,
+ )
+
+ for row in data.to_dict("records"):
+ artist = mpl.text.Text(
+ x=row["x"],
+ y=row["y"],
+ text=str(row.get("text", vals["text"])),
+ color=color,
+ fontsize=fontsize,
+ horizontalalignment=halign,
+ verticalalignment=valign,
+ transform=ax.transData + offset_trans,
+ **self.artist_kws,
+ )
+ ax.add_artist(artist)
+ ax_data[ax].append([row["x"], row["y"]])
+
+ for ax, ax_vals in ax_data.items():
+ ax.update_datalim(np.array(ax_vals))
diff --git a/seaborn/_statistics.py b/seaborn/_statistics.py
new file mode 100644
index 0000000000000000000000000000000000000000..40346b02697d2fe58887ac8d00048a16b5b5c6e3
--- /dev/null
+++ b/seaborn/_statistics.py
@@ -0,0 +1,698 @@
+"""Statistical transformations for visualization.
+
+This module is currently private, but is being written to eventually form part
+of the public API.
+
+The classes should behave roughly in the style of scikit-learn.
+
+- All data-independent parameters should be passed to the class constructor.
+- Each class should implement a default transformation that is exposed through
+ __call__. These are currently written for vector arguments, but I think
+ consuming a whole `plot_data` DataFrame and return it with transformed
+ variables would make more sense.
+- Some class have data-dependent preprocessing that should be cached and used
+ multiple times (think defining histogram bins off all data and then counting
+ observations within each bin multiple times per data subsets). These currently
+ have unique names, but it would be good to have a common name. Not quite
+ `fit`, but something similar.
+- Alternatively, the transform interface could take some information about grouping
+ variables and do a groupby internally.
+- Some classes should define alternate transforms that might make the most sense
+ with a different function. For example, KDE usually evaluates the distribution
+ on a regular grid, but it would be useful for it to transform at the actual
+ datapoints. Then again, this could be controlled by a parameter at the time of
+ class instantiation.
+
+"""
+from numbers import Number
+from statistics import NormalDist
+import numpy as np
+import pandas as pd
+try:
+ from scipy.stats import gaussian_kde
+ _no_scipy = False
+except ImportError:
+ from .external.kde import gaussian_kde
+ _no_scipy = True
+
+from .algorithms import bootstrap
+from .utils import _check_argument
+
+
+class KDE:
+ """Univariate and bivariate kernel density estimator."""
+ def __init__(
+ self, *,
+ bw_method=None,
+ bw_adjust=1,
+ gridsize=200,
+ cut=3,
+ clip=None,
+ cumulative=False,
+ ):
+ """Initialize the estimator with its parameters.
+
+ Parameters
+ ----------
+ bw_method : string, scalar, or callable, optional
+ Method for determining the smoothing bandwidth to use; passed to
+ :class:`scipy.stats.gaussian_kde`.
+ bw_adjust : number, optional
+ Factor that multiplicatively scales the value chosen using
+ ``bw_method``. Increasing will make the curve smoother. See Notes.
+ gridsize : int, optional
+ Number of points on each dimension of the evaluation grid.
+ cut : number, optional
+ Factor, multiplied by the smoothing bandwidth, that determines how
+ far the evaluation grid extends past the extreme datapoints. When
+ set to 0, truncate the curve at the data limits.
+ clip : pair of numbers or None, or a pair of such pairs
+ Do not evaluate the density outside of these limits.
+ cumulative : bool, optional
+ If True, estimate a cumulative distribution function. Requires scipy.
+
+ """
+ if clip is None:
+ clip = None, None
+
+ self.bw_method = bw_method
+ self.bw_adjust = bw_adjust
+ self.gridsize = gridsize
+ self.cut = cut
+ self.clip = clip
+ self.cumulative = cumulative
+
+ if cumulative and _no_scipy:
+ raise RuntimeError("Cumulative KDE evaluation requires scipy")
+
+ self.support = None
+
+ def _define_support_grid(self, x, bw, cut, clip, gridsize):
+ """Create the grid of evaluation points depending for vector x."""
+ clip_lo = -np.inf if clip[0] is None else clip[0]
+ clip_hi = +np.inf if clip[1] is None else clip[1]
+ gridmin = max(x.min() - bw * cut, clip_lo)
+ gridmax = min(x.max() + bw * cut, clip_hi)
+ return np.linspace(gridmin, gridmax, gridsize)
+
+ def _define_support_univariate(self, x, weights):
+ """Create a 1D grid of evaluation points."""
+ kde = self._fit(x, weights)
+ bw = np.sqrt(kde.covariance.squeeze())
+ grid = self._define_support_grid(
+ x, bw, self.cut, self.clip, self.gridsize
+ )
+ return grid
+
+ def _define_support_bivariate(self, x1, x2, weights):
+ """Create a 2D grid of evaluation points."""
+ clip = self.clip
+ if clip[0] is None or np.isscalar(clip[0]):
+ clip = (clip, clip)
+
+ kde = self._fit([x1, x2], weights)
+ bw = np.sqrt(np.diag(kde.covariance).squeeze())
+
+ grid1 = self._define_support_grid(
+ x1, bw[0], self.cut, clip[0], self.gridsize
+ )
+ grid2 = self._define_support_grid(
+ x2, bw[1], self.cut, clip[1], self.gridsize
+ )
+
+ return grid1, grid2
+
+ def define_support(self, x1, x2=None, weights=None, cache=True):
+ """Create the evaluation grid for a given data set."""
+ if x2 is None:
+ support = self._define_support_univariate(x1, weights)
+ else:
+ support = self._define_support_bivariate(x1, x2, weights)
+
+ if cache:
+ self.support = support
+
+ return support
+
+ def _fit(self, fit_data, weights=None):
+ """Fit the scipy kde while adding bw_adjust logic and version check."""
+ fit_kws = {"bw_method": self.bw_method}
+ if weights is not None:
+ fit_kws["weights"] = weights
+
+ kde = gaussian_kde(fit_data, **fit_kws)
+ kde.set_bandwidth(kde.factor * self.bw_adjust)
+
+ return kde
+
+ def _eval_univariate(self, x, weights=None):
+ """Fit and evaluate a univariate on univariate data."""
+ support = self.support
+ if support is None:
+ support = self.define_support(x, cache=False)
+
+ kde = self._fit(x, weights)
+
+ if self.cumulative:
+ s_0 = support[0]
+ density = np.array([
+ kde.integrate_box_1d(s_0, s_i) for s_i in support
+ ])
+ else:
+ density = kde(support)
+
+ return density, support
+
+ def _eval_bivariate(self, x1, x2, weights=None):
+ """Fit and evaluate a univariate on bivariate data."""
+ support = self.support
+ if support is None:
+ support = self.define_support(x1, x2, cache=False)
+
+ kde = self._fit([x1, x2], weights)
+
+ if self.cumulative:
+
+ grid1, grid2 = support
+ density = np.zeros((grid1.size, grid2.size))
+ p0 = grid1.min(), grid2.min()
+ for i, xi in enumerate(grid1):
+ for j, xj in enumerate(grid2):
+ density[i, j] = kde.integrate_box(p0, (xi, xj))
+
+ else:
+
+ xx1, xx2 = np.meshgrid(*support)
+ density = kde([xx1.ravel(), xx2.ravel()]).reshape(xx1.shape)
+
+ return density, support
+
+ def __call__(self, x1, x2=None, weights=None):
+ """Fit and evaluate on univariate or bivariate data."""
+ if x2 is None:
+ return self._eval_univariate(x1, weights)
+ else:
+ return self._eval_bivariate(x1, x2, weights)
+
+
+# Note: we no longer use this for univariate histograms in histplot,
+# preferring _stats.Hist. We'll deprecate this once we have a bivariate Stat class.
+class Histogram:
+ """Univariate and bivariate histogram estimator."""
+ def __init__(
+ self,
+ stat="count",
+ bins="auto",
+ binwidth=None,
+ binrange=None,
+ discrete=False,
+ cumulative=False,
+ ):
+ """Initialize the estimator with its parameters.
+
+ Parameters
+ ----------
+ stat : str
+ Aggregate statistic to compute in each bin.
+
+ - `count`: show the number of observations in each bin
+ - `frequency`: show the number of observations divided by the bin width
+ - `probability` or `proportion`: normalize such that bar heights sum to 1
+ - `percent`: normalize such that bar heights sum to 100
+ - `density`: normalize such that the total area of the histogram equals 1
+
+ bins : str, number, vector, or a pair of such values
+ Generic bin parameter that can be the name of a reference rule,
+ the number of bins, or the breaks of the bins.
+ Passed to :func:`numpy.histogram_bin_edges`.
+ binwidth : number or pair of numbers
+ Width of each bin, overrides ``bins`` but can be used with
+ ``binrange``.
+ binrange : pair of numbers or a pair of pairs
+ Lowest and highest value for bin edges; can be used either
+ with ``bins`` or ``binwidth``. Defaults to data extremes.
+ discrete : bool or pair of bools
+ If True, set ``binwidth`` and ``binrange`` such that bin
+ edges cover integer values in the dataset.
+ cumulative : bool
+ If True, return the cumulative statistic.
+
+ """
+ stat_choices = [
+ "count", "frequency", "density", "probability", "proportion", "percent",
+ ]
+ _check_argument("stat", stat_choices, stat)
+
+ self.stat = stat
+ self.bins = bins
+ self.binwidth = binwidth
+ self.binrange = binrange
+ self.discrete = discrete
+ self.cumulative = cumulative
+
+ self.bin_kws = None
+
+ def _define_bin_edges(self, x, weights, bins, binwidth, binrange, discrete):
+ """Inner function that takes bin parameters as arguments."""
+ if binrange is None:
+ start, stop = x.min(), x.max()
+ else:
+ start, stop = binrange
+
+ if discrete:
+ bin_edges = np.arange(start - .5, stop + 1.5)
+ elif binwidth is not None:
+ step = binwidth
+ bin_edges = np.arange(start, stop + step, step)
+ # Handle roundoff error (maybe there is a less clumsy way?)
+ if bin_edges.max() < stop or len(bin_edges) < 2:
+ bin_edges = np.append(bin_edges, bin_edges.max() + step)
+ else:
+ bin_edges = np.histogram_bin_edges(
+ x, bins, binrange, weights,
+ )
+ return bin_edges
+
+ def define_bin_params(self, x1, x2=None, weights=None, cache=True):
+ """Given data, return numpy.histogram parameters to define bins."""
+ if x2 is None:
+
+ bin_edges = self._define_bin_edges(
+ x1, weights, self.bins, self.binwidth, self.binrange, self.discrete,
+ )
+
+ if isinstance(self.bins, (str, Number)):
+ n_bins = len(bin_edges) - 1
+ bin_range = bin_edges.min(), bin_edges.max()
+ bin_kws = dict(bins=n_bins, range=bin_range)
+ else:
+ bin_kws = dict(bins=bin_edges)
+
+ else:
+
+ bin_edges = []
+ for i, x in enumerate([x1, x2]):
+
+ # Resolve out whether bin parameters are shared
+ # or specific to each variable
+
+ bins = self.bins
+ if not bins or isinstance(bins, (str, Number)):
+ pass
+ elif isinstance(bins[i], str):
+ bins = bins[i]
+ elif len(bins) == 2:
+ bins = bins[i]
+
+ binwidth = self.binwidth
+ if binwidth is None:
+ pass
+ elif not isinstance(binwidth, Number):
+ binwidth = binwidth[i]
+
+ binrange = self.binrange
+ if binrange is None:
+ pass
+ elif not isinstance(binrange[0], Number):
+ binrange = binrange[i]
+
+ discrete = self.discrete
+ if not isinstance(discrete, bool):
+ discrete = discrete[i]
+
+ # Define the bins for this variable
+
+ bin_edges.append(self._define_bin_edges(
+ x, weights, bins, binwidth, binrange, discrete,
+ ))
+
+ bin_kws = dict(bins=tuple(bin_edges))
+
+ if cache:
+ self.bin_kws = bin_kws
+
+ return bin_kws
+
+ def _eval_bivariate(self, x1, x2, weights):
+ """Inner function for histogram of two variables."""
+ bin_kws = self.bin_kws
+ if bin_kws is None:
+ bin_kws = self.define_bin_params(x1, x2, cache=False)
+
+ density = self.stat == "density"
+
+ hist, *bin_edges = np.histogram2d(
+ x1, x2, **bin_kws, weights=weights, density=density
+ )
+
+ area = np.outer(
+ np.diff(bin_edges[0]),
+ np.diff(bin_edges[1]),
+ )
+
+ if self.stat == "probability" or self.stat == "proportion":
+ hist = hist.astype(float) / hist.sum()
+ elif self.stat == "percent":
+ hist = hist.astype(float) / hist.sum() * 100
+ elif self.stat == "frequency":
+ hist = hist.astype(float) / area
+
+ if self.cumulative:
+ if self.stat in ["density", "frequency"]:
+ hist = (hist * area).cumsum(axis=0).cumsum(axis=1)
+ else:
+ hist = hist.cumsum(axis=0).cumsum(axis=1)
+
+ return hist, bin_edges
+
+ def _eval_univariate(self, x, weights):
+ """Inner function for histogram of one variable."""
+ bin_kws = self.bin_kws
+ if bin_kws is None:
+ bin_kws = self.define_bin_params(x, weights=weights, cache=False)
+
+ density = self.stat == "density"
+ hist, bin_edges = np.histogram(
+ x, **bin_kws, weights=weights, density=density,
+ )
+
+ if self.stat == "probability" or self.stat == "proportion":
+ hist = hist.astype(float) / hist.sum()
+ elif self.stat == "percent":
+ hist = hist.astype(float) / hist.sum() * 100
+ elif self.stat == "frequency":
+ hist = hist.astype(float) / np.diff(bin_edges)
+
+ if self.cumulative:
+ if self.stat in ["density", "frequency"]:
+ hist = (hist * np.diff(bin_edges)).cumsum()
+ else:
+ hist = hist.cumsum()
+
+ return hist, bin_edges
+
+ def __call__(self, x1, x2=None, weights=None):
+ """Count the occurrences in each bin, maybe normalize."""
+ if x2 is None:
+ return self._eval_univariate(x1, weights)
+ else:
+ return self._eval_bivariate(x1, x2, weights)
+
+
+class ECDF:
+ """Univariate empirical cumulative distribution estimator."""
+ def __init__(self, stat="proportion", complementary=False):
+ """Initialize the class with its parameters
+
+ Parameters
+ ----------
+ stat : {{"proportion", "percent", "count"}}
+ Distribution statistic to compute.
+ complementary : bool
+ If True, use the complementary CDF (1 - CDF)
+
+ """
+ _check_argument("stat", ["count", "percent", "proportion"], stat)
+ self.stat = stat
+ self.complementary = complementary
+
+ def _eval_bivariate(self, x1, x2, weights):
+ """Inner function for ECDF of two variables."""
+ raise NotImplementedError("Bivariate ECDF is not implemented")
+
+ def _eval_univariate(self, x, weights):
+ """Inner function for ECDF of one variable."""
+ sorter = x.argsort()
+ x = x[sorter]
+ weights = weights[sorter]
+ y = weights.cumsum()
+
+ if self.stat in ["percent", "proportion"]:
+ y = y / y.max()
+ if self.stat == "percent":
+ y = y * 100
+
+ x = np.r_[-np.inf, x]
+ y = np.r_[0, y]
+
+ if self.complementary:
+ y = y.max() - y
+
+ return y, x
+
+ def __call__(self, x1, x2=None, weights=None):
+ """Return proportion or count of observations below each sorted datapoint."""
+ x1 = np.asarray(x1)
+ if weights is None:
+ weights = np.ones_like(x1)
+ else:
+ weights = np.asarray(weights)
+
+ if x2 is None:
+ return self._eval_univariate(x1, weights)
+ else:
+ return self._eval_bivariate(x1, x2, weights)
+
+
+class EstimateAggregator:
+
+ def __init__(self, estimator, errorbar=None, **boot_kws):
+ """
+ Data aggregator that produces an estimate and error bar interval.
+
+ Parameters
+ ----------
+ estimator : callable or string
+ Function (or method name) that maps a vector to a scalar.
+ errorbar : string, (string, number) tuple, or callable
+ Name of errorbar method (either "ci", "pi", "se", or "sd"), or a tuple
+ with a method name and a level parameter, or a function that maps from a
+ vector to a (min, max) interval, or None to hide errorbar. See the
+ :doc:`errorbar tutorial </tutorial/error_bars>` for more information.
+ boot_kws
+ Additional keywords are passed to bootstrap when error_method is "ci".
+
+ """
+ self.estimator = estimator
+
+ method, level = _validate_errorbar_arg(errorbar)
+ self.error_method = method
+ self.error_level = level
+
+ self.boot_kws = boot_kws
+
+ def __call__(self, data, var):
+ """Aggregate over `var` column of `data` with estimate and error interval."""
+ vals = data[var]
+ if callable(self.estimator):
+ # You would think we could pass to vals.agg, and yet:
+ # https://github.com/mwaskom/seaborn/issues/2943
+ estimate = self.estimator(vals)
+ else:
+ estimate = vals.agg(self.estimator)
+
+ # Options that produce no error bars
+ if self.error_method is None:
+ err_min = err_max = np.nan
+ elif len(data) <= 1:
+ err_min = err_max = np.nan
+
+ # Generic errorbars from user-supplied function
+ elif callable(self.error_method):
+ err_min, err_max = self.error_method(vals)
+
+ # Parametric options
+ elif self.error_method == "sd":
+ half_interval = vals.std() * self.error_level
+ err_min, err_max = estimate - half_interval, estimate + half_interval
+ elif self.error_method == "se":
+ half_interval = vals.sem() * self.error_level
+ err_min, err_max = estimate - half_interval, estimate + half_interval
+
+ # Nonparametric options
+ elif self.error_method == "pi":
+ err_min, err_max = _percentile_interval(vals, self.error_level)
+ elif self.error_method == "ci":
+ units = data.get("units", None)
+ boots = bootstrap(vals, units=units, func=self.estimator, **self.boot_kws)
+ err_min, err_max = _percentile_interval(boots, self.error_level)
+
+ return pd.Series({var: estimate, f"{var}min": err_min, f"{var}max": err_max})
+
+
+class WeightedAggregator:
+
+ def __init__(self, estimator, errorbar=None, **boot_kws):
+ """
+ Data aggregator that produces a weighted estimate and error bar interval.
+
+ Parameters
+ ----------
+ estimator : string
+ Function (or method name) that maps a vector to a scalar. Currently
+ supports only "mean".
+ errorbar : string or (string, number) tuple
+ Name of errorbar method or a tuple with a method name and a level parameter.
+ Currently the only supported method is "ci".
+ boot_kws
+ Additional keywords are passed to bootstrap when error_method is "ci".
+
+ """
+ if estimator != "mean":
+ # Note that, while other weighted estimators may make sense (e.g. median),
+ # I'm not aware of an implementation in our dependencies. We can add one
+ # in seaborn later, if there is sufficient interest. For now, limit to mean.
+ raise ValueError(f"Weighted estimator must be 'mean', not {estimator!r}.")
+ self.estimator = estimator
+
+ method, level = _validate_errorbar_arg(errorbar)
+ if method is not None and method != "ci":
+ # As with the estimator, weighted 'sd' or 'pi' error bars may make sense.
+ # But we'll keep things simple for now and limit to (bootstrap) CI.
+ raise ValueError(f"Error bar method must be 'ci', not {method!r}.")
+ self.error_method = method
+ self.error_level = level
+
+ self.boot_kws = boot_kws
+
+ def __call__(self, data, var):
+ """Aggregate over `var` column of `data` with estimate and error interval."""
+ vals = data[var]
+ weights = data["weight"]
+
+ estimate = np.average(vals, weights=weights)
+
+ if self.error_method == "ci" and len(data) > 1:
+
+ def error_func(x, w):
+ return np.average(x, weights=w)
+
+ boots = bootstrap(vals, weights, func=error_func, **self.boot_kws)
+ err_min, err_max = _percentile_interval(boots, self.error_level)
+
+ else:
+ err_min = err_max = np.nan
+
+ return pd.Series({var: estimate, f"{var}min": err_min, f"{var}max": err_max})
+
+
+class LetterValues:
+
+ def __init__(self, k_depth, outlier_prop, trust_alpha):
+ """
+ Compute percentiles of a distribution using various tail stopping rules.
+
+ Parameters
+ ----------
+ k_depth: "tukey", "proportion", "trustworthy", or "full"
+ Stopping rule for choosing tail percentiled to show:
+
+ - tukey: Show a similar number of outliers as in a conventional boxplot.
+ - proportion: Show approximately `outlier_prop` outliers.
+ - trust_alpha: Use `trust_alpha` level for most extreme tail percentile.
+
+ outlier_prop: float
+ Parameter for `k_depth="proportion"` setting the expected outlier rate.
+ trust_alpha: float
+ Parameter for `k_depth="trustworthy"` setting the confidence threshold.
+
+ Notes
+ -----
+ Based on the proposal in this paper:
+ https://vita.had.co.nz/papers/letter-value-plot.pdf
+
+ """
+ k_options = ["tukey", "proportion", "trustworthy", "full"]
+ if isinstance(k_depth, str):
+ _check_argument("k_depth", k_options, k_depth)
+ elif not isinstance(k_depth, int):
+ err = (
+ "The `k_depth` parameter must be either an integer or string "
+ f"(one of {k_options}), not {k_depth!r}."
+ )
+ raise TypeError(err)
+
+ self.k_depth = k_depth
+ self.outlier_prop = outlier_prop
+ self.trust_alpha = trust_alpha
+
+ def _compute_k(self, n):
+
+ # Select the depth, i.e. number of boxes to draw, based on the method
+ if self.k_depth == "full":
+ # extend boxes to 100% of the data
+ k = int(np.log2(n)) + 1
+ elif self.k_depth == "tukey":
+ # This results with 5-8 points in each tail
+ k = int(np.log2(n)) - 3
+ elif self.k_depth == "proportion":
+ k = int(np.log2(n)) - int(np.log2(n * self.outlier_prop)) + 1
+ elif self.k_depth == "trustworthy":
+ normal_quantile_func = np.vectorize(NormalDist().inv_cdf)
+ point_conf = 2 * normal_quantile_func(1 - self.trust_alpha / 2) ** 2
+ k = int(np.log2(n / point_conf)) + 1
+ else:
+ # Allow having k directly specified as input
+ k = int(self.k_depth)
+
+ return max(k, 1)
+
+ def __call__(self, x):
+ """Evaluate the letter values."""
+ k = self._compute_k(len(x))
+ exp = np.arange(k + 1, 1, -1), np.arange(2, k + 2)
+ levels = k + 1 - np.concatenate([exp[0], exp[1][1:]])
+ percentiles = 100 * np.concatenate([0.5 ** exp[0], 1 - 0.5 ** exp[1]])
+ if self.k_depth == "full":
+ percentiles[0] = 0
+ percentiles[-1] = 100
+ values = np.percentile(x, percentiles)
+ fliers = np.asarray(x[(x < values.min()) | (x > values.max())])
+ median = np.percentile(x, 50)
+
+ return {
+ "k": k,
+ "levels": levels,
+ "percs": percentiles,
+ "values": values,
+ "fliers": fliers,
+ "median": median,
+ }
+
+
+def _percentile_interval(data, width):
+ """Return a percentile interval from data of a given width."""
+ edge = (100 - width) / 2
+ percentiles = edge, 100 - edge
+ return np.nanpercentile(data, percentiles)
+
+
+def _validate_errorbar_arg(arg):
+ """Check type and value of errorbar argument and assign default level."""
+ DEFAULT_LEVELS = {
+ "ci": 95,
+ "pi": 95,
+ "se": 1,
+ "sd": 1,
+ }
+
+ usage = "`errorbar` must be a callable, string, or (string, number) tuple"
+
+ if arg is None:
+ return None, None
+ elif callable(arg):
+ return arg, None
+ elif isinstance(arg, str):
+ method = arg
+ level = DEFAULT_LEVELS.get(method, None)
+ else:
+ try:
+ method, level = arg
+ except (ValueError, TypeError) as err:
+ raise err.__class__(usage) from err
+
+ _check_argument("errorbar", list(DEFAULT_LEVELS), method)
+ if level is not None and not isinstance(level, Number):
+ raise TypeError(usage)
+
+ return method, level
diff --git a/seaborn/_stats/__init__.py b/seaborn/_stats/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/seaborn/_stats/aggregation.py b/seaborn/_stats/aggregation.py
new file mode 100644
index 0000000000000000000000000000000000000000..7e7d60212a49383bbdf8f78bcb297d7f1bdbc561
--- /dev/null
+++ b/seaborn/_stats/aggregation.py
@@ -0,0 +1,130 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import ClassVar, Callable
+
+import pandas as pd
+from pandas import DataFrame
+
+from seaborn._core.scales import Scale
+from seaborn._core.groupby import GroupBy
+from seaborn._stats.base import Stat
+from seaborn._statistics import (
+ EstimateAggregator,
+ WeightedAggregator,
+)
+from seaborn._core.typing import Vector
+
+
+@dataclass
+class Agg(Stat):
+ """
+ Aggregate data along the value axis using given method.
+
+ Parameters
+ ----------
+ func : str or callable
+ Name of a :class:`pandas.Series` method or a vector -> scalar function.
+
+ See Also
+ --------
+ objects.Est : Aggregation with error bars.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Agg.rst
+
+ """
+ func: str | Callable[[Vector], float] = "mean"
+
+ group_by_orient: ClassVar[bool] = True
+
+ def __call__(
+ self, data: DataFrame, groupby: GroupBy, orient: str, scales: dict[str, Scale],
+ ) -> DataFrame:
+
+ var = {"x": "y", "y": "x"}.get(orient)
+ res = (
+ groupby
+ .agg(data, {var: self.func})
+ .dropna(subset=[var])
+ .reset_index(drop=True)
+ )
+ return res
+
+
+@dataclass
+class Est(Stat):
+ """
+ Calculate a point estimate and error bar interval.
+
+ For more information about the various `errorbar` choices, see the
+ :doc:`errorbar tutorial </tutorial/error_bars>`.
+
+ Additional variables:
+
+ - **weight**: When passed to a layer that uses this stat, a weighted estimate
+ will be computed. Note that use of weights currently limits the choice of
+ function and error bar method to `"mean"` and `"ci"`, respectively.
+
+ Parameters
+ ----------
+ func : str or callable
+ Name of a :class:`numpy.ndarray` method or a vector -> scalar function.
+ errorbar : str, (str, float) tuple, or callable
+ Name of errorbar method (one of "ci", "pi", "se" or "sd"), or a tuple
+ with a method name ane a level parameter, or a function that maps from a
+ vector to a (min, max) interval.
+ n_boot : int
+ Number of bootstrap samples to draw for "ci" errorbars.
+ seed : int
+ Seed for the PRNG used to draw bootstrap samples.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Est.rst
+
+ """
+ func: str | Callable[[Vector], float] = "mean"
+ errorbar: str | tuple[str, float] = ("ci", 95)
+ n_boot: int = 1000
+ seed: int | None = None
+
+ group_by_orient: ClassVar[bool] = True
+
+ def _process(
+ self, data: DataFrame, var: str, estimator: EstimateAggregator
+ ) -> DataFrame:
+ # Needed because GroupBy.apply assumes func is DataFrame -> DataFrame
+ # which we could probably make more general to allow Series return
+ res = estimator(data, var)
+ return pd.DataFrame([res])
+
+ def __call__(
+ self, data: DataFrame, groupby: GroupBy, orient: str, scales: dict[str, Scale],
+ ) -> DataFrame:
+
+ boot_kws = {"n_boot": self.n_boot, "seed": self.seed}
+ if "weight" in data:
+ engine = WeightedAggregator(self.func, self.errorbar, **boot_kws)
+ else:
+ engine = EstimateAggregator(self.func, self.errorbar, **boot_kws)
+
+ var = {"x": "y", "y": "x"}[orient]
+ res = (
+ groupby
+ .apply(data, self._process, var, engine)
+ .dropna(subset=[var])
+ .reset_index(drop=True)
+ )
+
+ res = res.fillna({f"{var}min": res[var], f"{var}max": res[var]})
+
+ return res
+
+
+@dataclass
+class Rolling(Stat):
+ ...
+
+ def __call__(self, data, groupby, orient, scales):
+ ...
diff --git a/seaborn/_stats/base.py b/seaborn/_stats/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..b80b228165406f2103f00ce9bb0143bf16c02002
--- /dev/null
+++ b/seaborn/_stats/base.py
@@ -0,0 +1,65 @@
+"""Base module for statistical transformations."""
+from __future__ import annotations
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import ClassVar, Any
+import warnings
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+ from pandas import DataFrame
+ from seaborn._core.groupby import GroupBy
+ from seaborn._core.scales import Scale
+
+
+@dataclass
+class Stat:
+ """Base class for objects that apply statistical transformations."""
+
+ # The class supports a partial-function application pattern. The object is
+ # initialized with desired parameters and the result is a callable that
+ # accepts and returns dataframes.
+
+ # The statistical transformation logic should not add any state to the instance
+ # beyond what is defined with the initialization parameters.
+
+ # Subclasses can declare whether the orient dimension should be used in grouping
+ # TODO consider whether this should be a parameter. Motivating example:
+ # use the same KDE class violin plots and univariate density estimation.
+ # In the former case, we would expect separate densities for each unique
+ # value on the orient axis, but we would not in the latter case.
+ group_by_orient: ClassVar[bool] = False
+
+ def _check_param_one_of(self, param: str, options: Iterable[Any]) -> None:
+ """Raise when parameter value is not one of a specified set."""
+ value = getattr(self, param)
+ if value not in options:
+ *most, last = options
+ option_str = ", ".join(f"{x!r}" for x in most[:-1]) + f" or {last!r}"
+ err = " ".join([
+ f"The `{param}` parameter for `{self.__class__.__name__}` must be",
+ f"one of {option_str}; not {value!r}.",
+ ])
+ raise ValueError(err)
+
+ def _check_grouping_vars(
+ self, param: str, data_vars: list[str], stacklevel: int = 2,
+ ) -> None:
+ """Warn if vars are named in parameter without being present in the data."""
+ param_vars = getattr(self, param)
+ undefined = set(param_vars) - set(data_vars)
+ if undefined:
+ param = f"{self.__class__.__name__}.{param}"
+ names = ", ".join(f"{x!r}" for x in undefined)
+ msg = f"Undefined variable(s) passed for {param}: {names}."
+ warnings.warn(msg, stacklevel=stacklevel)
+
+ def __call__(
+ self,
+ data: DataFrame,
+ groupby: GroupBy,
+ orient: str,
+ scales: dict[str, Scale],
+ ) -> DataFrame:
+ """Apply statistical transform to data subgroups and return combined result."""
+ return data
diff --git a/seaborn/_stats/counting.py b/seaborn/_stats/counting.py
new file mode 100644
index 0000000000000000000000000000000000000000..0c2fb7d4998ac6fcfd39bf79686c113f5481bb65
--- /dev/null
+++ b/seaborn/_stats/counting.py
@@ -0,0 +1,232 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import ClassVar
+
+import numpy as np
+import pandas as pd
+from pandas import DataFrame
+
+from seaborn._core.groupby import GroupBy
+from seaborn._core.scales import Scale
+from seaborn._stats.base import Stat
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+ from numpy.typing import ArrayLike
+
+
+@dataclass
+class Count(Stat):
+ """
+ Count distinct observations within groups.
+
+ See Also
+ --------
+ Hist : A more fully-featured transform including binning and/or normalization.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Count.rst
+
+ """
+ group_by_orient: ClassVar[bool] = True
+
+ def __call__(
+ self, data: DataFrame, groupby: GroupBy, orient: str, scales: dict[str, Scale],
+ ) -> DataFrame:
+
+ var = {"x": "y", "y": "x"}[orient]
+ res = (
+ groupby
+ .agg(data.assign(**{var: data[orient]}), {var: len})
+ .dropna(subset=["x", "y"])
+ .reset_index(drop=True)
+ )
+ return res
+
+
+@dataclass
+class Hist(Stat):
+ """
+ Bin observations, count them, and optionally normalize or cumulate.
+
+ Parameters
+ ----------
+ stat : str
+ Aggregate statistic to compute in each bin:
+
+ - `count`: the number of observations
+ - `density`: normalize so that the total area of the histogram equals 1
+ - `percent`: normalize so that bar heights sum to 100
+ - `probability` or `proportion`: normalize so that bar heights sum to 1
+ - `frequency`: divide the number of observations by the bin width
+
+ bins : str, int, or ArrayLike
+ Generic parameter that can be the name of a reference rule, the number
+ of bins, or the bin breaks. Passed to :func:`numpy.histogram_bin_edges`.
+ binwidth : float
+ Width of each bin; overrides `bins` but can be used with `binrange`.
+ Note that if `binwidth` does not evenly divide the bin range, the actual
+ bin width used will be only approximately equal to the parameter value.
+ binrange : (min, max)
+ Lowest and highest value for bin edges; can be used with either
+ `bins` (when a number) or `binwidth`. Defaults to data extremes.
+ common_norm : bool or list of variables
+ When not `False`, the normalization is applied across groups. Use
+ `True` to normalize across all groups, or pass variable name(s) that
+ define normalization groups.
+ common_bins : bool or list of variables
+ When not `False`, the same bins are used for all groups. Use `True` to
+ share bins across all groups, or pass variable name(s) to share within.
+ cumulative : bool
+ If True, cumulate the bin values.
+ discrete : bool
+ If True, set `binwidth` and `binrange` so that bins have unit width and
+ are centered on integer values
+
+ Notes
+ -----
+ The choice of bins for computing and plotting a histogram can exert
+ substantial influence on the insights that one is able to draw from the
+ visualization. If the bins are too large, they may erase important features.
+ On the other hand, bins that are too small may be dominated by random
+ variability, obscuring the shape of the true underlying distribution. The
+ default bin size is determined using a reference rule that depends on the
+ sample size and variance. This works well in many cases, (i.e., with
+ "well-behaved" data) but it fails in others. It is always a good to try
+ different bin sizes to be sure that you are not missing something important.
+ This function allows you to specify bins in several different ways, such as
+ by setting the total number of bins to use, the width of each bin, or the
+ specific locations where the bins should break.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Hist.rst
+
+ """
+ stat: str = "count"
+ bins: str | int | ArrayLike = "auto"
+ binwidth: float | None = None
+ binrange: tuple[float, float] | None = None
+ common_norm: bool | list[str] = True
+ common_bins: bool | list[str] = True
+ cumulative: bool = False
+ discrete: bool = False
+
+ def __post_init__(self):
+
+ stat_options = [
+ "count", "density", "percent", "probability", "proportion", "frequency"
+ ]
+ self._check_param_one_of("stat", stat_options)
+
+ def _define_bin_edges(self, vals, weight, bins, binwidth, binrange, discrete):
+ """Inner function that takes bin parameters as arguments."""
+ vals = vals.replace(-np.inf, np.nan).replace(np.inf, np.nan).dropna()
+
+ if binrange is None:
+ start, stop = vals.min(), vals.max()
+ else:
+ start, stop = binrange
+
+ if discrete:
+ bin_edges = np.arange(start - .5, stop + 1.5)
+ else:
+ if binwidth is not None:
+ bins = int(round((stop - start) / binwidth))
+ bin_edges = np.histogram_bin_edges(vals, bins, binrange, weight)
+
+ # TODO warning or cap on too many bins?
+
+ return bin_edges
+
+ def _define_bin_params(self, data, orient, scale_type):
+ """Given data, return numpy.histogram parameters to define bins."""
+ vals = data[orient]
+ weights = data.get("weight", None)
+
+ # TODO We'll want this for ordinal / discrete scales too
+ # (Do we need discrete as a parameter or just infer from scale?)
+ discrete = self.discrete or scale_type == "nominal"
+
+ bin_edges = self._define_bin_edges(
+ vals, weights, self.bins, self.binwidth, self.binrange, discrete,
+ )
+
+ if isinstance(self.bins, (str, int)):
+ n_bins = len(bin_edges) - 1
+ bin_range = bin_edges.min(), bin_edges.max()
+ bin_kws = dict(bins=n_bins, range=bin_range)
+ else:
+ bin_kws = dict(bins=bin_edges)
+
+ return bin_kws
+
+ def _get_bins_and_eval(self, data, orient, groupby, scale_type):
+
+ bin_kws = self._define_bin_params(data, orient, scale_type)
+ return groupby.apply(data, self._eval, orient, bin_kws)
+
+ def _eval(self, data, orient, bin_kws):
+
+ vals = data[orient]
+ weights = data.get("weight", None)
+
+ density = self.stat == "density"
+ hist, edges = np.histogram(vals, **bin_kws, weights=weights, density=density)
+
+ width = np.diff(edges)
+ center = edges[:-1] + width / 2
+
+ return pd.DataFrame({orient: center, "count": hist, "space": width})
+
+ def _normalize(self, data):
+
+ hist = data["count"]
+ if self.stat == "probability" or self.stat == "proportion":
+ hist = hist.astype(float) / hist.sum()
+ elif self.stat == "percent":
+ hist = hist.astype(float) / hist.sum() * 100
+ elif self.stat == "frequency":
+ hist = hist.astype(float) / data["space"]
+
+ if self.cumulative:
+ if self.stat in ["density", "frequency"]:
+ hist = (hist * data["space"]).cumsum()
+ else:
+ hist = hist.cumsum()
+
+ return data.assign(**{self.stat: hist})
+
+ def __call__(
+ self, data: DataFrame, groupby: GroupBy, orient: str, scales: dict[str, Scale],
+ ) -> DataFrame:
+
+ scale_type = scales[orient].__class__.__name__.lower()
+ grouping_vars = [str(v) for v in data if v in groupby.order]
+ if not grouping_vars or self.common_bins is True:
+ bin_kws = self._define_bin_params(data, orient, scale_type)
+ data = groupby.apply(data, self._eval, orient, bin_kws)
+ else:
+ if self.common_bins is False:
+ bin_groupby = GroupBy(grouping_vars)
+ else:
+ bin_groupby = GroupBy(self.common_bins)
+ self._check_grouping_vars("common_bins", grouping_vars)
+
+ data = bin_groupby.apply(
+ data, self._get_bins_and_eval, orient, groupby, scale_type,
+ )
+
+ if not grouping_vars or self.common_norm is True:
+ data = self._normalize(data)
+ else:
+ if self.common_norm is False:
+ norm_groupby = GroupBy(grouping_vars)
+ else:
+ norm_groupby = GroupBy(self.common_norm)
+ self._check_grouping_vars("common_norm", grouping_vars)
+ data = norm_groupby.apply(data, self._normalize)
+
+ other = {"x": "y", "y": "x"}[orient]
+ return data.assign(**{other: data[self.stat]})
diff --git a/seaborn/_stats/density.py b/seaborn/_stats/density.py
new file mode 100644
index 0000000000000000000000000000000000000000..e461387651556a28ded23d6583d78e8fff8e38b3
--- /dev/null
+++ b/seaborn/_stats/density.py
@@ -0,0 +1,214 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Any, Callable
+
+import numpy as np
+from numpy import ndarray
+import pandas as pd
+from pandas import DataFrame
+try:
+ from scipy.stats import gaussian_kde
+ _no_scipy = False
+except ImportError:
+ from seaborn.external.kde import gaussian_kde
+ _no_scipy = True
+
+from seaborn._core.groupby import GroupBy
+from seaborn._core.scales import Scale
+from seaborn._stats.base import Stat
+
+
+@dataclass
+class KDE(Stat):
+ """
+ Compute a univariate kernel density estimate.
+
+ Parameters
+ ----------
+ bw_adjust : float
+ Factor that multiplicatively scales the value chosen using
+ `bw_method`. Increasing will make the curve smoother. See Notes.
+ bw_method : string, scalar, or callable
+ Method for determining the smoothing bandwidth to use. Passed directly
+ to :class:`scipy.stats.gaussian_kde`; see there for options.
+ common_norm : bool or list of variables
+ If `True`, normalize so that the areas of all curves sums to 1.
+ If `False`, normalize each curve independently. If a list, defines
+ variable(s) to group by and normalize within.
+ common_grid : bool or list of variables
+ If `True`, all curves will share the same evaluation grid.
+ If `False`, each evaluation grid is independent. If a list, defines
+ variable(s) to group by and share a grid within.
+ gridsize : int or None
+ Number of points in the evaluation grid. If None, the density is
+ evaluated at the original datapoints.
+ cut : float
+ Factor, multiplied by the kernel bandwidth, that determines how far
+ the evaluation grid extends past the extreme datapoints. When set to 0,
+ the curve is truncated at the data limits.
+ cumulative : bool
+ If True, estimate a cumulative distribution function. Requires scipy.
+
+ Notes
+ -----
+ The *bandwidth*, or standard deviation of the smoothing kernel, is an
+ important parameter. Much like histogram bin width, using the wrong
+ bandwidth can produce a distorted representation. Over-smoothing can erase
+ true features, while under-smoothing can create false ones. The default
+ uses a rule-of-thumb that works best for distributions that are roughly
+ bell-shaped. It is a good idea to check the default by varying `bw_adjust`.
+
+ Because the smoothing is performed with a Gaussian kernel, the estimated
+ density curve can extend to values that may not make sense. For example, the
+ curve may be drawn over negative values when data that are naturally
+ positive. The `cut` parameter can be used to control the evaluation range,
+ but datasets that have many observations close to a natural boundary may be
+ better served by a different method.
+
+ Similar distortions may arise when a dataset is naturally discrete or "spiky"
+ (containing many repeated observations of the same value). KDEs will always
+ produce a smooth curve, which could be misleading.
+
+ The units on the density axis are a common source of confusion. While kernel
+ density estimation produces a probability distribution, the height of the curve
+ at each point gives a density, not a probability. A probability can be obtained
+ only by integrating the density across a range. The curve is normalized so
+ that the integral over all possible values is 1, meaning that the scale of
+ the density axis depends on the data values.
+
+ If scipy is installed, its cython-accelerated implementation will be used.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.KDE.rst
+
+ """
+ bw_adjust: float = 1
+ bw_method: str | float | Callable[[gaussian_kde], float] = "scott"
+ common_norm: bool | list[str] = True
+ common_grid: bool | list[str] = True
+ gridsize: int | None = 200
+ cut: float = 3
+ cumulative: bool = False
+
+ def __post_init__(self):
+
+ if self.cumulative and _no_scipy:
+ raise RuntimeError("Cumulative KDE evaluation requires scipy")
+
+ def _check_var_list_or_boolean(self, param: str, grouping_vars: Any) -> None:
+ """Do input checks on grouping parameters."""
+ value = getattr(self, param)
+ if not (
+ isinstance(value, bool)
+ or (isinstance(value, list) and all(isinstance(v, str) for v in value))
+ ):
+ param_name = f"{self.__class__.__name__}.{param}"
+ raise TypeError(f"{param_name} must be a boolean or list of strings.")
+ self._check_grouping_vars(param, grouping_vars, stacklevel=3)
+
+ def _fit(self, data: DataFrame, orient: str) -> gaussian_kde:
+ """Fit and return a KDE object."""
+ # TODO need to handle singular data
+
+ fit_kws: dict[str, Any] = {"bw_method": self.bw_method}
+ if "weight" in data:
+ fit_kws["weights"] = data["weight"]
+ kde = gaussian_kde(data[orient], **fit_kws)
+ kde.set_bandwidth(kde.factor * self.bw_adjust)
+
+ return kde
+
+ def _get_support(self, data: DataFrame, orient: str) -> ndarray:
+ """Define the grid that the KDE will be evaluated on."""
+ if self.gridsize is None:
+ return data[orient].to_numpy()
+
+ kde = self._fit(data, orient)
+ bw = np.sqrt(kde.covariance.squeeze())
+ gridmin = data[orient].min() - bw * self.cut
+ gridmax = data[orient].max() + bw * self.cut
+ return np.linspace(gridmin, gridmax, self.gridsize)
+
+ def _fit_and_evaluate(
+ self, data: DataFrame, orient: str, support: ndarray
+ ) -> DataFrame:
+ """Transform single group by fitting a KDE and evaluating on a support grid."""
+ empty = pd.DataFrame(columns=[orient, "weight", "density"], dtype=float)
+ if len(data) < 2:
+ return empty
+ try:
+ kde = self._fit(data, orient)
+ except np.linalg.LinAlgError:
+ return empty
+
+ if self.cumulative:
+ s_0 = support[0]
+ density = np.array([kde.integrate_box_1d(s_0, s_i) for s_i in support])
+ else:
+ density = kde(support)
+
+ weight = data["weight"].sum()
+ return pd.DataFrame({orient: support, "weight": weight, "density": density})
+
+ def _transform(
+ self, data: DataFrame, orient: str, grouping_vars: list[str]
+ ) -> DataFrame:
+ """Transform multiple groups by fitting KDEs and evaluating."""
+ empty = pd.DataFrame(columns=[*data.columns, "density"], dtype=float)
+ if len(data) < 2:
+ return empty
+ try:
+ support = self._get_support(data, orient)
+ except np.linalg.LinAlgError:
+ return empty
+
+ grouping_vars = [x for x in grouping_vars if data[x].nunique() > 1]
+ if not grouping_vars:
+ return self._fit_and_evaluate(data, orient, support)
+ groupby = GroupBy(grouping_vars)
+ return groupby.apply(data, self._fit_and_evaluate, orient, support)
+
+ def __call__(
+ self, data: DataFrame, groupby: GroupBy, orient: str, scales: dict[str, Scale],
+ ) -> DataFrame:
+
+ if "weight" not in data:
+ data = data.assign(weight=1)
+ data = data.dropna(subset=[orient, "weight"])
+
+ # Transform each group separately
+ grouping_vars = [str(v) for v in data if v in groupby.order]
+ if not grouping_vars or self.common_grid is True:
+ res = self._transform(data, orient, grouping_vars)
+ else:
+ if self.common_grid is False:
+ grid_vars = grouping_vars
+ else:
+ self._check_var_list_or_boolean("common_grid", grouping_vars)
+ grid_vars = [v for v in self.common_grid if v in grouping_vars]
+
+ res = (
+ GroupBy(grid_vars)
+ .apply(data, self._transform, orient, grouping_vars)
+ )
+
+ # Normalize, potentially within groups
+ if not grouping_vars or self.common_norm is True:
+ res = res.assign(group_weight=data["weight"].sum())
+ else:
+ if self.common_norm is False:
+ norm_vars = grouping_vars
+ else:
+ self._check_var_list_or_boolean("common_norm", grouping_vars)
+ norm_vars = [v for v in self.common_norm if v in grouping_vars]
+
+ res = res.join(
+ data.groupby(norm_vars)["weight"].sum().rename("group_weight"),
+ on=norm_vars,
+ )
+
+ res["density"] *= res.eval("weight / group_weight")
+ value = {"x": "y", "y": "x"}[orient]
+ res[value] = res["density"]
+ return res.drop(["weight", "group_weight"], axis=1)
diff --git a/seaborn/_stats/order.py b/seaborn/_stats/order.py
new file mode 100644
index 0000000000000000000000000000000000000000..c37c0985238efde6386e61055fca2d2f3ff2cc10
--- /dev/null
+++ b/seaborn/_stats/order.py
@@ -0,0 +1,78 @@
+
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import ClassVar, cast
+try:
+ from typing import Literal
+except ImportError:
+ from typing_extensions import Literal # type: ignore
+
+import numpy as np
+from pandas import DataFrame
+
+from seaborn._core.scales import Scale
+from seaborn._core.groupby import GroupBy
+from seaborn._stats.base import Stat
+from seaborn.utils import _version_predates
+
+
+# From https://github.com/numpy/numpy/blob/main/numpy/lib/function_base.pyi
+_MethodKind = Literal[
+ "inverted_cdf",
+ "averaged_inverted_cdf",
+ "closest_observation",
+ "interpolated_inverted_cdf",
+ "hazen",
+ "weibull",
+ "linear",
+ "median_unbiased",
+ "normal_unbiased",
+ "lower",
+ "higher",
+ "midpoint",
+ "nearest",
+]
+
+
+@dataclass
+class Perc(Stat):
+ """
+ Replace observations with percentile values.
+
+ Parameters
+ ----------
+ k : list of numbers or int
+ If a list of numbers, this gives the percentiles (in [0, 100]) to compute.
+ If an integer, compute `k` evenly-spaced percentiles between 0 and 100.
+ For example, `k=5` computes the 0, 25, 50, 75, and 100th percentiles.
+ method : str
+ Method for interpolating percentiles between observed datapoints.
+ See :func:`numpy.percentile` for valid options and more information.
+
+ Examples
+ --------
+ .. include:: ../docstrings/objects.Perc.rst
+
+ """
+ k: int | list[float] = 5
+ method: str = "linear"
+
+ group_by_orient: ClassVar[bool] = True
+
+ def _percentile(self, data: DataFrame, var: str) -> DataFrame:
+
+ k = list(np.linspace(0, 100, self.k)) if isinstance(self.k, int) else self.k
+ method = cast(_MethodKind, self.method)
+ values = data[var].dropna()
+ if _version_predates(np, "1.22"):
+ res = np.percentile(values, k, interpolation=method) # type: ignore
+ else:
+ res = np.percentile(data[var].dropna(), k, method=method)
+ return DataFrame({var: res, "percentile": k})
+
+ def __call__(
+ self, data: DataFrame, groupby: GroupBy, orient: str, scales: dict[str, Scale],
+ ) -> DataFrame:
+
+ var = {"x": "y", "y": "x"}[orient]
+ return groupby.apply(data, self._percentile, var)
diff --git a/seaborn/_stats/regression.py b/seaborn/_stats/regression.py
new file mode 100644
index 0000000000000000000000000000000000000000..9ec81a4e5c6ae4eca0baad56b23a5cc1e21a9399
--- /dev/null
+++ b/seaborn/_stats/regression.py
@@ -0,0 +1,50 @@
+from __future__ import annotations
+from dataclasses import dataclass
+
+import numpy as np
+import pandas as pd
+
+from seaborn._stats.base import Stat
+
+
+@dataclass
+class PolyFit(Stat):
+ """
+ Fit a polynomial of the given order and resample data onto predicted curve.
+ """
+ # This is a provisional class that is useful for building out functionality.
+ # It may or may not change substantially in form or dissappear as we think
+ # through the organization of the stats subpackage.
+
+ order: int = 2
+ gridsize: int = 100
+
+ def _fit_predict(self, data):
+
+ x = data["x"]
+ y = data["y"]
+ if x.nunique() <= self.order:
+ # TODO warn?
+ xx = yy = []
+ else:
+ p = np.polyfit(x, y, self.order)
+ xx = np.linspace(x.min(), x.max(), self.gridsize)
+ yy = np.polyval(p, xx)
+
+ return pd.DataFrame(dict(x=xx, y=yy))
+
+ # TODO we should have a way of identifying the method that will be applied
+ # and then only define __call__ on a base-class of stats with this pattern
+
+ def __call__(self, data, groupby, orient, scales):
+
+ return (
+ groupby
+ .apply(data.dropna(subset=["x", "y"]), self._fit_predict)
+ )
+
+
+@dataclass
+class OLSFit(Stat):
+
+ ...
diff --git a/seaborn/_testing.py b/seaborn/_testing.py
new file mode 100644
index 0000000000000000000000000000000000000000..c6f821cbe26f44a720cc8863fe6a863d61a275dd
--- /dev/null
+++ b/seaborn/_testing.py
@@ -0,0 +1,90 @@
+import numpy as np
+import matplotlib as mpl
+from matplotlib.colors import to_rgb, to_rgba
+from numpy.testing import assert_array_equal
+
+
+USE_PROPS = [
+ "alpha",
+ "edgecolor",
+ "facecolor",
+ "fill",
+ "hatch",
+ "height",
+ "linestyle",
+ "linewidth",
+ "paths",
+ "xy",
+ "xydata",
+ "sizes",
+ "zorder",
+]
+
+
+def assert_artists_equal(list1, list2):
+
+ assert len(list1) == len(list2)
+ for a1, a2 in zip(list1, list2):
+ assert a1.__class__ == a2.__class__
+ prop1 = a1.properties()
+ prop2 = a2.properties()
+ for key in USE_PROPS:
+ if key not in prop1:
+ continue
+ v1 = prop1[key]
+ v2 = prop2[key]
+ if key == "paths":
+ for p1, p2 in zip(v1, v2):
+ assert_array_equal(p1.vertices, p2.vertices)
+ assert_array_equal(p1.codes, p2.codes)
+ elif key == "color":
+ v1 = mpl.colors.to_rgba(v1)
+ v2 = mpl.colors.to_rgba(v2)
+ assert v1 == v2
+ elif isinstance(v1, np.ndarray):
+ assert_array_equal(v1, v2)
+ else:
+ assert v1 == v2
+
+
+def assert_legends_equal(leg1, leg2):
+
+ assert leg1.get_title().get_text() == leg2.get_title().get_text()
+ for t1, t2 in zip(leg1.get_texts(), leg2.get_texts()):
+ assert t1.get_text() == t2.get_text()
+
+ assert_artists_equal(
+ leg1.get_patches(), leg2.get_patches(),
+ )
+ assert_artists_equal(
+ leg1.get_lines(), leg2.get_lines(),
+ )
+
+
+def assert_plots_equal(ax1, ax2, labels=True):
+
+ assert_artists_equal(ax1.patches, ax2.patches)
+ assert_artists_equal(ax1.lines, ax2.lines)
+ assert_artists_equal(ax1.collections, ax2.collections)
+
+ if labels:
+ assert ax1.get_xlabel() == ax2.get_xlabel()
+ assert ax1.get_ylabel() == ax2.get_ylabel()
+
+
+def assert_colors_equal(a, b, check_alpha=True):
+
+ def handle_array(x):
+
+ if isinstance(x, np.ndarray):
+ if x.ndim > 1:
+ x = np.unique(x, axis=0).squeeze()
+ if x.ndim > 1:
+ raise ValueError("Color arrays must be 1 dimensional")
+ return x
+
+ a = handle_array(a)
+ b = handle_array(b)
+
+ f = to_rgba if check_alpha else to_rgb
+ assert f(a) == f(b)
diff --git a/seaborn/algorithms.py b/seaborn/algorithms.py
new file mode 100644
index 0000000000000000000000000000000000000000..2e34b9dd9cdffb5d82f56674fac4896de91a4d0a
--- /dev/null
+++ b/seaborn/algorithms.py
@@ -0,0 +1,120 @@
+"""Algorithms to support fitting routines in seaborn plotting functions."""
+import numpy as np
+import warnings
+
+
+def bootstrap(*args, **kwargs):
+ """Resample one or more arrays with replacement and store aggregate values.
+
+ Positional arguments are a sequence of arrays to bootstrap along the first
+ axis and pass to a summary function.
+
+ Keyword arguments:
+ n_boot : int, default=10000
+ Number of iterations
+ axis : int, default=None
+ Will pass axis to ``func`` as a keyword argument.
+ units : array, default=None
+ Array of sampling unit IDs. When used the bootstrap resamples units
+ and then observations within units instead of individual
+ datapoints.
+ func : string or callable, default="mean"
+ Function to call on the args that are passed in. If string, uses as
+ name of function in the numpy namespace. If nans are present in the
+ data, will try to use nan-aware version of named function.
+ seed : Generator | SeedSequence | RandomState | int | None
+ Seed for the random number generator; useful if you want
+ reproducible resamples.
+
+ Returns
+ -------
+ boot_dist: array
+ array of bootstrapped statistic values
+
+ """
+ # Ensure list of arrays are same length
+ if len(np.unique(list(map(len, args)))) > 1:
+ raise ValueError("All input arrays must have the same length")
+ n = len(args[0])
+
+ # Default keyword arguments
+ n_boot = kwargs.get("n_boot", 10000)
+ func = kwargs.get("func", "mean")
+ axis = kwargs.get("axis", None)
+ units = kwargs.get("units", None)
+ random_seed = kwargs.get("random_seed", None)
+ if random_seed is not None:
+ msg = "`random_seed` has been renamed to `seed` and will be removed"
+ warnings.warn(msg)
+ seed = kwargs.get("seed", random_seed)
+ if axis is None:
+ func_kwargs = dict()
+ else:
+ func_kwargs = dict(axis=axis)
+
+ # Initialize the resampler
+ if isinstance(seed, np.random.RandomState):
+ rng = seed
+ else:
+ rng = np.random.default_rng(seed)
+
+ # Coerce to arrays
+ args = list(map(np.asarray, args))
+ if units is not None:
+ units = np.asarray(units)
+
+ if isinstance(func, str):
+
+ # Allow named numpy functions
+ f = getattr(np, func)
+
+ # Try to use nan-aware version of function if necessary
+ missing_data = np.isnan(np.sum(np.column_stack(args)))
+
+ if missing_data and not func.startswith("nan"):
+ nanf = getattr(np, f"nan{func}", None)
+ if nanf is None:
+ msg = f"Data contain nans but no nan-aware version of `{func}` found"
+ warnings.warn(msg, UserWarning)
+ else:
+ f = nanf
+
+ else:
+ f = func
+
+ # Handle numpy changes
+ try:
+ integers = rng.integers
+ except AttributeError:
+ integers = rng.randint
+
+ # Do the bootstrap
+ if units is not None:
+ return _structured_bootstrap(args, n_boot, units, f,
+ func_kwargs, integers)
+
+ boot_dist = []
+ for i in range(int(n_boot)):
+ resampler = integers(0, n, n, dtype=np.intp) # intp is indexing dtype
+ sample = [a.take(resampler, axis=0) for a in args]
+ boot_dist.append(f(*sample, **func_kwargs))
+ return np.array(boot_dist)
+
+
+def _structured_bootstrap(args, n_boot, units, func, func_kwargs, integers):
+ """Resample units instead of datapoints."""
+ unique_units = np.unique(units)
+ n_units = len(unique_units)
+
+ args = [[a[units == unit] for unit in unique_units] for a in args]
+
+ boot_dist = []
+ for i in range(int(n_boot)):
+ resampler = integers(0, n_units, n_units, dtype=np.intp)
+ sample = [[a[i] for i in resampler] for a in args]
+ lengths = map(len, sample[0])
+ resampler = [integers(0, n, n, dtype=np.intp) for n in lengths]
+ sample = [[c.take(r, axis=0) for c, r in zip(a, resampler)] for a in sample]
+ sample = list(map(np.concatenate, sample))
+ boot_dist.append(func(*sample, **func_kwargs))
+ return np.array(boot_dist)
diff --git a/seaborn/axisgrid.py b/seaborn/axisgrid.py
new file mode 100644
index 0000000000000000000000000000000000000000..17d333bc89b79d3b43310493456176e9c69fab4e
--- /dev/null
+++ b/seaborn/axisgrid.py
@@ -0,0 +1,2401 @@
+from __future__ import annotations
+from itertools import product
+from inspect import signature
+import warnings
+from textwrap import dedent
+
+import numpy as np
+import pandas as pd
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+
+from ._base import VectorPlotter, variable_type, categorical_order
+from ._core.data import handle_data_source
+from ._compat import share_axis, get_legend_handles
+from . import utils
+from .utils import (
+ adjust_legend_subtitles,
+ set_hls_values,
+ _check_argument,
+ _draw_figure,
+ _disable_autolayout
+)
+from .palettes import color_palette, blend_palette
+from ._docstrings import (
+ DocstringComponents,
+ _core_docs,
+)
+
+__all__ = ["FacetGrid", "PairGrid", "JointGrid", "pairplot", "jointplot"]
+
+
+_param_docs = DocstringComponents.from_nested_components(
+ core=_core_docs["params"],
+)
+
+
+class _BaseGrid:
+ """Base class for grids of subplots."""
+
+ def set(self, **kwargs):
+ """Set attributes on each subplot Axes."""
+ for ax in self.axes.flat:
+ if ax is not None: # Handle removed axes
+ ax.set(**kwargs)
+ return self
+
+ @property
+ def fig(self):
+ """DEPRECATED: prefer the `figure` property."""
+ # Grid.figure is preferred because it matches the Axes attribute name.
+ # But as the maintanace burden on having this property is minimal,
+ # let's be slow about formally deprecating it. For now just note its deprecation
+ # in the docstring; add a warning in version 0.13, and eventually remove it.
+ return self._figure
+
+ @property
+ def figure(self):
+ """Access the :class:`matplotlib.figure.Figure` object underlying the grid."""
+ return self._figure
+
+ def apply(self, func, *args, **kwargs):
+ """
+ Pass the grid to a user-supplied function and return self.
+
+ The `func` must accept an object of this type for its first
+ positional argument. Additional arguments are passed through.
+ The return value of `func` is ignored; this method returns self.
+ See the `pipe` method if you want the return value.
+
+ Added in v0.12.0.
+
+ """
+ func(self, *args, **kwargs)
+ return self
+
+ def pipe(self, func, *args, **kwargs):
+ """
+ Pass the grid to a user-supplied function and return its value.
+
+ The `func` must accept an object of this type for its first
+ positional argument. Additional arguments are passed through.
+ The return value of `func` becomes the return value of this method.
+ See the `apply` method if you want to return self instead.
+
+ Added in v0.12.0.
+
+ """
+ return func(self, *args, **kwargs)
+
+ def savefig(self, *args, **kwargs):
+ """
+ Save an image of the plot.
+
+ This wraps :meth:`matplotlib.figure.Figure.savefig`, using bbox_inches="tight"
+ by default. Parameters are passed through to the matplotlib function.
+
+ """
+ kwargs = kwargs.copy()
+ kwargs.setdefault("bbox_inches", "tight")
+ self.figure.savefig(*args, **kwargs)
+
+
+class Grid(_BaseGrid):
+ """A grid that can have multiple subplots and an external legend."""
+ _margin_titles = False
+ _legend_out = True
+
+ def __init__(self):
+
+ self._tight_layout_rect = [0, 0, 1, 1]
+ self._tight_layout_pad = None
+
+ # This attribute is set externally and is a hack to handle newer functions that
+ # don't add proxy artists onto the Axes. We need an overall cleaner approach.
+ self._extract_legend_handles = False
+
+ def tight_layout(self, *args, **kwargs):
+ """Call fig.tight_layout within rect that exclude the legend."""
+ kwargs = kwargs.copy()
+ kwargs.setdefault("rect", self._tight_layout_rect)
+ if self._tight_layout_pad is not None:
+ kwargs.setdefault("pad", self._tight_layout_pad)
+ self._figure.tight_layout(*args, **kwargs)
+ return self
+
+ def add_legend(self, legend_data=None, title=None, label_order=None,
+ adjust_subtitles=False, **kwargs):
+ """Draw a legend, maybe placing it outside axes and resizing the figure.
+
+ Parameters
+ ----------
+ legend_data : dict
+ Dictionary mapping label names (or two-element tuples where the
+ second element is a label name) to matplotlib artist handles. The
+ default reads from ``self._legend_data``.
+ title : string
+ Title for the legend. The default reads from ``self._hue_var``.
+ label_order : list of labels
+ The order that the legend entries should appear in. The default
+ reads from ``self.hue_names``.
+ adjust_subtitles : bool
+ If True, modify entries with invisible artists to left-align
+ the labels and set the font size to that of a title.
+ kwargs : key, value pairings
+ Other keyword arguments are passed to the underlying legend methods
+ on the Figure or Axes object.
+
+ Returns
+ -------
+ self : Grid instance
+ Returns self for easy chaining.
+
+ """
+ # Find the data for the legend
+ if legend_data is None:
+ legend_data = self._legend_data
+ if label_order is None:
+ if self.hue_names is None:
+ label_order = list(legend_data.keys())
+ else:
+ label_order = list(map(utils.to_utf8, self.hue_names))
+
+ blank_handle = mpl.patches.Patch(alpha=0, linewidth=0)
+ handles = [legend_data.get(lab, blank_handle) for lab in label_order]
+ title = self._hue_var if title is None else title
+ title_size = mpl.rcParams["legend.title_fontsize"]
+
+ # Unpack nested labels from a hierarchical legend
+ labels = []
+ for entry in label_order:
+ if isinstance(entry, tuple):
+ _, label = entry
+ else:
+ label = entry
+ labels.append(label)
+
+ # Set default legend kwargs
+ kwargs.setdefault("scatterpoints", 1)
+
+ if self._legend_out:
+
+ kwargs.setdefault("frameon", False)
+ kwargs.setdefault("loc", "center right")
+
+ # Draw a full-figure legend outside the grid
+ figlegend = self._figure.legend(handles, labels, **kwargs)
+
+ self._legend = figlegend
+ figlegend.set_title(title, prop={"size": title_size})
+
+ if adjust_subtitles:
+ adjust_legend_subtitles(figlegend)
+
+ # Draw the plot to set the bounding boxes correctly
+ _draw_figure(self._figure)
+
+ # Calculate and set the new width of the figure so the legend fits
+ legend_width = figlegend.get_window_extent().width / self._figure.dpi
+ fig_width, fig_height = self._figure.get_size_inches()
+ self._figure.set_size_inches(fig_width + legend_width, fig_height)
+
+ # Draw the plot again to get the new transformations
+ _draw_figure(self._figure)
+
+ # Now calculate how much space we need on the right side
+ legend_width = figlegend.get_window_extent().width / self._figure.dpi
+ space_needed = legend_width / (fig_width + legend_width)
+ margin = .04 if self._margin_titles else .01
+ self._space_needed = margin + space_needed
+ right = 1 - self._space_needed
+
+ # Place the subplot axes to give space for the legend
+ self._figure.subplots_adjust(right=right)
+ self._tight_layout_rect[2] = right
+
+ else:
+ # Draw a legend in the first axis
+ ax = self.axes.flat[0]
+ kwargs.setdefault("loc", "best")
+
+ leg = ax.legend(handles, labels, **kwargs)
+ leg.set_title(title, prop={"size": title_size})
+ self._legend = leg
+
+ if adjust_subtitles:
+ adjust_legend_subtitles(leg)
+
+ return self
+
+ def _update_legend_data(self, ax):
+ """Extract the legend data from an axes object and save it."""
+ data = {}
+
+ # Get data directly from the legend, which is necessary
+ # for newer functions that don't add labeled proxy artists
+ if ax.legend_ is not None and self._extract_legend_handles:
+ handles = get_legend_handles(ax.legend_)
+ labels = [t.get_text() for t in ax.legend_.texts]
+ data.update({label: handle for handle, label in zip(handles, labels)})
+
+ handles, labels = ax.get_legend_handles_labels()
+ data.update({label: handle for handle, label in zip(handles, labels)})
+
+ self._legend_data.update(data)
+
+ # Now clear the legend
+ ax.legend_ = None
+
+ def _get_palette(self, data, hue, hue_order, palette):
+ """Get a list of colors for the hue variable."""
+ if hue is None:
+ palette = color_palette(n_colors=1)
+
+ else:
+ hue_names = categorical_order(data[hue], hue_order)
+ n_colors = len(hue_names)
+
+ # By default use either the current color palette or HUSL
+ if palette is None:
+ current_palette = utils.get_color_cycle()
+ if n_colors > len(current_palette):
+ colors = color_palette("husl", n_colors)
+ else:
+ colors = color_palette(n_colors=n_colors)
+
+ # Allow for palette to map from hue variable names
+ elif isinstance(palette, dict):
+ color_names = [palette[h] for h in hue_names]
+ colors = color_palette(color_names, n_colors)
+
+ # Otherwise act as if we just got a list of colors
+ else:
+ colors = color_palette(palette, n_colors)
+
+ palette = color_palette(colors, n_colors)
+
+ return palette
+
+ @property
+ def legend(self):
+ """The :class:`matplotlib.legend.Legend` object, if present."""
+ try:
+ return self._legend
+ except AttributeError:
+ return None
+
+ def tick_params(self, axis='both', **kwargs):
+ """Modify the ticks, tick labels, and gridlines.
+
+ Parameters
+ ----------
+ axis : {'x', 'y', 'both'}
+ The axis on which to apply the formatting.
+ kwargs : keyword arguments
+ Additional keyword arguments to pass to
+ :meth:`matplotlib.axes.Axes.tick_params`.
+
+ Returns
+ -------
+ self : Grid instance
+ Returns self for easy chaining.
+
+ """
+ for ax in self.figure.axes:
+ ax.tick_params(axis=axis, **kwargs)
+ return self
+
+
+_facet_docs = dict(
+
+ data=dedent("""\
+ data : DataFrame
+ Tidy ("long-form") dataframe where each column is a variable and each
+ row is an observation.\
+ """),
+ rowcol=dedent("""\
+ row, col : vectors or keys in ``data``
+ Variables that define subsets to plot on different facets.\
+ """),
+ rowcol_order=dedent("""\
+ {row,col}_order : vector of strings
+ Specify the order in which levels of the ``row`` and/or ``col`` variables
+ appear in the grid of subplots.\
+ """),
+ col_wrap=dedent("""\
+ col_wrap : int
+ "Wrap" the column variable at this width, so that the column facets
+ span multiple rows. Incompatible with a ``row`` facet.\
+ """),
+ share_xy=dedent("""\
+ share{x,y} : bool, 'col', or 'row' optional
+ If true, the facets will share y axes across columns and/or x axes
+ across rows.\
+ """),
+ height=dedent("""\
+ height : scalar
+ Height (in inches) of each facet. See also: ``aspect``.\
+ """),
+ aspect=dedent("""\
+ aspect : scalar
+ Aspect ratio of each facet, so that ``aspect * height`` gives the width
+ of each facet in inches.\
+ """),
+ palette=dedent("""\
+ palette : palette name, list, or dict
+ Colors to use for the different levels of the ``hue`` variable. Should
+ be something that can be interpreted by :func:`color_palette`, or a
+ dictionary mapping hue levels to matplotlib colors.\
+ """),
+ legend_out=dedent("""\
+ legend_out : bool
+ If ``True``, the figure size will be extended, and the legend will be
+ drawn outside the plot on the center right.\
+ """),
+ margin_titles=dedent("""\
+ margin_titles : bool
+ If ``True``, the titles for the row variable are drawn to the right of
+ the last column. This option is experimental and may not work in all
+ cases.\
+ """),
+ facet_kws=dedent("""\
+ facet_kws : dict
+ Additional parameters passed to :class:`FacetGrid`.
+ """),
+)
+
+
+class FacetGrid(Grid):
+ """Multi-plot grid for plotting conditional relationships."""
+
+ def __init__(
+ self, data, *,
+ row=None, col=None, hue=None, col_wrap=None,
+ sharex=True, sharey=True, height=3, aspect=1, palette=None,
+ row_order=None, col_order=None, hue_order=None, hue_kws=None,
+ dropna=False, legend_out=True, despine=True,
+ margin_titles=False, xlim=None, ylim=None, subplot_kws=None,
+ gridspec_kws=None,
+ ):
+
+ super().__init__()
+ data = handle_data_source(data)
+
+ # Determine the hue facet layer information
+ hue_var = hue
+ if hue is None:
+ hue_names = None
+ else:
+ hue_names = categorical_order(data[hue], hue_order)
+
+ colors = self._get_palette(data, hue, hue_order, palette)
+
+ # Set up the lists of names for the row and column facet variables
+ if row is None:
+ row_names = []
+ else:
+ row_names = categorical_order(data[row], row_order)
+
+ if col is None:
+ col_names = []
+ else:
+ col_names = categorical_order(data[col], col_order)
+
+ # Additional dict of kwarg -> list of values for mapping the hue var
+ hue_kws = hue_kws if hue_kws is not None else {}
+
+ # Make a boolean mask that is True anywhere there is an NA
+ # value in one of the faceting variables, but only if dropna is True
+ none_na = np.zeros(len(data), bool)
+ if dropna:
+ row_na = none_na if row is None else data[row].isnull()
+ col_na = none_na if col is None else data[col].isnull()
+ hue_na = none_na if hue is None else data[hue].isnull()
+ not_na = ~(row_na | col_na | hue_na)
+ else:
+ not_na = ~none_na
+
+ # Compute the grid shape
+ ncol = 1 if col is None else len(col_names)
+ nrow = 1 if row is None else len(row_names)
+ self._n_facets = ncol * nrow
+
+ self._col_wrap = col_wrap
+ if col_wrap is not None:
+ if row is not None:
+ err = "Cannot use `row` and `col_wrap` together."
+ raise ValueError(err)
+ ncol = col_wrap
+ nrow = int(np.ceil(len(col_names) / col_wrap))
+ self._ncol = ncol
+ self._nrow = nrow
+
+ # Calculate the base figure size
+ # This can get stretched later by a legend
+ # TODO this doesn't account for axis labels
+ figsize = (ncol * height * aspect, nrow * height)
+
+ # Validate some inputs
+ if col_wrap is not None:
+ margin_titles = False
+
+ # Build the subplot keyword dictionary
+ subplot_kws = {} if subplot_kws is None else subplot_kws.copy()
+ gridspec_kws = {} if gridspec_kws is None else gridspec_kws.copy()
+ if xlim is not None:
+ subplot_kws["xlim"] = xlim
+ if ylim is not None:
+ subplot_kws["ylim"] = ylim
+
+ # --- Initialize the subplot grid
+
+ with _disable_autolayout():
+ fig = plt.figure(figsize=figsize)
+
+ if col_wrap is None:
+
+ kwargs = dict(squeeze=False,
+ sharex=sharex, sharey=sharey,
+ subplot_kw=subplot_kws,
+ gridspec_kw=gridspec_kws)
+
+ axes = fig.subplots(nrow, ncol, **kwargs)
+
+ if col is None and row is None:
+ axes_dict = {}
+ elif col is None:
+ axes_dict = dict(zip(row_names, axes.flat))
+ elif row is None:
+ axes_dict = dict(zip(col_names, axes.flat))
+ else:
+ facet_product = product(row_names, col_names)
+ axes_dict = dict(zip(facet_product, axes.flat))
+
+ else:
+
+ # If wrapping the col variable we need to make the grid ourselves
+ if gridspec_kws:
+ warnings.warn("`gridspec_kws` ignored when using `col_wrap`")
+
+ n_axes = len(col_names)
+ axes = np.empty(n_axes, object)
+ axes[0] = fig.add_subplot(nrow, ncol, 1, **subplot_kws)
+ if sharex:
+ subplot_kws["sharex"] = axes[0]
+ if sharey:
+ subplot_kws["sharey"] = axes[0]
+ for i in range(1, n_axes):
+ axes[i] = fig.add_subplot(nrow, ncol, i + 1, **subplot_kws)
+
+ axes_dict = dict(zip(col_names, axes))
+
+ # --- Set up the class attributes
+
+ # Attributes that are part of the public API but accessed through
+ # a property so that Sphinx adds them to the auto class doc
+ self._figure = fig
+ self._axes = axes
+ self._axes_dict = axes_dict
+ self._legend = None
+
+ # Public attributes that aren't explicitly documented
+ # (It's not obvious that having them be public was a good idea)
+ self.data = data
+ self.row_names = row_names
+ self.col_names = col_names
+ self.hue_names = hue_names
+ self.hue_kws = hue_kws
+
+ # Next the private variables
+ self._nrow = nrow
+ self._row_var = row
+ self._ncol = ncol
+ self._col_var = col
+
+ self._margin_titles = margin_titles
+ self._margin_titles_texts = []
+ self._col_wrap = col_wrap
+ self._hue_var = hue_var
+ self._colors = colors
+ self._legend_out = legend_out
+ self._legend_data = {}
+ self._x_var = None
+ self._y_var = None
+ self._sharex = sharex
+ self._sharey = sharey
+ self._dropna = dropna
+ self._not_na = not_na
+
+ # --- Make the axes look good
+
+ self.set_titles()
+ self.tight_layout()
+
+ if despine:
+ self.despine()
+
+ if sharex in [True, 'col']:
+ for ax in self._not_bottom_axes:
+ for label in ax.get_xticklabels():
+ label.set_visible(False)
+ ax.xaxis.offsetText.set_visible(False)
+ ax.xaxis.label.set_visible(False)
+
+ if sharey in [True, 'row']:
+ for ax in self._not_left_axes:
+ for label in ax.get_yticklabels():
+ label.set_visible(False)
+ ax.yaxis.offsetText.set_visible(False)
+ ax.yaxis.label.set_visible(False)
+
+ __init__.__doc__ = dedent("""\
+ Initialize the matplotlib figure and FacetGrid object.
+
+ This class maps a dataset onto multiple axes arrayed in a grid of rows
+ and columns that correspond to *levels* of variables in the dataset.
+ The plots it produces are often called "lattice", "trellis", or
+ "small-multiple" graphics.
+
+ It can also represent levels of a third variable with the ``hue``
+ parameter, which plots different subsets of data in different colors.
+ This uses color to resolve elements on a third dimension, but only
+ draws subsets on top of each other and will not tailor the ``hue``
+ parameter for the specific visualization the way that axes-level
+ functions that accept ``hue`` will.
+
+ The basic workflow is to initialize the :class:`FacetGrid` object with
+ the dataset and the variables that are used to structure the grid. Then
+ one or more plotting functions can be applied to each subset by calling
+ :meth:`FacetGrid.map` or :meth:`FacetGrid.map_dataframe`. Finally, the
+ plot can be tweaked with other methods to do things like change the
+ axis labels, use different ticks, or add a legend. See the detailed
+ code examples below for more information.
+
+ .. warning::
+
+ When using seaborn functions that infer semantic mappings from a
+ dataset, care must be taken to synchronize those mappings across
+ facets (e.g., by defining the ``hue`` mapping with a palette dict or
+ setting the data type of the variables to ``category``). In most cases,
+ it will be better to use a figure-level function (e.g. :func:`relplot`
+ or :func:`catplot`) than to use :class:`FacetGrid` directly.
+
+ See the :ref:`tutorial <grid_tutorial>` for more information.
+
+ Parameters
+ ----------
+ {data}
+ row, col, hue : strings
+ Variables that define subsets of the data, which will be drawn on
+ separate facets in the grid. See the ``{{var}}_order`` parameters to
+ control the order of levels of this variable.
+ {col_wrap}
+ {share_xy}
+ {height}
+ {aspect}
+ {palette}
+ {{row,col,hue}}_order : lists
+ Order for the levels of the faceting variables. By default, this
+ will be the order that the levels appear in ``data`` or, if the
+ variables are pandas categoricals, the category order.
+ hue_kws : dictionary of param -> list of values mapping
+ Other keyword arguments to insert into the plotting call to let
+ other plot attributes vary across levels of the hue variable (e.g.
+ the markers in a scatterplot).
+ {legend_out}
+ despine : boolean
+ Remove the top and right spines from the plots.
+ {margin_titles}
+ {{x, y}}lim: tuples
+ Limits for each of the axes on each facet (only relevant when
+ share{{x, y}} is True).
+ subplot_kws : dict
+ Dictionary of keyword arguments passed to matplotlib subplot(s)
+ methods.
+ gridspec_kws : dict
+ Dictionary of keyword arguments passed to
+ :class:`matplotlib.gridspec.GridSpec`
+ (via :meth:`matplotlib.figure.Figure.subplots`).
+ Ignored if ``col_wrap`` is not ``None``.
+
+ See Also
+ --------
+ PairGrid : Subplot grid for plotting pairwise relationships
+ relplot : Combine a relational plot and a :class:`FacetGrid`
+ displot : Combine a distribution plot and a :class:`FacetGrid`
+ catplot : Combine a categorical plot and a :class:`FacetGrid`
+ lmplot : Combine a regression plot and a :class:`FacetGrid`
+
+ Examples
+ --------
+
+ .. note::
+
+ These examples use seaborn functions to demonstrate some of the
+ advanced features of the class, but in most cases you will want
+ to use figue-level functions (e.g. :func:`displot`, :func:`relplot`)
+ to make the plots shown here.
+
+ .. include:: ../docstrings/FacetGrid.rst
+
+ """).format(**_facet_docs)
+
+ def facet_data(self):
+ """Generator for name indices and data subsets for each facet.
+
+ Yields
+ ------
+ (i, j, k), data_ijk : tuple of ints, DataFrame
+ The ints provide an index into the {row, col, hue}_names attribute,
+ and the dataframe contains a subset of the full data corresponding
+ to each facet. The generator yields subsets that correspond with
+ the self.axes.flat iterator, or self.axes[i, j] when `col_wrap`
+ is None.
+
+ """
+ data = self.data
+
+ # Construct masks for the row variable
+ if self.row_names:
+ row_masks = [data[self._row_var] == n for n in self.row_names]
+ else:
+ row_masks = [np.repeat(True, len(self.data))]
+
+ # Construct masks for the column variable
+ if self.col_names:
+ col_masks = [data[self._col_var] == n for n in self.col_names]
+ else:
+ col_masks = [np.repeat(True, len(self.data))]
+
+ # Construct masks for the hue variable
+ if self.hue_names:
+ hue_masks = [data[self._hue_var] == n for n in self.hue_names]
+ else:
+ hue_masks = [np.repeat(True, len(self.data))]
+
+ # Here is the main generator loop
+ for (i, row), (j, col), (k, hue) in product(enumerate(row_masks),
+ enumerate(col_masks),
+ enumerate(hue_masks)):
+ data_ijk = data[row & col & hue & self._not_na]
+ yield (i, j, k), data_ijk
+
+ def map(self, func, *args, **kwargs):
+ """Apply a plotting function to each facet's subset of the data.
+
+ Parameters
+ ----------
+ func : callable
+ A plotting function that takes data and keyword arguments. It
+ must plot to the currently active matplotlib Axes and take a
+ `color` keyword argument. If faceting on the `hue` dimension,
+ it must also take a `label` keyword argument.
+ args : strings
+ Column names in self.data that identify variables with data to
+ plot. The data for each variable is passed to `func` in the
+ order the variables are specified in the call.
+ kwargs : keyword arguments
+ All keyword arguments are passed to the plotting function.
+
+ Returns
+ -------
+ self : object
+ Returns self.
+
+ """
+ # If color was a keyword argument, grab it here
+ kw_color = kwargs.pop("color", None)
+
+ # How we use the function depends on where it comes from
+ func_module = str(getattr(func, "__module__", ""))
+
+ # Check for categorical plots without order information
+ if func_module == "seaborn.categorical":
+ if "order" not in kwargs:
+ warning = ("Using the {} function without specifying "
+ "`order` is likely to produce an incorrect "
+ "plot.".format(func.__name__))
+ warnings.warn(warning)
+ if len(args) == 3 and "hue_order" not in kwargs:
+ warning = ("Using the {} function without specifying "
+ "`hue_order` is likely to produce an incorrect "
+ "plot.".format(func.__name__))
+ warnings.warn(warning)
+
+ # Iterate over the data subsets
+ for (row_i, col_j, hue_k), data_ijk in self.facet_data():
+
+ # If this subset is null, move on
+ if not data_ijk.values.size:
+ continue
+
+ # Get the current axis
+ modify_state = not func_module.startswith("seaborn")
+ ax = self.facet_axis(row_i, col_j, modify_state)
+
+ # Decide what color to plot with
+ kwargs["color"] = self._facet_color(hue_k, kw_color)
+
+ # Insert the other hue aesthetics if appropriate
+ for kw, val_list in self.hue_kws.items():
+ kwargs[kw] = val_list[hue_k]
+
+ # Insert a label in the keyword arguments for the legend
+ if self._hue_var is not None:
+ kwargs["label"] = utils.to_utf8(self.hue_names[hue_k])
+
+ # Get the actual data we are going to plot with
+ plot_data = data_ijk[list(args)]
+ if self._dropna:
+ plot_data = plot_data.dropna()
+ plot_args = [v for k, v in plot_data.items()]
+
+ # Some matplotlib functions don't handle pandas objects correctly
+ if func_module.startswith("matplotlib"):
+ plot_args = [v.values for v in plot_args]
+
+ # Draw the plot
+ self._facet_plot(func, ax, plot_args, kwargs)
+
+ # Finalize the annotations and layout
+ self._finalize_grid(args[:2])
+
+ return self
+
+ def map_dataframe(self, func, *args, **kwargs):
+ """Like ``.map`` but passes args as strings and inserts data in kwargs.
+
+ This method is suitable for plotting with functions that accept a
+ long-form DataFrame as a `data` keyword argument and access the
+ data in that DataFrame using string variable names.
+
+ Parameters
+ ----------
+ func : callable
+ A plotting function that takes data and keyword arguments. Unlike
+ the `map` method, a function used here must "understand" Pandas
+ objects. It also must plot to the currently active matplotlib Axes
+ and take a `color` keyword argument. If faceting on the `hue`
+ dimension, it must also take a `label` keyword argument.
+ args : strings
+ Column names in self.data that identify variables with data to
+ plot. The data for each variable is passed to `func` in the
+ order the variables are specified in the call.
+ kwargs : keyword arguments
+ All keyword arguments are passed to the plotting function.
+
+ Returns
+ -------
+ self : object
+ Returns self.
+
+ """
+
+ # If color was a keyword argument, grab it here
+ kw_color = kwargs.pop("color", None)
+
+ # Iterate over the data subsets
+ for (row_i, col_j, hue_k), data_ijk in self.facet_data():
+
+ # If this subset is null, move on
+ if not data_ijk.values.size:
+ continue
+
+ # Get the current axis
+ modify_state = not str(func.__module__).startswith("seaborn")
+ ax = self.facet_axis(row_i, col_j, modify_state)
+
+ # Decide what color to plot with
+ kwargs["color"] = self._facet_color(hue_k, kw_color)
+
+ # Insert the other hue aesthetics if appropriate
+ for kw, val_list in self.hue_kws.items():
+ kwargs[kw] = val_list[hue_k]
+
+ # Insert a label in the keyword arguments for the legend
+ if self._hue_var is not None:
+ kwargs["label"] = self.hue_names[hue_k]
+
+ # Stick the facet dataframe into the kwargs
+ if self._dropna:
+ data_ijk = data_ijk.dropna()
+ kwargs["data"] = data_ijk
+
+ # Draw the plot
+ self._facet_plot(func, ax, args, kwargs)
+
+ # For axis labels, prefer to use positional args for backcompat
+ # but also extract the x/y kwargs and use if no corresponding arg
+ axis_labels = [kwargs.get("x", None), kwargs.get("y", None)]
+ for i, val in enumerate(args[:2]):
+ axis_labels[i] = val
+ self._finalize_grid(axis_labels)
+
+ return self
+
+ def _facet_color(self, hue_index, kw_color):
+
+ color = self._colors[hue_index]
+ if kw_color is not None:
+ return kw_color
+ elif color is not None:
+ return color
+
+ def _facet_plot(self, func, ax, plot_args, plot_kwargs):
+
+ # Draw the plot
+ if str(func.__module__).startswith("seaborn"):
+ plot_kwargs = plot_kwargs.copy()
+ semantics = ["x", "y", "hue", "size", "style"]
+ for key, val in zip(semantics, plot_args):
+ plot_kwargs[key] = val
+ plot_args = []
+ plot_kwargs["ax"] = ax
+ func(*plot_args, **plot_kwargs)
+
+ # Sort out the supporting information
+ self._update_legend_data(ax)
+
+ def _finalize_grid(self, axlabels):
+ """Finalize the annotations and layout."""
+ self.set_axis_labels(*axlabels)
+ self.tight_layout()
+
+ def facet_axis(self, row_i, col_j, modify_state=True):
+ """Make the axis identified by these indices active and return it."""
+
+ # Calculate the actual indices of the axes to plot on
+ if self._col_wrap is not None:
+ ax = self.axes.flat[col_j]
+ else:
+ ax = self.axes[row_i, col_j]
+
+ # Get a reference to the axes object we want, and make it active
+ if modify_state:
+ plt.sca(ax)
+ return ax
+
+ def despine(self, **kwargs):
+ """Remove axis spines from the facets."""
+ utils.despine(self._figure, **kwargs)
+ return self
+
+ def set_axis_labels(self, x_var=None, y_var=None, clear_inner=True, **kwargs):
+ """Set axis labels on the left column and bottom row of the grid."""
+ if x_var is not None:
+ self._x_var = x_var
+ self.set_xlabels(x_var, clear_inner=clear_inner, **kwargs)
+ if y_var is not None:
+ self._y_var = y_var
+ self.set_ylabels(y_var, clear_inner=clear_inner, **kwargs)
+
+ return self
+
+ def set_xlabels(self, label=None, clear_inner=True, **kwargs):
+ """Label the x axis on the bottom row of the grid."""
+ if label is None:
+ label = self._x_var
+ for ax in self._bottom_axes:
+ ax.set_xlabel(label, **kwargs)
+ if clear_inner:
+ for ax in self._not_bottom_axes:
+ ax.set_xlabel("")
+ return self
+
+ def set_ylabels(self, label=None, clear_inner=True, **kwargs):
+ """Label the y axis on the left column of the grid."""
+ if label is None:
+ label = self._y_var
+ for ax in self._left_axes:
+ ax.set_ylabel(label, **kwargs)
+ if clear_inner:
+ for ax in self._not_left_axes:
+ ax.set_ylabel("")
+ return self
+
+ def set_xticklabels(self, labels=None, step=None, **kwargs):
+ """Set x axis tick labels of the grid."""
+ for ax in self.axes.flat:
+ curr_ticks = ax.get_xticks()
+ ax.set_xticks(curr_ticks)
+ if labels is None:
+ curr_labels = [label.get_text() for label in ax.get_xticklabels()]
+ if step is not None:
+ xticks = ax.get_xticks()[::step]
+ curr_labels = curr_labels[::step]
+ ax.set_xticks(xticks)
+ ax.set_xticklabels(curr_labels, **kwargs)
+ else:
+ ax.set_xticklabels(labels, **kwargs)
+ return self
+
+ def set_yticklabels(self, labels=None, **kwargs):
+ """Set y axis tick labels on the left column of the grid."""
+ for ax in self.axes.flat:
+ curr_ticks = ax.get_yticks()
+ ax.set_yticks(curr_ticks)
+ if labels is None:
+ curr_labels = [label.get_text() for label in ax.get_yticklabels()]
+ ax.set_yticklabels(curr_labels, **kwargs)
+ else:
+ ax.set_yticklabels(labels, **kwargs)
+ return self
+
+ def set_titles(self, template=None, row_template=None, col_template=None, **kwargs):
+ """Draw titles either above each facet or on the grid margins.
+
+ Parameters
+ ----------
+ template : string
+ Template for all titles with the formatting keys {col_var} and
+ {col_name} (if using a `col` faceting variable) and/or {row_var}
+ and {row_name} (if using a `row` faceting variable).
+ row_template:
+ Template for the row variable when titles are drawn on the grid
+ margins. Must have {row_var} and {row_name} formatting keys.
+ col_template:
+ Template for the column variable when titles are drawn on the grid
+ margins. Must have {col_var} and {col_name} formatting keys.
+
+ Returns
+ -------
+ self: object
+ Returns self.
+
+ """
+ args = dict(row_var=self._row_var, col_var=self._col_var)
+ kwargs["size"] = kwargs.pop("size", mpl.rcParams["axes.labelsize"])
+
+ # Establish default templates
+ if row_template is None:
+ row_template = "{row_var} = {row_name}"
+ if col_template is None:
+ col_template = "{col_var} = {col_name}"
+ if template is None:
+ if self._row_var is None:
+ template = col_template
+ elif self._col_var is None:
+ template = row_template
+ else:
+ template = " | ".join([row_template, col_template])
+
+ row_template = utils.to_utf8(row_template)
+ col_template = utils.to_utf8(col_template)
+ template = utils.to_utf8(template)
+
+ if self._margin_titles:
+
+ # Remove any existing title texts
+ for text in self._margin_titles_texts:
+ text.remove()
+ self._margin_titles_texts = []
+
+ if self.row_names is not None:
+ # Draw the row titles on the right edge of the grid
+ for i, row_name in enumerate(self.row_names):
+ ax = self.axes[i, -1]
+ args.update(dict(row_name=row_name))
+ title = row_template.format(**args)
+ text = ax.annotate(
+ title, xy=(1.02, .5), xycoords="axes fraction",
+ rotation=270, ha="left", va="center",
+ **kwargs
+ )
+ self._margin_titles_texts.append(text)
+
+ if self.col_names is not None:
+ # Draw the column titles as normal titles
+ for j, col_name in enumerate(self.col_names):
+ args.update(dict(col_name=col_name))
+ title = col_template.format(**args)
+ self.axes[0, j].set_title(title, **kwargs)
+
+ return self
+
+ # Otherwise title each facet with all the necessary information
+ if (self._row_var is not None) and (self._col_var is not None):
+ for i, row_name in enumerate(self.row_names):
+ for j, col_name in enumerate(self.col_names):
+ args.update(dict(row_name=row_name, col_name=col_name))
+ title = template.format(**args)
+ self.axes[i, j].set_title(title, **kwargs)
+ elif self.row_names is not None and len(self.row_names):
+ for i, row_name in enumerate(self.row_names):
+ args.update(dict(row_name=row_name))
+ title = template.format(**args)
+ self.axes[i, 0].set_title(title, **kwargs)
+ elif self.col_names is not None and len(self.col_names):
+ for i, col_name in enumerate(self.col_names):
+ args.update(dict(col_name=col_name))
+ title = template.format(**args)
+ # Index the flat array so col_wrap works
+ self.axes.flat[i].set_title(title, **kwargs)
+ return self
+
+ def refline(self, *, x=None, y=None, color='.5', linestyle='--', **line_kws):
+ """Add a reference line(s) to each facet.
+
+ Parameters
+ ----------
+ x, y : numeric
+ Value(s) to draw the line(s) at.
+ color : :mod:`matplotlib color <matplotlib.colors>`
+ Specifies the color of the reference line(s). Pass ``color=None`` to
+ use ``hue`` mapping.
+ linestyle : str
+ Specifies the style of the reference line(s).
+ line_kws : key, value mappings
+ Other keyword arguments are passed to :meth:`matplotlib.axes.Axes.axvline`
+ when ``x`` is not None and :meth:`matplotlib.axes.Axes.axhline` when ``y``
+ is not None.
+
+ Returns
+ -------
+ :class:`FacetGrid` instance
+ Returns ``self`` for easy method chaining.
+
+ """
+ line_kws['color'] = color
+ line_kws['linestyle'] = linestyle
+
+ if x is not None:
+ self.map(plt.axvline, x=x, **line_kws)
+
+ if y is not None:
+ self.map(plt.axhline, y=y, **line_kws)
+
+ return self
+
+ # ------ Properties that are part of the public API and documented by Sphinx
+
+ @property
+ def axes(self):
+ """An array of the :class:`matplotlib.axes.Axes` objects in the grid."""
+ return self._axes
+
+ @property
+ def ax(self):
+ """The :class:`matplotlib.axes.Axes` when no faceting variables are assigned."""
+ if self.axes.shape == (1, 1):
+ return self.axes[0, 0]
+ else:
+ err = (
+ "Use the `.axes` attribute when facet variables are assigned."
+ )
+ raise AttributeError(err)
+
+ @property
+ def axes_dict(self):
+ """A mapping of facet names to corresponding :class:`matplotlib.axes.Axes`.
+
+ If only one of ``row`` or ``col`` is assigned, each key is a string
+ representing a level of that variable. If both facet dimensions are
+ assigned, each key is a ``({row_level}, {col_level})`` tuple.
+
+ """
+ return self._axes_dict
+
+ # ------ Private properties, that require some computation to get
+
+ @property
+ def _inner_axes(self):
+ """Return a flat array of the inner axes."""
+ if self._col_wrap is None:
+ return self.axes[:-1, 1:].flat
+ else:
+ axes = []
+ n_empty = self._nrow * self._ncol - self._n_facets
+ for i, ax in enumerate(self.axes):
+ append = (
+ i % self._ncol
+ and i < (self._ncol * (self._nrow - 1))
+ and i < (self._ncol * (self._nrow - 1) - n_empty)
+ )
+ if append:
+ axes.append(ax)
+ return np.array(axes, object).flat
+
+ @property
+ def _left_axes(self):
+ """Return a flat array of the left column of axes."""
+ if self._col_wrap is None:
+ return self.axes[:, 0].flat
+ else:
+ axes = []
+ for i, ax in enumerate(self.axes):
+ if not i % self._ncol:
+ axes.append(ax)
+ return np.array(axes, object).flat
+
+ @property
+ def _not_left_axes(self):
+ """Return a flat array of axes that aren't on the left column."""
+ if self._col_wrap is None:
+ return self.axes[:, 1:].flat
+ else:
+ axes = []
+ for i, ax in enumerate(self.axes):
+ if i % self._ncol:
+ axes.append(ax)
+ return np.array(axes, object).flat
+
+ @property
+ def _bottom_axes(self):
+ """Return a flat array of the bottom row of axes."""
+ if self._col_wrap is None:
+ return self.axes[-1, :].flat
+ else:
+ axes = []
+ n_empty = self._nrow * self._ncol - self._n_facets
+ for i, ax in enumerate(self.axes):
+ append = (
+ i >= (self._ncol * (self._nrow - 1))
+ or i >= (self._ncol * (self._nrow - 1) - n_empty)
+ )
+ if append:
+ axes.append(ax)
+ return np.array(axes, object).flat
+
+ @property
+ def _not_bottom_axes(self):
+ """Return a flat array of axes that aren't on the bottom row."""
+ if self._col_wrap is None:
+ return self.axes[:-1, :].flat
+ else:
+ axes = []
+ n_empty = self._nrow * self._ncol - self._n_facets
+ for i, ax in enumerate(self.axes):
+ append = (
+ i < (self._ncol * (self._nrow - 1))
+ and i < (self._ncol * (self._nrow - 1) - n_empty)
+ )
+ if append:
+ axes.append(ax)
+ return np.array(axes, object).flat
+
+
+class PairGrid(Grid):
+ """Subplot grid for plotting pairwise relationships in a dataset.
+
+ This object maps each variable in a dataset onto a column and row in a
+ grid of multiple axes. Different axes-level plotting functions can be
+ used to draw bivariate plots in the upper and lower triangles, and the
+ marginal distribution of each variable can be shown on the diagonal.
+
+ Several different common plots can be generated in a single line using
+ :func:`pairplot`. Use :class:`PairGrid` when you need more flexibility.
+
+ See the :ref:`tutorial <grid_tutorial>` for more information.
+
+ """
+ def __init__(
+ self, data, *, hue=None, vars=None, x_vars=None, y_vars=None,
+ hue_order=None, palette=None, hue_kws=None, corner=False, diag_sharey=True,
+ height=2.5, aspect=1, layout_pad=.5, despine=True, dropna=False,
+ ):
+ """Initialize the plot figure and PairGrid object.
+
+ Parameters
+ ----------
+ data : DataFrame
+ Tidy (long-form) dataframe where each column is a variable and
+ each row is an observation.
+ hue : string (variable name)
+ Variable in ``data`` to map plot aspects to different colors. This
+ variable will be excluded from the default x and y variables.
+ vars : list of variable names
+ Variables within ``data`` to use, otherwise use every column with
+ a numeric datatype.
+ {x, y}_vars : lists of variable names
+ Variables within ``data`` to use separately for the rows and
+ columns of the figure; i.e. to make a non-square plot.
+ hue_order : list of strings
+ Order for the levels of the hue variable in the palette
+ palette : dict or seaborn color palette
+ Set of colors for mapping the ``hue`` variable. If a dict, keys
+ should be values in the ``hue`` variable.
+ hue_kws : dictionary of param -> list of values mapping
+ Other keyword arguments to insert into the plotting call to let
+ other plot attributes vary across levels of the hue variable (e.g.
+ the markers in a scatterplot).
+ corner : bool
+ If True, don't add axes to the upper (off-diagonal) triangle of the
+ grid, making this a "corner" plot.
+ height : scalar
+ Height (in inches) of each facet.
+ aspect : scalar
+ Aspect * height gives the width (in inches) of each facet.
+ layout_pad : scalar
+ Padding between axes; passed to ``fig.tight_layout``.
+ despine : boolean
+ Remove the top and right spines from the plots.
+ dropna : boolean
+ Drop missing values from the data before plotting.
+
+ See Also
+ --------
+ pairplot : Easily drawing common uses of :class:`PairGrid`.
+ FacetGrid : Subplot grid for plotting conditional relationships.
+
+ Examples
+ --------
+
+ .. include:: ../docstrings/PairGrid.rst
+
+ """
+
+ super().__init__()
+ data = handle_data_source(data)
+
+ # Sort out the variables that define the grid
+ numeric_cols = self._find_numeric_cols(data)
+ if hue in numeric_cols:
+ numeric_cols.remove(hue)
+ if vars is not None:
+ x_vars = list(vars)
+ y_vars = list(vars)
+ if x_vars is None:
+ x_vars = numeric_cols
+ if y_vars is None:
+ y_vars = numeric_cols
+
+ if np.isscalar(x_vars):
+ x_vars = [x_vars]
+ if np.isscalar(y_vars):
+ y_vars = [y_vars]
+
+ self.x_vars = x_vars = list(x_vars)
+ self.y_vars = y_vars = list(y_vars)
+ self.square_grid = self.x_vars == self.y_vars
+
+ if not x_vars:
+ raise ValueError("No variables found for grid columns.")
+ if not y_vars:
+ raise ValueError("No variables found for grid rows.")
+
+ # Create the figure and the array of subplots
+ figsize = len(x_vars) * height * aspect, len(y_vars) * height
+
+ with _disable_autolayout():
+ fig = plt.figure(figsize=figsize)
+
+ axes = fig.subplots(len(y_vars), len(x_vars),
+ sharex="col", sharey="row",
+ squeeze=False)
+
+ # Possibly remove upper axes to make a corner grid
+ # Note: setting up the axes is usually the most time-intensive part
+ # of using the PairGrid. We are foregoing the speed improvement that
+ # we would get by just not setting up the hidden axes so that we can
+ # avoid implementing fig.subplots ourselves. But worth thinking about.
+ self._corner = corner
+ if corner:
+ hide_indices = np.triu_indices_from(axes, 1)
+ for i, j in zip(*hide_indices):
+ axes[i, j].remove()
+ axes[i, j] = None
+
+ self._figure = fig
+ self.axes = axes
+ self.data = data
+
+ # Save what we are going to do with the diagonal
+ self.diag_sharey = diag_sharey
+ self.diag_vars = None
+ self.diag_axes = None
+
+ self._dropna = dropna
+
+ # Label the axes
+ self._add_axis_labels()
+
+ # Sort out the hue variable
+ self._hue_var = hue
+ if hue is None:
+ self.hue_names = hue_order = ["_nolegend_"]
+ self.hue_vals = pd.Series(["_nolegend_"] * len(data),
+ index=data.index)
+ else:
+ # We need hue_order and hue_names because the former is used to control
+ # the order of drawing and the latter is used to control the order of
+ # the legend. hue_names can become string-typed while hue_order must
+ # retain the type of the input data. This is messy but results from
+ # the fact that PairGrid can implement the hue-mapping logic itself
+ # (and was originally written exclusively that way) but now can delegate
+ # to the axes-level functions, while always handling legend creation.
+ # See GH2307
+ hue_names = hue_order = categorical_order(data[hue], hue_order)
+ if dropna:
+ # Filter NA from the list of unique hue names
+ hue_names = list(filter(pd.notnull, hue_names))
+ self.hue_names = hue_names
+ self.hue_vals = data[hue]
+
+ # Additional dict of kwarg -> list of values for mapping the hue var
+ self.hue_kws = hue_kws if hue_kws is not None else {}
+
+ self._orig_palette = palette
+ self._hue_order = hue_order
+ self.palette = self._get_palette(data, hue, hue_order, palette)
+ self._legend_data = {}
+
+ # Make the plot look nice
+ for ax in axes[:-1, :].flat:
+ if ax is None:
+ continue
+ for label in ax.get_xticklabels():
+ label.set_visible(False)
+ ax.xaxis.offsetText.set_visible(False)
+ ax.xaxis.label.set_visible(False)
+
+ for ax in axes[:, 1:].flat:
+ if ax is None:
+ continue
+ for label in ax.get_yticklabels():
+ label.set_visible(False)
+ ax.yaxis.offsetText.set_visible(False)
+ ax.yaxis.label.set_visible(False)
+
+ self._tight_layout_rect = [.01, .01, .99, .99]
+ self._tight_layout_pad = layout_pad
+ self._despine = despine
+ if despine:
+ utils.despine(fig=fig)
+ self.tight_layout(pad=layout_pad)
+
+ def map(self, func, **kwargs):
+ """Plot with the same function in every subplot.
+
+ Parameters
+ ----------
+ func : callable plotting function
+ Must take x, y arrays as positional arguments and draw onto the
+ "currently active" matplotlib Axes. Also needs to accept kwargs
+ called ``color`` and ``label``.
+
+ """
+ row_indices, col_indices = np.indices(self.axes.shape)
+ indices = zip(row_indices.flat, col_indices.flat)
+ self._map_bivariate(func, indices, **kwargs)
+
+ return self
+
+ def map_lower(self, func, **kwargs):
+ """Plot with a bivariate function on the lower diagonal subplots.
+
+ Parameters
+ ----------
+ func : callable plotting function
+ Must take x, y arrays as positional arguments and draw onto the
+ "currently active" matplotlib Axes. Also needs to accept kwargs
+ called ``color`` and ``label``.
+
+ """
+ indices = zip(*np.tril_indices_from(self.axes, -1))
+ self._map_bivariate(func, indices, **kwargs)
+ return self
+
+ def map_upper(self, func, **kwargs):
+ """Plot with a bivariate function on the upper diagonal subplots.
+
+ Parameters
+ ----------
+ func : callable plotting function
+ Must take x, y arrays as positional arguments and draw onto the
+ "currently active" matplotlib Axes. Also needs to accept kwargs
+ called ``color`` and ``label``.
+
+ """
+ indices = zip(*np.triu_indices_from(self.axes, 1))
+ self._map_bivariate(func, indices, **kwargs)
+ return self
+
+ def map_offdiag(self, func, **kwargs):
+ """Plot with a bivariate function on the off-diagonal subplots.
+
+ Parameters
+ ----------
+ func : callable plotting function
+ Must take x, y arrays as positional arguments and draw onto the
+ "currently active" matplotlib Axes. Also needs to accept kwargs
+ called ``color`` and ``label``.
+
+ """
+ if self.square_grid:
+ self.map_lower(func, **kwargs)
+ if not self._corner:
+ self.map_upper(func, **kwargs)
+ else:
+ indices = []
+ for i, (y_var) in enumerate(self.y_vars):
+ for j, (x_var) in enumerate(self.x_vars):
+ if x_var != y_var:
+ indices.append((i, j))
+ self._map_bivariate(func, indices, **kwargs)
+ return self
+
+ def map_diag(self, func, **kwargs):
+ """Plot with a univariate function on each diagonal subplot.
+
+ Parameters
+ ----------
+ func : callable plotting function
+ Must take an x array as a positional argument and draw onto the
+ "currently active" matplotlib Axes. Also needs to accept kwargs
+ called ``color`` and ``label``.
+
+ """
+ # Add special diagonal axes for the univariate plot
+ if self.diag_axes is None:
+ diag_vars = []
+ diag_axes = []
+ for i, y_var in enumerate(self.y_vars):
+ for j, x_var in enumerate(self.x_vars):
+ if x_var == y_var:
+
+ # Make the density axes
+ diag_vars.append(x_var)
+ ax = self.axes[i, j]
+ diag_ax = ax.twinx()
+ diag_ax.set_axis_off()
+ diag_axes.append(diag_ax)
+
+ # Work around matplotlib bug
+ # https://github.com/matplotlib/matplotlib/issues/15188
+ if not plt.rcParams.get("ytick.left", True):
+ for tick in ax.yaxis.majorTicks:
+ tick.tick1line.set_visible(False)
+
+ # Remove main y axis from density axes in a corner plot
+ if self._corner:
+ ax.yaxis.set_visible(False)
+ if self._despine:
+ utils.despine(ax=ax, left=True)
+ # TODO add optional density ticks (on the right)
+ # when drawing a corner plot?
+
+ if self.diag_sharey and diag_axes:
+ for ax in diag_axes[1:]:
+ share_axis(diag_axes[0], ax, "y")
+
+ self.diag_vars = diag_vars
+ self.diag_axes = diag_axes
+
+ if "hue" not in signature(func).parameters:
+ return self._map_diag_iter_hue(func, **kwargs)
+
+ # Loop over diagonal variables and axes, making one plot in each
+ for var, ax in zip(self.diag_vars, self.diag_axes):
+
+ plot_kwargs = kwargs.copy()
+ if str(func.__module__).startswith("seaborn"):
+ plot_kwargs["ax"] = ax
+ else:
+ plt.sca(ax)
+
+ vector = self.data[var]
+ if self._hue_var is not None:
+ hue = self.data[self._hue_var]
+ else:
+ hue = None
+
+ if self._dropna:
+ not_na = vector.notna()
+ if hue is not None:
+ not_na &= hue.notna()
+ vector = vector[not_na]
+ if hue is not None:
+ hue = hue[not_na]
+
+ plot_kwargs.setdefault("hue", hue)
+ plot_kwargs.setdefault("hue_order", self._hue_order)
+ plot_kwargs.setdefault("palette", self._orig_palette)
+ func(x=vector, **plot_kwargs)
+ ax.legend_ = None
+
+ self._add_axis_labels()
+ return self
+
+ def _map_diag_iter_hue(self, func, **kwargs):
+ """Put marginal plot on each diagonal axes, iterating over hue."""
+ # Plot on each of the diagonal axes
+ fixed_color = kwargs.pop("color", None)
+
+ for var, ax in zip(self.diag_vars, self.diag_axes):
+ hue_grouped = self.data[var].groupby(self.hue_vals, observed=True)
+
+ plot_kwargs = kwargs.copy()
+ if str(func.__module__).startswith("seaborn"):
+ plot_kwargs["ax"] = ax
+ else:
+ plt.sca(ax)
+
+ for k, label_k in enumerate(self._hue_order):
+
+ # Attempt to get data for this level, allowing for empty
+ try:
+ data_k = hue_grouped.get_group(label_k)
+ except KeyError:
+ data_k = pd.Series([], dtype=float)
+
+ if fixed_color is None:
+ color = self.palette[k]
+ else:
+ color = fixed_color
+
+ if self._dropna:
+ data_k = utils.remove_na(data_k)
+
+ if str(func.__module__).startswith("seaborn"):
+ func(x=data_k, label=label_k, color=color, **plot_kwargs)
+ else:
+ func(data_k, label=label_k, color=color, **plot_kwargs)
+
+ self._add_axis_labels()
+
+ return self
+
+ def _map_bivariate(self, func, indices, **kwargs):
+ """Draw a bivariate plot on the indicated axes."""
+ # This is a hack to handle the fact that new distribution plots don't add
+ # their artists onto the axes. This is probably superior in general, but
+ # we'll need a better way to handle it in the axisgrid functions.
+ from .distributions import histplot, kdeplot
+ if func is histplot or func is kdeplot:
+ self._extract_legend_handles = True
+
+ kws = kwargs.copy() # Use copy as we insert other kwargs
+ for i, j in indices:
+ x_var = self.x_vars[j]
+ y_var = self.y_vars[i]
+ ax = self.axes[i, j]
+ if ax is None: # i.e. we are in corner mode
+ continue
+ self._plot_bivariate(x_var, y_var, ax, func, **kws)
+ self._add_axis_labels()
+
+ if "hue" in signature(func).parameters:
+ self.hue_names = list(self._legend_data)
+
+ def _plot_bivariate(self, x_var, y_var, ax, func, **kwargs):
+ """Draw a bivariate plot on the specified axes."""
+ if "hue" not in signature(func).parameters:
+ self._plot_bivariate_iter_hue(x_var, y_var, ax, func, **kwargs)
+ return
+
+ kwargs = kwargs.copy()
+ if str(func.__module__).startswith("seaborn"):
+ kwargs["ax"] = ax
+ else:
+ plt.sca(ax)
+
+ if x_var == y_var:
+ axes_vars = [x_var]
+ else:
+ axes_vars = [x_var, y_var]
+
+ if self._hue_var is not None and self._hue_var not in axes_vars:
+ axes_vars.append(self._hue_var)
+
+ data = self.data[axes_vars]
+ if self._dropna:
+ data = data.dropna()
+
+ x = data[x_var]
+ y = data[y_var]
+ if self._hue_var is None:
+ hue = None
+ else:
+ hue = data.get(self._hue_var)
+
+ if "hue" not in kwargs:
+ kwargs.update({
+ "hue": hue, "hue_order": self._hue_order, "palette": self._orig_palette,
+ })
+ func(x=x, y=y, **kwargs)
+
+ self._update_legend_data(ax)
+
+ def _plot_bivariate_iter_hue(self, x_var, y_var, ax, func, **kwargs):
+ """Draw a bivariate plot while iterating over hue subsets."""
+ kwargs = kwargs.copy()
+ if str(func.__module__).startswith("seaborn"):
+ kwargs["ax"] = ax
+ else:
+ plt.sca(ax)
+
+ if x_var == y_var:
+ axes_vars = [x_var]
+ else:
+ axes_vars = [x_var, y_var]
+
+ hue_grouped = self.data.groupby(self.hue_vals, observed=True)
+ for k, label_k in enumerate(self._hue_order):
+
+ kws = kwargs.copy()
+
+ # Attempt to get data for this level, allowing for empty
+ try:
+ data_k = hue_grouped.get_group(label_k)
+ except KeyError:
+ data_k = pd.DataFrame(columns=axes_vars,
+ dtype=float)
+
+ if self._dropna:
+ data_k = data_k[axes_vars].dropna()
+
+ x = data_k[x_var]
+ y = data_k[y_var]
+
+ for kw, val_list in self.hue_kws.items():
+ kws[kw] = val_list[k]
+ kws.setdefault("color", self.palette[k])
+ if self._hue_var is not None:
+ kws["label"] = label_k
+
+ if str(func.__module__).startswith("seaborn"):
+ func(x=x, y=y, **kws)
+ else:
+ func(x, y, **kws)
+
+ self._update_legend_data(ax)
+
+ def _add_axis_labels(self):
+ """Add labels to the left and bottom Axes."""
+ for ax, label in zip(self.axes[-1, :], self.x_vars):
+ ax.set_xlabel(label)
+ for ax, label in zip(self.axes[:, 0], self.y_vars):
+ ax.set_ylabel(label)
+
+ def _find_numeric_cols(self, data):
+ """Find which variables in a DataFrame are numeric."""
+ numeric_cols = []
+ for col in data:
+ if variable_type(data[col]) == "numeric":
+ numeric_cols.append(col)
+ return numeric_cols
+
+
+class JointGrid(_BaseGrid):
+ """Grid for drawing a bivariate plot with marginal univariate plots.
+
+ Many plots can be drawn by using the figure-level interface :func:`jointplot`.
+ Use this class directly when you need more flexibility.
+
+ """
+
+ def __init__(
+ self, data=None, *,
+ x=None, y=None, hue=None,
+ height=6, ratio=5, space=.2,
+ palette=None, hue_order=None, hue_norm=None,
+ dropna=False, xlim=None, ylim=None, marginal_ticks=False,
+ ):
+
+ # Set up the subplot grid
+ f = plt.figure(figsize=(height, height))
+ gs = plt.GridSpec(ratio + 1, ratio + 1)
+
+ ax_joint = f.add_subplot(gs[1:, :-1])
+ ax_marg_x = f.add_subplot(gs[0, :-1], sharex=ax_joint)
+ ax_marg_y = f.add_subplot(gs[1:, -1], sharey=ax_joint)
+
+ self._figure = f
+ self.ax_joint = ax_joint
+ self.ax_marg_x = ax_marg_x
+ self.ax_marg_y = ax_marg_y
+
+ # Turn off tick visibility for the measure axis on the marginal plots
+ plt.setp(ax_marg_x.get_xticklabels(), visible=False)
+ plt.setp(ax_marg_y.get_yticklabels(), visible=False)
+ plt.setp(ax_marg_x.get_xticklabels(minor=True), visible=False)
+ plt.setp(ax_marg_y.get_yticklabels(minor=True), visible=False)
+
+ # Turn off the ticks on the density axis for the marginal plots
+ if not marginal_ticks:
+ plt.setp(ax_marg_x.yaxis.get_majorticklines(), visible=False)
+ plt.setp(ax_marg_x.yaxis.get_minorticklines(), visible=False)
+ plt.setp(ax_marg_y.xaxis.get_majorticklines(), visible=False)
+ plt.setp(ax_marg_y.xaxis.get_minorticklines(), visible=False)
+ plt.setp(ax_marg_x.get_yticklabels(), visible=False)
+ plt.setp(ax_marg_y.get_xticklabels(), visible=False)
+ plt.setp(ax_marg_x.get_yticklabels(minor=True), visible=False)
+ plt.setp(ax_marg_y.get_xticklabels(minor=True), visible=False)
+ ax_marg_x.yaxis.grid(False)
+ ax_marg_y.xaxis.grid(False)
+
+ # Process the input variables
+ p = VectorPlotter(data=data, variables=dict(x=x, y=y, hue=hue))
+ plot_data = p.plot_data.loc[:, p.plot_data.notna().any()]
+
+ # Possibly drop NA
+ if dropna:
+ plot_data = plot_data.dropna()
+
+ def get_var(var):
+ vector = plot_data.get(var, None)
+ if vector is not None:
+ vector = vector.rename(p.variables.get(var, None))
+ return vector
+
+ self.x = get_var("x")
+ self.y = get_var("y")
+ self.hue = get_var("hue")
+
+ for axis in "xy":
+ name = p.variables.get(axis, None)
+ if name is not None:
+ getattr(ax_joint, f"set_{axis}label")(name)
+
+ if xlim is not None:
+ ax_joint.set_xlim(xlim)
+ if ylim is not None:
+ ax_joint.set_ylim(ylim)
+
+ # Store the semantic mapping parameters for axes-level functions
+ self._hue_params = dict(palette=palette, hue_order=hue_order, hue_norm=hue_norm)
+
+ # Make the grid look nice
+ utils.despine(f)
+ if not marginal_ticks:
+ utils.despine(ax=ax_marg_x, left=True)
+ utils.despine(ax=ax_marg_y, bottom=True)
+ for axes in [ax_marg_x, ax_marg_y]:
+ for axis in [axes.xaxis, axes.yaxis]:
+ axis.label.set_visible(False)
+ f.tight_layout()
+ f.subplots_adjust(hspace=space, wspace=space)
+
+ def _inject_kwargs(self, func, kws, params):
+ """Add params to kws if they are accepted by func."""
+ func_params = signature(func).parameters
+ for key, val in params.items():
+ if key in func_params:
+ kws.setdefault(key, val)
+
+ def plot(self, joint_func, marginal_func, **kwargs):
+ """Draw the plot by passing functions for joint and marginal axes.
+
+ This method passes the ``kwargs`` dictionary to both functions. If you
+ need more control, call :meth:`JointGrid.plot_joint` and
+ :meth:`JointGrid.plot_marginals` directly with specific parameters.
+
+ Parameters
+ ----------
+ joint_func, marginal_func : callables
+ Functions to draw the bivariate and univariate plots. See methods
+ referenced above for information about the required characteristics
+ of these functions.
+ kwargs
+ Additional keyword arguments are passed to both functions.
+
+ Returns
+ -------
+ :class:`JointGrid` instance
+ Returns ``self`` for easy method chaining.
+
+ """
+ self.plot_marginals(marginal_func, **kwargs)
+ self.plot_joint(joint_func, **kwargs)
+ return self
+
+ def plot_joint(self, func, **kwargs):
+ """Draw a bivariate plot on the joint axes of the grid.
+
+ Parameters
+ ----------
+ func : plotting callable
+ If a seaborn function, it should accept ``x`` and ``y``. Otherwise,
+ it must accept ``x`` and ``y`` vectors of data as the first two
+ positional arguments, and it must plot on the "current" axes.
+ If ``hue`` was defined in the class constructor, the function must
+ accept ``hue`` as a parameter.
+ kwargs
+ Keyword argument are passed to the plotting function.
+
+ Returns
+ -------
+ :class:`JointGrid` instance
+ Returns ``self`` for easy method chaining.
+
+ """
+ kwargs = kwargs.copy()
+ if str(func.__module__).startswith("seaborn"):
+ kwargs["ax"] = self.ax_joint
+ else:
+ plt.sca(self.ax_joint)
+ if self.hue is not None:
+ kwargs["hue"] = self.hue
+ self._inject_kwargs(func, kwargs, self._hue_params)
+
+ if str(func.__module__).startswith("seaborn"):
+ func(x=self.x, y=self.y, **kwargs)
+ else:
+ func(self.x, self.y, **kwargs)
+
+ return self
+
+ def plot_marginals(self, func, **kwargs):
+ """Draw univariate plots on each marginal axes.
+
+ Parameters
+ ----------
+ func : plotting callable
+ If a seaborn function, it should accept ``x`` and ``y`` and plot
+ when only one of them is defined. Otherwise, it must accept a vector
+ of data as the first positional argument and determine its orientation
+ using the ``vertical`` parameter, and it must plot on the "current" axes.
+ If ``hue`` was defined in the class constructor, it must accept ``hue``
+ as a parameter.
+ kwargs
+ Keyword argument are passed to the plotting function.
+
+ Returns
+ -------
+ :class:`JointGrid` instance
+ Returns ``self`` for easy method chaining.
+
+ """
+ seaborn_func = (
+ str(func.__module__).startswith("seaborn")
+ # deprecated distplot has a legacy API, special case it
+ and not func.__name__ == "distplot"
+ )
+ func_params = signature(func).parameters
+ kwargs = kwargs.copy()
+ if self.hue is not None:
+ kwargs["hue"] = self.hue
+ self._inject_kwargs(func, kwargs, self._hue_params)
+
+ if "legend" in func_params:
+ kwargs.setdefault("legend", False)
+
+ if "orientation" in func_params:
+ # e.g. plt.hist
+ orient_kw_x = {"orientation": "vertical"}
+ orient_kw_y = {"orientation": "horizontal"}
+ elif "vertical" in func_params:
+ # e.g. sns.distplot (also how did this get backwards?)
+ orient_kw_x = {"vertical": False}
+ orient_kw_y = {"vertical": True}
+
+ if seaborn_func:
+ func(x=self.x, ax=self.ax_marg_x, **kwargs)
+ else:
+ plt.sca(self.ax_marg_x)
+ func(self.x, **orient_kw_x, **kwargs)
+
+ if seaborn_func:
+ func(y=self.y, ax=self.ax_marg_y, **kwargs)
+ else:
+ plt.sca(self.ax_marg_y)
+ func(self.y, **orient_kw_y, **kwargs)
+
+ self.ax_marg_x.yaxis.get_label().set_visible(False)
+ self.ax_marg_y.xaxis.get_label().set_visible(False)
+
+ return self
+
+ def refline(
+ self, *, x=None, y=None, joint=True, marginal=True,
+ color='.5', linestyle='--', **line_kws
+ ):
+ """Add a reference line(s) to joint and/or marginal axes.
+
+ Parameters
+ ----------
+ x, y : numeric
+ Value(s) to draw the line(s) at.
+ joint, marginal : bools
+ Whether to add the reference line(s) to the joint/marginal axes.
+ color : :mod:`matplotlib color <matplotlib.colors>`
+ Specifies the color of the reference line(s).
+ linestyle : str
+ Specifies the style of the reference line(s).
+ line_kws : key, value mappings
+ Other keyword arguments are passed to :meth:`matplotlib.axes.Axes.axvline`
+ when ``x`` is not None and :meth:`matplotlib.axes.Axes.axhline` when ``y``
+ is not None.
+
+ Returns
+ -------
+ :class:`JointGrid` instance
+ Returns ``self`` for easy method chaining.
+
+ """
+ line_kws['color'] = color
+ line_kws['linestyle'] = linestyle
+
+ if x is not None:
+ if joint:
+ self.ax_joint.axvline(x, **line_kws)
+ if marginal:
+ self.ax_marg_x.axvline(x, **line_kws)
+
+ if y is not None:
+ if joint:
+ self.ax_joint.axhline(y, **line_kws)
+ if marginal:
+ self.ax_marg_y.axhline(y, **line_kws)
+
+ return self
+
+ def set_axis_labels(self, xlabel="", ylabel="", **kwargs):
+ """Set axis labels on the bivariate axes.
+
+ Parameters
+ ----------
+ xlabel, ylabel : strings
+ Label names for the x and y variables.
+ kwargs : key, value mappings
+ Other keyword arguments are passed to the following functions:
+
+ - :meth:`matplotlib.axes.Axes.set_xlabel`
+ - :meth:`matplotlib.axes.Axes.set_ylabel`
+
+ Returns
+ -------
+ :class:`JointGrid` instance
+ Returns ``self`` for easy method chaining.
+
+ """
+ self.ax_joint.set_xlabel(xlabel, **kwargs)
+ self.ax_joint.set_ylabel(ylabel, **kwargs)
+ return self
+
+
+JointGrid.__init__.__doc__ = """\
+Set up the grid of subplots and store data internally for easy plotting.
+
+Parameters
+----------
+{params.core.data}
+{params.core.xy}
+height : number
+ Size of each side of the figure in inches (it will be square).
+ratio : number
+ Ratio of joint axes height to marginal axes height.
+space : number
+ Space between the joint and marginal axes
+dropna : bool
+ If True, remove missing observations before plotting.
+{{x, y}}lim : pairs of numbers
+ Set axis limits to these values before plotting.
+marginal_ticks : bool
+ If False, suppress ticks on the count/density axis of the marginal plots.
+{params.core.hue}
+ Note: unlike in :class:`FacetGrid` or :class:`PairGrid`, the axes-level
+ functions must support ``hue`` to use it in :class:`JointGrid`.
+{params.core.palette}
+{params.core.hue_order}
+{params.core.hue_norm}
+
+See Also
+--------
+{seealso.jointplot}
+{seealso.pairgrid}
+{seealso.pairplot}
+
+Examples
+--------
+
+.. include:: ../docstrings/JointGrid.rst
+
+""".format(
+ params=_param_docs,
+ seealso=_core_docs["seealso"],
+)
+
+
+def pairplot(
+ data, *,
+ hue=None, hue_order=None, palette=None,
+ vars=None, x_vars=None, y_vars=None,
+ kind="scatter", diag_kind="auto", markers=None,
+ height=2.5, aspect=1, corner=False, dropna=False,
+ plot_kws=None, diag_kws=None, grid_kws=None, size=None,
+):
+ """Plot pairwise relationships in a dataset.
+
+ By default, this function will create a grid of Axes such that each numeric
+ variable in ``data`` will by shared across the y-axes across a single row and
+ the x-axes across a single column. The diagonal plots are treated
+ differently: a univariate distribution plot is drawn to show the marginal
+ distribution of the data in each column.
+
+ It is also possible to show a subset of variables or plot different
+ variables on the rows and columns.
+
+ This is a high-level interface for :class:`PairGrid` that is intended to
+ make it easy to draw a few common styles. You should use :class:`PairGrid`
+ directly if you need more flexibility.
+
+ Parameters
+ ----------
+ data : `pandas.DataFrame`
+ Tidy (long-form) dataframe where each column is a variable and
+ each row is an observation.
+ hue : name of variable in ``data``
+ Variable in ``data`` to map plot aspects to different colors.
+ hue_order : list of strings
+ Order for the levels of the hue variable in the palette
+ palette : dict or seaborn color palette
+ Set of colors for mapping the ``hue`` variable. If a dict, keys
+ should be values in the ``hue`` variable.
+ vars : list of variable names
+ Variables within ``data`` to use, otherwise use every column with
+ a numeric datatype.
+ {x, y}_vars : lists of variable names
+ Variables within ``data`` to use separately for the rows and
+ columns of the figure; i.e. to make a non-square plot.
+ kind : {'scatter', 'kde', 'hist', 'reg'}
+ Kind of plot to make.
+ diag_kind : {'auto', 'hist', 'kde', None}
+ Kind of plot for the diagonal subplots. If 'auto', choose based on
+ whether or not ``hue`` is used.
+ markers : single matplotlib marker code or list
+ Either the marker to use for all scatterplot points or a list of markers
+ with a length the same as the number of levels in the hue variable so that
+ differently colored points will also have different scatterplot
+ markers.
+ height : scalar
+ Height (in inches) of each facet.
+ aspect : scalar
+ Aspect * height gives the width (in inches) of each facet.
+ corner : bool
+ If True, don't add axes to the upper (off-diagonal) triangle of the
+ grid, making this a "corner" plot.
+ dropna : boolean
+ Drop missing values from the data before plotting.
+ {plot, diag, grid}_kws : dicts
+ Dictionaries of keyword arguments. ``plot_kws`` are passed to the
+ bivariate plotting function, ``diag_kws`` are passed to the univariate
+ plotting function, and ``grid_kws`` are passed to the :class:`PairGrid`
+ constructor.
+
+ Returns
+ -------
+ grid : :class:`PairGrid`
+ Returns the underlying :class:`PairGrid` instance for further tweaking.
+
+ See Also
+ --------
+ PairGrid : Subplot grid for more flexible plotting of pairwise relationships.
+ JointGrid : Grid for plotting joint and marginal distributions of two variables.
+
+ Examples
+ --------
+
+ .. include:: ../docstrings/pairplot.rst
+
+ """
+ # Avoid circular import
+ from .distributions import histplot, kdeplot
+
+ # Handle deprecations
+ if size is not None:
+ height = size
+ msg = ("The `size` parameter has been renamed to `height`; "
+ "please update your code.")
+ warnings.warn(msg, UserWarning)
+
+ if not isinstance(data, pd.DataFrame):
+ raise TypeError(
+ f"'data' must be pandas DataFrame object, not: {type(data)}")
+
+ plot_kws = {} if plot_kws is None else plot_kws.copy()
+ diag_kws = {} if diag_kws is None else diag_kws.copy()
+ grid_kws = {} if grid_kws is None else grid_kws.copy()
+
+ # Resolve "auto" diag kind
+ if diag_kind == "auto":
+ if hue is None:
+ diag_kind = "kde" if kind == "kde" else "hist"
+ else:
+ diag_kind = "hist" if kind == "hist" else "kde"
+
+ # Set up the PairGrid
+ grid_kws.setdefault("diag_sharey", diag_kind == "hist")
+ grid = PairGrid(data, vars=vars, x_vars=x_vars, y_vars=y_vars, hue=hue,
+ hue_order=hue_order, palette=palette, corner=corner,
+ height=height, aspect=aspect, dropna=dropna, **grid_kws)
+
+ # Add the markers here as PairGrid has figured out how many levels of the
+ # hue variable are needed and we don't want to duplicate that process
+ if markers is not None:
+ if kind == "reg":
+ # Needed until regplot supports style
+ if grid.hue_names is None:
+ n_markers = 1
+ else:
+ n_markers = len(grid.hue_names)
+ if not isinstance(markers, list):
+ markers = [markers] * n_markers
+ if len(markers) != n_markers:
+ raise ValueError("markers must be a singleton or a list of "
+ "markers for each level of the hue variable")
+ grid.hue_kws = {"marker": markers}
+ elif kind == "scatter":
+ if isinstance(markers, str):
+ plot_kws["marker"] = markers
+ elif hue is not None:
+ plot_kws["style"] = data[hue]
+ plot_kws["markers"] = markers
+
+ # Draw the marginal plots on the diagonal
+ diag_kws = diag_kws.copy()
+ diag_kws.setdefault("legend", False)
+ if diag_kind == "hist":
+ grid.map_diag(histplot, **diag_kws)
+ elif diag_kind == "kde":
+ diag_kws.setdefault("fill", True)
+ diag_kws.setdefault("warn_singular", False)
+ grid.map_diag(kdeplot, **diag_kws)
+
+ # Maybe plot on the off-diagonals
+ if diag_kind is not None:
+ plotter = grid.map_offdiag
+ else:
+ plotter = grid.map
+
+ if kind == "scatter":
+ from .relational import scatterplot # Avoid circular import
+ plotter(scatterplot, **plot_kws)
+ elif kind == "reg":
+ from .regression import regplot # Avoid circular import
+ plotter(regplot, **plot_kws)
+ elif kind == "kde":
+ from .distributions import kdeplot # Avoid circular import
+ plot_kws.setdefault("warn_singular", False)
+ plotter(kdeplot, **plot_kws)
+ elif kind == "hist":
+ from .distributions import histplot # Avoid circular import
+ plotter(histplot, **plot_kws)
+
+ # Add a legend
+ if hue is not None:
+ grid.add_legend()
+
+ grid.tight_layout()
+
+ return grid
+
+
+def jointplot(
+ data=None, *, x=None, y=None, hue=None, kind="scatter",
+ height=6, ratio=5, space=.2, dropna=False, xlim=None, ylim=None,
+ color=None, palette=None, hue_order=None, hue_norm=None, marginal_ticks=False,
+ joint_kws=None, marginal_kws=None,
+ **kwargs
+):
+ # Avoid circular imports
+ from .relational import scatterplot
+ from .regression import regplot, residplot
+ from .distributions import histplot, kdeplot, _freedman_diaconis_bins
+
+ if kwargs.pop("ax", None) is not None:
+ msg = "Ignoring `ax`; jointplot is a figure-level function."
+ warnings.warn(msg, UserWarning, stacklevel=2)
+
+ # Set up empty default kwarg dicts
+ joint_kws = {} if joint_kws is None else joint_kws.copy()
+ joint_kws.update(kwargs)
+ marginal_kws = {} if marginal_kws is None else marginal_kws.copy()
+
+ # Handle deprecations of distplot-specific kwargs
+ distplot_keys = [
+ "rug", "fit", "hist_kws", "norm_hist" "hist_kws", "rug_kws",
+ ]
+ unused_keys = []
+ for key in distplot_keys:
+ if key in marginal_kws:
+ unused_keys.append(key)
+ marginal_kws.pop(key)
+ if unused_keys and kind != "kde":
+ msg = (
+ "The marginal plotting function has changed to `histplot`,"
+ " which does not accept the following argument(s): {}."
+ ).format(", ".join(unused_keys))
+ warnings.warn(msg, UserWarning)
+
+ # Validate the plot kind
+ plot_kinds = ["scatter", "hist", "hex", "kde", "reg", "resid"]
+ _check_argument("kind", plot_kinds, kind)
+
+ # Raise early if using `hue` with a kind that does not support it
+ if hue is not None and kind in ["hex", "reg", "resid"]:
+ msg = f"Use of `hue` with `kind='{kind}'` is not currently supported."
+ raise ValueError(msg)
+
+ # Make a colormap based off the plot color
+ # (Currently used only for kind="hex")
+ if color is None:
+ color = "C0"
+ color_rgb = mpl.colors.colorConverter.to_rgb(color)
+ colors = [set_hls_values(color_rgb, l=val) for val in np.linspace(1, 0, 12)]
+ cmap = blend_palette(colors, as_cmap=True)
+
+ # Matplotlib's hexbin plot is not na-robust
+ if kind == "hex":
+ dropna = True
+
+ # Initialize the JointGrid object
+ grid = JointGrid(
+ data=data, x=x, y=y, hue=hue,
+ palette=palette, hue_order=hue_order, hue_norm=hue_norm,
+ dropna=dropna, height=height, ratio=ratio, space=space,
+ xlim=xlim, ylim=ylim, marginal_ticks=marginal_ticks,
+ )
+
+ if grid.hue is not None:
+ marginal_kws.setdefault("legend", False)
+
+ # Plot the data using the grid
+ if kind.startswith("scatter"):
+
+ joint_kws.setdefault("color", color)
+ grid.plot_joint(scatterplot, **joint_kws)
+
+ if grid.hue is None:
+ marg_func = histplot
+ else:
+ marg_func = kdeplot
+ marginal_kws.setdefault("warn_singular", False)
+ marginal_kws.setdefault("fill", True)
+
+ marginal_kws.setdefault("color", color)
+ grid.plot_marginals(marg_func, **marginal_kws)
+
+ elif kind.startswith("hist"):
+
+ # TODO process pair parameters for bins, etc. and pass
+ # to both joint and marginal plots
+
+ joint_kws.setdefault("color", color)
+ grid.plot_joint(histplot, **joint_kws)
+
+ marginal_kws.setdefault("kde", False)
+ marginal_kws.setdefault("color", color)
+
+ marg_x_kws = marginal_kws.copy()
+ marg_y_kws = marginal_kws.copy()
+
+ pair_keys = "bins", "binwidth", "binrange"
+ for key in pair_keys:
+ if isinstance(joint_kws.get(key), tuple):
+ x_val, y_val = joint_kws[key]
+ marg_x_kws.setdefault(key, x_val)
+ marg_y_kws.setdefault(key, y_val)
+
+ histplot(data=data, x=x, hue=hue, **marg_x_kws, ax=grid.ax_marg_x)
+ histplot(data=data, y=y, hue=hue, **marg_y_kws, ax=grid.ax_marg_y)
+
+ elif kind.startswith("kde"):
+
+ joint_kws.setdefault("color", color)
+ joint_kws.setdefault("warn_singular", False)
+ grid.plot_joint(kdeplot, **joint_kws)
+
+ marginal_kws.setdefault("color", color)
+ if "fill" in joint_kws:
+ marginal_kws.setdefault("fill", joint_kws["fill"])
+
+ grid.plot_marginals(kdeplot, **marginal_kws)
+
+ elif kind.startswith("hex"):
+
+ x_bins = min(_freedman_diaconis_bins(grid.x), 50)
+ y_bins = min(_freedman_diaconis_bins(grid.y), 50)
+ gridsize = int(np.mean([x_bins, y_bins]))
+
+ joint_kws.setdefault("gridsize", gridsize)
+ joint_kws.setdefault("cmap", cmap)
+ grid.plot_joint(plt.hexbin, **joint_kws)
+
+ marginal_kws.setdefault("kde", False)
+ marginal_kws.setdefault("color", color)
+ grid.plot_marginals(histplot, **marginal_kws)
+
+ elif kind.startswith("reg"):
+
+ marginal_kws.setdefault("color", color)
+ marginal_kws.setdefault("kde", True)
+ grid.plot_marginals(histplot, **marginal_kws)
+
+ joint_kws.setdefault("color", color)
+ grid.plot_joint(regplot, **joint_kws)
+
+ elif kind.startswith("resid"):
+
+ joint_kws.setdefault("color", color)
+ grid.plot_joint(residplot, **joint_kws)
+
+ x, y = grid.ax_joint.collections[0].get_offsets().T
+ marginal_kws.setdefault("color", color)
+ histplot(x=x, hue=hue, ax=grid.ax_marg_x, **marginal_kws)
+ histplot(y=y, hue=hue, ax=grid.ax_marg_y, **marginal_kws)
+
+ # Make the main axes active in the matplotlib state machine
+ plt.sca(grid.ax_joint)
+
+ return grid
+
+
+jointplot.__doc__ = """\
+Draw a plot of two variables with bivariate and univariate graphs.
+
+This function provides a convenient interface to the :class:`JointGrid`
+class, with several canned plot kinds. This is intended to be a fairly
+lightweight wrapper; if you need more flexibility, you should use
+:class:`JointGrid` directly.
+
+Parameters
+----------
+{params.core.data}
+{params.core.xy}
+{params.core.hue}
+kind : {{ "scatter" | "kde" | "hist" | "hex" | "reg" | "resid" }}
+ Kind of plot to draw. See the examples for references to the underlying functions.
+height : numeric
+ Size of the figure (it will be square).
+ratio : numeric
+ Ratio of joint axes height to marginal axes height.
+space : numeric
+ Space between the joint and marginal axes
+dropna : bool
+ If True, remove observations that are missing from ``x`` and ``y``.
+{{x, y}}lim : pairs of numbers
+ Axis limits to set before plotting.
+{params.core.color}
+{params.core.palette}
+{params.core.hue_order}
+{params.core.hue_norm}
+marginal_ticks : bool
+ If False, suppress ticks on the count/density axis of the marginal plots.
+{{joint, marginal}}_kws : dicts
+ Additional keyword arguments for the plot components.
+kwargs
+ Additional keyword arguments are passed to the function used to
+ draw the plot on the joint Axes, superseding items in the
+ ``joint_kws`` dictionary.
+
+Returns
+-------
+{returns.jointgrid}
+
+See Also
+--------
+{seealso.jointgrid}
+{seealso.pairgrid}
+{seealso.pairplot}
+
+Examples
+--------
+
+.. include:: ../docstrings/jointplot.rst
+
+""".format(
+ params=_param_docs,
+ returns=_core_docs["returns"],
+ seealso=_core_docs["seealso"],
+)
diff --git a/seaborn/categorical.py b/seaborn/categorical.py
new file mode 100644
index 0000000000000000000000000000000000000000..a43c085ba28b0eb6ce34c0161102937216a01638
--- /dev/null
+++ b/seaborn/categorical.py
@@ -0,0 +1,3456 @@
+from collections import namedtuple
+from textwrap import dedent
+import warnings
+from colorsys import rgb_to_hls
+from functools import partial
+
+import numpy as np
+import pandas as pd
+
+import matplotlib as mpl
+from matplotlib.cbook import normalize_kwargs
+from matplotlib.collections import PatchCollection
+from matplotlib.markers import MarkerStyle
+from matplotlib.patches import Rectangle
+import matplotlib.pyplot as plt
+
+from seaborn._core.typing import default, deprecated
+from seaborn._base import VectorPlotter, infer_orient, categorical_order
+from seaborn._stats.density import KDE
+from seaborn import utils
+from seaborn.utils import (
+ desaturate,
+ _check_argument,
+ _draw_figure,
+ _default_color,
+ _get_patch_legend_artist,
+ _get_transform_functions,
+ _scatter_legend_artist,
+ _version_predates,
+)
+from seaborn._compat import groupby_apply_include_groups
+from seaborn._statistics import (
+ EstimateAggregator,
+ LetterValues,
+ WeightedAggregator,
+)
+from seaborn.palettes import light_palette
+from seaborn.axisgrid import FacetGrid, _facet_docs
+
+
+__all__ = [
+ "catplot",
+ "stripplot", "swarmplot",
+ "boxplot", "violinplot", "boxenplot",
+ "pointplot", "barplot", "countplot",
+]
+
+
+class _CategoricalPlotter(VectorPlotter):
+
+ wide_structure = {"x": "@columns", "y": "@values", "hue": "@columns"}
+ flat_structure = {"y": "@values"}
+
+ _legend_attributes = ["color"]
+
+ def __init__(
+ self,
+ data=None,
+ variables={},
+ order=None,
+ orient=None,
+ require_numeric=False,
+ color=None,
+ legend="auto",
+ ):
+
+ super().__init__(data=data, variables=variables)
+
+ # This method takes care of some bookkeeping that is necessary because the
+ # original categorical plots (prior to the 2021 refactor) had some rules that
+ # don't fit exactly into VectorPlotter logic. It may be wise to have a second
+ # round of refactoring that moves the logic deeper, but this will keep things
+ # relatively sensible for now.
+
+ # For wide data, orient determines assignment to x/y differently from the
+ # default VectorPlotter rules. If we do decide to make orient part of the
+ # _base variable assignment, we'll want to figure out how to express that.
+ if self.input_format == "wide" and orient in ["h", "y"]:
+ self.plot_data = self.plot_data.rename(columns={"x": "y", "y": "x"})
+ orig_variables = set(self.variables)
+ orig_x = self.variables.pop("x", None)
+ orig_y = self.variables.pop("y", None)
+ orig_x_type = self.var_types.pop("x", None)
+ orig_y_type = self.var_types.pop("y", None)
+ if "x" in orig_variables:
+ self.variables["y"] = orig_x
+ self.var_types["y"] = orig_x_type
+ if "y" in orig_variables:
+ self.variables["x"] = orig_y
+ self.var_types["x"] = orig_y_type
+
+ # Initially there was more special code for wide-form data where plots were
+ # multi-colored by default and then either palette or color could be used.
+ # We want to provide backwards compatibility for this behavior in a relatively
+ # simply way, so we delete the hue information when color is specified.
+ if (
+ self.input_format == "wide"
+ and "hue" in self.variables
+ and color is not None
+ ):
+ self.plot_data.drop("hue", axis=1)
+ self.variables.pop("hue")
+
+ # The concept of an "orientation" is important to the original categorical
+ # plots, but there's no provision for it in VectorPlotter, so we need it here.
+ # Note that it could be useful for the other functions in at least two ways
+ # (orienting a univariate distribution plot from long-form data and selecting
+ # the aggregation axis in lineplot), so we may want to eventually refactor it.
+ self.orient = infer_orient(
+ x=self.plot_data.get("x", None),
+ y=self.plot_data.get("y", None),
+ orient=orient,
+ require_numeric=False,
+ )
+
+ self.legend = legend
+
+ # Short-circuit in the case of an empty plot
+ if not self.has_xy_data:
+ return
+
+ # Categorical plots can be "univariate" in which case they get an anonymous
+ # category label on the opposite axis. Note: this duplicates code in the core
+ # scale_categorical function. We need to do it here because of the next line.
+ if self.orient not in self.variables:
+ self.variables[self.orient] = None
+ self.var_types[self.orient] = "categorical"
+ self.plot_data[self.orient] = ""
+
+ # Categorical variables have discrete levels that we need to track
+ cat_levels = categorical_order(self.plot_data[self.orient], order)
+ self.var_levels[self.orient] = cat_levels
+
+ def _hue_backcompat(self, color, palette, hue_order, force_hue=False):
+ """Implement backwards compatibility for hue parametrization.
+
+ Note: the force_hue parameter is used so that functions can be shown to
+ pass existing tests during refactoring and then tested for new behavior.
+ It can be removed after completion of the work.
+
+ """
+ # The original categorical functions applied a palette to the categorical axis
+ # by default. We want to require an explicit hue mapping, to be more consistent
+ # with how things work elsewhere now. I don't think there's any good way to
+ # do this gently -- because it's triggered by the default value of hue=None,
+ # users would always get a warning, unless we introduce some sentinel "default"
+ # argument for this change. That's possible, but asking users to set `hue=None`
+ # on every call is annoying.
+ # We are keeping the logic for implementing the old behavior in with the current
+ # system so that (a) we can punt on that decision and (b) we can ensure that
+ # refactored code passes old tests.
+ default_behavior = color is None or palette is not None
+ if force_hue and "hue" not in self.variables and default_behavior:
+ self._redundant_hue = True
+ self.plot_data["hue"] = self.plot_data[self.orient]
+ self.variables["hue"] = self.variables[self.orient]
+ self.var_types["hue"] = "categorical"
+ hue_order = self.var_levels[self.orient]
+
+ # Because we convert the categorical axis variable to string,
+ # we need to update a dictionary palette too
+ if isinstance(palette, dict):
+ palette = {str(k): v for k, v in palette.items()}
+
+ else:
+ if "hue" in self.variables:
+ redundant = (self.plot_data["hue"] == self.plot_data[self.orient]).all()
+ else:
+ redundant = False
+ self._redundant_hue = redundant
+
+ # Previously, categorical plots had a trick where color= could seed the palette.
+ # Because that's an explicit parameterization, we are going to give it one
+ # release cycle with a warning before removing.
+ if "hue" in self.variables and palette is None and color is not None:
+ if not isinstance(color, str):
+ color = mpl.colors.to_hex(color)
+ palette = f"dark:{color}"
+ msg = (
+ "\n\nSetting a gradient palette using color= is deprecated and will be "
+ f"removed in v0.14.0. Set `palette='{palette}'` for the same effect.\n"
+ )
+ warnings.warn(msg, FutureWarning, stacklevel=3)
+
+ return palette, hue_order
+
+ def _palette_without_hue_backcompat(self, palette, hue_order):
+ """Provide one cycle where palette= implies hue= when not provided"""
+ if "hue" not in self.variables and palette is not None:
+ msg = (
+ "\n\nPassing `palette` without assigning `hue` is deprecated "
+ f"and will be removed in v0.14.0. Assign the `{self.orient}` variable "
+ "to `hue` and set `legend=False` for the same effect.\n"
+ )
+ warnings.warn(msg, FutureWarning, stacklevel=3)
+
+ self.legend = False
+ self.plot_data["hue"] = self.plot_data[self.orient]
+ self.variables["hue"] = self.variables.get(self.orient)
+ self.var_types["hue"] = self.var_types.get(self.orient)
+
+ hue_order = self.var_levels.get(self.orient)
+ self._var_levels.pop("hue", None)
+
+ return hue_order
+
+ def _point_kwargs_backcompat(self, scale, join, kwargs):
+ """Provide two cycles where scale= and join= work, but redirect to kwargs."""
+ if scale is not deprecated:
+ lw = mpl.rcParams["lines.linewidth"] * 1.8 * scale
+ mew = lw * .75
+ ms = lw * 2
+
+ msg = (
+ "\n\n"
+ "The `scale` parameter is deprecated and will be removed in v0.15.0. "
+ "You can now control the size of each plot element using matplotlib "
+ "`Line2D` parameters (e.g., `linewidth`, `markersize`, etc.)."
+ "\n"
+ )
+ warnings.warn(msg, stacklevel=3)
+ kwargs.update(linewidth=lw, markeredgewidth=mew, markersize=ms)
+
+ if join is not deprecated:
+ msg = (
+ "\n\n"
+ "The `join` parameter is deprecated and will be removed in v0.15.0."
+ )
+ if not join:
+ msg += (
+ " You can remove the line between points with `linestyle='none'`."
+ )
+ kwargs.update(linestyle="")
+ msg += "\n"
+ warnings.warn(msg, stacklevel=3)
+
+ def _err_kws_backcompat(self, err_kws, errcolor, errwidth, capsize):
+ """Provide two cycles where existing signature-level err_kws are handled."""
+ def deprecate_err_param(name, key, val):
+ if val is deprecated:
+ return
+ suggest = f"err_kws={{'{key}': {val!r}}}"
+ msg = (
+ f"\n\nThe `{name}` parameter is deprecated. And will be removed "
+ f"in v0.15.0. Pass `{suggest}` instead.\n"
+ )
+ warnings.warn(msg, FutureWarning, stacklevel=4)
+ err_kws[key] = val
+
+ if errcolor is not None:
+ deprecate_err_param("errcolor", "color", errcolor)
+ deprecate_err_param("errwidth", "linewidth", errwidth)
+
+ if capsize is None:
+ capsize = 0
+ msg = (
+ "\n\nPassing `capsize=None` is deprecated and will be removed "
+ "in v0.15.0. Pass `capsize=0` to disable caps.\n"
+ )
+ warnings.warn(msg, FutureWarning, stacklevel=3)
+
+ return err_kws, capsize
+
+ def _violin_scale_backcompat(self, scale, scale_hue, density_norm, common_norm):
+ """Provide two cycles of backcompat for scale kwargs"""
+ if scale is not deprecated:
+ density_norm = scale
+ msg = (
+ "\n\nThe `scale` parameter has been renamed and will be removed "
+ f"in v0.15.0. Pass `density_norm={scale!r}` for the same effect."
+ )
+ warnings.warn(msg, FutureWarning, stacklevel=3)
+
+ if scale_hue is not deprecated:
+ common_norm = scale_hue
+ msg = (
+ "\n\nThe `scale_hue` parameter has been replaced and will be removed "
+ f"in v0.15.0. Pass `common_norm={not scale_hue}` for the same effect."
+ )
+ warnings.warn(msg, FutureWarning, stacklevel=3)
+
+ return density_norm, common_norm
+
+ def _violin_bw_backcompat(self, bw, bw_method):
+ """Provide two cycles of backcompat for violin bandwidth parameterization."""
+ if bw is not deprecated:
+ bw_method = bw
+ msg = dedent(f"""\n
+ The `bw` parameter is deprecated in favor of `bw_method`/`bw_adjust`.
+ Setting `bw_method={bw!r}`, but please see docs for the new parameters
+ and update your code. This will become an error in seaborn v0.15.0.
+ """)
+ warnings.warn(msg, FutureWarning, stacklevel=3)
+ return bw_method
+
+ def _boxen_scale_backcompat(self, scale, width_method):
+ """Provide two cycles of backcompat for scale kwargs"""
+ if scale is not deprecated:
+ width_method = scale
+ msg = (
+ "\n\nThe `scale` parameter has been renamed to `width_method` and "
+ f"will be removed in v0.15. Pass `width_method={scale!r}"
+ )
+ if scale == "area":
+ msg += ", but note that the result for 'area' will appear different."
+ else:
+ msg += " for the same effect."
+ warnings.warn(msg, FutureWarning, stacklevel=3)
+
+ return width_method
+
+ def _complement_color(self, color, base_color, hue_map):
+ """Allow a color to be set automatically using a basis of comparison."""
+ if color == "gray":
+ msg = (
+ 'Use "auto" to set automatic grayscale colors. From v0.14.0, '
+ '"gray" will default to matplotlib\'s definition.'
+ )
+ warnings.warn(msg, FutureWarning, stacklevel=3)
+ color = "auto"
+ elif color is None or color is default:
+ color = "auto"
+
+ if color != "auto":
+ return color
+
+ if hue_map.lookup_table is None:
+ if base_color is None:
+ return None
+ basis = [mpl.colors.to_rgb(base_color)]
+ else:
+ basis = [mpl.colors.to_rgb(c) for c in hue_map.lookup_table.values()]
+ unique_colors = np.unique(basis, axis=0)
+ light_vals = [rgb_to_hls(*rgb[:3])[1] for rgb in unique_colors]
+ lum = min(light_vals) * .6
+ return (lum, lum, lum)
+
+ def _map_prop_with_hue(self, name, value, fallback, plot_kws):
+ """Support pointplot behavior of modifying the marker/linestyle with hue."""
+ if value is default:
+ value = plot_kws.pop(name, fallback)
+
+ if "hue" in self.variables:
+ levels = self._hue_map.levels
+ if isinstance(value, list):
+ mapping = {k: v for k, v in zip(levels, value)}
+ else:
+ mapping = {k: value for k in levels}
+ else:
+ mapping = {None: value}
+
+ return mapping
+
+ def _adjust_cat_axis(self, ax, axis):
+ """Set ticks and limits for a categorical variable."""
+ # Note: in theory, this could happen in _attach for all categorical axes
+ # But two reasons not to do that:
+ # - If it happens before plotting, autoscaling messes up the plot limits
+ # - It would change existing plots from other seaborn functions
+ if self.var_types[axis] != "categorical":
+ return
+
+ # If both x/y data are empty, the correct way to set up the plot is
+ # somewhat undefined; because we don't add null category data to the plot in
+ # this case we don't *have* a categorical axis (yet), so best to just bail.
+ if self.plot_data[axis].empty:
+ return
+
+ # We can infer the total number of categories (including those from previous
+ # plots that are not part of the plot we are currently making) from the number
+ # of ticks, which matplotlib sets up while doing unit conversion. This feels
+ # slightly risky, as if we are relying on something that may be a matplotlib
+ # implementation detail. But I cannot think of a better way to keep track of
+ # the state from previous categorical calls (see GH2516 for context)
+ n = len(getattr(ax, f"get_{axis}ticks")())
+
+ if axis == "x":
+ ax.xaxis.grid(False)
+ ax.set_xlim(-.5, n - .5, auto=None)
+ else:
+ ax.yaxis.grid(False)
+ # Note limits that correspond to previously-inverted y axis
+ ax.set_ylim(n - .5, -.5, auto=None)
+
+ def _dodge_needed(self):
+ """Return True when use of `hue` would cause overlaps."""
+ groupers = list({self.orient, "col", "row"} & set(self.variables))
+ if "hue" in self.variables:
+ orient = self.plot_data[groupers].value_counts()
+ paired = self.plot_data[[*groupers, "hue"]].value_counts()
+ return orient.size != paired.size
+ return False
+
+ def _dodge(self, keys, data):
+ """Apply a dodge transform to coordinates in place."""
+ if "hue" not in self.variables:
+ # Short-circuit if hue variable was not assigned
+ # We could potentially warn when hue=None, dodge=True, user may be confused
+ # But I think it's fine to just treat it as a no-op.
+ return
+ hue_idx = self._hue_map.levels.index(keys["hue"])
+ n = len(self._hue_map.levels)
+ data["width"] /= n
+
+ full_width = data["width"] * n
+ offset = data["width"] * hue_idx + data["width"] / 2 - full_width / 2
+ data[self.orient] += offset
+
+ def _invert_scale(self, ax, data, vars=("x", "y")):
+ """Undo scaling after computation so data are plotted correctly."""
+ for var in vars:
+ _, inv = _get_transform_functions(ax, var[0])
+ if var == self.orient and "width" in data:
+ hw = data["width"] / 2
+ data["edge"] = inv(data[var] - hw)
+ data["width"] = inv(data[var] + hw) - data["edge"].to_numpy()
+ for suf in ["", "min", "max"]:
+ if (col := f"{var}{suf}") in data:
+ data[col] = inv(data[col])
+
+ def _configure_legend(self, ax, func, common_kws=None, semantic_kws=None):
+ if self.legend == "auto":
+ show_legend = not self._redundant_hue and self.input_format != "wide"
+ else:
+ show_legend = bool(self.legend)
+ if show_legend:
+ self.add_legend_data(ax, func, common_kws, semantic_kws=semantic_kws)
+ handles, _ = ax.get_legend_handles_labels()
+ if handles:
+ ax.legend(title=self.legend_title)
+
+ @property
+ def _native_width(self):
+ """Return unit of width separating categories on native numeric scale."""
+ # Categorical data always have a unit width
+ if self.var_types[self.orient] == "categorical":
+ return 1
+
+ # Otherwise, define the width as the smallest space between observations
+ unique_values = np.unique(self.comp_data[self.orient])
+ if len(unique_values) > 1:
+ native_width = np.nanmin(np.diff(unique_values))
+ else:
+ native_width = 1
+ return native_width
+
+ def _nested_offsets(self, width, dodge):
+ """Return offsets for each hue level for dodged plots."""
+ offsets = None
+ if "hue" in self.variables and self._hue_map.levels is not None:
+ n_levels = len(self._hue_map.levels)
+ if dodge:
+ each_width = width / n_levels
+ offsets = np.linspace(0, width - each_width, n_levels)
+ offsets -= offsets.mean()
+ else:
+ offsets = np.zeros(n_levels)
+ return offsets
+
+ # Note that the plotting methods here aim (in most cases) to produce the
+ # exact same artists as the original (pre 0.12) version of the code, so
+ # there is some weirdness that might not otherwise be clean or make sense in
+ # this context, such as adding empty artists for combinations of variables
+ # with no observations
+
+ def plot_strips(
+ self,
+ jitter,
+ dodge,
+ color,
+ plot_kws,
+ ):
+
+ width = .8 * self._native_width
+ offsets = self._nested_offsets(width, dodge)
+
+ if jitter is True:
+ jlim = 0.1
+ else:
+ jlim = float(jitter)
+ if "hue" in self.variables and dodge and self._hue_map.levels is not None:
+ jlim /= len(self._hue_map.levels)
+ jlim *= self._native_width
+ jitterer = partial(np.random.uniform, low=-jlim, high=+jlim)
+
+ iter_vars = [self.orient]
+ if dodge:
+ iter_vars.append("hue")
+
+ ax = self.ax
+ dodge_move = jitter_move = 0
+
+ if "marker" in plot_kws and not MarkerStyle(plot_kws["marker"]).is_filled():
+ plot_kws.pop("edgecolor", None)
+
+ for sub_vars, sub_data in self.iter_data(iter_vars,
+ from_comp_data=True,
+ allow_empty=True):
+
+ ax = self._get_axes(sub_vars)
+
+ if offsets is not None and (offsets != 0).any():
+ dodge_move = offsets[sub_data["hue"].map(self._hue_map.levels.index)]
+
+ jitter_move = jitterer(size=len(sub_data)) if len(sub_data) > 1 else 0
+
+ adjusted_data = sub_data[self.orient] + dodge_move + jitter_move
+ sub_data[self.orient] = adjusted_data
+ self._invert_scale(ax, sub_data)
+
+ points = ax.scatter(sub_data["x"], sub_data["y"], color=color, **plot_kws)
+ if "hue" in self.variables:
+ points.set_facecolors(self._hue_map(sub_data["hue"]))
+
+ self._configure_legend(ax, _scatter_legend_artist, common_kws=plot_kws)
+
+ def plot_swarms(
+ self,
+ dodge,
+ color,
+ warn_thresh,
+ plot_kws,
+ ):
+
+ width = .8 * self._native_width
+ offsets = self._nested_offsets(width, dodge)
+
+ iter_vars = [self.orient]
+ if dodge:
+ iter_vars.append("hue")
+
+ ax = self.ax
+ point_collections = {}
+ dodge_move = 0
+
+ if "marker" in plot_kws and not MarkerStyle(plot_kws["marker"]).is_filled():
+ plot_kws.pop("edgecolor", None)
+
+ for sub_vars, sub_data in self.iter_data(iter_vars,
+ from_comp_data=True,
+ allow_empty=True):
+
+ ax = self._get_axes(sub_vars)
+
+ if offsets is not None:
+ dodge_move = offsets[sub_data["hue"].map(self._hue_map.levels.index)]
+
+ if not sub_data.empty:
+ sub_data[self.orient] = sub_data[self.orient] + dodge_move
+
+ self._invert_scale(ax, sub_data)
+
+ points = ax.scatter(sub_data["x"], sub_data["y"], color=color, **plot_kws)
+ if "hue" in self.variables:
+ points.set_facecolors(self._hue_map(sub_data["hue"]))
+
+ if not sub_data.empty:
+ point_collections[(ax, sub_data[self.orient].iloc[0])] = points
+
+ beeswarm = Beeswarm(width=width, orient=self.orient, warn_thresh=warn_thresh)
+ for (ax, center), points in point_collections.items():
+ if points.get_offsets().shape[0] > 1:
+
+ def draw(points, renderer, *, center=center):
+
+ beeswarm(points, center)
+
+ if self.orient == "y":
+ scalex = False
+ scaley = ax.get_autoscaley_on()
+ else:
+ scalex = ax.get_autoscalex_on()
+ scaley = False
+
+ # This prevents us from undoing the nice categorical axis limits
+ # set in _adjust_cat_axis, because that method currently leave
+ # the autoscale flag in its original setting. It may be better
+ # to disable autoscaling there to avoid needing to do this.
+ fixed_scale = self.var_types[self.orient] == "categorical"
+ ax.update_datalim(points.get_datalim(ax.transData))
+ if not fixed_scale and (scalex or scaley):
+ ax.autoscale_view(scalex=scalex, scaley=scaley)
+
+ super(points.__class__, points).draw(renderer)
+
+ points.draw = draw.__get__(points)
+
+ _draw_figure(ax.figure)
+ self._configure_legend(ax, _scatter_legend_artist, plot_kws)
+
+ def plot_boxes(
+ self,
+ width,
+ dodge,
+ gap,
+ fill,
+ whis,
+ color,
+ linecolor,
+ linewidth,
+ fliersize,
+ plot_kws, # TODO rename user_kws?
+ ):
+
+ iter_vars = ["hue"]
+ value_var = {"x": "y", "y": "x"}[self.orient]
+
+ def get_props(element, artist=mpl.lines.Line2D):
+ return normalize_kwargs(plot_kws.pop(f"{element}props", {}), artist)
+
+ if not fill and linewidth is None:
+ linewidth = mpl.rcParams["lines.linewidth"]
+ bootstrap = plot_kws.pop("bootstrap", mpl.rcParams["boxplot.bootstrap"])
+ plot_kws.setdefault("shownotches", plot_kws.pop("notch", False))
+
+ box_artist = mpl.patches.Rectangle if fill else mpl.lines.Line2D
+ props = {
+ "box": get_props("box", box_artist),
+ "median": get_props("median"),
+ "whisker": get_props("whisker"),
+ "flier": get_props("flier"),
+ "cap": get_props("cap"),
+ }
+
+ props["median"].setdefault("solid_capstyle", "butt")
+ props["whisker"].setdefault("solid_capstyle", "butt")
+ props["flier"].setdefault("markersize", fliersize)
+
+ ax = self.ax
+
+ for sub_vars, sub_data in self.iter_data(iter_vars,
+ from_comp_data=True,
+ allow_empty=False):
+
+ ax = self._get_axes(sub_vars)
+
+ grouped = sub_data.groupby(self.orient)[value_var]
+ positions = sorted(sub_data[self.orient].unique().astype(float))
+ value_data = [x.to_numpy() for _, x in grouped]
+ stats = pd.DataFrame(mpl.cbook.boxplot_stats(value_data, whis=whis,
+ bootstrap=bootstrap))
+
+ orig_width = width * self._native_width
+ data = pd.DataFrame({self.orient: positions, "width": orig_width})
+ if dodge:
+ self._dodge(sub_vars, data)
+ if gap:
+ data["width"] *= 1 - gap
+ capwidth = plot_kws.get("capwidths", 0.5 * data["width"])
+
+ self._invert_scale(ax, data)
+ _, inv = _get_transform_functions(ax, value_var)
+ for stat in ["mean", "med", "q1", "q3", "cilo", "cihi", "whislo", "whishi"]:
+ stats[stat] = inv(stats[stat])
+ stats["fliers"] = stats["fliers"].map(inv)
+
+ linear_orient_scale = getattr(ax, f"get_{self.orient}scale")() == "linear"
+
+ maincolor = self._hue_map(sub_vars["hue"]) if "hue" in sub_vars else color
+ if fill:
+ boxprops = {
+ "facecolor": maincolor, "edgecolor": linecolor, **props["box"]
+ }
+ medianprops = {"color": linecolor, **props["median"]}
+ whiskerprops = {"color": linecolor, **props["whisker"]}
+ flierprops = {"markeredgecolor": linecolor, **props["flier"]}
+ capprops = {"color": linecolor, **props["cap"]}
+ else:
+ boxprops = {"color": maincolor, **props["box"]}
+ medianprops = {"color": maincolor, **props["median"]}
+ whiskerprops = {"color": maincolor, **props["whisker"]}
+ flierprops = {"markeredgecolor": maincolor, **props["flier"]}
+ capprops = {"color": maincolor, **props["cap"]}
+
+ if linewidth is not None:
+ for prop_dict in [boxprops, medianprops, whiskerprops, capprops]:
+ prop_dict.setdefault("linewidth", linewidth)
+
+ default_kws = dict(
+ bxpstats=stats.to_dict("records"),
+ positions=data[self.orient],
+ # Set width to 0 to avoid going out of domain
+ widths=data["width"] if linear_orient_scale else 0,
+ patch_artist=fill,
+ vert=self.orient == "x",
+ manage_ticks=False,
+ boxprops=boxprops,
+ medianprops=medianprops,
+ whiskerprops=whiskerprops,
+ flierprops=flierprops,
+ capprops=capprops,
+ # Added in matplotlib 3.6.0; see below
+ # capwidths=capwidth,
+ **(
+ {} if _version_predates(mpl, "3.6.0")
+ else {"capwidths": capwidth}
+ )
+ )
+ boxplot_kws = {**default_kws, **plot_kws}
+ artists = ax.bxp(**boxplot_kws)
+
+ # Reset artist widths after adding so everything stays positive
+ ori_idx = ["x", "y"].index(self.orient)
+
+ if not linear_orient_scale:
+ for i, box in enumerate(data.to_dict("records")):
+ p0 = box["edge"]
+ p1 = box["edge"] + box["width"]
+
+ if artists["boxes"]:
+ box_artist = artists["boxes"][i]
+ if fill:
+ box_verts = box_artist.get_path().vertices.T
+ else:
+ box_verts = box_artist.get_data()
+ box_verts[ori_idx][0] = p0
+ box_verts[ori_idx][3:] = p0
+ box_verts[ori_idx][1:3] = p1
+ if not fill:
+ # When fill is True, the data get changed in place
+ box_artist.set_data(box_verts)
+ ax.update_datalim(
+ np.transpose(box_verts),
+ updatex=self.orient == "x",
+ updatey=self.orient == "y",
+ )
+
+ if artists["medians"]:
+ verts = artists["medians"][i].get_xydata().T
+ verts[ori_idx][:] = p0, p1
+ artists["medians"][i].set_data(verts)
+
+ if artists["caps"]:
+ f_fwd, f_inv = _get_transform_functions(ax, self.orient)
+ for line in artists["caps"][2 * i:2 * i + 2]:
+ p0 = f_inv(f_fwd(box[self.orient]) - capwidth[i] / 2)
+ p1 = f_inv(f_fwd(box[self.orient]) + capwidth[i] / 2)
+ verts = line.get_xydata().T
+ verts[ori_idx][:] = p0, p1
+ line.set_data(verts)
+
+ ax.add_container(BoxPlotContainer(artists))
+
+ legend_artist = _get_patch_legend_artist(fill)
+ self._configure_legend(ax, legend_artist, boxprops)
+
+ def plot_boxens(
+ self,
+ width,
+ dodge,
+ gap,
+ fill,
+ color,
+ linecolor,
+ linewidth,
+ width_method,
+ k_depth,
+ outlier_prop,
+ trust_alpha,
+ showfliers,
+ box_kws,
+ flier_kws,
+ line_kws,
+ plot_kws,
+ ):
+
+ iter_vars = [self.orient, "hue"]
+ value_var = {"x": "y", "y": "x"}[self.orient]
+
+ estimator = LetterValues(k_depth, outlier_prop, trust_alpha)
+
+ width_method_options = ["exponential", "linear", "area"]
+ _check_argument("width_method", width_method_options, width_method)
+
+ box_kws = plot_kws if box_kws is None else {**plot_kws, **box_kws}
+ flier_kws = {} if flier_kws is None else flier_kws.copy()
+ line_kws = {} if line_kws is None else line_kws.copy()
+
+ if linewidth is None:
+ if fill:
+ linewidth = 0.5 * mpl.rcParams["lines.linewidth"]
+ else:
+ linewidth = mpl.rcParams["lines.linewidth"]
+
+ ax = self.ax
+
+ for sub_vars, sub_data in self.iter_data(iter_vars,
+ from_comp_data=True,
+ allow_empty=False):
+
+ ax = self._get_axes(sub_vars)
+ _, inv_ori = _get_transform_functions(ax, self.orient)
+ _, inv_val = _get_transform_functions(ax, value_var)
+
+ # Statistics
+ lv_data = estimator(sub_data[value_var])
+ n = lv_data["k"] * 2 - 1
+ vals = lv_data["values"]
+
+ pos_data = pd.DataFrame({
+ self.orient: [sub_vars[self.orient]],
+ "width": [width * self._native_width],
+ })
+ if dodge:
+ self._dodge(sub_vars, pos_data)
+ if gap:
+ pos_data["width"] *= 1 - gap
+
+ # Letter-value boxes
+ levels = lv_data["levels"]
+ exponent = (levels - 1 - lv_data["k"]).astype(float)
+ if width_method == "linear":
+ rel_widths = levels + 1
+ elif width_method == "exponential":
+ rel_widths = 2 ** exponent
+ elif width_method == "area":
+ tails = levels < (lv_data["k"] - 1)
+ rel_widths = 2 ** (exponent - tails) / np.diff(lv_data["values"])
+
+ center = pos_data[self.orient].item()
+ widths = rel_widths / rel_widths.max() * pos_data["width"].item()
+
+ box_vals = inv_val(vals)
+ box_pos = inv_ori(center - widths / 2)
+ box_heights = inv_val(vals[1:]) - inv_val(vals[:-1])
+ box_widths = inv_ori(center + widths / 2) - inv_ori(center - widths / 2)
+
+ maincolor = self._hue_map(sub_vars["hue"]) if "hue" in sub_vars else color
+ flier_colors = {
+ "facecolor": "none", "edgecolor": ".45" if fill else maincolor
+ }
+ if fill:
+ cmap = light_palette(maincolor, as_cmap=True)
+ boxcolors = cmap(2 ** ((exponent + 2) / 3))
+ else:
+ boxcolors = maincolor
+
+ boxen = []
+ for i in range(n):
+ if self.orient == "x":
+ xy = (box_pos[i], box_vals[i])
+ w, h = (box_widths[i], box_heights[i])
+ else:
+ xy = (box_vals[i], box_pos[i])
+ w, h = (box_heights[i], box_widths[i])
+ boxen.append(Rectangle(xy, w, h))
+
+ if fill:
+ box_colors = {"facecolors": boxcolors, "edgecolors": linecolor}
+ else:
+ box_colors = {"facecolors": "none", "edgecolors": boxcolors}
+
+ collection_kws = {**box_colors, "linewidth": linewidth, **box_kws}
+ ax.add_collection(PatchCollection(boxen, **collection_kws), autolim=False)
+ ax.update_datalim(
+ np.column_stack([box_vals, box_vals]),
+ updatex=self.orient == "y",
+ updatey=self.orient == "x",
+ )
+
+ # Median line
+ med = lv_data["median"]
+ hw = pos_data["width"].item() / 2
+ if self.orient == "x":
+ x, y = inv_ori([center - hw, center + hw]), inv_val([med, med])
+ else:
+ x, y = inv_val([med, med]), inv_ori([center - hw, center + hw])
+ default_kws = {
+ "color": linecolor if fill else maincolor,
+ "solid_capstyle": "butt",
+ "linewidth": 1.25 * linewidth,
+ }
+ ax.plot(x, y, **{**default_kws, **line_kws})
+
+ # Outliers ("fliers")
+ if showfliers:
+ vals = inv_val(lv_data["fliers"])
+ pos = np.full(len(vals), inv_ori(pos_data[self.orient].item()))
+ x, y = (pos, vals) if self.orient == "x" else (vals, pos)
+ ax.scatter(x, y, **{**flier_colors, "s": 25, **flier_kws})
+
+ ax.autoscale_view(scalex=self.orient == "y", scaley=self.orient == "x")
+
+ legend_artist = _get_patch_legend_artist(fill)
+ common_kws = {**box_kws, "linewidth": linewidth, "edgecolor": linecolor}
+ self._configure_legend(ax, legend_artist, common_kws)
+
+ def plot_violins(
+ self,
+ width,
+ dodge,
+ gap,
+ split,
+ color,
+ fill,
+ linecolor,
+ linewidth,
+ inner,
+ density_norm,
+ common_norm,
+ kde_kws,
+ inner_kws,
+ plot_kws,
+ ):
+
+ iter_vars = [self.orient, "hue"]
+ value_var = {"x": "y", "y": "x"}[self.orient]
+
+ inner_options = ["box", "quart", "stick", "point", None]
+ _check_argument("inner", inner_options, inner, prefix=True)
+ _check_argument("density_norm", ["area", "count", "width"], density_norm)
+
+ if linewidth is None:
+ if fill:
+ linewidth = 1.25 * mpl.rcParams["patch.linewidth"]
+ else:
+ linewidth = mpl.rcParams["lines.linewidth"]
+
+ if inner is not None and inner.startswith("box"):
+ box_width = inner_kws.pop("box_width", linewidth * 4.5)
+ whis_width = inner_kws.pop("whis_width", box_width / 3)
+ marker = inner_kws.pop("marker", "_" if self.orient == "x" else "|")
+
+ kde = KDE(**kde_kws)
+ ax = self.ax
+ violin_data = []
+
+ # Iterate through all the data splits once to compute the KDEs
+ for sub_vars, sub_data in self.iter_data(iter_vars,
+ from_comp_data=True,
+ allow_empty=False):
+
+ sub_data["weight"] = sub_data.get("weights", 1)
+ stat_data = kde._transform(sub_data, value_var, [])
+
+ maincolor = self._hue_map(sub_vars["hue"]) if "hue" in sub_vars else color
+ if not fill:
+ linecolor = maincolor
+ maincolor = "none"
+ default_kws = dict(
+ facecolor=maincolor,
+ edgecolor=linecolor,
+ linewidth=linewidth,
+ )
+
+ violin_data.append({
+ "position": sub_vars[self.orient],
+ "observations": sub_data[value_var],
+ "density": stat_data["density"],
+ "support": stat_data[value_var],
+ "kwargs": {**default_kws, **plot_kws},
+ "sub_vars": sub_vars,
+ "ax": self._get_axes(sub_vars),
+ })
+
+ # Once we've computed all the KDEs, get statistics for normalization
+ def vars_to_key(sub_vars):
+ return tuple((k, v) for k, v in sub_vars.items() if k != self.orient)
+
+ norm_keys = [vars_to_key(violin["sub_vars"]) for violin in violin_data]
+ if common_norm:
+ common_max_density = np.nanmax([v["density"].max() for v in violin_data])
+ common_max_count = np.nanmax([len(v["observations"]) for v in violin_data])
+ max_density = {key: common_max_density for key in norm_keys}
+ max_count = {key: common_max_count for key in norm_keys}
+ else:
+ with warnings.catch_warnings():
+ # Ignore warning when all violins are singular; it's not important
+ warnings.filterwarnings('ignore', "All-NaN (slice|axis) encountered")
+ max_density = {
+ key: np.nanmax([
+ v["density"].max() for v in violin_data
+ if vars_to_key(v["sub_vars"]) == key
+ ]) for key in norm_keys
+ }
+ max_count = {
+ key: np.nanmax([
+ len(v["observations"]) for v in violin_data
+ if vars_to_key(v["sub_vars"]) == key
+ ]) for key in norm_keys
+ }
+
+ real_width = width * self._native_width
+
+ # Now iterate through the violins again to apply the normalization and plot
+ for violin in violin_data:
+
+ index = pd.RangeIndex(0, max(len(violin["support"]), 1))
+ data = pd.DataFrame({
+ self.orient: violin["position"],
+ value_var: violin["support"],
+ "density": violin["density"],
+ "width": real_width,
+ }, index=index)
+
+ if dodge:
+ self._dodge(violin["sub_vars"], data)
+ if gap:
+ data["width"] *= 1 - gap
+
+ # Normalize the density across the distribution(s) and relative to the width
+ norm_key = vars_to_key(violin["sub_vars"])
+ hw = data["width"] / 2
+ peak_density = violin["density"].max()
+ if np.isnan(peak_density):
+ span = 1
+ elif density_norm == "area":
+ span = data["density"] / max_density[norm_key]
+ elif density_norm == "count":
+ count = len(violin["observations"])
+ span = data["density"] / peak_density * (count / max_count[norm_key])
+ elif density_norm == "width":
+ span = data["density"] / peak_density
+ span = span * hw * (2 if split else 1)
+
+ # Handle split violins (i.e. asymmetric spans)
+ right_side = (
+ 0 if "hue" not in self.variables
+ else self._hue_map.levels.index(violin["sub_vars"]["hue"]) % 2
+ )
+ if split:
+ offsets = (hw, span - hw) if right_side else (span - hw, hw)
+ else:
+ offsets = span, span
+
+ ax = violin["ax"]
+ _, invx = _get_transform_functions(ax, "x")
+ _, invy = _get_transform_functions(ax, "y")
+ inv_pos = {"x": invx, "y": invy}[self.orient]
+ inv_val = {"x": invx, "y": invy}[value_var]
+
+ linecolor = violin["kwargs"]["edgecolor"]
+
+ # Handle singular datasets (one or more observations with no variance
+ if np.isnan(peak_density):
+ pos = data[self.orient].iloc[0]
+ val = violin["observations"].mean()
+ if self.orient == "x":
+ x, y = [pos - offsets[0], pos + offsets[1]], [val, val]
+ else:
+ x, y = [val, val], [pos - offsets[0], pos + offsets[1]]
+ ax.plot(invx(x), invy(y), color=linecolor, linewidth=linewidth)
+ continue
+
+ # Plot the main violin body
+ plot_func = {"x": ax.fill_betweenx, "y": ax.fill_between}[self.orient]
+ plot_func(
+ inv_val(data[value_var]),
+ inv_pos(data[self.orient] - offsets[0]),
+ inv_pos(data[self.orient] + offsets[1]),
+ **violin["kwargs"]
+ )
+
+ # Adjust the observation data
+ obs = violin["observations"]
+ pos_dict = {self.orient: violin["position"], "width": real_width}
+ if dodge:
+ self._dodge(violin["sub_vars"], pos_dict)
+ if gap:
+ pos_dict["width"] *= (1 - gap)
+
+ # --- Plot the inner components
+ if inner is None:
+ continue
+
+ elif inner.startswith("point"):
+ pos = np.array([pos_dict[self.orient]] * len(obs))
+ if split:
+ pos += (-1 if right_side else 1) * pos_dict["width"] / 2
+ x, y = (pos, obs) if self.orient == "x" else (obs, pos)
+ kws = {
+ "color": linecolor,
+ "edgecolor": linecolor,
+ "s": (linewidth * 2) ** 2,
+ "zorder": violin["kwargs"].get("zorder", 2) + 1,
+ **inner_kws,
+ }
+ ax.scatter(invx(x), invy(y), **kws)
+
+ elif inner.startswith("stick"):
+ pos0 = np.interp(obs, data[value_var], data[self.orient] - offsets[0])
+ pos1 = np.interp(obs, data[value_var], data[self.orient] + offsets[1])
+ pos_pts = np.stack([inv_pos(pos0), inv_pos(pos1)])
+ val_pts = np.stack([inv_val(obs), inv_val(obs)])
+ segments = np.stack([pos_pts, val_pts]).transpose(2, 1, 0)
+ if self.orient == "y":
+ segments = segments[:, :, ::-1]
+ kws = {
+ "color": linecolor,
+ "linewidth": linewidth / 2,
+ **inner_kws,
+ }
+ lines = mpl.collections.LineCollection(segments, **kws)
+ ax.add_collection(lines, autolim=False)
+
+ elif inner.startswith("quart"):
+ stats = np.percentile(obs, [25, 50, 75])
+ pos0 = np.interp(stats, data[value_var], data[self.orient] - offsets[0])
+ pos1 = np.interp(stats, data[value_var], data[self.orient] + offsets[1])
+ pos_pts = np.stack([inv_pos(pos0), inv_pos(pos1)])
+ val_pts = np.stack([inv_val(stats), inv_val(stats)])
+ segments = np.stack([pos_pts, val_pts]).transpose(2, 0, 1)
+ if self.orient == "y":
+ segments = segments[:, ::-1, :]
+ dashes = [(1.25, .75), (2.5, 1), (1.25, .75)]
+ for i, segment in enumerate(segments):
+ kws = {
+ "color": linecolor,
+ "linewidth": linewidth,
+ "dashes": dashes[i],
+ **inner_kws,
+ }
+ ax.plot(*segment, **kws)
+
+ elif inner.startswith("box"):
+ stats = mpl.cbook.boxplot_stats(obs)[0]
+ pos = np.array(pos_dict[self.orient])
+ if split:
+ pos += (-1 if right_side else 1) * pos_dict["width"] / 2
+ pos = [pos, pos], [pos, pos], [pos]
+ val = (
+ [stats["whislo"], stats["whishi"]],
+ [stats["q1"], stats["q3"]],
+ [stats["med"]]
+ )
+ if self.orient == "x":
+ (x0, x1, x2), (y0, y1, y2) = pos, val
+ else:
+ (x0, x1, x2), (y0, y1, y2) = val, pos
+
+ if split:
+ offset = (1 if right_side else -1) * box_width / 72 / 2
+ dx, dy = (offset, 0) if self.orient == "x" else (0, -offset)
+ trans = ax.transData + mpl.transforms.ScaledTranslation(
+ dx, dy, ax.figure.dpi_scale_trans,
+ )
+ else:
+ trans = ax.transData
+ line_kws = {
+ "color": linecolor,
+ "transform": trans,
+ **inner_kws,
+ "linewidth": whis_width,
+ }
+ ax.plot(invx(x0), invy(y0), **line_kws)
+ line_kws["linewidth"] = box_width
+ ax.plot(invx(x1), invy(y1), **line_kws)
+ dot_kws = {
+ "marker": marker,
+ "markersize": box_width / 1.2,
+ "markeredgewidth": box_width / 5,
+ "transform": trans,
+ **inner_kws,
+ "markeredgecolor": "w",
+ "markerfacecolor": "w",
+ "color": linecolor, # simplify tests
+ }
+ ax.plot(invx(x2), invy(y2), **dot_kws)
+
+ legend_artist = _get_patch_legend_artist(fill)
+ common_kws = {**plot_kws, "linewidth": linewidth, "edgecolor": linecolor}
+ self._configure_legend(ax, legend_artist, common_kws)
+
+ def plot_points(
+ self,
+ aggregator,
+ markers,
+ linestyles,
+ dodge,
+ color,
+ capsize,
+ err_kws,
+ plot_kws,
+ ):
+
+ agg_var = {"x": "y", "y": "x"}[self.orient]
+ iter_vars = ["hue"]
+
+ plot_kws = normalize_kwargs(plot_kws, mpl.lines.Line2D)
+ plot_kws.setdefault("linewidth", mpl.rcParams["lines.linewidth"] * 1.8)
+ plot_kws.setdefault("markeredgewidth", plot_kws["linewidth"] * 0.75)
+ plot_kws.setdefault("markersize", plot_kws["linewidth"] * np.sqrt(2 * np.pi))
+
+ markers = self._map_prop_with_hue("marker", markers, "o", plot_kws)
+ linestyles = self._map_prop_with_hue("linestyle", linestyles, "-", plot_kws)
+
+ base_positions = self.var_levels[self.orient]
+ if self.var_types[self.orient] == "categorical":
+ min_cat_val = int(self.comp_data[self.orient].min())
+ max_cat_val = int(self.comp_data[self.orient].max())
+ base_positions = [i for i in range(min_cat_val, max_cat_val + 1)]
+
+ n_hue_levels = 0 if self._hue_map.levels is None else len(self._hue_map.levels)
+ if dodge is True:
+ dodge = .025 * n_hue_levels
+
+ ax = self.ax
+
+ for sub_vars, sub_data in self.iter_data(iter_vars,
+ from_comp_data=True,
+ allow_empty=True):
+
+ ax = self._get_axes(sub_vars)
+
+ ori_axis = getattr(ax, f"{self.orient}axis")
+ transform, _ = _get_transform_functions(ax, self.orient)
+ positions = transform(ori_axis.convert_units(base_positions))
+ agg_data = sub_data if sub_data.empty else (
+ sub_data
+ .groupby(self.orient)
+ .apply(aggregator, agg_var, **groupby_apply_include_groups(False))
+ .reindex(pd.Index(positions, name=self.orient))
+ .reset_index()
+ )
+
+ if dodge:
+ hue_idx = self._hue_map.levels.index(sub_vars["hue"])
+ step_size = dodge / (n_hue_levels - 1)
+ offset = -dodge / 2 + step_size * hue_idx
+ agg_data[self.orient] += offset * self._native_width
+
+ self._invert_scale(ax, agg_data)
+
+ sub_kws = plot_kws.copy()
+ sub_kws.update(
+ marker=markers[sub_vars.get("hue")],
+ linestyle=linestyles[sub_vars.get("hue")],
+ color=self._hue_map(sub_vars["hue"]) if "hue" in sub_vars else color,
+ )
+
+ line, = ax.plot(agg_data["x"], agg_data["y"], **sub_kws)
+
+ sub_err_kws = err_kws.copy()
+ line_props = line.properties()
+ for prop in ["color", "linewidth", "alpha", "zorder"]:
+ sub_err_kws.setdefault(prop, line_props[prop])
+ if aggregator.error_method is not None:
+ self.plot_errorbars(ax, agg_data, capsize, sub_err_kws)
+
+ legend_artist = partial(mpl.lines.Line2D, [], [])
+ semantic_kws = {"hue": {"marker": markers, "linestyle": linestyles}}
+ self._configure_legend(ax, legend_artist, sub_kws, semantic_kws)
+
+ def plot_bars(
+ self,
+ aggregator,
+ dodge,
+ gap,
+ width,
+ fill,
+ color,
+ capsize,
+ err_kws,
+ plot_kws,
+ ):
+
+ agg_var = {"x": "y", "y": "x"}[self.orient]
+ iter_vars = ["hue"]
+
+ ax = self.ax
+
+ if self._hue_map.levels is None:
+ dodge = False
+
+ if dodge and capsize is not None:
+ capsize = capsize / len(self._hue_map.levels)
+
+ if not fill:
+ plot_kws.setdefault("linewidth", 1.5 * mpl.rcParams["lines.linewidth"])
+
+ err_kws.setdefault("linewidth", 1.5 * mpl.rcParams["lines.linewidth"])
+
+ for sub_vars, sub_data in self.iter_data(iter_vars,
+ from_comp_data=True,
+ allow_empty=True):
+
+ ax = self._get_axes(sub_vars)
+
+ agg_data = sub_data if sub_data.empty else (
+ sub_data
+ .groupby(self.orient)
+ .apply(aggregator, agg_var, **groupby_apply_include_groups(False))
+ .reset_index()
+ )
+
+ agg_data["width"] = width * self._native_width
+ if dodge:
+ self._dodge(sub_vars, agg_data)
+ if gap:
+ agg_data["width"] *= 1 - gap
+
+ agg_data["edge"] = agg_data[self.orient] - agg_data["width"] / 2
+ self._invert_scale(ax, agg_data)
+
+ if self.orient == "x":
+ bar_func = ax.bar
+ kws = dict(
+ x=agg_data["edge"], height=agg_data["y"], width=agg_data["width"]
+ )
+ else:
+ bar_func = ax.barh
+ kws = dict(
+ y=agg_data["edge"], width=agg_data["x"], height=agg_data["width"]
+ )
+
+ main_color = self._hue_map(sub_vars["hue"]) if "hue" in sub_vars else color
+
+ # Set both color and facecolor for property cycle logic
+ kws["align"] = "edge"
+ if fill:
+ kws.update(color=main_color, facecolor=main_color)
+ else:
+ kws.update(color=main_color, edgecolor=main_color, facecolor="none")
+
+ bar_func(**{**kws, **plot_kws})
+
+ if aggregator.error_method is not None:
+ self.plot_errorbars(
+ ax, agg_data, capsize,
+ {"color": ".26" if fill else main_color, **err_kws}
+ )
+
+ legend_artist = _get_patch_legend_artist(fill)
+ self._configure_legend(ax, legend_artist, plot_kws)
+
+ def plot_errorbars(self, ax, data, capsize, err_kws):
+
+ var = {"x": "y", "y": "x"}[self.orient]
+ for row in data.to_dict("records"):
+
+ row = dict(row)
+ pos = np.array([row[self.orient], row[self.orient]])
+ val = np.array([row[f"{var}min"], row[f"{var}max"]])
+
+ if capsize:
+
+ cw = capsize * self._native_width / 2
+ scl, inv = _get_transform_functions(ax, self.orient)
+ cap = inv(scl(pos[0]) - cw), inv(scl(pos[1]) + cw)
+
+ pos = np.concatenate([
+ [*cap, np.nan], pos, [np.nan, *cap]
+ ])
+ val = np.concatenate([
+ [val[0], val[0], np.nan], val, [np.nan, val[-1], val[-1]],
+ ])
+
+ if self.orient == "x":
+ args = pos, val
+ else:
+ args = val, pos
+ ax.plot(*args, **err_kws)
+
+
+class _CategoricalAggPlotter(_CategoricalPlotter):
+
+ flat_structure = {"x": "@index", "y": "@values"}
+
+
+_categorical_docs = dict(
+
+ # Shared narrative docs
+ categorical_narrative=dedent("""\
+ See the :ref:`tutorial <categorical_tutorial>` for more information.
+
+ .. note::
+ By default, this function treats one of the variables as categorical
+ and draws data at ordinal positions (0, 1, ... n) on the relevant axis.
+ As of version 0.13.0, this can be disabled by setting `native_scale=True`.
+ """),
+
+ # Shared function parameters
+ input_params=dedent("""\
+ x, y, hue : names of variables in `data` or vector data
+ Inputs for plotting long-form data. See examples for interpretation.\
+ """),
+ categorical_data=dedent("""\
+ data : DataFrame, Series, dict, array, or list of arrays
+ Dataset for plotting. If `x` and `y` are absent, this is
+ interpreted as wide-form. Otherwise it is expected to be long-form.\
+ """),
+ order_vars=dedent("""\
+ order, hue_order : lists of strings
+ Order to plot the categorical levels in; otherwise the levels are
+ inferred from the data objects.\
+ """),
+ stat_api_params=dedent("""\
+ estimator : string or callable that maps vector -> scalar
+ Statistical function to estimate within each categorical bin.
+ errorbar : string, (string, number) tuple, callable or None
+ Name of errorbar method (either "ci", "pi", "se", or "sd"), or a tuple
+ with a method name and a level parameter, or a function that maps from a
+ vector to a (min, max) interval, or None to hide errorbar. See the
+ :doc:`errorbar tutorial </tutorial/error_bars>` for more information.
+
+ .. versionadded:: v0.12.0
+ n_boot : int
+ Number of bootstrap samples used to compute confidence intervals.
+ seed : int, `numpy.random.Generator`, or `numpy.random.RandomState`
+ Seed or random number generator for reproducible bootstrapping.
+ units : name of variable in `data` or vector data
+ Identifier of sampling units; used by the errorbar function to
+ perform a multilevel bootstrap and account for repeated measures
+ weights : name of variable in `data` or vector data
+ Data values or column used to compute weighted statistics.
+ Note that the use of weights may limit other statistical options.
+
+ .. versionadded:: v0.13.1\
+ """),
+ ci=dedent("""\
+ ci : float
+ Level of the confidence interval to show, in [0, 100].
+
+ .. deprecated:: v0.12.0
+ Use `errorbar=("ci", ...)`.\
+ """),
+ orient=dedent("""\
+ orient : "v" | "h" | "x" | "y"
+ Orientation of the plot (vertical or horizontal). This is usually
+ inferred based on the type of the input variables, but it can be used
+ to resolve ambiguity when both `x` and `y` are numeric or when
+ plotting wide-form data.
+
+ .. versionchanged:: v0.13.0
+ Added 'x'/'y' as options, equivalent to 'v'/'h'.\
+ """),
+ color=dedent("""\
+ color : matplotlib color
+ Single color for the elements in the plot.\
+ """),
+ palette=dedent("""\
+ palette : palette name, list, dict, or :class:`matplotlib.colors.Colormap`
+ Color palette that maps the hue variable. If the palette is a dictionary,
+ keys should be names of levels and values should be matplotlib colors.
+ The type/value will sometimes force a qualitative/quantitative mapping.\
+ """),
+ hue_norm=dedent("""\
+ hue_norm : tuple or :class:`matplotlib.colors.Normalize` object
+ Normalization in data units for colormap applied to the `hue`
+ variable when it is numeric. Not relevant if `hue` is categorical.
+
+ .. versionadded:: v0.12.0\
+ """),
+ saturation=dedent("""\
+ saturation : float
+ Proportion of the original saturation to draw fill colors in. Large
+ patches often look better with desaturated colors, but set this to
+ `1` if you want the colors to perfectly match the input values.\
+ """),
+ capsize=dedent("""\
+ capsize : float
+ Width of the "caps" on error bars, relative to bar spacing.\
+ """),
+ errcolor=dedent("""\
+ errcolor : matplotlib color
+ Color used for the error bar lines.
+
+ .. deprecated:: 0.13.0
+ Use `err_kws={'color': ...}`.\
+ """),
+ errwidth=dedent("""\
+ errwidth : float
+ Thickness of error bar lines (and caps), in points.
+
+ .. deprecated:: 0.13.0
+ Use `err_kws={'linewidth': ...}`.\
+ """),
+ fill=dedent("""\
+ fill : bool
+ If True, use a solid patch. Otherwise, draw as line art.
+
+ .. versionadded:: v0.13.0\
+ """),
+ gap=dedent("""\
+ gap : float
+ Shrink on the orient axis by this factor to add a gap between dodged elements.
+
+ .. versionadded:: 0.13.0\
+ """),
+ width=dedent("""\
+ width : float
+ Width allotted to each element on the orient axis. When `native_scale=True`,
+ it is relative to the minimum distance between two values in the native scale.\
+ """),
+ dodge=dedent("""\
+ dodge : "auto" or bool
+ When hue mapping is used, whether elements should be narrowed and shifted along
+ the orient axis to eliminate overlap. If `"auto"`, set to `True` when the
+ orient variable is crossed with the categorical variable or `False` otherwise.
+
+ .. versionchanged:: 0.13.0
+
+ Added `"auto"` mode as a new default.\
+ """),
+ linewidth=dedent("""\
+ linewidth : float
+ Width of the lines that frame the plot elements.\
+ """),
+ linecolor=dedent("""\
+ linecolor : color
+ Color to use for line elements, when `fill` is True.
+
+ .. versionadded:: v0.13.0\
+ """),
+ log_scale=dedent("""\
+ log_scale : bool or number, or pair of bools or numbers
+ Set axis scale(s) to log. A single value sets the data axis for any numeric
+ axes in the plot. A pair of values sets each axis independently.
+ Numeric values are interpreted as the desired base (default 10).
+ When `None` or `False`, seaborn defers to the existing Axes scale.
+
+ .. versionadded:: v0.13.0\
+ """),
+ native_scale=dedent("""\
+ native_scale : bool
+ When True, numeric or datetime values on the categorical axis will maintain
+ their original scaling rather than being converted to fixed indices.
+
+ .. versionadded:: v0.13.0\
+ """),
+ formatter=dedent("""\
+ formatter : callable
+ Function for converting categorical data into strings. Affects both grouping
+ and tick labels.
+
+ .. versionadded:: v0.13.0\
+ """),
+ legend=dedent("""\
+ legend : "auto", "brief", "full", or False
+ How to draw the legend. If "brief", numeric `hue` and `size`
+ variables will be represented with a sample of evenly spaced values.
+ If "full", every group will get an entry in the legend. If "auto",
+ choose between brief or full representation based on number of levels.
+ If `False`, no legend data is added and no legend is drawn.
+
+ .. versionadded:: v0.13.0\
+ """),
+ err_kws=dedent("""\
+ err_kws : dict
+ Parameters of :class:`matplotlib.lines.Line2D`, for the error bar artists.
+
+ .. versionadded:: v0.13.0\
+ """),
+ ax_in=dedent("""\
+ ax : matplotlib Axes
+ Axes object to draw the plot onto, otherwise uses the current Axes.\
+ """),
+ ax_out=dedent("""\
+ ax : matplotlib Axes
+ Returns the Axes object with the plot drawn onto it.\
+ """),
+
+ # Shared see also
+ boxplot=dedent("""\
+ boxplot : A traditional box-and-whisker plot with a similar API.\
+ """),
+ violinplot=dedent("""\
+ violinplot : A combination of boxplot and kernel density estimation.\
+ """),
+ stripplot=dedent("""\
+ stripplot : A scatterplot where one variable is categorical. Can be used
+ in conjunction with other plots to show each observation.\
+ """),
+ swarmplot=dedent("""\
+ swarmplot : A categorical scatterplot where the points do not overlap. Can
+ be used with other plots to show each observation.\
+ """),
+ barplot=dedent("""\
+ barplot : Show point estimates and confidence intervals using bars.\
+ """),
+ countplot=dedent("""\
+ countplot : Show the counts of observations in each categorical bin.\
+ """),
+ pointplot=dedent("""\
+ pointplot : Show point estimates and confidence intervals using dots.\
+ """),
+ catplot=dedent("""\
+ catplot : Combine a categorical plot with a :class:`FacetGrid`.\
+ """),
+ boxenplot=dedent("""\
+ boxenplot : An enhanced boxplot for larger datasets.\
+ """),
+
+)
+
+_categorical_docs.update(_facet_docs)
+
+
+def boxplot(
+ data=None, *, x=None, y=None, hue=None, order=None, hue_order=None,
+ orient=None, color=None, palette=None, saturation=.75, fill=True,
+ dodge="auto", width=.8, gap=0, whis=1.5, linecolor="auto", linewidth=None,
+ fliersize=None, hue_norm=None, native_scale=False, log_scale=None, formatter=None,
+ legend="auto", ax=None, **kwargs
+):
+
+ p = _CategoricalPlotter(
+ data=data,
+ variables=dict(x=x, y=y, hue=hue),
+ order=order,
+ orient=orient,
+ color=color,
+ legend=legend,
+ )
+
+ if ax is None:
+ ax = plt.gca()
+
+ if p.plot_data.empty:
+ return ax
+
+ if dodge == "auto":
+ # Needs to be before scale_categorical changes the coordinate series dtype
+ dodge = p._dodge_needed()
+
+ if p.var_types.get(p.orient) == "categorical" or not native_scale:
+ p.scale_categorical(p.orient, order=order, formatter=formatter)
+
+ p._attach(ax, log_scale=log_scale)
+
+ # Deprecations to remove in v0.14.0.
+ hue_order = p._palette_without_hue_backcompat(palette, hue_order)
+ palette, hue_order = p._hue_backcompat(color, palette, hue_order)
+
+ saturation = saturation if fill else 1
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm, saturation=saturation)
+ color = _default_color(
+ ax.fill_between, hue, color,
+ {k: v for k, v in kwargs.items() if k in ["c", "color", "fc", "facecolor"]},
+ saturation=saturation,
+ )
+ linecolor = p._complement_color(linecolor, color, p._hue_map)
+
+ p.plot_boxes(
+ width=width,
+ dodge=dodge,
+ gap=gap,
+ fill=fill,
+ whis=whis,
+ color=color,
+ linecolor=linecolor,
+ linewidth=linewidth,
+ fliersize=fliersize,
+ plot_kws=kwargs,
+ )
+
+ p._add_axis_labels(ax)
+ p._adjust_cat_axis(ax, axis=p.orient)
+
+ return ax
+
+
+boxplot.__doc__ = dedent("""\
+ Draw a box plot to show distributions with respect to categories.
+
+ A box plot (or box-and-whisker plot) shows the distribution of quantitative
+ data in a way that facilitates comparisons between variables or across
+ levels of a categorical variable. The box shows the quartiles of the
+ dataset while the whiskers extend to show the rest of the distribution,
+ except for points that are determined to be "outliers" using a method
+ that is a function of the inter-quartile range.
+
+ {categorical_narrative}
+
+ Parameters
+ ----------
+ {categorical_data}
+ {input_params}
+ {order_vars}
+ {orient}
+ {color}
+ {palette}
+ {saturation}
+ {fill}
+ {dodge}
+ {width}
+ {gap}
+ whis : float or pair of floats
+ Paramater that controls whisker length. If scalar, whiskers are drawn
+ to the farthest datapoint within *whis * IQR* from the nearest hinge.
+ If a tuple, it is interpreted as percentiles that whiskers represent.
+ {linecolor}
+ {linewidth}
+ fliersize : float
+ Size of the markers used to indicate outlier observations.
+ {hue_norm}
+ {log_scale}
+ {native_scale}
+ {formatter}
+ {legend}
+ {ax_in}
+ kwargs : key, value mappings
+ Other keyword arguments are passed through to
+ :meth:`matplotlib.axes.Axes.boxplot`.
+
+ Returns
+ -------
+ {ax_out}
+
+ See Also
+ --------
+ {violinplot}
+ {stripplot}
+ {swarmplot}
+ {catplot}
+
+ Examples
+ --------
+ .. include:: ../docstrings/boxplot.rst
+
+ """).format(**_categorical_docs)
+
+
+def violinplot(
+ data=None, *, x=None, y=None, hue=None, order=None, hue_order=None,
+ orient=None, color=None, palette=None, saturation=.75, fill=True,
+ inner="box", split=False, width=.8, dodge="auto", gap=0,
+ linewidth=None, linecolor="auto", cut=2, gridsize=100,
+ bw_method="scott", bw_adjust=1, density_norm="area", common_norm=False,
+ hue_norm=None, formatter=None, log_scale=None, native_scale=False,
+ legend="auto", scale=deprecated, scale_hue=deprecated, bw=deprecated,
+ inner_kws=None, ax=None, **kwargs,
+):
+
+ p = _CategoricalPlotter(
+ data=data,
+ variables=dict(x=x, y=y, hue=hue),
+ order=order,
+ orient=orient,
+ color=color,
+ legend=legend,
+ )
+
+ if ax is None:
+ ax = plt.gca()
+
+ if p.plot_data.empty:
+ return ax
+
+ if dodge == "auto":
+ # Needs to be before scale_categorical changes the coordinate series dtype
+ dodge = p._dodge_needed()
+
+ if p.var_types.get(p.orient) == "categorical" or not native_scale:
+ p.scale_categorical(p.orient, order=order, formatter=formatter)
+
+ p._attach(ax, log_scale=log_scale)
+
+ # Deprecations to remove in v0.14.0.
+ hue_order = p._palette_without_hue_backcompat(palette, hue_order)
+ palette, hue_order = p._hue_backcompat(color, palette, hue_order)
+
+ saturation = saturation if fill else 1
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm, saturation=saturation)
+ color = _default_color(
+ ax.fill_between, hue, color,
+ {k: v for k, v in kwargs.items() if k in ["c", "color", "fc", "facecolor"]},
+ saturation=saturation,
+ )
+ linecolor = p._complement_color(linecolor, color, p._hue_map)
+
+ density_norm, common_norm = p._violin_scale_backcompat(
+ scale, scale_hue, density_norm, common_norm,
+ )
+
+ bw_method = p._violin_bw_backcompat(bw, bw_method)
+ kde_kws = dict(cut=cut, gridsize=gridsize, bw_method=bw_method, bw_adjust=bw_adjust)
+ inner_kws = {} if inner_kws is None else inner_kws.copy()
+
+ p.plot_violins(
+ width=width,
+ dodge=dodge,
+ gap=gap,
+ split=split,
+ color=color,
+ fill=fill,
+ linecolor=linecolor,
+ linewidth=linewidth,
+ inner=inner,
+ density_norm=density_norm,
+ common_norm=common_norm,
+ kde_kws=kde_kws,
+ inner_kws=inner_kws,
+ plot_kws=kwargs,
+ )
+
+ p._add_axis_labels(ax)
+ p._adjust_cat_axis(ax, axis=p.orient)
+
+ return ax
+
+
+violinplot.__doc__ = dedent("""\
+ Draw a patch representing a KDE and add observations or box plot statistics.
+
+ A violin plot plays a similar role as a box-and-whisker plot. It shows the
+ distribution of data points after grouping by one (or more) variables.
+ Unlike a box plot, each violin is drawn using a kernel density estimate
+ of the underlying distribution.
+
+ {categorical_narrative}
+
+ Parameters
+ ----------
+ {categorical_data}
+ {input_params}
+ {order_vars}
+ {orient}
+ {color}
+ {palette}
+ {saturation}
+ {fill}
+ inner : {{"box", "quart", "point", "stick", None}}
+ Representation of the data in the violin interior. One of the following:
+
+ - `"box"`: draw a miniature box-and-whisker plot
+ - `"quart"`: show the quartiles of the data
+ - `"point"` or `"stick"`: show each observation
+ split : bool
+ Show an un-mirrored distribution, alternating sides when using `hue`.
+
+ .. versionchanged:: v0.13.0
+ Previously, this option required a `hue` variable with exactly two levels.
+ {width}
+ {dodge}
+ {gap}
+ {linewidth}
+ {linecolor}
+ cut : float
+ Distance, in units of bandwidth, to extend the density past extreme
+ datapoints. Set to 0 to limit the violin within the data range.
+ gridsize : int
+ Number of points in the discrete grid used to evaluate the KDE.
+ bw_method : {{"scott", "silverman", float}}
+ Either the name of a reference rule or the scale factor to use when
+ computing the kernel bandwidth. The actual kernel size will be
+ determined by multiplying the scale factor by the standard deviation of
+ the data within each group.
+
+ .. versionadded:: v0.13.0
+ bw_adjust: float
+ Factor that scales the bandwidth to use more or less smoothing.
+
+ .. versionadded:: v0.13.0
+ density_norm : {{"area", "count", "width"}}
+ Method that normalizes each density to determine the violin's width.
+ If `area`, each violin will have the same area. If `count`, the width
+ will be proportional to the number of observations. If `width`, each
+ violin will have the same width.
+
+ .. versionadded:: v0.13.0
+ common_norm : bool
+ When `True`, normalize the density across all violins.
+
+ .. versionadded:: v0.13.0
+ {hue_norm}
+ {formatter}
+ {log_scale}
+ {native_scale}
+ {legend}
+ scale : {{"area", "count", "width"}}
+ .. deprecated:: v0.13.0
+ See `density_norm`.
+ scale_hue : bool
+ .. deprecated:: v0.13.0
+ See `common_norm`.
+ bw : {{'scott', 'silverman', float}}
+ .. deprecated:: v0.13.0
+ See `bw_method` and `bw_adjust`.
+ inner_kws : dict of key, value mappings
+ Keyword arguments for the "inner" plot, passed to one of:
+
+ - :class:`matplotlib.collections.LineCollection` (with `inner="stick"`)
+ - :meth:`matplotlib.axes.Axes.scatter` (with `inner="point"`)
+ - :meth:`matplotlib.axes.Axes.plot` (with `inner="quart"` or `inner="box"`)
+
+ Additionally, with `inner="box"`, the keywords `box_width`, `whis_width`,
+ and `marker` receive special handling for the components of the "box" plot.
+
+ .. versionadded:: v0.13.0
+ {ax_in}
+ kwargs : key, value mappings
+ Keyword arguments for the violin patches, passsed through to
+ :meth:`matplotlib.axes.Axes.fill_between`.
+
+ Returns
+ -------
+ {ax_out}
+
+ See Also
+ --------
+ {boxplot}
+ {stripplot}
+ {swarmplot}
+ {catplot}
+
+ Examples
+ --------
+ .. include:: ../docstrings/violinplot.rst
+
+ """).format(**_categorical_docs)
+
+
+def boxenplot(
+ data=None, *, x=None, y=None, hue=None, order=None, hue_order=None,
+ orient=None, color=None, palette=None, saturation=.75, fill=True,
+ dodge="auto", width=.8, gap=0, linewidth=None, linecolor=None,
+ width_method="exponential", k_depth="tukey", outlier_prop=0.007, trust_alpha=0.05,
+ showfliers=True, hue_norm=None, log_scale=None, native_scale=False, formatter=None,
+ legend="auto", scale=deprecated, box_kws=None, flier_kws=None, line_kws=None,
+ ax=None, **kwargs,
+):
+
+ p = _CategoricalPlotter(
+ data=data,
+ variables=dict(x=x, y=y, hue=hue),
+ order=order,
+ orient=orient,
+ color=color,
+ legend=legend,
+ )
+
+ if ax is None:
+ ax = plt.gca()
+
+ if p.plot_data.empty:
+ return ax
+
+ if dodge == "auto":
+ # Needs to be before scale_categorical changes the coordinate series dtype
+ dodge = p._dodge_needed()
+
+ if p.var_types.get(p.orient) == "categorical" or not native_scale:
+ p.scale_categorical(p.orient, order=order, formatter=formatter)
+
+ p._attach(ax, log_scale=log_scale)
+
+ # Deprecations to remove in v0.14.0.
+ hue_order = p._palette_without_hue_backcompat(palette, hue_order)
+ palette, hue_order = p._hue_backcompat(color, palette, hue_order)
+
+ # Longer-term deprecations
+ width_method = p._boxen_scale_backcompat(scale, width_method)
+
+ saturation = saturation if fill else 1
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm, saturation=saturation)
+ color = _default_color(
+ ax.fill_between, hue, color,
+ {}, # TODO how to get default color?
+ # {k: v for k, v in kwargs.items() if k in ["c", "color", "fc", "facecolor"]},
+ saturation=saturation,
+ )
+ linecolor = p._complement_color(linecolor, color, p._hue_map)
+
+ p.plot_boxens(
+ width=width,
+ dodge=dodge,
+ gap=gap,
+ fill=fill,
+ color=color,
+ linecolor=linecolor,
+ linewidth=linewidth,
+ width_method=width_method,
+ k_depth=k_depth,
+ outlier_prop=outlier_prop,
+ trust_alpha=trust_alpha,
+ showfliers=showfliers,
+ box_kws=box_kws,
+ flier_kws=flier_kws,
+ line_kws=line_kws,
+ plot_kws=kwargs,
+ )
+
+ p._add_axis_labels(ax)
+ p._adjust_cat_axis(ax, axis=p.orient)
+
+ return ax
+
+
+boxenplot.__doc__ = dedent("""\
+ Draw an enhanced box plot for larger datasets.
+
+ This style of plot was originally named a "letter value" plot because it
+ shows a large number of quantiles that are defined as "letter values". It
+ is similar to a box plot in plotting a nonparametric representation of a
+ distribution in which all features correspond to actual observations. By
+ plotting more quantiles, it provides more information about the shape of
+ the distribution, particularly in the tails.
+
+ {categorical_narrative}
+
+ Parameters
+ ----------
+ {categorical_data}
+ {input_params}
+ {order_vars}
+ {orient}
+ {color}
+ {palette}
+ {saturation}
+ {fill}
+ {dodge}
+ {width}
+ {gap}
+ {linewidth}
+ {linecolor}
+ width_method : {{"exponential", "linear", "area"}}
+ Method to use for the width of the letter value boxes:
+
+ - `"exponential"`: Represent the corresponding percentile
+ - `"linear"`: Decrease by a constant amount for each box
+ - `"area"`: Represent the density of data points in that box
+ k_depth : {{"tukey", "proportion", "trustworthy", "full"}} or int
+ The number of levels to compute and draw in each tail:
+
+ - `"tukey"`: Use log2(n) - 3 levels, covering similar range as boxplot whiskers
+ - `"proportion"`: Leave approximately `outlier_prop` fliers
+ - `"trusthworthy"`: Extend to level with confidence of at least `trust_alpha`
+ - `"full"`: Use log2(n) + 1 levels and extend to most extreme points
+ outlier_prop : float
+ Proportion of data expected to be outliers; used when `k_depth="proportion"`.
+ trust_alpha : float
+ Confidence threshold for most extreme level; used when `k_depth="trustworthy"`.
+ showfliers : bool
+ If False, suppress the plotting of outliers.
+ {hue_norm}
+ {log_scale}
+ {native_scale}
+ {formatter}
+ {legend}
+ box_kws: dict
+ Keyword arguments for the box artists; passed to
+ :class:`matplotlib.patches.Rectangle`.
+
+ .. versionadded:: v0.12.0
+ line_kws: dict
+ Keyword arguments for the line denoting the median; passed to
+ :meth:`matplotlib.axes.Axes.plot`.
+
+ .. versionadded:: v0.12.0
+ flier_kws: dict
+ Keyword arguments for the scatter denoting the outlier observations;
+ passed to :meth:`matplotlib.axes.Axes.scatter`.
+
+ .. versionadded:: v0.12.0
+ {ax_in}
+ kwargs : key, value mappings
+ Other keyword arguments are passed to :class:`matplotlib.patches.Rectangle`,
+ superceded by those in `box_kws`.
+
+ Returns
+ -------
+ {ax_out}
+
+ See Also
+ --------
+ {violinplot}
+ {boxplot}
+ {catplot}
+
+ Notes
+ -----
+
+ For a more extensive explanation, you can read the paper that introduced the plot:
+ https://vita.had.co.nz/papers/letter-value-plot.html
+
+ Examples
+ --------
+ .. include:: ../docstrings/boxenplot.rst
+
+ """).format(**_categorical_docs)
+
+
+def stripplot(
+ data=None, *, x=None, y=None, hue=None, order=None, hue_order=None,
+ jitter=True, dodge=False, orient=None, color=None, palette=None,
+ size=5, edgecolor=default, linewidth=0,
+ hue_norm=None, log_scale=None, native_scale=False, formatter=None, legend="auto",
+ ax=None, **kwargs
+):
+
+ p = _CategoricalPlotter(
+ data=data,
+ variables=dict(x=x, y=y, hue=hue),
+ order=order,
+ orient=orient,
+ color=color,
+ legend=legend,
+ )
+
+ if ax is None:
+ ax = plt.gca()
+
+ if p.plot_data.empty:
+ return ax
+
+ if p.var_types.get(p.orient) == "categorical" or not native_scale:
+ p.scale_categorical(p.orient, order=order, formatter=formatter)
+
+ p._attach(ax, log_scale=log_scale)
+
+ # Deprecations to remove in v0.14.0.
+ hue_order = p._palette_without_hue_backcompat(palette, hue_order)
+ palette, hue_order = p._hue_backcompat(color, palette, hue_order)
+
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm)
+ color = _default_color(ax.scatter, hue, color, kwargs)
+ edgecolor = p._complement_color(edgecolor, color, p._hue_map)
+
+ kwargs.setdefault("zorder", 3)
+ size = kwargs.get("s", size)
+
+ kwargs.update(
+ s=size ** 2,
+ edgecolor=edgecolor,
+ linewidth=linewidth,
+ )
+
+ p.plot_strips(
+ jitter=jitter,
+ dodge=dodge,
+ color=color,
+ plot_kws=kwargs,
+ )
+
+ # XXX this happens inside a plotting method in the distribution plots
+ # but maybe it's better out here? Alternatively, we have an open issue
+ # suggesting that _attach could add default axes labels, which seems smart.
+ p._add_axis_labels(ax)
+ p._adjust_cat_axis(ax, axis=p.orient)
+
+ return ax
+
+
+stripplot.__doc__ = dedent("""\
+ Draw a categorical scatterplot using jitter to reduce overplotting.
+
+ A strip plot can be drawn on its own, but it is also a good complement
+ to a box or violin plot in cases where you want to show all observations
+ along with some representation of the underlying distribution.
+
+ {categorical_narrative}
+
+ Parameters
+ ----------
+ {categorical_data}
+ {input_params}
+ {order_vars}
+ jitter : float, `True`/`1` is special-cased
+ Amount of jitter (only along the categorical axis) to apply. This
+ can be useful when you have many points and they overlap, so that
+ it is easier to see the distribution. You can specify the amount
+ of jitter (half the width of the uniform random variable support),
+ or use `True` for a good default.
+ dodge : bool
+ When a `hue` variable is assigned, setting this to `True` will
+ separate the strips for different hue levels along the categorical
+ axis and narrow the amount of space allotedto each strip. Otherwise,
+ the points for each level will be plotted in the same strip.
+ {orient}
+ {color}
+ {palette}
+ size : float
+ Radius of the markers, in points.
+ edgecolor : matplotlib color, "gray" is special-cased
+ Color of the lines around each point. If you pass `"gray"`, the
+ brightness is determined by the color palette used for the body
+ of the points. Note that `stripplot` has `linewidth=0` by default,
+ so edge colors are only visible with nonzero line width.
+ {linewidth}
+ {hue_norm}
+ {log_scale}
+ {native_scale}
+ {formatter}
+ {legend}
+ {ax_in}
+ kwargs : key, value mappings
+ Other keyword arguments are passed through to
+ :meth:`matplotlib.axes.Axes.scatter`.
+
+ Returns
+ -------
+ {ax_out}
+
+ See Also
+ --------
+ {swarmplot}
+ {boxplot}
+ {violinplot}
+ {catplot}
+
+ Examples
+ --------
+ .. include:: ../docstrings/stripplot.rst
+
+ """).format(**_categorical_docs)
+
+
+def swarmplot(
+ data=None, *, x=None, y=None, hue=None, order=None, hue_order=None,
+ dodge=False, orient=None, color=None, palette=None,
+ size=5, edgecolor=None, linewidth=0, hue_norm=None, log_scale=None,
+ native_scale=False, formatter=None, legend="auto", warn_thresh=.05,
+ ax=None, **kwargs
+):
+
+ p = _CategoricalPlotter(
+ data=data,
+ variables=dict(x=x, y=y, hue=hue),
+ order=order,
+ orient=orient,
+ color=color,
+ legend=legend,
+ )
+
+ if ax is None:
+ ax = plt.gca()
+
+ if p.plot_data.empty:
+ return ax
+
+ if p.var_types.get(p.orient) == "categorical" or not native_scale:
+ p.scale_categorical(p.orient, order=order, formatter=formatter)
+
+ p._attach(ax, log_scale=log_scale)
+
+ if not p.has_xy_data:
+ return ax
+
+ # Deprecations to remove in v0.14.0.
+ hue_order = p._palette_without_hue_backcompat(palette, hue_order)
+ palette, hue_order = p._hue_backcompat(color, palette, hue_order)
+
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm)
+ color = _default_color(ax.scatter, hue, color, kwargs)
+ edgecolor = p._complement_color(edgecolor, color, p._hue_map)
+
+ kwargs.setdefault("zorder", 3)
+ size = kwargs.get("s", size)
+
+ if linewidth is None:
+ linewidth = size / 10
+
+ kwargs.update(dict(
+ s=size ** 2,
+ edgecolor=edgecolor,
+ linewidth=linewidth,
+ ))
+
+ p.plot_swarms(
+ dodge=dodge,
+ color=color,
+ warn_thresh=warn_thresh,
+ plot_kws=kwargs,
+ )
+
+ p._add_axis_labels(ax)
+ p._adjust_cat_axis(ax, axis=p.orient)
+
+ return ax
+
+
+swarmplot.__doc__ = dedent("""\
+ Draw a categorical scatterplot with points adjusted to be non-overlapping.
+
+ This function is similar to :func:`stripplot`, but the points are adjusted
+ (only along the categorical axis) so that they don't overlap. This gives a
+ better representation of the distribution of values, but it does not scale
+ well to large numbers of observations. This style of plot is sometimes
+ called a "beeswarm".
+
+ A swarm plot can be drawn on its own, but it is also a good complement
+ to a box or violin plot in cases where you want to show all observations
+ along with some representation of the underlying distribution.
+
+ {categorical_narrative}
+
+ Parameters
+ ----------
+ {categorical_data}
+ {input_params}
+ {order_vars}
+ dodge : bool
+ When a `hue` variable is assigned, setting this to `True` will
+ separate the swarms for different hue levels along the categorical
+ axis and narrow the amount of space allotedto each strip. Otherwise,
+ the points for each level will be plotted in the same swarm.
+ {orient}
+ {color}
+ {palette}
+ size : float
+ Radius of the markers, in points.
+ edgecolor : matplotlib color, "gray" is special-cased
+ Color of the lines around each point. If you pass `"gray"`, the
+ brightness is determined by the color palette used for the body
+ of the points.
+ {linewidth}
+ {log_scale}
+ {native_scale}
+ {formatter}
+ {legend}
+ {ax_in}
+ kwargs : key, value mappings
+ Other keyword arguments are passed through to
+ :meth:`matplotlib.axes.Axes.scatter`.
+
+ Returns
+ -------
+ {ax_out}
+
+ See Also
+ --------
+ {boxplot}
+ {violinplot}
+ {stripplot}
+ {catplot}
+
+ Examples
+ --------
+ .. include:: ../docstrings/swarmplot.rst
+
+ """).format(**_categorical_docs)
+
+
+def barplot(
+ data=None, *, x=None, y=None, hue=None, order=None, hue_order=None,
+ estimator="mean", errorbar=("ci", 95), n_boot=1000, seed=None, units=None,
+ weights=None, orient=None, color=None, palette=None, saturation=.75,
+ fill=True, hue_norm=None, width=.8, dodge="auto", gap=0, log_scale=None,
+ native_scale=False, formatter=None, legend="auto", capsize=0, err_kws=None,
+ ci=deprecated, errcolor=deprecated, errwidth=deprecated, ax=None, **kwargs,
+):
+
+ errorbar = utils._deprecate_ci(errorbar, ci)
+
+ # Be backwards compatible with len passed directly, which
+ # does not work in Series.agg (maybe a pandas bug?)
+ if estimator is len:
+ estimator = "size"
+
+ p = _CategoricalAggPlotter(
+ data=data,
+ variables=dict(x=x, y=y, hue=hue, units=units, weight=weights),
+ order=order,
+ orient=orient,
+ color=color,
+ legend=legend,
+ )
+
+ if ax is None:
+ ax = plt.gca()
+
+ if p.plot_data.empty:
+ return ax
+
+ if dodge == "auto":
+ # Needs to be before scale_categorical changes the coordinate series dtype
+ dodge = p._dodge_needed()
+
+ if p.var_types.get(p.orient) == "categorical" or not native_scale:
+ p.scale_categorical(p.orient, order=order, formatter=formatter)
+
+ p._attach(ax, log_scale=log_scale)
+
+ # Deprecations to remove in v0.14.0.
+ hue_order = p._palette_without_hue_backcompat(palette, hue_order)
+ palette, hue_order = p._hue_backcompat(color, palette, hue_order)
+
+ saturation = saturation if fill else 1
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm, saturation=saturation)
+ color = _default_color(ax.bar, hue, color, kwargs, saturation=saturation)
+
+ agg_cls = WeightedAggregator if "weight" in p.plot_data else EstimateAggregator
+ aggregator = agg_cls(estimator, errorbar, n_boot=n_boot, seed=seed)
+ err_kws = {} if err_kws is None else normalize_kwargs(err_kws, mpl.lines.Line2D)
+
+ # Deprecations to remove in v0.15.0.
+ err_kws, capsize = p._err_kws_backcompat(err_kws, errcolor, errwidth, capsize)
+
+ p.plot_bars(
+ aggregator=aggregator,
+ dodge=dodge,
+ width=width,
+ gap=gap,
+ color=color,
+ fill=fill,
+ capsize=capsize,
+ err_kws=err_kws,
+ plot_kws=kwargs,
+ )
+
+ p._add_axis_labels(ax)
+ p._adjust_cat_axis(ax, axis=p.orient)
+
+ return ax
+
+
+barplot.__doc__ = dedent("""\
+ Show point estimates and errors as rectangular bars.
+
+ A bar plot represents an aggregate or statistical estimate for a numeric
+ variable with the height of each rectangle and indicates the uncertainty
+ around that estimate using an error bar. Bar plots include 0 in the
+ axis range, and they are a good choice when 0 is a meaningful value
+ for the variable to take.
+
+ {categorical_narrative}
+
+ Parameters
+ ----------
+ {categorical_data}
+ {input_params}
+ {order_vars}
+ {stat_api_params}
+ {orient}
+ {color}
+ {palette}
+ {saturation}
+ {fill}
+ {hue_norm}
+ {width}
+ {dodge}
+ {gap}
+ {log_scale}
+ {native_scale}
+ {formatter}
+ {legend}
+ {capsize}
+ {err_kws}
+ {ci}
+ {errcolor}
+ {errwidth}
+ {ax_in}
+ kwargs : key, value mappings
+ Other parameters are passed through to :class:`matplotlib.patches.Rectangle`.
+
+ Returns
+ -------
+ {ax_out}
+
+ See Also
+ --------
+ {countplot}
+ {pointplot}
+ {catplot}
+
+ Notes
+ -----
+
+ For datasets where 0 is not a meaningful value, a :func:`pointplot` will
+ allow you to focus on differences between levels of one or more categorical
+ variables.
+
+ It is also important to keep in mind that a bar plot shows only the mean (or
+ other aggregate) value, but it is often more informative to show the
+ distribution of values at each level of the categorical variables. In those
+ cases, approaches such as a :func:`boxplot` or :func:`violinplot` may be
+ more appropriate.
+
+ Examples
+ --------
+ .. include:: ../docstrings/barplot.rst
+
+ """).format(**_categorical_docs)
+
+
+def pointplot(
+ data=None, *, x=None, y=None, hue=None, order=None, hue_order=None,
+ estimator="mean", errorbar=("ci", 95), n_boot=1000, seed=None, units=None,
+ weights=None, color=None, palette=None, hue_norm=None, markers=default,
+ linestyles=default, dodge=False, log_scale=None, native_scale=False,
+ orient=None, capsize=0, formatter=None, legend="auto", err_kws=None,
+ ci=deprecated, errwidth=deprecated, join=deprecated, scale=deprecated,
+ ax=None, **kwargs,
+):
+
+ errorbar = utils._deprecate_ci(errorbar, ci)
+
+ p = _CategoricalAggPlotter(
+ data=data,
+ variables=dict(x=x, y=y, hue=hue, units=units, weight=weights),
+ order=order,
+ orient=orient,
+ # Handle special backwards compatibility where pointplot originally
+ # did *not* default to multi-colored unless a palette was specified.
+ color="C0" if (color is None and palette is None) else color,
+ legend=legend,
+ )
+
+ if ax is None:
+ ax = plt.gca()
+
+ if p.plot_data.empty:
+ return ax
+
+ if p.var_types.get(p.orient) == "categorical" or not native_scale:
+ p.scale_categorical(p.orient, order=order, formatter=formatter)
+
+ p._attach(ax, log_scale=log_scale)
+
+ # Deprecations to remove in v0.14.0.
+ hue_order = p._palette_without_hue_backcompat(palette, hue_order)
+ palette, hue_order = p._hue_backcompat(color, palette, hue_order)
+
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm)
+ color = _default_color(ax.plot, hue, color, kwargs)
+
+ agg_cls = WeightedAggregator if "weight" in p.plot_data else EstimateAggregator
+ aggregator = agg_cls(estimator, errorbar, n_boot=n_boot, seed=seed)
+ err_kws = {} if err_kws is None else normalize_kwargs(err_kws, mpl.lines.Line2D)
+
+ # Deprecations to remove in v0.15.0.
+ p._point_kwargs_backcompat(scale, join, kwargs)
+ err_kws, capsize = p._err_kws_backcompat(err_kws, None, errwidth, capsize)
+
+ p.plot_points(
+ aggregator=aggregator,
+ markers=markers,
+ linestyles=linestyles,
+ dodge=dodge,
+ color=color,
+ capsize=capsize,
+ err_kws=err_kws,
+ plot_kws=kwargs,
+ )
+
+ p._add_axis_labels(ax)
+ p._adjust_cat_axis(ax, axis=p.orient)
+
+ return ax
+
+
+pointplot.__doc__ = dedent("""\
+ Show point estimates and errors using lines with markers.
+
+ A point plot represents an estimate of central tendency for a numeric
+ variable by the position of the dot and provides some indication of the
+ uncertainty around that estimate using error bars.
+
+ Point plots can be more useful than bar plots for focusing comparisons
+ between different levels of one or more categorical variables. They are
+ particularly adept at showing interactions: how the relationship between
+ levels of one categorical variable changes across levels of a second
+ categorical variable. The lines that join each point from the same `hue`
+ level allow interactions to be judged by differences in slope, which is
+ easier for the eyes than comparing the heights of several groups of points
+ or bars.
+
+ {categorical_narrative}
+
+ Parameters
+ ----------
+ {categorical_data}
+ {input_params}
+ {order_vars}
+ {stat_api_params}
+ {color}
+ {palette}
+ markers : string or list of strings
+ Markers to use for each of the `hue` levels.
+ linestyles : string or list of strings
+ Line styles to use for each of the `hue` levels.
+ dodge : bool or float
+ Amount to separate the points for each level of the `hue` variable along
+ the categorical axis. Setting to `True` will apply a small default.
+ {log_scale}
+ {native_scale}
+ {orient}
+ {capsize}
+ {formatter}
+ {legend}
+ {err_kws}
+ {ci}
+ {errwidth}
+ join : bool
+ If `True`, connect point estimates with a line.
+
+ .. deprecated:: v0.13.0
+ Set `linestyle="none"` to remove the lines between the points.
+ scale : float
+ Scale factor for the plot elements.
+
+ .. deprecated:: v0.13.0
+ Control element sizes with :class:`matplotlib.lines.Line2D` parameters.
+ {ax_in}
+ kwargs : key, value mappings
+ Other parameters are passed through to :class:`matplotlib.lines.Line2D`.
+
+ .. versionadded:: v0.13.0
+
+ Returns
+ -------
+ {ax_out}
+
+ See Also
+ --------
+ {barplot}
+ {catplot}
+
+ Notes
+ -----
+ It is important to keep in mind that a point plot shows only the mean (or
+ other estimator) value, but in many cases it may be more informative to
+ show the distribution of values at each level of the categorical variables.
+ In that case, other approaches such as a box or violin plot may be more
+ appropriate.
+
+ Examples
+ --------
+ .. include:: ../docstrings/pointplot.rst
+
+ """).format(**_categorical_docs)
+
+
+def countplot(
+ data=None, *, x=None, y=None, hue=None, order=None, hue_order=None,
+ orient=None, color=None, palette=None, saturation=.75, fill=True, hue_norm=None,
+ stat="count", width=.8, dodge="auto", gap=0, log_scale=None, native_scale=False,
+ formatter=None, legend="auto", ax=None, **kwargs
+):
+
+ if x is None and y is not None:
+ orient = "y"
+ x = 1 if list(y) else None
+ elif x is not None and y is None:
+ orient = "x"
+ y = 1 if list(x) else None
+ elif x is not None and y is not None:
+ raise TypeError("Cannot pass values for both `x` and `y`.")
+
+ p = _CategoricalAggPlotter(
+ data=data,
+ variables=dict(x=x, y=y, hue=hue),
+ order=order,
+ orient=orient,
+ color=color,
+ legend=legend,
+ )
+
+ if ax is None:
+ ax = plt.gca()
+
+ if p.plot_data.empty:
+ return ax
+
+ if dodge == "auto":
+ # Needs to be before scale_categorical changes the coordinate series dtype
+ dodge = p._dodge_needed()
+
+ if p.var_types.get(p.orient) == "categorical" or not native_scale:
+ p.scale_categorical(p.orient, order=order, formatter=formatter)
+
+ p._attach(ax, log_scale=log_scale)
+
+ # Deprecations to remove in v0.14.0.
+ hue_order = p._palette_without_hue_backcompat(palette, hue_order)
+ palette, hue_order = p._hue_backcompat(color, palette, hue_order)
+
+ saturation = saturation if fill else 1
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm, saturation=saturation)
+ color = _default_color(ax.bar, hue, color, kwargs, saturation)
+
+ count_axis = {"x": "y", "y": "x"}[p.orient]
+ if p.input_format == "wide":
+ p.plot_data[count_axis] = 1
+
+ _check_argument("stat", ["count", "percent", "probability", "proportion"], stat)
+ p.variables[count_axis] = stat
+ if stat != "count":
+ denom = 100 if stat == "percent" else 1
+ p.plot_data[count_axis] /= len(p.plot_data) / denom
+
+ aggregator = EstimateAggregator("sum", errorbar=None)
+
+ p.plot_bars(
+ aggregator=aggregator,
+ dodge=dodge,
+ width=width,
+ gap=gap,
+ color=color,
+ fill=fill,
+ capsize=0,
+ err_kws={},
+ plot_kws=kwargs,
+ )
+
+ p._add_axis_labels(ax)
+ p._adjust_cat_axis(ax, axis=p.orient)
+
+ return ax
+
+
+countplot.__doc__ = dedent("""\
+ Show the counts of observations in each categorical bin using bars.
+
+ A count plot can be thought of as a histogram across a categorical, instead
+ of quantitative, variable. The basic API and options are identical to those
+ for :func:`barplot`, so you can compare counts across nested variables.
+
+ Note that :func:`histplot` function offers similar functionality with additional
+ features (e.g. bar stacking), although its default behavior is somewhat different.
+
+ {categorical_narrative}
+
+ Parameters
+ ----------
+ {categorical_data}
+ {input_params}
+ {order_vars}
+ {orient}
+ {color}
+ {palette}
+ {saturation}
+ {hue_norm}
+ stat : {{'count', 'percent', 'proportion', 'probability'}}
+ Statistic to compute; when not `'count'`, bar heights will be normalized so that
+ they sum to 100 (for `'percent'`) or 1 (otherwise) across the plot.
+
+ .. versionadded:: v0.13.0
+ {width}
+ {dodge}
+ {log_scale}
+ {native_scale}
+ {formatter}
+ {legend}
+ {ax_in}
+ kwargs : key, value mappings
+ Other parameters are passed through to :class:`matplotlib.patches.Rectangle`.
+
+ Returns
+ -------
+ {ax_out}
+
+ See Also
+ --------
+ histplot : Bin and count observations with additional options.
+ {barplot}
+ {catplot}
+
+ Examples
+ --------
+ .. include:: ../docstrings/countplot.rst
+
+ """).format(**_categorical_docs)
+
+
+def catplot(
+ data=None, *, x=None, y=None, hue=None, row=None, col=None, kind="strip",
+ estimator="mean", errorbar=("ci", 95), n_boot=1000, seed=None, units=None,
+ weights=None, order=None, hue_order=None, row_order=None, col_order=None,
+ col_wrap=None, height=5, aspect=1, log_scale=None, native_scale=False,
+ formatter=None, orient=None, color=None, palette=None, hue_norm=None,
+ legend="auto", legend_out=True, sharex=True, sharey=True,
+ margin_titles=False, facet_kws=None, ci=deprecated, **kwargs
+):
+
+ # Check for attempt to plot onto specific axes and warn
+ if "ax" in kwargs:
+ msg = ("catplot is a figure-level function and does not accept "
+ f"target axes. You may wish to try {kind}plot")
+ warnings.warn(msg, UserWarning)
+ kwargs.pop("ax")
+
+ desaturated_kinds = ["bar", "count", "box", "violin", "boxen"]
+ undodged_kinds = ["strip", "swarm", "point"]
+
+ if kind in ["bar", "point", "count"]:
+ Plotter = _CategoricalAggPlotter
+ else:
+ Plotter = _CategoricalPlotter
+
+ if kind == "count":
+ if x is None and y is not None:
+ orient = "y"
+ x = 1
+ elif x is not None and y is None:
+ orient = "x"
+ y = 1
+ elif x is not None and y is not None:
+ raise ValueError("Cannot pass values for both `x` and `y`.")
+
+ p = Plotter(
+ data=data,
+ variables=dict(
+ x=x, y=y, hue=hue, row=row, col=col, units=units, weight=weights
+ ),
+ order=order,
+ orient=orient,
+ # Handle special backwards compatibility where pointplot originally
+ # did *not* default to multi-colored unless a palette was specified.
+ color="C0" if kind == "point" and palette is None and color is None else color,
+ legend=legend,
+ )
+
+ for var in ["row", "col"]:
+ # Handle faceting variables that lack name information
+ if var in p.variables and p.variables[var] is None:
+ p.variables[var] = f"_{var}_"
+
+ # Adapt the plot_data dataframe for use with FacetGrid
+ facet_data = p.plot_data.rename(columns=p.variables)
+ facet_data = facet_data.loc[:, ~facet_data.columns.duplicated()]
+
+ col_name = p.variables.get("col", None)
+ row_name = p.variables.get("row", None)
+
+ if facet_kws is None:
+ facet_kws = {}
+
+ g = FacetGrid(
+ data=facet_data, row=row_name, col=col_name, col_wrap=col_wrap,
+ row_order=row_order, col_order=col_order, sharex=sharex, sharey=sharey,
+ legend_out=legend_out, margin_titles=margin_titles,
+ height=height, aspect=aspect,
+ **facet_kws,
+ )
+
+ # Capture this here because scale_categorical is going to insert a (null)
+ # x variable even if it is empty. It's not clear whether that needs to
+ # happen or if disabling that is the cleaner solution.
+ has_xy_data = p.has_xy_data
+
+ if not native_scale or p.var_types[p.orient] == "categorical":
+ p.scale_categorical(p.orient, order=order, formatter=formatter)
+
+ p._attach(g, log_scale=log_scale)
+
+ if not has_xy_data:
+ return g
+
+ # Deprecations to remove in v0.14.0.
+ hue_order = p._palette_without_hue_backcompat(palette, hue_order)
+ palette, hue_order = p._hue_backcompat(color, palette, hue_order)
+
+ # Othe deprecations
+ errorbar = utils._deprecate_ci(errorbar, ci)
+
+ saturation = kwargs.pop(
+ "saturation",
+ 0.75 if kind in desaturated_kinds and kwargs.get("fill", True) else 1
+ )
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm, saturation=saturation)
+
+ # Set a default color
+ # Otherwise each artist will be plotted separately and trip the color cycle
+ if hue is None:
+ color = "C0" if color is None else color
+ if saturation < 1:
+ color = desaturate(color, saturation)
+
+ if kind in ["strip", "swarm"]:
+ kwargs = normalize_kwargs(kwargs, mpl.collections.PathCollection)
+ kwargs["edgecolor"] = p._complement_color(
+ kwargs.pop("edgecolor", default), color, p._hue_map
+ )
+
+ width = kwargs.pop("width", 0.8)
+ dodge = kwargs.pop("dodge", False if kind in undodged_kinds else "auto")
+ if dodge == "auto":
+ dodge = p._dodge_needed()
+
+ if "weight" in p.plot_data:
+ if kind not in ["bar", "point"]:
+ msg = f"The `weights` parameter has no effect with kind={kind!r}."
+ warnings.warn(msg, stacklevel=2)
+ agg_cls = WeightedAggregator
+ else:
+ agg_cls = EstimateAggregator
+
+ if kind == "strip":
+
+ jitter = kwargs.pop("jitter", True)
+ plot_kws = kwargs.copy()
+ plot_kws.setdefault("zorder", 3)
+ plot_kws.setdefault("linewidth", 0)
+ if "s" not in plot_kws:
+ plot_kws["s"] = plot_kws.pop("size", 5) ** 2
+
+ p.plot_strips(
+ jitter=jitter,
+ dodge=dodge,
+ color=color,
+ plot_kws=plot_kws,
+ )
+
+ elif kind == "swarm":
+
+ warn_thresh = kwargs.pop("warn_thresh", .05)
+ plot_kws = kwargs.copy()
+ plot_kws.setdefault("zorder", 3)
+ if "s" not in plot_kws:
+ plot_kws["s"] = plot_kws.pop("size", 5) ** 2
+
+ if plot_kws.setdefault("linewidth", 0) is None:
+ plot_kws["linewidth"] = np.sqrt(plot_kws["s"]) / 10
+
+ p.plot_swarms(
+ dodge=dodge,
+ color=color,
+ warn_thresh=warn_thresh,
+ plot_kws=plot_kws,
+ )
+
+ elif kind == "box":
+
+ plot_kws = kwargs.copy()
+ gap = plot_kws.pop("gap", 0)
+ fill = plot_kws.pop("fill", True)
+ whis = plot_kws.pop("whis", 1.5)
+ linewidth = plot_kws.pop("linewidth", None)
+ fliersize = plot_kws.pop("fliersize", 5)
+ linecolor = p._complement_color(
+ plot_kws.pop("linecolor", "auto"), color, p._hue_map
+ )
+
+ p.plot_boxes(
+ width=width,
+ dodge=dodge,
+ gap=gap,
+ fill=fill,
+ whis=whis,
+ color=color,
+ linecolor=linecolor,
+ linewidth=linewidth,
+ fliersize=fliersize,
+ plot_kws=plot_kws,
+ )
+
+ elif kind == "violin":
+
+ plot_kws = kwargs.copy()
+ gap = plot_kws.pop("gap", 0)
+ fill = plot_kws.pop("fill", True)
+ split = plot_kws.pop("split", False)
+ inner = plot_kws.pop("inner", "box")
+ density_norm = plot_kws.pop("density_norm", "area")
+ common_norm = plot_kws.pop("common_norm", False)
+
+ scale = plot_kws.pop("scale", deprecated)
+ scale_hue = plot_kws.pop("scale_hue", deprecated)
+ density_norm, common_norm = p._violin_scale_backcompat(
+ scale, scale_hue, density_norm, common_norm,
+ )
+
+ bw_method = p._violin_bw_backcompat(
+ plot_kws.pop("bw", deprecated), plot_kws.pop("bw_method", "scott")
+ )
+ kde_kws = dict(
+ cut=plot_kws.pop("cut", 2),
+ gridsize=plot_kws.pop("gridsize", 100),
+ bw_adjust=plot_kws.pop("bw_adjust", 1),
+ bw_method=bw_method,
+ )
+
+ inner_kws = plot_kws.pop("inner_kws", {}).copy()
+ linewidth = plot_kws.pop("linewidth", None)
+ linecolor = plot_kws.pop("linecolor", "auto")
+ linecolor = p._complement_color(linecolor, color, p._hue_map)
+
+ p.plot_violins(
+ width=width,
+ dodge=dodge,
+ gap=gap,
+ split=split,
+ color=color,
+ fill=fill,
+ linecolor=linecolor,
+ linewidth=linewidth,
+ inner=inner,
+ density_norm=density_norm,
+ common_norm=common_norm,
+ kde_kws=kde_kws,
+ inner_kws=inner_kws,
+ plot_kws=plot_kws,
+ )
+
+ elif kind == "boxen":
+
+ plot_kws = kwargs.copy()
+ gap = plot_kws.pop("gap", 0)
+ fill = plot_kws.pop("fill", True)
+ linecolor = plot_kws.pop("linecolor", "auto")
+ linewidth = plot_kws.pop("linewidth", None)
+ k_depth = plot_kws.pop("k_depth", "tukey")
+ width_method = plot_kws.pop("width_method", "exponential")
+ outlier_prop = plot_kws.pop("outlier_prop", 0.007)
+ trust_alpha = plot_kws.pop("trust_alpha", 0.05)
+ showfliers = plot_kws.pop("showfliers", True)
+ box_kws = plot_kws.pop("box_kws", {})
+ flier_kws = plot_kws.pop("flier_kws", {})
+ line_kws = plot_kws.pop("line_kws", {})
+ if "scale" in plot_kws:
+ width_method = p._boxen_scale_backcompat(
+ plot_kws["scale"], width_method
+ )
+ linecolor = p._complement_color(linecolor, color, p._hue_map)
+
+ p.plot_boxens(
+ width=width,
+ dodge=dodge,
+ gap=gap,
+ fill=fill,
+ color=color,
+ linecolor=linecolor,
+ linewidth=linewidth,
+ width_method=width_method,
+ k_depth=k_depth,
+ outlier_prop=outlier_prop,
+ trust_alpha=trust_alpha,
+ showfliers=showfliers,
+ box_kws=box_kws,
+ flier_kws=flier_kws,
+ line_kws=line_kws,
+ plot_kws=plot_kws,
+ )
+
+ elif kind == "point":
+
+ aggregator = agg_cls(estimator, errorbar, n_boot=n_boot, seed=seed)
+
+ markers = kwargs.pop("markers", default)
+ linestyles = kwargs.pop("linestyles", default)
+
+ # Deprecations to remove in v0.15.0.
+ # TODO Uncomment when removing deprecation backcompat
+ # capsize = kwargs.pop("capsize", 0)
+ # err_kws = normalize_kwargs(kwargs.pop("err_kws", {}), mpl.lines.Line2D)
+ p._point_kwargs_backcompat(
+ kwargs.pop("scale", deprecated),
+ kwargs.pop("join", deprecated),
+ kwargs
+ )
+ err_kws, capsize = p._err_kws_backcompat(
+ normalize_kwargs(kwargs.pop("err_kws", {}), mpl.lines.Line2D),
+ None,
+ errwidth=kwargs.pop("errwidth", deprecated),
+ capsize=kwargs.pop("capsize", 0),
+ )
+
+ p.plot_points(
+ aggregator=aggregator,
+ markers=markers,
+ linestyles=linestyles,
+ dodge=dodge,
+ color=color,
+ capsize=capsize,
+ err_kws=err_kws,
+ plot_kws=kwargs,
+ )
+
+ elif kind == "bar":
+
+ aggregator = agg_cls(estimator, errorbar, n_boot=n_boot, seed=seed)
+
+ err_kws, capsize = p._err_kws_backcompat(
+ normalize_kwargs(kwargs.pop("err_kws", {}), mpl.lines.Line2D),
+ errcolor=kwargs.pop("errcolor", deprecated),
+ errwidth=kwargs.pop("errwidth", deprecated),
+ capsize=kwargs.pop("capsize", 0),
+ )
+ gap = kwargs.pop("gap", 0)
+ fill = kwargs.pop("fill", True)
+
+ p.plot_bars(
+ aggregator=aggregator,
+ dodge=dodge,
+ width=width,
+ gap=gap,
+ color=color,
+ fill=fill,
+ capsize=capsize,
+ err_kws=err_kws,
+ plot_kws=kwargs,
+ )
+
+ elif kind == "count":
+
+ aggregator = EstimateAggregator("sum", errorbar=None)
+
+ count_axis = {"x": "y", "y": "x"}[p.orient]
+ p.plot_data[count_axis] = 1
+
+ stat_options = ["count", "percent", "probability", "proportion"]
+ stat = _check_argument("stat", stat_options, kwargs.pop("stat", "count"))
+ p.variables[count_axis] = stat
+ if stat != "count":
+ denom = 100 if stat == "percent" else 1
+ p.plot_data[count_axis] /= len(p.plot_data) / denom
+
+ gap = kwargs.pop("gap", 0)
+ fill = kwargs.pop("fill", True)
+
+ p.plot_bars(
+ aggregator=aggregator,
+ dodge=dodge,
+ width=width,
+ gap=gap,
+ color=color,
+ fill=fill,
+ capsize=0,
+ err_kws={},
+ plot_kws=kwargs,
+ )
+
+ else:
+ msg = (
+ f"Invalid `kind`: {kind!r}. Options are 'strip', 'swarm', "
+ "'box', 'boxen', 'violin', 'bar', 'count', and 'point'."
+ )
+ raise ValueError(msg)
+
+ for ax in g.axes.flat:
+ p._adjust_cat_axis(ax, axis=p.orient)
+
+ g.set_axis_labels(p.variables.get("x"), p.variables.get("y"))
+ g.set_titles()
+ g.tight_layout()
+
+ for ax in g.axes.flat:
+ g._update_legend_data(ax)
+ ax.legend_ = None
+
+ if legend == "auto":
+ show_legend = not p._redundant_hue and p.input_format != "wide"
+ else:
+ show_legend = bool(legend)
+ if show_legend:
+ g.add_legend(title=p.variables.get("hue"), label_order=hue_order)
+
+ if data is not None:
+ # Replace the dataframe on the FacetGrid for any subsequent maps
+ g.data = data
+
+ return g
+
+
+catplot.__doc__ = dedent("""\
+ Figure-level interface for drawing categorical plots onto a FacetGrid.
+
+ This function provides access to several axes-level functions that
+ show the relationship between a numerical and one or more categorical
+ variables using one of several visual representations. The `kind`
+ parameter selects the underlying axes-level function to use.
+
+ Categorical scatterplots:
+
+ - :func:`stripplot` (with `kind="strip"`; the default)
+ - :func:`swarmplot` (with `kind="swarm"`)
+
+ Categorical distribution plots:
+
+ - :func:`boxplot` (with `kind="box"`)
+ - :func:`violinplot` (with `kind="violin"`)
+ - :func:`boxenplot` (with `kind="boxen"`)
+
+ Categorical estimate plots:
+
+ - :func:`pointplot` (with `kind="point"`)
+ - :func:`barplot` (with `kind="bar"`)
+ - :func:`countplot` (with `kind="count"`)
+
+ Extra keyword arguments are passed to the underlying function, so you
+ should refer to the documentation for each to see kind-specific options.
+
+ {categorical_narrative}
+
+ After plotting, the :class:`FacetGrid` with the plot is returned and can
+ be used directly to tweak supporting plot details or add other layers.
+
+ Parameters
+ ----------
+ {categorical_data}
+ {input_params}
+ row, col : names of variables in `data` or vector data
+ Categorical variables that will determine the faceting of the grid.
+ kind : str
+ The kind of plot to draw, corresponds to the name of a categorical
+ axes-level plotting function. Options are: "strip", "swarm", "box", "violin",
+ "boxen", "point", "bar", or "count".
+ {stat_api_params}
+ {order_vars}
+ row_order, col_order : lists of strings
+ Order to organize the rows and/or columns of the grid in; otherwise the
+ orders are inferred from the data objects.
+ {col_wrap}
+ {height}
+ {aspect}
+ {native_scale}
+ {formatter}
+ {orient}
+ {color}
+ {palette}
+ {hue_norm}
+ {legend}
+ {legend_out}
+ {share_xy}
+ {margin_titles}
+ facet_kws : dict
+ Dictionary of other keyword arguments to pass to :class:`FacetGrid`.
+ kwargs : key, value pairings
+ Other keyword arguments are passed through to the underlying plotting
+ function.
+
+ Returns
+ -------
+ :class:`FacetGrid`
+ Returns the :class:`FacetGrid` object with the plot on it for further
+ tweaking.
+
+ Examples
+ --------
+ .. include:: ../docstrings/catplot.rst
+
+ """).format(**_categorical_docs)
+
+
+class Beeswarm:
+ """Modifies a scatterplot artist to show a beeswarm plot."""
+ def __init__(self, orient="x", width=0.8, warn_thresh=.05):
+
+ self.orient = orient
+ self.width = width
+ self.warn_thresh = warn_thresh
+
+ def __call__(self, points, center):
+ """Swarm `points`, a PathCollection, around the `center` position."""
+ # Convert from point size (area) to diameter
+
+ ax = points.axes
+ dpi = ax.figure.dpi
+
+ # Get the original positions of the points
+ orig_xy_data = points.get_offsets()
+
+ # Reset the categorical positions to the center line
+ cat_idx = 1 if self.orient == "y" else 0
+ orig_xy_data[:, cat_idx] = center
+
+ # Transform the data coordinates to point coordinates.
+ # We'll figure out the swarm positions in the latter
+ # and then convert back to data coordinates and replot
+ orig_x_data, orig_y_data = orig_xy_data.T
+ orig_xy = ax.transData.transform(orig_xy_data)
+
+ # Order the variables so that x is the categorical axis
+ if self.orient == "y":
+ orig_xy = orig_xy[:, [1, 0]]
+
+ # Add a column with each point's radius
+ sizes = points.get_sizes()
+ if sizes.size == 1:
+ sizes = np.repeat(sizes, orig_xy.shape[0])
+ edge = points.get_linewidth().item()
+ radii = (np.sqrt(sizes) + edge) / 2 * (dpi / 72)
+ orig_xy = np.c_[orig_xy, radii]
+
+ # Sort along the value axis to facilitate the beeswarm
+ sorter = np.argsort(orig_xy[:, 1])
+ orig_xyr = orig_xy[sorter]
+
+ # Adjust points along the categorical axis to prevent overlaps
+ new_xyr = np.empty_like(orig_xyr)
+ new_xyr[sorter] = self.beeswarm(orig_xyr)
+
+ # Transform the point coordinates back to data coordinates
+ if self.orient == "y":
+ new_xy = new_xyr[:, [1, 0]]
+ else:
+ new_xy = new_xyr[:, :2]
+ new_x_data, new_y_data = ax.transData.inverted().transform(new_xy).T
+
+ # Add gutters
+ t_fwd, t_inv = _get_transform_functions(ax, self.orient)
+ if self.orient == "y":
+ self.add_gutters(new_y_data, center, t_fwd, t_inv)
+ else:
+ self.add_gutters(new_x_data, center, t_fwd, t_inv)
+
+ # Reposition the points so they do not overlap
+ if self.orient == "y":
+ points.set_offsets(np.c_[orig_x_data, new_y_data])
+ else:
+ points.set_offsets(np.c_[new_x_data, orig_y_data])
+
+ def beeswarm(self, orig_xyr):
+ """Adjust x position of points to avoid overlaps."""
+ # In this method, `x` is always the categorical axis
+ # Center of the swarm, in point coordinates
+ midline = orig_xyr[0, 0]
+
+ # Start the swarm with the first point
+ swarm = np.atleast_2d(orig_xyr[0])
+
+ # Loop over the remaining points
+ for xyr_i in orig_xyr[1:]:
+
+ # Find the points in the swarm that could possibly
+ # overlap with the point we are currently placing
+ neighbors = self.could_overlap(xyr_i, swarm)
+
+ # Find positions that would be valid individually
+ # with respect to each of the swarm neighbors
+ candidates = self.position_candidates(xyr_i, neighbors)
+
+ # Sort candidates by their centrality
+ offsets = np.abs(candidates[:, 0] - midline)
+ candidates = candidates[np.argsort(offsets)]
+
+ # Find the first candidate that does not overlap any neighbors
+ new_xyr_i = self.first_non_overlapping_candidate(candidates, neighbors)
+
+ # Place it into the swarm
+ swarm = np.vstack([swarm, new_xyr_i])
+
+ return swarm
+
+ def could_overlap(self, xyr_i, swarm):
+ """Return a list of all swarm points that could overlap with target."""
+ # Because we work backwards through the swarm and can short-circuit,
+ # the for-loop is faster than vectorization
+ _, y_i, r_i = xyr_i
+ neighbors = []
+ for xyr_j in reversed(swarm):
+ _, y_j, r_j = xyr_j
+ if (y_i - y_j) < (r_i + r_j):
+ neighbors.append(xyr_j)
+ else:
+ break
+ return np.array(neighbors)[::-1]
+
+ def position_candidates(self, xyr_i, neighbors):
+ """Return a list of coordinates that might be valid by adjusting x."""
+ candidates = [xyr_i]
+ x_i, y_i, r_i = xyr_i
+ left_first = True
+ for x_j, y_j, r_j in neighbors:
+ dy = y_i - y_j
+ dx = np.sqrt(max((r_i + r_j) ** 2 - dy ** 2, 0)) * 1.05
+ cl, cr = (x_j - dx, y_i, r_i), (x_j + dx, y_i, r_i)
+ if left_first:
+ new_candidates = [cl, cr]
+ else:
+ new_candidates = [cr, cl]
+ candidates.extend(new_candidates)
+ left_first = not left_first
+ return np.array(candidates)
+
+ def first_non_overlapping_candidate(self, candidates, neighbors):
+ """Find the first candidate that does not overlap with the swarm."""
+
+ # If we have no neighbors, all candidates are good.
+ if len(neighbors) == 0:
+ return candidates[0]
+
+ neighbors_x = neighbors[:, 0]
+ neighbors_y = neighbors[:, 1]
+ neighbors_r = neighbors[:, 2]
+
+ for xyr_i in candidates:
+
+ x_i, y_i, r_i = xyr_i
+
+ dx = neighbors_x - x_i
+ dy = neighbors_y - y_i
+ sq_distances = np.square(dx) + np.square(dy)
+
+ sep_needed = np.square(neighbors_r + r_i)
+
+ # Good candidate does not overlap any of neighbors which means that
+ # squared distance between candidate and any of the neighbors has
+ # to be at least square of the summed radii
+ good_candidate = np.all(sq_distances >= sep_needed)
+
+ if good_candidate:
+ return xyr_i
+
+ raise RuntimeError(
+ "No non-overlapping candidates found. This should not happen."
+ )
+
+ def add_gutters(self, points, center, trans_fwd, trans_inv):
+ """Stop points from extending beyond their territory."""
+ half_width = self.width / 2
+ low_gutter = trans_inv(trans_fwd(center) - half_width)
+ off_low = points < low_gutter
+ if off_low.any():
+ points[off_low] = low_gutter
+ high_gutter = trans_inv(trans_fwd(center) + half_width)
+ off_high = points > high_gutter
+ if off_high.any():
+ points[off_high] = high_gutter
+
+ gutter_prop = (off_high + off_low).sum() / len(points)
+ if gutter_prop > self.warn_thresh:
+ msg = (
+ "{:.1%} of the points cannot be placed; you may want "
+ "to decrease the size of the markers or use stripplot."
+ ).format(gutter_prop)
+ warnings.warn(msg, UserWarning)
+
+ return points
+
+
+BoxPlotArtists = namedtuple("BoxPlotArtists", "box median whiskers caps fliers mean")
+
+
+class BoxPlotContainer:
+
+ def __init__(self, artist_dict):
+
+ self.boxes = artist_dict["boxes"]
+ self.medians = artist_dict["medians"]
+ self.whiskers = artist_dict["whiskers"]
+ self.caps = artist_dict["caps"]
+ self.fliers = artist_dict["fliers"]
+ self.means = artist_dict["means"]
+
+ self._label = None
+ self._children = [
+ *self.boxes,
+ *self.medians,
+ *self.whiskers,
+ *self.caps,
+ *self.fliers,
+ *self.means,
+ ]
+
+ def __repr__(self):
+ return f"<BoxPlotContainer object with {len(self.boxes)} boxes>"
+
+ def __getitem__(self, idx):
+ pair_slice = slice(2 * idx, 2 * idx + 2)
+ return BoxPlotArtists(
+ self.boxes[idx] if self.boxes else [],
+ self.medians[idx] if self.medians else [],
+ self.whiskers[pair_slice] if self.whiskers else [],
+ self.caps[pair_slice] if self.caps else [],
+ self.fliers[idx] if self.fliers else [],
+ self.means[idx]if self.means else [],
+ )
+
+ def __iter__(self):
+ yield from (self[i] for i in range(len(self.boxes)))
+
+ def get_label(self):
+ return self._label
+
+ def set_label(self, value):
+ self._label = value
+
+ def get_children(self):
+ return self._children
+
+ def remove(self):
+ for child in self._children:
+ child.remove()
diff --git a/seaborn/cm.py b/seaborn/cm.py
new file mode 100644
index 0000000000000000000000000000000000000000..df7ce61997882d7d7f734052292438e4234a5cc7
--- /dev/null
+++ b/seaborn/cm.py
@@ -0,0 +1,1586 @@
+from matplotlib import colors
+from seaborn._compat import register_colormap
+
+
+_rocket_lut = [
+ [ 0.01060815, 0.01808215, 0.10018654],
+ [ 0.01428972, 0.02048237, 0.10374486],
+ [ 0.01831941, 0.0229766 , 0.10738511],
+ [ 0.02275049, 0.02554464, 0.11108639],
+ [ 0.02759119, 0.02818316, 0.11483751],
+ [ 0.03285175, 0.03088792, 0.11863035],
+ [ 0.03853466, 0.03365771, 0.12245873],
+ [ 0.04447016, 0.03648425, 0.12631831],
+ [ 0.05032105, 0.03936808, 0.13020508],
+ [ 0.05611171, 0.04224835, 0.13411624],
+ [ 0.0618531 , 0.04504866, 0.13804929],
+ [ 0.06755457, 0.04778179, 0.14200206],
+ [ 0.0732236 , 0.05045047, 0.14597263],
+ [ 0.0788708 , 0.05305461, 0.14995981],
+ [ 0.08450105, 0.05559631, 0.15396203],
+ [ 0.09011319, 0.05808059, 0.15797687],
+ [ 0.09572396, 0.06050127, 0.16200507],
+ [ 0.10132312, 0.06286782, 0.16604287],
+ [ 0.10692823, 0.06517224, 0.17009175],
+ [ 0.1125315 , 0.06742194, 0.17414848],
+ [ 0.11813947, 0.06961499, 0.17821272],
+ [ 0.12375803, 0.07174938, 0.18228425],
+ [ 0.12938228, 0.07383015, 0.18636053],
+ [ 0.13501631, 0.07585609, 0.19044109],
+ [ 0.14066867, 0.0778224 , 0.19452676],
+ [ 0.14633406, 0.07973393, 0.1986151 ],
+ [ 0.15201338, 0.08159108, 0.20270523],
+ [ 0.15770877, 0.08339312, 0.20679668],
+ [ 0.16342174, 0.0851396 , 0.21088893],
+ [ 0.16915387, 0.08682996, 0.21498104],
+ [ 0.17489524, 0.08848235, 0.2190294 ],
+ [ 0.18065495, 0.09009031, 0.22303512],
+ [ 0.18643324, 0.09165431, 0.22699705],
+ [ 0.19223028, 0.09317479, 0.23091409],
+ [ 0.19804623, 0.09465217, 0.23478512],
+ [ 0.20388117, 0.09608689, 0.23860907],
+ [ 0.20973515, 0.09747934, 0.24238489],
+ [ 0.21560818, 0.09882993, 0.24611154],
+ [ 0.22150014, 0.10013944, 0.2497868 ],
+ [ 0.22741085, 0.10140876, 0.25340813],
+ [ 0.23334047, 0.10263737, 0.25697736],
+ [ 0.23928891, 0.10382562, 0.2604936 ],
+ [ 0.24525608, 0.10497384, 0.26395596],
+ [ 0.25124182, 0.10608236, 0.26736359],
+ [ 0.25724602, 0.10715148, 0.27071569],
+ [ 0.26326851, 0.1081815 , 0.27401148],
+ [ 0.26930915, 0.1091727 , 0.2772502 ],
+ [ 0.27536766, 0.11012568, 0.28043021],
+ [ 0.28144375, 0.11104133, 0.2835489 ],
+ [ 0.2875374 , 0.11191896, 0.28660853],
+ [ 0.29364846, 0.11275876, 0.2896085 ],
+ [ 0.29977678, 0.11356089, 0.29254823],
+ [ 0.30592213, 0.11432553, 0.29542718],
+ [ 0.31208435, 0.11505284, 0.29824485],
+ [ 0.31826327, 0.1157429 , 0.30100076],
+ [ 0.32445869, 0.11639585, 0.30369448],
+ [ 0.33067031, 0.11701189, 0.30632563],
+ [ 0.33689808, 0.11759095, 0.3088938 ],
+ [ 0.34314168, 0.11813362, 0.31139721],
+ [ 0.34940101, 0.11863987, 0.3138355 ],
+ [ 0.355676 , 0.11910909, 0.31620996],
+ [ 0.36196644, 0.1195413 , 0.31852037],
+ [ 0.36827206, 0.11993653, 0.32076656],
+ [ 0.37459292, 0.12029443, 0.32294825],
+ [ 0.38092887, 0.12061482, 0.32506528],
+ [ 0.38727975, 0.12089756, 0.3271175 ],
+ [ 0.39364518, 0.12114272, 0.32910494],
+ [ 0.40002537, 0.12134964, 0.33102734],
+ [ 0.40642019, 0.12151801, 0.33288464],
+ [ 0.41282936, 0.12164769, 0.33467689],
+ [ 0.41925278, 0.12173833, 0.33640407],
+ [ 0.42569057, 0.12178916, 0.33806605],
+ [ 0.43214263, 0.12179973, 0.33966284],
+ [ 0.43860848, 0.12177004, 0.34119475],
+ [ 0.44508855, 0.12169883, 0.34266151],
+ [ 0.45158266, 0.12158557, 0.34406324],
+ [ 0.45809049, 0.12142996, 0.34540024],
+ [ 0.46461238, 0.12123063, 0.34667231],
+ [ 0.47114798, 0.12098721, 0.34787978],
+ [ 0.47769736, 0.12069864, 0.34902273],
+ [ 0.48426077, 0.12036349, 0.35010104],
+ [ 0.49083761, 0.11998161, 0.35111537],
+ [ 0.49742847, 0.11955087, 0.35206533],
+ [ 0.50403286, 0.11907081, 0.35295152],
+ [ 0.51065109, 0.11853959, 0.35377385],
+ [ 0.51728314, 0.1179558 , 0.35453252],
+ [ 0.52392883, 0.11731817, 0.35522789],
+ [ 0.53058853, 0.11662445, 0.35585982],
+ [ 0.53726173, 0.11587369, 0.35642903],
+ [ 0.54394898, 0.11506307, 0.35693521],
+ [ 0.5506426 , 0.11420757, 0.35737863],
+ [ 0.55734473, 0.11330456, 0.35775059],
+ [ 0.56405586, 0.11235265, 0.35804813],
+ [ 0.57077365, 0.11135597, 0.35827146],
+ [ 0.5774991 , 0.11031233, 0.35841679],
+ [ 0.58422945, 0.10922707, 0.35848469],
+ [ 0.59096382, 0.10810205, 0.35847347],
+ [ 0.59770215, 0.10693774, 0.35838029],
+ [ 0.60444226, 0.10573912, 0.35820487],
+ [ 0.61118304, 0.10450943, 0.35794557],
+ [ 0.61792306, 0.10325288, 0.35760108],
+ [ 0.62466162, 0.10197244, 0.35716891],
+ [ 0.63139686, 0.10067417, 0.35664819],
+ [ 0.63812122, 0.09938212, 0.35603757],
+ [ 0.64483795, 0.0980891 , 0.35533555],
+ [ 0.65154562, 0.09680192, 0.35454107],
+ [ 0.65824241, 0.09552918, 0.3536529 ],
+ [ 0.66492652, 0.09428017, 0.3526697 ],
+ [ 0.67159578, 0.09306598, 0.35159077],
+ [ 0.67824099, 0.09192342, 0.3504148 ],
+ [ 0.684863 , 0.09085633, 0.34914061],
+ [ 0.69146268, 0.0898675 , 0.34776864],
+ [ 0.69803757, 0.08897226, 0.3462986 ],
+ [ 0.70457834, 0.0882129 , 0.34473046],
+ [ 0.71108138, 0.08761223, 0.3430635 ],
+ [ 0.7175507 , 0.08716212, 0.34129974],
+ [ 0.72398193, 0.08688725, 0.33943958],
+ [ 0.73035829, 0.0868623 , 0.33748452],
+ [ 0.73669146, 0.08704683, 0.33543669],
+ [ 0.74297501, 0.08747196, 0.33329799],
+ [ 0.74919318, 0.08820542, 0.33107204],
+ [ 0.75535825, 0.08919792, 0.32876184],
+ [ 0.76145589, 0.09050716, 0.32637117],
+ [ 0.76748424, 0.09213602, 0.32390525],
+ [ 0.77344838, 0.09405684, 0.32136808],
+ [ 0.77932641, 0.09634794, 0.31876642],
+ [ 0.78513609, 0.09892473, 0.31610488],
+ [ 0.79085854, 0.10184672, 0.313391 ],
+ [ 0.7965014 , 0.10506637, 0.31063031],
+ [ 0.80205987, 0.10858333, 0.30783 ],
+ [ 0.80752799, 0.11239964, 0.30499738],
+ [ 0.81291606, 0.11645784, 0.30213802],
+ [ 0.81820481, 0.12080606, 0.29926105],
+ [ 0.82341472, 0.12535343, 0.2963705 ],
+ [ 0.82852822, 0.13014118, 0.29347474],
+ [ 0.83355779, 0.13511035, 0.29057852],
+ [ 0.83850183, 0.14025098, 0.2876878 ],
+ [ 0.84335441, 0.14556683, 0.28480819],
+ [ 0.84813096, 0.15099892, 0.281943 ],
+ [ 0.85281737, 0.15657772, 0.27909826],
+ [ 0.85742602, 0.1622583 , 0.27627462],
+ [ 0.86196552, 0.16801239, 0.27346473],
+ [ 0.86641628, 0.17387796, 0.27070818],
+ [ 0.87079129, 0.17982114, 0.26797378],
+ [ 0.87507281, 0.18587368, 0.26529697],
+ [ 0.87925878, 0.19203259, 0.26268136],
+ [ 0.8833417 , 0.19830556, 0.26014181],
+ [ 0.88731387, 0.20469941, 0.25769539],
+ [ 0.89116859, 0.21121788, 0.2553592 ],
+ [ 0.89490337, 0.21785614, 0.25314362],
+ [ 0.8985026 , 0.22463251, 0.25108745],
+ [ 0.90197527, 0.23152063, 0.24918223],
+ [ 0.90530097, 0.23854541, 0.24748098],
+ [ 0.90848638, 0.24568473, 0.24598324],
+ [ 0.911533 , 0.25292623, 0.24470258],
+ [ 0.9144225 , 0.26028902, 0.24369359],
+ [ 0.91717106, 0.26773821, 0.24294137],
+ [ 0.91978131, 0.27526191, 0.24245973],
+ [ 0.92223947, 0.28287251, 0.24229568],
+ [ 0.92456587, 0.29053388, 0.24242622],
+ [ 0.92676657, 0.29823282, 0.24285536],
+ [ 0.92882964, 0.30598085, 0.24362274],
+ [ 0.93078135, 0.31373977, 0.24468803],
+ [ 0.93262051, 0.3215093 , 0.24606461],
+ [ 0.93435067, 0.32928362, 0.24775328],
+ [ 0.93599076, 0.33703942, 0.24972157],
+ [ 0.93752831, 0.34479177, 0.25199928],
+ [ 0.93899289, 0.35250734, 0.25452808],
+ [ 0.94036561, 0.36020899, 0.25734661],
+ [ 0.94167588, 0.36786594, 0.2603949 ],
+ [ 0.94291042, 0.37549479, 0.26369821],
+ [ 0.94408513, 0.3830811 , 0.26722004],
+ [ 0.94520419, 0.39062329, 0.27094924],
+ [ 0.94625977, 0.39813168, 0.27489742],
+ [ 0.94727016, 0.4055909 , 0.27902322],
+ [ 0.94823505, 0.41300424, 0.28332283],
+ [ 0.94914549, 0.42038251, 0.28780969],
+ [ 0.95001704, 0.42771398, 0.29244728],
+ [ 0.95085121, 0.43500005, 0.29722817],
+ [ 0.95165009, 0.44224144, 0.30214494],
+ [ 0.9524044 , 0.44944853, 0.3072105 ],
+ [ 0.95312556, 0.45661389, 0.31239776],
+ [ 0.95381595, 0.46373781, 0.31769923],
+ [ 0.95447591, 0.47082238, 0.32310953],
+ [ 0.95510255, 0.47787236, 0.32862553],
+ [ 0.95569679, 0.48489115, 0.33421404],
+ [ 0.95626788, 0.49187351, 0.33985601],
+ [ 0.95681685, 0.49882008, 0.34555431],
+ [ 0.9573439 , 0.50573243, 0.35130912],
+ [ 0.95784842, 0.51261283, 0.35711942],
+ [ 0.95833051, 0.51946267, 0.36298589],
+ [ 0.95879054, 0.52628305, 0.36890904],
+ [ 0.95922872, 0.53307513, 0.3748895 ],
+ [ 0.95964538, 0.53983991, 0.38092784],
+ [ 0.96004345, 0.54657593, 0.3870292 ],
+ [ 0.96042097, 0.55328624, 0.39319057],
+ [ 0.96077819, 0.55997184, 0.39941173],
+ [ 0.9611152 , 0.5666337 , 0.40569343],
+ [ 0.96143273, 0.57327231, 0.41203603],
+ [ 0.96173392, 0.57988594, 0.41844491],
+ [ 0.96201757, 0.58647675, 0.42491751],
+ [ 0.96228344, 0.59304598, 0.43145271],
+ [ 0.96253168, 0.5995944 , 0.43805131],
+ [ 0.96276513, 0.60612062, 0.44471698],
+ [ 0.96298491, 0.6126247 , 0.45145074],
+ [ 0.96318967, 0.61910879, 0.45824902],
+ [ 0.96337949, 0.6255736 , 0.46511271],
+ [ 0.96355923, 0.63201624, 0.47204746],
+ [ 0.96372785, 0.63843852, 0.47905028],
+ [ 0.96388426, 0.64484214, 0.4861196 ],
+ [ 0.96403203, 0.65122535, 0.4932578 ],
+ [ 0.96417332, 0.65758729, 0.50046894],
+ [ 0.9643063 , 0.66393045, 0.5077467 ],
+ [ 0.96443322, 0.67025402, 0.51509334],
+ [ 0.96455845, 0.67655564, 0.52251447],
+ [ 0.96467922, 0.68283846, 0.53000231],
+ [ 0.96479861, 0.68910113, 0.53756026],
+ [ 0.96492035, 0.69534192, 0.5451917 ],
+ [ 0.96504223, 0.7015636 , 0.5528892 ],
+ [ 0.96516917, 0.70776351, 0.5606593 ],
+ [ 0.96530224, 0.71394212, 0.56849894],
+ [ 0.96544032, 0.72010124, 0.57640375],
+ [ 0.96559206, 0.72623592, 0.58438387],
+ [ 0.96575293, 0.73235058, 0.59242739],
+ [ 0.96592829, 0.73844258, 0.60053991],
+ [ 0.96612013, 0.74451182, 0.60871954],
+ [ 0.96632832, 0.75055966, 0.61696136],
+ [ 0.96656022, 0.75658231, 0.62527295],
+ [ 0.96681185, 0.76258381, 0.63364277],
+ [ 0.96709183, 0.76855969, 0.64207921],
+ [ 0.96739773, 0.77451297, 0.65057302],
+ [ 0.96773482, 0.78044149, 0.65912731],
+ [ 0.96810471, 0.78634563, 0.66773889],
+ [ 0.96850919, 0.79222565, 0.6764046 ],
+ [ 0.96893132, 0.79809112, 0.68512266],
+ [ 0.96935926, 0.80395415, 0.69383201],
+ [ 0.9698028 , 0.80981139, 0.70252255],
+ [ 0.97025511, 0.81566605, 0.71120296],
+ [ 0.97071849, 0.82151775, 0.71987163],
+ [ 0.97120159, 0.82736371, 0.72851999],
+ [ 0.97169389, 0.83320847, 0.73716071],
+ [ 0.97220061, 0.83905052, 0.74578903],
+ [ 0.97272597, 0.84488881, 0.75440141],
+ [ 0.97327085, 0.85072354, 0.76299805],
+ [ 0.97383206, 0.85655639, 0.77158353],
+ [ 0.97441222, 0.86238689, 0.78015619],
+ [ 0.97501782, 0.86821321, 0.78871034],
+ [ 0.97564391, 0.87403763, 0.79725261],
+ [ 0.97628674, 0.87986189, 0.8057883 ],
+ [ 0.97696114, 0.88568129, 0.81430324],
+ [ 0.97765722, 0.89149971, 0.82280948],
+ [ 0.97837585, 0.89731727, 0.83130786],
+ [ 0.97912374, 0.90313207, 0.83979337],
+ [ 0.979891 , 0.90894778, 0.84827858],
+ [ 0.98067764, 0.91476465, 0.85676611],
+ [ 0.98137749, 0.92061729, 0.86536915]
+]
+
+
+_mako_lut = [
+ [ 0.04503935, 0.01482344, 0.02092227],
+ [ 0.04933018, 0.01709292, 0.02535719],
+ [ 0.05356262, 0.01950702, 0.03018802],
+ [ 0.05774337, 0.02205989, 0.03545515],
+ [ 0.06188095, 0.02474764, 0.04115287],
+ [ 0.06598247, 0.0275665 , 0.04691409],
+ [ 0.07005374, 0.03051278, 0.05264306],
+ [ 0.07409947, 0.03358324, 0.05834631],
+ [ 0.07812339, 0.03677446, 0.06403249],
+ [ 0.08212852, 0.0400833 , 0.06970862],
+ [ 0.08611731, 0.04339148, 0.07538208],
+ [ 0.09009161, 0.04664706, 0.08105568],
+ [ 0.09405308, 0.04985685, 0.08673591],
+ [ 0.09800301, 0.05302279, 0.09242646],
+ [ 0.10194255, 0.05614641, 0.09813162],
+ [ 0.10587261, 0.05922941, 0.103854 ],
+ [ 0.1097942 , 0.06227277, 0.10959847],
+ [ 0.11370826, 0.06527747, 0.11536893],
+ [ 0.11761516, 0.06824548, 0.12116393],
+ [ 0.12151575, 0.07117741, 0.12698763],
+ [ 0.12541095, 0.07407363, 0.1328442 ],
+ [ 0.12930083, 0.07693611, 0.13873064],
+ [ 0.13317849, 0.07976988, 0.14465095],
+ [ 0.13701138, 0.08259683, 0.15060265],
+ [ 0.14079223, 0.08542126, 0.15659379],
+ [ 0.14452486, 0.08824175, 0.16262484],
+ [ 0.14820351, 0.09106304, 0.16869476],
+ [ 0.15183185, 0.09388372, 0.17480366],
+ [ 0.15540398, 0.09670855, 0.18094993],
+ [ 0.15892417, 0.09953561, 0.18713384],
+ [ 0.16238588, 0.10236998, 0.19335329],
+ [ 0.16579435, 0.10520905, 0.19960847],
+ [ 0.16914226, 0.10805832, 0.20589698],
+ [ 0.17243586, 0.11091443, 0.21221911],
+ [ 0.17566717, 0.11378321, 0.21857219],
+ [ 0.17884322, 0.11666074, 0.2249565 ],
+ [ 0.18195582, 0.11955283, 0.23136943],
+ [ 0.18501213, 0.12245547, 0.23781116],
+ [ 0.18800459, 0.12537395, 0.24427914],
+ [ 0.19093944, 0.1283047 , 0.25077369],
+ [ 0.19381092, 0.13125179, 0.25729255],
+ [ 0.19662307, 0.13421303, 0.26383543],
+ [ 0.19937337, 0.13719028, 0.27040111],
+ [ 0.20206187, 0.14018372, 0.27698891],
+ [ 0.20469116, 0.14319196, 0.28359861],
+ [ 0.20725547, 0.14621882, 0.29022775],
+ [ 0.20976258, 0.14925954, 0.29687795],
+ [ 0.21220409, 0.15231929, 0.30354703],
+ [ 0.21458611, 0.15539445, 0.31023563],
+ [ 0.21690827, 0.15848519, 0.31694355],
+ [ 0.21916481, 0.16159489, 0.32366939],
+ [ 0.2213631 , 0.16471913, 0.33041431],
+ [ 0.22349947, 0.1678599 , 0.33717781],
+ [ 0.2255714 , 0.1710185 , 0.34395925],
+ [ 0.22758415, 0.17419169, 0.35075983],
+ [ 0.22953569, 0.17738041, 0.35757941],
+ [ 0.23142077, 0.18058733, 0.3644173 ],
+ [ 0.2332454 , 0.18380872, 0.37127514],
+ [ 0.2350092 , 0.18704459, 0.3781528 ],
+ [ 0.23670785, 0.190297 , 0.38504973],
+ [ 0.23834119, 0.19356547, 0.39196711],
+ [ 0.23991189, 0.19684817, 0.39890581],
+ [ 0.24141903, 0.20014508, 0.4058667 ],
+ [ 0.24286214, 0.20345642, 0.4128484 ],
+ [ 0.24423453, 0.20678459, 0.41985299],
+ [ 0.24554109, 0.21012669, 0.42688124],
+ [ 0.2467815 , 0.21348266, 0.43393244],
+ [ 0.24795393, 0.21685249, 0.4410088 ],
+ [ 0.24905614, 0.22023618, 0.448113 ],
+ [ 0.25007383, 0.22365053, 0.45519562],
+ [ 0.25098926, 0.22710664, 0.46223892],
+ [ 0.25179696, 0.23060342, 0.46925447],
+ [ 0.25249346, 0.23414353, 0.47623196],
+ [ 0.25307401, 0.23772973, 0.48316271],
+ [ 0.25353152, 0.24136961, 0.49001976],
+ [ 0.25386167, 0.24506548, 0.49679407],
+ [ 0.25406082, 0.2488164 , 0.50348932],
+ [ 0.25412435, 0.25262843, 0.51007843],
+ [ 0.25404842, 0.25650743, 0.51653282],
+ [ 0.25383134, 0.26044852, 0.52286845],
+ [ 0.2534705 , 0.26446165, 0.52903422],
+ [ 0.25296722, 0.2685428 , 0.53503572],
+ [ 0.2523226 , 0.27269346, 0.54085315],
+ [ 0.25153974, 0.27691629, 0.54645752],
+ [ 0.25062402, 0.28120467, 0.55185939],
+ [ 0.24958205, 0.28556371, 0.55701246],
+ [ 0.24842386, 0.28998148, 0.56194601],
+ [ 0.24715928, 0.29446327, 0.56660884],
+ [ 0.24580099, 0.29899398, 0.57104399],
+ [ 0.24436202, 0.30357852, 0.57519929],
+ [ 0.24285591, 0.30819938, 0.57913247],
+ [ 0.24129828, 0.31286235, 0.58278615],
+ [ 0.23970131, 0.3175495 , 0.5862272 ],
+ [ 0.23807973, 0.32226344, 0.58941872],
+ [ 0.23644557, 0.32699241, 0.59240198],
+ [ 0.2348113 , 0.33173196, 0.59518282],
+ [ 0.23318874, 0.33648036, 0.59775543],
+ [ 0.2315855 , 0.34122763, 0.60016456],
+ [ 0.23001121, 0.34597357, 0.60240251],
+ [ 0.2284748 , 0.35071512, 0.6044784 ],
+ [ 0.22698081, 0.35544612, 0.60642528],
+ [ 0.22553305, 0.36016515, 0.60825252],
+ [ 0.22413977, 0.36487341, 0.60994938],
+ [ 0.22280246, 0.36956728, 0.61154118],
+ [ 0.22152555, 0.37424409, 0.61304472],
+ [ 0.22030752, 0.37890437, 0.61446646],
+ [ 0.2191538 , 0.38354668, 0.61581561],
+ [ 0.21806257, 0.38817169, 0.61709794],
+ [ 0.21703799, 0.39277882, 0.61831922],
+ [ 0.21607792, 0.39736958, 0.61948028],
+ [ 0.21518463, 0.40194196, 0.62059763],
+ [ 0.21435467, 0.40649717, 0.62167507],
+ [ 0.21358663, 0.41103579, 0.62271724],
+ [ 0.21288172, 0.41555771, 0.62373011],
+ [ 0.21223835, 0.42006355, 0.62471794],
+ [ 0.21165312, 0.42455441, 0.62568371],
+ [ 0.21112526, 0.42903064, 0.6266318 ],
+ [ 0.21065161, 0.43349321, 0.62756504],
+ [ 0.21023306, 0.43794288, 0.62848279],
+ [ 0.20985996, 0.44238227, 0.62938329],
+ [ 0.20951045, 0.44680966, 0.63030696],
+ [ 0.20916709, 0.45122981, 0.63124483],
+ [ 0.20882976, 0.45564335, 0.63219599],
+ [ 0.20849798, 0.46005094, 0.63315928],
+ [ 0.20817199, 0.46445309, 0.63413391],
+ [ 0.20785149, 0.46885041, 0.63511876],
+ [ 0.20753716, 0.47324327, 0.63611321],
+ [ 0.20722876, 0.47763224, 0.63711608],
+ [ 0.20692679, 0.48201774, 0.63812656],
+ [ 0.20663156, 0.48640018, 0.63914367],
+ [ 0.20634336, 0.49078002, 0.64016638],
+ [ 0.20606303, 0.49515755, 0.6411939 ],
+ [ 0.20578999, 0.49953341, 0.64222457],
+ [ 0.20552612, 0.50390766, 0.64325811],
+ [ 0.20527189, 0.50828072, 0.64429331],
+ [ 0.20502868, 0.51265277, 0.64532947],
+ [ 0.20479718, 0.51702417, 0.64636539],
+ [ 0.20457804, 0.52139527, 0.64739979],
+ [ 0.20437304, 0.52576622, 0.64843198],
+ [ 0.20418396, 0.53013715, 0.64946117],
+ [ 0.20401238, 0.53450825, 0.65048638],
+ [ 0.20385896, 0.53887991, 0.65150606],
+ [ 0.20372653, 0.54325208, 0.65251978],
+ [ 0.20361709, 0.5476249 , 0.6535266 ],
+ [ 0.20353258, 0.55199854, 0.65452542],
+ [ 0.20347472, 0.55637318, 0.655515 ],
+ [ 0.20344718, 0.56074869, 0.65649508],
+ [ 0.20345161, 0.56512531, 0.65746419],
+ [ 0.20349089, 0.56950304, 0.65842151],
+ [ 0.20356842, 0.57388184, 0.65936642],
+ [ 0.20368663, 0.57826181, 0.66029768],
+ [ 0.20384884, 0.58264293, 0.6612145 ],
+ [ 0.20405904, 0.58702506, 0.66211645],
+ [ 0.20431921, 0.59140842, 0.66300179],
+ [ 0.20463464, 0.59579264, 0.66387079],
+ [ 0.20500731, 0.60017798, 0.66472159],
+ [ 0.20544449, 0.60456387, 0.66555409],
+ [ 0.20596097, 0.60894927, 0.66636568],
+ [ 0.20654832, 0.61333521, 0.66715744],
+ [ 0.20721003, 0.61772167, 0.66792838],
+ [ 0.20795035, 0.62210845, 0.66867802],
+ [ 0.20877302, 0.62649546, 0.66940555],
+ [ 0.20968223, 0.63088252, 0.6701105 ],
+ [ 0.21068163, 0.63526951, 0.67079211],
+ [ 0.21177544, 0.63965621, 0.67145005],
+ [ 0.21298582, 0.64404072, 0.67208182],
+ [ 0.21430361, 0.64842404, 0.67268861],
+ [ 0.21572716, 0.65280655, 0.67326978],
+ [ 0.21726052, 0.65718791, 0.6738255 ],
+ [ 0.21890636, 0.66156803, 0.67435491],
+ [ 0.220668 , 0.66594665, 0.67485792],
+ [ 0.22255447, 0.67032297, 0.67533374],
+ [ 0.22458372, 0.67469531, 0.67578061],
+ [ 0.22673713, 0.67906542, 0.67620044],
+ [ 0.22901625, 0.6834332 , 0.67659251],
+ [ 0.23142316, 0.68779836, 0.67695703],
+ [ 0.23395924, 0.69216072, 0.67729378],
+ [ 0.23663857, 0.69651881, 0.67760151],
+ [ 0.23946645, 0.70087194, 0.67788018],
+ [ 0.24242624, 0.70522162, 0.67813088],
+ [ 0.24549008, 0.70957083, 0.67835215],
+ [ 0.24863372, 0.71392166, 0.67854868],
+ [ 0.25187832, 0.71827158, 0.67872193],
+ [ 0.25524083, 0.72261873, 0.67887024],
+ [ 0.25870947, 0.72696469, 0.67898912],
+ [ 0.26229238, 0.73130855, 0.67907645],
+ [ 0.26604085, 0.73564353, 0.67914062],
+ [ 0.26993099, 0.73997282, 0.67917264],
+ [ 0.27397488, 0.74429484, 0.67917096],
+ [ 0.27822463, 0.74860229, 0.67914468],
+ [ 0.28264201, 0.75290034, 0.67907959],
+ [ 0.2873016 , 0.75717817, 0.67899164],
+ [ 0.29215894, 0.76144162, 0.67886578],
+ [ 0.29729823, 0.76567816, 0.67871894],
+ [ 0.30268199, 0.76989232, 0.67853896],
+ [ 0.30835665, 0.77407636, 0.67833512],
+ [ 0.31435139, 0.77822478, 0.67811118],
+ [ 0.3206671 , 0.78233575, 0.67786729],
+ [ 0.32733158, 0.78640315, 0.67761027],
+ [ 0.33437168, 0.79042043, 0.67734882],
+ [ 0.34182112, 0.79437948, 0.67709394],
+ [ 0.34968889, 0.79827511, 0.67685638],
+ [ 0.35799244, 0.80210037, 0.67664969],
+ [ 0.36675371, 0.80584651, 0.67649539],
+ [ 0.3759816 , 0.80950627, 0.67641393],
+ [ 0.38566792, 0.81307432, 0.67642947],
+ [ 0.39579804, 0.81654592, 0.67656899],
+ [ 0.40634556, 0.81991799, 0.67686215],
+ [ 0.41730243, 0.82318339, 0.67735255],
+ [ 0.4285828 , 0.82635051, 0.6780564 ],
+ [ 0.44012728, 0.82942353, 0.67900049],
+ [ 0.45189421, 0.83240398, 0.68021733],
+ [ 0.46378379, 0.83530763, 0.6817062 ],
+ [ 0.47573199, 0.83814472, 0.68347352],
+ [ 0.48769865, 0.84092197, 0.68552698],
+ [ 0.49962354, 0.84365379, 0.68783929],
+ [ 0.5114027 , 0.8463718 , 0.69029789],
+ [ 0.52301693, 0.84908401, 0.69288545],
+ [ 0.53447549, 0.85179048, 0.69561066],
+ [ 0.54578602, 0.8544913 , 0.69848331],
+ [ 0.55695565, 0.85718723, 0.70150427],
+ [ 0.56798832, 0.85987893, 0.70468261],
+ [ 0.57888639, 0.86256715, 0.70802931],
+ [ 0.5896541 , 0.8652532 , 0.71154204],
+ [ 0.60028928, 0.86793835, 0.71523675],
+ [ 0.61079441, 0.87062438, 0.71910895],
+ [ 0.62116633, 0.87331311, 0.72317003],
+ [ 0.63140509, 0.87600675, 0.72741689],
+ [ 0.64150735, 0.87870746, 0.73185717],
+ [ 0.65147219, 0.8814179 , 0.73648495],
+ [ 0.66129632, 0.8841403 , 0.74130658],
+ [ 0.67097934, 0.88687758, 0.74631123],
+ [ 0.68051833, 0.88963189, 0.75150483],
+ [ 0.68991419, 0.89240612, 0.75687187],
+ [ 0.69916533, 0.89520211, 0.76241714],
+ [ 0.70827373, 0.89802257, 0.76812286],
+ [ 0.71723995, 0.90086891, 0.77399039],
+ [ 0.72606665, 0.90374337, 0.7800041 ],
+ [ 0.73475675, 0.90664718, 0.78615802],
+ [ 0.74331358, 0.90958151, 0.79244474],
+ [ 0.75174143, 0.91254787, 0.79884925],
+ [ 0.76004473, 0.91554656, 0.80536823],
+ [ 0.76827704, 0.91856549, 0.81196513],
+ [ 0.77647029, 0.921603 , 0.81855729],
+ [ 0.78462009, 0.92466151, 0.82514119],
+ [ 0.79273542, 0.92773848, 0.83172131],
+ [ 0.8008109 , 0.93083672, 0.83829355],
+ [ 0.80885107, 0.93395528, 0.84485982],
+ [ 0.81685878, 0.9370938 , 0.85142101],
+ [ 0.82483206, 0.94025378, 0.8579751 ],
+ [ 0.83277661, 0.94343371, 0.86452477],
+ [ 0.84069127, 0.94663473, 0.87106853],
+ [ 0.84857662, 0.9498573 , 0.8776059 ],
+ [ 0.8564431 , 0.95309792, 0.88414253],
+ [ 0.86429066, 0.95635719, 0.89067759],
+ [ 0.87218969, 0.95960708, 0.89725384]
+]
+
+
+_vlag_lut = [
+ [ 0.13850039, 0.41331206, 0.74052025],
+ [ 0.15077609, 0.41762684, 0.73970427],
+ [ 0.16235219, 0.4219191 , 0.7389667 ],
+ [ 0.1733322 , 0.42619024, 0.73832537],
+ [ 0.18382538, 0.43044226, 0.73776764],
+ [ 0.19394034, 0.4346772 , 0.73725867],
+ [ 0.20367115, 0.43889576, 0.73685314],
+ [ 0.21313625, 0.44310003, 0.73648045],
+ [ 0.22231173, 0.44729079, 0.73619681],
+ [ 0.23125148, 0.45146945, 0.73597803],
+ [ 0.23998101, 0.45563715, 0.7358223 ],
+ [ 0.24853358, 0.45979489, 0.73571524],
+ [ 0.25691416, 0.4639437 , 0.73566943],
+ [ 0.26513894, 0.46808455, 0.73568319],
+ [ 0.27322194, 0.47221835, 0.73575497],
+ [ 0.28117543, 0.47634598, 0.73588332],
+ [ 0.28901021, 0.48046826, 0.73606686],
+ [ 0.2967358 , 0.48458597, 0.73630433],
+ [ 0.30436071, 0.48869986, 0.73659451],
+ [ 0.3118955 , 0.49281055, 0.73693255],
+ [ 0.31935389, 0.49691847, 0.73730851],
+ [ 0.32672701, 0.5010247 , 0.73774013],
+ [ 0.33402607, 0.50512971, 0.73821941],
+ [ 0.34125337, 0.50923419, 0.73874905],
+ [ 0.34840921, 0.51333892, 0.73933402],
+ [ 0.35551826, 0.51744353, 0.73994642],
+ [ 0.3625676 , 0.52154929, 0.74060763],
+ [ 0.36956356, 0.52565656, 0.74131327],
+ [ 0.37649902, 0.52976642, 0.74207698],
+ [ 0.38340273, 0.53387791, 0.74286286],
+ [ 0.39025859, 0.53799253, 0.7436962 ],
+ [ 0.39706821, 0.54211081, 0.744578 ],
+ [ 0.40384046, 0.54623277, 0.74549872],
+ [ 0.41058241, 0.55035849, 0.74645094],
+ [ 0.41728385, 0.55448919, 0.74745174],
+ [ 0.42395178, 0.55862494, 0.74849357],
+ [ 0.4305964 , 0.56276546, 0.74956387],
+ [ 0.4372044 , 0.56691228, 0.75068412],
+ [ 0.4437909 , 0.57106468, 0.75183427],
+ [ 0.45035117, 0.5752235 , 0.75302312],
+ [ 0.45687824, 0.57938983, 0.75426297],
+ [ 0.46339713, 0.58356191, 0.75551816],
+ [ 0.46988778, 0.58774195, 0.75682037],
+ [ 0.47635605, 0.59192986, 0.75816245],
+ [ 0.48281101, 0.5961252 , 0.75953212],
+ [ 0.4892374 , 0.60032986, 0.76095418],
+ [ 0.49566225, 0.60454154, 0.76238852],
+ [ 0.50206137, 0.60876307, 0.76387371],
+ [ 0.50845128, 0.61299312, 0.76538551],
+ [ 0.5148258 , 0.61723272, 0.76693475],
+ [ 0.52118385, 0.62148236, 0.76852436],
+ [ 0.52753571, 0.62574126, 0.77013939],
+ [ 0.53386831, 0.63001125, 0.77180152],
+ [ 0.54020159, 0.63429038, 0.7734803 ],
+ [ 0.54651272, 0.63858165, 0.77521306],
+ [ 0.55282975, 0.64288207, 0.77695608],
+ [ 0.55912585, 0.64719519, 0.77875327],
+ [ 0.56542599, 0.65151828, 0.78056551],
+ [ 0.57170924, 0.65585426, 0.78242747],
+ [ 0.57799572, 0.6602009 , 0.78430751],
+ [ 0.58426817, 0.66456073, 0.78623458],
+ [ 0.590544 , 0.66893178, 0.78818117],
+ [ 0.59680758, 0.67331643, 0.79017369],
+ [ 0.60307553, 0.67771273, 0.79218572],
+ [ 0.60934065, 0.68212194, 0.79422987],
+ [ 0.61559495, 0.68654548, 0.7963202 ],
+ [ 0.62185554, 0.69098125, 0.79842918],
+ [ 0.62810662, 0.69543176, 0.80058381],
+ [ 0.63436425, 0.69989499, 0.80275812],
+ [ 0.64061445, 0.70437326, 0.80497621],
+ [ 0.6468706 , 0.70886488, 0.80721641],
+ [ 0.65312213, 0.7133717 , 0.80949719],
+ [ 0.65937818, 0.71789261, 0.81180392],
+ [ 0.66563334, 0.72242871, 0.81414642],
+ [ 0.67189155, 0.72697967, 0.81651872],
+ [ 0.67815314, 0.73154569, 0.81892097],
+ [ 0.68441395, 0.73612771, 0.82136094],
+ [ 0.69068321, 0.74072452, 0.82382353],
+ [ 0.69694776, 0.7453385 , 0.82633199],
+ [ 0.70322431, 0.74996721, 0.8288583 ],
+ [ 0.70949595, 0.75461368, 0.83143221],
+ [ 0.7157774 , 0.75927574, 0.83402904],
+ [ 0.72206299, 0.76395461, 0.83665922],
+ [ 0.72835227, 0.76865061, 0.8393242 ],
+ [ 0.73465238, 0.7733628 , 0.84201224],
+ [ 0.74094862, 0.77809393, 0.84474951],
+ [ 0.74725683, 0.78284158, 0.84750915],
+ [ 0.75357103, 0.78760701, 0.85030217],
+ [ 0.75988961, 0.79239077, 0.85313207],
+ [ 0.76621987, 0.79719185, 0.85598668],
+ [ 0.77255045, 0.8020125 , 0.85888658],
+ [ 0.77889241, 0.80685102, 0.86181298],
+ [ 0.78524572, 0.81170768, 0.86476656],
+ [ 0.79159841, 0.81658489, 0.86776906],
+ [ 0.79796459, 0.82148036, 0.8707962 ],
+ [ 0.80434168, 0.82639479, 0.87385315],
+ [ 0.8107221 , 0.83132983, 0.87695392],
+ [ 0.81711301, 0.8362844 , 0.88008641],
+ [ 0.82351479, 0.84125863, 0.88325045],
+ [ 0.82992772, 0.84625263, 0.88644594],
+ [ 0.83634359, 0.85126806, 0.8896878 ],
+ [ 0.84277295, 0.85630293, 0.89295721],
+ [ 0.84921192, 0.86135782, 0.89626076],
+ [ 0.85566206, 0.866432 , 0.89959467],
+ [ 0.86211514, 0.87152627, 0.90297183],
+ [ 0.86857483, 0.87663856, 0.90638248],
+ [ 0.87504231, 0.88176648, 0.90981938],
+ [ 0.88151194, 0.88690782, 0.91328493],
+ [ 0.88797938, 0.89205857, 0.91677544],
+ [ 0.89443865, 0.89721298, 0.9202854 ],
+ [ 0.90088204, 0.90236294, 0.92380601],
+ [ 0.90729768, 0.90749778, 0.92732797],
+ [ 0.91367037, 0.91260329, 0.93083814],
+ [ 0.91998105, 0.91766106, 0.93431861],
+ [ 0.92620596, 0.92264789, 0.93774647],
+ [ 0.93231683, 0.9275351 , 0.94109192],
+ [ 0.93827772, 0.9322888 , 0.94432312],
+ [ 0.94404755, 0.93686925, 0.94740137],
+ [ 0.94958284, 0.94123072, 0.95027696],
+ [ 0.95482682, 0.9453245 , 0.95291103],
+ [ 0.9597248 , 0.94909728, 0.95525103],
+ [ 0.96422552, 0.95249273, 0.95723271],
+ [ 0.96826161, 0.95545812, 0.95882188],
+ [ 0.97178458, 0.95793984, 0.95995705],
+ [ 0.97474105, 0.95989142, 0.96059997],
+ [ 0.97708604, 0.96127366, 0.96071853],
+ [ 0.97877855, 0.96205832, 0.96030095],
+ [ 0.97978484, 0.96222949, 0.95935496],
+ [ 0.9805997 , 0.96155216, 0.95813083],
+ [ 0.98152619, 0.95993719, 0.95639322],
+ [ 0.9819726 , 0.95766608, 0.95399269],
+ [ 0.98191855, 0.9547873 , 0.95098107],
+ [ 0.98138514, 0.95134771, 0.94740644],
+ [ 0.98040845, 0.94739906, 0.94332125],
+ [ 0.97902107, 0.94300131, 0.93878672],
+ [ 0.97729348, 0.93820409, 0.93385135],
+ [ 0.9752533 , 0.933073 , 0.92858252],
+ [ 0.97297834, 0.92765261, 0.92302309],
+ [ 0.97049104, 0.92200317, 0.91723505],
+ [ 0.96784372, 0.91616744, 0.91126063],
+ [ 0.96507281, 0.91018664, 0.90514124],
+ [ 0.96222034, 0.90409203, 0.89890756],
+ [ 0.9593079 , 0.89791478, 0.89259122],
+ [ 0.95635626, 0.89167908, 0.88621654],
+ [ 0.95338303, 0.88540373, 0.87980238],
+ [ 0.95040174, 0.87910333, 0.87336339],
+ [ 0.94742246, 0.87278899, 0.86691076],
+ [ 0.94445249, 0.86646893, 0.86045277],
+ [ 0.94150476, 0.86014606, 0.85399191],
+ [ 0.93857394, 0.85382798, 0.84753642],
+ [ 0.93566206, 0.84751766, 0.84108935],
+ [ 0.93277194, 0.8412164 , 0.83465197],
+ [ 0.92990106, 0.83492672, 0.82822708],
+ [ 0.92704736, 0.82865028, 0.82181656],
+ [ 0.92422703, 0.82238092, 0.81541333],
+ [ 0.92142581, 0.81612448, 0.80902415],
+ [ 0.91864501, 0.80988032, 0.80264838],
+ [ 0.91587578, 0.80365187, 0.79629001],
+ [ 0.9131367 , 0.79743115, 0.78994 ],
+ [ 0.91041602, 0.79122265, 0.78360361],
+ [ 0.90771071, 0.78502727, 0.77728196],
+ [ 0.90501581, 0.77884674, 0.7709771 ],
+ [ 0.90235365, 0.77267117, 0.76467793],
+ [ 0.8997019 , 0.76650962, 0.75839484],
+ [ 0.89705346, 0.76036481, 0.752131 ],
+ [ 0.89444021, 0.75422253, 0.74587047],
+ [ 0.89183355, 0.74809474, 0.73962689],
+ [ 0.88923216, 0.74198168, 0.73340061],
+ [ 0.88665892, 0.73587283, 0.72717995],
+ [ 0.88408839, 0.72977904, 0.72097718],
+ [ 0.88153537, 0.72369332, 0.71478461],
+ [ 0.87899389, 0.7176179 , 0.70860487],
+ [ 0.87645157, 0.71155805, 0.7024439 ],
+ [ 0.8739399 , 0.70549893, 0.6962854 ],
+ [ 0.87142626, 0.6994551 , 0.69014561],
+ [ 0.8689268 , 0.69341868, 0.68401597],
+ [ 0.86643562, 0.687392 , 0.67789917],
+ [ 0.86394434, 0.68137863, 0.67179927],
+ [ 0.86147586, 0.67536728, 0.665704 ],
+ [ 0.85899928, 0.66937226, 0.6596292 ],
+ [ 0.85654668, 0.66337773, 0.6535577 ],
+ [ 0.85408818, 0.65739772, 0.64750494],
+ [ 0.85164413, 0.65142189, 0.64145983],
+ [ 0.84920091, 0.6454565 , 0.63542932],
+ [ 0.84676427, 0.63949827, 0.62941 ],
+ [ 0.84433231, 0.63354773, 0.62340261],
+ [ 0.84190106, 0.62760645, 0.61740899],
+ [ 0.83947935, 0.62166951, 0.61142404],
+ [ 0.8370538 , 0.61574332, 0.60545478],
+ [ 0.83463975, 0.60981951, 0.59949247],
+ [ 0.83221877, 0.60390724, 0.593547 ],
+ [ 0.82980985, 0.59799607, 0.58760751],
+ [ 0.82740268, 0.59209095, 0.58167944],
+ [ 0.82498638, 0.5861973 , 0.57576866],
+ [ 0.82258181, 0.5803034 , 0.56986307],
+ [ 0.82016611, 0.57442123, 0.56397539],
+ [ 0.81776305, 0.56853725, 0.55809173],
+ [ 0.81534551, 0.56266602, 0.55222741],
+ [ 0.81294293, 0.55679056, 0.5463651 ],
+ [ 0.81052113, 0.55092973, 0.54052443],
+ [ 0.80811509, 0.54506305, 0.53468464],
+ [ 0.80568952, 0.53921036, 0.52886622],
+ [ 0.80327506, 0.53335335, 0.52305077],
+ [ 0.80084727, 0.52750583, 0.51725256],
+ [ 0.79842217, 0.5216578 , 0.51146173],
+ [ 0.79599382, 0.51581223, 0.50568155],
+ [ 0.79355781, 0.50997127, 0.49991444],
+ [ 0.79112596, 0.50412707, 0.49415289],
+ [ 0.78867442, 0.49829386, 0.48841129],
+ [ 0.7862306 , 0.49245398, 0.48267247],
+ [ 0.7837687 , 0.48662309, 0.47695216],
+ [ 0.78130809, 0.4807883 , 0.47123805],
+ [ 0.77884467, 0.47495151, 0.46553236],
+ [ 0.77636283, 0.46912235, 0.45984473],
+ [ 0.77388383, 0.46328617, 0.45416141],
+ [ 0.77138912, 0.45745466, 0.44849398],
+ [ 0.76888874, 0.45162042, 0.44283573],
+ [ 0.76638802, 0.44577901, 0.43718292],
+ [ 0.76386116, 0.43994762, 0.43155211],
+ [ 0.76133542, 0.43410655, 0.42592523],
+ [ 0.75880631, 0.42825801, 0.42030488],
+ [ 0.75624913, 0.42241905, 0.41470727],
+ [ 0.7536919 , 0.41656866, 0.40911347],
+ [ 0.75112748, 0.41071104, 0.40352792],
+ [ 0.74854331, 0.40485474, 0.3979589 ],
+ [ 0.74594723, 0.39899309, 0.39240088],
+ [ 0.74334332, 0.39312199, 0.38685075],
+ [ 0.74073277, 0.38723941, 0.3813074 ],
+ [ 0.73809409, 0.38136133, 0.37578553],
+ [ 0.73544692, 0.37547129, 0.37027123],
+ [ 0.73278943, 0.36956954, 0.36476549],
+ [ 0.73011829, 0.36365761, 0.35927038],
+ [ 0.72743485, 0.35773314, 0.35378465],
+ [ 0.72472722, 0.35180504, 0.34831662],
+ [ 0.72200473, 0.34586421, 0.34285937],
+ [ 0.71927052, 0.33990649, 0.33741033],
+ [ 0.71652049, 0.33393396, 0.33197219],
+ [ 0.71375362, 0.32794602, 0.32654545],
+ [ 0.71096951, 0.32194148, 0.32113016],
+ [ 0.70816772, 0.31591904, 0.31572637],
+ [ 0.70534784, 0.30987734, 0.31033414],
+ [ 0.70250944, 0.30381489, 0.30495353],
+ [ 0.69965211, 0.2977301 , 0.2995846 ],
+ [ 0.6967754 , 0.29162126, 0.29422741],
+ [ 0.69388446, 0.28548074, 0.28887769],
+ [ 0.69097561, 0.2793096 , 0.28353795],
+ [ 0.68803513, 0.27311993, 0.27821876],
+ [ 0.6850794 , 0.26689144, 0.27290694],
+ [ 0.682108 , 0.26062114, 0.26760246],
+ [ 0.67911013, 0.2543177 , 0.26231367],
+ [ 0.67609393, 0.24796818, 0.25703372],
+ [ 0.67305921, 0.24156846, 0.25176238],
+ [ 0.67000176, 0.23511902, 0.24650278],
+ [ 0.66693423, 0.22859879, 0.24124404],
+ [ 0.6638441 , 0.22201742, 0.2359961 ],
+ [ 0.66080672, 0.21526712, 0.23069468]
+]
+
+
+_icefire_lut = [
+ [ 0.73936227, 0.90443867, 0.85757238],
+ [ 0.72888063, 0.89639109, 0.85488394],
+ [ 0.71834255, 0.88842162, 0.8521605 ],
+ [ 0.70773866, 0.88052939, 0.849422 ],
+ [ 0.69706215, 0.87271313, 0.84668315],
+ [ 0.68629021, 0.86497329, 0.84398721],
+ [ 0.67543654, 0.85730617, 0.84130969],
+ [ 0.66448539, 0.84971123, 0.83868005],
+ [ 0.65342679, 0.84218728, 0.83611512],
+ [ 0.64231804, 0.83471867, 0.83358584],
+ [ 0.63117745, 0.827294 , 0.83113431],
+ [ 0.62000484, 0.81991069, 0.82876741],
+ [ 0.60879435, 0.81256797, 0.82648905],
+ [ 0.59754118, 0.80526458, 0.82430414],
+ [ 0.58624247, 0.79799884, 0.82221573],
+ [ 0.57489525, 0.7907688 , 0.82022901],
+ [ 0.56349779, 0.78357215, 0.81834861],
+ [ 0.55204294, 0.77640827, 0.81657563],
+ [ 0.54052516, 0.76927562, 0.81491462],
+ [ 0.52894085, 0.76217215, 0.81336913],
+ [ 0.51728854, 0.75509528, 0.81194156],
+ [ 0.50555676, 0.74804469, 0.81063503],
+ [ 0.49373871, 0.7410187 , 0.80945242],
+ [ 0.48183174, 0.73401449, 0.80839675],
+ [ 0.46982587, 0.72703075, 0.80747097],
+ [ 0.45770893, 0.72006648, 0.80667756],
+ [ 0.44547249, 0.71311941, 0.80601991],
+ [ 0.43318643, 0.70617126, 0.80549278],
+ [ 0.42110294, 0.69916972, 0.80506683],
+ [ 0.40925101, 0.69211059, 0.80473246],
+ [ 0.3976693 , 0.68498786, 0.80448272],
+ [ 0.38632002, 0.67781125, 0.80431024],
+ [ 0.37523981, 0.67057537, 0.80420832],
+ [ 0.36442578, 0.66328229, 0.80417474],
+ [ 0.35385939, 0.65593699, 0.80420591],
+ [ 0.34358916, 0.64853177, 0.8043 ],
+ [ 0.33355526, 0.64107876, 0.80445484],
+ [ 0.32383062, 0.63356578, 0.80467091],
+ [ 0.31434372, 0.62600624, 0.8049475 ],
+ [ 0.30516161, 0.618389 , 0.80528692],
+ [ 0.29623491, 0.61072284, 0.80569021],
+ [ 0.28759072, 0.60300319, 0.80616055],
+ [ 0.27923924, 0.59522877, 0.80669803],
+ [ 0.27114651, 0.5874047 , 0.80730545],
+ [ 0.26337153, 0.57952055, 0.80799113],
+ [ 0.25588696, 0.57157984, 0.80875922],
+ [ 0.248686 , 0.56358255, 0.80961366],
+ [ 0.24180668, 0.55552289, 0.81055123],
+ [ 0.23526251, 0.54739477, 0.8115939 ],
+ [ 0.22921445, 0.53918506, 0.81267292],
+ [ 0.22397687, 0.53086094, 0.8137141 ],
+ [ 0.21977058, 0.52241482, 0.81457651],
+ [ 0.21658989, 0.51384321, 0.81528511],
+ [ 0.21452772, 0.50514155, 0.81577278],
+ [ 0.21372783, 0.49630865, 0.81589566],
+ [ 0.21409503, 0.48734861, 0.81566163],
+ [ 0.2157176 , 0.47827123, 0.81487615],
+ [ 0.21842857, 0.46909168, 0.81351614],
+ [ 0.22211705, 0.45983212, 0.81146983],
+ [ 0.22665681, 0.45052233, 0.80860217],
+ [ 0.23176013, 0.44119137, 0.80494325],
+ [ 0.23727775, 0.43187704, 0.80038017],
+ [ 0.24298285, 0.42261123, 0.79493267],
+ [ 0.24865068, 0.41341842, 0.78869164],
+ [ 0.25423116, 0.40433127, 0.78155831],
+ [ 0.25950239, 0.39535521, 0.77376848],
+ [ 0.2644736 , 0.38651212, 0.76524809],
+ [ 0.26901584, 0.37779582, 0.75621942],
+ [ 0.27318141, 0.36922056, 0.746605 ],
+ [ 0.27690355, 0.3607736 , 0.73659374],
+ [ 0.28023585, 0.35244234, 0.72622103],
+ [ 0.28306009, 0.34438449, 0.71500731],
+ [ 0.28535896, 0.33660243, 0.70303975],
+ [ 0.28708711, 0.32912157, 0.69034504],
+ [ 0.28816354, 0.32200604, 0.67684067],
+ [ 0.28862749, 0.31519824, 0.66278813],
+ [ 0.28847904, 0.30869064, 0.6482815 ],
+ [ 0.28770912, 0.30250126, 0.63331265],
+ [ 0.28640325, 0.29655509, 0.61811374],
+ [ 0.28458943, 0.29082155, 0.60280913],
+ [ 0.28233561, 0.28527482, 0.58742866],
+ [ 0.27967038, 0.2798938 , 0.57204225],
+ [ 0.27665361, 0.27465357, 0.55667809],
+ [ 0.27332564, 0.2695165 , 0.54145387],
+ [ 0.26973851, 0.26447054, 0.52634916],
+ [ 0.2659204 , 0.25949691, 0.511417 ],
+ [ 0.26190145, 0.25458123, 0.49668768],
+ [ 0.2577151 , 0.24971691, 0.48214874],
+ [ 0.25337618, 0.24490494, 0.46778758],
+ [ 0.24890842, 0.24013332, 0.45363816],
+ [ 0.24433654, 0.23539226, 0.4397245 ],
+ [ 0.23967922, 0.23067729, 0.4260591 ],
+ [ 0.23495608, 0.22598894, 0.41262952],
+ [ 0.23018113, 0.22132414, 0.39945577],
+ [ 0.22534609, 0.21670847, 0.38645794],
+ [ 0.22048761, 0.21211723, 0.37372555],
+ [ 0.2156198 , 0.20755389, 0.36125301],
+ [ 0.21074637, 0.20302717, 0.34903192],
+ [ 0.20586893, 0.19855368, 0.33701661],
+ [ 0.20101757, 0.19411573, 0.32529173],
+ [ 0.19619947, 0.18972425, 0.31383846],
+ [ 0.19140726, 0.18540157, 0.30260777],
+ [ 0.1866769 , 0.1811332 , 0.29166583],
+ [ 0.18201285, 0.17694992, 0.28088776],
+ [ 0.17745228, 0.17282141, 0.27044211],
+ [ 0.17300684, 0.16876921, 0.26024893],
+ [ 0.16868273, 0.16479861, 0.25034479],
+ [ 0.16448691, 0.16091728, 0.24075373],
+ [ 0.16043195, 0.15714351, 0.23141745],
+ [ 0.15652427, 0.15348248, 0.22238175],
+ [ 0.15277065, 0.14994111, 0.21368395],
+ [ 0.14918274, 0.14653431, 0.20529486],
+ [ 0.14577095, 0.14327403, 0.19720829],
+ [ 0.14254381, 0.14016944, 0.18944326],
+ [ 0.13951035, 0.13723063, 0.18201072],
+ [ 0.13667798, 0.13446606, 0.17493774],
+ [ 0.13405762, 0.13188822, 0.16820842],
+ [ 0.13165767, 0.12950667, 0.16183275],
+ [ 0.12948748, 0.12733187, 0.15580631],
+ [ 0.12755435, 0.1253723 , 0.15014098],
+ [ 0.12586516, 0.12363617, 0.1448459 ],
+ [ 0.12442647, 0.12213143, 0.13992571],
+ [ 0.12324241, 0.12086419, 0.13539995],
+ [ 0.12232067, 0.11984278, 0.13124644],
+ [ 0.12166209, 0.11907077, 0.12749671],
+ [ 0.12126982, 0.11855309, 0.12415079],
+ [ 0.12114244, 0.11829179, 0.1212385 ],
+ [ 0.12127766, 0.11828837, 0.11878534],
+ [ 0.12284806, 0.1179729 , 0.11772022],
+ [ 0.12619498, 0.11721796, 0.11770203],
+ [ 0.129968 , 0.11663788, 0.11792377],
+ [ 0.13410011, 0.11625146, 0.11839138],
+ [ 0.13855459, 0.11606618, 0.11910584],
+ [ 0.14333775, 0.11607038, 0.1200606 ],
+ [ 0.148417 , 0.11626929, 0.12125453],
+ [ 0.15377389, 0.11666192, 0.12268364],
+ [ 0.15941427, 0.11723486, 0.12433911],
+ [ 0.16533376, 0.11797856, 0.12621303],
+ [ 0.17152547, 0.11888403, 0.12829735],
+ [ 0.17797765, 0.11994436, 0.13058435],
+ [ 0.18468769, 0.12114722, 0.13306426],
+ [ 0.19165663, 0.12247737, 0.13572616],
+ [ 0.19884415, 0.12394381, 0.1385669 ],
+ [ 0.20627181, 0.12551883, 0.14157124],
+ [ 0.21394877, 0.12718055, 0.14472604],
+ [ 0.22184572, 0.12893119, 0.14802579],
+ [ 0.22994394, 0.13076731, 0.15146314],
+ [ 0.23823937, 0.13267611, 0.15502793],
+ [ 0.24676041, 0.13462172, 0.15870321],
+ [ 0.25546457, 0.13661751, 0.16248722],
+ [ 0.26433628, 0.13865956, 0.16637301],
+ [ 0.27341345, 0.14070412, 0.17034221],
+ [ 0.28264773, 0.14277192, 0.1743957 ],
+ [ 0.29202272, 0.14486161, 0.17852793],
+ [ 0.30159648, 0.14691224, 0.1827169 ],
+ [ 0.31129002, 0.14897583, 0.18695213],
+ [ 0.32111555, 0.15103351, 0.19119629],
+ [ 0.33107961, 0.1530674 , 0.19543758],
+ [ 0.34119892, 0.15504762, 0.1996803 ],
+ [ 0.35142388, 0.15701131, 0.20389086],
+ [ 0.36178937, 0.1589124 , 0.20807639],
+ [ 0.37229381, 0.16073993, 0.21223189],
+ [ 0.38288348, 0.16254006, 0.2163249 ],
+ [ 0.39359592, 0.16426336, 0.22036577],
+ [ 0.40444332, 0.16588767, 0.22434027],
+ [ 0.41537995, 0.16745325, 0.2282297 ],
+ [ 0.42640867, 0.16894939, 0.23202755],
+ [ 0.43754706, 0.17034847, 0.23572899],
+ [ 0.44878564, 0.1716535 , 0.23932344],
+ [ 0.4601126 , 0.17287365, 0.24278607],
+ [ 0.47151732, 0.17401641, 0.24610337],
+ [ 0.48300689, 0.17506676, 0.2492737 ],
+ [ 0.49458302, 0.17601892, 0.25227688],
+ [ 0.50623876, 0.17687777, 0.255096 ],
+ [ 0.5179623 , 0.17765528, 0.2577162 ],
+ [ 0.52975234, 0.17835232, 0.2601134 ],
+ [ 0.54159776, 0.17898292, 0.26226847],
+ [ 0.55348804, 0.17956232, 0.26416003],
+ [ 0.56541729, 0.18010175, 0.26575971],
+ [ 0.57736669, 0.180631 , 0.26704888],
+ [ 0.58932081, 0.18117827, 0.26800409],
+ [ 0.60127582, 0.18175888, 0.26858488],
+ [ 0.61319563, 0.1824336 , 0.2687872 ],
+ [ 0.62506376, 0.18324015, 0.26858301],
+ [ 0.63681202, 0.18430173, 0.26795276],
+ [ 0.64842603, 0.18565472, 0.26689463],
+ [ 0.65988195, 0.18734638, 0.26543435],
+ [ 0.67111966, 0.18948885, 0.26357955],
+ [ 0.68209194, 0.19216636, 0.26137175],
+ [ 0.69281185, 0.19535326, 0.25887063],
+ [ 0.70335022, 0.19891271, 0.25617971],
+ [ 0.71375229, 0.20276438, 0.25331365],
+ [ 0.72401436, 0.20691287, 0.25027366],
+ [ 0.73407638, 0.21145051, 0.24710661],
+ [ 0.74396983, 0.21631913, 0.24380715],
+ [ 0.75361506, 0.22163653, 0.24043996],
+ [ 0.7630579 , 0.22731637, 0.23700095],
+ [ 0.77222228, 0.23346231, 0.23356628],
+ [ 0.78115441, 0.23998404, 0.23013825],
+ [ 0.78979746, 0.24694858, 0.22678822],
+ [ 0.79819286, 0.25427223, 0.22352658],
+ [ 0.80630444, 0.26198807, 0.22040877],
+ [ 0.81417437, 0.27001406, 0.21744645],
+ [ 0.82177364, 0.27837336, 0.21468316],
+ [ 0.82915955, 0.28696963, 0.21210766],
+ [ 0.83628628, 0.2958499 , 0.20977813],
+ [ 0.84322168, 0.30491136, 0.20766435],
+ [ 0.84995458, 0.31415945, 0.2057863 ],
+ [ 0.85648867, 0.32358058, 0.20415327],
+ [ 0.86286243, 0.33312058, 0.20274969],
+ [ 0.86908321, 0.34276705, 0.20157271],
+ [ 0.87512876, 0.3525416 , 0.20064949],
+ [ 0.88100349, 0.36243385, 0.19999078],
+ [ 0.8866469 , 0.37249496, 0.1997976 ],
+ [ 0.89203964, 0.38273475, 0.20013431],
+ [ 0.89713496, 0.39318156, 0.20121514],
+ [ 0.90195099, 0.40380687, 0.20301555],
+ [ 0.90648379, 0.41460191, 0.20558847],
+ [ 0.9106967 , 0.42557857, 0.20918529],
+ [ 0.91463791, 0.43668557, 0.21367954],
+ [ 0.91830723, 0.44790913, 0.21916352],
+ [ 0.92171507, 0.45922856, 0.22568002],
+ [ 0.92491786, 0.4705936 , 0.23308207],
+ [ 0.92790792, 0.48200153, 0.24145932],
+ [ 0.93073701, 0.49341219, 0.25065486],
+ [ 0.93343918, 0.5048017 , 0.26056148],
+ [ 0.93602064, 0.51616486, 0.27118485],
+ [ 0.93850535, 0.52748892, 0.28242464],
+ [ 0.94092933, 0.53875462, 0.29416042],
+ [ 0.94330011, 0.5499628 , 0.30634189],
+ [ 0.94563159, 0.56110987, 0.31891624],
+ [ 0.94792955, 0.57219822, 0.33184256],
+ [ 0.95020929, 0.5832232 , 0.34508419],
+ [ 0.95247324, 0.59419035, 0.35859866],
+ [ 0.95471709, 0.60510869, 0.37236035],
+ [ 0.95698411, 0.61595766, 0.38629631],
+ [ 0.95923863, 0.62676473, 0.40043317],
+ [ 0.9615041 , 0.6375203 , 0.41474106],
+ [ 0.96371553, 0.64826619, 0.42928335],
+ [ 0.96591497, 0.65899621, 0.44380444],
+ [ 0.96809871, 0.66971662, 0.45830232],
+ [ 0.9702495 , 0.6804394 , 0.47280492],
+ [ 0.9723881 , 0.69115622, 0.48729272],
+ [ 0.97450723, 0.70187358, 0.50178034],
+ [ 0.9766108 , 0.712592 , 0.51626837],
+ [ 0.97871716, 0.72330511, 0.53074053],
+ [ 0.98082222, 0.73401769, 0.54520694],
+ [ 0.9829001 , 0.74474445, 0.5597019 ],
+ [ 0.98497466, 0.75547635, 0.57420239],
+ [ 0.98705581, 0.76621129, 0.58870185],
+ [ 0.98913325, 0.77695637, 0.60321626],
+ [ 0.99119918, 0.78771716, 0.61775821],
+ [ 0.9932672 , 0.79848979, 0.63231691],
+ [ 0.99535958, 0.80926704, 0.64687278],
+ [ 0.99740544, 0.82008078, 0.66150571],
+ [ 0.9992197 , 0.83100723, 0.6764127 ]
+]
+
+
+_flare_lut = [
+ [0.92907237, 0.68878959, 0.50411509],
+ [0.92891402, 0.68494686, 0.50173994],
+ [0.92864754, 0.68116207, 0.4993754],
+ [0.92836112, 0.67738527, 0.49701572],
+ [0.9280599, 0.67361354, 0.49466044],
+ [0.92775569, 0.66983999, 0.49230866],
+ [0.9274375, 0.66607098, 0.48996097],
+ [0.927111, 0.66230315, 0.48761688],
+ [0.92677996, 0.6585342, 0.485276],
+ [0.92644317, 0.65476476, 0.48293832],
+ [0.92609759, 0.65099658, 0.48060392],
+ [0.925747, 0.64722729, 0.47827244],
+ [0.92539502, 0.64345456, 0.47594352],
+ [0.92503106, 0.6396848, 0.47361782],
+ [0.92466877, 0.6359095, 0.47129427],
+ [0.92429828, 0.63213463, 0.46897349],
+ [0.92392172, 0.62835879, 0.46665526],
+ [0.92354597, 0.62457749, 0.46433898],
+ [0.9231622, 0.6207962, 0.46202524],
+ [0.92277222, 0.61701365, 0.45971384],
+ [0.92237978, 0.61322733, 0.45740444],
+ [0.92198615, 0.60943622, 0.45509686],
+ [0.92158735, 0.60564276, 0.45279137],
+ [0.92118373, 0.60184659, 0.45048789],
+ [0.92077582, 0.59804722, 0.44818634],
+ [0.92036413, 0.59424414, 0.44588663],
+ [0.91994924, 0.5904368, 0.44358868],
+ [0.91952943, 0.58662619, 0.4412926],
+ [0.91910675, 0.58281075, 0.43899817],
+ [0.91868096, 0.57899046, 0.4367054],
+ [0.91825103, 0.57516584, 0.43441436],
+ [0.91781857, 0.57133556, 0.43212486],
+ [0.9173814, 0.56750099, 0.4298371],
+ [0.91694139, 0.56366058, 0.42755089],
+ [0.91649756, 0.55981483, 0.42526631],
+ [0.91604942, 0.55596387, 0.42298339],
+ [0.9155979, 0.55210684, 0.42070204],
+ [0.9151409, 0.54824485, 0.4184247],
+ [0.91466138, 0.54438817, 0.41617858],
+ [0.91416896, 0.54052962, 0.41396347],
+ [0.91366559, 0.53666778, 0.41177769],
+ [0.91315173, 0.53280208, 0.40962196],
+ [0.91262605, 0.52893336, 0.40749715],
+ [0.91208866, 0.52506133, 0.40540404],
+ [0.91153952, 0.52118582, 0.40334346],
+ [0.91097732, 0.51730767, 0.4013163],
+ [0.910403, 0.51342591, 0.39932342],
+ [0.90981494, 0.50954168, 0.39736571],
+ [0.90921368, 0.5056543, 0.39544411],
+ [0.90859797, 0.50176463, 0.39355952],
+ [0.90796841, 0.49787195, 0.39171297],
+ [0.90732341, 0.4939774, 0.38990532],
+ [0.90666382, 0.49008006, 0.38813773],
+ [0.90598815, 0.486181, 0.38641107],
+ [0.90529624, 0.48228017, 0.38472641],
+ [0.90458808, 0.47837738, 0.38308489],
+ [0.90386248, 0.47447348, 0.38148746],
+ [0.90311921, 0.4705685, 0.37993524],
+ [0.90235809, 0.46666239, 0.37842943],
+ [0.90157824, 0.46275577, 0.37697105],
+ [0.90077904, 0.45884905, 0.37556121],
+ [0.89995995, 0.45494253, 0.37420106],
+ [0.89912041, 0.4510366, 0.37289175],
+ [0.8982602, 0.44713126, 0.37163458],
+ [0.89737819, 0.44322747, 0.37043052],
+ [0.89647387, 0.43932557, 0.36928078],
+ [0.89554477, 0.43542759, 0.36818855],
+ [0.89458871, 0.4315354, 0.36715654],
+ [0.89360794, 0.42764714, 0.36618273],
+ [0.89260152, 0.42376366, 0.36526813],
+ [0.8915687, 0.41988565, 0.36441384],
+ [0.89050882, 0.41601371, 0.36362102],
+ [0.8894159, 0.41215334, 0.36289639],
+ [0.888292, 0.40830288, 0.36223756],
+ [0.88713784, 0.40446193, 0.36164328],
+ [0.88595253, 0.40063149, 0.36111438],
+ [0.88473115, 0.39681635, 0.3606566],
+ [0.88347246, 0.39301805, 0.36027074],
+ [0.88217931, 0.38923439, 0.35995244],
+ [0.880851, 0.38546632, 0.35970244],
+ [0.87947728, 0.38172422, 0.35953127],
+ [0.87806542, 0.37800172, 0.35942941],
+ [0.87661509, 0.37429964, 0.35939659],
+ [0.87511668, 0.37062819, 0.35944178],
+ [0.87357554, 0.36698279, 0.35955811],
+ [0.87199254, 0.3633634, 0.35974223],
+ [0.87035691, 0.35978174, 0.36000516],
+ [0.86867647, 0.35623087, 0.36033559],
+ [0.86694949, 0.35271349, 0.36073358],
+ [0.86516775, 0.34923921, 0.36120624],
+ [0.86333996, 0.34580008, 0.36174113],
+ [0.86145909, 0.3424046, 0.36234402],
+ [0.85952586, 0.33905327, 0.36301129],
+ [0.85754536, 0.33574168, 0.36373567],
+ [0.855514, 0.33247568, 0.36451271],
+ [0.85344392, 0.32924217, 0.36533344],
+ [0.8513284, 0.32604977, 0.36620106],
+ [0.84916723, 0.32289973, 0.36711424],
+ [0.84696243, 0.31979068, 0.36806976],
+ [0.84470627, 0.31673295, 0.36907066],
+ [0.84240761, 0.31371695, 0.37010969],
+ [0.84005337, 0.31075974, 0.37119284],
+ [0.83765537, 0.30784814, 0.3723105],
+ [0.83520234, 0.30499724, 0.37346726],
+ [0.83270291, 0.30219766, 0.37465552],
+ [0.83014895, 0.29946081, 0.37587769],
+ [0.82754694, 0.29677989, 0.37712733],
+ [0.82489111, 0.29416352, 0.37840532],
+ [0.82218644, 0.29160665, 0.37970606],
+ [0.81942908, 0.28911553, 0.38102921],
+ [0.81662276, 0.28668665, 0.38236999],
+ [0.81376555, 0.28432371, 0.383727],
+ [0.81085964, 0.28202508, 0.38509649],
+ [0.8079055, 0.27979128, 0.38647583],
+ [0.80490309, 0.27762348, 0.3878626],
+ [0.80185613, 0.2755178, 0.38925253],
+ [0.79876118, 0.27347974, 0.39064559],
+ [0.79562644, 0.27149928, 0.39203532],
+ [0.79244362, 0.2695883, 0.39342447],
+ [0.78922456, 0.26773176, 0.3948046],
+ [0.78596161, 0.26594053, 0.39617873],
+ [0.7826624, 0.26420493, 0.39754146],
+ [0.77932717, 0.26252522, 0.39889102],
+ [0.77595363, 0.2609049, 0.4002279],
+ [0.77254999, 0.25933319, 0.40154704],
+ [0.76911107, 0.25781758, 0.40284959],
+ [0.76564158, 0.25635173, 0.40413341],
+ [0.76214598, 0.25492998, 0.40539471],
+ [0.75861834, 0.25356035, 0.40663694],
+ [0.75506533, 0.25223402, 0.40785559],
+ [0.75148963, 0.2509473, 0.40904966],
+ [0.74788835, 0.24970413, 0.41022028],
+ [0.74426345, 0.24850191, 0.41136599],
+ [0.74061927, 0.24733457, 0.41248516],
+ [0.73695678, 0.24620072, 0.41357737],
+ [0.73327278, 0.24510469, 0.41464364],
+ [0.72957096, 0.24404127, 0.4156828],
+ [0.72585394, 0.24300672, 0.41669383],
+ [0.7221226, 0.24199971, 0.41767651],
+ [0.71837612, 0.24102046, 0.41863486],
+ [0.71463236, 0.24004289, 0.41956983],
+ [0.7108932, 0.23906316, 0.42048681],
+ [0.70715842, 0.23808142, 0.42138647],
+ [0.70342811, 0.2370976, 0.42226844],
+ [0.69970218, 0.23611179, 0.42313282],
+ [0.69598055, 0.2351247, 0.42397678],
+ [0.69226314, 0.23413578, 0.42480327],
+ [0.68854988, 0.23314511, 0.42561234],
+ [0.68484064, 0.23215279, 0.42640419],
+ [0.68113541, 0.23115942, 0.42717615],
+ [0.67743412, 0.23016472, 0.42792989],
+ [0.67373662, 0.22916861, 0.42866642],
+ [0.67004287, 0.22817117, 0.42938576],
+ [0.66635279, 0.22717328, 0.43008427],
+ [0.66266621, 0.22617435, 0.43076552],
+ [0.65898313, 0.22517434, 0.43142956],
+ [0.65530349, 0.22417381, 0.43207427],
+ [0.65162696, 0.22317307, 0.4327001],
+ [0.64795375, 0.22217149, 0.43330852],
+ [0.64428351, 0.22116972, 0.43389854],
+ [0.64061624, 0.22016818, 0.43446845],
+ [0.63695183, 0.21916625, 0.43502123],
+ [0.63329016, 0.21816454, 0.43555493],
+ [0.62963102, 0.2171635, 0.43606881],
+ [0.62597451, 0.21616235, 0.43656529],
+ [0.62232019, 0.21516239, 0.43704153],
+ [0.61866821, 0.21416307, 0.43749868],
+ [0.61501835, 0.21316435, 0.43793808],
+ [0.61137029, 0.21216761, 0.4383556],
+ [0.60772426, 0.2111715, 0.43875552],
+ [0.60407977, 0.21017746, 0.43913439],
+ [0.60043678, 0.20918503, 0.43949412],
+ [0.59679524, 0.20819447, 0.43983393],
+ [0.59315487, 0.20720639, 0.44015254],
+ [0.58951566, 0.20622027, 0.44045213],
+ [0.58587715, 0.20523751, 0.44072926],
+ [0.5822395, 0.20425693, 0.44098758],
+ [0.57860222, 0.20328034, 0.44122241],
+ [0.57496549, 0.20230637, 0.44143805],
+ [0.57132875, 0.20133689, 0.4416298],
+ [0.56769215, 0.20037071, 0.44180142],
+ [0.5640552, 0.19940936, 0.44194923],
+ [0.56041794, 0.19845221, 0.44207535],
+ [0.55678004, 0.1975, 0.44217824],
+ [0.55314129, 0.19655316, 0.44225723],
+ [0.54950166, 0.19561118, 0.44231412],
+ [0.54585987, 0.19467771, 0.44234111],
+ [0.54221157, 0.19375869, 0.44233698],
+ [0.5385549, 0.19285696, 0.44229959],
+ [0.5348913, 0.19197036, 0.44222958],
+ [0.53122177, 0.1910974, 0.44212735],
+ [0.52754464, 0.19024042, 0.44199159],
+ [0.52386353, 0.18939409, 0.44182449],
+ [0.52017476, 0.18856368, 0.44162345],
+ [0.51648277, 0.18774266, 0.44139128],
+ [0.51278481, 0.18693492, 0.44112605],
+ [0.50908361, 0.18613639, 0.4408295],
+ [0.50537784, 0.18534893, 0.44050064],
+ [0.50166912, 0.18457008, 0.44014054],
+ [0.49795686, 0.18380056, 0.43974881],
+ [0.49424218, 0.18303865, 0.43932623],
+ [0.49052472, 0.18228477, 0.43887255],
+ [0.48680565, 0.1815371, 0.43838867],
+ [0.48308419, 0.18079663, 0.43787408],
+ [0.47936222, 0.18006056, 0.43733022],
+ [0.47563799, 0.17933127, 0.43675585],
+ [0.47191466, 0.17860416, 0.43615337],
+ [0.46818879, 0.17788392, 0.43552047],
+ [0.46446454, 0.17716458, 0.43486036],
+ [0.46073893, 0.17645017, 0.43417097],
+ [0.45701462, 0.17573691, 0.43345429],
+ [0.45329097, 0.17502549, 0.43271025],
+ [0.44956744, 0.17431649, 0.4319386],
+ [0.44584668, 0.17360625, 0.43114133],
+ [0.44212538, 0.17289906, 0.43031642],
+ [0.43840678, 0.17219041, 0.42946642],
+ [0.43469046, 0.17148074, 0.42859124],
+ [0.4309749, 0.17077192, 0.42769008],
+ [0.42726297, 0.17006003, 0.42676519],
+ [0.42355299, 0.16934709, 0.42581586],
+ [0.41984535, 0.16863258, 0.42484219],
+ [0.41614149, 0.16791429, 0.42384614],
+ [0.41244029, 0.16719372, 0.42282661],
+ [0.40874177, 0.16647061, 0.42178429],
+ [0.40504765, 0.16574261, 0.42072062],
+ [0.401357, 0.16501079, 0.41963528],
+ [0.397669, 0.16427607, 0.418528],
+ [0.39398585, 0.16353554, 0.41740053],
+ [0.39030735, 0.16278924, 0.41625344],
+ [0.3866314, 0.16203977, 0.41508517],
+ [0.38295904, 0.16128519, 0.41389849],
+ [0.37928736, 0.16052483, 0.41270599],
+ [0.37562649, 0.15974704, 0.41151182],
+ [0.37197803, 0.15895049, 0.41031532],
+ [0.36833779, 0.15813871, 0.40911916],
+ [0.36470944, 0.15730861, 0.40792149],
+ [0.36109117, 0.15646169, 0.40672362],
+ [0.35748213, 0.15559861, 0.40552633],
+ [0.353885, 0.15471714, 0.40432831],
+ [0.35029682, 0.15381967, 0.4031316],
+ [0.34671861, 0.1529053, 0.40193587],
+ [0.34315191, 0.15197275, 0.40074049],
+ [0.33959331, 0.15102466, 0.3995478],
+ [0.33604378, 0.15006017, 0.39835754],
+ [0.33250529, 0.14907766, 0.39716879],
+ [0.32897621, 0.14807831, 0.39598285],
+ [0.3254559, 0.14706248, 0.39480044],
+ [0.32194567, 0.14602909, 0.39362106],
+ [0.31844477, 0.14497857, 0.39244549],
+ [0.31494974, 0.14391333, 0.39127626],
+ [0.31146605, 0.14282918, 0.39011024],
+ [0.30798857, 0.1417297, 0.38895105],
+ [0.30451661, 0.14061515, 0.38779953],
+ [0.30105136, 0.13948445, 0.38665531],
+ [0.2975886, 0.1383403, 0.38552159],
+ [0.29408557, 0.13721193, 0.38442775]
+]
+
+
+_crest_lut = [
+ [0.6468274, 0.80289262, 0.56592265],
+ [0.64233318, 0.80081141, 0.56639461],
+ [0.63791969, 0.7987162, 0.56674976],
+ [0.6335316, 0.79661833, 0.56706128],
+ [0.62915226, 0.7945212, 0.56735066],
+ [0.62477862, 0.79242543, 0.56762143],
+ [0.62042003, 0.79032918, 0.56786129],
+ [0.61606327, 0.78823508, 0.56808666],
+ [0.61171322, 0.78614216, 0.56829092],
+ [0.60736933, 0.78405055, 0.56847436],
+ [0.60302658, 0.78196121, 0.56864272],
+ [0.59868708, 0.77987374, 0.56879289],
+ [0.59435366, 0.77778758, 0.56892099],
+ [0.59001953, 0.77570403, 0.56903477],
+ [0.58568753, 0.77362254, 0.56913028],
+ [0.58135593, 0.77154342, 0.56920908],
+ [0.57702623, 0.76946638, 0.56926895],
+ [0.57269165, 0.76739266, 0.5693172],
+ [0.56835934, 0.76532092, 0.56934507],
+ [0.56402533, 0.76325185, 0.56935664],
+ [0.55968429, 0.76118643, 0.56935732],
+ [0.55534159, 0.75912361, 0.56934052],
+ [0.55099572, 0.75706366, 0.56930743],
+ [0.54664626, 0.75500662, 0.56925799],
+ [0.54228969, 0.75295306, 0.56919546],
+ [0.53792417, 0.75090328, 0.56912118],
+ [0.53355172, 0.74885687, 0.5690324],
+ [0.52917169, 0.74681387, 0.56892926],
+ [0.52478243, 0.74477453, 0.56881287],
+ [0.52038338, 0.74273888, 0.56868323],
+ [0.5159739, 0.74070697, 0.56854039],
+ [0.51155269, 0.73867895, 0.56838507],
+ [0.50711872, 0.73665492, 0.56821764],
+ [0.50267118, 0.73463494, 0.56803826],
+ [0.49822926, 0.73261388, 0.56785146],
+ [0.49381422, 0.73058524, 0.56767484],
+ [0.48942421, 0.72854938, 0.56751036],
+ [0.48505993, 0.72650623, 0.56735752],
+ [0.48072207, 0.72445575, 0.56721583],
+ [0.4764113, 0.72239788, 0.56708475],
+ [0.47212827, 0.72033258, 0.56696376],
+ [0.46787361, 0.71825983, 0.56685231],
+ [0.46364792, 0.71617961, 0.56674986],
+ [0.45945271, 0.71409167, 0.56665625],
+ [0.45528878, 0.71199595, 0.56657103],
+ [0.45115557, 0.70989276, 0.5664931],
+ [0.44705356, 0.70778212, 0.56642189],
+ [0.44298321, 0.70566406, 0.56635683],
+ [0.43894492, 0.70353863, 0.56629734],
+ [0.43493911, 0.70140588, 0.56624286],
+ [0.43096612, 0.69926587, 0.5661928],
+ [0.42702625, 0.69711868, 0.56614659],
+ [0.42311977, 0.69496438, 0.56610368],
+ [0.41924689, 0.69280308, 0.56606355],
+ [0.41540778, 0.69063486, 0.56602564],
+ [0.41160259, 0.68845984, 0.56598944],
+ [0.40783143, 0.68627814, 0.56595436],
+ [0.40409434, 0.68408988, 0.56591994],
+ [0.40039134, 0.68189518, 0.56588564],
+ [0.39672238, 0.6796942, 0.56585103],
+ [0.39308781, 0.67748696, 0.56581581],
+ [0.38949137, 0.67527276, 0.56578084],
+ [0.38592889, 0.67305266, 0.56574422],
+ [0.38240013, 0.67082685, 0.56570561],
+ [0.37890483, 0.66859548, 0.56566462],
+ [0.37544276, 0.66635871, 0.56562081],
+ [0.37201365, 0.66411673, 0.56557372],
+ [0.36861709, 0.6618697, 0.5655231],
+ [0.36525264, 0.65961782, 0.56546873],
+ [0.36191986, 0.65736125, 0.56541032],
+ [0.35861935, 0.65509998, 0.56534768],
+ [0.35535621, 0.65283302, 0.56528211],
+ [0.35212361, 0.65056188, 0.56521171],
+ [0.34892097, 0.64828676, 0.56513633],
+ [0.34574785, 0.64600783, 0.56505539],
+ [0.34260357, 0.64372528, 0.5649689],
+ [0.33948744, 0.64143931, 0.56487679],
+ [0.33639887, 0.6391501, 0.56477869],
+ [0.33334501, 0.63685626, 0.56467661],
+ [0.33031952, 0.63455911, 0.564569],
+ [0.3273199, 0.63225924, 0.56445488],
+ [0.32434526, 0.62995682, 0.56433457],
+ [0.32139487, 0.62765201, 0.56420795],
+ [0.31846807, 0.62534504, 0.56407446],
+ [0.3155731, 0.62303426, 0.56393695],
+ [0.31270304, 0.62072111, 0.56379321],
+ [0.30985436, 0.61840624, 0.56364307],
+ [0.30702635, 0.61608984, 0.56348606],
+ [0.30421803, 0.61377205, 0.56332267],
+ [0.30143611, 0.61145167, 0.56315419],
+ [0.29867863, 0.60912907, 0.56298054],
+ [0.29593872, 0.60680554, 0.56280022],
+ [0.29321538, 0.60448121, 0.56261376],
+ [0.2905079, 0.60215628, 0.56242036],
+ [0.28782827, 0.5998285, 0.56222366],
+ [0.28516521, 0.59749996, 0.56202093],
+ [0.28251558, 0.59517119, 0.56181204],
+ [0.27987847, 0.59284232, 0.56159709],
+ [0.27726216, 0.59051189, 0.56137785],
+ [0.27466434, 0.58818027, 0.56115433],
+ [0.2720767, 0.58584893, 0.56092486],
+ [0.26949829, 0.58351797, 0.56068983],
+ [0.26693801, 0.58118582, 0.56045121],
+ [0.26439366, 0.57885288, 0.56020858],
+ [0.26185616, 0.57652063, 0.55996077],
+ [0.25932459, 0.57418919, 0.55970795],
+ [0.25681303, 0.57185614, 0.55945297],
+ [0.25431024, 0.56952337, 0.55919385],
+ [0.25180492, 0.56719255, 0.5589305],
+ [0.24929311, 0.56486397, 0.5586654],
+ [0.24678356, 0.56253666, 0.55839491],
+ [0.24426587, 0.56021153, 0.55812473],
+ [0.24174022, 0.55788852, 0.55785448],
+ [0.23921167, 0.55556705, 0.55758211],
+ [0.23668315, 0.55324675, 0.55730676],
+ [0.23414742, 0.55092825, 0.55703167],
+ [0.23160473, 0.54861143, 0.5567573],
+ [0.22905996, 0.54629572, 0.55648168],
+ [0.22651648, 0.54398082, 0.5562029],
+ [0.22396709, 0.54166721, 0.55592542],
+ [0.22141221, 0.53935481, 0.55564885],
+ [0.21885269, 0.53704347, 0.55537294],
+ [0.21629986, 0.53473208, 0.55509319],
+ [0.21374297, 0.53242154, 0.5548144],
+ [0.21118255, 0.53011166, 0.55453708],
+ [0.2086192, 0.52780237, 0.55426067],
+ [0.20605624, 0.52549322, 0.55398479],
+ [0.20350004, 0.5231837, 0.55370601],
+ [0.20094292, 0.52087429, 0.55342884],
+ [0.19838567, 0.51856489, 0.55315283],
+ [0.19582911, 0.51625531, 0.55287818],
+ [0.19327413, 0.51394542, 0.55260469],
+ [0.19072933, 0.51163448, 0.5523289],
+ [0.18819045, 0.50932268, 0.55205372],
+ [0.18565609, 0.50701014, 0.55177937],
+ [0.18312739, 0.50469666, 0.55150597],
+ [0.18060561, 0.50238204, 0.55123374],
+ [0.178092, 0.50006616, 0.55096224],
+ [0.17558808, 0.49774882, 0.55069118],
+ [0.17310341, 0.49542924, 0.5504176],
+ [0.17063111, 0.49310789, 0.55014445],
+ [0.1681728, 0.49078458, 0.54987159],
+ [0.1657302, 0.48845913, 0.54959882],
+ [0.16330517, 0.48613135, 0.54932605],
+ [0.16089963, 0.48380104, 0.54905306],
+ [0.15851561, 0.48146803, 0.54877953],
+ [0.15615526, 0.47913212, 0.54850526],
+ [0.15382083, 0.47679313, 0.54822991],
+ [0.15151471, 0.47445087, 0.54795318],
+ [0.14924112, 0.47210502, 0.54767411],
+ [0.1470032, 0.46975537, 0.54739226],
+ [0.14480101, 0.46740187, 0.54710832],
+ [0.14263736, 0.46504434, 0.54682188],
+ [0.14051521, 0.46268258, 0.54653253],
+ [0.13843761, 0.46031639, 0.54623985],
+ [0.13640774, 0.45794558, 0.5459434],
+ [0.13442887, 0.45556994, 0.54564272],
+ [0.1325044, 0.45318928, 0.54533736],
+ [0.13063777, 0.4508034, 0.54502674],
+ [0.12883252, 0.44841211, 0.5447104],
+ [0.12709242, 0.44601517, 0.54438795],
+ [0.1254209, 0.44361244, 0.54405855],
+ [0.12382162, 0.44120373, 0.54372156],
+ [0.12229818, 0.43878887, 0.54337634],
+ [0.12085453, 0.4363676, 0.54302253],
+ [0.11949938, 0.43393955, 0.54265715],
+ [0.11823166, 0.43150478, 0.54228104],
+ [0.11705496, 0.42906306, 0.54189388],
+ [0.115972, 0.42661431, 0.54149449],
+ [0.11498598, 0.42415835, 0.54108222],
+ [0.11409965, 0.42169502, 0.54065622],
+ [0.11331533, 0.41922424, 0.5402155],
+ [0.11263542, 0.41674582, 0.53975931],
+ [0.1120615, 0.4142597, 0.53928656],
+ [0.11159738, 0.41176567, 0.53879549],
+ [0.11125248, 0.40926325, 0.53828203],
+ [0.11101698, 0.40675289, 0.53774864],
+ [0.11089152, 0.40423445, 0.53719455],
+ [0.11085121, 0.4017095, 0.53662425],
+ [0.11087217, 0.39917938, 0.53604354],
+ [0.11095515, 0.39664394, 0.53545166],
+ [0.11110676, 0.39410282, 0.53484509],
+ [0.11131735, 0.39155635, 0.53422678],
+ [0.11158595, 0.38900446, 0.53359634],
+ [0.11191139, 0.38644711, 0.5329534],
+ [0.11229224, 0.38388426, 0.53229748],
+ [0.11273683, 0.38131546, 0.53162393],
+ [0.11323438, 0.37874109, 0.53093619],
+ [0.11378271, 0.37616112, 0.53023413],
+ [0.11437992, 0.37357557, 0.52951727],
+ [0.11502681, 0.37098429, 0.52878396],
+ [0.11572661, 0.36838709, 0.52803124],
+ [0.11646936, 0.36578429, 0.52726234],
+ [0.11725299, 0.3631759, 0.52647685],
+ [0.1180755, 0.36056193, 0.52567436],
+ [0.1189438, 0.35794203, 0.5248497],
+ [0.11984752, 0.35531657, 0.52400649],
+ [0.1207833, 0.35268564, 0.52314492],
+ [0.12174895, 0.35004927, 0.52226461],
+ [0.12274959, 0.34740723, 0.52136104],
+ [0.12377809, 0.34475975, 0.52043639],
+ [0.12482961, 0.34210702, 0.51949179],
+ [0.125902, 0.33944908, 0.51852688],
+ [0.12699998, 0.33678574, 0.51753708],
+ [0.12811691, 0.33411727, 0.51652464],
+ [0.12924811, 0.33144384, 0.51549084],
+ [0.13039157, 0.32876552, 0.51443538],
+ [0.13155228, 0.32608217, 0.51335321],
+ [0.13272282, 0.32339407, 0.51224759],
+ [0.13389954, 0.32070138, 0.51111946],
+ [0.13508064, 0.31800419, 0.50996862],
+ [0.13627149, 0.31530238, 0.50878942],
+ [0.13746376, 0.31259627, 0.50758645],
+ [0.13865499, 0.30988598, 0.50636017],
+ [0.13984364, 0.30717161, 0.50511042],
+ [0.14103515, 0.30445309, 0.50383119],
+ [0.14222093, 0.30173071, 0.50252813],
+ [0.14339946, 0.2990046, 0.50120127],
+ [0.14456941, 0.29627483, 0.49985054],
+ [0.14573579, 0.29354139, 0.49847009],
+ [0.14689091, 0.29080452, 0.49706566],
+ [0.1480336, 0.28806432, 0.49563732],
+ [0.1491628, 0.28532086, 0.49418508],
+ [0.15028228, 0.28257418, 0.49270402],
+ [0.15138673, 0.27982444, 0.49119848],
+ [0.15247457, 0.27707172, 0.48966925],
+ [0.15354487, 0.2743161, 0.48811641],
+ [0.15459955, 0.27155765, 0.4865371],
+ [0.15563716, 0.26879642, 0.4849321],
+ [0.1566572, 0.26603191, 0.48330429],
+ [0.15765823, 0.26326032, 0.48167456],
+ [0.15862147, 0.26048295, 0.48005785],
+ [0.15954301, 0.25770084, 0.47845341],
+ [0.16043267, 0.25491144, 0.4768626],
+ [0.16129262, 0.25211406, 0.4752857],
+ [0.1621119, 0.24931169, 0.47372076],
+ [0.16290577, 0.24649998, 0.47217025],
+ [0.16366819, 0.24368054, 0.47063302],
+ [0.1644021, 0.24085237, 0.46910949],
+ [0.16510882, 0.2380149, 0.46759982],
+ [0.16579015, 0.23516739, 0.46610429],
+ [0.1664433, 0.2323105, 0.46462219],
+ [0.16707586, 0.22944155, 0.46315508],
+ [0.16768475, 0.22656122, 0.46170223],
+ [0.16826815, 0.22366984, 0.46026308],
+ [0.16883174, 0.22076514, 0.45883891],
+ [0.16937589, 0.21784655, 0.45742976],
+ [0.16990129, 0.21491339, 0.45603578],
+ [0.1704074, 0.21196535, 0.45465677],
+ [0.17089473, 0.20900176, 0.4532928],
+ [0.17136819, 0.20602012, 0.45194524],
+ [0.17182683, 0.20302012, 0.45061386],
+ [0.17227059, 0.20000106, 0.44929865],
+ [0.17270583, 0.19695949, 0.44800165],
+ [0.17313804, 0.19389201, 0.44672488],
+ [0.17363177, 0.19076859, 0.44549087]
+]
+
+
+_lut_dict = dict(
+ rocket=_rocket_lut,
+ mako=_mako_lut,
+ icefire=_icefire_lut,
+ vlag=_vlag_lut,
+ flare=_flare_lut,
+ crest=_crest_lut,
+
+)
+
+for _name, _lut in _lut_dict.items():
+
+ _cmap = colors.ListedColormap(_lut, _name)
+ locals()[_name] = _cmap
+
+ _cmap_r = colors.ListedColormap(_lut[::-1], _name + "_r")
+ locals()[_name + "_r"] = _cmap_r
+
+ register_colormap(_name, _cmap)
+ register_colormap(_name + "_r", _cmap_r)
+
+del colors, register_colormap
diff --git a/seaborn/colors/__init__.py b/seaborn/colors/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..3d0bf1d56bdc5c0e724c8eeb95200297884337cc
--- /dev/null
+++ b/seaborn/colors/__init__.py
@@ -0,0 +1,2 @@
+from .xkcd_rgb import xkcd_rgb # noqa: F401
+from .crayons import crayons # noqa: F401
diff --git a/seaborn/colors/crayons.py b/seaborn/colors/crayons.py
new file mode 100644
index 0000000000000000000000000000000000000000..548af1f199355e00e2b1956aa992a48ed61d090a
--- /dev/null
+++ b/seaborn/colors/crayons.py
@@ -0,0 +1,120 @@
+crayons = {'Almond': '#EFDECD',
+ 'Antique Brass': '#CD9575',
+ 'Apricot': '#FDD9B5',
+ 'Aquamarine': '#78DBE2',
+ 'Asparagus': '#87A96B',
+ 'Atomic Tangerine': '#FFA474',
+ 'Banana Mania': '#FAE7B5',
+ 'Beaver': '#9F8170',
+ 'Bittersweet': '#FD7C6E',
+ 'Black': '#000000',
+ 'Blue': '#1F75FE',
+ 'Blue Bell': '#A2A2D0',
+ 'Blue Green': '#0D98BA',
+ 'Blue Violet': '#7366BD',
+ 'Blush': '#DE5D83',
+ 'Brick Red': '#CB4154',
+ 'Brown': '#B4674D',
+ 'Burnt Orange': '#FF7F49',
+ 'Burnt Sienna': '#EA7E5D',
+ 'Cadet Blue': '#B0B7C6',
+ 'Canary': '#FFFF99',
+ 'Caribbean Green': '#00CC99',
+ 'Carnation Pink': '#FFAACC',
+ 'Cerise': '#DD4492',
+ 'Cerulean': '#1DACD6',
+ 'Chestnut': '#BC5D58',
+ 'Copper': '#DD9475',
+ 'Cornflower': '#9ACEEB',
+ 'Cotton Candy': '#FFBCD9',
+ 'Dandelion': '#FDDB6D',
+ 'Denim': '#2B6CC4',
+ 'Desert Sand': '#EFCDB8',
+ 'Eggplant': '#6E5160',
+ 'Electric Lime': '#CEFF1D',
+ 'Fern': '#71BC78',
+ 'Forest Green': '#6DAE81',
+ 'Fuchsia': '#C364C5',
+ 'Fuzzy Wuzzy': '#CC6666',
+ 'Gold': '#E7C697',
+ 'Goldenrod': '#FCD975',
+ 'Granny Smith Apple': '#A8E4A0',
+ 'Gray': '#95918C',
+ 'Green': '#1CAC78',
+ 'Green Yellow': '#F0E891',
+ 'Hot Magenta': '#FF1DCE',
+ 'Inchworm': '#B2EC5D',
+ 'Indigo': '#5D76CB',
+ 'Jazzberry Jam': '#CA3767',
+ 'Jungle Green': '#3BB08F',
+ 'Laser Lemon': '#FEFE22',
+ 'Lavender': '#FCB4D5',
+ 'Macaroni and Cheese': '#FFBD88',
+ 'Magenta': '#F664AF',
+ 'Mahogany': '#CD4A4C',
+ 'Manatee': '#979AAA',
+ 'Mango Tango': '#FF8243',
+ 'Maroon': '#C8385A',
+ 'Mauvelous': '#EF98AA',
+ 'Melon': '#FDBCB4',
+ 'Midnight Blue': '#1A4876',
+ 'Mountain Meadow': '#30BA8F',
+ 'Navy Blue': '#1974D2',
+ 'Neon Carrot': '#FFA343',
+ 'Olive Green': '#BAB86C',
+ 'Orange': '#FF7538',
+ 'Orchid': '#E6A8D7',
+ 'Outer Space': '#414A4C',
+ 'Outrageous Orange': '#FF6E4A',
+ 'Pacific Blue': '#1CA9C9',
+ 'Peach': '#FFCFAB',
+ 'Periwinkle': '#C5D0E6',
+ 'Piggy Pink': '#FDDDE6',
+ 'Pine Green': '#158078',
+ 'Pink Flamingo': '#FC74FD',
+ 'Pink Sherbert': '#F78FA7',
+ 'Plum': '#8E4585',
+ 'Purple Heart': '#7442C8',
+ "Purple Mountains' Majesty": '#9D81BA',
+ 'Purple Pizzazz': '#FE4EDA',
+ 'Radical Red': '#FF496C',
+ 'Raw Sienna': '#D68A59',
+ 'Razzle Dazzle Rose': '#FF48D0',
+ 'Razzmatazz': '#E3256B',
+ 'Red': '#EE204D',
+ 'Red Orange': '#FF5349',
+ 'Red Violet': '#C0448F',
+ "Robin's Egg Blue": '#1FCECB',
+ 'Royal Purple': '#7851A9',
+ 'Salmon': '#FF9BAA',
+ 'Scarlet': '#FC2847',
+ "Screamin' Green": '#76FF7A',
+ 'Sea Green': '#93DFB8',
+ 'Sepia': '#A5694F',
+ 'Shadow': '#8A795D',
+ 'Shamrock': '#45CEA2',
+ 'Shocking Pink': '#FB7EFD',
+ 'Silver': '#CDC5C2',
+ 'Sky Blue': '#80DAEB',
+ 'Spring Green': '#ECEABE',
+ 'Sunglow': '#FFCF48',
+ 'Sunset Orange': '#FD5E53',
+ 'Tan': '#FAA76C',
+ 'Tickle Me Pink': '#FC89AC',
+ 'Timberwolf': '#DBD7D2',
+ 'Tropical Rain Forest': '#17806D',
+ 'Tumbleweed': '#DEAA88',
+ 'Turquoise Blue': '#77DDE7',
+ 'Unmellow Yellow': '#FFFF66',
+ 'Violet (Purple)': '#926EAE',
+ 'Violet Red': '#F75394',
+ 'Vivid Tangerine': '#FFA089',
+ 'Vivid Violet': '#8F509D',
+ 'White': '#FFFFFF',
+ 'Wild Blue Yonder': '#A2ADD0',
+ 'Wild Strawberry': '#FF43A4',
+ 'Wild Watermelon': '#FC6C85',
+ 'Wisteria': '#CDA4DE',
+ 'Yellow': '#FCE883',
+ 'Yellow Green': '#C5E384',
+ 'Yellow Orange': '#FFAE42'}
diff --git a/seaborn/colors/xkcd_rgb.py b/seaborn/colors/xkcd_rgb.py
new file mode 100644
index 0000000000000000000000000000000000000000..0f775cf6512c789ee4201cc41ed5c5fcc389a500
--- /dev/null
+++ b/seaborn/colors/xkcd_rgb.py
@@ -0,0 +1,949 @@
+xkcd_rgb = {'acid green': '#8ffe09',
+ 'adobe': '#bd6c48',
+ 'algae': '#54ac68',
+ 'algae green': '#21c36f',
+ 'almost black': '#070d0d',
+ 'amber': '#feb308',
+ 'amethyst': '#9b5fc0',
+ 'apple': '#6ecb3c',
+ 'apple green': '#76cd26',
+ 'apricot': '#ffb16d',
+ 'aqua': '#13eac9',
+ 'aqua blue': '#02d8e9',
+ 'aqua green': '#12e193',
+ 'aqua marine': '#2ee8bb',
+ 'aquamarine': '#04d8b2',
+ 'army green': '#4b5d16',
+ 'asparagus': '#77ab56',
+ 'aubergine': '#3d0734',
+ 'auburn': '#9a3001',
+ 'avocado': '#90b134',
+ 'avocado green': '#87a922',
+ 'azul': '#1d5dec',
+ 'azure': '#069af3',
+ 'baby blue': '#a2cffe',
+ 'baby green': '#8cff9e',
+ 'baby pink': '#ffb7ce',
+ 'baby poo': '#ab9004',
+ 'baby poop': '#937c00',
+ 'baby poop green': '#8f9805',
+ 'baby puke green': '#b6c406',
+ 'baby purple': '#ca9bf7',
+ 'baby shit brown': '#ad900d',
+ 'baby shit green': '#889717',
+ 'banana': '#ffff7e',
+ 'banana yellow': '#fafe4b',
+ 'barbie pink': '#fe46a5',
+ 'barf green': '#94ac02',
+ 'barney': '#ac1db8',
+ 'barney purple': '#a00498',
+ 'battleship grey': '#6b7c85',
+ 'beige': '#e6daa6',
+ 'berry': '#990f4b',
+ 'bile': '#b5c306',
+ 'black': '#000000',
+ 'bland': '#afa88b',
+ 'blood': '#770001',
+ 'blood orange': '#fe4b03',
+ 'blood red': '#980002',
+ 'blue': '#0343df',
+ 'blue blue': '#2242c7',
+ 'blue green': '#137e6d',
+ 'blue grey': '#607c8e',
+ 'blue purple': '#5729ce',
+ 'blue violet': '#5d06e9',
+ 'blue with a hint of purple': '#533cc6',
+ 'blue/green': '#0f9b8e',
+ 'blue/grey': '#758da3',
+ 'blue/purple': '#5a06ef',
+ 'blueberry': '#464196',
+ 'bluegreen': '#017a79',
+ 'bluegrey': '#85a3b2',
+ 'bluey green': '#2bb179',
+ 'bluey grey': '#89a0b0',
+ 'bluey purple': '#6241c7',
+ 'bluish': '#2976bb',
+ 'bluish green': '#10a674',
+ 'bluish grey': '#748b97',
+ 'bluish purple': '#703be7',
+ 'blurple': '#5539cc',
+ 'blush': '#f29e8e',
+ 'blush pink': '#fe828c',
+ 'booger': '#9bb53c',
+ 'booger green': '#96b403',
+ 'bordeaux': '#7b002c',
+ 'boring green': '#63b365',
+ 'bottle green': '#044a05',
+ 'brick': '#a03623',
+ 'brick orange': '#c14a09',
+ 'brick red': '#8f1402',
+ 'bright aqua': '#0bf9ea',
+ 'bright blue': '#0165fc',
+ 'bright cyan': '#41fdfe',
+ 'bright green': '#01ff07',
+ 'bright lavender': '#c760ff',
+ 'bright light blue': '#26f7fd',
+ 'bright light green': '#2dfe54',
+ 'bright lilac': '#c95efb',
+ 'bright lime': '#87fd05',
+ 'bright lime green': '#65fe08',
+ 'bright magenta': '#ff08e8',
+ 'bright olive': '#9cbb04',
+ 'bright orange': '#ff5b00',
+ 'bright pink': '#fe01b1',
+ 'bright purple': '#be03fd',
+ 'bright red': '#ff000d',
+ 'bright sea green': '#05ffa6',
+ 'bright sky blue': '#02ccfe',
+ 'bright teal': '#01f9c6',
+ 'bright turquoise': '#0ffef9',
+ 'bright violet': '#ad0afd',
+ 'bright yellow': '#fffd01',
+ 'bright yellow green': '#9dff00',
+ 'british racing green': '#05480d',
+ 'bronze': '#a87900',
+ 'brown': '#653700',
+ 'brown green': '#706c11',
+ 'brown grey': '#8d8468',
+ 'brown orange': '#b96902',
+ 'brown red': '#922b05',
+ 'brown yellow': '#b29705',
+ 'brownish': '#9c6d57',
+ 'brownish green': '#6a6e09',
+ 'brownish grey': '#86775f',
+ 'brownish orange': '#cb7723',
+ 'brownish pink': '#c27e79',
+ 'brownish purple': '#76424e',
+ 'brownish red': '#9e3623',
+ 'brownish yellow': '#c9b003',
+ 'browny green': '#6f6c0a',
+ 'browny orange': '#ca6b02',
+ 'bruise': '#7e4071',
+ 'bubble gum pink': '#ff69af',
+ 'bubblegum': '#ff6cb5',
+ 'bubblegum pink': '#fe83cc',
+ 'buff': '#fef69e',
+ 'burgundy': '#610023',
+ 'burnt orange': '#c04e01',
+ 'burnt red': '#9f2305',
+ 'burnt siena': '#b75203',
+ 'burnt sienna': '#b04e0f',
+ 'burnt umber': '#a0450e',
+ 'burnt yellow': '#d5ab09',
+ 'burple': '#6832e3',
+ 'butter': '#ffff81',
+ 'butter yellow': '#fffd74',
+ 'butterscotch': '#fdb147',
+ 'cadet blue': '#4e7496',
+ 'camel': '#c69f59',
+ 'camo': '#7f8f4e',
+ 'camo green': '#526525',
+ 'camouflage green': '#4b6113',
+ 'canary': '#fdff63',
+ 'canary yellow': '#fffe40',
+ 'candy pink': '#ff63e9',
+ 'caramel': '#af6f09',
+ 'carmine': '#9d0216',
+ 'carnation': '#fd798f',
+ 'carnation pink': '#ff7fa7',
+ 'carolina blue': '#8ab8fe',
+ 'celadon': '#befdb7',
+ 'celery': '#c1fd95',
+ 'cement': '#a5a391',
+ 'cerise': '#de0c62',
+ 'cerulean': '#0485d1',
+ 'cerulean blue': '#056eee',
+ 'charcoal': '#343837',
+ 'charcoal grey': '#3c4142',
+ 'chartreuse': '#c1f80a',
+ 'cherry': '#cf0234',
+ 'cherry red': '#f7022a',
+ 'chestnut': '#742802',
+ 'chocolate': '#3d1c02',
+ 'chocolate brown': '#411900',
+ 'cinnamon': '#ac4f06',
+ 'claret': '#680018',
+ 'clay': '#b66a50',
+ 'clay brown': '#b2713d',
+ 'clear blue': '#247afd',
+ 'cloudy blue': '#acc2d9',
+ 'cobalt': '#1e488f',
+ 'cobalt blue': '#030aa7',
+ 'cocoa': '#875f42',
+ 'coffee': '#a6814c',
+ 'cool blue': '#4984b8',
+ 'cool green': '#33b864',
+ 'cool grey': '#95a3a6',
+ 'copper': '#b66325',
+ 'coral': '#fc5a50',
+ 'coral pink': '#ff6163',
+ 'cornflower': '#6a79f7',
+ 'cornflower blue': '#5170d7',
+ 'cranberry': '#9e003a',
+ 'cream': '#ffffc2',
+ 'creme': '#ffffb6',
+ 'crimson': '#8c000f',
+ 'custard': '#fffd78',
+ 'cyan': '#00ffff',
+ 'dandelion': '#fedf08',
+ 'dark': '#1b2431',
+ 'dark aqua': '#05696b',
+ 'dark aquamarine': '#017371',
+ 'dark beige': '#ac9362',
+ 'dark blue': '#00035b',
+ 'dark blue green': '#005249',
+ 'dark blue grey': '#1f3b4d',
+ 'dark brown': '#341c02',
+ 'dark coral': '#cf524e',
+ 'dark cream': '#fff39a',
+ 'dark cyan': '#0a888a',
+ 'dark forest green': '#002d04',
+ 'dark fuchsia': '#9d0759',
+ 'dark gold': '#b59410',
+ 'dark grass green': '#388004',
+ 'dark green': '#033500',
+ 'dark green blue': '#1f6357',
+ 'dark grey': '#363737',
+ 'dark grey blue': '#29465b',
+ 'dark hot pink': '#d90166',
+ 'dark indigo': '#1f0954',
+ 'dark khaki': '#9b8f55',
+ 'dark lavender': '#856798',
+ 'dark lilac': '#9c6da5',
+ 'dark lime': '#84b701',
+ 'dark lime green': '#7ebd01',
+ 'dark magenta': '#960056',
+ 'dark maroon': '#3c0008',
+ 'dark mauve': '#874c62',
+ 'dark mint': '#48c072',
+ 'dark mint green': '#20c073',
+ 'dark mustard': '#a88905',
+ 'dark navy': '#000435',
+ 'dark navy blue': '#00022e',
+ 'dark olive': '#373e02',
+ 'dark olive green': '#3c4d03',
+ 'dark orange': '#c65102',
+ 'dark pastel green': '#56ae57',
+ 'dark peach': '#de7e5d',
+ 'dark periwinkle': '#665fd1',
+ 'dark pink': '#cb416b',
+ 'dark plum': '#3f012c',
+ 'dark purple': '#35063e',
+ 'dark red': '#840000',
+ 'dark rose': '#b5485d',
+ 'dark royal blue': '#02066f',
+ 'dark sage': '#598556',
+ 'dark salmon': '#c85a53',
+ 'dark sand': '#a88f59',
+ 'dark sea green': '#11875d',
+ 'dark seafoam': '#1fb57a',
+ 'dark seafoam green': '#3eaf76',
+ 'dark sky blue': '#448ee4',
+ 'dark slate blue': '#214761',
+ 'dark tan': '#af884a',
+ 'dark taupe': '#7f684e',
+ 'dark teal': '#014d4e',
+ 'dark turquoise': '#045c5a',
+ 'dark violet': '#34013f',
+ 'dark yellow': '#d5b60a',
+ 'dark yellow green': '#728f02',
+ 'darkblue': '#030764',
+ 'darkgreen': '#054907',
+ 'darkish blue': '#014182',
+ 'darkish green': '#287c37',
+ 'darkish pink': '#da467d',
+ 'darkish purple': '#751973',
+ 'darkish red': '#a90308',
+ 'deep aqua': '#08787f',
+ 'deep blue': '#040273',
+ 'deep brown': '#410200',
+ 'deep green': '#02590f',
+ 'deep lavender': '#8d5eb7',
+ 'deep lilac': '#966ebd',
+ 'deep magenta': '#a0025c',
+ 'deep orange': '#dc4d01',
+ 'deep pink': '#cb0162',
+ 'deep purple': '#36013f',
+ 'deep red': '#9a0200',
+ 'deep rose': '#c74767',
+ 'deep sea blue': '#015482',
+ 'deep sky blue': '#0d75f8',
+ 'deep teal': '#00555a',
+ 'deep turquoise': '#017374',
+ 'deep violet': '#490648',
+ 'denim': '#3b638c',
+ 'denim blue': '#3b5b92',
+ 'desert': '#ccad60',
+ 'diarrhea': '#9f8303',
+ 'dirt': '#8a6e45',
+ 'dirt brown': '#836539',
+ 'dirty blue': '#3f829d',
+ 'dirty green': '#667e2c',
+ 'dirty orange': '#c87606',
+ 'dirty pink': '#ca7b80',
+ 'dirty purple': '#734a65',
+ 'dirty yellow': '#cdc50a',
+ 'dodger blue': '#3e82fc',
+ 'drab': '#828344',
+ 'drab green': '#749551',
+ 'dried blood': '#4b0101',
+ 'duck egg blue': '#c3fbf4',
+ 'dull blue': '#49759c',
+ 'dull brown': '#876e4b',
+ 'dull green': '#74a662',
+ 'dull orange': '#d8863b',
+ 'dull pink': '#d5869d',
+ 'dull purple': '#84597e',
+ 'dull red': '#bb3f3f',
+ 'dull teal': '#5f9e8f',
+ 'dull yellow': '#eedc5b',
+ 'dusk': '#4e5481',
+ 'dusk blue': '#26538d',
+ 'dusky blue': '#475f94',
+ 'dusky pink': '#cc7a8b',
+ 'dusky purple': '#895b7b',
+ 'dusky rose': '#ba6873',
+ 'dust': '#b2996e',
+ 'dusty blue': '#5a86ad',
+ 'dusty green': '#76a973',
+ 'dusty lavender': '#ac86a8',
+ 'dusty orange': '#f0833a',
+ 'dusty pink': '#d58a94',
+ 'dusty purple': '#825f87',
+ 'dusty red': '#b9484e',
+ 'dusty rose': '#c0737a',
+ 'dusty teal': '#4c9085',
+ 'earth': '#a2653e',
+ 'easter green': '#8cfd7e',
+ 'easter purple': '#c071fe',
+ 'ecru': '#feffca',
+ 'egg shell': '#fffcc4',
+ 'eggplant': '#380835',
+ 'eggplant purple': '#430541',
+ 'eggshell': '#ffffd4',
+ 'eggshell blue': '#c4fff7',
+ 'electric blue': '#0652ff',
+ 'electric green': '#21fc0d',
+ 'electric lime': '#a8ff04',
+ 'electric pink': '#ff0490',
+ 'electric purple': '#aa23ff',
+ 'emerald': '#01a049',
+ 'emerald green': '#028f1e',
+ 'evergreen': '#05472a',
+ 'faded blue': '#658cbb',
+ 'faded green': '#7bb274',
+ 'faded orange': '#f0944d',
+ 'faded pink': '#de9dac',
+ 'faded purple': '#916e99',
+ 'faded red': '#d3494e',
+ 'faded yellow': '#feff7f',
+ 'fawn': '#cfaf7b',
+ 'fern': '#63a950',
+ 'fern green': '#548d44',
+ 'fire engine red': '#fe0002',
+ 'flat blue': '#3c73a8',
+ 'flat green': '#699d4c',
+ 'fluorescent green': '#08ff08',
+ 'fluro green': '#0aff02',
+ 'foam green': '#90fda9',
+ 'forest': '#0b5509',
+ 'forest green': '#06470c',
+ 'forrest green': '#154406',
+ 'french blue': '#436bad',
+ 'fresh green': '#69d84f',
+ 'frog green': '#58bc08',
+ 'fuchsia': '#ed0dd9',
+ 'gold': '#dbb40c',
+ 'golden': '#f5bf03',
+ 'golden brown': '#b27a01',
+ 'golden rod': '#f9bc08',
+ 'golden yellow': '#fec615',
+ 'goldenrod': '#fac205',
+ 'grape': '#6c3461',
+ 'grape purple': '#5d1451',
+ 'grapefruit': '#fd5956',
+ 'grass': '#5cac2d',
+ 'grass green': '#3f9b0b',
+ 'grassy green': '#419c03',
+ 'green': '#15b01a',
+ 'green apple': '#5edc1f',
+ 'green blue': '#06b48b',
+ 'green brown': '#544e03',
+ 'green grey': '#77926f',
+ 'green teal': '#0cb577',
+ 'green yellow': '#c9ff27',
+ 'green/blue': '#01c08d',
+ 'green/yellow': '#b5ce08',
+ 'greenblue': '#23c48b',
+ 'greenish': '#40a368',
+ 'greenish beige': '#c9d179',
+ 'greenish blue': '#0b8b87',
+ 'greenish brown': '#696112',
+ 'greenish cyan': '#2afeb7',
+ 'greenish grey': '#96ae8d',
+ 'greenish tan': '#bccb7a',
+ 'greenish teal': '#32bf84',
+ 'greenish turquoise': '#00fbb0',
+ 'greenish yellow': '#cdfd02',
+ 'greeny blue': '#42b395',
+ 'greeny brown': '#696006',
+ 'greeny grey': '#7ea07a',
+ 'greeny yellow': '#c6f808',
+ 'grey': '#929591',
+ 'grey blue': '#6b8ba4',
+ 'grey brown': '#7f7053',
+ 'grey green': '#789b73',
+ 'grey pink': '#c3909b',
+ 'grey purple': '#826d8c',
+ 'grey teal': '#5e9b8a',
+ 'grey/blue': '#647d8e',
+ 'grey/green': '#86a17d',
+ 'greyblue': '#77a1b5',
+ 'greyish': '#a8a495',
+ 'greyish blue': '#5e819d',
+ 'greyish brown': '#7a6a4f',
+ 'greyish green': '#82a67d',
+ 'greyish pink': '#c88d94',
+ 'greyish purple': '#887191',
+ 'greyish teal': '#719f91',
+ 'gross green': '#a0bf16',
+ 'gunmetal': '#536267',
+ 'hazel': '#8e7618',
+ 'heather': '#a484ac',
+ 'heliotrope': '#d94ff5',
+ 'highlighter green': '#1bfc06',
+ 'hospital green': '#9be5aa',
+ 'hot green': '#25ff29',
+ 'hot magenta': '#f504c9',
+ 'hot pink': '#ff028d',
+ 'hot purple': '#cb00f5',
+ 'hunter green': '#0b4008',
+ 'ice': '#d6fffa',
+ 'ice blue': '#d7fffe',
+ 'icky green': '#8fae22',
+ 'indian red': '#850e04',
+ 'indigo': '#380282',
+ 'indigo blue': '#3a18b1',
+ 'iris': '#6258c4',
+ 'irish green': '#019529',
+ 'ivory': '#ffffcb',
+ 'jade': '#1fa774',
+ 'jade green': '#2baf6a',
+ 'jungle green': '#048243',
+ 'kelley green': '#009337',
+ 'kelly green': '#02ab2e',
+ 'kermit green': '#5cb200',
+ 'key lime': '#aeff6e',
+ 'khaki': '#aaa662',
+ 'khaki green': '#728639',
+ 'kiwi': '#9cef43',
+ 'kiwi green': '#8ee53f',
+ 'lavender': '#c79fef',
+ 'lavender blue': '#8b88f8',
+ 'lavender pink': '#dd85d7',
+ 'lawn green': '#4da409',
+ 'leaf': '#71aa34',
+ 'leaf green': '#5ca904',
+ 'leafy green': '#51b73b',
+ 'leather': '#ac7434',
+ 'lemon': '#fdff52',
+ 'lemon green': '#adf802',
+ 'lemon lime': '#bffe28',
+ 'lemon yellow': '#fdff38',
+ 'lichen': '#8fb67b',
+ 'light aqua': '#8cffdb',
+ 'light aquamarine': '#7bfdc7',
+ 'light beige': '#fffeb6',
+ 'light blue': '#95d0fc',
+ 'light blue green': '#7efbb3',
+ 'light blue grey': '#b7c9e2',
+ 'light bluish green': '#76fda8',
+ 'light bright green': '#53fe5c',
+ 'light brown': '#ad8150',
+ 'light burgundy': '#a8415b',
+ 'light cyan': '#acfffc',
+ 'light eggplant': '#894585',
+ 'light forest green': '#4f9153',
+ 'light gold': '#fddc5c',
+ 'light grass green': '#9af764',
+ 'light green': '#96f97b',
+ 'light green blue': '#56fca2',
+ 'light greenish blue': '#63f7b4',
+ 'light grey': '#d8dcd6',
+ 'light grey blue': '#9dbcd4',
+ 'light grey green': '#b7e1a1',
+ 'light indigo': '#6d5acf',
+ 'light khaki': '#e6f2a2',
+ 'light lavendar': '#efc0fe',
+ 'light lavender': '#dfc5fe',
+ 'light light blue': '#cafffb',
+ 'light light green': '#c8ffb0',
+ 'light lilac': '#edc8ff',
+ 'light lime': '#aefd6c',
+ 'light lime green': '#b9ff66',
+ 'light magenta': '#fa5ff7',
+ 'light maroon': '#a24857',
+ 'light mauve': '#c292a1',
+ 'light mint': '#b6ffbb',
+ 'light mint green': '#a6fbb2',
+ 'light moss green': '#a6c875',
+ 'light mustard': '#f7d560',
+ 'light navy': '#155084',
+ 'light navy blue': '#2e5a88',
+ 'light neon green': '#4efd54',
+ 'light olive': '#acbf69',
+ 'light olive green': '#a4be5c',
+ 'light orange': '#fdaa48',
+ 'light pastel green': '#b2fba5',
+ 'light pea green': '#c4fe82',
+ 'light peach': '#ffd8b1',
+ 'light periwinkle': '#c1c6fc',
+ 'light pink': '#ffd1df',
+ 'light plum': '#9d5783',
+ 'light purple': '#bf77f6',
+ 'light red': '#ff474c',
+ 'light rose': '#ffc5cb',
+ 'light royal blue': '#3a2efe',
+ 'light sage': '#bcecac',
+ 'light salmon': '#fea993',
+ 'light sea green': '#98f6b0',
+ 'light seafoam': '#a0febf',
+ 'light seafoam green': '#a7ffb5',
+ 'light sky blue': '#c6fcff',
+ 'light tan': '#fbeeac',
+ 'light teal': '#90e4c1',
+ 'light turquoise': '#7ef4cc',
+ 'light urple': '#b36ff6',
+ 'light violet': '#d6b4fc',
+ 'light yellow': '#fffe7a',
+ 'light yellow green': '#ccfd7f',
+ 'light yellowish green': '#c2ff89',
+ 'lightblue': '#7bc8f6',
+ 'lighter green': '#75fd63',
+ 'lighter purple': '#a55af4',
+ 'lightgreen': '#76ff7b',
+ 'lightish blue': '#3d7afd',
+ 'lightish green': '#61e160',
+ 'lightish purple': '#a552e6',
+ 'lightish red': '#fe2f4a',
+ 'lilac': '#cea2fd',
+ 'liliac': '#c48efd',
+ 'lime': '#aaff32',
+ 'lime green': '#89fe05',
+ 'lime yellow': '#d0fe1d',
+ 'lipstick': '#d5174e',
+ 'lipstick red': '#c0022f',
+ 'macaroni and cheese': '#efb435',
+ 'magenta': '#c20078',
+ 'mahogany': '#4a0100',
+ 'maize': '#f4d054',
+ 'mango': '#ffa62b',
+ 'manilla': '#fffa86',
+ 'marigold': '#fcc006',
+ 'marine': '#042e60',
+ 'marine blue': '#01386a',
+ 'maroon': '#650021',
+ 'mauve': '#ae7181',
+ 'medium blue': '#2c6fbb',
+ 'medium brown': '#7f5112',
+ 'medium green': '#39ad48',
+ 'medium grey': '#7d7f7c',
+ 'medium pink': '#f36196',
+ 'medium purple': '#9e43a2',
+ 'melon': '#ff7855',
+ 'merlot': '#730039',
+ 'metallic blue': '#4f738e',
+ 'mid blue': '#276ab3',
+ 'mid green': '#50a747',
+ 'midnight': '#03012d',
+ 'midnight blue': '#020035',
+ 'midnight purple': '#280137',
+ 'military green': '#667c3e',
+ 'milk chocolate': '#7f4e1e',
+ 'mint': '#9ffeb0',
+ 'mint green': '#8fff9f',
+ 'minty green': '#0bf77d',
+ 'mocha': '#9d7651',
+ 'moss': '#769958',
+ 'moss green': '#658b38',
+ 'mossy green': '#638b27',
+ 'mud': '#735c12',
+ 'mud brown': '#60460f',
+ 'mud green': '#606602',
+ 'muddy brown': '#886806',
+ 'muddy green': '#657432',
+ 'muddy yellow': '#bfac05',
+ 'mulberry': '#920a4e',
+ 'murky green': '#6c7a0e',
+ 'mushroom': '#ba9e88',
+ 'mustard': '#ceb301',
+ 'mustard brown': '#ac7e04',
+ 'mustard green': '#a8b504',
+ 'mustard yellow': '#d2bd0a',
+ 'muted blue': '#3b719f',
+ 'muted green': '#5fa052',
+ 'muted pink': '#d1768f',
+ 'muted purple': '#805b87',
+ 'nasty green': '#70b23f',
+ 'navy': '#01153e',
+ 'navy blue': '#001146',
+ 'navy green': '#35530a',
+ 'neon blue': '#04d9ff',
+ 'neon green': '#0cff0c',
+ 'neon pink': '#fe019a',
+ 'neon purple': '#bc13fe',
+ 'neon red': '#ff073a',
+ 'neon yellow': '#cfff04',
+ 'nice blue': '#107ab0',
+ 'night blue': '#040348',
+ 'ocean': '#017b92',
+ 'ocean blue': '#03719c',
+ 'ocean green': '#3d9973',
+ 'ocher': '#bf9b0c',
+ 'ochre': '#bf9005',
+ 'ocre': '#c69c04',
+ 'off blue': '#5684ae',
+ 'off green': '#6ba353',
+ 'off white': '#ffffe4',
+ 'off yellow': '#f1f33f',
+ 'old pink': '#c77986',
+ 'old rose': '#c87f89',
+ 'olive': '#6e750e',
+ 'olive brown': '#645403',
+ 'olive drab': '#6f7632',
+ 'olive green': '#677a04',
+ 'olive yellow': '#c2b709',
+ 'orange': '#f97306',
+ 'orange brown': '#be6400',
+ 'orange pink': '#ff6f52',
+ 'orange red': '#fd411e',
+ 'orange yellow': '#ffad01',
+ 'orangeish': '#fd8d49',
+ 'orangered': '#fe420f',
+ 'orangey brown': '#b16002',
+ 'orangey red': '#fa4224',
+ 'orangey yellow': '#fdb915',
+ 'orangish': '#fc824a',
+ 'orangish brown': '#b25f03',
+ 'orangish red': '#f43605',
+ 'orchid': '#c875c4',
+ 'pale': '#fff9d0',
+ 'pale aqua': '#b8ffeb',
+ 'pale blue': '#d0fefe',
+ 'pale brown': '#b1916e',
+ 'pale cyan': '#b7fffa',
+ 'pale gold': '#fdde6c',
+ 'pale green': '#c7fdb5',
+ 'pale grey': '#fdfdfe',
+ 'pale lavender': '#eecffe',
+ 'pale light green': '#b1fc99',
+ 'pale lilac': '#e4cbff',
+ 'pale lime': '#befd73',
+ 'pale lime green': '#b1ff65',
+ 'pale magenta': '#d767ad',
+ 'pale mauve': '#fed0fc',
+ 'pale olive': '#b9cc81',
+ 'pale olive green': '#b1d27b',
+ 'pale orange': '#ffa756',
+ 'pale peach': '#ffe5ad',
+ 'pale pink': '#ffcfdc',
+ 'pale purple': '#b790d4',
+ 'pale red': '#d9544d',
+ 'pale rose': '#fdc1c5',
+ 'pale salmon': '#ffb19a',
+ 'pale sky blue': '#bdf6fe',
+ 'pale teal': '#82cbb2',
+ 'pale turquoise': '#a5fbd5',
+ 'pale violet': '#ceaefa',
+ 'pale yellow': '#ffff84',
+ 'parchment': '#fefcaf',
+ 'pastel blue': '#a2bffe',
+ 'pastel green': '#b0ff9d',
+ 'pastel orange': '#ff964f',
+ 'pastel pink': '#ffbacd',
+ 'pastel purple': '#caa0ff',
+ 'pastel red': '#db5856',
+ 'pastel yellow': '#fffe71',
+ 'pea': '#a4bf20',
+ 'pea green': '#8eab12',
+ 'pea soup': '#929901',
+ 'pea soup green': '#94a617',
+ 'peach': '#ffb07c',
+ 'peachy pink': '#ff9a8a',
+ 'peacock blue': '#016795',
+ 'pear': '#cbf85f',
+ 'periwinkle': '#8e82fe',
+ 'periwinkle blue': '#8f99fb',
+ 'perrywinkle': '#8f8ce7',
+ 'petrol': '#005f6a',
+ 'pig pink': '#e78ea5',
+ 'pine': '#2b5d34',
+ 'pine green': '#0a481e',
+ 'pink': '#ff81c0',
+ 'pink purple': '#db4bda',
+ 'pink red': '#f5054f',
+ 'pink/purple': '#ef1de7',
+ 'pinkish': '#d46a7e',
+ 'pinkish brown': '#b17261',
+ 'pinkish grey': '#c8aca9',
+ 'pinkish orange': '#ff724c',
+ 'pinkish purple': '#d648d7',
+ 'pinkish red': '#f10c45',
+ 'pinkish tan': '#d99b82',
+ 'pinky': '#fc86aa',
+ 'pinky purple': '#c94cbe',
+ 'pinky red': '#fc2647',
+ 'piss yellow': '#ddd618',
+ 'pistachio': '#c0fa8b',
+ 'plum': '#580f41',
+ 'plum purple': '#4e0550',
+ 'poison green': '#40fd14',
+ 'poo': '#8f7303',
+ 'poo brown': '#885f01',
+ 'poop': '#7f5e00',
+ 'poop brown': '#7a5901',
+ 'poop green': '#6f7c00',
+ 'powder blue': '#b1d1fc',
+ 'powder pink': '#ffb2d0',
+ 'primary blue': '#0804f9',
+ 'prussian blue': '#004577',
+ 'puce': '#a57e52',
+ 'puke': '#a5a502',
+ 'puke brown': '#947706',
+ 'puke green': '#9aae07',
+ 'puke yellow': '#c2be0e',
+ 'pumpkin': '#e17701',
+ 'pumpkin orange': '#fb7d07',
+ 'pure blue': '#0203e2',
+ 'purple': '#7e1e9c',
+ 'purple blue': '#632de9',
+ 'purple brown': '#673a3f',
+ 'purple grey': '#866f85',
+ 'purple pink': '#e03fd8',
+ 'purple red': '#990147',
+ 'purple/blue': '#5d21d0',
+ 'purple/pink': '#d725de',
+ 'purpleish': '#98568d',
+ 'purpleish blue': '#6140ef',
+ 'purpleish pink': '#df4ec8',
+ 'purpley': '#8756e4',
+ 'purpley blue': '#5f34e7',
+ 'purpley grey': '#947e94',
+ 'purpley pink': '#c83cb9',
+ 'purplish': '#94568c',
+ 'purplish blue': '#601ef9',
+ 'purplish brown': '#6b4247',
+ 'purplish grey': '#7a687f',
+ 'purplish pink': '#ce5dae',
+ 'purplish red': '#b0054b',
+ 'purply': '#983fb2',
+ 'purply blue': '#661aee',
+ 'purply pink': '#f075e6',
+ 'putty': '#beae8a',
+ 'racing green': '#014600',
+ 'radioactive green': '#2cfa1f',
+ 'raspberry': '#b00149',
+ 'raw sienna': '#9a6200',
+ 'raw umber': '#a75e09',
+ 'really light blue': '#d4ffff',
+ 'red': '#e50000',
+ 'red brown': '#8b2e16',
+ 'red orange': '#fd3c06',
+ 'red pink': '#fa2a55',
+ 'red purple': '#820747',
+ 'red violet': '#9e0168',
+ 'red wine': '#8c0034',
+ 'reddish': '#c44240',
+ 'reddish brown': '#7f2b0a',
+ 'reddish grey': '#997570',
+ 'reddish orange': '#f8481c',
+ 'reddish pink': '#fe2c54',
+ 'reddish purple': '#910951',
+ 'reddy brown': '#6e1005',
+ 'rich blue': '#021bf9',
+ 'rich purple': '#720058',
+ 'robin egg blue': '#8af1fe',
+ "robin's egg": '#6dedfd',
+ "robin's egg blue": '#98eff9',
+ 'rosa': '#fe86a4',
+ 'rose': '#cf6275',
+ 'rose pink': '#f7879a',
+ 'rose red': '#be013c',
+ 'rosy pink': '#f6688e',
+ 'rouge': '#ab1239',
+ 'royal': '#0c1793',
+ 'royal blue': '#0504aa',
+ 'royal purple': '#4b006e',
+ 'ruby': '#ca0147',
+ 'russet': '#a13905',
+ 'rust': '#a83c09',
+ 'rust brown': '#8b3103',
+ 'rust orange': '#c45508',
+ 'rust red': '#aa2704',
+ 'rusty orange': '#cd5909',
+ 'rusty red': '#af2f0d',
+ 'saffron': '#feb209',
+ 'sage': '#87ae73',
+ 'sage green': '#88b378',
+ 'salmon': '#ff796c',
+ 'salmon pink': '#fe7b7c',
+ 'sand': '#e2ca76',
+ 'sand brown': '#cba560',
+ 'sand yellow': '#fce166',
+ 'sandstone': '#c9ae74',
+ 'sandy': '#f1da7a',
+ 'sandy brown': '#c4a661',
+ 'sandy yellow': '#fdee73',
+ 'sap green': '#5c8b15',
+ 'sapphire': '#2138ab',
+ 'scarlet': '#be0119',
+ 'sea': '#3c9992',
+ 'sea blue': '#047495',
+ 'sea green': '#53fca1',
+ 'seafoam': '#80f9ad',
+ 'seafoam blue': '#78d1b6',
+ 'seafoam green': '#7af9ab',
+ 'seaweed': '#18d17b',
+ 'seaweed green': '#35ad6b',
+ 'sepia': '#985e2b',
+ 'shamrock': '#01b44c',
+ 'shamrock green': '#02c14d',
+ 'shit': '#7f5f00',
+ 'shit brown': '#7b5804',
+ 'shit green': '#758000',
+ 'shocking pink': '#fe02a2',
+ 'sick green': '#9db92c',
+ 'sickly green': '#94b21c',
+ 'sickly yellow': '#d0e429',
+ 'sienna': '#a9561e',
+ 'silver': '#c5c9c7',
+ 'sky': '#82cafc',
+ 'sky blue': '#75bbfd',
+ 'slate': '#516572',
+ 'slate blue': '#5b7c99',
+ 'slate green': '#658d6d',
+ 'slate grey': '#59656d',
+ 'slime green': '#99cc04',
+ 'snot': '#acbb0d',
+ 'snot green': '#9dc100',
+ 'soft blue': '#6488ea',
+ 'soft green': '#6fc276',
+ 'soft pink': '#fdb0c0',
+ 'soft purple': '#a66fb5',
+ 'spearmint': '#1ef876',
+ 'spring green': '#a9f971',
+ 'spruce': '#0a5f38',
+ 'squash': '#f2ab15',
+ 'steel': '#738595',
+ 'steel blue': '#5a7d9a',
+ 'steel grey': '#6f828a',
+ 'stone': '#ada587',
+ 'stormy blue': '#507b9c',
+ 'straw': '#fcf679',
+ 'strawberry': '#fb2943',
+ 'strong blue': '#0c06f7',
+ 'strong pink': '#ff0789',
+ 'sun yellow': '#ffdf22',
+ 'sunflower': '#ffc512',
+ 'sunflower yellow': '#ffda03',
+ 'sunny yellow': '#fff917',
+ 'sunshine yellow': '#fffd37',
+ 'swamp': '#698339',
+ 'swamp green': '#748500',
+ 'tan': '#d1b26f',
+ 'tan brown': '#ab7e4c',
+ 'tan green': '#a9be70',
+ 'tangerine': '#ff9408',
+ 'taupe': '#b9a281',
+ 'tea': '#65ab7c',
+ 'tea green': '#bdf8a3',
+ 'teal': '#029386',
+ 'teal blue': '#01889f',
+ 'teal green': '#25a36f',
+ 'tealish': '#24bca8',
+ 'tealish green': '#0cdc73',
+ 'terra cotta': '#c9643b',
+ 'terracota': '#cb6843',
+ 'terracotta': '#ca6641',
+ 'tiffany blue': '#7bf2da',
+ 'tomato': '#ef4026',
+ 'tomato red': '#ec2d01',
+ 'topaz': '#13bbaf',
+ 'toupe': '#c7ac7d',
+ 'toxic green': '#61de2a',
+ 'tree green': '#2a7e19',
+ 'true blue': '#010fcc',
+ 'true green': '#089404',
+ 'turquoise': '#06c2ac',
+ 'turquoise blue': '#06b1c4',
+ 'turquoise green': '#04f489',
+ 'turtle green': '#75b84f',
+ 'twilight': '#4e518b',
+ 'twilight blue': '#0a437a',
+ 'ugly blue': '#31668a',
+ 'ugly brown': '#7d7103',
+ 'ugly green': '#7a9703',
+ 'ugly pink': '#cd7584',
+ 'ugly purple': '#a442a0',
+ 'ugly yellow': '#d0c101',
+ 'ultramarine': '#2000b1',
+ 'ultramarine blue': '#1805db',
+ 'umber': '#b26400',
+ 'velvet': '#750851',
+ 'vermillion': '#f4320c',
+ 'very dark blue': '#000133',
+ 'very dark brown': '#1d0200',
+ 'very dark green': '#062e03',
+ 'very dark purple': '#2a0134',
+ 'very light blue': '#d5ffff',
+ 'very light brown': '#d3b683',
+ 'very light green': '#d1ffbd',
+ 'very light pink': '#fff4f2',
+ 'very light purple': '#f6cefc',
+ 'very pale blue': '#d6fffe',
+ 'very pale green': '#cffdbc',
+ 'vibrant blue': '#0339f8',
+ 'vibrant green': '#0add08',
+ 'vibrant purple': '#ad03de',
+ 'violet': '#9a0eea',
+ 'violet blue': '#510ac9',
+ 'violet pink': '#fb5ffc',
+ 'violet red': '#a50055',
+ 'viridian': '#1e9167',
+ 'vivid blue': '#152eff',
+ 'vivid green': '#2fef10',
+ 'vivid purple': '#9900fa',
+ 'vomit': '#a2a415',
+ 'vomit green': '#89a203',
+ 'vomit yellow': '#c7c10c',
+ 'warm blue': '#4b57db',
+ 'warm brown': '#964e02',
+ 'warm grey': '#978a84',
+ 'warm pink': '#fb5581',
+ 'warm purple': '#952e8f',
+ 'washed out green': '#bcf5a6',
+ 'water blue': '#0e87cc',
+ 'watermelon': '#fd4659',
+ 'weird green': '#3ae57f',
+ 'wheat': '#fbdd7e',
+ 'white': '#ffffff',
+ 'windows blue': '#3778bf',
+ 'wine': '#80013f',
+ 'wine red': '#7b0323',
+ 'wintergreen': '#20f986',
+ 'wisteria': '#a87dc2',
+ 'yellow': '#ffff14',
+ 'yellow brown': '#b79400',
+ 'yellow green': '#c0fb2d',
+ 'yellow ochre': '#cb9d06',
+ 'yellow orange': '#fcb001',
+ 'yellow tan': '#ffe36e',
+ 'yellow/green': '#c8fd3d',
+ 'yellowgreen': '#bbf90f',
+ 'yellowish': '#faee66',
+ 'yellowish brown': '#9b7a01',
+ 'yellowish green': '#b0dd16',
+ 'yellowish orange': '#ffab0f',
+ 'yellowish tan': '#fcfc81',
+ 'yellowy brown': '#ae8b0c',
+ 'yellowy green': '#bff128'}
diff --git a/seaborn/distributions.py b/seaborn/distributions.py
new file mode 100644
index 0000000000000000000000000000000000000000..f8ec166cf4b346417e1cb7e43f591e9dd0cc970f
--- /dev/null
+++ b/seaborn/distributions.py
@@ -0,0 +1,2531 @@
+"""Plotting functions for visualizing distributions."""
+from numbers import Number
+from functools import partial
+import math
+import textwrap
+import warnings
+
+import numpy as np
+import pandas as pd
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+import matplotlib.transforms as tx
+from matplotlib.cbook import normalize_kwargs
+from matplotlib.colors import to_rgba
+from matplotlib.collections import LineCollection
+
+from ._base import VectorPlotter
+
+# We have moved univariate histogram computation over to the new Hist class,
+# but still use the older Histogram for bivariate computation.
+from ._statistics import ECDF, Histogram, KDE
+from ._stats.counting import Hist
+
+from .axisgrid import (
+ FacetGrid,
+ _facet_docs,
+)
+from .utils import (
+ remove_na,
+ _get_transform_functions,
+ _kde_support,
+ _check_argument,
+ _assign_default_kwargs,
+ _default_color,
+)
+from .palettes import color_palette
+from .external import husl
+from .external.kde import gaussian_kde
+from ._docstrings import (
+ DocstringComponents,
+ _core_docs,
+)
+
+
+__all__ = ["displot", "histplot", "kdeplot", "ecdfplot", "rugplot", "distplot"]
+
+# ==================================================================================== #
+# Module documentation
+# ==================================================================================== #
+
+_dist_params = dict(
+
+ multiple="""
+multiple : {{"layer", "stack", "fill"}}
+ Method for drawing multiple elements when semantic mapping creates subsets.
+ Only relevant with univariate data.
+ """,
+ log_scale="""
+log_scale : bool or number, or pair of bools or numbers
+ Set axis scale(s) to log. A single value sets the data axis for any numeric
+ axes in the plot. A pair of values sets each axis independently.
+ Numeric values are interpreted as the desired base (default 10).
+ When `None` or `False`, seaborn defers to the existing Axes scale.
+ """,
+ legend="""
+legend : bool
+ If False, suppress the legend for semantic variables.
+ """,
+ cbar="""
+cbar : bool
+ If True, add a colorbar to annotate the color mapping in a bivariate plot.
+ Note: Does not currently support plots with a ``hue`` variable well.
+ """,
+ cbar_ax="""
+cbar_ax : :class:`matplotlib.axes.Axes`
+ Pre-existing axes for the colorbar.
+ """,
+ cbar_kws="""
+cbar_kws : dict
+ Additional parameters passed to :meth:`matplotlib.figure.Figure.colorbar`.
+ """,
+)
+
+_param_docs = DocstringComponents.from_nested_components(
+ core=_core_docs["params"],
+ facets=DocstringComponents(_facet_docs),
+ dist=DocstringComponents(_dist_params),
+ kde=DocstringComponents.from_function_params(KDE.__init__),
+ hist=DocstringComponents.from_function_params(Histogram.__init__),
+ ecdf=DocstringComponents.from_function_params(ECDF.__init__),
+)
+
+
+# ==================================================================================== #
+# Internal API
+# ==================================================================================== #
+
+
+class _DistributionPlotter(VectorPlotter):
+
+ wide_structure = {"x": "@values", "hue": "@columns"}
+ flat_structure = {"x": "@values"}
+
+ def __init__(
+ self,
+ data=None,
+ variables={},
+ ):
+
+ super().__init__(data=data, variables=variables)
+
+ @property
+ def univariate(self):
+ """Return True if only x or y are used."""
+ # TODO this could go down to core, but putting it here now.
+ # We'd want to be conceptually clear that univariate only applies
+ # to x/y and not to other semantics, which can exist.
+ # We haven't settled on a good conceptual name for x/y.
+ return bool({"x", "y"} - set(self.variables))
+
+ @property
+ def data_variable(self):
+ """Return the variable with data for univariate plots."""
+ # TODO This could also be in core, but it should have a better name.
+ if not self.univariate:
+ raise AttributeError("This is not a univariate plot")
+ return {"x", "y"}.intersection(self.variables).pop()
+
+ @property
+ def has_xy_data(self):
+ """Return True at least one of x or y is defined."""
+ # TODO see above points about where this should go
+ return bool({"x", "y"} & set(self.variables))
+
+ def _add_legend(
+ self,
+ ax_obj, artist, fill, element, multiple, alpha, artist_kws, legend_kws,
+ ):
+ """Add artists that reflect semantic mappings and put then in a legend."""
+ # TODO note that this doesn't handle numeric mappings like the relational plots
+ handles = []
+ labels = []
+ for level in self._hue_map.levels:
+ color = self._hue_map(level)
+
+ kws = self._artist_kws(
+ artist_kws, fill, element, multiple, color, alpha
+ )
+
+ # color gets added to the kws to workaround an issue with barplot's color
+ # cycle integration but it causes problems in this context where we are
+ # setting artist properties directly, so pop it off here
+ if "facecolor" in kws:
+ kws.pop("color", None)
+
+ handles.append(artist(**kws))
+ labels.append(level)
+
+ if isinstance(ax_obj, mpl.axes.Axes):
+ ax_obj.legend(handles, labels, title=self.variables["hue"], **legend_kws)
+ else: # i.e. a FacetGrid. TODO make this better
+ legend_data = dict(zip(labels, handles))
+ ax_obj.add_legend(
+ legend_data,
+ title=self.variables["hue"],
+ label_order=self.var_levels["hue"],
+ **legend_kws
+ )
+
+ def _artist_kws(self, kws, fill, element, multiple, color, alpha):
+ """Handle differences between artists in filled/unfilled plots."""
+ kws = kws.copy()
+ if fill:
+ kws = normalize_kwargs(kws, mpl.collections.PolyCollection)
+ kws.setdefault("facecolor", to_rgba(color, alpha))
+
+ if element == "bars":
+ # Make bar() interface with property cycle correctly
+ # https://github.com/matplotlib/matplotlib/issues/19385
+ kws["color"] = "none"
+
+ if multiple in ["stack", "fill"] or element == "bars":
+ kws.setdefault("edgecolor", mpl.rcParams["patch.edgecolor"])
+ else:
+ kws.setdefault("edgecolor", to_rgba(color, 1))
+ elif element == "bars":
+ kws["facecolor"] = "none"
+ kws["edgecolor"] = to_rgba(color, alpha)
+ else:
+ kws["color"] = to_rgba(color, alpha)
+ return kws
+
+ def _quantile_to_level(self, data, quantile):
+ """Return data levels corresponding to quantile cuts of mass."""
+ isoprop = np.asarray(quantile)
+ values = np.ravel(data)
+ sorted_values = np.sort(values)[::-1]
+ normalized_values = np.cumsum(sorted_values) / values.sum()
+ idx = np.searchsorted(normalized_values, 1 - isoprop)
+ levels = np.take(sorted_values, idx, mode="clip")
+ return levels
+
+ def _cmap_from_color(self, color):
+ """Return a sequential colormap given a color seed."""
+ # Like so much else here, this is broadly useful, but keeping it
+ # in this class to signify that I haven't thought overly hard about it...
+ r, g, b, _ = to_rgba(color)
+ h, s, _ = husl.rgb_to_husl(r, g, b)
+ xx = np.linspace(-1, 1, int(1.15 * 256))[:256]
+ ramp = np.zeros((256, 3))
+ ramp[:, 0] = h
+ ramp[:, 1] = s * np.cos(xx)
+ ramp[:, 2] = np.linspace(35, 80, 256)
+ colors = np.clip([husl.husl_to_rgb(*hsl) for hsl in ramp], 0, 1)
+ return mpl.colors.ListedColormap(colors[::-1])
+
+ def _default_discrete(self):
+ """Find default values for discrete hist estimation based on variable type."""
+ if self.univariate:
+ discrete = self.var_types[self.data_variable] == "categorical"
+ else:
+ discrete_x = self.var_types["x"] == "categorical"
+ discrete_y = self.var_types["y"] == "categorical"
+ discrete = discrete_x, discrete_y
+ return discrete
+
+ def _resolve_multiple(self, curves, multiple):
+ """Modify the density data structure to handle multiple densities."""
+
+ # Default baselines have all densities starting at 0
+ baselines = {k: np.zeros_like(v) for k, v in curves.items()}
+
+ # TODO we should have some central clearinghouse for checking if any
+ # "grouping" (terminnology?) semantics have been assigned
+ if "hue" not in self.variables:
+ return curves, baselines
+
+ if multiple in ("stack", "fill"):
+
+ # Setting stack or fill means that the curves share a
+ # support grid / set of bin edges, so we can make a dataframe
+ # Reverse the column order to plot from top to bottom
+ curves = pd.DataFrame(curves).iloc[:, ::-1]
+
+ # Find column groups that are nested within col/row variables
+ column_groups = {}
+ for i, keyd in enumerate(map(dict, curves.columns)):
+ facet_key = keyd.get("col", None), keyd.get("row", None)
+ column_groups.setdefault(facet_key, [])
+ column_groups[facet_key].append(i)
+
+ baselines = curves.copy()
+
+ for col_idxs in column_groups.values():
+ cols = curves.columns[col_idxs]
+
+ norm_constant = curves[cols].sum(axis="columns")
+
+ # Take the cumulative sum to stack
+ curves[cols] = curves[cols].cumsum(axis="columns")
+
+ # Normalize by row sum to fill
+ if multiple == "fill":
+ curves[cols] = curves[cols].div(norm_constant, axis="index")
+
+ # Define where each segment starts
+ baselines[cols] = curves[cols].shift(1, axis=1).fillna(0)
+
+ if multiple == "dodge":
+
+ # Account for the unique semantic (non-faceting) levels
+ # This will require rethiniking if we add other semantics!
+ hue_levels = self.var_levels["hue"]
+ n = len(hue_levels)
+ f_fwd, f_inv = self._get_scale_transforms(self.data_variable)
+ for key in curves:
+
+ level = dict(key)["hue"]
+ hist = curves[key].reset_index(name="heights")
+ level_idx = hue_levels.index(level)
+
+ a = f_fwd(hist["edges"])
+ b = f_fwd(hist["edges"] + hist["widths"])
+ w = (b - a) / n
+ new_min = f_inv(a + level_idx * w)
+ new_max = f_inv(a + (level_idx + 1) * w)
+ hist["widths"] = new_max - new_min
+ hist["edges"] = new_min
+
+ curves[key] = hist.set_index(["edges", "widths"])["heights"]
+
+ return curves, baselines
+
+ # -------------------------------------------------------------------------------- #
+ # Computation
+ # -------------------------------------------------------------------------------- #
+
+ def _compute_univariate_density(
+ self,
+ data_variable,
+ common_norm,
+ common_grid,
+ estimate_kws,
+ warn_singular=True,
+ ):
+
+ # Initialize the estimator object
+ estimator = KDE(**estimate_kws)
+
+ if set(self.variables) - {"x", "y"}:
+ if common_grid:
+ all_observations = self.comp_data.dropna()
+ estimator.define_support(all_observations[data_variable])
+ else:
+ common_norm = False
+
+ all_data = self.plot_data.dropna()
+ if common_norm and "weights" in all_data:
+ whole_weight = all_data["weights"].sum()
+ else:
+ whole_weight = len(all_data)
+
+ densities = {}
+
+ for sub_vars, sub_data in self.iter_data("hue", from_comp_data=True):
+
+ # Extract the data points from this sub set and remove nulls
+ observations = sub_data[data_variable]
+
+ # Extract the weights for this subset of observations
+ if "weights" in self.variables:
+ weights = sub_data["weights"]
+ part_weight = weights.sum()
+ else:
+ weights = None
+ part_weight = len(sub_data)
+
+ # Estimate the density of observations at this level
+ variance = np.nan_to_num(observations.var())
+ singular = len(observations) < 2 or math.isclose(variance, 0)
+ try:
+ if not singular:
+ # Convoluted approach needed because numerical failures
+ # can manifest in a few different ways.
+ density, support = estimator(observations, weights=weights)
+ except np.linalg.LinAlgError:
+ singular = True
+
+ if singular:
+ msg = (
+ "Dataset has 0 variance; skipping density estimate. "
+ "Pass `warn_singular=False` to disable this warning."
+ )
+ if warn_singular:
+ warnings.warn(msg, UserWarning, stacklevel=4)
+ continue
+
+ # Invert the scaling of the support points
+ _, f_inv = self._get_scale_transforms(self.data_variable)
+ support = f_inv(support)
+
+ # Apply a scaling factor so that the integral over all subsets is 1
+ if common_norm:
+ density *= part_weight / whole_weight
+
+ # Store the density for this level
+ key = tuple(sub_vars.items())
+ densities[key] = pd.Series(density, index=support)
+
+ return densities
+
+ # -------------------------------------------------------------------------------- #
+ # Plotting
+ # -------------------------------------------------------------------------------- #
+
+ def plot_univariate_histogram(
+ self,
+ multiple,
+ element,
+ fill,
+ common_norm,
+ common_bins,
+ shrink,
+ kde,
+ kde_kws,
+ color,
+ legend,
+ line_kws,
+ estimate_kws,
+ **plot_kws,
+ ):
+
+ # -- Default keyword dicts
+ kde_kws = {} if kde_kws is None else kde_kws.copy()
+ line_kws = {} if line_kws is None else line_kws.copy()
+ estimate_kws = {} if estimate_kws is None else estimate_kws.copy()
+
+ # -- Input checking
+ _check_argument("multiple", ["layer", "stack", "fill", "dodge"], multiple)
+ _check_argument("element", ["bars", "step", "poly"], element)
+
+ auto_bins_with_weights = (
+ "weights" in self.variables
+ and estimate_kws["bins"] == "auto"
+ and estimate_kws["binwidth"] is None
+ and not estimate_kws["discrete"]
+ )
+ if auto_bins_with_weights:
+ msg = (
+ "`bins` cannot be 'auto' when using weights. "
+ "Setting `bins=10`, but you will likely want to adjust."
+ )
+ warnings.warn(msg, UserWarning)
+ estimate_kws["bins"] = 10
+
+ # Simplify downstream code if we are not normalizing
+ if estimate_kws["stat"] == "count":
+ common_norm = False
+
+ orient = self.data_variable
+
+ # Now initialize the Histogram estimator
+ estimator = Hist(**estimate_kws)
+ histograms = {}
+
+ # Do pre-compute housekeeping related to multiple groups
+ all_data = self.comp_data.dropna()
+ all_weights = all_data.get("weights", None)
+
+ multiple_histograms = set(self.variables) - {"x", "y"}
+ if multiple_histograms:
+ if common_bins:
+ bin_kws = estimator._define_bin_params(all_data, orient, None)
+ else:
+ common_norm = False
+
+ if common_norm and all_weights is not None:
+ whole_weight = all_weights.sum()
+ else:
+ whole_weight = len(all_data)
+
+ # Estimate the smoothed kernel densities, for use later
+ if kde:
+ # TODO alternatively, clip at min/max bins?
+ kde_kws.setdefault("cut", 0)
+ kde_kws["cumulative"] = estimate_kws["cumulative"]
+ densities = self._compute_univariate_density(
+ self.data_variable,
+ common_norm,
+ common_bins,
+ kde_kws,
+ warn_singular=False,
+ )
+
+ # First pass through the data to compute the histograms
+ for sub_vars, sub_data in self.iter_data("hue", from_comp_data=True):
+
+ # Prepare the relevant data
+ key = tuple(sub_vars.items())
+ orient = self.data_variable
+
+ if "weights" in self.variables:
+ sub_data["weight"] = sub_data.pop("weights")
+ part_weight = sub_data["weight"].sum()
+ else:
+ part_weight = len(sub_data)
+
+ # Do the histogram computation
+ if not (multiple_histograms and common_bins):
+ bin_kws = estimator._define_bin_params(sub_data, orient, None)
+ res = estimator._normalize(estimator._eval(sub_data, orient, bin_kws))
+ heights = res[estimator.stat].to_numpy()
+ widths = res["space"].to_numpy()
+ edges = res[orient].to_numpy() - widths / 2
+
+ # Rescale the smoothed curve to match the histogram
+ if kde and key in densities:
+ density = densities[key]
+ if estimator.cumulative:
+ hist_norm = heights.max()
+ else:
+ hist_norm = (heights * widths).sum()
+ densities[key] *= hist_norm
+
+ # Convert edges back to original units for plotting
+ ax = self._get_axes(sub_vars)
+ _, inv = _get_transform_functions(ax, self.data_variable)
+ widths = inv(edges + widths) - inv(edges)
+ edges = inv(edges)
+
+ # Pack the histogram data and metadata together
+ edges = edges + (1 - shrink) / 2 * widths
+ widths *= shrink
+ index = pd.MultiIndex.from_arrays([
+ pd.Index(edges, name="edges"),
+ pd.Index(widths, name="widths"),
+ ])
+ hist = pd.Series(heights, index=index, name="heights")
+
+ # Apply scaling to normalize across groups
+ if common_norm:
+ hist *= part_weight / whole_weight
+
+ # Store the finalized histogram data for future plotting
+ histograms[key] = hist
+
+ # Modify the histogram and density data to resolve multiple groups
+ histograms, baselines = self._resolve_multiple(histograms, multiple)
+ if kde:
+ densities, _ = self._resolve_multiple(
+ densities, None if multiple == "dodge" else multiple
+ )
+
+ # Set autoscaling-related meta
+ sticky_stat = (0, 1) if multiple == "fill" else (0, np.inf)
+ if multiple == "fill":
+ # Filled plots should not have any margins
+ bin_vals = histograms.index.to_frame()
+ edges = bin_vals["edges"]
+ widths = bin_vals["widths"]
+ sticky_data = (
+ edges.min(),
+ edges.max() + widths.loc[edges.idxmax()]
+ )
+ else:
+ sticky_data = []
+
+ # --- Handle default visual attributes
+
+ # Note: default linewidth is determined after plotting
+
+ # Default alpha should depend on other parameters
+ if fill:
+ # Note: will need to account for other grouping semantics if added
+ if "hue" in self.variables and multiple == "layer":
+ default_alpha = .5 if element == "bars" else .25
+ elif kde:
+ default_alpha = .5
+ else:
+ default_alpha = .75
+ else:
+ default_alpha = 1
+ alpha = plot_kws.pop("alpha", default_alpha) # TODO make parameter?
+
+ hist_artists = []
+
+ # Go back through the dataset and draw the plots
+ for sub_vars, _ in self.iter_data("hue", reverse=True):
+
+ key = tuple(sub_vars.items())
+ hist = histograms[key].rename("heights").reset_index()
+ bottom = np.asarray(baselines[key])
+
+ ax = self._get_axes(sub_vars)
+
+ # Define the matplotlib attributes that depend on semantic mapping
+ if "hue" in self.variables:
+ sub_color = self._hue_map(sub_vars["hue"])
+ else:
+ sub_color = color
+
+ artist_kws = self._artist_kws(
+ plot_kws, fill, element, multiple, sub_color, alpha
+ )
+
+ if element == "bars":
+
+ # Use matplotlib bar plotting
+
+ plot_func = ax.bar if self.data_variable == "x" else ax.barh
+ artists = plot_func(
+ hist["edges"],
+ hist["heights"] - bottom,
+ hist["widths"],
+ bottom,
+ align="edge",
+ **artist_kws,
+ )
+
+ for bar in artists:
+ if self.data_variable == "x":
+ bar.sticky_edges.x[:] = sticky_data
+ bar.sticky_edges.y[:] = sticky_stat
+ else:
+ bar.sticky_edges.x[:] = sticky_stat
+ bar.sticky_edges.y[:] = sticky_data
+
+ hist_artists.extend(artists)
+
+ else:
+
+ # Use either fill_between or plot to draw hull of histogram
+ if element == "step":
+
+ final = hist.iloc[-1]
+ x = np.append(hist["edges"], final["edges"] + final["widths"])
+ y = np.append(hist["heights"], final["heights"])
+ b = np.append(bottom, bottom[-1])
+
+ if self.data_variable == "x":
+ step = "post"
+ drawstyle = "steps-post"
+ else:
+ step = "post" # fillbetweenx handles mapping internally
+ drawstyle = "steps-pre"
+
+ elif element == "poly":
+
+ x = hist["edges"] + hist["widths"] / 2
+ y = hist["heights"]
+ b = bottom
+
+ step = None
+ drawstyle = None
+
+ if self.data_variable == "x":
+ if fill:
+ artist = ax.fill_between(x, b, y, step=step, **artist_kws)
+ else:
+ artist, = ax.plot(x, y, drawstyle=drawstyle, **artist_kws)
+ artist.sticky_edges.x[:] = sticky_data
+ artist.sticky_edges.y[:] = sticky_stat
+ else:
+ if fill:
+ artist = ax.fill_betweenx(x, b, y, step=step, **artist_kws)
+ else:
+ artist, = ax.plot(y, x, drawstyle=drawstyle, **artist_kws)
+ artist.sticky_edges.x[:] = sticky_stat
+ artist.sticky_edges.y[:] = sticky_data
+
+ hist_artists.append(artist)
+
+ if kde:
+
+ # Add in the density curves
+
+ try:
+ density = densities[key]
+ except KeyError:
+ continue
+ support = density.index
+
+ if "x" in self.variables:
+ line_args = support, density
+ sticky_x, sticky_y = None, (0, np.inf)
+ else:
+ line_args = density, support
+ sticky_x, sticky_y = (0, np.inf), None
+
+ line_kws["color"] = to_rgba(sub_color, 1)
+ line, = ax.plot(
+ *line_args, **line_kws,
+ )
+
+ if sticky_x is not None:
+ line.sticky_edges.x[:] = sticky_x
+ if sticky_y is not None:
+ line.sticky_edges.y[:] = sticky_y
+
+ if element == "bars" and "linewidth" not in plot_kws:
+
+ # Now we handle linewidth, which depends on the scaling of the plot
+
+ # We will base everything on the minimum bin width
+ hist_metadata = pd.concat([
+ # Use .items for generality over dict or df
+ h.index.to_frame() for _, h in histograms.items()
+ ]).reset_index(drop=True)
+ thin_bar_idx = hist_metadata["widths"].idxmin()
+ binwidth = hist_metadata.loc[thin_bar_idx, "widths"]
+ left_edge = hist_metadata.loc[thin_bar_idx, "edges"]
+
+ # Set initial value
+ default_linewidth = math.inf
+
+ # Loop through subsets based only on facet variables
+ for sub_vars, _ in self.iter_data():
+
+ ax = self._get_axes(sub_vars)
+
+ # Needed in some cases to get valid transforms.
+ # Innocuous in other cases?
+ ax.autoscale_view()
+
+ # Convert binwidth from data coordinates to pixels
+ pts_x, pts_y = 72 / ax.figure.dpi * abs(
+ ax.transData.transform([left_edge + binwidth] * 2)
+ - ax.transData.transform([left_edge] * 2)
+ )
+ if self.data_variable == "x":
+ binwidth_points = pts_x
+ else:
+ binwidth_points = pts_y
+
+ # The relative size of the lines depends on the appearance
+ # This is a provisional value and may need more tweaking
+ default_linewidth = min(.1 * binwidth_points, default_linewidth)
+
+ # Set the attributes
+ for bar in hist_artists:
+
+ # Don't let the lines get too thick
+ max_linewidth = bar.get_linewidth()
+ if not fill:
+ max_linewidth *= 1.5
+
+ linewidth = min(default_linewidth, max_linewidth)
+
+ # If not filling, don't let lines disappear
+ if not fill:
+ min_linewidth = .5
+ linewidth = max(linewidth, min_linewidth)
+
+ bar.set_linewidth(linewidth)
+
+ # --- Finalize the plot ----
+
+ # Axis labels
+ ax = self.ax if self.ax is not None else self.facets.axes.flat[0]
+ default_x = default_y = ""
+ if self.data_variable == "x":
+ default_y = estimator.stat.capitalize()
+ if self.data_variable == "y":
+ default_x = estimator.stat.capitalize()
+ self._add_axis_labels(ax, default_x, default_y)
+
+ # Legend for semantic variables
+ if "hue" in self.variables and legend:
+
+ if fill or element == "bars":
+ artist = partial(mpl.patches.Patch)
+ else:
+ artist = partial(mpl.lines.Line2D, [], [])
+
+ ax_obj = self.ax if self.ax is not None else self.facets
+ self._add_legend(
+ ax_obj, artist, fill, element, multiple, alpha, plot_kws, {},
+ )
+
+ def plot_bivariate_histogram(
+ self,
+ common_bins, common_norm,
+ thresh, pthresh, pmax,
+ color, legend,
+ cbar, cbar_ax, cbar_kws,
+ estimate_kws,
+ **plot_kws,
+ ):
+
+ # Default keyword dicts
+ cbar_kws = {} if cbar_kws is None else cbar_kws.copy()
+
+ # Now initialize the Histogram estimator
+ estimator = Histogram(**estimate_kws)
+
+ # Do pre-compute housekeeping related to multiple groups
+ if set(self.variables) - {"x", "y"}:
+ all_data = self.comp_data.dropna()
+ if common_bins:
+ estimator.define_bin_params(
+ all_data["x"],
+ all_data["y"],
+ all_data.get("weights", None),
+ )
+ else:
+ common_norm = False
+
+ # -- Determine colormap threshold and norm based on the full data
+
+ full_heights = []
+ for _, sub_data in self.iter_data(from_comp_data=True):
+ sub_heights, _ = estimator(
+ sub_data["x"], sub_data["y"], sub_data.get("weights", None)
+ )
+ full_heights.append(sub_heights)
+
+ common_color_norm = not set(self.variables) - {"x", "y"} or common_norm
+
+ if pthresh is not None and common_color_norm:
+ thresh = self._quantile_to_level(full_heights, pthresh)
+
+ plot_kws.setdefault("vmin", 0)
+ if common_color_norm:
+ if pmax is not None:
+ vmax = self._quantile_to_level(full_heights, pmax)
+ else:
+ vmax = plot_kws.pop("vmax", max(map(np.max, full_heights)))
+ else:
+ vmax = None
+
+ # Get a default color
+ # (We won't follow the color cycle here, as multiple plots are unlikely)
+ if color is None:
+ color = "C0"
+
+ # --- Loop over data (subsets) and draw the histograms
+ for sub_vars, sub_data in self.iter_data("hue", from_comp_data=True):
+
+ if sub_data.empty:
+ continue
+
+ # Do the histogram computation
+ heights, (x_edges, y_edges) = estimator(
+ sub_data["x"],
+ sub_data["y"],
+ weights=sub_data.get("weights", None),
+ )
+
+ # Get the axes for this plot
+ ax = self._get_axes(sub_vars)
+
+ # Invert the scale for the edges
+ _, inv_x = _get_transform_functions(ax, "x")
+ _, inv_y = _get_transform_functions(ax, "y")
+ x_edges = inv_x(x_edges)
+ y_edges = inv_y(y_edges)
+
+ # Apply scaling to normalize across groups
+ if estimator.stat != "count" and common_norm:
+ heights *= len(sub_data) / len(all_data)
+
+ # Define the specific kwargs for this artist
+ artist_kws = plot_kws.copy()
+ if "hue" in self.variables:
+ color = self._hue_map(sub_vars["hue"])
+ cmap = self._cmap_from_color(color)
+ artist_kws["cmap"] = cmap
+ else:
+ cmap = artist_kws.pop("cmap", None)
+ if isinstance(cmap, str):
+ cmap = color_palette(cmap, as_cmap=True)
+ elif cmap is None:
+ cmap = self._cmap_from_color(color)
+ artist_kws["cmap"] = cmap
+
+ # Set the upper norm on the colormap
+ if not common_color_norm and pmax is not None:
+ vmax = self._quantile_to_level(heights, pmax)
+ if vmax is not None:
+ artist_kws["vmax"] = vmax
+
+ # Make cells at or below the threshold transparent
+ if not common_color_norm and pthresh:
+ thresh = self._quantile_to_level(heights, pthresh)
+ if thresh is not None:
+ heights = np.ma.masked_less_equal(heights, thresh)
+
+ # pcolormesh is going to turn the grid off, but we want to keep it
+ # I'm not sure if there's a better way to get the grid state
+ x_grid = any([l.get_visible() for l in ax.xaxis.get_gridlines()])
+ y_grid = any([l.get_visible() for l in ax.yaxis.get_gridlines()])
+
+ mesh = ax.pcolormesh(
+ x_edges,
+ y_edges,
+ heights.T,
+ **artist_kws,
+ )
+
+ # pcolormesh sets sticky edges, but we only want them if not thresholding
+ if thresh is not None:
+ mesh.sticky_edges.x[:] = []
+ mesh.sticky_edges.y[:] = []
+
+ # Add an optional colorbar
+ # Note, we want to improve this. When hue is used, it will stack
+ # multiple colorbars with redundant ticks in an ugly way.
+ # But it's going to take some work to have multiple colorbars that
+ # share ticks nicely.
+ if cbar:
+ ax.figure.colorbar(mesh, cbar_ax, ax, **cbar_kws)
+
+ # Reset the grid state
+ if x_grid:
+ ax.grid(True, axis="x")
+ if y_grid:
+ ax.grid(True, axis="y")
+
+ # --- Finalize the plot
+
+ ax = self.ax if self.ax is not None else self.facets.axes.flat[0]
+ self._add_axis_labels(ax)
+
+ if "hue" in self.variables and legend:
+
+ # TODO if possible, I would like to move the contour
+ # intensity information into the legend too and label the
+ # iso proportions rather than the raw density values
+
+ artist_kws = {}
+ artist = partial(mpl.patches.Patch)
+ ax_obj = self.ax if self.ax is not None else self.facets
+ self._add_legend(
+ ax_obj, artist, True, False, "layer", 1, artist_kws, {},
+ )
+
+ def plot_univariate_density(
+ self,
+ multiple,
+ common_norm,
+ common_grid,
+ warn_singular,
+ fill,
+ color,
+ legend,
+ estimate_kws,
+ **plot_kws,
+ ):
+
+ # Handle conditional defaults
+ if fill is None:
+ fill = multiple in ("stack", "fill")
+
+ # Preprocess the matplotlib keyword dictionaries
+ if fill:
+ artist = mpl.collections.PolyCollection
+ else:
+ artist = mpl.lines.Line2D
+ plot_kws = normalize_kwargs(plot_kws, artist)
+
+ # Input checking
+ _check_argument("multiple", ["layer", "stack", "fill"], multiple)
+
+ # Always share the evaluation grid when stacking
+ subsets = bool(set(self.variables) - {"x", "y"})
+ if subsets and multiple in ("stack", "fill"):
+ common_grid = True
+
+ # Do the computation
+ densities = self._compute_univariate_density(
+ self.data_variable,
+ common_norm,
+ common_grid,
+ estimate_kws,
+ warn_singular,
+ )
+
+ # Adjust densities based on the `multiple` rule
+ densities, baselines = self._resolve_multiple(densities, multiple)
+
+ # Control the interaction with autoscaling by defining sticky_edges
+ # i.e. we don't want autoscale margins below the density curve
+ sticky_density = (0, 1) if multiple == "fill" else (0, np.inf)
+
+ if multiple == "fill":
+ # Filled plots should not have any margins
+ sticky_support = densities.index.min(), densities.index.max()
+ else:
+ sticky_support = []
+
+ if fill:
+ if multiple == "layer":
+ default_alpha = .25
+ else:
+ default_alpha = .75
+ else:
+ default_alpha = 1
+ alpha = plot_kws.pop("alpha", default_alpha) # TODO make parameter?
+
+ # Now iterate through the subsets and draw the densities
+ # We go backwards so stacked densities read from top-to-bottom
+ for sub_vars, _ in self.iter_data("hue", reverse=True):
+
+ # Extract the support grid and density curve for this level
+ key = tuple(sub_vars.items())
+ try:
+ density = densities[key]
+ except KeyError:
+ continue
+ support = density.index
+ fill_from = baselines[key]
+
+ ax = self._get_axes(sub_vars)
+
+ if "hue" in self.variables:
+ sub_color = self._hue_map(sub_vars["hue"])
+ else:
+ sub_color = color
+
+ artist_kws = self._artist_kws(
+ plot_kws, fill, False, multiple, sub_color, alpha
+ )
+
+ # Either plot a curve with observation values on the x axis
+ if "x" in self.variables:
+
+ if fill:
+ artist = ax.fill_between(support, fill_from, density, **artist_kws)
+
+ else:
+ artist, = ax.plot(support, density, **artist_kws)
+
+ artist.sticky_edges.x[:] = sticky_support
+ artist.sticky_edges.y[:] = sticky_density
+
+ # Or plot a curve with observation values on the y axis
+ else:
+ if fill:
+ artist = ax.fill_betweenx(support, fill_from, density, **artist_kws)
+ else:
+ artist, = ax.plot(density, support, **artist_kws)
+
+ artist.sticky_edges.x[:] = sticky_density
+ artist.sticky_edges.y[:] = sticky_support
+
+ # --- Finalize the plot ----
+
+ ax = self.ax if self.ax is not None else self.facets.axes.flat[0]
+ default_x = default_y = ""
+ if self.data_variable == "x":
+ default_y = "Density"
+ if self.data_variable == "y":
+ default_x = "Density"
+ self._add_axis_labels(ax, default_x, default_y)
+
+ if "hue" in self.variables and legend:
+
+ if fill:
+ artist = partial(mpl.patches.Patch)
+ else:
+ artist = partial(mpl.lines.Line2D, [], [])
+
+ ax_obj = self.ax if self.ax is not None else self.facets
+ self._add_legend(
+ ax_obj, artist, fill, False, multiple, alpha, plot_kws, {},
+ )
+
+ def plot_bivariate_density(
+ self,
+ common_norm,
+ fill,
+ levels,
+ thresh,
+ color,
+ legend,
+ cbar,
+ warn_singular,
+ cbar_ax,
+ cbar_kws,
+ estimate_kws,
+ **contour_kws,
+ ):
+
+ contour_kws = contour_kws.copy()
+
+ estimator = KDE(**estimate_kws)
+
+ if not set(self.variables) - {"x", "y"}:
+ common_norm = False
+
+ all_data = self.plot_data.dropna()
+
+ # Loop through the subsets and estimate the KDEs
+ densities, supports = {}, {}
+
+ for sub_vars, sub_data in self.iter_data("hue", from_comp_data=True):
+
+ # Extract the data points from this sub set
+ observations = sub_data[["x", "y"]]
+ min_variance = observations.var().fillna(0).min()
+ observations = observations["x"], observations["y"]
+
+ # Extract the weights for this subset of observations
+ if "weights" in self.variables:
+ weights = sub_data["weights"]
+ else:
+ weights = None
+
+ # Estimate the density of observations at this level
+ singular = math.isclose(min_variance, 0)
+ try:
+ if not singular:
+ density, support = estimator(*observations, weights=weights)
+ except np.linalg.LinAlgError:
+ # Testing for 0 variance doesn't catch all cases where scipy raises,
+ # but we can also get a ValueError, so we need this convoluted approach
+ singular = True
+
+ if singular:
+ msg = (
+ "KDE cannot be estimated (0 variance or perfect covariance). "
+ "Pass `warn_singular=False` to disable this warning."
+ )
+ if warn_singular:
+ warnings.warn(msg, UserWarning, stacklevel=3)
+ continue
+
+ # Transform the support grid back to the original scale
+ ax = self._get_axes(sub_vars)
+ _, inv_x = _get_transform_functions(ax, "x")
+ _, inv_y = _get_transform_functions(ax, "y")
+ support = inv_x(support[0]), inv_y(support[1])
+
+ # Apply a scaling factor so that the integral over all subsets is 1
+ if common_norm:
+ density *= len(sub_data) / len(all_data)
+
+ key = tuple(sub_vars.items())
+ densities[key] = density
+ supports[key] = support
+
+ # Define a grid of iso-proportion levels
+ if thresh is None:
+ thresh = 0
+ if isinstance(levels, Number):
+ levels = np.linspace(thresh, 1, levels)
+ else:
+ if min(levels) < 0 or max(levels) > 1:
+ raise ValueError("levels must be in [0, 1]")
+
+ # Transform from iso-proportions to iso-densities
+ if common_norm:
+ common_levels = self._quantile_to_level(
+ list(densities.values()), levels,
+ )
+ draw_levels = {k: common_levels for k in densities}
+ else:
+ draw_levels = {
+ k: self._quantile_to_level(d, levels)
+ for k, d in densities.items()
+ }
+
+ # Define the coloring of the contours
+ if "hue" in self.variables:
+ for param in ["cmap", "colors"]:
+ if param in contour_kws:
+ msg = f"{param} parameter ignored when using hue mapping."
+ warnings.warn(msg, UserWarning)
+ contour_kws.pop(param)
+ else:
+
+ # Work out a default coloring of the contours
+ coloring_given = set(contour_kws) & {"cmap", "colors"}
+ if fill and not coloring_given:
+ cmap = self._cmap_from_color(color)
+ contour_kws["cmap"] = cmap
+ if not fill and not coloring_given:
+ contour_kws["colors"] = [color]
+
+ # Use our internal colormap lookup
+ cmap = contour_kws.pop("cmap", None)
+ if isinstance(cmap, str):
+ cmap = color_palette(cmap, as_cmap=True)
+ if cmap is not None:
+ contour_kws["cmap"] = cmap
+
+ # Loop through the subsets again and plot the data
+ for sub_vars, _ in self.iter_data("hue"):
+
+ if "hue" in sub_vars:
+ color = self._hue_map(sub_vars["hue"])
+ if fill:
+ contour_kws["cmap"] = self._cmap_from_color(color)
+ else:
+ contour_kws["colors"] = [color]
+
+ ax = self._get_axes(sub_vars)
+
+ # Choose the function to plot with
+ # TODO could add a pcolormesh based option as well
+ # Which would look something like element="raster"
+ if fill:
+ contour_func = ax.contourf
+ else:
+ contour_func = ax.contour
+
+ key = tuple(sub_vars.items())
+ if key not in densities:
+ continue
+ density = densities[key]
+ xx, yy = supports[key]
+
+ # Pop the label kwarg which is unused by contour_func (but warns)
+ contour_kws.pop("label", None)
+
+ cset = contour_func(
+ xx, yy, density,
+ levels=draw_levels[key],
+ **contour_kws,
+ )
+
+ # Add a color bar representing the contour heights
+ # Note: this shows iso densities, not iso proportions
+ # See more notes in histplot about how this could be improved
+ if cbar:
+ cbar_kws = {} if cbar_kws is None else cbar_kws
+ ax.figure.colorbar(cset, cbar_ax, ax, **cbar_kws)
+
+ # --- Finalize the plot
+ ax = self.ax if self.ax is not None else self.facets.axes.flat[0]
+ self._add_axis_labels(ax)
+
+ if "hue" in self.variables and legend:
+
+ # TODO if possible, I would like to move the contour
+ # intensity information into the legend too and label the
+ # iso proportions rather than the raw density values
+
+ artist_kws = {}
+ if fill:
+ artist = partial(mpl.patches.Patch)
+ else:
+ artist = partial(mpl.lines.Line2D, [], [])
+
+ ax_obj = self.ax if self.ax is not None else self.facets
+ self._add_legend(
+ ax_obj, artist, fill, False, "layer", 1, artist_kws, {},
+ )
+
+ def plot_univariate_ecdf(self, estimate_kws, legend, **plot_kws):
+
+ estimator = ECDF(**estimate_kws)
+
+ # Set the draw style to step the right way for the data variable
+ drawstyles = dict(x="steps-post", y="steps-pre")
+ plot_kws["drawstyle"] = drawstyles[self.data_variable]
+
+ # Loop through the subsets, transform and plot the data
+ for sub_vars, sub_data in self.iter_data(
+ "hue", reverse=True, from_comp_data=True,
+ ):
+
+ # Compute the ECDF
+ if sub_data.empty:
+ continue
+
+ observations = sub_data[self.data_variable]
+ weights = sub_data.get("weights", None)
+ stat, vals = estimator(observations, weights=weights)
+
+ # Assign attributes based on semantic mapping
+ artist_kws = plot_kws.copy()
+ if "hue" in self.variables:
+ artist_kws["color"] = self._hue_map(sub_vars["hue"])
+
+ # Return the data variable to the linear domain
+ ax = self._get_axes(sub_vars)
+ _, inv = _get_transform_functions(ax, self.data_variable)
+ vals = inv(vals)
+
+ # Manually set the minimum value on a "log" scale
+ if isinstance(inv.__self__, mpl.scale.LogTransform):
+ vals[0] = -np.inf
+
+ # Work out the orientation of the plot
+ if self.data_variable == "x":
+ plot_args = vals, stat
+ stat_variable = "y"
+ else:
+ plot_args = stat, vals
+ stat_variable = "x"
+
+ if estimator.stat == "count":
+ top_edge = len(observations)
+ else:
+ top_edge = 1
+
+ # Draw the line for this subset
+ artist, = ax.plot(*plot_args, **artist_kws)
+ sticky_edges = getattr(artist.sticky_edges, stat_variable)
+ sticky_edges[:] = 0, top_edge
+
+ # --- Finalize the plot ----
+ ax = self.ax if self.ax is not None else self.facets.axes.flat[0]
+ stat = estimator.stat.capitalize()
+ default_x = default_y = ""
+ if self.data_variable == "x":
+ default_y = stat
+ if self.data_variable == "y":
+ default_x = stat
+ self._add_axis_labels(ax, default_x, default_y)
+
+ if "hue" in self.variables and legend:
+ artist = partial(mpl.lines.Line2D, [], [])
+ alpha = plot_kws.get("alpha", 1)
+ ax_obj = self.ax if self.ax is not None else self.facets
+ self._add_legend(
+ ax_obj, artist, False, False, None, alpha, plot_kws, {},
+ )
+
+ def plot_rug(self, height, expand_margins, legend, **kws):
+
+ for sub_vars, sub_data, in self.iter_data(from_comp_data=True):
+
+ ax = self._get_axes(sub_vars)
+
+ kws.setdefault("linewidth", 1)
+
+ if expand_margins:
+ xmarg, ymarg = ax.margins()
+ if "x" in self.variables:
+ ymarg += height * 2
+ if "y" in self.variables:
+ xmarg += height * 2
+ ax.margins(x=xmarg, y=ymarg)
+
+ if "hue" in self.variables:
+ kws.pop("c", None)
+ kws.pop("color", None)
+
+ if "x" in self.variables:
+ self._plot_single_rug(sub_data, "x", height, ax, kws)
+ if "y" in self.variables:
+ self._plot_single_rug(sub_data, "y", height, ax, kws)
+
+ # --- Finalize the plot
+ self._add_axis_labels(ax)
+ if "hue" in self.variables and legend:
+ # TODO ideally i'd like the legend artist to look like a rug
+ legend_artist = partial(mpl.lines.Line2D, [], [])
+ self._add_legend(
+ ax, legend_artist, False, False, None, 1, {}, {},
+ )
+
+ def _plot_single_rug(self, sub_data, var, height, ax, kws):
+ """Draw a rugplot along one axis of the plot."""
+ vector = sub_data[var]
+ n = len(vector)
+
+ # Return data to linear domain
+ _, inv = _get_transform_functions(ax, var)
+ vector = inv(vector)
+
+ # We'll always add a single collection with varying colors
+ if "hue" in self.variables:
+ colors = self._hue_map(sub_data["hue"])
+ else:
+ colors = None
+
+ # Build the array of values for the LineCollection
+ if var == "x":
+
+ trans = tx.blended_transform_factory(ax.transData, ax.transAxes)
+ xy_pairs = np.column_stack([
+ np.repeat(vector, 2), np.tile([0, height], n)
+ ])
+
+ if var == "y":
+
+ trans = tx.blended_transform_factory(ax.transAxes, ax.transData)
+ xy_pairs = np.column_stack([
+ np.tile([0, height], n), np.repeat(vector, 2)
+ ])
+
+ # Draw the lines on the plot
+ line_segs = xy_pairs.reshape([n, 2, 2])
+ ax.add_collection(LineCollection(
+ line_segs, transform=trans, colors=colors, **kws
+ ))
+
+ ax.autoscale_view(scalex=var == "x", scaley=var == "y")
+
+
+# ==================================================================================== #
+# External API
+# ==================================================================================== #
+
+def histplot(
+ data=None, *,
+ # Vector variables
+ x=None, y=None, hue=None, weights=None,
+ # Histogram computation parameters
+ stat="count", bins="auto", binwidth=None, binrange=None,
+ discrete=None, cumulative=False, common_bins=True, common_norm=True,
+ # Histogram appearance parameters
+ multiple="layer", element="bars", fill=True, shrink=1,
+ # Histogram smoothing with a kernel density estimate
+ kde=False, kde_kws=None, line_kws=None,
+ # Bivariate histogram parameters
+ thresh=0, pthresh=None, pmax=None, cbar=False, cbar_ax=None, cbar_kws=None,
+ # Hue mapping parameters
+ palette=None, hue_order=None, hue_norm=None, color=None,
+ # Axes information
+ log_scale=None, legend=True, ax=None,
+ # Other appearance keywords
+ **kwargs,
+):
+
+ p = _DistributionPlotter(
+ data=data,
+ variables=dict(x=x, y=y, hue=hue, weights=weights),
+ )
+
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm)
+
+ if ax is None:
+ ax = plt.gca()
+
+ p._attach(ax, log_scale=log_scale)
+
+ if p.univariate: # Note, bivariate plots won't cycle
+ if fill:
+ method = ax.bar if element == "bars" else ax.fill_between
+ else:
+ method = ax.plot
+ color = _default_color(method, hue, color, kwargs)
+
+ if not p.has_xy_data:
+ return ax
+
+ # Default to discrete bins for categorical variables
+ if discrete is None:
+ discrete = p._default_discrete()
+
+ estimate_kws = dict(
+ stat=stat,
+ bins=bins,
+ binwidth=binwidth,
+ binrange=binrange,
+ discrete=discrete,
+ cumulative=cumulative,
+ )
+
+ if p.univariate:
+
+ p.plot_univariate_histogram(
+ multiple=multiple,
+ element=element,
+ fill=fill,
+ shrink=shrink,
+ common_norm=common_norm,
+ common_bins=common_bins,
+ kde=kde,
+ kde_kws=kde_kws,
+ color=color,
+ legend=legend,
+ estimate_kws=estimate_kws,
+ line_kws=line_kws,
+ **kwargs,
+ )
+
+ else:
+
+ p.plot_bivariate_histogram(
+ common_bins=common_bins,
+ common_norm=common_norm,
+ thresh=thresh,
+ pthresh=pthresh,
+ pmax=pmax,
+ color=color,
+ legend=legend,
+ cbar=cbar,
+ cbar_ax=cbar_ax,
+ cbar_kws=cbar_kws,
+ estimate_kws=estimate_kws,
+ **kwargs,
+ )
+
+ return ax
+
+
+histplot.__doc__ = """\
+Plot univariate or bivariate histograms to show distributions of datasets.
+
+A histogram is a classic visualization tool that represents the distribution
+of one or more variables by counting the number of observations that fall within
+discrete bins.
+
+This function can normalize the statistic computed within each bin to estimate
+frequency, density or probability mass, and it can add a smooth curve obtained
+using a kernel density estimate, similar to :func:`kdeplot`.
+
+More information is provided in the :ref:`user guide <tutorial_hist>`.
+
+Parameters
+----------
+{params.core.data}
+{params.core.xy}
+{params.core.hue}
+weights : vector or key in ``data``
+ If provided, weight the contribution of the corresponding data points
+ towards the count in each bin by these factors.
+{params.hist.stat}
+{params.hist.bins}
+{params.hist.binwidth}
+{params.hist.binrange}
+discrete : bool
+ If True, default to ``binwidth=1`` and draw the bars so that they are
+ centered on their corresponding data points. This avoids "gaps" that may
+ otherwise appear when using discrete (integer) data.
+cumulative : bool
+ If True, plot the cumulative counts as bins increase.
+common_bins : bool
+ If True, use the same bins when semantic variables produce multiple
+ plots. If using a reference rule to determine the bins, it will be computed
+ with the full dataset.
+common_norm : bool
+ If True and using a normalized statistic, the normalization will apply over
+ the full dataset. Otherwise, normalize each histogram independently.
+multiple : {{"layer", "dodge", "stack", "fill"}}
+ Approach to resolving multiple elements when semantic mapping creates subsets.
+ Only relevant with univariate data.
+element : {{"bars", "step", "poly"}}
+ Visual representation of the histogram statistic.
+ Only relevant with univariate data.
+fill : bool
+ If True, fill in the space under the histogram.
+ Only relevant with univariate data.
+shrink : number
+ Scale the width of each bar relative to the binwidth by this factor.
+ Only relevant with univariate data.
+kde : bool
+ If True, compute a kernel density estimate to smooth the distribution
+ and show on the plot as (one or more) line(s).
+ Only relevant with univariate data.
+kde_kws : dict
+ Parameters that control the KDE computation, as in :func:`kdeplot`.
+line_kws : dict
+ Parameters that control the KDE visualization, passed to
+ :meth:`matplotlib.axes.Axes.plot`.
+thresh : number or None
+ Cells with a statistic less than or equal to this value will be transparent.
+ Only relevant with bivariate data.
+pthresh : number or None
+ Like ``thresh``, but a value in [0, 1] such that cells with aggregate counts
+ (or other statistics, when used) up to this proportion of the total will be
+ transparent.
+pmax : number or None
+ A value in [0, 1] that sets that saturation point for the colormap at a value
+ such that cells below constitute this proportion of the total count (or
+ other statistic, when used).
+{params.dist.cbar}
+{params.dist.cbar_ax}
+{params.dist.cbar_kws}
+{params.core.palette}
+{params.core.hue_order}
+{params.core.hue_norm}
+{params.core.color}
+{params.dist.log_scale}
+{params.dist.legend}
+{params.core.ax}
+kwargs
+ Other keyword arguments are passed to one of the following matplotlib
+ functions:
+
+ - :meth:`matplotlib.axes.Axes.bar` (univariate, element="bars")
+ - :meth:`matplotlib.axes.Axes.fill_between` (univariate, other element, fill=True)
+ - :meth:`matplotlib.axes.Axes.plot` (univariate, other element, fill=False)
+ - :meth:`matplotlib.axes.Axes.pcolormesh` (bivariate)
+
+Returns
+-------
+{returns.ax}
+
+See Also
+--------
+{seealso.displot}
+{seealso.kdeplot}
+{seealso.rugplot}
+{seealso.ecdfplot}
+{seealso.jointplot}
+
+Notes
+-----
+
+The choice of bins for computing and plotting a histogram can exert
+substantial influence on the insights that one is able to draw from the
+visualization. If the bins are too large, they may erase important features.
+On the other hand, bins that are too small may be dominated by random
+variability, obscuring the shape of the true underlying distribution. The
+default bin size is determined using a reference rule that depends on the
+sample size and variance. This works well in many cases, (i.e., with
+"well-behaved" data) but it fails in others. It is always a good to try
+different bin sizes to be sure that you are not missing something important.
+This function allows you to specify bins in several different ways, such as
+by setting the total number of bins to use, the width of each bin, or the
+specific locations where the bins should break.
+
+Examples
+--------
+
+.. include:: ../docstrings/histplot.rst
+
+""".format(
+ params=_param_docs,
+ returns=_core_docs["returns"],
+ seealso=_core_docs["seealso"],
+)
+
+
+def kdeplot(
+ data=None, *, x=None, y=None, hue=None, weights=None,
+ palette=None, hue_order=None, hue_norm=None, color=None, fill=None,
+ multiple="layer", common_norm=True, common_grid=False, cumulative=False,
+ bw_method="scott", bw_adjust=1, warn_singular=True, log_scale=None,
+ levels=10, thresh=.05, gridsize=200, cut=3, clip=None,
+ legend=True, cbar=False, cbar_ax=None, cbar_kws=None, ax=None,
+ **kwargs,
+):
+
+ # --- Start with backwards compatability for versions < 0.11.0 ----------------
+
+ # Handle (past) deprecation of `data2`
+ if "data2" in kwargs:
+ msg = "`data2` has been removed (replaced by `y`); please update your code."
+ raise TypeError(msg)
+
+ # Handle deprecation of `vertical`
+ vertical = kwargs.pop("vertical", None)
+ if vertical is not None:
+ if vertical:
+ action_taken = "assigning data to `y`."
+ if x is None:
+ data, y = y, data
+ else:
+ x, y = y, x
+ else:
+ action_taken = "assigning data to `x`."
+ msg = textwrap.dedent(f"""\n
+ The `vertical` parameter is deprecated; {action_taken}
+ This will become an error in seaborn v0.14.0; please update your code.
+ """)
+ warnings.warn(msg, UserWarning, stacklevel=2)
+
+ # Handle deprecation of `bw`
+ bw = kwargs.pop("bw", None)
+ if bw is not None:
+ msg = textwrap.dedent(f"""\n
+ The `bw` parameter is deprecated in favor of `bw_method` and `bw_adjust`.
+ Setting `bw_method={bw}`, but please see the docs for the new parameters
+ and update your code. This will become an error in seaborn v0.14.0.
+ """)
+ warnings.warn(msg, UserWarning, stacklevel=2)
+ bw_method = bw
+
+ # Handle deprecation of `kernel`
+ if kwargs.pop("kernel", None) is not None:
+ msg = textwrap.dedent("""\n
+ Support for alternate kernels has been removed; using Gaussian kernel.
+ This will become an error in seaborn v0.14.0; please update your code.
+ """)
+ warnings.warn(msg, UserWarning, stacklevel=2)
+
+ # Handle deprecation of shade_lowest
+ shade_lowest = kwargs.pop("shade_lowest", None)
+ if shade_lowest is not None:
+ if shade_lowest:
+ thresh = 0
+ msg = textwrap.dedent(f"""\n
+ `shade_lowest` has been replaced by `thresh`; setting `thresh={thresh}.
+ This will become an error in seaborn v0.14.0; please update your code.
+ """)
+ warnings.warn(msg, UserWarning, stacklevel=2)
+
+ # Handle "soft" deprecation of shade `shade` is not really the right
+ # terminology here, but unlike some of the other deprecated parameters it
+ # is probably very commonly used and much hard to remove. This is therefore
+ # going to be a longer process where, first, `fill` will be introduced and
+ # be used throughout the documentation. In 0.12, when kwarg-only
+ # enforcement hits, we can remove the shade/shade_lowest out of the
+ # function signature all together and pull them out of the kwargs. Then we
+ # can actually fire a FutureWarning, and eventually remove.
+ shade = kwargs.pop("shade", None)
+ if shade is not None:
+ fill = shade
+ msg = textwrap.dedent(f"""\n
+ `shade` is now deprecated in favor of `fill`; setting `fill={shade}`.
+ This will become an error in seaborn v0.14.0; please update your code.
+ """)
+ warnings.warn(msg, FutureWarning, stacklevel=2)
+
+ # Handle `n_levels`
+ # This was never in the formal API but it was processed, and appeared in an
+ # example. We can treat as an alias for `levels` now and deprecate later.
+ levels = kwargs.pop("n_levels", levels)
+
+ # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
+
+ p = _DistributionPlotter(
+ data=data,
+ variables=dict(x=x, y=y, hue=hue, weights=weights),
+ )
+
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm)
+
+ if ax is None:
+ ax = plt.gca()
+
+ p._attach(ax, allowed_types=["numeric", "datetime"], log_scale=log_scale)
+
+ method = ax.fill_between if fill else ax.plot
+ color = _default_color(method, hue, color, kwargs)
+
+ if not p.has_xy_data:
+ return ax
+
+ # Pack the kwargs for statistics.KDE
+ estimate_kws = dict(
+ bw_method=bw_method,
+ bw_adjust=bw_adjust,
+ gridsize=gridsize,
+ cut=cut,
+ clip=clip,
+ cumulative=cumulative,
+ )
+
+ if p.univariate:
+
+ plot_kws = kwargs.copy()
+
+ p.plot_univariate_density(
+ multiple=multiple,
+ common_norm=common_norm,
+ common_grid=common_grid,
+ fill=fill,
+ color=color,
+ legend=legend,
+ warn_singular=warn_singular,
+ estimate_kws=estimate_kws,
+ **plot_kws,
+ )
+
+ else:
+
+ p.plot_bivariate_density(
+ common_norm=common_norm,
+ fill=fill,
+ levels=levels,
+ thresh=thresh,
+ legend=legend,
+ color=color,
+ warn_singular=warn_singular,
+ cbar=cbar,
+ cbar_ax=cbar_ax,
+ cbar_kws=cbar_kws,
+ estimate_kws=estimate_kws,
+ **kwargs,
+ )
+
+ return ax
+
+
+kdeplot.__doc__ = """\
+Plot univariate or bivariate distributions using kernel density estimation.
+
+A kernel density estimate (KDE) plot is a method for visualizing the
+distribution of observations in a dataset, analogous to a histogram. KDE
+represents the data using a continuous probability density curve in one or
+more dimensions.
+
+The approach is explained further in the :ref:`user guide <tutorial_kde>`.
+
+Relative to a histogram, KDE can produce a plot that is less cluttered and
+more interpretable, especially when drawing multiple distributions. But it
+has the potential to introduce distortions if the underlying distribution is
+bounded or not smooth. Like a histogram, the quality of the representation
+also depends on the selection of good smoothing parameters.
+
+Parameters
+----------
+{params.core.data}
+{params.core.xy}
+{params.core.hue}
+weights : vector or key in ``data``
+ If provided, weight the kernel density estimation using these values.
+{params.core.palette}
+{params.core.hue_order}
+{params.core.hue_norm}
+{params.core.color}
+fill : bool or None
+ If True, fill in the area under univariate density curves or between
+ bivariate contours. If None, the default depends on ``multiple``.
+{params.dist.multiple}
+common_norm : bool
+ If True, scale each conditional density by the number of observations
+ such that the total area under all densities sums to 1. Otherwise,
+ normalize each density independently.
+common_grid : bool
+ If True, use the same evaluation grid for each kernel density estimate.
+ Only relevant with univariate data.
+{params.kde.cumulative}
+{params.kde.bw_method}
+{params.kde.bw_adjust}
+warn_singular : bool
+ If True, issue a warning when trying to estimate the density of data
+ with zero variance.
+{params.dist.log_scale}
+levels : int or vector
+ Number of contour levels or values to draw contours at. A vector argument
+ must have increasing values in [0, 1]. Levels correspond to iso-proportions
+ of the density: e.g., 20% of the probability mass will lie below the
+ contour drawn for 0.2. Only relevant with bivariate data.
+thresh : number in [0, 1]
+ Lowest iso-proportion level at which to draw a contour line. Ignored when
+ ``levels`` is a vector. Only relevant with bivariate data.
+gridsize : int
+ Number of points on each dimension of the evaluation grid.
+{params.kde.cut}
+{params.kde.clip}
+{params.dist.legend}
+{params.dist.cbar}
+{params.dist.cbar_ax}
+{params.dist.cbar_kws}
+{params.core.ax}
+kwargs
+ Other keyword arguments are passed to one of the following matplotlib
+ functions:
+
+ - :meth:`matplotlib.axes.Axes.plot` (univariate, ``fill=False``),
+ - :meth:`matplotlib.axes.Axes.fill_between` (univariate, ``fill=True``),
+ - :meth:`matplotlib.axes.Axes.contour` (bivariate, ``fill=False``),
+ - :meth:`matplotlib.axes.contourf` (bivariate, ``fill=True``).
+
+Returns
+-------
+{returns.ax}
+
+See Also
+--------
+{seealso.displot}
+{seealso.histplot}
+{seealso.ecdfplot}
+{seealso.jointplot}
+{seealso.violinplot}
+
+Notes
+-----
+
+The *bandwidth*, or standard deviation of the smoothing kernel, is an
+important parameter. Misspecification of the bandwidth can produce a
+distorted representation of the data. Much like the choice of bin width in a
+histogram, an over-smoothed curve can erase true features of a
+distribution, while an under-smoothed curve can create false features out of
+random variability. The rule-of-thumb that sets the default bandwidth works
+best when the true distribution is smooth, unimodal, and roughly bell-shaped.
+It is always a good idea to check the default behavior by using ``bw_adjust``
+to increase or decrease the amount of smoothing.
+
+Because the smoothing algorithm uses a Gaussian kernel, the estimated density
+curve can extend to values that do not make sense for a particular dataset.
+For example, the curve may be drawn over negative values when smoothing data
+that are naturally positive. The ``cut`` and ``clip`` parameters can be used
+to control the extent of the curve, but datasets that have many observations
+close to a natural boundary may be better served by a different visualization
+method.
+
+Similar considerations apply when a dataset is naturally discrete or "spiky"
+(containing many repeated observations of the same value). Kernel density
+estimation will always produce a smooth curve, which would be misleading
+in these situations.
+
+The units on the density axis are a common source of confusion. While kernel
+density estimation produces a probability distribution, the height of the curve
+at each point gives a density, not a probability. A probability can be obtained
+only by integrating the density across a range. The curve is normalized so
+that the integral over all possible values is 1, meaning that the scale of
+the density axis depends on the data values.
+
+Examples
+--------
+
+.. include:: ../docstrings/kdeplot.rst
+
+""".format(
+ params=_param_docs,
+ returns=_core_docs["returns"],
+ seealso=_core_docs["seealso"],
+)
+
+
+def ecdfplot(
+ data=None, *,
+ # Vector variables
+ x=None, y=None, hue=None, weights=None,
+ # Computation parameters
+ stat="proportion", complementary=False,
+ # Hue mapping parameters
+ palette=None, hue_order=None, hue_norm=None,
+ # Axes information
+ log_scale=None, legend=True, ax=None,
+ # Other appearance keywords
+ **kwargs,
+):
+
+ p = _DistributionPlotter(
+ data=data,
+ variables=dict(x=x, y=y, hue=hue, weights=weights),
+ )
+
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm)
+
+ # We could support other semantics (size, style) here fairly easily
+ # But it would make distplot a bit more complicated.
+ # It's always possible to add features like that later, so I am going to defer.
+ # It will be even easier to wait until after there is a more general/abstract
+ # way to go from semantic specs to artist attributes.
+
+ if ax is None:
+ ax = plt.gca()
+
+ p._attach(ax, log_scale=log_scale)
+
+ color = kwargs.pop("color", kwargs.pop("c", None))
+ kwargs["color"] = _default_color(ax.plot, hue, color, kwargs)
+
+ if not p.has_xy_data:
+ return ax
+
+ # We could add this one day, but it's of dubious value
+ if not p.univariate:
+ raise NotImplementedError("Bivariate ECDF plots are not implemented")
+
+ estimate_kws = dict(
+ stat=stat,
+ complementary=complementary,
+ )
+
+ p.plot_univariate_ecdf(
+ estimate_kws=estimate_kws,
+ legend=legend,
+ **kwargs,
+ )
+
+ return ax
+
+
+ecdfplot.__doc__ = """\
+Plot empirical cumulative distribution functions.
+
+An ECDF represents the proportion or count of observations falling below each
+unique value in a dataset. Compared to a histogram or density plot, it has the
+advantage that each observation is visualized directly, meaning that there are
+no binning or smoothing parameters that need to be adjusted. It also aids direct
+comparisons between multiple distributions. A downside is that the relationship
+between the appearance of the plot and the basic properties of the distribution
+(such as its central tendency, variance, and the presence of any bimodality)
+may not be as intuitive.
+
+More information is provided in the :ref:`user guide <tutorial_ecdf>`.
+
+Parameters
+----------
+{params.core.data}
+{params.core.xy}
+{params.core.hue}
+weights : vector or key in ``data``
+ If provided, weight the contribution of the corresponding data points
+ towards the cumulative distribution using these values.
+{params.ecdf.stat}
+{params.ecdf.complementary}
+{params.core.palette}
+{params.core.hue_order}
+{params.core.hue_norm}
+{params.dist.log_scale}
+{params.dist.legend}
+{params.core.ax}
+kwargs
+ Other keyword arguments are passed to :meth:`matplotlib.axes.Axes.plot`.
+
+Returns
+-------
+{returns.ax}
+
+See Also
+--------
+{seealso.displot}
+{seealso.histplot}
+{seealso.kdeplot}
+{seealso.rugplot}
+
+Examples
+--------
+
+.. include:: ../docstrings/ecdfplot.rst
+
+""".format(
+ params=_param_docs,
+ returns=_core_docs["returns"],
+ seealso=_core_docs["seealso"],
+)
+
+
+def rugplot(
+ data=None, *, x=None, y=None, hue=None, height=.025, expand_margins=True,
+ palette=None, hue_order=None, hue_norm=None, legend=True, ax=None, **kwargs
+):
+
+ # A note: I think it would make sense to add multiple= to rugplot and allow
+ # rugs for different hue variables to be shifted orthogonal to the data axis
+ # But is this stacking, or dodging?
+
+ # A note: if we want to add a style semantic to rugplot,
+ # we could make an option that draws the rug using scatterplot
+
+ # A note, it would also be nice to offer some kind of histogram/density
+ # rugplot, since alpha blending doesn't work great in the large n regime
+
+ # --- Start with backwards compatability for versions < 0.11.0 ----------------
+
+ a = kwargs.pop("a", None)
+ axis = kwargs.pop("axis", None)
+
+ if a is not None:
+ data = a
+ msg = textwrap.dedent("""\n
+ The `a` parameter has been replaced; use `x`, `y`, and/or `data` instead.
+ Please update your code; This will become an error in seaborn v0.14.0.
+ """)
+ warnings.warn(msg, UserWarning, stacklevel=2)
+
+ if axis is not None:
+ if axis == "x":
+ x = data
+ elif axis == "y":
+ y = data
+ data = None
+ msg = textwrap.dedent(f"""\n
+ The `axis` parameter has been deprecated; use the `{axis}` parameter instead.
+ Please update your code; this will become an error in seaborn v0.14.0.
+ """)
+ warnings.warn(msg, UserWarning, stacklevel=2)
+
+ vertical = kwargs.pop("vertical", None)
+ if vertical is not None:
+ if vertical:
+ action_taken = "assigning data to `y`."
+ if x is None:
+ data, y = y, data
+ else:
+ x, y = y, x
+ else:
+ action_taken = "assigning data to `x`."
+ msg = textwrap.dedent(f"""\n
+ The `vertical` parameter is deprecated; {action_taken}
+ This will become an error in seaborn v0.14.0; please update your code.
+ """)
+ warnings.warn(msg, UserWarning, stacklevel=2)
+
+ # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
+
+ p = _DistributionPlotter(
+ data=data,
+ variables=dict(x=x, y=y, hue=hue),
+ )
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm)
+
+ if ax is None:
+ ax = plt.gca()
+
+ p._attach(ax)
+
+ color = kwargs.pop("color", kwargs.pop("c", None))
+ kwargs["color"] = _default_color(ax.plot, hue, color, kwargs)
+
+ if not p.has_xy_data:
+ return ax
+
+ p.plot_rug(height, expand_margins, legend, **kwargs)
+
+ return ax
+
+
+rugplot.__doc__ = """\
+Plot marginal distributions by drawing ticks along the x and y axes.
+
+This function is intended to complement other plots by showing the location
+of individual observations in an unobtrusive way.
+
+Parameters
+----------
+{params.core.data}
+{params.core.xy}
+{params.core.hue}
+height : float
+ Proportion of axes extent covered by each rug element. Can be negative.
+expand_margins : bool
+ If True, increase the axes margins by the height of the rug to avoid
+ overlap with other elements.
+{params.core.palette}
+{params.core.hue_order}
+{params.core.hue_norm}
+legend : bool
+ If False, do not add a legend for semantic variables.
+{params.core.ax}
+kwargs
+ Other keyword arguments are passed to
+ :meth:`matplotlib.collections.LineCollection`
+
+Returns
+-------
+{returns.ax}
+
+Examples
+--------
+
+.. include:: ../docstrings/rugplot.rst
+
+""".format(
+ params=_param_docs,
+ returns=_core_docs["returns"],
+)
+
+
+def displot(
+ data=None, *,
+ # Vector variables
+ x=None, y=None, hue=None, row=None, col=None, weights=None,
+ # Other plot parameters
+ kind="hist", rug=False, rug_kws=None, log_scale=None, legend=True,
+ # Hue-mapping parameters
+ palette=None, hue_order=None, hue_norm=None, color=None,
+ # Faceting parameters
+ col_wrap=None, row_order=None, col_order=None,
+ height=5, aspect=1, facet_kws=None,
+ **kwargs,
+):
+
+ p = _DistributionPlotter(
+ data=data,
+ variables=dict(x=x, y=y, hue=hue, weights=weights, row=row, col=col),
+ )
+
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm)
+
+ _check_argument("kind", ["hist", "kde", "ecdf"], kind)
+
+ # --- Initialize the FacetGrid object
+
+ # Check for attempt to plot onto specific axes and warn
+ if "ax" in kwargs:
+ msg = (
+ "`displot` is a figure-level function and does not accept "
+ "the ax= parameter. You may wish to try {}plot.".format(kind)
+ )
+ warnings.warn(msg, UserWarning)
+ kwargs.pop("ax")
+
+ for var in ["row", "col"]:
+ # Handle faceting variables that lack name information
+ if var in p.variables and p.variables[var] is None:
+ p.variables[var] = f"_{var}_"
+
+ # Adapt the plot_data dataframe for use with FacetGrid
+ grid_data = p.plot_data.rename(columns=p.variables)
+ grid_data = grid_data.loc[:, ~grid_data.columns.duplicated()]
+
+ col_name = p.variables.get("col")
+ row_name = p.variables.get("row")
+
+ if facet_kws is None:
+ facet_kws = {}
+
+ g = FacetGrid(
+ data=grid_data, row=row_name, col=col_name,
+ col_wrap=col_wrap, row_order=row_order,
+ col_order=col_order, height=height,
+ aspect=aspect,
+ **facet_kws,
+ )
+
+ # Now attach the axes object to the plotter object
+ if kind == "kde":
+ allowed_types = ["numeric", "datetime"]
+ else:
+ allowed_types = None
+ p._attach(g, allowed_types=allowed_types, log_scale=log_scale)
+
+ # Check for a specification that lacks x/y data and return early
+ if not p.has_xy_data:
+ return g
+
+ if color is None and hue is None:
+ color = "C0"
+ # XXX else warn if hue is not None?
+
+ kwargs["legend"] = legend
+
+ # --- Draw the plots
+
+ if kind == "hist":
+
+ hist_kws = kwargs.copy()
+
+ # Extract the parameters that will go directly to Histogram
+ estimate_defaults = {}
+ _assign_default_kwargs(estimate_defaults, Histogram.__init__, histplot)
+
+ estimate_kws = {}
+ for key, default_val in estimate_defaults.items():
+ estimate_kws[key] = hist_kws.pop(key, default_val)
+
+ # Handle derivative defaults
+ if estimate_kws["discrete"] is None:
+ estimate_kws["discrete"] = p._default_discrete()
+
+ hist_kws["estimate_kws"] = estimate_kws
+
+ hist_kws.setdefault("color", color)
+
+ if p.univariate:
+
+ _assign_default_kwargs(hist_kws, p.plot_univariate_histogram, histplot)
+ p.plot_univariate_histogram(**hist_kws)
+
+ else:
+
+ _assign_default_kwargs(hist_kws, p.plot_bivariate_histogram, histplot)
+ p.plot_bivariate_histogram(**hist_kws)
+
+ elif kind == "kde":
+
+ kde_kws = kwargs.copy()
+
+ # Extract the parameters that will go directly to KDE
+ estimate_defaults = {}
+ _assign_default_kwargs(estimate_defaults, KDE.__init__, kdeplot)
+
+ estimate_kws = {}
+ for key, default_val in estimate_defaults.items():
+ estimate_kws[key] = kde_kws.pop(key, default_val)
+
+ kde_kws["estimate_kws"] = estimate_kws
+ kde_kws["color"] = color
+
+ if p.univariate:
+
+ _assign_default_kwargs(kde_kws, p.plot_univariate_density, kdeplot)
+ p.plot_univariate_density(**kde_kws)
+
+ else:
+
+ _assign_default_kwargs(kde_kws, p.plot_bivariate_density, kdeplot)
+ p.plot_bivariate_density(**kde_kws)
+
+ elif kind == "ecdf":
+
+ ecdf_kws = kwargs.copy()
+
+ # Extract the parameters that will go directly to the estimator
+ estimate_kws = {}
+ estimate_defaults = {}
+ _assign_default_kwargs(estimate_defaults, ECDF.__init__, ecdfplot)
+ for key, default_val in estimate_defaults.items():
+ estimate_kws[key] = ecdf_kws.pop(key, default_val)
+
+ ecdf_kws["estimate_kws"] = estimate_kws
+ ecdf_kws["color"] = color
+
+ if p.univariate:
+
+ _assign_default_kwargs(ecdf_kws, p.plot_univariate_ecdf, ecdfplot)
+ p.plot_univariate_ecdf(**ecdf_kws)
+
+ else:
+
+ raise NotImplementedError("Bivariate ECDF plots are not implemented")
+
+ # All plot kinds can include a rug
+ if rug:
+ # TODO with expand_margins=True, each facet expands margins... annoying!
+ if rug_kws is None:
+ rug_kws = {}
+ _assign_default_kwargs(rug_kws, p.plot_rug, rugplot)
+ rug_kws["legend"] = False
+ if color is not None:
+ rug_kws["color"] = color
+ p.plot_rug(**rug_kws)
+
+ # Call FacetGrid annotation methods
+ # Note that the legend is currently set inside the plotting method
+ g.set_axis_labels(
+ x_var=p.variables.get("x", g.axes.flat[0].get_xlabel()),
+ y_var=p.variables.get("y", g.axes.flat[0].get_ylabel()),
+ )
+ g.set_titles()
+ g.tight_layout()
+
+ if data is not None and (x is not None or y is not None):
+ if not isinstance(data, pd.DataFrame):
+ data = pd.DataFrame(data)
+ g.data = pd.merge(
+ data,
+ g.data[g.data.columns.difference(data.columns)],
+ left_index=True,
+ right_index=True,
+ )
+ else:
+ wide_cols = {
+ k: f"_{k}_" if v is None else v for k, v in p.variables.items()
+ }
+ g.data = p.plot_data.rename(columns=wide_cols)
+
+ return g
+
+
+displot.__doc__ = """\
+Figure-level interface for drawing distribution plots onto a FacetGrid.
+
+This function provides access to several approaches for visualizing the
+univariate or bivariate distribution of data, including subsets of data
+defined by semantic mapping and faceting across multiple subplots. The
+``kind`` parameter selects the approach to use:
+
+- :func:`histplot` (with ``kind="hist"``; the default)
+- :func:`kdeplot` (with ``kind="kde"``)
+- :func:`ecdfplot` (with ``kind="ecdf"``; univariate-only)
+
+Additionally, a :func:`rugplot` can be added to any kind of plot to show
+individual observations.
+
+Extra keyword arguments are passed to the underlying function, so you should
+refer to the documentation for each to understand the complete set of options
+for making plots with this interface.
+
+See the :doc:`distribution plots tutorial <../tutorial/distributions>` for a more
+in-depth discussion of the relative strengths and weaknesses of each approach.
+The distinction between figure-level and axes-level functions is explained
+further in the :doc:`user guide <../tutorial/function_overview>`.
+
+Parameters
+----------
+{params.core.data}
+{params.core.xy}
+{params.core.hue}
+{params.facets.rowcol}
+weights : vector or key in ``data``
+ Observation weights used for computing the distribution function.
+kind : {{"hist", "kde", "ecdf"}}
+ Approach for visualizing the data. Selects the underlying plotting function
+ and determines the additional set of valid parameters.
+rug : bool
+ If True, show each observation with marginal ticks (as in :func:`rugplot`).
+rug_kws : dict
+ Parameters to control the appearance of the rug plot.
+{params.dist.log_scale}
+{params.dist.legend}
+{params.core.palette}
+{params.core.hue_order}
+{params.core.hue_norm}
+{params.core.color}
+{params.facets.col_wrap}
+{params.facets.rowcol_order}
+{params.facets.height}
+{params.facets.aspect}
+{params.facets.facet_kws}
+kwargs
+ Other keyword arguments are documented with the relevant axes-level function:
+
+ - :func:`histplot` (with ``kind="hist"``)
+ - :func:`kdeplot` (with ``kind="kde"``)
+ - :func:`ecdfplot` (with ``kind="ecdf"``)
+
+Returns
+-------
+{returns.facetgrid}
+
+See Also
+--------
+{seealso.histplot}
+{seealso.kdeplot}
+{seealso.rugplot}
+{seealso.ecdfplot}
+{seealso.jointplot}
+
+Examples
+--------
+
+See the API documentation for the axes-level functions for more details
+about the breadth of options available for each plot kind.
+
+.. include:: ../docstrings/displot.rst
+
+""".format(
+ params=_param_docs,
+ returns=_core_docs["returns"],
+ seealso=_core_docs["seealso"],
+)
+
+
+# =========================================================================== #
+# DEPRECATED FUNCTIONS LIVE BELOW HERE
+# =========================================================================== #
+
+
+def _freedman_diaconis_bins(a):
+ """Calculate number of hist bins using Freedman-Diaconis rule."""
+ # From https://stats.stackexchange.com/questions/798/
+ a = np.asarray(a)
+ if len(a) < 2:
+ return 1
+ iqr = np.subtract.reduce(np.nanpercentile(a, [75, 25]))
+ h = 2 * iqr / (len(a) ** (1 / 3))
+ # fall back to sqrt(a) bins if iqr is 0
+ if h == 0:
+ return int(np.sqrt(a.size))
+ else:
+ return int(np.ceil((a.max() - a.min()) / h))
+
+
+def distplot(a=None, bins=None, hist=True, kde=True, rug=False, fit=None,
+ hist_kws=None, kde_kws=None, rug_kws=None, fit_kws=None,
+ color=None, vertical=False, norm_hist=False, axlabel=None,
+ label=None, ax=None, x=None):
+ """
+ DEPRECATED
+
+ This function has been deprecated and will be removed in seaborn v0.14.0.
+ It has been replaced by :func:`histplot` and :func:`displot`, two functions
+ with a modern API and many more capabilities.
+
+ For a guide to updating, please see this notebook:
+
+ https://gist.github.com/mwaskom/de44147ed2974457ad6372750bbe5751
+
+ """
+
+ if kde and not hist:
+ axes_level_suggestion = (
+ "`kdeplot` (an axes-level function for kernel density plots)"
+ )
+ else:
+ axes_level_suggestion = (
+ "`histplot` (an axes-level function for histograms)"
+ )
+
+ msg = textwrap.dedent(f"""
+
+ `distplot` is a deprecated function and will be removed in seaborn v0.14.0.
+
+ Please adapt your code to use either `displot` (a figure-level function with
+ similar flexibility) or {axes_level_suggestion}.
+
+ For a guide to updating your code to use the new functions, please see
+ https://gist.github.com/mwaskom/de44147ed2974457ad6372750bbe5751
+ """)
+ warnings.warn(msg, UserWarning, stacklevel=2)
+
+ if ax is None:
+ ax = plt.gca()
+
+ # Intelligently label the support axis
+ label_ax = bool(axlabel)
+ if axlabel is None and hasattr(a, "name"):
+ axlabel = a.name
+ if axlabel is not None:
+ label_ax = True
+
+ # Support new-style API
+ if x is not None:
+ a = x
+
+ # Make a a 1-d float array
+ a = np.asarray(a, float)
+ if a.ndim > 1:
+ a = a.squeeze()
+
+ # Drop null values from array
+ a = remove_na(a)
+
+ # Decide if the hist is normed
+ norm_hist = norm_hist or kde or (fit is not None)
+
+ # Handle dictionary defaults
+ hist_kws = {} if hist_kws is None else hist_kws.copy()
+ kde_kws = {} if kde_kws is None else kde_kws.copy()
+ rug_kws = {} if rug_kws is None else rug_kws.copy()
+ fit_kws = {} if fit_kws is None else fit_kws.copy()
+
+ # Get the color from the current color cycle
+ if color is None:
+ if vertical:
+ line, = ax.plot(0, a.mean())
+ else:
+ line, = ax.plot(a.mean(), 0)
+ color = line.get_color()
+ line.remove()
+
+ # Plug the label into the right kwarg dictionary
+ if label is not None:
+ if hist:
+ hist_kws["label"] = label
+ elif kde:
+ kde_kws["label"] = label
+ elif rug:
+ rug_kws["label"] = label
+ elif fit:
+ fit_kws["label"] = label
+
+ if hist:
+ if bins is None:
+ bins = min(_freedman_diaconis_bins(a), 50)
+ hist_kws.setdefault("alpha", 0.4)
+ hist_kws.setdefault("density", norm_hist)
+
+ orientation = "horizontal" if vertical else "vertical"
+ hist_color = hist_kws.pop("color", color)
+ ax.hist(a, bins, orientation=orientation,
+ color=hist_color, **hist_kws)
+ if hist_color != color:
+ hist_kws["color"] = hist_color
+
+ axis = "y" if vertical else "x"
+
+ if kde:
+ kde_color = kde_kws.pop("color", color)
+ kdeplot(**{axis: a}, ax=ax, color=kde_color, **kde_kws)
+ if kde_color != color:
+ kde_kws["color"] = kde_color
+
+ if rug:
+ rug_color = rug_kws.pop("color", color)
+ rugplot(**{axis: a}, ax=ax, color=rug_color, **rug_kws)
+ if rug_color != color:
+ rug_kws["color"] = rug_color
+
+ if fit is not None:
+
+ def pdf(x):
+ return fit.pdf(x, *params)
+
+ fit_color = fit_kws.pop("color", "#282828")
+ gridsize = fit_kws.pop("gridsize", 200)
+ cut = fit_kws.pop("cut", 3)
+ clip = fit_kws.pop("clip", (-np.inf, np.inf))
+ bw = gaussian_kde(a).scotts_factor() * a.std(ddof=1)
+ x = _kde_support(a, bw, gridsize, cut, clip)
+ params = fit.fit(a)
+ y = pdf(x)
+ if vertical:
+ x, y = y, x
+ ax.plot(x, y, color=fit_color, **fit_kws)
+ if fit_color != "#282828":
+ fit_kws["color"] = fit_color
+
+ if label_ax:
+ if vertical:
+ ax.set_ylabel(axlabel)
+ else:
+ ax.set_xlabel(axlabel)
+
+ return ax
diff --git a/seaborn/external/__init__.py b/seaborn/external/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/seaborn/external/appdirs.py b/seaborn/external/appdirs.py
new file mode 100644
index 0000000000000000000000000000000000000000..70c382964824fe0fce175f44cf6061b44cd4f922
--- /dev/null
+++ b/seaborn/external/appdirs.py
@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+# Copyright (c) 2005-2010 ActiveState Software Inc.
+# Copyright (c) 2013 Eddy Petrișor
+
+# flake8: noqa
+
+"""
+This file is directly from
+https://github.com/ActiveState/appdirs/blob/3fe6a83776843a46f20c2e5587afcffe05e03b39/appdirs.py
+
+The license of https://github.com/ActiveState/appdirs copied below:
+
+
+# This is the MIT license
+
+Copyright (c) 2010 ActiveState Software Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+
+"""Utilities for determining application-specific dirs.
+
+See <https://github.com/ActiveState/appdirs> for details and usage.
+"""
+# Dev Notes:
+# - MSDN on where to store app data files:
+# http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120
+# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html
+# - XDG spec for Un*x: https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
+
+__version__ = "1.4.4"
+__version_info__ = tuple(int(segment) for segment in __version__.split("."))
+
+
+import sys
+import os
+
+unicode = str
+
+if sys.platform.startswith('java'):
+ import platform
+ os_name = platform.java_ver()[3][0]
+ if os_name.startswith('Windows'): # "Windows XP", "Windows 7", etc.
+ system = 'win32'
+ elif os_name.startswith('Mac'): # "Mac OS X", etc.
+ system = 'darwin'
+ else: # "Linux", "SunOS", "FreeBSD", etc.
+ # Setting this to "linux2" is not ideal, but only Windows or Mac
+ # are actually checked for and the rest of the module expects
+ # *sys.platform* style strings.
+ system = 'linux2'
+else:
+ system = sys.platform
+
+
+def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True):
+ r"""Return full path to the user-specific cache dir for this application.
+
+ "appname" is the name of application.
+ If None, just the system directory is returned.
+ "appauthor" (only used on Windows) is the name of the
+ appauthor or distributing body for this application. Typically
+ it is the owning company name. This falls back to appname. You may
+ pass False to disable it.
+ "version" is an optional version path element to append to the
+ path. You might want to use this if you want multiple versions
+ of your app to be able to run independently. If used, this
+ would typically be "<major>.<minor>".
+ Only applied when appname is present.
+ "opinion" (boolean) can be False to disable the appending of
+ "Cache" to the base app data dir for Windows. See
+ discussion below.
+
+ Typical user cache directories are:
+ Mac OS X: ~/Library/Caches/<AppName>
+ Unix: ~/.cache/<AppName> (XDG default)
+ Win XP: C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Cache
+ Vista: C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Cache
+
+ On Windows the only suggestion in the MSDN docs is that local settings go in
+ the `CSIDL_LOCAL_APPDATA` directory. This is identical to the non-roaming
+ app data dir (the default returned by `user_data_dir` above). Apps typically
+ put cache data somewhere *under* the given dir here. Some examples:
+ ...\Mozilla\Firefox\Profiles\<ProfileName>\Cache
+ ...\Acme\SuperApp\Cache\1.0
+ OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value.
+ This can be disabled with the `opinion=False` option.
+ """
+ if system == "win32":
+ if appauthor is None:
+ appauthor = appname
+ path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA"))
+ if appname:
+ if appauthor is not False:
+ path = os.path.join(path, appauthor, appname)
+ else:
+ path = os.path.join(path, appname)
+ if opinion:
+ path = os.path.join(path, "Cache")
+ elif system == 'darwin':
+ path = os.path.expanduser('~/Library/Caches')
+ if appname:
+ path = os.path.join(path, appname)
+ else:
+ path = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
+ if appname:
+ path = os.path.join(path, appname)
+ if appname and version:
+ path = os.path.join(path, version)
+ return path
+
+
+#---- internal support stuff
+
+def _get_win_folder_from_registry(csidl_name):
+ """This is a fallback technique at best. I'm not sure if using the
+ registry for this guarantees us the correct answer for all CSIDL_*
+ names.
+ """
+ import winreg as _winreg
+
+ shell_folder_name = {
+ "CSIDL_APPDATA": "AppData",
+ "CSIDL_COMMON_APPDATA": "Common AppData",
+ "CSIDL_LOCAL_APPDATA": "Local AppData",
+ }[csidl_name]
+
+ key = _winreg.OpenKey(
+ _winreg.HKEY_CURRENT_USER,
+ r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
+ )
+ dir, type = _winreg.QueryValueEx(key, shell_folder_name)
+ return dir
+
+
+def _get_win_folder_with_pywin32(csidl_name):
+ from win32com.shell import shellcon, shell
+ dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0)
+ # Try to make this a unicode path because SHGetFolderPath does
+ # not return unicode strings when there is unicode data in the
+ # path.
+ try:
+ dir = unicode(dir)
+
+ # Downgrade to short path name if have highbit chars. See
+ # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
+ has_high_char = False
+ for c in dir:
+ if ord(c) > 255:
+ has_high_char = True
+ break
+ if has_high_char:
+ try:
+ import win32api
+ dir = win32api.GetShortPathName(dir)
+ except ImportError:
+ pass
+ except UnicodeError:
+ pass
+ return dir
+
+
+def _get_win_folder_with_ctypes(csidl_name):
+ import ctypes
+
+ csidl_const = {
+ "CSIDL_APPDATA": 26,
+ "CSIDL_COMMON_APPDATA": 35,
+ "CSIDL_LOCAL_APPDATA": 28,
+ }[csidl_name]
+
+ buf = ctypes.create_unicode_buffer(1024)
+ ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf)
+
+ # Downgrade to short path name if have highbit chars. See
+ # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
+ has_high_char = False
+ for c in buf:
+ if ord(c) > 255:
+ has_high_char = True
+ break
+ if has_high_char:
+ buf2 = ctypes.create_unicode_buffer(1024)
+ if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024):
+ buf = buf2
+
+ return buf.value
+
+def _get_win_folder_with_jna(csidl_name):
+ import array
+ from com.sun import jna
+ from com.sun.jna.platform import win32
+
+ buf_size = win32.WinDef.MAX_PATH * 2
+ buf = array.zeros('c', buf_size)
+ shell = win32.Shell32.INSTANCE
+ shell.SHGetFolderPath(None, getattr(win32.ShlObj, csidl_name), None, win32.ShlObj.SHGFP_TYPE_CURRENT, buf)
+ dir = jna.Native.toString(buf.tostring()).rstrip("\0")
+
+ # Downgrade to short path name if have highbit chars. See
+ # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
+ has_high_char = False
+ for c in dir:
+ if ord(c) > 255:
+ has_high_char = True
+ break
+ if has_high_char:
+ buf = array.zeros('c', buf_size)
+ kernel = win32.Kernel32.INSTANCE
+ if kernel.GetShortPathName(dir, buf, buf_size):
+ dir = jna.Native.toString(buf.tostring()).rstrip("\0")
+
+ return dir
+
+if system == "win32":
+ try:
+ import win32com.shell
+ _get_win_folder = _get_win_folder_with_pywin32
+ except ImportError:
+ try:
+ from ctypes import windll
+ _get_win_folder = _get_win_folder_with_ctypes
+ except ImportError:
+ try:
+ import com.sun.jna
+ _get_win_folder = _get_win_folder_with_jna
+ except ImportError:
+ _get_win_folder = _get_win_folder_from_registry
diff --git a/seaborn/external/docscrape.py b/seaborn/external/docscrape.py
new file mode 100644
index 0000000000000000000000000000000000000000..99dc3ff797f5faf21ec4f53bf0c6cd036e38c9c9
--- /dev/null
+++ b/seaborn/external/docscrape.py
@@ -0,0 +1,715 @@
+"""Extract reference documentation from the NumPy source tree.
+
+Copyright (C) 2008 Stefan van der Walt <stefan@mentat.za.net>, Pauli Virtanen <pav@iki.fi>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+"""
+import inspect
+import textwrap
+import re
+import pydoc
+from warnings import warn
+from collections import namedtuple
+from collections.abc import Callable, Mapping
+import copy
+import sys
+
+
+def strip_blank_lines(l):
+ "Remove leading and trailing blank lines from a list of lines"
+ while l and not l[0].strip():
+ del l[0]
+ while l and not l[-1].strip():
+ del l[-1]
+ return l
+
+
+class Reader:
+ """A line-based string reader.
+
+ """
+ def __init__(self, data):
+ """
+ Parameters
+ ----------
+ data : str
+ String with lines separated by '\n'.
+
+ """
+ if isinstance(data, list):
+ self._str = data
+ else:
+ self._str = data.split('\n') # store string as list of lines
+
+ self.reset()
+
+ def __getitem__(self, n):
+ return self._str[n]
+
+ def reset(self):
+ self._l = 0 # current line nr
+
+ def read(self):
+ if not self.eof():
+ out = self[self._l]
+ self._l += 1
+ return out
+ else:
+ return ''
+
+ def seek_next_non_empty_line(self):
+ for l in self[self._l:]:
+ if l.strip():
+ break
+ else:
+ self._l += 1
+
+ def eof(self):
+ return self._l >= len(self._str)
+
+ def read_to_condition(self, condition_func):
+ start = self._l
+ for line in self[start:]:
+ if condition_func(line):
+ return self[start:self._l]
+ self._l += 1
+ if self.eof():
+ return self[start:self._l+1]
+ return []
+
+ def read_to_next_empty_line(self):
+ self.seek_next_non_empty_line()
+
+ def is_empty(line):
+ return not line.strip()
+
+ return self.read_to_condition(is_empty)
+
+ def read_to_next_unindented_line(self):
+ def is_unindented(line):
+ return (line.strip() and (len(line.lstrip()) == len(line)))
+ return self.read_to_condition(is_unindented)
+
+ def peek(self, n=0):
+ if self._l + n < len(self._str):
+ return self[self._l + n]
+ else:
+ return ''
+
+ def is_empty(self):
+ return not ''.join(self._str).strip()
+
+
+class ParseError(Exception):
+ def __str__(self):
+ message = self.args[0]
+ if hasattr(self, 'docstring'):
+ message = f"{message} in {self.docstring!r}"
+ return message
+
+
+Parameter = namedtuple('Parameter', ['name', 'type', 'desc'])
+
+
+class NumpyDocString(Mapping):
+ """Parses a numpydoc string to an abstract representation
+
+ Instances define a mapping from section title to structured data.
+
+ """
+
+ sections = {
+ 'Signature': '',
+ 'Summary': [''],
+ 'Extended Summary': [],
+ 'Parameters': [],
+ 'Returns': [],
+ 'Yields': [],
+ 'Receives': [],
+ 'Raises': [],
+ 'Warns': [],
+ 'Other Parameters': [],
+ 'Attributes': [],
+ 'Methods': [],
+ 'See Also': [],
+ 'Notes': [],
+ 'Warnings': [],
+ 'References': '',
+ 'Examples': '',
+ 'index': {}
+ }
+
+ def __init__(self, docstring, config={}):
+ orig_docstring = docstring
+ docstring = textwrap.dedent(docstring).split('\n')
+
+ self._doc = Reader(docstring)
+ self._parsed_data = copy.deepcopy(self.sections)
+
+ try:
+ self._parse()
+ except ParseError as e:
+ e.docstring = orig_docstring
+ raise
+
+ def __getitem__(self, key):
+ return self._parsed_data[key]
+
+ def __setitem__(self, key, val):
+ if key not in self._parsed_data:
+ self._error_location(f"Unknown section {key}", error=False)
+ else:
+ self._parsed_data[key] = val
+
+ def __iter__(self):
+ return iter(self._parsed_data)
+
+ def __len__(self):
+ return len(self._parsed_data)
+
+ def _is_at_section(self):
+ self._doc.seek_next_non_empty_line()
+
+ if self._doc.eof():
+ return False
+
+ l1 = self._doc.peek().strip() # e.g. Parameters
+
+ if l1.startswith('.. index::'):
+ return True
+
+ l2 = self._doc.peek(1).strip() # ---------- or ==========
+ return l2.startswith('-'*len(l1)) or l2.startswith('='*len(l1))
+
+ def _strip(self, doc):
+ i = 0
+ j = 0
+ for i, line in enumerate(doc):
+ if line.strip():
+ break
+
+ for j, line in enumerate(doc[::-1]):
+ if line.strip():
+ break
+
+ return doc[i:len(doc)-j]
+
+ def _read_to_next_section(self):
+ section = self._doc.read_to_next_empty_line()
+
+ while not self._is_at_section() and not self._doc.eof():
+ if not self._doc.peek(-1).strip(): # previous line was empty
+ section += ['']
+
+ section += self._doc.read_to_next_empty_line()
+
+ return section
+
+ def _read_sections(self):
+ while not self._doc.eof():
+ data = self._read_to_next_section()
+ name = data[0].strip()
+
+ if name.startswith('..'): # index section
+ yield name, data[1:]
+ elif len(data) < 2:
+ yield StopIteration
+ else:
+ yield name, self._strip(data[2:])
+
+ def _parse_param_list(self, content, single_element_is_type=False):
+ r = Reader(content)
+ params = []
+ while not r.eof():
+ header = r.read().strip()
+ if ' : ' in header:
+ arg_name, arg_type = header.split(' : ')[:2]
+ else:
+ if single_element_is_type:
+ arg_name, arg_type = '', header
+ else:
+ arg_name, arg_type = header, ''
+
+ desc = r.read_to_next_unindented_line()
+ desc = dedent_lines(desc)
+ desc = strip_blank_lines(desc)
+
+ params.append(Parameter(arg_name, arg_type, desc))
+
+ return params
+
+ # See also supports the following formats.
+ #
+ # <FUNCNAME>
+ # <FUNCNAME> SPACE* COLON SPACE+ <DESC> SPACE*
+ # <FUNCNAME> ( COMMA SPACE+ <FUNCNAME>)+ (COMMA | PERIOD)? SPACE*
+ # <FUNCNAME> ( COMMA SPACE+ <FUNCNAME>)* SPACE* COLON SPACE+ <DESC> SPACE*
+
+ # <FUNCNAME> is one of
+ # <PLAIN_FUNCNAME>
+ # COLON <ROLE> COLON BACKTICK <PLAIN_FUNCNAME> BACKTICK
+ # where
+ # <PLAIN_FUNCNAME> is a legal function name, and
+ # <ROLE> is any nonempty sequence of word characters.
+ # Examples: func_f1 :meth:`func_h1` :obj:`~baz.obj_r` :class:`class_j`
+ # <DESC> is a string describing the function.
+
+ _role = r":(?P<role>\w+):"
+ _funcbacktick = r"`(?P<name>(?:~\w+\.)?[a-zA-Z0-9_\.-]+)`"
+ _funcplain = r"(?P<name2>[a-zA-Z0-9_\.-]+)"
+ _funcname = r"(" + _role + _funcbacktick + r"|" + _funcplain + r")"
+ _funcnamenext = _funcname.replace('role', 'rolenext')
+ _funcnamenext = _funcnamenext.replace('name', 'namenext')
+ _description = r"(?P<description>\s*:(\s+(?P<desc>\S+.*))?)?\s*$"
+ _func_rgx = re.compile(r"^\s*" + _funcname + r"\s*")
+ _line_rgx = re.compile(
+ r"^\s*" +
+ r"(?P<allfuncs>" + # group for all function names
+ _funcname +
+ r"(?P<morefuncs>([,]\s+" + _funcnamenext + r")*)" +
+ r")" + # end of "allfuncs"
+ r"(?P<trailing>[,\.])?" + # Some function lists have a trailing comma (or period) '\s*'
+ _description)
+
+ # Empty <DESC> elements are replaced with '..'
+ empty_description = '..'
+
+ def _parse_see_also(self, content):
+ """
+ func_name : Descriptive text
+ continued text
+ another_func_name : Descriptive text
+ func_name1, func_name2, :meth:`func_name`, func_name3
+
+ """
+
+ items = []
+
+ def parse_item_name(text):
+ """Match ':role:`name`' or 'name'."""
+ m = self._func_rgx.match(text)
+ if not m:
+ raise ParseError(f"{text} is not a item name")
+ role = m.group('role')
+ name = m.group('name') if role else m.group('name2')
+ return name, role, m.end()
+
+ rest = []
+ for line in content:
+ if not line.strip():
+ continue
+
+ line_match = self._line_rgx.match(line)
+ description = None
+ if line_match:
+ description = line_match.group('desc')
+ if line_match.group('trailing') and description:
+ self._error_location(
+ 'Unexpected comma or period after function list at index %d of '
+ 'line "%s"' % (line_match.end('trailing'), line),
+ error=False)
+ if not description and line.startswith(' '):
+ rest.append(line.strip())
+ elif line_match:
+ funcs = []
+ text = line_match.group('allfuncs')
+ while True:
+ if not text.strip():
+ break
+ name, role, match_end = parse_item_name(text)
+ funcs.append((name, role))
+ text = text[match_end:].strip()
+ if text and text[0] == ',':
+ text = text[1:].strip()
+ rest = list(filter(None, [description]))
+ items.append((funcs, rest))
+ else:
+ raise ParseError(f"{line} is not a item name")
+ return items
+
+ def _parse_index(self, section, content):
+ """
+ .. index: default
+ :refguide: something, else, and more
+
+ """
+ def strip_each_in(lst):
+ return [s.strip() for s in lst]
+
+ out = {}
+ section = section.split('::')
+ if len(section) > 1:
+ out['default'] = strip_each_in(section[1].split(','))[0]
+ for line in content:
+ line = line.split(':')
+ if len(line) > 2:
+ out[line[1]] = strip_each_in(line[2].split(','))
+ return out
+
+ def _parse_summary(self):
+ """Grab signature (if given) and summary"""
+ if self._is_at_section():
+ return
+
+ # If several signatures present, take the last one
+ while True:
+ summary = self._doc.read_to_next_empty_line()
+ summary_str = " ".join([s.strip() for s in summary]).strip()
+ compiled = re.compile(r'^([\w., ]+=)?\s*[\w\.]+\(.*\)$')
+ if compiled.match(summary_str):
+ self['Signature'] = summary_str
+ if not self._is_at_section():
+ continue
+ break
+
+ if summary is not None:
+ self['Summary'] = summary
+
+ if not self._is_at_section():
+ self['Extended Summary'] = self._read_to_next_section()
+
+ def _parse(self):
+ self._doc.reset()
+ self._parse_summary()
+
+ sections = list(self._read_sections())
+ section_names = {section for section, content in sections}
+
+ has_returns = 'Returns' in section_names
+ has_yields = 'Yields' in section_names
+ # We could do more tests, but we are not. Arbitrarily.
+ if has_returns and has_yields:
+ msg = 'Docstring contains both a Returns and Yields section.'
+ raise ValueError(msg)
+ if not has_yields and 'Receives' in section_names:
+ msg = 'Docstring contains a Receives section but not Yields.'
+ raise ValueError(msg)
+
+ for (section, content) in sections:
+ if not section.startswith('..'):
+ section = (s.capitalize() for s in section.split(' '))
+ section = ' '.join(section)
+ if self.get(section):
+ self._error_location(f"The section {section} appears twice")
+
+ if section in ('Parameters', 'Other Parameters', 'Attributes',
+ 'Methods'):
+ self[section] = self._parse_param_list(content)
+ elif section in ('Returns', 'Yields', 'Raises', 'Warns', 'Receives'):
+ self[section] = self._parse_param_list(
+ content, single_element_is_type=True)
+ elif section.startswith('.. index::'):
+ self['index'] = self._parse_index(section, content)
+ elif section == 'See Also':
+ self['See Also'] = self._parse_see_also(content)
+ else:
+ self[section] = content
+
+ def _error_location(self, msg, error=True):
+ if hasattr(self, '_obj'):
+ # we know where the docs came from:
+ try:
+ filename = inspect.getsourcefile(self._obj)
+ except TypeError:
+ filename = None
+ msg = msg + f" in the docstring of {self._obj} in {filename}."
+ if error:
+ raise ValueError(msg)
+ else:
+ warn(msg)
+
+ # string conversion routines
+
+ def _str_header(self, name, symbol='-'):
+ return [name, len(name)*symbol]
+
+ def _str_indent(self, doc, indent=4):
+ out = []
+ for line in doc:
+ out += [' '*indent + line]
+ return out
+
+ def _str_signature(self):
+ if self['Signature']:
+ return [self['Signature'].replace('*', r'\*')] + ['']
+ else:
+ return ['']
+
+ def _str_summary(self):
+ if self['Summary']:
+ return self['Summary'] + ['']
+ else:
+ return []
+
+ def _str_extended_summary(self):
+ if self['Extended Summary']:
+ return self['Extended Summary'] + ['']
+ else:
+ return []
+
+ def _str_param_list(self, name):
+ out = []
+ if self[name]:
+ out += self._str_header(name)
+ for param in self[name]:
+ parts = []
+ if param.name:
+ parts.append(param.name)
+ if param.type:
+ parts.append(param.type)
+ out += [' : '.join(parts)]
+ if param.desc and ''.join(param.desc).strip():
+ out += self._str_indent(param.desc)
+ out += ['']
+ return out
+
+ def _str_section(self, name):
+ out = []
+ if self[name]:
+ out += self._str_header(name)
+ out += self[name]
+ out += ['']
+ return out
+
+ def _str_see_also(self, func_role):
+ if not self['See Also']:
+ return []
+ out = []
+ out += self._str_header("See Also")
+ out += ['']
+ last_had_desc = True
+ for funcs, desc in self['See Also']:
+ assert isinstance(funcs, list)
+ links = []
+ for func, role in funcs:
+ if role:
+ link = f':{role}:`{func}`'
+ elif func_role:
+ link = f':{func_role}:`{func}`'
+ else:
+ link = f"`{func}`_"
+ links.append(link)
+ link = ', '.join(links)
+ out += [link]
+ if desc:
+ out += self._str_indent([' '.join(desc)])
+ last_had_desc = True
+ else:
+ last_had_desc = False
+ out += self._str_indent([self.empty_description])
+
+ if last_had_desc:
+ out += ['']
+ out += ['']
+ return out
+
+ def _str_index(self):
+ idx = self['index']
+ out = []
+ output_index = False
+ default_index = idx.get('default', '')
+ if default_index:
+ output_index = True
+ out += [f'.. index:: {default_index}']
+ for section, references in idx.items():
+ if section == 'default':
+ continue
+ output_index = True
+ out += [f" :{section}: {', '.join(references)}"]
+ if output_index:
+ return out
+ else:
+ return ''
+
+ def __str__(self, func_role=''):
+ out = []
+ out += self._str_signature()
+ out += self._str_summary()
+ out += self._str_extended_summary()
+ for param_list in ('Parameters', 'Returns', 'Yields', 'Receives',
+ 'Other Parameters', 'Raises', 'Warns'):
+ out += self._str_param_list(param_list)
+ out += self._str_section('Warnings')
+ out += self._str_see_also(func_role)
+ for s in ('Notes', 'References', 'Examples'):
+ out += self._str_section(s)
+ for param_list in ('Attributes', 'Methods'):
+ out += self._str_param_list(param_list)
+ out += self._str_index()
+ return '\n'.join(out)
+
+
+def indent(str, indent=4):
+ indent_str = ' '*indent
+ if str is None:
+ return indent_str
+ lines = str.split('\n')
+ return '\n'.join(indent_str + l for l in lines)
+
+
+def dedent_lines(lines):
+ """Deindent a list of lines maximally"""
+ return textwrap.dedent("\n".join(lines)).split("\n")
+
+
+def header(text, style='-'):
+ return text + '\n' + style*len(text) + '\n'
+
+
+class FunctionDoc(NumpyDocString):
+ def __init__(self, func, role='func', doc=None, config={}):
+ self._f = func
+ self._role = role # e.g. "func" or "meth"
+
+ if doc is None:
+ if func is None:
+ raise ValueError("No function or docstring given")
+ doc = inspect.getdoc(func) or ''
+ NumpyDocString.__init__(self, doc, config)
+
+ if not self['Signature'] and func is not None:
+ func, func_name = self.get_func()
+ try:
+ try:
+ signature = str(inspect.signature(func))
+ except (AttributeError, ValueError):
+ # try to read signature, backward compat for older Python
+ if sys.version_info[0] >= 3:
+ argspec = inspect.getfullargspec(func)
+ else:
+ argspec = inspect.getargspec(func)
+ signature = inspect.formatargspec(*argspec)
+ signature = f'{func_name}{signature}'
+ except TypeError:
+ signature = f'{func_name}()'
+ self['Signature'] = signature
+
+ def get_func(self):
+ func_name = getattr(self._f, '__name__', self.__class__.__name__)
+ if inspect.isclass(self._f):
+ func = getattr(self._f, '__call__', self._f.__init__)
+ else:
+ func = self._f
+ return func, func_name
+
+ def __str__(self):
+ out = ''
+
+ func, func_name = self.get_func()
+
+ roles = {'func': 'function',
+ 'meth': 'method'}
+
+ if self._role:
+ if self._role not in roles:
+ print(f"Warning: invalid role {self._role}")
+ out += f".. {roles.get(self._role, '')}:: {func_name}\n \n\n"
+
+ out += super().__str__(func_role=self._role)
+ return out
+
+
+class ClassDoc(NumpyDocString):
+
+ extra_public_methods = ['__call__']
+
+ def __init__(self, cls, doc=None, modulename='', func_doc=FunctionDoc,
+ config={}):
+ if not inspect.isclass(cls) and cls is not None:
+ raise ValueError(f"Expected a class or None, but got {cls!r}")
+ self._cls = cls
+
+ if 'sphinx' in sys.modules:
+ from sphinx.ext.autodoc import ALL
+ else:
+ ALL = object()
+
+ self.show_inherited_members = config.get(
+ 'show_inherited_class_members', True)
+
+ if modulename and not modulename.endswith('.'):
+ modulename += '.'
+ self._mod = modulename
+
+ if doc is None:
+ if cls is None:
+ raise ValueError("No class or documentation string given")
+ doc = pydoc.getdoc(cls)
+
+ NumpyDocString.__init__(self, doc)
+
+ _members = config.get('members', [])
+ if _members is ALL:
+ _members = None
+ _exclude = config.get('exclude-members', [])
+
+ if config.get('show_class_members', True) and _exclude is not ALL:
+ def splitlines_x(s):
+ if not s:
+ return []
+ else:
+ return s.splitlines()
+ for field, items in [('Methods', self.methods),
+ ('Attributes', self.properties)]:
+ if not self[field]:
+ doc_list = []
+ for name in sorted(items):
+ if (name in _exclude or
+ (_members and name not in _members)):
+ continue
+ try:
+ doc_item = pydoc.getdoc(getattr(self._cls, name))
+ doc_list.append(
+ Parameter(name, '', splitlines_x(doc_item)))
+ except AttributeError:
+ pass # method doesn't exist
+ self[field] = doc_list
+
+ @property
+ def methods(self):
+ if self._cls is None:
+ return []
+ return [name for name, func in inspect.getmembers(self._cls)
+ if ((not name.startswith('_')
+ or name in self.extra_public_methods)
+ and isinstance(func, Callable)
+ and self._is_show_member(name))]
+
+ @property
+ def properties(self):
+ if self._cls is None:
+ return []
+ return [name for name, func in inspect.getmembers(self._cls)
+ if (not name.startswith('_') and
+ (func is None or isinstance(func, property) or
+ inspect.isdatadescriptor(func))
+ and self._is_show_member(name))]
+
+ def _is_show_member(self, name):
+ if self.show_inherited_members:
+ return True # show all class members
+ if name not in self._cls.__dict__:
+ return False # class member is inherited, we do not show it
+ return True
diff --git a/seaborn/external/husl.py b/seaborn/external/husl.py
new file mode 100644
index 0000000000000000000000000000000000000000..63e98cbb71640f24a0d5e0eda697bc97d12ffc5b
--- /dev/null
+++ b/seaborn/external/husl.py
@@ -0,0 +1,313 @@
+import operator
+import math
+
+__version__ = "2.1.0"
+
+
+m = [
+ [3.2406, -1.5372, -0.4986],
+ [-0.9689, 1.8758, 0.0415],
+ [0.0557, -0.2040, 1.0570]
+]
+
+m_inv = [
+ [0.4124, 0.3576, 0.1805],
+ [0.2126, 0.7152, 0.0722],
+ [0.0193, 0.1192, 0.9505]
+]
+
+# Hard-coded D65 illuminant
+refX = 0.95047
+refY = 1.00000
+refZ = 1.08883
+refU = 0.19784
+refV = 0.46834
+lab_e = 0.008856
+lab_k = 903.3
+
+
+# Public API
+
+def husl_to_rgb(h, s, l):
+ return lch_to_rgb(*husl_to_lch([h, s, l]))
+
+
+def husl_to_hex(h, s, l):
+ return rgb_to_hex(husl_to_rgb(h, s, l))
+
+
+def rgb_to_husl(r, g, b):
+ return lch_to_husl(rgb_to_lch(r, g, b))
+
+
+def hex_to_husl(hex):
+ return rgb_to_husl(*hex_to_rgb(hex))
+
+
+def huslp_to_rgb(h, s, l):
+ return lch_to_rgb(*huslp_to_lch([h, s, l]))
+
+
+def huslp_to_hex(h, s, l):
+ return rgb_to_hex(huslp_to_rgb(h, s, l))
+
+
+def rgb_to_huslp(r, g, b):
+ return lch_to_huslp(rgb_to_lch(r, g, b))
+
+
+def hex_to_huslp(hex):
+ return rgb_to_huslp(*hex_to_rgb(hex))
+
+
+def lch_to_rgb(l, c, h):
+ return xyz_to_rgb(luv_to_xyz(lch_to_luv([l, c, h])))
+
+
+def rgb_to_lch(r, g, b):
+ return luv_to_lch(xyz_to_luv(rgb_to_xyz([r, g, b])))
+
+
+def max_chroma(L, H):
+ hrad = math.radians(H)
+ sinH = (math.sin(hrad))
+ cosH = (math.cos(hrad))
+ sub1 = (math.pow(L + 16, 3.0) / 1560896.0)
+ sub2 = sub1 if sub1 > 0.008856 else (L / 903.3)
+ result = float("inf")
+ for row in m:
+ m1 = row[0]
+ m2 = row[1]
+ m3 = row[2]
+ top = ((0.99915 * m1 + 1.05122 * m2 + 1.14460 * m3) * sub2)
+ rbottom = (0.86330 * m3 - 0.17266 * m2)
+ lbottom = (0.12949 * m3 - 0.38848 * m1)
+ bottom = (rbottom * sinH + lbottom * cosH) * sub2
+
+ for t in (0.0, 1.0):
+ C = (L * (top - 1.05122 * t) / (bottom + 0.17266 * sinH * t))
+ if C > 0.0 and C < result:
+ result = C
+ return result
+
+
+def _hrad_extremum(L):
+ lhs = (math.pow(L, 3.0) + 48.0 * math.pow(L, 2.0) + 768.0 * L + 4096.0) / 1560896.0
+ rhs = 1107.0 / 125000.0
+ sub = lhs if lhs > rhs else 10.0 * L / 9033.0
+ chroma = float("inf")
+ result = None
+ for row in m:
+ for limit in (0.0, 1.0):
+ [m1, m2, m3] = row
+ top = -3015466475.0 * m3 * sub + 603093295.0 * m2 * sub - 603093295.0 * limit
+ bottom = 1356959916.0 * m1 * sub - 452319972.0 * m3 * sub
+ hrad = math.atan2(top, bottom)
+ # This is a math hack to deal with tan quadrants, I'm too lazy to figure
+ # out how to do this properly
+ if limit == 0.0:
+ hrad += math.pi
+ test = max_chroma(L, math.degrees(hrad))
+ if test < chroma:
+ chroma = test
+ result = hrad
+ return result
+
+
+def max_chroma_pastel(L):
+ H = math.degrees(_hrad_extremum(L))
+ return max_chroma(L, H)
+
+
+def dot_product(a, b):
+ return sum(map(operator.mul, a, b))
+
+
+def f(t):
+ if t > lab_e:
+ return (math.pow(t, 1.0 / 3.0))
+ else:
+ return (7.787 * t + 16.0 / 116.0)
+
+
+def f_inv(t):
+ if math.pow(t, 3.0) > lab_e:
+ return (math.pow(t, 3.0))
+ else:
+ return (116.0 * t - 16.0) / lab_k
+
+
+def from_linear(c):
+ if c <= 0.0031308:
+ return 12.92 * c
+ else:
+ return (1.055 * math.pow(c, 1.0 / 2.4) - 0.055)
+
+
+def to_linear(c):
+ a = 0.055
+
+ if c > 0.04045:
+ return (math.pow((c + a) / (1.0 + a), 2.4))
+ else:
+ return (c / 12.92)
+
+
+def rgb_prepare(triple):
+ ret = []
+ for ch in triple:
+ ch = round(ch, 3)
+
+ if ch < -0.0001 or ch > 1.0001:
+ raise Exception(f"Illegal RGB value {ch:f}")
+
+ if ch < 0:
+ ch = 0
+ if ch > 1:
+ ch = 1
+
+ # Fix for Python 3 which by default rounds 4.5 down to 4.0
+ # instead of Python 2 which is rounded to 5.0 which caused
+ # a couple off by one errors in the tests. Tests now all pass
+ # in Python 2 and Python 3
+ ret.append(int(round(ch * 255 + 0.001, 0)))
+
+ return ret
+
+
+def hex_to_rgb(hex):
+ if hex.startswith('#'):
+ hex = hex[1:]
+ r = int(hex[0:2], 16) / 255.0
+ g = int(hex[2:4], 16) / 255.0
+ b = int(hex[4:6], 16) / 255.0
+ return [r, g, b]
+
+
+def rgb_to_hex(triple):
+ [r, g, b] = triple
+ return '#%02x%02x%02x' % tuple(rgb_prepare([r, g, b]))
+
+
+def xyz_to_rgb(triple):
+ xyz = map(lambda row: dot_product(row, triple), m)
+ return list(map(from_linear, xyz))
+
+
+def rgb_to_xyz(triple):
+ rgbl = list(map(to_linear, triple))
+ return list(map(lambda row: dot_product(row, rgbl), m_inv))
+
+
+def xyz_to_luv(triple):
+ X, Y, Z = triple
+
+ if X == Y == Z == 0.0:
+ return [0.0, 0.0, 0.0]
+
+ varU = (4.0 * X) / (X + (15.0 * Y) + (3.0 * Z))
+ varV = (9.0 * Y) / (X + (15.0 * Y) + (3.0 * Z))
+ L = 116.0 * f(Y / refY) - 16.0
+
+ # Black will create a divide-by-zero error
+ if L == 0.0:
+ return [0.0, 0.0, 0.0]
+
+ U = 13.0 * L * (varU - refU)
+ V = 13.0 * L * (varV - refV)
+
+ return [L, U, V]
+
+
+def luv_to_xyz(triple):
+ L, U, V = triple
+
+ if L == 0:
+ return [0.0, 0.0, 0.0]
+
+ varY = f_inv((L + 16.0) / 116.0)
+ varU = U / (13.0 * L) + refU
+ varV = V / (13.0 * L) + refV
+ Y = varY * refY
+ X = 0.0 - (9.0 * Y * varU) / ((varU - 4.0) * varV - varU * varV)
+ Z = (9.0 * Y - (15.0 * varV * Y) - (varV * X)) / (3.0 * varV)
+
+ return [X, Y, Z]
+
+
+def luv_to_lch(triple):
+ L, U, V = triple
+
+ C = (math.pow(math.pow(U, 2) + math.pow(V, 2), (1.0 / 2.0)))
+ hrad = (math.atan2(V, U))
+ H = math.degrees(hrad)
+ if H < 0.0:
+ H = 360.0 + H
+
+ return [L, C, H]
+
+
+def lch_to_luv(triple):
+ L, C, H = triple
+
+ Hrad = math.radians(H)
+ U = (math.cos(Hrad) * C)
+ V = (math.sin(Hrad) * C)
+
+ return [L, U, V]
+
+
+def husl_to_lch(triple):
+ H, S, L = triple
+
+ if L > 99.9999999:
+ return [100, 0.0, H]
+ if L < 0.00000001:
+ return [0.0, 0.0, H]
+
+ mx = max_chroma(L, H)
+ C = mx / 100.0 * S
+
+ return [L, C, H]
+
+
+def lch_to_husl(triple):
+ L, C, H = triple
+
+ if L > 99.9999999:
+ return [H, 0.0, 100.0]
+ if L < 0.00000001:
+ return [H, 0.0, 0.0]
+
+ mx = max_chroma(L, H)
+ S = C / mx * 100.0
+
+ return [H, S, L]
+
+
+def huslp_to_lch(triple):
+ H, S, L = triple
+
+ if L > 99.9999999:
+ return [100, 0.0, H]
+ if L < 0.00000001:
+ return [0.0, 0.0, H]
+
+ mx = max_chroma_pastel(L)
+ C = mx / 100.0 * S
+
+ return [L, C, H]
+
+
+def lch_to_huslp(triple):
+ L, C, H = triple
+
+ if L > 99.9999999:
+ return [H, 0.0, 100.0]
+ if L < 0.00000001:
+ return [H, 0.0, 0.0]
+
+ mx = max_chroma_pastel(L)
+ S = C / mx * 100.0
+
+ return [H, S, L]
diff --git a/seaborn/external/kde.py b/seaborn/external/kde.py
new file mode 100644
index 0000000000000000000000000000000000000000..6add4e19127895817b42f8602b5deb43ba3b725d
--- /dev/null
+++ b/seaborn/external/kde.py
@@ -0,0 +1,380 @@
+"""
+This module was copied from the scipy project.
+
+In the process of copying, some methods were removed because they depended on
+other parts of scipy (especially on compiled components), allowing seaborn to
+have a simple and pure Python implementation. These include:
+
+- integrate_gaussian
+- integrate_box
+- integrate_box_1d
+- integrate_kde
+- logpdf
+- resample
+
+Additionally, the numpy.linalg module was substituted for scipy.linalg,
+and the examples section (with doctests) was removed from the docstring
+
+The original scipy license is copied below:
+
+Copyright (c) 2001-2002 Enthought, Inc. 2003-2019, SciPy Developers.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+ contributors may be used to endorse or promote products derived
+ from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+
+# -------------------------------------------------------------------------------
+#
+# Define classes for (uni/multi)-variate kernel density estimation.
+#
+# Currently, only Gaussian kernels are implemented.
+#
+# Written by: Robert Kern
+#
+# Date: 2004-08-09
+#
+# Modified: 2005-02-10 by Robert Kern.
+# Contributed to SciPy
+# 2005-10-07 by Robert Kern.
+# Some fixes to match the new scipy_core
+#
+# Copyright 2004-2005 by Enthought, Inc.
+#
+# -------------------------------------------------------------------------------
+
+import numpy as np
+from numpy import (asarray, atleast_2d, reshape, zeros, newaxis, dot, exp, pi,
+ sqrt, power, atleast_1d, sum, ones, cov)
+from numpy import linalg
+
+
+__all__ = ['gaussian_kde']
+
+
+class gaussian_kde:
+ """Representation of a kernel-density estimate using Gaussian kernels.
+
+ Kernel density estimation is a way to estimate the probability density
+ function (PDF) of a random variable in a non-parametric way.
+ `gaussian_kde` works for both uni-variate and multi-variate data. It
+ includes automatic bandwidth determination. The estimation works best for
+ a unimodal distribution; bimodal or multi-modal distributions tend to be
+ oversmoothed.
+
+ Parameters
+ ----------
+ dataset : array_like
+ Datapoints to estimate from. In case of univariate data this is a 1-D
+ array, otherwise a 2-D array with shape (# of dims, # of data).
+ bw_method : str, scalar or callable, optional
+ The method used to calculate the estimator bandwidth. This can be
+ 'scott', 'silverman', a scalar constant or a callable. If a scalar,
+ this will be used directly as `kde.factor`. If a callable, it should
+ take a `gaussian_kde` instance as only parameter and return a scalar.
+ If None (default), 'scott' is used. See Notes for more details.
+ weights : array_like, optional
+ weights of datapoints. This must be the same shape as dataset.
+ If None (default), the samples are assumed to be equally weighted
+
+ Attributes
+ ----------
+ dataset : ndarray
+ The dataset with which `gaussian_kde` was initialized.
+ d : int
+ Number of dimensions.
+ n : int
+ Number of datapoints.
+ neff : int
+ Effective number of datapoints.
+
+ .. versionadded:: 1.2.0
+ factor : float
+ The bandwidth factor, obtained from `kde.covariance_factor`, with which
+ the covariance matrix is multiplied.
+ covariance : ndarray
+ The covariance matrix of `dataset`, scaled by the calculated bandwidth
+ (`kde.factor`).
+ inv_cov : ndarray
+ The inverse of `covariance`.
+
+ Methods
+ -------
+ evaluate
+ __call__
+ integrate_gaussian
+ integrate_box_1d
+ integrate_box
+ integrate_kde
+ pdf
+ logpdf
+ resample
+ set_bandwidth
+ covariance_factor
+
+ Notes
+ -----
+ Bandwidth selection strongly influences the estimate obtained from the KDE
+ (much more so than the actual shape of the kernel). Bandwidth selection
+ can be done by a "rule of thumb", by cross-validation, by "plug-in
+ methods" or by other means; see [3]_, [4]_ for reviews. `gaussian_kde`
+ uses a rule of thumb, the default is Scott's Rule.
+
+ Scott's Rule [1]_, implemented as `scotts_factor`, is::
+
+ n**(-1./(d+4)),
+
+ with ``n`` the number of data points and ``d`` the number of dimensions.
+ In the case of unequally weighted points, `scotts_factor` becomes::
+
+ neff**(-1./(d+4)),
+
+ with ``neff`` the effective number of datapoints.
+ Silverman's Rule [2]_, implemented as `silverman_factor`, is::
+
+ (n * (d + 2) / 4.)**(-1. / (d + 4)).
+
+ or in the case of unequally weighted points::
+
+ (neff * (d + 2) / 4.)**(-1. / (d + 4)).
+
+ Good general descriptions of kernel density estimation can be found in [1]_
+ and [2]_, the mathematics for this multi-dimensional implementation can be
+ found in [1]_.
+
+ With a set of weighted samples, the effective number of datapoints ``neff``
+ is defined by::
+
+ neff = sum(weights)^2 / sum(weights^2)
+
+ as detailed in [5]_.
+
+ References
+ ----------
+ .. [1] D.W. Scott, "Multivariate Density Estimation: Theory, Practice, and
+ Visualization", John Wiley & Sons, New York, Chicester, 1992.
+ .. [2] B.W. Silverman, "Density Estimation for Statistics and Data
+ Analysis", Vol. 26, Monographs on Statistics and Applied Probability,
+ Chapman and Hall, London, 1986.
+ .. [3] B.A. Turlach, "Bandwidth Selection in Kernel Density Estimation: A
+ Review", CORE and Institut de Statistique, Vol. 19, pp. 1-33, 1993.
+ .. [4] D.M. Bashtannyk and R.J. Hyndman, "Bandwidth selection for kernel
+ conditional density estimation", Computational Statistics & Data
+ Analysis, Vol. 36, pp. 279-298, 2001.
+ .. [5] Gray P. G., 1969, Journal of the Royal Statistical Society.
+ Series A (General), 132, 272
+
+ """
+ def __init__(self, dataset, bw_method=None, weights=None):
+ self.dataset = atleast_2d(asarray(dataset))
+ if not self.dataset.size > 1:
+ raise ValueError("`dataset` input should have multiple elements.")
+
+ self.d, self.n = self.dataset.shape
+
+ if weights is not None:
+ self._weights = atleast_1d(weights).astype(float)
+ self._weights /= sum(self._weights)
+ if self.weights.ndim != 1:
+ raise ValueError("`weights` input should be one-dimensional.")
+ if len(self._weights) != self.n:
+ raise ValueError("`weights` input should be of length n")
+ self._neff = 1/sum(self._weights**2)
+
+ self.set_bandwidth(bw_method=bw_method)
+
+ def evaluate(self, points):
+ """Evaluate the estimated pdf on a set of points.
+
+ Parameters
+ ----------
+ points : (# of dimensions, # of points)-array
+ Alternatively, a (# of dimensions,) vector can be passed in and
+ treated as a single point.
+
+ Returns
+ -------
+ values : (# of points,)-array
+ The values at each point.
+
+ Raises
+ ------
+ ValueError : if the dimensionality of the input points is different than
+ the dimensionality of the KDE.
+
+ """
+ points = atleast_2d(asarray(points))
+
+ d, m = points.shape
+ if d != self.d:
+ if d == 1 and m == self.d:
+ # points was passed in as a row vector
+ points = reshape(points, (self.d, 1))
+ m = 1
+ else:
+ msg = f"points have dimension {d}, dataset has dimension {self.d}"
+ raise ValueError(msg)
+
+ output_dtype = np.common_type(self.covariance, points)
+ result = zeros((m,), dtype=output_dtype)
+
+ whitening = linalg.cholesky(self.inv_cov)
+ scaled_dataset = dot(whitening, self.dataset)
+ scaled_points = dot(whitening, points)
+
+ if m >= self.n:
+ # there are more points than data, so loop over data
+ for i in range(self.n):
+ diff = scaled_dataset[:, i, newaxis] - scaled_points
+ energy = sum(diff * diff, axis=0) / 2.0
+ result += self.weights[i]*exp(-energy)
+ else:
+ # loop over points
+ for i in range(m):
+ diff = scaled_dataset - scaled_points[:, i, newaxis]
+ energy = sum(diff * diff, axis=0) / 2.0
+ result[i] = sum(exp(-energy)*self.weights, axis=0)
+
+ result = result / self._norm_factor
+
+ return result
+
+ __call__ = evaluate
+
+ def scotts_factor(self):
+ """Compute Scott's factor.
+
+ Returns
+ -------
+ s : float
+ Scott's factor.
+ """
+ return power(self.neff, -1./(self.d+4))
+
+ def silverman_factor(self):
+ """Compute the Silverman factor.
+
+ Returns
+ -------
+ s : float
+ The silverman factor.
+ """
+ return power(self.neff*(self.d+2.0)/4.0, -1./(self.d+4))
+
+ # Default method to calculate bandwidth, can be overwritten by subclass
+ covariance_factor = scotts_factor
+ covariance_factor.__doc__ = """Computes the coefficient (`kde.factor`) that
+ multiplies the data covariance matrix to obtain the kernel covariance
+ matrix. The default is `scotts_factor`. A subclass can overwrite this
+ method to provide a different method, or set it through a call to
+ `kde.set_bandwidth`."""
+
+ def set_bandwidth(self, bw_method=None):
+ """Compute the estimator bandwidth with given method.
+
+ The new bandwidth calculated after a call to `set_bandwidth` is used
+ for subsequent evaluations of the estimated density.
+
+ Parameters
+ ----------
+ bw_method : str, scalar or callable, optional
+ The method used to calculate the estimator bandwidth. This can be
+ 'scott', 'silverman', a scalar constant or a callable. If a
+ scalar, this will be used directly as `kde.factor`. If a callable,
+ it should take a `gaussian_kde` instance as only parameter and
+ return a scalar. If None (default), nothing happens; the current
+ `kde.covariance_factor` method is kept.
+
+ Notes
+ -----
+ .. versionadded:: 0.11
+
+ """
+ if bw_method is None:
+ pass
+ elif bw_method == 'scott':
+ self.covariance_factor = self.scotts_factor
+ elif bw_method == 'silverman':
+ self.covariance_factor = self.silverman_factor
+ elif np.isscalar(bw_method) and not isinstance(bw_method, str):
+ self._bw_method = 'use constant'
+ self.covariance_factor = lambda: bw_method
+ elif callable(bw_method):
+ self._bw_method = bw_method
+ self.covariance_factor = lambda: self._bw_method(self)
+ else:
+ msg = "`bw_method` should be 'scott', 'silverman', a scalar " \
+ "or a callable."
+ raise ValueError(msg)
+
+ self._compute_covariance()
+
+ def _compute_covariance(self):
+ """Computes the covariance matrix for each Gaussian kernel using
+ covariance_factor().
+ """
+ self.factor = self.covariance_factor()
+ # Cache covariance and inverse covariance of the data
+ if not hasattr(self, '_data_inv_cov'):
+ self._data_covariance = atleast_2d(cov(self.dataset, rowvar=1,
+ bias=False,
+ aweights=self.weights))
+ self._data_inv_cov = linalg.inv(self._data_covariance)
+
+ self.covariance = self._data_covariance * self.factor**2
+ self.inv_cov = self._data_inv_cov / self.factor**2
+ self._norm_factor = sqrt(linalg.det(2*pi*self.covariance))
+
+ def pdf(self, x):
+ """
+ Evaluate the estimated pdf on a provided set of points.
+
+ Notes
+ -----
+ This is an alias for `gaussian_kde.evaluate`. See the ``evaluate``
+ docstring for more details.
+
+ """
+ return self.evaluate(x)
+
+ @property
+ def weights(self):
+ try:
+ return self._weights
+ except AttributeError:
+ self._weights = ones(self.n)/self.n
+ return self._weights
+
+ @property
+ def neff(self):
+ try:
+ return self._neff
+ except AttributeError:
+ self._neff = 1/sum(self.weights**2)
+ return self._neff
diff --git a/seaborn/external/version.py b/seaborn/external/version.py
new file mode 100644
index 0000000000000000000000000000000000000000..7eb57d32ce3e811d4460b1b9a93513a986347e25
--- /dev/null
+++ b/seaborn/external/version.py
@@ -0,0 +1,461 @@
+"""Extract reference documentation from the pypa/packaging source tree.
+
+In the process of copying, some unused methods / classes were removed.
+These include:
+
+- parse()
+- anything involving LegacyVersion
+
+This software is made available under the terms of *either* of the licenses
+found in LICENSE.APACHE or LICENSE.BSD. Contributions to this software is made
+under the terms of *both* these licenses.
+
+Vendored from:
+- https://github.com/pypa/packaging/
+- commit ba07d8287b4554754ac7178d177033ea3f75d489 (09/09/2021)
+"""
+
+
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+
+import collections
+import itertools
+import re
+from typing import Callable, Optional, SupportsInt, Tuple, Union
+
+__all__ = ["Version", "InvalidVersion", "VERSION_PATTERN"]
+
+
+# Vendored from https://github.com/pypa/packaging/blob/main/packaging/_structures.py
+
+class InfinityType:
+ def __repr__(self) -> str:
+ return "Infinity"
+
+ def __hash__(self) -> int:
+ return hash(repr(self))
+
+ def __lt__(self, other: object) -> bool:
+ return False
+
+ def __le__(self, other: object) -> bool:
+ return False
+
+ def __eq__(self, other: object) -> bool:
+ return isinstance(other, self.__class__)
+
+ def __ne__(self, other: object) -> bool:
+ return not isinstance(other, self.__class__)
+
+ def __gt__(self, other: object) -> bool:
+ return True
+
+ def __ge__(self, other: object) -> bool:
+ return True
+
+ def __neg__(self: object) -> "NegativeInfinityType":
+ return NegativeInfinity
+
+
+Infinity = InfinityType()
+
+
+class NegativeInfinityType:
+ def __repr__(self) -> str:
+ return "-Infinity"
+
+ def __hash__(self) -> int:
+ return hash(repr(self))
+
+ def __lt__(self, other: object) -> bool:
+ return True
+
+ def __le__(self, other: object) -> bool:
+ return True
+
+ def __eq__(self, other: object) -> bool:
+ return isinstance(other, self.__class__)
+
+ def __ne__(self, other: object) -> bool:
+ return not isinstance(other, self.__class__)
+
+ def __gt__(self, other: object) -> bool:
+ return False
+
+ def __ge__(self, other: object) -> bool:
+ return False
+
+ def __neg__(self: object) -> InfinityType:
+ return Infinity
+
+
+NegativeInfinity = NegativeInfinityType()
+
+
+# Vendored from https://github.com/pypa/packaging/blob/main/packaging/version.py
+
+InfiniteTypes = Union[InfinityType, NegativeInfinityType]
+PrePostDevType = Union[InfiniteTypes, Tuple[str, int]]
+SubLocalType = Union[InfiniteTypes, int, str]
+LocalType = Union[
+ NegativeInfinityType,
+ Tuple[
+ Union[
+ SubLocalType,
+ Tuple[SubLocalType, str],
+ Tuple[NegativeInfinityType, SubLocalType],
+ ],
+ ...,
+ ],
+]
+CmpKey = Tuple[
+ int, Tuple[int, ...], PrePostDevType, PrePostDevType, PrePostDevType, LocalType
+]
+LegacyCmpKey = Tuple[int, Tuple[str, ...]]
+VersionComparisonMethod = Callable[
+ [Union[CmpKey, LegacyCmpKey], Union[CmpKey, LegacyCmpKey]], bool
+]
+
+_Version = collections.namedtuple(
+ "_Version", ["epoch", "release", "dev", "pre", "post", "local"]
+)
+
+
+
+class InvalidVersion(ValueError):
+ """
+ An invalid version was found, users should refer to PEP 440.
+ """
+
+
+class _BaseVersion:
+ _key: Union[CmpKey, LegacyCmpKey]
+
+ def __hash__(self) -> int:
+ return hash(self._key)
+
+ # Please keep the duplicated `isinstance` check
+ # in the six comparisons hereunder
+ # unless you find a way to avoid adding overhead function calls.
+ def __lt__(self, other: "_BaseVersion") -> bool:
+ if not isinstance(other, _BaseVersion):
+ return NotImplemented
+
+ return self._key < other._key
+
+ def __le__(self, other: "_BaseVersion") -> bool:
+ if not isinstance(other, _BaseVersion):
+ return NotImplemented
+
+ return self._key <= other._key
+
+ def __eq__(self, other: object) -> bool:
+ if not isinstance(other, _BaseVersion):
+ return NotImplemented
+
+ return self._key == other._key
+
+ def __ge__(self, other: "_BaseVersion") -> bool:
+ if not isinstance(other, _BaseVersion):
+ return NotImplemented
+
+ return self._key >= other._key
+
+ def __gt__(self, other: "_BaseVersion") -> bool:
+ if not isinstance(other, _BaseVersion):
+ return NotImplemented
+
+ return self._key > other._key
+
+ def __ne__(self, other: object) -> bool:
+ if not isinstance(other, _BaseVersion):
+ return NotImplemented
+
+ return self._key != other._key
+
+
+# Deliberately not anchored to the start and end of the string, to make it
+# easier for 3rd party code to reuse
+VERSION_PATTERN = r"""
+ v?
+ (?:
+ (?:(?P<epoch>[0-9]+)!)? # epoch
+ (?P<release>[0-9]+(?:\.[0-9]+)*) # release segment
+ (?P<pre> # pre-release
+ [-_\.]?
+ (?P<pre_l>(a|b|c|rc|alpha|beta|pre|preview))
+ [-_\.]?
+ (?P<pre_n>[0-9]+)?
+ )?
+ (?P<post> # post release
+ (?:-(?P<post_n1>[0-9]+))
+ |
+ (?:
+ [-_\.]?
+ (?P<post_l>post|rev|r)
+ [-_\.]?
+ (?P<post_n2>[0-9]+)?
+ )
+ )?
+ (?P<dev> # dev release
+ [-_\.]?
+ (?P<dev_l>dev)
+ [-_\.]?
+ (?P<dev_n>[0-9]+)?
+ )?
+ )
+ (?:\+(?P<local>[a-z0-9]+(?:[-_\.][a-z0-9]+)*))? # local version
+"""
+
+
+class Version(_BaseVersion):
+
+ _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE)
+
+ def __init__(self, version: str) -> None:
+
+ # Validate the version and parse it into pieces
+ match = self._regex.search(version)
+ if not match:
+ raise InvalidVersion(f"Invalid version: '{version}'")
+
+ # Store the parsed out pieces of the version
+ self._version = _Version(
+ epoch=int(match.group("epoch")) if match.group("epoch") else 0,
+ release=tuple(int(i) for i in match.group("release").split(".")),
+ pre=_parse_letter_version(match.group("pre_l"), match.group("pre_n")),
+ post=_parse_letter_version(
+ match.group("post_l"), match.group("post_n1") or match.group("post_n2")
+ ),
+ dev=_parse_letter_version(match.group("dev_l"), match.group("dev_n")),
+ local=_parse_local_version(match.group("local")),
+ )
+
+ # Generate a key which will be used for sorting
+ self._key = _cmpkey(
+ self._version.epoch,
+ self._version.release,
+ self._version.pre,
+ self._version.post,
+ self._version.dev,
+ self._version.local,
+ )
+
+ def __repr__(self) -> str:
+ return f"<Version('{self}')>"
+
+ def __str__(self) -> str:
+ parts = []
+
+ # Epoch
+ if self.epoch != 0:
+ parts.append(f"{self.epoch}!")
+
+ # Release segment
+ parts.append(".".join(str(x) for x in self.release))
+
+ # Pre-release
+ if self.pre is not None:
+ parts.append("".join(str(x) for x in self.pre))
+
+ # Post-release
+ if self.post is not None:
+ parts.append(f".post{self.post}")
+
+ # Development release
+ if self.dev is not None:
+ parts.append(f".dev{self.dev}")
+
+ # Local version segment
+ if self.local is not None:
+ parts.append(f"+{self.local}")
+
+ return "".join(parts)
+
+ @property
+ def epoch(self) -> int:
+ _epoch: int = self._version.epoch
+ return _epoch
+
+ @property
+ def release(self) -> Tuple[int, ...]:
+ _release: Tuple[int, ...] = self._version.release
+ return _release
+
+ @property
+ def pre(self) -> Optional[Tuple[str, int]]:
+ _pre: Optional[Tuple[str, int]] = self._version.pre
+ return _pre
+
+ @property
+ def post(self) -> Optional[int]:
+ return self._version.post[1] if self._version.post else None
+
+ @property
+ def dev(self) -> Optional[int]:
+ return self._version.dev[1] if self._version.dev else None
+
+ @property
+ def local(self) -> Optional[str]:
+ if self._version.local:
+ return ".".join(str(x) for x in self._version.local)
+ else:
+ return None
+
+ @property
+ def public(self) -> str:
+ return str(self).split("+", 1)[0]
+
+ @property
+ def base_version(self) -> str:
+ parts = []
+
+ # Epoch
+ if self.epoch != 0:
+ parts.append(f"{self.epoch}!")
+
+ # Release segment
+ parts.append(".".join(str(x) for x in self.release))
+
+ return "".join(parts)
+
+ @property
+ def is_prerelease(self) -> bool:
+ return self.dev is not None or self.pre is not None
+
+ @property
+ def is_postrelease(self) -> bool:
+ return self.post is not None
+
+ @property
+ def is_devrelease(self) -> bool:
+ return self.dev is not None
+
+ @property
+ def major(self) -> int:
+ return self.release[0] if len(self.release) >= 1 else 0
+
+ @property
+ def minor(self) -> int:
+ return self.release[1] if len(self.release) >= 2 else 0
+
+ @property
+ def micro(self) -> int:
+ return self.release[2] if len(self.release) >= 3 else 0
+
+
+def _parse_letter_version(
+ letter: str, number: Union[str, bytes, SupportsInt]
+) -> Optional[Tuple[str, int]]:
+
+ if letter:
+ # We consider there to be an implicit 0 in a pre-release if there is
+ # not a numeral associated with it.
+ if number is None:
+ number = 0
+
+ # We normalize any letters to their lower case form
+ letter = letter.lower()
+
+ # We consider some words to be alternate spellings of other words and
+ # in those cases we want to normalize the spellings to our preferred
+ # spelling.
+ if letter == "alpha":
+ letter = "a"
+ elif letter == "beta":
+ letter = "b"
+ elif letter in ["c", "pre", "preview"]:
+ letter = "rc"
+ elif letter in ["rev", "r"]:
+ letter = "post"
+
+ return letter, int(number)
+ if not letter and number:
+ # We assume if we are given a number, but we are not given a letter
+ # then this is using the implicit post release syntax (e.g. 1.0-1)
+ letter = "post"
+
+ return letter, int(number)
+
+ return None
+
+
+_local_version_separators = re.compile(r"[\._-]")
+
+
+def _parse_local_version(local: str) -> Optional[LocalType]:
+ """
+ Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve").
+ """
+ if local is not None:
+ return tuple(
+ part.lower() if not part.isdigit() else int(part)
+ for part in _local_version_separators.split(local)
+ )
+ return None
+
+
+def _cmpkey(
+ epoch: int,
+ release: Tuple[int, ...],
+ pre: Optional[Tuple[str, int]],
+ post: Optional[Tuple[str, int]],
+ dev: Optional[Tuple[str, int]],
+ local: Optional[Tuple[SubLocalType]],
+) -> CmpKey:
+
+ # When we compare a release version, we want to compare it with all of the
+ # trailing zeros removed. So we'll use a reverse the list, drop all the now
+ # leading zeros until we come to something non zero, then take the rest
+ # re-reverse it back into the correct order and make it a tuple and use
+ # that for our sorting key.
+ _release = tuple(
+ reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release))))
+ )
+
+ # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0.
+ # We'll do this by abusing the pre segment, but we _only_ want to do this
+ # if there is not a pre or a post segment. If we have one of those then
+ # the normal sorting rules will handle this case correctly.
+ if pre is None and post is None and dev is not None:
+ _pre: PrePostDevType = NegativeInfinity
+ # Versions without a pre-release (except as noted above) should sort after
+ # those with one.
+ elif pre is None:
+ _pre = Infinity
+ else:
+ _pre = pre
+
+ # Versions without a post segment should sort before those with one.
+ if post is None:
+ _post: PrePostDevType = NegativeInfinity
+
+ else:
+ _post = post
+
+ # Versions without a development segment should sort after those with one.
+ if dev is None:
+ _dev: PrePostDevType = Infinity
+
+ else:
+ _dev = dev
+
+ if local is None:
+ # Versions without a local segment should sort before those with one.
+ _local: LocalType = NegativeInfinity
+ else:
+ # Versions with a local segment need that segment parsed to implement
+ # the sorting rules in PEP440.
+ # - Alpha numeric segments sort before numeric segments
+ # - Alpha numeric segments sort lexicographically
+ # - Numeric segments sort numerically
+ # - Shorter versions sort before longer versions when the prefixes
+ # match exactly
+ _local = tuple(
+ (i, "") if isinstance(i, int) else (NegativeInfinity, i) for i in local
+ )
+
+ return epoch, _release, _pre, _post, _dev, _local
diff --git a/seaborn/matrix.py b/seaborn/matrix.py
new file mode 100644
index 0000000000000000000000000000000000000000..6b99c118b62858c31b1c7db499fb0af4b4d99cb3
--- /dev/null
+++ b/seaborn/matrix.py
@@ -0,0 +1,1262 @@
+"""Functions to visualize matrices of data."""
+import warnings
+
+import matplotlib as mpl
+from matplotlib.collections import LineCollection
+import matplotlib.pyplot as plt
+from matplotlib import gridspec
+import numpy as np
+import pandas as pd
+try:
+ from scipy.cluster import hierarchy
+ _no_scipy = False
+except ImportError:
+ _no_scipy = True
+
+from . import cm
+from .axisgrid import Grid
+from ._compat import get_colormap
+from .utils import (
+ despine,
+ axis_ticklabels_overlap,
+ relative_luminance,
+ to_utf8,
+ _draw_figure,
+)
+
+
+__all__ = ["heatmap", "clustermap"]
+
+
+def _index_to_label(index):
+ """Convert a pandas index or multiindex to an axis label."""
+ if isinstance(index, pd.MultiIndex):
+ return "-".join(map(to_utf8, index.names))
+ else:
+ return index.name
+
+
+def _index_to_ticklabels(index):
+ """Convert a pandas index or multiindex into ticklabels."""
+ if isinstance(index, pd.MultiIndex):
+ return ["-".join(map(to_utf8, i)) for i in index.values]
+ else:
+ return index.values
+
+
+def _convert_colors(colors):
+ """Convert either a list of colors or nested lists of colors to RGB."""
+ to_rgb = mpl.colors.to_rgb
+
+ try:
+ to_rgb(colors[0])
+ # If this works, there is only one level of colors
+ return list(map(to_rgb, colors))
+ except ValueError:
+ # If we get here, we have nested lists
+ return [list(map(to_rgb, color_list)) for color_list in colors]
+
+
+def _matrix_mask(data, mask):
+ """Ensure that data and mask are compatible and add missing values.
+
+ Values will be plotted for cells where ``mask`` is ``False``.
+
+ ``data`` is expected to be a DataFrame; ``mask`` can be an array or
+ a DataFrame.
+
+ """
+ if mask is None:
+ mask = np.zeros(data.shape, bool)
+
+ if isinstance(mask, np.ndarray):
+ # For array masks, ensure that shape matches data then convert
+ if mask.shape != data.shape:
+ raise ValueError("Mask must have the same shape as data.")
+
+ mask = pd.DataFrame(mask,
+ index=data.index,
+ columns=data.columns,
+ dtype=bool)
+
+ elif isinstance(mask, pd.DataFrame):
+ # For DataFrame masks, ensure that semantic labels match data
+ if not mask.index.equals(data.index) \
+ and mask.columns.equals(data.columns):
+ err = "Mask must have the same index and columns as data."
+ raise ValueError(err)
+
+ # Add any cells with missing data to the mask
+ # This works around an issue where `plt.pcolormesh` doesn't represent
+ # missing data properly
+ mask = mask | pd.isnull(data)
+
+ return mask
+
+
+class _HeatMapper:
+ """Draw a heatmap plot of a matrix with nice labels and colormaps."""
+
+ def __init__(self, data, vmin, vmax, cmap, center, robust, annot, fmt,
+ annot_kws, cbar, cbar_kws,
+ xticklabels=True, yticklabels=True, mask=None):
+ """Initialize the plotting object."""
+ # We always want to have a DataFrame with semantic information
+ # and an ndarray to pass to matplotlib
+ if isinstance(data, pd.DataFrame):
+ plot_data = data.values
+ else:
+ plot_data = np.asarray(data)
+ data = pd.DataFrame(plot_data)
+
+ # Validate the mask and convert to DataFrame
+ mask = _matrix_mask(data, mask)
+
+ plot_data = np.ma.masked_where(np.asarray(mask), plot_data)
+
+ # Get good names for the rows and columns
+ xtickevery = 1
+ if isinstance(xticklabels, int):
+ xtickevery = xticklabels
+ xticklabels = _index_to_ticklabels(data.columns)
+ elif xticklabels is True:
+ xticklabels = _index_to_ticklabels(data.columns)
+ elif xticklabels is False:
+ xticklabels = []
+
+ ytickevery = 1
+ if isinstance(yticklabels, int):
+ ytickevery = yticklabels
+ yticklabels = _index_to_ticklabels(data.index)
+ elif yticklabels is True:
+ yticklabels = _index_to_ticklabels(data.index)
+ elif yticklabels is False:
+ yticklabels = []
+
+ if not len(xticklabels):
+ self.xticks = []
+ self.xticklabels = []
+ elif isinstance(xticklabels, str) and xticklabels == "auto":
+ self.xticks = "auto"
+ self.xticklabels = _index_to_ticklabels(data.columns)
+ else:
+ self.xticks, self.xticklabels = self._skip_ticks(xticklabels,
+ xtickevery)
+
+ if not len(yticklabels):
+ self.yticks = []
+ self.yticklabels = []
+ elif isinstance(yticklabels, str) and yticklabels == "auto":
+ self.yticks = "auto"
+ self.yticklabels = _index_to_ticklabels(data.index)
+ else:
+ self.yticks, self.yticklabels = self._skip_ticks(yticklabels,
+ ytickevery)
+
+ # Get good names for the axis labels
+ xlabel = _index_to_label(data.columns)
+ ylabel = _index_to_label(data.index)
+ self.xlabel = xlabel if xlabel is not None else ""
+ self.ylabel = ylabel if ylabel is not None else ""
+
+ # Determine good default values for the colormapping
+ self._determine_cmap_params(plot_data, vmin, vmax,
+ cmap, center, robust)
+
+ # Sort out the annotations
+ if annot is None or annot is False:
+ annot = False
+ annot_data = None
+ else:
+ if isinstance(annot, bool):
+ annot_data = plot_data
+ else:
+ annot_data = np.asarray(annot)
+ if annot_data.shape != plot_data.shape:
+ err = "`data` and `annot` must have same shape."
+ raise ValueError(err)
+ annot = True
+
+ # Save other attributes to the object
+ self.data = data
+ self.plot_data = plot_data
+
+ self.annot = annot
+ self.annot_data = annot_data
+
+ self.fmt = fmt
+ self.annot_kws = {} if annot_kws is None else annot_kws.copy()
+ self.cbar = cbar
+ self.cbar_kws = {} if cbar_kws is None else cbar_kws.copy()
+
+ def _determine_cmap_params(self, plot_data, vmin, vmax,
+ cmap, center, robust):
+ """Use some heuristics to set good defaults for colorbar and range."""
+
+ # plot_data is a np.ma.array instance
+ calc_data = plot_data.astype(float).filled(np.nan)
+ if vmin is None:
+ if robust:
+ vmin = np.nanpercentile(calc_data, 2)
+ else:
+ vmin = np.nanmin(calc_data)
+ if vmax is None:
+ if robust:
+ vmax = np.nanpercentile(calc_data, 98)
+ else:
+ vmax = np.nanmax(calc_data)
+ self.vmin, self.vmax = vmin, vmax
+
+ # Choose default colormaps if not provided
+ if cmap is None:
+ if center is None:
+ self.cmap = cm.rocket
+ else:
+ self.cmap = cm.icefire
+ elif isinstance(cmap, str):
+ self.cmap = get_colormap(cmap)
+ elif isinstance(cmap, list):
+ self.cmap = mpl.colors.ListedColormap(cmap)
+ else:
+ self.cmap = cmap
+
+ # Recenter a divergent colormap
+ if center is not None:
+
+ # Copy bad values
+ # in mpl<3.2 only masked values are honored with "bad" color spec
+ # (see https://github.com/matplotlib/matplotlib/pull/14257)
+ bad = self.cmap(np.ma.masked_invalid([np.nan]))[0]
+
+ # under/over values are set for sure when cmap extremes
+ # do not map to the same color as +-inf
+ under = self.cmap(-np.inf)
+ over = self.cmap(np.inf)
+ under_set = under != self.cmap(0)
+ over_set = over != self.cmap(self.cmap.N - 1)
+
+ vrange = max(vmax - center, center - vmin)
+ normlize = mpl.colors.Normalize(center - vrange, center + vrange)
+ cmin, cmax = normlize([vmin, vmax])
+ cc = np.linspace(cmin, cmax, 256)
+ self.cmap = mpl.colors.ListedColormap(self.cmap(cc))
+ self.cmap.set_bad(bad)
+ if under_set:
+ self.cmap.set_under(under)
+ if over_set:
+ self.cmap.set_over(over)
+
+ def _annotate_heatmap(self, ax, mesh):
+ """Add textual labels with the value in each cell."""
+ mesh.update_scalarmappable()
+ height, width = self.annot_data.shape
+ xpos, ypos = np.meshgrid(np.arange(width) + .5, np.arange(height) + .5)
+ for x, y, m, color, val in zip(xpos.flat, ypos.flat,
+ mesh.get_array().flat, mesh.get_facecolors(),
+ self.annot_data.flat):
+ if m is not np.ma.masked:
+ lum = relative_luminance(color)
+ text_color = ".15" if lum > .408 else "w"
+ annotation = ("{:" + self.fmt + "}").format(val)
+ text_kwargs = dict(color=text_color, ha="center", va="center")
+ text_kwargs.update(self.annot_kws)
+ ax.text(x, y, annotation, **text_kwargs)
+
+ def _skip_ticks(self, labels, tickevery):
+ """Return ticks and labels at evenly spaced intervals."""
+ n = len(labels)
+ if tickevery == 0:
+ ticks, labels = [], []
+ elif tickevery == 1:
+ ticks, labels = np.arange(n) + .5, labels
+ else:
+ start, end, step = 0, n, tickevery
+ ticks = np.arange(start, end, step) + .5
+ labels = labels[start:end:step]
+ return ticks, labels
+
+ def _auto_ticks(self, ax, labels, axis):
+ """Determine ticks and ticklabels that minimize overlap."""
+ transform = ax.figure.dpi_scale_trans.inverted()
+ bbox = ax.get_window_extent().transformed(transform)
+ size = [bbox.width, bbox.height][axis]
+ axis = [ax.xaxis, ax.yaxis][axis]
+ tick, = axis.set_ticks([0])
+ fontsize = tick.label1.get_size()
+ max_ticks = int(size // (fontsize / 72))
+ if max_ticks < 1:
+ return [], []
+ tick_every = len(labels) // max_ticks + 1
+ tick_every = 1 if tick_every == 0 else tick_every
+ ticks, labels = self._skip_ticks(labels, tick_every)
+ return ticks, labels
+
+ def plot(self, ax, cax, kws):
+ """Draw the heatmap on the provided Axes."""
+ # Remove all the Axes spines
+ despine(ax=ax, left=True, bottom=True)
+
+ # setting vmin/vmax in addition to norm is deprecated
+ # so avoid setting if norm is set
+ if kws.get("norm") is None:
+ kws.setdefault("vmin", self.vmin)
+ kws.setdefault("vmax", self.vmax)
+
+ # Draw the heatmap
+ mesh = ax.pcolormesh(self.plot_data, cmap=self.cmap, **kws)
+
+ # Set the axis limits
+ ax.set(xlim=(0, self.data.shape[1]), ylim=(0, self.data.shape[0]))
+
+ # Invert the y axis to show the plot in matrix form
+ ax.invert_yaxis()
+
+ # Possibly add a colorbar
+ if self.cbar:
+ cb = ax.figure.colorbar(mesh, cax, ax, **self.cbar_kws)
+ cb.outline.set_linewidth(0)
+ # If rasterized is passed to pcolormesh, also rasterize the
+ # colorbar to avoid white lines on the PDF rendering
+ if kws.get('rasterized', False):
+ cb.solids.set_rasterized(True)
+
+ # Add row and column labels
+ if isinstance(self.xticks, str) and self.xticks == "auto":
+ xticks, xticklabels = self._auto_ticks(ax, self.xticklabels, 0)
+ else:
+ xticks, xticklabels = self.xticks, self.xticklabels
+
+ if isinstance(self.yticks, str) and self.yticks == "auto":
+ yticks, yticklabels = self._auto_ticks(ax, self.yticklabels, 1)
+ else:
+ yticks, yticklabels = self.yticks, self.yticklabels
+
+ ax.set(xticks=xticks, yticks=yticks)
+ xtl = ax.set_xticklabels(xticklabels)
+ ytl = ax.set_yticklabels(yticklabels, rotation="vertical")
+ plt.setp(ytl, va="center") # GH2484
+
+ # Possibly rotate them if they overlap
+ _draw_figure(ax.figure)
+
+ if axis_ticklabels_overlap(xtl):
+ plt.setp(xtl, rotation="vertical")
+ if axis_ticklabels_overlap(ytl):
+ plt.setp(ytl, rotation="horizontal")
+
+ # Add the axis labels
+ ax.set(xlabel=self.xlabel, ylabel=self.ylabel)
+
+ # Annotate the cells with the formatted values
+ if self.annot:
+ self._annotate_heatmap(ax, mesh)
+
+
+def heatmap(
+ data, *,
+ vmin=None, vmax=None, cmap=None, center=None, robust=False,
+ annot=None, fmt=".2g", annot_kws=None,
+ linewidths=0, linecolor="white",
+ cbar=True, cbar_kws=None, cbar_ax=None,
+ square=False, xticklabels="auto", yticklabels="auto",
+ mask=None, ax=None,
+ **kwargs
+):
+ """Plot rectangular data as a color-encoded matrix.
+
+ This is an Axes-level function and will draw the heatmap into the
+ currently-active Axes if none is provided to the ``ax`` argument. Part of
+ this Axes space will be taken and used to plot a colormap, unless ``cbar``
+ is False or a separate Axes is provided to ``cbar_ax``.
+
+ Parameters
+ ----------
+ data : rectangular dataset
+ 2D dataset that can be coerced into an ndarray. If a Pandas DataFrame
+ is provided, the index/column information will be used to label the
+ columns and rows.
+ vmin, vmax : floats, optional
+ Values to anchor the colormap, otherwise they are inferred from the
+ data and other keyword arguments.
+ cmap : matplotlib colormap name or object, or list of colors, optional
+ The mapping from data values to color space. If not provided, the
+ default will depend on whether ``center`` is set.
+ center : float, optional
+ The value at which to center the colormap when plotting divergent data.
+ Using this parameter will change the default ``cmap`` if none is
+ specified.
+ robust : bool, optional
+ If True and ``vmin`` or ``vmax`` are absent, the colormap range is
+ computed with robust quantiles instead of the extreme values.
+ annot : bool or rectangular dataset, optional
+ If True, write the data value in each cell. If an array-like with the
+ same shape as ``data``, then use this to annotate the heatmap instead
+ of the data. Note that DataFrames will match on position, not index.
+ fmt : str, optional
+ String formatting code to use when adding annotations.
+ annot_kws : dict of key, value mappings, optional
+ Keyword arguments for :meth:`matplotlib.axes.Axes.text` when ``annot``
+ is True.
+ linewidths : float, optional
+ Width of the lines that will divide each cell.
+ linecolor : color, optional
+ Color of the lines that will divide each cell.
+ cbar : bool, optional
+ Whether to draw a colorbar.
+ cbar_kws : dict of key, value mappings, optional
+ Keyword arguments for :meth:`matplotlib.figure.Figure.colorbar`.
+ cbar_ax : matplotlib Axes, optional
+ Axes in which to draw the colorbar, otherwise take space from the
+ main Axes.
+ square : bool, optional
+ If True, set the Axes aspect to "equal" so each cell will be
+ square-shaped.
+ xticklabels, yticklabels : "auto", bool, list-like, or int, optional
+ If True, plot the column names of the dataframe. If False, don't plot
+ the column names. If list-like, plot these alternate labels as the
+ xticklabels. If an integer, use the column names but plot only every
+ n label. If "auto", try to densely plot non-overlapping labels.
+ mask : bool array or DataFrame, optional
+ If passed, data will not be shown in cells where ``mask`` is True.
+ Cells with missing values are automatically masked.
+ ax : matplotlib Axes, optional
+ Axes in which to draw the plot, otherwise use the currently-active
+ Axes.
+ kwargs : other keyword arguments
+ All other keyword arguments are passed to
+ :meth:`matplotlib.axes.Axes.pcolormesh`.
+
+ Returns
+ -------
+ ax : matplotlib Axes
+ Axes object with the heatmap.
+
+ See Also
+ --------
+ clustermap : Plot a matrix using hierarchical clustering to arrange the
+ rows and columns.
+
+ Examples
+ --------
+
+ .. include:: ../docstrings/heatmap.rst
+
+ """
+ # Initialize the plotter object
+ plotter = _HeatMapper(data, vmin, vmax, cmap, center, robust, annot, fmt,
+ annot_kws, cbar, cbar_kws, xticklabels,
+ yticklabels, mask)
+
+ # Add the pcolormesh kwargs here
+ kwargs["linewidths"] = linewidths
+ kwargs["edgecolor"] = linecolor
+
+ # Draw the plot and return the Axes
+ if ax is None:
+ ax = plt.gca()
+ if square:
+ ax.set_aspect("equal")
+ plotter.plot(ax, cbar_ax, kwargs)
+ return ax
+
+
+class _DendrogramPlotter:
+ """Object for drawing tree of similarities between data rows/columns"""
+
+ def __init__(self, data, linkage, metric, method, axis, label, rotate):
+ """Plot a dendrogram of the relationships between the columns of data
+
+ Parameters
+ ----------
+ data : pandas.DataFrame
+ Rectangular data
+ """
+ self.axis = axis
+ if self.axis == 1:
+ data = data.T
+
+ if isinstance(data, pd.DataFrame):
+ array = data.values
+ else:
+ array = np.asarray(data)
+ data = pd.DataFrame(array)
+
+ self.array = array
+ self.data = data
+
+ self.shape = self.data.shape
+ self.metric = metric
+ self.method = method
+ self.axis = axis
+ self.label = label
+ self.rotate = rotate
+
+ if linkage is None:
+ self.linkage = self.calculated_linkage
+ else:
+ self.linkage = linkage
+ self.dendrogram = self.calculate_dendrogram()
+
+ # Dendrogram ends are always at multiples of 5, who knows why
+ ticks = 10 * np.arange(self.data.shape[0]) + 5
+
+ if self.label:
+ ticklabels = _index_to_ticklabels(self.data.index)
+ ticklabels = [ticklabels[i] for i in self.reordered_ind]
+ if self.rotate:
+ self.xticks = []
+ self.yticks = ticks
+ self.xticklabels = []
+
+ self.yticklabels = ticklabels
+ self.ylabel = _index_to_label(self.data.index)
+ self.xlabel = ''
+ else:
+ self.xticks = ticks
+ self.yticks = []
+ self.xticklabels = ticklabels
+ self.yticklabels = []
+ self.ylabel = ''
+ self.xlabel = _index_to_label(self.data.index)
+ else:
+ self.xticks, self.yticks = [], []
+ self.yticklabels, self.xticklabels = [], []
+ self.xlabel, self.ylabel = '', ''
+
+ self.dependent_coord = self.dendrogram['dcoord']
+ self.independent_coord = self.dendrogram['icoord']
+
+ def _calculate_linkage_scipy(self):
+ linkage = hierarchy.linkage(self.array, method=self.method,
+ metric=self.metric)
+ return linkage
+
+ def _calculate_linkage_fastcluster(self):
+ import fastcluster
+ # Fastcluster has a memory-saving vectorized version, but only
+ # with certain linkage methods, and mostly with euclidean metric
+ # vector_methods = ('single', 'centroid', 'median', 'ward')
+ euclidean_methods = ('centroid', 'median', 'ward')
+ euclidean = self.metric == 'euclidean' and self.method in \
+ euclidean_methods
+ if euclidean or self.method == 'single':
+ return fastcluster.linkage_vector(self.array,
+ method=self.method,
+ metric=self.metric)
+ else:
+ linkage = fastcluster.linkage(self.array, method=self.method,
+ metric=self.metric)
+ return linkage
+
+ @property
+ def calculated_linkage(self):
+
+ try:
+ return self._calculate_linkage_fastcluster()
+ except ImportError:
+ if np.prod(self.shape) >= 10000:
+ msg = ("Clustering large matrix with scipy. Installing "
+ "`fastcluster` may give better performance.")
+ warnings.warn(msg)
+
+ return self._calculate_linkage_scipy()
+
+ def calculate_dendrogram(self):
+ """Calculates a dendrogram based on the linkage matrix
+
+ Made a separate function, not a property because don't want to
+ recalculate the dendrogram every time it is accessed.
+
+ Returns
+ -------
+ dendrogram : dict
+ Dendrogram dictionary as returned by scipy.cluster.hierarchy
+ .dendrogram. The important key-value pairing is
+ "reordered_ind" which indicates the re-ordering of the matrix
+ """
+ return hierarchy.dendrogram(self.linkage, no_plot=True,
+ color_threshold=-np.inf)
+
+ @property
+ def reordered_ind(self):
+ """Indices of the matrix, reordered by the dendrogram"""
+ return self.dendrogram['leaves']
+
+ def plot(self, ax, tree_kws):
+ """Plots a dendrogram of the similarities between data on the axes
+
+ Parameters
+ ----------
+ ax : matplotlib.axes.Axes
+ Axes object upon which the dendrogram is plotted
+
+ """
+ tree_kws = {} if tree_kws is None else tree_kws.copy()
+ tree_kws.setdefault("linewidths", .5)
+ tree_kws.setdefault("colors", tree_kws.pop("color", (.2, .2, .2)))
+
+ if self.rotate and self.axis == 0:
+ coords = zip(self.dependent_coord, self.independent_coord)
+ else:
+ coords = zip(self.independent_coord, self.dependent_coord)
+ lines = LineCollection([list(zip(x, y)) for x, y in coords],
+ **tree_kws)
+
+ ax.add_collection(lines)
+ number_of_leaves = len(self.reordered_ind)
+ max_dependent_coord = max(map(max, self.dependent_coord))
+
+ if self.rotate:
+ ax.yaxis.set_ticks_position('right')
+
+ # Constants 10 and 1.05 come from
+ # `scipy.cluster.hierarchy._plot_dendrogram`
+ ax.set_ylim(0, number_of_leaves * 10)
+ ax.set_xlim(0, max_dependent_coord * 1.05)
+
+ ax.invert_xaxis()
+ ax.invert_yaxis()
+ else:
+ # Constants 10 and 1.05 come from
+ # `scipy.cluster.hierarchy._plot_dendrogram`
+ ax.set_xlim(0, number_of_leaves * 10)
+ ax.set_ylim(0, max_dependent_coord * 1.05)
+
+ despine(ax=ax, bottom=True, left=True)
+
+ ax.set(xticks=self.xticks, yticks=self.yticks,
+ xlabel=self.xlabel, ylabel=self.ylabel)
+ xtl = ax.set_xticklabels(self.xticklabels)
+ ytl = ax.set_yticklabels(self.yticklabels, rotation='vertical')
+
+ # Force a draw of the plot to avoid matplotlib window error
+ _draw_figure(ax.figure)
+
+ if len(ytl) > 0 and axis_ticklabels_overlap(ytl):
+ plt.setp(ytl, rotation="horizontal")
+ if len(xtl) > 0 and axis_ticklabels_overlap(xtl):
+ plt.setp(xtl, rotation="vertical")
+ return self
+
+
+def dendrogram(
+ data, *,
+ linkage=None, axis=1, label=True, metric='euclidean',
+ method='average', rotate=False, tree_kws=None, ax=None
+):
+ """Draw a tree diagram of relationships within a matrix
+
+ Parameters
+ ----------
+ data : pandas.DataFrame
+ Rectangular data
+ linkage : numpy.array, optional
+ Linkage matrix
+ axis : int, optional
+ Which axis to use to calculate linkage. 0 is rows, 1 is columns.
+ label : bool, optional
+ If True, label the dendrogram at leaves with column or row names
+ metric : str, optional
+ Distance metric. Anything valid for scipy.spatial.distance.pdist
+ method : str, optional
+ Linkage method to use. Anything valid for
+ scipy.cluster.hierarchy.linkage
+ rotate : bool, optional
+ When plotting the matrix, whether to rotate it 90 degrees
+ counter-clockwise, so the leaves face right
+ tree_kws : dict, optional
+ Keyword arguments for the ``matplotlib.collections.LineCollection``
+ that is used for plotting the lines of the dendrogram tree.
+ ax : matplotlib axis, optional
+ Axis to plot on, otherwise uses current axis
+
+ Returns
+ -------
+ dendrogramplotter : _DendrogramPlotter
+ A Dendrogram plotter object.
+
+ Notes
+ -----
+ Access the reordered dendrogram indices with
+ dendrogramplotter.reordered_ind
+
+ """
+ if _no_scipy:
+ raise RuntimeError("dendrogram requires scipy to be installed")
+
+ plotter = _DendrogramPlotter(data, linkage=linkage, axis=axis,
+ metric=metric, method=method,
+ label=label, rotate=rotate)
+ if ax is None:
+ ax = plt.gca()
+
+ return plotter.plot(ax=ax, tree_kws=tree_kws)
+
+
+class ClusterGrid(Grid):
+
+ def __init__(self, data, pivot_kws=None, z_score=None, standard_scale=None,
+ figsize=None, row_colors=None, col_colors=None, mask=None,
+ dendrogram_ratio=None, colors_ratio=None, cbar_pos=None):
+ """Grid object for organizing clustered heatmap input on to axes"""
+ if _no_scipy:
+ raise RuntimeError("ClusterGrid requires scipy to be available")
+
+ if isinstance(data, pd.DataFrame):
+ self.data = data
+ else:
+ self.data = pd.DataFrame(data)
+
+ self.data2d = self.format_data(self.data, pivot_kws, z_score,
+ standard_scale)
+
+ self.mask = _matrix_mask(self.data2d, mask)
+
+ self._figure = plt.figure(figsize=figsize)
+
+ self.row_colors, self.row_color_labels = \
+ self._preprocess_colors(data, row_colors, axis=0)
+ self.col_colors, self.col_color_labels = \
+ self._preprocess_colors(data, col_colors, axis=1)
+
+ try:
+ row_dendrogram_ratio, col_dendrogram_ratio = dendrogram_ratio
+ except TypeError:
+ row_dendrogram_ratio = col_dendrogram_ratio = dendrogram_ratio
+
+ try:
+ row_colors_ratio, col_colors_ratio = colors_ratio
+ except TypeError:
+ row_colors_ratio = col_colors_ratio = colors_ratio
+
+ width_ratios = self.dim_ratios(self.row_colors,
+ row_dendrogram_ratio,
+ row_colors_ratio)
+ height_ratios = self.dim_ratios(self.col_colors,
+ col_dendrogram_ratio,
+ col_colors_ratio)
+
+ nrows = 2 if self.col_colors is None else 3
+ ncols = 2 if self.row_colors is None else 3
+
+ self.gs = gridspec.GridSpec(nrows, ncols,
+ width_ratios=width_ratios,
+ height_ratios=height_ratios)
+
+ self.ax_row_dendrogram = self._figure.add_subplot(self.gs[-1, 0])
+ self.ax_col_dendrogram = self._figure.add_subplot(self.gs[0, -1])
+ self.ax_row_dendrogram.set_axis_off()
+ self.ax_col_dendrogram.set_axis_off()
+
+ self.ax_row_colors = None
+ self.ax_col_colors = None
+
+ if self.row_colors is not None:
+ self.ax_row_colors = self._figure.add_subplot(
+ self.gs[-1, 1])
+ if self.col_colors is not None:
+ self.ax_col_colors = self._figure.add_subplot(
+ self.gs[1, -1])
+
+ self.ax_heatmap = self._figure.add_subplot(self.gs[-1, -1])
+ if cbar_pos is None:
+ self.ax_cbar = self.cax = None
+ else:
+ # Initialize the colorbar axes in the gridspec so that tight_layout
+ # works. We will move it where it belongs later. This is a hack.
+ self.ax_cbar = self._figure.add_subplot(self.gs[0, 0])
+ self.cax = self.ax_cbar # Backwards compatibility
+ self.cbar_pos = cbar_pos
+
+ self.dendrogram_row = None
+ self.dendrogram_col = None
+
+ def _preprocess_colors(self, data, colors, axis):
+ """Preprocess {row/col}_colors to extract labels and convert colors."""
+ labels = None
+
+ if colors is not None:
+ if isinstance(colors, (pd.DataFrame, pd.Series)):
+
+ # If data is unindexed, raise
+ if (not hasattr(data, "index") and axis == 0) or (
+ not hasattr(data, "columns") and axis == 1
+ ):
+ axis_name = "col" if axis else "row"
+ msg = (f"{axis_name}_colors indices can't be matched with data "
+ f"indices. Provide {axis_name}_colors as a non-indexed "
+ "datatype, e.g. by using `.to_numpy()``")
+ raise TypeError(msg)
+
+ # Ensure colors match data indices
+ if axis == 0:
+ colors = colors.reindex(data.index)
+ else:
+ colors = colors.reindex(data.columns)
+
+ # Replace na's with white color
+ # TODO We should set these to transparent instead
+ colors = colors.astype(object).fillna('white')
+
+ # Extract color values and labels from frame/series
+ if isinstance(colors, pd.DataFrame):
+ labels = list(colors.columns)
+ colors = colors.T.values
+ else:
+ if colors.name is None:
+ labels = [""]
+ else:
+ labels = [colors.name]
+ colors = colors.values
+
+ colors = _convert_colors(colors)
+
+ return colors, labels
+
+ def format_data(self, data, pivot_kws, z_score=None,
+ standard_scale=None):
+ """Extract variables from data or use directly."""
+
+ # Either the data is already in 2d matrix format, or need to do a pivot
+ if pivot_kws is not None:
+ data2d = data.pivot(**pivot_kws)
+ else:
+ data2d = data
+
+ if z_score is not None and standard_scale is not None:
+ raise ValueError(
+ 'Cannot perform both z-scoring and standard-scaling on data')
+
+ if z_score is not None:
+ data2d = self.z_score(data2d, z_score)
+ if standard_scale is not None:
+ data2d = self.standard_scale(data2d, standard_scale)
+ return data2d
+
+ @staticmethod
+ def z_score(data2d, axis=1):
+ """Standarize the mean and variance of the data axis
+
+ Parameters
+ ----------
+ data2d : pandas.DataFrame
+ Data to normalize
+ axis : int
+ Which axis to normalize across. If 0, normalize across rows, if 1,
+ normalize across columns.
+
+ Returns
+ -------
+ normalized : pandas.DataFrame
+ Noramlized data with a mean of 0 and variance of 1 across the
+ specified axis.
+ """
+ if axis == 1:
+ z_scored = data2d
+ else:
+ z_scored = data2d.T
+
+ z_scored = (z_scored - z_scored.mean()) / z_scored.std()
+
+ if axis == 1:
+ return z_scored
+ else:
+ return z_scored.T
+
+ @staticmethod
+ def standard_scale(data2d, axis=1):
+ """Divide the data by the difference between the max and min
+
+ Parameters
+ ----------
+ data2d : pandas.DataFrame
+ Data to normalize
+ axis : int
+ Which axis to normalize across. If 0, normalize across rows, if 1,
+ normalize across columns.
+
+ Returns
+ -------
+ standardized : pandas.DataFrame
+ Noramlized data with a mean of 0 and variance of 1 across the
+ specified axis.
+
+ """
+ # Normalize these values to range from 0 to 1
+ if axis == 1:
+ standardized = data2d
+ else:
+ standardized = data2d.T
+
+ subtract = standardized.min()
+ standardized = (standardized - subtract) / (
+ standardized.max() - standardized.min())
+
+ if axis == 1:
+ return standardized
+ else:
+ return standardized.T
+
+ def dim_ratios(self, colors, dendrogram_ratio, colors_ratio):
+ """Get the proportions of the figure taken up by each axes."""
+ ratios = [dendrogram_ratio]
+
+ if colors is not None:
+ # Colors are encoded as rgb, so there is an extra dimension
+ if np.ndim(colors) > 2:
+ n_colors = len(colors)
+ else:
+ n_colors = 1
+
+ ratios += [n_colors * colors_ratio]
+
+ # Add the ratio for the heatmap itself
+ ratios.append(1 - sum(ratios))
+
+ return ratios
+
+ @staticmethod
+ def color_list_to_matrix_and_cmap(colors, ind, axis=0):
+ """Turns a list of colors into a numpy matrix and matplotlib colormap
+
+ These arguments can now be plotted using heatmap(matrix, cmap)
+ and the provided colors will be plotted.
+
+ Parameters
+ ----------
+ colors : list of matplotlib colors
+ Colors to label the rows or columns of a dataframe.
+ ind : list of ints
+ Ordering of the rows or columns, to reorder the original colors
+ by the clustered dendrogram order
+ axis : int
+ Which axis this is labeling
+
+ Returns
+ -------
+ matrix : numpy.array
+ A numpy array of integer values, where each indexes into the cmap
+ cmap : matplotlib.colors.ListedColormap
+
+ """
+ try:
+ mpl.colors.to_rgb(colors[0])
+ except ValueError:
+ # We have a 2D color structure
+ m, n = len(colors), len(colors[0])
+ if not all(len(c) == n for c in colors[1:]):
+ raise ValueError("Multiple side color vectors must have same size")
+ else:
+ # We have one vector of colors
+ m, n = 1, len(colors)
+ colors = [colors]
+
+ # Map from unique colors to colormap index value
+ unique_colors = {}
+ matrix = np.zeros((m, n), int)
+ for i, inner in enumerate(colors):
+ for j, color in enumerate(inner):
+ idx = unique_colors.setdefault(color, len(unique_colors))
+ matrix[i, j] = idx
+
+ # Reorder for clustering and transpose for axis
+ matrix = matrix[:, ind]
+ if axis == 0:
+ matrix = matrix.T
+
+ cmap = mpl.colors.ListedColormap(list(unique_colors))
+ return matrix, cmap
+
+ def plot_dendrograms(self, row_cluster, col_cluster, metric, method,
+ row_linkage, col_linkage, tree_kws):
+ # Plot the row dendrogram
+ if row_cluster:
+ self.dendrogram_row = dendrogram(
+ self.data2d, metric=metric, method=method, label=False, axis=0,
+ ax=self.ax_row_dendrogram, rotate=True, linkage=row_linkage,
+ tree_kws=tree_kws
+ )
+ else:
+ self.ax_row_dendrogram.set_xticks([])
+ self.ax_row_dendrogram.set_yticks([])
+ # PLot the column dendrogram
+ if col_cluster:
+ self.dendrogram_col = dendrogram(
+ self.data2d, metric=metric, method=method, label=False,
+ axis=1, ax=self.ax_col_dendrogram, linkage=col_linkage,
+ tree_kws=tree_kws
+ )
+ else:
+ self.ax_col_dendrogram.set_xticks([])
+ self.ax_col_dendrogram.set_yticks([])
+ despine(ax=self.ax_row_dendrogram, bottom=True, left=True)
+ despine(ax=self.ax_col_dendrogram, bottom=True, left=True)
+
+ def plot_colors(self, xind, yind, **kws):
+ """Plots color labels between the dendrogram and the heatmap
+
+ Parameters
+ ----------
+ heatmap_kws : dict
+ Keyword arguments heatmap
+
+ """
+ # Remove any custom colormap and centering
+ # TODO this code has consistently caused problems when we
+ # have missed kwargs that need to be excluded that it might
+ # be better to rewrite *in*clusively.
+ kws = kws.copy()
+ kws.pop('cmap', None)
+ kws.pop('norm', None)
+ kws.pop('center', None)
+ kws.pop('annot', None)
+ kws.pop('vmin', None)
+ kws.pop('vmax', None)
+ kws.pop('robust', None)
+ kws.pop('xticklabels', None)
+ kws.pop('yticklabels', None)
+
+ # Plot the row colors
+ if self.row_colors is not None:
+ matrix, cmap = self.color_list_to_matrix_and_cmap(
+ self.row_colors, yind, axis=0)
+
+ # Get row_color labels
+ if self.row_color_labels is not None:
+ row_color_labels = self.row_color_labels
+ else:
+ row_color_labels = False
+
+ heatmap(matrix, cmap=cmap, cbar=False, ax=self.ax_row_colors,
+ xticklabels=row_color_labels, yticklabels=False, **kws)
+
+ # Adjust rotation of labels
+ if row_color_labels is not False:
+ plt.setp(self.ax_row_colors.get_xticklabels(), rotation=90)
+ else:
+ despine(self.ax_row_colors, left=True, bottom=True)
+
+ # Plot the column colors
+ if self.col_colors is not None:
+ matrix, cmap = self.color_list_to_matrix_and_cmap(
+ self.col_colors, xind, axis=1)
+
+ # Get col_color labels
+ if self.col_color_labels is not None:
+ col_color_labels = self.col_color_labels
+ else:
+ col_color_labels = False
+
+ heatmap(matrix, cmap=cmap, cbar=False, ax=self.ax_col_colors,
+ xticklabels=False, yticklabels=col_color_labels, **kws)
+
+ # Adjust rotation of labels, place on right side
+ if col_color_labels is not False:
+ self.ax_col_colors.yaxis.tick_right()
+ plt.setp(self.ax_col_colors.get_yticklabels(), rotation=0)
+ else:
+ despine(self.ax_col_colors, left=True, bottom=True)
+
+ def plot_matrix(self, colorbar_kws, xind, yind, **kws):
+ self.data2d = self.data2d.iloc[yind, xind]
+ self.mask = self.mask.iloc[yind, xind]
+
+ # Try to reorganize specified tick labels, if provided
+ xtl = kws.pop("xticklabels", "auto")
+ try:
+ xtl = np.asarray(xtl)[xind]
+ except (TypeError, IndexError):
+ pass
+ ytl = kws.pop("yticklabels", "auto")
+ try:
+ ytl = np.asarray(ytl)[yind]
+ except (TypeError, IndexError):
+ pass
+
+ # Reorganize the annotations to match the heatmap
+ annot = kws.pop("annot", None)
+ if annot is None or annot is False:
+ pass
+ else:
+ if isinstance(annot, bool):
+ annot_data = self.data2d
+ else:
+ annot_data = np.asarray(annot)
+ if annot_data.shape != self.data2d.shape:
+ err = "`data` and `annot` must have same shape."
+ raise ValueError(err)
+ annot_data = annot_data[yind][:, xind]
+ annot = annot_data
+
+ # Setting ax_cbar=None in clustermap call implies no colorbar
+ kws.setdefault("cbar", self.ax_cbar is not None)
+ heatmap(self.data2d, ax=self.ax_heatmap, cbar_ax=self.ax_cbar,
+ cbar_kws=colorbar_kws, mask=self.mask,
+ xticklabels=xtl, yticklabels=ytl, annot=annot, **kws)
+
+ ytl = self.ax_heatmap.get_yticklabels()
+ ytl_rot = None if not ytl else ytl[0].get_rotation()
+ self.ax_heatmap.yaxis.set_ticks_position('right')
+ self.ax_heatmap.yaxis.set_label_position('right')
+ if ytl_rot is not None:
+ ytl = self.ax_heatmap.get_yticklabels()
+ plt.setp(ytl, rotation=ytl_rot)
+
+ tight_params = dict(h_pad=.02, w_pad=.02)
+ if self.ax_cbar is None:
+ self._figure.tight_layout(**tight_params)
+ else:
+ # Turn the colorbar axes off for tight layout so that its
+ # ticks don't interfere with the rest of the plot layout.
+ # Then move it.
+ self.ax_cbar.set_axis_off()
+ self._figure.tight_layout(**tight_params)
+ self.ax_cbar.set_axis_on()
+ self.ax_cbar.set_position(self.cbar_pos)
+
+ def plot(self, metric, method, colorbar_kws, row_cluster, col_cluster,
+ row_linkage, col_linkage, tree_kws, **kws):
+
+ # heatmap square=True sets the aspect ratio on the axes, but that is
+ # not compatible with the multi-axes layout of clustergrid
+ if kws.get("square", False):
+ msg = "``square=True`` ignored in clustermap"
+ warnings.warn(msg)
+ kws.pop("square")
+
+ colorbar_kws = {} if colorbar_kws is None else colorbar_kws
+
+ self.plot_dendrograms(row_cluster, col_cluster, metric, method,
+ row_linkage=row_linkage, col_linkage=col_linkage,
+ tree_kws=tree_kws)
+ try:
+ xind = self.dendrogram_col.reordered_ind
+ except AttributeError:
+ xind = np.arange(self.data2d.shape[1])
+ try:
+ yind = self.dendrogram_row.reordered_ind
+ except AttributeError:
+ yind = np.arange(self.data2d.shape[0])
+
+ self.plot_colors(xind, yind, **kws)
+ self.plot_matrix(colorbar_kws, xind, yind, **kws)
+ return self
+
+
+def clustermap(
+ data, *,
+ pivot_kws=None, method='average', metric='euclidean',
+ z_score=None, standard_scale=None, figsize=(10, 10),
+ cbar_kws=None, row_cluster=True, col_cluster=True,
+ row_linkage=None, col_linkage=None,
+ row_colors=None, col_colors=None, mask=None,
+ dendrogram_ratio=.2, colors_ratio=0.03,
+ cbar_pos=(.02, .8, .05, .18), tree_kws=None,
+ **kwargs
+):
+ """
+ Plot a matrix dataset as a hierarchically-clustered heatmap.
+
+ This function requires scipy to be available.
+
+ Parameters
+ ----------
+ data : 2D array-like
+ Rectangular data for clustering. Cannot contain NAs.
+ pivot_kws : dict, optional
+ If `data` is a tidy dataframe, can provide keyword arguments for
+ pivot to create a rectangular dataframe.
+ method : str, optional
+ Linkage method to use for calculating clusters. See
+ :func:`scipy.cluster.hierarchy.linkage` documentation for more
+ information.
+ metric : str, optional
+ Distance metric to use for the data. See
+ :func:`scipy.spatial.distance.pdist` documentation for more options.
+ To use different metrics (or methods) for rows and columns, you may
+ construct each linkage matrix yourself and provide them as
+ `{row,col}_linkage`.
+ z_score : int or None, optional
+ Either 0 (rows) or 1 (columns). Whether or not to calculate z-scores
+ for the rows or the columns. Z scores are: z = (x - mean)/std, so
+ values in each row (column) will get the mean of the row (column)
+ subtracted, then divided by the standard deviation of the row (column).
+ This ensures that each row (column) has mean of 0 and variance of 1.
+ standard_scale : int or None, optional
+ Either 0 (rows) or 1 (columns). Whether or not to standardize that
+ dimension, meaning for each row or column, subtract the minimum and
+ divide each by its maximum.
+ figsize : tuple of (width, height), optional
+ Overall size of the figure.
+ cbar_kws : dict, optional
+ Keyword arguments to pass to `cbar_kws` in :func:`heatmap`, e.g. to
+ add a label to the colorbar.
+ {row,col}_cluster : bool, optional
+ If ``True``, cluster the {rows, columns}.
+ {row,col}_linkage : :class:`numpy.ndarray`, optional
+ Precomputed linkage matrix for the rows or columns. See
+ :func:`scipy.cluster.hierarchy.linkage` for specific formats.
+ {row,col}_colors : list-like or pandas DataFrame/Series, optional
+ List of colors to label for either the rows or columns. Useful to evaluate
+ whether samples within a group are clustered together. Can use nested lists or
+ DataFrame for multiple color levels of labeling. If given as a
+ :class:`pandas.DataFrame` or :class:`pandas.Series`, labels for the colors are
+ extracted from the DataFrames column names or from the name of the Series.
+ DataFrame/Series colors are also matched to the data by their index, ensuring
+ colors are drawn in the correct order.
+ mask : bool array or DataFrame, optional
+ If passed, data will not be shown in cells where `mask` is True.
+ Cells with missing values are automatically masked. Only used for
+ visualizing, not for calculating.
+ {dendrogram,colors}_ratio : float, or pair of floats, optional
+ Proportion of the figure size devoted to the two marginal elements. If
+ a pair is given, they correspond to (row, col) ratios.
+ cbar_pos : tuple of (left, bottom, width, height), optional
+ Position of the colorbar axes in the figure. Setting to ``None`` will
+ disable the colorbar.
+ tree_kws : dict, optional
+ Parameters for the :class:`matplotlib.collections.LineCollection`
+ that is used to plot the lines of the dendrogram tree.
+ kwargs : other keyword arguments
+ All other keyword arguments are passed to :func:`heatmap`.
+
+ Returns
+ -------
+ :class:`ClusterGrid`
+ A :class:`ClusterGrid` instance.
+
+ See Also
+ --------
+ heatmap : Plot rectangular data as a color-encoded matrix.
+
+ Notes
+ -----
+ The returned object has a ``savefig`` method that should be used if you
+ want to save the figure object without clipping the dendrograms.
+
+ To access the reordered row indices, use:
+ ``clustergrid.dendrogram_row.reordered_ind``
+
+ Column indices, use:
+ ``clustergrid.dendrogram_col.reordered_ind``
+
+ Examples
+ --------
+
+ .. include:: ../docstrings/clustermap.rst
+
+ """
+ if _no_scipy:
+ raise RuntimeError("clustermap requires scipy to be available")
+
+ plotter = ClusterGrid(data, pivot_kws=pivot_kws, figsize=figsize,
+ row_colors=row_colors, col_colors=col_colors,
+ z_score=z_score, standard_scale=standard_scale,
+ mask=mask, dendrogram_ratio=dendrogram_ratio,
+ colors_ratio=colors_ratio, cbar_pos=cbar_pos)
+
+ return plotter.plot(metric=metric, method=method,
+ colorbar_kws=cbar_kws,
+ row_cluster=row_cluster, col_cluster=col_cluster,
+ row_linkage=row_linkage, col_linkage=col_linkage,
+ tree_kws=tree_kws, **kwargs)
diff --git a/seaborn/miscplot.py b/seaborn/miscplot.py
new file mode 100644
index 0000000000000000000000000000000000000000..3fb290c812f8de293c9731ecd6bc83ee88fd2239
--- /dev/null
+++ b/seaborn/miscplot.py
@@ -0,0 +1,45 @@
+import numpy as np
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+import matplotlib.ticker as ticker
+
+__all__ = ["palplot", "dogplot"]
+
+
+def palplot(pal, size=1):
+ """Plot the values in a color palette as a horizontal array.
+
+ Parameters
+ ----------
+ pal : sequence of matplotlib colors
+ colors, i.e. as returned by seaborn.color_palette()
+ size :
+ scaling factor for size of plot
+
+ """
+ n = len(pal)
+ _, ax = plt.subplots(1, 1, figsize=(n * size, size))
+ ax.imshow(np.arange(n).reshape(1, n),
+ cmap=mpl.colors.ListedColormap(list(pal)),
+ interpolation="nearest", aspect="auto")
+ ax.set_xticks(np.arange(n) - .5)
+ ax.set_yticks([-.5, .5])
+ # Ensure nice border between colors
+ ax.set_xticklabels(["" for _ in range(n)])
+ # The proper way to set no ticks
+ ax.yaxis.set_major_locator(ticker.NullLocator())
+
+
+def dogplot(*_, **__):
+ """Who's a good boy?"""
+ from urllib.request import urlopen
+ from io import BytesIO
+
+ url = "https://github.com/mwaskom/seaborn-data/raw/master/png/img{}.png"
+ pic = np.random.randint(2, 7)
+ data = BytesIO(urlopen(url.format(pic)).read())
+ img = plt.imread(data)
+ f, ax = plt.subplots(figsize=(5, 5), dpi=100)
+ f.subplots_adjust(0, 0, 1, 1)
+ ax.imshow(img)
+ ax.set_axis_off()
diff --git a/seaborn/objects.py b/seaborn/objects.py
new file mode 100644
index 0000000000000000000000000000000000000000..123e57f0a936e8e73c684dd647b9813da86a3f60
--- /dev/null
+++ b/seaborn/objects.py
@@ -0,0 +1,49 @@
+"""
+A declarative, object-oriented interface for creating statistical graphics.
+
+The seaborn.objects namespace contains a number of classes that can be composed
+together to build a customized visualization.
+
+The main object is :class:`Plot`, which is the starting point for all figures.
+Pass :class:`Plot` a dataset and specify assignments from its variables to
+roles in the plot. Build up the visualization by calling its methods.
+
+There are four other general types of objects in this interface:
+
+- :class:`Mark` subclasses, which create matplotlib artists for visualization
+- :class:`Stat` subclasses, which apply statistical transforms before plotting
+- :class:`Move` subclasses, which make further adjustments to reduce overplotting
+
+These classes are passed to :meth:`Plot.add` to define a layer in the plot.
+Each layer has a :class:`Mark` and optional :class:`Stat` and/or :class:`Move`.
+Plots can have multiple layers.
+
+The other general type of object is a :class:`Scale` subclass, which provide an
+interface for controlling the mappings between data values and visual properties.
+Pass :class:`Scale` objects to :meth:`Plot.scale`.
+
+See the documentation for other :class:`Plot` methods to learn about the many
+ways that a plot can be enhanced and customized.
+
+"""
+from seaborn._core.plot import Plot # noqa: F401
+
+from seaborn._marks.base import Mark # noqa: F401
+from seaborn._marks.area import Area, Band # noqa: F401
+from seaborn._marks.bar import Bar, Bars # noqa: F401
+from seaborn._marks.dot import Dot, Dots # noqa: F401
+from seaborn._marks.line import Dash, Line, Lines, Path, Paths, Range # noqa: F401
+from seaborn._marks.text import Text # noqa: F401
+
+from seaborn._stats.base import Stat # noqa: F401
+from seaborn._stats.aggregation import Agg, Est # noqa: F401
+from seaborn._stats.counting import Count, Hist # noqa: F401
+from seaborn._stats.density import KDE # noqa: F401
+from seaborn._stats.order import Perc # noqa: F401
+from seaborn._stats.regression import PolyFit # noqa: F401
+
+from seaborn._core.moves import Dodge, Jitter, Norm, Shift, Stack, Move # noqa: F401
+
+from seaborn._core.scales import ( # noqa: F401
+ Boolean, Continuous, Nominal, Temporal, Scale
+)
diff --git a/seaborn/palettes.py b/seaborn/palettes.py
new file mode 100644
index 0000000000000000000000000000000000000000..f7f4298436f6fd83d819f314c0ad2cbe0db4b257
--- /dev/null
+++ b/seaborn/palettes.py
@@ -0,0 +1,841 @@
+import colorsys
+from itertools import cycle
+
+import numpy as np
+import matplotlib as mpl
+
+from .external import husl
+
+from .utils import desaturate, get_color_cycle
+from .colors import xkcd_rgb, crayons
+from ._compat import get_colormap
+
+
+__all__ = ["color_palette", "hls_palette", "husl_palette", "mpl_palette",
+ "dark_palette", "light_palette", "diverging_palette",
+ "blend_palette", "xkcd_palette", "crayon_palette",
+ "cubehelix_palette", "set_color_codes"]
+
+
+SEABORN_PALETTES = dict(
+ deep=["#4C72B0", "#DD8452", "#55A868", "#C44E52", "#8172B3",
+ "#937860", "#DA8BC3", "#8C8C8C", "#CCB974", "#64B5CD"],
+ deep6=["#4C72B0", "#55A868", "#C44E52",
+ "#8172B3", "#CCB974", "#64B5CD"],
+ muted=["#4878D0", "#EE854A", "#6ACC64", "#D65F5F", "#956CB4",
+ "#8C613C", "#DC7EC0", "#797979", "#D5BB67", "#82C6E2"],
+ muted6=["#4878D0", "#6ACC64", "#D65F5F",
+ "#956CB4", "#D5BB67", "#82C6E2"],
+ pastel=["#A1C9F4", "#FFB482", "#8DE5A1", "#FF9F9B", "#D0BBFF",
+ "#DEBB9B", "#FAB0E4", "#CFCFCF", "#FFFEA3", "#B9F2F0"],
+ pastel6=["#A1C9F4", "#8DE5A1", "#FF9F9B",
+ "#D0BBFF", "#FFFEA3", "#B9F2F0"],
+ bright=["#023EFF", "#FF7C00", "#1AC938", "#E8000B", "#8B2BE2",
+ "#9F4800", "#F14CC1", "#A3A3A3", "#FFC400", "#00D7FF"],
+ bright6=["#023EFF", "#1AC938", "#E8000B",
+ "#8B2BE2", "#FFC400", "#00D7FF"],
+ dark=["#001C7F", "#B1400D", "#12711C", "#8C0800", "#591E71",
+ "#592F0D", "#A23582", "#3C3C3C", "#B8850A", "#006374"],
+ dark6=["#001C7F", "#12711C", "#8C0800",
+ "#591E71", "#B8850A", "#006374"],
+ colorblind=["#0173B2", "#DE8F05", "#029E73", "#D55E00", "#CC78BC",
+ "#CA9161", "#FBAFE4", "#949494", "#ECE133", "#56B4E9"],
+ colorblind6=["#0173B2", "#029E73", "#D55E00",
+ "#CC78BC", "#ECE133", "#56B4E9"]
+)
+
+
+MPL_QUAL_PALS = {
+ "tab10": 10, "tab20": 20, "tab20b": 20, "tab20c": 20,
+ "Set1": 9, "Set2": 8, "Set3": 12,
+ "Accent": 8, "Paired": 12,
+ "Pastel1": 9, "Pastel2": 8, "Dark2": 8,
+}
+
+
+QUAL_PALETTE_SIZES = MPL_QUAL_PALS.copy()
+QUAL_PALETTE_SIZES.update({k: len(v) for k, v in SEABORN_PALETTES.items()})
+QUAL_PALETTES = list(QUAL_PALETTE_SIZES.keys())
+
+
+class _ColorPalette(list):
+ """Set the color palette in a with statement, otherwise be a list."""
+ def __enter__(self):
+ """Open the context."""
+ from .rcmod import set_palette
+ self._orig_palette = color_palette()
+ set_palette(self)
+ return self
+
+ def __exit__(self, *args):
+ """Close the context."""
+ from .rcmod import set_palette
+ set_palette(self._orig_palette)
+
+ def as_hex(self):
+ """Return a color palette with hex codes instead of RGB values."""
+ hex = [mpl.colors.rgb2hex(rgb) for rgb in self]
+ return _ColorPalette(hex)
+
+ def _repr_html_(self):
+ """Rich display of the color palette in an HTML frontend."""
+ s = 55
+ n = len(self)
+ html = f'<svg width="{n * s}" height="{s}">'
+ for i, c in enumerate(self.as_hex()):
+ html += (
+ f'<rect x="{i * s}" y="0" width="{s}" height="{s}" style="fill:{c};'
+ 'stroke-width:2;stroke:rgb(255,255,255)"/>'
+ )
+ html += '</svg>'
+ return html
+
+
+def _patch_colormap_display():
+ """Simplify the rich display of matplotlib color maps in a notebook."""
+ def _repr_png_(self):
+ """Generate a PNG representation of the Colormap."""
+ import io
+ from PIL import Image
+ import numpy as np
+ IMAGE_SIZE = (400, 50)
+ X = np.tile(np.linspace(0, 1, IMAGE_SIZE[0]), (IMAGE_SIZE[1], 1))
+ pixels = self(X, bytes=True)
+ png_bytes = io.BytesIO()
+ Image.fromarray(pixels).save(png_bytes, format='png')
+ return png_bytes.getvalue()
+
+ def _repr_html_(self):
+ """Generate an HTML representation of the Colormap."""
+ import base64
+ png_bytes = self._repr_png_()
+ png_base64 = base64.b64encode(png_bytes).decode('ascii')
+ return ('<img '
+ + 'alt="' + self.name + ' color map" '
+ + 'title="' + self.name + '"'
+ + 'src="data:image/png;base64,' + png_base64 + '">')
+
+ mpl.colors.Colormap._repr_png_ = _repr_png_
+ mpl.colors.Colormap._repr_html_ = _repr_html_
+
+
+def color_palette(palette=None, n_colors=None, desat=None, as_cmap=False):
+ """Return a list of colors or continuous colormap defining a palette.
+
+ Possible ``palette`` values include:
+ - Name of a seaborn palette (deep, muted, bright, pastel, dark, colorblind)
+ - Name of matplotlib colormap
+ - 'husl' or 'hls'
+ - 'ch:<cubehelix arguments>'
+ - 'light:<color>', 'dark:<color>', 'blend:<color>,<color>',
+ - A sequence of colors in any format matplotlib accepts
+
+ Calling this function with ``palette=None`` will return the current
+ matplotlib color cycle.
+
+ This function can also be used in a ``with`` statement to temporarily
+ set the color cycle for a plot or set of plots.
+
+ See the :ref:`tutorial <palette_tutorial>` for more information.
+
+ Parameters
+ ----------
+ palette : None, string, or sequence, optional
+ Name of palette or None to return current palette. If a sequence, input
+ colors are used but possibly cycled and desaturated.
+ n_colors : int, optional
+ Number of colors in the palette. If ``None``, the default will depend
+ on how ``palette`` is specified. Named palettes default to 6 colors,
+ but grabbing the current palette or passing in a list of colors will
+ not change the number of colors unless this is specified. Asking for
+ more colors than exist in the palette will cause it to cycle. Ignored
+ when ``as_cmap`` is True.
+ desat : float, optional
+ Proportion to desaturate each color by.
+ as_cmap : bool
+ If True, return a :class:`matplotlib.colors.ListedColormap`.
+
+ Returns
+ -------
+ list of RGB tuples or :class:`matplotlib.colors.ListedColormap`
+
+ See Also
+ --------
+ set_palette : Set the default color cycle for all plots.
+ set_color_codes : Reassign color codes like ``"b"``, ``"g"``, etc. to
+ colors from one of the seaborn palettes.
+
+ Examples
+ --------
+
+ .. include:: ../docstrings/color_palette.rst
+
+ """
+ if palette is None:
+ palette = get_color_cycle()
+ if n_colors is None:
+ n_colors = len(palette)
+
+ elif not isinstance(palette, str):
+ palette = palette
+ if n_colors is None:
+ n_colors = len(palette)
+ else:
+
+ if n_colors is None:
+ # Use all colors in a qualitative palette or 6 of another kind
+ n_colors = QUAL_PALETTE_SIZES.get(palette, 6)
+
+ if palette in SEABORN_PALETTES:
+ # Named "seaborn variant" of matplotlib default color cycle
+ palette = SEABORN_PALETTES[palette]
+
+ elif palette == "hls":
+ # Evenly spaced colors in cylindrical RGB space
+ palette = hls_palette(n_colors, as_cmap=as_cmap)
+
+ elif palette == "husl":
+ # Evenly spaced colors in cylindrical Lab space
+ palette = husl_palette(n_colors, as_cmap=as_cmap)
+
+ elif palette.lower() == "jet":
+ # Paternalism
+ raise ValueError("No.")
+
+ elif palette.startswith("ch:"):
+ # Cubehelix palette with params specified in string
+ args, kwargs = _parse_cubehelix_args(palette)
+ palette = cubehelix_palette(n_colors, *args, **kwargs, as_cmap=as_cmap)
+
+ elif palette.startswith("light:"):
+ # light palette to color specified in string
+ _, color = palette.split(":")
+ reverse = color.endswith("_r")
+ if reverse:
+ color = color[:-2]
+ palette = light_palette(color, n_colors, reverse=reverse, as_cmap=as_cmap)
+
+ elif palette.startswith("dark:"):
+ # light palette to color specified in string
+ _, color = palette.split(":")
+ reverse = color.endswith("_r")
+ if reverse:
+ color = color[:-2]
+ palette = dark_palette(color, n_colors, reverse=reverse, as_cmap=as_cmap)
+
+ elif palette.startswith("blend:"):
+ # blend palette between colors specified in string
+ _, colors = palette.split(":")
+ colors = colors.split(",")
+ palette = blend_palette(colors, n_colors, as_cmap=as_cmap)
+
+ else:
+ try:
+ # Perhaps a named matplotlib colormap?
+ palette = mpl_palette(palette, n_colors, as_cmap=as_cmap)
+ except (ValueError, KeyError): # Error class changed in mpl36
+ raise ValueError(f"{palette!r} is not a valid palette name")
+
+ if desat is not None:
+ palette = [desaturate(c, desat) for c in palette]
+
+ if not as_cmap:
+
+ # Always return as many colors as we asked for
+ pal_cycle = cycle(palette)
+ palette = [next(pal_cycle) for _ in range(n_colors)]
+
+ # Always return in r, g, b tuple format
+ try:
+ palette = map(mpl.colors.colorConverter.to_rgb, palette)
+ palette = _ColorPalette(palette)
+ except ValueError:
+ raise ValueError(f"Could not generate a palette for {palette}")
+
+ return palette
+
+
+def hls_palette(n_colors=6, h=.01, l=.6, s=.65, as_cmap=False): # noqa
+ """
+ Return hues with constant lightness and saturation in the HLS system.
+
+ The hues are evenly sampled along a circular path. The resulting palette will be
+ appropriate for categorical or cyclical data.
+
+ The `h`, `l`, and `s` values should be between 0 and 1.
+
+ .. note::
+ While the separation of the resulting colors will be mathematically
+ constant, the HLS system does not construct a perceptually-uniform space,
+ so their apparent intensity will vary.
+
+ Parameters
+ ----------
+ n_colors : int
+ Number of colors in the palette.
+ h : float
+ The value of the first hue.
+ l : float
+ The lightness value.
+ s : float
+ The saturation intensity.
+ as_cmap : bool
+ If True, return a matplotlib colormap object.
+
+ Returns
+ -------
+ palette
+ list of RGB tuples or :class:`matplotlib.colors.ListedColormap`
+
+ See Also
+ --------
+ husl_palette : Make a palette using evenly spaced hues in the HUSL system.
+
+ Examples
+ --------
+ .. include:: ../docstrings/hls_palette.rst
+
+ """
+ if as_cmap:
+ n_colors = 256
+ hues = np.linspace(0, 1, int(n_colors) + 1)[:-1]
+ hues += h
+ hues %= 1
+ hues -= hues.astype(int)
+ palette = [colorsys.hls_to_rgb(h_i, l, s) for h_i in hues]
+ if as_cmap:
+ return mpl.colors.ListedColormap(palette, "hls")
+ else:
+ return _ColorPalette(palette)
+
+
+def husl_palette(n_colors=6, h=.01, s=.9, l=.65, as_cmap=False): # noqa
+ """
+ Return hues with constant lightness and saturation in the HUSL system.
+
+ The hues are evenly sampled along a circular path. The resulting palette will be
+ appropriate for categorical or cyclical data.
+
+ The `h`, `l`, and `s` values should be between 0 and 1.
+
+ This function is similar to :func:`hls_palette`, but it uses a nonlinear color
+ space that is more perceptually uniform.
+
+ Parameters
+ ----------
+ n_colors : int
+ Number of colors in the palette.
+ h : float
+ The value of the first hue.
+ l : float
+ The lightness value.
+ s : float
+ The saturation intensity.
+ as_cmap : bool
+ If True, return a matplotlib colormap object.
+
+ Returns
+ -------
+ palette
+ list of RGB tuples or :class:`matplotlib.colors.ListedColormap`
+
+ See Also
+ --------
+ hls_palette : Make a palette using evenly spaced hues in the HSL system.
+
+ Examples
+ --------
+ .. include:: ../docstrings/husl_palette.rst
+
+ """
+ if as_cmap:
+ n_colors = 256
+ hues = np.linspace(0, 1, int(n_colors) + 1)[:-1]
+ hues += h
+ hues %= 1
+ hues *= 359
+ s *= 99
+ l *= 99 # noqa
+ palette = [_color_to_rgb((h_i, s, l), input="husl") for h_i in hues]
+ if as_cmap:
+ return mpl.colors.ListedColormap(palette, "hsl")
+ else:
+ return _ColorPalette(palette)
+
+
+def mpl_palette(name, n_colors=6, as_cmap=False):
+ """
+ Return a palette or colormap from the matplotlib registry.
+
+ For continuous palettes, evenly-spaced discrete samples are chosen while
+ excluding the minimum and maximum value in the colormap to provide better
+ contrast at the extremes.
+
+ For qualitative palettes (e.g. those from colorbrewer), exact values are
+ indexed (rather than interpolated), but fewer than `n_colors` can be returned
+ if the palette does not define that many.
+
+ Parameters
+ ----------
+ name : string
+ Name of the palette. This should be a named matplotlib colormap.
+ n_colors : int
+ Number of discrete colors in the palette.
+
+ Returns
+ -------
+ list of RGB tuples or :class:`matplotlib.colors.ListedColormap`
+
+ Examples
+ --------
+ .. include:: ../docstrings/mpl_palette.rst
+
+ """
+ if name.endswith("_d"):
+ sub_name = name[:-2]
+ if sub_name.endswith("_r"):
+ reverse = True
+ sub_name = sub_name[:-2]
+ else:
+ reverse = False
+ pal = color_palette(sub_name, 2) + ["#333333"]
+ if reverse:
+ pal = pal[::-1]
+ cmap = blend_palette(pal, n_colors, as_cmap=True)
+ else:
+ cmap = get_colormap(name)
+
+ if name in MPL_QUAL_PALS:
+ bins = np.linspace(0, 1, MPL_QUAL_PALS[name])[:n_colors]
+ else:
+ bins = np.linspace(0, 1, int(n_colors) + 2)[1:-1]
+ palette = list(map(tuple, cmap(bins)[:, :3]))
+
+ if as_cmap:
+ return cmap
+ else:
+ return _ColorPalette(palette)
+
+
+def _color_to_rgb(color, input):
+ """Add some more flexibility to color choices."""
+ if input == "hls":
+ color = colorsys.hls_to_rgb(*color)
+ elif input == "husl":
+ color = husl.husl_to_rgb(*color)
+ color = tuple(np.clip(color, 0, 1))
+ elif input == "xkcd":
+ color = xkcd_rgb[color]
+
+ return mpl.colors.to_rgb(color)
+
+
+def dark_palette(color, n_colors=6, reverse=False, as_cmap=False, input="rgb"):
+ """Make a sequential palette that blends from dark to ``color``.
+
+ This kind of palette is good for data that range between relatively
+ uninteresting low values and interesting high values.
+
+ The ``color`` parameter can be specified in a number of ways, including
+ all options for defining a color in matplotlib and several additional
+ color spaces that are handled by seaborn. You can also use the database
+ of named colors from the XKCD color survey.
+
+ If you are using the IPython notebook, you can also choose this palette
+ interactively with the :func:`choose_dark_palette` function.
+
+ Parameters
+ ----------
+ color : base color for high values
+ hex, rgb-tuple, or html color name
+ n_colors : int, optional
+ number of colors in the palette
+ reverse : bool, optional
+ if True, reverse the direction of the blend
+ as_cmap : bool, optional
+ If True, return a :class:`matplotlib.colors.ListedColormap`.
+ input : {'rgb', 'hls', 'husl', xkcd'}
+ Color space to interpret the input color. The first three options
+ apply to tuple inputs and the latter applies to string inputs.
+
+ Returns
+ -------
+ palette
+ list of RGB tuples or :class:`matplotlib.colors.ListedColormap`
+
+ See Also
+ --------
+ light_palette : Create a sequential palette with bright low values.
+ diverging_palette : Create a diverging palette with two colors.
+
+ Examples
+ --------
+ .. include:: ../docstrings/dark_palette.rst
+
+ """
+ rgb = _color_to_rgb(color, input)
+ hue, sat, _ = husl.rgb_to_husl(*rgb)
+ gray_s, gray_l = .15 * sat, 15
+ gray = _color_to_rgb((hue, gray_s, gray_l), input="husl")
+ colors = [rgb, gray] if reverse else [gray, rgb]
+ return blend_palette(colors, n_colors, as_cmap)
+
+
+def light_palette(color, n_colors=6, reverse=False, as_cmap=False, input="rgb"):
+ """Make a sequential palette that blends from light to ``color``.
+
+ The ``color`` parameter can be specified in a number of ways, including
+ all options for defining a color in matplotlib and several additional
+ color spaces that are handled by seaborn. You can also use the database
+ of named colors from the XKCD color survey.
+
+ If you are using a Jupyter notebook, you can also choose this palette
+ interactively with the :func:`choose_light_palette` function.
+
+ Parameters
+ ----------
+ color : base color for high values
+ hex code, html color name, or tuple in `input` space.
+ n_colors : int, optional
+ number of colors in the palette
+ reverse : bool, optional
+ if True, reverse the direction of the blend
+ as_cmap : bool, optional
+ If True, return a :class:`matplotlib.colors.ListedColormap`.
+ input : {'rgb', 'hls', 'husl', xkcd'}
+ Color space to interpret the input color. The first three options
+ apply to tuple inputs and the latter applies to string inputs.
+
+ Returns
+ -------
+ palette
+ list of RGB tuples or :class:`matplotlib.colors.ListedColormap`
+
+ See Also
+ --------
+ dark_palette : Create a sequential palette with dark low values.
+ diverging_palette : Create a diverging palette with two colors.
+
+ Examples
+ --------
+ .. include:: ../docstrings/light_palette.rst
+
+ """
+ rgb = _color_to_rgb(color, input)
+ hue, sat, _ = husl.rgb_to_husl(*rgb)
+ gray_s, gray_l = .15 * sat, 95
+ gray = _color_to_rgb((hue, gray_s, gray_l), input="husl")
+ colors = [rgb, gray] if reverse else [gray, rgb]
+ return blend_palette(colors, n_colors, as_cmap)
+
+
+def diverging_palette(h_neg, h_pos, s=75, l=50, sep=1, n=6, # noqa
+ center="light", as_cmap=False):
+ """Make a diverging palette between two HUSL colors.
+
+ If you are using the IPython notebook, you can also choose this palette
+ interactively with the :func:`choose_diverging_palette` function.
+
+ Parameters
+ ----------
+ h_neg, h_pos : float in [0, 359]
+ Anchor hues for negative and positive extents of the map.
+ s : float in [0, 100], optional
+ Anchor saturation for both extents of the map.
+ l : float in [0, 100], optional
+ Anchor lightness for both extents of the map.
+ sep : int, optional
+ Size of the intermediate region.
+ n : int, optional
+ Number of colors in the palette (if not returning a cmap)
+ center : {"light", "dark"}, optional
+ Whether the center of the palette is light or dark
+ as_cmap : bool, optional
+ If True, return a :class:`matplotlib.colors.ListedColormap`.
+
+ Returns
+ -------
+ palette
+ list of RGB tuples or :class:`matplotlib.colors.ListedColormap`
+
+ See Also
+ --------
+ dark_palette : Create a sequential palette with dark values.
+ light_palette : Create a sequential palette with light values.
+
+ Examples
+ --------
+ .. include: ../docstrings/diverging_palette.rst
+
+ """
+ palfunc = dict(dark=dark_palette, light=light_palette)[center]
+ n_half = int(128 - (sep // 2))
+ neg = palfunc((h_neg, s, l), n_half, reverse=True, input="husl")
+ pos = palfunc((h_pos, s, l), n_half, input="husl")
+ midpoint = dict(light=[(.95, .95, .95)], dark=[(.133, .133, .133)])[center]
+ mid = midpoint * sep
+ pal = blend_palette(np.concatenate([neg, mid, pos]), n, as_cmap=as_cmap)
+ return pal
+
+
+def blend_palette(colors, n_colors=6, as_cmap=False, input="rgb"):
+ """Make a palette that blends between a list of colors.
+
+ Parameters
+ ----------
+ colors : sequence of colors in various formats interpreted by `input`
+ hex code, html color name, or tuple in `input` space.
+ n_colors : int, optional
+ Number of colors in the palette.
+ as_cmap : bool, optional
+ If True, return a :class:`matplotlib.colors.ListedColormap`.
+
+ Returns
+ -------
+ palette
+ list of RGB tuples or :class:`matplotlib.colors.ListedColormap`
+
+ Examples
+ --------
+ .. include: ../docstrings/blend_palette.rst
+
+ """
+ colors = [_color_to_rgb(color, input) for color in colors]
+ name = "blend"
+ pal = mpl.colors.LinearSegmentedColormap.from_list(name, colors)
+ if not as_cmap:
+ rgb_array = pal(np.linspace(0, 1, int(n_colors)))[:, :3] # no alpha
+ pal = _ColorPalette(map(tuple, rgb_array))
+ return pal
+
+
+def xkcd_palette(colors):
+ """Make a palette with color names from the xkcd color survey.
+
+ See xkcd for the full list of colors: https://xkcd.com/color/rgb/
+
+ This is just a simple wrapper around the `seaborn.xkcd_rgb` dictionary.
+
+ Parameters
+ ----------
+ colors : list of strings
+ List of keys in the `seaborn.xkcd_rgb` dictionary.
+
+ Returns
+ -------
+ palette
+ A list of colors as RGB tuples.
+
+ See Also
+ --------
+ crayon_palette : Make a palette with Crayola crayon colors.
+
+ """
+ palette = [xkcd_rgb[name] for name in colors]
+ return color_palette(palette, len(palette))
+
+
+def crayon_palette(colors):
+ """Make a palette with color names from Crayola crayons.
+
+ Colors are taken from here:
+ https://en.wikipedia.org/wiki/List_of_Crayola_crayon_colors
+
+ This is just a simple wrapper around the `seaborn.crayons` dictionary.
+
+ Parameters
+ ----------
+ colors : list of strings
+ List of keys in the `seaborn.crayons` dictionary.
+
+ Returns
+ -------
+ palette
+ A list of colors as RGB tuples.
+
+ See Also
+ --------
+ xkcd_palette : Make a palette with named colors from the XKCD color survey.
+
+ """
+ palette = [crayons[name] for name in colors]
+ return color_palette(palette, len(palette))
+
+
+def cubehelix_palette(n_colors=6, start=0, rot=.4, gamma=1.0, hue=0.8,
+ light=.85, dark=.15, reverse=False, as_cmap=False):
+ """Make a sequential palette from the cubehelix system.
+
+ This produces a colormap with linearly-decreasing (or increasing)
+ brightness. That means that information will be preserved if printed to
+ black and white or viewed by someone who is colorblind. "cubehelix" is
+ also available as a matplotlib-based palette, but this function gives the
+ user more control over the look of the palette and has a different set of
+ defaults.
+
+ In addition to using this function, it is also possible to generate a
+ cubehelix palette generally in seaborn using a string starting with
+ `ch:` and containing other parameters (e.g. `"ch:s=.25,r=-.5"`).
+
+ Parameters
+ ----------
+ n_colors : int
+ Number of colors in the palette.
+ start : float, 0 <= start <= 3
+ The hue value at the start of the helix.
+ rot : float
+ Rotations around the hue wheel over the range of the palette.
+ gamma : float 0 <= gamma
+ Nonlinearity to emphasize dark (gamma < 1) or light (gamma > 1) colors.
+ hue : float, 0 <= hue <= 1
+ Saturation of the colors.
+ dark : float 0 <= dark <= 1
+ Intensity of the darkest color in the palette.
+ light : float 0 <= light <= 1
+ Intensity of the lightest color in the palette.
+ reverse : bool
+ If True, the palette will go from dark to light.
+ as_cmap : bool
+ If True, return a :class:`matplotlib.colors.ListedColormap`.
+
+ Returns
+ -------
+ palette
+ list of RGB tuples or :class:`matplotlib.colors.ListedColormap`
+
+ See Also
+ --------
+ choose_cubehelix_palette : Launch an interactive widget to select cubehelix
+ palette parameters.
+ dark_palette : Create a sequential palette with dark low values.
+ light_palette : Create a sequential palette with bright low values.
+
+ References
+ ----------
+ Green, D. A. (2011). "A colour scheme for the display of astronomical
+ intensity images". Bulletin of the Astromical Society of India, Vol. 39,
+ p. 289-295.
+
+ Examples
+ --------
+ .. include:: ../docstrings/cubehelix_palette.rst
+
+ """
+ def get_color_function(p0, p1):
+ # Copied from matplotlib because it lives in private module
+ def color(x):
+ # Apply gamma factor to emphasise low or high intensity values
+ xg = x ** gamma
+
+ # Calculate amplitude and angle of deviation from the black
+ # to white diagonal in the plane of constant
+ # perceived intensity.
+ a = hue * xg * (1 - xg) / 2
+
+ phi = 2 * np.pi * (start / 3 + rot * x)
+
+ return xg + a * (p0 * np.cos(phi) + p1 * np.sin(phi))
+ return color
+
+ cdict = {
+ "red": get_color_function(-0.14861, 1.78277),
+ "green": get_color_function(-0.29227, -0.90649),
+ "blue": get_color_function(1.97294, 0.0),
+ }
+
+ cmap = mpl.colors.LinearSegmentedColormap("cubehelix", cdict)
+
+ x = np.linspace(light, dark, int(n_colors))
+ pal = cmap(x)[:, :3].tolist()
+ if reverse:
+ pal = pal[::-1]
+
+ if as_cmap:
+ x_256 = np.linspace(light, dark, 256)
+ if reverse:
+ x_256 = x_256[::-1]
+ pal_256 = cmap(x_256)
+ cmap = mpl.colors.ListedColormap(pal_256, "seaborn_cubehelix")
+ return cmap
+ else:
+ return _ColorPalette(pal)
+
+
+def _parse_cubehelix_args(argstr):
+ """Turn stringified cubehelix params into args/kwargs."""
+
+ if argstr.startswith("ch:"):
+ argstr = argstr[3:]
+
+ if argstr.endswith("_r"):
+ reverse = True
+ argstr = argstr[:-2]
+ else:
+ reverse = False
+
+ if not argstr:
+ return [], {"reverse": reverse}
+
+ all_args = argstr.split(",")
+
+ args = [float(a.strip(" ")) for a in all_args if "=" not in a]
+
+ kwargs = [a.split("=") for a in all_args if "=" in a]
+ kwargs = {k.strip(" "): float(v.strip(" ")) for k, v in kwargs}
+
+ kwarg_map = dict(
+ s="start", r="rot", g="gamma",
+ h="hue", l="light", d="dark", # noqa: E741
+ )
+
+ kwargs = {kwarg_map.get(k, k): v for k, v in kwargs.items()}
+
+ if reverse:
+ kwargs["reverse"] = True
+
+ return args, kwargs
+
+
+def set_color_codes(palette="deep"):
+ """Change how matplotlib color shorthands are interpreted.
+
+ Calling this will change how shorthand codes like "b" or "g"
+ are interpreted by matplotlib in subsequent plots.
+
+ Parameters
+ ----------
+ palette : {deep, muted, pastel, dark, bright, colorblind}
+ Named seaborn palette to use as the source of colors.
+
+ See Also
+ --------
+ set : Color codes can be set through the high-level seaborn style
+ manager.
+ set_palette : Color codes can also be set through the function that
+ sets the matplotlib color cycle.
+
+ """
+ if palette == "reset":
+ colors = [
+ (0., 0., 1.),
+ (0., .5, 0.),
+ (1., 0., 0.),
+ (.75, 0., .75),
+ (.75, .75, 0.),
+ (0., .75, .75),
+ (0., 0., 0.)
+ ]
+ elif not isinstance(palette, str):
+ err = "set_color_codes requires a named seaborn palette"
+ raise TypeError(err)
+ elif palette in SEABORN_PALETTES:
+ if not palette.endswith("6"):
+ palette = palette + "6"
+ colors = SEABORN_PALETTES[palette] + [(.1, .1, .1)]
+ else:
+ err = f"Cannot set colors with palette '{palette}'"
+ raise ValueError(err)
+
+ for code, color in zip("bgrmyck", colors):
+ rgb = mpl.colors.colorConverter.to_rgb(color)
+ mpl.colors.colorConverter.colors[code] = rgb
diff --git a/seaborn/rcmod.py b/seaborn/rcmod.py
new file mode 100644
index 0000000000000000000000000000000000000000..de238323147e393bfee469e1bc3fafec157cb28f
--- /dev/null
+++ b/seaborn/rcmod.py
@@ -0,0 +1,533 @@
+"""Control plot style and scaling using the matplotlib rcParams interface."""
+import functools
+import matplotlib as mpl
+from cycler import cycler
+from . import palettes
+
+
+__all__ = ["set_theme", "set", "reset_defaults", "reset_orig",
+ "axes_style", "set_style", "plotting_context", "set_context",
+ "set_palette"]
+
+
+_style_keys = [
+
+ "axes.facecolor",
+ "axes.edgecolor",
+ "axes.grid",
+ "axes.axisbelow",
+ "axes.labelcolor",
+
+ "figure.facecolor",
+
+ "grid.color",
+ "grid.linestyle",
+
+ "text.color",
+
+ "xtick.color",
+ "ytick.color",
+ "xtick.direction",
+ "ytick.direction",
+ "lines.solid_capstyle",
+
+ "patch.edgecolor",
+ "patch.force_edgecolor",
+
+ "image.cmap",
+ "font.family",
+ "font.sans-serif",
+
+ "xtick.bottom",
+ "xtick.top",
+ "ytick.left",
+ "ytick.right",
+
+ "axes.spines.left",
+ "axes.spines.bottom",
+ "axes.spines.right",
+ "axes.spines.top",
+
+]
+
+_context_keys = [
+
+ "font.size",
+ "axes.labelsize",
+ "axes.titlesize",
+ "xtick.labelsize",
+ "ytick.labelsize",
+ "legend.fontsize",
+ "legend.title_fontsize",
+
+ "axes.linewidth",
+ "grid.linewidth",
+ "lines.linewidth",
+ "lines.markersize",
+ "patch.linewidth",
+
+ "xtick.major.width",
+ "ytick.major.width",
+ "xtick.minor.width",
+ "ytick.minor.width",
+
+ "xtick.major.size",
+ "ytick.major.size",
+ "xtick.minor.size",
+ "ytick.minor.size",
+
+]
+
+
+def set_theme(context="notebook", style="darkgrid", palette="deep",
+ font="sans-serif", font_scale=1, color_codes=True, rc=None):
+ """
+ Set aspects of the visual theme for all matplotlib and seaborn plots.
+
+ This function changes the global defaults for all plots using the
+ matplotlib rcParams system. The themeing is decomposed into several distinct
+ sets of parameter values.
+
+ The options are illustrated in the :doc:`aesthetics <../tutorial/aesthetics>`
+ and :doc:`color palette <../tutorial/color_palettes>` tutorials.
+
+ Parameters
+ ----------
+ context : string or dict
+ Scaling parameters, see :func:`plotting_context`.
+ style : string or dict
+ Axes style parameters, see :func:`axes_style`.
+ palette : string or sequence
+ Color palette, see :func:`color_palette`.
+ font : string
+ Font family, see matplotlib font manager.
+ font_scale : float, optional
+ Separate scaling factor to independently scale the size of the
+ font elements.
+ color_codes : bool
+ If ``True`` and ``palette`` is a seaborn palette, remap the shorthand
+ color codes (e.g. "b", "g", "r", etc.) to the colors from this palette.
+ rc : dict or None
+ Dictionary of rc parameter mappings to override the above.
+
+ Examples
+ --------
+
+ .. include:: ../docstrings/set_theme.rst
+
+ """
+ set_context(context, font_scale)
+ set_style(style, rc={"font.family": font})
+ set_palette(palette, color_codes=color_codes)
+ if rc is not None:
+ mpl.rcParams.update(rc)
+
+
+def set(*args, **kwargs):
+ """
+ Alias for :func:`set_theme`, which is the preferred interface.
+
+ This function may be removed in the future.
+ """
+ set_theme(*args, **kwargs)
+
+
+def reset_defaults():
+ """Restore all RC params to default settings."""
+ mpl.rcParams.update(mpl.rcParamsDefault)
+
+
+def reset_orig():
+ """Restore all RC params to original settings (respects custom rc)."""
+ from . import _orig_rc_params
+ mpl.rcParams.update(_orig_rc_params)
+
+
+def axes_style(style=None, rc=None):
+ """
+ Get the parameters that control the general style of the plots.
+
+ The style parameters control properties like the color of the background and
+ whether a grid is enabled by default. This is accomplished using the
+ matplotlib rcParams system.
+
+ The options are illustrated in the
+ :doc:`aesthetics tutorial <../tutorial/aesthetics>`.
+
+ This function can also be used as a context manager to temporarily
+ alter the global defaults. See :func:`set_theme` or :func:`set_style`
+ to modify the global defaults for all plots.
+
+ Parameters
+ ----------
+ style : None, dict, or one of {darkgrid, whitegrid, dark, white, ticks}
+ A dictionary of parameters or the name of a preconfigured style.
+ rc : dict, optional
+ Parameter mappings to override the values in the preset seaborn
+ style dictionaries. This only updates parameters that are
+ considered part of the style definition.
+
+ Examples
+ --------
+
+ .. include:: ../docstrings/axes_style.rst
+
+ """
+ if style is None:
+ style_dict = {k: mpl.rcParams[k] for k in _style_keys}
+
+ elif isinstance(style, dict):
+ style_dict = style
+
+ else:
+ styles = ["white", "dark", "whitegrid", "darkgrid", "ticks"]
+ if style not in styles:
+ raise ValueError(f"style must be one of {', '.join(styles)}")
+
+ # Define colors here
+ dark_gray = ".15"
+ light_gray = ".8"
+
+ # Common parameters
+ style_dict = {
+
+ "figure.facecolor": "white",
+ "axes.labelcolor": dark_gray,
+
+ "xtick.direction": "out",
+ "ytick.direction": "out",
+ "xtick.color": dark_gray,
+ "ytick.color": dark_gray,
+
+ "axes.axisbelow": True,
+ "grid.linestyle": "-",
+
+
+ "text.color": dark_gray,
+ "font.family": ["sans-serif"],
+ "font.sans-serif": ["Arial", "DejaVu Sans", "Liberation Sans",
+ "Bitstream Vera Sans", "sans-serif"],
+
+
+ "lines.solid_capstyle": "round",
+ "patch.edgecolor": "w",
+ "patch.force_edgecolor": True,
+
+ "image.cmap": "rocket",
+
+ "xtick.top": False,
+ "ytick.right": False,
+
+ }
+
+ # Set grid on or off
+ if "grid" in style:
+ style_dict.update({
+ "axes.grid": True,
+ })
+ else:
+ style_dict.update({
+ "axes.grid": False,
+ })
+
+ # Set the color of the background, spines, and grids
+ if style.startswith("dark"):
+ style_dict.update({
+
+ "axes.facecolor": "#EAEAF2",
+ "axes.edgecolor": "white",
+ "grid.color": "white",
+
+ "axes.spines.left": True,
+ "axes.spines.bottom": True,
+ "axes.spines.right": True,
+ "axes.spines.top": True,
+
+ })
+
+ elif style == "whitegrid":
+ style_dict.update({
+
+ "axes.facecolor": "white",
+ "axes.edgecolor": light_gray,
+ "grid.color": light_gray,
+
+ "axes.spines.left": True,
+ "axes.spines.bottom": True,
+ "axes.spines.right": True,
+ "axes.spines.top": True,
+
+ })
+
+ elif style in ["white", "ticks"]:
+ style_dict.update({
+
+ "axes.facecolor": "white",
+ "axes.edgecolor": dark_gray,
+ "grid.color": light_gray,
+
+ "axes.spines.left": True,
+ "axes.spines.bottom": True,
+ "axes.spines.right": True,
+ "axes.spines.top": True,
+
+ })
+
+ # Show or hide the axes ticks
+ if style == "ticks":
+ style_dict.update({
+ "xtick.bottom": True,
+ "ytick.left": True,
+ })
+ else:
+ style_dict.update({
+ "xtick.bottom": False,
+ "ytick.left": False,
+ })
+
+ # Remove entries that are not defined in the base list of valid keys
+ # This lets us handle matplotlib <=/> 2.0
+ style_dict = {k: v for k, v in style_dict.items() if k in _style_keys}
+
+ # Override these settings with the provided rc dictionary
+ if rc is not None:
+ rc = {k: v for k, v in rc.items() if k in _style_keys}
+ style_dict.update(rc)
+
+ # Wrap in an _AxesStyle object so this can be used in a with statement
+ style_object = _AxesStyle(style_dict)
+
+ return style_object
+
+
+def set_style(style=None, rc=None):
+ """
+ Set the parameters that control the general style of the plots.
+
+ The style parameters control properties like the color of the background and
+ whether a grid is enabled by default. This is accomplished using the
+ matplotlib rcParams system.
+
+ The options are illustrated in the
+ :doc:`aesthetics tutorial <../tutorial/aesthetics>`.
+
+ See :func:`axes_style` to get the parameter values.
+
+ Parameters
+ ----------
+ style : dict, or one of {darkgrid, whitegrid, dark, white, ticks}
+ A dictionary of parameters or the name of a preconfigured style.
+ rc : dict, optional
+ Parameter mappings to override the values in the preset seaborn
+ style dictionaries. This only updates parameters that are
+ considered part of the style definition.
+
+ Examples
+ --------
+
+ .. include:: ../docstrings/set_style.rst
+
+ """
+ style_object = axes_style(style, rc)
+ mpl.rcParams.update(style_object)
+
+
+def plotting_context(context=None, font_scale=1, rc=None):
+ """
+ Get the parameters that control the scaling of plot elements.
+
+ These parameters correspond to label size, line thickness, etc. For more
+ information, see the :doc:`aesthetics tutorial <../tutorial/aesthetics>`.
+
+ The base context is "notebook", and the other contexts are "paper", "talk",
+ and "poster", which are version of the notebook parameters scaled by different
+ values. Font elements can also be scaled independently of (but relative to)
+ the other values.
+
+ This function can also be used as a context manager to temporarily
+ alter the global defaults. See :func:`set_theme` or :func:`set_context`
+ to modify the global defaults for all plots.
+
+ Parameters
+ ----------
+ context : None, dict, or one of {paper, notebook, talk, poster}
+ A dictionary of parameters or the name of a preconfigured set.
+ font_scale : float, optional
+ Separate scaling factor to independently scale the size of the
+ font elements.
+ rc : dict, optional
+ Parameter mappings to override the values in the preset seaborn
+ context dictionaries. This only updates parameters that are
+ considered part of the context definition.
+
+ Examples
+ --------
+
+ .. include:: ../docstrings/plotting_context.rst
+
+ """
+ if context is None:
+ context_dict = {k: mpl.rcParams[k] for k in _context_keys}
+
+ elif isinstance(context, dict):
+ context_dict = context
+
+ else:
+
+ contexts = ["paper", "notebook", "talk", "poster"]
+ if context not in contexts:
+ raise ValueError(f"context must be in {', '.join(contexts)}")
+
+ # Set up dictionary of default parameters
+ texts_base_context = {
+
+ "font.size": 12,
+ "axes.labelsize": 12,
+ "axes.titlesize": 12,
+ "xtick.labelsize": 11,
+ "ytick.labelsize": 11,
+ "legend.fontsize": 11,
+ "legend.title_fontsize": 12,
+
+ }
+
+ base_context = {
+
+ "axes.linewidth": 1.25,
+ "grid.linewidth": 1,
+ "lines.linewidth": 1.5,
+ "lines.markersize": 6,
+ "patch.linewidth": 1,
+
+ "xtick.major.width": 1.25,
+ "ytick.major.width": 1.25,
+ "xtick.minor.width": 1,
+ "ytick.minor.width": 1,
+
+ "xtick.major.size": 6,
+ "ytick.major.size": 6,
+ "xtick.minor.size": 4,
+ "ytick.minor.size": 4,
+
+ }
+ base_context.update(texts_base_context)
+
+ # Scale all the parameters by the same factor depending on the context
+ scaling = dict(paper=.8, notebook=1, talk=1.5, poster=2)[context]
+ context_dict = {k: v * scaling for k, v in base_context.items()}
+
+ # Now independently scale the fonts
+ font_keys = texts_base_context.keys()
+ font_dict = {k: context_dict[k] * font_scale for k in font_keys}
+ context_dict.update(font_dict)
+
+ # Override these settings with the provided rc dictionary
+ if rc is not None:
+ rc = {k: v for k, v in rc.items() if k in _context_keys}
+ context_dict.update(rc)
+
+ # Wrap in a _PlottingContext object so this can be used in a with statement
+ context_object = _PlottingContext(context_dict)
+
+ return context_object
+
+
+def set_context(context=None, font_scale=1, rc=None):
+ """
+ Set the parameters that control the scaling of plot elements.
+
+ These parameters correspond to label size, line thickness, etc.
+ Calling this function modifies the global matplotlib `rcParams`. For more
+ information, see the :doc:`aesthetics tutorial <../tutorial/aesthetics>`.
+
+ The base context is "notebook", and the other contexts are "paper", "talk",
+ and "poster", which are version of the notebook parameters scaled by different
+ values. Font elements can also be scaled independently of (but relative to)
+ the other values.
+
+ See :func:`plotting_context` to get the parameter values.
+
+ Parameters
+ ----------
+ context : dict, or one of {paper, notebook, talk, poster}
+ A dictionary of parameters or the name of a preconfigured set.
+ font_scale : float, optional
+ Separate scaling factor to independently scale the size of the
+ font elements.
+ rc : dict, optional
+ Parameter mappings to override the values in the preset seaborn
+ context dictionaries. This only updates parameters that are
+ considered part of the context definition.
+
+ Examples
+ --------
+
+ .. include:: ../docstrings/set_context.rst
+
+ """
+ context_object = plotting_context(context, font_scale, rc)
+ mpl.rcParams.update(context_object)
+
+
+class _RCAesthetics(dict):
+ def __enter__(self):
+ rc = mpl.rcParams
+ self._orig = {k: rc[k] for k in self._keys}
+ self._set(self)
+
+ def __exit__(self, exc_type, exc_value, exc_tb):
+ self._set(self._orig)
+
+ def __call__(self, func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ with self:
+ return func(*args, **kwargs)
+ return wrapper
+
+
+class _AxesStyle(_RCAesthetics):
+ """Light wrapper on a dict to set style temporarily."""
+ _keys = _style_keys
+ _set = staticmethod(set_style)
+
+
+class _PlottingContext(_RCAesthetics):
+ """Light wrapper on a dict to set context temporarily."""
+ _keys = _context_keys
+ _set = staticmethod(set_context)
+
+
+def set_palette(palette, n_colors=None, desat=None, color_codes=False):
+ """Set the matplotlib color cycle using a seaborn palette.
+
+ Parameters
+ ----------
+ palette : seaborn color palette | matplotlib colormap | hls | husl
+ Palette definition. Should be something :func:`color_palette` can process.
+ n_colors : int
+ Number of colors in the cycle. The default number of colors will depend
+ on the format of ``palette``, see the :func:`color_palette`
+ documentation for more information.
+ desat : float
+ Proportion to desaturate each color by.
+ color_codes : bool
+ If ``True`` and ``palette`` is a seaborn palette, remap the shorthand
+ color codes (e.g. "b", "g", "r", etc.) to the colors from this palette.
+
+ See Also
+ --------
+ color_palette : build a color palette or set the color cycle temporarily
+ in a ``with`` statement.
+ set_context : set parameters to scale plot elements
+ set_style : set the default parameters for figure style
+
+ """
+ colors = palettes.color_palette(palette, n_colors, desat)
+ cyl = cycler('color', colors)
+ mpl.rcParams['axes.prop_cycle'] = cyl
+ if color_codes:
+ try:
+ palettes.set_color_codes(palette)
+ except (ValueError, TypeError):
+ pass
diff --git a/seaborn/regression.py b/seaborn/regression.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e5503a422820191a124fa9a08b6d1319faffd6f
--- /dev/null
+++ b/seaborn/regression.py
@@ -0,0 +1,940 @@
+"""Plotting functions for linear models (broadly construed)."""
+import copy
+from textwrap import dedent
+import warnings
+import numpy as np
+import pandas as pd
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+
+try:
+ import statsmodels
+ assert statsmodels
+ _has_statsmodels = True
+except ImportError:
+ _has_statsmodels = False
+
+from . import utils
+from . import algorithms as algo
+from .axisgrid import FacetGrid, _facet_docs
+
+
+__all__ = ["lmplot", "regplot", "residplot"]
+
+
+class _LinearPlotter:
+ """Base class for plotting relational data in tidy format.
+
+ To get anything useful done you'll have to inherit from this, but setup
+ code that can be abstracted out should be put here.
+
+ """
+ def establish_variables(self, data, **kws):
+ """Extract variables from data or use directly."""
+ self.data = data
+
+ # Validate the inputs
+ any_strings = any([isinstance(v, str) for v in kws.values()])
+ if any_strings and data is None:
+ raise ValueError("Must pass `data` if using named variables.")
+
+ # Set the variables
+ for var, val in kws.items():
+ if isinstance(val, str):
+ vector = data[val]
+ elif isinstance(val, list):
+ vector = np.asarray(val)
+ else:
+ vector = val
+ if vector is not None and vector.shape != (1,):
+ vector = np.squeeze(vector)
+ if np.ndim(vector) > 1:
+ err = "regplot inputs must be 1d"
+ raise ValueError(err)
+ setattr(self, var, vector)
+
+ def dropna(self, *vars):
+ """Remove observations with missing data."""
+ vals = [getattr(self, var) for var in vars]
+ vals = [v for v in vals if v is not None]
+ not_na = np.all(np.column_stack([pd.notnull(v) for v in vals]), axis=1)
+ for var in vars:
+ val = getattr(self, var)
+ if val is not None:
+ setattr(self, var, val[not_na])
+
+ def plot(self, ax):
+ raise NotImplementedError
+
+
+class _RegressionPlotter(_LinearPlotter):
+ """Plotter for numeric independent variables with regression model.
+
+ This does the computations and drawing for the `regplot` function, and
+ is thus also used indirectly by `lmplot`.
+ """
+ def __init__(self, x, y, data=None, x_estimator=None, x_bins=None,
+ x_ci="ci", scatter=True, fit_reg=True, ci=95, n_boot=1000,
+ units=None, seed=None, order=1, logistic=False, lowess=False,
+ robust=False, logx=False, x_partial=None, y_partial=None,
+ truncate=False, dropna=True, x_jitter=None, y_jitter=None,
+ color=None, label=None):
+
+ # Set member attributes
+ self.x_estimator = x_estimator
+ self.ci = ci
+ self.x_ci = ci if x_ci == "ci" else x_ci
+ self.n_boot = n_boot
+ self.seed = seed
+ self.scatter = scatter
+ self.fit_reg = fit_reg
+ self.order = order
+ self.logistic = logistic
+ self.lowess = lowess
+ self.robust = robust
+ self.logx = logx
+ self.truncate = truncate
+ self.x_jitter = x_jitter
+ self.y_jitter = y_jitter
+ self.color = color
+ self.label = label
+
+ # Validate the regression options:
+ if sum((order > 1, logistic, robust, lowess, logx)) > 1:
+ raise ValueError("Mutually exclusive regression options.")
+
+ # Extract the data vals from the arguments or passed dataframe
+ self.establish_variables(data, x=x, y=y, units=units,
+ x_partial=x_partial, y_partial=y_partial)
+
+ # Drop null observations
+ if dropna:
+ self.dropna("x", "y", "units", "x_partial", "y_partial")
+
+ # Regress nuisance variables out of the data
+ if self.x_partial is not None:
+ self.x = self.regress_out(self.x, self.x_partial)
+ if self.y_partial is not None:
+ self.y = self.regress_out(self.y, self.y_partial)
+
+ # Possibly bin the predictor variable, which implies a point estimate
+ if x_bins is not None:
+ self.x_estimator = np.mean if x_estimator is None else x_estimator
+ x_discrete, x_bins = self.bin_predictor(x_bins)
+ self.x_discrete = x_discrete
+ else:
+ self.x_discrete = self.x
+
+ # Disable regression in case of singleton inputs
+ if len(self.x) <= 1:
+ self.fit_reg = False
+
+ # Save the range of the x variable for the grid later
+ if self.fit_reg:
+ self.x_range = self.x.min(), self.x.max()
+
+ @property
+ def scatter_data(self):
+ """Data where each observation is a point."""
+ x_j = self.x_jitter
+ if x_j is None:
+ x = self.x
+ else:
+ x = self.x + np.random.uniform(-x_j, x_j, len(self.x))
+
+ y_j = self.y_jitter
+ if y_j is None:
+ y = self.y
+ else:
+ y = self.y + np.random.uniform(-y_j, y_j, len(self.y))
+
+ return x, y
+
+ @property
+ def estimate_data(self):
+ """Data with a point estimate and CI for each discrete x value."""
+ x, y = self.x_discrete, self.y
+ vals = sorted(np.unique(x))
+ points, cis = [], []
+
+ for val in vals:
+
+ # Get the point estimate of the y variable
+ _y = y[x == val]
+ est = self.x_estimator(_y)
+ points.append(est)
+
+ # Compute the confidence interval for this estimate
+ if self.x_ci is None:
+ cis.append(None)
+ else:
+ units = None
+ if self.x_ci == "sd":
+ sd = np.std(_y)
+ _ci = est - sd, est + sd
+ else:
+ if self.units is not None:
+ units = self.units[x == val]
+ boots = algo.bootstrap(_y,
+ func=self.x_estimator,
+ n_boot=self.n_boot,
+ units=units,
+ seed=self.seed)
+ _ci = utils.ci(boots, self.x_ci)
+ cis.append(_ci)
+
+ return vals, points, cis
+
+ def _check_statsmodels(self):
+ """Check whether statsmodels is installed if any boolean options require it."""
+ options = "logistic", "robust", "lowess"
+ err = "`{}=True` requires statsmodels, an optional dependency, to be installed."
+ for option in options:
+ if getattr(self, option) and not _has_statsmodels:
+ raise RuntimeError(err.format(option))
+
+ def fit_regression(self, ax=None, x_range=None, grid=None):
+ """Fit the regression model."""
+ self._check_statsmodels()
+
+ # Create the grid for the regression
+ if grid is None:
+ if self.truncate:
+ x_min, x_max = self.x_range
+ else:
+ if ax is None:
+ x_min, x_max = x_range
+ else:
+ x_min, x_max = ax.get_xlim()
+ grid = np.linspace(x_min, x_max, 100)
+ ci = self.ci
+
+ # Fit the regression
+ if self.order > 1:
+ yhat, yhat_boots = self.fit_poly(grid, self.order)
+ elif self.logistic:
+ from statsmodels.genmod.generalized_linear_model import GLM
+ from statsmodels.genmod.families import Binomial
+ yhat, yhat_boots = self.fit_statsmodels(grid, GLM,
+ family=Binomial())
+ elif self.lowess:
+ ci = None
+ grid, yhat = self.fit_lowess()
+ elif self.robust:
+ from statsmodels.robust.robust_linear_model import RLM
+ yhat, yhat_boots = self.fit_statsmodels(grid, RLM)
+ elif self.logx:
+ yhat, yhat_boots = self.fit_logx(grid)
+ else:
+ yhat, yhat_boots = self.fit_fast(grid)
+
+ # Compute the confidence interval at each grid point
+ if ci is None:
+ err_bands = None
+ else:
+ err_bands = utils.ci(yhat_boots, ci, axis=0)
+
+ return grid, yhat, err_bands
+
+ def fit_fast(self, grid):
+ """Low-level regression and prediction using linear algebra."""
+ def reg_func(_x, _y):
+ return np.linalg.pinv(_x).dot(_y)
+
+ X, y = np.c_[np.ones(len(self.x)), self.x], self.y
+ grid = np.c_[np.ones(len(grid)), grid]
+ yhat = grid.dot(reg_func(X, y))
+ if self.ci is None:
+ return yhat, None
+
+ beta_boots = algo.bootstrap(X, y,
+ func=reg_func,
+ n_boot=self.n_boot,
+ units=self.units,
+ seed=self.seed).T
+ yhat_boots = grid.dot(beta_boots).T
+ return yhat, yhat_boots
+
+ def fit_poly(self, grid, order):
+ """Regression using numpy polyfit for higher-order trends."""
+ def reg_func(_x, _y):
+ return np.polyval(np.polyfit(_x, _y, order), grid)
+
+ x, y = self.x, self.y
+ yhat = reg_func(x, y)
+ if self.ci is None:
+ return yhat, None
+
+ yhat_boots = algo.bootstrap(x, y,
+ func=reg_func,
+ n_boot=self.n_boot,
+ units=self.units,
+ seed=self.seed)
+ return yhat, yhat_boots
+
+ def fit_statsmodels(self, grid, model, **kwargs):
+ """More general regression function using statsmodels objects."""
+ import statsmodels.tools.sm_exceptions as sme
+ X, y = np.c_[np.ones(len(self.x)), self.x], self.y
+ grid = np.c_[np.ones(len(grid)), grid]
+
+ def reg_func(_x, _y):
+ err_classes = (sme.PerfectSeparationError,)
+ try:
+ with warnings.catch_warnings():
+ if hasattr(sme, "PerfectSeparationWarning"):
+ # statsmodels>=0.14.0
+ warnings.simplefilter("error", sme.PerfectSeparationWarning)
+ err_classes = (*err_classes, sme.PerfectSeparationWarning)
+ yhat = model(_y, _x, **kwargs).fit().predict(grid)
+ except err_classes:
+ yhat = np.empty(len(grid))
+ yhat.fill(np.nan)
+ return yhat
+
+ yhat = reg_func(X, y)
+ if self.ci is None:
+ return yhat, None
+
+ yhat_boots = algo.bootstrap(X, y,
+ func=reg_func,
+ n_boot=self.n_boot,
+ units=self.units,
+ seed=self.seed)
+ return yhat, yhat_boots
+
+ def fit_lowess(self):
+ """Fit a locally-weighted regression, which returns its own grid."""
+ from statsmodels.nonparametric.smoothers_lowess import lowess
+ grid, yhat = lowess(self.y, self.x).T
+ return grid, yhat
+
+ def fit_logx(self, grid):
+ """Fit the model in log-space."""
+ X, y = np.c_[np.ones(len(self.x)), self.x], self.y
+ grid = np.c_[np.ones(len(grid)), np.log(grid)]
+
+ def reg_func(_x, _y):
+ _x = np.c_[_x[:, 0], np.log(_x[:, 1])]
+ return np.linalg.pinv(_x).dot(_y)
+
+ yhat = grid.dot(reg_func(X, y))
+ if self.ci is None:
+ return yhat, None
+
+ beta_boots = algo.bootstrap(X, y,
+ func=reg_func,
+ n_boot=self.n_boot,
+ units=self.units,
+ seed=self.seed).T
+ yhat_boots = grid.dot(beta_boots).T
+ return yhat, yhat_boots
+
+ def bin_predictor(self, bins):
+ """Discretize a predictor by assigning value to closest bin."""
+ x = np.asarray(self.x)
+ if np.isscalar(bins):
+ percentiles = np.linspace(0, 100, bins + 2)[1:-1]
+ bins = np.percentile(x, percentiles)
+ else:
+ bins = np.ravel(bins)
+
+ dist = np.abs(np.subtract.outer(x, bins))
+ x_binned = bins[np.argmin(dist, axis=1)].ravel()
+
+ return x_binned, bins
+
+ def regress_out(self, a, b):
+ """Regress b from a keeping a's original mean."""
+ a_mean = a.mean()
+ a = a - a_mean
+ b = b - b.mean()
+ b = np.c_[b]
+ a_prime = a - b.dot(np.linalg.pinv(b).dot(a))
+ return np.asarray(a_prime + a_mean).reshape(a.shape)
+
+ def plot(self, ax, scatter_kws, line_kws):
+ """Draw the full plot."""
+ # Insert the plot label into the correct set of keyword arguments
+ if self.scatter:
+ scatter_kws["label"] = self.label
+ else:
+ line_kws["label"] = self.label
+
+ # Use the current color cycle state as a default
+ if self.color is None:
+ lines, = ax.plot([], [])
+ color = lines.get_color()
+ lines.remove()
+ else:
+ color = self.color
+
+ # Ensure that color is hex to avoid matplotlib weirdness
+ color = mpl.colors.rgb2hex(mpl.colors.colorConverter.to_rgb(color))
+
+ # Let color in keyword arguments override overall plot color
+ scatter_kws.setdefault("color", color)
+ line_kws.setdefault("color", color)
+
+ # Draw the constituent plots
+ if self.scatter:
+ self.scatterplot(ax, scatter_kws)
+
+ if self.fit_reg:
+ self.lineplot(ax, line_kws)
+
+ # Label the axes
+ if hasattr(self.x, "name"):
+ ax.set_xlabel(self.x.name)
+ if hasattr(self.y, "name"):
+ ax.set_ylabel(self.y.name)
+
+ def scatterplot(self, ax, kws):
+ """Draw the data."""
+ # Treat the line-based markers specially, explicitly setting larger
+ # linewidth than is provided by the seaborn style defaults.
+ # This would ideally be handled better in matplotlib (i.e., distinguish
+ # between edgewidth for solid glyphs and linewidth for line glyphs
+ # but this should do for now.
+ line_markers = ["1", "2", "3", "4", "+", "x", "|", "_"]
+ if self.x_estimator is None:
+ if "marker" in kws and kws["marker"] in line_markers:
+ lw = mpl.rcParams["lines.linewidth"]
+ else:
+ lw = mpl.rcParams["lines.markeredgewidth"]
+ kws.setdefault("linewidths", lw)
+
+ if not hasattr(kws['color'], 'shape') or kws['color'].shape[1] < 4:
+ kws.setdefault("alpha", .8)
+
+ x, y = self.scatter_data
+ ax.scatter(x, y, **kws)
+ else:
+ # TODO abstraction
+ ci_kws = {"color": kws["color"]}
+ if "alpha" in kws:
+ ci_kws["alpha"] = kws["alpha"]
+ ci_kws["linewidth"] = mpl.rcParams["lines.linewidth"] * 1.75
+ kws.setdefault("s", 50)
+
+ xs, ys, cis = self.estimate_data
+ if [ci for ci in cis if ci is not None]:
+ for x, ci in zip(xs, cis):
+ ax.plot([x, x], ci, **ci_kws)
+ ax.scatter(xs, ys, **kws)
+
+ def lineplot(self, ax, kws):
+ """Draw the model."""
+ # Fit the regression model
+ grid, yhat, err_bands = self.fit_regression(ax)
+ edges = grid[0], grid[-1]
+
+ # Get set default aesthetics
+ fill_color = kws["color"]
+ lw = kws.pop("lw", mpl.rcParams["lines.linewidth"] * 1.5)
+ kws.setdefault("linewidth", lw)
+
+ # Draw the regression line and confidence interval
+ line, = ax.plot(grid, yhat, **kws)
+ if not self.truncate:
+ line.sticky_edges.x[:] = edges # Prevent mpl from adding margin
+ if err_bands is not None:
+ ax.fill_between(grid, *err_bands, facecolor=fill_color, alpha=.15)
+
+
+_regression_docs = dict(
+
+ model_api=dedent("""\
+ There are a number of mutually exclusive options for estimating the
+ regression model. See the :ref:`tutorial <regression_tutorial>` for more
+ information.\
+ """),
+ regplot_vs_lmplot=dedent("""\
+ The :func:`regplot` and :func:`lmplot` functions are closely related, but
+ the former is an axes-level function while the latter is a figure-level
+ function that combines :func:`regplot` and :class:`FacetGrid`.\
+ """),
+ x_estimator=dedent("""\
+ x_estimator : callable that maps vector -> scalar, optional
+ Apply this function to each unique value of ``x`` and plot the
+ resulting estimate. This is useful when ``x`` is a discrete variable.
+ If ``x_ci`` is given, this estimate will be bootstrapped and a
+ confidence interval will be drawn.\
+ """),
+ x_bins=dedent("""\
+ x_bins : int or vector, optional
+ Bin the ``x`` variable into discrete bins and then estimate the central
+ tendency and a confidence interval. This binning only influences how
+ the scatterplot is drawn; the regression is still fit to the original
+ data. This parameter is interpreted either as the number of
+ evenly-sized (not necessary spaced) bins or the positions of the bin
+ centers. When this parameter is used, it implies that the default of
+ ``x_estimator`` is ``numpy.mean``.\
+ """),
+ x_ci=dedent("""\
+ x_ci : "ci", "sd", int in [0, 100] or None, optional
+ Size of the confidence interval used when plotting a central tendency
+ for discrete values of ``x``. If ``"ci"``, defer to the value of the
+ ``ci`` parameter. If ``"sd"``, skip bootstrapping and show the
+ standard deviation of the observations in each bin.\
+ """),
+ scatter=dedent("""\
+ scatter : bool, optional
+ If ``True``, draw a scatterplot with the underlying observations (or
+ the ``x_estimator`` values).\
+ """),
+ fit_reg=dedent("""\
+ fit_reg : bool, optional
+ If ``True``, estimate and plot a regression model relating the ``x``
+ and ``y`` variables.\
+ """),
+ ci=dedent("""\
+ ci : int in [0, 100] or None, optional
+ Size of the confidence interval for the regression estimate. This will
+ be drawn using translucent bands around the regression line. The
+ confidence interval is estimated using a bootstrap; for large
+ datasets, it may be advisable to avoid that computation by setting
+ this parameter to None.\
+ """),
+ n_boot=dedent("""\
+ n_boot : int, optional
+ Number of bootstrap resamples used to estimate the ``ci``. The default
+ value attempts to balance time and stability; you may want to increase
+ this value for "final" versions of plots.\
+ """),
+ units=dedent("""\
+ units : variable name in ``data``, optional
+ If the ``x`` and ``y`` observations are nested within sampling units,
+ those can be specified here. This will be taken into account when
+ computing the confidence intervals by performing a multilevel bootstrap
+ that resamples both units and observations (within unit). This does not
+ otherwise influence how the regression is estimated or drawn.\
+ """),
+ seed=dedent("""\
+ seed : int, numpy.random.Generator, or numpy.random.RandomState, optional
+ Seed or random number generator for reproducible bootstrapping.\
+ """),
+ order=dedent("""\
+ order : int, optional
+ If ``order`` is greater than 1, use ``numpy.polyfit`` to estimate a
+ polynomial regression.\
+ """),
+ logistic=dedent("""\
+ logistic : bool, optional
+ If ``True``, assume that ``y`` is a binary variable and use
+ ``statsmodels`` to estimate a logistic regression model. Note that this
+ is substantially more computationally intensive than linear regression,
+ so you may wish to decrease the number of bootstrap resamples
+ (``n_boot``) or set ``ci`` to None.\
+ """),
+ lowess=dedent("""\
+ lowess : bool, optional
+ If ``True``, use ``statsmodels`` to estimate a nonparametric lowess
+ model (locally weighted linear regression). Note that confidence
+ intervals cannot currently be drawn for this kind of model.\
+ """),
+ robust=dedent("""\
+ robust : bool, optional
+ If ``True``, use ``statsmodels`` to estimate a robust regression. This
+ will de-weight outliers. Note that this is substantially more
+ computationally intensive than standard linear regression, so you may
+ wish to decrease the number of bootstrap resamples (``n_boot``) or set
+ ``ci`` to None.\
+ """),
+ logx=dedent("""\
+ logx : bool, optional
+ If ``True``, estimate a linear regression of the form y ~ log(x), but
+ plot the scatterplot and regression model in the input space. Note that
+ ``x`` must be positive for this to work.\
+ """),
+ xy_partial=dedent("""\
+ {x,y}_partial : strings in ``data`` or matrices
+ Confounding variables to regress out of the ``x`` or ``y`` variables
+ before plotting.\
+ """),
+ truncate=dedent("""\
+ truncate : bool, optional
+ If ``True``, the regression line is bounded by the data limits. If
+ ``False``, it extends to the ``x`` axis limits.
+ """),
+ xy_jitter=dedent("""\
+ {x,y}_jitter : floats, optional
+ Add uniform random noise of this size to either the ``x`` or ``y``
+ variables. The noise is added to a copy of the data after fitting the
+ regression, and only influences the look of the scatterplot. This can
+ be helpful when plotting variables that take discrete values.\
+ """),
+ scatter_line_kws=dedent("""\
+ {scatter,line}_kws : dictionaries
+ Additional keyword arguments to pass to ``plt.scatter`` and
+ ``plt.plot``.\
+ """),
+)
+_regression_docs.update(_facet_docs)
+
+
+def lmplot(
+ data, *,
+ x=None, y=None, hue=None, col=None, row=None,
+ palette=None, col_wrap=None, height=5, aspect=1, markers="o",
+ sharex=None, sharey=None, hue_order=None, col_order=None, row_order=None,
+ legend=True, legend_out=None, x_estimator=None, x_bins=None,
+ x_ci="ci", scatter=True, fit_reg=True, ci=95, n_boot=1000,
+ units=None, seed=None, order=1, logistic=False, lowess=False,
+ robust=False, logx=False, x_partial=None, y_partial=None,
+ truncate=True, x_jitter=None, y_jitter=None, scatter_kws=None,
+ line_kws=None, facet_kws=None,
+):
+
+ if facet_kws is None:
+ facet_kws = {}
+
+ def facet_kw_deprecation(key, val):
+ msg = (
+ f"{key} is deprecated from the `lmplot` function signature. "
+ "Please update your code to pass it using `facet_kws`."
+ )
+ if val is not None:
+ warnings.warn(msg, UserWarning)
+ facet_kws[key] = val
+
+ facet_kw_deprecation("sharex", sharex)
+ facet_kw_deprecation("sharey", sharey)
+ facet_kw_deprecation("legend_out", legend_out)
+
+ if data is None:
+ raise TypeError("Missing required keyword argument `data`.")
+
+ # Reduce the dataframe to only needed columns
+ need_cols = [x, y, hue, col, row, units, x_partial, y_partial]
+ cols = np.unique([a for a in need_cols if a is not None]).tolist()
+ data = data[cols]
+
+ # Initialize the grid
+ facets = FacetGrid(
+ data, row=row, col=col, hue=hue,
+ palette=palette,
+ row_order=row_order, col_order=col_order, hue_order=hue_order,
+ height=height, aspect=aspect, col_wrap=col_wrap,
+ **facet_kws,
+ )
+
+ # Add the markers here as FacetGrid has figured out how many levels of the
+ # hue variable are needed and we don't want to duplicate that process
+ if facets.hue_names is None:
+ n_markers = 1
+ else:
+ n_markers = len(facets.hue_names)
+ if not isinstance(markers, list):
+ markers = [markers] * n_markers
+ if len(markers) != n_markers:
+ raise ValueError("markers must be a singleton or a list of markers "
+ "for each level of the hue variable")
+ facets.hue_kws = {"marker": markers}
+
+ def update_datalim(data, x, y, ax, **kws):
+ xys = data[[x, y]].to_numpy().astype(float)
+ ax.update_datalim(xys, updatey=False)
+ ax.autoscale_view(scaley=False)
+
+ facets.map_dataframe(update_datalim, x=x, y=y)
+
+ # Draw the regression plot on each facet
+ regplot_kws = dict(
+ x_estimator=x_estimator, x_bins=x_bins, x_ci=x_ci,
+ scatter=scatter, fit_reg=fit_reg, ci=ci, n_boot=n_boot, units=units,
+ seed=seed, order=order, logistic=logistic, lowess=lowess,
+ robust=robust, logx=logx, x_partial=x_partial, y_partial=y_partial,
+ truncate=truncate, x_jitter=x_jitter, y_jitter=y_jitter,
+ scatter_kws=scatter_kws, line_kws=line_kws,
+ )
+ facets.map_dataframe(regplot, x=x, y=y, **regplot_kws)
+ facets.set_axis_labels(x, y)
+
+ # Add a legend
+ if legend and (hue is not None) and (hue not in [col, row]):
+ facets.add_legend()
+ return facets
+
+
+lmplot.__doc__ = dedent("""\
+ Plot data and regression model fits across a FacetGrid.
+
+ This function combines :func:`regplot` and :class:`FacetGrid`. It is
+ intended as a convenient interface to fit regression models across
+ conditional subsets of a dataset.
+
+ When thinking about how to assign variables to different facets, a general
+ rule is that it makes sense to use ``hue`` for the most important
+ comparison, followed by ``col`` and ``row``. However, always think about
+ your particular dataset and the goals of the visualization you are
+ creating.
+
+ {model_api}
+
+ The parameters to this function span most of the options in
+ :class:`FacetGrid`, although there may be occasional cases where you will
+ want to use that class and :func:`regplot` directly.
+
+ Parameters
+ ----------
+ {data}
+ x, y : strings, optional
+ Input variables; these should be column names in ``data``.
+ hue, col, row : strings
+ Variables that define subsets of the data, which will be drawn on
+ separate facets in the grid. See the ``*_order`` parameters to control
+ the order of levels of this variable.
+ {palette}
+ {col_wrap}
+ {height}
+ {aspect}
+ markers : matplotlib marker code or list of marker codes, optional
+ Markers for the scatterplot. If a list, each marker in the list will be
+ used for each level of the ``hue`` variable.
+ {share_xy}
+
+ .. deprecated:: 0.12.0
+ Pass using the `facet_kws` dictionary.
+
+ {{hue,col,row}}_order : lists, optional
+ Order for the levels of the faceting variables. By default, this will
+ be the order that the levels appear in ``data`` or, if the variables
+ are pandas categoricals, the category order.
+ legend : bool, optional
+ If ``True`` and there is a ``hue`` variable, add a legend.
+ {legend_out}
+
+ .. deprecated:: 0.12.0
+ Pass using the `facet_kws` dictionary.
+
+ {x_estimator}
+ {x_bins}
+ {x_ci}
+ {scatter}
+ {fit_reg}
+ {ci}
+ {n_boot}
+ {units}
+ {seed}
+ {order}
+ {logistic}
+ {lowess}
+ {robust}
+ {logx}
+ {xy_partial}
+ {truncate}
+ {xy_jitter}
+ {scatter_line_kws}
+ facet_kws : dict
+ Dictionary of keyword arguments for :class:`FacetGrid`.
+
+ See Also
+ --------
+ regplot : Plot data and a conditional model fit.
+ FacetGrid : Subplot grid for plotting conditional relationships.
+ pairplot : Combine :func:`regplot` and :class:`PairGrid` (when used with
+ ``kind="reg"``).
+
+ Notes
+ -----
+
+ {regplot_vs_lmplot}
+
+ Examples
+ --------
+
+ .. include:: ../docstrings/lmplot.rst
+
+ """).format(**_regression_docs)
+
+
+def regplot(
+ data=None, *, x=None, y=None,
+ x_estimator=None, x_bins=None, x_ci="ci",
+ scatter=True, fit_reg=True, ci=95, n_boot=1000, units=None,
+ seed=None, order=1, logistic=False, lowess=False, robust=False,
+ logx=False, x_partial=None, y_partial=None,
+ truncate=True, dropna=True, x_jitter=None, y_jitter=None,
+ label=None, color=None, marker="o",
+ scatter_kws=None, line_kws=None, ax=None
+):
+
+ plotter = _RegressionPlotter(x, y, data, x_estimator, x_bins, x_ci,
+ scatter, fit_reg, ci, n_boot, units, seed,
+ order, logistic, lowess, robust, logx,
+ x_partial, y_partial, truncate, dropna,
+ x_jitter, y_jitter, color, label)
+
+ if ax is None:
+ ax = plt.gca()
+
+ scatter_kws = {} if scatter_kws is None else copy.copy(scatter_kws)
+ scatter_kws["marker"] = marker
+ line_kws = {} if line_kws is None else copy.copy(line_kws)
+ plotter.plot(ax, scatter_kws, line_kws)
+ return ax
+
+
+regplot.__doc__ = dedent("""\
+ Plot data and a linear regression model fit.
+
+ {model_api}
+
+ Parameters
+ ----------
+ x, y: string, series, or vector array
+ Input variables. If strings, these should correspond with column names
+ in ``data``. When pandas objects are used, axes will be labeled with
+ the series name.
+ {data}
+ {x_estimator}
+ {x_bins}
+ {x_ci}
+ {scatter}
+ {fit_reg}
+ {ci}
+ {n_boot}
+ {units}
+ {seed}
+ {order}
+ {logistic}
+ {lowess}
+ {robust}
+ {logx}
+ {xy_partial}
+ {truncate}
+ {xy_jitter}
+ label : string
+ Label to apply to either the scatterplot or regression line (if
+ ``scatter`` is ``False``) for use in a legend.
+ color : matplotlib color
+ Color to apply to all plot elements; will be superseded by colors
+ passed in ``scatter_kws`` or ``line_kws``.
+ marker : matplotlib marker code
+ Marker to use for the scatterplot glyphs.
+ {scatter_line_kws}
+ ax : matplotlib Axes, optional
+ Axes object to draw the plot onto, otherwise uses the current Axes.
+
+ Returns
+ -------
+ ax : matplotlib Axes
+ The Axes object containing the plot.
+
+ See Also
+ --------
+ lmplot : Combine :func:`regplot` and :class:`FacetGrid` to plot multiple
+ linear relationships in a dataset.
+ jointplot : Combine :func:`regplot` and :class:`JointGrid` (when used with
+ ``kind="reg"``).
+ pairplot : Combine :func:`regplot` and :class:`PairGrid` (when used with
+ ``kind="reg"``).
+ residplot : Plot the residuals of a linear regression model.
+
+ Notes
+ -----
+
+ {regplot_vs_lmplot}
+
+
+ It's also easy to combine :func:`regplot` and :class:`JointGrid` or
+ :class:`PairGrid` through the :func:`jointplot` and :func:`pairplot`
+ functions, although these do not directly accept all of :func:`regplot`'s
+ parameters.
+
+ Examples
+ --------
+
+ .. include:: ../docstrings/regplot.rst
+
+ """).format(**_regression_docs)
+
+
+def residplot(
+ data=None, *, x=None, y=None,
+ x_partial=None, y_partial=None, lowess=False,
+ order=1, robust=False, dropna=True, label=None, color=None,
+ scatter_kws=None, line_kws=None, ax=None
+):
+ """Plot the residuals of a linear regression.
+
+ This function will regress y on x (possibly as a robust or polynomial
+ regression) and then draw a scatterplot of the residuals. You can
+ optionally fit a lowess smoother to the residual plot, which can
+ help in determining if there is structure to the residuals.
+
+ Parameters
+ ----------
+ data : DataFrame, optional
+ DataFrame to use if `x` and `y` are column names.
+ x : vector or string
+ Data or column name in `data` for the predictor variable.
+ y : vector or string
+ Data or column name in `data` for the response variable.
+ {x, y}_partial : vectors or string(s) , optional
+ These variables are treated as confounding and are removed from
+ the `x` or `y` variables before plotting.
+ lowess : boolean, optional
+ Fit a lowess smoother to the residual scatterplot.
+ order : int, optional
+ Order of the polynomial to fit when calculating the residuals.
+ robust : boolean, optional
+ Fit a robust linear regression when calculating the residuals.
+ dropna : boolean, optional
+ If True, ignore observations with missing data when fitting and
+ plotting.
+ label : string, optional
+ Label that will be used in any plot legends.
+ color : matplotlib color, optional
+ Color to use for all elements of the plot.
+ {scatter, line}_kws : dictionaries, optional
+ Additional keyword arguments passed to scatter() and plot() for drawing
+ the components of the plot.
+ ax : matplotlib axis, optional
+ Plot into this axis, otherwise grab the current axis or make a new
+ one if not existing.
+
+ Returns
+ -------
+ ax: matplotlib axes
+ Axes with the regression plot.
+
+ See Also
+ --------
+ regplot : Plot a simple linear regression model.
+ jointplot : Draw a :func:`residplot` with univariate marginal distributions
+ (when used with ``kind="resid"``).
+
+ Examples
+ --------
+
+ .. include:: ../docstrings/residplot.rst
+
+ """
+ plotter = _RegressionPlotter(x, y, data, ci=None,
+ order=order, robust=robust,
+ x_partial=x_partial, y_partial=y_partial,
+ dropna=dropna, color=color, label=label)
+
+ if ax is None:
+ ax = plt.gca()
+
+ # Calculate the residual from a linear regression
+ _, yhat, _ = plotter.fit_regression(grid=plotter.x)
+ plotter.y = plotter.y - yhat
+
+ # Set the regression option on the plotter
+ if lowess:
+ plotter.lowess = True
+ else:
+ plotter.fit_reg = False
+
+ # Plot a horizontal line at 0
+ ax.axhline(0, ls=":", c=".2")
+
+ # Draw the scatterplot
+ scatter_kws = {} if scatter_kws is None else scatter_kws.copy()
+ line_kws = {} if line_kws is None else line_kws.copy()
+ plotter.plot(ax, scatter_kws, line_kws)
+ return ax
diff --git a/seaborn/relational.py b/seaborn/relational.py
new file mode 100644
index 0000000000000000000000000000000000000000..ff0701c7938d29396f1e055fdfec998e66acdd29
--- /dev/null
+++ b/seaborn/relational.py
@@ -0,0 +1,982 @@
+from functools import partial
+import warnings
+
+import numpy as np
+import pandas as pd
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+from matplotlib.cbook import normalize_kwargs
+
+from ._base import (
+ VectorPlotter,
+)
+from .utils import (
+ adjust_legend_subtitles,
+ _default_color,
+ _deprecate_ci,
+ _get_transform_functions,
+ _scatter_legend_artist,
+)
+from ._compat import groupby_apply_include_groups
+from ._statistics import EstimateAggregator, WeightedAggregator
+from .axisgrid import FacetGrid, _facet_docs
+from ._docstrings import DocstringComponents, _core_docs
+
+
+__all__ = ["relplot", "scatterplot", "lineplot"]
+
+
+_relational_narrative = DocstringComponents(dict(
+
+ # --- Introductory prose
+ main_api="""
+The relationship between `x` and `y` can be shown for different subsets
+of the data using the `hue`, `size`, and `style` parameters. These
+parameters control what visual semantics are used to identify the different
+subsets. It is possible to show up to three dimensions independently by
+using all three semantic types, but this style of plot can be hard to
+interpret and is often ineffective. Using redundant semantics (i.e. both
+`hue` and `style` for the same variable) can be helpful for making
+graphics more accessible.
+
+See the :ref:`tutorial <relational_tutorial>` for more information.
+ """,
+
+ relational_semantic="""
+The default treatment of the `hue` (and to a lesser extent, `size`)
+semantic, if present, depends on whether the variable is inferred to
+represent "numeric" or "categorical" data. In particular, numeric variables
+are represented with a sequential colormap by default, and the legend
+entries show regular "ticks" with values that may or may not exist in the
+data. This behavior can be controlled through various parameters, as
+described and illustrated below.
+ """,
+))
+
+_relational_docs = dict(
+
+ # --- Shared function parameters
+ data_vars="""
+x, y : names of variables in `data` or vector data
+ Input data variables; must be numeric. Can pass data directly or
+ reference columns in `data`.
+ """,
+ data="""
+data : DataFrame, array, or list of arrays
+ Input data structure. If `x` and `y` are specified as names, this
+ should be a "long-form" DataFrame containing those columns. Otherwise
+ it is treated as "wide-form" data and grouping variables are ignored.
+ See the examples for the various ways this parameter can be specified
+ and the different effects of each.
+ """,
+ palette="""
+palette : string, list, dict, or matplotlib colormap
+ An object that determines how colors are chosen when `hue` is used.
+ It can be the name of a seaborn palette or matplotlib colormap, a list
+ of colors (anything matplotlib understands), a dict mapping levels
+ of the `hue` variable to colors, or a matplotlib colormap object.
+ """,
+ hue_order="""
+hue_order : list
+ Specified order for the appearance of the `hue` variable levels,
+ otherwise they are determined from the data. Not relevant when the
+ `hue` variable is numeric.
+ """,
+ hue_norm="""
+hue_norm : tuple or :class:`matplotlib.colors.Normalize` object
+ Normalization in data units for colormap applied to the `hue`
+ variable when it is numeric. Not relevant if `hue` is categorical.
+ """,
+ sizes="""
+sizes : list, dict, or tuple
+ An object that determines how sizes are chosen when `size` is used.
+ List or dict arguments should provide a size for each unique data value,
+ which forces a categorical interpretation. The argument may also be a
+ min, max tuple.
+ """,
+ size_order="""
+size_order : list
+ Specified order for appearance of the `size` variable levels,
+ otherwise they are determined from the data. Not relevant when the
+ `size` variable is numeric.
+ """,
+ size_norm="""
+size_norm : tuple or Normalize object
+ Normalization in data units for scaling plot objects when the
+ `size` variable is numeric.
+ """,
+ dashes="""
+dashes : boolean, list, or dictionary
+ Object determining how to draw the lines for different levels of the
+ `style` variable. Setting to `True` will use default dash codes, or
+ you can pass a list of dash codes or a dictionary mapping levels of the
+ `style` variable to dash codes. Setting to `False` will use solid
+ lines for all subsets. Dashes are specified as in matplotlib: a tuple
+ of `(segment, gap)` lengths, or an empty string to draw a solid line.
+ """,
+ markers="""
+markers : boolean, list, or dictionary
+ Object determining how to draw the markers for different levels of the
+ `style` variable. Setting to `True` will use default markers, or
+ you can pass a list of markers or a dictionary mapping levels of the
+ `style` variable to markers. Setting to `False` will draw
+ marker-less lines. Markers are specified as in matplotlib.
+ """,
+ style_order="""
+style_order : list
+ Specified order for appearance of the `style` variable levels
+ otherwise they are determined from the data. Not relevant when the
+ `style` variable is numeric.
+ """,
+ units="""
+units : vector or key in `data`
+ Grouping variable identifying sampling units. When used, a separate
+ line will be drawn for each unit with appropriate semantics, but no
+ legend entry will be added. Useful for showing distribution of
+ experimental replicates when exact identities are not needed.
+ """,
+ estimator="""
+estimator : name of pandas method or callable or None
+ Method for aggregating across multiple observations of the `y`
+ variable at the same `x` level. If `None`, all observations will
+ be drawn.
+ """,
+ ci="""
+ci : int or "sd" or None
+ Size of the confidence interval to draw when aggregating.
+
+ .. deprecated:: 0.12.0
+ Use the new `errorbar` parameter for more flexibility.
+
+ """,
+ n_boot="""
+n_boot : int
+ Number of bootstraps to use for computing the confidence interval.
+ """,
+ seed="""
+seed : int, numpy.random.Generator, or numpy.random.RandomState
+ Seed or random number generator for reproducible bootstrapping.
+ """,
+ legend="""
+legend : "auto", "brief", "full", or False
+ How to draw the legend. If "brief", numeric `hue` and `size`
+ variables will be represented with a sample of evenly spaced values.
+ If "full", every group will get an entry in the legend. If "auto",
+ choose between brief or full representation based on number of levels.
+ If `False`, no legend data is added and no legend is drawn.
+ """,
+ ax_in="""
+ax : matplotlib Axes
+ Axes object to draw the plot onto, otherwise uses the current Axes.
+ """,
+ ax_out="""
+ax : matplotlib Axes
+ Returns the Axes object with the plot drawn onto it.
+ """,
+
+)
+
+
+_param_docs = DocstringComponents.from_nested_components(
+ core=_core_docs["params"],
+ facets=DocstringComponents(_facet_docs),
+ rel=DocstringComponents(_relational_docs),
+ stat=DocstringComponents.from_function_params(EstimateAggregator.__init__),
+)
+
+
+class _RelationalPlotter(VectorPlotter):
+
+ wide_structure = {
+ "x": "@index", "y": "@values", "hue": "@columns", "style": "@columns",
+ }
+
+ # TODO where best to define default parameters?
+ sort = True
+
+
+class _LinePlotter(_RelationalPlotter):
+
+ _legend_attributes = ["color", "linewidth", "marker", "dashes"]
+
+ def __init__(
+ self, *,
+ data=None, variables={},
+ estimator=None, n_boot=None, seed=None, errorbar=None,
+ sort=True, orient="x", err_style=None, err_kws=None, legend=None
+ ):
+
+ # TODO this is messy, we want the mapping to be agnostic about
+ # the kind of plot to draw, but for the time being we need to set
+ # this information so the SizeMapping can use it
+ self._default_size_range = (
+ np.r_[.5, 2] * mpl.rcParams["lines.linewidth"]
+ )
+
+ super().__init__(data=data, variables=variables)
+
+ self.estimator = estimator
+ self.errorbar = errorbar
+ self.n_boot = n_boot
+ self.seed = seed
+ self.sort = sort
+ self.orient = orient
+ self.err_style = err_style
+ self.err_kws = {} if err_kws is None else err_kws
+
+ self.legend = legend
+
+ def plot(self, ax, kws):
+ """Draw the plot onto an axes, passing matplotlib kwargs."""
+
+ # Draw a test plot, using the passed in kwargs. The goal here is to
+ # honor both (a) the current state of the plot cycler and (b) the
+ # specified kwargs on all the lines we will draw, overriding when
+ # relevant with the data semantics. Note that we won't cycle
+ # internally; in other words, if `hue` is not used, all elements will
+ # have the same color, but they will have the color that you would have
+ # gotten from the corresponding matplotlib function, and calling the
+ # function will advance the axes property cycle.
+
+ kws = normalize_kwargs(kws, mpl.lines.Line2D)
+ kws.setdefault("markeredgewidth", 0.75)
+ kws.setdefault("markeredgecolor", "w")
+
+ # Set default error kwargs
+ err_kws = self.err_kws.copy()
+ if self.err_style == "band":
+ err_kws.setdefault("alpha", .2)
+ elif self.err_style == "bars":
+ pass
+ elif self.err_style is not None:
+ err = "`err_style` must be 'band' or 'bars', not {}"
+ raise ValueError(err.format(self.err_style))
+
+ # Initialize the aggregation object
+ weighted = "weight" in self.plot_data
+ agg = (WeightedAggregator if weighted else EstimateAggregator)(
+ self.estimator, self.errorbar, n_boot=self.n_boot, seed=self.seed,
+ )
+
+ # TODO abstract variable to aggregate over here-ish. Better name?
+ orient = self.orient
+ if orient not in {"x", "y"}:
+ err = f"`orient` must be either 'x' or 'y', not {orient!r}."
+ raise ValueError(err)
+ other = {"x": "y", "y": "x"}[orient]
+
+ # TODO How to handle NA? We don't want NA to propagate through to the
+ # estimate/CI when some values are present, but we would also like
+ # matplotlib to show "gaps" in the line when all values are missing.
+ # This is straightforward absent aggregation, but complicated with it.
+ # If we want to use nas, we need to conditionalize dropna in iter_data.
+
+ # Loop over the semantic subsets and add to the plot
+ grouping_vars = "hue", "size", "style"
+ for sub_vars, sub_data in self.iter_data(grouping_vars, from_comp_data=True):
+
+ if self.sort:
+ sort_vars = ["units", orient, other]
+ sort_cols = [var for var in sort_vars if var in self.variables]
+ sub_data = sub_data.sort_values(sort_cols)
+
+ if (
+ self.estimator is not None
+ and sub_data[orient].value_counts().max() > 1
+ ):
+ if "units" in self.variables:
+ # TODO eventually relax this constraint
+ err = "estimator must be None when specifying units"
+ raise ValueError(err)
+ grouped = sub_data.groupby(orient, sort=self.sort)
+ # Could pass as_index=False instead of reset_index,
+ # but that fails on a corner case with older pandas.
+ sub_data = (
+ grouped
+ .apply(agg, other, **groupby_apply_include_groups(False))
+ .reset_index()
+ )
+ else:
+ sub_data[f"{other}min"] = np.nan
+ sub_data[f"{other}max"] = np.nan
+
+ # Apply inverse axis scaling
+ for var in "xy":
+ _, inv = _get_transform_functions(ax, var)
+ for col in sub_data.filter(regex=f"^{var}"):
+ sub_data[col] = inv(sub_data[col])
+
+ # --- Draw the main line(s)
+
+ if "units" in self.variables: # XXX why not add to grouping variables?
+ lines = []
+ for _, unit_data in sub_data.groupby("units"):
+ lines.extend(ax.plot(unit_data["x"], unit_data["y"], **kws))
+ else:
+ lines = ax.plot(sub_data["x"], sub_data["y"], **kws)
+
+ for line in lines:
+
+ if "hue" in sub_vars:
+ line.set_color(self._hue_map(sub_vars["hue"]))
+
+ if "size" in sub_vars:
+ line.set_linewidth(self._size_map(sub_vars["size"]))
+
+ if "style" in sub_vars:
+ attributes = self._style_map(sub_vars["style"])
+ if "dashes" in attributes:
+ line.set_dashes(attributes["dashes"])
+ if "marker" in attributes:
+ line.set_marker(attributes["marker"])
+
+ line_color = line.get_color()
+ line_alpha = line.get_alpha()
+ line_capstyle = line.get_solid_capstyle()
+
+ # --- Draw the confidence intervals
+
+ if self.estimator is not None and self.errorbar is not None:
+
+ # TODO handling of orientation will need to happen here
+
+ if self.err_style == "band":
+
+ func = {"x": ax.fill_between, "y": ax.fill_betweenx}[orient]
+ func(
+ sub_data[orient],
+ sub_data[f"{other}min"], sub_data[f"{other}max"],
+ color=line_color, **err_kws
+ )
+
+ elif self.err_style == "bars":
+
+ error_param = {
+ f"{other}err": (
+ sub_data[other] - sub_data[f"{other}min"],
+ sub_data[f"{other}max"] - sub_data[other],
+ )
+ }
+ ebars = ax.errorbar(
+ sub_data["x"], sub_data["y"], **error_param,
+ linestyle="", color=line_color, alpha=line_alpha,
+ **err_kws
+ )
+
+ # Set the capstyle properly on the error bars
+ for obj in ebars.get_children():
+ if isinstance(obj, mpl.collections.LineCollection):
+ obj.set_capstyle(line_capstyle)
+
+ # Finalize the axes details
+ self._add_axis_labels(ax)
+ if self.legend:
+ legend_artist = partial(mpl.lines.Line2D, xdata=[], ydata=[])
+ attrs = {"hue": "color", "size": "linewidth", "style": None}
+ self.add_legend_data(ax, legend_artist, kws, attrs)
+ handles, _ = ax.get_legend_handles_labels()
+ if handles:
+ legend = ax.legend(title=self.legend_title)
+ adjust_legend_subtitles(legend)
+
+
+class _ScatterPlotter(_RelationalPlotter):
+
+ _legend_attributes = ["color", "s", "marker"]
+
+ def __init__(self, *, data=None, variables={}, legend=None):
+
+ # TODO this is messy, we want the mapping to be agnostic about
+ # the kind of plot to draw, but for the time being we need to set
+ # this information so the SizeMapping can use it
+ self._default_size_range = (
+ np.r_[.5, 2] * np.square(mpl.rcParams["lines.markersize"])
+ )
+
+ super().__init__(data=data, variables=variables)
+
+ self.legend = legend
+
+ def plot(self, ax, kws):
+
+ # --- Determine the visual attributes of the plot
+
+ data = self.comp_data.dropna()
+ if data.empty:
+ return
+
+ kws = normalize_kwargs(kws, mpl.collections.PathCollection)
+
+ # Define the vectors of x and y positions
+ empty = np.full(len(data), np.nan)
+ x = data.get("x", empty)
+ y = data.get("y", empty)
+
+ # Apply inverse scaling to the coordinate variables
+ _, inv_x = _get_transform_functions(ax, "x")
+ _, inv_y = _get_transform_functions(ax, "y")
+ x, y = inv_x(x), inv_y(y)
+
+ if "style" in self.variables:
+ # Use a representative marker so scatter sets the edgecolor
+ # properly for line art markers. We currently enforce either
+ # all or none line art so this works.
+ example_level = self._style_map.levels[0]
+ example_marker = self._style_map(example_level, "marker")
+ kws.setdefault("marker", example_marker)
+
+ # Conditionally set the marker edgecolor based on whether the marker is "filled"
+ # See https://github.com/matplotlib/matplotlib/issues/17849 for context
+ m = kws.get("marker", mpl.rcParams.get("marker", "o"))
+ if not isinstance(m, mpl.markers.MarkerStyle):
+ # TODO in more recent matplotlib (which?) can pass a MarkerStyle here
+ m = mpl.markers.MarkerStyle(m)
+ if m.is_filled():
+ kws.setdefault("edgecolor", "w")
+
+ # Draw the scatter plot
+ points = ax.scatter(x=x, y=y, **kws)
+
+ # Apply the mapping from semantic variables to artist attributes
+
+ if "hue" in self.variables:
+ points.set_facecolors(self._hue_map(data["hue"]))
+
+ if "size" in self.variables:
+ points.set_sizes(self._size_map(data["size"]))
+
+ if "style" in self.variables:
+ p = [self._style_map(val, "path") for val in data["style"]]
+ points.set_paths(p)
+
+ # Apply dependent default attributes
+
+ if "linewidth" not in kws:
+ sizes = points.get_sizes()
+ linewidth = .08 * np.sqrt(np.percentile(sizes, 10))
+ points.set_linewidths(linewidth)
+ kws["linewidth"] = linewidth
+
+ # Finalize the axes details
+ self._add_axis_labels(ax)
+ if self.legend:
+ attrs = {"hue": "color", "size": "s", "style": None}
+ self.add_legend_data(ax, _scatter_legend_artist, kws, attrs)
+ handles, _ = ax.get_legend_handles_labels()
+ if handles:
+ legend = ax.legend(title=self.legend_title)
+ adjust_legend_subtitles(legend)
+
+
+def lineplot(
+ data=None, *,
+ x=None, y=None, hue=None, size=None, style=None, units=None, weights=None,
+ palette=None, hue_order=None, hue_norm=None,
+ sizes=None, size_order=None, size_norm=None,
+ dashes=True, markers=None, style_order=None,
+ estimator="mean", errorbar=("ci", 95), n_boot=1000, seed=None,
+ orient="x", sort=True, err_style="band", err_kws=None,
+ legend="auto", ci="deprecated", ax=None, **kwargs
+):
+
+ # Handle deprecation of ci parameter
+ errorbar = _deprecate_ci(errorbar, ci)
+
+ p = _LinePlotter(
+ data=data,
+ variables=dict(
+ x=x, y=y, hue=hue, size=size, style=style, units=units, weight=weights
+ ),
+ estimator=estimator, n_boot=n_boot, seed=seed, errorbar=errorbar,
+ sort=sort, orient=orient, err_style=err_style, err_kws=err_kws,
+ legend=legend,
+ )
+
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm)
+ p.map_size(sizes=sizes, order=size_order, norm=size_norm)
+ p.map_style(markers=markers, dashes=dashes, order=style_order)
+
+ if ax is None:
+ ax = plt.gca()
+
+ if "style" not in p.variables and not {"ls", "linestyle"} & set(kwargs): # XXX
+ kwargs["dashes"] = "" if dashes is None or isinstance(dashes, bool) else dashes
+
+ if not p.has_xy_data:
+ return ax
+
+ p._attach(ax)
+
+ # Other functions have color as an explicit param,
+ # and we should probably do that here too
+ color = kwargs.pop("color", kwargs.pop("c", None))
+ kwargs["color"] = _default_color(ax.plot, hue, color, kwargs)
+
+ p.plot(ax, kwargs)
+ return ax
+
+
+lineplot.__doc__ = """\
+Draw a line plot with possibility of several semantic groupings.
+
+{narrative.main_api}
+
+{narrative.relational_semantic}
+
+By default, the plot aggregates over multiple `y` values at each value of
+`x` and shows an estimate of the central tendency and a confidence
+interval for that estimate.
+
+Parameters
+----------
+{params.core.data}
+{params.core.xy}
+hue : vector or key in `data`
+ Grouping variable that will produce lines with different colors.
+ Can be either categorical or numeric, although color mapping will
+ behave differently in latter case.
+size : vector or key in `data`
+ Grouping variable that will produce lines with different widths.
+ Can be either categorical or numeric, although size mapping will
+ behave differently in latter case.
+style : vector or key in `data`
+ Grouping variable that will produce lines with different dashes
+ and/or markers. Can have a numeric dtype but will always be treated
+ as categorical.
+{params.rel.units}
+weights : vector or key in `data`
+ Data values or column used to compute weighted estimation.
+ Note that use of weights currently limits the choice of statistics
+ to a 'mean' estimator and 'ci' errorbar.
+{params.core.palette}
+{params.core.hue_order}
+{params.core.hue_norm}
+{params.rel.sizes}
+{params.rel.size_order}
+{params.rel.size_norm}
+{params.rel.dashes}
+{params.rel.markers}
+{params.rel.style_order}
+{params.rel.estimator}
+{params.stat.errorbar}
+{params.rel.n_boot}
+{params.rel.seed}
+orient : "x" or "y"
+ Dimension along which the data are sorted / aggregated. Equivalently,
+ the "independent variable" of the resulting function.
+sort : boolean
+ If True, the data will be sorted by the x and y variables, otherwise
+ lines will connect points in the order they appear in the dataset.
+err_style : "band" or "bars"
+ Whether to draw the confidence intervals with translucent error bands
+ or discrete error bars.
+err_kws : dict of keyword arguments
+ Additional parameters to control the aesthetics of the error bars. The
+ kwargs are passed either to :meth:`matplotlib.axes.Axes.fill_between`
+ or :meth:`matplotlib.axes.Axes.errorbar`, depending on `err_style`.
+{params.rel.legend}
+{params.rel.ci}
+{params.core.ax}
+kwargs : key, value mappings
+ Other keyword arguments are passed down to
+ :meth:`matplotlib.axes.Axes.plot`.
+
+Returns
+-------
+{returns.ax}
+
+See Also
+--------
+{seealso.scatterplot}
+{seealso.pointplot}
+
+Examples
+--------
+
+.. include:: ../docstrings/lineplot.rst
+
+""".format(
+ narrative=_relational_narrative,
+ params=_param_docs,
+ returns=_core_docs["returns"],
+ seealso=_core_docs["seealso"],
+)
+
+
+def scatterplot(
+ data=None, *,
+ x=None, y=None, hue=None, size=None, style=None,
+ palette=None, hue_order=None, hue_norm=None,
+ sizes=None, size_order=None, size_norm=None,
+ markers=True, style_order=None, legend="auto", ax=None,
+ **kwargs
+):
+
+ p = _ScatterPlotter(
+ data=data,
+ variables=dict(x=x, y=y, hue=hue, size=size, style=style),
+ legend=legend
+ )
+
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm)
+ p.map_size(sizes=sizes, order=size_order, norm=size_norm)
+ p.map_style(markers=markers, order=style_order)
+
+ if ax is None:
+ ax = plt.gca()
+
+ if not p.has_xy_data:
+ return ax
+
+ p._attach(ax)
+
+ color = kwargs.pop("color", None)
+ kwargs["color"] = _default_color(ax.scatter, hue, color, kwargs)
+
+ p.plot(ax, kwargs)
+
+ return ax
+
+
+scatterplot.__doc__ = """\
+Draw a scatter plot with possibility of several semantic groupings.
+
+{narrative.main_api}
+
+{narrative.relational_semantic}
+
+Parameters
+----------
+{params.core.data}
+{params.core.xy}
+hue : vector or key in `data`
+ Grouping variable that will produce points with different colors.
+ Can be either categorical or numeric, although color mapping will
+ behave differently in latter case.
+size : vector or key in `data`
+ Grouping variable that will produce points with different sizes.
+ Can be either categorical or numeric, although size mapping will
+ behave differently in latter case.
+style : vector or key in `data`
+ Grouping variable that will produce points with different markers.
+ Can have a numeric dtype but will always be treated as categorical.
+{params.core.palette}
+{params.core.hue_order}
+{params.core.hue_norm}
+{params.rel.sizes}
+{params.rel.size_order}
+{params.rel.size_norm}
+{params.rel.markers}
+{params.rel.style_order}
+{params.rel.legend}
+{params.core.ax}
+kwargs : key, value mappings
+ Other keyword arguments are passed down to
+ :meth:`matplotlib.axes.Axes.scatter`.
+
+Returns
+-------
+{returns.ax}
+
+See Also
+--------
+{seealso.lineplot}
+{seealso.stripplot}
+{seealso.swarmplot}
+
+Examples
+--------
+
+.. include:: ../docstrings/scatterplot.rst
+
+""".format(
+ narrative=_relational_narrative,
+ params=_param_docs,
+ returns=_core_docs["returns"],
+ seealso=_core_docs["seealso"],
+)
+
+
+def relplot(
+ data=None, *,
+ x=None, y=None, hue=None, size=None, style=None, units=None, weights=None,
+ row=None, col=None, col_wrap=None, row_order=None, col_order=None,
+ palette=None, hue_order=None, hue_norm=None,
+ sizes=None, size_order=None, size_norm=None,
+ markers=None, dashes=None, style_order=None,
+ legend="auto", kind="scatter", height=5, aspect=1, facet_kws=None,
+ **kwargs
+):
+
+ if kind == "scatter":
+
+ Plotter = _ScatterPlotter
+ func = scatterplot
+ markers = True if markers is None else markers
+
+ elif kind == "line":
+
+ Plotter = _LinePlotter
+ func = lineplot
+ dashes = True if dashes is None else dashes
+
+ else:
+ err = f"Plot kind {kind} not recognized"
+ raise ValueError(err)
+
+ # Check for attempt to plot onto specific axes and warn
+ if "ax" in kwargs:
+ msg = (
+ "relplot is a figure-level function and does not accept "
+ "the `ax` parameter. You may wish to try {}".format(kind + "plot")
+ )
+ warnings.warn(msg, UserWarning)
+ kwargs.pop("ax")
+
+ # Use the full dataset to map the semantics
+ variables = dict(x=x, y=y, hue=hue, size=size, style=style)
+ if kind == "line":
+ variables["units"] = units
+ variables["weight"] = weights
+ else:
+ if units is not None:
+ msg = "The `units` parameter has no effect with kind='scatter'."
+ warnings.warn(msg, stacklevel=2)
+ if weights is not None:
+ msg = "The `weights` parameter has no effect with kind='scatter'."
+ warnings.warn(msg, stacklevel=2)
+ p = Plotter(
+ data=data,
+ variables=variables,
+ legend=legend,
+ )
+ p.map_hue(palette=palette, order=hue_order, norm=hue_norm)
+ p.map_size(sizes=sizes, order=size_order, norm=size_norm)
+ p.map_style(markers=markers, dashes=dashes, order=style_order)
+
+ # Extract the semantic mappings
+ if "hue" in p.variables:
+ palette = p._hue_map.lookup_table
+ hue_order = p._hue_map.levels
+ hue_norm = p._hue_map.norm
+ else:
+ palette = hue_order = hue_norm = None
+
+ if "size" in p.variables:
+ sizes = p._size_map.lookup_table
+ size_order = p._size_map.levels
+ size_norm = p._size_map.norm
+
+ if "style" in p.variables:
+ style_order = p._style_map.levels
+ if markers:
+ markers = {k: p._style_map(k, "marker") for k in style_order}
+ else:
+ markers = None
+ if dashes:
+ dashes = {k: p._style_map(k, "dashes") for k in style_order}
+ else:
+ dashes = None
+ else:
+ markers = dashes = style_order = None
+
+ # Now extract the data that would be used to draw a single plot
+ variables = p.variables
+ plot_data = p.plot_data
+
+ # Define the common plotting parameters
+ plot_kws = dict(
+ palette=palette, hue_order=hue_order, hue_norm=hue_norm,
+ sizes=sizes, size_order=size_order, size_norm=size_norm,
+ markers=markers, dashes=dashes, style_order=style_order,
+ legend=False,
+ )
+ plot_kws.update(kwargs)
+ if kind == "scatter":
+ plot_kws.pop("dashes")
+
+ # Add the grid semantics onto the plotter
+ grid_variables = dict(
+ x=x, y=y, row=row, col=col, hue=hue, size=size, style=style,
+ )
+ if kind == "line":
+ grid_variables.update(units=units, weights=weights)
+ p.assign_variables(data, grid_variables)
+
+ # Define the named variables for plotting on each facet
+ # Rename the variables with a leading underscore to avoid
+ # collisions with faceting variable names
+ plot_variables = {v: f"_{v}" for v in variables}
+ if "weight" in plot_variables:
+ plot_variables["weights"] = plot_variables.pop("weight")
+ plot_kws.update(plot_variables)
+
+ # Pass the row/col variables to FacetGrid with their original
+ # names so that the axes titles render correctly
+ for var in ["row", "col"]:
+ # Handle faceting variables that lack name information
+ if var in p.variables and p.variables[var] is None:
+ p.variables[var] = f"_{var}_"
+ grid_kws = {v: p.variables.get(v) for v in ["row", "col"]}
+
+ # Rename the columns of the plot_data structure appropriately
+ new_cols = plot_variables.copy()
+ new_cols.update(grid_kws)
+ full_data = p.plot_data.rename(columns=new_cols)
+
+ # Set up the FacetGrid object
+ facet_kws = {} if facet_kws is None else facet_kws.copy()
+ g = FacetGrid(
+ data=full_data.dropna(axis=1, how="all"),
+ **grid_kws,
+ col_wrap=col_wrap, row_order=row_order, col_order=col_order,
+ height=height, aspect=aspect, dropna=False,
+ **facet_kws
+ )
+
+ # Draw the plot
+ g.map_dataframe(func, **plot_kws)
+
+ # Label the axes, using the original variables
+ # Pass "" when the variable name is None to overwrite internal variables
+ g.set_axis_labels(variables.get("x") or "", variables.get("y") or "")
+
+ if legend:
+ # Replace the original plot data so the legend uses numeric data with
+ # the correct type, since we force a categorical mapping above.
+ p.plot_data = plot_data
+
+ # Handle the additional non-semantic keyword arguments out here.
+ # We're selective because some kwargs may be seaborn function specific
+ # and not relevant to the matplotlib artists going into the legend.
+ # Ideally, we will have a better solution where we don't need to re-make
+ # the legend out here and will have parity with the axes-level functions.
+ keys = ["c", "color", "alpha", "m", "marker"]
+ if kind == "scatter":
+ legend_artist = _scatter_legend_artist
+ keys += ["s", "facecolor", "fc", "edgecolor", "ec", "linewidth", "lw"]
+ else:
+ legend_artist = partial(mpl.lines.Line2D, xdata=[], ydata=[])
+ keys += [
+ "markersize", "ms",
+ "markeredgewidth", "mew",
+ "markeredgecolor", "mec",
+ "linestyle", "ls",
+ "linewidth", "lw",
+ ]
+
+ common_kws = {k: v for k, v in kwargs.items() if k in keys}
+ attrs = {"hue": "color", "style": None}
+ if kind == "scatter":
+ attrs["size"] = "s"
+ elif kind == "line":
+ attrs["size"] = "linewidth"
+ p.add_legend_data(g.axes.flat[0], legend_artist, common_kws, attrs)
+ if p.legend_data:
+ g.add_legend(legend_data=p.legend_data,
+ label_order=p.legend_order,
+ title=p.legend_title,
+ adjust_subtitles=True)
+
+ # Rename the columns of the FacetGrid's `data` attribute
+ # to match the original column names
+ orig_cols = {
+ f"_{k}": f"_{k}_" if v is None else v for k, v in variables.items()
+ }
+ grid_data = g.data.rename(columns=orig_cols)
+ if data is not None and (x is not None or y is not None):
+ if not isinstance(data, pd.DataFrame):
+ data = pd.DataFrame(data)
+ g.data = pd.merge(
+ data,
+ grid_data[grid_data.columns.difference(data.columns)],
+ left_index=True,
+ right_index=True,
+ )
+ else:
+ g.data = grid_data
+
+ return g
+
+
+relplot.__doc__ = """\
+Figure-level interface for drawing relational plots onto a FacetGrid.
+
+This function provides access to several different axes-level functions
+that show the relationship between two variables with semantic mappings
+of subsets. The `kind` parameter selects the underlying axes-level
+function to use:
+
+- :func:`scatterplot` (with `kind="scatter"`; the default)
+- :func:`lineplot` (with `kind="line"`)
+
+Extra keyword arguments are passed to the underlying function, so you
+should refer to the documentation for each to see kind-specific options.
+
+{narrative.main_api}
+
+{narrative.relational_semantic}
+
+After plotting, the :class:`FacetGrid` with the plot is returned and can
+be used directly to tweak supporting plot details or add other layers.
+
+Parameters
+----------
+{params.core.data}
+{params.core.xy}
+hue : vector or key in `data`
+ Grouping variable that will produce elements with different colors.
+ Can be either categorical or numeric, although color mapping will
+ behave differently in latter case.
+size : vector or key in `data`
+ Grouping variable that will produce elements with different sizes.
+ Can be either categorical or numeric, although size mapping will
+ behave differently in latter case.
+style : vector or key in `data`
+ Grouping variable that will produce elements with different styles.
+ Can have a numeric dtype but will always be treated as categorical.
+{params.rel.units}
+weights : vector or key in `data`
+ Data values or column used to compute weighted estimation.
+ Note that use of weights currently limits the choice of statistics
+ to a 'mean' estimator and 'ci' errorbar.
+{params.facets.rowcol}
+{params.facets.col_wrap}
+row_order, col_order : lists of strings
+ Order to organize the rows and/or columns of the grid in, otherwise the
+ orders are inferred from the data objects.
+{params.core.palette}
+{params.core.hue_order}
+{params.core.hue_norm}
+{params.rel.sizes}
+{params.rel.size_order}
+{params.rel.size_norm}
+{params.rel.style_order}
+{params.rel.dashes}
+{params.rel.markers}
+{params.rel.legend}
+kind : string
+ Kind of plot to draw, corresponding to a seaborn relational plot.
+ Options are `"scatter"` or `"line"`.
+{params.facets.height}
+{params.facets.aspect}
+facet_kws : dict
+ Dictionary of other keyword arguments to pass to :class:`FacetGrid`.
+kwargs : key, value pairings
+ Other keyword arguments are passed through to the underlying plotting
+ function.
+
+Returns
+-------
+{returns.facetgrid}
+
+Examples
+--------
+
+.. include:: ../docstrings/relplot.rst
+
+""".format(
+ narrative=_relational_narrative,
+ params=_param_docs,
+ returns=_core_docs["returns"],
+)
diff --git a/seaborn/utils.py b/seaborn/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..98720ba36d7f76a7bcd8afc0c93de3cd9c489d12
--- /dev/null
+++ b/seaborn/utils.py
@@ -0,0 +1,897 @@
+"""Utility functions, mostly for internal use."""
+import os
+import inspect
+import warnings
+import colorsys
+from contextlib import contextmanager
+from urllib.request import urlopen, urlretrieve
+from types import ModuleType
+
+import numpy as np
+import pandas as pd
+import matplotlib as mpl
+from matplotlib.colors import to_rgb
+import matplotlib.pyplot as plt
+from matplotlib.cbook import normalize_kwargs
+
+from seaborn._core.typing import deprecated
+from seaborn.external.version import Version
+from seaborn.external.appdirs import user_cache_dir
+
+__all__ = ["desaturate", "saturate", "set_hls_values", "move_legend",
+ "despine", "get_dataset_names", "get_data_home", "load_dataset"]
+
+DATASET_SOURCE = "https://raw.githubusercontent.com/mwaskom/seaborn-data/master"
+DATASET_NAMES_URL = f"{DATASET_SOURCE}/dataset_names.txt"
+
+
+def ci_to_errsize(cis, heights):
+ """Convert intervals to error arguments relative to plot heights.
+
+ Parameters
+ ----------
+ cis : 2 x n sequence
+ sequence of confidence interval limits
+ heights : n sequence
+ sequence of plot heights
+
+ Returns
+ -------
+ errsize : 2 x n array
+ sequence of error size relative to height values in correct
+ format as argument for plt.bar
+
+ """
+ cis = np.atleast_2d(cis).reshape(2, -1)
+ heights = np.atleast_1d(heights)
+ errsize = []
+ for i, (low, high) in enumerate(np.transpose(cis)):
+ h = heights[i]
+ elow = h - low
+ ehigh = high - h
+ errsize.append([elow, ehigh])
+
+ errsize = np.asarray(errsize).T
+ return errsize
+
+
+def _draw_figure(fig):
+ """Force draw of a matplotlib figure, accounting for back-compat."""
+ # See https://github.com/matplotlib/matplotlib/issues/19197 for context
+ fig.canvas.draw()
+ if fig.stale:
+ try:
+ fig.draw(fig.canvas.get_renderer())
+ except AttributeError:
+ pass
+
+
+def _default_color(method, hue, color, kws, saturation=1):
+ """If needed, get a default color by using the matplotlib property cycle."""
+
+ if hue is not None:
+ # This warning is probably user-friendly, but it's currently triggered
+ # in a FacetGrid context and I don't want to mess with that logic right now
+ # if color is not None:
+ # msg = "`color` is ignored when `hue` is assigned."
+ # warnings.warn(msg)
+ return None
+
+ kws = kws.copy()
+ kws.pop("label", None)
+
+ if color is not None:
+ if saturation < 1:
+ color = desaturate(color, saturation)
+ return color
+
+ elif method.__name__ == "plot":
+
+ color = normalize_kwargs(kws, mpl.lines.Line2D).get("color")
+ scout, = method([], [], scalex=False, scaley=False, color=color)
+ color = scout.get_color()
+ scout.remove()
+
+ elif method.__name__ == "scatter":
+
+ # Matplotlib will raise if the size of x/y don't match s/c,
+ # and the latter might be in the kws dict
+ scout_size = max(
+ np.atleast_1d(kws.get(key, [])).shape[0]
+ for key in ["s", "c", "fc", "facecolor", "facecolors"]
+ )
+ scout_x = scout_y = np.full(scout_size, np.nan)
+
+ scout = method(scout_x, scout_y, **kws)
+ facecolors = scout.get_facecolors()
+
+ if not len(facecolors):
+ # Handle bug in matplotlib <= 3.2 (I think)
+ # This will limit the ability to use non color= kwargs to specify
+ # a color in versions of matplotlib with the bug, but trying to
+ # work out what the user wanted by re-implementing the broken logic
+ # of inspecting the kwargs is probably too brittle.
+ single_color = False
+ else:
+ single_color = np.unique(facecolors, axis=0).shape[0] == 1
+
+ # Allow the user to specify an array of colors through various kwargs
+ if "c" not in kws and single_color:
+ color = to_rgb(facecolors[0])
+
+ scout.remove()
+
+ elif method.__name__ == "bar":
+
+ # bar() needs masked, not empty data, to generate a patch
+ scout, = method([np.nan], [np.nan], **kws)
+ color = to_rgb(scout.get_facecolor())
+ scout.remove()
+ # Axes.bar adds both a patch and a container
+ method.__self__.containers.pop(-1)
+
+ elif method.__name__ == "fill_between":
+
+ kws = normalize_kwargs(kws, mpl.collections.PolyCollection)
+ scout = method([], [], **kws)
+ facecolor = scout.get_facecolor()
+ color = to_rgb(facecolor[0])
+ scout.remove()
+
+ if saturation < 1:
+ color = desaturate(color, saturation)
+
+ return color
+
+
+def desaturate(color, prop):
+ """Decrease the saturation channel of a color by some percent.
+
+ Parameters
+ ----------
+ color : matplotlib color
+ hex, rgb-tuple, or html color name
+ prop : float
+ saturation channel of color will be multiplied by this value
+
+ Returns
+ -------
+ new_color : rgb tuple
+ desaturated color code in RGB tuple representation
+
+ """
+ # Check inputs
+ if not 0 <= prop <= 1:
+ raise ValueError("prop must be between 0 and 1")
+
+ # Get rgb tuple rep
+ rgb = to_rgb(color)
+
+ # Short circuit to avoid floating point issues
+ if prop == 1:
+ return rgb
+
+ # Convert to hls
+ h, l, s = colorsys.rgb_to_hls(*rgb)
+
+ # Desaturate the saturation channel
+ s *= prop
+
+ # Convert back to rgb
+ new_color = colorsys.hls_to_rgb(h, l, s)
+
+ return new_color
+
+
+def saturate(color):
+ """Return a fully saturated color with the same hue.
+
+ Parameters
+ ----------
+ color : matplotlib color
+ hex, rgb-tuple, or html color name
+
+ Returns
+ -------
+ new_color : rgb tuple
+ saturated color code in RGB tuple representation
+
+ """
+ return set_hls_values(color, s=1)
+
+
+def set_hls_values(color, h=None, l=None, s=None): # noqa
+ """Independently manipulate the h, l, or s channels of a color.
+
+ Parameters
+ ----------
+ color : matplotlib color
+ hex, rgb-tuple, or html color name
+ h, l, s : floats between 0 and 1, or None
+ new values for each channel in hls space
+
+ Returns
+ -------
+ new_color : rgb tuple
+ new color code in RGB tuple representation
+
+ """
+ # Get an RGB tuple representation
+ rgb = to_rgb(color)
+ vals = list(colorsys.rgb_to_hls(*rgb))
+ for i, val in enumerate([h, l, s]):
+ if val is not None:
+ vals[i] = val
+
+ rgb = colorsys.hls_to_rgb(*vals)
+ return rgb
+
+
+def axlabel(xlabel, ylabel, **kwargs):
+ """Grab current axis and label it.
+
+ DEPRECATED: will be removed in a future version.
+
+ """
+ msg = "This function is deprecated and will be removed in a future version"
+ warnings.warn(msg, FutureWarning)
+ ax = plt.gca()
+ ax.set_xlabel(xlabel, **kwargs)
+ ax.set_ylabel(ylabel, **kwargs)
+
+
+def remove_na(vector):
+ """Helper method for removing null values from data vectors.
+
+ Parameters
+ ----------
+ vector : vector object
+ Must implement boolean masking with [] subscript syntax.
+
+ Returns
+ -------
+ clean_clean : same type as ``vector``
+ Vector of data with null values removed. May be a copy or a view.
+
+ """
+ return vector[pd.notnull(vector)]
+
+
+def get_color_cycle():
+ """Return the list of colors in the current matplotlib color cycle
+
+ Parameters
+ ----------
+ None
+
+ Returns
+ -------
+ colors : list
+ List of matplotlib colors in the current cycle, or dark gray if
+ the current color cycle is empty.
+ """
+ cycler = mpl.rcParams['axes.prop_cycle']
+ return cycler.by_key()['color'] if 'color' in cycler.keys else [".15"]
+
+
+def despine(fig=None, ax=None, top=True, right=True, left=False,
+ bottom=False, offset=None, trim=False):
+ """Remove the top and right spines from plot(s).
+
+ fig : matplotlib figure, optional
+ Figure to despine all axes of, defaults to the current figure.
+ ax : matplotlib axes, optional
+ Specific axes object to despine. Ignored if fig is provided.
+ top, right, left, bottom : boolean, optional
+ If True, remove that spine.
+ offset : int or dict, optional
+ Absolute distance, in points, spines should be moved away
+ from the axes (negative values move spines inward). A single value
+ applies to all spines; a dict can be used to set offset values per
+ side.
+ trim : bool, optional
+ If True, limit spines to the smallest and largest major tick
+ on each non-despined axis.
+
+ Returns
+ -------
+ None
+
+ """
+ # Get references to the axes we want
+ if fig is None and ax is None:
+ axes = plt.gcf().axes
+ elif fig is not None:
+ axes = fig.axes
+ elif ax is not None:
+ axes = [ax]
+
+ for ax_i in axes:
+ for side in ["top", "right", "left", "bottom"]:
+ # Toggle the spine objects
+ is_visible = not locals()[side]
+ ax_i.spines[side].set_visible(is_visible)
+ if offset is not None and is_visible:
+ try:
+ val = offset.get(side, 0)
+ except AttributeError:
+ val = offset
+ ax_i.spines[side].set_position(('outward', val))
+
+ # Potentially move the ticks
+ if left and not right:
+ maj_on = any(
+ t.tick1line.get_visible()
+ for t in ax_i.yaxis.majorTicks
+ )
+ min_on = any(
+ t.tick1line.get_visible()
+ for t in ax_i.yaxis.minorTicks
+ )
+ ax_i.yaxis.set_ticks_position("right")
+ for t in ax_i.yaxis.majorTicks:
+ t.tick2line.set_visible(maj_on)
+ for t in ax_i.yaxis.minorTicks:
+ t.tick2line.set_visible(min_on)
+
+ if bottom and not top:
+ maj_on = any(
+ t.tick1line.get_visible()
+ for t in ax_i.xaxis.majorTicks
+ )
+ min_on = any(
+ t.tick1line.get_visible()
+ for t in ax_i.xaxis.minorTicks
+ )
+ ax_i.xaxis.set_ticks_position("top")
+ for t in ax_i.xaxis.majorTicks:
+ t.tick2line.set_visible(maj_on)
+ for t in ax_i.xaxis.minorTicks:
+ t.tick2line.set_visible(min_on)
+
+ if trim:
+ # clip off the parts of the spines that extend past major ticks
+ xticks = np.asarray(ax_i.get_xticks())
+ if xticks.size:
+ firsttick = np.compress(xticks >= min(ax_i.get_xlim()),
+ xticks)[0]
+ lasttick = np.compress(xticks <= max(ax_i.get_xlim()),
+ xticks)[-1]
+ ax_i.spines['bottom'].set_bounds(firsttick, lasttick)
+ ax_i.spines['top'].set_bounds(firsttick, lasttick)
+ newticks = xticks.compress(xticks <= lasttick)
+ newticks = newticks.compress(newticks >= firsttick)
+ ax_i.set_xticks(newticks)
+
+ yticks = np.asarray(ax_i.get_yticks())
+ if yticks.size:
+ firsttick = np.compress(yticks >= min(ax_i.get_ylim()),
+ yticks)[0]
+ lasttick = np.compress(yticks <= max(ax_i.get_ylim()),
+ yticks)[-1]
+ ax_i.spines['left'].set_bounds(firsttick, lasttick)
+ ax_i.spines['right'].set_bounds(firsttick, lasttick)
+ newticks = yticks.compress(yticks <= lasttick)
+ newticks = newticks.compress(newticks >= firsttick)
+ ax_i.set_yticks(newticks)
+
+
+def move_legend(obj, loc, **kwargs):
+ """
+ Recreate a plot's legend at a new location.
+
+ The name is a slight misnomer. Matplotlib legends do not expose public
+ control over their position parameters. So this function creates a new legend,
+ copying over the data from the original object, which is then removed.
+
+ Parameters
+ ----------
+ obj : the object with the plot
+ This argument can be either a seaborn or matplotlib object:
+
+ - :class:`seaborn.FacetGrid` or :class:`seaborn.PairGrid`
+ - :class:`matplotlib.axes.Axes` or :class:`matplotlib.figure.Figure`
+
+ loc : str or int
+ Location argument, as in :meth:`matplotlib.axes.Axes.legend`.
+
+ kwargs
+ Other keyword arguments are passed to :meth:`matplotlib.axes.Axes.legend`.
+
+ Examples
+ --------
+
+ .. include:: ../docstrings/move_legend.rst
+
+ """
+ # This is a somewhat hackish solution that will hopefully be obviated by
+ # upstream improvements to matplotlib legends that make them easier to
+ # modify after creation.
+
+ from seaborn.axisgrid import Grid # Avoid circular import
+
+ # Locate the legend object and a method to recreate the legend
+ if isinstance(obj, Grid):
+ old_legend = obj.legend
+ legend_func = obj.figure.legend
+ elif isinstance(obj, mpl.axes.Axes):
+ old_legend = obj.legend_
+ legend_func = obj.legend
+ elif isinstance(obj, mpl.figure.Figure):
+ if obj.legends:
+ old_legend = obj.legends[-1]
+ else:
+ old_legend = None
+ legend_func = obj.legend
+ else:
+ err = "`obj` must be a seaborn Grid or matplotlib Axes or Figure instance."
+ raise TypeError(err)
+
+ if old_legend is None:
+ err = f"{obj} has no legend attached."
+ raise ValueError(err)
+
+ # Extract the components of the legend we need to reuse
+ # Import here to avoid a circular import
+ from seaborn._compat import get_legend_handles
+ handles = get_legend_handles(old_legend)
+ labels = [t.get_text() for t in old_legend.get_texts()]
+
+ # Handle the case where the user is trying to override the labels
+ if (new_labels := kwargs.pop("labels", None)) is not None:
+ if len(new_labels) != len(labels):
+ err = "Length of new labels does not match existing legend."
+ raise ValueError(err)
+ labels = new_labels
+
+ # Extract legend properties that can be passed to the recreation method
+ # (Vexingly, these don't all round-trip)
+ legend_kws = inspect.signature(mpl.legend.Legend).parameters
+ props = {k: v for k, v in old_legend.properties().items() if k in legend_kws}
+
+ # Delegate default bbox_to_anchor rules to matplotlib
+ props.pop("bbox_to_anchor")
+
+ # Try to propagate the existing title and font properties; respect new ones too
+ title = props.pop("title")
+ if "title" in kwargs:
+ title.set_text(kwargs.pop("title"))
+ title_kwargs = {k: v for k, v in kwargs.items() if k.startswith("title_")}
+ for key, val in title_kwargs.items():
+ title.set(**{key[6:]: val})
+ kwargs.pop(key)
+
+ # Try to respect the frame visibility
+ kwargs.setdefault("frameon", old_legend.legendPatch.get_visible())
+
+ # Remove the old legend and create the new one
+ props.update(kwargs)
+ old_legend.remove()
+ new_legend = legend_func(handles, labels, loc=loc, **props)
+ new_legend.set_title(title.get_text(), title.get_fontproperties())
+
+ # Let the Grid object continue to track the correct legend object
+ if isinstance(obj, Grid):
+ obj._legend = new_legend
+
+
+def _kde_support(data, bw, gridsize, cut, clip):
+ """Establish support for a kernel density estimate."""
+ support_min = max(data.min() - bw * cut, clip[0])
+ support_max = min(data.max() + bw * cut, clip[1])
+ support = np.linspace(support_min, support_max, gridsize)
+
+ return support
+
+
+def ci(a, which=95, axis=None):
+ """Return a percentile range from an array of values."""
+ p = 50 - which / 2, 50 + which / 2
+ return np.nanpercentile(a, p, axis)
+
+
+def get_dataset_names():
+ """Report available example datasets, useful for reporting issues.
+
+ Requires an internet connection.
+
+ """
+ with urlopen(DATASET_NAMES_URL) as resp:
+ txt = resp.read()
+
+ dataset_names = [name.strip() for name in txt.decode().split("\n")]
+ return list(filter(None, dataset_names))
+
+
+def get_data_home(data_home=None):
+ """Return a path to the cache directory for example datasets.
+
+ This directory is used by :func:`load_dataset`.
+
+ If the ``data_home`` argument is not provided, it will use a directory
+ specified by the `SEABORN_DATA` environment variable (if it exists)
+ or otherwise default to an OS-appropriate user cache location.
+
+ """
+ if data_home is None:
+ data_home = os.environ.get("SEABORN_DATA", user_cache_dir("seaborn"))
+ data_home = os.path.expanduser(data_home)
+ if not os.path.exists(data_home):
+ os.makedirs(data_home)
+ return data_home
+
+
+def load_dataset(name, cache=True, data_home=None, **kws):
+ """Load an example dataset from the online repository (requires internet).
+
+ This function provides quick access to a small number of example datasets
+ that are useful for documenting seaborn or generating reproducible examples
+ for bug reports. It is not necessary for normal usage.
+
+ Note that some of the datasets have a small amount of preprocessing applied
+ to define a proper ordering for categorical variables.
+
+ Use :func:`get_dataset_names` to see a list of available datasets.
+
+ Parameters
+ ----------
+ name : str
+ Name of the dataset (``{name}.csv`` on
+ https://github.com/mwaskom/seaborn-data).
+ cache : boolean, optional
+ If True, try to load from the local cache first, and save to the cache
+ if a download is required.
+ data_home : string, optional
+ The directory in which to cache data; see :func:`get_data_home`.
+ kws : keys and values, optional
+ Additional keyword arguments are passed to passed through to
+ :func:`pandas.read_csv`.
+
+ Returns
+ -------
+ df : :class:`pandas.DataFrame`
+ Tabular data, possibly with some preprocessing applied.
+
+ """
+ # A common beginner mistake is to assume that one's personal data needs
+ # to be passed through this function to be usable with seaborn.
+ # Let's provide a more helpful error than you would otherwise get.
+ if isinstance(name, pd.DataFrame):
+ err = (
+ "This function accepts only strings (the name of an example dataset). "
+ "You passed a pandas DataFrame. If you have your own dataset, "
+ "it is not necessary to use this function before plotting."
+ )
+ raise TypeError(err)
+
+ url = f"{DATASET_SOURCE}/{name}.csv"
+
+ if cache:
+ cache_path = os.path.join(get_data_home(data_home), os.path.basename(url))
+ if not os.path.exists(cache_path):
+ if name not in get_dataset_names():
+ raise ValueError(f"'{name}' is not one of the example datasets.")
+ urlretrieve(url, cache_path)
+ full_path = cache_path
+ else:
+ full_path = url
+
+ df = pd.read_csv(full_path, **kws)
+
+ if df.iloc[-1].isnull().all():
+ df = df.iloc[:-1]
+
+ # Set some columns as a categorical type with ordered levels
+
+ if name == "tips":
+ df["day"] = pd.Categorical(df["day"], ["Thur", "Fri", "Sat", "Sun"])
+ df["sex"] = pd.Categorical(df["sex"], ["Male", "Female"])
+ df["time"] = pd.Categorical(df["time"], ["Lunch", "Dinner"])
+ df["smoker"] = pd.Categorical(df["smoker"], ["Yes", "No"])
+
+ elif name == "flights":
+ months = df["month"].str[:3]
+ df["month"] = pd.Categorical(months, months.unique())
+
+ elif name == "exercise":
+ df["time"] = pd.Categorical(df["time"], ["1 min", "15 min", "30 min"])
+ df["kind"] = pd.Categorical(df["kind"], ["rest", "walking", "running"])
+ df["diet"] = pd.Categorical(df["diet"], ["no fat", "low fat"])
+
+ elif name == "titanic":
+ df["class"] = pd.Categorical(df["class"], ["First", "Second", "Third"])
+ df["deck"] = pd.Categorical(df["deck"], list("ABCDEFG"))
+
+ elif name == "penguins":
+ df["sex"] = df["sex"].str.title()
+
+ elif name == "diamonds":
+ df["color"] = pd.Categorical(
+ df["color"], ["D", "E", "F", "G", "H", "I", "J"],
+ )
+ df["clarity"] = pd.Categorical(
+ df["clarity"], ["IF", "VVS1", "VVS2", "VS1", "VS2", "SI1", "SI2", "I1"],
+ )
+ df["cut"] = pd.Categorical(
+ df["cut"], ["Ideal", "Premium", "Very Good", "Good", "Fair"],
+ )
+
+ elif name == "taxis":
+ df["pickup"] = pd.to_datetime(df["pickup"])
+ df["dropoff"] = pd.to_datetime(df["dropoff"])
+
+ elif name == "seaice":
+ df["Date"] = pd.to_datetime(df["Date"])
+
+ elif name == "dowjones":
+ df["Date"] = pd.to_datetime(df["Date"])
+
+ return df
+
+
+def axis_ticklabels_overlap(labels):
+ """Return a boolean for whether the list of ticklabels have overlaps.
+
+ Parameters
+ ----------
+ labels : list of matplotlib ticklabels
+
+ Returns
+ -------
+ overlap : boolean
+ True if any of the labels overlap.
+
+ """
+ if not labels:
+ return False
+ try:
+ bboxes = [l.get_window_extent() for l in labels]
+ overlaps = [b.count_overlaps(bboxes) for b in bboxes]
+ return max(overlaps) > 1
+ except RuntimeError:
+ # Issue on macos backend raises an error in the above code
+ return False
+
+
+def axes_ticklabels_overlap(ax):
+ """Return booleans for whether the x and y ticklabels on an Axes overlap.
+
+ Parameters
+ ----------
+ ax : matplotlib Axes
+
+ Returns
+ -------
+ x_overlap, y_overlap : booleans
+ True when the labels on that axis overlap.
+
+ """
+ return (axis_ticklabels_overlap(ax.get_xticklabels()),
+ axis_ticklabels_overlap(ax.get_yticklabels()))
+
+
+def locator_to_legend_entries(locator, limits, dtype):
+ """Return levels and formatted levels for brief numeric legends."""
+ raw_levels = locator.tick_values(*limits).astype(dtype)
+
+ # The locator can return ticks outside the limits, clip them here
+ raw_levels = [l for l in raw_levels if l >= limits[0] and l <= limits[1]]
+
+ class dummy_axis:
+ def get_view_interval(self):
+ return limits
+
+ if isinstance(locator, mpl.ticker.LogLocator):
+ formatter = mpl.ticker.LogFormatter()
+ else:
+ formatter = mpl.ticker.ScalarFormatter()
+ # Avoid having an offset/scientific notation which we don't currently
+ # have any way of representing in the legend
+ formatter.set_useOffset(False)
+ formatter.set_scientific(False)
+ formatter.axis = dummy_axis()
+
+ formatted_levels = formatter.format_ticks(raw_levels)
+
+ return raw_levels, formatted_levels
+
+
+def relative_luminance(color):
+ """Calculate the relative luminance of a color according to W3C standards
+
+ Parameters
+ ----------
+ color : matplotlib color or sequence of matplotlib colors
+ Hex code, rgb-tuple, or html color name.
+
+ Returns
+ -------
+ luminance : float(s) between 0 and 1
+
+ """
+ rgb = mpl.colors.colorConverter.to_rgba_array(color)[:, :3]
+ rgb = np.where(rgb <= .03928, rgb / 12.92, ((rgb + .055) / 1.055) ** 2.4)
+ lum = rgb.dot([.2126, .7152, .0722])
+ try:
+ return lum.item()
+ except ValueError:
+ return lum
+
+
+def to_utf8(obj):
+ """Return a string representing a Python object.
+
+ Strings (i.e. type ``str``) are returned unchanged.
+
+ Byte strings (i.e. type ``bytes``) are returned as UTF-8-decoded strings.
+
+ For other objects, the method ``__str__()`` is called, and the result is
+ returned as a string.
+
+ Parameters
+ ----------
+ obj : object
+ Any Python object
+
+ Returns
+ -------
+ s : str
+ UTF-8-decoded string representation of ``obj``
+
+ """
+ if isinstance(obj, str):
+ return obj
+ try:
+ return obj.decode(encoding="utf-8")
+ except AttributeError: # obj is not bytes-like
+ return str(obj)
+
+
+def _check_argument(param, options, value, prefix=False):
+ """Raise if value for param is not in options."""
+ if prefix and value is not None:
+ failure = not any(value.startswith(p) for p in options if isinstance(p, str))
+ else:
+ failure = value not in options
+ if failure:
+ raise ValueError(
+ f"The value for `{param}` must be one of {options}, "
+ f"but {repr(value)} was passed."
+ )
+ return value
+
+
+def _assign_default_kwargs(kws, call_func, source_func):
+ """Assign default kwargs for call_func using values from source_func."""
+ # This exists so that axes-level functions and figure-level functions can
+ # both call a Plotter method while having the default kwargs be defined in
+ # the signature of the axes-level function.
+ # An alternative would be to have a decorator on the method that sets its
+ # defaults based on those defined in the axes-level function.
+ # Then the figure-level function would not need to worry about defaults.
+ # I am not sure which is better.
+ needed = inspect.signature(call_func).parameters
+ defaults = inspect.signature(source_func).parameters
+
+ for param in needed:
+ if param in defaults and param not in kws:
+ kws[param] = defaults[param].default
+
+ return kws
+
+
+def adjust_legend_subtitles(legend):
+ """
+ Make invisible-handle "subtitles" entries look more like titles.
+
+ Note: This function is not part of the public API and may be changed or removed.
+
+ """
+ # Legend title not in rcParams until 3.0
+ font_size = plt.rcParams.get("legend.title_fontsize", None)
+ hpackers = legend.findobj(mpl.offsetbox.VPacker)[0].get_children()
+ for hpack in hpackers:
+ draw_area, text_area = hpack.get_children()
+ handles = draw_area.get_children()
+ if not all(artist.get_visible() for artist in handles):
+ draw_area.set_width(0)
+ for text in text_area.get_children():
+ if font_size is not None:
+ text.set_size(font_size)
+
+
+def _deprecate_ci(errorbar, ci):
+ """
+ Warn on usage of ci= and convert to appropriate errorbar= arg.
+
+ ci was deprecated when errorbar was added in 0.12. It should not be removed
+ completely for some time, but it can be moved out of function definitions
+ (and extracted from kwargs) after one cycle.
+
+ """
+ if ci is not deprecated and ci != "deprecated":
+ if ci is None:
+ errorbar = None
+ elif ci == "sd":
+ errorbar = "sd"
+ else:
+ errorbar = ("ci", ci)
+ msg = (
+ "\n\nThe `ci` parameter is deprecated. "
+ f"Use `errorbar={repr(errorbar)}` for the same effect.\n"
+ )
+ warnings.warn(msg, FutureWarning, stacklevel=3)
+
+ return errorbar
+
+
+def _get_transform_functions(ax, axis):
+ """Return the forward and inverse transforms for a given axis."""
+ axis_obj = getattr(ax, f"{axis}axis")
+ transform = axis_obj.get_transform()
+ return transform.transform, transform.inverted().transform
+
+
+@contextmanager
+def _disable_autolayout():
+ """Context manager for preventing rc-controlled auto-layout behavior."""
+ # This is a workaround for an issue in matplotlib, for details see
+ # https://github.com/mwaskom/seaborn/issues/2914
+ # The only affect of this rcParam is to set the default value for
+ # layout= in plt.figure, so we could just do that instead.
+ # But then we would need to own the complexity of the transition
+ # from tight_layout=True -> layout="tight". This seems easier,
+ # but can be removed when (if) that is simpler on the matplotlib side,
+ # or if the layout algorithms are improved to handle figure legends.
+ orig_val = mpl.rcParams["figure.autolayout"]
+ try:
+ mpl.rcParams["figure.autolayout"] = False
+ yield
+ finally:
+ mpl.rcParams["figure.autolayout"] = orig_val
+
+
+def _version_predates(lib: ModuleType, version: str) -> bool:
+ """Helper function for checking version compatibility."""
+ return Version(lib.__version__) < Version(version)
+
+
+def _scatter_legend_artist(**kws):
+
+ kws = normalize_kwargs(kws, mpl.collections.PathCollection)
+
+ edgecolor = kws.pop("edgecolor", None)
+ rc = mpl.rcParams
+ line_kws = {
+ "linestyle": "",
+ "marker": kws.pop("marker", "o"),
+ "markersize": np.sqrt(kws.pop("s", rc["lines.markersize"] ** 2)),
+ "markerfacecolor": kws.pop("facecolor", kws.get("color")),
+ "markeredgewidth": kws.pop("linewidth", 0),
+ **kws,
+ }
+
+ if edgecolor is not None:
+ if edgecolor == "face":
+ line_kws["markeredgecolor"] = line_kws["markerfacecolor"]
+ else:
+ line_kws["markeredgecolor"] = edgecolor
+
+ return mpl.lines.Line2D([], [], **line_kws)
+
+
+def _get_patch_legend_artist(fill):
+
+ def legend_artist(**kws):
+
+ color = kws.pop("color", None)
+ if color is not None:
+ if fill:
+ kws["facecolor"] = color
+ else:
+ kws["edgecolor"] = color
+ kws["facecolor"] = "none"
+
+ return mpl.patches.Rectangle((0, 0), 0, 0, **kws)
+
+ return legend_artist
diff --git a/seaborn/widgets.py b/seaborn/widgets.py
new file mode 100644
index 0000000000000000000000000000000000000000..502812af57f5fa2c7e8163c33f472b594f506c79
--- /dev/null
+++ b/seaborn/widgets.py
@@ -0,0 +1,426 @@
+import numpy as np
+import matplotlib.pyplot as plt
+from matplotlib.colors import LinearSegmentedColormap
+
+try:
+ from ipywidgets import interact, FloatSlider, IntSlider
+except ImportError:
+ def interact(f):
+ msg = "Interactive palettes require `ipywidgets`, which is not installed."
+ raise ImportError(msg)
+
+from .miscplot import palplot
+from .palettes import (color_palette, dark_palette, light_palette,
+ diverging_palette, cubehelix_palette)
+
+
+__all__ = ["choose_colorbrewer_palette", "choose_cubehelix_palette",
+ "choose_dark_palette", "choose_light_palette",
+ "choose_diverging_palette"]
+
+
+def _init_mutable_colormap():
+ """Create a matplotlib colormap that will be updated by the widgets."""
+ greys = color_palette("Greys", 256)
+ cmap = LinearSegmentedColormap.from_list("interactive", greys)
+ cmap._init()
+ cmap._set_extremes()
+ return cmap
+
+
+def _update_lut(cmap, colors):
+ """Change the LUT values in a matplotlib colormap in-place."""
+ cmap._lut[:256] = colors
+ cmap._set_extremes()
+
+
+def _show_cmap(cmap):
+ """Show a continuous matplotlib colormap."""
+ from .rcmod import axes_style # Avoid circular import
+ with axes_style("white"):
+ f, ax = plt.subplots(figsize=(8.25, .75))
+ ax.set(xticks=[], yticks=[])
+ x = np.linspace(0, 1, 256)[np.newaxis, :]
+ ax.pcolormesh(x, cmap=cmap)
+
+
+def choose_colorbrewer_palette(data_type, as_cmap=False):
+ """Select a palette from the ColorBrewer set.
+
+ These palettes are built into matplotlib and can be used by name in
+ many seaborn functions, or by passing the object returned by this function.
+
+ Parameters
+ ----------
+ data_type : {'sequential', 'diverging', 'qualitative'}
+ This describes the kind of data you want to visualize. See the seaborn
+ color palette docs for more information about how to choose this value.
+ Note that you can pass substrings (e.g. 'q' for 'qualitative.
+
+ as_cmap : bool
+ If True, the return value is a matplotlib colormap rather than a
+ list of discrete colors.
+
+ Returns
+ -------
+ pal or cmap : list of colors or matplotlib colormap
+ Object that can be passed to plotting functions.
+
+ See Also
+ --------
+ dark_palette : Create a sequential palette with dark low values.
+ light_palette : Create a sequential palette with bright low values.
+ diverging_palette : Create a diverging palette from selected colors.
+ cubehelix_palette : Create a sequential palette or colormap using the
+ cubehelix system.
+
+
+ """
+ if data_type.startswith("q") and as_cmap:
+ raise ValueError("Qualitative palettes cannot be colormaps.")
+
+ pal = []
+ if as_cmap:
+ cmap = _init_mutable_colormap()
+
+ if data_type.startswith("s"):
+ opts = ["Greys", "Reds", "Greens", "Blues", "Oranges", "Purples",
+ "BuGn", "BuPu", "GnBu", "OrRd", "PuBu", "PuRd", "RdPu", "YlGn",
+ "PuBuGn", "YlGnBu", "YlOrBr", "YlOrRd"]
+ variants = ["regular", "reverse", "dark"]
+
+ @interact
+ def choose_sequential(name=opts, n=(2, 18),
+ desat=FloatSlider(min=0, max=1, value=1),
+ variant=variants):
+ if variant == "reverse":
+ name += "_r"
+ elif variant == "dark":
+ name += "_d"
+
+ if as_cmap:
+ colors = color_palette(name, 256, desat)
+ _update_lut(cmap, np.c_[colors, np.ones(256)])
+ _show_cmap(cmap)
+ else:
+ pal[:] = color_palette(name, n, desat)
+ palplot(pal)
+
+ elif data_type.startswith("d"):
+ opts = ["RdBu", "RdGy", "PRGn", "PiYG", "BrBG",
+ "RdYlBu", "RdYlGn", "Spectral"]
+ variants = ["regular", "reverse"]
+
+ @interact
+ def choose_diverging(name=opts, n=(2, 16),
+ desat=FloatSlider(min=0, max=1, value=1),
+ variant=variants):
+ if variant == "reverse":
+ name += "_r"
+ if as_cmap:
+ colors = color_palette(name, 256, desat)
+ _update_lut(cmap, np.c_[colors, np.ones(256)])
+ _show_cmap(cmap)
+ else:
+ pal[:] = color_palette(name, n, desat)
+ palplot(pal)
+
+ elif data_type.startswith("q"):
+ opts = ["Set1", "Set2", "Set3", "Paired", "Accent",
+ "Pastel1", "Pastel2", "Dark2"]
+
+ @interact
+ def choose_qualitative(name=opts, n=(2, 16),
+ desat=FloatSlider(min=0, max=1, value=1)):
+ pal[:] = color_palette(name, n, desat)
+ palplot(pal)
+
+ if as_cmap:
+ return cmap
+ return pal
+
+
+def choose_dark_palette(input="husl", as_cmap=False):
+ """Launch an interactive widget to create a dark sequential palette.
+
+ This corresponds with the :func:`dark_palette` function. This kind
+ of palette is good for data that range between relatively uninteresting
+ low values and interesting high values.
+
+ Requires IPython 2+ and must be used in the notebook.
+
+ Parameters
+ ----------
+ input : {'husl', 'hls', 'rgb'}
+ Color space for defining the seed value. Note that the default is
+ different than the default input for :func:`dark_palette`.
+ as_cmap : bool
+ If True, the return value is a matplotlib colormap rather than a
+ list of discrete colors.
+
+ Returns
+ -------
+ pal or cmap : list of colors or matplotlib colormap
+ Object that can be passed to plotting functions.
+
+ See Also
+ --------
+ dark_palette : Create a sequential palette with dark low values.
+ light_palette : Create a sequential palette with bright low values.
+ cubehelix_palette : Create a sequential palette or colormap using the
+ cubehelix system.
+
+ """
+ pal = []
+ if as_cmap:
+ cmap = _init_mutable_colormap()
+
+ if input == "rgb":
+ @interact
+ def choose_dark_palette_rgb(r=(0., 1.),
+ g=(0., 1.),
+ b=(0., 1.),
+ n=(3, 17)):
+ color = r, g, b
+ if as_cmap:
+ colors = dark_palette(color, 256, input="rgb")
+ _update_lut(cmap, colors)
+ _show_cmap(cmap)
+ else:
+ pal[:] = dark_palette(color, n, input="rgb")
+ palplot(pal)
+
+ elif input == "hls":
+ @interact
+ def choose_dark_palette_hls(h=(0., 1.),
+ l=(0., 1.), # noqa: E741
+ s=(0., 1.),
+ n=(3, 17)):
+ color = h, l, s
+ if as_cmap:
+ colors = dark_palette(color, 256, input="hls")
+ _update_lut(cmap, colors)
+ _show_cmap(cmap)
+ else:
+ pal[:] = dark_palette(color, n, input="hls")
+ palplot(pal)
+
+ elif input == "husl":
+ @interact
+ def choose_dark_palette_husl(h=(0, 359),
+ s=(0, 99),
+ l=(0, 99), # noqa: E741
+ n=(3, 17)):
+ color = h, s, l
+ if as_cmap:
+ colors = dark_palette(color, 256, input="husl")
+ _update_lut(cmap, colors)
+ _show_cmap(cmap)
+ else:
+ pal[:] = dark_palette(color, n, input="husl")
+ palplot(pal)
+
+ if as_cmap:
+ return cmap
+ return pal
+
+
+def choose_light_palette(input="husl", as_cmap=False):
+ """Launch an interactive widget to create a light sequential palette.
+
+ This corresponds with the :func:`light_palette` function. This kind
+ of palette is good for data that range between relatively uninteresting
+ low values and interesting high values.
+
+ Requires IPython 2+ and must be used in the notebook.
+
+ Parameters
+ ----------
+ input : {'husl', 'hls', 'rgb'}
+ Color space for defining the seed value. Note that the default is
+ different than the default input for :func:`light_palette`.
+ as_cmap : bool
+ If True, the return value is a matplotlib colormap rather than a
+ list of discrete colors.
+
+ Returns
+ -------
+ pal or cmap : list of colors or matplotlib colormap
+ Object that can be passed to plotting functions.
+
+ See Also
+ --------
+ light_palette : Create a sequential palette with bright low values.
+ dark_palette : Create a sequential palette with dark low values.
+ cubehelix_palette : Create a sequential palette or colormap using the
+ cubehelix system.
+
+ """
+ pal = []
+ if as_cmap:
+ cmap = _init_mutable_colormap()
+
+ if input == "rgb":
+ @interact
+ def choose_light_palette_rgb(r=(0., 1.),
+ g=(0., 1.),
+ b=(0., 1.),
+ n=(3, 17)):
+ color = r, g, b
+ if as_cmap:
+ colors = light_palette(color, 256, input="rgb")
+ _update_lut(cmap, colors)
+ _show_cmap(cmap)
+ else:
+ pal[:] = light_palette(color, n, input="rgb")
+ palplot(pal)
+
+ elif input == "hls":
+ @interact
+ def choose_light_palette_hls(h=(0., 1.),
+ l=(0., 1.), # noqa: E741
+ s=(0., 1.),
+ n=(3, 17)):
+ color = h, l, s
+ if as_cmap:
+ colors = light_palette(color, 256, input="hls")
+ _update_lut(cmap, colors)
+ _show_cmap(cmap)
+ else:
+ pal[:] = light_palette(color, n, input="hls")
+ palplot(pal)
+
+ elif input == "husl":
+ @interact
+ def choose_light_palette_husl(h=(0, 359),
+ s=(0, 99),
+ l=(0, 99), # noqa: E741
+ n=(3, 17)):
+ color = h, s, l
+ if as_cmap:
+ colors = light_palette(color, 256, input="husl")
+ _update_lut(cmap, colors)
+ _show_cmap(cmap)
+ else:
+ pal[:] = light_palette(color, n, input="husl")
+ palplot(pal)
+
+ if as_cmap:
+ return cmap
+ return pal
+
+
+def choose_diverging_palette(as_cmap=False):
+ """Launch an interactive widget to choose a diverging color palette.
+
+ This corresponds with the :func:`diverging_palette` function. This kind
+ of palette is good for data that range between interesting low values
+ and interesting high values with a meaningful midpoint. (For example,
+ change scores relative to some baseline value).
+
+ Requires IPython 2+ and must be used in the notebook.
+
+ Parameters
+ ----------
+ as_cmap : bool
+ If True, the return value is a matplotlib colormap rather than a
+ list of discrete colors.
+
+ Returns
+ -------
+ pal or cmap : list of colors or matplotlib colormap
+ Object that can be passed to plotting functions.
+
+ See Also
+ --------
+ diverging_palette : Create a diverging color palette or colormap.
+ choose_colorbrewer_palette : Interactively choose palettes from the
+ colorbrewer set, including diverging palettes.
+
+ """
+ pal = []
+ if as_cmap:
+ cmap = _init_mutable_colormap()
+
+ @interact
+ def choose_diverging_palette(
+ h_neg=IntSlider(min=0,
+ max=359,
+ value=220),
+ h_pos=IntSlider(min=0,
+ max=359,
+ value=10),
+ s=IntSlider(min=0, max=99, value=74),
+ l=IntSlider(min=0, max=99, value=50), # noqa: E741
+ sep=IntSlider(min=1, max=50, value=10),
+ n=(2, 16),
+ center=["light", "dark"]
+ ):
+ if as_cmap:
+ colors = diverging_palette(h_neg, h_pos, s, l, sep, 256, center)
+ _update_lut(cmap, colors)
+ _show_cmap(cmap)
+ else:
+ pal[:] = diverging_palette(h_neg, h_pos, s, l, sep, n, center)
+ palplot(pal)
+
+ if as_cmap:
+ return cmap
+ return pal
+
+
+def choose_cubehelix_palette(as_cmap=False):
+ """Launch an interactive widget to create a sequential cubehelix palette.
+
+ This corresponds with the :func:`cubehelix_palette` function. This kind
+ of palette is good for data that range between relatively uninteresting
+ low values and interesting high values. The cubehelix system allows the
+ palette to have more hue variance across the range, which can be helpful
+ for distinguishing a wider range of values.
+
+ Requires IPython 2+ and must be used in the notebook.
+
+ Parameters
+ ----------
+ as_cmap : bool
+ If True, the return value is a matplotlib colormap rather than a
+ list of discrete colors.
+
+ Returns
+ -------
+ pal or cmap : list of colors or matplotlib colormap
+ Object that can be passed to plotting functions.
+
+ See Also
+ --------
+ cubehelix_palette : Create a sequential palette or colormap using the
+ cubehelix system.
+
+ """
+ pal = []
+ if as_cmap:
+ cmap = _init_mutable_colormap()
+
+ @interact
+ def choose_cubehelix(n_colors=IntSlider(min=2, max=16, value=9),
+ start=FloatSlider(min=0, max=3, value=0),
+ rot=FloatSlider(min=-1, max=1, value=.4),
+ gamma=FloatSlider(min=0, max=5, value=1),
+ hue=FloatSlider(min=0, max=1, value=.8),
+ light=FloatSlider(min=0, max=1, value=.85),
+ dark=FloatSlider(min=0, max=1, value=.15),
+ reverse=False):
+
+ if as_cmap:
+ colors = cubehelix_palette(256, start, rot, gamma,
+ hue, light, dark, reverse)
+ _update_lut(cmap, np.c_[colors, np.ones(256)])
+ _show_cmap(cmap)
+ else:
+ pal[:] = cubehelix_palette(n_colors, start, rot, gamma,
+ hue, light, dark, reverse)
+ palplot(pal)
+
+ if as_cmap:
+ return cmap
+ return pal
diff --git a/wandb/__init__.py b/wandb/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4fb4b50214942d249659fdb97ecb321e8c152c6d
--- /dev/null
+++ b/wandb/__init__.py
@@ -0,0 +1,253 @@
+"""Use wandb to track machine learning work.
+
+The most commonly used functions/objects are:
+ - wandb.init — initialize a new run at the top of your training script
+ - wandb.config — track hyperparameters and metadata
+ - wandb.log — log metrics and media over time within your training loop
+
+For guides and examples, see https://docs.wandb.ai.
+
+For scripts and interactive notebooks, see https://github.com/wandb/examples.
+
+For reference documentation, see https://docs.wandb.com/ref/python.
+"""
+__version__ = "0.16.3.dev1"
+_minimum_core_version = "0.17.0b8"
+
+# Used with pypi checks and other messages related to pip
+_wandb_module = "wandb"
+
+from typing import Optional
+
+from wandb.errors import Error
+
+# This needs to be early as other modules call it.
+from wandb.errors.term import termsetup, termlog, termerror, termwarn
+
+from wandb import sdk as wandb_sdk
+
+import wandb
+
+wandb.wandb_lib = wandb_sdk.lib # type: ignore
+
+init = wandb_sdk.init
+setup = wandb_sdk.setup
+_attach = wandb_sdk._attach
+_sync = wandb_sdk._sync
+_teardown = wandb_sdk.teardown
+watch = wandb_sdk.watch
+unwatch = wandb_sdk.unwatch
+finish = wandb_sdk.finish
+join = finish
+login = wandb_sdk.login
+helper = wandb_sdk.helper
+sweep = wandb_sdk.sweep
+controller = wandb_sdk.controller
+require = wandb_sdk.require
+Artifact = wandb_sdk.Artifact
+AlertLevel = wandb_sdk.AlertLevel
+Settings = wandb_sdk.Settings
+Config = wandb_sdk.Config
+
+from wandb.apis import InternalApi, PublicApi
+from wandb.errors import CommError, UsageError
+
+_preinit = wandb.wandb_lib.preinit # type: ignore
+_lazyloader = wandb.wandb_lib.lazyloader # type: ignore
+
+# Call import module hook to set up any needed require hooks
+wandb.sdk.wandb_require._import_module_hook()
+
+from wandb import wandb_torch
+
+# Move this (keras.__init__ expects it at top level)
+from wandb.sdk.data_types._private import _cleanup_media_tmp_dir
+
+_cleanup_media_tmp_dir()
+
+from wandb.data_types import Graph
+from wandb.data_types import Image
+from wandb.data_types import Plotly
+
+# from wandb.data_types import Bokeh # keeping out of top level for now since Bokeh plots have poor UI
+from wandb.data_types import Video
+from wandb.data_types import Audio
+from wandb.data_types import Table
+from wandb.data_types import Html
+from wandb.data_types import Object3D
+from wandb.data_types import Molecule
+from wandb.data_types import Histogram
+from wandb.data_types import Classes
+from wandb.data_types import JoinedTable
+
+from wandb.wandb_agent import agent
+
+# from wandb.core import *
+from wandb.viz import visualize
+from wandb import plot
+from wandb import plots # deprecating this
+from wandb.integration.sagemaker import sagemaker_auth
+from wandb.sdk.internal import profiler
+
+# Artifact import types
+from wandb.sdk.artifacts.artifact_ttl import ArtifactTTL
+
+# Used to make sure we don't use some code in the incorrect process context
+_IS_INTERNAL_PROCESS = False
+
+
+def _set_internal_process(disable=False):
+ global _IS_INTERNAL_PROCESS
+ if _IS_INTERNAL_PROCESS is None:
+ return
+ if disable:
+ _IS_INTERNAL_PROCESS = None
+ return
+ _IS_INTERNAL_PROCESS = True
+
+
+def _assert_is_internal_process():
+ if _IS_INTERNAL_PROCESS is None:
+ return
+ assert _IS_INTERNAL_PROCESS
+
+
+def _assert_is_user_process():
+ if _IS_INTERNAL_PROCESS is None:
+ return
+ assert not _IS_INTERNAL_PROCESS
+
+
+# toplevel:
+# save()
+# restore()
+# login()
+# sweep()
+# agent()
+
+# globals
+Api = PublicApi
+api = InternalApi()
+run: Optional["wandb_sdk.wandb_run.Run"] = None
+config = _preinit.PreInitObject("wandb.config", wandb_sdk.wandb_config.Config)
+summary = _preinit.PreInitObject("wandb.summary", wandb_sdk.wandb_summary.Summary)
+log = _preinit.PreInitCallable("wandb.log", wandb_sdk.wandb_run.Run.log) # type: ignore
+save = _preinit.PreInitCallable("wandb.save", wandb_sdk.wandb_run.Run.save) # type: ignore
+restore = wandb_sdk.wandb_run.restore
+use_artifact = _preinit.PreInitCallable(
+ "wandb.use_artifact", wandb_sdk.wandb_run.Run.use_artifact # type: ignore
+)
+log_artifact = _preinit.PreInitCallable(
+ "wandb.log_artifact", wandb_sdk.wandb_run.Run.log_artifact # type: ignore
+)
+log_model = _preinit.PreInitCallable(
+ "wandb.log_model", wandb_sdk.wandb_run.Run.log_model # type: ignore
+)
+use_model = _preinit.PreInitCallable(
+ "wandb.use_model", wandb_sdk.wandb_run.Run.use_model # type: ignore
+)
+link_model = _preinit.PreInitCallable(
+ "wandb.link_model", wandb_sdk.wandb_run.Run.link_model # type: ignore
+)
+define_metric = _preinit.PreInitCallable(
+ "wandb.define_metric", wandb_sdk.wandb_run.Run.define_metric # type: ignore
+)
+
+mark_preempting = _preinit.PreInitCallable(
+ "wandb.mark_preempting", wandb_sdk.wandb_run.Run.mark_preempting # type: ignore
+)
+
+plot_table = _preinit.PreInitCallable(
+ "wandb.plot_table", wandb_sdk.wandb_run.Run.plot_table
+)
+alert = _preinit.PreInitCallable("wandb.alert", wandb_sdk.wandb_run.Run.alert) # type: ignore
+
+# record of patched libraries
+patched = {"tensorboard": [], "keras": [], "gym": []} # type: ignore
+
+keras = _lazyloader.LazyLoader("wandb.keras", globals(), "wandb.integration.keras")
+sklearn = _lazyloader.LazyLoader("wandb.sklearn", globals(), "wandb.sklearn")
+tensorflow = _lazyloader.LazyLoader(
+ "wandb.tensorflow", globals(), "wandb.integration.tensorflow"
+)
+xgboost = _lazyloader.LazyLoader(
+ "wandb.xgboost", globals(), "wandb.integration.xgboost"
+)
+catboost = _lazyloader.LazyLoader(
+ "wandb.catboost", globals(), "wandb.integration.catboost"
+)
+tensorboard = _lazyloader.LazyLoader(
+ "wandb.tensorboard", globals(), "wandb.integration.tensorboard"
+)
+gym = _lazyloader.LazyLoader("wandb.gym", globals(), "wandb.integration.gym")
+lightgbm = _lazyloader.LazyLoader(
+ "wandb.lightgbm", globals(), "wandb.integration.lightgbm"
+)
+docker = _lazyloader.LazyLoader("wandb.docker", globals(), "wandb.docker")
+jupyter = _lazyloader.LazyLoader("wandb.jupyter", globals(), "wandb.jupyter")
+sacred = _lazyloader.LazyLoader("wandb.sacred", globals(), "wandb.integration.sacred")
+
+
+def ensure_configured():
+ global api
+ api = InternalApi()
+
+
+def set_trace():
+ import pdb # TODO: support other debuggers
+
+ # frame = sys._getframe().f_back
+ pdb.set_trace() # TODO: pass the parent stack...
+
+
+def load_ipython_extension(ipython):
+ ipython.register_magics(wandb.jupyter.WandBMagics)
+
+
+if wandb_sdk.lib.ipython.in_notebook():
+ from IPython import get_ipython # type: ignore[import-not-found]
+
+ load_ipython_extension(get_ipython())
+
+
+from .analytics import Sentry as _Sentry
+
+if "dev" in __version__:
+ import os
+
+ os.environ["WANDB_ERROR_REPORTING"] = os.environ.get(
+ "WANDB_ERROR_REPORTING", "false"
+ )
+
+_sentry = _Sentry()
+_sentry.setup()
+
+
+__all__ = (
+ "__version__",
+ "init",
+ "setup",
+ "save",
+ "sweep",
+ "controller",
+ "agent",
+ "config",
+ "log",
+ "summary",
+ "join",
+ "Api",
+ "Graph",
+ "Image",
+ "Plotly",
+ "Video",
+ "Audio",
+ "Table",
+ "Html",
+ "Object3D",
+ "Molecule",
+ "Histogram",
+ "ArtifactTTL",
+ "log_model",
+ "use_model",
+ "link_model",
+)
diff --git a/wandb/__main__.py b/wandb/__main__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e32b1ef386cb475c83b09fb527a9621a7370847a
--- /dev/null
+++ b/wandb/__main__.py
@@ -0,0 +1,3 @@
+from wandb.cli import cli
+
+cli.cli(prog_name="python -m wandb")
diff --git a/wandb/_globals.py b/wandb/_globals.py
new file mode 100644
index 0000000000000000000000000000000000000000..7cbf0c90b7f36d53ed0ad69a9f2ab6564ae8068e
--- /dev/null
+++ b/wandb/_globals.py
@@ -0,0 +1,19 @@
+# WARNING: This is an anti-pattern file and we should avoid
+# adding to it and remove entries whenever possible. This file
+# contains global objects which need to be referenced by multiple
+# submodules. If you need a global object, seriously reconsider. This
+# file is intended to be a stop gap to help during code migrations (eg.
+# when moving to typing a module) to avoid circular references. Anything
+# added here is pure tech debt. Use with care. - Tim
+
+_glob_datatypes_callback = None
+
+
+def _datatypes_set_callback(cb):
+ global _glob_datatypes_callback
+ _glob_datatypes_callback = cb
+
+
+def _datatypes_callback(fname):
+ if _glob_datatypes_callback:
+ _glob_datatypes_callback(fname)
diff --git a/wandb/agents/__init__.py b/wandb/agents/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/agents/pyagent.py b/wandb/agents/pyagent.py
new file mode 100644
index 0000000000000000000000000000000000000000..c4825555fee3e329d2fb375af48fb917b5f09b5b
--- /dev/null
+++ b/wandb/agents/pyagent.py
@@ -0,0 +1,355 @@
+"""Agent - Agent object.
+
+Manage wandb agent.
+
+"""
+
+
+import ctypes
+import logging
+import os
+import queue
+import socket
+import threading
+import time
+
+import wandb
+from wandb import wandb_sdk
+from wandb.apis import InternalApi
+from wandb.sdk.launch.sweeps import utils as sweep_utils
+
+logger = logging.getLogger(__name__)
+
+
+def _terminate_thread(thread):
+ if not thread.is_alive():
+ return
+ if hasattr(thread, "_terminated"):
+ return
+ thread._terminated = True
+ tid = getattr(thread, "_thread_id", None)
+ if tid is None:
+ for k, v in threading._active.items():
+ if v is thread:
+ tid = k
+ if tid is None:
+ # This should never happen
+ return
+ logger.debug(f"Terminating thread: {tid}")
+ res = ctypes.pythonapi.PyThreadState_SetAsyncExc(
+ ctypes.c_long(tid), ctypes.py_object(Exception)
+ )
+ if res == 0:
+ # This should never happen
+ return
+ elif res != 1:
+ # Revert
+ logger.debug(f"Termination failed for thread {tid}")
+ ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(tid), None)
+
+
+class Job:
+ def __init__(self, command):
+ self.command = command
+ job_type = command.get("type")
+ self.type = job_type
+ self.run_id = command.get("run_id")
+ self.config = command.get("args")
+
+ def __repr__(self):
+ if self.type == "run":
+ return f"Job({self.run_id},{self.config})"
+ elif self.type == "stop":
+ return f"stop({self.run_id})"
+ else:
+ return "exit"
+
+
+class RunStatus:
+ QUEUED = "QUEUED"
+ RUNNING = "RUNNING"
+ STOPPED = "STOPPED"
+ ERRORED = "ERRORED"
+ DONE = "DONE"
+
+
+class Agent:
+ FLAPPING_MAX_SECONDS = 60
+ FLAPPING_MAX_FAILURES = 3
+ MAX_INITIAL_FAILURES = 5
+
+ def __init__(
+ self, sweep_id=None, project=None, entity=None, function=None, count=None
+ ):
+ self._sweep_path = sweep_id
+ self._sweep_id = None
+ self._project = project
+ self._entity = entity
+ self._function = function
+ self._count = count
+ # glob_config = os.path.expanduser('~/.config/wandb/settings')
+ # loc_config = 'wandb/settings'
+ # files = (glob_config, loc_config)
+ self._api = InternalApi()
+ self._agent_id = None
+ self._max_initial_failures = wandb.env.get_agent_max_initial_failures(
+ self.MAX_INITIAL_FAILURES
+ )
+ # if the directory to log to is not set, set it
+ if os.environ.get(wandb.env.DIR) is None:
+ os.environ[wandb.env.DIR] = os.path.abspath(os.getcwd())
+
+ def _init(self):
+ # These are not in constructor so that Agent instance can be rerun
+ self._run_threads = {}
+ self._run_status = {}
+ self._queue = queue.Queue()
+ self._exit_flag = False
+ self._exceptions = {}
+ self._start_time = time.time()
+
+ def _register(self):
+ logger.debug("Agent._register()")
+ agent = self._api.register_agent(socket.gethostname(), sweep_id=self._sweep_id)
+ self._agent_id = agent["id"]
+ logger.debug(f"agent_id = {self._agent_id}")
+
+ def _setup(self):
+ logger.debug("Agent._setup()")
+ self._init()
+ parts = dict(entity=self._entity, project=self._project, name=self._sweep_path)
+ err = sweep_utils.parse_sweep_id(parts)
+ if err:
+ wandb.termerror(err)
+ return
+ entity = parts.get("entity") or self._entity
+ project = parts.get("project") or self._project
+ sweep_id = parts.get("name") or self._sweep_id
+ if sweep_id:
+ os.environ[wandb.env.SWEEP_ID] = sweep_id
+ if entity:
+ wandb.env.set_entity(entity)
+ if project:
+ wandb.env.set_project(project)
+ if sweep_id:
+ self._sweep_id = sweep_id
+ self._register()
+
+ def _stop_run(self, run_id):
+ logger.debug(f"Stopping run {run_id}.")
+ self._run_status[run_id] = RunStatus.STOPPED
+ thread = self._run_threads.get(run_id)
+ if thread:
+ _terminate_thread(thread)
+
+ def _stop_all_runs(self):
+ logger.debug("Stopping all runs.")
+ for run in list(self._run_threads.keys()):
+ self._stop_run(run)
+
+ def _exit(self):
+ self._stop_all_runs()
+ self._exit_flag = True
+ # _terminate_thread(self._main_thread)
+
+ def _heartbeat(self):
+ while True:
+ if self._exit_flag:
+ return
+ # if not self._main_thread.is_alive():
+ # return
+ run_status = {
+ run: True
+ for run, status in self._run_status.items()
+ if status in (RunStatus.QUEUED, RunStatus.RUNNING)
+ }
+ commands = self._api.agent_heartbeat(self._agent_id, {}, run_status)
+ if commands:
+ job = Job(commands[0])
+ logger.debug(f"Job received: {job}")
+ if job.type in ["run", "resume"]:
+ self._queue.put(job)
+ self._run_status[job.run_id] = RunStatus.QUEUED
+ elif job.type == "stop":
+ self._stop_run(job.run_id)
+ elif job.type == "exit":
+ self._exit()
+ return
+ time.sleep(5)
+
+ def _run_jobs_from_queue(self): # noqa:C901
+ global _INSTANCES
+ _INSTANCES += 1
+ try:
+ waiting = False
+ count = 0
+ while True:
+ if self._exit_flag:
+ return
+ try:
+ try:
+ job = self._queue.get(timeout=5)
+ if self._exit_flag:
+ logger.debug("Exiting main loop due to exit flag.")
+ wandb.termlog("Sweep Agent: Exiting.")
+ return
+ except queue.Empty:
+ if not waiting:
+ logger.debug("Paused.")
+ wandb.termlog("Sweep Agent: Waiting for job.")
+ waiting = True
+ time.sleep(5)
+ if self._exit_flag:
+ logger.debug("Exiting main loop due to exit flag.")
+ wandb.termlog("Sweep Agent: Exiting.")
+ return
+ continue
+ if waiting:
+ logger.debug("Resumed.")
+ wandb.termlog("Job received.")
+ waiting = False
+ count += 1
+ run_id = job.run_id
+ if self._run_status[run_id] == RunStatus.STOPPED:
+ continue
+ logger.debug(f"Spawning new thread for run {run_id}.")
+ thread = threading.Thread(target=self._run_job, args=(job,))
+ self._run_threads[run_id] = thread
+ thread.start()
+ self._run_status[run_id] = RunStatus.RUNNING
+ thread.join()
+ logger.debug(f"Thread joined for run {run_id}.")
+ if self._run_status[run_id] == RunStatus.RUNNING:
+ self._run_status[run_id] = RunStatus.DONE
+ elif self._run_status[run_id] == RunStatus.ERRORED:
+ exc = self._exceptions[run_id]
+ logger.error(f"Run {run_id} errored: {repr(exc)}")
+ wandb.termerror(f"Run {run_id} errored: {repr(exc)}")
+ if os.getenv(wandb.env.AGENT_DISABLE_FLAPPING) == "true":
+ self._exit_flag = True
+ return
+ elif (
+ time.time() - self._start_time < self.FLAPPING_MAX_SECONDS
+ ) and (len(self._exceptions) >= self.FLAPPING_MAX_FAILURES):
+ msg = "Detected {} failed runs in the first {} seconds, killing sweep.".format(
+ self.FLAPPING_MAX_FAILURES, self.FLAPPING_MAX_SECONDS
+ )
+ logger.error(msg)
+ wandb.termerror(msg)
+ wandb.termlog(
+ "To disable this check set WANDB_AGENT_DISABLE_FLAPPING=true"
+ )
+ self._exit_flag = True
+ return
+ if (
+ self._max_initial_failures < len(self._exceptions)
+ and len(self._exceptions) >= count
+ ):
+ msg = "Detected {} failed runs in a row at start, killing sweep.".format(
+ self._max_initial_failures
+ )
+ logger.error(msg)
+ wandb.termerror(msg)
+ wandb.termlog(
+ "To change this value set WANDB_AGENT_MAX_INITIAL_FAILURES=val"
+ )
+ self._exit_flag = True
+ return
+ if self._count and self._count == count:
+ logger.debug("Exiting main loop because max count reached.")
+ self._exit_flag = True
+ return
+ except KeyboardInterrupt:
+ logger.debug("Ctrl + C detected. Stopping sweep.")
+ wandb.termlog("Ctrl + C detected. Stopping sweep.")
+ self._exit()
+ return
+ except Exception as e:
+ if self._exit_flag:
+ logger.debug("Exiting main loop due to exit flag.")
+ wandb.termlog("Sweep Agent: Killed.")
+ return
+ else:
+ raise e
+ finally:
+ _INSTANCES -= 1
+
+ def _run_job(self, job):
+ try:
+ run_id = job.run_id
+
+ config_file = os.path.join(
+ "wandb", "sweep-" + self._sweep_id, "config-" + run_id + ".yaml"
+ )
+ os.environ[wandb.env.RUN_ID] = run_id
+ base_dir = os.environ.get(wandb.env.DIR, "")
+ sweep_param_path = os.path.join(base_dir, config_file)
+ os.environ[wandb.env.SWEEP_PARAM_PATH] = sweep_param_path
+ wandb.wandb_lib.config_util.save_config_file_from_dict(
+ sweep_param_path, job.config
+ )
+ os.environ[wandb.env.SWEEP_ID] = self._sweep_id
+ wandb_sdk.wandb_setup._setup(_reset=True)
+
+ wandb.termlog(f"Agent Starting Run: {run_id} with config:")
+ for k, v in job.config.items():
+ wandb.termlog("\t{}: {}".format(k, v["value"]))
+
+ self._function()
+ wandb.finish()
+ except KeyboardInterrupt as ki:
+ raise ki
+ except Exception as e:
+ wandb.finish(exit_code=1)
+ if self._run_status[run_id] == RunStatus.RUNNING:
+ self._run_status[run_id] = RunStatus.ERRORED
+ self._exceptions[run_id] = e
+ finally:
+ # clean up the environment changes made
+ os.environ.pop(wandb.env.RUN_ID, None)
+ os.environ.pop(wandb.env.SWEEP_ID, None)
+ os.environ.pop(wandb.env.SWEEP_PARAM_PATH, None)
+
+ def run(self):
+ logger.info(
+ "Starting sweep agent: entity={}, project={}, count={}".format(
+ self._entity, self._project, self._count
+ )
+ )
+ self._setup()
+ # self._main_thread = threading.Thread(target=self._run_jobs_from_queue)
+ self._heartbeat_thread = threading.Thread(target=self._heartbeat)
+ self._heartbeat_thread.daemon = True
+ # self._main_thread.start()
+ self._heartbeat_thread.start()
+ # self._main_thread.join()
+ self._run_jobs_from_queue()
+
+
+def pyagent(sweep_id, function, entity=None, project=None, count=None):
+ """Generic agent entrypoint, used for CLI or jupyter.
+
+ Arguments:
+ sweep_id (dict): Sweep ID generated by CLI or sweep API
+ function (func, optional): A function to call instead of the "program"
+ entity (str, optional): W&B Entity
+ project (str, optional): W&B Project
+ count (int, optional): the number of trials to run.
+ """
+ if not callable(function):
+ raise Exception("function paramter must be callable!")
+ agent = Agent(
+ sweep_id,
+ function=function,
+ entity=entity,
+ project=project,
+ count=count,
+ )
+ agent.run()
+
+
+_INSTANCES = 0
+
+
+def is_running():
+ return bool(_INSTANCES)
diff --git a/wandb/analytics/__init__.py b/wandb/analytics/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab265f84acb6b30517cca1a08e24ccdfd90c50e8
--- /dev/null
+++ b/wandb/analytics/__init__.py
@@ -0,0 +1,3 @@
+__all__ = ("Sentry",)
+
+from .sentry import Sentry
diff --git a/wandb/analytics/sentry.py b/wandb/analytics/sentry.py
new file mode 100644
index 0000000000000000000000000000000000000000..5cde185c203192d367bc45673766afd89cc15d44
--- /dev/null
+++ b/wandb/analytics/sentry.py
@@ -0,0 +1,265 @@
+__all__ = ("Sentry",)
+
+
+import atexit
+import functools
+import os
+import pathlib
+import sys
+from types import TracebackType
+from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Type, Union
+from urllib.parse import quote
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+import sentry_sdk # type: ignore
+import sentry_sdk.utils # type: ignore
+
+import wandb
+import wandb.env
+import wandb.util
+
+if TYPE_CHECKING:
+ import wandb.sdk.internal.settings_static
+
+SENTRY_DEFAULT_DSN = (
+ "https://2592b1968ea94cca9b5ef5e348e094a7@o151352.ingest.sentry.io/4504800232407040"
+)
+
+SessionStatus = Literal["ok", "exited", "crashed", "abnormal"]
+
+
+def _safe_noop(func: Callable) -> Callable:
+ """Decorator to ensure that Sentry methods do nothing if disabled and don't raise."""
+
+ @functools.wraps(func)
+ def wrapper(self: Type["Sentry"], *args: Any, **kwargs: Any) -> Any:
+ if self._disabled:
+ return None
+ try:
+ return func(self, *args, **kwargs)
+ except Exception as e:
+ # do not call self.exception here to avoid infinite recursion
+ if func.__name__ != "exception":
+ self.exception(f"Error in {func.__name__}: {e}")
+ return None
+
+ return wrapper
+
+
+class Sentry:
+ _disabled: bool
+
+ def __init__(self) -> None:
+ self._disabled = not wandb.env.error_reporting_enabled()
+ self._sent_messages: set = set()
+
+ self.dsn = os.environ.get(wandb.env.SENTRY_DSN, SENTRY_DEFAULT_DSN)
+
+ self.hub: Optional[sentry_sdk.hub.Hub] = None
+
+ # ensure we always end the Sentry session
+ atexit.register(self.end_session)
+
+ @property
+ def environment(self) -> str:
+ """Return the environment we're running in."""
+ # check if we're in a git repo
+ is_git = pathlib.Path(__file__).parent.parent.parent.joinpath(".git").exists()
+
+ # these match the environments for gorilla
+ return "development" if is_git else "production"
+
+ @_safe_noop
+ def setup(self) -> None:
+ """Setup Sentry SDK.
+
+ We use lower-level APIs (i.e., not sentry_sdk.init) here
+ to avoid the possibility of interfering with the user's
+ own Sentry SDK setup.
+ """
+ client = sentry_sdk.Client(
+ dsn=self.dsn,
+ default_integrations=False,
+ environment=self.environment,
+ release=wandb.__version__,
+ )
+ self.hub = sentry_sdk.Hub(client)
+
+ @_safe_noop
+ def message(self, message: str, repeat: bool = True) -> None:
+ """Send a message to Sentry."""
+ if not repeat and message in self._sent_messages:
+ return
+ self._sent_messages.add(message)
+ self.hub.capture_message(message) # type: ignore
+
+ @_safe_noop
+ def exception(
+ self,
+ exc: Union[
+ str,
+ BaseException,
+ Tuple[
+ Optional[Type[BaseException]],
+ Optional[BaseException],
+ Optional[TracebackType],
+ ],
+ None,
+ ],
+ handled: bool = False,
+ status: Optional["SessionStatus"] = None,
+ ) -> None:
+ """Log an exception to Sentry."""
+ error = Exception(exc) if isinstance(exc, str) else exc
+ # based on self.hub.capture_exception(_exc)
+ if error is not None:
+ exc_info = sentry_sdk.utils.exc_info_from_error(error)
+ else:
+ exc_info = sys.exc_info()
+
+ event, hint = sentry_sdk.utils.event_from_exception(
+ exc_info,
+ client_options=self.hub.client.options, # type: ignore
+ mechanism={"type": "generic", "handled": handled},
+ )
+ try:
+ self.hub.capture_event(event, hint=hint) # type: ignore
+ except Exception:
+ pass
+
+ # if the status is not explicitly set, we'll set it to "crashed" if the exception
+ # was unhandled, or "errored" if it was handled
+ status = status or ("crashed" if not handled else "errored") # type: ignore
+ self.mark_session(status=status)
+
+ client, _ = self.hub._stack[-1] # type: ignore
+ client.flush()
+
+ return None
+
+ def reraise(self, exc: Any) -> None:
+ """Re-raise an exception after logging it to Sentry.
+
+ Use this for top-level exceptions when you want the user to see the traceback.
+
+ Must be called from within an exception handler.
+ """
+ self.exception(exc)
+ # this will messily add this "reraise" function to the stack trace,
+ # but hopefully it's not too bad
+ raise exc.with_traceback(sys.exc_info()[2])
+
+ @_safe_noop
+ def start_session(self) -> None:
+ """Start a new session."""
+ assert self.hub is not None
+ # get the current client and scope
+ _, scope = self.hub._stack[-1]
+ session = scope._session
+
+ # if there's no session, start one
+ if session is None:
+ self.hub.start_session()
+
+ @_safe_noop
+ def end_session(self) -> None:
+ """End the current session."""
+ assert self.hub is not None
+ # get the current client and scope
+ client, scope = self.hub._stack[-1]
+ session = scope._session
+
+ if session is not None and client is not None:
+ self.hub.end_session()
+ client.flush()
+
+ @_safe_noop
+ def mark_session(self, status: Optional["SessionStatus"] = None) -> None:
+ """Mark the current session with a status."""
+ assert self.hub is not None
+ _, scope = self.hub._stack[-1]
+ session = scope._session
+
+ if session is not None:
+ session.update(status=status)
+
+ @_safe_noop
+ def configure_scope(
+ self,
+ tags: Optional[Dict[str, Any]] = None,
+ process_context: Optional[str] = None,
+ ) -> None:
+ """Configure the Sentry scope for the current thread.
+
+ This function should be called at the beginning of every thread that
+ will send events to Sentry. It sets the tags that will be applied to
+ all events sent from this thread. It also tries to start a session
+ if one doesn't already exist for this thread.
+ """
+ assert self.hub is not None
+ settings_tags = (
+ "entity",
+ "project",
+ "run_id",
+ "run_url",
+ "sweep_url",
+ "sweep_id",
+ "deployment",
+ "_disable_service",
+ "_require_core",
+ "launch",
+ )
+
+ with self.hub.configure_scope() as scope:
+ scope.set_tag("platform", wandb.util.get_platform_name())
+
+ # set context
+ if process_context:
+ scope.set_tag("process_context", process_context)
+
+ # apply settings tags
+ if tags is None:
+ return None
+
+ for tag in settings_tags:
+ val = tags.get(tag, None)
+ if val not in (None, ""):
+ scope.set_tag(tag, val)
+
+ if tags.get("_colab", None):
+ python_runtime = "colab"
+ elif tags.get("_jupyter", None):
+ python_runtime = "jupyter"
+ elif tags.get("_ipython", None):
+ python_runtime = "ipython"
+ else:
+ python_runtime = "python"
+ scope.set_tag("python_runtime", python_runtime)
+
+ # Construct run_url and sweep_url given run_id and sweep_id
+ for obj in ("run", "sweep"):
+ obj_id, obj_url = f"{obj}_id", f"{obj}_url"
+ if tags.get(obj_url, None):
+ continue
+
+ try:
+ app_url = wandb.util.app_url(tags["base_url"]) # type: ignore
+ entity, project = (quote(tags[k]) for k in ("entity", "project")) # type: ignore
+ scope.set_tag(
+ obj_url,
+ f"{app_url}/{entity}/{project}/{obj}s/{tags[obj_id]}",
+ )
+ except Exception:
+ pass
+
+ email = tags.get("email")
+ if email:
+ scope.user = {"email": email} # noqa
+
+ # todo: add back the option to pass general tags see: c645f625d1c1a3db4a6b0e2aa8e924fee101904c (wandb/util.py)
+
+ self.start_session()
diff --git a/wandb/apis/__init__.py b/wandb/apis/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4daf1e8773ee37ed9af6499243b96ddb9b79aff8
--- /dev/null
+++ b/wandb/apis/__init__.py
@@ -0,0 +1,48 @@
+"""api."""
+
+from typing import Callable
+
+import requests
+from urllib3.exceptions import InsecureRequestWarning
+
+import wandb
+from wandb import env, util
+
+
+def _disable_ssl() -> Callable[[], None]:
+ # Because third party libraries may also use requests, we monkey patch it globally
+ # and turn off urllib3 warnings instead printing a global warning to the user.
+ wandb.termwarn(
+ "Disabling SSL verification. Connections to this server are not verified and may be insecure!"
+ )
+
+ requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
+ old_merge_environment_settings = requests.Session.merge_environment_settings
+
+ def merge_environment_settings(self, url, proxies, stream, verify, cert):
+ settings = old_merge_environment_settings(
+ self, url, proxies, stream, verify, cert
+ )
+ settings["verify"] = False
+ return settings
+
+ requests.Session.merge_environment_settings = merge_environment_settings
+
+ def reset():
+ requests.Session.merge_environment_settings = old_merge_environment_settings
+
+ return reset
+
+
+if env.ssl_disabled():
+ _disable_ssl()
+
+
+reset_path = util.vendor_setup()
+
+from .internal import Api as InternalApi # noqa
+from .public import Api as PublicApi # noqa
+
+reset_path()
+
+__all__ = ["InternalApi", "PublicApi"]
diff --git a/wandb/apis/attrs.py b/wandb/apis/attrs.py
new file mode 100644
index 0000000000000000000000000000000000000000..1e58161e8303a05d7b3ebf42bc07880bdbb326af
--- /dev/null
+++ b/wandb/apis/attrs.py
@@ -0,0 +1,40 @@
+from typing import Any, MutableMapping
+
+import wandb
+from wandb.sdk.lib import ipython
+
+
+class Attrs:
+ def __init__(self, attrs: MutableMapping[str, Any]):
+ self._attrs = attrs
+
+ def snake_to_camel(self, string):
+ camel = "".join([i.title() for i in string.split("_")])
+ return camel[0].lower() + camel[1:]
+
+ def display(self, height=420, hidden=False) -> bool:
+ """Display this object in jupyter."""
+ html = self.to_html(height, hidden)
+ if html is None:
+ wandb.termwarn("This object does not support `.display()`")
+ return False
+ if ipython.in_jupyter():
+ ipython.display_html(html)
+ return True
+ else:
+ wandb.termwarn(".display() only works in jupyter environments")
+ return False
+
+ def to_html(self, *args, **kwargs):
+ return None
+
+ def __getattr__(self, name):
+ key = self.snake_to_camel(name)
+ if key == "user":
+ raise AttributeError
+ if key in self._attrs.keys():
+ return self._attrs[key]
+ elif name in self._attrs.keys():
+ return self._attrs[name]
+ else:
+ raise AttributeError(f"{repr(self)!r} object has no attribute {name!r}")
diff --git a/wandb/apis/importers/__init__.py b/wandb/apis/importers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..dd28c1feba9e8d15867abb097528ac1e2bb0d272
--- /dev/null
+++ b/wandb/apis/importers/__init__.py
@@ -0,0 +1,4 @@
+from wandb.util import get_module
+
+if get_module("mlflow"):
+ from .mlflow import MlflowImporter, MlflowRun # noqa: F401
diff --git a/wandb/apis/importers/base.py b/wandb/apis/importers/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..0350081abcd9a5443731a4717a3638f64933af6c
--- /dev/null
+++ b/wandb/apis/importers/base.py
@@ -0,0 +1,400 @@
+import json
+import os
+import queue
+import sys
+from dataclasses import dataclass
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+from unittest.mock import patch
+
+from google.protobuf.json_format import ParseDict
+from tqdm import tqdm
+
+import wandb
+from wandb.proto import wandb_internal_pb2 as pb
+from wandb.proto import wandb_settings_pb2
+from wandb.proto import wandb_telemetry_pb2 as telem_pb
+from wandb.sdk.interface.interface import file_policy_to_enum
+from wandb.sdk.interface.interface_queue import InterfaceQueue
+from wandb.sdk.internal import context
+from wandb.sdk.internal.sender import SendManager
+from wandb.sdk.internal.settings_static import SettingsStatic
+from wandb.util import cast_dictlike_to_dict, coalesce
+
+if sys.version_info >= (3, 8):
+ from typing import Protocol
+else:
+ from typing_extensions import Protocol
+
+
+with patch("click.echo"):
+ from wandb.apis.reports import Report
+
+
+class ImporterRun(Protocol):
+ def run_id(self) -> str:
+ ...
+
+ def entity(self) -> str:
+ ...
+
+ def project(self) -> str:
+ ...
+
+ def config(self) -> Dict[str, Any]:
+ ...
+
+ def summary(self) -> Dict[str, float]:
+ ...
+
+ def metrics(self) -> Iterable[Dict[str, float]]:
+ """Metrics for the run.
+
+ We expect metrics in this shape:
+
+ [
+ {'metric1': 1, 'metric2': 1, '_step': 0},
+ {'metric1': 2, 'metric2': 4, '_step': 1},
+ {'metric1': 3, 'metric2': 9, '_step': 2},
+ ...
+ ]
+
+ You can also submit metrics in this shape:
+ [
+ {'metric1': 1, '_step': 0},
+ {'metric2': 1, '_step': 0},
+ {'metric1': 2, '_step': 1},
+ {'metric2': 4, '_step': 1},
+ ...
+ ]
+ """
+ ...
+
+ def run_group(self) -> Optional[str]:
+ ...
+
+ def job_type(self) -> Optional[str]:
+ ...
+
+ def display_name(self) -> str:
+ ...
+
+ def notes(self) -> Optional[str]:
+ ...
+
+ def tags(self) -> Optional[List[str]]:
+ ...
+
+ def artifacts(self) -> Optional[Iterable[wandb.Artifact]]: # type: ignore
+ ...
+
+ def used_artifacts(self) -> Optional[Iterable[wandb.Artifact]]: # type: ignore
+ ...
+
+ def os_version(self) -> Optional[str]:
+ ...
+
+ def python_version(self) -> Optional[str]:
+ ...
+
+ def cuda_version(self) -> Optional[str]:
+ ...
+
+ def program(self) -> Optional[str]:
+ ...
+
+ def host(self) -> Optional[str]:
+ ...
+
+ def username(self) -> Optional[str]:
+ ...
+
+ def executable(self) -> Optional[str]:
+ ...
+
+ def gpus_used(self) -> Optional[str]:
+ ...
+
+ def cpus_used(self) -> Optional[int]: # can we get the model?
+ ...
+
+ def memory_used(self) -> Optional[int]:
+ ...
+
+ def runtime(self) -> Optional[int]:
+ ...
+
+ def start_time(self) -> Optional[int]:
+ ...
+
+ def code_path(self) -> Optional[str]:
+ ...
+
+ def cli_version(self) -> Optional[str]:
+ ...
+
+ def files(self) -> Optional[Iterable[Tuple[str, str]]]:
+ ...
+
+ def logs(self) -> Optional[Iterable[str]]:
+ ...
+
+
+class Importer(Protocol):
+ def collect_runs(self, *args, **kwargs) -> Iterable[ImporterRun]:
+ ...
+
+ def collect_reports(self, *args, **kwargs) -> Iterable[Report]:
+ ...
+
+ def import_run(self, run: ImporterRun) -> None:
+ ...
+
+ def import_report(self, report: Report) -> None:
+ ...
+
+
+@dataclass
+class RecordMaker:
+ run: ImporterRun
+ interface: InterfaceQueue = InterfaceQueue()
+
+ @property
+ def run_dir(self) -> str:
+ return f"./wandb-importer/{self.run.run_id()}"
+
+ def _make_run_record(self) -> pb.Record:
+ run = pb.RunRecord()
+ run.run_id = self.run.run_id()
+ run.entity = self.run.entity()
+ run.project = self.run.project()
+ run.display_name = coalesce(self.run.display_name())
+ run.notes = coalesce(self.run.notes(), "")
+ run.tags.extend(coalesce(self.run.tags(), []))
+ run.start_time.FromMilliseconds(self.run.start_time())
+
+ host = self.run.host()
+ if host is not None:
+ run.host = host
+
+ runtime = self.run.runtime()
+ if runtime is not None:
+ run.runtime = runtime
+
+ run_group = self.run.run_group()
+ if run_group is not None:
+ run.run_group = run_group
+
+ config = self.run.config()
+ if "_wandb" not in config:
+ config["_wandb"] = {}
+
+ # how do I get this automatically?
+ config["_wandb"]["code_path"] = self.run.code_path()
+ config["_wandb"]["python_version"] = self.run.python_version()
+ config["_wandb"]["cli_version"] = self.run.cli_version()
+
+ self.interface._make_config(
+ data=config,
+ obj=run.config,
+ ) # is there a better way?
+ return self.interface._make_record(run=run)
+
+ def _make_output_record(self, line) -> pb.Record:
+ output_raw = pb.OutputRawRecord()
+ output_raw.output_type = pb.OutputRawRecord.OutputType.STDOUT
+ output_raw.line = line
+ return self.interface._make_record(output_raw=output_raw)
+
+ def _make_summary_record(self) -> pb.Record:
+ d: dict = {
+ **self.run.summary(),
+ "_runtime": self.run.runtime(), # quirk of runtime -- it has to be here!
+ # '_timestamp': self.run.start_time()/1000,
+ }
+ d = cast_dictlike_to_dict(d)
+ summary = self.interface._make_summary_from_dict(d)
+ return self.interface._make_record(summary=summary)
+
+ def _make_history_records(self) -> Iterable[pb.Record]:
+ for _, metrics in enumerate(self.run.metrics()):
+ history = pb.HistoryRecord()
+ for k, v in metrics.items():
+ item = history.item.add()
+ item.key = k
+ item.value_json = json.dumps(v)
+ yield self.interface._make_record(history=history)
+
+ def _make_files_record(
+ self,
+ files_dict,
+ ) -> pb.Record:
+ files_record = pb.FilesRecord()
+ for path, policy in files_dict["files"]:
+ f = files_record.files.add()
+ f.path = path
+ f.policy = file_policy_to_enum(policy) # is this always "end"?
+ return self.interface._make_record(files=files_record)
+
+ def _make_metadata_files_record(self) -> pb.Record:
+ files = self.run.files()
+ if files is None:
+ metadata_fname = self._make_metadata_file()
+ files = [(metadata_fname, "end")]
+
+ files_dict = {"files": files}
+ return self._make_files_record(files_dict)
+
+ def _make_artifact_record(self, artifact, use_artifact=False) -> pb.Record:
+ proto = self.interface._make_artifact(artifact)
+ proto.run_id = self.run.run_id()
+ proto.project = self.run.project()
+ proto.entity = self.run.entity()
+ proto.user_created = use_artifact
+ proto.use_after_commit = use_artifact
+ proto.finalize = True
+ for tag in ["latest", "imported"]:
+ proto.aliases.append(tag)
+ return self.interface._make_record(artifact=proto)
+
+ def _make_telem_record(self) -> pb.Record:
+ telem = telem_pb.TelemetryRecord()
+
+ feature = telem_pb.Feature()
+ feature.importer_mlflow = True
+ telem.feature.CopyFrom(feature)
+
+ cli_version = self.run.cli_version()
+ if cli_version:
+ telem.cli_version = cli_version
+
+ python_version = self.run.python_version()
+ if python_version:
+ telem.python_version = python_version
+
+ return self.interface._make_record(telemetry=telem)
+
+ def _make_metadata_file(self) -> str:
+ missing_text = "This data was not captured"
+
+ d = {}
+ d["os"] = coalesce(self.run.os_version(), missing_text)
+ d["python"] = coalesce(self.run.python_version(), missing_text)
+ d["program"] = coalesce(self.run.program(), missing_text)
+ d["cuda"] = coalesce(self.run.cuda_version(), missing_text)
+ d["host"] = coalesce(self.run.host(), missing_text)
+ d["username"] = coalesce(self.run.username(), missing_text)
+ d["executable"] = coalesce(self.run.executable(), missing_text)
+
+ gpus_used = self.run.gpus_used()
+ if gpus_used is not None:
+ d["gpu_devices"] = json.dumps(gpus_used)
+ d["gpu_count"] = json.dumps(len(gpus_used))
+
+ cpus_used = self.run.cpus_used()
+ if cpus_used is not None:
+ d["cpu_count"] = json.dumps(self.run.cpus_used())
+
+ mem_used = self.run.memory_used()
+ if mem_used is not None:
+ d["memory"] = json.dumps({"total": self.run.memory_used()})
+
+ fname = f"{self.run_dir}/files/wandb-metadata.json"
+ with open(fname, "w") as f:
+ f.write(json.dumps(d))
+ return fname
+
+
+def send_run_with_send_manager(
+ run: ImporterRun,
+ overrides: Optional[Dict[str, Any]] = None,
+ settings_override: Optional[SettingsStatic] = None,
+) -> None:
+ # does this need to be here for pmap?
+ if overrides:
+ for k, v in overrides.items():
+ # `lambda: v` won't work!
+ # https://stackoverflow.com/questions/10802002/why-deepcopy-doesnt-create-new-references-to-lambda-function
+ setattr(run, k, lambda v=v: v)
+ _settings_override = coalesce(settings_override, {})
+ rm = RecordMaker(run)
+
+ root_dir = rm.run_dir
+ default_settings = {
+ "files_dir": os.path.join(root_dir, "files"),
+ "root_dir": root_dir,
+ "_start_time": 0,
+ "git_remote": None,
+ "resume": False,
+ "program": None,
+ "ignore_globs": (),
+ "run_id": None,
+ "entity": None,
+ "project": None,
+ "run_group": None,
+ "job_type": None,
+ "run_tags": None,
+ "run_name": None,
+ "run_notes": None,
+ "save_code": None,
+ "email": None,
+ "silent": None,
+ "_offline": None,
+ "_sync": True,
+ "_live_policy_rate_limit": None,
+ "_live_policy_wait_time": None,
+ "disable_job_creation": False,
+ "_async_upload_concurrency_limit": None,
+ }
+ combined_settings = {**default_settings, **_settings_override}
+ settings_message = wandb_settings_pb2.Settings()
+ ParseDict(combined_settings, settings_message)
+
+ settings = SettingsStatic(settings_message)
+
+ record_q: queue.Queue = queue.Queue()
+ result_q: queue.Queue = queue.Queue()
+ interface = InterfaceQueue(record_q=record_q)
+ context_keeper = context.ContextKeeper()
+
+ with SendManager(settings, record_q, result_q, interface, context_keeper) as sm:
+ wandb.termlog(">> Make run record")
+ sm.send(rm._make_run_record())
+
+ wandb.termlog(">> Use Artifacts")
+ used_artifacts = run.used_artifacts()
+ if used_artifacts is not None:
+ for artifact in tqdm(
+ used_artifacts, desc="Used artifacts", unit="artifacts", leave=False
+ ):
+ sm.send(rm._make_artifact_record(artifact, use_artifact=True))
+
+ wandb.termlog(">> Log Artifacts")
+ artifacts = run.artifacts()
+ if artifacts is not None:
+ for artifact in tqdm(
+ artifacts, desc="Logged artifacts", unit="artifacts", leave=False
+ ):
+ sm.send(rm._make_artifact_record(artifact))
+
+ wandb.termlog(">> Log Metadata")
+ sm.send(rm._make_metadata_files_record())
+
+ wandb.termlog(">> Log History")
+ for history_record in tqdm(
+ rm._make_history_records(), desc="History", unit="steps", leave=False
+ ):
+ sm.send(history_record)
+
+ wandb.termlog(">> Log Summary")
+ sm.send(rm._make_summary_record())
+
+ wandb.termlog(">> Log Output")
+ # if hasattr(run, "_logs"):
+ # lines = run._logs
+ lines = run.logs()
+ if lines is not None:
+ for line in tqdm(lines, desc="Stdout", unit="lines", leave=False):
+ sm.send(rm._make_output_record(line))
+
+ wandb.termlog(">> Log Telem")
+ sm.send(rm._make_telem_record())
diff --git a/wandb/apis/importers/mlflow.py b/wandb/apis/importers/mlflow.py
new file mode 100644
index 0000000000000000000000000000000000000000..e4d0f4c78120bbb98d83f0399fa6c89e212fefcc
--- /dev/null
+++ b/wandb/apis/importers/mlflow.py
@@ -0,0 +1,230 @@
+import re
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+from unittest.mock import patch
+
+from packaging.version import Version # type: ignore
+from tqdm.auto import tqdm
+
+import wandb
+from wandb import Artifact
+from wandb.util import coalesce, get_module
+
+from .base import ImporterRun, send_run_with_send_manager
+
+with patch("click.echo"):
+ from wandb.apis.reports import Report
+
+mlflow = get_module(
+ "mlflow",
+ required="To use the MlflowImporter, please install mlflow: `pip install mlflow`",
+)
+
+mlflow_version = Version(mlflow.__version__)
+
+
+class MlflowRun:
+ def __init__(self, run, mlflow_client):
+ self.run = run
+ self.mlflow_client = mlflow_client
+
+ def run_id(self) -> str:
+ return self.run.info.run_id
+
+ def entity(self) -> str:
+ return self.run.info.user_id
+
+ def project(self) -> str:
+ return "imported-from-mlflow"
+
+ def config(self) -> Dict[str, Any]:
+ return self.run.data.params
+
+ def summary(self) -> Dict[str, float]:
+ return self.run.data.metrics
+
+ def metrics(self) -> Iterable[Dict[str, float]]:
+ d: Dict[int, Dict[str, float]] = defaultdict(dict)
+ for k in self.run.data.metrics.keys():
+ metric = self.mlflow_client.get_metric_history(self.run.info.run_id, k)
+ for item in metric:
+ d[item.step][item.key] = item.value
+
+ for k, v in d.items():
+ yield {"_step": k, **v}
+
+ def run_group(self) -> Optional[str]:
+ # this is nesting? Parent at `run.info.tags.get("mlflow.parentRunId")`
+ return f"Experiment {self.run.info.experiment_id}"
+
+ def job_type(self) -> Optional[str]:
+ # Is this the right approach?
+ return f"User {self.run.info.user_id}"
+
+ def display_name(self) -> str:
+ if mlflow_version < Version("1.30.0"):
+ return self.run.data.tags["mlflow.runName"]
+ return self.run.info.run_name
+
+ def notes(self) -> Optional[str]:
+ return self.run.data.tags.get("mlflow.note.content")
+
+ def tags(self) -> Optional[List[str]]:
+ mlflow_tags = {
+ k: v for k, v in self.run.data.tags.items() if not k.startswith("mlflow.")
+ }
+ return [f"{k}={v}" for k, v in mlflow_tags.items()]
+
+ def artifacts(self) -> Optional[Iterable[Artifact]]: # type: ignore
+ if mlflow_version < Version("2.0.0"):
+ dir_path = self.mlflow_client.download_artifacts(
+ run_id=self.run.info.run_id, path=""
+ )
+ else:
+ dir_path = mlflow.artifacts.download_artifacts(run_id=self.run.info.run_id)
+
+ artifact_name = self._handle_incompatible_strings(self.display_name())
+ art = wandb.Artifact(artifact_name, "imported-artifacts")
+ art.add_dir(dir_path)
+
+ return [art]
+
+ def used_artifacts(self) -> Optional[Iterable[Artifact]]: # type: ignore
+ ...
+
+ def os_version(self) -> Optional[str]:
+ ...
+
+ def python_version(self) -> Optional[str]:
+ ...
+
+ def cuda_version(self) -> Optional[str]:
+ ...
+
+ def program(self) -> Optional[str]:
+ ...
+
+ def host(self) -> Optional[str]:
+ ...
+
+ def username(self) -> Optional[str]:
+ ...
+
+ def executable(self) -> Optional[str]:
+ ...
+
+ def gpus_used(self) -> Optional[str]:
+ ...
+
+ def cpus_used(self) -> Optional[int]: # can we get the model?
+ ...
+
+ def memory_used(self) -> Optional[int]:
+ ...
+
+ def runtime(self) -> Optional[int]:
+ end_time = (
+ self.run.info.end_time // 1000
+ if self.run.info.end_time is not None
+ else self.start_time()
+ )
+ return end_time - self.start_time()
+
+ def start_time(self) -> Optional[int]:
+ return self.run.info.start_time // 1000
+
+ def code_path(self) -> Optional[str]:
+ ...
+
+ def cli_version(self) -> Optional[str]:
+ ...
+
+ def files(self) -> Optional[Iterable[Tuple[str, str]]]:
+ ...
+
+ def logs(self) -> Optional[Iterable[str]]:
+ ...
+
+ @staticmethod
+ def _handle_incompatible_strings(s: str) -> str:
+ valid_chars = r"[^a-zA-Z0-9_\-\.]"
+ replacement = "__"
+
+ return re.sub(valid_chars, replacement, s)
+
+
+class MlflowImporter:
+ def __init__(self, mlflow_tracking_uri, mlflow_registry_uri=None) -> None:
+ self.mlflow_tracking_uri = mlflow_tracking_uri
+
+ mlflow.set_tracking_uri(self.mlflow_tracking_uri)
+ if mlflow_registry_uri:
+ mlflow.set_registry_uri(mlflow_registry_uri)
+ self.mlflow_client = mlflow.tracking.MlflowClient(mlflow_tracking_uri)
+
+ def collect_runs(self, limit: Optional[int] = None) -> Iterable[MlflowRun]:
+ if mlflow_version < Version("1.28.0"):
+ experiments = self.mlflow_client.list_experiments()
+ else:
+ experiments = self.mlflow_client.search_experiments()
+
+ runs = (
+ run
+ for exp in experiments
+ for run in self.mlflow_client.search_runs(exp.experiment_id)
+ )
+ for i, run in enumerate(runs):
+ if limit and i >= limit:
+ break
+ yield MlflowRun(run, self.mlflow_client)
+
+ def import_run(
+ self,
+ run: ImporterRun,
+ overrides: Optional[Dict[str, Any]] = None,
+ ) -> None:
+ mlflow.set_tracking_uri(self.mlflow_tracking_uri)
+ send_run_with_send_manager(run, overrides)
+
+ def import_runs(
+ self,
+ runs: Iterable[ImporterRun],
+ overrides: Optional[Dict[str, Any]] = None,
+ pool_kwargs: Optional[Dict[str, Any]] = None,
+ ) -> None:
+ _overrides = coalesce(overrides, {})
+ _pool_kwargs = coalesce(pool_kwargs, {})
+ runs = list(self.collect_runs())
+
+ with ThreadPoolExecutor(**_pool_kwargs) as exc:
+ futures = {
+ exc.submit(self.import_run, run, overrides=_overrides): run
+ for run in runs
+ }
+ with tqdm(desc="Importing runs", total=len(futures), unit="run") as pbar:
+ for future in as_completed(futures):
+ run = futures[future]
+ try:
+ future.result()
+ except Exception as e:
+ wandb.termerror(f"Failed to import {run.display_name()}: {e}")
+ raise e
+ else:
+ pbar.set_description(
+ f"Imported Run: {run.run_group()} {run.display_name()}"
+ )
+ finally:
+ pbar.update(1)
+
+ def import_all_runs(
+ self,
+ limit: Optional[int] = None,
+ overrides: Optional[Dict[str, Any]] = None,
+ pool_kwargs: Optional[Dict[str, Any]] = None,
+ ) -> None:
+ runs = self.collect_runs(limit)
+ self.import_runs(runs, overrides, pool_kwargs)
+
+ def import_report(self, report: Report):
+ raise NotImplementedError("MLFlow does not have a reports concept")
diff --git a/wandb/apis/internal.py b/wandb/apis/internal.py
new file mode 100644
index 0000000000000000000000000000000000000000..e31192831d89056f32101cca4b5c3dae87dc109c
--- /dev/null
+++ b/wandb/apis/internal.py
@@ -0,0 +1,228 @@
+from typing import Any
+
+from wandb.sdk.internal.internal_api import Api as InternalApi
+
+
+class Api:
+ """Internal proxy to the official internal API."""
+
+ # TODO: Move these methods to PublicApi.
+
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ self._api_args = args
+ self._api_kwargs = kwargs
+ self._api = None
+
+ def __getstate__(self):
+ """Use for serializing.
+
+ self._api is not serializable, so it's dropped
+ """
+ state = self.__dict__.copy()
+ del state["_api"]
+ return state
+
+ def __setstate__(self, state):
+ """Used for deserializing.
+
+ Don't need to set self._api because it's constructed when needed.
+ """
+ self.__dict__.update(state)
+ self._api = None
+
+ @property
+ def api(self) -> InternalApi:
+ # This is a property in order to delay construction of Internal API
+ # for as long as possible. If constructed in constructor, then the
+ # whole InternalAPI is started when simply importing wandb.
+ if self._api is None:
+ self._api = InternalApi(*self._api_args, **self._api_kwargs)
+ return self._api
+
+ @property
+ def api_key(self):
+ return self.api.api_key
+
+ @property
+ def api_url(self):
+ return self.api.api_url
+
+ @property
+ def app_url(self):
+ return self.api.app_url
+
+ @property
+ def default_entity(self):
+ return self.api.default_entity
+
+ @property
+ def git(self):
+ return self.api.git
+
+ def file_current(self, *args):
+ return self.api.file_current(*args)
+
+ def download_file(self, *args, **kwargs):
+ return self.api.download_file(*args, **kwargs)
+
+ def download_write_file(self, *args, **kwargs):
+ return self.api.download_write_file(*args, **kwargs)
+
+ def set_current_run_id(self, run_id):
+ return self.api.set_current_run_id(run_id)
+
+ def viewer(self):
+ return self.api.viewer()
+
+ def max_cli_version(self):
+ return self.api.max_cli_version()
+
+ def viewer_server_info(self):
+ return self.api.viewer_server_info()
+
+ def list_projects(self, entity=None):
+ return self.api.list_projects(entity=entity)
+
+ def format_project(self, project):
+ return self.api.format_project(project)
+
+ def upsert_project(self, project, id=None, description=None, entity=None):
+ return self.api.upsert_project(
+ project, id=id, description=description, entity=entity
+ )
+
+ def upsert_run(self, *args, **kwargs):
+ return self.api.upsert_run(*args, **kwargs)
+
+ def settings(self, *args, **kwargs):
+ return self.api.settings(*args, **kwargs)
+
+ def clear_setting(
+ self, key: str, globally: bool = False, persist: bool = False
+ ) -> None:
+ return self.api.clear_setting(key, globally, persist)
+
+ def set_setting(
+ self, key: str, value: Any, globally: bool = False, persist: bool = False
+ ) -> None:
+ return self.api.set_setting(key, value, globally, persist)
+
+ def parse_slug(self, *args, **kwargs):
+ return self.api.parse_slug(*args, **kwargs)
+
+ def download_url(self, *args, **kwargs):
+ return self.api.download_url(*args, **kwargs)
+
+ def download_urls(self, *args, **kwargs):
+ return self.api.download_urls(*args, **kwargs)
+
+ def create_anonymous_api_key(self) -> str:
+ return self.api.create_anonymous_api_key()
+
+ def push(self, *args, **kwargs):
+ return self.api.push(*args, **kwargs)
+
+ def sweep(self, *args, **kwargs):
+ return self.api.sweep(*args, **kwargs)
+
+ def upsert_sweep(self, *args, **kwargs):
+ return self.api.upsert_sweep(*args, **kwargs)
+
+ def set_sweep_state(self, *args, **kwargs):
+ return self.api.set_sweep_state(*args, **kwargs)
+
+ def get_sweep_state(self, *args, **kwargs):
+ return self.api.get_sweep_state(*args, **kwargs)
+
+ def stop_sweep(self, *args, **kwargs):
+ return self.api.stop_sweep(*args, **kwargs)
+
+ def cancel_sweep(self, *args, **kwargs):
+ return self.api.cancel_sweep(*args, **kwargs)
+
+ def pause_sweep(self, *args, **kwargs):
+ return self.api.pause_sweep(*args, **kwargs)
+
+ def resume_sweep(self, *args, **kwargs):
+ return self.api.resume_sweep(*args, **kwargs)
+
+ def register_agent(self, *args, **kwargs):
+ return self.api.register_agent(*args, **kwargs)
+
+ def agent_heartbeat(self, *args, **kwargs):
+ return self.api.agent_heartbeat(*args, **kwargs)
+
+ def use_artifact(self, *args, **kwargs):
+ return self.api.use_artifact(*args, **kwargs)
+
+ def create_artifact(self, *args, **kwargs):
+ return self.api.create_artifact(*args, **kwargs)
+
+ def complete_multipart_upload_artifact(self, *args, **kwargs):
+ return self.api.complete_multipart_upload_artifact(*args, **kwargs)
+
+ def run_config(self, *args, **kwargs):
+ return self.api.run_config(*args, **kwargs)
+
+ def upload_file_retry(self, *args, **kwargs):
+ return self.api.upload_file_retry(*args, **kwargs)
+
+ def upload_multipart_file_chunk_retry(self, *args, **kwargs):
+ return self.api.upload_multipart_file_chunk_retry(*args, **kwargs)
+
+ async def upload_file_retry_async(self, *args, **kwargs):
+ return await self.api.upload_file_retry_async(*args, **kwargs)
+
+ def get_run_info(self, *args, **kwargs):
+ return self.api.get_run_info(*args, **kwargs)
+
+ def get_run_state(self, *args, **kwargs):
+ return self.api.get_run_state(*args, **kwargs)
+
+ def entity_is_team(self, *args, **kwargs):
+ return self.api.entity_is_team(*args, **kwargs)
+
+ def get_project_run_queues(self, *args, **kwargs):
+ return self.api.get_project_run_queues(*args, **kwargs)
+
+ def push_to_run_queue(self, *args, **kwargs):
+ return self.api.push_to_run_queue(*args, **kwargs)
+
+ def pop_from_run_queue(self, *args, **kwargs):
+ return self.api.pop_from_run_queue(*args, **kwargs)
+
+ def ack_run_queue_item(self, *args, **kwargs):
+ return self.api.ack_run_queue_item(*args, **kwargs)
+
+ def create_launch_agent(self, *args, **kwargs):
+ return self.api.create_launch_agent(*args, **kwargs)
+
+ def create_default_resource_config(self, *args, **kwargs):
+ return self.api.create_default_resource_config(*args, **kwargs)
+
+ def create_run_queue(self, *args, **kwargs):
+ return self.api.create_run_queue(*args, **kwargs)
+
+ def update_launch_agent_status(self, *args, **kwargs):
+ return self.api.update_launch_agent_status(*args, **kwargs)
+
+ def launch_agent_introspection(self, *args, **kwargs):
+ return self.api.launch_agent_introspection(*args, **kwargs)
+
+ def fail_run_queue_item_introspection(self, *args, **kwargs):
+ return self.api.fail_run_queue_item_introspection(*args, **kwargs)
+
+ def fail_run_queue_item(self, *args, **kwargs):
+ return self.api.fail_run_queue_item(*args, **kwargs)
+
+ def update_run_queue_item_warning(self, *args, **kwargs):
+ return self.api.update_run_queue_item_warning(*args, **kwargs)
+
+ def get_launch_agent(self, *args, **kwargs):
+ return self.api.get_launch_agent(*args, **kwargs)
+
+ def stop_run(self, *args, **kwargs):
+ return self.api.stop_run(*args, **kwargs)
+
+
+__all__ = ["Api"]
diff --git a/wandb/apis/normalize.py b/wandb/apis/normalize.py
new file mode 100644
index 0000000000000000000000000000000000000000..29b8819139b188d92176e0c2abe4cc962c76e1cb
--- /dev/null
+++ b/wandb/apis/normalize.py
@@ -0,0 +1,89 @@
+"""normalize."""
+
+import ast
+import sys
+from functools import wraps
+from typing import Callable, List, TypeVar
+
+import requests
+from wandb_gql.client import RetryError
+
+from wandb import env
+from wandb.errors import CommError, Error
+
+_F = TypeVar("_F", bound=Callable)
+
+
+def parse_backend_error_messages(response: requests.Response) -> List[str]:
+ errors = []
+ try:
+ data = response.json()
+ except ValueError:
+ return errors
+
+ if "errors" in data and isinstance(data["errors"], list):
+ for error in data["errors"]:
+ # Our tests and potentially some api endpoints return a string error?
+ if isinstance(error, str):
+ error = {"message": error}
+ if "message" in error:
+ errors.append(error["message"])
+ return errors
+
+
+def normalize_exceptions(func: _F) -> _F:
+ """Function decorator for catching common errors and re-raising as wandb.Error."""
+
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ message = "Whoa, you found a bug."
+ try:
+ return func(*args, **kwargs)
+ except requests.HTTPError as error:
+ errors = parse_backend_error_messages(error.response)
+ if errors:
+ message = " ".join(errors)
+ message += (
+ f" (Error {error.response.status_code}: {error.response.reason})"
+ )
+ else:
+ message = error.response
+ raise CommError(message, error)
+ except RetryError as err:
+ if (
+ "response" in dir(err.last_exception)
+ and err.last_exception.response is not None
+ ):
+ try:
+ message = err.last_exception.response.json().get(
+ "errors", [{"message": message}]
+ )[0]["message"]
+ except ValueError:
+ message = err.last_exception.response.text
+ else:
+ message = err.last_exception
+
+ if env.is_debug():
+ raise err.last_exception.with_traceback(sys.exc_info()[2])
+ else:
+ raise CommError(message, err.last_exception).with_traceback(
+ sys.exc_info()[2]
+ )
+ except Error as err:
+ raise err
+ except Exception as err:
+ # gql raises server errors with dict's as strings...
+ if len(err.args) > 0:
+ payload = err.args[0]
+ else:
+ payload = err
+ if str(payload).startswith("{"):
+ message = ast.literal_eval(str(payload))["message"]
+ else:
+ message = str(err)
+ if env.is_debug():
+ raise
+ else:
+ raise CommError(message, err).with_traceback(sys.exc_info()[2])
+
+ return wrapper
diff --git a/wandb/apis/paginator.py b/wandb/apis/paginator.py
new file mode 100644
index 0000000000000000000000000000000000000000..e27d52b3bcb3103f1ec0bef7f5ba0fca52ac37c6
--- /dev/null
+++ b/wandb/apis/paginator.py
@@ -0,0 +1,81 @@
+from typing import TYPE_CHECKING, Any, MutableMapping, Optional
+
+if TYPE_CHECKING:
+ from wandb_gql import Client
+
+
+class Paginator:
+ QUERY = None
+
+ def __init__(
+ self,
+ client: "Client",
+ variables: MutableMapping[str, Any],
+ per_page: Optional[int] = None,
+ ):
+ self.client = client
+ self.variables = variables
+ # We don't allow unbounded paging
+ self.per_page = per_page
+ if self.per_page is None:
+ self.per_page = 50
+ self.objects = []
+ self.index = -1
+ self.last_response = None
+
+ def __iter__(self):
+ self.index = -1
+ return self
+
+ def __len__(self):
+ if self.length is None:
+ self._load_page()
+ if self.length is None:
+ raise ValueError("Object doesn't provide length")
+ return self.length
+
+ @property
+ def length(self):
+ raise NotImplementedError
+
+ @property
+ def more(self):
+ raise NotImplementedError
+
+ @property
+ def cursor(self):
+ raise NotImplementedError
+
+ def convert_objects(self):
+ raise NotImplementedError
+
+ def update_variables(self):
+ self.variables.update({"perPage": self.per_page, "cursor": self.cursor})
+
+ def _load_page(self):
+ if not self.more:
+ return False
+ self.update_variables()
+ self.last_response = self.client.execute(
+ self.QUERY, variable_values=self.variables
+ )
+ self.objects.extend(self.convert_objects())
+ return True
+
+ def __getitem__(self, index):
+ loaded = True
+ stop = index.stop if isinstance(index, slice) else index
+ while loaded and stop > len(self.objects) - 1:
+ loaded = self._load_page()
+ return self.objects[index]
+
+ def __next__(self):
+ self.index += 1
+ if len(self.objects) <= self.index:
+ if not self._load_page():
+ raise StopIteration
+ if len(self.objects) <= self.index:
+ raise StopIteration
+ return self.objects[self.index]
+
+ next = __next__
diff --git a/wandb/apis/public/__init__.py b/wandb/apis/public/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..29b3c3d24e7dca3f0e0a03a6c7dce1b50ec866af
--- /dev/null
+++ b/wandb/apis/public/__init__.py
@@ -0,0 +1,34 @@
+from wandb.apis.public.api import Api, RetryingClient, requests
+from wandb.apis.public.artifacts import (
+ ARTIFACT_FILES_FRAGMENT,
+ ARTIFACTS_TYPES_FRAGMENT,
+ ArtifactCollection,
+ ArtifactCollections,
+ ArtifactFiles,
+ Artifacts,
+ ArtifactType,
+ ArtifactTypes,
+ RunArtifacts,
+)
+from wandb.apis.public.files import FILE_FRAGMENT, File, Files
+from wandb.apis.public.history import HistoryScan, SampledHistoryScan
+from wandb.apis.public.jobs import (
+ Job,
+ QueuedRun,
+ RunQueue,
+ RunQueueAccessType,
+ RunQueuePrioritizationMode,
+ RunQueueResourceType,
+)
+from wandb.apis.public.projects import PROJECT_FRAGMENT, Project, Projects
+from wandb.apis.public.query_generator import QueryGenerator
+from wandb.apis.public.reports import (
+ BetaReport,
+ PanelMetricsHelper,
+ PythonMongoishQueryGenerator,
+ Reports,
+)
+from wandb.apis.public.runs import RUN_FRAGMENT, Run, Runs
+from wandb.apis.public.sweeps import SWEEP_FRAGMENT, Sweep
+from wandb.apis.public.teams import Member, Team
+from wandb.apis.public.users import User
diff --git a/wandb/apis/public/api.py b/wandb/apis/public/api.py
new file mode 100644
index 0000000000000000000000000000000000000000..8b31ccbc30c4a05c016e84f2a5409eb56f7a84b2
--- /dev/null
+++ b/wandb/apis/public/api.py
@@ -0,0 +1,1047 @@
+"""Use the Public API to export or update data that you have saved to W&B.
+
+Before using this API, you'll want to log data from your script — check the
+[Quickstart](https://docs.wandb.ai/quickstart) for more details.
+
+You might use the Public API to
+ - update metadata or metrics for an experiment after it has been completed,
+ - pull down your results as a dataframe for post-hoc analysis in a Jupyter notebook, or
+ - check your saved model artifacts for those tagged as `ready-to-deploy`.
+
+For more on using the Public API, check out [our guide](https://docs.wandb.com/guides/track/public-api-guide).
+"""
+import json
+import logging
+import os
+import urllib
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+import requests
+from wandb_gql import Client, gql
+from wandb_gql.client import RetryError
+
+import wandb
+from wandb import env, util
+from wandb.apis import public
+from wandb.apis.internal import Api as InternalApi
+from wandb.apis.normalize import normalize_exceptions
+from wandb.apis.public.const import RETRY_TIMEDELTA
+from wandb.sdk.internal.thread_local_settings import _thread_local_api_settings
+from wandb.sdk.launch.utils import LAUNCH_DEFAULT_PROJECT
+from wandb.sdk.lib import retry, runid
+from wandb.sdk.lib.gql_request import GraphQLSession
+
+if TYPE_CHECKING:
+ import wandb.apis.reports
+ import wandb.apis.reports.util
+
+
+logger = logging.getLogger(__name__)
+
+
+class RetryingClient:
+ INFO_QUERY = gql(
+ """
+ query ServerInfo{
+ serverInfo {
+ cliVersionInfo
+ latestLocalVersionInfo {
+ outOfDate
+ latestVersionString
+ versionOnThisInstanceString
+ }
+ }
+ }
+ """
+ )
+
+ def __init__(self, client: Client):
+ self._server_info = None
+ self._client = client
+
+ @property
+ def app_url(self):
+ return util.app_url(self._client.transport.url.replace("/graphql", "")) + "/"
+
+ @retry.retriable(
+ retry_timedelta=RETRY_TIMEDELTA,
+ check_retry_fn=util.no_retry_auth,
+ retryable_exceptions=(RetryError, requests.RequestException),
+ )
+ def execute(self, *args, **kwargs):
+ try:
+ return self._client.execute(*args, **kwargs)
+ except requests.exceptions.ReadTimeout:
+ if "timeout" not in kwargs:
+ timeout = self._client.transport.default_timeout
+ wandb.termwarn(
+ f"A graphql request initiated by the public wandb API timed out (timeout={timeout} sec). "
+ f"Create a new API with an integer timeout larger than {timeout}, e.g., `api = wandb.Api(timeout={timeout + 10})` "
+ f"to increase the graphql timeout."
+ )
+ raise
+
+ @property
+ def server_info(self):
+ if self._server_info is None:
+ self._server_info = self.execute(self.INFO_QUERY).get("serverInfo")
+ return self._server_info
+
+ def version_supported(self, min_version):
+ try:
+ from packaging.version import Version as parse_version # noqa: N813
+ except ImportError:
+ from pkg_resources import parse_version
+
+ return parse_version(min_version) <= parse_version(
+ self.server_info["cliVersionInfo"]["max_cli_version"]
+ )
+
+
+class Api:
+ """Used for querying the wandb server.
+
+ Examples:
+ Most common way to initialize
+ >>> wandb.Api()
+
+ Arguments:
+ overrides: (dict) You can set `base_url` if you are using a wandb server
+ other than https://api.wandb.ai.
+ You can also set defaults for `entity`, `project`, and `run`.
+ """
+
+ _HTTP_TIMEOUT = env.get_http_timeout(19)
+ VIEWER_QUERY = gql(
+ """
+ query Viewer{
+ viewer {
+ id
+ flags
+ entity
+ username
+ email
+ admin
+ apiKeys {
+ edges {
+ node {
+ id
+ name
+ description
+ }
+ }
+ }
+ teams {
+ edges {
+ node {
+ name
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+ USERS_QUERY = gql(
+ """
+ query SearchUsers($query: String) {
+ users(query: $query) {
+ edges {
+ node {
+ id
+ flags
+ entity
+ admin
+ email
+ deletedAt
+ username
+ apiKeys {
+ edges {
+ node {
+ id
+ name
+ description
+ }
+ }
+ }
+ teams {
+ edges {
+ node {
+ name
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+
+ CREATE_PROJECT = gql(
+ """
+ mutation upsertModel(
+ $description: String
+ $entityName: String
+ $id: String
+ $name: String
+ $framework: String
+ $access: String
+ $views: JSONString
+ ) {
+ upsertModel(
+ input: {
+ description: $description
+ entityName: $entityName
+ id: $id
+ name: $name
+ framework: $framework
+ access: $access
+ views: $views
+ }
+ ) {
+ project {
+ id
+ name
+ entityName
+ description
+ access
+ views
+ }
+ model {
+ id
+ name
+ entityName
+ description
+ access
+ views
+ }
+ inserted
+ }
+ }
+ """
+ )
+
+ def __init__(
+ self,
+ overrides=None,
+ timeout: Optional[int] = None,
+ api_key: Optional[str] = None,
+ ) -> None:
+ self.settings = InternalApi().settings()
+ _overrides = overrides or {}
+ self._api_key = api_key
+ if self.api_key is None and _thread_local_api_settings.cookies is None:
+ wandb.login(host=_overrides.get("base_url"))
+ self.settings.update(_overrides)
+ if "username" in _overrides and "entity" not in _overrides:
+ wandb.termwarn(
+ 'Passing "username" to Api is deprecated. please use "entity" instead.'
+ )
+ self.settings["entity"] = _overrides["username"]
+ self.settings["base_url"] = self.settings["base_url"].rstrip("/")
+
+ self._viewer = None
+ self._projects = {}
+ self._runs = {}
+ self._sweeps = {}
+ self._reports = {}
+ self._default_entity = None
+ self._timeout = timeout if timeout is not None else self._HTTP_TIMEOUT
+ auth = None
+ if not _thread_local_api_settings.cookies:
+ auth = ("api", self.api_key)
+ proxies = self.settings.get("_proxies") or json.loads(
+ os.environ.get("WANDB__PROXIES", "{}")
+ )
+ self._base_client = Client(
+ transport=GraphQLSession(
+ headers={
+ "User-Agent": self.user_agent,
+ "Use-Admin-Privileges": "true",
+ **(_thread_local_api_settings.headers or {}),
+ },
+ use_json=True,
+ # this timeout won't apply when the DNS lookup fails. in that case, it will be 60s
+ # https://bugs.python.org/issue22889
+ timeout=self._timeout,
+ auth=auth,
+ url="%s/graphql" % self.settings["base_url"],
+ cookies=_thread_local_api_settings.cookies,
+ proxies=proxies,
+ )
+ )
+ self._client = RetryingClient(self._base_client)
+
+ def create_project(self, name: str, entity: str):
+ self.client.execute(self.CREATE_PROJECT, {"entityName": entity, "name": name})
+
+ def create_run(self, **kwargs):
+ """Create a new run."""
+ if kwargs.get("entity") is None:
+ kwargs["entity"] = self.default_entity
+ return public.Run.create(self, **kwargs)
+
+ def create_report(
+ self,
+ project: str,
+ entity: str = "",
+ title: Optional[str] = "Untitled Report",
+ description: Optional[str] = "",
+ width: Optional[str] = "readable",
+ blocks: Optional["wandb.apis.reports.util.Block"] = None,
+ ) -> "wandb.apis.reports.Report":
+ if entity == "":
+ entity = self.default_entity or ""
+ if blocks is None:
+ blocks = []
+ return wandb.apis.reports.Report(
+ project, entity, title, description, width, blocks
+ ).save()
+
+ def create_run_queue(
+ self,
+ name: str,
+ type: "public.RunQueueResourceType",
+ entity: Optional[str] = None,
+ prioritization_mode: Optional["public.RunQueuePrioritizationMode"] = None,
+ config: Optional[dict] = None,
+ template_variables: Optional[dict] = None,
+ ) -> "public.RunQueue":
+ """Create a new run queue (launch).
+
+ Arguments:
+ name: (str) Name of the queue to create
+ type: (str) Type of resource to be used for the queue. One of "local-container", "local-process", "kubernetes", "sagemaker", or "gcp-vertex".
+ entity: (str) Optional name of the entity to create the queue. If None, will use the configured or default entity.
+ prioritization_mode: (str) Optional version of prioritization to use. Either "V0" or None
+ config: (dict) Optional default resource configuration to be used for the queue. Use handlebars (eg. "{{var}}") to specify template variables.
+ template_variables (dict): A dictionary of template variable schemas to be used with the config. Expected format of:
+ {
+ "var-name": {
+ "schema": {
+ "type": "<string | number | integer>",
+ "default": <optional value>,
+ "minimum": <optional minimum>,
+ "maximum": <optional maximum>,
+ "enum": [..."<options>"]
+ }
+ }
+ }
+
+ Returns:
+ The newly created `RunQueue`
+
+ Raises:
+ ValueError if any of the parameters are invalid
+ wandb.Error on wandb API errors
+ """
+ # TODO(np): Need to check server capabilities for this feature
+ # 0. assert params are valid/normalized
+ if entity is None:
+ entity = self.settings["entity"] or self.default_entity
+ if entity is None:
+ raise ValueError(
+ "entity must be passed as a parameter, or set in settings"
+ )
+
+ if len(name) == 0:
+ raise ValueError("name must be non-empty")
+ if len(name) > 64:
+ raise ValueError("name must be less than 64 characters")
+
+ if type not in [
+ "local-container",
+ "local-process",
+ "kubernetes",
+ "sagemaker",
+ "gcp-vertex",
+ ]:
+ raise ValueError(
+ "resource_type must be one of 'local-container', 'local-process', 'kubernetes', 'sagemaker', or 'gcp-vertex'"
+ )
+
+ if prioritization_mode:
+ prioritization_mode = prioritization_mode.upper()
+ if prioritization_mode not in ["V0"]:
+ raise ValueError("prioritization_mode must be 'V0' if present")
+
+ if config is None:
+ config = {}
+
+ # 1. create required default launch project in the entity
+ self.create_project(LAUNCH_DEFAULT_PROJECT, entity)
+
+ api = InternalApi(
+ default_settings={
+ "entity": entity,
+ "project": self.project(LAUNCH_DEFAULT_PROJECT),
+ },
+ retry_timedelta=RETRY_TIMEDELTA,
+ )
+
+ # 2. create default resource config, receive config id
+ config_json = json.dumps({"resource_args": {type: config}})
+ create_config_result = api.create_default_resource_config(
+ entity, type, config_json, template_variables
+ )
+ if not create_config_result["success"]:
+ raise wandb.Error("failed to create default resource config")
+ config_id = create_config_result["defaultResourceConfigID"]
+
+ # 3. create run queue
+ create_queue_result = api.create_run_queue(
+ entity,
+ LAUNCH_DEFAULT_PROJECT,
+ name,
+ "PROJECT",
+ prioritization_mode,
+ config_id,
+ )
+ if not create_queue_result["success"]:
+ raise wandb.Error("failed to create run queue")
+
+ return public.RunQueue(
+ client=self.client,
+ name=name,
+ entity=entity,
+ prioritization_mode=prioritization_mode,
+ _access="PROJECT",
+ _default_resource_config_id=config_id,
+ _default_resource_config=config,
+ )
+
+ def load_report(self, path: str) -> "wandb.apis.reports.Report":
+ """Get report at a given path.
+
+ Arguments:
+ path: (str) Path to the target report in the form `entity/project/reports/reportId`.
+ You can get this by copy-pasting the URL after your wandb url. For example:
+ `megatruong/report-editing/reports/My-fabulous-report-title--VmlldzoxOTc1Njk0`
+
+ Returns:
+ A `BetaReport` object which represents the report at `path`
+
+ Raises:
+ wandb.Error if path is invalid
+ """
+ return wandb.apis.reports.Report.from_url(path)
+
+ def create_user(self, email, admin=False):
+ """Create a new user.
+
+ Arguments:
+ email: (str) The email address of the user
+ admin: (bool) Whether this user should be a global instance admin
+
+ Returns:
+ A `User` object
+ """
+ return public.User.create(self, email, admin)
+
+ def sync_tensorboard(self, root_dir, run_id=None, project=None, entity=None):
+ """Sync a local directory containing tfevent files to wandb."""
+ from wandb.sync import SyncManager # TODO: circular import madness
+
+ run_id = run_id or runid.generate_id()
+ project = project or self.settings.get("project") or "uncategorized"
+ entity = entity or self.default_entity
+ # TODO: pipe through log_path to inform the user how to debug
+ sm = SyncManager(
+ project=project,
+ entity=entity,
+ run_id=run_id,
+ mark_synced=False,
+ app_url=self.client.app_url,
+ view=False,
+ verbose=False,
+ sync_tensorboard=True,
+ )
+ sm.add(root_dir)
+ sm.start()
+ while not sm.is_done():
+ _ = sm.poll()
+ return self.run("/".join([entity, project, run_id]))
+
+ @property
+ def client(self):
+ return self._client
+
+ @property
+ def user_agent(self):
+ return "W&B Public Client %s" % wandb.__version__
+
+ @property
+ def api_key(self):
+ # just use thread local api key if it's set
+ if _thread_local_api_settings.api_key:
+ return _thread_local_api_settings.api_key
+ if self._api_key is not None:
+ return self._api_key
+ auth = requests.utils.get_netrc_auth(self.settings["base_url"])
+ key = None
+ if auth:
+ key = auth[-1]
+ # Environment should take precedence
+ if os.getenv("WANDB_API_KEY"):
+ key = os.environ["WANDB_API_KEY"]
+ self._api_key = key # memoize key
+ return key
+
+ @property
+ def default_entity(self):
+ if self._default_entity is None:
+ res = self._client.execute(self.VIEWER_QUERY)
+ self._default_entity = (res.get("viewer") or {}).get("entity")
+ return self._default_entity
+
+ @property
+ def viewer(self):
+ if self._viewer is None:
+ self._viewer = public.User(
+ self._client, self._client.execute(self.VIEWER_QUERY).get("viewer")
+ )
+ self._default_entity = self._viewer.entity
+ return self._viewer
+
+ def flush(self):
+ """Flush the local cache.
+
+ The api object keeps a local cache of runs, so if the state of the run may
+ change while executing your script you must clear the local cache with
+ `api.flush()` to get the latest values associated with the run.
+ """
+ self._runs = {}
+
+ def from_path(self, path):
+ """Return a run, sweep, project or report from a path.
+
+ Examples:
+ ```
+ project = api.from_path("my_project")
+ team_project = api.from_path("my_team/my_project")
+ run = api.from_path("my_team/my_project/runs/id")
+ sweep = api.from_path("my_team/my_project/sweeps/id")
+ report = api.from_path("my_team/my_project/reports/My-Report-Vm11dsdf")
+ ```
+
+ Arguments:
+ path: (str) The path to the project, run, sweep or report
+
+ Returns:
+ A `Project`, `Run`, `Sweep`, or `BetaReport` instance.
+
+ Raises:
+ wandb.Error if path is invalid or the object doesn't exist
+ """
+ parts = path.strip("/ ").split("/")
+ if len(parts) == 1:
+ return self.project(path)
+ elif len(parts) == 2:
+ return self.project(parts[1], parts[0])
+ elif len(parts) == 3:
+ return self.run(path)
+ elif len(parts) == 4:
+ if parts[2].startswith("run"):
+ return self.run(path)
+ elif parts[2].startswith("sweep"):
+ return self.sweep(path)
+ elif parts[2].startswith("report"):
+ if "--" not in parts[-1]:
+ if "-" in parts[-1]:
+ raise wandb.Error(
+ "Invalid report path, should be team/project/reports/Name--XXXX"
+ )
+ else:
+ parts[-1] = "--" + parts[-1]
+ name, id = parts[-1].split("--")
+ return public.BetaReport(
+ self.client,
+ {
+ "display_name": urllib.parse.unquote(name.replace("-", " ")),
+ "id": id,
+ "spec": "{}",
+ },
+ parts[0],
+ parts[1],
+ )
+ raise wandb.Error(
+ "Invalid path, should be TEAM/PROJECT/TYPE/ID where TYPE is runs, sweeps, or reports"
+ )
+
+ def _parse_project_path(self, path):
+ """Return project and entity for project specified by path."""
+ project = self.settings["project"] or "uncategorized"
+ entity = self.settings["entity"] or self.default_entity
+ if path is None:
+ return entity, project
+ parts = path.split("/", 1)
+ if len(parts) == 1:
+ return entity, path
+ return parts
+
+ def _parse_path(self, path):
+ """Parse url, filepath, or docker paths.
+
+ Allows paths in the following formats:
+ - url: entity/project/runs/id
+ - path: entity/project/id
+ - docker: entity/project:id
+
+ Entity is optional and will fall back to the current logged-in user.
+ """
+ project = self.settings["project"] or "uncategorized"
+ entity = self.settings["entity"] or self.default_entity
+ parts = (
+ path.replace("/runs/", "/").replace("/sweeps/", "/").strip("/ ").split("/")
+ )
+ if ":" in parts[-1]:
+ id = parts[-1].split(":")[-1]
+ parts[-1] = parts[-1].split(":")[0]
+ elif parts[-1]:
+ id = parts[-1]
+ if len(parts) == 1 and project != "uncategorized":
+ pass
+ elif len(parts) > 1:
+ project = parts[1]
+ if entity and id == project:
+ project = parts[0]
+ else:
+ entity = parts[0]
+ if len(parts) == 3:
+ entity = parts[0]
+ else:
+ project = parts[0]
+ return entity, project, id
+
+ def _parse_artifact_path(self, path):
+ """Return project, entity and artifact name for project specified by path."""
+ project = self.settings["project"] or "uncategorized"
+ entity = self.settings["entity"] or self.default_entity
+ if path is None:
+ return entity, project
+ parts = path.split("/")
+ if len(parts) > 3:
+ raise ValueError("Invalid artifact path: %s" % path)
+ elif len(parts) == 1:
+ return entity, project, path
+ elif len(parts) == 2:
+ return entity, parts[0], parts[1]
+ return parts
+
+ def projects(self, entity=None, per_page=200):
+ """Get projects for a given entity.
+
+ Arguments:
+ entity: (str) Name of the entity requested. If None, will fall back to
+ default entity passed to `Api`. If no default entity, will raise a `ValueError`.
+ per_page: (int) Sets the page size for query pagination. None will use the default size.
+ Usually there is no reason to change this.
+
+ Returns:
+ A `Projects` object which is an iterable collection of `Project` objects.
+
+ """
+ if entity is None:
+ entity = self.settings["entity"] or self.default_entity
+ if entity is None:
+ raise ValueError(
+ "entity must be passed as a parameter, or set in settings"
+ )
+ if entity not in self._projects:
+ self._projects[entity] = public.Projects(
+ self.client, entity, per_page=per_page
+ )
+ return self._projects[entity]
+
+ def project(self, name, entity=None):
+ if entity is None:
+ entity = self.settings["entity"] or self.default_entity
+ return public.Project(self.client, entity, name, {})
+
+ def reports(self, path="", name=None, per_page=50):
+ """Get reports for a given project path.
+
+ WARNING: This api is in beta and will likely change in a future release
+
+ Arguments:
+ path: (str) path to project the report resides in, should be in the form: "entity/project"
+ name: (str) optional name of the report requested.
+ per_page: (int) Sets the page size for query pagination. None will use the default size.
+ Usually there is no reason to change this.
+
+ Returns:
+ A `Reports` object which is an iterable collection of `BetaReport` objects.
+ """
+ entity, project, _ = self._parse_path(path + "/fake_run")
+
+ if name:
+ name = urllib.parse.unquote(name)
+ key = "/".join([entity, project, str(name)])
+ else:
+ key = "/".join([entity, project])
+
+ if key not in self._reports:
+ self._reports[key] = public.Reports(
+ self.client,
+ public.Project(self.client, entity, project, {}),
+ name=name,
+ per_page=per_page,
+ )
+ return self._reports[key]
+
+ def create_team(self, team, admin_username=None):
+ """Create a new team.
+
+ Arguments:
+ team: (str) The name of the team
+ admin_username: (str) optional username of the admin user of the team, defaults to the current user.
+
+ Returns:
+ A `Team` object
+ """
+ return public.Team.create(self, team, admin_username)
+
+ def team(self, team):
+ return public.Team(self.client, team)
+
+ def user(self, username_or_email):
+ """Return a user from a username or email address.
+
+ Note: This function only works for Local Admins, if you are trying to get your own user object, please use `api.viewer`.
+
+ Arguments:
+ username_or_email: (str) The username or email address of the user
+
+ Returns:
+ A `User` object or None if a user couldn't be found
+ """
+ res = self._client.execute(self.USERS_QUERY, {"query": username_or_email})
+ if len(res["users"]["edges"]) == 0:
+ return None
+ elif len(res["users"]["edges"]) > 1:
+ wandb.termwarn(
+ "Found multiple users, returning the first user matching {}".format(
+ username_or_email
+ )
+ )
+ return public.User(self._client, res["users"]["edges"][0]["node"])
+
+ def users(self, username_or_email):
+ """Return all users from a partial username or email address query.
+
+ Note: This function only works for Local Admins, if you are trying to get your own user object, please use `api.viewer`.
+
+ Arguments:
+ username_or_email: (str) The prefix or suffix of the user you want to find
+
+ Returns:
+ An array of `User` objects
+ """
+ res = self._client.execute(self.USERS_QUERY, {"query": username_or_email})
+ return [
+ public.User(self._client, edge["node"]) for edge in res["users"]["edges"]
+ ]
+
+ def runs(
+ self,
+ path: Optional[str] = None,
+ filters: Optional[Dict[str, Any]] = None,
+ order: str = "-created_at",
+ per_page: int = 50,
+ include_sweeps: bool = True,
+ ):
+ """Return a set of runs from a project that match the filters provided.
+
+ You can filter by `config.*`, `summary_metrics.*`, `tags`, `state`, `entity`, `createdAt`, etc.
+
+ Examples:
+ Find runs in my_project where config.experiment_name has been set to "foo"
+ ```
+ api.runs(path="my_entity/my_project", filters={"config.experiment_name": "foo"})
+ ```
+
+ Find runs in my_project where config.experiment_name has been set to "foo" or "bar"
+ ```
+ api.runs(
+ path="my_entity/my_project",
+ filters={"$or": [{"config.experiment_name": "foo"}, {"config.experiment_name": "bar"}]}
+ )
+ ```
+
+ Find runs in my_project where config.experiment_name matches a regex (anchors are not supported)
+ ```
+ api.runs(
+ path="my_entity/my_project",
+ filters={"config.experiment_name": {"$regex": "b.*"}}
+ )
+ ```
+
+ Find runs in my_project where the run name matches a regex (anchors are not supported)
+ ```
+ api.runs(
+ path="my_entity/my_project",
+ filters={"display_name": {"$regex": "^foo.*"}}
+ )
+ ```
+
+ Find runs in my_project sorted by ascending loss
+ ```
+ api.runs(path="my_entity/my_project", order="+summary_metrics.loss")
+ ```
+
+ Arguments:
+ path: (str) path to project, should be in the form: "entity/project"
+ filters: (dict) queries for specific runs using the MongoDB query language.
+ You can filter by run properties such as config.key, summary_metrics.key, state, entity, createdAt, etc.
+ For example: {"config.experiment_name": "foo"} would find runs with a config entry
+ of experiment name set to "foo"
+ You can compose operations to make more complicated queries,
+ see Reference for the language is at https://docs.mongodb.com/manual/reference/operator/query
+ order: (str) Order can be `created_at`, `heartbeat_at`, `config.*.value`, or `summary_metrics.*`.
+ If you prepend order with a + order is ascending.
+ If you prepend order with a - order is descending (default).
+ The default order is run.created_at from newest to oldest.
+
+ Returns:
+ A `Runs` object, which is an iterable collection of `Run` objects.
+ """
+ entity, project = self._parse_project_path(path)
+ filters = filters or {}
+ key = (path or "") + str(filters) + str(order)
+ if not self._runs.get(key):
+ self._runs[key] = public.Runs(
+ self.client,
+ entity,
+ project,
+ filters=filters,
+ order=order,
+ per_page=per_page,
+ include_sweeps=include_sweeps,
+ )
+ return self._runs[key]
+
+ @normalize_exceptions
+ def run(self, path=""):
+ """Return a single run by parsing path in the form entity/project/run_id.
+
+ Arguments:
+ path: (str) path to run in the form `entity/project/run_id`.
+ If `api.entity` is set, this can be in the form `project/run_id`
+ and if `api.project` is set this can just be the run_id.
+
+ Returns:
+ A `Run` object.
+ """
+ entity, project, run_id = self._parse_path(path)
+ if not self._runs.get(path):
+ self._runs[path] = public.Run(self.client, entity, project, run_id)
+ return self._runs[path]
+
+ def queued_run(
+ self,
+ entity,
+ project,
+ queue_name,
+ run_queue_item_id,
+ project_queue=None,
+ priority=None,
+ ):
+ """Return a single queued run based on the path.
+
+ Parses paths of the form entity/project/queue_id/run_queue_item_id.
+ """
+ return public.QueuedRun(
+ self.client,
+ entity,
+ project,
+ queue_name,
+ run_queue_item_id,
+ project_queue=project_queue,
+ priority=priority,
+ )
+
+ def run_queue(
+ self,
+ entity,
+ name,
+ ):
+ """Return the named `RunQueue` for entity.
+
+ To create a new `RunQueue`, use `wandb.Api().create_run_queue(...)`.
+ """
+ return public.RunQueue(
+ self.client,
+ name,
+ entity,
+ )
+
+ @normalize_exceptions
+ def sweep(self, path=""):
+ """Return a sweep by parsing path in the form `entity/project/sweep_id`.
+
+ Arguments:
+ path: (str, optional) path to sweep in the form entity/project/sweep_id. If `api.entity`
+ is set, this can be in the form project/sweep_id and if `api.project` is set
+ this can just be the sweep_id.
+
+ Returns:
+ A `Sweep` object.
+ """
+ entity, project, sweep_id = self._parse_path(path)
+ if not self._sweeps.get(path):
+ self._sweeps[path] = public.Sweep(self.client, entity, project, sweep_id)
+ return self._sweeps[path]
+
+ @normalize_exceptions
+ def artifact_types(self, project=None):
+ entity, project = self._parse_project_path(project)
+ return public.ArtifactTypes(self.client, entity, project)
+
+ @normalize_exceptions
+ def artifact_type(self, type_name, project=None):
+ entity, project = self._parse_project_path(project)
+ return public.ArtifactType(self.client, entity, project, type_name)
+
+ @normalize_exceptions
+ def artifact_collections(self, project_name: str, type_name: str, per_page=50):
+ entity, project = self._parse_project_path(project_name)
+ return public.ArtifactCollections(
+ self.client, entity, project, type_name, per_page=per_page
+ )
+
+ @normalize_exceptions
+ def artifact_collection(self, type_name: str, name: str):
+ """Return a single artifact collection by type and parsing path in the form `entity/project/name`.
+
+ Arguments:
+ name: (str) An artifact collection name. May be prefixed with entity/project.
+ type: (str) The type of artifact collection to fetch.
+
+ Returns:
+ An `ArtifactCollection` object.
+ """
+ entity, project, collection_name = self._parse_artifact_path(name)
+ return public.ArtifactCollection(
+ self.client, entity, project, collection_name, type_name
+ )
+
+ @normalize_exceptions
+ def artifact_versions(self, type_name, name, per_page=50):
+ """Deprecated, use artifacts(type_name, name) instead."""
+ wandb.termwarn(
+ "Api.artifact_versions(type_name, name) is deprecated, use Api.artifacts(type_name, name) instead."
+ )
+ return self.artifacts(type_name, name, per_page=per_page)
+
+ @normalize_exceptions
+ def artifacts(self, type_name, name, per_page=50):
+ entity, project, collection_name = self._parse_artifact_path(name)
+ return public.Artifacts(
+ self.client, entity, project, collection_name, type_name, per_page=per_page
+ )
+
+ @normalize_exceptions
+ def artifact(self, name, type=None):
+ """Return a single artifact by parsing path in the form `entity/project/name`.
+
+ Arguments:
+ name: (str) An artifact name. May be prefixed with entity/project. Valid names
+ can be in the following forms:
+ name:version
+ name:alias
+ type: (str, optional) The type of artifact to fetch.
+
+ Returns:
+ A `Artifact` object.
+ """
+ if name is None:
+ raise ValueError("You must specify name= to fetch an artifact.")
+ entity, project, artifact_name = self._parse_artifact_path(name)
+ artifact = wandb.Artifact._from_name(
+ entity, project, artifact_name, self.client
+ )
+ if type is not None and artifact.type != type:
+ raise ValueError(
+ f"type {type} specified but this artifact is of type {artifact.type}"
+ )
+ return artifact
+
+ @normalize_exceptions
+ def job(self, name, path=None):
+ if name is None:
+ raise ValueError("You must specify name= to fetch a job.")
+ elif name.count("/") != 2 or ":" not in name:
+ raise ValueError(
+ "Invalid job specification. A job must be of the form: <entity>/<project>/<job-name>:<alias-or-version>"
+ )
+ return public.Job(self, name, path)
+
+ @normalize_exceptions
+ def list_jobs(self, entity, project):
+ if entity is None:
+ raise ValueError("Specify an entity when listing jobs")
+ if project is None:
+ raise ValueError("Specify a project when listing jobs")
+
+ query = gql(
+ """
+ query ArtifactOfType(
+ $entityName: String!,
+ $projectName: String!,
+ $artifactTypeName: String!,
+ ) {
+ project(name: $projectName, entityName: $entityName) {
+ artifactType(name: $artifactTypeName) {
+ artifactCollections {
+ edges {
+ node {
+ artifacts {
+ edges {
+ node {
+ id
+ state
+ aliases {
+ alias
+ }
+ artifactSequence {
+ name
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+
+ try:
+ artifact_query = self._client.execute(
+ query,
+ {
+ "projectName": project,
+ "entityName": entity,
+ "artifactTypeName": "job",
+ },
+ )
+
+ if not artifact_query or not artifact_query["project"]:
+ wandb.termerror(
+ f"Project: '{project}' not found in entity: '{entity}' or access denied."
+ )
+ return []
+
+ if artifact_query["project"]["artifactType"] is None:
+ return []
+
+ artifacts = artifact_query["project"]["artifactType"][
+ "artifactCollections"
+ ]["edges"]
+
+ return [x["node"]["artifacts"] for x in artifacts]
+ except requests.exceptions.HTTPError:
+ return False
diff --git a/wandb/apis/public/artifacts.py b/wandb/apis/public/artifacts.py
new file mode 100644
index 0000000000000000000000000000000000000000..696ee0f33e89269670f53a3bd89b61da88e1007f
--- /dev/null
+++ b/wandb/apis/public/artifacts.py
@@ -0,0 +1,812 @@
+"""Public API: artifacts."""
+import json
+from typing import TYPE_CHECKING, Any, Mapping, Optional, Sequence
+
+from wandb_gql import Client, gql
+
+import wandb
+from wandb.apis import public
+from wandb.apis.normalize import normalize_exceptions
+from wandb.apis.paginator import Paginator
+
+if TYPE_CHECKING:
+ from wandb.apis.public import RetryingClient, Run
+
+
+ARTIFACTS_TYPES_FRAGMENT = """
+fragment ArtifactTypesFragment on ArtifactTypeConnection {
+ edges {
+ node {
+ id
+ name
+ description
+ createdAt
+ }
+ cursor
+ }
+ pageInfo {
+ endCursor
+ hasNextPage
+ }
+}
+"""
+
+# TODO, factor out common file fragment
+ARTIFACT_FILES_FRAGMENT = """fragment ArtifactFilesFragment on Artifact {
+ files(names: $fileNames, after: $fileCursor, first: $fileLimit) {
+ edges {
+ node {
+ id
+ name: displayName
+ url
+ sizeBytes
+ storagePath
+ mimetype
+ updatedAt
+ digest
+ md5
+ }
+ cursor
+ }
+ pageInfo {
+ endCursor
+ hasNextPage
+ }
+ }
+}"""
+
+
+class ArtifactTypes(Paginator):
+ QUERY = gql(
+ """
+ query ProjectArtifacts(
+ $entityName: String!,
+ $projectName: String!,
+ $cursor: String,
+ ) {
+ project(name: $projectName, entityName: $entityName) {
+ artifactTypes(after: $cursor) {
+ ...ArtifactTypesFragment
+ }
+ }
+ }
+ %s
+ """
+ % ARTIFACTS_TYPES_FRAGMENT
+ )
+
+ def __init__(
+ self,
+ client: Client,
+ entity: str,
+ project: str,
+ per_page: Optional[int] = 50,
+ ):
+ self.entity = entity
+ self.project = project
+
+ variable_values = {
+ "entityName": entity,
+ "projectName": project,
+ }
+
+ super().__init__(client, variable_values, per_page)
+
+ @property
+ def length(self):
+ # TODO
+ return None
+
+ @property
+ def more(self):
+ if self.last_response:
+ return self.last_response["project"]["artifactTypes"]["pageInfo"][
+ "hasNextPage"
+ ]
+ else:
+ return True
+
+ @property
+ def cursor(self):
+ if self.last_response:
+ return self.last_response["project"]["artifactTypes"]["edges"][-1]["cursor"]
+ else:
+ return None
+
+ def update_variables(self):
+ self.variables.update({"cursor": self.cursor})
+
+ def convert_objects(self):
+ if self.last_response["project"] is None:
+ return []
+ return [
+ ArtifactType(
+ self.client, self.entity, self.project, r["node"]["name"], r["node"]
+ )
+ for r in self.last_response["project"]["artifactTypes"]["edges"]
+ ]
+
+
+class ArtifactType:
+ def __init__(
+ self,
+ client: Client,
+ entity: str,
+ project: str,
+ type_name: str,
+ attrs: Optional[Mapping[str, Any]] = None,
+ ):
+ self.client = client
+ self.entity = entity
+ self.project = project
+ self.type = type_name
+ self._attrs = attrs
+ if self._attrs is None:
+ self.load()
+
+ def load(self):
+ query = gql(
+ """
+ query ProjectArtifactType(
+ $entityName: String!,
+ $projectName: String!,
+ $artifactTypeName: String!
+ ) {
+ project(name: $projectName, entityName: $entityName) {
+ artifactType(name: $artifactTypeName) {
+ id
+ name
+ description
+ createdAt
+ }
+ }
+ }
+ """
+ )
+ response: Optional[Mapping[str, Any]] = self.client.execute(
+ query,
+ variable_values={
+ "entityName": self.entity,
+ "projectName": self.project,
+ "artifactTypeName": self.type,
+ },
+ )
+ if (
+ response is None
+ or response.get("project") is None
+ or response["project"].get("artifactType") is None
+ ):
+ raise ValueError("Could not find artifact type %s" % self.type)
+ self._attrs = response["project"]["artifactType"]
+ return self._attrs
+
+ @property
+ def id(self):
+ return self._attrs["id"]
+
+ @property
+ def name(self):
+ return self._attrs["name"]
+
+ @normalize_exceptions
+ def collections(self, per_page=50):
+ """Artifact collections."""
+ return ArtifactCollections(self.client, self.entity, self.project, self.type)
+
+ def collection(self, name):
+ return ArtifactCollection(
+ self.client, self.entity, self.project, name, self.type
+ )
+
+ def __repr__(self):
+ return f"<ArtifactType {self.type}>"
+
+
+class ArtifactCollections(Paginator):
+ def __init__(
+ self,
+ client: Client,
+ entity: str,
+ project: str,
+ type_name: str,
+ per_page: Optional[int] = 50,
+ ):
+ self.entity = entity
+ self.project = project
+ self.type_name = type_name
+
+ variable_values = {
+ "entityName": entity,
+ "projectName": project,
+ "artifactTypeName": type_name,
+ }
+
+ self.QUERY = gql(
+ """
+ query ProjectArtifactCollections(
+ $entityName: String!,
+ $projectName: String!,
+ $artifactTypeName: String!
+ $cursor: String,
+ ) {
+ project(name: $projectName, entityName: $entityName) {
+ artifactType(name: $artifactTypeName) {
+ artifactCollections: %s(after: $cursor) {
+ pageInfo {
+ endCursor
+ hasNextPage
+ }
+ totalCount
+ edges {
+ node {
+ id
+ name
+ description
+ createdAt
+ }
+ cursor
+ }
+ }
+ }
+ }
+ }
+ """
+ % artifact_collection_plural_edge_name(
+ server_supports_artifact_collections_gql_edges(client)
+ )
+ )
+
+ super().__init__(client, variable_values, per_page)
+
+ @property
+ def length(self):
+ if self.last_response:
+ return self.last_response["project"]["artifactType"]["artifactCollections"][
+ "totalCount"
+ ]
+ else:
+ return None
+
+ @property
+ def more(self):
+ if self.last_response:
+ return self.last_response["project"]["artifactType"]["artifactCollections"][
+ "pageInfo"
+ ]["hasNextPage"]
+ else:
+ return True
+
+ @property
+ def cursor(self):
+ if self.last_response:
+ return self.last_response["project"]["artifactType"]["artifactCollections"][
+ "edges"
+ ][-1]["cursor"]
+ else:
+ return None
+
+ def update_variables(self):
+ self.variables.update({"cursor": self.cursor})
+
+ def convert_objects(self):
+ return [
+ ArtifactCollection(
+ self.client,
+ self.entity,
+ self.project,
+ r["node"]["name"],
+ self.type_name,
+ )
+ for r in self.last_response["project"]["artifactType"][
+ "artifactCollections"
+ ]["edges"]
+ ]
+
+
+class ArtifactCollection:
+ def __init__(
+ self,
+ client: Client,
+ entity: str,
+ project: str,
+ name: str,
+ type: str,
+ attrs: Optional[Mapping[str, Any]] = None,
+ ):
+ self.client = client
+ self.entity = entity
+ self.project = project
+ self.name = name
+ self.type = type
+ self._attrs = attrs
+ if self._attrs is None:
+ self.load()
+ self._aliases = [a["node"]["alias"] for a in self._attrs["aliases"]["edges"]]
+
+ @property
+ def id(self):
+ return self._attrs["id"]
+
+ @normalize_exceptions
+ def artifacts(self, per_page=50):
+ """Artifacts."""
+ return Artifacts(
+ self.client,
+ self.entity,
+ self.project,
+ self.name,
+ self.type,
+ per_page=per_page,
+ )
+
+ @property
+ def aliases(self):
+ """Artifact Collection Aliases."""
+ return self._aliases
+
+ def load(self):
+ query = gql(
+ """
+ query ArtifactCollection(
+ $entityName: String!,
+ $projectName: String!,
+ $artifactTypeName: String!,
+ $artifactCollectionName: String!,
+ $cursor: String,
+ $perPage: Int = 1000
+ ) {
+ project(name: $projectName, entityName: $entityName) {
+ artifactType(name: $artifactTypeName) {
+ artifactCollection: %s(name: $artifactCollectionName) {
+ id
+ name
+ description
+ createdAt
+ aliases(after: $cursor, first: $perPage){
+ edges {
+ node {
+ alias
+ }
+ cursor
+ }
+ pageInfo {
+ endCursor
+ hasNextPage
+ }
+ }
+ }
+ }
+ }
+ }
+ """
+ % artifact_collection_edge_name(
+ server_supports_artifact_collections_gql_edges(self.client)
+ )
+ )
+ response = self.client.execute(
+ query,
+ variable_values={
+ "entityName": self.entity,
+ "projectName": self.project,
+ "artifactTypeName": self.type,
+ "artifactCollectionName": self.name,
+ },
+ )
+ if (
+ response is None
+ or response.get("project") is None
+ or response["project"].get("artifactType") is None
+ or response["project"]["artifactType"].get("artifactCollection") is None
+ ):
+ raise ValueError("Could not find artifact type %s" % self.type)
+ self._attrs = response["project"]["artifactType"]["artifactCollection"]
+ return self._attrs
+
+ @normalize_exceptions
+ def is_sequence(self) -> bool:
+ """Return True if this is a sequence."""
+ query = gql(
+ """
+ query FindSequence($entity: String!, $project: String!, $collection: String!, $type: String!) {
+ project(name: $project, entityName: $entity) {
+ artifactType(name: $type) {
+ __typename
+ artifactSequence(name: $collection) {
+ __typename
+ }
+ }
+ }
+ }
+ """
+ )
+ variables = {
+ "entity": self.entity,
+ "project": self.project,
+ "collection": self.name,
+ "type": self.type,
+ }
+ res = self.client.execute(query, variable_values=variables)
+ sequence = res["project"]["artifactType"]["artifactSequence"]
+ return sequence is not None and sequence["__typename"] == "ArtifactSequence"
+
+ @normalize_exceptions
+ def delete(self):
+ """Delete the entire artifact collection."""
+ if self.is_sequence():
+ mutation = gql(
+ """
+ mutation deleteArtifactSequence($id: ID!) {
+ deleteArtifactSequence(input: {
+ artifactSequenceID: $id
+ }) {
+ artifactCollection {
+ state
+ }
+ }
+ }
+ """
+ )
+ else:
+ mutation = gql(
+ """
+ mutation deleteArtifactPortfolio($id: ID!) {
+ deleteArtifactPortfolio(input: {
+ artifactPortfolioID: $id
+ }) {
+ artifactCollection {
+ state
+ }
+ }
+ }
+ """
+ )
+ self.client.execute(mutation, variable_values={"id": self.id})
+
+ def __repr__(self):
+ return f"<ArtifactCollection {self.name} ({self.type})>"
+
+
+class Artifacts(Paginator):
+ """An iterable collection of artifact versions associated with a project and optional filter.
+
+ This is generally used indirectly via the `Api`.artifact_versions method.
+ """
+
+ def __init__(
+ self,
+ client: Client,
+ entity: str,
+ project: str,
+ collection_name: str,
+ type: str,
+ filters: Optional[Mapping[str, Any]] = None,
+ order: Optional[str] = None,
+ per_page: int = 50,
+ ):
+ self.entity = entity
+ self.collection_name = collection_name
+ self.type = type
+ self.project = project
+ self.filters = {"state": "COMMITTED"} if filters is None else filters
+ self.order = order
+ variables = {
+ "project": self.project,
+ "entity": self.entity,
+ "order": self.order,
+ "type": self.type,
+ "collection": self.collection_name,
+ "filters": json.dumps(self.filters),
+ }
+ self.QUERY = gql(
+ """
+ query Artifacts($project: String!, $entity: String!, $type: String!, $collection: String!, $cursor: String, $perPage: Int = 50, $order: String, $filters: JSONString) {{
+ project(name: $project, entityName: $entity) {{
+ artifactType(name: $type) {{
+ artifactCollection: {}(name: $collection) {{
+ name
+ artifacts(filters: $filters, after: $cursor, first: $perPage, order: $order) {{
+ totalCount
+ edges {{
+ node {{
+ ...ArtifactFragment
+ }}
+ version
+ cursor
+ }}
+ pageInfo {{
+ endCursor
+ hasNextPage
+ }}
+ }}
+ }}
+ }}
+ }}
+ }}
+ {}
+ """.format(
+ artifact_collection_edge_name(
+ server_supports_artifact_collections_gql_edges(client)
+ ),
+ wandb.Artifact._get_gql_artifact_fragment(),
+ )
+ )
+ super().__init__(client, variables, per_page)
+
+ @property
+ def length(self):
+ if self.last_response:
+ return self.last_response["project"]["artifactType"]["artifactCollection"][
+ "artifacts"
+ ]["totalCount"]
+ else:
+ return None
+
+ @property
+ def more(self):
+ if self.last_response:
+ return self.last_response["project"]["artifactType"]["artifactCollection"][
+ "artifacts"
+ ]["pageInfo"]["hasNextPage"]
+ else:
+ return True
+
+ @property
+ def cursor(self):
+ if self.last_response:
+ return self.last_response["project"]["artifactType"]["artifactCollection"][
+ "artifacts"
+ ]["edges"][-1]["cursor"]
+ else:
+ return None
+
+ def convert_objects(self):
+ if self.last_response["project"]["artifactType"]["artifactCollection"] is None:
+ return []
+ return [
+ wandb.Artifact._from_attrs(
+ self.entity,
+ self.project,
+ self.collection_name + ":" + a["version"],
+ a["node"],
+ self.client,
+ )
+ for a in self.last_response["project"]["artifactType"][
+ "artifactCollection"
+ ]["artifacts"]["edges"]
+ ]
+
+
+class RunArtifacts(Paginator):
+ def __init__(
+ self, client: Client, run: "Run", mode="logged", per_page: Optional[int] = 50
+ ):
+ output_query = gql(
+ """
+ query RunOutputArtifacts(
+ $entity: String!, $project: String!, $runName: String!, $cursor: String, $perPage: Int,
+ ) {
+ project(name: $project, entityName: $entity) {
+ run(name: $runName) {
+ outputArtifacts(after: $cursor, first: $perPage) {
+ totalCount
+ edges {
+ node {
+ ...ArtifactFragment
+ }
+ cursor
+ }
+ pageInfo {
+ endCursor
+ hasNextPage
+ }
+ }
+ }
+ }
+ }
+ """
+ + wandb.Artifact._get_gql_artifact_fragment()
+ )
+
+ input_query = gql(
+ """
+ query RunInputArtifacts(
+ $entity: String!, $project: String!, $runName: String!, $cursor: String, $perPage: Int,
+ ) {
+ project(name: $project, entityName: $entity) {
+ run(name: $runName) {
+ inputArtifacts(after: $cursor, first: $perPage) {
+ totalCount
+ edges {
+ node {
+ ...ArtifactFragment
+ }
+ cursor
+ }
+ pageInfo {
+ endCursor
+ hasNextPage
+ }
+ }
+ }
+ }
+ }
+ """
+ + wandb.Artifact._get_gql_artifact_fragment()
+ )
+
+ self.run = run
+ if mode == "logged":
+ self.run_key = "outputArtifacts"
+ self.QUERY = output_query
+ elif mode == "used":
+ self.run_key = "inputArtifacts"
+ self.QUERY = input_query
+ else:
+ raise ValueError("mode must be logged or used")
+
+ variable_values = {
+ "entity": run.entity,
+ "project": run.project,
+ "runName": run.id,
+ }
+
+ super().__init__(client, variable_values, per_page)
+
+ @property
+ def length(self):
+ if self.last_response:
+ return self.last_response["project"]["run"][self.run_key]["totalCount"]
+ else:
+ return None
+
+ @property
+ def more(self):
+ if self.last_response:
+ return self.last_response["project"]["run"][self.run_key]["pageInfo"][
+ "hasNextPage"
+ ]
+ else:
+ return True
+
+ @property
+ def cursor(self):
+ if self.last_response:
+ return self.last_response["project"]["run"][self.run_key]["edges"][-1][
+ "cursor"
+ ]
+ else:
+ return None
+
+ def convert_objects(self):
+ return [
+ wandb.Artifact._from_attrs(
+ r["node"]["artifactSequence"]["project"]["entityName"],
+ r["node"]["artifactSequence"]["project"]["name"],
+ "{}:v{}".format(
+ r["node"]["artifactSequence"]["name"], r["node"]["versionIndex"]
+ ),
+ r["node"],
+ self.client,
+ )
+ for r in self.last_response["project"]["run"][self.run_key]["edges"]
+ ]
+
+
+class ArtifactFiles(Paginator):
+ QUERY = gql(
+ """
+ query ArtifactFiles(
+ $entityName: String!,
+ $projectName: String!,
+ $artifactTypeName: String!,
+ $artifactName: String!
+ $fileNames: [String!],
+ $fileCursor: String,
+ $fileLimit: Int = 50
+ ) {
+ project(name: $projectName, entityName: $entityName) {
+ artifactType(name: $artifactTypeName) {
+ artifact(name: $artifactName) {
+ ...ArtifactFilesFragment
+ }
+ }
+ }
+ }
+ %s
+ """
+ % ARTIFACT_FILES_FRAGMENT
+ )
+
+ def __init__(
+ self,
+ client: Client,
+ artifact: "wandb.Artifact",
+ names: Optional[Sequence[str]] = None,
+ per_page: int = 50,
+ ):
+ self.artifact = artifact
+ variables = {
+ "entityName": artifact.source_entity,
+ "projectName": artifact.source_project,
+ "artifactTypeName": artifact.type,
+ "artifactName": artifact.source_name,
+ "fileNames": names,
+ }
+ # The server must advertise at least SDK 0.12.21
+ # to get storagePath
+ if not client.version_supported("0.12.21"):
+ self.QUERY = gql(self.QUERY.loc.source.body.replace("storagePath\n", ""))
+ super().__init__(client, variables, per_page)
+
+ @property
+ def path(self):
+ return [self.artifact.entity, self.artifact.project, self.artifact.name]
+
+ @property
+ def length(self):
+ return self.artifact.file_count
+
+ @property
+ def more(self):
+ if self.last_response:
+ return self.last_response["project"]["artifactType"]["artifact"]["files"][
+ "pageInfo"
+ ]["hasNextPage"]
+ else:
+ return True
+
+ @property
+ def cursor(self):
+ if self.last_response:
+ return self.last_response["project"]["artifactType"]["artifact"]["files"][
+ "edges"
+ ][-1]["cursor"]
+ else:
+ return None
+
+ def update_variables(self):
+ self.variables.update({"fileLimit": self.per_page, "fileCursor": self.cursor})
+
+ def convert_objects(self):
+ return [
+ public.File(self.client, r["node"])
+ for r in self.last_response["project"]["artifactType"]["artifact"]["files"][
+ "edges"
+ ]
+ ]
+
+ def __repr__(self):
+ return "<ArtifactFiles {} ({})>".format("/".join(self.path), len(self))
+
+
+def server_supports_artifact_collections_gql_edges(
+ client: "RetryingClient", warn: bool = False
+) -> bool:
+ # TODO: Validate this version
+ # Edges were merged into core on Mar 2, 2022: https://github.com/wandb/core/commit/81c90b29eaacfe0a96dc1ebd83c53560ca763e8b
+ # CLI version was bumped to "0.12.11" on Mar 3, 2022: https://github.com/wandb/core/commit/328396fa7c89a2178d510a1be9c0d4451f350d7b
+ supported = client.version_supported("0.12.11") # edges were merged on
+ if not supported and warn:
+ # First local release to include the above is 0.9.50: https://github.com/wandb/local/releases/tag/0.9.50
+ wandb.termwarn(
+ "W&B Local Server version does not support ArtifactCollection gql edges; falling back to using legacy ArtifactSequence. Please update server to at least version 0.9.50."
+ )
+ return supported
+
+
+def artifact_collection_edge_name(server_supports_artifact_collections: bool) -> str:
+ return (
+ "artifactCollection"
+ if server_supports_artifact_collections
+ else "artifactSequence"
+ )
+
+
+def artifact_collection_plural_edge_name(
+ server_supports_artifact_collections: bool,
+) -> str:
+ return (
+ "artifactCollections"
+ if server_supports_artifact_collections
+ else "artifactSequences"
+ )
diff --git a/wandb/apis/public/const.py b/wandb/apis/public/const.py
new file mode 100644
index 0000000000000000000000000000000000000000..77af2da729d553a25f97a864546639f520b8817a
--- /dev/null
+++ b/wandb/apis/public/const.py
@@ -0,0 +1,4 @@
+import datetime
+
+# Only retry requests for 20 seconds in the public api
+RETRY_TIMEDELTA = datetime.timedelta(seconds=20)
diff --git a/wandb/apis/public/files.py b/wandb/apis/public/files.py
new file mode 100644
index 0000000000000000000000000000000000000000..cdb5ed6946a2be20e9d13a7fda5c61157e81304a
--- /dev/null
+++ b/wandb/apis/public/files.py
@@ -0,0 +1,185 @@
+"""Public API: files."""
+import io
+import os
+
+import requests
+from wandb_gql import gql
+from wandb_gql.client import RetryError
+
+import wandb
+from wandb import util
+from wandb.apis.attrs import Attrs
+from wandb.apis.normalize import normalize_exceptions
+from wandb.apis.paginator import Paginator
+from wandb.apis.public.const import RETRY_TIMEDELTA
+from wandb.sdk.lib import retry
+
+FILE_FRAGMENT = """fragment RunFilesFragment on Run {
+ files(names: $fileNames, after: $fileCursor, first: $fileLimit) {
+ edges {
+ node {
+ id
+ name
+ url(upload: $upload)
+ directUrl
+ sizeBytes
+ mimetype
+ updatedAt
+ md5
+ }
+ cursor
+ }
+ pageInfo {
+ endCursor
+ hasNextPage
+ }
+ }
+}"""
+
+
+class Files(Paginator):
+ """An iterable collection of `File` objects."""
+
+ QUERY = gql(
+ """
+ query RunFiles($project: String!, $entity: String!, $name: String!, $fileCursor: String,
+ $fileLimit: Int = 50, $fileNames: [String] = [], $upload: Boolean = false) {
+ project(name: $project, entityName: $entity) {
+ run(name: $name) {
+ fileCount
+ ...RunFilesFragment
+ }
+ }
+ }
+ %s
+ """
+ % FILE_FRAGMENT
+ )
+
+ def __init__(self, client, run, names=None, per_page=50, upload=False):
+ self.run = run
+ variables = {
+ "project": run.project,
+ "entity": run.entity,
+ "name": run.id,
+ "fileNames": names or [],
+ "upload": upload,
+ }
+ super().__init__(client, variables, per_page)
+
+ @property
+ def length(self):
+ if self.last_response:
+ return self.last_response["project"]["run"]["fileCount"]
+ else:
+ return None
+
+ @property
+ def more(self):
+ if self.last_response:
+ return self.last_response["project"]["run"]["files"]["pageInfo"][
+ "hasNextPage"
+ ]
+ else:
+ return True
+
+ @property
+ def cursor(self):
+ if self.last_response:
+ return self.last_response["project"]["run"]["files"]["edges"][-1]["cursor"]
+ else:
+ return None
+
+ def update_variables(self):
+ self.variables.update({"fileLimit": self.per_page, "fileCursor": self.cursor})
+
+ def convert_objects(self):
+ return [
+ File(self.client, r["node"])
+ for r in self.last_response["project"]["run"]["files"]["edges"]
+ ]
+
+ def __repr__(self):
+ return "<Files {} ({})>".format("/".join(self.run.path), len(self))
+
+
+class File(Attrs):
+ """File is a class associated with a file saved by wandb.
+
+ Attributes:
+ name (string): filename
+ url (string): path to file
+ direct_url (string): path to file in the bucket
+ md5 (string): md5 of file
+ mimetype (string): mimetype of file
+ updated_at (string): timestamp of last update
+ size (int): size of file in bytes
+
+ """
+
+ def __init__(self, client, attrs):
+ self.client = client
+ self._attrs = attrs
+ super().__init__(dict(attrs))
+
+ @property
+ def size(self):
+ size_bytes = self._attrs["sizeBytes"]
+ if size_bytes is not None:
+ return int(size_bytes)
+ return 0
+
+ @normalize_exceptions
+ @retry.retriable(
+ retry_timedelta=RETRY_TIMEDELTA,
+ check_retry_fn=util.no_retry_auth,
+ retryable_exceptions=(RetryError, requests.RequestException),
+ )
+ def download(
+ self, root: str = ".", replace: bool = False, exist_ok: bool = False
+ ) -> io.TextIOWrapper:
+ """Downloads a file previously saved by a run from the wandb server.
+
+ Arguments:
+ replace (boolean): If `True`, download will overwrite a local file
+ if it exists. Defaults to `False`.
+ root (str): Local directory to save the file. Defaults to ".".
+ exist_ok (boolean): If `True`, will not raise ValueError if file already
+ exists and will not re-download unless replace=True. Defaults to `False`.
+
+ Raises:
+ `ValueError` if file already exists, replace=False and exist_ok=False.
+ """
+ path = os.path.join(root, self.name)
+ if os.path.exists(path) and not replace:
+ if exist_ok:
+ return open(path)
+ else:
+ raise ValueError(
+ "File already exists, pass replace=True to overwrite or exist_ok=True to leave it as is and don't error."
+ )
+
+ util.download_file_from_url(path, self.url, wandb.Api().api_key)
+ return open(path)
+
+ @normalize_exceptions
+ def delete(self):
+ mutation = gql(
+ """
+ mutation deleteFiles($files: [ID!]!) {
+ deleteFiles(input: {
+ files: $files
+ }) {
+ success
+ }
+ }
+ """
+ )
+ self.client.execute(mutation, variable_values={"files": [self.id]})
+
+ def __repr__(self):
+ return "<File {} ({}) {}>".format(
+ self.name,
+ self.mimetype,
+ util.to_human_size(self.size, units=util.POW_2_BYTES),
+ )
diff --git a/wandb/apis/public/history.py b/wandb/apis/public/history.py
new file mode 100644
index 0000000000000000000000000000000000000000..c7dc65f5e9afeae74cf5d94306e9bef0e0dccd25
--- /dev/null
+++ b/wandb/apis/public/history.py
@@ -0,0 +1,148 @@
+"""Public API: history."""
+import json
+
+import requests
+from wandb_gql import gql
+from wandb_gql.client import RetryError
+
+from wandb import util
+from wandb.apis.normalize import normalize_exceptions
+from wandb.sdk.lib import retry
+
+
+class HistoryScan:
+ QUERY = gql(
+ """
+ query HistoryPage($entity: String!, $project: String!, $run: String!, $minStep: Int64!, $maxStep: Int64!, $pageSize: Int!) {
+ project(name: $project, entityName: $entity) {
+ run(name: $run) {
+ history(minStep: $minStep, maxStep: $maxStep, samples: $pageSize)
+ }
+ }
+ }
+ """
+ )
+
+ def __init__(self, client, run, min_step, max_step, page_size=1000):
+ self.client = client
+ self.run = run
+ self.page_size = page_size
+ self.min_step = min_step
+ self.max_step = max_step
+ self.page_offset = min_step # minStep for next page
+ self.scan_offset = 0 # index within current page of rows
+ self.rows = [] # current page of rows
+
+ def __iter__(self):
+ self.page_offset = self.min_step
+ self.scan_offset = 0
+ self.rows = []
+ return self
+
+ def __next__(self):
+ while True:
+ if self.scan_offset < len(self.rows):
+ row = self.rows[self.scan_offset]
+ self.scan_offset += 1
+ return row
+ if self.page_offset >= self.max_step:
+ raise StopIteration()
+ self._load_next()
+
+ next = __next__
+
+ @normalize_exceptions
+ @retry.retriable(
+ check_retry_fn=util.no_retry_auth,
+ retryable_exceptions=(RetryError, requests.RequestException),
+ )
+ def _load_next(self):
+ max_step = self.page_offset + self.page_size
+ if max_step > self.max_step:
+ max_step = self.max_step
+ variables = {
+ "entity": self.run.entity,
+ "project": self.run.project,
+ "run": self.run.id,
+ "minStep": int(self.page_offset),
+ "maxStep": int(max_step),
+ "pageSize": int(self.page_size),
+ }
+
+ res = self.client.execute(self.QUERY, variable_values=variables)
+ res = res["project"]["run"]["history"]
+ self.rows = [json.loads(row) for row in res]
+ self.page_offset += self.page_size
+ self.scan_offset = 0
+
+
+class SampledHistoryScan:
+ QUERY = gql(
+ """
+ query SampledHistoryPage($entity: String!, $project: String!, $run: String!, $spec: JSONString!) {
+ project(name: $project, entityName: $entity) {
+ run(name: $run) {
+ sampledHistory(specs: [$spec])
+ }
+ }
+ }
+ """
+ )
+
+ def __init__(self, client, run, keys, min_step, max_step, page_size=1000):
+ self.client = client
+ self.run = run
+ self.keys = keys
+ self.page_size = page_size
+ self.min_step = min_step
+ self.max_step = max_step
+ self.page_offset = min_step # minStep for next page
+ self.scan_offset = 0 # index within current page of rows
+ self.rows = [] # current page of rows
+
+ def __iter__(self):
+ self.page_offset = self.min_step
+ self.scan_offset = 0
+ self.rows = []
+ return self
+
+ def __next__(self):
+ while True:
+ if self.scan_offset < len(self.rows):
+ row = self.rows[self.scan_offset]
+ self.scan_offset += 1
+ return row
+ if self.page_offset >= self.max_step:
+ raise StopIteration()
+ self._load_next()
+
+ next = __next__
+
+ @normalize_exceptions
+ @retry.retriable(
+ check_retry_fn=util.no_retry_auth,
+ retryable_exceptions=(RetryError, requests.RequestException),
+ )
+ def _load_next(self):
+ max_step = self.page_offset + self.page_size
+ if max_step > self.max_step:
+ max_step = self.max_step
+ variables = {
+ "entity": self.run.entity,
+ "project": self.run.project,
+ "run": self.run.id,
+ "spec": json.dumps(
+ {
+ "keys": self.keys,
+ "minStep": int(self.page_offset),
+ "maxStep": int(max_step),
+ "samples": int(self.page_size),
+ }
+ ),
+ }
+
+ res = self.client.execute(self.QUERY, variable_values=variables)
+ res = res["project"]["run"]["sampledHistory"]
+ self.rows = res[0]
+ self.page_offset += self.page_size
+ self.scan_offset = 0
diff --git a/wandb/apis/public/jobs.py b/wandb/apis/public/jobs.py
new file mode 100644
index 0000000000000000000000000000000000000000..64aeac0a8d94565e9e074ff9c08e87b89f75f90d
--- /dev/null
+++ b/wandb/apis/public/jobs.py
@@ -0,0 +1,633 @@
+"""Public API: jobs."""
+import json
+import os
+import shutil
+import sys
+import time
+from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+from wandb_gql import gql
+
+import wandb
+from wandb import util
+from wandb.apis import public
+from wandb.apis.normalize import normalize_exceptions
+from wandb.errors import CommError
+from wandb.sdk.artifacts.artifact_state import ArtifactState
+from wandb.sdk.data_types._dtypes import InvalidType, Type, TypeRegistry
+from wandb.sdk.launch.errors import LaunchError
+from wandb.sdk.launch.utils import (
+ LAUNCH_DEFAULT_PROJECT,
+ _fetch_git_repo,
+ apply_patch,
+ convert_jupyter_notebook_to_script,
+)
+
+if TYPE_CHECKING:
+ from wandb.apis.public import Api, RetryingClient
+
+
+class Job:
+ _name: str
+ _input_types: Type
+ _output_types: Type
+ _entity: str
+ _project: str
+ _entrypoint: List[str]
+ _notebook_job: bool
+ _partial: bool
+
+ def __init__(self, api: "Api", name, path: Optional[str] = None) -> None:
+ try:
+ self._job_artifact = api.artifact(name, type="job")
+ except CommError:
+ raise CommError(f"Job artifact {name} not found")
+ if path:
+ self._fpath = path
+ self._job_artifact.download(root=path)
+ else:
+ self._fpath = self._job_artifact.download()
+ self._name = name
+ self._api = api
+ self._entity = api.default_entity
+
+ with open(os.path.join(self._fpath, "wandb-job.json")) as f:
+ self._job_info: Mapping[str, Any] = json.load(f)
+ source_info = self._job_info.get("source", {})
+ # only use notebook job if entrypoint not set and notebook is set
+ self._notebook_job = source_info.get("notebook", False)
+ self._entrypoint = source_info.get("entrypoint")
+ self._args = source_info.get("args")
+ self._partial = self._job_info.get("_partial", False)
+ self._requirements_file = os.path.join(self._fpath, "requirements.frozen.txt")
+ self._input_types = TypeRegistry.type_from_dict(
+ self._job_info.get("input_types")
+ )
+ self._output_types = TypeRegistry.type_from_dict(
+ self._job_info.get("output_types")
+ )
+ if self._job_info.get("source_type") == "artifact":
+ self._set_configure_launch_project(self._configure_launch_project_artifact)
+ if self._job_info.get("source_type") == "repo":
+ self._set_configure_launch_project(self._configure_launch_project_repo)
+ if self._job_info.get("source_type") == "image":
+ self._set_configure_launch_project(self._configure_launch_project_container)
+
+ @property
+ def name(self):
+ return self._name
+
+ def _set_configure_launch_project(self, func):
+ self.configure_launch_project = func
+
+ def _get_code_artifact(self, artifact_string):
+ artifact_string, base_url, is_id = util.parse_artifact_string(artifact_string)
+ if is_id:
+ code_artifact = wandb.Artifact._from_id(artifact_string, self._api._client)
+ else:
+ code_artifact = self._api.artifact(name=artifact_string, type="code")
+ if code_artifact is None:
+ raise LaunchError("No code artifact found")
+ if code_artifact.state == ArtifactState.DELETED:
+ raise LaunchError(
+ f"Job {self.name} references deleted code artifact {code_artifact.name}"
+ )
+ return code_artifact
+
+ def _configure_launch_project_notebook(self, launch_project):
+ new_fname = convert_jupyter_notebook_to_script(
+ self._entrypoint[-1], launch_project.project_dir
+ )
+ new_entrypoint = self._entrypoint
+ new_entrypoint[-1] = new_fname
+ launch_project.set_entry_point(new_entrypoint)
+
+ def _configure_launch_project_repo(self, launch_project):
+ git_info = self._job_info.get("source", {}).get("git", {})
+ _fetch_git_repo(
+ launch_project.project_dir,
+ git_info["remote"],
+ git_info["commit"],
+ )
+ if os.path.exists(os.path.join(self._fpath, "diff.patch")):
+ with open(os.path.join(self._fpath, "diff.patch")) as f:
+ apply_patch(f.read(), launch_project.project_dir)
+ shutil.copy(self._requirements_file, launch_project.project_dir)
+ launch_project.python_version = self._job_info.get("runtime")
+ if self._notebook_job:
+ self._configure_launch_project_notebook(launch_project)
+ else:
+ launch_project.set_entry_point(self._entrypoint)
+
+ def _configure_launch_project_artifact(self, launch_project):
+ artifact_string = self._job_info.get("source", {}).get("artifact")
+ if artifact_string is None:
+ raise LaunchError(f"Job {self.name} had no source artifact")
+
+ code_artifact = self._get_code_artifact(artifact_string)
+ launch_project.python_version = self._job_info.get("runtime")
+ shutil.copy(self._requirements_file, launch_project.project_dir)
+
+ code_artifact.download(launch_project.project_dir)
+
+ if self._notebook_job:
+ self._configure_launch_project_notebook(launch_project)
+ else:
+ launch_project.set_entry_point(self._entrypoint)
+
+ def _configure_launch_project_container(self, launch_project):
+ launch_project.docker_image = self._job_info.get("source", {}).get("image")
+ if launch_project.docker_image is None:
+ raise LaunchError(
+ "Job had malformed source dictionary without an image key"
+ )
+ if self._entrypoint:
+ launch_project.set_entry_point(self._entrypoint)
+
+ def set_entrypoint(self, entrypoint: List[str]):
+ self._entrypoint = entrypoint
+
+ def call(
+ self,
+ config,
+ project=None,
+ entity=None,
+ queue=None,
+ resource="local-container",
+ resource_args=None,
+ template_variables=None,
+ project_queue=None,
+ priority=None,
+ ):
+ from wandb.sdk.launch import _launch_add
+
+ run_config = {}
+ for key, item in config.items():
+ if util._is_artifact_object(item):
+ if isinstance(item, wandb.Artifact) and item.is_draft():
+ raise ValueError("Cannot queue jobs with unlogged artifacts")
+ run_config[key] = util.artifact_to_json(item)
+
+ run_config.update(config)
+
+ assigned_config_type = self._input_types.assign(run_config)
+ if self._partial:
+ wandb.termwarn(
+ "Launching manually created job for the first time, can't verify types"
+ )
+ else:
+ if isinstance(assigned_config_type, InvalidType):
+ raise TypeError(self._input_types.explain(run_config))
+
+ queued_run = _launch_add.launch_add(
+ job=self._name,
+ config={"overrides": {"run_config": run_config}},
+ template_variables=template_variables,
+ project=project or self._project,
+ entity=entity or self._entity,
+ queue_name=queue,
+ resource=resource,
+ project_queue=project_queue,
+ resource_args=resource_args,
+ priority=priority,
+ )
+ return queued_run
+
+
+class QueuedRun:
+ """A single queued run associated with an entity and project. Call `run = queued_run.wait_until_running()` or `run = queued_run.wait_until_finished()` to access the run."""
+
+ def __init__(
+ self,
+ client,
+ entity,
+ project,
+ queue_name,
+ run_queue_item_id,
+ project_queue=LAUNCH_DEFAULT_PROJECT,
+ priority=None,
+ ):
+ self.client = client
+ self._entity = entity
+ self._project = project
+ self._queue_name = queue_name
+ self._run_queue_item_id = run_queue_item_id
+ self.sweep = None
+ self._run = None
+ self.project_queue = project_queue
+ self.priority = priority
+
+ @property
+ def queue_name(self):
+ return self._queue_name
+
+ @property
+ def id(self):
+ return self._run_queue_item_id
+
+ @property
+ def project(self):
+ return self._project
+
+ @property
+ def entity(self):
+ return self._entity
+
+ @property
+ def state(self):
+ item = self._get_item()
+ if item:
+ return item["state"].lower()
+
+ raise ValueError(
+ f"Could not find QueuedRunItem associated with id: {self.id} on queue {self.queue_name} at itemId: {self.id}"
+ )
+
+ @normalize_exceptions
+ def _get_run_queue_item_legacy(self) -> Dict:
+ query = gql(
+ """
+ query GetRunQueueItem($projectName: String!, $entityName: String!, $runQueue: String!) {
+ project(name: $projectName, entityName: $entityName) {
+ runQueue(name:$runQueue) {
+ runQueueItems {
+ edges {
+ node {
+ id
+ state
+ associatedRunId
+ }
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+ variable_values = {
+ "projectName": self.project_queue,
+ "entityName": self._entity,
+ "runQueue": self.queue_name,
+ }
+ res = self.client.execute(query, variable_values)
+
+ for item in res["project"]["runQueue"]["runQueueItems"]["edges"]:
+ if str(item["node"]["id"]) == str(self.id):
+ return item["node"]
+
+ @normalize_exceptions
+ def _get_item(self):
+ query = gql(
+ """
+ query GetRunQueueItem($projectName: String!, $entityName: String!, $runQueue: String!, $itemId: ID!) {
+ project(name: $projectName, entityName: $entityName) {
+ runQueue(name: $runQueue) {
+ runQueueItem(id: $itemId) {
+ id
+ state
+ associatedRunId
+ }
+ }
+ }
+ }
+ """
+ )
+ variable_values = {
+ "projectName": self.project_queue,
+ "entityName": self._entity,
+ "runQueue": self.queue_name,
+ "itemId": self.id,
+ }
+ try:
+ res = self.client.execute(query, variable_values) # exception w/ old server
+ if res["project"]["runQueue"].get("runQueueItem") is not None:
+ return res["project"]["runQueue"]["runQueueItem"]
+ except Exception as e:
+ if "Cannot query field" not in str(e):
+ raise LaunchError(f"Unknown exception: {e}")
+
+ return self._get_run_queue_item_legacy()
+
+ @normalize_exceptions
+ def wait_until_finished(self):
+ if not self._run:
+ self.wait_until_running()
+
+ self._run.wait_until_finished()
+ # refetch run to get updated summary
+ self._run.load(force=True)
+ return self._run
+
+ @normalize_exceptions
+ def delete(self, delete_artifacts=False):
+ """Delete the given queued run from the wandb backend."""
+ query = gql(
+ """
+ query fetchRunQueuesFromProject($entityName: String!, $projectName: String!, $runQueueName: String!) {
+ project(name: $projectName, entityName: $entityName) {
+ runQueue(name: $runQueueName) {
+ id
+ }
+ }
+ }
+ """
+ )
+
+ res = self.client.execute(
+ query,
+ variable_values={
+ "entityName": self.entity,
+ "projectName": self.project_queue,
+ "runQueueName": self.queue_name,
+ },
+ )
+
+ if res["project"].get("runQueue") is not None:
+ queue_id = res["project"]["runQueue"]["id"]
+
+ mutation = gql(
+ """
+ mutation DeleteFromRunQueue(
+ $queueID: ID!,
+ $runQueueItemId: ID!
+ ) {
+ deleteFromRunQueue(input: {
+ queueID: $queueID
+ runQueueItemId: $runQueueItemId
+ }) {
+ success
+ clientMutationId
+ }
+ }
+ """
+ )
+ self.client.execute(
+ mutation,
+ variable_values={
+ "queueID": queue_id,
+ "runQueueItemId": self._run_queue_item_id,
+ },
+ )
+
+ @normalize_exceptions
+ def wait_until_running(self):
+ if self._run is not None:
+ return self._run
+
+ while True:
+ # sleep here to hide an ugly warning
+ time.sleep(2)
+ item = self._get_item()
+ if item and item["associatedRunId"] is not None:
+ try:
+ self._run = public.Run(
+ self.client,
+ self._entity,
+ self.project,
+ item["associatedRunId"],
+ None,
+ )
+ self._run_id = item["associatedRunId"]
+ return self._run
+ except ValueError as e:
+ print(e)
+ elif item:
+ wandb.termlog("Waiting for run to start")
+
+ time.sleep(3)
+
+ def __repr__(self):
+ return f"<QueuedRun {self.queue_name} ({self.id})"
+
+
+RunQueueResourceType = Literal[
+ "local-container", "local-process", "kubernetes", "sagemaker", "gcp-vertex"
+]
+RunQueueAccessType = Literal["project", "user"]
+RunQueuePrioritizationMode = Literal["DISABLED", "V0"]
+
+
+class RunQueue:
+ def __init__(
+ self,
+ client: "RetryingClient",
+ name: str,
+ entity: str,
+ prioritization_mode: Optional[RunQueuePrioritizationMode] = None,
+ _access: Optional[RunQueueAccessType] = None,
+ _default_resource_config_id: Optional[int] = None,
+ _default_resource_config: Optional[dict] = None,
+ ) -> None:
+ self._name: str = name
+ self._client = client
+ self._entity = entity
+ self._prioritization_mode = prioritization_mode
+ self._access = _access
+ self._default_resource_config_id = _default_resource_config_id
+ self._default_resource_config = _default_resource_config
+ self._template_variables = None
+ self._type = None
+ self._items = None
+ self._id = None
+
+ @property
+ def name(self):
+ return self._name
+
+ @property
+ def entity(self):
+ return self._entity
+
+ @property
+ def prioritization_mode(self) -> RunQueuePrioritizationMode:
+ if self._prioritization_mode is None:
+ self._get_metadata()
+ return self._prioritization_mode
+
+ @property
+ def access(self) -> RunQueueAccessType:
+ if self._access is None:
+ self._get_metadata()
+ return self._access
+
+ @property
+ def type(self) -> RunQueueResourceType:
+ if self._type is None:
+ if self._default_resource_config_id is None:
+ self._get_metadata()
+ self._get_default_resource_config()
+ return self._type
+
+ @property
+ def default_resource_config(self):
+ if self._default_resource_config is None:
+ if self._default_resource_config_id is None:
+ self._get_metadata()
+ self._get_default_resource_config()
+ return self._default_resource_config
+
+ @property
+ def template_variables(self):
+ if self._template_variables is None:
+ if self._default_resource_config_id is None:
+ self._get_metadata()
+ self._get_default_resource_config()
+ return self._template_variables
+
+ @property
+ def id(self) -> str:
+ if self._id is None:
+ self._get_metadata()
+ return self._id
+
+ @property
+ def items(self) -> List[QueuedRun]:
+ """Up to the first 100 queued runs. Modifying this list will not modify the queue or any enqueued items!"""
+ # TODO(np): Add a paginated interface
+ if self._items is None:
+ self._get_items()
+ return self._items
+
+ @normalize_exceptions
+ def delete(self):
+ """Delete the run queue from the wandb backend."""
+ query = gql(
+ """
+ mutation DeleteRunQueue($id: ID!) {
+ deleteRunQueues(input: {queueIDs: [$id]}) {
+ success
+ clientMutationId
+ }
+ }
+ """
+ )
+ variable_values = {"id": self.id}
+ res = self._client.execute(query, variable_values)
+ if res["deleteRunQueues"]["success"]:
+ self._id = None
+ self._access = None
+ self._default_resource_config_id = None
+ self._default_resource_config = None
+ self._items = None
+ else:
+ raise CommError(f"Failed to delete run queue {self.name}")
+
+ def __repr__(self):
+ return f"<RunQueue {self._entity}/{self._name}>"
+
+ @normalize_exceptions
+ def _get_metadata(self):
+ query = gql(
+ """
+ query GetRunQueueMetadata($projectName: String!, $entityName: String!, $runQueue: String!) {
+ project(name: $projectName, entityName: $entityName) {
+ runQueue(name: $runQueue) {
+ id
+ access
+ defaultResourceConfigID
+ prioritizationMode
+ }
+ }
+ }
+ """
+ )
+ variable_values = {
+ "projectName": LAUNCH_DEFAULT_PROJECT,
+ "entityName": self._entity,
+ "runQueue": self._name,
+ }
+ res = self._client.execute(query, variable_values)
+ self._id = res["project"]["runQueue"]["id"]
+ self._access = res["project"]["runQueue"]["access"]
+ self._default_resource_config_id = res["project"]["runQueue"][
+ "defaultResourceConfigID"
+ ]
+ if self._default_resource_config_id is None:
+ self._default_resource_config = {}
+ self._prioritization_mode = res["project"]["runQueue"]["prioritizationMode"]
+
+ @normalize_exceptions
+ def _get_default_resource_config(self):
+ query = gql(
+ """
+ query GetDefaultResourceConfig($entityName: String!, $id: ID!) {
+ entity(name: $entityName) {
+ defaultResourceConfig(id: $id) {
+ config
+ resource
+ templateVariables {
+ name
+ schema
+ }
+ }
+ }
+ }
+ """
+ )
+ variable_values = {
+ "entityName": self._entity,
+ "id": self._default_resource_config_id,
+ }
+ res = self._client.execute(query, variable_values)
+ self._type = res["entity"]["defaultResourceConfig"]["resource"]
+ self._default_resource_config = res["entity"]["defaultResourceConfig"]["config"]
+ self._template_variables = res["entity"]["defaultResourceConfig"][
+ "templateVariables"
+ ]
+
+ @normalize_exceptions
+ def _get_items(self):
+ query = gql(
+ """
+ query GetRunQueueItems($projectName: String!, $entityName: String!, $runQueue: String!) {
+ project(name: $projectName, entityName: $entityName) {
+ runQueue(name: $runQueue) {
+ runQueueItems(first: 100) {
+ edges {
+ node {
+ id
+ }
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+ variable_values = {
+ "projectName": LAUNCH_DEFAULT_PROJECT,
+ "entityName": self._entity,
+ "runQueue": self._name,
+ }
+ res = self._client.execute(query, variable_values)
+ self._items = []
+ for item in res["project"]["runQueue"]["runQueueItems"]["edges"]:
+ self._items.append(
+ QueuedRun(
+ self._client,
+ self._entity,
+ LAUNCH_DEFAULT_PROJECT,
+ self._name,
+ item["node"]["id"],
+ )
+ )
+
+ @classmethod
+ def create(
+ cls,
+ name: str,
+ resource: "RunQueueResourceType",
+ entity: Optional[str] = None,
+ prioritization_mode: Optional["RunQueuePrioritizationMode"] = None,
+ config: Optional[dict] = None,
+ template_variables: Optional[dict] = None,
+ ) -> "RunQueue":
+ public_api = Api()
+ return public_api.create_run_queue(
+ name, resource, entity, prioritization_mode, config, template_variables
+ )
diff --git a/wandb/apis/public/projects.py b/wandb/apis/public/projects.py
new file mode 100644
index 0000000000000000000000000000000000000000..357bb9c7bacd51c8255ce54a6b176492175c16eb
--- /dev/null
+++ b/wandb/apis/public/projects.py
@@ -0,0 +1,161 @@
+"""Public API: projects."""
+from wandb_gql import gql
+
+from wandb.apis import public
+from wandb.apis.attrs import Attrs
+from wandb.apis.normalize import normalize_exceptions
+from wandb.apis.paginator import Paginator
+from wandb.sdk.lib import ipython
+
+PROJECT_FRAGMENT = """fragment ProjectFragment on Project {
+ id
+ name
+ entityName
+ createdAt
+ isBenchmark
+}"""
+
+
+class Projects(Paginator):
+ """An iterable collection of `Project` objects."""
+
+ QUERY = gql(
+ """
+ query Projects($entity: String, $cursor: String, $perPage: Int = 50) {
+ models(entityName: $entity, after: $cursor, first: $perPage) {
+ edges {
+ node {
+ ...ProjectFragment
+ }
+ cursor
+ }
+ pageInfo {
+ endCursor
+ hasNextPage
+ }
+ }
+ }
+ %s
+ """
+ % PROJECT_FRAGMENT
+ )
+
+ def __init__(self, client, entity, per_page=50):
+ self.client = client
+ self.entity = entity
+ variables = {
+ "entity": self.entity,
+ }
+ super().__init__(client, variables, per_page)
+
+ @property
+ def length(self):
+ return None
+
+ @property
+ def more(self):
+ if self.last_response:
+ return self.last_response["models"]["pageInfo"]["hasNextPage"]
+ else:
+ return True
+
+ @property
+ def cursor(self):
+ if self.last_response:
+ return self.last_response["models"]["edges"][-1]["cursor"]
+ else:
+ return None
+
+ def convert_objects(self):
+ return [
+ Project(self.client, self.entity, p["node"]["name"], p["node"])
+ for p in self.last_response["models"]["edges"]
+ ]
+
+ def __repr__(self):
+ return f"<Projects {self.entity}>"
+
+
+class Project(Attrs):
+ """A project is a namespace for runs."""
+
+ def __init__(self, client, entity, project, attrs):
+ super().__init__(dict(attrs))
+ self.client = client
+ self.name = project
+ self.entity = entity
+
+ @property
+ def path(self):
+ return [self.entity, self.name]
+
+ @property
+ def url(self):
+ return self.client.app_url + "/".join(self.path + ["workspace"])
+
+ def to_html(self, height=420, hidden=False):
+ """Generate HTML containing an iframe displaying this project."""
+ url = self.url + "?jupyter=true"
+ style = f"border:none;width:100%;height:{height}px;"
+ prefix = ""
+ if hidden:
+ style += "display:none;"
+ prefix = ipython.toggle_button("project")
+ return prefix + f"<iframe src={url!r} style={style!r}></iframe>"
+
+ def _repr_html_(self) -> str:
+ return self.to_html()
+
+ def __repr__(self):
+ return "<Project {}>".format("/".join(self.path))
+
+ @normalize_exceptions
+ def artifacts_types(self, per_page=50):
+ return public.ArtifactTypes(self.client, self.entity, self.name)
+
+ @normalize_exceptions
+ def sweeps(self):
+ query = gql(
+ """
+ query GetSweeps($project: String!, $entity: String!) {
+ project(name: $project, entityName: $entity) {
+ totalSweeps
+ sweeps {
+ edges {
+ node {
+ ...SweepFragment
+ }
+ cursor
+ }
+ pageInfo {
+ endCursor
+ hasNextPage
+ }
+ }
+ }
+ }
+ %s
+ """
+ % public.SWEEP_FRAGMENT
+ )
+ variable_values = {"project": self.name, "entity": self.entity}
+ ret = self.client.execute(query, variable_values)
+ if ret["project"]["totalSweeps"] < 1:
+ return []
+
+ return [
+ # match format of existing public sweep apis
+ public.Sweep(
+ self.client,
+ self.entity,
+ self.name,
+ e["node"]["name"],
+ attrs={
+ "id": e["node"]["id"],
+ "name": e["node"]["name"],
+ "bestLoss": e["node"]["bestLoss"],
+ "config": e["node"]["config"],
+ },
+ )
+ for e in ret["project"]["sweeps"]["edges"]
+ ]
diff --git a/wandb/apis/public/query_generator.py b/wandb/apis/public/query_generator.py
new file mode 100644
index 0000000000000000000000000000000000000000..258064d5da18b4d87cbedbaa199df43accb0a6a0
--- /dev/null
+++ b/wandb/apis/public/query_generator.py
@@ -0,0 +1,166 @@
+class QueryGenerator:
+ """QueryGenerator is a helper object to write filters for runs."""
+
+ INDIVIDUAL_OP_TO_MONGO = {
+ "!=": "$ne",
+ ">": "$gt",
+ ">=": "$gte",
+ "<": "$lt",
+ "<=": "$lte",
+ "IN": "$in",
+ "NIN": "$nin",
+ "REGEX": "$regex",
+ }
+ MONGO_TO_INDIVIDUAL_OP = {v: k for k, v in INDIVIDUAL_OP_TO_MONGO.items()}
+
+ GROUP_OP_TO_MONGO = {"AND": "$and", "OR": "$or"}
+ MONGO_TO_GROUP_OP = {v: k for k, v in GROUP_OP_TO_MONGO.items()}
+
+ def __init__(self):
+ pass
+
+ @classmethod
+ def format_order_key(cls, key: str):
+ if key.startswith("+") or key.startswith("-"):
+ direction = key[0]
+ key = key[1:]
+ else:
+ direction = "-"
+ parts = key.split(".")
+ if len(parts) == 1:
+ # Assume the user meant summary_metrics if not a run column
+ if parts[0] not in ["createdAt", "updatedAt", "name", "sweep"]:
+ return direction + "summary_metrics." + parts[0]
+ # Assume summary metrics if prefix isn't known
+ elif parts[0] not in ["config", "summary_metrics", "tags"]:
+ return direction + ".".join(["summary_metrics"] + parts)
+ else:
+ return direction + ".".join(parts)
+
+ def _is_group(self, op):
+ return op.get("filters") is not None
+
+ def _is_individual(self, op):
+ return op.get("key") is not None
+
+ def _to_mongo_op_value(self, op, value):
+ if op == "=":
+ return value
+ else:
+ return {self.INDIVIDUAL_OP_TO_MONGO[op]: value}
+
+ def key_to_server_path(self, key):
+ if key["section"] == "config":
+ return "config." + key["name"]
+ elif key["section"] == "summary":
+ return "summary_metrics." + key["name"]
+ elif key["section"] == "keys_info":
+ return "keys_info.keys." + key["name"]
+ elif key["section"] == "run":
+ return key["name"]
+ elif key["section"] == "tags":
+ return "tags." + key["name"]
+ raise ValueError("Invalid key: %s" % key)
+
+ def server_path_to_key(self, path):
+ if path.startswith("config."):
+ return {"section": "config", "name": path.split("config.", 1)[1]}
+ elif path.startswith("summary_metrics."):
+ return {"section": "summary", "name": path.split("summary_metrics.", 1)[1]}
+ elif path.startswith("keys_info.keys."):
+ return {"section": "keys_info", "name": path.split("keys_info.keys.", 1)[1]}
+ elif path.startswith("tags."):
+ return {"section": "tags", "name": path.split("tags.", 1)[1]}
+ else:
+ return {"section": "run", "name": path}
+
+ def keys_to_order(self, keys):
+ orders = []
+ for key in keys["keys"]:
+ order = self.key_to_server_path(key["key"])
+ if key.get("ascending"):
+ order = "+" + order
+ else:
+ order = "-" + order
+ orders.append(order)
+ # return ",".join(orders)
+ return orders
+
+ def order_to_keys(self, order):
+ keys = []
+ for k in order: # orderstr.split(","):
+ name = k[1:]
+ if k[0] == "+":
+ ascending = True
+ elif k[0] == "-":
+ ascending = False
+ else:
+ raise Exception("you must sort by ascending(+) or descending(-)")
+
+ key = {"key": {"section": "run", "name": name}, "ascending": ascending}
+ keys.append(key)
+
+ return {"keys": keys}
+
+ def _to_mongo_individual(self, filter):
+ if filter["key"]["name"] == "":
+ return None
+
+ if filter.get("value") is None and filter["op"] != "=" and filter["op"] != "!=":
+ return None
+
+ if filter.get("disabled") is not None and filter["disabled"]:
+ return None
+
+ if filter["key"]["section"] == "tags":
+ if filter["op"] == "IN":
+ return {"tags": {"$in": filter["value"]}}
+ if filter["value"] is False:
+ return {
+ "$or": [{"tags": None}, {"tags": {"$ne": filter["key"]["name"]}}]
+ }
+ else:
+ return {"tags": filter["key"]["name"]}
+ path = self.key_to_server_path(filter["key"])
+ if path is None:
+ return path
+ return {path: self._to_mongo_op_value(filter["op"], filter["value"])}
+
+ def filter_to_mongo(self, filter):
+ if self._is_individual(filter):
+ return self._to_mongo_individual(filter)
+ elif self._is_group(filter):
+ return {
+ self.GROUP_OP_TO_MONGO[filter["op"]]: [
+ self.filter_to_mongo(f) for f in filter["filters"]
+ ]
+ }
+
+ def mongo_to_filter(self, filter):
+ # Returns {"op": "OR", "filters": [{"op": "AND", "filters": []}]}
+ if filter is None:
+ return None # this covers the case where self.filter_to_mongo returns None.
+
+ group_op = None
+ for key in filter.keys():
+ # if self.MONGO_TO_GROUP_OP[key]:
+ if key in self.MONGO_TO_GROUP_OP:
+ group_op = key
+ break
+ if group_op is not None:
+ return {
+ "op": self.MONGO_TO_GROUP_OP[group_op],
+ "filters": [self.mongo_to_filter(f) for f in filter[group_op]],
+ }
+ else:
+ for k, v in filter.items():
+ if isinstance(v, dict):
+ # TODO: do we always have one key in this case?
+ op = next(iter(v.keys()))
+ return {
+ "key": self.server_path_to_key(k),
+ "op": self.MONGO_TO_INDIVIDUAL_OP[op],
+ "value": v[op],
+ }
+ else:
+ return {"key": self.server_path_to_key(k), "op": "=", "value": v}
diff --git a/wandb/apis/public/reports.py b/wandb/apis/public/reports.py
new file mode 100644
index 0000000000000000000000000000000000000000..d96bb7d1036ede3bfd384f25864350a880cbf9e4
--- /dev/null
+++ b/wandb/apis/public/reports.py
@@ -0,0 +1,468 @@
+"""Public API: reports."""
+import ast
+import json
+import sys
+import urllib
+
+from wandb_gql import gql
+
+import wandb
+from wandb.apis import public
+from wandb.apis.attrs import Attrs
+from wandb.apis.paginator import Paginator
+from wandb.sdk.lib import ipython
+
+
+class Reports(Paginator):
+ """Reports is an iterable collection of `BetaReport` objects."""
+
+ QUERY = gql(
+ """
+ query ProjectViews($project: String!, $entity: String!, $reportCursor: String,
+ $reportLimit: Int!, $viewType: String = "runs", $viewName: String) {
+ project(name: $project, entityName: $entity) {
+ allViews(viewType: $viewType, viewName: $viewName, first:
+ $reportLimit, after: $reportCursor) {
+ edges {
+ node {
+ id
+ name
+ displayName
+ description
+ user {
+ username
+ photoUrl
+ }
+ spec
+ updatedAt
+ }
+ cursor
+ }
+ pageInfo {
+ endCursor
+ hasNextPage
+ }
+
+ }
+ }
+ }
+ """
+ )
+
+ def __init__(self, client, project, name=None, entity=None, per_page=50):
+ self.project = project
+ self.name = name
+ variables = {
+ "project": project.name,
+ "entity": project.entity,
+ "viewName": self.name,
+ }
+ super().__init__(client, variables, per_page)
+
+ @property
+ def length(self):
+ # TODO: Add the count the backend
+ if self.last_response:
+ return len(self.objects)
+ else:
+ return None
+
+ @property
+ def more(self):
+ if self.last_response:
+ return self.last_response["project"]["allViews"]["pageInfo"]["hasNextPage"]
+ else:
+ return True
+
+ @property
+ def cursor(self):
+ if self.last_response:
+ return self.last_response["project"]["allViews"]["edges"][-1]["cursor"]
+ else:
+ return None
+
+ def update_variables(self):
+ self.variables.update(
+ {"reportCursor": self.cursor, "reportLimit": self.per_page}
+ )
+
+ def convert_objects(self):
+ if self.last_response["project"] is None:
+ raise ValueError(
+ f"Project {self.variables['project']} does not exist under entity {self.variables['entity']}"
+ )
+ return [
+ BetaReport(
+ self.client,
+ r["node"],
+ entity=self.project.entity,
+ project=self.project.name,
+ )
+ for r in self.last_response["project"]["allViews"]["edges"]
+ ]
+
+ def __repr__(self):
+ return "<Reports {}>".format("/".join(self.project.path))
+
+
+class BetaReport(Attrs):
+ """BetaReport is a class associated with reports created in wandb.
+
+ WARNING: this API will likely change in a future release
+
+ Attributes:
+ name (string): report name
+ description (string): report description;
+ user (User): the user that created the report
+ spec (dict): the spec off the report;
+ updated_at (string): timestamp of last update
+ """
+
+ def __init__(self, client, attrs, entity=None, project=None):
+ self.client = client
+ self.project = project
+ self.entity = entity
+ self.query_generator = public.QueryGenerator()
+ super().__init__(dict(attrs))
+ self._attrs["spec"] = json.loads(self._attrs["spec"])
+
+ @property
+ def sections(self):
+ return self.spec["panelGroups"]
+
+ def runs(self, section, per_page=50, only_selected=True):
+ run_set_idx = section.get("openRunSet", 0)
+ run_set = section["runSets"][run_set_idx]
+ order = self.query_generator.key_to_server_path(run_set["sort"]["key"])
+ if run_set["sort"].get("ascending"):
+ order = "+" + order
+ else:
+ order = "-" + order
+ filters = self.query_generator.filter_to_mongo(run_set["filters"])
+ if only_selected:
+ # TODO: handle this not always existing
+ filters["$or"][0]["$and"].append(
+ {"name": {"$in": run_set["selections"]["tree"]}}
+ )
+ return public.Runs(
+ self.client,
+ self.entity,
+ self.project,
+ filters=filters,
+ order=order,
+ per_page=per_page,
+ )
+
+ @property
+ def updated_at(self):
+ return self._attrs["updatedAt"]
+
+ @property
+ def url(self):
+ return self.client.app_url + "/".join(
+ [
+ self.entity,
+ self.project,
+ "reports",
+ "--".join(
+ [
+ urllib.parse.quote(self.display_name.replace(" ", "-")),
+ self.id.replace("=", ""),
+ ]
+ ),
+ ]
+ )
+
+ def to_html(self, height=1024, hidden=False):
+ """Generate HTML containing an iframe displaying this report."""
+ url = self.url + "?jupyter=true"
+ style = f"border:none;width:100%;height:{height}px;"
+ prefix = ""
+ if hidden:
+ style += "display:none;"
+ prefix = ipython.toggle_button("report")
+ return prefix + f"<iframe src={url!r} style={style!r}></iframe>"
+
+ def _repr_html_(self) -> str:
+ return self.to_html()
+
+
+class PythonMongoishQueryGenerator:
+ SPACER = "----------"
+ DECIMAL_SPACER = ";;;"
+ FRONTEND_NAME_MAPPING = {
+ "ID": "name",
+ "Name": "displayName",
+ "Tags": "tags",
+ "State": "state",
+ "CreatedTimestamp": "createdAt",
+ "Runtime": "duration",
+ "User": "username",
+ "Sweep": "sweep",
+ "Group": "group",
+ "JobType": "jobType",
+ "Hostname": "host",
+ "UsingArtifact": "inputArtifacts",
+ "OutputtingArtifact": "outputArtifacts",
+ "Step": "_step",
+ "Relative Time (Wall)": "_absolute_runtime",
+ "Relative Time (Process)": "_runtime",
+ "Wall Time": "_timestamp",
+ # "GroupedRuns": "__wb_group_by_all"
+ }
+ FRONTEND_NAME_MAPPING_REVERSED = {v: k for k, v in FRONTEND_NAME_MAPPING.items()}
+ AST_OPERATORS = {
+ ast.Lt: "$lt",
+ ast.LtE: "$lte",
+ ast.Gt: "$gt",
+ ast.GtE: "$gte",
+ ast.Eq: "=",
+ ast.Is: "=",
+ ast.NotEq: "$ne",
+ ast.IsNot: "$ne",
+ ast.In: "$in",
+ ast.NotIn: "$nin",
+ ast.And: "$and",
+ ast.Or: "$or",
+ ast.Not: "$not",
+ }
+
+ if sys.version_info >= (3, 8):
+ AST_FIELDS = {
+ ast.Constant: "value",
+ ast.Name: "id",
+ ast.List: "elts",
+ ast.Tuple: "elts",
+ }
+ else:
+ AST_FIELDS = {
+ ast.Str: "s",
+ ast.Num: "n",
+ ast.Name: "id",
+ ast.List: "elts",
+ ast.Tuple: "elts",
+ ast.NameConstant: "value",
+ }
+
+ def __init__(self, run_set):
+ self.run_set = run_set
+ self.panel_metrics_helper = PanelMetricsHelper()
+
+ def _handle_compare(self, node):
+ # only left side can be a col
+ left = self.front_to_back(self._handle_fields(node.left))
+ op = self._handle_ops(node.ops[0])
+ right = self._handle_fields(node.comparators[0])
+
+ # Eq has no op for some reason
+ if op == "=":
+ return {left: right}
+ else:
+ return {left: {op: right}}
+
+ def _handle_fields(self, node):
+ result = getattr(node, self.AST_FIELDS.get(type(node)))
+ if isinstance(result, list):
+ return [self._handle_fields(node) for node in result]
+ elif isinstance(result, str):
+ return self._unconvert(result)
+ return result
+
+ def _handle_ops(self, node):
+ return self.AST_OPERATORS.get(type(node))
+
+ def _replace_numeric_dots(self, s):
+ numeric_dots = []
+ for i, (left, mid, right) in enumerate(zip(s, s[1:], s[2:]), 1):
+ if mid == ".":
+ if (
+ left.isdigit()
+ and right.isdigit() # 1.2
+ or left.isdigit()
+ and right == " " # 1.
+ or left == " "
+ and right.isdigit() # .2
+ ):
+ numeric_dots.append(i)
+ # Edge: Catch number ending in dot at end of string
+ if s[-2].isdigit() and s[-1] == ".":
+ numeric_dots.append(len(s) - 1)
+ numeric_dots = [-1] + numeric_dots + [len(s)]
+
+ substrs = []
+ for start, stop in zip(numeric_dots, numeric_dots[1:]):
+ substrs.append(s[start + 1 : stop])
+ substrs.append(self.DECIMAL_SPACER)
+ substrs = substrs[:-1]
+ return "".join(substrs)
+
+ def _convert(self, filterstr):
+ _conversion = (
+ self._replace_numeric_dots(filterstr) # temporarily sub numeric dots
+ .replace(".", self.SPACER) # Allow dotted fields
+ .replace(self.DECIMAL_SPACER, ".") # add them back
+ )
+ return "(" + _conversion + ")"
+
+ def _unconvert(self, field_name):
+ return field_name.replace(self.SPACER, ".") # Allow dotted fields
+
+ def python_to_mongo(self, filterstr):
+ try:
+ tree = ast.parse(self._convert(filterstr), mode="eval")
+ except SyntaxError as e:
+ raise ValueError(
+ "Invalid python comparison expression; form something like `my_col == 123`"
+ ) from e
+
+ multiple_filters = hasattr(tree.body, "op")
+
+ if multiple_filters:
+ op = self.AST_OPERATORS.get(type(tree.body.op))
+ values = [self._handle_compare(v) for v in tree.body.values]
+ else:
+ op = "$and"
+ values = [self._handle_compare(tree.body)]
+ return {"$or": [{op: values}]}
+
+ def front_to_back(self, name):
+ name, *rest = name.split(".")
+ rest = "." + ".".join(rest) if rest else ""
+
+ if name in self.FRONTEND_NAME_MAPPING:
+ return self.FRONTEND_NAME_MAPPING[name]
+ elif name in self.FRONTEND_NAME_MAPPING_REVERSED:
+ return name
+ elif name in self.run_set._runs_config:
+ return f"config.{name}.value{rest}"
+ else: # assume summary metrics
+ return f"summary_metrics.{name}{rest}"
+
+ def back_to_front(self, name):
+ if name in self.FRONTEND_NAME_MAPPING_REVERSED:
+ return self.FRONTEND_NAME_MAPPING_REVERSED[name]
+ elif name in self.FRONTEND_NAME_MAPPING:
+ return name
+ elif (
+ name.startswith("config.") and ".value" in name
+ ): # may be brittle: originally "endswith", but that doesn't work with nested keys...
+ # strip is weird sometimes (??)
+ return name.replace("config.", "").replace(".value", "")
+ elif name.startswith("summary_metrics."):
+ return name.replace("summary_metrics.", "")
+ wandb.termerror(f"Unknown token: {name}")
+ return name
+
+ # These are only used for ParallelCoordinatesPlot because it has weird backend names...
+ def pc_front_to_back(self, name):
+ name, *rest = name.split(".")
+ rest = "." + ".".join(rest) if rest else ""
+ if name is None:
+ return None
+ elif name in self.panel_metrics_helper.FRONTEND_NAME_MAPPING:
+ return "summary:" + self.panel_metrics_helper.FRONTEND_NAME_MAPPING[name]
+ elif name in self.FRONTEND_NAME_MAPPING:
+ return self.FRONTEND_NAME_MAPPING[name]
+ elif name in self.FRONTEND_NAME_MAPPING_REVERSED:
+ return name
+ elif name in self.run_set._runs_config:
+ return f"config:{name}.value{rest}"
+ else: # assume summary metrics
+ return f"summary:{name}{rest}"
+
+ def pc_back_to_front(self, name):
+ if name is None:
+ return None
+ elif "summary:" in name:
+ name = name.replace("summary:", "")
+ return self.panel_metrics_helper.FRONTEND_NAME_MAPPING_REVERSED.get(
+ name, name
+ )
+ elif name in self.FRONTEND_NAME_MAPPING_REVERSED:
+ return self.FRONTEND_NAME_MAPPING_REVERSED[name]
+ elif name in self.FRONTEND_NAME_MAPPING:
+ return name
+ elif name.startswith("config:") and ".value" in name:
+ return name.replace("config:", "").replace(".value", "")
+ elif name.startswith("summary_metrics."):
+ return name.replace("summary_metrics.", "")
+ return name
+
+
+class PanelMetricsHelper:
+ FRONTEND_NAME_MAPPING = {
+ "Step": "_step",
+ "Relative Time (Wall)": "_absolute_runtime",
+ "Relative Time (Process)": "_runtime",
+ "Wall Time": "_timestamp",
+ }
+ FRONTEND_NAME_MAPPING_REVERSED = {v: k for k, v in FRONTEND_NAME_MAPPING.items()}
+
+ RUN_MAPPING = {"Created Timestamp": "createdAt", "Latest Timestamp": "heartbeatAt"}
+ RUN_MAPPING_REVERSED = {v: k for k, v in RUN_MAPPING.items()}
+
+ def front_to_back(self, name):
+ if name in self.FRONTEND_NAME_MAPPING:
+ return self.FRONTEND_NAME_MAPPING[name]
+ return name
+
+ def back_to_front(self, name):
+ if name in self.FRONTEND_NAME_MAPPING_REVERSED:
+ return self.FRONTEND_NAME_MAPPING_REVERSED[name]
+ return name
+
+ # ScatterPlot and ParallelCoords have weird conventions
+ def special_front_to_back(self, name):
+ if name is None:
+ return name
+
+ name, *rest = name.split(".")
+ rest = "." + ".".join(rest) if rest else ""
+
+ # special case for config
+ if name.startswith("c::"):
+ name = name[3:]
+ return f"config:{name}.value{rest}"
+
+ # special case for summary
+ if name.startswith("s::"):
+ name = name[3:] + rest
+ return f"summary:{name}"
+
+ name = name + rest
+ if name in self.RUN_MAPPING:
+ return "run:" + self.RUN_MAPPING[name]
+ if name in self.FRONTEND_NAME_MAPPING:
+ return "summary:" + self.FRONTEND_NAME_MAPPING[name]
+ if name == "Index":
+ return name
+ return "summary:" + name
+
+ def special_back_to_front(self, name):
+ if name is not None:
+ kind, rest = name.split(":", 1)
+
+ if kind == "config":
+ pieces = rest.split(".")
+ if len(pieces) <= 1:
+ raise ValueError(f"Invalid name: {name}")
+ elif len(pieces) == 2:
+ name = pieces[0]
+ elif len(pieces) >= 3:
+ name = pieces[:1] + pieces[2:]
+ name = ".".join(name)
+ return f"c::{name}"
+
+ elif kind == "summary":
+ name = rest
+ return f"s::{name}"
+
+ if name is None:
+ return name
+ elif "summary:" in name:
+ name = name.replace("summary:", "")
+ return self.FRONTEND_NAME_MAPPING_REVERSED.get(name, name)
+ elif "run:" in name:
+ name = name.replace("run:", "")
+ return self.RUN_MAPPING_REVERSED[name]
+ return name
diff --git a/wandb/apis/public/runs.py b/wandb/apis/public/runs.py
new file mode 100644
index 0000000000000000000000000000000000000000..6b706a5669efe59135fa7258e6166699e8741cab
--- /dev/null
+++ b/wandb/apis/public/runs.py
@@ -0,0 +1,802 @@
+"""Public API: runs."""
+import json
+import os
+import tempfile
+import time
+import urllib
+from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional
+
+from wandb_gql import gql
+
+import wandb
+from wandb import env, util
+from wandb.apis import public
+from wandb.apis.attrs import Attrs
+from wandb.apis.internal import Api as InternalApi
+from wandb.apis.normalize import normalize_exceptions
+from wandb.apis.paginator import Paginator
+from wandb.apis.public.const import RETRY_TIMEDELTA
+from wandb.sdk.lib import ipython, json_util, runid
+from wandb.sdk.lib.paths import LogicalPath
+
+if TYPE_CHECKING:
+ from wandb.apis.public import RetryingClient
+
+WANDB_INTERNAL_KEYS = {"_wandb", "wandb_version"}
+
+RUN_FRAGMENT = """fragment RunFragment on Run {
+ id
+ tags
+ name
+ displayName
+ sweepName
+ state
+ config
+ group
+ jobType
+ commit
+ readOnly
+ createdAt
+ heartbeatAt
+ description
+ notes
+ systemMetrics
+ summaryMetrics
+ historyLineCount
+ user {
+ name
+ username
+ }
+ historyKeys
+}"""
+
+
+class Runs(Paginator):
+ """An iterable collection of runs associated with a project and optional filter.
+
+ This is generally used indirectly via the `Api`.runs method.
+ """
+
+ QUERY = gql(
+ """
+ query Runs($project: String!, $entity: String!, $cursor: String, $perPage: Int = 50, $order: String, $filters: JSONString) {
+ project(name: $project, entityName: $entity) {
+ runCount(filters: $filters)
+ readOnly
+ runs(filters: $filters, after: $cursor, first: $perPage, order: $order) {
+ edges {
+ node {
+ ...RunFragment
+ }
+ cursor
+ }
+ pageInfo {
+ endCursor
+ hasNextPage
+ }
+ }
+ }
+ }
+ %s
+ """
+ % RUN_FRAGMENT
+ )
+
+ def __init__(
+ self,
+ client: "RetryingClient",
+ entity: str,
+ project: str,
+ filters: Optional[Dict[str, Any]] = None,
+ order: Optional[str] = None,
+ per_page: int = 50,
+ include_sweeps: bool = True,
+ ):
+ self.entity = entity
+ self.project = project
+ self.filters = filters or {}
+ self.order = order
+ self._sweeps = {}
+ self._include_sweeps = include_sweeps
+ variables = {
+ "project": self.project,
+ "entity": self.entity,
+ "order": self.order,
+ "filters": json.dumps(self.filters),
+ }
+ super().__init__(client, variables, per_page)
+
+ @property
+ def length(self):
+ if self.last_response:
+ return self.last_response["project"]["runCount"]
+ else:
+ return None
+
+ @property
+ def more(self):
+ if self.last_response:
+ return self.last_response["project"]["runs"]["pageInfo"]["hasNextPage"]
+ else:
+ return True
+
+ @property
+ def cursor(self):
+ if self.last_response:
+ return self.last_response["project"]["runs"]["edges"][-1]["cursor"]
+ else:
+ return None
+
+ def convert_objects(self):
+ objs = []
+ if self.last_response is None or self.last_response.get("project") is None:
+ raise ValueError("Could not find project %s" % self.project)
+ for run_response in self.last_response["project"]["runs"]["edges"]:
+ run = Run(
+ self.client,
+ self.entity,
+ self.project,
+ run_response["node"]["name"],
+ run_response["node"],
+ include_sweeps=self._include_sweeps,
+ )
+ objs.append(run)
+
+ if self._include_sweeps and run.sweep_name:
+ if run.sweep_name in self._sweeps:
+ sweep = self._sweeps[run.sweep_name]
+ else:
+ sweep = public.Sweep.get(
+ self.client,
+ self.entity,
+ self.project,
+ run.sweep_name,
+ withRuns=False,
+ )
+ self._sweeps[run.sweep_name] = sweep
+
+ if sweep is None:
+ continue
+ run.sweep = sweep
+
+ return objs
+
+ def __repr__(self):
+ return f"<Runs {self.entity}/{self.project}>"
+
+
+class Run(Attrs):
+ """A single run associated with an entity and project.
+
+ Attributes:
+ tags ([str]): a list of tags associated with the run
+ url (str): the url of this run
+ id (str): unique identifier for the run (defaults to eight characters)
+ name (str): the name of the run
+ state (str): one of: running, finished, crashed, killed, preempting, preempted
+ config (dict): a dict of hyperparameters associated with the run
+ created_at (str): ISO timestamp when the run was started
+ system_metrics (dict): the latest system metrics recorded for the run
+ summary (dict): A mutable dict-like property that holds the current summary.
+ Calling update will persist any changes.
+ project (str): the project associated with the run
+ entity (str): the name of the entity associated with the run
+ user (str): the name of the user who created the run
+ path (str): Unique identifier [entity]/[project]/[run_id]
+ notes (str): Notes about the run
+ read_only (boolean): Whether the run is editable
+ history_keys (str): Keys of the history metrics that have been logged
+ with `wandb.log({key: value})`
+ metadata (str): Metadata about the run from wandb-metadata.json
+ """
+
+ def __init__(
+ self,
+ client: "RetryingClient",
+ entity: str,
+ project: str,
+ run_id: str,
+ attrs: Optional[Mapping] = None,
+ include_sweeps: bool = True,
+ ):
+ """Initialize a Run object.
+
+ Run is always initialized by calling api.runs() where api is an instance of
+ wandb.Api.
+ """
+ _attrs = attrs or {}
+ super().__init__(dict(_attrs))
+ self.client = client
+ self._entity = entity
+ self.project = project
+ self._files = {}
+ self._base_dir = env.get_dir(tempfile.gettempdir())
+ self.id = run_id
+ self.sweep = None
+ self._include_sweeps = include_sweeps
+ self.dir = os.path.join(self._base_dir, *self.path)
+ try:
+ os.makedirs(self.dir)
+ except OSError:
+ pass
+ self._summary = None
+ self._metadata: Optional[Dict[str, Any]] = None
+ self._state = _attrs.get("state", "not found")
+
+ self.load(force=not _attrs)
+
+ @property
+ def state(self):
+ return self._state
+
+ @property
+ def entity(self):
+ return self._entity
+
+ @property
+ def username(self):
+ wandb.termwarn("Run.username is deprecated. Please use Run.entity instead.")
+ return self._entity
+
+ @property
+ def storage_id(self):
+ # For compatibility with wandb.Run, which has storage IDs
+ # in self.storage_id and names in self.id.
+
+ return self._attrs.get("id")
+
+ @property
+ def id(self):
+ return self._attrs.get("name")
+
+ @id.setter
+ def id(self, new_id):
+ attrs = self._attrs
+ attrs["name"] = new_id
+ return new_id
+
+ @property
+ def name(self):
+ return self._attrs.get("displayName")
+
+ @name.setter
+ def name(self, new_name):
+ self._attrs["displayName"] = new_name
+ return new_name
+
+ @classmethod
+ def create(cls, api, run_id=None, project=None, entity=None):
+ """Create a run for the given project."""
+ run_id = run_id or runid.generate_id()
+ project = project or api.settings.get("project") or "uncategorized"
+ mutation = gql(
+ """
+ mutation UpsertBucket($project: String, $entity: String, $name: String!) {
+ upsertBucket(input: {modelName: $project, entityName: $entity, name: $name}) {
+ bucket {
+ project {
+ name
+ entity { name }
+ }
+ id
+ name
+ }
+ inserted
+ }
+ }
+ """
+ )
+ variables = {"entity": entity, "project": project, "name": run_id}
+ res = api.client.execute(mutation, variable_values=variables)
+ res = res["upsertBucket"]["bucket"]
+ return Run(
+ api.client,
+ res["project"]["entity"]["name"],
+ res["project"]["name"],
+ res["name"],
+ {
+ "id": res["id"],
+ "config": "{}",
+ "systemMetrics": "{}",
+ "summaryMetrics": "{}",
+ "tags": [],
+ "description": None,
+ "notes": None,
+ "state": "running",
+ },
+ )
+
+ def load(self, force=False):
+ query = gql(
+ """
+ query Run($project: String!, $entity: String!, $name: String!) {
+ project(name: $project, entityName: $entity) {
+ run(name: $name) {
+ ...RunFragment
+ }
+ }
+ }
+ %s
+ """
+ % RUN_FRAGMENT
+ )
+ if force or not self._attrs:
+ response = self._exec(query)
+ if (
+ response is None
+ or response.get("project") is None
+ or response["project"].get("run") is None
+ ):
+ raise ValueError("Could not find run %s" % self)
+ self._attrs = response["project"]["run"]
+ self._state = self._attrs["state"]
+
+ if self._include_sweeps and self.sweep_name and not self.sweep:
+ # There may be a lot of runs. Don't bother pulling them all
+ # just for the sake of this one.
+ self.sweep = public.Sweep.get(
+ self.client,
+ self.entity,
+ self.project,
+ self.sweep_name,
+ withRuns=False,
+ )
+
+ try:
+ self._attrs["summaryMetrics"] = (
+ json.loads(self._attrs["summaryMetrics"])
+ if self._attrs.get("summaryMetrics")
+ else {}
+ )
+ except json.decoder.JSONDecodeError:
+ # ignore invalid utf-8 or control characters
+ self._attrs["summaryMetrics"] = json.loads(
+ self._attrs["summaryMetrics"],
+ strict=False,
+ )
+ self._attrs["systemMetrics"] = (
+ json.loads(self._attrs["systemMetrics"])
+ if self._attrs.get("systemMetrics")
+ else {}
+ )
+ if self._attrs.get("user"):
+ self.user = public.User(self.client, self._attrs["user"])
+ config_user, config_raw = {}, {}
+ for key, value in json.loads(self._attrs.get("config") or "{}").items():
+ config = config_raw if key in WANDB_INTERNAL_KEYS else config_user
+ if isinstance(value, dict) and "value" in value:
+ config[key] = value["value"]
+ else:
+ config[key] = value
+ config_raw.update(config_user)
+ self._attrs["config"] = config_user
+ self._attrs["rawconfig"] = config_raw
+ return self._attrs
+
+ @normalize_exceptions
+ def wait_until_finished(self):
+ query = gql(
+ """
+ query RunState($project: String!, $entity: String!, $name: String!) {
+ project(name: $project, entityName: $entity) {
+ run(name: $name) {
+ state
+ }
+ }
+ }
+ """
+ )
+ while True:
+ res = self._exec(query)
+ state = res["project"]["run"]["state"]
+ if state in ["finished", "crashed", "failed"]:
+ print(f"Run finished with status: {state}")
+ self._attrs["state"] = state
+ self._state = state
+ return
+ time.sleep(5)
+
+ @normalize_exceptions
+ def update(self):
+ """Persist changes to the run object to the wandb backend."""
+ mutation = gql(
+ """
+ mutation UpsertBucket($id: String!, $description: String, $display_name: String, $notes: String, $tags: [String!], $config: JSONString!, $groupName: String) {
+ upsertBucket(input: {id: $id, description: $description, displayName: $display_name, notes: $notes, tags: $tags, config: $config, groupName: $groupName}) {
+ bucket {
+ ...RunFragment
+ }
+ }
+ }
+ %s
+ """
+ % RUN_FRAGMENT
+ )
+ _ = self._exec(
+ mutation,
+ id=self.storage_id,
+ tags=self.tags,
+ description=self.description,
+ notes=self.notes,
+ display_name=self.display_name,
+ config=self.json_config,
+ groupName=self.group,
+ )
+ self.summary.update()
+
+ @normalize_exceptions
+ def delete(self, delete_artifacts=False):
+ """Delete the given run from the wandb backend."""
+ mutation = gql(
+ """
+ mutation DeleteRun(
+ $id: ID!,
+ {}
+ ) {{
+ deleteRun(input: {{
+ id: $id,
+ {}
+ }}) {{
+ clientMutationId
+ }}
+ }}
+ """.format(
+ "$deleteArtifacts: Boolean" if delete_artifacts else "",
+ "deleteArtifacts: $deleteArtifacts" if delete_artifacts else "",
+ )
+ )
+
+ self.client.execute(
+ mutation,
+ variable_values={
+ "id": self.storage_id,
+ "deleteArtifacts": delete_artifacts,
+ },
+ )
+
+ def save(self):
+ self.update()
+
+ @property
+ def json_config(self):
+ config = {}
+ for k, v in self.config.items():
+ config[k] = {"value": v, "desc": None}
+ return json.dumps(config)
+
+ def _exec(self, query, **kwargs):
+ """Execute a query against the cloud backend."""
+ variables = {"entity": self.entity, "project": self.project, "name": self.id}
+ variables.update(kwargs)
+ return self.client.execute(query, variable_values=variables)
+
+ def _sampled_history(self, keys, x_axis="_step", samples=500):
+ spec = {"keys": [x_axis] + keys, "samples": samples}
+ query = gql(
+ """
+ query RunSampledHistory($project: String!, $entity: String!, $name: String!, $specs: [JSONString!]!) {
+ project(name: $project, entityName: $entity) {
+ run(name: $name) { sampledHistory(specs: $specs) }
+ }
+ }
+ """
+ )
+
+ response = self._exec(query, specs=[json.dumps(spec)])
+ # sampledHistory returns one list per spec, we only send one spec
+ return response["project"]["run"]["sampledHistory"][0]
+
+ def _full_history(self, samples=500, stream="default"):
+ node = "history" if stream == "default" else "events"
+ query = gql(
+ """
+ query RunFullHistory($project: String!, $entity: String!, $name: String!, $samples: Int) {
+ project(name: $project, entityName: $entity) {
+ run(name: $name) { %s(samples: $samples) }
+ }
+ }
+ """
+ % node
+ )
+
+ response = self._exec(query, samples=samples)
+ return [json.loads(line) for line in response["project"]["run"][node]]
+
+ @normalize_exceptions
+ def files(self, names=None, per_page=50):
+ """Return a file path for each file named.
+
+ Arguments:
+ names (list): names of the requested files, if empty returns all files
+ per_page (int): number of results per page.
+
+ Returns:
+ A `Files` object, which is an iterator over `File` objects.
+ """
+ return public.Files(self.client, self, names or [], per_page)
+
+ @normalize_exceptions
+ def file(self, name):
+ """Return the path of a file with a given name in the artifact.
+
+ Arguments:
+ name (str): name of requested file.
+
+ Returns:
+ A `File` matching the name argument.
+ """
+ return public.Files(self.client, self, [name])[0]
+
+ @normalize_exceptions
+ def upload_file(self, path, root="."):
+ """Upload a file.
+
+ Arguments:
+ path (str): name of file to upload.
+ root (str): the root path to save the file relative to. i.e.
+ If you want to have the file saved in the run as "my_dir/file.txt"
+ and you're currently in "my_dir" you would set root to "../".
+
+ Returns:
+ A `File` matching the name argument.
+ """
+ api = InternalApi(
+ default_settings={"entity": self.entity, "project": self.project},
+ retry_timedelta=RETRY_TIMEDELTA,
+ )
+ api.set_current_run_id(self.id)
+ root = os.path.abspath(root)
+ name = os.path.relpath(path, root)
+ with open(os.path.join(root, name), "rb") as f:
+ api.push({LogicalPath(name): f})
+ return public.Files(self.client, self, [name])[0]
+
+ @normalize_exceptions
+ def history(
+ self, samples=500, keys=None, x_axis="_step", pandas=True, stream="default"
+ ):
+ """Return sampled history metrics for a run.
+
+ This is simpler and faster if you are ok with the history records being sampled.
+
+ Arguments:
+ samples : (int, optional) The number of samples to return
+ pandas : (bool, optional) Return a pandas dataframe
+ keys : (list, optional) Only return metrics for specific keys
+ x_axis : (str, optional) Use this metric as the xAxis defaults to _step
+ stream : (str, optional) "default" for metrics, "system" for machine metrics
+
+ Returns:
+ pandas.DataFrame: If pandas=True returns a `pandas.DataFrame` of history
+ metrics.
+ list of dicts: If pandas=False returns a list of dicts of history metrics.
+ """
+ if keys is not None and not isinstance(keys, list):
+ wandb.termerror("keys must be specified in a list")
+ return []
+ if keys is not None and len(keys) > 0 and not isinstance(keys[0], str):
+ wandb.termerror("keys argument must be a list of strings")
+ return []
+
+ if keys and stream != "default":
+ wandb.termerror("stream must be default when specifying keys")
+ return []
+ elif keys:
+ lines = self._sampled_history(keys=keys, x_axis=x_axis, samples=samples)
+ else:
+ lines = self._full_history(samples=samples, stream=stream)
+ if pandas:
+ pandas = util.get_module("pandas")
+ if pandas:
+ lines = pandas.DataFrame.from_records(lines)
+ else:
+ print("Unable to load pandas, call history with pandas=False")
+ return lines
+
+ @normalize_exceptions
+ def scan_history(self, keys=None, page_size=1000, min_step=None, max_step=None):
+ """Returns an iterable collection of all history records for a run.
+
+ Example:
+ Export all the loss values for an example run
+
+ ```python
+ run = api.run("l2k2/examples-numpy-boston/i0wt6xua")
+ history = run.scan_history(keys=["Loss"])
+ losses = [row["Loss"] for row in history]
+ ```
+
+
+ Arguments:
+ keys ([str], optional): only fetch these keys, and only fetch rows that have all of keys defined.
+ page_size (int, optional): size of pages to fetch from the api
+
+ Returns:
+ An iterable collection over history records (dict).
+ """
+ if keys is not None and not isinstance(keys, list):
+ wandb.termerror("keys must be specified in a list")
+ return []
+ if keys is not None and len(keys) > 0 and not isinstance(keys[0], str):
+ wandb.termerror("keys argument must be a list of strings")
+ return []
+
+ last_step = self.lastHistoryStep
+ # set defaults for min/max step
+ if min_step is None:
+ min_step = 0
+ if max_step is None:
+ max_step = last_step + 1
+ # if the max step is past the actual last step, clamp it down
+ if max_step > last_step:
+ max_step = last_step + 1
+ if keys is None:
+ return public.HistoryScan(
+ run=self,
+ client=self.client,
+ page_size=page_size,
+ min_step=min_step,
+ max_step=max_step,
+ )
+ else:
+ return public.SampledHistoryScan(
+ run=self,
+ client=self.client,
+ keys=keys,
+ page_size=page_size,
+ min_step=min_step,
+ max_step=max_step,
+ )
+
+ @normalize_exceptions
+ def logged_artifacts(self, per_page=100):
+ return public.RunArtifacts(self.client, self, mode="logged", per_page=per_page)
+
+ @normalize_exceptions
+ def used_artifacts(self, per_page=100):
+ return public.RunArtifacts(self.client, self, mode="used", per_page=per_page)
+
+ @normalize_exceptions
+ def use_artifact(self, artifact, use_as=None):
+ """Declare an artifact as an input to a run.
+
+ Arguments:
+ artifact (`Artifact`): An artifact returned from
+ `wandb.Api().artifact(name)`
+ use_as (string, optional): A string identifying
+ how the artifact is used in the script. Used
+ to easily differentiate artifacts used in a
+ run, when using the beta wandb launch
+ feature's artifact swapping functionality.
+
+ Returns:
+ A `Artifact` object.
+ """
+ api = InternalApi(
+ default_settings={"entity": self.entity, "project": self.project},
+ retry_timedelta=RETRY_TIMEDELTA,
+ )
+ api.set_current_run_id(self.id)
+
+ if isinstance(artifact, wandb.Artifact) and not artifact.is_draft():
+ api.use_artifact(artifact.id, use_as=use_as or artifact.name)
+ return artifact
+ elif isinstance(artifact, wandb.Artifact) and artifact.is_draft():
+ raise ValueError(
+ "Only existing artifacts are accepted by this api. "
+ "Manually create one with `wandb artifact put`"
+ )
+ else:
+ raise ValueError("You must pass a wandb.Api().artifact() to use_artifact")
+
+ @normalize_exceptions
+ def log_artifact(self, artifact, aliases=None):
+ """Declare an artifact as output of a run.
+
+ Arguments:
+ artifact (`Artifact`): An artifact returned from
+ `wandb.Api().artifact(name)`
+ aliases (list, optional): Aliases to apply to this artifact
+ Returns:
+ A `Artifact` object.
+ """
+ api = InternalApi(
+ default_settings={"entity": self.entity, "project": self.project},
+ retry_timedelta=RETRY_TIMEDELTA,
+ )
+ api.set_current_run_id(self.id)
+
+ if isinstance(artifact, wandb.Artifact) and not artifact.is_draft():
+ if (
+ self.entity != artifact.source_entity
+ or self.project != artifact.source_project
+ ):
+ raise ValueError("A run can't log an artifact to a different project.")
+ artifact_collection_name = artifact.source_name.split(":")[0]
+ api.create_artifact(
+ artifact.type,
+ artifact_collection_name,
+ artifact.digest,
+ aliases=aliases,
+ )
+ return artifact
+ elif isinstance(artifact, wandb.Artifact) and artifact.is_draft():
+ raise ValueError(
+ "Only existing artifacts are accepted by this api. "
+ "Manually create one with `wandb artifact put`"
+ )
+ else:
+ raise ValueError("You must pass a wandb.Api().artifact() to use_artifact")
+
+ @property
+ def summary(self):
+ if self._summary is None:
+ from wandb.old.summary import HTTPSummary
+
+ # TODO: fix the outdir issue
+ self._summary = HTTPSummary(self, self.client, summary=self.summary_metrics)
+ return self._summary
+
+ @property
+ def path(self):
+ return [
+ urllib.parse.quote_plus(str(self.entity)),
+ urllib.parse.quote_plus(str(self.project)),
+ urllib.parse.quote_plus(str(self.id)),
+ ]
+
+ @property
+ def url(self):
+ path = self.path
+ path.insert(2, "runs")
+ return self.client.app_url + "/".join(path)
+
+ @property
+ def metadata(self):
+ if self._metadata is None:
+ try:
+ f = self.file("wandb-metadata.json")
+ contents = util.download_file_into_memory(f.url, wandb.Api().api_key)
+ self._metadata = json_util.loads(contents)
+ except: # noqa: E722
+ # file doesn't exist, or can't be downloaded, or can't be parsed
+ pass
+ return self._metadata
+
+ @property
+ def lastHistoryStep(self): # noqa: N802
+ query = gql(
+ """
+ query RunHistoryKeys($project: String!, $entity: String!, $name: String!) {
+ project(name: $project, entityName: $entity) {
+ run(name: $name) { historyKeys }
+ }
+ }
+ """
+ )
+ response = self._exec(query)
+ if (
+ response is None
+ or response.get("project") is None
+ or response["project"].get("run") is None
+ or response["project"]["run"].get("historyKeys") is None
+ ):
+ return -1
+ history_keys = response["project"]["run"]["historyKeys"]
+ return history_keys["lastStep"] if "lastStep" in history_keys else -1
+
+ def to_html(self, height=420, hidden=False):
+ """Generate HTML containing an iframe displaying this run."""
+ url = self.url + "?jupyter=true"
+ style = f"border:none;width:100%;height:{height}px;"
+ prefix = ""
+ if hidden:
+ style += "display:none;"
+ prefix = ipython.toggle_button()
+ return prefix + f"<iframe src={url!r} style={style!r}></iframe>"
+
+ def _repr_html_(self) -> str:
+ return self.to_html()
+
+ def __repr__(self):
+ return "<Run {} ({})>".format("/".join(self.path), self.state)
diff --git a/wandb/apis/public/sweeps.py b/wandb/apis/public/sweeps.py
new file mode 100644
index 0000000000000000000000000000000000000000..c3ac50ae40b189aee49879be55943d402c494da0
--- /dev/null
+++ b/wandb/apis/public/sweeps.py
@@ -0,0 +1,239 @@
+"""Public API: sweeps."""
+import urllib
+from typing import Optional
+
+from wandb_gql import gql
+
+import wandb
+from wandb import util
+from wandb.apis import public
+from wandb.apis.attrs import Attrs
+from wandb.sdk.lib import ipython
+
+SWEEP_FRAGMENT = """fragment SweepFragment on Sweep {
+ id
+ name
+ method
+ state
+ description
+ displayName
+ bestLoss
+ config
+ createdAt
+ updatedAt
+ runCount
+}
+"""
+
+
+class Sweep(Attrs):
+ """A set of runs associated with a sweep.
+
+ Examples:
+ Instantiate with:
+ ```
+ api = wandb.Api()
+ sweep = api.sweep(path/to/sweep)
+ ```
+
+ Attributes:
+ runs: (`Runs`) list of runs
+ id: (str) sweep id
+ project: (str) name of project
+ config: (str) dictionary of sweep configuration
+ state: (str) the state of the sweep
+ expected_run_count: (int) number of expected runs for the sweep
+ """
+
+ QUERY = gql(
+ """
+ query Sweep($project: String, $entity: String, $name: String!) {
+ project(name: $project, entityName: $entity) {
+ sweep(sweepName: $name) {
+ id
+ name
+ state
+ runCountExpected
+ bestLoss
+ config
+ }
+ }
+ }
+ """
+ )
+
+ LEGACY_QUERY = gql(
+ """
+ query Sweep($project: String, $entity: String, $name: String!) {
+ project(name: $project, entityName: $entity) {
+ sweep(sweepName: $name) {
+ id
+ name
+ state
+ bestLoss
+ config
+ }
+ }
+ }
+ """
+ )
+
+ def __init__(self, client, entity, project, sweep_id, attrs=None):
+ # TODO: Add agents / flesh this out.
+ super().__init__(dict(attrs or {}))
+ self.client = client
+ self._entity = entity
+ self.project = project
+ self.id = sweep_id
+ self.runs = []
+
+ self.load(force=not attrs)
+
+ @property
+ def entity(self):
+ return self._entity
+
+ @property
+ def username(self):
+ wandb.termwarn("Sweep.username is deprecated. please use Sweep.entity instead.")
+ return self._entity
+
+ @property
+ def config(self):
+ return util.load_yaml(self._attrs["config"])
+
+ def load(self, force: bool = False):
+ if force or not self._attrs:
+ sweep = self.get(self.client, self.entity, self.project, self.id)
+ if sweep is None:
+ raise ValueError("Could not find sweep %s" % self)
+ self._attrs = sweep._attrs
+ self.runs = sweep.runs
+
+ return self._attrs
+
+ @property
+ def order(self):
+ if self._attrs.get("config") and self.config.get("metric"):
+ sort_order = self.config["metric"].get("goal", "minimize")
+ prefix = "+" if sort_order == "minimize" else "-"
+ return public.QueryGenerator.format_order_key(
+ prefix + self.config["metric"]["name"]
+ )
+
+ def best_run(self, order=None):
+ """Return the best run sorted by the metric defined in config or the order passed in."""
+ if order is None:
+ order = self.order
+ else:
+ order = public.QueryGenerator.format_order_key(order)
+ if order is None:
+ wandb.termwarn(
+ "No order specified and couldn't find metric in sweep config, returning most recent run"
+ )
+ else:
+ wandb.termlog("Sorting runs by %s" % order)
+ filters = {"$and": [{"sweep": self.id}]}
+ try:
+ return public.Runs(
+ self.client,
+ self.entity,
+ self.project,
+ order=order,
+ filters=filters,
+ per_page=1,
+ )[0]
+ except IndexError:
+ return None
+
+ @property
+ def expected_run_count(self) -> Optional[int]:
+ """Return the number of expected runs in the sweep or None for infinite runs."""
+ return self._attrs.get("runCountExpected")
+
+ @property
+ def path(self):
+ return [
+ urllib.parse.quote_plus(str(self.entity)),
+ urllib.parse.quote_plus(str(self.project)),
+ urllib.parse.quote_plus(str(self.id)),
+ ]
+
+ @property
+ def url(self):
+ path = self.path
+ path.insert(2, "sweeps")
+ return self.client.app_url + "/".join(path)
+
+ @property
+ def name(self):
+ return self.config.get("name") or self.id
+
+ @classmethod
+ def get(
+ cls,
+ client,
+ entity=None,
+ project=None,
+ sid=None,
+ order=None,
+ query=None,
+ **kwargs,
+ ):
+ """Execute a query against the cloud backend."""
+ if query is None:
+ query = cls.QUERY
+
+ variables = {
+ "entity": entity,
+ "project": project,
+ "name": sid,
+ }
+ variables.update(kwargs)
+
+ response = None
+ try:
+ response = client.execute(query, variable_values=variables)
+ except Exception:
+ # Don't handle exception, rely on legacy query
+ # TODO(gst): Implement updated introspection workaround
+ query = cls.LEGACY_QUERY
+ response = client.execute(query, variable_values=variables)
+
+ if (
+ not response
+ or not response.get("project")
+ or not response["project"].get("sweep")
+ ):
+ return None
+
+ sweep_response = response["project"]["sweep"]
+ sweep = cls(client, entity, project, sid, attrs=sweep_response)
+ sweep.runs = public.Runs(
+ client,
+ entity,
+ project,
+ order=order,
+ per_page=10,
+ filters={"$and": [{"sweep": sweep.id}]},
+ )
+
+ return sweep
+
+ def to_html(self, height=420, hidden=False):
+ """Generate HTML containing an iframe displaying this sweep."""
+ url = self.url + "?jupyter=true"
+ style = f"border:none;width:100%;height:{height}px;"
+ prefix = ""
+ if hidden:
+ style += "display:none;"
+ prefix = ipython.toggle_button("sweep")
+ return prefix + f"<iframe src={url!r} style={style!r}></iframe>"
+
+ def _repr_html_(self) -> str:
+ return self.to_html()
+
+ def __repr__(self):
+ return "<Sweep {} ({})>".format(
+ "/".join(self.path), self._attrs.get("state", "Unknown State")
+ )
diff --git a/wandb/apis/public/teams.py b/wandb/apis/public/teams.py
new file mode 100644
index 0000000000000000000000000000000000000000..759e93d4342f8d9b82b497aab36a90564f51ef5f
--- /dev/null
+++ b/wandb/apis/public/teams.py
@@ -0,0 +1,197 @@
+"""Public API: teams."""
+import requests
+from wandb_gql import gql
+
+from wandb.apis.attrs import Attrs
+
+
+class Member(Attrs):
+ DELETE_MEMBER_MUTATION = gql(
+ """
+ mutation DeleteInvite($id: String, $entityName: String) {
+ deleteInvite(input: {id: $id, entityName: $entityName}) {
+ success
+ }
+ }
+ """
+ )
+
+ def __init__(self, client, team, attrs):
+ super().__init__(attrs)
+ self._client = client
+ self.team = team
+
+ def delete(self):
+ """Remove a member from a team.
+
+ Returns:
+ Boolean indicating success
+ """
+ try:
+ return self._client.execute(
+ self.DELETE_MEMBER_MUTATION, {"id": self.id, "entityName": self.team}
+ )["deleteInvite"]["success"]
+ except requests.exceptions.HTTPError:
+ return False
+
+ def __repr__(self):
+ return f"<Member {self.name} ({self.account_type})>"
+
+
+class Team(Attrs):
+ CREATE_TEAM_MUTATION = gql(
+ """
+ mutation CreateTeam($teamName: String!, $teamAdminUserName: String) {
+ createTeam(input: {teamName: $teamName, teamAdminUserName: $teamAdminUserName}) {
+ entity {
+ id
+ name
+ available
+ photoUrl
+ limits
+ }
+ }
+ }
+ """
+ )
+ CREATE_INVITE_MUTATION = gql(
+ """
+ mutation CreateInvite($entityName: String!, $email: String, $username: String, $admin: Boolean) {
+ createInvite(input: {entityName: $entityName, email: $email, username: $username, admin: $admin}) {
+ invite {
+ id
+ name
+ email
+ createdAt
+ toUser {
+ name
+ }
+ }
+ }
+ }
+ """
+ )
+ TEAM_QUERY = gql(
+ """
+ query Entity($name: String!) {
+ entity(name: $name) {
+ id
+ name
+ available
+ photoUrl
+ readOnly
+ readOnlyAdmin
+ isTeam
+ privateOnly
+ storageBytes
+ codeSavingEnabled
+ defaultAccess
+ isPaid
+ members {
+ id
+ admin
+ pending
+ email
+ username
+ name
+ photoUrl
+ accountType
+ apiKey
+ }
+ }
+ }
+ """
+ )
+ CREATE_SERVICE_ACCOUNT_MUTATION = gql(
+ """
+ mutation CreateServiceAccount($entityName: String!, $description: String!) {
+ createServiceAccount(
+ input: {description: $description, entityName: $entityName}
+ ) {
+ user {
+ id
+ }
+ }
+ }
+ """
+ )
+
+ def __init__(self, client, name, attrs=None):
+ super().__init__(attrs or {})
+ self._client = client
+ self.name = name
+ self.load()
+
+ @classmethod
+ def create(cls, api, team, admin_username=None):
+ """Create a new team.
+
+ Arguments:
+ api: (`Api`) The api instance to use
+ team: (str) The name of the team
+ admin_username: (str) optional username of the admin user of the team, defaults to the current user.
+
+ Returns:
+ A `Team` object
+ """
+ try:
+ api.client.execute(
+ cls.CREATE_TEAM_MUTATION,
+ {"teamName": team, "teamAdminUserName": admin_username},
+ )
+ except requests.exceptions.HTTPError:
+ pass
+ return Team(api.client, team)
+
+ def invite(self, username_or_email, admin=False):
+ """Invite a user to a team.
+
+ Arguments:
+ username_or_email: (str) The username or email address of the user you want to invite
+ admin: (bool) Whether to make this user a team admin, defaults to False
+
+ Returns:
+ True on success, False if user was already invited or didn't exist
+ """
+ variables = {"entityName": self.name, "admin": admin}
+ if "@" in username_or_email:
+ variables["email"] = username_or_email
+ else:
+ variables["username"] = username_or_email
+ try:
+ self._client.execute(self.CREATE_INVITE_MUTATION, variables)
+ except requests.exceptions.HTTPError:
+ return False
+ return True
+
+ def create_service_account(self, description):
+ """Create a service account for the team.
+
+ Arguments:
+ description: (str) A description for this service account
+
+ Returns:
+ The service account `Member` object, or None on failure
+ """
+ try:
+ self._client.execute(
+ self.CREATE_SERVICE_ACCOUNT_MUTATION,
+ {"description": description, "entityName": self.name},
+ )
+ self.load(True)
+ return self.members[-1]
+ except requests.exceptions.HTTPError:
+ return None
+
+ def load(self, force=False):
+ if force or not self._attrs:
+ response = self._client.execute(self.TEAM_QUERY, {"name": self.name})
+ self._attrs = response["entity"]
+ self._attrs["members"] = [
+ Member(self._client, self.name, member)
+ for member in self._attrs["members"]
+ ]
+ return self._attrs
+
+ def __repr__(self):
+ return f"<Team {self.name}>"
diff --git a/wandb/apis/public/users.py b/wandb/apis/public/users.py
new file mode 100644
index 0000000000000000000000000000000000000000..6afbe51df85354bd5a0c9ac12587a841a5f5b117
--- /dev/null
+++ b/wandb/apis/public/users.py
@@ -0,0 +1,135 @@
+"""Public API: users."""
+import requests
+from wandb_gql import gql
+
+import wandb
+from wandb.apis.attrs import Attrs
+
+
+class User(Attrs):
+ CREATE_USER_MUTATION = gql(
+ """
+ mutation CreateUserFromAdmin($email: String!, $admin: Boolean) {
+ createUser(input: {email: $email, admin: $admin}) {
+ user {
+ id
+ name
+ username
+ email
+ admin
+ }
+ }
+ }
+ """
+ )
+
+ DELETE_API_KEY_MUTATION = gql(
+ """
+ mutation DeleteApiKey($id: String!) {
+ deleteApiKey(input: {id: $id}) {
+ success
+ }
+ }
+ """
+ )
+ GENERATE_API_KEY_MUTATION = gql(
+ """
+ mutation GenerateApiKey($description: String) {
+ generateApiKey(input: {description: $description}) {
+ apiKey {
+ id
+ name
+ }
+ }
+ }
+ """
+ )
+
+ def __init__(self, client, attrs):
+ super().__init__(attrs)
+ self._client = client
+ self._user_api = None
+
+ @property
+ def user_api(self):
+ """An instance of the api using credentials from the user."""
+ if self._user_api is None and len(self.api_keys) > 0:
+ self._user_api = wandb.Api(api_key=self.api_keys[0])
+ return self._user_api
+
+ @classmethod
+ def create(cls, api, email, admin=False):
+ """Create a new user.
+
+ Arguments:
+ api: (`Api`) The api instance to use
+ email: (str) The name of the team
+ admin: (bool) Whether this user should be a global instance admin
+
+ Returns:
+ A `User` object
+ """
+ res = api.client.execute(
+ cls.CREATE_USER_MUTATION,
+ {"email": email, "admin": admin},
+ )
+ return User(api.client, res["createUser"]["user"])
+
+ @property
+ def api_keys(self):
+ if self._attrs.get("apiKeys") is None:
+ return []
+ return [k["node"]["name"] for k in self._attrs["apiKeys"]["edges"]]
+
+ @property
+ def teams(self):
+ if self._attrs.get("teams") is None:
+ return []
+ return [k["node"]["name"] for k in self._attrs["teams"]["edges"]]
+
+ def delete_api_key(self, api_key):
+ """Delete a user's api key.
+
+ Returns:
+ Boolean indicating success
+
+ Raises:
+ ValueError if the api_key couldn't be found
+ """
+ idx = self.api_keys.index(api_key)
+ try:
+ self._client.execute(
+ self.DELETE_API_KEY_MUTATION,
+ {"id": self._attrs["apiKeys"]["edges"][idx]["node"]["id"]},
+ )
+ except requests.exceptions.HTTPError:
+ return False
+ return True
+
+ def generate_api_key(self, description=None):
+ """Generate a new api key.
+
+ Returns:
+ The new api key, or None on failure
+ """
+ try:
+ # We must make this call using credentials from the original user
+ key = self.user_api.client.execute(
+ self.GENERATE_API_KEY_MUTATION, {"description": description}
+ )["generateApiKey"]["apiKey"]
+ self._attrs["apiKeys"]["edges"].append({"node": key})
+ return key["name"]
+ except (requests.exceptions.HTTPError, AttributeError):
+ return None
+
+ def __repr__(self):
+ if "email" in self._attrs:
+ return f"<User {self._attrs['email']}>"
+ elif "username" in self._attrs:
+ return f"<User {self._attrs['username']}>"
+ elif "id" in self._attrs:
+ return f"<User {self._attrs['id']}>"
+ elif "name" in self._attrs:
+ return f"<User {self._attrs['name']!r}>"
+ else:
+ return "<User ???>"
diff --git a/wandb/apis/reports/__init__.py b/wandb/apis/reports/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..48f67a65f026585f9aaeb0e4a05ee2643801e22a
--- /dev/null
+++ b/wandb/apis/reports/__init__.py
@@ -0,0 +1,30 @@
+# flake8: noqa
+from inspect import cleandoc
+
+from ... import termlog
+from . import blocks, helpers, panels, templates
+from .blocks import *
+from .helpers import LineKey, PCColumn
+from .panels import *
+from .report import Report
+from .runset import Runset
+from .templates import *
+from .util import InlineCode, InlineLaTeX, Link
+
+termlog(
+ cleandoc(
+ """
+ Thanks for trying out the Report API!
+ For a tutorial, check out https://colab.research.google.com/drive/1CzyJx1nuOS4pdkXa2XPaRQyZdmFmLmXV
+
+ Try out tab completion to see what's available.
+ ∟ everything: `wr.<tab>`
+ ∟ panels: `wr.panels.<tab>`
+ ∟ blocks: `wr.blocks.<tab>`
+ ∟ helpers: `wr.helpers.<tab>`
+ ∟ templates: `wr.templates.<tab>`
+
+ For bugs/feature requests, please create an issue on github: https://github.com/wandb/wandb/issues
+ """
+ )
+)
diff --git a/wandb/apis/reports/_blocks.py b/wandb/apis/reports/_blocks.py
new file mode 100644
index 0000000000000000000000000000000000000000..6543b9436a4a518fe085e383a08fecee42ad5777
--- /dev/null
+++ b/wandb/apis/reports/_blocks.py
@@ -0,0 +1,1410 @@
+import inspect
+import re
+import urllib
+from typing import List as LList
+from typing import Optional, Union
+
+from ... import __version__ as wandb_ver
+from ... import termwarn
+from ..public import Api as PublicApi
+from ._panels import UnknownPanel, WeavePanel, panel_mapping, weave_panels
+from .runset import Runset
+from .util import (
+ Attr,
+ Base,
+ Block,
+ InlineCode,
+ InlineLaTeX,
+ Link,
+ Panel,
+ coalesce,
+ fix_collisions,
+ nested_get,
+ nested_set,
+ weave_inputs,
+)
+from .validators import OneOf, TypeValidator
+
+
+class UnknownBlock(Block):
+ pass
+
+
+class PanelGrid(Block):
+ runsets: list = Attr(
+ json_path="spec.metadata.runSets",
+ validators=[TypeValidator(Runset, how="keys")],
+ )
+ panels: list = Attr(
+ json_path="spec.metadata.panelBankSectionConfig.panels",
+ validators=[TypeValidator(Panel, how="keys")],
+ )
+ custom_run_colors: dict = Attr(
+ json_path="spec.metadata.customRunColors",
+ validators=[
+ TypeValidator(Union[str, tuple], how="keys"),
+ TypeValidator(str, how="values"),
+ ],
+ )
+ active_runset: Union[str, None] = Attr(json_path="spec.metadata.openRunSet")
+
+ def __init__(
+ self,
+ runsets=None,
+ panels=None,
+ custom_run_colors=None,
+ active_runset=None,
+ *args,
+ **kwargs,
+ ):
+ super().__init__(*args, **kwargs)
+ self._spec = self._default_panel_grid_spec()
+ self.runsets = coalesce(runsets, self._default_runsets())
+ self.panels = coalesce(panels, self._default_panels())
+ self.custom_run_colors = coalesce(custom_run_colors, {})
+ self.active_runset = active_runset
+
+ @active_runset.getter
+ def active_runset(self):
+ json_path = self._get_path("active_runset")
+ index = nested_get(self, json_path)
+ if index is None:
+ return None
+ else:
+ return self.runsets[index].name
+
+ @active_runset.setter
+ def active_runset(self, name):
+ json_path = self._get_path("active_runset")
+ index = None
+ for i, rs in enumerate(self.runsets):
+ if rs.name == name:
+ index = i
+ break
+ nested_set(self, json_path, index)
+
+ @panels.getter
+ def panels(self):
+ json_path = self._get_path("panels")
+ specs = nested_get(self, json_path)
+ panels = []
+ for pspec in specs:
+ cls = panel_mapping.get(pspec["viewType"], UnknownPanel)
+ if cls is UnknownPanel:
+ termwarn(
+ inspect.cleandoc(
+ f"""
+ UNKNOWN PANEL DETECTED
+ This can happen if we have added new panels, but you are using an older version of the SDK.
+ If your report is loading normally, you can safely ignore this message (but we recommend not touching UnknownPanel)
+ If you think this is an error, please file a bug report including your SDK version ({wandb_ver}) and this spec ({pspec})
+ """
+ )
+ )
+ if cls is WeavePanel:
+ for cls in weave_panels:
+ try:
+ cls.from_json(pspec)
+ except Exception:
+ pass
+ else:
+ break
+ panels.append(cls.from_json(pspec))
+ return panels
+
+ @panels.setter
+ def panels(self, new_panels):
+ json_path = self._get_path("panels")
+ new_specs = [p.spec for p in fix_collisions(new_panels)]
+ nested_set(self, json_path, new_specs)
+
+ @runsets.getter
+ def runsets(self):
+ json_path = self._get_path("runsets")
+ specs = nested_get(self, json_path)
+ return [Runset.from_json(spec) for spec in specs]
+
+ @runsets.setter
+ def runsets(self, new_runsets):
+ json_path = self._get_path("runsets")
+ new_specs = [rs.spec for rs in new_runsets]
+ nested_set(self, json_path, new_specs)
+
+ @custom_run_colors.getter
+ def custom_run_colors(self):
+ json_path = self._get_path("custom_run_colors")
+ id_colors = nested_get(self, json_path)
+
+ def is_groupid(s):
+ for rs in self.runsets:
+ if rs.spec["id"] in s:
+ return True
+ return False
+
+ def groupid_to_ordertuple(groupid):
+ rs = self.runsets[0]
+ if "-run:" in groupid:
+ id, rest = groupid.split("-run:", 1)
+ else:
+ id, rest = groupid.split("-", 1)
+ kvs = rest.split("-")
+ kvs = [rs.pm_query_generator.pc_back_to_front(v) for v in kvs]
+ keys, ordertuple = zip(*[kv.split(":") for kv in kvs])
+ rs_name = self._get_rs_by_id(id).name
+ return (rs_name, *ordertuple)
+
+ def run_id_to_name(id):
+ for rs in self.runsets:
+ try:
+ run = PublicApi().run(f"{rs.entity}/{rs.project}/{id}")
+ except Exception:
+ pass
+ else:
+ return run.name
+ raise ValueError("Unable to find this run!")
+
+ color_settings = {}
+ for id, c in id_colors.items():
+ if id == "ref":
+ continue
+ if is_groupid(id):
+ key = groupid_to_ordertuple(id)
+ else:
+ key = run_id_to_name(id)
+ color_settings[key] = c
+ return color_settings
+
+ @custom_run_colors.setter
+ def custom_run_colors(self, new_custom_run_colors):
+ json_path = self._get_path("custom_run_colors")
+ color_settings = {}
+
+ def ordertuple_to_groupid(ordertuple):
+ rs_name, rest = ordertuple[0], ordertuple[1:]
+ rs = self._get_rs_by_name(rs_name)
+ id = rs.spec["id"]
+ keys = [rs.pm_query_generator.pc_front_to_back(k) for k in rs.groupby]
+ kvs = [f"{k}:{v}" for k, v in zip(keys, rest)]
+ linked = "-".join(kvs)
+ return f"{id}-{linked}"
+
+ def run_name_to_id(name):
+ for rs in self.runsets:
+ runs = PublicApi().runs(
+ path=f"{rs.entity}/{rs.project}", filters={"display_name": name}
+ )
+ if len(runs) > 1:
+ termwarn(
+ "Multiple runs with the same name found! Using the first one."
+ )
+ for run in runs:
+ if run.name == name:
+ return run.id
+ raise ValueError("Unable to find this run!")
+
+ for name, c in new_custom_run_colors.items():
+ if isinstance(name, tuple):
+ key = ordertuple_to_groupid(name)
+ else:
+ key = run_name_to_id(name)
+ color_settings[key] = c
+ nested_set(self, json_path, color_settings)
+
+ def _get_rs_by_id(self, id):
+ for rs in self.runsets:
+ if rs.spec["id"] == id:
+ return rs
+
+ def _get_rs_by_name(self, name):
+ for rs in self.runsets:
+ if rs.name == name:
+ return rs
+
+ @staticmethod
+ def _default_panel_grid_spec():
+ return {
+ "type": "panel-grid",
+ "children": [{"text": ""}],
+ "metadata": {
+ "openViz": True,
+ "panels": {
+ "views": {"0": {"name": "Panels", "defaults": [], "config": []}},
+ "tabs": ["0"],
+ },
+ "panelBankConfig": {
+ "state": 0,
+ "settings": {
+ "autoOrganizePrefix": 2,
+ "showEmptySections": False,
+ "sortAlphabetically": False,
+ },
+ "sections": [
+ {
+ "name": "Hidden Panels",
+ "isOpen": False,
+ "panels": [],
+ "type": "flow",
+ "flowConfig": {
+ "snapToColumns": True,
+ "columnsPerPage": 3,
+ "rowsPerPage": 2,
+ "gutterWidth": 16,
+ "boxWidth": 460,
+ "boxHeight": 300,
+ },
+ "sorted": 0,
+ "localPanelSettings": {
+ "xAxis": "_step",
+ "smoothingWeight": 0,
+ "smoothingType": "exponential",
+ "ignoreOutliers": False,
+ "xAxisActive": False,
+ "smoothingActive": False,
+ },
+ }
+ ],
+ },
+ "panelBankSectionConfig": {
+ "name": "Report Panels",
+ "isOpen": False,
+ "panels": [],
+ "type": "grid",
+ "flowConfig": {
+ "snapToColumns": True,
+ "columnsPerPage": 3,
+ "rowsPerPage": 2,
+ "gutterWidth": 16,
+ "boxWidth": 460,
+ "boxHeight": 300,
+ },
+ "sorted": 0,
+ "localPanelSettings": {
+ "xAxis": "_step",
+ "smoothingWeight": 0,
+ "smoothingType": "exponential",
+ "ignoreOutliers": False,
+ "xAxisActive": False,
+ "smoothingActive": False,
+ },
+ },
+ "customRunColors": {},
+ "runSets": [],
+ "openRunSet": 0,
+ "name": "unused-name",
+ },
+ }
+
+ @staticmethod
+ def _default_runsets():
+ return [Runset()]
+
+ @staticmethod
+ def _default_panels():
+ return []
+
+
+class List(Base):
+ @classmethod
+ def from_json(cls, spec: dict) -> "Union[CheckedList, OrderedList, UnorderedList]":
+ items = []
+ for item in spec["children"]:
+ text = []
+ for elem in item["children"][0]["children"]:
+ if elem.get("type") == "latex":
+ text.append(InlineLaTeX(elem["content"]))
+ elif elem.get("type") == "link":
+ text.append(Link(elem["children"][0]["text"], elem["url"]))
+ elif elem.get("inlineCode"):
+ text.append(InlineCode(elem["text"]))
+ elif elem.get("text"):
+ text.append(elem["text"])
+ items.append(text)
+ checked = [item.get("checked") for item in spec["children"]]
+ ordered = spec.get("ordered")
+
+ # NAND: Either checked or ordered or neither (unordered), never both
+ if all(x is None for x in checked):
+ checked = None
+ if checked is not None and ordered is not None:
+ raise ValueError(
+ "Lists can be checked, ordered or neither (unordered), but not both!"
+ )
+
+ if checked:
+ return CheckedList(items, checked)
+ elif ordered:
+ return OrderedList(items)
+ else:
+ return UnorderedList(items)
+
+
+class CheckedList(Block, List):
+ items: list = Attr()
+ checked: list = Attr()
+
+ def __init__(self, items=None, checked=None, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ if items is not None and checked is not None and len(items) != len(checked):
+ raise ValueError("Items and checked lists must be the same length!")
+
+ self.items = coalesce(items, [""])
+ self.checked = coalesce(checked, [False for _ in self.items])
+
+ @property
+ def spec(self) -> dict:
+ children = []
+ for item, check in zip(self.items, self.checked):
+ if isinstance(item, list):
+ content = [
+ t.spec if not isinstance(t, str) else {"text": t} for t in item
+ ]
+ else:
+ content = [{"text": item}]
+ children.append(
+ {
+ "type": "list-item",
+ "children": [{"type": "paragraph", "children": content}],
+ "checked": check,
+ }
+ )
+
+ return {"type": "list", "children": children}
+
+
+class OrderedList(Block, List):
+ items: list = Attr()
+
+ def __init__(self, items=None, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.items = coalesce(items, [""])
+
+ @property
+ def spec(self) -> dict:
+ children = []
+ for item in self.items:
+ if isinstance(item, list):
+ content = [
+ t.spec if not isinstance(t, str) else {"text": t} for t in item
+ ]
+ else:
+ content = [{"text": item}]
+ children.append(
+ {
+ "type": "list-item",
+ "children": [{"type": "paragraph", "children": content}],
+ "ordered": True,
+ }
+ )
+
+ return {"type": "list", "ordered": True, "children": children}
+
+
+class UnorderedList(Block, List):
+ items: list = Attr()
+
+ def __init__(self, items=None, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.items = coalesce(items, [""])
+
+ @property
+ def spec(self) -> dict:
+ children = []
+ for item in self.items:
+ if isinstance(item, list):
+ content = [
+ t.spec if not isinstance(t, str) else {"text": t} for t in item
+ ]
+ else:
+ content = [{"text": item}]
+ children.append(
+ {
+ "type": "list-item",
+ "children": [{"type": "paragraph", "children": content}],
+ }
+ )
+
+ return {"type": "list", "children": children}
+
+
+class Heading(Base):
+ @classmethod
+ def from_json(cls, spec: dict) -> "Union[H1,H2,H3]":
+ level = spec["level"]
+ level_mapping = {1: H1, 2: H2, 3: H3}
+ if level not in level_mapping:
+ raise ValueError(f"`level` must be one of {list(level_mapping.keys())}")
+
+ if isinstance(spec["children"], str):
+ text = spec["children"]
+ else:
+ text = []
+ for elem in spec["children"]:
+ if elem.get("type") == "latex":
+ text.append(InlineLaTeX(elem["content"]))
+ elif elem.get("type") == "link":
+ text.append(Link(elem["children"][0]["text"], elem["url"]))
+ elif elem.get("inlineCode"):
+ text.append(InlineCode(elem["text"]))
+ elif elem.get("text"):
+ text.append(elem["text"])
+ if not isinstance(text, list):
+ text = [text]
+ return level_mapping[level](text)
+
+
+class H1(Block, Heading):
+ text: Union[str, list, Link] = Attr()
+
+ def __init__(self, text="", *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.text = text
+
+ @property
+ def spec(self) -> dict:
+ if isinstance(self.text, list):
+ content = [
+ t.spec if not isinstance(t, str) else {"text": t} for t in self.text
+ ]
+ else:
+ content = [{"text": self.text}]
+ return {
+ "type": "heading",
+ "children": content,
+ "level": 1,
+ }
+
+
+class H2(Block, Heading):
+ text: Union[str, list, Link] = Attr()
+
+ def __init__(self, text="", *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.text = text
+
+ @property
+ def spec(self) -> dict:
+ if isinstance(self.text, list):
+ content = [
+ t.spec if not isinstance(t, str) else {"text": t} for t in self.text
+ ]
+ else:
+ content = [{"text": self.text}]
+ return {
+ "type": "heading",
+ "children": content,
+ "level": 2,
+ }
+
+
+class H3(Block, Heading):
+ text: Union[str, list, Link] = Attr()
+
+ def __init__(self, text="", *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.text = text
+
+ @property
+ def spec(self) -> dict:
+ if isinstance(self.text, list):
+ content = [
+ t.spec if not isinstance(t, str) else {"text": t} for t in self.text
+ ]
+ else:
+ content = [{"text": self.text}]
+ return {
+ "type": "heading",
+ "children": content,
+ "level": 3,
+ }
+
+
+class BlockQuote(Block):
+ text: str = Attr()
+
+ def __init__(self, text="", *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.text = text
+
+ @classmethod
+ def from_json(cls, spec: dict) -> "BlockQuote":
+ text = spec["children"][0]["text"]
+ return cls(text)
+
+ @property
+ def spec(self) -> dict:
+ return {"type": "block-quote", "children": [{"text": self.text}]}
+
+
+class CalloutBlock(Block):
+ text: Union[str, list] = Attr()
+
+ def __init__(self, text=" ", *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.text = text
+
+ def __post_init__(self) -> None:
+ if isinstance(self.text, str):
+ self.text = self.text.split("\n")
+
+ @classmethod
+ def from_json(cls, spec: dict) -> "CalloutBlock":
+ text = [child["children"][0]["text"] for child in spec["children"]]
+ return cls(text)
+
+ @property
+ def spec(self) -> dict:
+ return {
+ "type": "callout-block",
+ "children": [
+ {"type": "callout-line", "children": [{"text": text}]}
+ for text in self.text
+ ],
+ }
+
+
+class CodeBlock(Block):
+ code: Union[str, list] = Attr()
+ language: str = Attr()
+
+ def __init__(self, code=" ", language=None, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.code = code
+ self.language = coalesce(language, "python")
+
+ def __post_init__(self) -> None:
+ if isinstance(self.code, str):
+ self.code = self.code.split("\n")
+
+ @classmethod
+ def from_json(cls, spec: dict) -> "CodeBlock":
+ code = [child["children"][0]["text"] for child in spec["children"]]
+ language = spec.get("language", "python")
+ return cls(code, language)
+
+ @property
+ def spec(self) -> dict:
+ language = self.language.lower()
+ return {
+ "type": "code-block",
+ "children": [
+ {
+ "type": "code-line",
+ "children": [{"text": text}],
+ "language": language,
+ }
+ for text in self.code
+ ],
+ "language": language,
+ }
+
+
+class MarkdownBlock(Block):
+ text: Union[str, list] = Attr()
+
+ def __init__(self, text="", *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.text = text
+ if isinstance(self.text, list):
+ self.text = "\n".join(self.text)
+
+ @classmethod
+ def from_json(cls, spec: dict) -> "MarkdownBlock":
+ text = spec["content"]
+ return cls(text)
+
+ @property
+ def spec(self) -> dict:
+ return {
+ "type": "markdown-block",
+ "children": [{"text": ""}],
+ "content": self.text,
+ }
+
+
+class LaTeXBlock(Block):
+ text: Union[str, list] = Attr()
+
+ def __init__(self, text="", *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.text = text
+ if isinstance(self.text, list):
+ self.text = "\n".join(self.text)
+
+ @classmethod
+ def from_json(cls, spec: dict) -> "LaTeXBlock":
+ text = spec["content"]
+ return cls(text)
+
+ @property
+ def spec(self) -> dict:
+ return {
+ "type": "latex",
+ "children": [{"text": ""}],
+ "content": self.text,
+ "block": True,
+ }
+
+
+class Gallery(Block):
+ ids: list = Attr(validators=[TypeValidator(str, how="keys")])
+
+ def __init__(self, ids, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.ids = ids
+
+ @classmethod
+ def from_json(cls, spec: dict) -> "Gallery":
+ ids = spec["ids"]
+ return cls(ids)
+
+ @classmethod
+ def from_report_urls(cls, urls: LList[str]) -> "Gallery":
+ from .report import Report
+
+ ids = [Report._url_to_report_id(url) for url in urls]
+ return cls(ids)
+
+ @property
+ def spec(self) -> dict:
+ return {"type": "gallery", "children": [{"text": ""}], "ids": self.ids}
+
+
+class Image(Block):
+ url: str = Attr()
+ caption: Optional[str] = Attr()
+
+ def __init__(self, url, caption=None, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.url = url
+ self.caption = caption
+
+ @classmethod
+ def from_json(cls, spec: dict) -> "Image":
+ url = spec["url"]
+ caption = spec["children"][0]["text"] if spec.get("hasCaption") else None
+ return cls(url, caption)
+
+ @property
+ def spec(self) -> dict:
+ if self.caption:
+ return {
+ "type": "image",
+ "children": [{"text": self.caption}],
+ "url": self.url,
+ "hasCaption": True,
+ }
+ else:
+ return {"type": "image", "children": [{"text": ""}], "url": self.url}
+
+
+class WeaveBlockSummaryTable(Block):
+ """This is a hacky solution to support the most common way of getting Weave tables for now..."""
+
+ entity: str = Attr()
+ project: str = Attr()
+ table_name: str = Attr()
+
+ def __init__(self, entity, project, table_name, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.entity = entity
+ self.project = project
+ self.table_name = table_name
+
+ @classmethod
+ def from_json(cls, spec: dict) -> "WeaveBlockSummaryTable":
+ entity = spec["config"]["panelConfig"]["exp"]["fromOp"]["inputs"]["obj"][
+ "fromOp"
+ ]["inputs"]["run"]["fromOp"]["inputs"]["project"]["fromOp"]["inputs"][
+ "entityName"
+ ][
+ "val"
+ ]
+ project = spec["config"]["panelConfig"]["exp"]["fromOp"]["inputs"]["obj"][
+ "fromOp"
+ ]["inputs"]["run"]["fromOp"]["inputs"]["project"]["fromOp"]["inputs"][
+ "projectName"
+ ][
+ "val"
+ ]
+ table_name = spec["config"]["panelConfig"]["exp"]["fromOp"]["inputs"]["key"][
+ "val"
+ ]
+ return cls(entity, project, table_name)
+
+ @property
+ def spec(self) -> dict:
+ return {
+ "type": "weave-panel",
+ "children": [{"text": ""}],
+ "config": {
+ "panelConfig": {
+ "exp": {
+ "nodeType": "output",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": {
+ "type": "typedDict",
+ "propertyTypes": {"project": "project"},
+ },
+ },
+ "value": {
+ "type": "list",
+ "objectType": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {"run": "run"},
+ },
+ "value": {
+ "type": "union",
+ "members": [
+ {
+ "type": "file",
+ "extension": "json",
+ "wbObjectType": {
+ "type": "table",
+ "columnTypes": {},
+ },
+ },
+ "none",
+ ],
+ },
+ },
+ },
+ },
+ "fromOp": {
+ "name": "pick",
+ "inputs": {
+ "obj": {
+ "nodeType": "output",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": {
+ "type": "typedDict",
+ "propertyTypes": {"project": "project"},
+ },
+ },
+ "value": {
+ "type": "list",
+ "objectType": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {"run": "run"},
+ },
+ "value": {
+ "type": "union",
+ "members": [
+ {
+ "type": "typedDict",
+ "propertyTypes": {
+ "_wandb": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "runtime": "number"
+ },
+ }
+ },
+ },
+ {
+ "type": "typedDict",
+ "propertyTypes": {
+ "_step": "number",
+ "table": {
+ "type": "file",
+ "extension": "json",
+ "wbObjectType": {
+ "type": "table",
+ "columnTypes": {},
+ },
+ },
+ "_wandb": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "runtime": "number"
+ },
+ },
+ "_runtime": "number",
+ "_timestamp": "number",
+ },
+ },
+ {
+ "type": "typedDict",
+ "propertyTypes": {},
+ },
+ ],
+ },
+ },
+ },
+ },
+ "fromOp": {
+ "name": "run-summary",
+ "inputs": {
+ "run": {
+ "nodeType": "output",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "project": "project"
+ },
+ },
+ },
+ "value": {
+ "type": "list",
+ "objectType": "run",
+ },
+ },
+ "fromOp": {
+ "name": "project-runs",
+ "inputs": {
+ "project": {
+ "nodeType": "output",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": "project",
+ },
+ "fromOp": {
+ "name": "root-project",
+ "inputs": {
+ "entityName": {
+ "nodeType": "const",
+ "type": "string",
+ "val": self.entity,
+ },
+ "projectName": {
+ "nodeType": "const",
+ "type": "string",
+ "val": self.project,
+ },
+ },
+ },
+ }
+ },
+ },
+ }
+ },
+ },
+ },
+ "key": {
+ "nodeType": "const",
+ "type": "string",
+ "val": self.table_name,
+ },
+ },
+ },
+ "__userInput": True,
+ }
+ }
+ },
+ }
+
+
+class WeaveBlockArtifact(Block):
+ """This is a hacky solution to support the most common way of getting Weave artifacts for now..."""
+
+ entity: str = Attr()
+ project: str = Attr()
+ artifact: str = Attr()
+ tab: str = Attr(
+ validators=[OneOf(["overview", "metadata", "usage", "files", "lineage"])]
+ )
+
+ def __init__(self, entity, project, artifact, tab="overview", *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.entity = entity
+ self.project = project
+ self.artifact = artifact
+ self.tab = tab
+
+ @classmethod
+ def from_json(cls, spec: dict) -> "WeaveBlockSummaryTable":
+ inputs = weave_inputs(spec)
+ entity = inputs["project"]["fromOp"]["inputs"]["entityName"]["val"]
+ project = inputs["project"]["fromOp"]["inputs"]["projectName"]["val"]
+ artifact = inputs["artifactName"]["val"]
+ tab = spec["config"]["panelConfig"]["panelConfig"]["tabConfigs"]["overview"][
+ "selectedTab"
+ ]
+ return cls(entity, project, artifact, tab)
+
+ @property
+ def spec(self) -> dict:
+ return {
+ "type": "weave-panel",
+ "children": [{"text": ""}],
+ "config": {
+ "panelConfig": {
+ "exp": {
+ "nodeType": "output",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "project": "project",
+ "artifactName": "string",
+ },
+ },
+ },
+ "value": "artifact",
+ },
+ "fromOp": {
+ "name": "project-artifact",
+ "inputs": {
+ "project": {
+ "nodeType": "output",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": "project",
+ },
+ "fromOp": {
+ "name": "root-project",
+ "inputs": {
+ "entityName": {
+ "nodeType": "const",
+ "type": "string",
+ "val": self.entity,
+ },
+ "projectName": {
+ "nodeType": "const",
+ "type": "string",
+ "val": self.project,
+ },
+ },
+ },
+ },
+ "artifactName": {
+ "nodeType": "const",
+ "type": "string",
+ "val": self.artifact,
+ },
+ },
+ },
+ "__userInput": True,
+ },
+ "panelInputType": {
+ "type": "tagged",
+ "tag": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "project": "project",
+ "artifactName": "string",
+ },
+ },
+ },
+ "value": "artifact",
+ },
+ "panelConfig": {
+ "tabConfigs": {"overview": {"selectedTab": self.tab}}
+ },
+ }
+ },
+ }
+
+
+class WeaveBlockArtifactVersionedFile(Block):
+ """This is a hacky solution to support the most common way of getting Weave artifact verions for now..."""
+
+ entity: str = Attr()
+ project: str = Attr()
+ artifact: str = Attr()
+ version: str = Attr()
+ file: str = Attr()
+
+ def __init__(self, entity, project, artifact, version, file, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.entity = entity
+ self.project = project
+ self.artifact = artifact
+ self.version = version
+ self.file = file
+
+ @classmethod
+ def from_json(cls, spec: dict) -> "WeaveBlockSummaryTable":
+ inputs = weave_inputs(spec)
+ entity = inputs["artifactVersion"]["fromOp"]["inputs"]["project"]["fromOp"][
+ "inputs"
+ ]["entityName"]["val"]
+ project = inputs["artifactVersion"]["fromOp"]["inputs"]["project"]["fromOp"][
+ "inputs"
+ ]["projectName"]["val"]
+ artifact = inputs["artifactVersion"]["fromOp"]["inputs"]["artifactName"]["val"]
+ version = inputs["artifactVersion"]["fromOp"]["inputs"]["artifactVersionAlias"][
+ "val"
+ ]
+ file = inputs["path"]["val"]
+ return cls(entity, project, artifact, version, file)
+
+ @property
+ def spec(self) -> dict:
+ return {
+ "type": "weave-panel",
+ "children": [{"text": ""}],
+ "config": {
+ "panelConfig": {
+ "exp": {
+ "nodeType": "output",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "project": "project",
+ "artifactName": "string",
+ "artifactVersionAlias": "string",
+ },
+ },
+ },
+ "value": {
+ "type": "file",
+ "extension": "json",
+ "wbObjectType": {"type": "table", "columnTypes": {}},
+ },
+ },
+ "fromOp": {
+ "name": "artifactVersion-file",
+ "inputs": {
+ "artifactVersion": {
+ "nodeType": "output",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "project": "project",
+ "artifactName": "string",
+ "artifactVersionAlias": "string",
+ },
+ },
+ },
+ "value": "artifactVersion",
+ },
+ "fromOp": {
+ "name": "project-artifactVersion",
+ "inputs": {
+ "project": {
+ "nodeType": "output",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": "project",
+ },
+ "fromOp": {
+ "name": "root-project",
+ "inputs": {
+ "entityName": {
+ "nodeType": "const",
+ "type": "string",
+ "val": self.entity,
+ },
+ "projectName": {
+ "nodeType": "const",
+ "type": "string",
+ "val": self.project,
+ },
+ },
+ },
+ },
+ "artifactName": {
+ "nodeType": "const",
+ "type": "string",
+ "val": self.artifact,
+ },
+ "artifactVersionAlias": {
+ "nodeType": "const",
+ "type": "string",
+ "val": self.version,
+ },
+ },
+ },
+ },
+ "path": {
+ "nodeType": "const",
+ "type": "string",
+ "val": self.file,
+ },
+ },
+ },
+ "__userInput": True,
+ }
+ }
+ },
+ }
+
+
+class HorizontalRule(Block):
+ @classmethod
+ def from_json(cls, spec: dict) -> "HorizontalRule":
+ return cls()
+
+ @property
+ def spec(self):
+ return {"type": "horizontal-rule", "children": [{"text": ""}]}
+
+
+class TableOfContents(Block):
+ @classmethod
+ def from_json(cls, spec: dict) -> "TableOfContents":
+ return cls()
+
+ @property
+ def spec(self) -> dict:
+ return {"type": "table-of-contents", "children": [{"text": ""}]}
+
+
+class SoundCloud(Block):
+ url: str = Attr()
+
+ def __init__(self, url, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.url = url
+
+ @classmethod
+ def from_json(cls, spec: dict) -> "SoundCloud":
+ quoted_url = spec["html"].split("url=")[-1].split("&show_artwork")[0]
+ url = urllib.parse.unquote(quoted_url)
+ return cls(url)
+
+ @property
+ def spec(self) -> dict:
+ quoted_url = urllib.parse.quote(self.url)
+ return {
+ "type": "soundcloud",
+ "html": f'<iframe width="100%" height="400" scrolling="no" frameborder="no" src="https://w.soundcloud.com/player/?visual=true&url={quoted_url}&show_artwork=true"></iframe>',
+ "children": [{"text": ""}],
+ }
+
+
+class Twitter(Block):
+ embed_html: str = Attr()
+
+ def __init__(self, embed_html, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.embed_html = embed_html
+ if self.embed_html:
+ pattern = r" <script[\s\S]+?/script>"
+ self.embed_html = re.sub(pattern, "\n", self.embed_html)
+
+ @classmethod
+ def from_json(cls, spec: dict) -> "Twitter":
+ embed_html = spec["html"]
+ return cls(embed_html)
+
+ @property
+ def spec(self) -> dict:
+ return {"type": "twitter", "html": self.embed_html, "children": [{"text": ""}]}
+
+
+class Spotify(Block):
+ spotify_id: str = Attr()
+
+ def __init__(self, spotify_id, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.spotify_id = spotify_id
+
+ @classmethod
+ def from_json(cls, spec: dict) -> "Spotify":
+ return cls(spec["spotifyID"])
+
+ @classmethod
+ def from_url(cls, url: str) -> "Spotify":
+ spotify_id = url.split("/")[-1].split("?")[0]
+ return cls(spotify_id)
+
+ @property
+ def spec(self) -> dict:
+ return {
+ "type": "spotify",
+ "spotifyType": "track",
+ "spotifyID": self.spotify_id,
+ "children": [{"text": ""}],
+ }
+
+
+class Video(Block):
+ url: str = Attr()
+
+ def __init__(self, url, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.url = url
+
+ @classmethod
+ def from_json(cls, spec: dict) -> "Video":
+ return cls(spec["url"])
+
+ @property
+ def spec(self) -> dict:
+ return {
+ "type": "video",
+ "url": self.url,
+ "children": [{"text": ""}],
+ }
+
+
+class P(Block):
+ text: Union[str, InlineLaTeX, InlineCode, Link, list] = Attr()
+
+ def __init__(self, text="", *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.text = text
+
+ @classmethod
+ def from_json(cls, spec):
+ if isinstance(spec["children"], str):
+ text = spec["children"]
+ else:
+ text = []
+ for elem in spec["children"]:
+ if elem.get("type") == "latex":
+ text.append(InlineLaTeX(elem["content"]))
+ elif elem.get("type") == "link":
+ text.append(Link(elem["children"][0]["text"], elem["url"]))
+ elif elem.get("inlineCode"):
+ text.append(InlineCode(elem["text"]))
+ elif elem.get("text"):
+ text.append(elem["text"])
+
+ if not isinstance(text, list):
+ text = [text]
+ return cls(text)
+
+ @property
+ def spec(self) -> dict:
+ if isinstance(self.text, list):
+ content = [
+ t.spec if not isinstance(t, str) else {"text": t} for t in self.text
+ ]
+ else:
+ content = [{"text": self.text}]
+
+ return {"type": "paragraph", "children": content}
+
+
+class WeaveBlock(Block):
+ def __init__(self, spec, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._spec = spec
+
+ @classmethod
+ def from_json(cls, spec):
+ obj = cls(spec=spec)
+ obj._spec = spec
+ return obj
+
+ @property
+ def spec(self):
+ return self._spec
+
+
+block_mapping = {
+ "block-quote": BlockQuote,
+ "callout-block": CalloutBlock,
+ "code-block": CodeBlock,
+ "gallery": Gallery,
+ "heading": Heading,
+ "horizontal-rule": HorizontalRule,
+ "image": Image,
+ "latex": LaTeXBlock,
+ "list": List,
+ "markdown-block": MarkdownBlock,
+ "panel-grid": PanelGrid,
+ "paragraph": P,
+ "table-of-contents": TableOfContents,
+ "weave-panel": WeaveBlock,
+ "video": Video,
+ "spotify": Spotify,
+ "twitter": Twitter,
+ "soundcloud": SoundCloud,
+}
+
+weave_blocks = [
+ WeaveBlockSummaryTable,
+ WeaveBlockArtifactVersionedFile,
+ WeaveBlockArtifact,
+ WeaveBlock,
+]
diff --git a/wandb/apis/reports/_helpers.py b/wandb/apis/reports/_helpers.py
new file mode 100644
index 0000000000000000000000000000000000000000..b158e23a460a2108da4f0cd6ea6f5e6cee466a62
--- /dev/null
+++ b/wandb/apis/reports/_helpers.py
@@ -0,0 +1,70 @@
+from typing import Optional
+
+from ..public import PanelMetricsHelper, Run
+from .runset import Runset
+from .util import Attr, Base, Panel, nested_get, nested_set
+
+
+class LineKey:
+ def __init__(self, key: str) -> None:
+ self.key = key
+
+ def __hash__(self) -> int:
+ return hash(self.key)
+
+ def __repr__(self) -> str:
+ return f"LineKey(key={self.key!r})"
+
+ @classmethod
+ def from_run(cls, run: "Run", metric: str) -> "LineKey":
+ key = f"{run.id}:{metric}"
+ return cls(key)
+
+ @classmethod
+ def from_panel_agg(cls, runset: "Runset", panel: "Panel", metric: str) -> "LineKey":
+ key = f"{runset.id}-config:group:{panel.groupby}:null:{metric}"
+ return cls(key)
+
+ @classmethod
+ def from_runset_agg(cls, runset: "Runset", metric: str) -> "LineKey":
+ groupby = runset.groupby
+ if runset.groupby is None:
+ groupby = "null"
+
+ key = f"{runset.id}-run:group:{groupby}:{metric}"
+ return cls(key)
+
+
+class PCColumn(Base):
+ metric: str = Attr(json_path="spec.accessor")
+ name: Optional[str] = Attr(json_path="spec.displayName")
+ ascending: Optional[bool] = Attr(json_path="spec.inverted")
+ log_scale: Optional[bool] = Attr(json_path="spec.log")
+
+ def __init__(
+ self, metric, name=None, ascending=None, log_scale=None, *args, **kwargs
+ ):
+ super().__init__(*args, **kwargs)
+ self.panel_metrics_helper = PanelMetricsHelper()
+ self.metric = metric
+ self.name = name
+ self.ascending = ascending
+ self.log_scale = log_scale
+
+ @classmethod
+ def from_json(cls, spec):
+ obj = cls(metric=spec["accessor"])
+ obj._spec = spec
+ return obj
+
+ @metric.getter
+ def metric(self):
+ json_path = self._get_path("metric")
+ value = nested_get(self, json_path)
+ return self.panel_metrics_helper.special_back_to_front(value)
+
+ @metric.setter
+ def metric(self, value):
+ json_path = self._get_path("metric")
+ value = self.panel_metrics_helper.special_front_to_back(value)
+ nested_set(self, json_path, value)
diff --git a/wandb/apis/reports/_panels.py b/wandb/apis/reports/_panels.py
new file mode 100644
index 0000000000000000000000000000000000000000..77637509043cfbfb004732d53d183a9ec2ce05ed
--- /dev/null
+++ b/wandb/apis/reports/_panels.py
@@ -0,0 +1,1282 @@
+from typing import Optional, Union
+
+from .helpers import LineKey, PCColumn
+from .util import Attr, Panel, coalesce, nested_get, nested_set
+from .validators import (
+ AGGFUNCS,
+ CODE_COMPARE_DIFF,
+ FONT_SIZES,
+ LEGEND_POSITIONS,
+ LINEPLOT_STYLES,
+ RANGEFUNCS,
+ SMOOTHING_TYPES,
+ Length,
+ OneOf,
+ TypeValidator,
+)
+
+
+class UnknownPanel(Panel):
+ @property
+ def view_type(self) -> str:
+ return "UNKNOWN PANEL"
+
+
+class LinePlot(Panel):
+ title: Optional[str] = Attr(
+ json_path="spec.config.chartTitle",
+ )
+ x: Optional[str] = Attr(json_path="spec.config.xAxis")
+ y: Optional[Union[list, str]] = Attr(json_path="spec.config.metrics")
+ range_x: Union[list, tuple] = Attr(
+ json_path=["spec.config.xAxisMin", "spec.config.xAxisMax"],
+ validators=[
+ Length(2),
+ TypeValidator(Optional[Union[int, float]], how="keys"),
+ ],
+ )
+ range_y: Union[list, tuple] = Attr(
+ json_path=["spec.config.yAxisMin", "spec.config.yAxisMax"],
+ validators=[
+ Length(2),
+ TypeValidator(Optional[Union[int, float]], how="keys"),
+ ],
+ )
+ log_x: Optional[bool] = Attr(json_path="spec.config.xLogScale")
+ log_y: Optional[bool] = Attr(json_path="spec.config.yLogScale")
+ title_x: Optional[str] = Attr(json_path="spec.config.xAxisTitle")
+ title_y: Optional[str] = Attr(json_path="spec.config.yAxisTitle")
+ ignore_outliers: Optional[bool] = Attr(json_path="spec.config.ignoreOutliers")
+ groupby: Optional[str] = Attr(json_path="spec.config.groupBy")
+ groupby_aggfunc: Optional[str] = Attr(
+ json_path="spec.config.groupAgg",
+ validators=[OneOf(AGGFUNCS)],
+ )
+ groupby_rangefunc: Optional[str] = Attr(
+ json_path="spec.config.groupArea",
+ validators=[OneOf(RANGEFUNCS)],
+ )
+ smoothing_factor: Optional[float] = Attr(json_path="spec.config.smoothingWeight")
+ smoothing_type: Optional[str] = Attr(
+ json_path="spec.config.smoothingType",
+ validators=[OneOf(SMOOTHING_TYPES)],
+ )
+ smoothing_show_original: Optional[bool] = Attr(
+ json_path="spec.config.showOriginalAfterSmoothing"
+ )
+ max_runs_to_show: Optional[int] = Attr(json_path="spec.config.limit")
+ custom_expressions: Optional[str] = Attr(json_path="spec.config.expressions")
+ plot_type: Optional[str] = Attr(
+ json_path="spec.config.plotType",
+ validators=[OneOf(LINEPLOT_STYLES)],
+ )
+ font_size: Optional[str] = Attr(
+ json_path="spec.config.fontSize",
+ validators=[OneOf(FONT_SIZES)],
+ )
+ legend_position: Optional[str] = Attr(
+ json_path="spec.config.legendPosition",
+ validators=[OneOf(LEGEND_POSITIONS)],
+ )
+ legend_template: Optional[str] = Attr(json_path="spec.config.legendTemplate")
+ # Attr( json_path="spec.config.startingXAxis")
+ # Attr( json_path="spec.config.useLocalSmoothing")
+ # Attr( json_path="spec.config.useGlobalSmoothingWeight")
+ # Attr( json_path="spec.config.legendFields")
+ aggregate: Optional[bool] = Attr(json_path="spec.config.aggregate")
+ # Attr( json_path="spec.config.aggregateMetrics")
+ # Attr( json_path="spec.config.metricRegex")
+ # Attr( json_path="spec.config.useMetricRegex")
+ # Attr( json_path="spec.config.yAxisAutoRange")
+ group_runs_limit: Optional[int] = Attr(json_path="spec.config.groupRunsLimit")
+ xaxis_expression: Optional[str] = Attr(json_path="spec.config.xExpression")
+ # Attr( json_path="spec.config.colorEachMetricDifferently")
+ # Attr( json_path="spec.config.showLegend")
+
+ # line_titles: Optional[dict] = Attr(
+ # json_path="spec.config.overrideSeriesTitles",
+ # validators=[
+ # TypeValidator(LineKey, how="keys"),
+ # TypeValidator(str, how="values"),
+ # ],
+ # )
+ # line_marks: Optional[dict] = Attr(
+ # json_path="spec.config.overrideMarks",
+ # validators=[
+ # TypeValidator(LineKey, how="keys"),
+ # OneOf(MARKS, how="values"),
+ # ],
+ # )
+ # line_colors: Optional[dict] = Attr(
+ # json_path="spec.config.overrideColors",
+ # validators=[
+ # TypeValidator(LineKey, how="keys"),
+ # ],
+ # )
+ # line_widths: Optional[dict] = Attr(
+ # json_path="spec.config.overrideLineWidths",
+ # validators=[
+ # TypeValidator(LineKey, how="keys"),
+ # TypeValidator(Union[int, float], how="values"),
+ # Between(0.5, 3.0, how="values"),
+ # ],
+ # )
+
+ def __init__(
+ self,
+ title: Optional[str] = None,
+ x: Optional[str] = None,
+ y: Optional[Union[list, str]] = None,
+ range_x: Union[list, tuple] = (None, None),
+ range_y: Union[list, tuple] = (None, None),
+ log_x: Optional[bool] = None,
+ log_y: Optional[bool] = None,
+ title_x: Optional[str] = None,
+ title_y: Optional[str] = None,
+ ignore_outliers: Optional[bool] = None,
+ groupby: Optional[str] = None,
+ groupby_aggfunc: Optional[str] = None,
+ groupby_rangefunc: Optional[str] = None,
+ smoothing_factor: Optional[float] = None,
+ smoothing_type: Optional[str] = None,
+ smoothing_show_original: Optional[bool] = None,
+ max_runs_to_show: Optional[int] = None,
+ custom_expressions: Optional[str] = None,
+ plot_type: Optional[str] = None,
+ font_size: Optional[str] = None,
+ legend_position: Optional[str] = None,
+ legend_template: Optional[str] = None,
+ aggregate: Optional[bool] = None,
+ group_runs_limit: Optional[int] = None,
+ xaxis_expression: Optional[str] = None,
+ # line_titles: Optional[dict] = None,
+ # line_marks: Optional[dict] = None,
+ # line_colors: Optional[dict] = None,
+ # line_widths: Optional[dict] = None,
+ *args,
+ **kwargs,
+ ):
+ super().__init__(*args, **kwargs)
+ self.title = title
+ self.x = x
+ self.y = y
+ self.range_x = range_x
+ self.range_y = range_y
+ self.log_x = log_x
+ self.log_y = log_y
+ self.title_x = title_x
+ self.title_y = title_y
+ self.ignore_outliers = ignore_outliers
+ self.groupby = groupby
+ self.groupby_aggfunc = groupby_aggfunc
+ self.groupby_rangefunc = groupby_rangefunc
+ self.smoothing_factor = smoothing_factor
+ self.smoothing_type = smoothing_type
+ self.smoothing_show_original = smoothing_show_original
+ self.max_runs_to_show = max_runs_to_show
+ self.custom_expressions = custom_expressions
+ self.plot_type = plot_type
+ self.font_size = font_size
+ self.legend_position = legend_position
+ self.legend_template = legend_template
+ self.aggregate = aggregate
+ self.group_runs_limit = group_runs_limit
+ self.xaxis_expression = xaxis_expression
+ # self.line_titles = line_titles
+ # self.line_marks = line_marks
+ # self.line_colors = line_colors
+ # self.line_widths = line_widths
+
+ @x.getter
+ def x(self):
+ json_path = self._get_path("x")
+ value = nested_get(self, json_path)
+ return self.panel_metrics_helper.back_to_front(value)
+
+ @x.setter
+ def x(self, value):
+ if value is None:
+ value = "Step"
+
+ json_path = self._get_path("x")
+ value = self.panel_metrics_helper.front_to_back(value)
+ nested_set(self, json_path, value)
+
+ @y.getter
+ def y(self):
+ json_path = self._get_path("y")
+ value = nested_get(self, json_path)
+ if value is None:
+ return value
+ return [self.panel_metrics_helper.back_to_front(v) for v in value]
+
+ @y.setter
+ def y(self, value):
+ json_path = self._get_path("y")
+ if value is not None:
+ if not isinstance(value, list):
+ value = [value]
+ value = [self.panel_metrics_helper.front_to_back(v) for v in value]
+ nested_set(self, json_path, value)
+
+ @property
+ def view_type(self):
+ return "Run History Line Plot"
+
+
+class ScatterPlot(Panel):
+ title: Optional[str] = Attr(json_path="spec.config.chartTitle")
+ x: Optional[str] = Attr(json_path="spec.config.xAxis")
+ y: Optional[str] = Attr(json_path="spec.config.yAxis")
+ z: Optional[str] = Attr(json_path="spec.config.zAxis")
+ range_x: Union[list, tuple] = Attr(
+ json_path=["spec.config.xAxisMin", "spec.config.xAxisMax"],
+ validators=[
+ Length(2),
+ TypeValidator(Optional[Union[int, float]], how="keys"),
+ ],
+ )
+ range_y: Union[list, tuple] = Attr(
+ json_path=["spec.config.yAxisMin", "spec.config.yAxisMax"],
+ validators=[
+ Length(2),
+ TypeValidator(Optional[Union[int, float]], how="keys"),
+ ],
+ )
+ range_z: Union[list, tuple] = Attr(
+ json_path=["spec.config.zAxisMin", "spec.config.zAxisMax"],
+ validators=[
+ Length(2),
+ TypeValidator(Optional[Union[int, float]], how="keys"),
+ ],
+ )
+ log_x: Optional[bool] = Attr(json_path="spec.config.xAxisLogScale")
+ log_y: Optional[bool] = Attr(json_path="spec.config.yAxisLogScale")
+ log_z: Optional[bool] = Attr(json_path="spec.config.zAxisLogScale")
+ running_ymin: Optional[bool] = Attr(json_path="spec.config.showMinYAxisLine")
+ running_ymax: Optional[bool] = Attr(json_path="spec.config.showMaxYAxisLine")
+ running_ymean: Optional[bool] = Attr(json_path="spec.config.showAvgYAxisLine")
+ legend_template: Optional[str] = Attr(json_path="spec.config.legendTemplate")
+ gradient: Optional[dict] = Attr(
+ json_path="spec.config.customGradient",
+ )
+ # color: ... = Attr(json_path="spec.config.color")
+ # range_color: ... = Attr(
+ # ["spec.config.minColor", "spec.config.maxColor"],
+ # (list, tuple),
+ # validators=[Length(2), TypeValidator((int, float), how='keys')],
+ # )
+
+ # Attr(json_path="spec.config.legendFields")
+ font_size: Optional[str] = Attr(
+ json_path="spec.config.fontSize",
+ validators=[OneOf(FONT_SIZES)],
+ )
+ # Attr(json_path="spec.config.yAxisLineSmoothingWeight")
+ regression: Optional[bool] = Attr(json_path="spec.config.showLinearRegression")
+
+ def __init__(
+ self,
+ title=None,
+ x=None,
+ y=None,
+ z=None,
+ range_x=(None, None),
+ range_y=(None, None),
+ range_z=(None, None),
+ log_x=None,
+ log_y=None,
+ log_z=None,
+ running_ymin=None,
+ running_ymax=None,
+ running_ymean=None,
+ legend_template=None,
+ gradient=None,
+ font_size=None,
+ regression=None,
+ *args,
+ **kwargs,
+ ):
+ super().__init__(*args, **kwargs)
+ self.title = title
+ self.x = x
+ self.y = y
+ self.z = z
+ self.range_x = range_x
+ self.range_y = range_y
+ self.range_z = range_z
+ self.log_x = log_x
+ self.log_y = log_y
+ self.log_z = log_z
+ self.running_ymin = running_ymin
+ self.running_ymax = running_ymax
+ self.running_ymean = running_ymean
+ self.legend_template = legend_template
+ self.gradient = gradient
+ self.font_size = font_size
+ self.regression = regression
+
+ @x.getter
+ def x(self):
+ json_path = self._get_path("x")
+ value = nested_get(self, json_path)
+ return self.panel_metrics_helper.special_back_to_front(value)
+
+ @x.setter
+ def x(self, value):
+ json_path = self._get_path("x")
+ value = self.panel_metrics_helper.special_front_to_back(value)
+ nested_set(self, json_path, value)
+
+ @y.getter
+ def y(self):
+ json_path = self._get_path("y")
+ value = nested_get(self, json_path)
+ return self.panel_metrics_helper.special_back_to_front(value)
+
+ @y.setter
+ def y(self, value):
+ json_path = self._get_path("y")
+ value = self.panel_metrics_helper.special_front_to_back(value)
+ nested_set(self, json_path, value)
+
+ @z.getter
+ def z(self):
+ json_path = self._get_path("z")
+ value = nested_get(self, json_path)
+ return self.panel_metrics_helper.special_back_to_front(value)
+
+ @z.setter
+ def z(self, value):
+ json_path = self._get_path("z")
+ value = self.panel_metrics_helper.special_front_to_back(value)
+ nested_set(self, json_path, value)
+
+ @property
+ def view_type(self) -> str:
+ return "Scatter Plot"
+
+
+class BarPlot(Panel):
+ title: Optional[str] = Attr(json_path="spec.config.chartTitle")
+ metrics: Optional[Union[list, str]] = Attr(
+ json_path="spec.config.metrics",
+ validators=[TypeValidator(str, how="keys")],
+ )
+ orientation: str = Attr(
+ json_path="spec.config.vertical", validators=[OneOf(["v", "h"])]
+ )
+ range_x: Union[list, tuple] = Attr(
+ json_path=["spec.config.xAxisMin", "spec.config.xAxisMax"],
+ validators=[
+ Length(2),
+ TypeValidator(Optional[Union[int, float]], how="keys"),
+ ],
+ )
+ title_x: Optional[str] = Attr(json_path="spec.config.xAxisTitle")
+ title_y: Optional[str] = Attr(json_path="spec.config.yAxisTitle")
+ groupby: Optional[str] = Attr(json_path="spec.config.groupBy")
+ groupby_aggfunc: Optional[str] = Attr(
+ json_path="spec.config.groupAgg",
+ validators=[OneOf(AGGFUNCS)],
+ )
+ groupby_rangefunc: Optional[str] = Attr(
+ json_path="spec.config.groupArea",
+ validators=[OneOf(RANGEFUNCS)],
+ )
+ max_runs_to_show: Optional[int] = Attr(json_path="spec.config.limit")
+ max_bars_to_show: Optional[int] = Attr(json_path="spec.config.barLimit")
+ custom_expressions: Optional[str] = Attr(json_path="spec.config.expressions")
+ legend_template: Optional[str] = Attr(json_path="spec.config.legendTemplate")
+ font_size: Optional[str] = Attr(
+ json_path="spec.config.fontSize",
+ validators=[OneOf(FONT_SIZES)],
+ )
+ # Attr(json_path="spec.config.limit")
+ # Attr(json_path="spec.config.barLimit")
+ # Attr(json_path="spec.config.aggregate")
+ # Attr(json_path="spec.config.aggregateMetrics")
+ # Attr(json_path="spec.config.groupRunsLimit")
+ # Attr(json_path="spec.config.plotStyle")
+ # Attr(json_path="spec.config.legendFields")
+ # Attr(json_path="spec.config.colorEachMetricDifferently")
+
+ line_titles: Optional[dict] = Attr(
+ json_path="spec.config.overrideSeriesTitles",
+ validators=[
+ TypeValidator(LineKey, how="keys"),
+ TypeValidator(str, how="values"),
+ ],
+ )
+ line_colors: Optional[dict] = Attr(
+ json_path="spec.config.overrideColors",
+ validators=[
+ TypeValidator(LineKey, how="keys"),
+ ],
+ )
+
+ def __init__(
+ self,
+ title=None,
+ metrics=None,
+ orientation="h",
+ range_x=(None, None),
+ title_x=None,
+ title_y=None,
+ groupby=None,
+ groupby_aggfunc=None,
+ groupby_rangefunc=None,
+ max_runs_to_show=None,
+ max_bars_to_show=None,
+ custom_expressions=None,
+ legend_template=None,
+ font_size=None,
+ line_titles=None,
+ line_colors=None,
+ *args,
+ **kwargs,
+ ):
+ super().__init__(*args, **kwargs)
+ self.title = title
+ self.metrics = metrics
+ self.orientation = orientation
+ self.range_x = range_x
+ self.title_x = title_x
+ self.title_y = title_y
+ self.groupby = groupby
+ self.groupby_aggfunc = groupby_aggfunc
+ self.groupby_rangefunc = groupby_rangefunc
+ self.max_runs_to_show = max_runs_to_show
+ self.max_bars_to_show = max_bars_to_show
+ self.custom_expressions = custom_expressions
+ self.legend_template = legend_template
+ self.font_size = font_size
+ self.line_titles = line_titles
+ self.line_colors = line_colors
+
+ @metrics.getter
+ def metrics(self):
+ json_path = self._get_path("metrics")
+ value = nested_get(self, json_path)
+ if value is None:
+ return value
+ return [self.panel_metrics_helper.back_to_front(v) for v in value]
+
+ @metrics.setter
+ def metrics(self, value):
+ json_path = self._get_path("metrics")
+ if value is not None:
+ if not isinstance(value, list):
+ value = [value]
+ value = [self.panel_metrics_helper.front_to_back(v) for v in value]
+ nested_set(self, json_path, value)
+
+ @orientation.getter
+ def orientation(self):
+ json_path = self._get_path("orientation")
+ value = nested_get(self, json_path)
+ return "v" if value is True else "h"
+
+ @orientation.setter
+ def orientation(self, value):
+ json_path = self._get_path("orientation")
+ value = True if value == "v" else False
+ nested_set(self, json_path, value)
+
+ @property
+ def view_type(self) -> str:
+ return "Bar Chart"
+
+
+class ScalarChart(Panel):
+ title: Optional[str] = Attr(json_path="spec.config.chartTitle")
+ metric: str = Attr(json_path="spec.config.metrics")
+ groupby_aggfunc: Optional[str] = Attr(
+ json_path="spec.config.groupAgg",
+ validators=[OneOf(AGGFUNCS)],
+ )
+ groupby_rangefunc: Optional[str] = Attr(
+ json_path="spec.config.groupArea",
+ validators=[OneOf(RANGEFUNCS)],
+ )
+ custom_expressions: Optional[str] = Attr(json_path="spec.config.expressions")
+ legend_template: Optional[str] = Attr(json_path="spec.config.legendTemplate")
+
+ # Attr(json_path="spec.config.aggregate")
+ # Attr(json_path="spec.config.aggregateMetrics")
+ # Attr(json_path="spec.config.groupBy")
+ # Attr(json_path="spec.config.groupRunsLimit")
+ # Attr(json_path="spec.config.legendFields")
+ # Attr(json_path="spec.config.showLegend")
+ font_size: Optional[str] = Attr(
+ json_path="spec.config.fontSize",
+ validators=[OneOf(FONT_SIZES)],
+ )
+
+ def __init__(
+ self,
+ title=None,
+ metric=None,
+ groupby_aggfunc=None,
+ groupby_rangefunc=None,
+ custom_expressions=None,
+ legend_template=None,
+ font_size=None,
+ *args,
+ **kwargs,
+ ):
+ super().__init__(*args, **kwargs)
+ self.title = title
+ self.metric = coalesce(metric, "")
+ self.groupby_aggfunc = groupby_aggfunc
+ self.groupby_rangefunc = groupby_rangefunc
+ self.custom_expressions = custom_expressions
+ self.legend_template = legend_template
+ self.font_size = font_size
+
+ @metric.getter
+ def metric(self):
+ json_path = self._get_path("metric")
+ value = nested_get(self, json_path)[0]
+ return self.panel_metrics_helper.back_to_front(value)
+
+ @metric.setter
+ def metric(self, new_metrics):
+ json_path = self._get_path("metric")
+ new_metrics = self.panel_metrics_helper.front_to_back(new_metrics)
+ nested_set(self, json_path, [new_metrics])
+
+ @property
+ def view_type(self) -> str:
+ return "Scalar Chart"
+
+
+class CodeComparer(Panel):
+ diff: Optional[str] = Attr(
+ json_path="spec.config.diff",
+ validators=[OneOf(CODE_COMPARE_DIFF)],
+ )
+
+ def __init__(self, diff=None, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.diff = diff
+
+ @property
+ def view_type(self) -> str:
+ return "Code Comparer"
+
+
+class ParallelCoordinatesPlot(Panel):
+ columns: list = Attr(
+ json_path="spec.config.columns",
+ validators=[TypeValidator(Union[PCColumn, str], how="keys")],
+ )
+ title: Optional[str] = Attr(json_path="spec.config.chartTitle")
+ gradient: Optional[list] = Attr(json_path="spec.config.customGradient")
+
+ # Attr(json_path="spec.config.dimensions")
+ # Attr(json_path="spec.config.gradientColor")
+ # Attr(json_path="spec.config.legendFields")
+ font_size: Optional[str] = Attr(
+ json_path="spec.config.fontSize",
+ validators=[OneOf(FONT_SIZES)],
+ )
+
+ def __init__(self, columns=None, title=None, font_size=None, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.columns = coalesce(columns, [])
+ self.title = title
+ self.font_size = font_size
+
+ @columns.getter
+ def columns(self):
+ json_path = self._get_path("columns")
+ specs = nested_get(self, json_path)
+ return [PCColumn.from_json(cspec) for cspec in specs]
+
+ @columns.setter
+ def columns(self, new_columns):
+ json_path = self._get_path("columns")
+ cols = []
+ for c in new_columns:
+ if isinstance(c, PCColumn):
+ cols.append(c)
+ else:
+ cols.append(PCColumn(c))
+ specs = [c.spec for c in cols]
+ nested_set(self, json_path, specs)
+
+ @property
+ def view_type(self) -> str:
+ return "Parallel Coordinates Plot"
+
+
+class ParameterImportancePlot(Panel):
+ with_respect_to: str = Attr(json_path="spec.config.targetKey")
+
+ def __init__(self, with_respect_to=None, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.with_respect_to = coalesce(with_respect_to, "Created Timestamp")
+
+ @with_respect_to.getter
+ def with_respect_to(self):
+ json_path = self._get_path("with_respect_to")
+ value = nested_get(self, json_path)
+ return self.panel_metrics_helper.back_to_front(value)
+
+ @with_respect_to.setter
+ def with_respect_to(self, value):
+ json_path = self._get_path("with_respect_to")
+ value = self.panel_metrics_helper.front_to_back(value)
+ nested_set(self, json_path, value)
+
+ @property
+ def view_type(self) -> str:
+ return "Parameter Importance"
+
+
+class RunComparer(Panel):
+ def __init__(self, diff_only=None, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.diff_only = diff_only
+
+ diff_only: Optional[str] = Attr(
+ json_path="spec.config.diffOnly",
+ validators=[OneOf(["split", None])],
+ )
+
+ @property
+ def view_type(self) -> str:
+ return "Run Comparer"
+
+
+class MediaBrowser(Panel):
+ num_columns: Optional[int] = Attr(json_path="spec.config.columnCount")
+ media_keys: Optional[str] = Attr(json_path="spec.config.mediaKeys")
+ # Attr(json_path="spec.config.chartTitle")
+ # Attr(json_path="spec.config.stepIndex")
+ # Attr(json_path="spec.config.mediaIndex")
+ # Attr(json_path="spec.config.actualSize")
+ # Attr(json_path="spec.config.fitToDimension")
+ # Attr(json_path="spec.config.pixelated")
+ # Attr(json_path="spec.config.mode")
+ # Attr(json_path="spec.config.gallerySettings")
+ # Attr(json_path="spec.config.gridSettings")
+ # Attr(json_path="spec.config.selection")
+ # Attr(json_path="spec.config.page")
+ # Attr(json_path="spec.config.tileLayout")
+ # Attr(json_path="spec.config.stepStrideLength")
+ # Attr(json_path="spec.config.snapToExistingStep")
+ # Attr(json_path="spec.config.maxGalleryItems")
+ # Attr(json_path="spec.config.maxYAxisCount")
+ # Attr(json_path="spec.config.moleculeConfig")
+ # Attr(json_path="spec.config.segmentationMaskConfig")
+ # Attr(json_path="spec.config.boundingBoxConfig")
+
+ def __init__(self, num_columns=None, media_keys=None, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.num_columns = num_columns
+ self.media_keys = media_keys
+
+ @property
+ def view_type(self) -> str:
+ return "Media Browser"
+
+
+class MarkdownPanel(Panel):
+ markdown: Optional[str] = Attr(json_path="spec.config.value")
+
+ def __init__(self, markdown=None, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.markdown = markdown
+
+ @property
+ def view_type(self) -> str:
+ return "Markdown Panel"
+
+
+class ConfusionMatrix(Panel):
+ @property
+ def view_type(self) -> str:
+ return "Confusion Matrix"
+
+
+class DataFrames(Panel):
+ @property
+ def view_type(self) -> str:
+ return "Data Frame Table"
+
+
+class MultiRunTable(Panel):
+ @property
+ def view_type(self) -> str:
+ return "Multi Run Table"
+
+
+class Vega(Panel):
+ @property
+ def view_type(self) -> str:
+ return "Vega"
+
+
+class CustomChart(Panel):
+ query: dict = Attr(json_path="spec.config.userQuery.queryFields")
+ chart_name: str = Attr(json_path="spec.config.panelDefId")
+ chart_fields: dict = Attr(json_path="spec.config.fieldSettings")
+ chart_strings: dict = Attr(json_path="spec.config.stringSettings")
+
+ def __init__(
+ self,
+ query=None,
+ chart_name="",
+ chart_fields=None,
+ chart_strings=None,
+ *args,
+ **kwargs,
+ ):
+ super().__init__(*args, **kwargs)
+ self.spec["config"] = {"transform": {"name": "tableWithLeafColNames"}}
+ self.query = coalesce(query, {})
+ self.chart_name = chart_name
+ self.chart_fields = coalesce(chart_fields, {})
+ self.chart_strings = coalesce(chart_strings, {})
+
+ @classmethod
+ def from_table(cls, table_name, chart_fields=None, chart_strings=None):
+ return CustomChart(
+ query={"summaryTable": {"tableKey": table_name}},
+ chart_fields=chart_fields,
+ chart_strings=chart_strings,
+ )
+
+ @property
+ def view_type(self) -> str:
+ return "Vega2"
+
+ @query.getter
+ def query(self):
+ def fields_to_dict(fields):
+ d = {}
+ for field in fields:
+ keys = set(field.keys())
+ name = field["name"]
+
+ if keys == {"name", "fields"}:
+ d[name] = {}
+ elif keys == {"name", "value"}:
+ d[name] = field["value"]
+ elif keys == {"name", "args", "fields"}:
+ d[name] = fields_to_dict(field["args"])
+ return d
+
+ fields = nested_get(self, self._get_path("query"))
+ return fields_to_dict(fields)
+
+ @query.setter
+ def query(self, d):
+ def dict_to_fields(d):
+ fields = []
+ for k, v in d.items():
+ if isinstance(v, dict) and len(v) > 0:
+ field = {"name": k, "args": dict_to_fields(v), "fields": []}
+ elif isinstance(v, dict) and len(v) == 0 or v is None:
+ field = {"name": k, "fields": []}
+ else:
+ field = {"name": k, "value": v}
+ fields.append(field)
+ return fields
+
+ d.setdefault("id", [])
+ d.setdefault("name", [])
+
+ _query = [
+ {
+ "args": [
+ {"name": "runSets", "value": r"${runSets}"},
+ {"name": "limit", "value": 500},
+ ],
+ "fields": dict_to_fields(d),
+ "name": "runSets",
+ }
+ ]
+ nested_set(self, self._get_path("query"), _query)
+
+
+class Vega3(Panel):
+ @property
+ def view_type(self) -> str:
+ return "Vega3"
+
+
+class WeavePanel(Panel):
+ @property
+ def view_type(self) -> str:
+ return "Weave"
+
+
+class WeavePanelSummaryTable(Panel):
+ table_name: Optional[str] = Attr(
+ json_path="spec.config.panel2Config.exp.fromOp.inputs.key.val"
+ )
+
+ def __init__(self, table_name, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._spec["config"] = self._default_config()
+ self.table_name = table_name
+
+ @classmethod
+ def from_json(cls, spec):
+ table_name = spec["config"]["panel2Config"]["exp"]["fromOp"]["inputs"]["key"][
+ "val"
+ ]
+ return cls(table_name)
+
+ @property
+ def view_type(self) -> str:
+ return "Weave"
+
+ @staticmethod
+ def _default_config():
+ return {
+ "panel2Config": {
+ "exp": {
+ "nodeType": "output",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "project": "project",
+ "filter": "string",
+ "order": "string",
+ },
+ },
+ },
+ "value": {
+ "type": "list",
+ "objectType": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {"run": "run"},
+ },
+ "value": {
+ "type": "union",
+ "members": [
+ {
+ "type": "file",
+ "extension": "json",
+ "wbObjectType": {
+ "type": "table",
+ "columnTypes": {},
+ },
+ },
+ "none",
+ ],
+ },
+ },
+ "maxLength": 50,
+ },
+ },
+ "fromOp": {
+ "name": "pick",
+ "inputs": {
+ "obj": {
+ "nodeType": "output",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "project": "project",
+ "filter": "string",
+ "order": "string",
+ },
+ },
+ },
+ "value": {
+ "type": "list",
+ "objectType": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {"run": "run"},
+ },
+ "value": {
+ "type": "union",
+ "members": [
+ {
+ "type": "typedDict",
+ "propertyTypes": {
+ "_wandb": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "runtime": "number"
+ },
+ }
+ },
+ },
+ {
+ "type": "typedDict",
+ "propertyTypes": {
+ "_step": "number",
+ "table": {
+ "type": "file",
+ "extension": "json",
+ "wbObjectType": {
+ "type": "table",
+ "columnTypes": {},
+ },
+ },
+ "_wandb": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "runtime": "number"
+ },
+ },
+ "_runtime": "number",
+ "_timestamp": "number",
+ },
+ },
+ ],
+ },
+ },
+ "maxLength": 50,
+ },
+ },
+ "fromOp": {
+ "name": "run-summary",
+ "inputs": {
+ "run": {
+ "nodeType": "var",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "project": "project",
+ "filter": "string",
+ "order": "string",
+ },
+ },
+ },
+ "value": {
+ "type": "list",
+ "objectType": "run",
+ "maxLength": 50,
+ },
+ },
+ "varName": "runs",
+ }
+ },
+ },
+ },
+ "key": {
+ "nodeType": "const",
+ "type": "string",
+ "val": "",
+ },
+ },
+ },
+ "__userInput": True,
+ }
+ }
+ }
+
+
+class WeavePanelArtifact(Panel):
+ artifact: Optional[str] = Attr(
+ json_path="spec.config.panel2Config.exp.fromOp.inputs.artifactName.val"
+ )
+ tab: str = Attr(
+ json_path="spec.config.panel2Config.panelConfig.tabConfigs.overview.selectedTab",
+ validators=[OneOf(["overview", "metadata", "usage", "files", "lineage"])],
+ )
+
+ def __init__(self, artifact, tab="overview", *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._spec["config"] = self._default_config()
+ self.artifact = artifact
+ self.tab = tab
+
+ @classmethod
+ def from_json(cls, spec):
+ artifact = spec["config"]["panel2Config"]["exp"]["fromOp"]["inputs"][
+ "artifactName"
+ ]["val"]
+ tab = spec["config"]["panel2Config"]["panelConfig"]["tabConfigs"]["overview"][
+ "selectedTab"
+ ]
+ return cls(artifact, tab)
+
+ @property
+ def view_type(self) -> str:
+ return "Weave"
+
+ @staticmethod
+ def _default_config():
+ return {
+ "panel2Config": {
+ "exp": {
+ "nodeType": "output",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "project": "project",
+ "artifactName": "string",
+ },
+ },
+ },
+ "value": "artifact",
+ },
+ "fromOp": {
+ "name": "project-artifact",
+ "inputs": {
+ "project": {
+ "nodeType": "var",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": "project",
+ },
+ "varName": "project",
+ },
+ "artifactName": {
+ "nodeType": "const",
+ "type": "string",
+ "val": "",
+ },
+ },
+ },
+ "__userInput": True,
+ },
+ "panelInputType": {
+ "type": "tagged",
+ "tag": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "project": "project",
+ "artifactName": "string",
+ },
+ },
+ },
+ "value": "artifact",
+ },
+ "panelConfig": {
+ "tabConfigs": {"overview": {"selectedTab": "overview"}}
+ },
+ }
+ }
+
+
+class WeavePanelArtifactVersionedFile(Panel):
+ artifact: str = Attr(
+ json_path="spec.config.panel2Config.exp.fromOp.inputs.artifactVersion.fromOp.inputs.artifactName.val"
+ )
+ version: str = Attr(
+ json_path="spec.config.panel2Config.exp.fromOp.inputs.artifactVersion.fromOp.inputs.artifactVersionAlias.val",
+ )
+ file: str = Attr(json_path="spec.config.panel2Config.exp.fromOp.inputs.path.val")
+
+ def __init__(self, artifact, version, file, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._spec["config"] = self._default_config()
+ self.artifact = artifact
+ self.version = version
+ self.file = file
+
+ @classmethod
+ def from_json(cls, spec):
+ artifact = spec["config"]["panel2Config"]["exp"]["fromOp"]["inputs"][
+ "artifactVersion"
+ ]["fromOp"]["inputs"]["artifactName"]["val"]
+ version = spec["config"]["panel2Config"]["exp"]["fromOp"]["inputs"][
+ "artifactVersion"
+ ]["fromOp"]["inputs"]["artifactVersionAlias"]["val"]
+ file = spec["config"]["panel2Config"]["exp"]["fromOp"]["inputs"]["path"]["val"]
+ return cls(artifact, version, file)
+
+ @property
+ def view_type(self) -> str:
+ return "Weave"
+
+ @staticmethod
+ def _default_config():
+ return {
+ "panel2Config": {
+ "exp": {
+ "nodeType": "output",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "project": "project",
+ "artifactName": "string",
+ "artifactVersionAlias": "string",
+ },
+ },
+ },
+ "value": {
+ "type": "file",
+ "extension": "json",
+ "wbObjectType": {"type": "table", "columnTypes": {}},
+ },
+ },
+ "fromOp": {
+ "name": "artifactVersion-file",
+ "inputs": {
+ "artifactVersion": {
+ "nodeType": "output",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "project": "project",
+ "artifactName": "string",
+ "artifactVersionAlias": "string",
+ },
+ },
+ },
+ "value": "artifactVersion",
+ },
+ "fromOp": {
+ "name": "project-artifactVersion",
+ "inputs": {
+ "project": {
+ "nodeType": "var",
+ "type": {
+ "type": "tagged",
+ "tag": {
+ "type": "typedDict",
+ "propertyTypes": {
+ "entityName": "string",
+ "projectName": "string",
+ },
+ },
+ "value": "project",
+ },
+ "varName": "project",
+ },
+ "artifactName": {
+ "nodeType": "const",
+ "type": "string",
+ "val": "",
+ },
+ "artifactVersionAlias": {
+ "nodeType": "const",
+ "type": "string",
+ "val": "",
+ },
+ },
+ },
+ },
+ "path": {"nodeType": "const", "type": "string", "val": ""},
+ },
+ },
+ "__userInput": True,
+ }
+ }
+ }
+
+
+panel_mapping = {
+ # Panels with config
+ "Run History Line Plot": LinePlot,
+ "Scatter Plot": ScatterPlot,
+ "Bar Chart": BarPlot,
+ "Scalar Chart": ScalarChart,
+ "Code Comparer": CodeComparer,
+ "Parallel Coordinates Plot": ParallelCoordinatesPlot,
+ "Parameter Importance": ParameterImportancePlot,
+ "Run Comparer": RunComparer,
+ "Media Browser": MediaBrowser,
+ "Markdown Panel": MarkdownPanel,
+ # Panels with no config
+ "Confusion Matrix": ConfusionMatrix,
+ "Data Frame Table": DataFrames,
+ "Multi Run Table": MultiRunTable,
+ "Vega": Vega,
+ "Vega2": CustomChart,
+ "Vega3": Vega3,
+ "Weave": WeavePanel,
+}
+
+weave_panels = [
+ WeavePanelSummaryTable,
+ WeavePanelArtifactVersionedFile,
+ WeavePanelArtifact,
+ WeavePanel,
+]
diff --git a/wandb/apis/reports/_templates.py b/wandb/apis/reports/_templates.py
new file mode 100644
index 0000000000000000000000000000000000000000..1fae98633ecc5f8e5fb11d3878ceec955ccf81df
--- /dev/null
+++ b/wandb/apis/reports/_templates.py
@@ -0,0 +1,478 @@
+# We can import from the top after dropping support for Python 3.6
+# import wandb.apis.reports as wr
+from .util import coalesce
+
+
+def create_example_header():
+ """Create an example header with image at top."""
+ import wandb.apis.reports as wr
+
+ return [
+ wr.P(),
+ wr.HorizontalRule(),
+ wr.P(),
+ wr.Image(
+ "https://camo.githubusercontent.com/83839f20c90facc062330f8fee5a7ab910fdd04b80b4c4c7e89d6d8137543540/68747470733a2f2f692e696d6775722e636f6d2f676236423469672e706e67"
+ ),
+ wr.P(),
+ wr.HorizontalRule(),
+ wr.P(),
+ ]
+
+
+def create_example_footer():
+ """Create an example footer with image and text at bottom."""
+ import wandb.apis.reports as wr
+
+ return [
+ wr.P(),
+ wr.HorizontalRule(),
+ wr.P(),
+ wr.H1("Disclaimer"),
+ wr.P(
+ "The views and opinions expressed in this report are those of the authors and do not necessarily reflect the official policy or position of Weights & Biases. blah blah blah blah blah boring text at the bottom"
+ ),
+ wr.P(),
+ wr.HorizontalRule(),
+ ]
+
+
+def create_enterprise_report(
+ project=None,
+ title="Untitled Report",
+ description="",
+ header=None,
+ body=None,
+ footer=None,
+):
+ """Create an example enterprise report with a header and footer.
+
+ Can be used to add custom branding to reports.
+ """
+ import wandb.apis.reports as wr
+
+ project = coalesce(project, "default-project")
+ header = coalesce(header, create_example_header())
+ body = coalesce(body, [])
+ footer = coalesce(footer, create_example_footer())
+
+ return wr.Report(
+ project=project,
+ title=title,
+ description=description,
+ blocks=[*header, *body, *footer],
+ )
+
+
+def create_customer_landing_page(
+ project=None,
+ company_name="My Company",
+ main_contact="My Contact (name@email.com)",
+ slack_link="https://company.slack.com",
+):
+ """Create an example customer landing page using data from Andrew's demo."""
+ import wandb.apis.reports as wr
+
+ project = coalesce(project, "default-project")
+
+ return wr.Report(
+ project,
+ title=f"Weights & Biases @ {company_name}",
+ description=f"The developer-first MLOps platform is now available at {company_name}!\nReach out to {main_contact} for an account, and join your dedicated slack channel at:\n{slack_link}",
+ blocks=[
+ wr.P(),
+ wr.HorizontalRule(),
+ wr.TableOfContents(),
+ wr.P(),
+ wr.HorizontalRule(),
+ wr.H1(text=["What is Weights & Biases?"]),
+ wr.P(
+ text=[
+ "Weights & Biases (W&B) is the developer-first MLOps platform to build better models faster. Over 200,000+ ML practitioners at 500+ companies use W&B to optimize their ML workflows in Natural Language, Computer Vision, Reinforcement Learning, Tabular ML, Finance, and more!"
+ ]
+ ),
+ wr.P(),
+ wr.H2(text=["Why do you need W&B?"]),
+ wr.P(
+ text=[
+ "ML is a highly experimental field. Often we try many different datasets, model architectures, optimizers, hyperparameters, etc."
+ ]
+ ),
+ wr.P(
+ text=["Experimentation is great, but it can get messy. Have you ever:"]
+ ),
+ wr.UnorderedList(
+ items=[
+ ["Logged experiments in a sketchy spreadsheet?"],
+ [
+ "Built an amazing model but could not reproduce it for a colleague / model validation?"
+ ],
+ ["Wondered why your model is making strange predictions?"],
+ ["Fumbled with tuning hyperparameters?"],
+ [
+ "Struggled explaining to a colleague the impact of what you're doing?"
+ ],
+ ]
+ ),
+ wr.P(
+ text=["If that sounds familiar, W&B might be a good solution for you!"]
+ ),
+ wr.P(),
+ wr.H2(
+ text=[
+ "What does W&B do?",
+ wr.Link(text="", url="https://wandb.ai/site/experiment-tracking"),
+ ]
+ ),
+ wr.P(
+ text=[
+ wr.Link(text="", url="https://wandb.ai/site/experiment-tracking"),
+ "W&B has lightweight and flexible tools for... (expand to see more)",
+ ]
+ ),
+ wr.H3(
+ text=[
+ wr.Link(
+ text="Experiment tracking",
+ url="https://wandb.ai/site/experiment-tracking",
+ )
+ ]
+ ),
+ wr.PanelGrid(
+ runsets=[
+ wr.Runset(
+ entity="megatruong",
+ project="whirlwind_test4",
+ name="Run set",
+ query="",
+ filters={
+ "$or": [
+ {
+ "$and": [
+ {"state": {"$ne": "crashed"}},
+ {
+ "config.Learner.value.opt_func": {
+ "$ne": None
+ }
+ },
+ ]
+ }
+ ]
+ },
+ groupby=["Learner.opt_func"],
+ order=["-CreatedTimestamp"],
+ )
+ ],
+ panels=[
+ wr.LinePlot(
+ x="Step",
+ y=["gradients/layers.0.4.0.bn1.bias"],
+ log_y=False,
+ groupby="None",
+ layout={"x": 16, "y": 12, "w": 8, "h": 6},
+ ),
+ wr.LinePlot(
+ x="Step",
+ y=["gradients/layers.0.1.weight"],
+ log_y=False,
+ groupby="None",
+ layout={"x": 8, "y": 12, "w": 8, "h": 6},
+ ),
+ wr.LinePlot(
+ x="Step",
+ y=["gradients/layers.0.1.bias"],
+ log_y=False,
+ groupby="None",
+ layout={"x": 0, "y": 12, "w": 8, "h": 6},
+ ),
+ wr.LinePlot(
+ x="Step",
+ y=["train_loss"],
+ log_y=False,
+ groupby="None",
+ layout={"x": 16, "y": 0, "w": 8, "h": 6},
+ ),
+ wr.LinePlot(
+ x="Step",
+ y=["valid_loss"],
+ log_y=False,
+ groupby="None",
+ layout={"x": 16, "y": 6, "w": 8, "h": 6},
+ ),
+ wr.LinePlot(
+ x="Step",
+ y=["top_k_accuracy"],
+ log_y=False,
+ groupby="None",
+ layout={"x": 8, "y": 0, "w": 8, "h": 6},
+ ),
+ wr.LinePlot(
+ x="Step",
+ y=["mom_0"],
+ log_y=False,
+ groupby="None",
+ layout={"x": 0, "y": 6, "w": 8, "h": 6},
+ ),
+ wr.LinePlot(
+ x="Step",
+ y=["lr_0"],
+ log_y=False,
+ groupby="None",
+ layout={"x": 8, "y": 6, "w": 8, "h": 6},
+ ),
+ wr.LinePlot(
+ x="Step",
+ y=["accuracy"],
+ log_y=False,
+ groupby="None",
+ layout={"x": 0, "y": 0, "w": 8, "h": 6},
+ ),
+ ],
+ custom_run_colors={
+ ("Run set", "megatruong"): "rgb(83, 135, 221)",
+ ("Run set", "fastai.optimizer.ranger"): "rgb(83, 135, 221)",
+ ("Run set", "fastai.optimizer.Adam"): "rgb(229, 116, 57)",
+ },
+ ),
+ wr.P(
+ text=[
+ wr.Link(
+ text="",
+ url="https://assets.website-files.com/5ac6b7f2924c656f2b13a88c/6066c22135b8983b61ad7939_weights-and-biases-logo.svg",
+ )
+ ]
+ ),
+ wr.H3(
+ text=[
+ wr.Link(
+ text="Dataset and model versioning, evaluation, and reproduction",
+ url="https://wandb.ai/site/artifacts",
+ )
+ ]
+ ),
+ wr.WeaveBlockArtifact(
+ entity="megatruong",
+ project="whirlwind_test4",
+ artifact="camvid_learner",
+ tab="lineage",
+ ),
+ wr.P(),
+ wr.H3(
+ text=[
+ wr.Link(
+ text="Hyperparameter optimization",
+ url="https://wandb.ai/site/sweeps",
+ )
+ ]
+ ),
+ wr.P(text=[wr.Link(text="", url="https://wandb.ai/site/sweeps")]),
+ wr.PanelGrid(
+ runsets=[
+ wr.Runset(
+ entity="wandb",
+ project="cartpole",
+ name="Run set",
+ query="sweep",
+ filters={"$or": [{"$and": []}]},
+ order=["-CreatedTimestamp"],
+ )
+ ],
+ panels=[
+ wr.MediaBrowser(layout={"x": 0, "y": 10, "w": 24, "h": 10}),
+ wr.ParallelCoordinatesPlot(
+ columns=[
+ wr.PCColumn(metric="c::activation"),
+ wr.PCColumn(metric="c::lr", log_scale=True),
+ wr.PCColumn(
+ metric="c::target_model_update",
+ log_scale=True,
+ ),
+ wr.PCColumn(metric="c::n_hidden", log_scale=True),
+ wr.PCColumn(metric="test_reward"),
+ ],
+ layout={"x": 0, "y": 0, "w": 24, "h": 10},
+ ),
+ ],
+ ),
+ wr.H3(
+ text=[
+ wr.Link(
+ text="Model visualization and analysis",
+ url="https://wandb.ai/site/tables",
+ )
+ ]
+ ),
+ wr.P(text=[wr.Link(text="", url="https://wandb.ai/site/tables")]),
+ wr.PanelGrid(
+ runsets=[
+ wr.Runset(
+ entity="megatruong",
+ project="whirlwind_test4",
+ name="Run set",
+ query="",
+ filters={"$or": [{"$and": []}]},
+ order=["-CreatedTimestamp"],
+ )
+ ],
+ panels=[
+ wr.WeavePanelSummaryTable(
+ table_name="valid_table",
+ layout={"x": 7, "y": 0, "w": 7, "h": 13},
+ ),
+ wr.WeavePanelSummaryTable(
+ table_name="img_table",
+ layout={"x": 0, "y": 0, "w": 7, "h": 13},
+ ),
+ wr.WeavePanelSummaryTable(
+ table_name="image_table",
+ layout={"x": 14, "y": 0, "w": 10, "h": 13},
+ ),
+ ],
+ ),
+ wr.P(),
+ wr.PanelGrid(
+ runsets=[
+ wr.Runset(
+ entity="wandb",
+ project="wandb_spacy_integration",
+ name="Run set",
+ query="",
+ filters={"$or": [{"$and": []}]},
+ order=["-CreatedTimestamp"],
+ )
+ ],
+ panels=[
+ wr.WeavePanelSummaryTable(
+ table_name="spaCy NER table",
+ layout={"x": 0, "y": 0, "w": 24, "h": 10},
+ ),
+ wr.WeavePanelSummaryTable(
+ table_name="per annotation scores",
+ layout={"x": 7, "y": 10, "w": 17, "h": 8},
+ ),
+ wr.WeavePanelSummaryTable(
+ table_name="metrics", layout={"x": 0, "y": 10, "w": 7, "h": 8}
+ ),
+ ],
+ ),
+ wr.H3(
+ text=[
+ wr.Link(
+ text="ML team collaboration and sharing results",
+ url="https://wandb.ai/site/reports",
+ )
+ ]
+ ),
+ wr.H2(text=["How do I get access?"]),
+ wr.P(text=[f"Ask {main_contact} to help:"]),
+ wr.OrderedList(
+ items=[
+ ["Set up your account"],
+ [
+ "Get added to the ",
+ wr.Link(
+ text="joint slack channel",
+ url=slack_link,
+ ),
+ ],
+ ]
+ ),
+ wr.HorizontalRule(),
+ wr.H1(text=["Getting Started"]),
+ wr.P(text=["W&B has two components:"]),
+ wr.OrderedList(
+ items=[
+ ["A centrally managed MLOps platform and UI"],
+ [
+ "The ",
+ wr.InlineCode(code="wandb"),
+ " SDK (",
+ wr.Link(text="github", url="https://github.com/wandb/client"),
+ ", ",
+ wr.Link(text="pypi", url="https://pypi.org/project/wandb/"),
+ ", ",
+ wr.Link(
+ text="conda-forge",
+ url="https://anaconda.org/conda-forge/wandb",
+ ),
+ ")",
+ ],
+ ]
+ ),
+ wr.P(),
+ wr.H3(text=["1. Install the SDK"]),
+ wr.CodeBlock(code=["pip install wandb"], language="python"),
+ wr.P(),
+ wr.H3(text=["2. Log in to W&B"]),
+ wr.P(text=["You will be prompted to get and set your API key in the UI."]),
+ wr.CodeBlock(code=["wandb.login()"], language="python"),
+ wr.P(),
+ wr.H3(text=["3. Setup an experiment"]),
+ wr.P(
+ text=[
+ "Add this to the beginning of your scripts (or top of your notebook)."
+ ]
+ ),
+ wr.CodeBlock(code=["wandb.init()"], language="python"),
+ wr.P(),
+ wr.P(
+ text=[
+ wr.Link(
+ text="For more details on options and advanced usage, see the docs.",
+ url="https://docs.wandb.ai/ref/python/init",
+ )
+ ]
+ ),
+ wr.P(),
+ wr.H3(text=["4. Log anything!"]),
+ wr.P(text=["You can log metrics anywhere in your script, for example"]),
+ wr.CodeBlock(code=['wandb.log({"loss": model_loss})'], language="python"),
+ wr.P(),
+ wr.P(
+ text=[
+ "Log metrics, graphs, dataframes, images with segmentation masks or bounding boxes, videos, point clouds, custom HTML, and more! ",
+ wr.Link(
+ text="For more details on logging, including advanced types, see the docs.",
+ url="https://docs.wandb.ai/guides/track/log",
+ ),
+ ]
+ ),
+ wr.P(text=["W&B also helps you reproduce results by capturing:"]),
+ wr.UnorderedList(
+ items=[
+ ["git state (repo, commit)"],
+ ["requirements (requirements.txt, conda_env.yml)"],
+ ["logs, including stdout"],
+ [
+ "hardware metrics (CPU, GPU, network, memory utilization, temperature, throughput)"
+ ],
+ ["and more!"],
+ ]
+ ),
+ wr.P(),
+ wr.H3(text=["Putting everything together:"]),
+ wr.CodeBlock(
+ code=[
+ "wandb.login()",
+ "",
+ "wandb.init()",
+ "for i in range(1000):",
+ ' wandb.log({"metric": i})',
+ ],
+ language="python",
+ ),
+ wr.P(),
+ wr.H1(text=["What else is possible with W&B?"]),
+ wr.H2(text=["Example projects"]),
+ wr.Gallery(
+ ids=[
+ "Vmlldzo4ODc0MDc=",
+ "Vmlldzo4NDI3NzM=",
+ "Vmlldzo2MDIzMTg=",
+ "VmlldzoyMjA3MjY=",
+ "Vmlldzo1NjM4OA==",
+ ]
+ ),
+ wr.P(),
+ ],
+ )
diff --git a/wandb/apis/reports/blocks.py b/wandb/apis/reports/blocks.py
new file mode 100644
index 0000000000000000000000000000000000000000..102acbae6885e44e7a62c4b3f712ad01eb8bcb2b
--- /dev/null
+++ b/wandb/apis/reports/blocks.py
@@ -0,0 +1,27 @@
+# flake8: noqa
+from ._blocks import (
+ H1,
+ H2,
+ H3,
+ BlockQuote,
+ CalloutBlock,
+ CheckedList,
+ CodeBlock,
+ Gallery,
+ HorizontalRule,
+ Image,
+ LaTeXBlock,
+ MarkdownBlock,
+ OrderedList,
+ P,
+ PanelGrid,
+ SoundCloud,
+ Spotify,
+ TableOfContents,
+ UnorderedList,
+ Video,
+ WeaveBlockArtifact,
+ WeaveBlockArtifactVersionedFile,
+ WeaveBlockSummaryTable,
+ Twitter,
+)
diff --git a/wandb/apis/reports/helpers.py b/wandb/apis/reports/helpers.py
new file mode 100644
index 0000000000000000000000000000000000000000..4a6bf8721edebda6f4099990eea7b25d7985045e
--- /dev/null
+++ b/wandb/apis/reports/helpers.py
@@ -0,0 +1,2 @@
+# flake8: noqa
+from ._helpers import LineKey, PCColumn
diff --git a/wandb/apis/reports/mutations.py b/wandb/apis/reports/mutations.py
new file mode 100644
index 0000000000000000000000000000000000000000..20c523f85190bc7aa80bc393e0c2215c56393d68
--- /dev/null
+++ b/wandb/apis/reports/mutations.py
@@ -0,0 +1,73 @@
+from wandb_gql import gql
+
+VIEW_REPORT = gql(
+ """
+ query SpecificReport($reportId: ID!) {
+ view(id: $reportId) {
+ id
+ type
+ name
+ displayName
+ description
+ project {
+ id
+ name
+ entityName
+ }
+ createdAt
+ updatedAt
+ spec
+ previewUrl
+ user {
+ name
+ username
+ userInfo
+ }
+ }
+ }
+ """
+)
+UPSERT_VIEW = gql(
+ """
+ mutation upsertView(
+ $id: ID
+ $entityName: String
+ $projectName: String
+ $type: String
+ $name: String
+ $displayName: String
+ $description: String
+ $spec: String!
+ ) {
+ upsertView(
+ input: {
+ id: $id
+ entityName: $entityName
+ projectName: $projectName
+ name: $name
+ displayName: $displayName
+ description: $description
+ type: $type
+ createdUsing: WANDB_SDK
+ spec: $spec
+ }
+ ) {
+ view {
+ id
+ type
+ name
+ displayName
+ description
+ project {
+ id
+ name
+ entityName
+ }
+ spec
+ updatedAt
+ }
+ inserted
+ }
+ }
+"""
+)
diff --git a/wandb/apis/reports/panels.py b/wandb/apis/reports/panels.py
new file mode 100644
index 0000000000000000000000000000000000000000..9a671e00ad082cda73dcc69646365ca522db10f5
--- /dev/null
+++ b/wandb/apis/reports/panels.py
@@ -0,0 +1,17 @@
+# flake8: noqa
+from ._panels import (
+ BarPlot,
+ CodeComparer,
+ CustomChart,
+ LinePlot,
+ MarkdownPanel,
+ MediaBrowser,
+ ParallelCoordinatesPlot,
+ ParameterImportancePlot,
+ RunComparer,
+ ScalarChart,
+ ScatterPlot,
+ WeavePanelArtifact,
+ WeavePanelArtifactVersionedFile,
+ WeavePanelSummaryTable,
+)
diff --git a/wandb/apis/reports/report.py b/wandb/apis/reports/report.py
new file mode 100644
index 0000000000000000000000000000000000000000..843b498b7123f471d2b26ee8bad136cafb7453e2
--- /dev/null
+++ b/wandb/apis/reports/report.py
@@ -0,0 +1,268 @@
+import base64
+import inspect
+import json
+import re
+import urllib
+from copy import deepcopy
+from typing import List as LList
+
+from ... import __version__ as wandb_ver
+from ... import termlog, termwarn
+from ...sdk.lib import ipython
+from ..public import Api as PublicApi
+from ..public import RetryingClient
+from ._blocks import P, PanelGrid, UnknownBlock, WeaveBlock, block_mapping, weave_blocks
+from .mutations import UPSERT_VIEW, VIEW_REPORT
+from .runset import Runset
+from .util import Attr, Base, Block, coalesce, generate_name, nested_get, nested_set
+from .validators import OneOf, TypeValidator
+
+
+class Report(Base):
+ project: str = Attr(json_path="viewspec.project.name")
+ entity: str = Attr(json_path="viewspec.project.entityName")
+ title: str = Attr(json_path="viewspec.displayName")
+ description: str = Attr(json_path="viewspec.description")
+ width: str = Attr(
+ json_path="viewspec.spec.width",
+ validators=[OneOf(["readable", "fixed", "fluid"])],
+ )
+ blocks: list = Attr(
+ json_path="viewspec.spec.blocks",
+ validators=[TypeValidator(Block, how="keys")],
+ )
+
+ def __init__(
+ self,
+ project,
+ entity=None,
+ title="Untitled Report",
+ description="",
+ width="readable",
+ blocks=None,
+ _api=None,
+ *args,
+ **kwargs,
+ ):
+ super().__init__(*args, **kwargs)
+ self._viewspec = self._default_viewspec()
+ self._orig_viewspec = deepcopy(self._viewspec)
+ self._api = PublicApi() if _api is None else _api
+
+ self.project = project
+ self.entity = coalesce(entity, self._api.default_entity, "")
+ self.title = title
+ self.description = description
+ self.width = width
+ self.blocks = coalesce(blocks, [])
+
+ @classmethod
+ def from_url(cls, url, api=None):
+ if api is None:
+ api = PublicApi()
+ report_id = cls._url_to_report_id(url)
+ r = api.client.execute(VIEW_REPORT, variable_values={"reportId": report_id})
+ viewspec = r["view"]
+ viewspec["spec"] = json.loads(viewspec["spec"])
+ return cls.from_json(viewspec)
+
+ @staticmethod
+ def _url_to_report_id(url):
+ path_msg = "Path must be `entity/project/reports/report_title--report_id`"
+ try:
+ report_path, *_ = url.split("?")
+ report_path = report_path.replace("---", "--")
+
+ if "--" not in report_path:
+ raise ValueError(path_msg)
+
+ *_, report_id = report_path.split("--")
+ if len(report_id) == 0:
+ raise ValueError("Invalid report id")
+
+ report_id = report_id.strip()
+
+ """
+ Server does not generate IDs with correct padding, so decode with default validate=False.
+ Then re-encode it with correct padding.
+ https://stackoverflow.com/questions/2941995/python-ignore-incorrect-padding-error-when-base64-decoding
+
+ Corresponding core app logic that strips the padding in url
+ https://github.com/wandb/core/blob/b563437c1f3237ec35b1fb388ac14abbab7b4279/frontends/app/src/util/url/shared.ts#L33-L78
+ """
+ report_id = base64.b64encode(base64.b64decode(report_id + "==")).decode(
+ "utf-8"
+ )
+
+ except ValueError as e:
+ raise ValueError(path_msg) from e
+ else:
+ return report_id
+
+ @blocks.getter
+ def blocks(self):
+ json_path = self._get_path("blocks")
+ block_specs = nested_get(self, json_path)
+ blocks = []
+ for bspec in block_specs:
+ cls = block_mapping.get(bspec["type"], UnknownBlock)
+ if cls is UnknownBlock:
+ termwarn(
+ inspect.cleandoc(
+ f"""
+ UNKNOWN BLOCK DETECTED
+ This can happen if we have added new blocks, but you are using an older version of the SDK.
+ If your report is loading normally, you can safely ignore this message (but we recommend not touching UnknownBlock)
+ If you think this is an error, please file a bug report including your SDK version ({wandb_ver}) and this spec ({bspec})
+ """
+ )
+ )
+ if cls is WeaveBlock:
+ for cls in weave_blocks:
+ try:
+ cls.from_json(bspec)
+ except Exception:
+ pass
+ else:
+ break
+ blocks.append(cls.from_json(bspec))
+ return blocks[1:-1] # accounts for hidden p blocks
+
+ @blocks.setter
+ def blocks(self, new_blocks):
+ json_path = self._get_path("blocks")
+ new_block_specs = (
+ [P("").spec] + [b.spec for b in new_blocks] + [P("").spec]
+ ) # hidden p blocks
+ nested_set(self, json_path, new_block_specs)
+
+ @staticmethod
+ def _default_viewspec():
+ return {
+ "id": None,
+ "name": None,
+ "spec": {
+ "version": 5,
+ "panelSettings": {},
+ "blocks": [],
+ "width": "readable",
+ "authors": [],
+ "discussionThreads": [],
+ "ref": {},
+ },
+ }
+
+ @classmethod
+ def from_json(cls, viewspec):
+ obj = cls(project=viewspec["project"]["name"])
+ obj._viewspec = viewspec
+ obj._orig_viewspec = deepcopy(obj._viewspec)
+ return obj
+
+ @property
+ def viewspec(self):
+ return self._viewspec
+
+ @property
+ def modified(self) -> bool:
+ return self._viewspec != self._orig_viewspec
+
+ @property
+ def spec(self) -> dict:
+ return self._viewspec["spec"]
+
+ @property
+ def client(self) -> "RetryingClient":
+ return self._api.client
+
+ @property
+ def id(self) -> str:
+ return self._viewspec["id"]
+
+ @property
+ def name(self) -> str:
+ return self._viewspec["name"]
+
+ @property
+ def panel_grids(self) -> "LList[PanelGrid]":
+ return [b for b in self.blocks if isinstance(b, PanelGrid)]
+
+ @property
+ def runsets(self) -> "LList[Runset]":
+ return [rs for pg in self.panel_grids for rs in pg.runsets]
+
+ @property
+ def url(self) -> str:
+ title = re.sub(r"\W", "-", self.title)
+ title = re.sub(r"-+", "-", title)
+ title = urllib.parse.quote(title)
+ id = self.id.replace("=", "")
+ app_url = self._api.client.app_url
+ if not app_url.endswith("/"):
+ app_url = app_url + "/"
+ return f"{app_url}{self.entity}/{self.project}/reports/{title}--{id}"
+
+ def save(self, draft: bool = False, clone: bool = False) -> "Report":
+ if not self.modified:
+ termwarn("Report has not been modified")
+
+ # create project if not exists
+ projects = self._api.projects(self.entity)
+ is_new_project = True
+ for p in projects:
+ if p.name == self.project:
+ is_new_project = False
+ break
+
+ if is_new_project:
+ self._api.create_project(self.project, self.entity)
+
+ # All panel grids must have at least one runset
+ for pg in self.panel_grids:
+ if not pg.runsets:
+ pg.runsets = PanelGrid._default_runsets()
+
+ # Check runsets with `None` for project and replace with the report's project.
+ # We have to do this here because RunSets don't know about their report until they're added to it.
+ for rs in self.runsets:
+ rs.entity = coalesce(rs.entity, self._api.default_entity)
+ rs.project = coalesce(rs.project, self.project)
+
+ r = self._api.client.execute(
+ UPSERT_VIEW,
+ variable_values={
+ "id": None if clone or not self.id else self.id,
+ "name": generate_name() if clone or not self.name else self.name,
+ "entityName": self.entity,
+ "projectName": self.project,
+ "description": self.description,
+ "displayName": self.title,
+ "type": "runs/draft" if draft else "runs",
+ "spec": json.dumps(self.spec),
+ },
+ )
+
+ viewspec = r["upsertView"]["view"]
+ viewspec["spec"] = json.loads(viewspec["spec"])
+ if clone:
+ return Report.from_json(viewspec)
+ else:
+ self._viewspec["id"] = viewspec["id"]
+ self._viewspec["name"] = viewspec["name"]
+ return self
+
+ def to_html(self, height: int = 1024, hidden: bool = False) -> str:
+ """Generate HTML containing an iframe displaying this report."""
+ try:
+ url = self.url + "?jupyter=true"
+ style = f"border:none;width:100%;height:{height}px;"
+ prefix = ""
+ if hidden:
+ style += "display:none;"
+ prefix = ipython.toggle_button("report")
+ return prefix + f"<iframe src={url!r} style={style!r}></iframe>"
+ except AttributeError:
+ termlog("HTML repr will be available after you save the report!")
+
+ def _repr_html_(self) -> str:
+ return self.to_html()
diff --git a/wandb/apis/reports/runset.py b/wandb/apis/reports/runset.py
new file mode 100644
index 0000000000000000000000000000000000000000..0ed96540df6ec1ac40db57ca653bd8ac91925eff
--- /dev/null
+++ b/wandb/apis/reports/runset.py
@@ -0,0 +1,144 @@
+from typing import Any, Dict, Optional, TypeVar
+
+from ..public import Api as PublicApi
+from ..public import PythonMongoishQueryGenerator, QueryGenerator, Runs
+from .util import Attr, Base, coalesce, generate_name, nested_get, nested_set
+
+T = TypeVar("T")
+
+
+class Runset(Base):
+ entity: Optional[str] = Attr(json_path="spec.project.entityName")
+ project: Optional[str] = Attr(json_path="spec.project.name")
+ name: str = Attr(json_path="spec.name")
+ query: str = Attr(json_path="spec.search.query")
+ filters: dict = Attr(json_path="spec.filters")
+ groupby: list = Attr(json_path="spec.grouping")
+ order: list = Attr(json_path="spec.sort")
+
+ def __init__(
+ self,
+ entity=None,
+ project=None,
+ name="Run set",
+ query="",
+ filters=None,
+ groupby=None,
+ order=None,
+ *args,
+ **kwargs,
+ ):
+ super().__init__(*args, **kwargs)
+ self._spec = self._default_runset_spec()
+ self.query_generator = QueryGenerator()
+ self.pm_query_generator = PythonMongoishQueryGenerator(self)
+
+ self.entity = coalesce(entity, PublicApi().default_entity, "")
+ self.project = project # If the project is None, it will be updated to the report's project on save. See: Report.save
+ self.name = name
+ self.query = query
+ self.filters = coalesce(filters, self._default_filters())
+ self.groupby = coalesce(groupby, self._default_groupby())
+ self.order = coalesce(order, self._default_order())
+
+ @classmethod
+ def from_json(cls, spec: Dict[str, Any]) -> T:
+ """This has a custom implementation because sometimes runsets are missing the project field."""
+ obj = cls()
+ obj._spec = spec
+
+ project = spec.get("project")
+ if project:
+ obj.entity = project.get(
+ "entityName", coalesce(PublicApi().default_entity, "")
+ )
+ obj.project = project.get("name")
+ else:
+ obj.entity = coalesce(PublicApi().default_entity, "")
+ obj.project = None
+
+ return obj
+
+ @filters.getter
+ def filters(self):
+ json_path = self._get_path("filters")
+ filter_specs = nested_get(self, json_path)
+ return self.query_generator.filter_to_mongo(filter_specs)
+
+ @filters.setter
+ def filters(self, new_filters):
+ json_path = self._get_path("filters")
+ new_filter_specs = self.query_generator.mongo_to_filter(new_filters)
+ nested_set(self, json_path, new_filter_specs)
+
+ def set_filters_with_python_expr(self, expr):
+ self.filters = self.pm_query_generator.python_to_mongo(expr)
+ return self
+
+ @groupby.getter
+ def groupby(self):
+ json_path = self._get_path("groupby")
+ groupby_specs = nested_get(self, json_path)
+ cols = [self.query_generator.key_to_server_path(k) for k in groupby_specs]
+ return [self.pm_query_generator.back_to_front(c) for c in cols]
+
+ @groupby.setter
+ def groupby(self, new_groupby):
+ json_path = self._get_path("groupby")
+ cols = [self.pm_query_generator.front_to_back(g) for g in new_groupby]
+ new_groupby_specs = [self.query_generator.server_path_to_key(c) for c in cols]
+ nested_set(self, json_path, new_groupby_specs)
+
+ @order.getter
+ def order(self):
+ json_path = self._get_path("order")
+ order_specs = nested_get(self, json_path)
+ cols = self.query_generator.keys_to_order(order_specs)
+ return [c[0] + self.pm_query_generator.back_to_front(c[1:]) for c in cols]
+
+ @order.setter
+ def order(self, new_orders):
+ json_path = self._get_path("order")
+ cols = [o[0] + self.pm_query_generator.front_to_back(o[1:]) for o in new_orders]
+ new_order_specs = self.query_generator.order_to_keys(cols)
+ nested_set(self, json_path, new_order_specs)
+
+ @property
+ def _runs_config(self) -> dict:
+ return {k: v for run in self.runs for k, v in run.config.items()}
+
+ @property
+ def runs(self) -> Runs:
+ return PublicApi().runs(
+ path=f"{self.entity}/{self.project}", filters=self.filters
+ )
+
+ @staticmethod
+ def _default_filters():
+ return {"$or": [{"$and": []}]}
+
+ @staticmethod
+ def _default_groupby():
+ return []
+
+ @staticmethod
+ def _default_order():
+ return ["-CreatedTimestamp"]
+
+ @staticmethod
+ def _default_runset_spec():
+ return {
+ "id": generate_name(),
+ "runFeed": {
+ "version": 2,
+ "columnVisible": {"run:name": False},
+ "columnPinned": {},
+ "columnWidths": {},
+ "columnOrder": [],
+ "pageSize": 10,
+ "onlyShowSelected": False,
+ },
+ "enabled": True,
+ "selections": {"root": 1, "bounds": [], "tree": []},
+ "expandedRowAddresses": [],
+ }
diff --git a/wandb/apis/reports/templates.py b/wandb/apis/reports/templates.py
new file mode 100644
index 0000000000000000000000000000000000000000..b4387eac728e6b03f1b346907e98255b19795840
--- /dev/null
+++ b/wandb/apis/reports/templates.py
@@ -0,0 +1,7 @@
+# flake8: noqa
+from ._templates import (
+ create_customer_landing_page,
+ create_enterprise_report,
+ create_example_footer,
+ create_example_header,
+)
diff --git a/wandb/apis/reports/util.py b/wandb/apis/reports/util.py
new file mode 100644
index 0000000000000000000000000000000000000000..7c8f2003a31c5593233ea94e1b7b615012251b64
--- /dev/null
+++ b/wandb/apis/reports/util.py
@@ -0,0 +1,406 @@
+import random
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ Generic,
+ List,
+ Optional,
+ Tuple,
+ TypeVar,
+ Union,
+ get_type_hints,
+)
+
+from ..public import PanelMetricsHelper
+from .validators import UNDEFINED_TYPE, TypeValidator, Validator
+
+# Func = TypeVar("Func")
+T = TypeVar("T")
+V = TypeVar("V")
+Func = Callable[[T], V]
+
+
+def generate_name(length: int = 12) -> str:
+ """Generate a random name.
+
+ This implementation roughly based the following snippet in core:
+ https://github.com/wandb/core/blob/master/lib/js/cg/src/utils/string.ts#L39-L44.
+ """
+
+ # Borrowed from numpy: https://github.com/numpy/numpy/blob/v1.23.0/numpy/core/numeric.py#L2069-L2123
+ def base_repr(number: int, base: int, padding: int = 0) -> str:
+ digits = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ if base > len(digits):
+ raise ValueError("Bases greater than 36 not handled in base_repr.")
+ elif base < 2:
+ raise ValueError("Bases less than 2 not handled in base_repr.")
+
+ num = abs(number)
+ res = []
+ while num:
+ res.append(digits[num % base])
+ num //= base
+ if padding:
+ res.append("0" * padding)
+ if number < 0:
+ res.append("-")
+ return "".join(reversed(res or "0"))
+
+ rand = random.random()
+ rand = int(float(str(rand)[2:]))
+ rand36 = base_repr(rand, 36)
+ return rand36.lower()[:length]
+
+
+def coalesce(*arg: Any) -> Any:
+ """Return the first non-none value in the list of arguments.
+
+ Similar to ?? in C#.
+ """
+ return next((a for a in arg if a is not None), None)
+
+
+def nested_get(json: dict, keys: str) -> Any:
+ """Get the element at the terminal node of a nested JSON dict based on `path`.
+
+ The first item of the path can be an object.
+ """
+ keys = keys.split(".")
+ if len(keys) == 1:
+ return vars(json)[keys[0]]
+ else:
+ rv = json
+ for key in keys:
+ if isinstance(rv, Base):
+ if not hasattr(rv, key):
+ setattr(rv, key, {})
+ rv = getattr(rv, key)
+ else:
+ if key not in rv:
+ rv[key] = None
+ rv = rv[key]
+ return rv
+
+
+def nested_set(json: dict, keys: str, value: Any) -> None:
+ """Set the element at the terminal node of a nested JSON dict based on `path`.
+
+ The first item of the path can be an object.
+
+ If nodes do not exist, they are created along the way.
+ """
+ keys = keys.split(".")
+ if len(keys) == 1:
+ vars(json)[keys[0]] = value
+ else:
+ for key in keys[:-1]:
+ if isinstance(json, Base):
+ if not hasattr(json, key):
+ setattr(json, key, {})
+ json = getattr(json, key)
+ else:
+ json = json.setdefault(key, {})
+ json[keys[-1]] = value
+
+
+class Property(Generic[T]):
+ """Property descriptor with a default getter and setter."""
+
+ def __init__(
+ self, fget: Optional[Func] = None, fset: Optional[Func] = None
+ ) -> None:
+ self.fget = fget or self.default_fget
+ self.fset = fset or self.default_fset
+ self.name = ""
+
+ def __set_name__(self, owner: Any, name: str) -> None:
+ self.name = name
+
+ def __get__(self, obj: Any, objtype: Optional[Any] = None) -> T:
+ if obj is None:
+ return self
+ if self.fget is None:
+ raise AttributeError(f"unreadable attribute {self.name}")
+ return self.fget(obj)
+
+ def __set__(self, obj: Any, value: Any) -> None:
+ if self.fset is None:
+ raise AttributeError(f"can't set attribute {self.name}")
+ self.fset(obj, value)
+
+ def getter(self, fget: Func) -> Func:
+ prop = type(self)(fget, self.fset)
+ prop.name = self.name
+ return prop
+
+ def setter(self, fset: Func) -> Func:
+ prop = type(self)(self.fget, fset)
+ prop.name = self.name
+ return prop
+
+ def default_fget(self, obj: Any) -> Any:
+ return obj.__dict__[self.name]
+
+ def default_fset(self, obj: Any, value: Any) -> None:
+ obj.__dict__[self.name] = value
+
+
+class Validated(Property):
+ def __init__(
+ self, *args: Func, validators: Optional[List[Validator]] = None, **kwargs: Func
+ ) -> None:
+ super().__init__(*args, **kwargs)
+ if validators is None:
+ validators = []
+ self.validators = validators
+
+ def __set__(self, instance: Any, value: Any) -> None:
+ if not isinstance(value, type(self)):
+ for validator in self.validators:
+ validator(self, value)
+ super().__set__(instance, value)
+
+
+class Typed(Validated):
+ def __set_name__(self, owner: Any, name: str) -> None:
+ super().__set_name__(owner, name)
+ self.type = get_type_hints(owner).get(name, UNDEFINED_TYPE)
+
+ if self.type is not UNDEFINED_TYPE:
+ self.validators = [TypeValidator(attr_type=self.type)] + self.validators
+
+
+class JSONLinked(Property):
+ """Property that is linked to one or more JSON keys."""
+
+ def __init__(
+ self,
+ *args: Func,
+ json_path: Optional[Union[str, List[str]]] = None,
+ **kwargs: Func,
+ ) -> None:
+ super().__init__(*args, **kwargs)
+ self.path_or_name = json_path
+
+ def __set_name__(self, owner: Any, name: str) -> None:
+ if self.path_or_name is None:
+ self.path_or_name = name
+ super().__set_name__(owner, name)
+
+ def getter(self, fget: Func) -> Func:
+ prop = type(self)(fget, self.fset, json_path=self.path_or_name)
+ prop.name = self.name
+ return prop
+
+ def setter(self, fset: Func) -> Func:
+ prop = type(self)(self.fget, fset, json_path=self.path_or_name)
+ prop.name = self.name
+ return prop
+
+ def default_fget(self, obj: Any) -> Union[Any, List[Any]]:
+ if isinstance(self.path_or_name, str):
+ return nested_get(obj, self.path_or_name)
+ elif isinstance(self.path_or_name, list):
+ return [nested_get(obj, p) for p in self.path_or_name]
+ else:
+ raise TypeError(f"Unexpected type for path {type(self.path_or_name)!r}")
+
+ def default_fset(self, obj: Any, value: Any) -> None:
+ if isinstance(self.path_or_name, str):
+ nested_set(obj, self.path_or_name, value)
+ elif isinstance(self.path_or_name, list):
+ for p, v in zip(self.path_or_name, value):
+ nested_set(obj, p, v)
+ else:
+ raise TypeError(f"Unexpected type for path {type(self.path_or_name)!r}")
+
+
+class Attr(Typed, JSONLinked):
+ def getter(self, fget: Func) -> Func:
+ prop = type(self)(
+ fget, self.fset, json_path=self.path_or_name, validators=self.validators
+ )
+ prop.name = self.name
+ return prop
+
+ def setter(self, fset: Func) -> Func:
+ prop = type(self)(
+ self.fget, fset, json_path=self.path_or_name, validators=self.validators
+ )
+ prop.name = self.name
+ return prop
+
+
+class SubclassOnlyABC:
+ def __new__(cls, *args: Any, **kwargs: Any) -> T:
+ if SubclassOnlyABC in cls.__bases__:
+ raise TypeError(f"Abstract class {cls.__name__} cannot be instantiated")
+
+ return super().__new__(cls)
+
+
+class ShortReprMixin:
+ def __repr__(self) -> str:
+ clas = self.__class__.__name__
+ props = {
+ k: getattr(self, k)
+ for k, v in self.__class__.__dict__.items()
+ if isinstance(v, Attr)
+ }
+ settings = [
+ f"{k}={v!r}" for k, v in props.items() if not self._is_interesting(v)
+ ]
+ return "{}({})".format(clas, ", ".join(settings))
+
+ @staticmethod
+ def _is_interesting(x: Any) -> bool:
+ if isinstance(x, (list, tuple)):
+ return all(v is None for v in x)
+ return x is None or x == {}
+
+
+class Base(SubclassOnlyABC, ShortReprMixin):
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ super().__init__(*args, **kwargs)
+ self._spec = {}
+
+ @property
+ def spec(self) -> Dict[str, Any]:
+ return self._spec
+
+ @classmethod
+ def from_json(cls, spec: Dict[str, Any]) -> T:
+ obj = cls()
+ obj._spec = spec
+ return obj
+
+ def _get_path(self, var: str) -> str:
+ return vars(type(self))[var].path_or_name
+
+
+class Panel(Base, SubclassOnlyABC):
+ layout: dict = Attr(json_path="spec.layout")
+
+ def __init__(
+ self, layout: Optional[Dict[str, int]] = None, *args: Any, **kwargs: Any
+ ) -> None:
+ super().__init__(*args, **kwargs)
+ self._spec["viewType"] = self.view_type
+ self._spec["__id__"] = generate_name()
+ self.layout = coalesce(layout, self._default_panel_layout())
+ self.panel_metrics_helper = PanelMetricsHelper()
+
+ @property
+ def view_type(self) -> str:
+ return "UNKNOWN PANEL"
+
+ @property
+ def config(self) -> Dict[str, Any]:
+ return self._spec["config"]
+
+ @staticmethod
+ def _default_panel_layout() -> Dict[str, int]:
+ return {"x": 0, "y": 0, "w": 8, "h": 6}
+
+ @layout.setter
+ def layout(self, d: Dict[str, int]) -> None:
+ d["x"] = coalesce(d.get("x"), self._default_panel_layout()["x"])
+ d["y"] = coalesce(d.get("y"), self._default_panel_layout()["y"])
+ d["w"] = coalesce(d.get("w"), self._default_panel_layout()["w"])
+ d["h"] = coalesce(d.get("h"), self._default_panel_layout()["h"])
+
+ # json_path = self._get_path("layout")
+ # can't use _get_path because it's not on the obj... if only we had dataclass...
+ json_path = "spec.layout"
+ nested_set(self, json_path, d)
+
+
+class Block(Base, SubclassOnlyABC):
+ pass
+
+
+def fix_collisions(panels: List[Panel]) -> List[Panel]:
+ x_max = 24
+
+ for i, p1 in enumerate(panels):
+ for p2 in panels[i:]:
+ if collides(p1, p2):
+ # try to move right
+ x, y = shift(p1, p2)
+ if p2.layout["x"] + p2.layout["w"] + x <= x_max:
+ p2.layout["x"] += x
+
+ # if you hit right right bound, move down
+ else:
+ p2.layout["y"] += y
+
+ # then check if you can move left again to cleanup layout
+ p2.layout["x"] = 0
+ return panels
+
+
+def collides(p1: Panel, p2: Panel) -> bool:
+ l1, l2 = p1.layout, p2.layout
+
+ if (
+ (p1.spec["__id__"] == p2.spec["__id__"])
+ or (l1["x"] + l1["w"] <= l2["x"])
+ or (l1["x"] >= l2["w"] + l2["x"])
+ or (l1["y"] + l1["h"] <= l2["y"])
+ or (l1["y"] >= l2["y"] + l2["h"])
+ ):
+ return False
+
+ return True
+
+
+def shift(p1: Panel, p2: Panel) -> Tuple[Panel, Panel]:
+ l1, l2 = p1.layout, p2.layout
+
+ x = l1["x"] + l1["w"] - l2["x"]
+ y = l1["y"] + l1["h"] - l2["y"]
+
+ return x, y
+
+
+class InlineLaTeX(Base):
+ latex: str = Attr()
+
+ def __init__(self, latex="", *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.latex = latex
+
+ @property
+ def spec(self) -> dict:
+ return {"type": "latex", "children": [{"text": ""}], "content": self.latex}
+
+
+class InlineCode(Base):
+ code: str = Attr()
+
+ def __init__(self, code="", *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.code = code
+
+ @property
+ def spec(self) -> dict:
+ return {"text": self.code, "inlineCode": True}
+
+
+class Link(Base):
+ text: str = Attr()
+ url: str = Attr()
+
+ def __init__(self, text, url, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.text = text
+ self.url = url
+
+ @property
+ def spec(self) -> dict:
+ return {"type": "link", "url": self.url, "children": [{"text": self.text}]}
+
+
+def weave_inputs(spec):
+ return spec["config"]["panelConfig"]["exp"]["fromOp"]["inputs"]
diff --git a/wandb/apis/reports/validators.py b/wandb/apis/reports/validators.py
new file mode 100644
index 0000000000000000000000000000000000000000..d7cfa1a4ca55f9e00311a6979bbaf2e3b4d77a8b
--- /dev/null
+++ b/wandb/apis/reports/validators.py
@@ -0,0 +1,131 @@
+from abc import ABC, abstractmethod
+from typing import TypeVar, Union
+
+LINEPLOT_STYLES = ["line", "stacked-area", "pct-area", None]
+BARPLOT_STYLES = ["bar", "boxplot", "violin", None]
+FONT_SIZES = ["small", "medium", "large", "auto", None]
+LEGEND_POSITIONS = ["north", "south", "east", "west", None]
+LEGEND_ORIENTATIONS = ["horizontal", "vertical", None]
+AGGFUNCS = ["mean", "min", "max", "median", "sum", "samples", None]
+RANGEFUNCS = ["minmax", "stddev", "stderr", "none", "samples", None]
+MARKS = ["solid", "dashed", "dotted", "dotdash", "dotdotdash", None]
+TIMESTEPS = ["seconds", "minutes", "hours", "days", None]
+SMOOTHING_TYPES = ["exponential", "gaussian", "average", "none", None]
+CODE_COMPARE_DIFF = ["split", "unified", None]
+
+
+UNDEFINED_TYPE = TypeVar("UNDEFINED_TYPE")
+
+
+class Validator(ABC):
+ def __init__(self, how=None):
+ self.how = how
+
+ @abstractmethod
+ def call(self, attr_name, value):
+ pass
+
+ def __call__(self, attr, value):
+ attr_name = attr.name
+ if value is None and self.how in {"keys", "values"}:
+ return
+ if self.how == "keys":
+ attr_name += " keys"
+ for v in value:
+ self.call(attr_name, v)
+ elif self.how == "values":
+ attr_name += " values"
+ for v in value.values():
+ self.call(attr_name, v)
+ elif self.how is None:
+ attr_name += " object"
+ self.call(attr_name, value)
+ else:
+ raise ValueError(
+ 'Validator setting `how` must be one of ("keys", "values", None)'
+ )
+
+
+class TypeValidator(Validator):
+ def __init__(self, attr_type, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.attr_type = attr_type
+ try:
+ origin = attr_type.__origin__
+ subtypes = attr_type.__args__
+ except AttributeError: # normal types
+ self.attr_type = (attr_type,)
+ else:
+ if origin is Union:
+ self.attr_type = subtypes
+ else:
+ raise TypeError(f"{attr_type} is not currently supported.")
+
+ def call(self, attr_name, value):
+ if not isinstance(value, self.attr_type):
+ raise TypeError(
+ f"{attr_name} must be of type {self.attr_type!r} (got {type(value)!r})"
+ )
+
+
+class OneOf(Validator):
+ def __init__(self, options, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.options = options
+
+ def call(self, attr_name, value):
+ if value not in self.options:
+ raise ValueError(
+ f"{attr_name} must be one of {self.options!r} (got {value!r})"
+ )
+
+
+class Length(Validator):
+ def __init__(self, k, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.k = k
+
+ def call(self, attr_name, value):
+ if len(value) != self.k:
+ raise ValueError(
+ f"{attr_name} must have exactly {self.k} elements (got {len(value)!r}, elems: {value!r})"
+ )
+
+
+class Between(Validator):
+ def __init__(self, lb, ub, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.lb = lb
+ self.ub = ub
+
+ def call(self, attr_name, value):
+ if not self.lb <= value <= self.ub:
+ raise ValueError(
+ f"{attr_name} must be between [{self.lb}, {self.ub}] inclusive (got {value})"
+ )
+
+
+class OrderString(TypeValidator):
+ def __init__(self):
+ super().__init__(attr_type=str)
+
+ def call(self, attr_name, value):
+ super().call(attr_name, value)
+
+ if value[0] not in {"+", "-"}:
+ raise ValueError(
+ f'{attr_name} must be prefixed with "+" or "-" to indicate ascending or descending order'
+ )
+
+
+class LayoutDict(Validator):
+ def call(self, attr_name, value):
+ if set(value.keys()) != {"x", "y", "w", "h"}:
+ raise ValueError(
+ f"{attr_name} must be a dict containing exactly the keys `x`, y`, `w`, `h`"
+ )
+ for k, v in value.items():
+ if not isinstance(v, int):
+ raise ValueError(
+ f"{attr_name} key `{k}` must be of type {int} (got {type(v)!r})"
+ )
diff --git a/wandb/beta/workflows.py b/wandb/beta/workflows.py
new file mode 100644
index 0000000000000000000000000000000000000000..943c343b45e533279a1cc1c09c4be78c5b7ae849
--- /dev/null
+++ b/wandb/beta/workflows.py
@@ -0,0 +1,283 @@
+import json
+import os
+from typing import Any, Dict, List, Optional, Union
+
+import wandb
+import wandb.data_types as data_types
+from wandb.data_types import _SavedModel
+from wandb.sdk.artifacts.artifact import Artifact
+from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+
+
+def _add_any(
+ artifact: Artifact,
+ path_or_obj: Union[
+ str, ArtifactManifestEntry, data_types.WBValue
+ ], # todo: add dataframe
+ name: Optional[str],
+) -> Any:
+ """Add an object to an artifact.
+
+ High-level wrapper to add object(s) to an artifact - calls any of the .add* methods
+ under Artifact depending on the type of object that's passed in. This will probably
+ be moved to the Artifact class in the future.
+
+ Args:
+ artifact: `Artifact` - artifact created with `wandb.Artifact(...)`
+ path_or_obj: `Union[str, ArtifactManifestEntry, data_types.WBValue]` - either a
+ str or valid object which indicates what to add to an artifact.
+
+ name: `str` - the name of the object which is added to an artifact.
+
+ Returns:
+ Type[Any] - Union[None, ArtifactManifestEntry, etc]
+
+ """
+ if isinstance(path_or_obj, ArtifactManifestEntry):
+ return artifact.add_reference(path_or_obj, name)
+ elif isinstance(path_or_obj, data_types.WBValue):
+ return artifact.add(path_or_obj, name)
+ elif isinstance(path_or_obj, str):
+ if os.path.isdir(path_or_obj):
+ return artifact.add_dir(path_or_obj)
+ elif os.path.isfile(path_or_obj):
+ return artifact.add_file(path_or_obj)
+ else:
+ with artifact.new_file(name) as f:
+ f.write(json.dumps(path_or_obj, sort_keys=True))
+ else:
+ raise ValueError(
+ f"Expected `path_or_obj` to be instance of `ArtifactManifestEntry`, `WBValue`, or `str, found {type(path_or_obj)}"
+ )
+
+
+def _log_artifact_version(
+ name: str,
+ type: str,
+ entries: Dict[str, Union[str, ArtifactManifestEntry, data_types.WBValue]],
+ aliases: Optional[Union[str, List[str]]] = None,
+ description: Optional[str] = None,
+ metadata: Optional[dict] = None,
+ project: Optional[str] = None,
+ scope_project: Optional[bool] = None,
+ job_type: str = "auto",
+) -> Artifact:
+ """Create an artifact, populate it, and log it with a run.
+
+ If a run is not present, we create one.
+
+ Args:
+ name: `str` - name of the artifact. If not scoped to a project, name will be
+ suffixed by "-{run_id}".
+ type: `str` - type of the artifact, used in the UI to group artifacts of the
+ same type.
+ entries: `Dict` - dictionary containing the named objects we want added to this
+ artifact.
+ description: `str` - text description of artifact.
+ metadata: `Dict` - users can pass in artifact-specific metadata here, will be
+ visible in the UI.
+ project: `str` - project under which to place this artifact.
+ scope_project: `bool` - if True, we will not suffix `name` with "-{run_id}".
+ job_type: `str` - Only applied if run is not present and we create one.
+ Used to identify runs of a certain job type, i.e "evaluation".
+
+ Returns:
+ Artifact
+
+ """
+ if wandb.run is None:
+ run = wandb.init(
+ project=project, job_type=job_type, settings=wandb.Settings(silent="true")
+ )
+ else:
+ run = wandb.run
+
+ if not scope_project:
+ name = f"{name}-{run.id}"
+
+ if metadata is None:
+ metadata = {}
+
+ art = wandb.Artifact(name, type, description, metadata, False, None)
+
+ for path in entries:
+ _add_any(art, entries[path], path)
+
+ # "latest" should always be present as an alias
+ aliases = wandb.util._resolve_aliases(aliases)
+ run.log_artifact(art, aliases=aliases)
+
+ return art
+
+
+def log_model(
+ model_obj: Any,
+ name: str = "model",
+ aliases: Optional[Union[str, List[str]]] = None,
+ description: Optional[str] = None,
+ metadata: Optional[dict] = None,
+ project: Optional[str] = None,
+ scope_project: Optional[bool] = None,
+ **kwargs: Dict[str, Any],
+) -> "_SavedModel":
+ """Log a model object to enable model-centric workflows in the UI.
+
+ Supported frameworks include PyTorch, Keras, Tensorflow, Scikit-learn, etc. Under
+ the hood, we create a model artifact, bind it to the run that produced this model,
+ associate it with the latest metrics logged with `wandb.log(...)` and more.
+
+ Args:
+ model_obj: any model object created with the following ML frameworks: PyTorch,
+ Keras, Tensorflow, Scikit-learn. name: `str` - name of the model artifact
+ that will be created to house this model_obj.
+ aliases: `str, List[str]` - optional alias(es) that will be applied on this
+ model and allow for unique identification. The alias "latest" will always be
+ applied to the latest version of a model.
+ description: `str` - text description/notes about the model - will be visible in
+ the Model Card UI.
+ metadata: `Dict` - model-specific metadata goes here - will be visible the UI.
+ project: `str` - project under which to place this artifact.
+ scope_project: `bool` - If true, name of this model artifact will not be
+ suffixed by `-{run_id}`.
+
+ Returns:
+ _SavedModel instance
+
+ Example:
+ ```python
+ import torch.nn as nn
+ import torch.nn.functional as F
+
+
+ class Net(nn.Module):
+ def __init__(self):
+ super(Net, self).__init__()
+ self.fc1 = nn.Linear(10, 10)
+
+ def forward(self, x):
+ x = self.fc1(x)
+ x = F.relu(x)
+ return x
+
+
+ model = Net()
+ sm = log_model(model, "my-simple-model", aliases=["best"])
+ ```
+
+ """
+ model = data_types._SavedModel.init(model_obj, **kwargs)
+ _ = _log_artifact_version(
+ name=name,
+ type="model",
+ entries={
+ "index": model,
+ },
+ aliases=aliases,
+ description=description,
+ metadata=metadata,
+ project=project,
+ scope_project=scope_project,
+ job_type="log_model",
+ )
+ # TODO: handle offline mode appropriately.
+ return model
+
+
+def use_model(aliased_path: str) -> "_SavedModel":
+ """Fetch a saved model from an alias.
+
+ Under the hood, we use the alias to fetch the model artifact containing the
+ serialized model files and rebuild the model object from these files. We also
+ declare the fetched model artifact as an input to the run (with `run.use_artifact`).
+
+ Args:
+ aliased_path: `str` - the following forms are valid: "name:version",
+ "name:alias". May be prefixed with "entity/project".
+
+ Returns:
+ _SavedModel instance
+
+ Example:
+ ```python
+ # Assuming you have previously logged a model with the name "my-simple-model":
+ sm = use_model("my-simple-model:latest")
+ model = sm.model_obj()
+ ```
+ """
+ if ":" not in aliased_path:
+ raise ValueError(
+ "aliased_path must be of the form 'name:alias' or 'name:version'."
+ )
+
+ # Returns a _SavedModel instance
+ if wandb.run:
+ run = wandb.run
+ artifact = run.use_artifact(aliased_path)
+ sm = artifact.get("index")
+
+ if sm is None or not isinstance(sm, _SavedModel):
+ raise ValueError(
+ "Deserialization into model object failed: _SavedModel instance could not be initialized properly."
+ )
+
+ return sm
+ else:
+ raise ValueError(
+ "use_model can only be called inside a run. Please call wandb.init() before use_model(...)"
+ )
+
+
+def link_model(
+ model: "_SavedModel",
+ target_path: str,
+ aliases: Optional[Union[str, List[str]]] = None,
+) -> None:
+ """Link the given model to a portfolio.
+
+ A portfolio is a promoted collection which contains (in this case) model artifacts.
+ Linking to a portfolio allows for useful model-centric workflows in the UI.
+
+ Args:
+ model: `_SavedModel` - an instance of _SavedModel, most likely from the output
+ of `log_model` or `use_model`.
+ target_path: `str` - the target portfolio. The following forms are valid for the
+ string: {portfolio}, {project/portfolio},{entity}/{project}/{portfolio}.
+ aliases: `str, List[str]` - optional alias(es) that will only be applied on this
+ linked model inside the portfolio. The alias "latest" will always be applied
+ to the latest version of a model.
+
+ Returns:
+ None
+
+ Example:
+ sm = use_model("my-simple-model:latest")
+ link_model(sm, "my-portfolio")
+
+ """
+ aliases = wandb.util._resolve_aliases(aliases)
+
+ if wandb.run:
+ run = wandb.run
+
+ # _artifact_source, if it exists, points to a Public Artifact.
+ # Its existence means that _SavedModel was deserialized from a logged artifact, most likely from `use_model`.
+ if model._artifact_source:
+ artifact = model._artifact_source.artifact
+ # If the _SavedModel has been added to a Local Artifact (most likely through `.add(WBValue)`), then
+ # model._artifact_target will point to that Local Artifact.
+ elif model._artifact_target and model._artifact_target.artifact._final:
+ artifact = model._artifact_target.artifact
+ else:
+ raise ValueError(
+ "Linking requires that the given _SavedModel belongs to an artifact"
+ )
+
+ run.link_artifact(artifact, target_path, aliases)
+
+ else:
+ if model._artifact_source is not None:
+ model._artifact_source.artifact.link(target_path, aliases)
+ else:
+ raise ValueError(
+ "Linking requires that the given _SavedModel belongs to a logged artifact."
+ )
diff --git a/wandb/bin/apple_gpu_stats b/wandb/bin/apple_gpu_stats
new file mode 100755
index 0000000000000000000000000000000000000000..97ed32d7cc7b78e85534fc840845d57d0ea311f9
Binary files /dev/null and b/wandb/bin/apple_gpu_stats differ
diff --git a/wandb/catboost/__init__.py b/wandb/catboost/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..5ff79bd186c7107d95259223ed2a538bd97fa337
--- /dev/null
+++ b/wandb/catboost/__init__.py
@@ -0,0 +1,9 @@
+"""Compatibility catboost module.
+
+In the future use:
+ from wandb.integration.catboost import WandbCallback
+"""
+
+from wandb.integration.catboost import WandbCallback, log_summary
+
+__all__ = ["log_summary", "WandbCallback"]
diff --git a/wandb/cli/__init__.py b/wandb/cli/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/cli/cli.py b/wandb/cli/cli.py
new file mode 100644
index 0000000000000000000000000000000000000000..22c0b9429c115a2987c4e5eb5d375ec3e3304c7a
--- /dev/null
+++ b/wandb/cli/cli.py
@@ -0,0 +1,2890 @@
+#!/usr/bin/env python
+
+import asyncio
+import configparser
+import datetime
+import getpass
+import json
+import logging
+import os
+import pathlib
+import shlex
+import shutil
+import subprocess
+import sys
+import tempfile
+import textwrap
+import time
+import traceback
+from functools import wraps
+from typing import Any, Dict, Optional
+
+import click
+import yaml
+from click.exceptions import ClickException
+
+# pycreds has a find_executable that works in windows
+from dockerpycreds.utils import find_executable
+
+import wandb
+import wandb.env
+
+# from wandb.old.core import wandb_dir
+import wandb.sdk.verify.verify as wandb_verify
+from wandb import Config, Error, env, util, wandb_agent, wandb_sdk
+from wandb.apis import InternalApi, PublicApi
+from wandb.apis.public import RunQueue
+from wandb.integration.magic import magic_install
+from wandb.sdk.artifacts.artifact_file_cache import get_artifact_file_cache
+from wandb.sdk.launch import utils as launch_utils
+from wandb.sdk.launch._launch_add import _launch_add
+from wandb.sdk.launch.errors import ExecutionError, LaunchError
+from wandb.sdk.launch.sweeps import utils as sweep_utils
+from wandb.sdk.launch.sweeps.scheduler import Scheduler
+from wandb.sdk.lib import filesystem
+from wandb.sdk.lib.wburls import wburls
+from wandb.sync import SyncManager, get_run_from_path, get_runs
+
+# Send cli logs to wandb/debug-cli.<username>.log by default and fallback to a temp dir.
+_wandb_dir = wandb.old.core.wandb_dir(env.get_dir())
+if not os.path.exists(_wandb_dir):
+ _wandb_dir = tempfile.gettempdir()
+
+try:
+ _username = getpass.getuser()
+except KeyError:
+ # getuser() could raise KeyError in restricted environments like
+ # chroot jails or docker containers. Return user id in these cases.
+ _username = str(os.getuid())
+
+_wandb_log_path = os.path.join(_wandb_dir, f"debug-cli.{_username}.log")
+
+logging.basicConfig(
+ filename=_wandb_log_path,
+ level=logging.INFO,
+ format="%(asctime)s %(levelname)s %(message)s",
+ datefmt="%Y-%m-%d %H:%M:%S",
+)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+logger = logging.getLogger("wandb")
+
+# Click Contexts
+CONTEXT = {"default_map": {}}
+RUN_CONTEXT = {
+ "default_map": {},
+ "allow_extra_args": True,
+ "ignore_unknown_options": True,
+}
+
+
+def cli_unsupported(argument):
+ wandb.termerror(f"Unsupported argument `{argument}`")
+ sys.exit(1)
+
+
+class ClickWandbException(ClickException):
+ def format_message(self):
+ # log_file = util.get_log_file_path()
+ log_file = ""
+ orig_type = f"{self.orig_type.__module__}.{self.orig_type.__name__}"
+ if issubclass(self.orig_type, Error):
+ return click.style(str(self.message), fg="red")
+ else:
+ return (
+ f"An Exception was raised, see {log_file} for full traceback.\n"
+ f"{orig_type}: {self.message}"
+ )
+
+
+def display_error(func):
+ """Function decorator for catching common errors and re-raising as wandb.Error."""
+
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ try:
+ return func(*args, **kwargs)
+ except wandb.Error as e:
+ exc_type, exc_value, exc_traceback = sys.exc_info()
+ lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
+ logger.error("".join(lines))
+ wandb.termerror(f"Find detailed error logs at: {_wandb_log_path}")
+ click_exc = ClickWandbException(e)
+ click_exc.orig_type = exc_type
+ raise click_exc.with_traceback(sys.exc_info()[2])
+
+ return wrapper
+
+
+_api = None # caching api instance allows patching from unit tests
+
+
+def _get_cling_api(reset=None):
+ """Get a reference to the internal api with cling settings."""
+ global _api
+ if reset:
+ _api = None
+ wandb_sdk.wandb_setup._setup(_reset=True)
+ if _api is None:
+ # TODO(jhr): make a settings object that is better for non runs.
+ # only override the necessary setting
+ wandb.setup(settings=dict(_cli_only_mode=True))
+ _api = InternalApi()
+ return _api
+
+
+def prompt_for_project(ctx, entity):
+ """Ask the user for a project, creating one if necessary."""
+ result = ctx.invoke(projects, entity=entity, display=False)
+ api = _get_cling_api()
+ try:
+ if len(result) == 0:
+ project = click.prompt("Enter a name for your first project")
+ # description = editor()
+ project = api.upsert_project(project, entity=entity)["name"]
+ else:
+ project_names = [project["name"] for project in result] + ["Create New"]
+ wandb.termlog("Which project should we use?")
+ result = util.prompt_choices(project_names)
+ if result:
+ project = result
+ else:
+ project = "Create New"
+ # TODO: check with the server if the project exists
+ if project == "Create New":
+ project = click.prompt(
+ "Enter a name for your new project", value_proc=api.format_project
+ )
+ # description = editor()
+ project = api.upsert_project(project, entity=entity)["name"]
+
+ except wandb.errors.CommError as e:
+ raise ClickException(str(e))
+
+ return project
+
+
+class RunGroup(click.Group):
+ @display_error
+ def get_command(self, ctx, cmd_name):
+ # TODO: check if cmd_name is a file in the current dir and not require `run`?
+ rv = click.Group.get_command(self, ctx, cmd_name)
+ if rv is not None:
+ return rv
+ return None
+
+
+@click.command(cls=RunGroup, invoke_without_command=True)
+@click.version_option(version=wandb.__version__)
+@click.pass_context
+def cli(ctx):
+ if ctx.invoked_subcommand is None:
+ click.echo(ctx.get_help())
+
+
+@cli.command(context_settings=CONTEXT, help="List projects", hidden=True)
+@click.option(
+ "--entity",
+ "-e",
+ default=None,
+ envvar=env.ENTITY,
+ help="The entity to scope the listing to.",
+)
+@display_error
+def projects(entity, display=True):
+ api = _get_cling_api()
+ projects = api.list_projects(entity=entity)
+ if len(projects) == 0:
+ message = "No projects found for %s" % entity
+ else:
+ message = 'Latest projects for "%s"' % entity
+ if display:
+ click.echo(click.style(message, bold=True))
+ for project in projects:
+ click.echo(
+ "".join(
+ (
+ click.style(project["name"], fg="blue", bold=True),
+ " - ",
+ str(project["description"] or "").split("\n")[0],
+ )
+ )
+ )
+ return projects
+
+
+@cli.command(context_settings=CONTEXT, help="Login to Weights & Biases")
+@click.argument("key", nargs=-1)
+@click.option("--cloud", is_flag=True, help="Login to the cloud instead of local")
+@click.option("--host", default=None, help="Login to a specific instance of W&B")
+@click.option(
+ "--relogin", default=None, is_flag=True, help="Force relogin if already logged in."
+)
+@click.option("--anonymously", default=False, is_flag=True, help="Log in anonymously")
+@click.option("--verify", default=False, is_flag=True, help="Verify login credentials")
+@display_error
+def login(key, host, cloud, relogin, anonymously, verify, no_offline=False):
+ # TODO: handle no_offline
+ anon_mode = "must" if anonymously else "never"
+
+ wandb_sdk.wandb_login._handle_host_wandb_setting(host, cloud)
+ # A change in click or the test harness means key can be none...
+ key = key[0] if key is not None and len(key) > 0 else None
+ if key:
+ relogin = True
+
+ login_settings = dict(
+ _cli_only_mode=True,
+ _disable_viewer=relogin and not verify,
+ anonymous=anon_mode,
+ )
+ if host is not None:
+ login_settings["base_url"] = host
+
+ try:
+ wandb.setup(settings=login_settings)
+ except TypeError as e:
+ wandb.termerror(str(e))
+ sys.exit(1)
+
+ wandb.login(
+ relogin=relogin,
+ key=key,
+ anonymous=anon_mode,
+ host=host,
+ force=True,
+ verify=verify,
+ )
+
+
+@cli.command(
+ context_settings=CONTEXT, help="Run a wandb service", name="service", hidden=True
+)
+@click.option(
+ "--sock-port", default=None, type=int, help="The host port to bind socket service."
+)
+@click.option("--port-filename", default=None, help="Save allocated port to file.")
+@click.option("--address", default=None, help="The address to bind service.")
+@click.option("--pid", default=None, type=int, help="The parent process id to monitor.")
+@click.option("--debug", is_flag=True, help="log debug info")
+@click.option("--serve-sock", is_flag=True, help="use socket mode")
+@display_error
+def service(
+ sock_port=None,
+ port_filename=None,
+ address=None,
+ pid=None,
+ debug=False,
+ serve_sock=False,
+):
+ from wandb.sdk.service.server import WandbServer
+
+ server = WandbServer(
+ sock_port=sock_port,
+ port_fname=port_filename,
+ address=address,
+ pid=pid,
+ debug=debug,
+ serve_sock=serve_sock,
+ )
+ server.serve()
+
+
+@cli.command(
+ context_settings=CONTEXT, help="Configure a directory with Weights & Biases"
+)
+@click.option("--project", "-p", help="The project to use.")
+@click.option("--entity", "-e", help="The entity to scope the project to.")
+# TODO(jhr): Enable these with settings rework
+# @click.option("--setting", "-s", help="enable an arbitrary setting.", multiple=True)
+# @click.option('--show', is_flag=True, help="Show settings")
+@click.option("--reset", is_flag=True, help="Reset settings")
+@click.option(
+ "--mode",
+ "-m",
+ help=' Can be "online", "offline" or "disabled". Defaults to online.',
+)
+@click.pass_context
+@display_error
+def init(ctx, project, entity, reset, mode):
+ from wandb.old.core import __stage_dir__, _set_stage_dir, wandb_dir
+
+ if __stage_dir__ is None:
+ _set_stage_dir("wandb")
+
+ # non-interactive init
+ if reset or project or entity or mode:
+ api = InternalApi()
+ if reset:
+ api.clear_setting("entity", persist=True)
+ api.clear_setting("project", persist=True)
+ api.clear_setting("mode", persist=True)
+ # TODO(jhr): clear more settings?
+ if entity:
+ api.set_setting("entity", entity, persist=True)
+ if project:
+ api.set_setting("project", project, persist=True)
+ if mode:
+ api.set_setting("mode", mode, persist=True)
+ return
+
+ if os.path.isdir(wandb_dir()) and os.path.exists(
+ os.path.join(wandb_dir(), "settings")
+ ):
+ click.confirm(
+ click.style(
+ "This directory has been configured previously, should we re-configure it?",
+ bold=True,
+ ),
+ abort=True,
+ )
+ else:
+ click.echo(
+ click.style("Let's setup this directory for W&B!", fg="green", bold=True)
+ )
+ api = _get_cling_api()
+ if api.api_key is None:
+ ctx.invoke(login)
+ api = _get_cling_api(reset=True)
+
+ viewer = api.viewer()
+
+ # Viewer can be `None` in case your API information became invalid, or
+ # in testing if you switch hosts.
+ if not viewer:
+ click.echo(
+ click.style(
+ "Your login information seems to be invalid: can you log in again please?",
+ fg="red",
+ bold=True,
+ )
+ )
+ ctx.invoke(login)
+ api = _get_cling_api(reset=True)
+
+ # This shouldn't happen.
+ viewer = api.viewer()
+ if not viewer:
+ click.echo(
+ click.style(
+ "We're sorry, there was a problem logging you in. "
+ "Please send us a note at support@wandb.com and tell us how this happened.",
+ fg="red",
+ bold=True,
+ )
+ )
+ sys.exit(1)
+
+ # At this point we should be logged in successfully.
+ if len(viewer["teams"]["edges"]) > 1:
+ team_names = [e["node"]["name"] for e in viewer["teams"]["edges"]] + [
+ "Manual entry"
+ ]
+ wandb.termlog(
+ "Which team should we use?",
+ )
+ result = util.prompt_choices(team_names)
+ # result can be empty on click
+ if result:
+ entity = result
+ else:
+ entity = "Manual Entry"
+ if entity == "Manual Entry":
+ entity = click.prompt("Enter the name of the team you want to use")
+ else:
+ entity = viewer.get("entity") or click.prompt(
+ "What username or team should we use?"
+ )
+
+ # TODO: this error handling sucks and the output isn't pretty
+ try:
+ project = prompt_for_project(ctx, entity)
+ except ClickWandbException:
+ raise ClickException(f"Could not find team: {entity}")
+
+ api.set_setting("entity", entity, persist=True)
+ api.set_setting("project", project, persist=True)
+ api.set_setting("base_url", api.settings().get("base_url"), persist=True)
+
+ filesystem.mkdir_exists_ok(wandb_dir())
+ with open(os.path.join(wandb_dir(), ".gitignore"), "w") as file:
+ file.write("*\n!settings")
+
+ click.echo(
+ click.style("This directory is configured! Next, track a run:\n", fg="green")
+ + textwrap.dedent(
+ """\
+ * In your training script:
+ {code1}
+ {code2}
+ * then `{run}`.
+ """
+ ).format(
+ code1=click.style("import wandb", bold=True),
+ code2=click.style('wandb.init(project="%s")' % project, bold=True),
+ run=click.style("python <train.py>", bold=True),
+ )
+ )
+
+
+@cli.group()
+def beta():
+ """Beta versions of wandb CLI commands. Requires wandb-core."""
+ # this is the future that requires wandb-core!
+ from wandb.util import get_core_path
+
+ if not get_core_path():
+ click.echo(
+ "wandb beta commands require wandb-core, please install with `pip install wandb-core`"
+ )
+ sys.exit(1)
+
+
+@beta.command(
+ name="sync",
+ context_settings=CONTEXT,
+ help="Upload a training run to W&B",
+)
+@click.pass_context
+@click.argument("wandb_dir", nargs=1, type=click.Path(exists=True))
+@click.option("--id", "run_id", help="The run you want to upload to.")
+@click.option("--project", "-p", help="The project you want to upload to.")
+@click.option("--entity", "-e", help="The entity to scope to.")
+@click.option("--skip-console", is_flag=True, default=False, help="Skip console logs")
+@click.option("--append", is_flag=True, default=False, help="Append run")
+@click.option(
+ "--include",
+ "-i",
+ help="Glob to include. Can be used multiple times.",
+ multiple=True,
+)
+@click.option(
+ "--exclude",
+ "-e",
+ help="Glob to exclude. Can be used multiple times.",
+ multiple=True,
+)
+@click.option(
+ "--mark-synced/--no-mark-synced",
+ is_flag=True,
+ default=True,
+ help="Mark runs as synced",
+)
+@click.option(
+ "--skip-synced/--no-skip-synced",
+ is_flag=True,
+ default=True,
+ help="Skip synced runs",
+)
+@click.option(
+ "--dry-run", is_flag=True, help="Perform a dry run without uploading anything."
+)
+@display_error
+def sync_beta(
+ ctx,
+ wandb_dir=None,
+ run_id: Optional[str] = None,
+ project: Optional[str] = None,
+ entity: Optional[str] = None,
+ skip_console: bool = False,
+ append: bool = False,
+ include: Optional[str] = None,
+ exclude: Optional[str] = None,
+ skip_synced: bool = True,
+ mark_synced: bool = True,
+ dry_run: bool = False,
+):
+ import concurrent.futures
+ from multiprocessing import cpu_count
+
+ paths = set()
+
+ # TODO: test file discovery logic
+ # include and exclude globs are evaluated relative to the provided base_path
+ if include:
+ for pattern in include:
+ matching_dirs = list(pathlib.Path(wandb_dir).glob(pattern))
+ for d in matching_dirs:
+ if not d.is_dir():
+ continue
+ wandb_files = [p for p in d.glob("*.wandb") if p.is_file()]
+ if len(wandb_files) > 1:
+ print(f"Multiple wandb files found in directory {d}, skipping")
+ elif len(wandb_files) == 1:
+ paths.add(d)
+ else:
+ paths.update({p.parent for p in pathlib.Path(wandb_dir).glob("**/*.wandb")})
+
+ for pattern in exclude:
+ matching_dirs = list(pathlib.Path(wandb_dir).glob(pattern))
+ for d in matching_dirs:
+ if not d.is_dir():
+ continue
+ if d in paths:
+ paths.remove(d)
+
+ # remove paths that are already synced, if requested
+ if skip_synced:
+ synced_paths = set()
+ for path in paths:
+ wandb_synced_files = [p for p in path.glob("*.wandb.synced") if p.is_file()]
+ if len(wandb_synced_files) > 1:
+ print(
+ f"Multiple wandb.synced files found in directory {path}, skipping"
+ )
+ elif len(wandb_synced_files) == 1:
+ synced_paths.add(path)
+ paths -= synced_paths
+
+ if run_id and len(paths) > 1:
+ # TODO: handle this more gracefully
+ click.echo("id can only be set for a single run.", err=True)
+ sys.exit(1)
+
+ if not paths:
+ click.echo("No runs to sync.")
+ return
+
+ click.echo("Found runs:")
+ for path in paths:
+ click.echo(f" {path}")
+
+ if dry_run:
+ return
+
+ wandb.sdk.wandb_setup.setup()
+
+ # TODO: make it thread-safe in the Rust code
+ with concurrent.futures.ProcessPoolExecutor(
+ max_workers=min(len(paths), cpu_count())
+ ) as executor:
+ futures = []
+ for path in paths:
+ # we already know there is only one wandb file in the directory
+ wandb_file = [p for p in path.glob("*.wandb") if p.is_file()][0]
+ future = executor.submit(
+ wandb._sync,
+ wandb_file,
+ run_id=run_id,
+ project=project,
+ entity=entity,
+ skip_console=skip_console,
+ append=append,
+ mark_synced=mark_synced,
+ )
+ futures.append(future)
+
+ # Wait for tasks to complete
+ for _ in concurrent.futures.as_completed(futures):
+ pass
+
+
+@cli.command(
+ context_settings=CONTEXT, help="Upload an offline training directory to W&B"
+)
+@click.pass_context
+@click.argument("path", nargs=-1, type=click.Path(exists=True))
+@click.option("--view", is_flag=True, default=False, help="View runs", hidden=True)
+@click.option("--verbose", is_flag=True, default=False, help="Verbose", hidden=True)
+@click.option("--id", "run_id", help="The run you want to upload to.")
+@click.option("--project", "-p", help="The project you want to upload to.")
+@click.option("--entity", "-e", help="The entity to scope to.")
+@click.option(
+ "--job_type",
+ "job_type",
+ help="Specifies the type of run for grouping related runs together.",
+)
+@click.option(
+ "--sync-tensorboard/--no-sync-tensorboard",
+ is_flag=True,
+ default=None,
+ help="Stream tfevent files to wandb.",
+)
+@click.option("--include-globs", help="Comma seperated list of globs to include.")
+@click.option("--exclude-globs", help="Comma seperated list of globs to exclude.")
+@click.option(
+ "--include-online/--no-include-online",
+ is_flag=True,
+ default=None,
+ help="Include online runs",
+)
+@click.option(
+ "--include-offline/--no-include-offline",
+ is_flag=True,
+ default=None,
+ help="Include offline runs",
+)
+@click.option(
+ "--include-synced/--no-include-synced",
+ is_flag=True,
+ default=None,
+ help="Include synced runs",
+)
+@click.option(
+ "--mark-synced/--no-mark-synced",
+ is_flag=True,
+ default=True,
+ help="Mark runs as synced",
+)
+@click.option("--sync-all", is_flag=True, default=False, help="Sync all runs")
+@click.option("--clean", is_flag=True, default=False, help="Delete synced runs")
+@click.option(
+ "--clean-old-hours",
+ default=24,
+ help="Delete runs created before this many hours. To be used alongside --clean flag.",
+ type=int,
+)
+@click.option(
+ "--clean-force",
+ is_flag=True,
+ default=False,
+ help="Clean without confirmation prompt.",
+)
+@click.option("--ignore", hidden=True)
+@click.option("--show", default=5, help="Number of runs to show")
+@click.option("--append", is_flag=True, default=False, help="Append run")
+@click.option("--skip-console", is_flag=True, default=False, help="Skip console logs")
+@display_error
+def sync(
+ ctx,
+ path=None,
+ view=None,
+ verbose=None,
+ run_id=None,
+ project=None,
+ entity=None,
+ job_type=None, # trace this back to SyncManager
+ sync_tensorboard=None,
+ include_globs=None,
+ exclude_globs=None,
+ include_online=None,
+ include_offline=None,
+ include_synced=None,
+ mark_synced=None,
+ sync_all=None,
+ ignore=None,
+ show=None,
+ clean=None,
+ clean_old_hours=24,
+ clean_force=None,
+ append=None,
+ skip_console=None,
+):
+ api = _get_cling_api()
+ if api.api_key is None:
+ wandb.termlog("Login to W&B to sync offline runs")
+ ctx.invoke(login, no_offline=True)
+ api = _get_cling_api(reset=True)
+
+ if ignore:
+ exclude_globs = ignore
+ if include_globs:
+ include_globs = include_globs.split(",")
+ if exclude_globs:
+ exclude_globs = exclude_globs.split(",")
+
+ def _summary():
+ all_items = get_runs(
+ include_online=True,
+ include_offline=True,
+ include_synced=True,
+ include_unsynced=True,
+ )
+ sync_items = get_runs(
+ include_online=include_online if include_online is not None else True,
+ include_offline=include_offline if include_offline is not None else True,
+ include_synced=include_synced if include_synced is not None else False,
+ include_unsynced=True,
+ exclude_globs=exclude_globs,
+ include_globs=include_globs,
+ )
+ synced = []
+ unsynced = []
+ for item in all_items:
+ (synced if item.synced else unsynced).append(item)
+ if sync_items:
+ wandb.termlog(f"Number of runs to be synced: {len(sync_items)}")
+ if show and show < len(sync_items):
+ wandb.termlog(f"Showing {show} runs to be synced:")
+ for item in sync_items[: (show or len(sync_items))]:
+ wandb.termlog(f" {item}")
+ else:
+ wandb.termlog("No runs to be synced.")
+ if synced:
+ clean_cmd = click.style("wandb sync --clean", fg="yellow")
+ wandb.termlog(
+ f"NOTE: use {clean_cmd} to delete {len(synced)} synced runs from local directory."
+ )
+ if unsynced:
+ sync_cmd = click.style("wandb sync --sync-all", fg="yellow")
+ wandb.termlog(
+ f"NOTE: use {sync_cmd} to sync {len(unsynced)} unsynced runs from local directory."
+ )
+
+ def _sync_path(_path, _sync_tensorboard):
+ if run_id and len(_path) > 1:
+ wandb.termerror("id can only be set for a single run.")
+ sys.exit(1)
+ sm = SyncManager(
+ project=project,
+ entity=entity,
+ run_id=run_id,
+ job_type=job_type,
+ mark_synced=mark_synced,
+ app_url=api.app_url,
+ view=view,
+ verbose=verbose,
+ sync_tensorboard=_sync_tensorboard,
+ log_path=_wandb_log_path,
+ append=append,
+ skip_console=skip_console,
+ )
+ for p in _path:
+ sm.add(p)
+ sm.start()
+ while not sm.is_done():
+ _ = sm.poll()
+
+ def _sync_all():
+ sync_items = get_runs(
+ include_online=include_online if include_online is not None else True,
+ include_offline=include_offline if include_offline is not None else True,
+ include_synced=include_synced if include_synced is not None else False,
+ include_unsynced=True,
+ exclude_globs=exclude_globs,
+ include_globs=include_globs,
+ )
+ if not sync_items:
+ wandb.termerror("Nothing to sync.")
+ else:
+ # When syncing run directories, default to not syncing tensorboard
+ sync_tb = sync_tensorboard if sync_tensorboard is not None else False
+ _sync_path(sync_items, sync_tb)
+
+ def _clean():
+ if path:
+ runs = list(map(get_run_from_path, path))
+ if not clean_force:
+ click.confirm(
+ click.style(
+ f"Are you sure you want to remove {len(runs)} runs?",
+ bold=True,
+ ),
+ abort=True,
+ )
+ for run in runs:
+ shutil.rmtree(run.path)
+ click.echo(click.style("Success!", fg="green"))
+ return
+ runs = get_runs(
+ include_online=include_online if include_online is not None else True,
+ include_offline=include_offline if include_offline is not None else True,
+ include_synced=include_synced if include_synced is not None else True,
+ include_unsynced=False,
+ exclude_globs=exclude_globs,
+ include_globs=include_globs,
+ )
+ since = datetime.datetime.now() - datetime.timedelta(hours=clean_old_hours)
+ old_runs = [run for run in runs if run.datetime < since]
+ old_runs.sort(key=lambda _run: _run.datetime)
+ if old_runs:
+ click.echo(
+ f"Found {len(runs)} runs, {len(old_runs)} are older than {clean_old_hours} hours"
+ )
+ for run in old_runs:
+ click.echo(run.path)
+ if not clean_force:
+ click.confirm(
+ click.style(
+ f"Are you sure you want to remove {len(old_runs)} runs?",
+ bold=True,
+ ),
+ abort=True,
+ )
+ for run in old_runs:
+ shutil.rmtree(run.path)
+ click.echo(click.style("Success!", fg="green"))
+ else:
+ click.echo(
+ click.style(
+ f"No runs older than {clean_old_hours} hours found", fg="red"
+ )
+ )
+
+ if sync_all:
+ _sync_all()
+ elif clean:
+ _clean()
+ elif path:
+ # When syncing a specific path, default to syncing tensorboard
+ sync_tb = sync_tensorboard if sync_tensorboard is not None else True
+ _sync_path(path, sync_tb)
+ else:
+ _summary()
+
+
+@cli.command(
+ context_settings=CONTEXT,
+ help="Initialize a hyperparameter sweep. Search for hyperparameters that optimizes a cost function of a machine learning model by testing various combinations.",
+)
+@click.option(
+ "--project",
+ "-p",
+ default=None,
+ help="""The name of the project where W&B runs created from the sweep are sent to. If the project is not specified, the run is sent to a project labeled Uncategorized.""",
+)
+@click.option(
+ "--entity",
+ "-e",
+ default=None,
+ help="""The username or team name where you want to send W&B runs created by the sweep to. Ensure that the entity you specify already exists. If you don't specify an entity, the run will be sent to your default entity, which is usually your username.""",
+)
+@click.option("--controller", is_flag=True, default=False, help="Run local controller")
+@click.option("--verbose", is_flag=True, default=False, help="Display verbose output")
+@click.option(
+ "--name",
+ default=None,
+ help="The name of the sweep. The sweep ID is used if no name is specified.",
+)
+@click.option("--program", default=None, help="Set sweep program")
+@click.option("--settings", default=None, help="Set sweep settings", hidden=True)
+@click.option("--update", default=None, help="Update pending sweep")
+@click.option(
+ "--stop",
+ is_flag=True,
+ default=False,
+ help="Finish a sweep to stop running new runs and let currently running runs finish.",
+)
+@click.option(
+ "--cancel",
+ is_flag=True,
+ default=False,
+ help="Cancel a sweep to kill all running runs and stop running new runs.",
+)
+@click.option(
+ "--pause",
+ is_flag=True,
+ default=False,
+ help="Pause a sweep to temporarily stop running new runs.",
+)
+@click.option(
+ "--resume",
+ is_flag=True,
+ default=False,
+ help="Resume a sweep to continue running new runs.",
+)
+@click.argument("config_yaml_or_sweep_id")
+@click.pass_context
+@display_error
+def sweep(
+ ctx,
+ project,
+ entity,
+ controller,
+ verbose,
+ name,
+ program,
+ settings,
+ update,
+ stop,
+ cancel,
+ pause,
+ resume,
+ config_yaml_or_sweep_id,
+):
+ state_args = "stop", "cancel", "pause", "resume"
+ lcls = locals()
+ is_state_change_command = sum(lcls[k] for k in state_args)
+ if is_state_change_command > 1:
+ raise Exception("Only one state flag (stop/cancel/pause/resume) is allowed.")
+ elif is_state_change_command == 1:
+ sweep_id = config_yaml_or_sweep_id
+ api = _get_cling_api()
+ if api.api_key is None:
+ wandb.termlog("Login to W&B to use the sweep feature")
+ ctx.invoke(login, no_offline=True)
+ api = _get_cling_api(reset=True)
+ parts = dict(entity=entity, project=project, name=sweep_id)
+ err = sweep_utils.parse_sweep_id(parts)
+ if err:
+ wandb.termerror(err)
+ return
+ entity = parts.get("entity") or entity
+ project = parts.get("project") or project
+ sweep_id = parts.get("name") or sweep_id
+ state = [s for s in state_args if lcls[s]][0]
+ ings = {
+ "stop": "Stopping",
+ "cancel": "Cancelling",
+ "pause": "Pausing",
+ "resume": "Resuming",
+ }
+ wandb.termlog(f"{ings[state]} sweep {entity}/{project}/{sweep_id}")
+ getattr(api, "%s_sweep" % state)(sweep_id, entity=entity, project=project)
+ wandb.termlog("Done.")
+ return
+ else:
+ config_yaml = config_yaml_or_sweep_id
+
+ def _parse_settings(settings):
+ """Parse settings from json or comma separated assignments."""
+ ret = {}
+ # TODO(jhr): merge with magic:_parse_magic
+ if settings.find("=") > 0:
+ for item in settings.split(","):
+ kv = item.split("=")
+ if len(kv) != 2:
+ wandb.termwarn(
+ "Unable to parse sweep settings key value pair", repeat=False
+ )
+ ret.update(dict([kv]))
+ return ret
+ wandb.termwarn("Unable to parse settings parameter", repeat=False)
+ return ret
+
+ api = _get_cling_api()
+ if api.api_key is None:
+ wandb.termlog("Login to W&B to use the sweep feature")
+ ctx.invoke(login, no_offline=True)
+ api = _get_cling_api(reset=True)
+
+ sweep_obj_id = None
+ if update:
+ parts = dict(entity=entity, project=project, name=update)
+ err = sweep_utils.parse_sweep_id(parts)
+ if err:
+ wandb.termerror(err)
+ return
+ entity = parts.get("entity") or entity
+ project = parts.get("project") or project
+ sweep_id = parts.get("name") or update
+
+ has_project = (project or api.settings("project")) is not None
+ has_entity = (entity or api.settings("entity")) is not None
+
+ termerror_msg = (
+ "Sweep lookup requires a valid %s, and none was specified. \n"
+ "Either set a default %s in wandb/settings, or, if invoking \n`wandb sweep` "
+ "from the command line, specify the full sweep path via: \n\n"
+ " wandb sweep {username}/{projectname}/{sweepid}\n\n"
+ )
+
+ if not has_entity:
+ wandb.termerror(termerror_msg % (("entity",) * 2))
+ return
+
+ if not has_project:
+ wandb.termerror(termerror_msg % (("project",) * 2))
+ return
+
+ found = api.sweep(sweep_id, "{}", entity=entity, project=project)
+ if not found:
+ wandb.termerror(f"Could not find sweep {entity}/{project}/{sweep_id}")
+ return
+ sweep_obj_id = found["id"]
+
+ action = "Updating" if sweep_obj_id else "Creating"
+ wandb.termlog(f"{action} sweep from: {config_yaml}")
+ config = sweep_utils.load_sweep_config(config_yaml)
+
+ # Set or override parameters
+ if name:
+ config["name"] = name
+ if program:
+ config["program"] = program
+ if settings:
+ settings = _parse_settings(settings)
+ if settings:
+ config.setdefault("settings", {})
+ config["settings"].update(settings)
+ if controller:
+ config.setdefault("controller", {})
+ config["controller"]["type"] = "local"
+
+ is_local = config.get("controller", {}).get("type") == "local"
+ if is_local:
+ from wandb import controller as wandb_controller
+
+ tuner = wandb_controller()
+ err = tuner._validate(config)
+ if err:
+ wandb.termerror(f"Error in sweep file: {err}")
+ return
+
+ env = os.environ
+ entity = (
+ entity
+ or env.get("WANDB_ENTITY")
+ or config.get("entity")
+ or api.settings("entity")
+ )
+ project = (
+ project
+ or env.get("WANDB_PROJECT")
+ or config.get("project")
+ or api.settings("project")
+ or util.auto_project_name(config.get("program"))
+ )
+
+ sweep_id, warnings = api.upsert_sweep(
+ config,
+ project=project,
+ entity=entity,
+ obj_id=sweep_obj_id,
+ )
+ sweep_utils.handle_sweep_config_violations(warnings)
+
+ # Log nicely formatted sweep information
+ styled_id = click.style(sweep_id, fg="yellow")
+ wandb.termlog(f"{action} sweep with ID: {styled_id}")
+
+ sweep_url = wandb_sdk.wandb_sweep._get_sweep_url(api, sweep_id)
+ if sweep_url:
+ styled_url = click.style(sweep_url, underline=True, fg="blue")
+ wandb.termlog(f"View sweep at: {styled_url}")
+
+ # re-probe entity and project if it was auto-detected by upsert_sweep
+ entity = entity or env.get("WANDB_ENTITY")
+ project = project or env.get("WANDB_PROJECT")
+
+ if entity and project:
+ sweep_path = f"{entity}/{project}/{sweep_id}"
+ elif project:
+ sweep_path = f"{project}/{sweep_id}"
+ else:
+ sweep_path = sweep_id
+
+ if sweep_path.find(" ") >= 0:
+ sweep_path = f"{sweep_path!r}"
+
+ styled_path = click.style(f"wandb agent {sweep_path}", fg="yellow")
+ wandb.termlog(f"Run sweep agent with: {styled_path}")
+ if controller:
+ wandb.termlog("Starting wandb controller...")
+ from wandb import controller as wandb_controller
+
+ tuner = wandb_controller(sweep_id)
+ tuner.run(verbose=verbose)
+
+
+@cli.command(
+ context_settings=CONTEXT,
+ no_args_is_help=True,
+ help="Run a W&B launch sweep (Experimental).",
+)
+@click.option(
+ "--queue",
+ "-q",
+ default=None,
+ help="The name of a queue to push the sweep to",
+)
+@click.option(
+ "--project",
+ "-p",
+ default=None,
+ help="Name of the project which the agent will watch. "
+ "If passed in, will override the project value passed in using a config file",
+)
+@click.option(
+ "--entity",
+ "-e",
+ default=None,
+ help="The entity to use. Defaults to current logged-in user",
+)
+@click.option(
+ "--resume_id",
+ "-r",
+ default=None,
+ help="Resume a launch sweep by passing an 8-char sweep id. Queue required",
+)
+@click.argument("config", required=False, type=click.Path(exists=True))
+@click.pass_context
+@display_error
+def launch_sweep(
+ ctx,
+ project,
+ entity,
+ queue,
+ config,
+ resume_id,
+):
+ api = _get_cling_api()
+ env = os.environ
+ if api.api_key is None:
+ wandb.termlog("Login to W&B to use the sweep feature")
+ ctx.invoke(login, no_offline=True)
+ api = _get_cling_api(reset=True)
+
+ entity = entity or env.get("WANDB_ENTITY") or api.settings("entity")
+ if entity is None:
+ wandb.termerror("Must specify entity when using launch")
+ return
+
+ project = project or env.get("WANDB_PROJECT") or api.settings("project")
+ if project is None:
+ wandb.termerror("A project must be configured when using launch")
+ return
+
+ # get personal username, not team name or service account, default to entity
+ author = api.viewer().get("username") or entity
+
+ # if not sweep_config XOR resume_id
+ if not (config or resume_id):
+ wandb.termerror("'config' and/or 'resume_id' required")
+ return
+
+ parsed_user_config = sweep_utils.load_launch_sweep_config(config)
+ # Rip special keys out of config, store in scheduler run_config
+ launch_args: Dict[str, Any] = parsed_user_config.pop("launch", {})
+ scheduler_args: Dict[str, Any] = parsed_user_config.pop("scheduler", {})
+ settings: Dict[str, Any] = scheduler_args.pop("settings", {})
+
+ scheduler_job: Optional[str] = scheduler_args.get("job")
+ if scheduler_job:
+ wandb.termwarn(
+ "Using a scheduler job for launch sweeps is *experimental* and may change without warning"
+ )
+ queue: Optional[str] = queue or launch_args.get("queue")
+
+ sweep_config, sweep_obj_id = None, None
+ if not resume_id:
+ sweep_config = parsed_user_config
+
+ # check method
+ method = sweep_config.get("method")
+ if scheduler_job and not method:
+ sweep_config["method"] = "custom"
+ elif scheduler_job and method != "custom":
+ # TODO(gst): Check if using Anaconda2
+ wandb.termwarn(
+ "Use 'method': 'custom' in the sweep config when using scheduler jobs, "
+ "or omit it entirely. For jobs using the wandb optimization engine (WandbScheduler), "
+ "set the method in the sweep config under scheduler.settings.method "
+ )
+ settings["method"] = method
+
+ if settings.get("method"):
+ # assume WandbScheduler, and user is using this right
+ sweep_config["method"] = settings["method"]
+
+ else: # Resuming an existing sweep
+ found = api.sweep(resume_id, "{}", entity=entity, project=project)
+ if not found:
+ wandb.termerror(f"Could not find sweep {entity}/{project}/{resume_id}")
+ return
+
+ if found.get("state") == "RUNNING":
+ wandb.termerror(
+ f"Cannot resume sweep {entity}/{project}/{resume_id}, it is already running"
+ )
+ return
+
+ sweep_obj_id = found["id"]
+ sweep_config = yaml.safe_load(found["config"])
+ wandb.termlog(f"Resuming from existing sweep {entity}/{project}/{resume_id}")
+ if len(parsed_user_config.keys()) > 0:
+ wandb.termwarn(
+ "Sweep parameters loaded from resumed sweep, ignoring provided config"
+ )
+
+ prev_scheduler = json.loads(found.get("scheduler") or "{}")
+ run_spec = json.loads(prev_scheduler.get("run_spec", "{}"))
+ if (
+ scheduler_job
+ and run_spec.get("job")
+ and run_spec.get("job") != scheduler_job
+ ):
+ wandb.termerror(
+ f"Resuming a launch sweep with a different scheduler job is not supported. Job loaded from sweep: {run_spec.get('job')}, job in config: {scheduler_job}"
+ )
+ return
+
+ prev_scheduler_args, prev_settings = sweep_utils.get_previous_args(run_spec)
+ # Passed in scheduler_args and settings override previous
+ scheduler_args.update(prev_scheduler_args)
+ settings.update(prev_settings)
+ if not queue:
+ wandb.termerror(
+ "Launch-sweeps require setting a 'queue', use --queue option or a 'queue' key in the 'launch' section in the config"
+ )
+ return
+
+ entrypoint = Scheduler.ENTRYPOINT if not scheduler_job else None
+ args = sweep_utils.construct_scheduler_args(
+ return_job=scheduler_job is not None,
+ sweep_config=sweep_config,
+ queue=queue,
+ project=project,
+ author=author,
+ )
+ if not args:
+ return
+
+ # validate training job existence
+ if not sweep_utils.check_job_exists(PublicApi(), sweep_config.get("job")):
+ return False
+
+ # validate scheduler job existence, if present
+ if not sweep_utils.check_job_exists(PublicApi(), scheduler_job):
+ return False
+
+ # Set run overrides for the Scheduler
+ overrides = {"run_config": {}}
+ if launch_args:
+ overrides["run_config"]["launch"] = launch_args
+ if scheduler_args:
+ overrides["run_config"]["scheduler"] = scheduler_args
+ if settings:
+ overrides["run_config"]["settings"] = settings
+
+ if scheduler_job:
+ overrides["run_config"]["sweep_args"] = args
+ else:
+ overrides["args"] = args
+
+ # configure scheduler job resource
+ resource = scheduler_args.get("resource")
+ if resource:
+ if resource == "local-process" and scheduler_job:
+ wandb.termerror(
+ "Scheduler jobs cannot be run with the 'local-process' resource"
+ )
+ return
+ if resource == "local-process" and scheduler_args.get("docker_image"):
+ wandb.termerror(
+ "Scheduler jobs cannot be run with the 'local-process' resource and a docker image"
+ )
+ return
+ else: # no resource set, default local-process if not scheduler job, else container
+ resource = "local-process" if not scheduler_job else "local-container"
+
+ # Launch job spec for the Scheduler
+ launch_scheduler_spec = launch_utils.construct_launch_spec(
+ uri=Scheduler.PLACEHOLDER_URI,
+ api=api,
+ name="Scheduler.WANDB_SWEEP_ID",
+ project=project,
+ entity=entity,
+ docker_image=scheduler_args.get("docker_image"),
+ resource=resource,
+ entry_point=entrypoint,
+ resource_args=scheduler_args.get("resource_args", {}),
+ repository=launch_args.get("registry", {}).get("url", None),
+ job=scheduler_job,
+ version=None,
+ launch_config={"overrides": overrides},
+ run_id="WANDB_SWEEP_ID", # scheduler inits run with sweep_id=run_id
+ author=None, # author gets passed into scheduler override args
+ )
+ launch_scheduler_with_queue = json.dumps(
+ {
+ "queue": queue,
+ "run_queue_project": launch_utils.LAUNCH_DEFAULT_PROJECT,
+ "run_spec": json.dumps(launch_scheduler_spec),
+ }
+ )
+
+ sweep_id, warnings = api.upsert_sweep(
+ sweep_config,
+ project=project,
+ entity=entity,
+ obj_id=sweep_obj_id, # if resuming
+ launch_scheduler=launch_scheduler_with_queue,
+ state="PENDING",
+ )
+ sweep_utils.handle_sweep_config_violations(warnings)
+ # Log nicely formatted sweep information
+ styled_id = click.style(sweep_id, fg="yellow")
+ wandb.termlog(f"{'Resumed' if resume_id else 'Created'} sweep with ID: {styled_id}")
+ sweep_url = wandb_sdk.wandb_sweep._get_sweep_url(api, sweep_id)
+ if sweep_url:
+ styled_url = click.style(sweep_url, underline=True, fg="blue")
+ wandb.termlog(f"View sweep at: {styled_url}")
+ wandb.termlog(f"Scheduler added to launch queue ({queue})")
+
+
+@cli.command(help=f"Launch or queue a W&B Job. See {wburls.get('cli_launch')}")
+@click.option("--uri", "-u", metavar="(str)", default=None, hidden=True)
+@click.option(
+ "--job",
+ "-j",
+ metavar="(str)",
+ default=None,
+ help="Name of the job to launch. If passed in, launch does not require a uri.",
+)
+@click.option(
+ "--entry-point",
+ "-E",
+ metavar="NAME",
+ default=None,
+ help="""Entry point within project. [default: main]. If the entry point is not found,
+ attempts to run the project file with the specified name as a script,
+ using 'python' to run .py files and the default shell (specified by
+ environment variable $SHELL) to run .sh files. If passed in, will override the entrypoint value passed in using a config file.""",
+)
+@click.option(
+ "--git-version",
+ "-g",
+ metavar="GIT-VERSION",
+ hidden=True,
+ help="Version of the project to run, as a Git commit reference for Git projects.",
+)
+@click.option(
+ "--name",
+ envvar="WANDB_NAME",
+ help="""Name of the run under which to launch the run. If not
+ specified, a random run name will be used to launch run. If passed in, will override the name passed in using a config file.""",
+)
+@click.option(
+ "--entity",
+ "-e",
+ metavar="(str)",
+ default=None,
+ help="""Name of the target entity which the new run will be sent to. Defaults to using the entity set by local wandb/settings folder.
+ If passed in, will override the entity value passed in using a config file.""",
+)
+@click.option(
+ "--project",
+ "-p",
+ metavar="(str)",
+ default=None,
+ help="""Name of the target project which the new run will be sent to. Defaults to using the project name given by the source uri
+ or for github runs, the git repo name. If passed in, will override the project value passed in using a config file.""",
+)
+@click.option(
+ "--resource",
+ "-r",
+ metavar="BACKEND",
+ default=None,
+ help="""Execution resource to use for run. Supported values: 'local-process', 'local-container', 'kubernetes', 'sagemaker', 'gcp-vertex'.
+ This is now a required parameter if pushing to a queue with no resource configuration.
+ If passed in, will override the resource value passed in using a config file.""",
+)
+@click.option(
+ "--docker-image",
+ "-d",
+ default=None,
+ metavar="DOCKER IMAGE",
+ help="""Specific docker image you'd like to use. In the form name:tag.
+ If passed in, will override the docker image value passed in using a config file.""",
+)
+@click.option(
+ "--config",
+ "-c",
+ metavar="FILE",
+ help="""Path to JSON file (must end in '.json') or JSON string which will be passed
+ as a launch config. Dictation how the launched run will be configured.""",
+)
+@click.option(
+ "--set-var",
+ "-v",
+ "cli_template_vars",
+ default=None,
+ multiple=True,
+ help="""Set template variable values for queues with allow listing enabled,
+ as key-value pairs e.g. `--set-var key1=value1 --set-var key2=value2`""",
+)
+@click.option(
+ "--queue",
+ "-q",
+ is_flag=False,
+ flag_value="default",
+ default=None,
+ help="""Name of run queue to push to. If none, launches single run directly. If supplied without
+ an argument (`--queue`), defaults to queue 'default'. Else, if name supplied, specified run queue must exist under the
+ project and entity supplied.""",
+)
+@click.option(
+ "--async",
+ "run_async",
+ is_flag=True,
+ help="""Flag to run the job asynchronously. Defaults to false, i.e. unless --async is set, wandb launch will wait for
+ the job to finish. This option is incompatible with --queue; asynchronous options when running with an agent should be
+ set on wandb launch-agent.""",
+)
+@click.option(
+ "--resource-args",
+ "-R",
+ metavar="FILE",
+ help="""Path to JSON file (must end in '.json') or JSON string which will be passed
+ as resource args to the compute resource. The exact content which should be
+ provided is different for each execution backend. See documentation for layout of this file.""",
+)
+@click.option(
+ "--build",
+ "-b",
+ is_flag=True,
+ hidden=True,
+ help="Flag to build an associated job and push to queue as an image job.",
+)
+@click.option(
+ "--repository",
+ "-rg",
+ is_flag=False,
+ default=None,
+ hidden=True,
+ help="Name of a remote repository. Will be used to push a built image to.",
+)
+# TODO: this is only included for back compat. But we should remove this in the future
+@click.option(
+ "--project-queue",
+ "-pq",
+ default=None,
+ hidden=True,
+ help="Name of the project containing the queue to push to. If none, defaults to entity level queues.",
+)
+@click.option(
+ "--dockerfile",
+ "-D",
+ default=None,
+ help="Path to the Dockerfile used to build the job, relative to the job's root",
+)
+@click.option(
+ "--priority",
+ "-P",
+ default=None,
+ type=click.Choice(["critical", "high", "medium", "low"]),
+ help="""When --queue is passed, set the priority of the job. Launch jobs with higher priority
+ are served first. The order, from highest to lowest priority, is: critical, high, medium, low""",
+)
+@display_error
+def launch(
+ uri,
+ job,
+ entry_point,
+ git_version,
+ name,
+ resource,
+ entity,
+ project,
+ docker_image,
+ config,
+ cli_template_vars,
+ queue,
+ run_async,
+ resource_args,
+ build,
+ repository,
+ project_queue,
+ dockerfile,
+ priority,
+):
+ """Start a W&B run from the given URI.
+
+ The URI can bea wandb URI, a GitHub repo uri, or a local path). In the case of a
+ wandb URI the arguments used in the original run will be used by default. These
+ arguments can be overridden using the args option, or specifying those arguments in
+ the config's 'overrides' key, 'args' field as a list of strings.
+
+ Running `wandb launch [URI]` will launch the run directly. To add the run to a
+ queue, run `wandb launch [URI] --queue [optional queuename]`.
+ """
+ logger.info(
+ f"=== Launch called with kwargs {locals()} CLI Version: {wandb.__version__}==="
+ )
+ from wandb.sdk.launch._launch import _launch
+
+ api = _get_cling_api()
+ wandb._sentry.configure_scope(process_context="launch_cli")
+
+ if run_async and queue is not None:
+ raise LaunchError(
+ "Cannot use both --async and --queue with wandb launch, see help for details."
+ )
+
+ if queue and docker_image and not project:
+ raise LaunchError(
+ "Cannot use --queue and --docker together without a project. Please specify a project with --project or -p."
+ )
+
+ if priority is not None and queue is None:
+ raise LaunchError("--priority flag requires --queue to be set")
+
+ if resource_args is not None:
+ resource_args = util.load_json_yaml_dict(resource_args)
+ if resource_args is None:
+ raise LaunchError("Invalid format for resource-args")
+ else:
+ resource_args = {}
+
+ if entry_point is not None:
+ entry_point = shlex.split(entry_point)
+
+ if config is not None:
+ config = util.load_json_yaml_dict(config)
+ if config is None:
+ raise LaunchError("Invalid format for config")
+ else:
+ config = {}
+
+ resource = resource or config.get("resource")
+
+ if build and queue is None:
+ raise LaunchError("Build flag requires a queue to be set")
+
+ try:
+ launch_utils.check_logged_in(api)
+ except Exception:
+ wandb.termerror(f"Error running job: {traceback.format_exc()}")
+
+ run_id = config.get("run_id")
+
+ if dockerfile:
+ if "overrides" in config:
+ config["overrides"]["dockerfile"] = dockerfile
+ else:
+ config["overrides"] = {"dockerfile": dockerfile}
+
+ if priority is not None:
+ priority_map = {
+ "critical": 0,
+ "high": 1,
+ "medium": 2,
+ "low": 3,
+ }
+ priority = priority_map[priority.lower()]
+
+ template_variables = None
+ if cli_template_vars:
+ if queue is None:
+ raise LaunchError("'--set-var' flag requires queue to be set")
+ if entity is None:
+ entity = launch_utils.get_default_entity(api, config)
+ public_api = PublicApi()
+ runqueue = RunQueue(client=public_api.client, name=queue, entity=entity)
+ template_variables = launch_utils.fetch_and_validate_template_variables(
+ runqueue, cli_template_vars
+ )
+
+ if queue is None:
+ # direct launch
+ try:
+ run = asyncio.run(
+ _launch(
+ api,
+ uri,
+ job,
+ project=project,
+ entity=entity,
+ docker_image=docker_image,
+ name=name,
+ entry_point=entry_point,
+ version=git_version,
+ resource=resource,
+ resource_args=resource_args,
+ launch_config=config,
+ synchronous=(not run_async),
+ run_id=run_id,
+ repository=repository,
+ )
+ )
+ if asyncio.run(run.get_status()).state in [
+ "failed",
+ "stopped",
+ "preempted",
+ ]:
+ wandb.termerror("Launched run exited with non-zero status")
+ sys.exit(1)
+ except LaunchError as e:
+ logger.error("=== %s ===", e)
+ wandb._sentry.exception(e)
+ sys.exit(e)
+ except ExecutionError as e:
+ logger.error("=== %s ===", e)
+ wandb._sentry.exception(e)
+ sys.exit(e)
+ except asyncio.CancelledError:
+ sys.exit(0)
+ else:
+ try:
+ asyncio.run(
+ _launch_add(
+ api,
+ uri,
+ job,
+ config,
+ template_variables,
+ project,
+ entity,
+ queue,
+ resource,
+ entry_point,
+ name,
+ git_version,
+ docker_image,
+ project_queue,
+ resource_args,
+ build=build,
+ run_id=run_id,
+ repository=repository,
+ priority=priority,
+ )
+ )
+ except Exception as e:
+ wandb._sentry.exception(e)
+ raise e
+
+
+@cli.command(
+ context_settings=CONTEXT,
+ help="Run a W&B launch agent.",
+)
+@click.pass_context
+@click.option(
+ "--queue",
+ "-q",
+ "queues",
+ default=None,
+ multiple=True,
+ metavar="<queue(s)>",
+ help="The name of a queue for the agent to watch. Multiple -q flags supported.",
+)
+@click.option(
+ "--project",
+ "-p",
+ default=None,
+ help="""Name of the project which the agent will watch.
+ If passed in, will override the project value passed in using a config file.""",
+)
+@click.option(
+ "--entity",
+ "-e",
+ default=None,
+ help="The entity to use. Defaults to current logged-in user",
+)
+@click.option(
+ "--log-file",
+ "-l",
+ default=None,
+ help=(
+ "Destination for internal agent logs. Use - for stdout. "
+ "By default all agents logs will go to debug.log in your wandb/ "
+ "subdirectory or WANDB_DIR if set."
+ ),
+)
+@click.option(
+ "--max-jobs",
+ "-j",
+ default=None,
+ help="The maximum number of launch jobs this agent can run in parallel. Defaults to 1. Set to -1 for no upper limit",
+)
+@click.option(
+ "--config", "-c", default=None, help="path to the agent config yaml to use"
+)
+@click.option(
+ "--url",
+ "-u",
+ default=None,
+ hidden=True,
+ help="a wandb client registration URL, this is generated in the UI",
+)
+@display_error
+def launch_agent(
+ ctx,
+ project=None,
+ entity=None,
+ queues=None,
+ max_jobs=None,
+ config=None,
+ url=None,
+ log_file=None,
+):
+ logger.info(
+ f"=== Launch-agent called with kwargs {locals()} CLI Version: {wandb.__version__} ==="
+ )
+ if url is not None:
+ raise LaunchError(
+ "--url is not supported in this version, upgrade with: pip install -u wandb"
+ )
+
+ import wandb.sdk.launch._launch as _launch
+
+ if log_file is not None:
+ _launch.set_launch_logfile(log_file)
+
+ api = _get_cling_api()
+ wandb._sentry.configure_scope(process_context="launch_agent")
+ agent_config, api = _launch.resolve_agent_config(
+ entity, project, max_jobs, queues, config
+ )
+
+ if len(agent_config.get("queues")) == 0:
+ raise LaunchError(
+ "To launch an agent please specify a queue or a list of queues in the configuration file or cli."
+ )
+
+ launch_utils.check_logged_in(api)
+
+ wandb.termlog("Starting launch agent ✨")
+ try:
+ _launch.create_and_run_agent(api, agent_config)
+ except Exception as e:
+ wandb._sentry.exception(e)
+ raise e
+
+
+@cli.command(context_settings=CONTEXT, help="Run the W&B agent")
+@click.pass_context
+@click.option(
+ "--project",
+ "-p",
+ default=None,
+ help="""The name of the project where W&B runs created from the sweep are sent to. If the project is not specified, the run is sent to a project labeled 'Uncategorized'.""",
+)
+@click.option(
+ "--entity",
+ "-e",
+ default=None,
+ help="""The username or team name where you want to send W&B runs created by the sweep to. Ensure that the entity you specify already exists. If you don't specify an entity, the run will be sent to your default entity, which is usually your username.""",
+)
+@click.option(
+ "--count", default=None, type=int, help="The max number of runs for this agent."
+)
+@click.argument("sweep_id")
+@display_error
+def agent(ctx, project, entity, count, sweep_id):
+ api = _get_cling_api()
+ if api.api_key is None:
+ wandb.termlog("Login to W&B to use the sweep agent feature")
+ ctx.invoke(login, no_offline=True)
+ api = _get_cling_api(reset=True)
+
+ wandb.termlog("Starting wandb agent 🕵ï¸")
+ wandb_agent.agent(sweep_id, entity=entity, project=project, count=count)
+
+ # you can send local commands like so:
+ # agent_api.command({'type': 'run', 'program': 'train.py',
+ # 'args': ['--max_epochs=10']})
+
+
+@cli.command(
+ context_settings=RUN_CONTEXT, help="Run a W&B launch sweep scheduler (Experimental)"
+)
+@click.pass_context
+@click.argument("sweep_id")
+@display_error
+def scheduler(
+ ctx,
+ sweep_id,
+):
+ api = InternalApi()
+ if api.api_key is None:
+ wandb.termlog("Login to W&B to use the sweep scheduler feature")
+ ctx.invoke(login, no_offline=True)
+ api = InternalApi(reset=True)
+
+ wandb._sentry.configure_scope(process_context="sweep_scheduler")
+ wandb.termlog("Starting a Launch Scheduler 🚀")
+ from wandb.sdk.launch.sweeps import load_scheduler
+
+ # TODO(gst): remove this monstrosity
+ # Future-proofing hack to pull any kwargs that get passed in through the CLI
+ kwargs = {}
+ for i, _arg in enumerate(ctx.args):
+ if isinstance(_arg, str) and _arg.startswith("--"):
+ # convert input kwargs from hyphens to underscores
+ _key = _arg[2:].replace("-", "_")
+ _args = ctx.args[i + 1]
+ if str.isdigit(_args):
+ _args = int(_args)
+ kwargs[_key] = _args
+ try:
+ sweep_type = kwargs.get("sweep_type", "wandb")
+ _scheduler = load_scheduler(scheduler_type=sweep_type)(
+ api,
+ sweep_id=sweep_id,
+ **kwargs,
+ )
+ _scheduler.start()
+ except Exception as e:
+ wandb._sentry.exception(e)
+ raise e
+
+
+@cli.group(help="Commands for managing and viewing W&B jobs")
+def job() -> None:
+ pass
+
+
+@job.command("list", help="List jobs in a project")
+@click.option(
+ "--project",
+ "-p",
+ envvar=env.PROJECT,
+ help="The project you want to list jobs from.",
+)
+@click.option(
+ "--entity",
+ "-e",
+ default="models",
+ envvar=env.ENTITY,
+ help="The entity the jobs belong to",
+)
+def _list(project, entity):
+ wandb.termlog(f"Listing jobs in {entity}/{project}")
+ public_api = PublicApi()
+ try:
+ jobs = public_api.list_jobs(entity=entity, project=project)
+ except wandb.errors.CommError as e:
+ wandb.termerror(f"{e}")
+ return
+
+ if len(jobs) == 0:
+ wandb.termlog("No jobs found")
+ return
+
+ for job in jobs:
+ aliases = []
+ if len(job["edges"]) == 0:
+ # deleted?
+ continue
+
+ name = job["edges"][0]["node"]["artifactSequence"]["name"]
+ for version in job["edges"]:
+ aliases += [x["alias"] for x in version["node"]["aliases"]]
+
+ # only list the most recent 10 job versions
+ aliases_str = ",".join(aliases[::-1])
+ wandb.termlog(f"{name} -- versions ({len(aliases)}): {aliases_str}")
+
+
+@job.command(
+ help="Describe a launch job. Provide the launch job in the form of: entity/project/job-name:alias-or-version"
+)
+@click.argument("job")
+def describe(job):
+ public_api = PublicApi()
+ try:
+ job = public_api.job(name=job)
+ except wandb.errors.CommError as e:
+ wandb.termerror(f"{e}")
+ return
+
+ for key in job._job_info:
+ if key.startswith("_"):
+ continue
+ wandb.termlog(f"{key}: {job._job_info[key]}")
+
+
+@job.command(
+ no_args_is_help=True,
+)
+@click.option(
+ "--project",
+ "-p",
+ envvar=env.PROJECT,
+ help="The project you want to list jobs from.",
+)
+@click.option(
+ "--entity",
+ "-e",
+ envvar=env.ENTITY,
+ help="The entity the jobs belong to",
+)
+@click.option(
+ "--name",
+ "-n",
+ help="Name for the job",
+)
+@click.option(
+ "--description",
+ "-d",
+ help="Description for the job",
+)
+@click.option(
+ "--alias",
+ "-a",
+ "aliases",
+ help="Alias for the job",
+ multiple=True,
+ default=tuple(),
+)
+@click.option(
+ "--entry-point",
+ "-E",
+ "entrypoint",
+ help="Codepath to the main script, required for repo jobs",
+)
+@click.option(
+ "--git-hash",
+ "-g",
+ "git_hash",
+ type=str,
+ help="Hash to a specific git commit.",
+)
+@click.option(
+ "--runtime",
+ "-r",
+ type=str,
+ help="Python runtime to execute the job",
+)
+@click.argument(
+ "job_type",
+ type=click.Choice(("git", "code", "image")),
+)
+@click.argument("path")
+def create(
+ path,
+ project,
+ entity,
+ name,
+ job_type,
+ description,
+ aliases,
+ entrypoint,
+ git_hash,
+ runtime,
+):
+ """Create a job from a source, without a wandb run.
+
+ Jobs can be of three types, git, code, or image.
+
+ git: A git source, with an entrypoint either in the path or provided explicitly pointing to the main python executable.
+ code: A code path, containing a requirements.txt file.
+ image: A docker image.
+ """
+ from wandb.sdk.launch.create_job import _create_job
+
+ api = _get_cling_api()
+ wandb._sentry.configure_scope(process_context="job_create")
+
+ entity = entity or os.getenv("WANDB_ENTITY") or api.default_entity
+ if not entity:
+ wandb.termerror("No entity provided, use --entity or set WANDB_ENTITY")
+ return
+
+ project = project or os.getenv("WANDB_PROJECT")
+ if not project:
+ wandb.termerror("No project provided, use --project or set WANDB_PROJECT")
+ return
+
+ if entrypoint is None and job_type in ["git", "code"]:
+ wandb.termwarn(
+ f"No entrypoint provided for {job_type} job, defaulting to main.py"
+ )
+ entrypoint = "main.py"
+
+ artifact, action, aliases = _create_job(
+ api=api,
+ path=path,
+ entity=entity,
+ project=project,
+ name=name,
+ job_type=job_type,
+ description=description,
+ aliases=list(aliases),
+ entrypoint=entrypoint,
+ git_hash=git_hash,
+ runtime=runtime,
+ )
+ if not artifact:
+ wandb.termerror("Job creation failed")
+ return
+
+ artifact_path = f"{entity}/{project}/{artifact.name}"
+ msg = f"{action} job: {click.style(artifact_path, fg='yellow')}"
+ if len(aliases) == 1:
+ alias_str = click.style(aliases[0], fg="yellow")
+ msg += f", with alias: {alias_str}"
+ elif len(aliases) > 1:
+ alias_str = click.style(", ".join(aliases), fg="yellow")
+ msg += f", with aliases: {alias_str}"
+
+ wandb.termlog(msg)
+ web_url = util.app_url(api.settings().get("base_url"))
+ url = click.style(f"{web_url}/{entity}/{project}/jobs", underline=True)
+ wandb.termlog(f"View all jobs in project '{project}' here: {url}\n")
+
+
+@cli.command(context_settings=CONTEXT, help="Run the W&B local sweep controller")
+@click.option("--verbose", is_flag=True, default=False, help="Display verbose output")
+@click.argument("sweep_id")
+@display_error
+def controller(verbose, sweep_id):
+ click.echo("Starting wandb controller...")
+ from wandb import controller as wandb_controller
+
+ tuner = wandb_controller(sweep_id)
+ tuner.run(verbose=verbose)
+
+
+@cli.command(context_settings=RUN_CONTEXT, name="docker-run")
+@click.pass_context
+@click.argument("docker_run_args", nargs=-1)
+def docker_run(ctx, docker_run_args):
+ """Wrap `docker run` and adds WANDB_API_KEY and WANDB_DOCKER environment variables.
+
+ This will also set the runtime to nvidia if the nvidia-docker executable is present
+ on the system and --runtime wasn't set.
+
+ See `docker run --help` for more details.
+ """
+ api = InternalApi()
+ args = list(docker_run_args)
+ if len(args) > 0 and args[0] == "run":
+ args.pop(0)
+ if len([a for a in args if a.startswith("--runtime")]) == 0 and find_executable(
+ "nvidia-docker"
+ ):
+ args = ["--runtime", "nvidia"] + args
+ # TODO: image_from_docker_args uses heuristics to find the docker image arg, there are likely cases
+ # where this won't work
+ image = util.image_from_docker_args(args)
+ resolved_image = None
+ if image:
+ resolved_image = wandb.docker.image_id(image)
+ if resolved_image:
+ args = ["-e", "WANDB_DOCKER=%s" % resolved_image] + args
+ else:
+ wandb.termlog(
+ "Couldn't detect image argument, running command without the WANDB_DOCKER env variable"
+ )
+ if api.api_key:
+ args = ["-e", "WANDB_API_KEY=%s" % api.api_key] + args
+ else:
+ wandb.termlog(
+ "Not logged in, run `wandb login` from the host machine to enable result logging"
+ )
+ subprocess.call(["docker", "run"] + args)
+
+
+@cli.command(context_settings=RUN_CONTEXT)
+@click.pass_context
+@click.argument("docker_run_args", nargs=-1)
+@click.argument("docker_image", required=False)
+@click.option(
+ "--nvidia/--no-nvidia",
+ default=find_executable("nvidia-docker") is not None,
+ help="Use the nvidia runtime, defaults to nvidia if nvidia-docker is present",
+)
+@click.option(
+ "--digest", is_flag=True, default=False, help="Output the image digest and exit"
+)
+@click.option(
+ "--jupyter/--no-jupyter", default=False, help="Run jupyter lab in the container"
+)
+@click.option(
+ "--dir", default="/app", help="Which directory to mount the code in the container"
+)
+@click.option("--no-dir", is_flag=True, help="Don't mount the current directory")
+@click.option(
+ "--shell", default="/bin/bash", help="The shell to start the container with"
+)
+@click.option("--port", default="8888", help="The host port to bind jupyter on")
+@click.option("--cmd", help="The command to run in the container")
+@click.option(
+ "--no-tty", is_flag=True, default=False, help="Run the command without a tty"
+)
+@display_error
+def docker(
+ ctx,
+ docker_run_args,
+ docker_image,
+ nvidia,
+ digest,
+ jupyter,
+ dir,
+ no_dir,
+ shell,
+ port,
+ cmd,
+ no_tty,
+):
+ """Run your code in a docker container.
+
+ W&B docker lets you run your code in a docker image ensuring wandb is configured. It
+ adds the WANDB_DOCKER and WANDB_API_KEY environment variables to your container and
+ mounts the current directory in /app by default. You can pass additional args which
+ will be added to `docker run` before the image name is declared, we'll choose a
+ default image for you if one isn't passed:
+
+ ```sh
+ wandb docker -v /mnt/dataset:/app/data
+ wandb docker gcr.io/kubeflow-images-public/tensorflow-1.12.0-notebook-cpu:v0.4.0 --jupyter
+ wandb docker wandb/deepo:keras-gpu --no-tty --cmd "python train.py --epochs=5"
+ ```
+
+ By default, we override the entrypoint to check for the existence of wandb and
+ install it if not present. If you pass the --jupyter flag we will ensure jupyter is
+ installed and start jupyter lab on port 8888. If we detect nvidia-docker on your
+ system we will use the nvidia runtime. If you just want wandb to set environment
+ variable to an existing docker run command, see the wandb docker-run command.
+ """
+ api = InternalApi()
+ if not find_executable("docker"):
+ raise ClickException("Docker not installed, install it from https://docker.com")
+ args = list(docker_run_args)
+ image = docker_image or ""
+ # remove run for users used to nvidia-docker
+ if len(args) > 0 and args[0] == "run":
+ args.pop(0)
+ if image == "" and len(args) > 0:
+ image = args.pop(0)
+ # If the user adds docker args without specifying an image (should be rare)
+ if not util.docker_image_regex(image.split("@")[0]):
+ if image:
+ args = args + [image]
+ image = wandb.docker.default_image(gpu=nvidia)
+ subprocess.call(["docker", "pull", image])
+ _, repo_name, tag = wandb.docker.parse(image)
+
+ resolved_image = wandb.docker.image_id(image)
+ if resolved_image is None:
+ raise ClickException(
+ "Couldn't find image locally or in a registry, try running `docker pull %s`"
+ % image
+ )
+ if digest:
+ sys.stdout.write(resolved_image)
+ exit(0)
+
+ existing = wandb.docker.shell(["ps", "-f", "ancestor=%s" % resolved_image, "-q"])
+ if existing:
+ if click.confirm(
+ "Found running container with the same image, do you want to attach?"
+ ):
+ subprocess.call(["docker", "attach", existing.split("\n")[0]])
+ exit(0)
+ cwd = os.getcwd()
+ command = [
+ "docker",
+ "run",
+ "-e",
+ "LANG=C.UTF-8",
+ "-e",
+ "WANDB_DOCKER=%s" % resolved_image,
+ "--ipc=host",
+ "-v",
+ wandb.docker.entrypoint + ":/wandb-entrypoint.sh",
+ "--entrypoint",
+ "/wandb-entrypoint.sh",
+ ]
+ if nvidia:
+ command.extend(["--runtime", "nvidia"])
+ if not no_dir:
+ # TODO: We should default to the working directory if defined
+ command.extend(["-v", cwd + ":" + dir, "-w", dir])
+ if api.api_key:
+ command.extend(["-e", "WANDB_API_KEY=%s" % api.api_key])
+ else:
+ wandb.termlog(
+ "Couldn't find WANDB_API_KEY, run `wandb login` to enable streaming metrics"
+ )
+ if jupyter:
+ command.extend(["-e", "WANDB_ENSURE_JUPYTER=1", "-p", port + ":8888"])
+ no_tty = True
+ cmd = (
+ "jupyter lab --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --notebook-dir %s"
+ % dir
+ )
+ command.extend(args)
+ if no_tty:
+ command.extend([image, shell, "-c", cmd])
+ else:
+ if cmd:
+ command.extend(["-e", "WANDB_COMMAND=%s" % cmd])
+ command.extend(["-it", image, shell])
+ wandb.termlog("Launching docker container \U0001F6A2")
+ subprocess.call(command)
+
+
+@cli.command(
+ context_settings=RUN_CONTEXT,
+ help="Start a local W&B container (deprecated, see wandb server --help)",
+ hidden=True,
+)
+@click.pass_context
+@click.option("--port", "-p", default="8080", help="The host port to bind W&B local on")
+@click.option(
+ "--env", "-e", default=[], multiple=True, help="Env vars to pass to wandb/local"
+)
+@click.option(
+ "--daemon/--no-daemon", default=True, help="Run or don't run in daemon mode"
+)
+@click.option(
+ "--upgrade", is_flag=True, default=False, help="Upgrade to the most recent version"
+)
+@click.option(
+ "--edge", is_flag=True, default=False, help="Run the bleeding edge", hidden=True
+)
+@display_error
+def local(ctx, *args, **kwargs):
+ wandb.termwarn("`wandb local` has been replaced with `wandb server start`.")
+ ctx.invoke(start, *args, **kwargs)
+
+
+@cli.group(help="Commands for operating a local W&B server")
+def server():
+ pass
+
+
+@server.command(context_settings=RUN_CONTEXT, help="Start a local W&B server")
+@click.pass_context
+@click.option(
+ "--port", "-p", default="8080", help="The host port to bind W&B server on"
+)
+@click.option(
+ "--env", "-e", default=[], multiple=True, help="Env vars to pass to wandb/local"
+)
+@click.option(
+ "--daemon/--no-daemon", default=True, help="Run or don't run in daemon mode"
+)
+@click.option(
+ "--upgrade",
+ is_flag=True,
+ default=False,
+ help="Upgrade to the most recent version",
+ hidden=True,
+)
+@click.option(
+ "--edge", is_flag=True, default=False, help="Run the bleeding edge", hidden=True
+)
+@display_error
+def start(ctx, port, env, daemon, upgrade, edge):
+ api = InternalApi()
+ if not find_executable("docker"):
+ raise ClickException("Docker not installed, install it from https://docker.com")
+ local_image_sha = wandb.docker.image_id("wandb/local").split("wandb/local")[-1]
+ registry_image_sha = wandb.docker.image_id_from_registry("wandb/local").split(
+ "wandb/local"
+ )[-1]
+ if local_image_sha != registry_image_sha:
+ if upgrade:
+ subprocess.call(["docker", "pull", "wandb/local"])
+ else:
+ wandb.termlog(
+ "A new version of the W&B server is available, upgrade by calling `wandb server start --upgrade`"
+ )
+ running = subprocess.check_output(
+ ["docker", "ps", "--filter", "name=wandb-local", "--format", "{{.ID}}"]
+ )
+ if running != b"":
+ if upgrade:
+ subprocess.call(["docker", "stop", "wandb-local"])
+ else:
+ wandb.termerror(
+ "A container named wandb-local is already running, run `docker stop wandb-local` if you want to start a new instance"
+ )
+ exit(1)
+ image = "docker.pkg.github.com/wandb/core/local" if edge else "wandb/local"
+ username = getpass.getuser()
+ env_vars = ["-e", "LOCAL_USERNAME=%s" % username]
+ for e in env:
+ env_vars.append("-e")
+ env_vars.append(e)
+ command = [
+ "docker",
+ "run",
+ "--rm",
+ "-v",
+ "wandb:/vol",
+ "-p",
+ port + ":8080",
+ "--name",
+ "wandb-local",
+ ] + env_vars
+ host = f"http://localhost:{port}"
+ api.set_setting("base_url", host, globally=True, persist=True)
+ if daemon:
+ command += ["-d"]
+ command += [image]
+
+ # DEVNULL is only in py3
+ try:
+ from subprocess import DEVNULL
+ except ImportError:
+ DEVNULL = open(os.devnull, "wb") # noqa: N806
+ code = subprocess.call(command, stdout=DEVNULL)
+ if daemon:
+ if code != 0:
+ wandb.termerror(
+ "Failed to launch the W&B server container, see the above error."
+ )
+ exit(1)
+ else:
+ wandb.termlog("W&B server started at http://localhost:%s \U0001F680" % port)
+ wandb.termlog("You can stop the server by running `wandb server stop`")
+ if not api.api_key:
+ # Let the server start before potentially launching a browser
+ time.sleep(2)
+ ctx.invoke(login, host=host)
+
+
+@server.command(context_settings=RUN_CONTEXT, help="Stop a local W&B server")
+def stop():
+ if not find_executable("docker"):
+ raise ClickException("Docker not installed, install it from https://docker.com")
+ subprocess.call(["docker", "stop", "wandb-local"])
+
+
+@cli.group(help="Commands for interacting with artifacts")
+def artifact():
+ pass
+
+
+@artifact.command(context_settings=CONTEXT, help="Upload an artifact to wandb")
+@click.argument("path")
+@click.option(
+ "--name", "-n", help="The name of the artifact to push: project/artifact_name"
+)
+@click.option("--description", "-d", help="A description of this artifact")
+@click.option("--type", "-t", default="dataset", help="The type of the artifact")
+@click.option(
+ "--alias",
+ "-a",
+ default=["latest"],
+ multiple=True,
+ help="An alias to apply to this artifact",
+)
+@click.option("--id", "run_id", help="The run you want to upload to.")
+@click.option(
+ "--resume",
+ is_flag=True,
+ default=None,
+ help="Resume the last run from your current directory.",
+)
+@display_error
+def put(path, name, description, type, alias, run_id, resume):
+ if name is None:
+ name = os.path.basename(path)
+ public_api = PublicApi()
+ entity, project, artifact_name = public_api._parse_artifact_path(name)
+ if project is None:
+ project = click.prompt("Enter the name of the project you want to use")
+ # TODO: settings nightmare...
+ api = InternalApi()
+ api.set_setting("entity", entity)
+ api.set_setting("project", project)
+ artifact = wandb.Artifact(name=artifact_name, type=type, description=description)
+ artifact_path = f"{entity}/{project}/{artifact_name}:{alias[0]}"
+ if os.path.isdir(path):
+ wandb.termlog(f'Uploading directory {path} to: "{artifact_path}" ({type})')
+ artifact.add_dir(path)
+ elif os.path.isfile(path):
+ wandb.termlog(f'Uploading file {path} to: "{artifact_path}" ({type})')
+ artifact.add_file(path)
+ elif "://" in path:
+ wandb.termlog(
+ f'Logging reference artifact from {path} to: "{artifact_path}" ({type})'
+ )
+ artifact.add_reference(path)
+ else:
+ raise ClickException("Path argument must be a file or directory")
+
+ with wandb.init(
+ entity=entity,
+ project=project,
+ config={"path": path},
+ job_type="cli_put",
+ id=run_id,
+ resume=resume,
+ ) as run:
+ run.log_artifact(artifact, aliases=alias)
+ artifact.wait()
+
+ wandb.termlog(
+ "Artifact uploaded, use this artifact in a run by adding:\n", prefix=False
+ )
+ wandb.termlog(
+ f' artifact = run.use_artifact("{artifact.source_qualified_name}")\n',
+ prefix=False,
+ )
+
+
+@artifact.command(context_settings=CONTEXT, help="Download an artifact from wandb")
+@click.argument("path")
+@click.option("--root", help="The directory you want to download the artifact to")
+@click.option("--type", help="The type of artifact you are downloading")
+@display_error
+def get(path, root, type):
+ public_api = PublicApi()
+ entity, project, artifact_name = public_api._parse_artifact_path(path)
+ if project is None:
+ project = click.prompt("Enter the name of the project you want to use")
+
+ try:
+ artifact_parts = artifact_name.split(":")
+ if len(artifact_parts) > 1:
+ version = artifact_parts[1]
+ artifact_name = artifact_parts[0]
+ else:
+ version = "latest"
+ full_path = f"{entity}/{project}/{artifact_name}:{version}"
+ wandb.termlog(
+ "Downloading {type} artifact {full_path}".format(
+ type=type or "dataset", full_path=full_path
+ )
+ )
+ artifact = public_api.artifact(full_path, type=type)
+ path = artifact.download(root=root)
+ wandb.termlog("Artifact downloaded to %s" % path)
+ except ValueError:
+ raise ClickException("Unable to download artifact")
+
+
+@artifact.command(
+ context_settings=CONTEXT, help="List all artifacts in a wandb project"
+)
+@click.argument("path")
+@click.option("--type", "-t", help="The type of artifacts to list")
+@display_error
+def ls(path, type):
+ public_api = PublicApi()
+ if type is not None:
+ types = [public_api.artifact_type(type, path)]
+ else:
+ types = public_api.artifact_types(path)
+
+ for kind in types:
+ for collection in kind.collections():
+ versions = public_api.artifact_versions(
+ kind.type,
+ "/".join([kind.entity, kind.project, collection.name]),
+ per_page=1,
+ )
+ latest = next(versions)
+ print(
+ "{:<15s}{:<15s}{:>15s} {:<20s}".format(
+ kind.type,
+ latest.updated_at,
+ util.to_human_size(latest.size),
+ latest.name,
+ )
+ )
+
+
+@artifact.group(help="Commands for interacting with the artifact cache")
+def cache():
+ pass
+
+
+@cache.command(
+ context_settings=CONTEXT,
+ help="Clean up less frequently used files from the artifacts cache",
+)
+@click.argument("target_size")
+@click.option("--remove-temp/--no-remove-temp", default=False, help="Remove temp files")
+@display_error
+def cleanup(target_size, remove_temp):
+ target_size = util.from_human_size(target_size)
+ cache = get_artifact_file_cache()
+ reclaimed_bytes = cache.cleanup(target_size, remove_temp)
+ print(f"Reclaimed {util.to_human_size(reclaimed_bytes)} of space")
+
+
+@cli.command(context_settings=CONTEXT, help="Pull files from Weights & Biases")
+@click.argument("run", envvar=env.RUN_ID)
+@click.option(
+ "--project", "-p", envvar=env.PROJECT, help="The project you want to download."
+)
+@click.option(
+ "--entity",
+ "-e",
+ default="models",
+ envvar=env.ENTITY,
+ help="The entity to scope the listing to.",
+)
+@display_error
+def pull(run, project, entity):
+ api = InternalApi()
+ project, run = api.parse_slug(run, project=project)
+ urls = api.download_urls(project, run=run, entity=entity)
+ if len(urls) == 0:
+ raise ClickException("Run has no files")
+ click.echo(f"Downloading: {click.style(project, bold=True)}/{run}")
+
+ for name in urls:
+ if api.file_current(name, urls[name]["md5"]):
+ click.echo("File %s is up to date" % name)
+ else:
+ length, response = api.download_file(urls[name]["url"])
+ # TODO: I had to add this because some versions in CI broke click.progressbar
+ sys.stdout.write("File %s\r" % name)
+ dirname = os.path.dirname(name)
+ if dirname != "":
+ filesystem.mkdir_exists_ok(dirname)
+ with click.progressbar(
+ length=length,
+ label="File %s" % name,
+ fill_char=click.style("&", fg="green"),
+ ) as bar:
+ with open(name, "wb") as f:
+ for data in response.iter_content(chunk_size=4096):
+ f.write(data)
+ bar.update(len(data))
+
+
+@cli.command(
+ context_settings=CONTEXT, help="Restore code, config and docker state for a run"
+)
+@click.pass_context
+@click.argument("run", envvar=env.RUN_ID)
+@click.option("--no-git", is_flag=True, default=False, help="Don't restore git state")
+@click.option(
+ "--branch/--no-branch",
+ default=True,
+ help="Whether to create a branch or checkout detached",
+)
+@click.option(
+ "--project", "-p", envvar=env.PROJECT, help="The project you wish to upload to."
+)
+@click.option(
+ "--entity", "-e", envvar=env.ENTITY, help="The entity to scope the listing to."
+)
+@display_error
+def restore(ctx, run, no_git, branch, project, entity):
+ from wandb.old.core import wandb_dir
+
+ api = _get_cling_api()
+ if ":" in run:
+ if "/" in run:
+ entity, rest = run.split("/", 1)
+ else:
+ rest = run
+ project, run = rest.split(":", 1)
+ elif run.count("/") > 1:
+ entity, run = run.split("/", 1)
+
+ project, run = api.parse_slug(run, project=project)
+ commit, json_config, patch_content, metadata = api.run_config(
+ project, run=run, entity=entity
+ )
+ repo = metadata.get("git", {}).get("repo")
+ image = metadata.get("docker")
+ restore_message = (
+ """`wandb restore` needs to be run from the same git repository as the original run.
+Run `git clone %s` and restore from there or pass the --no-git flag."""
+ % repo
+ )
+ if no_git:
+ commit = None
+ elif not api.git.enabled:
+ if repo:
+ raise ClickException(restore_message)
+ elif image:
+ wandb.termlog(
+ "Original run has no git history. Just restoring config and docker"
+ )
+
+ if commit and api.git.enabled:
+ wandb.termlog(f"Fetching origin and finding commit: {commit}")
+ subprocess.check_call(["git", "fetch", "--all"])
+ try:
+ api.git.repo.commit(commit)
+ except ValueError:
+ wandb.termlog(f"Couldn't find original commit: {commit}")
+ commit = None
+ files = api.download_urls(project, run=run, entity=entity)
+ for filename in files:
+ if filename.startswith("upstream_diff_") and filename.endswith(
+ ".patch"
+ ):
+ commit = filename[len("upstream_diff_") : -len(".patch")]
+ try:
+ api.git.repo.commit(commit)
+ except ValueError:
+ commit = None
+ else:
+ break
+
+ if commit:
+ wandb.termlog(f"Falling back to upstream commit: {commit}")
+ patch_path, _ = api.download_write_file(files[filename])
+ else:
+ raise ClickException(restore_message)
+ else:
+ if patch_content:
+ patch_path = os.path.join(wandb_dir(), "diff.patch")
+ with open(patch_path, "w") as f:
+ f.write(patch_content)
+ else:
+ patch_path = None
+
+ branch_name = "wandb/%s" % run
+ if branch and branch_name not in api.git.repo.branches:
+ api.git.repo.git.checkout(commit, b=branch_name)
+ wandb.termlog("Created branch %s" % click.style(branch_name, bold=True))
+ elif branch:
+ wandb.termlog(
+ "Using existing branch, run `git branch -D %s` from master for a clean checkout"
+ % branch_name
+ )
+ api.git.repo.git.checkout(branch_name)
+ else:
+ wandb.termlog("Checking out %s in detached mode" % commit)
+ api.git.repo.git.checkout(commit)
+
+ if patch_path:
+ # we apply the patch from the repository root so git doesn't exclude
+ # things outside the current directory
+ root = api.git.root
+ patch_rel_path = os.path.relpath(patch_path, start=root)
+ # --reject is necessary or else this fails any time a binary file
+ # occurs in the diff
+ exit_code = subprocess.call(
+ ["git", "apply", "--reject", patch_rel_path], cwd=root
+ )
+ if exit_code == 0:
+ wandb.termlog("Applied patch")
+ else:
+ wandb.termerror(
+ "Failed to apply patch, try un-staging any un-committed changes"
+ )
+
+ filesystem.mkdir_exists_ok(wandb_dir())
+ config_path = os.path.join(wandb_dir(), "config.yaml")
+ config = Config()
+ for k, v in json_config.items():
+ if k not in ("_wandb", "wandb_version"):
+ config[k] = v
+ s = b"wandb_version: 1"
+ s += b"\n\n" + yaml.dump(
+ config._as_dict(),
+ Dumper=yaml.SafeDumper,
+ default_flow_style=False,
+ allow_unicode=True,
+ encoding="utf-8",
+ )
+ s = s.decode("utf-8")
+ with open(config_path, "w") as f:
+ f.write(s)
+
+ wandb.termlog("Restored config variables to %s" % config_path)
+ if image:
+ if not metadata["program"].startswith("<") and metadata.get("args") is not None:
+ # TODO: we may not want to default to python here.
+ runner = util.find_runner(metadata["program"]) or ["python"]
+ command = runner + [metadata["program"]] + metadata["args"]
+ cmd = " ".join(command)
+ else:
+ wandb.termlog("Couldn't find original command, just restoring environment")
+ cmd = None
+ wandb.termlog("Docker image found, attempting to start")
+ ctx.invoke(docker, docker_run_args=[image], cmd=cmd)
+
+ return commit, json_config, patch_content, repo, metadata
+
+
+@cli.command(context_settings=CONTEXT, help="Run any script with wandb", hidden=True)
+@click.pass_context
+@click.argument("program")
+@click.argument("args", nargs=-1)
+@display_error
+def magic(ctx, program, args):
+ def magic_run(cmd, globals, locals):
+ try:
+ exec(cmd, globals, locals)
+ finally:
+ pass
+
+ sys.argv[:] = args
+ sys.argv.insert(0, program)
+ sys.path.insert(0, os.path.dirname(program))
+ try:
+ with open(program, "rb") as fp:
+ code = compile(fp.read(), program, "exec")
+ except OSError:
+ click.echo(click.style("Could not launch program: %s" % program, fg="red"))
+ sys.exit(1)
+ globs = {
+ "__file__": program,
+ "__name__": "__main__",
+ "__package__": None,
+ "wandb_magic_install": magic_install,
+ }
+ prep = (
+ """
+import __main__
+__main__.__file__ = "%s"
+wandb_magic_install()
+"""
+ % program
+ )
+ magic_run(prep, globs, None)
+ magic_run(code, globs, None)
+
+
+@cli.command("online", help="Enable W&B sync")
+@display_error
+def online():
+ api = InternalApi()
+ try:
+ api.clear_setting("disabled", persist=True)
+ api.clear_setting("mode", persist=True)
+ except configparser.Error:
+ pass
+ click.echo(
+ "W&B online. Running your script from this directory will now sync to the cloud."
+ )
+
+
+@cli.command("offline", help="Disable W&B sync")
+@display_error
+def offline():
+ api = InternalApi()
+ try:
+ api.set_setting("disabled", "true", persist=True)
+ api.set_setting("mode", "offline", persist=True)
+ click.echo(
+ "W&B offline. Running your script from this directory will only write metadata locally. Use wandb disabled to completely turn off W&B."
+ )
+ except configparser.Error:
+ click.echo(
+ "Unable to write config, copy and paste the following in your terminal to turn off W&B:\nexport WANDB_MODE=offline"
+ )
+
+
+@cli.command("on", hidden=True)
+@click.pass_context
+@display_error
+def on(ctx):
+ ctx.invoke(online)
+
+
+@cli.command("off", hidden=True)
+@click.pass_context
+@display_error
+def off(ctx):
+ ctx.invoke(offline)
+
+
+@cli.command("status", help="Show configuration settings")
+@click.option(
+ "--settings/--no-settings", help="Show the current settings", default=True
+)
+def status(settings):
+ api = _get_cling_api()
+ if settings:
+ click.echo(click.style("Current Settings", bold=True))
+ settings = api.settings()
+ click.echo(
+ json.dumps(settings, sort_keys=True, indent=2, separators=(",", ": "))
+ )
+
+
+@cli.command("disabled", help="Disable W&B.")
+@click.option(
+ "--service",
+ is_flag=True,
+ show_default=True,
+ default=True,
+ help="Disable W&B service",
+)
+def disabled(service):
+ api = InternalApi()
+ try:
+ api.set_setting("mode", "disabled", persist=True)
+ click.echo("W&B disabled.")
+ os.environ[wandb.env._DISABLE_SERVICE] = str(service)
+ except configparser.Error:
+ click.echo(
+ "Unable to write config, copy and paste the following in your terminal to turn off W&B:\nexport WANDB_MODE=disabled"
+ )
+
+
+@cli.command("enabled", help="Enable W&B.")
+@click.option(
+ "--service",
+ is_flag=True,
+ show_default=True,
+ default=True,
+ help="Enable W&B service",
+)
+def enabled(service):
+ api = InternalApi()
+ try:
+ api.set_setting("mode", "online", persist=True)
+ click.echo("W&B enabled.")
+ os.environ[wandb.env._DISABLE_SERVICE] = str(not service)
+ except configparser.Error:
+ click.echo(
+ "Unable to write config, copy and paste the following in your terminal to turn on W&B:\nexport WANDB_MODE=online"
+ )
+
+
+@cli.command("gc", hidden=True, context_settings={"ignore_unknown_options": True})
+@click.argument("args", nargs=-1)
+def gc(args):
+ click.echo(
+ "`wandb gc` command has been removed. Use `wandb sync --clean` to clean up synced runs."
+ )
+
+
+@cli.command(context_settings=CONTEXT, help="Verify your local instance")
+@click.option("--host", default=None, help="Test a specific instance of W&B")
+def verify(host):
+ # TODO: (kdg) Build this all into a WandbVerify object, and clean this up.
+ os.environ["WANDB_SILENT"] = "true"
+ os.environ["WANDB_PROJECT"] = "verify"
+ api = _get_cling_api()
+ reinit = False
+ if host is None:
+ host = api.settings("base_url")
+ print(f"Default host selected: {host}")
+ # if the given host does not match the default host, re-run init
+ elif host != api.settings("base_url"):
+ reinit = True
+
+ tmp_dir = tempfile.mkdtemp()
+ print(
+ "Find detailed logs for this test at: {}".format(os.path.join(tmp_dir, "wandb"))
+ )
+ os.chdir(tmp_dir)
+ os.environ["WANDB_BASE_URL"] = host
+ wandb.login(host=host)
+ if reinit:
+ api = _get_cling_api(reset=True)
+ if not wandb_verify.check_host(host):
+ sys.exit(1)
+ if not wandb_verify.check_logged_in(api, host):
+ sys.exit(1)
+ url_success, url = wandb_verify.check_graphql_put(api, host)
+ large_post_success = wandb_verify.check_large_post()
+ wandb_verify.check_secure_requests(
+ api.settings("base_url"),
+ "Checking requests to base url",
+ "Connections are not made over https. SSL required for secure communications.",
+ )
+ if url:
+ wandb_verify.check_secure_requests(
+ url,
+ "Checking requests made over signed URLs",
+ "Signed URL requests not made over https. SSL is required for secure communications.",
+ )
+ wandb_verify.check_cors_configuration(url, host)
+ wandb_verify.check_wandb_version(api)
+ check_run_success = wandb_verify.check_run(api)
+ check_artifacts_success = wandb_verify.check_artifacts()
+ if not (
+ check_artifacts_success
+ and check_run_success
+ and large_post_success
+ and url_success
+ ):
+ sys.exit(1)
+
+
+@cli.group("import", help="Commands for importing data from other systems")
+def importer():
+ pass
+
+
+@importer.command("mlflow", help="Import from MLFlow")
+@click.option("--mlflow-tracking-uri", help="MLFlow Tracking URI")
+@click.option(
+ "--target-entity", required=True, help="Override default entity to import data into"
+)
+@click.option(
+ "--target-project",
+ required=True,
+ help="Override default project to import data into",
+)
+def mlflow(mlflow_tracking_uri, target_entity, target_project):
+ from wandb.apis.importers import MlflowImporter
+
+ importer = MlflowImporter(mlflow_tracking_uri=mlflow_tracking_uri)
+ overrides = {
+ "entity": target_entity,
+ "project": target_project,
+ }
+
+ importer.import_all_parallel(overrides=overrides)
diff --git a/wandb/data_types.py b/wandb/data_types.py
new file mode 100644
index 0000000000000000000000000000000000000000..2c09035c956eea43cfd49591a5260c75c027fe35
--- /dev/null
+++ b/wandb/data_types.py
@@ -0,0 +1,2136 @@
+"""This module defines data types for logging rich, interactive visualizations to W&B.
+
+Data types include common media types, like images, audio, and videos,
+flexible containers for information, like tables and HTML, and more.
+
+For more on logging media, see [our guide](https://docs.wandb.com/guides/track/log/media)
+
+For more on logging structured data for interactive dataset and model analysis,
+see [our guide to W&B Tables](https://docs.wandb.com/guides/data-vis).
+
+All of these special data types are subclasses of WBValue. All the data types
+serialize to JSON, since that is what wandb uses to save the objects locally
+and upload them to the W&B server.
+"""
+
+import base64
+import binascii
+import codecs
+import datetime
+import hashlib
+import json
+import logging
+import os
+import pprint
+from decimal import Decimal
+from typing import Optional
+
+import wandb
+from wandb import util
+from wandb.sdk.lib import filesystem
+
+from .sdk.data_types import _dtypes
+from .sdk.data_types._private import MEDIA_TMP
+from .sdk.data_types.base_types.media import (
+ BatchableMedia,
+ Media,
+ _numpy_arrays_to_lists,
+)
+from .sdk.data_types.base_types.wb_value import WBValue
+from .sdk.data_types.helper_types.bounding_boxes_2d import BoundingBoxes2D
+from .sdk.data_types.helper_types.classes import Classes
+from .sdk.data_types.helper_types.image_mask import ImageMask
+from .sdk.data_types.histogram import Histogram
+from .sdk.data_types.html import Html
+from .sdk.data_types.image import Image
+from .sdk.data_types.molecule import Molecule
+from .sdk.data_types.object_3d import Object3D
+from .sdk.data_types.plotly import Plotly
+from .sdk.data_types.saved_model import _SavedModel
+from .sdk.data_types.trace_tree import WBTraceTree
+from .sdk.data_types.video import Video
+from .sdk.lib import runid
+
+# Note: we are importing everything from the sdk/data_types to maintain a namespace for now.
+# Once we fully type this file and move it all into sdk, then we will need to clean up the
+# other internal imports
+
+__all__ = [
+ # Untyped Exports
+ "Audio",
+ "Table",
+ "Bokeh",
+ # Typed Exports
+ "Histogram",
+ "Html",
+ "Image",
+ "Molecule",
+ "Object3D",
+ "Plotly",
+ "Video",
+ "WBTraceTree",
+ "_SavedModel",
+ # Typed Legacy Exports (I'd like to remove these)
+ "ImageMask",
+ "BoundingBoxes2D",
+ "Classes",
+]
+
+
+class _TableLinkMixin:
+ def set_table(self, table):
+ self._table = table
+
+
+class _TableKey(str, _TableLinkMixin):
+ def set_table(self, table, col_name):
+ assert col_name in table.columns
+ self._table = table
+ self._col_name = col_name
+
+
+class _TableIndex(int, _TableLinkMixin):
+ def get_row(self):
+ row = {}
+ if self._table:
+ row = {
+ c: self._table.data[self][i] for i, c in enumerate(self._table.columns)
+ }
+
+ return row
+
+
+def _json_helper(val, artifact):
+ if isinstance(val, WBValue):
+ return val.to_json(artifact)
+ elif val.__class__ == dict:
+ res = {}
+ for key in val:
+ res[key] = _json_helper(val[key], artifact)
+ return res
+
+ if hasattr(val, "tolist"):
+ py_val = val.tolist()
+ if val.__class__.__name__ == "datetime64" and isinstance(py_val, int):
+ # when numpy datetime64 .tolist() returns an int, it is nanoseconds.
+ # need to convert to milliseconds
+ return _json_helper(py_val / int(1e6), artifact)
+ return _json_helper(py_val, artifact)
+ elif hasattr(val, "item"):
+ return _json_helper(val.item(), artifact)
+
+ if isinstance(val, datetime.datetime):
+ if val.tzinfo is None:
+ val = datetime.datetime(
+ val.year,
+ val.month,
+ val.day,
+ val.hour,
+ val.minute,
+ val.second,
+ val.microsecond,
+ tzinfo=datetime.timezone.utc,
+ )
+ return int(val.timestamp() * 1000)
+ elif isinstance(val, datetime.date):
+ return int(
+ datetime.datetime(
+ val.year, val.month, val.day, tzinfo=datetime.timezone.utc
+ ).timestamp()
+ * 1000
+ )
+ elif isinstance(val, (list, tuple)):
+ return [_json_helper(i, artifact) for i in val]
+ elif isinstance(val, Decimal):
+ return float(val)
+ else:
+ return util.json_friendly(val)[0]
+
+
+class Table(Media):
+ """The Table class used to display and analyze tabular data.
+
+ Unlike traditional spreadsheets, Tables support numerous types of data:
+ scalar values, strings, numpy arrays, and most subclasses of `wandb.data_types.Media`.
+ This means you can embed `Images`, `Video`, `Audio`, and other sorts of rich, annotated media
+ directly in Tables, alongside other traditional scalar values.
+
+ This class is the primary class used to generate the Table Visualizer
+ in the UI: https://docs.wandb.ai/guides/data-vis/tables.
+
+ Tables can be constructed with initial data using the `data` or
+ `dataframe` parameters:
+ <!--yeadoc-test:table-construct-dataframe-->
+ ```python
+ import pandas as pd
+ import wandb
+
+ data = {"users": ["geoff", "juergen", "ada"], "feature_01": [1, 117, 42]}
+ df = pd.DataFrame(data)
+
+ tbl = wandb.Table(data=df)
+ assert all(tbl.get_column("users") == df["users"])
+ assert all(tbl.get_column("feature_01") == df["feature_01"])
+ ```
+
+ Additionally, users can add data to Tables incrementally by using the
+ `add_data`, `add_column`, and `add_computed_column` functions for
+ adding rows, columns, and columns computed from data in other columns, respectively:
+ <!--yeadoc-test:table-construct-rowwise-->
+ ```python
+ import wandb
+
+ tbl = wandb.Table(columns=["user"])
+
+ users = ["geoff", "juergen", "ada"]
+
+ [tbl.add_data(user) for user in users]
+ assert tbl.get_column("user") == users
+
+
+ def get_user_name_length(index, row):
+ return {"feature_01": len(row["user"])}
+
+
+ tbl.add_computed_columns(get_user_name_length)
+ assert tbl.get_column("feature_01") == [5, 7, 3]
+ ```
+
+ Tables can be logged directly to runs using `run.log({"my_table": table})`
+ or added to artifacts using `artifact.add(table, "my_table")`:
+ <!--yeadoc-test:table-logging-direct-->
+ ```python
+ import numpy as np
+ import wandb
+
+ wandb.init()
+
+ tbl = wandb.Table(columns=["image", "label"])
+
+ images = np.random.randint(0, 255, [2, 100, 100, 3], dtype=np.uint8)
+ labels = ["panda", "gibbon"]
+ [tbl.add_data(wandb.Image(image), label) for image, label in zip(images, labels)]
+
+ wandb.log({"classifier_out": tbl})
+ ```
+
+ Tables added directly to runs as above will produce a corresponding Table Visualizer in the
+ Workspace which can be used for further analysis and exporting to reports.
+
+ Tables added to artifacts can be viewed in the Artifact Tab and will render
+ an equivalent Table Visualizer directly in the artifact browser.
+
+ Tables expect each value for a column to be of the same type. By default, a column supports
+ optional values, but not mixed values. If you absolutely need to mix types,
+ you can enable the `allow_mixed_types` flag which will disable type checking
+ on the data. This will result in some table analytics features being disabled
+ due to lack of consistent typing.
+
+ Arguments:
+ columns: (List[str]) Names of the columns in the table.
+ Defaults to ["Input", "Output", "Expected"].
+ data: (List[List[any]]) 2D row-oriented array of values.
+ dataframe: (pandas.DataFrame) DataFrame object used to create the table.
+ When set, `data` and `columns` arguments are ignored.
+ optional: (Union[bool,List[bool]]) Determines if `None` values are allowed. Default to True
+ - If a singular bool value, then the optionality is enforced for all
+ columns specified at construction time
+ - If a list of bool values, then the optionality is applied to each
+ column - should be the same length as `columns`
+ applies to all columns. A list of bool values applies to each respective column.
+ allow_mixed_types: (bool) Determines if columns are allowed to have mixed types
+ (disables type validation). Defaults to False
+ """
+
+ MAX_ROWS = 10000
+ MAX_ARTIFACT_ROWS = 200000
+ _MAX_EMBEDDING_DIMENSIONS = 150
+ _log_type = "table"
+
+ def __init__(
+ self,
+ columns=None,
+ data=None,
+ rows=None,
+ dataframe=None,
+ dtype=None,
+ optional=True,
+ allow_mixed_types=False,
+ ):
+ """Initialize a Table object.
+
+ Rows is kept for legacy reasons, we use data to mimic the Pandas api.
+ """
+ super().__init__()
+ self._pk_col = None
+ self._fk_cols = set()
+ if allow_mixed_types:
+ dtype = _dtypes.AnyType
+
+ # This is kept for legacy reasons (tss: personally, I think we should remove this)
+ if columns is None:
+ columns = ["Input", "Output", "Expected"]
+
+ # Explicit dataframe option
+ if dataframe is not None:
+ self._init_from_dataframe(dataframe, columns, optional, dtype)
+ else:
+ # Expected pattern
+ if data is not None:
+ if util.is_numpy_array(data):
+ self._init_from_ndarray(data, columns, optional, dtype)
+ elif util.is_pandas_data_frame(data):
+ self._init_from_dataframe(data, columns, optional, dtype)
+ else:
+ self._init_from_list(data, columns, optional, dtype)
+
+ # legacy
+ elif rows is not None:
+ self._init_from_list(rows, columns, optional, dtype)
+
+ # Default empty case
+ else:
+ self._init_from_list([], columns, optional, dtype)
+
+ @staticmethod
+ def _assert_valid_columns(columns):
+ valid_col_types = [str, int]
+ assert isinstance(columns, list), "columns argument expects a `list` object"
+ assert len(columns) == 0 or all(
+ [type(col) in valid_col_types for col in columns]
+ ), "columns argument expects list of strings or ints"
+
+ def _init_from_list(self, data, columns, optional=True, dtype=None):
+ assert isinstance(data, list), "data argument expects a `list` object"
+ self.data = []
+ self._assert_valid_columns(columns)
+ self.columns = columns
+ self._make_column_types(dtype, optional)
+ for row in data:
+ self.add_data(*row)
+
+ def _init_from_ndarray(self, ndarray, columns, optional=True, dtype=None):
+ assert util.is_numpy_array(
+ ndarray
+ ), "ndarray argument expects a `numpy.ndarray` object"
+ self.data = []
+ self._assert_valid_columns(columns)
+ self.columns = columns
+ self._make_column_types(dtype, optional)
+ for row in ndarray:
+ self.add_data(*row)
+
+ def _init_from_dataframe(self, dataframe, columns, optional=True, dtype=None):
+ assert util.is_pandas_data_frame(
+ dataframe
+ ), "dataframe argument expects a `pandas.core.frame.DataFrame` object"
+ self.data = []
+ columns = list(dataframe.columns)
+ self._assert_valid_columns(columns)
+ self.columns = columns
+ self._make_column_types(dtype, optional)
+ for row in range(len(dataframe)):
+ self.add_data(*tuple(dataframe[col].values[row] for col in self.columns))
+
+ def _make_column_types(self, dtype=None, optional=True):
+ if dtype is None:
+ dtype = _dtypes.UnknownType()
+
+ if optional.__class__ != list:
+ optional = [optional for _ in range(len(self.columns))]
+
+ if dtype.__class__ != list:
+ dtype = [dtype for _ in range(len(self.columns))]
+
+ self._column_types = _dtypes.TypedDictType({})
+ for col_name, opt, dt in zip(self.columns, optional, dtype):
+ self.cast(col_name, dt, opt)
+
+ def cast(self, col_name, dtype, optional=False):
+ """Cast a column to a specific type.
+
+ Arguments:
+ col_name: (str) - name of the column to cast
+ dtype: (class, wandb.wandb_sdk.interface._dtypes.Type, any) - the target dtype. Can be one of
+ normal python class, internal WB type, or an example object (e.g. an instance of wandb.Image or wandb.Classes)
+ optional: (bool) - if the column should allow Nones
+ """
+ assert col_name in self.columns
+
+ wbtype = _dtypes.TypeRegistry.type_from_dtype(dtype)
+
+ if optional:
+ wbtype = _dtypes.OptionalType(wbtype)
+
+ # Cast each value in the row, raising an error if there are invalid entries.
+ col_ndx = self.columns.index(col_name)
+ for row in self.data:
+ result_type = wbtype.assign(row[col_ndx])
+ if isinstance(result_type, _dtypes.InvalidType):
+ raise TypeError(
+ "Existing data {}, of type {} cannot be cast to {}".format(
+ row[col_ndx],
+ _dtypes.TypeRegistry.type_of(row[col_ndx]),
+ wbtype,
+ )
+ )
+ wbtype = result_type
+
+ # Assert valid options
+ is_pk = isinstance(wbtype, _PrimaryKeyType)
+ is_fk = isinstance(wbtype, _ForeignKeyType)
+ is_fi = isinstance(wbtype, _ForeignIndexType)
+ if is_pk or is_fk or is_fi:
+ assert (
+ not optional
+ ), "Primary keys, foreign keys, and foreign indexes cannot be optional"
+
+ if (is_fk or is_fk) and id(wbtype.params["table"]) == id(self):
+ raise AssertionError("Cannot set a foreign table reference to same table")
+
+ if is_pk:
+ assert (
+ self._pk_col is None
+ ), "Cannot have multiple primary keys - {} is already set as the primary key.".format(
+ self._pk_col
+ )
+
+ # Update the column type
+ self._column_types.params["type_map"][col_name] = wbtype
+
+ # Wrap the data if needed
+ self._update_keys()
+ return wbtype
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def _eq_debug(self, other, should_assert=False):
+ eq = isinstance(other, Table)
+ assert not should_assert or eq, "Found type {}, expected {}".format(
+ other.__class__, Table
+ )
+ eq = eq and len(self.data) == len(other.data)
+ assert not should_assert or eq, "Found {} rows, expected {}".format(
+ len(other.data), len(self.data)
+ )
+ eq = eq and self.columns == other.columns
+ assert not should_assert or eq, "Found columns {}, expected {}".format(
+ other.columns, self.columns
+ )
+ eq = eq and self._column_types == other._column_types
+ assert (
+ not should_assert or eq
+ ), "Found column type {}, expected column type {}".format(
+ other._column_types, self._column_types
+ )
+ if eq:
+ for row_ndx in range(len(self.data)):
+ for col_ndx in range(len(self.data[row_ndx])):
+ _eq = self.data[row_ndx][col_ndx] == other.data[row_ndx][col_ndx]
+ # equal if all are equal
+ if util.is_numpy_array(_eq):
+ _eq = ((_eq * -1) + 1).sum() == 0
+ eq = eq and _eq
+ assert (
+ not should_assert or eq
+ ), "Unequal data at row_ndx {} col_ndx {}: found {}, expected {}".format(
+ row_ndx,
+ col_ndx,
+ other.data[row_ndx][col_ndx],
+ self.data[row_ndx][col_ndx],
+ )
+ if not eq:
+ return eq
+ return eq
+
+ def __eq__(self, other):
+ return self._eq_debug(other)
+
+ def add_row(self, *row):
+ """Deprecated: use add_data instead."""
+ logging.warning("add_row is deprecated, use add_data")
+ self.add_data(*row)
+
+ def add_data(self, *data):
+ """Add a row of data to the table.
+
+ Argument length should match column length.
+ """
+ if len(data) != len(self.columns):
+ raise ValueError(
+ "This table expects {} columns: {}, found {}".format(
+ len(self.columns), self.columns, len(data)
+ )
+ )
+
+ # Special case to pre-emptively cast a column as a key.
+ # Needed as String.assign(Key) is invalid
+ for ndx, item in enumerate(data):
+ if isinstance(item, _TableLinkMixin):
+ self.cast(
+ self.columns[ndx],
+ _dtypes.TypeRegistry.type_of(item),
+ optional=False,
+ )
+
+ # Update the table's column types
+ result_type = self._get_updated_result_type(data)
+ self._column_types = result_type
+
+ # rows need to be mutable
+ if isinstance(data, tuple):
+ data = list(data)
+ # Add the new data
+ self.data.append(data)
+
+ # Update the wrapper values if needed
+ self._update_keys(force_last=True)
+
+ def _get_updated_result_type(self, row):
+ """Return an updated result type based on incoming row.
+
+ Raises:
+ TypeError: if the assignment is invalid.
+ """
+ incoming_row_dict = {
+ col_key: row[ndx] for ndx, col_key in enumerate(self.columns)
+ }
+ current_type = self._column_types
+ result_type = current_type.assign(incoming_row_dict)
+ if isinstance(result_type, _dtypes.InvalidType):
+ raise TypeError(
+ "Data row contained incompatible types:\n{}".format(
+ current_type.explain(incoming_row_dict)
+ )
+ )
+ return result_type
+
+ def _to_table_json(self, max_rows=None, warn=True):
+ # separate this method for easier testing
+ if max_rows is None:
+ max_rows = Table.MAX_ROWS
+ n_rows = len(self.data)
+ if n_rows > max_rows and warn:
+ if wandb.run and (
+ wandb.run.settings.table_raise_on_max_row_limit_exceeded
+ or wandb.run.settings.strict
+ ):
+ raise ValueError(
+ f"Table row limit exceeded: table has {n_rows} rows, limit is {max_rows}. "
+ f"To increase the maximum number of allowed rows in a wandb.Table, override "
+ f"the limit with `wandb.Table.MAX_ARTIFACT_ROWS = X` and try again. Note: "
+ f"this may cause slower queries in the W&B UI."
+ )
+ logging.warning("Truncating wandb.Table object to %i rows." % max_rows)
+ return {"columns": self.columns, "data": self.data[:max_rows]}
+
+ def bind_to_run(self, *args, **kwargs):
+ # We set `warn=False` since Tables will now always be logged to both
+ # files and artifacts. The file limit will never practically matter and
+ # this code path will be ultimately removed. The 10k limit warning confuses
+ # users given that we publicly say 200k is the limit.
+ data = self._to_table_json(warn=False)
+ tmp_path = os.path.join(MEDIA_TMP.name, runid.generate_id() + ".table.json")
+ data = _numpy_arrays_to_lists(data)
+ with codecs.open(tmp_path, "w", encoding="utf-8") as fp:
+ util.json_dump_safer(data, fp)
+ self._set_file(tmp_path, is_tmp=True, extension=".table.json")
+ super().bind_to_run(*args, **kwargs)
+
+ @classmethod
+ def get_media_subdir(cls):
+ return os.path.join("media", "table")
+
+ @classmethod
+ def from_json(cls, json_obj, source_artifact):
+ data = []
+ column_types = None
+ np_deserialized_columns = {}
+ timestamp_column_indices = set()
+ if json_obj.get("column_types") is not None:
+ column_types = _dtypes.TypeRegistry.type_from_dict(
+ json_obj["column_types"], source_artifact
+ )
+ for col_name in column_types.params["type_map"]:
+ col_type = column_types.params["type_map"][col_name]
+ ndarray_type = None
+ if isinstance(col_type, _dtypes.NDArrayType):
+ ndarray_type = col_type
+ elif isinstance(col_type, _dtypes.UnionType):
+ for t in col_type.params["allowed_types"]:
+ if isinstance(t, _dtypes.NDArrayType):
+ ndarray_type = t
+ elif isinstance(t, _dtypes.TimestampType):
+ timestamp_column_indices.add(
+ json_obj["columns"].index(col_name)
+ )
+
+ elif isinstance(col_type, _dtypes.TimestampType):
+ timestamp_column_indices.add(json_obj["columns"].index(col_name))
+
+ if (
+ ndarray_type is not None
+ and ndarray_type._get_serialization_path() is not None
+ ):
+ serialization_path = ndarray_type._get_serialization_path()
+ np = util.get_module(
+ "numpy",
+ required="Deserializing numpy columns requires numpy to be installed",
+ )
+ deserialized = np.load(
+ source_artifact.get_entry(serialization_path["path"]).download()
+ )
+ np_deserialized_columns[
+ json_obj["columns"].index(col_name)
+ ] = deserialized[serialization_path["key"]]
+ ndarray_type._clear_serialization_path()
+
+ for r_ndx, row in enumerate(json_obj["data"]):
+ row_data = []
+ for c_ndx, item in enumerate(row):
+ cell = item
+ if c_ndx in timestamp_column_indices and isinstance(item, (int, float)):
+ cell = datetime.datetime.fromtimestamp(
+ item / 1000, tz=datetime.timezone.utc
+ )
+ elif c_ndx in np_deserialized_columns:
+ cell = np_deserialized_columns[c_ndx][r_ndx]
+ elif isinstance(item, dict) and "_type" in item:
+ obj = WBValue.init_from_json(item, source_artifact)
+ if obj is not None:
+ cell = obj
+ row_data.append(cell)
+ data.append(row_data)
+
+ # construct Table with dtypes for each column if type information exists
+ dtypes = None
+ if column_types is not None:
+ dtypes = [
+ column_types.params["type_map"][str(col)] for col in json_obj["columns"]
+ ]
+
+ new_obj = cls(columns=json_obj["columns"], data=data, dtype=dtypes)
+
+ if column_types is not None:
+ new_obj._column_types = column_types
+
+ new_obj._update_keys()
+ return new_obj
+
+ def to_json(self, run_or_artifact):
+ json_dict = super().to_json(run_or_artifact)
+
+ if isinstance(run_or_artifact, wandb.wandb_sdk.wandb_run.Run):
+ json_dict.update(
+ {
+ "_type": "table-file",
+ "ncols": len(self.columns),
+ "nrows": len(self.data),
+ }
+ )
+
+ elif isinstance(run_or_artifact, wandb.Artifact):
+ artifact = run_or_artifact
+ mapped_data = []
+ data = self._to_table_json(Table.MAX_ARTIFACT_ROWS)["data"]
+
+ ndarray_col_ndxs = set()
+ for col_ndx, col_name in enumerate(self.columns):
+ col_type = self._column_types.params["type_map"][col_name]
+ ndarray_type = None
+ if isinstance(col_type, _dtypes.NDArrayType):
+ ndarray_type = col_type
+ elif isinstance(col_type, _dtypes.UnionType):
+ for t in col_type.params["allowed_types"]:
+ if isinstance(t, _dtypes.NDArrayType):
+ ndarray_type = t
+
+ # Do not serialize 1d arrays - these are likely embeddings and
+ # will not have the same cost as higher dimensional arrays
+ is_1d_array = (
+ ndarray_type is not None
+ and "shape" in ndarray_type._params
+ and isinstance(ndarray_type._params["shape"], list)
+ and len(ndarray_type._params["shape"]) == 1
+ and ndarray_type._params["shape"][0]
+ <= self._MAX_EMBEDDING_DIMENSIONS
+ )
+ if is_1d_array:
+ self._column_types.params["type_map"][col_name] = _dtypes.ListType(
+ _dtypes.NumberType, ndarray_type._params["shape"][0]
+ )
+ elif ndarray_type is not None:
+ np = util.get_module(
+ "numpy",
+ required="Serializing numpy requires numpy to be installed",
+ )
+ file_name = f"{str(col_name)}_{runid.generate_id()}.npz"
+ npz_file_name = os.path.join(MEDIA_TMP.name, file_name)
+ np.savez_compressed(
+ npz_file_name,
+ **{
+ str(col_name): self.get_column(col_name, convert_to="numpy")
+ },
+ )
+ entry = artifact.add_file(
+ npz_file_name, "media/serialized_data/" + file_name, is_tmp=True
+ )
+ ndarray_type._set_serialization_path(entry.path, str(col_name))
+ ndarray_col_ndxs.add(col_ndx)
+
+ for row in data:
+ mapped_row = []
+ for ndx, v in enumerate(row):
+ if ndx in ndarray_col_ndxs:
+ mapped_row.append(None)
+ else:
+ mapped_row.append(_json_helper(v, artifact))
+ mapped_data.append(mapped_row)
+
+ json_dict.update(
+ {
+ "_type": Table._log_type,
+ "columns": self.columns,
+ "data": mapped_data,
+ "ncols": len(self.columns),
+ "nrows": len(mapped_data),
+ "column_types": self._column_types.to_json(artifact),
+ }
+ )
+ else:
+ raise ValueError("to_json accepts wandb_run.Run or wandb_artifact.Artifact")
+
+ return json_dict
+
+ def iterrows(self):
+ """Iterate over rows as (ndx, row).
+
+ Yields:
+ ------
+ index : int
+ The index of the row. Using this value in other WandB tables
+ will automatically build a relationship between the tables
+ row : List[any]
+ The data of the row.
+ """
+ for ndx in range(len(self.data)):
+ index = _TableIndex(ndx)
+ index.set_table(self)
+ yield index, self.data[ndx]
+
+ def set_pk(self, col_name):
+ # TODO: Docs
+ assert col_name in self.columns
+ self.cast(col_name, _PrimaryKeyType())
+
+ def set_fk(self, col_name, table, table_col):
+ # TODO: Docs
+ assert col_name in self.columns
+ assert col_name != self._pk_col
+ self.cast(col_name, _ForeignKeyType(table, table_col))
+
+ def _update_keys(self, force_last=False):
+ """Update the known key-like columns based on the current column types.
+
+ If the state has been updated since the last update, we wrap the data
+ appropriately in the Key classes.
+
+ Arguments:
+ force_last: (bool) Determines wrapping the last column of data even if there
+ are no key updates.
+ """
+ _pk_col = None
+ _fk_cols = set()
+
+ # Buildup the known keys from column types
+ c_types = self._column_types.params["type_map"]
+ for t in c_types:
+ if isinstance(c_types[t], _PrimaryKeyType):
+ _pk_col = t
+ elif isinstance(c_types[t], _ForeignKeyType) or isinstance(
+ c_types[t], _ForeignIndexType
+ ):
+ _fk_cols.add(t)
+
+ # If there are updates to perform, safely update them
+ has_update = _pk_col != self._pk_col or _fk_cols != self._fk_cols
+ if has_update:
+ # If we removed the PK
+ if _pk_col is None and self._pk_col is not None:
+ raise AssertionError(
+ f"Cannot unset primary key (column {self._pk_col})"
+ )
+ # If there is a removed FK
+ if len(self._fk_cols - _fk_cols) > 0:
+ raise AssertionError(
+ "Cannot unset foreign key. Attempted to unset ({})".format(
+ self._fk_cols - _fk_cols
+ )
+ )
+
+ self._pk_col = _pk_col
+ self._fk_cols = _fk_cols
+
+ # Apply updates to data only if there are update or the caller
+ # requested the final row to be updated
+ if has_update or force_last:
+ self._apply_key_updates(not has_update)
+
+ def _apply_key_updates(self, only_last=False):
+ """Appropriately wrap the underlying data in special key classes.
+
+ Arguments:
+ only_last: only apply the updates to the last row (used for performance when
+ the caller knows that the only new data is the last row and no updates were
+ applied to the column types)
+ """
+ c_types = self._column_types.params["type_map"]
+
+ # Define a helper function which will wrap the data of a single row
+ # in the appropriate class wrapper.
+ def update_row(row_ndx):
+ for fk_col in self._fk_cols:
+ col_ndx = self.columns.index(fk_col)
+
+ # Wrap the Foreign Keys
+ if isinstance(c_types[fk_col], _ForeignKeyType) and not isinstance(
+ self.data[row_ndx][col_ndx], _TableKey
+ ):
+ self.data[row_ndx][col_ndx] = _TableKey(self.data[row_ndx][col_ndx])
+ self.data[row_ndx][col_ndx].set_table(
+ c_types[fk_col].params["table"],
+ c_types[fk_col].params["col_name"],
+ )
+
+ # Wrap the Foreign Indexes
+ elif isinstance(c_types[fk_col], _ForeignIndexType) and not isinstance(
+ self.data[row_ndx][col_ndx], _TableIndex
+ ):
+ self.data[row_ndx][col_ndx] = _TableIndex(
+ self.data[row_ndx][col_ndx]
+ )
+ self.data[row_ndx][col_ndx].set_table(
+ c_types[fk_col].params["table"]
+ )
+
+ # Wrap the Primary Key
+ if self._pk_col is not None:
+ col_ndx = self.columns.index(self._pk_col)
+ self.data[row_ndx][col_ndx] = _TableKey(self.data[row_ndx][col_ndx])
+ self.data[row_ndx][col_ndx].set_table(self, self._pk_col)
+
+ if only_last:
+ update_row(len(self.data) - 1)
+ else:
+ for row_ndx in range(len(self.data)):
+ update_row(row_ndx)
+
+ def add_column(self, name, data, optional=False):
+ """Add a column of data to the table.
+
+ Arguments:
+ name: (str) - the unique name of the column
+ data: (list | np.array) - a column of homogenous data
+ optional: (bool) - if null-like values are permitted
+ """
+ assert isinstance(name, str) and name not in self.columns
+ is_np = util.is_numpy_array(data)
+ assert isinstance(data, list) or is_np
+ assert isinstance(optional, bool)
+ is_first_col = len(self.columns) == 0
+ assert is_first_col or len(data) == len(
+ self.data
+ ), f"Expected length {len(self.data)}, found {len(data)}"
+
+ # Add the new data
+ for ndx in range(max(len(data), len(self.data))):
+ if is_first_col:
+ self.data.append([])
+ if is_np:
+ self.data[ndx].append(data[ndx])
+ else:
+ self.data[ndx].append(data[ndx])
+ # add the column
+ self.columns.append(name)
+
+ try:
+ self.cast(name, _dtypes.UnknownType(), optional=optional)
+ except TypeError as err:
+ # Undo the changes
+ if is_first_col:
+ self.data = []
+ self.columns = []
+ else:
+ for ndx in range(len(self.data)):
+ self.data[ndx] = self.data[ndx][:-1]
+ self.columns = self.columns[:-1]
+ raise err
+
+ def get_column(self, name, convert_to=None):
+ """Retrieve a column of data from the table.
+
+ Arguments:
+ name: (str) - the name of the column
+ convert_to: (str, optional)
+ - "numpy": will convert the underlying data to numpy object
+ """
+ assert name in self.columns
+ assert convert_to is None or convert_to == "numpy"
+ if convert_to == "numpy":
+ np = util.get_module(
+ "numpy", required="Converting to numpy requires installing numpy"
+ )
+ col = []
+ col_ndx = self.columns.index(name)
+ for row in self.data:
+ item = row[col_ndx]
+ if convert_to is not None and isinstance(item, WBValue):
+ item = item.to_data_array()
+ col.append(item)
+ if convert_to == "numpy":
+ col = np.array(col)
+ return col
+
+ def get_index(self):
+ """Return an array of row indexes for use in other tables to create links."""
+ ndxs = []
+ for ndx in range(len(self.data)):
+ index = _TableIndex(ndx)
+ index.set_table(self)
+ ndxs.append(index)
+ return ndxs
+
+ def get_dataframe(self):
+ """Returns a pandas.DataFrame of the table."""
+ pd = util.get_module(
+ "pandas",
+ required="Converting to pandas.DataFrame requires installing pandas",
+ )
+ return pd.DataFrame.from_records(self.data, columns=self.columns)
+
+ def index_ref(self, index):
+ """Get a reference to a particular row index in the table."""
+ assert index < len(self.data)
+ _index = _TableIndex(index)
+ _index.set_table(self)
+ return _index
+
+ def add_computed_columns(self, fn):
+ """Add one or more computed columns based on existing data.
+
+ Args:
+ fn: A function which accepts one or two parameters, ndx (int) and row (dict),
+ which is expected to return a dict representing new columns for that row, keyed
+ by the new column names.
+
+ `ndx` is an integer representing the index of the row. Only included if `include_ndx`
+ is set to `True`.
+
+ `row` is a dictionary keyed by existing columns
+ """
+ new_columns = {}
+ for ndx, row in self.iterrows():
+ row_dict = {self.columns[i]: row[i] for i in range(len(self.columns))}
+ new_row_dict = fn(ndx, row_dict)
+ assert isinstance(new_row_dict, dict)
+ for key in new_row_dict:
+ new_columns[key] = new_columns.get(key, [])
+ new_columns[key].append(new_row_dict[key])
+ for new_col_name in new_columns:
+ self.add_column(new_col_name, new_columns[new_col_name])
+
+
+class _PartitionTablePartEntry:
+ """Helper class for PartitionTable to track its parts."""
+
+ def __init__(self, entry, source_artifact):
+ self.entry = entry
+ self.source_artifact = source_artifact
+ self._part = None
+
+ def get_part(self):
+ if self._part is None:
+ self._part = self.source_artifact.get(self.entry.path)
+ return self._part
+
+ def free(self):
+ self._part = None
+
+
+class PartitionedTable(Media):
+ """A table which is composed of multiple sub-tables.
+
+ Currently, PartitionedTable is designed to point to a directory within an artifact.
+ """
+
+ _log_type = "partitioned-table"
+
+ def __init__(self, parts_path):
+ """Initialize a PartitionedTable.
+
+ Args:
+ parts_path (str): path to a directory of tables in the artifact.
+ """
+ super().__init__()
+ self.parts_path = parts_path
+ self._loaded_part_entries = {}
+
+ def to_json(self, artifact_or_run):
+ json_obj = {
+ "_type": PartitionedTable._log_type,
+ }
+ if isinstance(artifact_or_run, wandb.wandb_sdk.wandb_run.Run):
+ artifact_entry_url = self._get_artifact_entry_ref_url()
+ if artifact_entry_url is None:
+ raise ValueError(
+ "PartitionedTables must first be added to an Artifact before logging to a Run"
+ )
+ json_obj["artifact_path"] = artifact_entry_url
+ else:
+ json_obj["parts_path"] = self.parts_path
+ return json_obj
+
+ @classmethod
+ def from_json(cls, json_obj, source_artifact):
+ instance = cls(json_obj["parts_path"])
+ entries = source_artifact.manifest.get_entries_in_directory(
+ json_obj["parts_path"]
+ )
+ for entry in entries:
+ instance._add_part_entry(entry, source_artifact)
+ return instance
+
+ def iterrows(self):
+ """Iterate over rows as (ndx, row).
+
+ Yields:
+ ------
+ index : int
+ The index of the row.
+ row : List[any]
+ The data of the row.
+ """
+ columns = None
+ ndx = 0
+ for entry_path in self._loaded_part_entries:
+ part = self._loaded_part_entries[entry_path].get_part()
+ if columns is None:
+ columns = part.columns
+ elif columns != part.columns:
+ raise ValueError(
+ "Table parts have non-matching columns. {} != {}".format(
+ columns, part.columns
+ )
+ )
+ for _, row in part.iterrows():
+ yield ndx, row
+ ndx += 1
+
+ self._loaded_part_entries[entry_path].free()
+
+ def _add_part_entry(self, entry, source_artifact):
+ self._loaded_part_entries[entry.path] = _PartitionTablePartEntry(
+ entry, source_artifact
+ )
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.parts_path == other.parts_path
+
+ def bind_to_run(self, *args, **kwargs):
+ raise ValueError("PartitionedTables cannot be bound to runs")
+
+
+class Audio(BatchableMedia):
+ """Wandb class for audio clips.
+
+ Arguments:
+ data_or_path: (string or numpy array) A path to an audio file
+ or a numpy array of audio data.
+ sample_rate: (int) Sample rate, required when passing in raw
+ numpy array of audio data.
+ caption: (string) Caption to display with audio.
+ """
+
+ _log_type = "audio-file"
+
+ def __init__(self, data_or_path, sample_rate=None, caption=None):
+ """Accept a path to an audio file or a numpy array of audio data."""
+ super().__init__()
+ self._duration = None
+ self._sample_rate = sample_rate
+ self._caption = caption
+
+ if isinstance(data_or_path, str):
+ if self.path_is_reference(data_or_path):
+ self._path = data_or_path
+ self._sha256 = hashlib.sha256(data_or_path.encode("utf-8")).hexdigest()
+ self._is_tmp = False
+ else:
+ self._set_file(data_or_path, is_tmp=False)
+ else:
+ if sample_rate is None:
+ raise ValueError(
+ 'Argument "sample_rate" is required when instantiating wandb.Audio with raw data.'
+ )
+
+ soundfile = util.get_module(
+ "soundfile",
+ required='Raw audio requires the soundfile package. To get it, run "pip install soundfile"',
+ )
+
+ tmp_path = os.path.join(MEDIA_TMP.name, runid.generate_id() + ".wav")
+ soundfile.write(tmp_path, data_or_path, sample_rate)
+ self._duration = len(data_or_path) / float(sample_rate)
+
+ self._set_file(tmp_path, is_tmp=True)
+
+ @classmethod
+ def get_media_subdir(cls):
+ return os.path.join("media", "audio")
+
+ @classmethod
+ def from_json(cls, json_obj, source_artifact):
+ return cls(
+ source_artifact.get_entry(json_obj["path"]).download(),
+ caption=json_obj["caption"],
+ )
+
+ def bind_to_run(
+ self, run, key, step, id_=None, ignore_copy_err: Optional[bool] = None
+ ):
+ if self.path_is_reference(self._path):
+ raise ValueError(
+ "Audio media created by a reference to external storage cannot currently be added to a run"
+ )
+
+ return super().bind_to_run(run, key, step, id_, ignore_copy_err)
+
+ def to_json(self, run):
+ json_dict = super().to_json(run)
+ json_dict.update(
+ {
+ "_type": self._log_type,
+ "caption": self._caption,
+ }
+ )
+ return json_dict
+
+ @classmethod
+ def seq_to_json(cls, seq, run, key, step):
+ audio_list = list(seq)
+
+ util.get_module(
+ "soundfile",
+ required="wandb.Audio requires the soundfile package. To get it, run: pip install soundfile",
+ )
+ base_path = os.path.join(run.dir, "media", "audio")
+ filesystem.mkdir_exists_ok(base_path)
+ meta = {
+ "_type": "audio",
+ "count": len(audio_list),
+ "audio": [a.to_json(run) for a in audio_list],
+ }
+ sample_rates = cls.sample_rates(audio_list)
+ if sample_rates:
+ meta["sampleRates"] = sample_rates
+ durations = cls.durations(audio_list)
+ if durations:
+ meta["durations"] = durations
+ captions = cls.captions(audio_list)
+ if captions:
+ meta["captions"] = captions
+
+ return meta
+
+ @classmethod
+ def durations(cls, audio_list):
+ return [a._duration for a in audio_list]
+
+ @classmethod
+ def sample_rates(cls, audio_list):
+ return [a._sample_rate for a in audio_list]
+
+ @classmethod
+ def captions(cls, audio_list):
+ captions = [a._caption for a in audio_list]
+ if all(c is None for c in captions):
+ return False
+ else:
+ return ["" if c is None else c for c in captions]
+
+ def resolve_ref(self):
+ if self.path_is_reference(self._path):
+ # this object was already created using a ref:
+ return self._path
+ source_artifact = self._artifact_source.artifact
+
+ resolved_name = source_artifact._local_path_to_name(self._path)
+ if resolved_name is not None:
+ target_entry = source_artifact.manifest.get_entry_by_path(resolved_name)
+ if target_entry is not None:
+ return target_entry.ref
+
+ return None
+
+ def __eq__(self, other):
+ if self.path_is_reference(self._path) or self.path_is_reference(other._path):
+ # one or more of these objects is an unresolved reference -- we'll compare
+ # their reference paths instead of their SHAs:
+ return (
+ self.resolve_ref() == other.resolve_ref()
+ and self._caption == other._caption
+ )
+
+ return super().__eq__(other) and self._caption == other._caption
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+
+class JoinedTable(Media):
+ """Join two tables for visualization in the Artifact UI.
+
+ Arguments:
+ table1 (str, wandb.Table, ArtifactManifestEntry):
+ the path to a wandb.Table in an artifact, the table object, or ArtifactManifestEntry
+ table2 (str, wandb.Table):
+ the path to a wandb.Table in an artifact, the table object, or ArtifactManifestEntry
+ join_key (str, [str, str]):
+ key or keys to perform the join
+ """
+
+ _log_type = "joined-table"
+
+ def __init__(self, table1, table2, join_key):
+ super().__init__()
+
+ if not isinstance(join_key, str) and (
+ not isinstance(join_key, list) or len(join_key) != 2
+ ):
+ raise ValueError(
+ "JoinedTable join_key should be a string or a list of two strings"
+ )
+
+ if not self._validate_table_input(table1):
+ raise ValueError(
+ "JoinedTable table1 should be an artifact path to a table or wandb.Table object"
+ )
+
+ if not self._validate_table_input(table2):
+ raise ValueError(
+ "JoinedTable table2 should be an artifact path to a table or wandb.Table object"
+ )
+
+ self._table1 = table1
+ self._table2 = table2
+ self._join_key = join_key
+
+ @classmethod
+ def from_json(cls, json_obj, source_artifact):
+ t1 = source_artifact.get(json_obj["table1"])
+ if t1 is None:
+ t1 = json_obj["table1"]
+
+ t2 = source_artifact.get(json_obj["table2"])
+ if t2 is None:
+ t2 = json_obj["table2"]
+
+ return cls(
+ t1,
+ t2,
+ json_obj["join_key"],
+ )
+
+ @staticmethod
+ def _validate_table_input(table):
+ """Helper method to validate that the table input is one of the 3 supported types."""
+ return (
+ (isinstance(table, str) and table.endswith(".table.json"))
+ or isinstance(table, Table)
+ or isinstance(table, PartitionedTable)
+ or (hasattr(table, "ref_url") and table.ref_url().endswith(".table.json"))
+ )
+
+ def _ensure_table_in_artifact(self, table, artifact, table_ndx):
+ """Helper method to add the table to the incoming artifact. Returns the path."""
+ if isinstance(table, Table) or isinstance(table, PartitionedTable):
+ table_name = f"t{table_ndx}_{str(id(self))}"
+ if (
+ table._artifact_source is not None
+ and table._artifact_source.name is not None
+ ):
+ table_name = os.path.basename(table._artifact_source.name)
+ entry = artifact.add(table, table_name)
+ table = entry.path
+ # Check if this is an ArtifactManifestEntry
+ elif hasattr(table, "ref_url"):
+ # Give the new object a unique, yet deterministic name
+ name = binascii.hexlify(base64.standard_b64decode(table.digest)).decode(
+ "ascii"
+ )[:20]
+ entry = artifact.add_reference(
+ table.ref_url(), "{}.{}.json".format(name, table.name.split(".")[-2])
+ )[0]
+ table = entry.path
+
+ err_str = "JoinedTable table:{} not found in artifact. Add a table to the artifact using Artifact#add(<table>, {}) before adding this JoinedTable"
+ if table not in artifact._manifest.entries:
+ raise ValueError(err_str.format(table, table))
+
+ return table
+
+ def to_json(self, artifact_or_run):
+ json_obj = {
+ "_type": JoinedTable._log_type,
+ }
+ if isinstance(artifact_or_run, wandb.wandb_sdk.wandb_run.Run):
+ artifact_entry_url = self._get_artifact_entry_ref_url()
+ if artifact_entry_url is None:
+ raise ValueError(
+ "JoinedTables must first be added to an Artifact before logging to a Run"
+ )
+ json_obj["artifact_path"] = artifact_entry_url
+ else:
+ table1 = self._ensure_table_in_artifact(self._table1, artifact_or_run, 1)
+ table2 = self._ensure_table_in_artifact(self._table2, artifact_or_run, 2)
+ json_obj.update(
+ {
+ "table1": table1,
+ "table2": table2,
+ "join_key": self._join_key,
+ }
+ )
+ return json_obj
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def _eq_debug(self, other, should_assert=False):
+ eq = isinstance(other, JoinedTable)
+ assert not should_assert or eq, "Found type {}, expected {}".format(
+ other.__class__, JoinedTable
+ )
+ eq = eq and self._join_key == other._join_key
+ assert not should_assert or eq, "Found {} join key, expected {}".format(
+ other._join_key, self._join_key
+ )
+ eq = eq and self._table1._eq_debug(other._table1, should_assert)
+ eq = eq and self._table2._eq_debug(other._table2, should_assert)
+ return eq
+
+ def __eq__(self, other):
+ return self._eq_debug(other, False)
+
+ def bind_to_run(self, *args, **kwargs):
+ raise ValueError("JoinedTables cannot be bound to runs")
+
+
+class Bokeh(Media):
+ """Wandb class for Bokeh plots.
+
+ Arguments:
+ val: Bokeh plot
+ """
+
+ _log_type = "bokeh-file"
+
+ def __init__(self, data_or_path):
+ super().__init__()
+ bokeh = util.get_module("bokeh", required=True)
+ if isinstance(data_or_path, str) and os.path.exists(data_or_path):
+ with open(data_or_path) as file:
+ b_json = json.load(file)
+ self.b_obj = bokeh.document.Document.from_json(b_json)
+ self._set_file(data_or_path, is_tmp=False, extension=".bokeh.json")
+ elif isinstance(data_or_path, bokeh.model.Model):
+ _data = bokeh.document.Document()
+ _data.add_root(data_or_path)
+ # serialize/deserialize pairing followed by sorting attributes ensures
+ # that the file's sha's are equivalent in subsequent calls
+ self.b_obj = bokeh.document.Document.from_json(_data.to_json())
+ b_json = self.b_obj.to_json()
+ if "references" in b_json["roots"]:
+ b_json["roots"]["references"].sort(key=lambda x: x["id"])
+
+ tmp_path = os.path.join(MEDIA_TMP.name, runid.generate_id() + ".bokeh.json")
+ with codecs.open(tmp_path, "w", encoding="utf-8") as fp:
+ util.json_dump_safer(b_json, fp)
+ self._set_file(tmp_path, is_tmp=True, extension=".bokeh.json")
+ elif not isinstance(data_or_path, bokeh.document.Document):
+ raise TypeError(
+ "Bokeh constructor accepts Bokeh document/model or path to Bokeh json file"
+ )
+
+ def get_media_subdir(self):
+ return os.path.join("media", "bokeh")
+
+ def to_json(self, run):
+ # TODO: (tss) this is getting redundant for all the media objects. We can probably
+ # pull this into Media#to_json and remove this type override for all the media types.
+ # There are only a few cases where the type is different between artifacts and runs.
+ json_dict = super().to_json(run)
+ json_dict["_type"] = self._log_type
+ return json_dict
+
+ @classmethod
+ def from_json(cls, json_obj, source_artifact):
+ return cls(source_artifact.get_entry(json_obj["path"]).download())
+
+
+def _nest(thing):
+ # Use tensorflows nest function if available, otherwise just wrap object in an array"""
+
+ tfutil = util.get_module("tensorflow.python.util")
+ if tfutil:
+ return tfutil.nest.flatten(thing)
+ else:
+ return [thing]
+
+
+class Graph(Media):
+ """Wandb class for graphs.
+
+ This class is typically used for saving and displaying neural net models. It
+ represents the graph as an array of nodes and edges. The nodes can have
+ labels that can be visualized by wandb.
+
+ Examples:
+ Import a keras model:
+ ```
+ Graph.from_keras(keras_model)
+ ```
+
+ Attributes:
+ format (string): Format to help wandb display the graph nicely.
+ nodes ([wandb.Node]): List of wandb.Nodes
+ nodes_by_id (dict): dict of ids -> nodes
+ edges ([(wandb.Node, wandb.Node)]): List of pairs of nodes interpreted as edges
+ loaded (boolean): Flag to tell whether the graph is completely loaded
+ root (wandb.Node): root node of the graph
+ """
+
+ _log_type = "graph-file"
+
+ def __init__(self, format="keras"):
+ super().__init__()
+ # LB: TODO: I think we should factor criterion and criterion_passed out
+ self.format = format
+ self.nodes = []
+ self.nodes_by_id = {}
+ self.edges = []
+ self.loaded = False
+ self.criterion = None
+ self.criterion_passed = False
+ self.root = None # optional root Node if applicable
+
+ def _to_graph_json(self, run=None):
+ # Needs to be its own function for tests
+ return {
+ "format": self.format,
+ "nodes": [node.to_json() for node in self.nodes],
+ "edges": [edge.to_json() for edge in self.edges],
+ }
+
+ def bind_to_run(self, *args, **kwargs):
+ data = self._to_graph_json()
+ tmp_path = os.path.join(MEDIA_TMP.name, runid.generate_id() + ".graph.json")
+ data = _numpy_arrays_to_lists(data)
+ with codecs.open(tmp_path, "w", encoding="utf-8") as fp:
+ util.json_dump_safer(data, fp)
+ self._set_file(tmp_path, is_tmp=True, extension=".graph.json")
+ if self.is_bound():
+ return
+ super().bind_to_run(*args, **kwargs)
+
+ @classmethod
+ def get_media_subdir(cls):
+ return os.path.join("media", "graph")
+
+ def to_json(self, run):
+ json_dict = super().to_json(run)
+ json_dict["_type"] = self._log_type
+ return json_dict
+
+ def __getitem__(self, nid):
+ return self.nodes_by_id[nid]
+
+ def pprint(self):
+ for edge in self.edges:
+ pprint.pprint(edge.attributes)
+ for node in self.nodes:
+ pprint.pprint(node.attributes)
+
+ def add_node(self, node=None, **node_kwargs):
+ if node is None:
+ node = Node(**node_kwargs)
+ elif node_kwargs:
+ raise ValueError(
+ f"Only pass one of either node ({node}) or other keyword arguments ({node_kwargs})"
+ )
+ self.nodes.append(node)
+ self.nodes_by_id[node.id] = node
+
+ return node
+
+ def add_edge(self, from_node, to_node):
+ edge = Edge(from_node, to_node)
+ self.edges.append(edge)
+
+ return edge
+
+ @classmethod
+ def from_keras(cls, model):
+ graph = cls()
+ # Shamelessly copied (then modified) from keras/keras/utils/layer_utils.py
+ sequential_like = cls._is_sequential(model)
+
+ relevant_nodes = None
+ if not sequential_like:
+ relevant_nodes = []
+ for v in model._nodes_by_depth.values():
+ relevant_nodes += v
+
+ layers = model.layers
+ for i in range(len(layers)):
+ node = Node.from_keras(layers[i])
+ if hasattr(layers[i], "_inbound_nodes"):
+ for in_node in layers[i]._inbound_nodes:
+ if relevant_nodes and in_node not in relevant_nodes:
+ # node is not part of the current network
+ continue
+ for in_layer in _nest(in_node.inbound_layers):
+ inbound_keras_node = Node.from_keras(in_layer)
+
+ if inbound_keras_node.id not in graph.nodes_by_id:
+ graph.add_node(inbound_keras_node)
+ inbound_node = graph.nodes_by_id[inbound_keras_node.id]
+
+ graph.add_edge(inbound_node, node)
+ graph.add_node(node)
+ return graph
+
+ @classmethod
+ def _is_sequential(cls, model):
+ sequential_like = True
+
+ if (
+ model.__class__.__name__ != "Sequential"
+ and hasattr(model, "_is_graph_network")
+ and model._is_graph_network
+ ):
+ nodes_by_depth = model._nodes_by_depth.values()
+ nodes = []
+ for v in nodes_by_depth:
+ # TensorFlow2 doesn't insure inbound is always a list
+ inbound = v[0].inbound_layers
+ if not hasattr(inbound, "__len__"):
+ inbound = [inbound]
+ if (len(v) > 1) or (len(v) == 1 and len(inbound) > 1):
+ # if the model has multiple nodes
+ # or if the nodes have multiple inbound_layers
+ # the model is no longer sequential
+ sequential_like = False
+ break
+ nodes += v
+ if sequential_like:
+ # search for shared layers
+ for layer in model.layers:
+ flag = False
+ if hasattr(layer, "_inbound_nodes"):
+ for node in layer._inbound_nodes:
+ if node in nodes:
+ if flag:
+ sequential_like = False
+ break
+ else:
+ flag = True
+ if not sequential_like:
+ break
+ return sequential_like
+
+
+class Node(WBValue):
+ """Node used in `Graph`."""
+
+ def __init__(
+ self,
+ id=None,
+ name=None,
+ class_name=None,
+ size=None,
+ parameters=None,
+ output_shape=None,
+ is_output=None,
+ num_parameters=None,
+ node=None,
+ ):
+ self._attributes = {"name": None}
+ self.in_edges = {} # indexed by source node id
+ self.out_edges = {} # indexed by dest node id
+ # optional object (e.g. PyTorch Parameter or Module) that this Node represents
+ self.obj = None
+
+ if node is not None:
+ self._attributes.update(node._attributes)
+ del self._attributes["id"]
+ self.obj = node.obj
+
+ if id is not None:
+ self.id = id
+ if name is not None:
+ self.name = name
+ if class_name is not None:
+ self.class_name = class_name
+ if size is not None:
+ self.size = size
+ if parameters is not None:
+ self.parameters = parameters
+ if output_shape is not None:
+ self.output_shape = output_shape
+ if is_output is not None:
+ self.is_output = is_output
+ if num_parameters is not None:
+ self.num_parameters = num_parameters
+
+ def to_json(self, run=None):
+ return self._attributes
+
+ def __repr__(self):
+ return repr(self._attributes)
+
+ @property
+ def id(self):
+ """Must be unique in the graph."""
+ return self._attributes.get("id")
+
+ @id.setter
+ def id(self, val):
+ self._attributes["id"] = val
+ return val
+
+ @property
+ def name(self):
+ """Usually the type of layer or sublayer."""
+ return self._attributes.get("name")
+
+ @name.setter
+ def name(self, val):
+ self._attributes["name"] = val
+ return val
+
+ @property
+ def class_name(self):
+ """Usually the type of layer or sublayer."""
+ return self._attributes.get("class_name")
+
+ @class_name.setter
+ def class_name(self, val):
+ self._attributes["class_name"] = val
+ return val
+
+ @property
+ def functions(self):
+ return self._attributes.get("functions", [])
+
+ @functions.setter
+ def functions(self, val):
+ self._attributes["functions"] = val
+ return val
+
+ @property
+ def parameters(self):
+ return self._attributes.get("parameters", [])
+
+ @parameters.setter
+ def parameters(self, val):
+ self._attributes["parameters"] = val
+ return val
+
+ @property
+ def size(self):
+ return self._attributes.get("size")
+
+ @size.setter
+ def size(self, val):
+ """Tensor size."""
+ self._attributes["size"] = tuple(val)
+ return val
+
+ @property
+ def output_shape(self):
+ return self._attributes.get("output_shape")
+
+ @output_shape.setter
+ def output_shape(self, val):
+ """Tensor output_shape."""
+ self._attributes["output_shape"] = val
+ return val
+
+ @property
+ def is_output(self):
+ return self._attributes.get("is_output")
+
+ @is_output.setter
+ def is_output(self, val):
+ """Tensor is_output."""
+ self._attributes["is_output"] = val
+ return val
+
+ @property
+ def num_parameters(self):
+ return self._attributes.get("num_parameters")
+
+ @num_parameters.setter
+ def num_parameters(self, val):
+ """Tensor num_parameters."""
+ self._attributes["num_parameters"] = val
+ return val
+
+ @property
+ def child_parameters(self):
+ return self._attributes.get("child_parameters")
+
+ @child_parameters.setter
+ def child_parameters(self, val):
+ """Tensor child_parameters."""
+ self._attributes["child_parameters"] = val
+ return val
+
+ @property
+ def is_constant(self):
+ return self._attributes.get("is_constant")
+
+ @is_constant.setter
+ def is_constant(self, val):
+ """Tensor is_constant."""
+ self._attributes["is_constant"] = val
+ return val
+
+ @classmethod
+ def from_keras(cls, layer):
+ node = cls()
+
+ try:
+ output_shape = layer.output_shape
+ except AttributeError:
+ output_shape = ["multiple"]
+
+ node.id = layer.name
+ node.name = layer.name
+ node.class_name = layer.__class__.__name__
+ node.output_shape = output_shape
+ node.num_parameters = layer.count_params()
+
+ return node
+
+
+class Edge(WBValue):
+ """Edge used in `Graph`."""
+
+ def __init__(self, from_node, to_node):
+ self._attributes = {}
+ self.from_node = from_node
+ self.to_node = to_node
+
+ def __repr__(self):
+ temp_attr = dict(self._attributes)
+ del temp_attr["from_node"]
+ del temp_attr["to_node"]
+ temp_attr["from_id"] = self.from_node.id
+ temp_attr["to_id"] = self.to_node.id
+ return str(temp_attr)
+
+ def to_json(self, run=None):
+ return [self.from_node.id, self.to_node.id]
+
+ @property
+ def name(self):
+ """Optional, not necessarily unique."""
+ return self._attributes.get("name")
+
+ @name.setter
+ def name(self, val):
+ self._attributes["name"] = val
+ return val
+
+ @property
+ def from_node(self):
+ return self._attributes.get("from_node")
+
+ @from_node.setter
+ def from_node(self, val):
+ self._attributes["from_node"] = val
+ return val
+
+ @property
+ def to_node(self):
+ return self._attributes.get("to_node")
+
+ @to_node.setter
+ def to_node(self, val):
+ self._attributes["to_node"] = val
+ return val
+
+
+# Custom dtypes for typing system
+class _ImageFileType(_dtypes.Type):
+ name = "image-file"
+ legacy_names = ["wandb.Image"]
+ types = [Image]
+
+ def __init__(
+ self,
+ box_layers=None,
+ box_score_keys=None,
+ mask_layers=None,
+ class_map=None,
+ **kwargs,
+ ):
+ box_layers = box_layers or {}
+ box_score_keys = box_score_keys or []
+ mask_layers = mask_layers or {}
+ class_map = class_map or {}
+
+ if isinstance(box_layers, _dtypes.ConstType):
+ box_layers = box_layers._params["val"]
+ if not isinstance(box_layers, dict):
+ raise TypeError("box_layers must be a dict")
+ else:
+ box_layers = _dtypes.ConstType(
+ {layer_key: set(box_layers[layer_key]) for layer_key in box_layers}
+ )
+
+ if isinstance(mask_layers, _dtypes.ConstType):
+ mask_layers = mask_layers._params["val"]
+ if not isinstance(mask_layers, dict):
+ raise TypeError("mask_layers must be a dict")
+ else:
+ mask_layers = _dtypes.ConstType(
+ {layer_key: set(mask_layers[layer_key]) for layer_key in mask_layers}
+ )
+
+ if isinstance(box_score_keys, _dtypes.ConstType):
+ box_score_keys = box_score_keys._params["val"]
+ if not isinstance(box_score_keys, list) and not isinstance(box_score_keys, set):
+ raise TypeError("box_score_keys must be a list or a set")
+ else:
+ box_score_keys = _dtypes.ConstType(set(box_score_keys))
+
+ if isinstance(class_map, _dtypes.ConstType):
+ class_map = class_map._params["val"]
+ if not isinstance(class_map, dict):
+ raise TypeError("class_map must be a dict")
+ else:
+ class_map = _dtypes.ConstType(class_map)
+
+ self.params.update(
+ {
+ "box_layers": box_layers,
+ "box_score_keys": box_score_keys,
+ "mask_layers": mask_layers,
+ "class_map": class_map,
+ }
+ )
+
+ def assign_type(self, wb_type=None):
+ if isinstance(wb_type, _ImageFileType):
+ box_layers_self = self.params["box_layers"].params["val"] or {}
+ box_score_keys_self = self.params["box_score_keys"].params["val"] or []
+ mask_layers_self = self.params["mask_layers"].params["val"] or {}
+ class_map_self = self.params["class_map"].params["val"] or {}
+
+ box_layers_other = wb_type.params["box_layers"].params["val"] or {}
+ box_score_keys_other = wb_type.params["box_score_keys"].params["val"] or []
+ mask_layers_other = wb_type.params["mask_layers"].params["val"] or {}
+ class_map_other = wb_type.params["class_map"].params["val"] or {}
+
+ # Merge the class_ids from each set of box_layers
+ box_layers = {
+ str(key): set(
+ list(box_layers_self.get(key, []))
+ + list(box_layers_other.get(key, []))
+ )
+ for key in set(
+ list(box_layers_self.keys()) + list(box_layers_other.keys())
+ )
+ }
+
+ # Merge the class_ids from each set of mask_layers
+ mask_layers = {
+ str(key): set(
+ list(mask_layers_self.get(key, []))
+ + list(mask_layers_other.get(key, []))
+ )
+ for key in set(
+ list(mask_layers_self.keys()) + list(mask_layers_other.keys())
+ )
+ }
+
+ # Merge the box score keys
+ box_score_keys = set(list(box_score_keys_self) + list(box_score_keys_other))
+
+ # Merge the class_map
+ class_map = {
+ str(key): class_map_self.get(key, class_map_other.get(key, None))
+ for key in set(
+ list(class_map_self.keys()) + list(class_map_other.keys())
+ )
+ }
+
+ return _ImageFileType(box_layers, box_score_keys, mask_layers, class_map)
+
+ return _dtypes.InvalidType()
+
+ @classmethod
+ def from_obj(cls, py_obj):
+ if not isinstance(py_obj, Image):
+ raise TypeError("py_obj must be a wandb.Image")
+ else:
+ if hasattr(py_obj, "_boxes") and py_obj._boxes:
+ box_layers = {
+ str(key): set(py_obj._boxes[key]._class_labels.keys())
+ for key in py_obj._boxes.keys()
+ }
+ box_score_keys = {
+ key
+ for val in py_obj._boxes.values()
+ for box in val._val
+ for key in box.get("scores", {}).keys()
+ }
+
+ else:
+ box_layers = {}
+ box_score_keys = set()
+
+ if hasattr(py_obj, "_masks") and py_obj._masks:
+ mask_layers = {
+ str(key): set(
+ py_obj._masks[key]._val["class_labels"].keys()
+ if hasattr(py_obj._masks[key], "_val")
+ else []
+ )
+ for key in py_obj._masks.keys()
+ }
+ else:
+ mask_layers = {}
+
+ if hasattr(py_obj, "_classes") and py_obj._classes:
+ class_set = {
+ str(item["id"]): item["name"] for item in py_obj._classes._class_set
+ }
+ else:
+ class_set = {}
+
+ return cls(box_layers, box_score_keys, mask_layers, class_set)
+
+
+class _TableType(_dtypes.Type):
+ name = "table"
+ legacy_names = ["wandb.Table"]
+ types = [Table]
+
+ def __init__(self, column_types=None):
+ if column_types is None:
+ column_types = _dtypes.UnknownType()
+ if isinstance(column_types, dict):
+ column_types = _dtypes.TypedDictType(column_types)
+ elif not (
+ isinstance(column_types, _dtypes.TypedDictType)
+ or isinstance(column_types, _dtypes.UnknownType)
+ ):
+ raise TypeError("column_types must be a dict or TypedDictType")
+
+ self.params.update({"column_types": column_types})
+
+ def assign_type(self, wb_type=None):
+ if isinstance(wb_type, _TableType):
+ column_types = self.params["column_types"].assign_type(
+ wb_type.params["column_types"]
+ )
+ if not isinstance(column_types, _dtypes.InvalidType):
+ return _TableType(column_types)
+
+ return _dtypes.InvalidType()
+
+ @classmethod
+ def from_obj(cls, py_obj):
+ if not isinstance(py_obj, Table):
+ raise TypeError("py_obj must be a wandb.Table")
+ else:
+ return cls(py_obj._column_types)
+
+
+class _ForeignKeyType(_dtypes.Type):
+ name = "foreignKey"
+ legacy_names = ["wandb.TableForeignKey"]
+ types = [_TableKey]
+
+ def __init__(self, table, col_name):
+ assert isinstance(table, Table)
+ assert isinstance(col_name, str)
+ assert col_name in table.columns
+ self.params.update({"table": table, "col_name": col_name})
+
+ def assign_type(self, wb_type=None):
+ if isinstance(wb_type, _dtypes.StringType):
+ return self
+ elif (
+ isinstance(wb_type, _ForeignKeyType)
+ and id(self.params["table"]) == id(wb_type.params["table"])
+ and self.params["col_name"] == wb_type.params["col_name"]
+ ):
+ return self
+
+ return _dtypes.InvalidType()
+
+ @classmethod
+ def from_obj(cls, py_obj):
+ if not isinstance(py_obj, _TableKey):
+ raise TypeError("py_obj must be a _TableKey")
+ else:
+ return cls(py_obj._table, py_obj._col_name)
+
+ def to_json(self, artifact=None):
+ res = super().to_json(artifact)
+ if artifact is not None:
+ table_name = f"media/tables/t_{runid.generate_id()}"
+ entry = artifact.add(self.params["table"], table_name)
+ res["params"]["table"] = entry.path
+ else:
+ raise AssertionError(
+ "_ForeignKeyType does not support serialization without an artifact"
+ )
+ return res
+
+ @classmethod
+ def from_json(
+ cls,
+ json_dict,
+ artifact,
+ ):
+ table = None
+ col_name = None
+ if artifact is None:
+ raise AssertionError(
+ "_ForeignKeyType does not support deserialization without an artifact"
+ )
+ else:
+ table = artifact.get(json_dict["params"]["table"])
+ col_name = json_dict["params"]["col_name"]
+
+ if table is None:
+ raise AssertionError("Unable to deserialize referenced table")
+
+ return cls(table, col_name)
+
+
+class _ForeignIndexType(_dtypes.Type):
+ name = "foreignIndex"
+ legacy_names = ["wandb.TableForeignIndex"]
+ types = [_TableIndex]
+
+ def __init__(self, table):
+ assert isinstance(table, Table)
+ self.params.update({"table": table})
+
+ def assign_type(self, wb_type=None):
+ if isinstance(wb_type, _dtypes.NumberType):
+ return self
+ elif isinstance(wb_type, _ForeignIndexType) and id(self.params["table"]) == id(
+ wb_type.params["table"]
+ ):
+ return self
+
+ return _dtypes.InvalidType()
+
+ @classmethod
+ def from_obj(cls, py_obj):
+ if not isinstance(py_obj, _TableIndex):
+ raise TypeError("py_obj must be a _TableIndex")
+ else:
+ return cls(py_obj._table)
+
+ def to_json(self, artifact=None):
+ res = super().to_json(artifact)
+ if artifact is not None:
+ table_name = f"media/tables/t_{runid.generate_id()}"
+ entry = artifact.add(self.params["table"], table_name)
+ res["params"]["table"] = entry.path
+ else:
+ raise AssertionError(
+ "_ForeignIndexType does not support serialization without an artifact"
+ )
+ return res
+
+ @classmethod
+ def from_json(
+ cls,
+ json_dict,
+ artifact,
+ ):
+ table = None
+ if artifact is None:
+ raise AssertionError(
+ "_ForeignIndexType does not support deserialization without an artifact"
+ )
+ else:
+ table = artifact.get(json_dict["params"]["table"])
+
+ if table is None:
+ raise AssertionError("Unable to deserialize referenced table")
+
+ return cls(table)
+
+
+class _PrimaryKeyType(_dtypes.Type):
+ name = "primaryKey"
+ legacy_names = ["wandb.TablePrimaryKey"]
+
+ def assign_type(self, wb_type=None):
+ if isinstance(wb_type, _dtypes.StringType) or isinstance(
+ wb_type, _PrimaryKeyType
+ ):
+ return self
+ return _dtypes.InvalidType()
+
+ @classmethod
+ def from_obj(cls, py_obj):
+ if not isinstance(py_obj, _TableKey):
+ raise TypeError("py_obj must be a wandb.Table")
+ else:
+ return cls()
+
+
+class _AudioFileType(_dtypes.Type):
+ name = "audio-file"
+ types = [Audio]
+
+
+class _BokehFileType(_dtypes.Type):
+ name = "bokeh-file"
+ types = [Bokeh]
+
+
+class _JoinedTableType(_dtypes.Type):
+ name = "joined-table"
+ types = [JoinedTable]
+
+
+class _PartitionedTableType(_dtypes.Type):
+ name = "partitioned-table"
+ types = [PartitionedTable]
+
+
+_dtypes.TypeRegistry.add(_AudioFileType)
+_dtypes.TypeRegistry.add(_BokehFileType)
+_dtypes.TypeRegistry.add(_ImageFileType)
+_dtypes.TypeRegistry.add(_TableType)
+_dtypes.TypeRegistry.add(_JoinedTableType)
+_dtypes.TypeRegistry.add(_PartitionedTableType)
+_dtypes.TypeRegistry.add(_ForeignKeyType)
+_dtypes.TypeRegistry.add(_PrimaryKeyType)
+_dtypes.TypeRegistry.add(_ForeignIndexType)
diff --git a/wandb/docker/__init__.py b/wandb/docker/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..17ce1f070bc4592801cb34751e9666439b711aad
--- /dev/null
+++ b/wandb/docker/__init__.py
@@ -0,0 +1,342 @@
+import json
+import logging
+import os
+import subprocess
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import requests
+from dockerpycreds.utils import find_executable # type: ignore
+
+from wandb.docker import auth, www_authenticate
+from wandb.errors import Error
+
+
+class DockerError(Error):
+ """Raised when attempting to execute a docker command."""
+
+ def __init__(
+ self,
+ command_launched: List[str],
+ return_code: int,
+ stdout: Optional[bytes] = None,
+ stderr: Optional[bytes] = None,
+ ) -> None:
+ command_launched_str = " ".join(command_launched)
+ error_msg = (
+ f"The docker command executed was `{command_launched_str}`.\n"
+ f"It returned with code {return_code}\n"
+ )
+ if stdout is not None:
+ error_msg += f"The content of stdout is '{stdout.decode()}'\n"
+ else:
+ error_msg += (
+ "The content of stdout can be found above the "
+ "stacktrace (it wasn't captured).\n"
+ )
+ if stderr is not None:
+ error_msg += f"The content of stderr is '{stderr.decode()}'\n"
+ else:
+ error_msg += (
+ "The content of stderr can be found above the "
+ "stacktrace (it wasn't captured)."
+ )
+ super().__init__(error_msg)
+
+
+entrypoint = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), "wandb-entrypoint.sh"
+)
+auth_config = auth.load_config()
+log = logging.getLogger(__name__)
+
+
+def shell(cmd: List[str]) -> Optional[str]:
+ """Simple wrapper for calling docker,.
+
+ returning None on error and the output on success
+ """
+ try:
+ return (
+ subprocess.check_output(["docker"] + cmd, stderr=subprocess.STDOUT)
+ .decode("utf8")
+ .strip()
+ )
+ except subprocess.CalledProcessError as e:
+ print(e)
+ return None
+
+
+_buildx_installed = None
+
+
+def is_buildx_installed() -> bool:
+ """Return `True` if docker buildx is installed and working."""
+ global _buildx_installed
+ if _buildx_installed is not None:
+ return _buildx_installed # type: ignore
+ if not find_executable("docker"):
+ _buildx_installed = False
+ else:
+ help_output = shell(["buildx", "--help"])
+ _buildx_installed = help_output is not None and "buildx" in help_output
+ return _buildx_installed
+
+
+def is_docker_installed() -> bool:
+ """Return `True` if docker is installed and working, else `False`."""
+ try:
+ # Run the docker --version command
+ result = subprocess.run(
+ ["docker", "--version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE
+ )
+ if result.returncode == 0:
+ return True
+ else:
+ return False
+ except FileNotFoundError:
+ # If docker command is not found
+ return False
+
+
+def build(
+ tags: List[str], file: str, context_path: str, platform: Optional[str] = None
+) -> str:
+ use_buildx = is_buildx_installed()
+ command = ["buildx", "build"] if use_buildx else ["build"]
+ command += ["--load"] if should_add_load_argument(platform) and use_buildx else []
+ if platform:
+ command += ["--platform", platform]
+ build_tags = []
+ for tag in tags:
+ build_tags += ["-t", tag]
+ args = ["docker"] + command + build_tags + ["-f", file, context_path]
+ stdout = run_command_live_output(
+ args,
+ )
+ return stdout
+
+
+def should_add_load_argument(platform: Optional[str]) -> bool:
+ # the load option does not work when multiple platforms are specified:
+ # https://github.com/docker/buildx/issues/59
+ if platform is None or (platform and "," not in platform):
+ return True
+ return False
+
+
+def run_command_live_output(args: List[Any]) -> str:
+ with subprocess.Popen(
+ args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ universal_newlines=True,
+ bufsize=1,
+ ) as process:
+ stdout = ""
+ while True:
+ chunk = os.read(process.stdout.fileno(), 4096) # type: ignore
+ if not chunk:
+ break
+ index = chunk.find(b"\r")
+ if index != -1:
+ print(chunk.decode(), end="")
+ else:
+ stdout += chunk.decode()
+ print(chunk.decode(), end="\r")
+
+ print(stdout)
+
+ return_code = process.wait()
+ if return_code != 0:
+ raise DockerError(args, return_code, stdout.encode())
+
+ return stdout
+
+
+def run(
+ args: List[Any],
+ capture_stdout: bool = True,
+ capture_stderr: bool = True,
+ input: Optional[bytes] = None,
+ return_stderr: bool = False,
+ env: Optional[Dict[str, str]] = None,
+) -> Union[str, Tuple[str, str]]:
+ args = [str(x) for x in args]
+ subprocess_env = dict(os.environ)
+ subprocess_env.update(env or {})
+ if args[1] == "buildx":
+ subprocess_env["DOCKER_CLI_EXPERIMENTAL"] = "enabled"
+ stdout_dest: Optional[int] = subprocess.PIPE if capture_stdout else None
+ stderr_dest: Optional[int] = subprocess.PIPE if capture_stderr else None
+
+ completed_process = subprocess.run(
+ args, input=input, stdout=stdout_dest, stderr=stderr_dest, env=subprocess_env
+ )
+ if completed_process.returncode != 0:
+ raise DockerError(
+ args,
+ completed_process.returncode,
+ completed_process.stdout,
+ completed_process.stderr,
+ )
+
+ if return_stderr:
+ return (
+ _post_process_stream(completed_process.stdout),
+ _post_process_stream(completed_process.stderr),
+ )
+ else:
+ return _post_process_stream(completed_process.stdout)
+
+
+def _post_process_stream(stream: Optional[bytes]) -> str:
+ if stream is None:
+ return ""
+ decoded_stream = stream.decode()
+ if len(decoded_stream) != 0 and decoded_stream[-1] == "\n":
+ decoded_stream = decoded_stream[:-1]
+ return decoded_stream
+
+
+def default_image(gpu: bool = False) -> str:
+ tag = "all"
+ if not gpu:
+ tag += "-cpu"
+ return "wandb/deepo:%s" % tag
+
+
+def parse_repository_tag(repo_name: str) -> Tuple[str, Optional[str]]:
+ parts = repo_name.rsplit("@", 1)
+ if len(parts) == 2:
+ return parts[0], parts[1]
+ parts = repo_name.rsplit(":", 1)
+ if len(parts) == 2 and "/" not in parts[1]:
+ return parts[0], parts[1]
+ return repo_name, None
+
+
+def parse(image_name: str) -> Tuple[str, str, str]:
+ repository, tag = parse_repository_tag(image_name)
+ registry, repo_name = auth.resolve_repository_name(repository)
+ if registry == "docker.io":
+ registry = "index.docker.io"
+ return registry, repo_name, (tag or "latest")
+
+
+def auth_token(registry: str, repo: str) -> Dict[str, str]:
+ """Make a request to the root of a v2 docker registry to get the auth url.
+
+ Always returns a dictionary, if there's no token key we couldn't authenticate
+ """
+ # TODO: Cache tokens?
+ auth_info = auth_config.resolve_authconfig(registry)
+ if auth_info:
+ normalized = {k.lower(): v for k, v in auth_info.items()}
+ normalized_auth_info = (
+ normalized.get("username"),
+ normalized.get("password"),
+ )
+ else:
+ normalized_auth_info = None
+ response = requests.get(f"https://{registry}/v2/", timeout=3)
+ if response.headers.get("www-authenticate"):
+ try:
+ info: Dict = www_authenticate.parse(response.headers["www-authenticate"])
+ except ValueError:
+ info = {}
+ else:
+ log.error(
+ f"Received {response} when attempting to authenticate with {registry}"
+ )
+ info = {}
+ if info.get("bearer"):
+ res = requests.get(
+ info["bearer"]["realm"]
+ + "?service={}&scope=repository:{}:pull".format(
+ info["bearer"]["service"], repo
+ ),
+ auth=normalized_auth_info, # type: ignore
+ timeout=3,
+ )
+ res.raise_for_status()
+ result_json: Dict[str, str] = res.json()
+ return result_json
+ return {}
+
+
+def image_id_from_registry(image_name: str) -> Optional[str]:
+ """Get the docker id from a public or private registry."""
+ registry, repository, tag = parse(image_name)
+ res = None
+ try:
+ token = auth_token(registry, repository).get("token")
+ # dockerhub is crazy
+ if registry == "index.docker.io":
+ registry = "registry-1.docker.io"
+ res = requests.head(
+ f"https://{registry}/v2/{repository}/manifests/{tag}",
+ headers={
+ "Authorization": f"Bearer {token}",
+ "Accept": "application/vnd.docker.distribution.manifest.v2+json",
+ },
+ timeout=5,
+ )
+ res.raise_for_status()
+ except requests.RequestException:
+ log.error(f"Received {res} when attempting to get digest for {image_name}")
+ return None
+ return "@".join([registry + "/" + repository, res.headers["Docker-Content-Digest"]])
+
+
+def image_id(image_name: str) -> Optional[str]:
+ """Retreve the image id from the local docker daemon or remote registry."""
+ if "@sha256:" in image_name:
+ return image_name
+ else:
+ digests = shell(["inspect", image_name, "--format", "{{json .RepoDigests}}"])
+ try:
+ if digests is None:
+ raise ValueError
+ im_id: str = json.loads(digests)[0]
+ return im_id
+ except (ValueError, IndexError):
+ return image_id_from_registry(image_name)
+
+
+def get_image_uid(image_name: str) -> int:
+ """Retrieve the image default uid through brute force."""
+ image_uid = shell(["run", image_name, "id", "-u"])
+ return int(image_uid) if image_uid else -1
+
+
+def push(image: str, tag: str) -> Optional[str]:
+ """Push an image to a remote registry."""
+ return shell(["push", f"{image}:{tag}"])
+
+
+def login(username: str, password: str, registry: str) -> Optional[str]:
+ """Login to a registry."""
+ return shell(["login", "--username", username, "--password", password, registry])
+
+
+def tag(image_name: str, tag: str) -> Optional[str]:
+ """Tag an image."""
+ return shell(["tag", image_name, tag])
+
+
+__all__ = [
+ "shell",
+ "build",
+ "run",
+ "image_id",
+ "image_id_from_registry",
+ "is_docker_installed",
+ "auth_token",
+ "parse",
+ "parse_repository_tag",
+ "default_image",
+ "get_image_uid",
+ "push",
+ "login",
+ "tag",
+]
diff --git a/wandb/docker/auth.py b/wandb/docker/auth.py
new file mode 100644
index 0000000000000000000000000000000000000000..5ce97d18ab7d21acd6750e59b48764a75a98e25c
--- /dev/null
+++ b/wandb/docker/auth.py
@@ -0,0 +1,436 @@
+# originally: https://github.com/docker/docker-py/blob/master/docker/auth.py
+import base64
+import json
+import logging
+import os
+import platform
+from typing import Any, Dict, Mapping, Optional, Tuple, Union
+
+import dockerpycreds # type: ignore
+
+IS_WINDOWS_PLATFORM = platform.system() == "Windows"
+DOCKER_CONFIG_FILENAME = os.path.join(".docker", "config.json")
+LEGACY_DOCKER_CONFIG_FILENAME = ".dockercfg"
+INDEX_NAME = "docker.io"
+INDEX_URL = f"https://index.{INDEX_NAME}/v1/"
+TOKEN_USERNAME = "<token>"
+
+log = logging.getLogger(__name__)
+
+
+class DockerError(Exception):
+ """Base class from which all other exceptions inherit.
+
+ If you want to catch all errors that the Docker SDK might raise,
+ catch this base exception.
+ """
+
+
+class InvalidConfigFileError(DockerError):
+ pass
+
+
+class InvalidRepositoryError(DockerError):
+ pass
+
+
+def find_config_file(config_path: Optional[str] = None) -> Optional[str]:
+ paths = list(
+ filter(
+ None,
+ [
+ config_path, # 1
+ config_path_from_environment(), # 2
+ os.path.join(home_dir(), DOCKER_CONFIG_FILENAME), # 3
+ os.path.join(home_dir(), LEGACY_DOCKER_CONFIG_FILENAME), # 4
+ ],
+ )
+ )
+
+ log.debug(f"Trying paths: {repr(paths)}")
+
+ for path in paths:
+ if os.path.exists(path):
+ log.debug(f"Found file at path: {path}")
+ return path
+
+ log.debug("No config file found")
+
+ return None
+
+
+def config_path_from_environment() -> Optional[str]:
+ config_dir = os.environ.get("DOCKER_CONFIG")
+ if not config_dir:
+ return None
+ return os.path.join(config_dir, os.path.basename(DOCKER_CONFIG_FILENAME))
+
+
+def home_dir() -> str:
+ """Get the user's home directory.
+
+ Uses the same logic as the Docker Engine client - use %USERPROFILE% on Windows,
+ $HOME/getuid on POSIX.
+ """
+ if IS_WINDOWS_PLATFORM:
+ return os.environ.get("USERPROFILE", "")
+ else:
+ return os.path.expanduser("~")
+
+
+def load_general_config(config_path: Optional[str] = None) -> Dict:
+ config_file = find_config_file(config_path)
+
+ if not config_file:
+ return {}
+
+ try:
+ with open(config_file) as f:
+ conf: Dict = json.load(f)
+ return conf
+ except (OSError, ValueError) as e:
+ # In the case of a legacy `.dockercfg` file, we won't
+ # be able to load any JSON data.
+ log.debug(e)
+
+ log.debug("All parsing attempts failed - returning empty config")
+ return {}
+
+
+def resolve_repository_name(repo_name: str) -> Tuple[str, str]:
+ if "://" in repo_name:
+ raise InvalidRepositoryError(
+ f"Repository name cannot contain a scheme ({repo_name})"
+ )
+
+ index_name, remote_name = split_repo_name(repo_name)
+ if index_name[0] == "-" or index_name[-1] == "-":
+ raise InvalidRepositoryError(
+ f"Invalid index name ({index_name}). Cannot begin or end with a hyphen."
+ )
+ return resolve_index_name(index_name), remote_name
+
+
+def resolve_index_name(index_name: str) -> str:
+ index_name = convert_to_hostname(index_name)
+ if index_name == "index." + INDEX_NAME:
+ index_name = INDEX_NAME
+ return index_name
+
+
+def split_repo_name(repo_name: str) -> Tuple[str, str]:
+ parts = repo_name.split("/", 1)
+ if len(parts) == 1 or (
+ "." not in parts[0] and ":" not in parts[0] and parts[0] != "localhost"
+ ):
+ # This is a docker index repo (ex: username/foobar or ubuntu)
+ return INDEX_NAME, repo_name
+ return parts[0], parts[1]
+
+
+def get_credential_store(authconfig: Dict, registry: str) -> Optional[str]:
+ if not isinstance(authconfig, AuthConfig):
+ authconfig = AuthConfig(authconfig)
+ return authconfig.get_credential_store(registry)
+
+
+class AuthConfig(dict):
+ def __init__(self, dct: Dict, credstore_env: Optional[Mapping] = None) -> None:
+ super().__init__(dct)
+ if "auths" not in dct:
+ dct["auths"] = {}
+ self.update(dct)
+ self._credstore_env = credstore_env
+ self._stores: Dict[str, dockerpycreds.Store] = dict()
+
+ @classmethod
+ def parse_auth(
+ cls,
+ entries: Dict[str, Dict[str, Any]],
+ raise_on_error: bool = False,
+ ) -> Dict[str, Dict[str, Any]]:
+ """Parse authentication entries.
+
+ Arguments:
+ entries: Dict of authentication entries.
+ raise_on_error: If set to true, an invalid format will raise
+ InvalidConfigFileError
+ Returns:
+ Authentication registry.
+ """
+ conf = {}
+ for registry, entry in entries.items():
+ if not isinstance(entry, dict):
+ log.debug(f"Config entry for key {registry} is not auth config") # type: ignore
+ # We sometimes fall back to parsing the whole config as if it
+ # was the auth config by itself, for legacy purposes. In that
+ # case, we fail silently and return an empty conf if any of the
+ # keys is not formatted properly.
+ if raise_on_error:
+ raise InvalidConfigFileError(
+ f"Invalid configuration for registry {registry}"
+ )
+ return {}
+ if "identitytoken" in entry:
+ log.debug(f"Found an IdentityToken entry for registry {registry}")
+ conf[registry] = {"IdentityToken": entry["identitytoken"]}
+ continue # Other values are irrelevant if we have a token
+
+ if "auth" not in entry:
+ # Starting with engine v1.11 (API 1.23), an empty dictionary is
+ # a valid value in the auth's config.
+ # https://github.com/docker/compose/issues/3265
+ log.debug(
+ f"Auth data for {registry} is absent. Client might be using a "
+ "credentials store instead."
+ )
+ conf[registry] = {}
+ continue
+
+ username, password = decode_auth(entry["auth"])
+ log.debug(
+ f"Found entry (registry={repr(registry)}, username={repr(username)})"
+ )
+
+ conf[registry] = {
+ "username": username,
+ "password": password,
+ "email": entry.get("email"),
+ "serveraddress": registry,
+ }
+ return conf
+
+ @classmethod
+ def load_config(
+ cls,
+ config_path: Optional[str],
+ config_dict: Optional[Dict[str, Any]],
+ credstore_env: Optional[Mapping] = None,
+ ) -> "AuthConfig":
+ """Load authentication data from a Docker configuration file.
+
+ If the config_path is not passed in it looks for a configuration file in the
+ root directory.
+
+ Lookup priority:
+ explicit config_path parameter > DOCKER_CONFIG environment
+ variable > ~/.docker/config.json > ~/.dockercfg.
+ """
+ if not config_dict:
+ config_file = find_config_file(config_path)
+
+ if not config_file:
+ return cls({}, credstore_env)
+ try:
+ with open(config_file) as f:
+ config_dict = json.load(f)
+ except (OSError, KeyError, ValueError) as e:
+ # Likely missing new Docker config file, or it's in an
+ # unknown format, continue to attempt to read old location
+ # and format.
+ log.debug(e)
+ return cls(_load_legacy_config(config_file), credstore_env)
+
+ res = {}
+ assert isinstance(config_dict, Dict) # worship mypy
+ if config_dict.get("auths"):
+ log.debug("Found 'auths' section")
+ res.update(
+ {"auths": cls.parse_auth(config_dict.pop("auths"), raise_on_error=True)}
+ )
+ if config_dict.get("credsStore"):
+ log.debug("Found 'credsStore' section")
+ res.update({"credsStore": config_dict.pop("credsStore")})
+ if config_dict.get("credHelpers"):
+ log.debug("Found 'credHelpers' section")
+ res.update({"credHelpers": config_dict.pop("credHelpers")})
+ if res:
+ return cls(res, credstore_env)
+
+ log.debug(
+ "Couldn't find auth-related section ; attempting to interpret "
+ "as auth-only file"
+ )
+ return cls({"auths": cls.parse_auth(config_dict)}, credstore_env)
+
+ @property
+ def auths(self) -> Dict[str, Dict[str, Any]]:
+ return self.get("auths", {}) # type: ignore
+
+ @property
+ def creds_store(self) -> Optional[str]:
+ return self.get("credsStore", None) # type: ignore
+
+ @property
+ def cred_helpers(self) -> Dict:
+ return self.get("credHelpers", {}) # type: ignore
+
+ @property
+ def is_empty(self) -> bool:
+ return not self.auths and not self.creds_store and not self.cred_helpers
+
+ def resolve_authconfig(
+ self, registry: Optional[str] = None
+ ) -> Optional[Dict[str, Any]]:
+ """Return the authentication data for a specific registry.
+
+ As with the Docker client, legacy entries in the config with full URLs are
+ stripped down to hostnames before checking for a match. Returns None if no match
+ was found.
+ """
+ if self.creds_store or self.cred_helpers:
+ store_name = self.get_credential_store(registry)
+ if store_name is not None:
+ log.debug(f"Using credentials store {store_name!r}")
+ cfg = self._resolve_authconfig_credstore(registry, store_name)
+ if cfg is not None:
+ return cfg
+ log.debug("No entry in credstore - fetching from auth dict")
+
+ # Default to the public index server
+ registry = resolve_index_name(registry) if registry else INDEX_NAME
+ log.debug(f"Looking for auth entry for {repr(registry)}")
+
+ if registry in self.auths:
+ log.debug(f"Found {repr(registry)}")
+ return self.auths[registry]
+
+ for key, conf in self.auths.items():
+ if resolve_index_name(key) == registry:
+ log.debug(f"Found {repr(key)}")
+ return conf
+
+ log.debug("No entry found")
+ return None
+
+ def _resolve_authconfig_credstore(
+ self, registry: Optional[str], credstore_name: str
+ ) -> Optional[Dict[str, Any]]:
+ if not registry or registry == INDEX_NAME:
+ # The ecosystem is a little schizophrenic with recker.io VS
+ # docker.io - in that case, it seems the full URL is necessary.
+ registry = INDEX_URL
+ log.debug(f"Looking for auth entry for {repr(registry)}")
+ store = self._get_store_instance(credstore_name)
+ try:
+ data = store.get(registry)
+ res = {
+ "ServerAddress": registry,
+ }
+ if data["Username"] == TOKEN_USERNAME:
+ res["IdentityToken"] = data["Secret"]
+ else:
+ res.update({"Username": data["Username"], "Password": data["Secret"]})
+ return res
+ except (dockerpycreds.CredentialsNotFound, ValueError):
+ log.debug("No entry found")
+ return None
+ except dockerpycreds.StoreError as e:
+ raise DockerError(f"Credentials store error: {repr(e)}")
+
+ def _get_store_instance(self, name: str) -> "dockerpycreds.Store":
+ if name not in self._stores:
+ self._stores[name] = dockerpycreds.Store(
+ name, environment=self._credstore_env
+ )
+ return self._stores[name]
+
+ def get_credential_store(self, registry: Optional[str]) -> Optional[str]:
+ if not registry or registry == INDEX_NAME:
+ registry = INDEX_URL
+
+ return self.cred_helpers.get(registry) or self.creds_store
+
+ def get_all_credentials(self) -> Dict[str, Dict[str, Any]]:
+ auth_data = self.auths.copy()
+ if self.creds_store:
+ # Retrieve all credentials from the default store
+ store = self._get_store_instance(self.creds_store)
+ for k in store.list().keys():
+ auth_data[k] = self._resolve_authconfig_credstore(k, self.creds_store) # type: ignore
+
+ # credHelpers entries take priority over all others
+ for reg, store_name in self.cred_helpers.items():
+ auth_data[reg] = self._resolve_authconfig_credstore(reg, store_name) # type: ignore
+
+ return auth_data
+
+ def add_auth(self, reg: str, data: Dict[str, Any]) -> None:
+ self["auths"][reg] = data
+
+
+def resolve_authconfig(
+ authconfig: Dict,
+ registry: Optional[str] = None,
+ credstore_env: Optional[Mapping] = None,
+) -> Optional[Dict[str, Any]]:
+ if not isinstance(authconfig, AuthConfig):
+ authconfig = AuthConfig(authconfig, credstore_env)
+ return authconfig.resolve_authconfig(registry)
+
+
+def convert_to_hostname(url: str) -> str:
+ return url.replace("http://", "").replace("https://", "").split("/", 1)[0]
+
+
+def decode_auth(auth: Union[str, bytes]) -> Tuple[str, str]:
+ if isinstance(auth, str):
+ auth = auth.encode("ascii")
+ s = base64.b64decode(auth)
+ login, pwd = s.split(b":", 1)
+ return login.decode("utf8"), pwd.decode("utf8")
+
+
+def parse_auth(
+ entries: Dict, raise_on_error: bool = False
+) -> Dict[str, Dict[str, Any]]:
+ """Parse authentication entries.
+
+ Arguments:
+ entries: Dict of authentication entries.
+ raise_on_error: If set to true, an invalid format will raise
+ InvalidConfigFileError
+ Returns:
+ Authentication registry.
+ """
+ return AuthConfig.parse_auth(entries, raise_on_error)
+
+
+def load_config(
+ config_path: Optional[str] = None,
+ config_dict: Optional[Dict[str, Any]] = None,
+ credstore_env: Optional[Mapping] = None,
+) -> AuthConfig:
+ return AuthConfig.load_config(config_path, config_dict, credstore_env)
+
+
+def _load_legacy_config(
+ config_file: str,
+) -> Dict[str, Dict[str, Union[str, Dict[str, str]]]]:
+ log.debug("Attempting to parse legacy auth file format")
+ try:
+ data = []
+ with open(config_file) as f:
+ for line in f.readlines():
+ data.append(line.strip().split(" = ")[1])
+ if len(data) < 2:
+ # Not enough data
+ raise InvalidConfigFileError("Invalid or empty configuration file!")
+
+ username, password = decode_auth(data[0])
+ return {
+ "auths": {
+ INDEX_NAME: {
+ "username": username,
+ "password": password,
+ "email": data[1],
+ "serveraddress": INDEX_URL,
+ }
+ }
+ }
+ except Exception as e:
+ log.debug(e)
+ pass
+
+ log.debug("All parsing attempts failed - returning empty config")
+ return {}
diff --git a/wandb/docker/wandb-entrypoint.sh b/wandb/docker/wandb-entrypoint.sh
new file mode 100755
index 0000000000000000000000000000000000000000..f2d4af61de2eb53973f8ee5eb6e619521d66a63a
--- /dev/null
+++ b/wandb/docker/wandb-entrypoint.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+set -e
+
+wandb="\x1b[34m\x1b[1mwandb\x1b[0m"
+/bin/echo -e "${wandb}: Checking image for required packages."
+
+if ! [ -x "$(command -v python)" ]; then
+ /bin/echo -e "${wandb}: python not installed, can't use wandb with this image."
+ exit 1
+fi
+
+if ! [ -x "$(command -v wandb)" ]; then
+ /bin/echo -e "${wandb}: wandb not installed, installing."
+ pip install wandb --upgrade
+else
+ ver=$(wandb --version)
+ /bin/echo -e "${wandb}: Found $ver"
+fi
+
+if [ "$WANDB_ENSURE_JUPYTER" = "1" ]; then
+ if ! [ -x "$(command -v jupyter-lab)" ]; then
+ /bin/echo -e "${wandb}: jupyter not installed, installing."
+ pip install jupyterlab
+ /bin/echo -e "${wandb}: starting jupyter, you can access it at: http://127.0.0.1:8888"
+ fi
+fi
+
+if ! [ -z "$WANDB_COMMAND" ]; then
+ /bin/echo $WANDB_COMMAND >> ~/.bash_history
+ /bin/echo -e "${wandb}: Command added to history, press up arrow to access it."
+ /bin/echo -e "${wandb}: $WANDB_COMMAND"
+fi
+exec "$@"
\ No newline at end of file
diff --git a/wandb/docker/www_authenticate.py b/wandb/docker/www_authenticate.py
new file mode 100644
index 0000000000000000000000000000000000000000..b64b6da4ddd5a2f1780a06f09cc2d309e9607d96
--- /dev/null
+++ b/wandb/docker/www_authenticate.py
@@ -0,0 +1,94 @@
+# Taken from: https://github.com/alexsdutton/www-authenticate
+import re
+from collections import OrderedDict
+from typing import Any, Optional
+
+_tokens = (
+ ("token", re.compile(r"""^([!#$%&'*+\-.^_`|~\w/]+(?:={1,2}$)?)""")),
+ ("token", re.compile(r'''^"((?:[^"\\]|\\\\|\\")+)"''')),
+ (None, re.compile(r"^\s+")),
+ ("equals", re.compile(r"^(=)")),
+ ("comma", re.compile(r"^(,)")),
+)
+
+
+def _casefold(value: str) -> str:
+ try:
+ return value.casefold()
+ except AttributeError:
+ return value.lower()
+
+
+class CaseFoldedOrderedDict(OrderedDict):
+ def __getitem__(self, key: str) -> Any:
+ return super().__getitem__(_casefold(key))
+
+ def __setitem__(self, key: str, value: Any) -> None:
+ super().__setitem__(_casefold(key), value)
+
+ def __contains__(self, key: object) -> bool:
+ return super().__contains__(_casefold(key)) # type: ignore
+
+ def get(self, key: str, default: Optional[Any] = None) -> Any:
+ return super().get(_casefold(key), default)
+
+ def pop(self, key: str, default: Optional[Any] = None) -> Any:
+ return super().pop(_casefold(key), default)
+
+
+def _group_pairs(tokens: list) -> None:
+ i = 0
+ while i < len(tokens) - 2:
+ if (
+ tokens[i][0] == "token"
+ and tokens[i + 1][0] == "equals"
+ and tokens[i + 2][0] == "token"
+ ):
+ tokens[i : i + 3] = [("pair", (tokens[i][1], tokens[i + 2][1]))]
+ i += 1
+
+
+def _group_challenges(tokens: list) -> list:
+ challenges = []
+ while tokens:
+ j = 1
+ if len(tokens) == 1:
+ pass
+ elif tokens[1][0] == "comma":
+ pass
+ elif tokens[1][0] == "token":
+ j = 2
+ else:
+ while j < len(tokens) and tokens[j][0] == "pair":
+ j += 2
+ j -= 1
+ challenges.append((tokens[0][1], tokens[1:j]))
+ tokens[: j + 1] = []
+ return challenges
+
+
+def parse(value: str) -> CaseFoldedOrderedDict:
+ tokens = []
+ while value:
+ for token_name, pattern in _tokens:
+ match = pattern.match(value)
+ if match:
+ value = value[match.end() :]
+ if token_name:
+ tokens.append((token_name, match.group(1)))
+ break
+ else:
+ raise ValueError("Failed to parse value")
+ _group_pairs(tokens)
+
+ challenges = CaseFoldedOrderedDict()
+ for name, tokens in _group_challenges(tokens): # noqa: B020
+ args, kwargs = [], {}
+ for token_name, value in tokens:
+ if token_name == "token":
+ args.append(value)
+ elif token_name == "pair":
+ kwargs[value[0]] = value[1]
+ challenges[name] = (args and args[0]) or kwargs or None
+
+ return challenges
diff --git a/wandb/env.py b/wandb/env.py
new file mode 100644
index 0000000000000000000000000000000000000000..50d7460724f35977244098ea57c986b54dd90e82
--- /dev/null
+++ b/wandb/env.py
@@ -0,0 +1,466 @@
+#!/usr/bin/env python
+
+"""All of W&B's environment variables.
+
+Getters and putters for all of them should go here. That
+way it'll be easier to avoid typos with names and be
+consistent about environment variables' semantics.
+
+Environment variables are not the authoritative source for
+these values in many cases.
+"""
+
+import json
+import os
+import sys
+from distutils.util import strtobool
+from pathlib import Path
+from typing import List, MutableMapping, Optional, Union
+
+import appdirs # type: ignore
+
+Env = Optional[MutableMapping]
+
+CONFIG_PATHS = "WANDB_CONFIG_PATHS"
+SWEEP_PARAM_PATH = "WANDB_SWEEP_PARAM_PATH"
+SHOW_RUN = "WANDB_SHOW_RUN"
+DEBUG = "WANDB_DEBUG"
+SILENT = "WANDB_SILENT"
+QUIET = "WANDB_QUIET"
+INITED = "WANDB_INITED"
+DIR = "WANDB_DIR"
+# Deprecate DESCRIPTION in a future release
+DESCRIPTION = "WANDB_DESCRIPTION"
+NAME = "WANDB_NAME"
+NOTEBOOK_NAME = "WANDB_NOTEBOOK_NAME"
+NOTES = "WANDB_NOTES"
+USERNAME = "WANDB_USERNAME"
+USER_EMAIL = "WANDB_USER_EMAIL"
+PROJECT = "WANDB_PROJECT"
+ENTITY = "WANDB_ENTITY"
+BASE_URL = "WANDB_BASE_URL"
+APP_URL = "WANDB_APP_URL"
+PROGRAM = "WANDB_PROGRAM"
+ARGS = "WANDB_ARGS"
+MODE = "WANDB_MODE"
+START_METHOD = "WANDB_START_METHOD"
+RESUME = "WANDB_RESUME"
+RUN_ID = "WANDB_RUN_ID"
+RUN_STORAGE_ID = "WANDB_RUN_STORAGE_ID"
+RUN_GROUP = "WANDB_RUN_GROUP"
+RUN_DIR = "WANDB_RUN_DIR"
+SWEEP_ID = "WANDB_SWEEP_ID"
+HTTP_TIMEOUT = "WANDB_HTTP_TIMEOUT"
+FILE_PUSHER_TIMEOUT = "WANDB_FILE_PUSHER_TIMEOUT"
+API_KEY = "WANDB_API_KEY"
+JOB_TYPE = "WANDB_JOB_TYPE"
+DISABLE_CODE = "WANDB_DISABLE_CODE"
+DISABLE_GIT = "WANDB_DISABLE_GIT"
+GIT_ROOT = "WANDB_GIT_ROOT"
+SAVE_CODE = "WANDB_SAVE_CODE"
+TAGS = "WANDB_TAGS"
+IGNORE = "WANDB_IGNORE_GLOBS"
+ERROR_REPORTING = "WANDB_ERROR_REPORTING"
+DOCKER = "WANDB_DOCKER"
+AGENT_REPORT_INTERVAL = "WANDB_AGENT_REPORT_INTERVAL"
+AGENT_KILL_DELAY = "WANDB_AGENT_KILL_DELAY"
+AGENT_DISABLE_FLAPPING = "WANDB_AGENT_DISABLE_FLAPPING"
+AGENT_MAX_INITIAL_FAILURES = "WANDB_AGENT_MAX_INITIAL_FAILURES"
+CRASH_NOSYNC_TIME = "WANDB_CRASH_NOSYNC_TIME"
+MAGIC = "WANDB_MAGIC"
+HOST = "WANDB_HOST"
+ANONYMOUS = "WANDB_ANONYMOUS"
+JUPYTER = "WANDB_JUPYTER"
+CONFIG_DIR = "WANDB_CONFIG_DIR"
+DATA_DIR = "WANDB_DATA_DIR"
+ARTIFACT_DIR = "WANDB_ARTIFACT_DIR"
+ARTIFACT_FETCH_FILE_URL_BATCH_SIZE = "WANDB_ARTIFACT_FETCH_FILE_URL_BATCH_SIZE"
+CACHE_DIR = "WANDB_CACHE_DIR"
+DISABLE_SSL = "WANDB_INSECURE_DISABLE_SSL"
+SERVICE = "WANDB_SERVICE"
+_DISABLE_SERVICE = "WANDB_DISABLE_SERVICE"
+SENTRY_DSN = "WANDB_SENTRY_DSN"
+INIT_TIMEOUT = "WANDB_INIT_TIMEOUT"
+GIT_COMMIT = "WANDB_GIT_COMMIT"
+GIT_REMOTE_URL = "WANDB_GIT_REMOTE_URL"
+_EXECUTABLE = "WANDB_EXECUTABLE"
+LAUNCH_QUEUE_NAME = "WANDB_LAUNCH_QUEUE_NAME"
+LAUNCH_QUEUE_ENTITY = "WANDB_LAUNCH_QUEUE_ENTITY"
+LAUNCH_TRACE_ID = "WANDB_LAUNCH_TRACE_ID"
+
+# For testing, to be removed in future version
+USE_V1_ARTIFACTS = "_WANDB_USE_V1_ARTIFACTS"
+
+
+def immutable_keys() -> List[str]:
+ """These are env keys that shouldn't change within a single process.
+
+ We use this to maintain certain values between multiple calls to wandb.init within a single process.
+ """
+ return [
+ DIR,
+ ENTITY,
+ PROJECT,
+ API_KEY,
+ IGNORE,
+ DISABLE_CODE,
+ DISABLE_GIT,
+ DOCKER,
+ MODE,
+ BASE_URL,
+ ERROR_REPORTING,
+ CRASH_NOSYNC_TIME,
+ MAGIC,
+ USERNAME,
+ USER_EMAIL,
+ DIR,
+ SILENT,
+ CONFIG_PATHS,
+ ANONYMOUS,
+ RUN_GROUP,
+ JOB_TYPE,
+ TAGS,
+ RESUME,
+ AGENT_REPORT_INTERVAL,
+ HTTP_TIMEOUT,
+ HOST,
+ DATA_DIR,
+ ARTIFACT_DIR,
+ ARTIFACT_FETCH_FILE_URL_BATCH_SIZE,
+ CACHE_DIR,
+ USE_V1_ARTIFACTS,
+ DISABLE_SSL,
+ ]
+
+
+def _env_as_bool(
+ var: str, default: Optional[str] = None, env: Optional[Env] = None
+) -> bool:
+ if env is None:
+ env = os.environ
+ val = env.get(var, default)
+ try:
+ val = bool(strtobool(val)) # type: ignore
+ except (AttributeError, ValueError):
+ pass
+ return val if isinstance(val, bool) else False
+
+
+def is_debug(default: Optional[str] = None, env: Optional[Env] = None) -> bool:
+ return _env_as_bool(DEBUG, default=default, env=env)
+
+
+def error_reporting_enabled() -> bool:
+ return _env_as_bool(ERROR_REPORTING, default="True")
+
+
+def ssl_disabled() -> bool:
+ return _env_as_bool(DISABLE_SSL, default="False")
+
+
+def get_error_reporting(
+ default: Union[bool, str] = True,
+ env: Optional[Env] = None,
+) -> Union[bool, str]:
+ if env is None:
+ env = os.environ
+
+ return env.get(ERROR_REPORTING, default)
+
+
+def get_run(default: Optional[str] = None, env: Optional[Env] = None) -> Optional[str]:
+ if env is None:
+ env = os.environ
+
+ return env.get(RUN_ID, default)
+
+
+def get_args(
+ default: Optional[List[str]] = None, env: Optional[Env] = None
+) -> Optional[List[str]]:
+ if env is None:
+ env = os.environ
+ if env.get(ARGS):
+ try:
+ return json.loads(env.get(ARGS, "[]")) # type: ignore
+ except ValueError:
+ return None
+ else:
+ return default or sys.argv[1:]
+
+
+def get_docker(
+ default: Optional[str] = None, env: Optional[Env] = None
+) -> Optional[str]:
+ if env is None:
+ env = os.environ
+
+ return env.get(DOCKER, default)
+
+
+def get_http_timeout(default: int = 20, env: Optional[Env] = None) -> int:
+ if env is None:
+ env = os.environ
+
+ return int(env.get(HTTP_TIMEOUT, default))
+
+
+def get_file_pusher_timeout(
+ default: Optional[int] = None,
+ env: Optional[Env] = None,
+) -> Optional[int]:
+ if env is None:
+ env = os.environ
+
+ timeout = env.get(FILE_PUSHER_TIMEOUT, default)
+ return int(timeout) if timeout else None
+
+
+def get_ignore(
+ default: Optional[List[str]] = None, env: Optional[Env] = None
+) -> Optional[List[str]]:
+ if env is None:
+ env = os.environ
+ ignore = env.get(IGNORE)
+ if ignore is not None:
+ return ignore.split(",")
+ else:
+ return default
+
+
+def get_project(
+ default: Optional[str] = None, env: Optional[Env] = None
+) -> Optional[str]:
+ if env is None:
+ env = os.environ
+
+ return env.get(PROJECT, default)
+
+
+def get_username(
+ default: Optional[str] = None, env: Optional[Env] = None
+) -> Optional[str]:
+ if env is None:
+ env = os.environ
+
+ return env.get(USERNAME, default)
+
+
+def get_user_email(
+ default: Optional[str] = None, env: Optional[Env] = None
+) -> Optional[str]:
+ if env is None:
+ env = os.environ
+
+ return env.get(USER_EMAIL, default)
+
+
+def get_entity(
+ default: Optional[str] = None, env: Optional[Env] = None
+) -> Optional[str]:
+ if env is None:
+ env = os.environ
+
+ return env.get(ENTITY, default)
+
+
+def get_base_url(
+ default: Optional[str] = None, env: Optional[Env] = None
+) -> Optional[str]:
+ if env is None:
+ env = os.environ
+
+ base_url = env.get(BASE_URL, default)
+
+ return base_url.rstrip("/") if base_url is not None else base_url
+
+
+def get_app_url(
+ default: Optional[str] = None, env: Optional[Env] = None
+) -> Optional[str]:
+ if env is None:
+ env = os.environ
+
+ return env.get(APP_URL, default)
+
+
+def get_show_run(default: Optional[str] = None, env: Optional[Env] = None) -> bool:
+ if env is None:
+ env = os.environ
+
+ return bool(env.get(SHOW_RUN, default))
+
+
+def get_description(
+ default: Optional[str] = None, env: Optional[Env] = None
+) -> Optional[str]:
+ if env is None:
+ env = os.environ
+
+ return env.get(DESCRIPTION, default)
+
+
+def get_tags(default: str = "", env: Optional[Env] = None) -> List[str]:
+ if env is None:
+ env = os.environ
+
+ return [tag for tag in env.get(TAGS, default).split(",") if tag]
+
+
+def get_dir(default: Optional[str] = None, env: Optional[Env] = None) -> Optional[str]:
+ if env is None:
+ env = os.environ
+ return env.get(DIR, default)
+
+
+def get_config_paths(
+ default: Optional[str] = None, env: Optional[Env] = None
+) -> Optional[str]:
+ if env is None:
+ env = os.environ
+ return env.get(CONFIG_PATHS, default)
+
+
+def get_agent_report_interval(
+ default: Optional[str] = None, env: Optional[Env] = None
+) -> Optional[int]:
+ if env is None:
+ env = os.environ
+ val = env.get(AGENT_REPORT_INTERVAL, default)
+ try:
+ val = int(val) # type: ignore
+ except ValueError:
+ val = None # silently ignore env format errors, caller should handle.
+ return val
+
+
+def get_agent_kill_delay(
+ default: Optional[str] = None, env: Optional[Env] = None
+) -> Optional[int]:
+ if env is None:
+ env = os.environ
+ val = env.get(AGENT_KILL_DELAY, default)
+ try:
+ val = int(val) # type: ignore
+ except ValueError:
+ val = None # silently ignore env format errors, caller should handle.
+ return val
+
+
+def get_crash_nosync_time(
+ default: Optional[str] = None, env: Optional[Env] = None
+) -> Optional[int]:
+ if env is None:
+ env = os.environ
+ val = env.get(CRASH_NOSYNC_TIME, default)
+ try:
+ val = int(val) # type: ignore
+ except ValueError:
+ val = None # silently ignore env format errors, caller should handle.
+ return val
+
+
+def get_magic(
+ default: Optional[str] = None, env: Optional[Env] = None
+) -> Optional[str]:
+ if env is None:
+ env = os.environ
+ val = env.get(MAGIC, default)
+ return val
+
+
+def get_data_dir(env: Optional[Env] = None) -> str:
+ default_dir = appdirs.user_data_dir("wandb")
+ if env is None:
+ env = os.environ
+ val = env.get(DATA_DIR, default_dir)
+ return val
+
+
+def get_artifact_dir(env: Optional[Env] = None) -> str:
+ default_dir = os.path.join(".", "artifacts")
+ if env is None:
+ env = os.environ
+ val = env.get(ARTIFACT_DIR, default_dir)
+ return os.path.abspath(val)
+
+
+def get_artifact_fetch_file_url_batch_size(env: Optional[Env] = None) -> int:
+ default_batch_size = 5000
+ if env is None:
+ env = os.environ
+ val = int(env.get(ARTIFACT_FETCH_FILE_URL_BATCH_SIZE, default_batch_size))
+ return val
+
+
+def get_cache_dir(env: Optional[Env] = None) -> Path:
+ env = env or os.environ
+ return Path(env.get(CACHE_DIR, appdirs.user_cache_dir("wandb")))
+
+
+def get_use_v1_artifacts(env: Optional[Env] = None) -> bool:
+ if env is None:
+ env = os.environ
+ val = bool(env.get(USE_V1_ARTIFACTS, False))
+ return val
+
+
+def get_agent_max_initial_failures(
+ default: Optional[int] = None, env: Optional[Env] = None
+) -> Optional[int]:
+ if env is None:
+ env = os.environ
+ val = env.get(AGENT_MAX_INITIAL_FAILURES, default)
+ try:
+ val = int(val) # type: ignore
+ except ValueError:
+ val = default
+ return val
+
+
+def set_entity(value: str, env: Optional[Env] = None) -> None:
+ if env is None:
+ env = os.environ
+ env[ENTITY] = value
+
+
+def set_project(value: str, env: Optional[Env] = None) -> None:
+ if env is None:
+ env = os.environ
+ env[PROJECT] = value or "uncategorized"
+
+
+def should_save_code() -> bool:
+ save_code = _env_as_bool(SAVE_CODE, default="False")
+ code_disabled = _env_as_bool(DISABLE_CODE, default="False")
+ return save_code and not code_disabled
+
+
+def disable_git(env: Optional[Env] = None) -> bool:
+ if env is None:
+ env = os.environ
+ val = env.get(DISABLE_GIT, default="False")
+ if isinstance(val, str):
+ val = False if val.lower() == "false" else True
+ return val
+
+
+def get_launch_queue_name(env: Optional[Env] = None) -> Optional[str]:
+ if env is None:
+ env = os.environ
+ val = env.get(LAUNCH_QUEUE_NAME, None)
+ return val
+
+
+def get_launch_queue_entity(env: Optional[Env] = None) -> Optional[str]:
+ if env is None:
+ env = os.environ
+ val = env.get(LAUNCH_QUEUE_ENTITY, None)
+ return val
+
+
+def get_launch_trace_id(env: Optional[Env] = None) -> Optional[str]:
+ if env is None:
+ env = os.environ
+ val = env.get(LAUNCH_TRACE_ID, None)
+ return val
diff --git a/wandb/errors/__init__.py b/wandb/errors/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..a0307f1f75e46f0c51f23ef54fc2038aedb24615
--- /dev/null
+++ b/wandb/errors/__init__.py
@@ -0,0 +1,41 @@
+__all__ = [
+ "Error",
+ "CommError",
+ "AuthenticationError",
+ "UsageError",
+ "UnsupportedError",
+]
+
+from typing import Optional
+
+
+class Error(Exception):
+ """Base W&B Error."""
+
+ def __init__(self, message, context: Optional[dict] = None) -> None:
+ super().__init__(message)
+ self.message = message
+ # sentry context capture
+ if context:
+ self.context = context
+
+
+class CommError(Error):
+ """Error communicating with W&B servers."""
+
+ def __init__(self, msg, exc=None) -> None:
+ self.exc = exc
+ self.message = msg
+ super().__init__(self.message)
+
+
+class AuthenticationError(CommError):
+ """Raised when authentication fails."""
+
+
+class UsageError(Error):
+ """Raised when an invalid usage of the SDK API is detected."""
+
+
+class UnsupportedError(UsageError):
+ """Raised when trying to use a feature that is not supported."""
diff --git a/wandb/errors/term.py b/wandb/errors/term.py
new file mode 100644
index 0000000000000000000000000000000000000000..382ea631e31ae548d5ce9832a34d3207e8875433
--- /dev/null
+++ b/wandb/errors/term.py
@@ -0,0 +1,95 @@
+import logging
+import sys
+from typing import Any
+
+import click
+
+LOG_STRING = click.style("wandb", fg="blue", bold=True)
+LOG_STRING_NOCOLOR = "wandb"
+ERROR_STRING = click.style("ERROR", bg="red", fg="green")
+WARN_STRING = click.style("WARNING", fg="yellow")
+PRINTED_MESSAGES = set() # type: ignore
+
+_silent = False
+_show_info = True
+_show_warnings = True
+_show_errors = True
+_logger = None
+
+
+def termsetup(settings, logger) -> None:
+ global _silent, _show_info, _show_warnings, _show_errors, _logger
+ _silent = settings.silent
+ _show_info = settings.show_info
+ _show_warnings = settings.show_warnings
+ _show_errors = settings.show_errors
+ _logger = logger
+
+
+def termlog(
+ string: str = "", newline: bool = True, repeat: bool = True, prefix: bool = True
+) -> None:
+ """Log to standard error with formatting.
+
+ Arguments:
+ string (str, optional): The string to print
+ newline (bool, optional): Print a newline at the end of the string
+ repeat (bool, optional): If set to False only prints the string once per process
+ """
+ _log(
+ string=string,
+ newline=newline,
+ repeat=repeat,
+ prefix=prefix,
+ silent=not _show_info,
+ )
+
+
+def termwarn(string: str, **kwargs: Any) -> None:
+ string = "\n".join([f"{WARN_STRING} {s}" for s in string.split("\n")])
+ _log(
+ string=string,
+ newline=True,
+ silent=not _show_warnings,
+ level=logging.WARNING,
+ **kwargs,
+ )
+
+
+def termerror(string: str, **kwargs: Any) -> None:
+ string = "\n".join([f"{ERROR_STRING} {s}" for s in string.split("\n")])
+ _log(
+ string=string,
+ newline=True,
+ silent=not _show_errors,
+ level=logging.ERROR,
+ **kwargs,
+ )
+
+
+def _log(
+ string="", newline=True, repeat=True, prefix=True, silent=False, level=logging.INFO
+):
+ global _logger
+ silent = silent or _silent
+ if string:
+ if prefix:
+ line = "\n".join([f"{LOG_STRING}: {s}" for s in string.split("\n")])
+ else:
+ line = string
+ else:
+ line = ""
+ if not repeat and line in PRINTED_MESSAGES:
+ return
+ # Repeated line tracking limited to 1k messages
+ if len(PRINTED_MESSAGES) < 1000:
+ PRINTED_MESSAGES.add(line)
+ if silent:
+ if level == logging.ERROR:
+ _logger.error(line)
+ elif level == logging.WARNING:
+ _logger.warning(line)
+ else:
+ _logger.info(line)
+ else:
+ click.echo(line, file=sys.stderr, nl=newline)
diff --git a/wandb/errors/util.py b/wandb/errors/util.py
new file mode 100644
index 0000000000000000000000000000000000000000..8a6d03c7e8f7e1ef103f139ae8523b4a09e78413
--- /dev/null
+++ b/wandb/errors/util.py
@@ -0,0 +1,57 @@
+from typing import Optional
+
+from wandb.proto import wandb_internal_pb2 as pb
+
+from . import AuthenticationError, CommError, Error, UnsupportedError, UsageError
+
+to_exception_map = {
+ pb.ErrorInfo.UNKNOWN: Error,
+ pb.ErrorInfo.COMMUNICATION: CommError,
+ pb.ErrorInfo.AUTHENTICATION: AuthenticationError,
+ pb.ErrorInfo.USAGE: UsageError,
+ pb.ErrorInfo.UNSUPPORTED: UnsupportedError,
+}
+
+from_exception_map = {v: k for k, v in to_exception_map.items()}
+
+
+class ProtobufErrorHandler:
+ """Converts protobuf errors to exceptions and vice versa."""
+
+ @staticmethod
+ def to_exception(error: pb.ErrorInfo) -> Optional[Error]:
+ """Convert a protobuf error to an exception.
+
+ Args:
+ error: The protobuf error to convert.
+
+ Returns:
+ The corresponding exception.
+
+ """
+ if not error.SerializeToString():
+ return None
+
+ if error.code in to_exception_map:
+ return to_exception_map[error.code](error.message)
+ return Error(error.message)
+
+ @classmethod
+ def from_exception(cls, exc: Error) -> "pb.ErrorInfo":
+ """Convert an wandb error to a protobuf error message.
+
+ Args:
+ exc: The exception to convert.
+
+ Returns:
+ The corresponding protobuf error message.
+ """
+ if not isinstance(exc, Error):
+ raise ValueError("exc must be a subclass of wandb.errors.Error")
+
+ code = None
+ for subclass in type(exc).__mro__:
+ if subclass in from_exception_map:
+ code = from_exception_map[subclass] # type: ignore
+ break
+ return pb.ErrorInfo(code=code, message=str(exc)) # type: ignore
diff --git a/wandb/fastai/__init__.py b/wandb/fastai/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..3cf89b7f51ce606e3c24876259f63c0292964dc1
--- /dev/null
+++ b/wandb/fastai/__init__.py
@@ -0,0 +1,9 @@
+"""Compatibility fastai module.
+
+In the future use:
+ from wandb.integration.fastai import WandbCallback
+"""
+
+from wandb.integration.fastai import WandbCallback
+
+__all__ = ["WandbCallback"]
diff --git a/wandb/filesync/__init__.py b/wandb/filesync/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/filesync/dir_watcher.py b/wandb/filesync/dir_watcher.py
new file mode 100644
index 0000000000000000000000000000000000000000..d952e15ee7c10900602cc0554139d749c26937b0
--- /dev/null
+++ b/wandb/filesync/dir_watcher.py
@@ -0,0 +1,403 @@
+import abc
+import fnmatch
+import glob
+import logging
+import os
+import queue
+import time
+from typing import TYPE_CHECKING, Any, Mapping, MutableMapping, MutableSet, Optional
+
+from wandb import util
+from wandb.sdk.interface.interface import GlobStr
+from wandb.sdk.lib.paths import LogicalPath
+
+if TYPE_CHECKING:
+ import wandb.vendor.watchdog_0_9_0.observers.api as wd_api
+ import wandb.vendor.watchdog_0_9_0.observers.polling as wd_polling
+ import wandb.vendor.watchdog_0_9_0.watchdog.events as wd_events
+ from wandb.sdk.interface.interface import PolicyName
+ from wandb.sdk.internal.file_pusher import FilePusher
+ from wandb.sdk.internal.settings_static import SettingsStatic
+else:
+ wd_polling = util.vendor_import("wandb_watchdog.observers.polling")
+ wd_events = util.vendor_import("wandb_watchdog.events")
+
+PathStr = str # TODO(spencerpearson): would be nice to use Path here
+
+
+logger = logging.getLogger(__name__)
+
+
+class FileEventHandler(abc.ABC):
+ def __init__(
+ self,
+ file_path: PathStr,
+ save_name: LogicalPath,
+ file_pusher: "FilePusher",
+ *args: Any,
+ **kwargs: Any,
+ ) -> None:
+ self.file_path = file_path
+ # Convert windows paths to unix paths
+ self.save_name = LogicalPath(save_name)
+ self._file_pusher = file_pusher
+ self._last_sync: Optional[float] = None
+
+ @property
+ @abc.abstractmethod
+ def policy(self) -> "PolicyName":
+ raise NotImplementedError
+
+ @abc.abstractmethod
+ def on_modified(self, force: bool = False) -> None:
+ raise NotImplementedError
+
+ @abc.abstractmethod
+ def finish(self) -> None:
+ raise NotImplementedError
+
+ def on_renamed(self, new_path: PathStr, new_name: LogicalPath) -> None:
+ self.file_path = new_path
+ self.save_name = new_name
+ self.on_modified()
+
+
+class PolicyNow(FileEventHandler):
+ """This policy only uploads files now."""
+
+ def on_modified(self, force: bool = False) -> None:
+ # only upload if we've never uploaded or when .save is called
+ if self._last_sync is None or force:
+ self._file_pusher.file_changed(self.save_name, self.file_path)
+ self._last_sync = os.path.getmtime(self.file_path)
+
+ def finish(self) -> None:
+ pass
+
+ @property
+ def policy(self) -> "PolicyName":
+ return "now"
+
+
+class PolicyEnd(FileEventHandler):
+ """This policy only updates at the end of the run."""
+
+ def on_modified(self, force: bool = False) -> None:
+ pass
+
+ # TODO: make sure we call this
+ def finish(self) -> None:
+ # We use copy=False to avoid possibly expensive copies, and because
+ # user files shouldn't still be changing at the end of the run.
+ self._last_sync = os.path.getmtime(self.file_path)
+ self._file_pusher.file_changed(self.save_name, self.file_path, copy=False)
+
+ @property
+ def policy(self) -> "PolicyName":
+ return "end"
+
+
+class PolicyLive(FileEventHandler):
+ """Event handler that uploads respecting throttling.
+
+ Uploads files every RATE_LIMIT_SECONDS, which changes as the size increases to deal
+ with throttling.
+ """
+
+ RATE_LIMIT_SECONDS = 15
+ unit_dict = dict(util.POW_10_BYTES)
+ # Wait to upload until size has increased 20% from last upload
+ RATE_LIMIT_SIZE_INCREASE = 1.2
+
+ def __init__(
+ self,
+ file_path: PathStr,
+ save_name: LogicalPath,
+ file_pusher: "FilePusher",
+ settings: Optional["SettingsStatic"] = None,
+ *args: Any,
+ **kwargs: Any,
+ ) -> None:
+ super().__init__(file_path, save_name, file_pusher, *args, **kwargs)
+ self._last_uploaded_time: Optional[float] = None
+ self._last_uploaded_size: int = 0
+ if settings is not None:
+ if settings._live_policy_rate_limit is not None:
+ self.RATE_LIMIT_SECONDS = settings._live_policy_rate_limit
+ self._min_wait_time: Optional[float] = settings._live_policy_wait_time
+ else:
+ self._min_wait_time = None
+
+ @property
+ def current_size(self) -> int:
+ return os.path.getsize(self.file_path)
+
+ @classmethod
+ def min_wait_for_size(cls, size: int) -> float:
+ if size < 10 * cls.unit_dict["MB"]:
+ return 60
+ elif size < 100 * cls.unit_dict["MB"]:
+ return 5 * 60
+ elif size < cls.unit_dict["GB"]:
+ return 10 * 60
+ else:
+ return 20 * 60
+
+ def should_update(self) -> bool:
+ if self._last_uploaded_time is not None:
+ # Check rate limit by time elapsed
+ time_elapsed = time.time() - self._last_uploaded_time
+ # if more than 15 seconds has passed potentially upload it
+ if time_elapsed < self.RATE_LIMIT_SECONDS:
+ return False
+
+ # Check rate limit by size increase
+ if float(self._last_uploaded_size) > 0:
+ size_increase = self.current_size / float(self._last_uploaded_size)
+ if size_increase < self.RATE_LIMIT_SIZE_INCREASE:
+ return False
+ return time_elapsed > (
+ self._min_wait_time or self.min_wait_for_size(self.current_size)
+ )
+
+ # if the file has never been uploaded, we'll upload it
+ return True
+
+ def on_modified(self, force: bool = False) -> None:
+ if self.current_size == 0:
+ return
+ if self._last_sync == os.path.getmtime(self.file_path):
+ return
+ if force or self.should_update():
+ self.save_file()
+
+ def save_file(self) -> None:
+ self._last_sync = os.path.getmtime(self.file_path)
+ self._last_uploaded_time = time.time()
+ self._last_uploaded_size = self.current_size
+ self._file_pusher.file_changed(self.save_name, self.file_path)
+
+ def finish(self) -> None:
+ self.on_modified(force=True)
+
+ @property
+ def policy(self) -> "PolicyName":
+ return "live"
+
+
+class DirWatcher:
+ def __init__(
+ self,
+ settings: "SettingsStatic",
+ file_pusher: "FilePusher",
+ file_dir: Optional[PathStr] = None,
+ ) -> None:
+ self._file_count = 0
+ self._dir = file_dir or settings.files_dir
+ self._settings = settings
+ self._savename_file_policies: MutableMapping[LogicalPath, PolicyName] = {}
+ self._user_file_policies: Mapping[PolicyName, MutableSet[GlobStr]] = {
+ "end": set(),
+ "live": set(),
+ "now": set(),
+ }
+ self._file_pusher = file_pusher
+ self._file_event_handlers: MutableMapping[LogicalPath, FileEventHandler] = {}
+ self._file_observer = wd_polling.PollingObserver()
+ self._file_observer.schedule(
+ self._per_file_event_handler(), self._dir, recursive=True
+ )
+ self._file_observer.start()
+ logger.info("watching files in: %s", settings.files_dir)
+
+ @property
+ def emitter(self) -> Optional["wd_api.EventEmitter"]:
+ try:
+ return next(iter(self._file_observer.emitters))
+ except StopIteration:
+ return None
+
+ def update_policy(self, path: GlobStr, policy: "PolicyName") -> None:
+ # When we're dealing with one of our own media files, there's no need
+ # to store the policy in memory. _get_file_event_handler will always
+ # return PolicyNow. Using the path makes syncing historic runs much
+ # faster if the name happens to include glob escapable characters. In
+ # the future we may add a flag to "files" records that indicates it's
+ # policy is not dynamic and doesn't need to be stored / checked.
+ save_name = LogicalPath(
+ os.path.relpath(os.path.join(self._dir, path), self._dir)
+ )
+ if save_name.startswith("media/"):
+ pass
+ elif path == glob.escape(path):
+ self._savename_file_policies[save_name] = policy
+ else:
+ self._user_file_policies[policy].add(path)
+ for src_path in glob.glob(os.path.join(self._dir, path)):
+ save_name = LogicalPath(os.path.relpath(src_path, self._dir))
+ feh = self._get_file_event_handler(src_path, save_name)
+ # handle the case where the policy changed
+ if feh.policy != policy:
+ try:
+ del self._file_event_handlers[save_name]
+ except KeyError:
+ # TODO: probably should do locking, but this handles moved files for now
+ pass
+ feh = self._get_file_event_handler(src_path, save_name)
+ feh.on_modified(force=True)
+
+ def _per_file_event_handler(self) -> "wd_events.FileSystemEventHandler":
+ """Create a Watchdog file event handler that does different things for every file."""
+ file_event_handler = wd_events.PatternMatchingEventHandler()
+ file_event_handler.on_created = self._on_file_created
+ file_event_handler.on_modified = self._on_file_modified
+ file_event_handler.on_moved = self._on_file_moved
+ file_event_handler._patterns = [os.path.join(self._dir, os.path.normpath("*"))]
+ # Ignore hidden files/folders
+ # TODO: what other files should we skip?
+ file_event_handler._ignore_patterns = [
+ "*.tmp",
+ "*.wandb",
+ "wandb-summary.json",
+ os.path.join(self._dir, ".*"),
+ os.path.join(self._dir, "*/.*"),
+ ]
+ for glb in self._settings.ignore_globs:
+ file_event_handler._ignore_patterns.append(os.path.join(self._dir, glb))
+
+ return file_event_handler
+
+ def _on_file_created(self, event: "wd_events.FileCreatedEvent") -> None:
+ logger.info("file/dir created: %s", event.src_path)
+ if os.path.isdir(event.src_path):
+ return None
+ self._file_count += 1
+ # We do the directory scan less often as it grows
+ if self._file_count % 100 == 0:
+ emitter = self.emitter
+ if emitter:
+ emitter._timeout = int(self._file_count / 100) + 1
+ save_name = LogicalPath(os.path.relpath(event.src_path, self._dir))
+ self._get_file_event_handler(event.src_path, save_name).on_modified()
+
+ # TODO(spencerpearson): this pattern repeats so many times we should have a method/function for it
+ # def _save_name(self, path: PathStr) -> LogicalPath:
+ # return LogicalPath(os.path.relpath(path, self._dir))
+
+ def _on_file_modified(self, event: "wd_events.FileModifiedEvent") -> None:
+ logger.info(f"file/dir modified: { event.src_path}")
+ if os.path.isdir(event.src_path):
+ return None
+ save_name = LogicalPath(os.path.relpath(event.src_path, self._dir))
+ self._get_file_event_handler(event.src_path, save_name).on_modified()
+
+ def _on_file_moved(self, event: "wd_events.FileMovedEvent") -> None:
+ # TODO: test me...
+ logger.info(f"file/dir moved: {event.src_path} -> {event.dest_path}")
+ if os.path.isdir(event.dest_path):
+ return None
+ old_save_name = LogicalPath(os.path.relpath(event.src_path, self._dir))
+ new_save_name = LogicalPath(os.path.relpath(event.dest_path, self._dir))
+
+ # We have to move the existing file handler to the new name
+ handler = self._get_file_event_handler(event.src_path, old_save_name)
+ self._file_event_handlers[new_save_name] = handler
+ del self._file_event_handlers[old_save_name]
+
+ handler.on_renamed(event.dest_path, new_save_name)
+
+ def _get_file_event_handler(
+ self, file_path: PathStr, save_name: LogicalPath
+ ) -> FileEventHandler:
+ """Get or create an event handler for a particular file.
+
+ file_path: the file's actual path
+ save_name: its path relative to the run directory (aka the watch directory)
+ """
+ # Always return PolicyNow for any of our media files.
+ if save_name.startswith("media/"):
+ return PolicyNow(file_path, save_name, self._file_pusher, self._settings)
+ if save_name not in self._file_event_handlers:
+ # TODO: we can use PolicyIgnore if there are files we never want to sync
+ if "tfevents" in save_name or "graph.pbtxt" in save_name:
+ self._file_event_handlers[save_name] = PolicyLive(
+ file_path, save_name, self._file_pusher, self._settings
+ )
+ elif save_name in self._savename_file_policies:
+ policy_name = self._savename_file_policies[save_name]
+ make_handler = (
+ PolicyLive
+ if policy_name == "live"
+ else PolicyNow
+ if policy_name == "now"
+ else PolicyEnd
+ )
+ self._file_event_handlers[save_name] = make_handler(
+ file_path, save_name, self._file_pusher, self._settings
+ )
+ else:
+ make_handler = PolicyEnd
+ for policy, globs in self._user_file_policies.items():
+ if policy == "end":
+ continue
+ # Convert set to list to avoid RuntimeError's
+ # TODO: we may need to add locks
+ for g in list(globs):
+ paths = glob.glob(os.path.join(self._dir, g))
+ if any(save_name in p for p in paths):
+ if policy == "live":
+ make_handler = PolicyLive
+ elif policy == "now":
+ make_handler = PolicyNow
+ self._file_event_handlers[save_name] = make_handler(
+ file_path, save_name, self._file_pusher, self._settings
+ )
+ return self._file_event_handlers[save_name]
+
+ def finish(self) -> None:
+ logger.info("shutting down directory watcher")
+ try:
+ # avoid hanging if we crashed before the observer was started
+ if self._file_observer.is_alive():
+ # rather unfortunately we need to manually do a final scan of the dir
+ # with `queue_events`, then iterate through all events before stopping
+ # the observer to catch all files written. First we need to prevent the
+ # existing thread from consuming our final events, then we process them
+ self._file_observer._timeout = 0
+ self._file_observer._stopped_event.set()
+ self._file_observer.join()
+ self.emitter.queue_events(0) # type: ignore[union-attr]
+ while True:
+ try:
+ self._file_observer.dispatch_events(
+ self._file_observer.event_queue, 0
+ )
+ except queue.Empty:
+ break
+ # Calling stop unschedules any inflight events so we handled them above
+ self._file_observer.stop()
+ # TODO: py2 TypeError: PyCObject_AsVoidPtr called with null pointer
+ except TypeError:
+ pass
+ # TODO: py3 SystemError: <built-in function stop> returned an error
+ except SystemError:
+ pass
+
+ # Ensure we've at least noticed every file in the run directory. Sometimes
+ # we miss things because asynchronously watching filesystems isn't reliable.
+ logger.info("scan: %s", self._dir)
+
+ for dirpath, _, filenames in os.walk(self._dir):
+ for fname in filenames:
+ file_path = os.path.join(dirpath, fname)
+ save_name = LogicalPath(os.path.relpath(file_path, self._dir))
+ ignored = False
+ for glb in self._settings.ignore_globs:
+ if len(fnmatch.filter([save_name], glb)) > 0:
+ ignored = True
+ logger.info("ignored: %s matching glob %s", save_name, glb)
+ break
+ if ignored:
+ continue
+ logger.info("scan save: %s %s", file_path, save_name)
+ self._get_file_event_handler(file_path, save_name).finish()
diff --git a/wandb/filesync/stats.py b/wandb/filesync/stats.py
new file mode 100644
index 0000000000000000000000000000000000000000..95c3523a37a2a82ce17bd40a6b4a667db04ff0ff
--- /dev/null
+++ b/wandb/filesync/stats.py
@@ -0,0 +1,100 @@
+import threading
+from typing import MutableMapping, NamedTuple
+
+import wandb
+
+
+class FileStats(NamedTuple):
+ deduped: bool
+ total: int
+ uploaded: int
+ failed: bool
+ artifact_file: bool
+
+
+class Summary(NamedTuple):
+ uploaded_bytes: int
+ total_bytes: int
+ deduped_bytes: int
+
+
+class FileCountsByCategory(NamedTuple):
+ artifact: int
+ wandb: int
+ media: int
+ other: int
+
+
+class Stats:
+ def __init__(self) -> None:
+ self._stats: MutableMapping[str, FileStats] = {}
+ self._lock = threading.Lock()
+
+ def init_file(
+ self, save_name: str, size: int, is_artifact_file: bool = False
+ ) -> None:
+ with self._lock:
+ self._stats[save_name] = FileStats(
+ deduped=False,
+ total=size,
+ uploaded=0,
+ failed=False,
+ artifact_file=is_artifact_file,
+ )
+
+ def set_file_deduped(self, save_name: str) -> None:
+ with self._lock:
+ orig = self._stats[save_name]
+ self._stats[save_name] = orig._replace(
+ deduped=True,
+ uploaded=orig.total,
+ )
+
+ def update_uploaded_file(self, save_name: str, total_uploaded: int) -> None:
+ with self._lock:
+ self._stats[save_name] = self._stats[save_name]._replace(
+ uploaded=total_uploaded,
+ )
+
+ def update_failed_file(self, save_name: str) -> None:
+ with self._lock:
+ self._stats[save_name] = self._stats[save_name]._replace(
+ uploaded=0,
+ failed=True,
+ )
+
+ def summary(self) -> Summary:
+ # Need to use list to ensure we get a copy, since other threads may
+ # modify this while we iterate
+ with self._lock:
+ stats = list(self._stats.values())
+ return Summary(
+ uploaded_bytes=sum(f.uploaded for f in stats),
+ total_bytes=sum(f.total for f in stats),
+ deduped_bytes=sum(f.total for f in stats if f.deduped),
+ )
+
+ def file_counts_by_category(self) -> FileCountsByCategory:
+ artifact_files = 0
+ wandb_files = 0
+ media_files = 0
+ other_files = 0
+ # Need to use list to ensure we get a copy, since other threads may
+ # modify this while we iterate
+ with self._lock:
+ file_stats = list(self._stats.items())
+ for save_name, stats in file_stats:
+ if stats.artifact_file:
+ artifact_files += 1
+ elif wandb.wandb_lib.filenames.is_wandb_file(save_name): # type: ignore[attr-defined] # TODO(spencerpearson): this is probably synonymous with wandb.sdk.lib.filenames...?
+ wandb_files += 1
+ elif save_name.startswith("media"):
+ media_files += 1
+ else:
+ other_files += 1
+ return FileCountsByCategory(
+ artifact=artifact_files,
+ wandb=wandb_files,
+ media=media_files,
+ other=other_files,
+ )
diff --git a/wandb/filesync/step_checksum.py b/wandb/filesync/step_checksum.py
new file mode 100644
index 0000000000000000000000000000000000000000..5a6d2a08c3ce13f623c16937213834acb881727d
--- /dev/null
+++ b/wandb/filesync/step_checksum.py
@@ -0,0 +1,145 @@
+"""Batching file prepare requests to our API."""
+
+import concurrent.futures
+import functools
+import os
+import queue
+import shutil
+import threading
+from typing import TYPE_CHECKING, NamedTuple, Optional, Union, cast
+
+from wandb.filesync import step_upload
+from wandb.sdk.lib import filesystem, runid
+from wandb.sdk.lib.paths import LogicalPath
+
+if TYPE_CHECKING:
+ import tempfile
+
+ from wandb.filesync import stats
+ from wandb.sdk.artifacts.artifact_manifest import ArtifactManifest
+ from wandb.sdk.artifacts.artifact_saver import SaveFn, SaveFnAsync
+ from wandb.sdk.internal import internal_api
+
+
+class RequestUpload(NamedTuple):
+ path: str
+ save_name: LogicalPath
+ copy: bool
+
+
+class RequestStoreManifestFiles(NamedTuple):
+ manifest: "ArtifactManifest"
+ artifact_id: str
+ save_fn: "SaveFn"
+ save_fn_async: "SaveFnAsync"
+
+
+class RequestCommitArtifact(NamedTuple):
+ artifact_id: str
+ finalize: bool
+ before_commit: step_upload.PreCommitFn
+ result_future: "concurrent.futures.Future[None]"
+
+
+class RequestFinish(NamedTuple):
+ callback: Optional[step_upload.OnRequestFinishFn]
+
+
+Event = Union[
+ RequestUpload, RequestStoreManifestFiles, RequestCommitArtifact, RequestFinish
+]
+
+
+class StepChecksum:
+ def __init__(
+ self,
+ api: "internal_api.Api",
+ tempdir: "tempfile.TemporaryDirectory",
+ request_queue: "queue.Queue[Event]",
+ output_queue: "queue.Queue[step_upload.Event]",
+ stats: "stats.Stats",
+ ) -> None:
+ self._api = api
+ self._tempdir = tempdir
+ self._request_queue = request_queue
+ self._output_queue = output_queue
+ self._stats = stats
+
+ self._thread = threading.Thread(target=self._thread_body)
+ self._thread.daemon = True
+
+ def _thread_body(self) -> None:
+ while True:
+ req = self._request_queue.get()
+ if isinstance(req, RequestUpload):
+ path = req.path
+ if req.copy:
+ path = os.path.join(
+ self._tempdir.name,
+ f"{runid.generate_id()}-{req.save_name}",
+ )
+ filesystem.mkdir_exists_ok(os.path.dirname(path))
+ try:
+ # certain linux distros throw an exception when copying
+ # large files: https://bugs.python.org/issue43743
+ shutil.copy2(req.path, path)
+ except OSError:
+ shutil._USE_CP_SENDFILE = False # type: ignore[attr-defined]
+ shutil.copy2(req.path, path)
+ self._stats.init_file(req.save_name, os.path.getsize(path))
+ self._output_queue.put(
+ step_upload.RequestUpload(
+ path,
+ req.save_name,
+ None,
+ None,
+ req.copy,
+ None,
+ None,
+ None,
+ )
+ )
+ elif isinstance(req, RequestStoreManifestFiles):
+ for entry in req.manifest.entries.values():
+ if entry.local_path:
+ self._stats.init_file(
+ entry.local_path,
+ cast(int, entry.size),
+ is_artifact_file=True,
+ )
+ self._output_queue.put(
+ step_upload.RequestUpload(
+ entry.local_path,
+ entry.path,
+ req.artifact_id,
+ entry.digest,
+ False,
+ functools.partial(req.save_fn, entry),
+ functools.partial(req.save_fn_async, entry),
+ entry.digest,
+ )
+ )
+ elif isinstance(req, RequestCommitArtifact):
+ self._output_queue.put(
+ step_upload.RequestCommitArtifact(
+ req.artifact_id,
+ req.finalize,
+ req.before_commit,
+ req.result_future,
+ )
+ )
+ elif isinstance(req, RequestFinish):
+ break
+ else:
+ raise Exception("internal error")
+
+ self._output_queue.put(step_upload.RequestFinish(req.callback))
+
+ def start(self) -> None:
+ self._thread.start()
+
+ def is_alive(self) -> bool:
+ return self._thread.is_alive()
+
+ def finish(self) -> None:
+ self._request_queue.put(RequestFinish(None))
diff --git a/wandb/filesync/step_prepare.py b/wandb/filesync/step_prepare.py
new file mode 100644
index 0000000000000000000000000000000000000000..6b6824200ce3f43caf2477e3aaf66bb4a66fcedf
--- /dev/null
+++ b/wandb/filesync/step_prepare.py
@@ -0,0 +1,199 @@
+"""Batching file prepare requests to our API."""
+
+import asyncio
+import functools
+import queue
+import threading
+import time
+from typing import (
+ TYPE_CHECKING,
+ Callable,
+ Dict,
+ List,
+ Mapping,
+ NamedTuple,
+ Optional,
+ Sequence,
+ Tuple,
+ Union,
+)
+
+if TYPE_CHECKING:
+ from wandb.sdk.internal.internal_api import (
+ Api,
+ CreateArtifactFileSpecInput,
+ CreateArtifactFilesResponseFile,
+ )
+
+
+# Request for a file to be prepared.
+class RequestPrepare(NamedTuple):
+ file_spec: "CreateArtifactFileSpecInput"
+ response_channel: Union[
+ "queue.Queue[ResponsePrepare]",
+ Tuple["asyncio.AbstractEventLoop", "asyncio.Future[ResponsePrepare]"],
+ ]
+
+
+class RequestFinish(NamedTuple):
+ pass
+
+
+class ResponsePrepare(NamedTuple):
+ birth_artifact_id: str
+ upload_url: Optional[str]
+ upload_headers: Sequence[str]
+ upload_id: Optional[str]
+ storage_path: Optional[str]
+ multipart_upload_urls: Optional[Dict[int, str]]
+
+
+Request = Union[RequestPrepare, RequestFinish]
+
+
+def _clamp(x: float, low: float, high: float) -> float:
+ return max(low, min(x, high))
+
+
+def gather_batch(
+ request_queue: "queue.Queue[Request]",
+ batch_time: float,
+ inter_event_time: float,
+ max_batch_size: int,
+ clock: Callable[[], float] = time.monotonic,
+) -> Tuple[bool, Sequence[RequestPrepare]]:
+ batch_start_time = clock()
+ remaining_time = batch_time
+
+ first_request = request_queue.get()
+ if isinstance(first_request, RequestFinish):
+ return True, []
+
+ batch: List[RequestPrepare] = [first_request]
+
+ while remaining_time > 0 and len(batch) < max_batch_size:
+ try:
+ request = request_queue.get(
+ timeout=_clamp(
+ x=inter_event_time,
+ low=1e-12, # 0 = "block forever", so just use something tiny
+ high=remaining_time,
+ ),
+ )
+ if isinstance(request, RequestFinish):
+ return True, batch
+
+ batch.append(request)
+ remaining_time = batch_time - (clock() - batch_start_time)
+
+ except queue.Empty:
+ break
+
+ return False, batch
+
+
+def prepare_response(response: "CreateArtifactFilesResponseFile") -> ResponsePrepare:
+ multipart_resp = response.get("uploadMultipartUrls")
+ part_list = multipart_resp["uploadUrlParts"] if multipart_resp else []
+ multipart_parts = {u["partNumber"]: u["uploadUrl"] for u in part_list} or None
+
+ return ResponsePrepare(
+ birth_artifact_id=response["artifact"]["id"],
+ upload_url=response["uploadUrl"],
+ upload_headers=response["uploadHeaders"],
+ upload_id=multipart_resp and multipart_resp.get("uploadID"),
+ storage_path=response.get("storagePath"),
+ multipart_upload_urls=multipart_parts,
+ )
+
+
+class StepPrepare:
+ """A thread that batches requests to our file prepare API.
+
+ Any number of threads may call prepare_async() in parallel. The PrepareBatcher thread
+ will batch requests up and send them all to the backend at once.
+ """
+
+ def __init__(
+ self,
+ api: "Api",
+ batch_time: float,
+ inter_event_time: float,
+ max_batch_size: int,
+ request_queue: Optional["queue.Queue[Request]"] = None,
+ ) -> None:
+ self._api = api
+ self._inter_event_time = inter_event_time
+ self._batch_time = batch_time
+ self._max_batch_size = max_batch_size
+ self._request_queue: queue.Queue[Request] = request_queue or queue.Queue()
+ self._thread = threading.Thread(target=self._thread_body)
+ self._thread.daemon = True
+
+ def _thread_body(self) -> None:
+ while True:
+ finish, batch = gather_batch(
+ request_queue=self._request_queue,
+ batch_time=self._batch_time,
+ inter_event_time=self._inter_event_time,
+ max_batch_size=self._max_batch_size,
+ )
+ if batch:
+ batch_response = self._prepare_batch(batch)
+ # send responses
+ for prepare_request in batch:
+ name = prepare_request.file_spec["name"]
+ response_file = batch_response[name]
+ response = prepare_response(response_file)
+ if isinstance(prepare_request.response_channel, queue.Queue):
+ prepare_request.response_channel.put(response)
+ else:
+ loop, future = prepare_request.response_channel
+ loop.call_soon_threadsafe(future.set_result, response)
+ if finish:
+ break
+
+ def _prepare_batch(
+ self, batch: Sequence[RequestPrepare]
+ ) -> Mapping[str, "CreateArtifactFilesResponseFile"]:
+ """Execute the prepareFiles API call.
+
+ Arguments:
+ batch: List of RequestPrepare objects
+ Returns:
+ dict of (save_name: ResponseFile) pairs where ResponseFile is a dict with
+ an uploadUrl key. The value of the uploadUrl key is None if the file
+ already exists, or a url string if the file should be uploaded.
+ """
+ return self._api.create_artifact_files([req.file_spec for req in batch])
+
+ def prepare_async(
+ self, file_spec: "CreateArtifactFileSpecInput"
+ ) -> "asyncio.Future[ResponsePrepare]":
+ """Request the backend to prepare a file for upload."""
+ response: asyncio.Future[ResponsePrepare] = asyncio.Future()
+ self._request_queue.put(
+ RequestPrepare(file_spec, (asyncio.get_event_loop(), response))
+ )
+ return response
+
+ @functools.wraps(prepare_async)
+ def prepare_sync(
+ self, file_spec: "CreateArtifactFileSpecInput"
+ ) -> "queue.Queue[ResponsePrepare]":
+ response_queue: queue.Queue[ResponsePrepare] = queue.Queue()
+ self._request_queue.put(RequestPrepare(file_spec, response_queue))
+ return response_queue
+
+ def start(self) -> None:
+ self._thread.start()
+
+ def finish(self) -> None:
+ self._request_queue.put(RequestFinish())
+
+ def is_alive(self) -> bool:
+ return self._thread.is_alive()
+
+ def shutdown(self) -> None:
+ self.finish()
+ self._thread.join()
diff --git a/wandb/filesync/step_upload.py b/wandb/filesync/step_upload.py
new file mode 100644
index 0000000000000000000000000000000000000000..7fcf694122caf94899fa15d5f464dfe4e267ed27
--- /dev/null
+++ b/wandb/filesync/step_upload.py
@@ -0,0 +1,392 @@
+"""Batching file prepare requests to our API."""
+
+import asyncio
+import concurrent.futures
+import logging
+import queue
+import sys
+import threading
+from typing import (
+ TYPE_CHECKING,
+ Awaitable,
+ Callable,
+ MutableMapping,
+ MutableSequence,
+ MutableSet,
+ NamedTuple,
+ Optional,
+ Union,
+)
+
+from wandb.errors.term import termerror
+from wandb.filesync import upload_job
+from wandb.sdk.lib.paths import LogicalPath
+
+if TYPE_CHECKING:
+ from wandb.filesync import stats
+ from wandb.sdk.internal import file_stream, internal_api, progress
+ from wandb.sdk.internal.settings_static import SettingsStatic
+
+ if sys.version_info >= (3, 8):
+ from typing import TypedDict
+ else:
+ from typing_extensions import TypedDict
+
+ class ArtifactStatus(TypedDict):
+ finalize: bool
+ pending_count: int
+ commit_requested: bool
+ pre_commit_callbacks: MutableSet["PreCommitFn"]
+ result_futures: MutableSet["concurrent.futures.Future[None]"]
+
+
+PreCommitFn = Callable[[], None]
+OnRequestFinishFn = Callable[[], None]
+SaveFn = Callable[["progress.ProgressFn"], bool]
+SaveFnAsync = Callable[["progress.ProgressFn"], Awaitable[bool]]
+
+logger = logging.getLogger(__name__)
+
+
+class RequestUpload(NamedTuple):
+ path: str
+ save_name: LogicalPath
+ artifact_id: Optional[str]
+ md5: Optional[str]
+ copied: bool
+ save_fn: Optional[SaveFn]
+ save_fn_async: Optional[SaveFnAsync]
+ digest: Optional[str]
+
+
+class RequestCommitArtifact(NamedTuple):
+ artifact_id: str
+ finalize: bool
+ before_commit: PreCommitFn
+ result_future: "concurrent.futures.Future[None]"
+
+
+class RequestFinish(NamedTuple):
+ callback: Optional[OnRequestFinishFn]
+
+
+class EventJobDone(NamedTuple):
+ job: RequestUpload
+ exc: Optional[BaseException]
+
+
+Event = Union[RequestUpload, RequestCommitArtifact, RequestFinish, EventJobDone]
+
+
+class AsyncExecutor:
+ """Runs async file uploads in a background thread."""
+
+ def __init__(
+ self,
+ pool: concurrent.futures.ThreadPoolExecutor,
+ concurrency_limit: Optional[int],
+ ) -> None:
+ self.loop = asyncio.new_event_loop()
+ self.loop.set_default_executor(pool)
+ self.loop_thread = threading.Thread(
+ target=self.loop.run_forever,
+ daemon=True,
+ name="wandb-upload-async",
+ )
+
+ self.concurrency_limiter = asyncio.Semaphore(
+ value=concurrency_limit or 128,
+ # Before Python 3.10: if we don't set `loop=loop`,
+ # then the Semaphore will bind to the wrong event loop,
+ # causing errors when a coroutine tries to wait for it;
+ # see https://pastebin.com/XcrS9suX .
+ # After 3.10: the `loop` argument doesn't exist.
+ # So we need to only conditionally pass in `loop`.
+ **({} if sys.version_info >= (3, 10) else {"loop": self.loop}),
+ )
+
+ def start(self) -> None:
+ self.loop_thread.start()
+
+ def stop(self) -> None:
+ self.loop.call_soon_threadsafe(self.loop.stop)
+
+ def submit(self, coro: Awaitable[None]) -> None:
+ async def run_with_limiter() -> None:
+ async with self.concurrency_limiter:
+ await coro
+
+ asyncio.run_coroutine_threadsafe(run_with_limiter(), self.loop)
+
+
+class StepUpload:
+ def __init__(
+ self,
+ api: "internal_api.Api",
+ stats: "stats.Stats",
+ event_queue: "queue.Queue[Event]",
+ max_threads: int,
+ file_stream: "file_stream.FileStreamApi",
+ settings: Optional["SettingsStatic"] = None,
+ ) -> None:
+ self._api = api
+ self._stats = stats
+ self._event_queue = event_queue
+ self._file_stream = file_stream
+
+ self._thread = threading.Thread(target=self._thread_body)
+ self._thread.daemon = True
+
+ self._pool = concurrent.futures.ThreadPoolExecutor(
+ thread_name_prefix="wandb-upload",
+ max_workers=max_threads,
+ )
+
+ self._async_executor = (
+ AsyncExecutor(
+ pool=self._pool,
+ concurrency_limit=settings._async_upload_concurrency_limit,
+ )
+ if settings is not None and settings._async_upload_concurrency_limit
+ else None
+ )
+
+ # Indexed by files' `save_name`'s, which are their ID's in the Run.
+ self._running_jobs: MutableMapping[LogicalPath, RequestUpload] = {}
+ self._pending_jobs: MutableSequence[RequestUpload] = []
+
+ self._artifacts: MutableMapping[str, ArtifactStatus] = {}
+
+ self.silent = bool(settings.silent) if settings else False
+
+ def _thread_body(self) -> None:
+ event: Optional[Event]
+ # Wait for event in the queue, and process one by one until a
+ # finish event is received
+ finish_callback = None
+ while True:
+ event = self._event_queue.get()
+ if isinstance(event, RequestFinish):
+ finish_callback = event.callback
+ break
+ self._handle_event(event)
+
+ # We've received a finish event. At this point, further Upload requests
+ # are invalid.
+
+ # After a finish event is received, iterate through the event queue
+ # one by one and process all remaining events.
+ while True:
+ try:
+ event = self._event_queue.get(True, 0.2)
+ except queue.Empty:
+ event = None
+ if event:
+ self._handle_event(event)
+ elif not self._running_jobs:
+ # Queue was empty and no jobs left.
+ self._pool.shutdown(wait=False)
+ if self._async_executor:
+ self._async_executor.stop()
+ if finish_callback:
+ finish_callback()
+ break
+
+ def _handle_event(self, event: Event) -> None:
+ if isinstance(event, EventJobDone):
+ job = event.job
+
+ if event.exc is not None:
+ logger.exception(
+ "Failed to upload file: %s", job.path, exc_info=event.exc
+ )
+
+ if job.artifact_id:
+ if event.exc is None:
+ self._artifacts[job.artifact_id]["pending_count"] -= 1
+ self._maybe_commit_artifact(job.artifact_id)
+ else:
+ if not self.silent:
+ termerror(
+ "Uploading artifact file failed. Artifact won't be committed."
+ )
+ self._fail_artifact_futures(job.artifact_id, event.exc)
+ self._running_jobs.pop(job.save_name)
+ # If we have any pending jobs, start one now
+ if self._pending_jobs:
+ event = self._pending_jobs.pop(0)
+ self._start_upload_job(event)
+ elif isinstance(event, RequestCommitArtifact):
+ if event.artifact_id not in self._artifacts:
+ self._init_artifact(event.artifact_id)
+ self._artifacts[event.artifact_id]["commit_requested"] = True
+ self._artifacts[event.artifact_id]["finalize"] = event.finalize
+ self._artifacts[event.artifact_id]["pre_commit_callbacks"].add(
+ event.before_commit
+ )
+ self._artifacts[event.artifact_id]["result_futures"].add(
+ event.result_future
+ )
+ self._maybe_commit_artifact(event.artifact_id)
+ elif isinstance(event, RequestUpload):
+ if event.artifact_id is not None:
+ if event.artifact_id not in self._artifacts:
+ self._init_artifact(event.artifact_id)
+ self._artifacts[event.artifact_id]["pending_count"] += 1
+ self._start_upload_job(event)
+ else:
+ raise Exception("Programming error: unhandled event: %s" % str(event))
+
+ def _start_upload_job(self, event: RequestUpload) -> None:
+ # Operations on a single backend file must be serialized. if
+ # we're already uploading this file, put the event on the
+ # end of the queue
+ if event.save_name in self._running_jobs:
+ self._pending_jobs.append(event)
+ return
+
+ if self._async_executor and event.save_fn_async is not None:
+ # (The `and save_fn_async is not None` is because the async code path
+ # doesn't support all uploads yet: even if the user has requested async,
+ # we sometimes need to use the sync method instead.)
+ self._spawn_upload_async(
+ event,
+ async_executor=self._async_executor,
+ )
+ else:
+ self._spawn_upload_sync(event)
+
+ def _spawn_upload_sync(self, event: RequestUpload) -> None:
+ """Spawn an upload job, and handles the bookkeeping of `self._running_jobs`.
+
+ Context: it's important that, whenever we add an entry to `self._running_jobs`,
+ we ensure that a corresponding `EventJobDone` message will eventually get handled;
+ otherwise, the `_running_jobs` entry will never get removed, and the StepUpload
+ will never shut down.
+
+ The sole purpose of this function is to make sure that the code that adds an entry
+ to `self._running_jobs` is textually right next to the code that eventually enqueues
+ the `EventJobDone` message. This should help keep them in sync.
+ """
+ # Adding the entry to `self._running_jobs` MUST happen in the main thread,
+ # NOT in the job that gets submitted to the thread-pool, to guard against
+ # this sequence of events:
+ # - StepUpload receives a RequestUpload
+ # ...and therefore spawns a thread to do the upload
+ # - StepUpload receives a RequestFinish
+ # ...and checks `self._running_jobs` to see if there are any tasks to wait for...
+ # ...and there are none, because the addition to `self._running_jobs` happens in
+ # the background thread, which the scheduler hasn't yet run...
+ # ...so the StepUpload shuts down. Even though we haven't uploaded the file!
+ #
+ # This would be very bad!
+ # So, this line has to happen _outside_ the `pool.submit()`.
+ self._running_jobs[event.save_name] = event
+
+ def run_and_notify() -> None:
+ try:
+ self._do_upload_sync(event)
+ finally:
+ self._event_queue.put(EventJobDone(event, exc=sys.exc_info()[1]))
+
+ self._pool.submit(run_and_notify)
+
+ def _spawn_upload_async(
+ self,
+ event: RequestUpload,
+ async_executor: AsyncExecutor,
+ ) -> None:
+ """Equivalent to _spawn_upload_sync, but uses the async event loop instead of a thread, and requires `event.save_fn_async`.
+
+ Raises:
+ AssertionError: if `event.save_fn_async` is None.
+ """
+ assert event.save_fn_async is not None
+
+ self._running_jobs[event.save_name] = event
+
+ async def run_and_notify() -> None:
+ try:
+ await self._do_upload_async(event)
+ finally:
+ self._event_queue.put(EventJobDone(event, exc=sys.exc_info()[1]))
+
+ async_executor.submit(run_and_notify())
+
+ def _do_upload_sync(self, event: RequestUpload) -> None:
+ job = upload_job.UploadJob(
+ self._stats,
+ self._api,
+ self._file_stream,
+ self.silent,
+ event.save_name,
+ event.path,
+ event.artifact_id,
+ event.md5,
+ event.copied,
+ event.save_fn,
+ event.digest,
+ )
+ job.run()
+
+ async def _do_upload_async(self, event: RequestUpload) -> None:
+ """Upload a file and returns when it's done. Requires `event.save_fn_async`."""
+ assert event.save_fn_async is not None
+ job = upload_job.UploadJobAsync(
+ stats=self._stats,
+ api=self._api,
+ file_stream=self._file_stream,
+ silent=self.silent,
+ request=event,
+ save_fn_async=event.save_fn_async,
+ )
+ await job.run()
+
+ def _init_artifact(self, artifact_id: str) -> None:
+ self._artifacts[artifact_id] = {
+ "finalize": False,
+ "pending_count": 0,
+ "commit_requested": False,
+ "pre_commit_callbacks": set(),
+ "result_futures": set(),
+ }
+
+ def _maybe_commit_artifact(self, artifact_id: str) -> None:
+ artifact_status = self._artifacts[artifact_id]
+ if (
+ artifact_status["pending_count"] == 0
+ and artifact_status["commit_requested"]
+ ):
+ try:
+ for pre_callback in artifact_status["pre_commit_callbacks"]:
+ pre_callback()
+ if artifact_status["finalize"]:
+ self._api.commit_artifact(artifact_id)
+ except Exception as exc:
+ termerror(
+ f"Committing artifact failed. Artifact {artifact_id} won't be finalized."
+ )
+ termerror(str(exc))
+ self._fail_artifact_futures(artifact_id, exc)
+ else:
+ self._resolve_artifact_futures(artifact_id)
+
+ def _fail_artifact_futures(self, artifact_id: str, exc: BaseException) -> None:
+ futures = self._artifacts[artifact_id]["result_futures"]
+ for result_future in futures:
+ result_future.set_exception(exc)
+ futures.clear()
+
+ def _resolve_artifact_futures(self, artifact_id: str) -> None:
+ futures = self._artifacts[artifact_id]["result_futures"]
+ for result_future in futures:
+ result_future.set_result(None)
+ futures.clear()
+
+ def start(self) -> None:
+ self._thread.start()
+ if self._async_executor:
+ self._async_executor.start()
+
+ def is_alive(self) -> bool:
+ return self._thread.is_alive()
diff --git a/wandb/filesync/upload_job.py b/wandb/filesync/upload_job.py
new file mode 100644
index 0000000000000000000000000000000000000000..8dbd214d25cff52117b892295225e057f7c38f76
--- /dev/null
+++ b/wandb/filesync/upload_job.py
@@ -0,0 +1,218 @@
+import asyncio
+import logging
+import os
+from typing import TYPE_CHECKING, Optional
+
+import wandb
+from wandb.sdk.lib.paths import LogicalPath
+
+if TYPE_CHECKING:
+ from wandb.filesync import dir_watcher, stats, step_upload
+ from wandb.sdk.internal import file_stream, internal_api
+
+
+logger = logging.getLogger(__name__)
+
+
+class UploadJob:
+ def __init__(
+ self,
+ stats: "stats.Stats",
+ api: "internal_api.Api",
+ file_stream: "file_stream.FileStreamApi",
+ silent: bool,
+ save_name: LogicalPath,
+ path: "dir_watcher.PathStr",
+ artifact_id: Optional[str],
+ md5: Optional[str],
+ copied: bool,
+ save_fn: Optional["step_upload.SaveFn"],
+ digest: Optional[str],
+ ) -> None:
+ """A file uploader.
+
+ Arguments:
+ push_function: function(save_name, actual_path) which actually uploads
+ the file.
+ save_name: string logical location of the file relative to the run
+ directory.
+ path: actual string path of the file to upload on the filesystem.
+ """
+ self._stats = stats
+ self._api = api
+ self._file_stream = file_stream
+ self.silent = silent
+ self.save_name = save_name
+ self.save_path = path
+ self.artifact_id = artifact_id
+ self.md5 = md5
+ self.copied = copied
+ self.save_fn = save_fn
+ self.digest = digest
+ super().__init__()
+
+ def run(self) -> None:
+ success = False
+ try:
+ self.push()
+ success = True
+ finally:
+ if self.copied and os.path.isfile(self.save_path):
+ os.remove(self.save_path)
+ if success:
+ self._file_stream.push_success(self.artifact_id, self.save_name) # type: ignore
+
+ def push(self) -> None:
+ if self.save_fn:
+ # Retry logic must happen in save_fn currently
+ try:
+ deduped = self.save_fn(
+ lambda _, t: self._stats.update_uploaded_file(self.save_path, t)
+ )
+ except Exception as e:
+ self._stats.update_failed_file(self.save_path)
+ logger.exception("Failed to upload file: %s", self.save_path)
+ wandb._sentry.exception(e)
+ message = str(e)
+ # TODO: this is usually XML, but could be JSON
+ if hasattr(e, "response"):
+ message = e.response.content
+ wandb.termerror(
+ f'Error uploading "{self.save_path}": {type(e).__name__}, {message}'
+ )
+ raise
+
+ if deduped:
+ logger.info("Skipped uploading %s", self.save_path)
+ self._stats.set_file_deduped(self.save_path)
+ else:
+ logger.info("Uploaded file %s", self.save_path)
+ return
+
+ if self.md5:
+ # This is the new artifact manifest upload flow, in which we create the
+ # database entry for the manifest file before creating it. This is used for
+ # artifact L0 files. Which now is only artifact_manifest.json
+ _, response = self._api.create_artifact_manifest(
+ self.save_name, self.md5, self.artifact_id
+ )
+ upload_url = response["uploadUrl"]
+ upload_headers = response["uploadHeaders"]
+ else:
+ # The classic file upload flow. We get a signed url and upload the file
+ # then the backend handles the cloud storage metadata callback to create the
+ # file entry. This flow has aged like a fine wine.
+ project = self._api.get_project()
+ _, upload_headers, result = self._api.upload_urls(project, [self.save_name])
+ file_info = result[self.save_name]
+ upload_url = file_info["uploadUrl"]
+
+ if upload_url is None:
+ logger.info("Skipped uploading %s", self.save_path)
+ self._stats.set_file_deduped(self.save_name)
+ else:
+ extra_headers = {}
+ for upload_header in upload_headers:
+ key, val = upload_header.split(":", 1)
+ extra_headers[key] = val
+ # Copied from push TODO(artifacts): clean up
+ # If the upload URL is relative, fill it in with the base URL,
+ # since its a proxied file store like the on-prem VM.
+ if upload_url.startswith("/"):
+ upload_url = f"{self._api.api_url}{upload_url}"
+ try:
+ with open(self.save_path, "rb") as f:
+ self._api.upload_file_retry(
+ upload_url,
+ f,
+ lambda _, t: self.progress(t),
+ extra_headers=extra_headers,
+ )
+ logger.info("Uploaded file %s", self.save_path)
+ except Exception as e:
+ self._stats.update_failed_file(self.save_name)
+ logger.exception("Failed to upload file: %s", self.save_path)
+ wandb._sentry.exception(e)
+ if not self.silent:
+ wandb.termerror(
+ f'Error uploading "{self.save_name}": {type(e).__name__}, {e}'
+ )
+ raise
+
+ def progress(self, total_bytes: int) -> None:
+ self._stats.update_uploaded_file(self.save_name, total_bytes)
+
+
+class UploadJobAsync:
+ """Roughly an async equivalent of UploadJob.
+
+ Important differences:
+ - `run` is a coroutine
+ - If `run()` fails, it falls back to the synchronous UploadJob
+ """
+
+ def __init__(
+ self,
+ stats: "stats.Stats",
+ api: "internal_api.Api",
+ file_stream: "file_stream.FileStreamApi",
+ silent: bool,
+ request: "step_upload.RequestUpload",
+ save_fn_async: "step_upload.SaveFnAsync",
+ ) -> None:
+ self._stats = stats
+ self._api = api
+ self._file_stream = file_stream
+ self.silent = silent
+ self._request = request
+ self._save_fn_async = save_fn_async
+
+ async def run(self) -> None:
+ try:
+ deduped = await self._save_fn_async(
+ lambda _, t: self._stats.update_uploaded_file(self._request.path, t)
+ )
+ except Exception as e:
+ # Async uploads aren't yet (2023-01) battle-tested.
+ # Fall back to the "normal" synchronous upload.
+ loop = asyncio.get_event_loop()
+ logger.exception("async upload failed", exc_info=e)
+ loop.run_in_executor(None, wandb._sentry.exception, e)
+ wandb.termwarn(
+ "Async file upload failed; falling back to sync", repeat=False
+ )
+ sync_job = UploadJob(
+ self._stats,
+ self._api,
+ self._file_stream,
+ self.silent,
+ self._request.save_name,
+ self._request.path,
+ self._request.artifact_id,
+ self._request.md5,
+ self._request.copied,
+ self._request.save_fn,
+ self._request.digest,
+ )
+
+ await loop.run_in_executor(None, sync_job.run)
+ else:
+ self._file_stream.push_success(
+ self._request.artifact_id, # type: ignore
+ self._request.save_name,
+ )
+
+ if deduped:
+ logger.info("Skipped uploading %s", self._request.path)
+ self._stats.set_file_deduped(self._request.path)
+ else:
+ logger.info("Uploaded file %s", self._request.path)
+ finally:
+ # If we fell back to the sync impl, the file will have already been deleted.
+ # Doesn't matter, we only try to delete it if it exists.
+ if self._request.copied:
+ try:
+ os.remove(self._request.path)
+ except OSError:
+ # The file has already been deleted, we don't have permissions, or something else we can't fix.
+ pass
diff --git a/wandb/integration/__init__.py b/wandb/integration/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/integration/catboost/__init__.py b/wandb/integration/catboost/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..ccd732b7e26cead03e12f0d6ab00f6d3165c2ba4
--- /dev/null
+++ b/wandb/integration/catboost/__init__.py
@@ -0,0 +1,5 @@
+"""W&B callback for CatBoost."""
+
+from .catboost import WandbCallback, log_summary
+
+__all__ = ["log_summary", "WandbCallback"]
diff --git a/wandb/integration/catboost/catboost.py b/wandb/integration/catboost/catboost.py
new file mode 100644
index 0000000000000000000000000000000000000000..a4619dc4e3aa807d8afa5444b19ee1b1c64f837a
--- /dev/null
+++ b/wandb/integration/catboost/catboost.py
@@ -0,0 +1,178 @@
+"""catboost init."""
+
+from pathlib import Path
+from types import SimpleNamespace
+from typing import List, Union
+
+from catboost import CatBoostClassifier, CatBoostRegressor # type: ignore
+
+import wandb
+from wandb.sdk.lib import telemetry as wb_telemetry
+
+
+class WandbCallback:
+ """`WandbCallback` automatically integrates CatBoost with wandb.
+
+ Arguments:
+ - metric_period: (int) if you are passing `metric_period` to your CatBoost model please pass the same value here (default=1).
+
+ Passing `WandbCallback` to CatBoost will:
+ - log training and validation metrics at every `metric_period`
+ - log iteration at every `metric_period`
+
+ Example:
+ ```
+ train_pool = Pool(train[features], label=train["label"], cat_features=cat_features)
+ test_pool = Pool(test[features], label=test["label"], cat_features=cat_features)
+
+ model = CatBoostRegressor(
+ iterations=100,
+ loss_function="Cox",
+ eval_metric="Cox",
+ )
+
+ model.fit(
+ train_pool,
+ eval_set=test_pool,
+ callbacks=[WandbCallback()],
+ )
+ ```
+ """
+
+ def __init__(self, metric_period: int = 1):
+ if wandb.run is None:
+ raise wandb.Error("You must call `wandb.init()` before `WandbCallback()`")
+
+ with wb_telemetry.context() as tel:
+ tel.feature.catboost_wandb_callback = True
+
+ self.metric_period: int = metric_period
+
+ def after_iteration(self, info: SimpleNamespace) -> bool:
+ if info.iteration % self.metric_period == 0:
+ for data, metric in info.metrics.items():
+ for metric_name, log in metric.items():
+ # todo: replace with wandb.run._log once available
+ wandb.log({f"{data}-{metric_name}": log[-1]}, commit=False)
+ # todo: replace with wandb.run._log once available
+ wandb.log({f"iteration@metric-period-{self.metric_period}": info.iteration})
+
+ return True
+
+
+def _checkpoint_artifact(
+ model: Union[CatBoostClassifier, CatBoostRegressor], aliases: List[str]
+) -> None:
+ """Upload model checkpoint as W&B artifact."""
+ if wandb.run is None:
+ raise wandb.Error(
+ "You must call `wandb.init()` before `_checkpoint_artifact()`"
+ )
+
+ model_name = f"model_{wandb.run.id}"
+ # save the model in the default `cbm` format
+ model_path = Path(wandb.run.dir) / "model"
+
+ model.save_model(model_path)
+
+ model_artifact = wandb.Artifact(name=model_name, type="model")
+ model_artifact.add_file(str(model_path))
+ wandb.log_artifact(model_artifact, aliases=aliases)
+
+
+def _log_feature_importance(
+ model: Union[CatBoostClassifier, CatBoostRegressor]
+) -> None:
+ """Log feature importance with default settings."""
+ if wandb.run is None:
+ raise wandb.Error(
+ "You must call `wandb.init()` before `_checkpoint_artifact()`"
+ )
+
+ feat_df = model.get_feature_importance(prettified=True)
+
+ fi_data = [
+ [feat, feat_imp]
+ for feat, feat_imp in zip(feat_df["Feature Id"], feat_df["Importances"])
+ ]
+ table = wandb.Table(data=fi_data, columns=["Feature", "Importance"])
+ # todo: replace with wandb.run._log once available
+ wandb.log(
+ {
+ "Feature Importance": wandb.plot.bar(
+ table, "Feature", "Importance", title="Feature Importance"
+ )
+ },
+ commit=False,
+ )
+
+
+def log_summary(
+ model: Union[CatBoostClassifier, CatBoostRegressor],
+ log_all_params: bool = True,
+ save_model_checkpoint: bool = False,
+ log_feature_importance: bool = True,
+) -> None:
+ """`log_summary` logs useful metrics about catboost model after training is done.
+
+ Arguments:
+ model: it can be CatBoostClassifier or CatBoostRegressor.
+ log_all_params: (boolean) if True (default) log the model hyperparameters as W&B config.
+ save_model_checkpoint: (boolean) if True saves the model upload as W&B artifacts.
+ log_feature_importance: (boolean) if True (default) logs feature importance as W&B bar chart using the default setting of `get_feature_importance`.
+
+ Using this along with `wandb_callback` will:
+
+ - save the hyperparameters as W&B config,
+ - log `best_iteration` and `best_score` as `wandb.summary`,
+ - save and upload your trained model to Weights & Biases Artifacts (when `save_model_checkpoint = True`)
+ - log feature importance plot.
+
+ Example:
+ ```python
+ train_pool = Pool(train[features], label=train["label"], cat_features=cat_features)
+ test_pool = Pool(test[features], label=test["label"], cat_features=cat_features)
+
+ model = CatBoostRegressor(
+ iterations=100,
+ loss_function="Cox",
+ eval_metric="Cox",
+ )
+
+ model.fit(
+ train_pool,
+ eval_set=test_pool,
+ callbacks=[WandbCallback()],
+ )
+
+ log_summary(model)
+ ```
+ """
+ if wandb.run is None:
+ raise wandb.Error("You must call `wandb.init()` before `log_summary()`")
+
+ if not (isinstance(model, (CatBoostClassifier, CatBoostRegressor))):
+ raise wandb.Error(
+ "Model should be an instance of CatBoostClassifier or CatBoostRegressor"
+ )
+
+ with wb_telemetry.context() as tel:
+ tel.feature.catboost_log_summary = True
+
+ # log configs
+ params = model.get_all_params()
+ if log_all_params:
+ wandb.config.update(params)
+
+ # log best score and iteration
+ wandb.run.summary["best_iteration"] = model.get_best_iteration()
+ wandb.run.summary["best_score"] = model.get_best_score()
+
+ # log model
+ if save_model_checkpoint:
+ aliases = ["best"] if params["use_best_model"] else ["last"]
+ _checkpoint_artifact(model, aliases=aliases)
+
+ # Feature importance
+ if log_feature_importance:
+ _log_feature_importance(model)
diff --git a/wandb/integration/cohere/__init__.py b/wandb/integration/cohere/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..2d367dc6988bda4251745dc6b3610a3e92b4c85e
--- /dev/null
+++ b/wandb/integration/cohere/__init__.py
@@ -0,0 +1,3 @@
+__all__ = ("autolog",)
+
+from .cohere import autolog
diff --git a/wandb/integration/cohere/cohere.py b/wandb/integration/cohere/cohere.py
new file mode 100644
index 0000000000000000000000000000000000000000..91f9a43e23150a6882dbb87512e6fbe657a7b8d4
--- /dev/null
+++ b/wandb/integration/cohere/cohere.py
@@ -0,0 +1,21 @@
+import logging
+
+from wandb.sdk.integration_utils.auto_logging import AutologAPI
+
+from .resolver import CohereRequestResponseResolver
+
+logger = logging.getLogger(__name__)
+
+
+autolog = AutologAPI(
+ name="Cohere",
+ symbols=(
+ "Client.generate",
+ "Client.chat",
+ "Client.classify",
+ "Client.summarize",
+ "Client.rerank",
+ ),
+ resolver=CohereRequestResponseResolver(),
+ telemetry_feature="cohere_autolog",
+)
diff --git a/wandb/integration/cohere/resolver.py b/wandb/integration/cohere/resolver.py
new file mode 100644
index 0000000000000000000000000000000000000000..6cfcdf020b87c8a5a3c15f164226ac50f8670d4d
--- /dev/null
+++ b/wandb/integration/cohere/resolver.py
@@ -0,0 +1,347 @@
+import logging
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+import wandb
+from wandb.sdk.integration_utils.auto_logging import Response
+from wandb.sdk.lib.runid import generate_id
+
+logger = logging.getLogger(__name__)
+
+
+def subset_dict(
+ original_dict: Dict[str, Any], keys_subset: Sequence[str]
+) -> Dict[str, Any]:
+ """Create a subset of a dictionary using a subset of keys.
+
+ :param original_dict: The original dictionary.
+ :param keys_subset: The subset of keys to extract.
+ :return: A dictionary containing only the specified keys.
+ """
+ return {key: original_dict[key] for key in keys_subset if key in original_dict}
+
+
+def reorder_and_convert_dict_list_to_table(
+ data: List[Dict[str, Any]], order: List[str]
+) -> Tuple[List[str], List[List[Any]]]:
+ """Convert a list of dictionaries to a pair of column names and corresponding values, with the option to order specific dictionaries.
+
+ :param data: A list of dictionaries.
+ :param order: A list of keys specifying the desired order for specific dictionaries. The remaining dictionaries will be ordered based on their original order.
+ :return: A pair of column names and corresponding values.
+ """
+ final_columns = []
+ keys_present = set()
+
+ # First, add all ordered keys to the final columns
+ for key in order:
+ if key not in keys_present:
+ final_columns.append(key)
+ keys_present.add(key)
+
+ # Then, add any keys present in the dictionaries but not in the order
+ for d in data:
+ for key in d:
+ if key not in keys_present:
+ final_columns.append(key)
+ keys_present.add(key)
+
+ # Then, construct the table of values
+ values = []
+ for d in data:
+ row = []
+ for key in final_columns:
+ row.append(d.get(key, None))
+ values.append(row)
+
+ return final_columns, values
+
+
+def flatten_dict(
+ dictionary: Dict[str, Any], parent_key: str = "", sep: str = "-"
+) -> Dict[str, Any]:
+ """Flatten a nested dictionary, joining keys using a specified separator.
+
+ :param dictionary: The dictionary to flatten.
+ :param parent_key: The base key to prepend to each key.
+ :param sep: The separator to use when joining keys.
+ :return: A flattened dictionary.
+ """
+ flattened_dict = {}
+ for key, value in dictionary.items():
+ new_key = f"{parent_key}{sep}{key}" if parent_key else key
+ if isinstance(value, dict):
+ flattened_dict.update(flatten_dict(value, new_key, sep=sep))
+ else:
+ flattened_dict[new_key] = value
+ return flattened_dict
+
+
+def collect_common_keys(list_of_dicts: List[Dict[str, Any]]) -> Dict[str, List[Any]]:
+ """Collect the common keys of a list of dictionaries. For each common key, put its values into a list in the order they appear in the original dictionaries.
+
+ :param list_of_dicts: The list of dictionaries to inspect.
+ :return: A dictionary with each common key and its corresponding list of values.
+ """
+ common_keys = set.intersection(*map(set, list_of_dicts))
+ common_dict = {key: [] for key in common_keys}
+ for d in list_of_dicts:
+ for key in common_keys:
+ common_dict[key].append(d[key])
+ return common_dict
+
+
+class CohereRequestResponseResolver:
+ """Class to resolve the request/response from the Cohere API and convert it to a dictionary that can be logged."""
+
+ def __call__(
+ self,
+ args: Sequence[Any],
+ kwargs: Dict[str, Any],
+ response: Response,
+ start_time: float,
+ time_elapsed: float,
+ ) -> Optional[Dict[str, Any]]:
+ """Process the response from the Cohere API and convert it to a dictionary that can be logged.
+
+ :param args: The arguments of the original function.
+ :param kwargs: The keyword arguments of the original function.
+ :param response: The response from the Cohere API.
+ :param start_time: The start time of the request.
+ :param time_elapsed: The time elapsed for the request.
+ :return: A dictionary containing the parsed response and timing information.
+ """
+ try:
+ # Each of the different endpoints map to one specific response type
+ # We want to 'type check' the response without directly importing the packages type
+ # It may make more sense to pass the invoked symbol from the AutologAPI instead
+ response_type = str(type(response)).split("'")[1].split(".")[-1]
+
+ # Initialize parsed_response to None to handle the case where the response type is unsupported
+ parsed_response = None
+ if response_type == "Generations":
+ parsed_response = self._resolve_generate_response(response)
+ # TODO: Remove hard-coded default model name
+ table_column_order = [
+ "start_time",
+ "query_id",
+ "model",
+ "prompt",
+ "text",
+ "token_likelihoods",
+ "likelihood",
+ "time_elapsed_(seconds)",
+ "end_time",
+ ]
+ default_model = "command"
+ elif response_type == "Chat":
+ parsed_response = self._resolve_chat_response(response)
+ table_column_order = [
+ "start_time",
+ "query_id",
+ "model",
+ "conversation_id",
+ "response_id",
+ "query",
+ "text",
+ "prompt",
+ "preamble",
+ "chat_history",
+ "chatlog",
+ "time_elapsed_(seconds)",
+ "end_time",
+ ]
+ default_model = "command"
+ elif response_type == "Classifications":
+ parsed_response = self._resolve_classify_response(response)
+ kwargs = self._resolve_classify_kwargs(kwargs)
+ table_column_order = [
+ "start_time",
+ "query_id",
+ "model",
+ "id",
+ "input",
+ "prediction",
+ "confidence",
+ "time_elapsed_(seconds)",
+ "end_time",
+ ]
+ default_model = "embed-english-v2.0"
+ elif response_type == "SummarizeResponse":
+ parsed_response = self._resolve_summarize_response(response)
+ table_column_order = [
+ "start_time",
+ "query_id",
+ "model",
+ "response_id",
+ "text",
+ "additional_command",
+ "summary",
+ "time_elapsed_(seconds)",
+ "end_time",
+ "length",
+ "format",
+ ]
+ default_model = "summarize-xlarge"
+ elif response_type == "Reranking":
+ parsed_response = self._resolve_rerank_response(response)
+ table_column_order = [
+ "start_time",
+ "query_id",
+ "model",
+ "id",
+ "query",
+ "top_n",
+ # This is a nested dict key that got flattened
+ "document-text",
+ "relevance_score",
+ "index",
+ "time_elapsed_(seconds)",
+ "end_time",
+ ]
+ default_model = "rerank-english-v2.0"
+ else:
+ logger.info(f"Unsupported Cohere response object: {response}")
+
+ return self._resolve(
+ args,
+ kwargs,
+ parsed_response,
+ start_time,
+ time_elapsed,
+ response_type,
+ table_column_order,
+ default_model,
+ )
+ except Exception as e:
+ logger.warning(f"Failed to resolve request/response: {e}")
+ return None
+
+ # These helper functions process the response from different endpoints of the Cohere API.
+ # Since the response objects for different endpoints have different structures,
+ # we need different logic to process them.
+
+ def _resolve_generate_response(self, response: Response) -> List[Dict[str, Any]]:
+ return_list = []
+ for _response in response:
+ # Built in Cohere.*.Generations function to color token_likelihoods and return a dict of response data
+ _response_dict = _response._visualize_helper()
+ try:
+ _response_dict["token_likelihoods"] = wandb.Html(
+ _response_dict["token_likelihoods"]
+ )
+ except (KeyError, ValueError):
+ pass
+ return_list.append(_response_dict)
+
+ return return_list
+
+ def _resolve_chat_response(self, response: Response) -> List[Dict[str, Any]]:
+ return [
+ subset_dict(
+ response.__dict__,
+ [
+ "response_id",
+ "generation_id",
+ "query",
+ "text",
+ "conversation_id",
+ "prompt",
+ "chatlog",
+ "preamble",
+ ],
+ )
+ ]
+
+ def _resolve_classify_response(self, response: Response) -> List[Dict[str, Any]]:
+ # The labels key is a dict returning the scores for the classification probability for each label provided
+ # We flatten this nested dict for ease of consumption in the wandb UI
+ return [flatten_dict(_response.__dict__) for _response in response]
+
+ def _resolve_classify_kwargs(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
+ # Example texts look strange when rendered in Wandb UI as it is a list of text and label
+ # We extract each value into its own column
+ example_texts = []
+ example_labels = []
+ for example in kwargs["examples"]:
+ example_texts.append(example.text)
+ example_labels.append(example.label)
+ kwargs.pop("examples")
+ kwargs["example_texts"] = example_texts
+ kwargs["example_labels"] = example_labels
+ return kwargs
+
+ def _resolve_summarize_response(self, response: Response) -> List[Dict[str, Any]]:
+ return [{"response_id": response.id, "summary": response.summary}]
+
+ def _resolve_rerank_response(self, response: Response) -> List[Dict[str, Any]]:
+ # The documents key contains a dict containing the content of the document which is at least "text"
+ # We flatten this nested dict for ease of consumption in the wandb UI
+ flattened_response_dicts = [
+ flatten_dict(_response.__dict__) for _response in response
+ ]
+ # ReRank returns each document provided a top_n value so we aggregate into one view so users can paginate a row
+ # As opposed to each row being one of the top_n responses
+ return_dict = collect_common_keys(flattened_response_dicts)
+ return_dict["id"] = response.id
+ return [return_dict]
+
+ def _resolve(
+ self,
+ args: Sequence[Any],
+ kwargs: Dict[str, Any],
+ parsed_response: List[Dict[str, Any]],
+ start_time: float,
+ time_elapsed: float,
+ response_type: str,
+ table_column_order: List[str],
+ default_model: str,
+ ) -> Dict[str, Any]:
+ """Convert a list of dictionaries to a pair of column names and corresponding values, with the option to order specific dictionaries.
+
+ :param args: The arguments passed to the API client.
+ :param kwargs: The keyword arguments passed to the API client.
+ :param parsed_response: The parsed response from the API.
+ :param start_time: The start time of the API request.
+ :param time_elapsed: The time elapsed during the API request.
+ :param response_type: The type of the API response.
+ :param table_column_order: The desired order of columns in the resulting table.
+ :param default_model: The default model to use if not specified in the response.
+ :return: A dictionary containing the formatted response.
+ """
+ # Args[0] is the client object where we can grab specific metadata about the underlying API status
+ query_id = generate_id(length=16)
+ parsed_args = subset_dict(
+ args[0].__dict__,
+ ["api_version", "batch_size", "max_retries", "num_workers", "timeout"],
+ )
+
+ start_time_dt = datetime.fromtimestamp(start_time)
+ end_time_dt = datetime.fromtimestamp(start_time + time_elapsed)
+
+ timings = {
+ "start_time": start_time_dt,
+ "end_time": end_time_dt,
+ "time_elapsed_(seconds)": time_elapsed,
+ }
+
+ packed_data = []
+ for _parsed_response in parsed_response:
+ _packed_dict = {
+ "query_id": query_id,
+ **kwargs,
+ **_parsed_response,
+ **timings,
+ **parsed_args,
+ }
+ if "model" not in _packed_dict:
+ _packed_dict["model"] = default_model
+ packed_data.append(_packed_dict)
+
+ columns, data = reorder_and_convert_dict_list_to_table(
+ packed_data, table_column_order
+ )
+
+ request_response_table = wandb.Table(data=data, columns=columns)
+
+ return {f"{response_type}": request_response_table}
diff --git a/wandb/integration/diffusers/__init__.py b/wandb/integration/diffusers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..5bcf7980133c41393fdc22db3bbff89be29be94e
--- /dev/null
+++ b/wandb/integration/diffusers/__init__.py
@@ -0,0 +1,3 @@
+from .autologger import autolog
+
+__all__ = ["autolog"]
diff --git a/wandb/integration/diffusers/autologger.py b/wandb/integration/diffusers/autologger.py
new file mode 100644
index 0000000000000000000000000000000000000000..c48de8cfa60e305453c1db43c3227b60cc115db5
--- /dev/null
+++ b/wandb/integration/diffusers/autologger.py
@@ -0,0 +1,71 @@
+import logging
+
+from wandb.sdk.integration_utils.auto_logging import AutologAPI
+
+from .pipeline_resolver import DiffusersPipelineResolver
+
+logger = logging.getLogger(__name__)
+
+autolog = AutologAPI(
+ name="diffusers",
+ symbols=(
+ "DiffusionPipeline.__call__",
+ "AutoPipelineForText2Image.__call__",
+ "AutoPipelineForImage2Image.__call__",
+ "AutoPipelineForInpainting.__call__",
+ "StableDiffusionPipeline.__call__",
+ "KandinskyCombinedPipeline.__call__",
+ "KandinskyV22CombinedPipeline.__call__",
+ "LatentConsistencyModelPipeline.__call__",
+ "LDMTextToImagePipeline.__call__",
+ "StableDiffusionPanoramaPipeline.__call__",
+ "StableDiffusionParadigmsPipeline.__call__",
+ "PixArtAlphaPipeline.__call__",
+ "StableDiffusionSAGPipeline.__call__",
+ "SemanticStableDiffusionPipeline.__call__",
+ "WuerstchenCombinedPipeline.__call__",
+ "AltDiffusionPipeline.__call__",
+ "StableDiffusionAttendAndExcitePipeline.__call__",
+ "StableDiffusionXLPipeline.__call__",
+ "StableDiffusionXLImg2ImgPipeline.__call__",
+ "IFPipeline.__call__",
+ "BlipDiffusionPipeline.__call__",
+ "BlipDiffusionControlNetPipeline.__call__",
+ "StableDiffusionControlNetPipeline.__call__",
+ "StableDiffusionControlNetImg2ImgPipeline.__call__",
+ "StableDiffusionControlNetInpaintPipeline.__call__",
+ "CycleDiffusionPipeline.__call__",
+ "StableDiffusionInstructPix2PixPipeline.__call__",
+ "PaintByExamplePipeline.__call__",
+ "RePaintPipeline.__call__",
+ "KandinskyImg2ImgCombinedPipeline.__call__",
+ "KandinskyInpaintCombinedPipeline.__call__",
+ "KandinskyV22Img2ImgCombinedPipeline.__call__",
+ "KandinskyV22InpaintCombinedPipeline.__call__",
+ "AnimateDiffPipeline.__call__",
+ "AudioLDMPipeline.__call__",
+ "AudioLDM2Pipeline.__call__",
+ "MusicLDMPipeline.__call__",
+ "StableDiffusionPix2PixZeroPipeline.__call__",
+ "PNDMPipeline.__call__",
+ "ShapEPipeline.__call__",
+ "StableDiffusionImg2ImgPipeline.__call__",
+ "StableDiffusionInpaintPipeline.__call__",
+ "StableDiffusionDepth2ImgPipeline.__call__",
+ "StableDiffusionImageVariationPipeline.__call__",
+ "StableDiffusionPipelineSafe.__call__",
+ "StableDiffusionUpscalePipeline.__call__",
+ "StableDiffusionAdapterPipeline.__call__",
+ "StableDiffusionGLIGENPipeline.__call__",
+ "StableDiffusionModelEditingPipeline.__call__",
+ "VersatileDiffusionTextToImagePipeline.__call__",
+ "VersatileDiffusionImageVariationPipeline.__call__",
+ "VersatileDiffusionDualGuidedPipeline.__call__",
+ "LDMPipeline.__call__",
+ "TextToVideoSDPipeline.__call__",
+ "TextToVideoZeroPipeline.__call__",
+ "StableVideoDiffusionPipeline.__call__",
+ ),
+ resolver=DiffusersPipelineResolver(),
+ telemetry_feature="diffusers_autolog",
+)
diff --git a/wandb/integration/diffusers/pipeline_resolver.py b/wandb/integration/diffusers/pipeline_resolver.py
new file mode 100644
index 0000000000000000000000000000000000000000..37cf75fd72dbe7baaff704abbdf16656afe6a1b2
--- /dev/null
+++ b/wandb/integration/diffusers/pipeline_resolver.py
@@ -0,0 +1,50 @@
+from typing import Any, Dict, Sequence
+
+from wandb.sdk.integration_utils.auto_logging import Response
+
+from .resolvers import (
+ SUPPORTED_MULTIMODAL_PIPELINES,
+ SUPPORTED_SDXL_PIPELINES,
+ DiffusersMultiModalPipelineResolver,
+ SDXLResolver,
+)
+
+
+class DiffusersPipelineResolver:
+ """Resolver for `DiffusionPipeline` request and responses from [HuggingFace Diffusers](https://huggingface.co/docs/diffusers/index), providing necessary data transformations, formatting, and logging.
+
+ This is based off `wandb.sdk.integration_utils.auto_logging.RequestResponseResolver`.
+ """
+
+ def __init__(self) -> None:
+ self.wandb_table = None
+
+ def __call__(
+ self,
+ args: Sequence[Any],
+ kwargs: Dict[str, Any],
+ response: Response,
+ start_time: float,
+ time_elapsed: float,
+ ) -> Any:
+ """Main call method for the `DiffusersPipelineResolver` class.
+
+ Arguments:
+ args: (Sequence[Any]) List of arguments.
+ kwargs: (Dict[str, Any]) Dictionary of keyword arguments.
+ response: (wandb.sdk.integration_utils.auto_logging.Response) The response from
+ the request.
+ start_time: (float) Time when request started.
+ time_elapsed: (float) Time elapsed for the request.
+
+ Returns:
+ Packed data as a dictionary for logging to wandb, None if an exception occurred.
+ """
+ pipeline_name = args[0].__class__.__name__
+ resolver = None
+ if pipeline_name in SUPPORTED_MULTIMODAL_PIPELINES:
+ resolver = DiffusersMultiModalPipelineResolver(pipeline_name)
+ elif pipeline_name in SUPPORTED_SDXL_PIPELINES:
+ resolver = SDXLResolver(pipeline_name)
+ loggable_dict = resolver(args, kwargs, response, start_time, time_elapsed)
+ return loggable_dict
diff --git a/wandb/integration/diffusers/resolvers/__init__.py b/wandb/integration/diffusers/resolvers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..3660810a382a20e1f9bf68cc9f0b0adf1c92889e
--- /dev/null
+++ b/wandb/integration/diffusers/resolvers/__init__.py
@@ -0,0 +1,12 @@
+from .multimodal import (
+ SUPPORTED_MULTIMODAL_PIPELINES,
+ DiffusersMultiModalPipelineResolver,
+)
+from .sdxl import SUPPORTED_SDXL_PIPELINES, SDXLResolver
+
+__all__ = [
+ "SUPPORTED_MULTIMODAL_PIPELINES",
+ "SUPPORTED_SDXL_PIPELINES",
+ "DiffusersMultiModalPipelineResolver",
+ "SDXLResolver",
+]
diff --git a/wandb/integration/diffusers/resolvers/multimodal.py b/wandb/integration/diffusers/resolvers/multimodal.py
new file mode 100644
index 0000000000000000000000000000000000000000..0f9a5221f88e7520786aba2448b9b24c8cd9417f
--- /dev/null
+++ b/wandb/integration/diffusers/resolvers/multimodal.py
@@ -0,0 +1,719 @@
+import logging
+from typing import Any, Dict, List, Sequence
+
+import wandb
+from wandb.sdk.integration_utils.auto_logging import Response
+
+from .utils import (
+ chunkify,
+ get_updated_kwargs,
+ postprocess_np_arrays_for_video,
+ postprocess_pils_to_np,
+)
+
+logger = logging.getLogger(__name__)
+
+
+SUPPORTED_MULTIMODAL_PIPELINES = {
+ "BlipDiffusionPipeline": {
+ "table-schema": [
+ "Reference-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Source-Subject-Category",
+ "Target-Subject-Category",
+ "Generated-Image",
+ ],
+ "kwarg-logging": [
+ "reference_image",
+ "prompt",
+ "neg_prompt",
+ "source_subject_category",
+ "target_subject_category",
+ ],
+ "kwarg-actions": [wandb.Image, None, None, None, None],
+ },
+ "BlipDiffusionControlNetPipeline": {
+ "table-schema": [
+ "Reference-Image",
+ "Control-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Source-Subject-Category",
+ "Target-Subject-Category",
+ "Generated-Image",
+ ],
+ "kwarg-logging": [
+ "reference_image",
+ "condtioning_image",
+ "prompt",
+ "neg_prompt",
+ "source_subject_category",
+ "target_subject_category",
+ ],
+ "kwarg-actions": [wandb.Image, wandb.Image, None, None, None, None],
+ },
+ "StableDiffusionControlNetPipeline": {
+ "table-schema": [
+ "Control-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["image", "prompt", "negative_prompt"],
+ "kwarg-actions": [wandb.Image, None, None],
+ },
+ "StableDiffusionControlNetImg2ImgPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Control-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["image", "control_image", "prompt", "negative_prompt"],
+ "kwarg-actions": [wandb.Image, wandb.Image, None, None],
+ },
+ "StableDiffusionControlNetInpaintPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Mask-Image",
+ "Control-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": [
+ "image",
+ "mask_image",
+ "control_image",
+ "prompt",
+ "negative_prompt",
+ ],
+ "kwarg-actions": [wandb.Image, wandb.Image, wandb.Image, None, None],
+ },
+ "CycleDiffusionPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Prompt",
+ "Source-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": [
+ "image",
+ "prompt",
+ "source_prompt",
+ ],
+ "kwarg-actions": [wandb.Image, None, None],
+ },
+ "StableDiffusionInstructPix2PixPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": [
+ "image",
+ "prompt",
+ "negative_prompt",
+ ],
+ "kwarg-actions": [wandb.Image, None, None],
+ },
+ "PaintByExamplePipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Example-Image",
+ "Mask-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": [
+ "image",
+ "example_image",
+ "mask_image",
+ ],
+ "kwarg-actions": [wandb.Image, wandb.Image, wandb.Image],
+ },
+ "RePaintPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Mask-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": [
+ "image",
+ "mask_image",
+ ],
+ "kwarg-actions": [wandb.Image, wandb.Image],
+ },
+ "StableDiffusionPipeline": {
+ "table-schema": ["Prompt", "Negative-Prompt", "Generated-Image"],
+ "kwarg-logging": ["prompt", "negative_prompt"],
+ "kwarg-actions": [None, None],
+ },
+ "KandinskyCombinedPipeline": {
+ "table-schema": ["Prompt", "Negative-Prompt", "Generated-Image"],
+ "kwarg-logging": ["prompt", "negative_prompt"],
+ "kwarg-actions": [None, None],
+ },
+ "KandinskyV22CombinedPipeline": {
+ "table-schema": ["Prompt", "Negative-Prompt", "Generated-Image"],
+ "kwarg-logging": ["prompt", "negative_prompt"],
+ "kwarg-actions": [None, None],
+ },
+ "LatentConsistencyModelPipeline": {
+ "table-schema": ["Prompt", "Generated-Image"],
+ "kwarg-logging": ["prompt"],
+ "kwarg-actions": [None],
+ },
+ "LDMTextToImagePipeline": {
+ "table-schema": ["Prompt", "Generated-Image"],
+ "kwarg-logging": ["prompt"],
+ "kwarg-actions": [None],
+ },
+ "StableDiffusionPanoramaPipeline": {
+ "table-schema": ["Prompt", "Negative-Prompt", "Generated-Image"],
+ "kwarg-logging": ["prompt", "negative_prompt"],
+ "kwarg-actions": [None, None],
+ },
+ "PixArtAlphaPipeline": {
+ "table-schema": ["Prompt", "Negative-Prompt", "Generated-Image"],
+ "kwarg-logging": ["prompt", "negative_prompt"],
+ "kwarg-actions": [None, None],
+ },
+ "StableDiffusionSAGPipeline": {
+ "table-schema": ["Prompt", "Negative-Prompt", "Generated-Image"],
+ "kwarg-logging": ["prompt", "negative_prompt"],
+ "kwarg-actions": [None, None],
+ },
+ "SemanticStableDiffusionPipeline": {
+ "table-schema": ["Prompt", "Negative-Prompt", "Generated-Image"],
+ "kwarg-logging": ["prompt", "negative_prompt"],
+ "kwarg-actions": [None, None],
+ },
+ "WuerstchenCombinedPipeline": {
+ "table-schema": ["Prompt", "Negative-Prompt", "Generated-Image"],
+ "kwarg-logging": ["prompt", "negative_prompt"],
+ "kwarg-actions": [None, None],
+ },
+ "IFPipeline": {
+ "table-schema": ["Prompt", "Negative-Prompt", "Generated-Image"],
+ "kwarg-logging": ["prompt", "negative_prompt"],
+ "kwarg-actions": [None, None],
+ },
+ "AltDiffusionPipeline": {
+ "table-schema": ["Prompt", "Negative-Prompt", "Generated-Image"],
+ "kwarg-logging": ["prompt", "negative_prompt"],
+ "kwarg-actions": [None, None],
+ },
+ "StableDiffusionAttendAndExcitePipeline": {
+ "table-schema": ["Prompt", "Negative-Prompt", "Generated-Image"],
+ "kwarg-logging": ["prompt", "negative_prompt"],
+ "kwarg-actions": [None, None],
+ },
+ "KandinskyImg2ImgCombinedPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["image", "prompt", "negative_prompt"],
+ "kwarg-actions": [wandb.Image, None, None],
+ },
+ "KandinskyInpaintCombinedPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["image", "prompt", "negative_prompt"],
+ "kwarg-actions": [wandb.Image, None, None],
+ },
+ "KandinskyV22Img2ImgCombinedPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["image", "prompt", "negative_prompt"],
+ "kwarg-actions": [wandb.Image, None, None],
+ },
+ "KandinskyV22InpaintCombinedPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["image", "prompt", "negative_prompt"],
+ "kwarg-actions": [wandb.Image, None, None],
+ },
+ "AnimateDiffPipeline": {
+ "table-schema": [
+ "Prompt",
+ "Negative-Prompt",
+ "Number-of-Frames",
+ "Generated-Video",
+ ],
+ "kwarg-logging": ["prompt", "negative_prompt", "num_frames"],
+ "kwarg-actions": [None, None, None],
+ "output-type": "video",
+ },
+ "StableVideoDiffusionPipeline": {
+ "table-schema": [
+ "Input-Image",
+ "Frames-Per-Second",
+ "Generated-Video",
+ ],
+ "kwarg-logging": ["image", "fps"],
+ "kwarg-actions": [wandb.Image, None],
+ "output-type": "video",
+ },
+ "AudioLDMPipeline": {
+ "table-schema": [
+ "Prompt",
+ "Negative-Prompt",
+ "Audio-Length-in-Seconds",
+ "Generated-Audio",
+ ],
+ "kwarg-logging": ["prompt", "negative_prompt", "audio_length_in_s"],
+ "kwarg-actions": [None, None, None],
+ "output-type": "audio",
+ },
+ "AudioLDM2Pipeline": {
+ "table-schema": [
+ "Prompt",
+ "Negative-Prompt",
+ "Audio-Length-in-Seconds",
+ "Generated-Audio",
+ ],
+ "kwarg-logging": ["prompt", "negative_prompt", "audio_length_in_s"],
+ "kwarg-actions": [None, None, None],
+ "output-type": "audio",
+ },
+ "MusicLDMPipeline": {
+ "table-schema": [
+ "Prompt",
+ "Negative-Prompt",
+ "Audio-Length-in-Seconds",
+ "Generated-Audio",
+ ],
+ "kwarg-logging": ["prompt", "negative_prompt", "audio_length_in_s"],
+ "kwarg-actions": [None, None, None],
+ "output-type": "audio",
+ },
+ "StableDiffusionPix2PixZeroPipeline": {
+ "table-schema": [
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["prompt", "negative_prompt"],
+ "kwarg-actions": [None, None],
+ },
+ "PNDMPipeline": {
+ "table-schema": [
+ "Batch-Size",
+ "Number-of-Inference-Steps",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["batch_size", "num_inference_steps"],
+ "kwarg-actions": [None, None],
+ },
+ "ShapEPipeline": {
+ "table-schema": [
+ "Prompt",
+ "Generated-Video",
+ ],
+ "kwarg-logging": ["prompt"],
+ "kwarg-actions": [None],
+ "output-type": "video",
+ },
+ "StableDiffusionImg2ImgPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["image", "prompt", "negative_prompt"],
+ "kwarg-actions": [wandb.Image, None, None],
+ },
+ "StableDiffusionInpaintPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Mask-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["image", "mask_image", "prompt", "negative_prompt"],
+ "kwarg-actions": [wandb.Image, wandb.Image, None, None],
+ },
+ "StableDiffusionDepth2ImgPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["image", "prompt", "negative_prompt"],
+ "kwarg-actions": [wandb.Image, None, None],
+ },
+ "StableDiffusionImageVariationPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Generated-Image",
+ ],
+ "kwarg-logging": [
+ "image",
+ ],
+ "kwarg-actions": [wandb.Image],
+ },
+ "StableDiffusionPipelineSafe": {
+ "table-schema": [
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["prompt", "negative_prompt"],
+ "kwarg-actions": [None, None],
+ },
+ "StableDiffusionUpscalePipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Upscaled-Image",
+ ],
+ "kwarg-logging": ["image", "prompt", "negative_prompt"],
+ "kwarg-actions": [wandb.Image, None, None],
+ },
+ "StableDiffusionAdapterPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["image", "prompt", "negative_prompt"],
+ "kwarg-actions": [wandb.Image, None, None],
+ },
+ "StableDiffusionGLIGENPipeline": {
+ "table-schema": [
+ "Prompt",
+ "GLIGEN-Phrases",
+ "GLIGEN-Boxes",
+ "GLIGEN-Inpaint-Image",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": [
+ "prompt",
+ "gligen_phrases",
+ "gligen_boxes",
+ "gligen_inpaint_image",
+ "negative_prompt",
+ ],
+ "kwarg-actions": [None, None, None, wandb.Image, None],
+ },
+ "VersatileDiffusionTextToImagePipeline": {
+ "table-schema": [
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["prompt", "negative_prompt"],
+ "kwarg-actions": [None, None],
+ },
+ "VersatileDiffusionImageVariationPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["image", "negative_prompt"],
+ "kwarg-actions": [wandb.Image, None],
+ },
+ "VersatileDiffusionDualGuidedPipeline": {
+ "table-schema": [
+ "Source-Image",
+ "Prompt",
+ "Negative-Prompt",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["image", "prompt", "negative_prompt"],
+ "kwarg-actions": [wandb.Image, None, None],
+ },
+ "LDMPipeline": {
+ "table-schema": [
+ "Batch-Size",
+ "Number-of-Inference-Steps",
+ "Generated-Image",
+ ],
+ "kwarg-logging": ["batch_size", "num_inference_steps"],
+ "kwarg-actions": [None, None],
+ },
+ "TextToVideoSDPipeline": {
+ "table-schema": [
+ "Prompt",
+ "Negative-Prompt",
+ "Number-of-Frames",
+ "Generated-Video",
+ ],
+ "kwarg-logging": ["prompt", "negative_prompt", "num_frames"],
+ "output-type": "video",
+ },
+ "TextToVideoZeroPipeline": {
+ "table-schema": [
+ "Prompt",
+ "Negative-Prompt",
+ "Number-of-Frames",
+ "Generated-Video",
+ ],
+ "kwarg-logging": ["prompt", "negative_prompt", "video_length"],
+ },
+}
+
+
+class DiffusersMultiModalPipelineResolver:
+ """Resolver for request and responses from [HuggingFace Diffusers](https://huggingface.co/docs/diffusers/index) multi-modal Diffusion Pipelines, providing necessary data transformations, formatting, and logging.
+
+ This resolver is internally involved in the
+ `__call__` for `wandb.integration.diffusers.pipeline_resolver.DiffusersPipelineResolver`.
+ This is based on `wandb.sdk.integration_utils.auto_logging.RequestResponseResolver`.
+
+ Arguments:
+ pipeline_name: (str) The name of the Diffusion Pipeline.
+ """
+
+ def __init__(self, pipeline_name: str) -> None:
+ self.pipeline_name = pipeline_name
+ columns = []
+ if pipeline_name in SUPPORTED_MULTIMODAL_PIPELINES:
+ columns += SUPPORTED_MULTIMODAL_PIPELINES[pipeline_name]["table-schema"]
+ else:
+ wandb.Error("Pipeline not supported for logging")
+ self.wandb_table = wandb.Table(columns=columns)
+
+ def __call__(
+ self,
+ args: Sequence[Any],
+ kwargs: Dict[str, Any],
+ response: Response,
+ start_time: float,
+ time_elapsed: float,
+ ) -> Any:
+ """Main call method for the `DiffusersPipelineResolver` class.
+
+ Arguments:
+ args: (Sequence[Any]) List of arguments.
+ kwargs: (Dict[str, Any]) Dictionary of keyword arguments.
+ response: (wandb.sdk.integration_utils.auto_logging.Response) The response from
+ the request.
+ start_time: (float) Time when request started.
+ time_elapsed: (float) Time elapsed for the request.
+
+ Returns:
+ Packed data as a dictionary for logging to wandb, None if an exception occurred.
+ """
+ try:
+ # Get the pipeline and the args
+ pipeline, args = args[0], args[1:]
+
+ # Update the Kwargs so that they can be logged easily
+ kwargs = get_updated_kwargs(pipeline, args, kwargs)
+
+ # Get the pipeline configs
+ pipeline_configs = dict(pipeline.config)
+ pipeline_configs["pipeline-name"] = self.pipeline_name
+
+ wandb.config.update(
+ {"workflow": {"pipeline": pipeline_configs, "params": kwargs}}
+ )
+
+ # Return the WandB loggable dict
+ loggable_dict = self.prepare_loggable_dict(response, kwargs)
+ return loggable_dict
+ except Exception as e:
+ logger.warning(e)
+ return None
+
+ def get_output_images(self, response: Response) -> List:
+ """Unpack the generated images, audio, video, etc. from the Diffusion Pipeline's response.
+
+ Arguments:
+ response: (wandb.sdk.integration_utils.auto_logging.Response) The response from
+ the request.
+
+ Returns:
+ List of generated images, audio, video, etc.
+ """
+ if "output-type" not in SUPPORTED_MULTIMODAL_PIPELINES[self.pipeline_name]:
+ return response.images
+ else:
+ if (
+ SUPPORTED_MULTIMODAL_PIPELINES[self.pipeline_name]["output-type"]
+ == "video"
+ ):
+ if self.pipeline_name in ["ShapEPipeline"]:
+ return response.images
+ return response.frames
+ elif (
+ SUPPORTED_MULTIMODAL_PIPELINES[self.pipeline_name]["output-type"]
+ == "audio"
+ ):
+ return response.audios
+
+ def log_media(self, image: Any, loggable_kwarg_chunks: List, idx: int) -> None:
+ """Log the generated images, audio, video, etc. from the Diffusion Pipeline's response along with an optional caption to a media panel in the run.
+
+ Arguments:
+ image: (Any) The generated images, audio, video, etc. from the Diffusion
+ Pipeline's response.
+ loggable_kwarg_chunks: (List) Loggable chunks of kwargs.
+ """
+ if "output-type" not in SUPPORTED_MULTIMODAL_PIPELINES[self.pipeline_name]:
+ try:
+ prompt_index = SUPPORTED_MULTIMODAL_PIPELINES[self.pipeline_name][
+ "kwarg-logging"
+ ].index("prompt")
+ caption = loggable_kwarg_chunks[prompt_index][idx]
+ except ValueError:
+ caption = None
+ wandb.log({"Generated-Image": wandb.Image(image, caption=caption)})
+ else:
+ if (
+ SUPPORTED_MULTIMODAL_PIPELINES[self.pipeline_name]["output-type"]
+ == "video"
+ ):
+ try:
+ prompt_index = SUPPORTED_MULTIMODAL_PIPELINES[self.pipeline_name][
+ "kwarg-logging"
+ ].index("prompt")
+ caption = loggable_kwarg_chunks[prompt_index][idx]
+ except ValueError:
+ caption = None
+ wandb.log(
+ {
+ "Generated-Video": wandb.Video(
+ postprocess_pils_to_np(image), fps=4, caption=caption
+ )
+ }
+ )
+ elif (
+ SUPPORTED_MULTIMODAL_PIPELINES[self.pipeline_name]["output-type"]
+ == "audio"
+ ):
+ try:
+ prompt_index = SUPPORTED_MULTIMODAL_PIPELINES[self.pipeline_name][
+ "kwarg-logging"
+ ].index("prompt")
+ caption = loggable_kwarg_chunks[prompt_index][idx]
+ except ValueError:
+ caption = None
+ wandb.log(
+ {
+ "Generated-Audio": wandb.Audio(
+ image, sample_rate=16000, caption=caption
+ )
+ }
+ )
+
+ def add_data_to_table(
+ self, image: Any, loggable_kwarg_chunks: List, idx: int
+ ) -> None:
+ """Populate the row of the `wandb.Table`.
+
+ Arguments:
+ image: (Any) The generated images, audio, video, etc. from the Diffusion
+ Pipeline's response.
+ loggable_kwarg_chunks: (List) Loggable chunks of kwargs.
+ idx: (int) Chunk index.
+ """
+ table_row = []
+ kwarg_actions = SUPPORTED_MULTIMODAL_PIPELINES[self.pipeline_name][
+ "kwarg-actions"
+ ]
+ for column_idx, loggable_kwarg_chunk in enumerate(loggable_kwarg_chunks):
+ if kwarg_actions[column_idx] is None:
+ table_row.append(
+ loggable_kwarg_chunk[idx]
+ if loggable_kwarg_chunk[idx] is not None
+ else ""
+ )
+ else:
+ table_row.append(kwarg_actions[column_idx](loggable_kwarg_chunk[idx]))
+ if "output-type" not in SUPPORTED_MULTIMODAL_PIPELINES[self.pipeline_name]:
+ table_row.append(wandb.Image(image))
+ else:
+ if (
+ SUPPORTED_MULTIMODAL_PIPELINES[self.pipeline_name]["output-type"]
+ == "video"
+ ):
+ table_row.append(wandb.Video(postprocess_pils_to_np(image), fps=4))
+ elif (
+ SUPPORTED_MULTIMODAL_PIPELINES[self.pipeline_name]["output-type"]
+ == "audio"
+ ):
+ table_row.append(wandb.Audio(image, sample_rate=16000))
+ self.wandb_table.add_data(*table_row)
+
+ def prepare_loggable_dict(
+ self, response: Response, kwargs: Dict[str, Any]
+ ) -> Dict[str, Any]:
+ """Prepare the loggable dictionary, which is the packed data as a dictionary for logging to wandb, None if an exception occurred.
+
+ Arguments:
+ response: (wandb.sdk.integration_utils.auto_logging.Response) The response from
+ the request.
+ kwargs: (Dict[str, Any]) Dictionary of keyword arguments.
+
+ Returns:
+ Packed data as a dictionary for logging to wandb, None if an exception occurred.
+ """
+ # Unpack the generated images, audio, video, etc. from the Diffusion Pipeline's response.
+ images = self.get_output_images(response)
+
+ # Account for exception pipelines for text-to-video
+ if self.pipeline_name in ["TextToVideoSDPipeline", "TextToVideoZeroPipeline"]:
+ video = postprocess_np_arrays_for_video(
+ images, normalize=self.pipeline_name == "TextToVideoZeroPipeline"
+ )
+ wandb.log(
+ {"Generated-Video": wandb.Video(video, fps=4, caption=kwargs["prompt"])}
+ )
+ loggable_kwarg_ids = SUPPORTED_MULTIMODAL_PIPELINES[self.pipeline_name][
+ "kwarg-logging"
+ ]
+ table_row = [
+ kwargs[loggable_kwarg_ids[idx]]
+ for idx in range(len(loggable_kwarg_ids))
+ ]
+ table_row.append(wandb.Video(video, fps=4))
+ self.wandb_table.add_data(*table_row)
+ else:
+ loggable_kwarg_ids = SUPPORTED_MULTIMODAL_PIPELINES[self.pipeline_name][
+ "kwarg-logging"
+ ]
+ # chunkify loggable kwargs
+ loggable_kwarg_chunks = []
+ for loggable_kwarg_id in loggable_kwarg_ids:
+ loggable_kwarg_chunks.append(
+ kwargs[loggable_kwarg_id]
+ if isinstance(kwargs[loggable_kwarg_id], list)
+ else [kwargs[loggable_kwarg_id]]
+ )
+ # chunkify the generated media
+ images = chunkify(images, len(loggable_kwarg_chunks[0]))
+ for idx in range(len(loggable_kwarg_chunks[0])):
+ for image in images[idx]:
+ # Log media to media panel
+ self.log_media(image, loggable_kwarg_chunks, idx)
+ # Populate the row of the wandb_table
+ self.add_data_to_table(image, loggable_kwarg_chunks, idx)
+
+ return {"Result-Table": self.wandb_table}
diff --git a/wandb/integration/diffusers/resolvers/sdxl.py b/wandb/integration/diffusers/resolvers/sdxl.py
new file mode 100644
index 0000000000000000000000000000000000000000..998ad22dc5cc622a11ddc803d35ff58843e3fadb
--- /dev/null
+++ b/wandb/integration/diffusers/resolvers/sdxl.py
@@ -0,0 +1,274 @@
+import logging
+from typing import TYPE_CHECKING, Any, Dict, List, Sequence
+
+import wandb
+from wandb.sdk.integration_utils.auto_logging import Response
+from wandb.util import get_module
+
+from .utils import chunkify, get_updated_kwargs
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+ torch_float_tensor = get_module("torch.FloatTensor")
+
+
+SUPPORTED_SDXL_PIPELINES = [
+ "StableDiffusionXLPipeline",
+ "StableDiffusionXLImg2ImgPipeline",
+]
+
+TEXT_TO_IMAGE_COLUMNS = [
+ "Workflow-Stage",
+ "Prompt",
+ "Negative-Prompt",
+ "Prompt-2",
+ "Negative-Prompt-2",
+ "Generated-Image",
+]
+
+
+def decode_sdxl_t2i_latents(pipeline: Any, latents: "torch_float_tensor") -> List:
+ """Decode latents generated by [`diffusers.StableDiffusionXLPipeline`](https://huggingface.co/docs/diffusers/main/en/api/pipelines/stable_diffusion/stable_diffusion_xl#stable-diffusion-xl).
+
+ Arguments:
+ pipeline: (diffusers.DiffusionPipeline) The Diffusion Pipeline from
+ [`diffusers`](https://huggingface.co/docs/diffusers).
+ latents (torch.FloatTensor): The generated latents.
+
+ Returns:
+ List of `PIL` images corresponding to the generated latents.
+ """
+ torch = get_module(
+ "torch",
+ required="Please ensure PyTorch is installed. You can check out https://pytorch.org/get-started/locally/#start-locally for installation instructions.",
+ )
+ with torch.no_grad():
+ needs_upcasting = (
+ pipeline.vae.dtype == torch.float16 and pipeline.vae.config.force_upcast
+ )
+ if needs_upcasting:
+ pipeline.upcast_vae()
+ latents = latents.to(
+ next(iter(pipeline.vae.post_quant_conv.parameters())).dtype
+ )
+ images = pipeline.vae.decode(
+ latents / pipeline.vae.config.scaling_factor, return_dict=False
+ )[0]
+ if needs_upcasting:
+ pipeline.vae.to(dtype=torch.float16)
+ if pipeline.watermark is not None:
+ images = pipeline.watermark.apply_watermark(images)
+ images = pipeline.image_processor.postprocess(images, output_type="pil")
+ pipeline.maybe_free_model_hooks()
+ return images
+
+
+class SDXLResolver:
+ """Resolver for request and responses from [`diffusers.StableDiffusionXLPipeline`](https://huggingface.co/docs/diffusers/main/en/api/pipelines/stable_diffusion/stable_diffusion_xl#stable-diffusion-xl) multi-modal Diffusion Pipelines, providing necessary data transformations, formatting, and logging.
+
+ Arguments:
+ pipeline_name: (str) The name of the Diffusion Pipeline.
+ """
+
+ def __init__(self, pipeline_name: str) -> None:
+ self.pipeline_name = pipeline_name
+ self.wandb_table = None
+ self.task = None
+ self.workflow_stage = None
+
+ def __call__(
+ self,
+ args: Sequence[Any],
+ kwargs: Dict[str, Any],
+ response: Response,
+ start_time: float,
+ time_elapsed: float,
+ ) -> Any:
+ """Main call method for the `DiffusersPipelineResolver` class.
+
+ Arguments:
+ args: (Sequence[Any]) List of arguments.
+ kwargs: (Dict[str, Any]) Dictionary of keyword arguments.
+ response: (wandb.sdk.integration_utils.auto_logging.Response) The response from
+ the request.
+ start_time: (float) Time when request started.
+ time_elapsed: (float) Time elapsed for the request.
+
+ Returns:
+ Packed data as a dictionary for logging to wandb, None if an exception occurred.
+ """
+ try:
+ # Get the pipeline and the args
+ pipeline, args = args[0], args[1:]
+
+ # Update the Kwargs so that they can be logged easily
+ kwargs = get_updated_kwargs(pipeline, args, kwargs)
+
+ # Get the pipeline configs
+ pipeline_configs = dict(pipeline.config)
+ pipeline_configs["pipeline-name"] = self.pipeline_name
+
+ # Return the WandB loggable dict
+ loggable_dict = self.prepare_loggable_dict(
+ pipeline, pipeline_configs, response, kwargs
+ )
+ return loggable_dict
+ except Exception as e:
+ logger.warning(e)
+ return None
+
+ def create_wandb_table(self, pipeline_configs: Dict[str, Any]) -> None:
+ """Create the `wandb.Table` with the specified schema.
+
+ Arguments:
+ pipeline_configs: (Dict[str, Any]) The configs corresponding to the
+ architecture of the pipeline.
+ """
+ columns = []
+ if self.pipeline_name == "StableDiffusionXLPipeline":
+ columns += TEXT_TO_IMAGE_COLUMNS
+ self.task = "text_to_image"
+ self.workflow_stage = "Base"
+ elif self.pipeline_name == "StableDiffusionXLImg2ImgPipeline":
+ if (
+ pipeline_configs["_name_or_path"]
+ == "stabilityai/stable-diffusion-xl-refiner-1.0"
+ ):
+ columns += TEXT_TO_IMAGE_COLUMNS
+ self.task = "text_to_image"
+ self.workflow_stage = "Refiner"
+ self.wandb_table = wandb.Table(columns=columns)
+
+ def prepare_loggable_dict_for_text_to_image(
+ self,
+ pipeline: Any,
+ workflow_stage: str,
+ response: Response,
+ kwargs: Dict[str, Any],
+ ) -> Dict[str, Any]:
+ """Prepare the loggable dictionary for text-to-image workflow.
+
+ Arguments:
+ pipeline: (Any) The Diffusion Pipeline.
+ workflow_stage: (str) The name of the workflow stage.
+ response: (wandb.sdk.integration_utils.auto_logging.Response) The response from
+ the request.
+ kwargs: (Dict[str, Any]) Dictionary of keyword arguments.
+
+ """
+ prompt_logging = (
+ kwargs["prompt"]
+ if isinstance(kwargs["prompt"], list)
+ else [kwargs["prompt"]]
+ )
+ prompt2_logging = (
+ kwargs["prompt_2"]
+ if isinstance(kwargs["prompt_2"], list)
+ else [kwargs["prompt_2"]]
+ )
+ negative_prompt_logging = (
+ kwargs["negative_prompt"]
+ if isinstance(kwargs["negative_prompt"], list)
+ else [kwargs["negative_prompt"]]
+ )
+ negative_prompt2_logging = (
+ kwargs["negative_prompt_2"]
+ if isinstance(kwargs["negative_prompt_2"], list)
+ else [kwargs["negative_prompt_2"]]
+ )
+ images = (
+ decode_sdxl_t2i_latents(pipeline, response.images)
+ if kwargs["output_type"] == "latent"
+ else response.images
+ )
+ images = chunkify(images, len(prompt_logging))
+ for idx in range(len(prompt_logging)):
+ for image in images[idx]:
+ wandb.log(
+ {
+ f"Generated-Image/{workflow_stage}": wandb.Image(
+ image,
+ caption=f"Prompt-1: {prompt_logging[idx]}\nPrompt-2: {prompt2_logging[idx]}",
+ )
+ }
+ )
+ self.wandb_table.add_data(
+ workflow_stage,
+ prompt_logging[idx] if prompt_logging[idx] is not None else "",
+ negative_prompt_logging[idx]
+ if negative_prompt_logging[idx] is not None
+ else "",
+ prompt2_logging[idx] if prompt2_logging[idx] is not None else "",
+ negative_prompt2_logging[idx]
+ if negative_prompt2_logging[idx] is not None
+ else "",
+ wandb.Image(image),
+ )
+
+ def update_wandb_configs(
+ self, pipeline_configs: Dict[str, Any], kwargs: Dict[str, Any]
+ ) -> None:
+ """Update the configs of the W&B run.
+
+ Arguments:
+ pipeline_configs: (Dict[str, Any]) The configs corresponding to the
+ architecture of the pipeline.
+ kwargs: (Dict[str, Any]) Dictionary of keyword arguments.
+
+ """
+ if "workflow" not in wandb.config:
+ wandb.config.update(
+ {
+ "workflow": [
+ {
+ "pipeline": pipeline_configs,
+ "params": kwargs,
+ "stage": self.workflow_stage,
+ }
+ ]
+ }
+ )
+ else:
+ existing_workflow = wandb.config.workflow
+ updated_workflow = existing_workflow + [
+ {
+ "pipeline": pipeline_configs,
+ "params": kwargs,
+ "stage": self.workflow_stage,
+ }
+ ]
+ wandb.config.update(
+ {
+ "workflow": updated_workflow,
+ },
+ allow_val_change=True,
+ )
+
+ def prepare_loggable_dict(
+ self,
+ pipeline: Any,
+ pipeline_configs: Dict[str, Any],
+ response: Response,
+ kwargs: Dict[str, Any],
+ ) -> Dict[str, Any]:
+ """Prepare the loggable dictionary, which is the packed data as a dictionary for logging to wandb, None if an exception occurred.
+
+ Arguments:
+ pipeline: (Any) The Diffusion Pipeline.
+ pipeline_configs: (Dict[str, Any]) The configs corresponding to the
+ architecture of the pipeline.
+ response: (wandb.sdk.integration_utils.auto_logging.Response) The
+ response from the request.
+ kwargs: (Dict[str, Any]) Dictionary of keyword arguments.
+
+ Returns:
+ Packed data as a dictionary for logging to wandb, None if an exception occurred.
+ """
+ self.create_wandb_table(pipeline_configs)
+ if self.task == "text_to_image":
+ self.prepare_loggable_dict_for_text_to_image(
+ pipeline, self.workflow_stage, response, kwargs
+ )
+ self.update_wandb_configs(pipeline_configs, kwargs)
+ return {f"{self.workflow_stage}/Result-Table": self.wandb_table}
diff --git a/wandb/integration/diffusers/resolvers/utils.py b/wandb/integration/diffusers/resolvers/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..07b0806c0f043791a8d94d649e7be9064b22f83d
--- /dev/null
+++ b/wandb/integration/diffusers/resolvers/utils.py
@@ -0,0 +1,61 @@
+import inspect
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence
+
+import wandb
+from wandb.util import get_module
+
+if TYPE_CHECKING:
+ np_array = get_module("numpy.array")
+
+
+def chunkify(input_list, chunk_size) -> List:
+ chunk_size = max(1, chunk_size)
+ return [
+ input_list[i : i + chunk_size] for i in range(0, len(input_list), chunk_size)
+ ]
+
+
+def get_updated_kwargs(
+ pipeline: Any, args: Sequence[Any], kwargs: Dict[str, Any]
+) -> Dict[str, Any]:
+ pipeline_call_parameters = list(
+ inspect.signature(pipeline.__call__).parameters.items()
+ )
+ for idx, arg in enumerate(args):
+ kwargs[pipeline_call_parameters[idx][0]] = arg
+ for pipeline_parameter in pipeline_call_parameters:
+ if pipeline_parameter[0] not in kwargs:
+ kwargs[pipeline_parameter[0]] = pipeline_parameter[1].default
+ if "generator" in kwargs:
+ generator = kwargs.pop("generator", None)
+ kwargs["seed"] = (
+ generator.get_state().to("cpu").tolist()[0]
+ if generator is not None
+ else None
+ )
+ if "ip_adapter_image" in kwargs:
+ if kwargs["ip_adapter_image"] is not None:
+ wandb.log({"IP-Adapter-Image": wandb.Image(kwargs["ip_adapter_image"])})
+ return kwargs
+
+
+def postprocess_pils_to_np(image: List) -> "np_array":
+ np = get_module(
+ "numpy",
+ required="Please ensure NumPy is installed. You can run `pip install numpy` to install it.",
+ )
+ return np.stack(
+ [np.transpose(np.array(img).astype("uint8"), axes=(2, 0, 1)) for img in image],
+ axis=0,
+ )
+
+
+def postprocess_np_arrays_for_video(
+ images: List["np_array"], normalize: Optional[bool] = False
+) -> "np_array":
+ np = get_module(
+ "numpy",
+ required="Please ensure NumPy is installed. You can run `pip install numpy` to install it.",
+ )
+ images = [(img * 255).astype("uint8") for img in images] if normalize else images
+ return np.transpose(np.stack((images), axis=0), axes=(0, 3, 1, 2))
diff --git a/wandb/integration/fastai/__init__.py b/wandb/integration/fastai/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..43142e4dafc8992398a3e52ca4d6f1ab71d89734
--- /dev/null
+++ b/wandb/integration/fastai/__init__.py
@@ -0,0 +1,248 @@
+"""Hooks that add fast.ai v1 Learners to Weights & Biases through a callback.
+
+Requested logged data can be configured through the callback constructor.
+
+Examples:
+ WandbCallback can be used when initializing the Learner::
+
+ ```
+ from wandb.fastai import WandbCallback
+ [...]
+ learn = Learner(data, ..., callback_fns=WandbCallback)
+ learn.fit(epochs)
+ ```
+
+ Custom parameters can be given using functools.partial::
+
+ ```
+ from wandb.fastai import WandbCallback
+ from functools import partial
+ [...]
+ learn = Learner(data, ..., callback_fns=partial(WandbCallback, ...))
+ learn.fit(epochs)
+ ```
+
+ Finally, it is possible to use WandbCallback only when starting
+ training. In this case it must be instantiated::
+
+ ```
+ learn.fit(..., callbacks=WandbCallback(learn))
+ ```
+
+ or, with custom parameters::
+
+ ```
+ learn.fit(..., callbacks=WandbCallback(learn, ...))
+ ```
+"""
+import random
+import sys
+from pathlib import Path
+from typing import Any, Optional
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+import fastai
+from fastai.callbacks import TrackerCallback
+
+import wandb
+
+try:
+ import matplotlib
+
+ if wandb.wandb_lib.ipython._get_python_type() != "jupyter": # type: ignore[attr-defined]
+ matplotlib.use("Agg") # non-interactive backend (avoid tkinter issues)
+ import matplotlib.pyplot as plt
+except ImportError:
+ print("Warning: matplotlib required if logging sample image predictions")
+
+
+class WandbCallback(TrackerCallback):
+ """Callback for saving model topology, losses & metrics.
+
+ Optionally logs weights, gradients, sample predictions and best trained model.
+
+ Arguments:
+ learn (fastai.basic_train.Learner): the fast.ai learner to hook.
+ log (str): "gradients", "parameters", "all", or None. Losses & metrics are always logged.
+ save_model (bool): save model at the end of each epoch. It will also load best model at the end of training.
+ monitor (str): metric to monitor for saving best model. None uses default TrackerCallback monitor value.
+ mode (str): "auto", "min" or "max" to compare "monitor" values and define best model.
+ input_type (str): "images" or None. Used to display sample predictions.
+ validation_data (list): data used for sample predictions if input_type is set.
+ predictions (int): number of predictions to make if input_type is set and validation_data is None.
+ seed (int): initialize random generator for sample predictions if input_type is set and validation_data is None.
+ """
+
+ # Record if watch has been called previously (even in another instance)
+ _watch_called = False
+
+ def __init__(
+ self,
+ learn: "fastai.basic_train.Learner",
+ log: Optional[Literal["gradients", "parameters", "all"]] = "gradients",
+ save_model: bool = True,
+ monitor: Optional[str] = None,
+ mode: Literal["auto", "min", "max"] = "auto",
+ input_type: Optional[Literal["images"]] = None,
+ validation_data: Optional[list] = None,
+ predictions: int = 36,
+ seed: int = 12345,
+ ) -> None:
+ # Check if wandb.init has been called
+ if wandb.run is None:
+ raise ValueError("You must call wandb.init() before WandbCallback()")
+
+ # Adapted from fast.ai "SaveModelCallback"
+ if monitor is None:
+ # use default TrackerCallback monitor value
+ super().__init__(learn, mode=mode)
+ else:
+ super().__init__(learn, monitor=monitor, mode=mode)
+ self.save_model = save_model
+ self.model_path = Path(wandb.run.dir) / "bestmodel.pth"
+
+ self.log = log
+ self.input_type = input_type
+ self.best = None
+
+ # Select items for sample predictions to see evolution along training
+ self.validation_data = validation_data
+ if input_type and not self.validation_data:
+ wandb_random = random.Random(seed) # For repeatability
+ predictions = min(predictions, len(learn.data.valid_ds))
+ indices = wandb_random.sample(range(len(learn.data.valid_ds)), predictions)
+ self.validation_data = [learn.data.valid_ds[i] for i in indices]
+
+ def on_train_begin(self, **kwargs: Any) -> None:
+ """Call watch method to log model topology, gradients & weights."""
+ # Set self.best, method inherited from "TrackerCallback" by "SaveModelCallback"
+ super().on_train_begin()
+
+ # Ensure we don't call "watch" multiple times
+ if not WandbCallback._watch_called:
+ WandbCallback._watch_called = True
+
+ # Logs model topology and optionally gradients and weights
+ wandb.watch(self.learn.model, log=self.log)
+
+ def on_epoch_end(
+ self, epoch: int, smooth_loss: float, last_metrics: list, **kwargs: Any
+ ) -> None:
+ """Log training loss, validation loss and custom metrics & log prediction samples & save model."""
+ if self.save_model:
+ # Adapted from fast.ai "SaveModelCallback"
+ current = self.get_monitor_value()
+ if current is not None and self.operator(current, self.best):
+ print(
+ "Better model found at epoch {} with {} value: {}.".format(
+ epoch, self.monitor, current
+ )
+ )
+ self.best = current
+
+ # Save within wandb folder
+ with self.model_path.open("wb") as model_file:
+ self.learn.save(model_file)
+
+ # Log sample predictions if learn.predict is available
+ if self.validation_data:
+ try:
+ self._wandb_log_predictions()
+ except FastaiError as e:
+ wandb.termwarn(e.message)
+ self.validation_data = None # prevent from trying again on next loop
+ except Exception as e:
+ wandb.termwarn(f"Unable to log prediction samples.\n{e}")
+ self.validation_data = None # prevent from trying again on next loop
+
+ # Log losses & metrics
+ # Adapted from fast.ai "CSVLogger"
+ logs = {
+ name: stat
+ for name, stat in list(
+ zip(self.learn.recorder.names, [epoch, smooth_loss] + last_metrics)
+ )
+ }
+ wandb.log(logs)
+
+ def on_train_end(self, **kwargs: Any) -> None:
+ """Load the best model."""
+ if self.save_model:
+ # Adapted from fast.ai "SaveModelCallback"
+ if self.model_path.is_file():
+ with self.model_path.open("rb") as model_file:
+ self.learn.load(model_file, purge=False)
+ print(f"Loaded best saved model from {self.model_path}")
+
+ def _wandb_log_predictions(self) -> None:
+ """Log prediction samples."""
+ pred_log = []
+
+ if self.validation_data is None:
+ return
+
+ for x, y in self.validation_data:
+ try:
+ pred = self.learn.predict(x)
+ except Exception:
+ raise FastaiError(
+ 'Unable to run "predict" method from Learner to log prediction samples.'
+ )
+
+ # scalar -> likely to be a category
+ # tensor of dim 1 -> likely to be multicategory
+ if not pred[1].shape or pred[1].dim() == 1:
+ pred_log.append(
+ wandb.Image(
+ x.data,
+ caption=f"Ground Truth: {y}\nPrediction: {pred[0]}",
+ )
+ )
+
+ # most vision datasets have a "show" function we can use
+ elif hasattr(x, "show"):
+ # log input data
+ pred_log.append(wandb.Image(x.data, caption="Input data", grouping=3))
+
+ # log label and prediction
+ for im, capt in ((pred[0], "Prediction"), (y, "Ground Truth")):
+ # Resize plot to image resolution
+ # from https://stackoverflow.com/a/13714915
+ my_dpi = 100
+ fig = plt.figure(frameon=False, dpi=my_dpi)
+ h, w = x.size
+ fig.set_size_inches(w / my_dpi, h / my_dpi)
+ ax = plt.Axes(fig, [0.0, 0.0, 1.0, 1.0])
+ ax.set_axis_off()
+ fig.add_axes(ax)
+
+ # Superpose label or prediction to input image
+ x.show(ax=ax, y=im)
+ pred_log.append(wandb.Image(fig, caption=capt))
+ plt.close(fig)
+
+ # likely to be an image
+ elif hasattr(y, "shape") and (
+ (len(y.shape) == 2) or (len(y.shape) == 3 and y.shape[0] in [1, 3, 4])
+ ):
+ pred_log.extend(
+ [
+ wandb.Image(x.data, caption="Input data", grouping=3),
+ wandb.Image(pred[0].data, caption="Prediction"),
+ wandb.Image(y.data, caption="Ground Truth"),
+ ]
+ )
+
+ # we just log input data
+ else:
+ pred_log.append(wandb.Image(x.data, caption="Input data"))
+
+ wandb.log({"Prediction Samples": pred_log}, commit=False)
+
+
+class FastaiError(wandb.Error):
+ pass
diff --git a/wandb/integration/gym/__init__.py b/wandb/integration/gym/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..b0b3744dbc0e40a20b592fd2abba258a0571c3e8
--- /dev/null
+++ b/wandb/integration/gym/__init__.py
@@ -0,0 +1,84 @@
+import re
+import sys
+from typing import Optional
+
+import wandb
+import wandb.util
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+
+_gym_version_lt_0_26: Optional[bool] = None
+_required_error_msg = (
+ "Couldn't import the gymnasium python package, "
+ "install with `pip install gymnasium`"
+)
+GymLib = Literal["gym", "gymnasium"]
+
+
+def monitor():
+ """Monitor a gym environment.
+
+ Supports both gym and gymnasium.
+ """
+ gym_lib: Optional[GymLib] = None
+
+ # gym is not maintained anymore, gymnasium is the drop-in replacement - prefer it
+ if wandb.util.get_module("gymnasium") is not None:
+ gym_lib = "gymnasium"
+ elif wandb.util.get_module("gym") is not None:
+ gym_lib = "gym"
+
+ if gym_lib is None:
+ raise wandb.Error(_required_error_msg)
+
+ vcr = wandb.util.get_module(
+ f"{gym_lib}.wrappers.monitoring.video_recorder",
+ required=_required_error_msg,
+ )
+
+ global _gym_version_lt_0_26
+
+ if _gym_version_lt_0_26 is None:
+ if gym_lib == "gym":
+ import gym
+ else:
+ import gymnasium as gym # type: ignore
+ from pkg_resources import parse_version
+
+ if parse_version(gym.__version__) < parse_version("0.26.0"):
+ _gym_version_lt_0_26 = True
+ else:
+ _gym_version_lt_0_26 = False
+
+ # breaking change in gym 0.26.0
+ vcr_recorder_attribute = "ImageEncoder" if _gym_version_lt_0_26 else "VideoRecorder"
+ recorder = getattr(vcr, vcr_recorder_attribute)
+ path = "output_path" if _gym_version_lt_0_26 else "path"
+
+ recorder.orig_close = recorder.close
+
+ def close(self):
+ recorder.orig_close(self)
+ if not self.enabled:
+ return
+ if wandb.run:
+ m = re.match(r".+(video\.\d+).+", getattr(self, path))
+ key = m.group(1) if m else "videos"
+ wandb.log({key: wandb.Video(getattr(self, path))})
+
+ def del_(self):
+ self.orig_close()
+
+ if not _gym_version_lt_0_26:
+ recorder.__del__ = del_
+ recorder.close = close
+ wandb.patched["gym"].append(
+ [
+ f"{gym_lib}.wrappers.monitoring.video_recorder.{vcr_recorder_attribute}",
+ "close",
+ ]
+ )
diff --git a/wandb/integration/huggingface/__init__.py b/wandb/integration/huggingface/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..943249ebe22686fcbdfaa6708c33e15ec875bccc
--- /dev/null
+++ b/wandb/integration/huggingface/__init__.py
@@ -0,0 +1,3 @@
+__all__ = ("autolog",)
+
+from .huggingface import autolog
diff --git a/wandb/integration/huggingface/huggingface.py b/wandb/integration/huggingface/huggingface.py
new file mode 100644
index 0000000000000000000000000000000000000000..d44cf2cbb51d009821e26924ab944a953a1ff866
--- /dev/null
+++ b/wandb/integration/huggingface/huggingface.py
@@ -0,0 +1,18 @@
+import logging
+
+from wandb.sdk.integration_utils.auto_logging import AutologAPI
+
+from .resolver import HuggingFacePipelineRequestResponseResolver
+
+logger = logging.getLogger(__name__)
+
+resolver = HuggingFacePipelineRequestResponseResolver()
+
+autolog = AutologAPI(
+ name="transformers",
+ symbols=("Pipeline.__call__",),
+ resolver=resolver,
+ telemetry_feature="hf_pipeline_autolog",
+)
+
+autolog.get_latest_id = resolver.get_latest_id
diff --git a/wandb/integration/huggingface/resolver.py b/wandb/integration/huggingface/resolver.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d40310adc64949e2881515ac7c57860cec01cb5
--- /dev/null
+++ b/wandb/integration/huggingface/resolver.py
@@ -0,0 +1,213 @@
+import logging
+import os
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
+
+import pytz
+
+import wandb
+from wandb.sdk.integration_utils.auto_logging import Response
+from wandb.sdk.lib.runid import generate_id
+
+logger = logging.getLogger(__name__)
+
+SUPPORTED_PIPELINE_TASKS = [
+ "text-classification",
+ "sentiment-analysis",
+ "question-answering",
+ "summarization",
+ "translation",
+ "text2text-generation",
+ "text-generation",
+ # "conversational",
+]
+
+PIPELINES_WITH_TOP_K = [
+ "text-classification",
+ "sentiment-analysis",
+ "question-answering",
+]
+
+
+class HuggingFacePipelineRequestResponseResolver:
+ """Resolver for HuggingFace's pipeline request and responses, providing necessary data transformations and formatting.
+
+ This is based off (from wandb.sdk.integration_utils.auto_logging import RequestResponseResolver)
+ """
+
+ autolog_id = None
+
+ def __call__(
+ self,
+ args: Sequence[Any],
+ kwargs: Dict[str, Any],
+ response: Response,
+ start_time: float,
+ time_elapsed: float,
+ ) -> Optional[Dict[str, Any]]:
+ """Main call method for this class.
+
+ :param args: list of arguments
+ :param kwargs: dictionary of keyword arguments
+ :param response: the response from the request
+ :param start_time: time when request started
+ :param time_elapsed: time elapsed for the request
+ :returns: packed data as a dictionary for logging to wandb, None if an exception occurred
+ """
+ try:
+ pipe, input_data = args[:2]
+ task = pipe.task
+
+ # Translation tasks are in the form of `translation_x_to_y`
+ if task in SUPPORTED_PIPELINE_TASKS or task.startswith("translation"):
+ model = self._get_model(pipe)
+ if model is None:
+ return None
+ model_alias = model.name_or_path
+ timestamp = datetime.now(pytz.utc)
+
+ input_data, response = self._transform_task_specific_data(
+ task, input_data, response
+ )
+ formatted_data = self._format_data(task, input_data, response, kwargs)
+ packed_data = self._create_table(
+ formatted_data, model_alias, timestamp, time_elapsed
+ )
+ table_name = os.environ.get("WANDB_AUTOLOG_TABLE_NAME", f"{task}")
+ # TODO: Let users decide the name in a way that does not use an environment variable
+
+ return {
+ table_name: wandb.Table(
+ columns=packed_data[0], data=packed_data[1:]
+ )
+ }
+
+ logger.warning(
+ f"The task: `{task}` is not yet supported.\nPlease contact `wandb` to notify us if you would like support for this task"
+ )
+ except Exception as e:
+ logger.warning(e)
+ return None
+
+ # TODO: This should have a dependency on PreTrainedModel. i.e. isinstance(PreTrainedModel)
+ # from transformers.modeling_utils import PreTrainedModel
+ # We do not want this dependency explicity in our codebase so we make a very general assumption about
+ # the structure of the pipeline which may have unintended consequences
+ def _get_model(self, pipe) -> Optional[Any]:
+ """Extracts model from the pipeline.
+
+ :param pipe: the HuggingFace pipeline
+ :returns: Model if available, None otherwise
+ """
+ model = pipe.model
+ try:
+ return model.model
+ except AttributeError:
+ logger.info(
+ "Model does not have a `.model` attribute. Assuming `pipe.model` is the correct model."
+ )
+ return model
+
+ @staticmethod
+ def _transform_task_specific_data(
+ task: str, input_data: Union[List[Any], Any], response: Union[List[Any], Any]
+ ) -> Tuple[Union[List[Any], Any], Union[List[Any], Any]]:
+ """Transform input and response data based on specific tasks.
+
+ :param task: the task name
+ :param input_data: the input data
+ :param response: the response data
+ :returns: tuple of transformed input_data and response
+ """
+ if task == "question-answering":
+ input_data = input_data if isinstance(input_data, list) else [input_data]
+ input_data = [data.__dict__ for data in input_data]
+ elif task == "conversational":
+ # We only grab the latest input/output pair from the conversation
+ # Logging the whole conversation renders strangely.
+ input_data = input_data if isinstance(input_data, list) else [input_data]
+ input_data = [data.__dict__["past_user_inputs"][-1] for data in input_data]
+
+ response = response if isinstance(response, list) else [response]
+ response = [data.__dict__["generated_responses"][-1] for data in response]
+ return input_data, response
+
+ def _format_data(
+ self,
+ task: str,
+ input_data: Union[List[Any], Any],
+ response: Union[List[Any], Any],
+ kwargs: Dict[str, Any],
+ ) -> List[Dict[str, Any]]:
+ """Formats input data, response, and kwargs into a list of dictionaries.
+
+ :param task: the task name
+ :param input_data: the input data
+ :param response: the response data
+ :param kwargs: dictionary of keyword arguments
+ :returns: list of dictionaries containing formatted data
+ """
+ input_data = input_data if isinstance(input_data, list) else [input_data]
+ response = response if isinstance(response, list) else [response]
+
+ formatted_data = []
+ for i_text, r_text in zip(input_data, response):
+ # Unpack single element responses for better rendering in wandb UI when it is a task without top_k
+ # top_k = 1 would unpack the response into a single element while top_k > 1 would be a list
+ # this would cause the UI to not properly concatenate the tables of the same task by omitting the elements past the first
+ if (
+ (isinstance(r_text, list))
+ and (len(r_text) == 1)
+ and task not in PIPELINES_WITH_TOP_K
+ ):
+ r_text = r_text[0]
+ formatted_data.append(
+ {"input": i_text, "response": r_text, "kwargs": kwargs}
+ )
+ return formatted_data
+
+ def _create_table(
+ self,
+ formatted_data: List[Dict[str, Any]],
+ model_alias: str,
+ timestamp: float,
+ time_elapsed: float,
+ ) -> List[List[Any]]:
+ """Creates a table from formatted data, model alias, timestamp, and elapsed time.
+
+ :param formatted_data: list of dictionaries containing formatted data
+ :param model_alias: alias of the model
+ :param timestamp: timestamp of the data
+ :param time_elapsed: time elapsed from the beginning
+ :returns: list of lists, representing a table of data. [0]th element = columns. [1]st element = data
+ """
+ header = [
+ "ID",
+ "Model Alias",
+ "Timestamp",
+ "Elapsed Time",
+ "Input",
+ "Response",
+ "Kwargs",
+ ]
+ table = [header]
+ autolog_id = generate_id(length=16)
+
+ for data in formatted_data:
+ row = [
+ autolog_id,
+ model_alias,
+ timestamp,
+ time_elapsed,
+ data["input"],
+ data["response"],
+ data["kwargs"],
+ ]
+ table.append(row)
+
+ self.autolog_id = autolog_id
+
+ return table
+
+ def get_latest_id(self):
+ return self.autolog_id
diff --git a/wandb/integration/keras/__init__.py b/wandb/integration/keras/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..8a244b71d7e2c6c1ccaf075f19aa52a270f7adf6
--- /dev/null
+++ b/wandb/integration/keras/__init__.py
@@ -0,0 +1,13 @@
+"""Tools for integrating `wandb` with [`Keras`](https://keras.io/).
+
+Keras is a deep learning API for [`TensorFlow`](https://www.tensorflow.org/).
+"""
+__all__ = (
+ "WandbCallback",
+ "WandbMetricsLogger",
+ "WandbModelCheckpoint",
+ "WandbEvalCallback",
+)
+
+from .callbacks import WandbEvalCallback, WandbMetricsLogger, WandbModelCheckpoint
+from .keras import WandbCallback # todo: legacy callback to be deprecated
diff --git a/wandb/integration/keras/callbacks/__init__.py b/wandb/integration/keras/callbacks/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..6035bd4c9c5067e553404a5a4bb818ef582a36a8
--- /dev/null
+++ b/wandb/integration/keras/callbacks/__init__.py
@@ -0,0 +1,5 @@
+__all__ = ("WandbMetricsLogger", "WandbModelCheckpoint", "WandbEvalCallback")
+
+from .metrics_logger import WandbMetricsLogger
+from .model_checkpoint import WandbModelCheckpoint
+from .tables_builder import WandbEvalCallback
diff --git a/wandb/integration/keras/callbacks/metrics_logger.py b/wandb/integration/keras/callbacks/metrics_logger.py
new file mode 100644
index 0000000000000000000000000000000000000000..281283749f363dc7bb683bc0eab45a8650beb5fe
--- /dev/null
+++ b/wandb/integration/keras/callbacks/metrics_logger.py
@@ -0,0 +1,130 @@
+import sys
+from typing import Any, Dict, Optional, Union
+
+import tensorflow as tf # type: ignore
+from tensorflow.keras import callbacks # type: ignore
+
+import wandb
+from wandb.integration.keras.keras import patch_tf_keras
+from wandb.sdk.lib import telemetry
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+
+LogStrategy = Literal["epoch", "batch"]
+
+
+patch_tf_keras()
+
+
+class WandbMetricsLogger(callbacks.Callback):
+ """Logger that sends system metrics to W&B.
+
+ `WandbMetricsLogger` automatically logs the `logs` dictionary that callback methods
+ take as argument to wandb.
+
+ This callback automatically logs the following to a W&B run page:
+ * system (CPU/GPU/TPU) metrics,
+ * train and validation metrics defined in `model.compile`,
+ * learning rate (both for a fixed value or a learning rate scheduler)
+
+ Notes:
+ If you resume training by passing `initial_epoch` to `model.fit` and you are using a
+ learning rate scheduler, make sure to pass `initial_global_step` to
+ `WandbMetricsLogger`. The `initial_global_step` is `step_size * initial_step`, where
+ `step_size` is number of training steps per epoch. `step_size` can be calculated as
+ the product of the cardinality of the training dataset and the batch size.
+
+ Arguments:
+ log_freq: ("epoch", "batch", or int) if "epoch", logs metrics
+ at the end of each epoch. If "batch", logs metrics at the end
+ of each batch. If an integer, logs metrics at the end of that
+ many batches. Defaults to "epoch".
+ initial_global_step: (int) Use this argument to correcly log the
+ learning rate when you resume training from some `initial_epoch`,
+ and a learning rate scheduler is used. This can be computed as
+ `step_size * initial_step`. Defaults to 0.
+ """
+
+ def __init__(
+ self,
+ log_freq: Union[LogStrategy, int] = "epoch",
+ initial_global_step: int = 0,
+ *args: Any,
+ **kwargs: Any,
+ ) -> None:
+ super().__init__(*args, **kwargs)
+
+ if wandb.run is None:
+ raise wandb.Error(
+ "You must call `wandb.init()` before WandbMetricsLogger()"
+ )
+
+ with telemetry.context(run=wandb.run) as tel:
+ tel.feature.keras_metrics_logger = True
+
+ if log_freq == "batch":
+ log_freq = 1
+
+ self.logging_batch_wise = isinstance(log_freq, int)
+ self.log_freq: Any = log_freq if self.logging_batch_wise else None
+ self.global_batch = 0
+ self.global_step = initial_global_step
+
+ if self.logging_batch_wise:
+ # define custom x-axis for batch logging.
+ wandb.define_metric("batch/batch_step")
+ # set all batch metrics to be logged against batch_step.
+ wandb.define_metric("batch/*", step_metric="batch/batch_step")
+ else:
+ # define custom x-axis for epoch-wise logging.
+ wandb.define_metric("epoch/epoch")
+ # set all epoch-wise metrics to be logged against epoch.
+ wandb.define_metric("epoch/*", step_metric="epoch/epoch")
+
+ def _get_lr(self) -> Union[float, None]:
+ if isinstance(self.model.optimizer.learning_rate, tf.Variable):
+ return float(self.model.optimizer.learning_rate.numpy().item())
+ try:
+ return float(
+ self.model.optimizer.learning_rate(step=self.global_step).numpy().item()
+ )
+ except Exception:
+ wandb.termerror("Unable to log learning rate.", repeat=False)
+ return None
+
+ def on_epoch_end(self, epoch: int, logs: Optional[Dict[str, Any]] = None) -> None:
+ """Called at the end of an epoch."""
+ logs = dict() if logs is None else {f"epoch/{k}": v for k, v in logs.items()}
+
+ logs["epoch/epoch"] = epoch
+
+ lr = self._get_lr()
+ if lr is not None:
+ logs["epoch/learning_rate"] = lr
+
+ wandb.log(logs)
+
+ def on_batch_end(self, batch: int, logs: Optional[Dict[str, Any]] = None) -> None:
+ self.global_step += 1
+ """An alias for `on_train_batch_end` for backwards compatibility."""
+ if self.logging_batch_wise and batch % self.log_freq == 0:
+ logs = {f"batch/{k}": v for k, v in logs.items()} if logs else {}
+ logs["batch/batch_step"] = self.global_batch
+
+ lr = self._get_lr()
+ if lr is not None:
+ logs["batch/learning_rate"] = lr
+
+ wandb.log(logs)
+
+ self.global_batch += self.log_freq
+
+ def on_train_batch_end(
+ self, batch: int, logs: Optional[Dict[str, Any]] = None
+ ) -> None:
+ """Called at the end of a training batch in `fit` methods."""
+ self.on_batch_end(batch, logs if logs else {})
diff --git a/wandb/integration/keras/callbacks/model_checkpoint.py b/wandb/integration/keras/callbacks/model_checkpoint.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d8383f63cb3bb338401d49863a36c08253eceb6
--- /dev/null
+++ b/wandb/integration/keras/callbacks/model_checkpoint.py
@@ -0,0 +1,200 @@
+import os
+import string
+import sys
+from typing import Any, Dict, List, Optional, Union
+
+import tensorflow as tf # type: ignore
+from tensorflow.keras import callbacks # type: ignore
+
+import wandb
+from wandb.sdk.lib import telemetry
+from wandb.sdk.lib.paths import StrPath
+
+from ..keras import patch_tf_keras
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+
+Mode = Literal["auto", "min", "max"]
+SaveStrategy = Literal["epoch"]
+
+patch_tf_keras()
+
+
+class WandbModelCheckpoint(callbacks.ModelCheckpoint):
+ """A checkpoint that periodically saves a Keras model or model weights.
+
+ Saved weights are uploaded to W&B as a `wandb.Artifact`.
+
+ Since this callback is subclassed from `tf.keras.callbacks.ModelCheckpoint`, the
+ checkpointing logic is taken care of by the parent callback. You can learn more
+ here: https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/ModelCheckpoint
+
+ This callback is to be used in conjunction with training using `model.fit()` to save
+ a model or weights (in a checkpoint file) at some interval. The model checkpoints
+ will be logged as W&B Artifacts. You can learn more here:
+ https://docs.wandb.ai/guides/artifacts
+
+ This callback provides the following features:
+ - Save the model that has achieved "best performance" based on "monitor".
+ - Save the model at the end of every epoch regardless of the performance.
+ - Save the model at the end of epoch or after a fixed number of training batches.
+ - Save only model weights, or save the whole model.
+ - Save the model either in SavedModel format or in `.h5` format.
+
+ Arguments:
+ filepath: (Union[str, os.PathLike]) path to save the model file. `filepath`
+ can contain named formatting options, which will be filled by the value
+ of `epoch` and keys in `logs` (passed in `on_epoch_end`). For example:
+ if `filepath` is `model-{epoch:02d}-{val_loss:.2f}`, then the
+ model checkpoints will be saved with the epoch number and the
+ validation loss in the filename.
+ monitor: (str) The metric name to monitor. Default to "val_loss".
+ verbose: (int) Verbosity mode, 0 or 1. Mode 0 is silent, and mode 1
+ displays messages when the callback takes an action.
+ save_best_only: (bool) if `save_best_only=True`, it only saves when the model
+ is considered the "best" and the latest best model according to the
+ quantity monitored will not be overwritten. If `filepath` doesn't contain
+ formatting options like `{epoch}` then `filepath` will be overwritten by
+ each new better model locally. The model logged as an artifact will still be
+ associated with the correct `monitor`. Artifacts will be uploaded
+ continuously and versioned separately as a new best model is found.
+ save_weights_only: (bool) if True, then only the model's weights will be saved.
+ mode: (Mode) one of {'auto', 'min', 'max'}. For `val_acc`, this should be `max`,
+ for `val_loss` this should be `min`, etc.
+ save_freq: (Union[SaveStrategy, int]) `epoch` or integer. When using `'epoch'`,
+ the callback saves the model after each epoch. When using an integer, the
+ callback saves the model at end of this many batches.
+ Note that when monitoring validation metrics such as `val_acc` or `val_loss`,
+ save_freq must be set to "epoch" as those metrics are only available at the
+ end of an epoch.
+ options: (Optional[str]) Optional `tf.train.CheckpointOptions` object if
+ `save_weights_only` is true or optional `tf.saved_model.SaveOptions`
+ object if `save_weights_only` is false.
+ initial_value_threshold: (Optional[float]) Floating point initial "best" value of the metric
+ to be monitored.
+ """
+
+ def __init__(
+ self,
+ filepath: StrPath,
+ monitor: str = "val_loss",
+ verbose: int = 0,
+ save_best_only: bool = False,
+ save_weights_only: bool = False,
+ mode: Mode = "auto",
+ save_freq: Union[SaveStrategy, int] = "epoch",
+ options: Optional[str] = None,
+ initial_value_threshold: Optional[float] = None,
+ **kwargs: Any,
+ ) -> None:
+ super().__init__(
+ filepath=filepath,
+ monitor=monitor,
+ verbose=verbose,
+ save_best_only=save_best_only,
+ save_weights_only=save_weights_only,
+ mode=mode,
+ save_freq=save_freq,
+ options=options,
+ initial_value_threshold=initial_value_threshold,
+ **kwargs,
+ )
+ if wandb.run is None:
+ raise wandb.Error(
+ "You must call `wandb.init()` before `WandbModelCheckpoint()`"
+ )
+ with telemetry.context(run=wandb.run) as tel:
+ tel.feature.keras_model_checkpoint = True
+
+ self.save_weights_only = save_weights_only
+
+ # User-friendly warning when trying to save the best model.
+ if self.save_best_only:
+ self._check_filepath()
+
+ self._is_old_tf_keras_version: Optional[bool] = None
+
+ def on_train_batch_end(
+ self, batch: int, logs: Optional[Dict[str, float]] = None
+ ) -> None:
+ if self._should_save_on_batch(batch):
+ if self.is_old_tf_keras_version:
+ # Save the model and get filepath
+ self._save_model(epoch=self._current_epoch, logs=logs)
+ filepath = self._get_file_path(epoch=self._current_epoch, logs=logs)
+ else:
+ # Save the model and get filepath
+ self._save_model(epoch=self._current_epoch, batch=batch, logs=logs)
+ filepath = self._get_file_path(
+ epoch=self._current_epoch, batch=batch, logs=logs
+ )
+ # Log the model as artifact
+ aliases = ["latest", f"epoch_{self._current_epoch}_batch_{batch}"]
+ self._log_ckpt_as_artifact(filepath, aliases=aliases)
+
+ def on_epoch_end(self, epoch: int, logs: Optional[Dict[str, float]] = None) -> None:
+ super().on_epoch_end(epoch, logs)
+ # Check if model checkpoint is created at the end of epoch.
+ if self.save_freq == "epoch":
+ # Get filepath where the model checkpoint is saved.
+ if self.is_old_tf_keras_version:
+ filepath = self._get_file_path(epoch=epoch, logs=logs)
+ else:
+ filepath = self._get_file_path(epoch=epoch, batch=None, logs=logs)
+ # Log the model as artifact
+ aliases = ["latest", f"epoch_{epoch}"]
+ self._log_ckpt_as_artifact(filepath, aliases=aliases)
+
+ def _log_ckpt_as_artifact(
+ self, filepath: str, aliases: Optional[List[str]] = None
+ ) -> None:
+ """Log model checkpoint as W&B Artifact."""
+ try:
+ assert wandb.run is not None
+ model_checkpoint_artifact = wandb.Artifact(
+ f"run_{wandb.run.id}_model", type="model"
+ )
+ if os.path.isfile(filepath):
+ model_checkpoint_artifact.add_file(filepath)
+ elif os.path.isdir(filepath):
+ model_checkpoint_artifact.add_dir(filepath)
+ else:
+ raise FileNotFoundError(f"No such file or directory {filepath}")
+ wandb.log_artifact(model_checkpoint_artifact, aliases=aliases or [])
+ except ValueError:
+ # This error occurs when `save_best_only=True` and the model
+ # checkpoint is not saved for that epoch/batch. Since TF/Keras
+ # is giving friendly log, we can avoid clustering the stdout.
+ pass
+
+ def _check_filepath(self) -> None:
+ placeholders = []
+ for tup in string.Formatter().parse(self.filepath):
+ if tup[1] is not None:
+ placeholders.append(tup[1])
+ if len(placeholders) == 0:
+ wandb.termwarn(
+ "When using `save_best_only`, ensure that the `filepath` argument "
+ "contains formatting placeholders like `{epoch:02d}` or `{batch:02d}`. "
+ "This ensures correct interpretation of the logged artifacts.",
+ repeat=False,
+ )
+
+ @property
+ def is_old_tf_keras_version(self) -> Optional[bool]:
+ if self._is_old_tf_keras_version is None:
+ from pkg_resources import parse_version
+
+ try:
+ if parse_version(tf.keras.__version__) < parse_version("2.6.0"):
+ self._is_old_tf_keras_version = True
+ else:
+ self._is_old_tf_keras_version = False
+ except AttributeError:
+ self._is_old_tf_keras_version = False
+
+ return self._is_old_tf_keras_version
diff --git a/wandb/integration/keras/callbacks/tables_builder.py b/wandb/integration/keras/callbacks/tables_builder.py
new file mode 100644
index 0000000000000000000000000000000000000000..16e0421ae83ee72781f2cd3d0e0ff6e40141fa82
--- /dev/null
+++ b/wandb/integration/keras/callbacks/tables_builder.py
@@ -0,0 +1,226 @@
+import abc
+from typing import Any, Dict, List, Optional
+
+from tensorflow.keras.callbacks import Callback # type: ignore
+
+import wandb
+from wandb.sdk.lib import telemetry
+
+
+class WandbEvalCallback(Callback, abc.ABC):
+ """Abstract base class to build Keras callbacks for model prediction visualization.
+
+ You can build callbacks for visualizing model predictions `on_epoch_end`
+ that can be passed to `model.fit()` for classification, object detection,
+ segmentation, etc. tasks.
+
+ To use this, inherit from this base callback class and implement the
+ `add_ground_truth` and `add_model_prediction` methods.
+
+ The base class will take care of the following:
+ - Initialize `data_table` for logging the ground truth and
+ `pred_table` for predictions.
+ - The data uploaded to `data_table` is used as a reference for the
+ `pred_table`. This is to reduce the memory footprint. The `data_table_ref`
+ is a list that can be used to access the referenced data.
+ Check out the example below to see how it's done.
+ - Log the tables to W&B as W&B Artifacts.
+ - Each new `pred_table` is logged as a new version with aliases.
+
+ Example:
+ ```python
+ class WandbClfEvalCallback(WandbEvalCallback):
+ def __init__(self, validation_data, data_table_columns, pred_table_columns):
+ super().__init__(data_table_columns, pred_table_columns)
+
+ self.x = validation_data[0]
+ self.y = validation_data[1]
+
+ def add_ground_truth(self):
+ for idx, (image, label) in enumerate(zip(self.x, self.y)):
+ self.data_table.add_data(idx, wandb.Image(image), label)
+
+ def add_model_predictions(self, epoch):
+ preds = self.model.predict(self.x, verbose=0)
+ preds = tf.argmax(preds, axis=-1)
+
+ data_table_ref = self.data_table_ref
+ table_idxs = data_table_ref.get_index()
+
+ for idx in table_idxs:
+ pred = preds[idx]
+ self.pred_table.add_data(
+ epoch,
+ data_table_ref.data[idx][0],
+ data_table_ref.data[idx][1],
+ data_table_ref.data[idx][2],
+ pred,
+ )
+
+
+ model.fit(
+ x,
+ y,
+ epochs=2,
+ validation_data=(x, y),
+ callbacks=[
+ WandbClfEvalCallback(
+ validation_data=(x, y),
+ data_table_columns=["idx", "image", "label"],
+ pred_table_columns=["epoch", "idx", "image", "label", "pred"],
+ )
+ ],
+ )
+ ```
+
+ To have more fine-grained control, you can override the `on_train_begin` and
+ `on_epoch_end` methods. If you want to log the samples after N batched, you
+ can implement `on_train_batch_end` method.
+ """
+
+ def __init__(
+ self,
+ data_table_columns: List[str],
+ pred_table_columns: List[str],
+ *args: Any,
+ **kwargs: Any,
+ ) -> None:
+ super().__init__(*args, **kwargs)
+
+ if wandb.run is None:
+ raise wandb.Error(
+ "You must call `wandb.init()` first before using this callback."
+ )
+
+ with telemetry.context(run=wandb.run) as tel:
+ tel.feature.keras_wandb_eval_callback = True
+
+ self.data_table_columns = data_table_columns
+ self.pred_table_columns = pred_table_columns
+
+ def on_train_begin(self, logs: Optional[Dict[str, float]] = None) -> None:
+ # Initialize the data_table
+ self.init_data_table(column_names=self.data_table_columns)
+ # Log the ground truth data
+ self.add_ground_truth(logs)
+ # Log the data_table as W&B Artifacts
+ self.log_data_table()
+
+ def on_epoch_end(self, epoch: int, logs: Optional[Dict[str, float]] = None) -> None:
+ # Initialize the pred_table
+ self.init_pred_table(column_names=self.pred_table_columns)
+ # Log the model prediction
+ self.add_model_predictions(epoch, logs)
+ # Log the pred_table as W&B Artifacts
+ self.log_pred_table()
+
+ @abc.abstractmethod
+ def add_ground_truth(self, logs: Optional[Dict[str, float]] = None) -> None:
+ """Add ground truth data to `data_table`.
+
+ Use this method to write the logic for adding validation/training data to
+ `data_table` initialized using `init_data_table` method.
+
+ Example:
+ ```python
+ for idx, data in enumerate(dataloader):
+ self.data_table.add_data(idx, data)
+ ```
+ This method is called once `on_train_begin` or equivalent hook.
+ """
+ raise NotImplementedError(f"{self.__class__.__name__}.add_ground_truth")
+
+ @abc.abstractmethod
+ def add_model_predictions(
+ self, epoch: int, logs: Optional[Dict[str, float]] = None
+ ) -> None:
+ """Add a prediction from a model to `pred_table`.
+
+ Use this method to write the logic for adding model prediction for validation/
+ training data to `pred_table` initialized using `init_pred_table` method.
+
+ Example:
+ ```python
+ # Assuming the dataloader is not shuffling the samples.
+ for idx, data in enumerate(dataloader):
+ preds = model.predict(data)
+ self.pred_table.add_data(
+ self.data_table_ref.data[idx][0], self.data_table_ref.data[idx][1], preds
+ )
+ ```
+ This method is called `on_epoch_end` or equivalent hook.
+ """
+ raise NotImplementedError(f"{self.__class__.__name__}.add_model_predictions")
+
+ def init_data_table(self, column_names: List[str]) -> None:
+ """Initialize the W&B Tables for validation data.
+
+ Call this method `on_train_begin` or equivalent hook. This is followed by adding
+ data to the table row or column wise.
+
+ Args:
+ column_names: (list) Column names for W&B Tables.
+ """
+ self.data_table = wandb.Table(columns=column_names, allow_mixed_types=True)
+
+ def init_pred_table(self, column_names: List[str]) -> None:
+ """Initialize the W&B Tables for model evaluation.
+
+ Call this method `on_epoch_end` or equivalent hook. This is followed by adding
+ data to the table row or column wise.
+
+ Args:
+ column_names: (list) Column names for W&B Tables.
+ """
+ self.pred_table = wandb.Table(columns=column_names)
+
+ def log_data_table(
+ self, name: str = "val", type: str = "dataset", table_name: str = "val_data"
+ ) -> None:
+ """Log the `data_table` as W&B artifact and call `use_artifact` on it.
+
+ This lets the evaluation table use the reference of already uploaded data
+ (images, text, scalar, etc.) without re-uploading.
+
+ Args:
+ name: (str) A human-readable name for this artifact, which is how you can
+ identify this artifact in the UI or reference it in use_artifact calls.
+ (default is 'val')
+ type: (str) The type of the artifact, which is used to organize and
+ differentiate artifacts. (default is 'dataset')
+ table_name: (str) The name of the table as will be displayed in the UI.
+ (default is 'val_data').
+ """
+ data_artifact = wandb.Artifact(name, type=type)
+ data_artifact.add(self.data_table, table_name)
+
+ # Calling `use_artifact` uploads the data to W&B.
+ assert wandb.run is not None
+ wandb.run.use_artifact(data_artifact)
+ data_artifact.wait()
+
+ # We get the reference table.
+ self.data_table_ref = data_artifact.get(table_name)
+
+ def log_pred_table(
+ self,
+ type: str = "evaluation",
+ table_name: str = "eval_data",
+ aliases: Optional[List[str]] = None,
+ ) -> None:
+ """Log the W&B Tables for model evaluation.
+
+ The table will be logged multiple times creating new version. Use this
+ to compare models at different intervals interactively.
+
+ Args:
+ type: (str) The type of the artifact, which is used to organize and
+ differentiate artifacts. (default is 'evaluation')
+ table_name: (str) The name of the table as will be displayed in the UI.
+ (default is 'eval_data')
+ aliases: (List[str]) List of aliases for the prediction table.
+ """
+ assert wandb.run is not None
+ pred_artifact = wandb.Artifact(f"run_{wandb.run.id}_pred", type=type)
+ pred_artifact.add(self.pred_table, table_name)
+ wandb.run.log_artifact(pred_artifact, aliases=aliases or ["latest"])
diff --git a/wandb/integration/keras/keras.py b/wandb/integration/keras/keras.py
new file mode 100644
index 0000000000000000000000000000000000000000..b4d3cf611ef3941498abeadb2e12f95def78de35
--- /dev/null
+++ b/wandb/integration/keras/keras.py
@@ -0,0 +1,1078 @@
+"""keras init."""
+
+import logging
+import operator
+import os
+import shutil
+import sys
+from itertools import chain
+
+import numpy as np
+import tensorflow as tf
+import tensorflow.keras.backend as K # noqa: N812
+
+import wandb
+from wandb.sdk.integration_utils.data_logging import ValidationDataLogger
+from wandb.sdk.lib.deprecate import Deprecated, deprecate
+from wandb.util import add_import_hook
+
+
+def _check_keras_version():
+ from keras import __version__ as keras_version
+ from pkg_resources import parse_version
+
+ if parse_version(keras_version) < parse_version("2.4.0"):
+ wandb.termwarn(
+ f"Keras version {keras_version} is not fully supported. Required keras >= 2.4.0"
+ )
+
+
+def _can_compute_flops() -> bool:
+ """FLOPS computation is restricted to TF 2.x as it requires tf.compat.v1."""
+ from pkg_resources import parse_version
+
+ if parse_version(tf.__version__) >= parse_version("2.0.0"):
+ return True
+
+ return False
+
+
+if "keras" in sys.modules:
+ _check_keras_version()
+else:
+ add_import_hook("keras", _check_keras_version)
+
+
+logger = logging.getLogger(__name__)
+
+
+def is_dataset(data):
+ dataset_ops = wandb.util.get_module("tensorflow.python.data.ops.dataset_ops")
+ if dataset_ops and hasattr(dataset_ops, "DatasetV2"):
+ dataset_types = (dataset_ops.DatasetV2,)
+ if hasattr(dataset_ops, "DatasetV1"):
+ dataset_types = dataset_types + (dataset_ops.DatasetV1,)
+ return isinstance(data, dataset_types)
+ else:
+ return False
+
+
+def is_generator_like(data):
+ # Checks if data is a generator, Sequence, or Iterator.
+
+ types = (tf.keras.utils.Sequence,)
+ iterator_ops = wandb.util.get_module("tensorflow.python.data.ops.iterator_ops")
+ if iterator_ops:
+ types = types + (iterator_ops.Iterator,)
+ # EagerIterator was in tensorflow < 2
+ if hasattr(iterator_ops, "EagerIterator"):
+ types = types + (iterator_ops.EagerIterator,)
+ elif hasattr(iterator_ops, "IteratorV2"):
+ types = types + (iterator_ops.IteratorV2,)
+ return hasattr(data, "next") or hasattr(data, "__next__") or isinstance(data, types)
+
+
+def patch_tf_keras(): # noqa: C901
+ from pkg_resources import parse_version
+ from tensorflow.python.eager import context
+
+ if (
+ parse_version("2.6.0")
+ <= parse_version(tf.__version__)
+ < parse_version("2.13.0")
+ ):
+ keras_engine = "keras.engine"
+ try:
+ from keras.engine import training
+ from keras.engine import training_arrays_v1 as training_arrays
+ from keras.engine import training_generator_v1 as training_generator
+ except (ImportError, AttributeError):
+ wandb.termerror("Unable to patch Tensorflow/Keras")
+ logger.exception("exception while trying to patch_tf_keras")
+ return
+ else:
+ keras_engine = "tensorflow.python.keras.engine"
+
+ from tensorflow.python.keras.engine import training
+
+ try:
+ from tensorflow.python.keras.engine import (
+ training_arrays_v1 as training_arrays,
+ )
+ from tensorflow.python.keras.engine import (
+ training_generator_v1 as training_generator,
+ )
+ except (ImportError, AttributeError):
+ try:
+ from tensorflow.python.keras.engine import (
+ training_arrays,
+ training_generator,
+ )
+ except (ImportError, AttributeError):
+ wandb.termerror("Unable to patch Tensorflow/Keras")
+ logger.exception("exception while trying to patch_tf_keras")
+ return
+
+ # Tensorflow 2.1
+ training_v2_1 = wandb.util.get_module("tensorflow.python.keras.engine.training_v2")
+ # Tensorflow 2.2
+ training_v2_2 = wandb.util.get_module(f"{keras_engine}.training_v1")
+
+ if training_v2_1:
+ old_v2 = training_v2_1.Loop.fit
+ elif training_v2_2:
+ old_v2 = training.Model.fit
+
+ old_arrays = training_arrays.fit_loop
+ old_generator = training_generator.fit_generator
+
+ def set_wandb_attrs(cbk, val_data):
+ if isinstance(cbk, WandbCallback):
+ if is_generator_like(val_data):
+ cbk.generator = val_data
+ elif is_dataset(val_data):
+ if context.executing_eagerly():
+ cbk.generator = iter(val_data)
+ else:
+ wandb.termwarn(
+ "Found a validation dataset in graph mode, can't patch Keras."
+ )
+ elif isinstance(val_data, tuple) and isinstance(val_data[0], tf.Tensor):
+ # Graph mode dataset generator
+ def gen():
+ while True:
+ yield K.get_session().run(val_data)
+
+ cbk.generator = gen()
+ else:
+ cbk.validation_data = val_data
+
+ def new_arrays(*args, **kwargs):
+ cbks = kwargs.get("callbacks", [])
+ val_inputs = kwargs.get("val_inputs")
+ val_targets = kwargs.get("val_targets")
+ # TODO: these could be generators, why index 0?
+ if val_inputs and val_targets:
+ for cbk in cbks:
+ set_wandb_attrs(cbk, (val_inputs[0], val_targets[0]))
+ return old_arrays(*args, **kwargs)
+
+ def new_generator(*args, **kwargs):
+ cbks = kwargs.get("callbacks", [])
+ val_data = kwargs.get("validation_data")
+ if val_data:
+ for cbk in cbks:
+ set_wandb_attrs(cbk, val_data)
+ return old_generator(*args, **kwargs)
+
+ def new_v2(*args, **kwargs):
+ cbks = kwargs.get("callbacks", [])
+ val_data = kwargs.get("validation_data")
+ if val_data:
+ for cbk in cbks:
+ set_wandb_attrs(cbk, val_data)
+ return old_v2(*args, **kwargs)
+
+ training_arrays.orig_fit_loop = old_arrays
+ training_arrays.fit_loop = new_arrays
+ training_generator.orig_fit_generator = old_generator
+ training_generator.fit_generator = new_generator
+ wandb.patched["keras"].append([f"{keras_engine}.training_arrays", "fit_loop"])
+ wandb.patched["keras"].append(
+ [f"{keras_engine}.training_generator", "fit_generator"]
+ )
+
+ if training_v2_1:
+ training_v2_1.Loop.fit = new_v2
+ wandb.patched["keras"].append(
+ ["tensorflow.python.keras.engine.training_v2.Loop", "fit"]
+ )
+ elif training_v2_2:
+ training.Model.fit = new_v2
+ wandb.patched["keras"].append([f"{keras_engine}.training.Model", "fit"])
+
+
+def _array_has_dtype(array):
+ return hasattr(array, "dtype")
+
+
+def _update_if_numeric(metrics, key, values):
+ if not _array_has_dtype(values):
+ _warn_not_logging(key)
+ return
+
+ if not is_numeric_array(values):
+ _warn_not_logging_non_numeric(key)
+ return
+
+ metrics[key] = wandb.Histogram(values)
+
+
+def is_numeric_array(array):
+ return np.issubdtype(array.dtype, np.number)
+
+
+def _warn_not_logging_non_numeric(name):
+ wandb.termwarn(
+ f"Non-numeric values found in layer: {name}, not logging this layer",
+ repeat=False,
+ )
+
+
+def _warn_not_logging(name):
+ wandb.termwarn(
+ f"Layer {name} has undetermined datatype not logging this layer",
+ repeat=False,
+ )
+
+
+tf_logger = tf.get_logger()
+
+patch_tf_keras()
+
+
+### For gradient logging ###
+
+
+def _get_custom_optimizer_parent_class():
+ from pkg_resources import parse_version
+
+ if parse_version(tf.__version__) >= parse_version("2.9.0"):
+ custom_optimizer_parent_class = tf.keras.optimizers.legacy.Optimizer
+ else:
+ custom_optimizer_parent_class = tf.keras.optimizers.Optimizer
+
+ return custom_optimizer_parent_class
+
+
+_custom_optimizer_parent_class = _get_custom_optimizer_parent_class()
+
+
+class _CustomOptimizer(_custom_optimizer_parent_class):
+ def __init__(self):
+ super().__init__(name="CustomOptimizer")
+ self._resource_apply_dense = tf.function(self._resource_apply_dense)
+ self._resource_apply_sparse = tf.function(self._resource_apply_sparse)
+
+ def _resource_apply_dense(self, grad, var):
+ var.assign(grad)
+
+ # this needs to be implemented to prevent a NotImplementedError when
+ # using Lookup layers.
+ def _resource_apply_sparse(self, grad, var, indices):
+ pass
+
+ def get_config(self):
+ return super().get_config()
+
+
+class _GradAccumulatorCallback(tf.keras.callbacks.Callback):
+ """Accumulates gradients during a fit() call when used in conjunction with the CustomOptimizer above."""
+
+ def set_model(self, model):
+ super().set_model(model)
+ self.og_weights = model.get_weights()
+ self.grads = [np.zeros(tuple(w.shape)) for w in model.trainable_weights]
+
+ def on_batch_end(self, batch, logs=None):
+ for g, w in zip(self.grads, self.model.trainable_weights):
+ g += w.numpy()
+ self.model.set_weights(self.og_weights)
+
+ def get_grads(self):
+ return [g.copy() for g in self.grads]
+
+
+###
+
+
+class WandbCallback(tf.keras.callbacks.Callback):
+ """`WandbCallback` automatically integrates keras with wandb.
+
+ Example:
+ ```python
+ model.fit(
+ X_train,
+ y_train,
+ validation_data=(X_test, y_test),
+ callbacks=[WandbCallback()],
+ )
+ ```
+
+ `WandbCallback` will automatically log history data from any
+ metrics collected by keras: loss and anything passed into `keras_model.compile()`.
+
+ `WandbCallback` will set summary metrics for the run associated with the "best" training
+ step, where "best" is defined by the `monitor` and `mode` attributes. This defaults
+ to the epoch with the minimum `val_loss`. `WandbCallback` will by default save the model
+ associated with the best `epoch`.
+
+ `WandbCallback` can optionally log gradient and parameter histograms.
+
+ `WandbCallback` can optionally save training and validation data for wandb to visualize.
+
+ Arguments:
+ monitor: (str) name of metric to monitor. Defaults to `val_loss`.
+ mode: (str) one of {`auto`, `min`, `max`}.
+ `min` - save model when monitor is minimized
+ `max` - save model when monitor is maximized
+ `auto` - try to guess when to save the model (default).
+ save_model:
+ True - save a model when monitor beats all previous epochs
+ False - don't save models
+ save_graph: (boolean) if True save model graph to wandb (default to True).
+ save_weights_only: (boolean) if True, then only the model's weights will be
+ saved (`model.save_weights(filepath)`), else the full model
+ is saved (`model.save(filepath)`).
+ log_weights: (boolean) if True save histograms of the model's layer's weights.
+ log_gradients: (boolean) if True log histograms of the training gradients
+ training_data: (tuple) Same format `(X,y)` as passed to `model.fit`. This is needed
+ for calculating gradients - this is mandatory if `log_gradients` is `True`.
+ validation_data: (tuple) Same format `(X,y)` as passed to `model.fit`. A set of data
+ for wandb to visualize. If this is set, every epoch, wandb will
+ make a small number of predictions and save the results for later visualization. In case
+ you are working with image data, please also set `input_type` and `output_type` in order
+ to log correctly.
+ generator: (generator) a generator that returns validation data for wandb to visualize. This
+ generator should return tuples `(X,y)`. Either `validate_data` or generator should
+ be set for wandb to visualize specific data examples. In case you are working with image data,
+ please also set `input_type` and `output_type` in order to log correctly.
+ validation_steps: (int) if `validation_data` is a generator, how many
+ steps to run the generator for the full validation set.
+ labels: (list) If you are visualizing your data with wandb this list of labels
+ will convert numeric output to understandable string if you are building a
+ multiclass classifier. If you are making a binary classifier you can pass in
+ a list of two labels ["label for false", "label for true"]. If `validate_data`
+ and generator are both false, this won't do anything.
+ predictions: (int) the number of predictions to make for visualization each epoch, max
+ is 100.
+ input_type: (string) type of the model input to help visualization. can be one of:
+ (`image`, `images`, `segmentation_mask`, `auto`).
+ output_type: (string) type of the model output to help visualization. can be one of:
+ (`image`, `images`, `segmentation_mask`, `label`).
+ log_evaluation: (boolean) if True, save a Table containing validation data and the
+ model's predictions at each epoch. See `validation_indexes`,
+ `validation_row_processor`, and `output_row_processor` for additional details.
+ class_colors: ([float, float, float]) if the input or output is a segmentation mask,
+ an array containing an rgb tuple (range 0-1) for each class.
+ log_batch_frequency: (integer) if None, callback will log every epoch.
+ If set to integer, callback will log training metrics every `log_batch_frequency`
+ batches.
+ log_best_prefix: (string) if None, no extra summary metrics will be saved.
+ If set to a string, the monitored metric and epoch will be prepended with this value
+ and stored as summary metrics.
+ validation_indexes: ([wandb.data_types._TableLinkMixin]) an ordered list of index keys to associate
+ with each validation example. If log_evaluation is True and `validation_indexes` is provided,
+ then a Table of validation data will not be created and instead each prediction will
+ be associated with the row represented by the `TableLinkMixin`. The most common way to obtain
+ such keys are is use `Table.get_index()` which will return a list of row keys.
+ validation_row_processor: (Callable) a function to apply to the validation data, commonly used to visualize the data.
+ The function will receive an `ndx` (int) and a `row` (dict). If your model has a single input,
+ then `row["input"]` will be the input data for the row. Else, it will be keyed based on the name of the
+ input slot. If your fit function takes a single target, then `row["target"]` will be the target data for the row. Else,
+ it will be keyed based on the name of the output slots. For example, if your input data is a single ndarray,
+ but you wish to visualize the data as an Image, then you can provide `lambda ndx, row: {"img": wandb.Image(row["input"])}`
+ as the processor. Ignored if log_evaluation is False or `validation_indexes` are present.
+ output_row_processor: (Callable) same as `validation_row_processor`, but applied to the model's output. `row["output"]` will contain
+ the results of the model output.
+ infer_missing_processors: (bool) Determines if `validation_row_processor` and `output_row_processor`
+ should be inferred if missing. Defaults to True. If `labels` are provided, we will attempt to infer classification-type
+ processors where appropriate.
+ log_evaluation_frequency: (int) Determines the frequency which evaluation results will be logged. Default 0 (only at the end of training).
+ Set to 1 to log every epoch, 2 to log every other epoch, and so on. Has no effect when log_evaluation is False.
+ compute_flops: (bool) Compute the FLOPs of your Keras Sequential or Functional model in GigaFLOPs unit.
+ """
+
+ def __init__(
+ self,
+ monitor="val_loss",
+ verbose=0,
+ mode="auto",
+ save_weights_only=False,
+ log_weights=False,
+ log_gradients=False,
+ save_model=True,
+ training_data=None,
+ validation_data=None,
+ labels=None,
+ predictions=36,
+ generator=None,
+ input_type=None,
+ output_type=None,
+ log_evaluation=False,
+ validation_steps=None,
+ class_colors=None,
+ log_batch_frequency=None,
+ log_best_prefix="best_",
+ save_graph=True,
+ validation_indexes=None,
+ validation_row_processor=None,
+ prediction_row_processor=None,
+ infer_missing_processors=True,
+ log_evaluation_frequency=0,
+ compute_flops=False,
+ **kwargs,
+ ):
+ if wandb.run is None:
+ raise wandb.Error("You must call wandb.init() before WandbCallback()")
+ with wandb.wandb_lib.telemetry.context(run=wandb.run) as tel:
+ tel.feature.keras = True
+ self.validation_data = None
+ # This is kept around for legacy reasons
+ if validation_data is not None:
+ if is_generator_like(validation_data):
+ generator = validation_data
+ else:
+ self.validation_data = validation_data
+ if labels is None:
+ labels = []
+ self.labels = labels
+ self.predictions = min(predictions, 100)
+
+ self.monitor = monitor
+ self.verbose = verbose
+ self.save_weights_only = save_weights_only
+ self.save_graph = save_graph
+
+ wandb.save("model-best.h5")
+ self.filepath = os.path.join(wandb.run.dir, "model-best.h5")
+ self.save_model = save_model
+ if save_model:
+ deprecate(
+ field_name=Deprecated.keras_callback__save_model,
+ warning_message=(
+ "The save_model argument by default saves the model in the HDF5 format that cannot save "
+ "custom objects like subclassed models and custom layers. This behavior will be deprecated "
+ "in a future release in favor of the SavedModel format. Meanwhile, the HDF5 model is saved "
+ "as W&B files and the SavedModel as W&B Artifacts."
+ ),
+ )
+
+ self.save_model_as_artifact = True
+ self.log_weights = log_weights
+ self.log_gradients = log_gradients
+ self.training_data = training_data
+ self.generator = generator
+ self._graph_rendered = False
+
+ data_type = kwargs.get("data_type", None)
+ if data_type is not None:
+ deprecate(
+ field_name=Deprecated.keras_callback__data_type,
+ warning_message=(
+ "The data_type argument of wandb.keras.WandbCallback is deprecated "
+ "and will be removed in a future release. Please use input_type instead.\n"
+ "Setting input_type = data_type."
+ ),
+ )
+ input_type = data_type
+ self.input_type = input_type
+ self.output_type = output_type
+ self.log_evaluation = log_evaluation
+ self.validation_steps = validation_steps
+ self.class_colors = np.array(class_colors) if class_colors is not None else None
+ self.log_batch_frequency = log_batch_frequency
+ self.log_best_prefix = log_best_prefix
+ self.compute_flops = compute_flops
+
+ self._prediction_batch_size = None
+
+ if self.log_gradients:
+ if int(tf.__version__.split(".")[0]) < 2:
+ raise Exception("Gradient logging requires tensorflow 2.0 or higher.")
+ if self.training_data is None:
+ raise ValueError(
+ "training_data argument is required for gradient logging."
+ )
+ if isinstance(self.training_data, (list, tuple)):
+ if len(self.training_data) != 2:
+ raise ValueError("training data must be a tuple of length two")
+ self._training_data_x, self._training_data_y = self.training_data
+ else:
+ self._training_data_x = (
+ self.training_data
+ ) # generator, tf.data.Dataset etc
+ self._training_data_y = None
+
+ # From Keras
+ if mode not in ["auto", "min", "max"]:
+ print(f"WandbCallback mode {mode} is unknown, fallback to auto mode.")
+ mode = "auto"
+
+ if mode == "min":
+ self.monitor_op = operator.lt
+ self.best = float("inf")
+ elif mode == "max":
+ self.monitor_op = operator.gt
+ self.best = float("-inf")
+ else:
+ if "acc" in self.monitor or self.monitor.startswith("fmeasure"):
+ self.monitor_op = operator.gt
+ self.best = float("-inf")
+ else:
+ self.monitor_op = operator.lt
+ self.best = float("inf")
+ # Get the previous best metric for resumed runs
+ previous_best = wandb.run.summary.get(f"{self.log_best_prefix}{self.monitor}")
+ if previous_best is not None:
+ self.best = previous_best
+
+ self._validation_data_logger = None
+ self._validation_indexes = validation_indexes
+ self._validation_row_processor = validation_row_processor
+ self._prediction_row_processor = prediction_row_processor
+ self._infer_missing_processors = infer_missing_processors
+ self._log_evaluation_frequency = log_evaluation_frequency
+ self._model_trained_since_last_eval = False
+
+ def _build_grad_accumulator_model(self):
+ inputs = self.model.inputs
+ outputs = self.model(inputs)
+ grad_acc_model = tf.keras.models.Model(inputs, outputs)
+ grad_acc_model.compile(loss=self.model.loss, optimizer=_CustomOptimizer())
+
+ # make sure magic doesn't think this is a user model
+ grad_acc_model._wandb_internal_model = True
+
+ self._grad_accumulator_model = grad_acc_model
+ self._grad_accumulator_callback = _GradAccumulatorCallback()
+
+ def _implements_train_batch_hooks(self):
+ return self.log_batch_frequency is not None
+
+ def _implements_test_batch_hooks(self):
+ return self.log_batch_frequency is not None
+
+ def _implements_predict_batch_hooks(self):
+ return self.log_batch_frequency is not None
+
+ def set_params(self, params):
+ self.params = params
+
+ def set_model(self, model):
+ self.model = model
+ if self.input_type == "auto" and len(model.inputs) == 1:
+ self.input_type = wandb.util.guess_data_type(
+ model.inputs[0].shape, risky=True
+ )
+ if self.input_type and self.output_type is None and len(model.outputs) == 1:
+ self.output_type = wandb.util.guess_data_type(model.outputs[0].shape)
+ if self.log_gradients:
+ self._build_grad_accumulator_model()
+
+ def _attempt_evaluation_log(self, commit=True):
+ if self.log_evaluation and self._validation_data_logger:
+ try:
+ if not self.model:
+ wandb.termwarn("WandbCallback unable to read model from trainer")
+ else:
+ self._validation_data_logger.log_predictions(
+ predictions=self._validation_data_logger.make_predictions(
+ self.model.predict
+ ),
+ commit=commit,
+ )
+ self._model_trained_since_last_eval = False
+ except Exception as e:
+ wandb.termwarn("Error durring prediction logging for epoch: " + str(e))
+
+ def on_epoch_end(self, epoch, logs=None):
+ if logs is None:
+ logs = {}
+ if self.log_weights:
+ wandb.log(self._log_weights(), commit=False)
+
+ if self.log_gradients:
+ wandb.log(self._log_gradients(), commit=False)
+
+ if self.input_type in (
+ "image",
+ "images",
+ "segmentation_mask",
+ ) or self.output_type in ("image", "images", "segmentation_mask"):
+ if self.generator:
+ self.validation_data = next(self.generator)
+ if self.validation_data is None:
+ wandb.termwarn(
+ "No validation_data set, pass a generator to the callback."
+ )
+ elif self.validation_data and len(self.validation_data) > 0:
+ wandb.log(
+ {"examples": self._log_images(num_images=self.predictions)},
+ commit=False,
+ )
+
+ if (
+ self._log_evaluation_frequency > 0
+ and epoch % self._log_evaluation_frequency == 0
+ ):
+ self._attempt_evaluation_log(commit=False)
+
+ wandb.log({"epoch": epoch}, commit=False)
+ wandb.log(logs, commit=True)
+
+ self.current = logs.get(self.monitor)
+ if self.current and self.monitor_op(self.current, self.best):
+ if self.log_best_prefix:
+ wandb.run.summary[
+ f"{self.log_best_prefix}{self.monitor}"
+ ] = self.current
+ wandb.run.summary["{}{}".format(self.log_best_prefix, "epoch")] = epoch
+ if self.verbose and not self.save_model:
+ print(
+ "Epoch %05d: %s improved from %0.5f to %0.5f"
+ % (epoch, self.monitor, self.best, self.current)
+ )
+ if self.save_model:
+ self._save_model(epoch)
+
+ if self.save_model and self.save_model_as_artifact:
+ self._save_model_as_artifact(epoch)
+
+ self.best = self.current
+
+ # This is what keras used pre tensorflow.keras
+ def on_batch_begin(self, batch, logs=None):
+ pass
+
+ # This is what keras used pre tensorflow.keras
+ def on_batch_end(self, batch, logs=None):
+ if self.save_graph and not self._graph_rendered:
+ # Couldn't do this in train_begin because keras may still not be built
+ wandb.run.summary["graph"] = wandb.Graph.from_keras(self.model)
+ self._graph_rendered = True
+
+ if self.log_batch_frequency and batch % self.log_batch_frequency == 0:
+ wandb.log(logs, commit=True)
+
+ def on_train_batch_begin(self, batch, logs=None):
+ self._model_trained_since_last_eval = True
+
+ def on_train_batch_end(self, batch, logs=None):
+ if self.save_graph and not self._graph_rendered:
+ # Couldn't do this in train_begin because keras may still not be built
+ wandb.run.summary["graph"] = wandb.Graph.from_keras(self.model)
+ self._graph_rendered = True
+
+ if self.log_batch_frequency and batch % self.log_batch_frequency == 0:
+ wandb.log(logs, commit=True)
+
+ def on_test_begin(self, logs=None):
+ pass
+
+ def on_test_end(self, logs=None):
+ pass
+
+ def on_test_batch_begin(self, batch, logs=None):
+ pass
+
+ def on_test_batch_end(self, batch, logs=None):
+ pass
+
+ def on_train_begin(self, logs=None):
+ if self.log_evaluation:
+ try:
+ validation_data = None
+ if self.validation_data:
+ validation_data = self.validation_data
+ elif self.generator:
+ if not self.validation_steps:
+ wandb.termwarn(
+ "WandbCallback is unable to log validation data. "
+ "When using a generator for validation_data, you must pass validation_steps"
+ )
+ else:
+ x = None
+ y_true = None
+ for _ in range(self.validation_steps):
+ bx, by_true = next(self.generator)
+ if x is None:
+ x, y_true = bx, by_true
+ else:
+ x, y_true = (
+ np.append(x, bx, axis=0),
+ np.append(y_true, by_true, axis=0),
+ )
+ validation_data = (x, y_true)
+ else:
+ wandb.termwarn(
+ "WandbCallback is unable to read validation_data from trainer "
+ "and therefore cannot log validation data. Ensure Keras is properly "
+ "patched by calling `from wandb.keras import WandbCallback` at the top of your script."
+ )
+ if validation_data:
+ self._validation_data_logger = ValidationDataLogger(
+ inputs=validation_data[0],
+ targets=validation_data[1],
+ indexes=self._validation_indexes,
+ validation_row_processor=self._validation_row_processor,
+ prediction_row_processor=self._prediction_row_processor,
+ class_labels=self.labels,
+ infer_missing_processors=self._infer_missing_processors,
+ )
+ except Exception as e:
+ wandb.termwarn(
+ "Error initializing ValidationDataLogger in WandbCallback. "
+ f"Skipping logging validation data. Error: {str(e)}"
+ )
+
+ if self.compute_flops and _can_compute_flops():
+ try:
+ wandb.summary["GFLOPs"] = self.get_flops()
+ except Exception as e:
+ wandb.termwarn("Unable to compute FLOPs for this model.")
+ logger.exception(e)
+
+ def on_train_end(self, logs=None):
+ if self._model_trained_since_last_eval:
+ self._attempt_evaluation_log()
+
+ def on_predict_begin(self, logs=None):
+ pass
+
+ def on_predict_end(self, logs=None):
+ pass
+
+ def on_predict_batch_begin(self, batch, logs=None):
+ pass
+
+ def on_predict_batch_end(self, batch, logs=None):
+ pass
+
+ def _logits_to_captions(self, logits):
+ if logits[0].shape[-1] == 1:
+ # Scalar output from the model
+ # TODO: handle validation_y
+ if len(self.labels) == 2:
+ # User has named true and false
+ captions = [
+ self.labels[1] if logits[0] > 0.5 else self.labels[0]
+ for logit in logits
+ ]
+ else:
+ if len(self.labels) != 0:
+ wandb.termwarn(
+ "keras model is producing a single output, "
+ 'so labels should be a length two array: ["False label", "True label"].'
+ )
+ captions = [logit[0] for logit in logits]
+ else:
+ # Vector output from the model
+ # TODO: handle validation_y
+ labels = np.argmax(np.stack(logits), axis=1)
+
+ if len(self.labels) > 0:
+ # User has named the categories in self.labels
+ captions = []
+ for label in labels:
+ try:
+ captions.append(self.labels[label])
+ except IndexError:
+ captions.append(label)
+ else:
+ captions = labels
+ return captions
+
+ def _masks_to_pixels(self, masks):
+ # if its a binary mask, just return it as grayscale instead of picking the argmax
+ if len(masks[0].shape) == 2 or masks[0].shape[-1] == 1:
+ return masks
+ class_colors = (
+ self.class_colors
+ if self.class_colors is not None
+ else np.array(wandb.util.class_colors(masks[0].shape[2]))
+ )
+ imgs = class_colors[np.argmax(masks, axis=-1)]
+ return imgs
+
+ def _log_images(self, num_images=36):
+ validation_X = self.validation_data[0] # noqa: N806
+ validation_y = self.validation_data[1]
+
+ validation_length = len(validation_X)
+
+ if validation_length > num_images:
+ # pick some data at random
+ indices = np.random.choice(validation_length, num_images, replace=False)
+ else:
+ indices = range(validation_length)
+
+ test_data = []
+ test_output = []
+ for i in indices:
+ test_example = validation_X[i]
+ test_data.append(test_example)
+ test_output.append(validation_y[i])
+
+ if self.model.stateful:
+ predictions = self.model.predict(np.stack(test_data), batch_size=1)
+ self.model.reset_states()
+ else:
+ predictions = self.model.predict(
+ np.stack(test_data), batch_size=self._prediction_batch_size
+ )
+ if len(predictions) != len(test_data):
+ self._prediction_batch_size = 1
+ predictions = self.model.predict(
+ np.stack(test_data), batch_size=self._prediction_batch_size
+ )
+
+ if self.input_type == "label":
+ if self.output_type in ("image", "images", "segmentation_mask"):
+ captions = self._logits_to_captions(test_data)
+ output_image_data = (
+ self._masks_to_pixels(predictions)
+ if self.output_type == "segmentation_mask"
+ else predictions
+ )
+ reference_image_data = (
+ self._masks_to_pixels(test_output)
+ if self.output_type == "segmentation_mask"
+ else test_output
+ )
+ output_images = [
+ wandb.Image(data, caption=captions[i], grouping=2)
+ for i, data in enumerate(output_image_data)
+ ]
+ reference_images = [
+ wandb.Image(data, caption=captions[i])
+ for i, data in enumerate(reference_image_data)
+ ]
+ return list(chain.from_iterable(zip(output_images, reference_images)))
+ elif self.input_type in ("image", "images", "segmentation_mask"):
+ input_image_data = (
+ self._masks_to_pixels(test_data)
+ if self.input_type == "segmentation_mask"
+ else test_data
+ )
+ if self.output_type == "label":
+ # we just use the predicted label as the caption for now
+ captions = self._logits_to_captions(predictions)
+ return [
+ wandb.Image(data, caption=captions[i])
+ for i, data in enumerate(test_data)
+ ]
+ elif self.output_type in ("image", "images", "segmentation_mask"):
+ output_image_data = (
+ self._masks_to_pixels(predictions)
+ if self.output_type == "segmentation_mask"
+ else predictions
+ )
+ reference_image_data = (
+ self._masks_to_pixels(test_output)
+ if self.output_type == "segmentation_mask"
+ else test_output
+ )
+ input_images = [
+ wandb.Image(data, grouping=3)
+ for i, data in enumerate(input_image_data)
+ ]
+ output_images = [
+ wandb.Image(data) for i, data in enumerate(output_image_data)
+ ]
+ reference_images = [
+ wandb.Image(data) for i, data in enumerate(reference_image_data)
+ ]
+ return list(
+ chain.from_iterable(
+ zip(input_images, output_images, reference_images)
+ )
+ )
+ else:
+ # unknown output, just log the input images
+ return [wandb.Image(img) for img in test_data]
+ elif self.output_type in ("image", "images", "segmentation_mask"):
+ # unknown input, just log the predicted and reference outputs without captions
+ output_image_data = (
+ self._masks_to_pixels(predictions)
+ if self.output_type == "segmentation_mask"
+ else predictions
+ )
+ reference_image_data = (
+ self._masks_to_pixels(test_output)
+ if self.output_type == "segmentation_mask"
+ else test_output
+ )
+ output_images = [
+ wandb.Image(data, grouping=2)
+ for i, data in enumerate(output_image_data)
+ ]
+ reference_images = [
+ wandb.Image(data) for i, data in enumerate(reference_image_data)
+ ]
+ return list(chain.from_iterable(zip(output_images, reference_images)))
+
+ def _log_weights(self):
+ metrics = {}
+ for layer in self.model.layers:
+ weights = layer.get_weights()
+ if len(weights) == 1:
+ _update_if_numeric(
+ metrics, "parameters/" + layer.name + ".weights", weights[0]
+ )
+ elif len(weights) == 2:
+ _update_if_numeric(
+ metrics, "parameters/" + layer.name + ".weights", weights[0]
+ )
+ _update_if_numeric(
+ metrics, "parameters/" + layer.name + ".bias", weights[1]
+ )
+ return metrics
+
+ def _log_gradients(self):
+ # Suppress callback warnings grad accumulator
+ og_level = tf_logger.level
+ tf_logger.setLevel("ERROR")
+
+ self._grad_accumulator_model.fit(
+ self._training_data_x,
+ self._training_data_y,
+ verbose=0,
+ callbacks=[self._grad_accumulator_callback],
+ )
+ tf_logger.setLevel(og_level)
+ weights = self.model.trainable_weights
+ grads = self._grad_accumulator_callback.grads
+ metrics = {}
+ for weight, grad in zip(weights, grads):
+ metrics[
+ "gradients/" + weight.name.split(":")[0] + ".gradient"
+ ] = wandb.Histogram(grad)
+ return metrics
+
+ def _log_dataframe(self):
+ x, y_true, y_pred = None, None, None
+
+ if self.validation_data:
+ x, y_true = self.validation_data[0], self.validation_data[1]
+ y_pred = self.model.predict(x)
+ elif self.generator:
+ if not self.validation_steps:
+ wandb.termwarn(
+ "when using a generator for validation data with dataframes, "
+ "you must pass validation_steps. skipping"
+ )
+ return None
+
+ for _ in range(self.validation_steps):
+ bx, by_true = next(self.generator)
+ by_pred = self.model.predict(bx)
+ if x is None:
+ x, y_true, y_pred = bx, by_true, by_pred
+ else:
+ x, y_true, y_pred = (
+ np.append(x, bx, axis=0),
+ np.append(y_true, by_true, axis=0),
+ np.append(y_pred, by_pred, axis=0),
+ )
+
+ if self.input_type in ("image", "images") and self.output_type == "label":
+ return wandb.image_categorizer_dataframe(
+ x=x, y_true=y_true, y_pred=y_pred, labels=self.labels
+ )
+ elif (
+ self.input_type in ("image", "images")
+ and self.output_type == "segmentation_mask"
+ ):
+ return wandb.image_segmentation_dataframe(
+ x=x,
+ y_true=y_true,
+ y_pred=y_pred,
+ labels=self.labels,
+ class_colors=self.class_colors,
+ )
+ else:
+ wandb.termwarn(
+ f"unknown dataframe type for input_type={self.input_type} and output_type={self.output_type}"
+ )
+ return None
+
+ def _save_model(self, epoch):
+ if wandb.run.disabled:
+ return
+ if self.verbose > 0:
+ print(
+ "Epoch %05d: %s improved from %0.5f to %0.5f,"
+ " saving model to %s"
+ % (epoch, self.monitor, self.best, self.current, self.filepath)
+ )
+
+ try:
+ if self.save_weights_only:
+ self.model.save_weights(self.filepath, overwrite=True)
+ else:
+ self.model.save(self.filepath, overwrite=True)
+ # Was getting `RuntimeError: Unable to create link` in TF 1.13.1
+ # also saw `TypeError: can't pickle _thread.RLock objects`
+ except (ImportError, RuntimeError, TypeError, AttributeError) as e:
+ wandb.termerror(
+ "Can't save model in the h5py format. The model will be saved as "
+ "as an W&B Artifact in the 'tf' format."
+ )
+ logger.exception(e)
+
+ def _save_model_as_artifact(self, epoch):
+ if wandb.run.disabled:
+ return
+
+ # Save the model in the SavedModel format.
+ # TODO: Replace this manual artifact creation with the `log_model` method
+ # after `log_model` is released from beta.
+ self.model.save(self.filepath[:-3], overwrite=True, save_format="tf")
+
+ # Log the model as artifact.
+ name = wandb.util.make_artifact_name_safe(f"model-{wandb.run.name}")
+ model_artifact = wandb.Artifact(name, type="model")
+ model_artifact.add_dir(self.filepath[:-3])
+ wandb.run.log_artifact(model_artifact, aliases=["latest", f"epoch_{epoch}"])
+
+ # Remove the SavedModel from wandb dir as we don't want to log it to save memory.
+ shutil.rmtree(self.filepath[:-3])
+
+ def get_flops(self) -> float:
+ """Calculate FLOPS [GFLOPs] for a tf.keras.Model or tf.keras.Sequential model in inference mode.
+
+ It uses tf.compat.v1.profiler under the hood.
+ """
+ if not hasattr(self, "model"):
+ raise wandb.Error("self.model must be set before using this method.")
+
+ if not isinstance(
+ self.model, (tf.keras.models.Sequential, tf.keras.models.Model)
+ ):
+ raise ValueError(
+ "Calculating FLOPS is only supported for "
+ "`tf.keras.Model` and `tf.keras.Sequential` instances."
+ )
+
+ from tensorflow.python.framework.convert_to_constants import (
+ convert_variables_to_constants_v2_as_graph,
+ )
+
+ # Compute FLOPs for one sample
+ batch_size = 1
+ inputs = [
+ tf.TensorSpec([batch_size] + inp.shape[1:], inp.dtype)
+ for inp in self.model.inputs
+ ]
+
+ # convert tf.keras model into frozen graph to count FLOPs about operations used at inference
+ real_model = tf.function(self.model).get_concrete_function(inputs)
+ frozen_func, _ = convert_variables_to_constants_v2_as_graph(real_model)
+
+ # Calculate FLOPs with tf.profiler
+ run_meta = tf.compat.v1.RunMetadata()
+ opts = (
+ tf.compat.v1.profiler.ProfileOptionBuilder(
+ tf.compat.v1.profiler.ProfileOptionBuilder().float_operation()
+ )
+ .with_empty_output()
+ .build()
+ )
+
+ flops = tf.compat.v1.profiler.profile(
+ graph=frozen_func.graph, run_meta=run_meta, cmd="scope", options=opts
+ )
+
+ # convert to GFLOPs
+ return (flops.total_float_ops / 1e9) / 2
diff --git a/wandb/integration/kfp/__init__.py b/wandb/integration/kfp/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..1f3a362ca0fc895bce03dccdfeaa3274a9b5963d
--- /dev/null
+++ b/wandb/integration/kfp/__init__.py
@@ -0,0 +1,6 @@
+__all__ = ["wandb_log", "unpatch_kfp"]
+
+from .kfp_patch import patch_kfp, unpatch_kfp
+from .wandb_logging import wandb_log
+
+patch_kfp()
diff --git a/wandb/integration/kfp/helpers.py b/wandb/integration/kfp/helpers.py
new file mode 100644
index 0000000000000000000000000000000000000000..feebcc62ea3688a2a93818adb445f6454a6d9dc2
--- /dev/null
+++ b/wandb/integration/kfp/helpers.py
@@ -0,0 +1,28 @@
+import json
+
+
+def add_wandb_visualization(run, mlpipeline_ui_metadata_path):
+ """NOTE: To use this, you must modify your component to have an output called `mlpipeline_ui_metadata_path` AND call `wandb.init` yourself inside that component.
+
+ Example usage:
+
+ def my_component(..., mlpipeline_ui_metadata_path: OutputPath()):
+ import wandb
+ from wandb.integration.kfp.helpers import add_wandb_visualization
+
+ with wandb.init() as run:
+ add_wandb_visualization(run, mlpipeline_ui_metadata_path)
+
+ ... # the rest of your code here
+ """
+
+ def get_iframe_html(run):
+ return f'<iframe src="{run.url}?kfp=true" style="border:none;width:100%;height:100%;min-width:900px;min-height:600px;"></iframe>'
+
+ iframe_html = get_iframe_html(run)
+ metadata = {
+ "outputs": [{"type": "markdown", "storage": "inline", "source": iframe_html}]
+ }
+
+ with open(mlpipeline_ui_metadata_path, "w") as metadata_file:
+ json.dump(metadata, metadata_file)
diff --git a/wandb/integration/kfp/kfp_patch.py b/wandb/integration/kfp/kfp_patch.py
new file mode 100644
index 0000000000000000000000000000000000000000..f93b0d06a7202015c8d1999d11e108191755485b
--- /dev/null
+++ b/wandb/integration/kfp/kfp_patch.py
@@ -0,0 +1,324 @@
+import inspect
+import itertools
+import textwrap
+from typing import Callable, List, Mapping, Optional
+
+from pkg_resources import parse_version
+
+import wandb
+
+try:
+ from kfp import __version__ as kfp_version
+ from kfp.components import structures
+ from kfp.components._components import _create_task_factory_from_component_spec
+ from kfp.components._python_op import _func_to_component_spec
+
+ MIN_KFP_VERSION = "1.6.1"
+
+ if parse_version(kfp_version) < parse_version(MIN_KFP_VERSION):
+ wandb.termwarn(
+ f"Your version of kfp {kfp_version} may not work. This integration requires kfp>={MIN_KFP_VERSION}"
+ )
+
+except ImportError:
+ wandb.termerror("kfp not found! Please `pip install kfp`")
+
+from .wandb_logging import wandb_log
+
+decorator_code = inspect.getsource(wandb_log)
+wandb_logging_extras = f"""
+import typing
+from typing import NamedTuple
+
+import collections
+from collections import namedtuple
+
+import kfp
+from kfp import components
+from kfp.components import InputPath, OutputPath
+
+import wandb
+
+{decorator_code}
+"""
+
+
+def full_path_exists(full_func):
+ def get_parent_child_pairs(full_func):
+ components = full_func.split(".")
+ parents, children = [], []
+ for i, _ in enumerate(components[:-1], 1):
+ parent = ".".join(components[:i])
+ child = components[i]
+ parents.append(parent)
+ children.append(child)
+ return zip(parents, children)
+
+ for parent, child in get_parent_child_pairs(full_func):
+ module = wandb.util.get_module(parent)
+ if not module or not hasattr(module, child) or getattr(module, child) is None:
+ return False
+ return True
+
+
+def patch(module_name, func):
+ module = wandb.util.get_module(module_name)
+ success = False
+
+ full_func = f"{module_name}.{func.__name__}"
+ if not full_path_exists(full_func):
+ wandb.termerror(
+ f"Failed to patch {module_name}.{func.__name__}! Please check if this package/module is installed!"
+ )
+ else:
+ wandb.patched.setdefault(module.__name__, [])
+ # if already patched, do not patch again
+ if [module, func.__name__] not in wandb.patched[module.__name__]:
+ setattr(module, f"orig_{func.__name__}", getattr(module, func.__name__))
+ setattr(module, func.__name__, func)
+ wandb.patched[module.__name__].append([module, func.__name__])
+ success = True
+
+ return success
+
+
+def unpatch(module_name):
+ if module_name in wandb.patched:
+ for module, func in wandb.patched[module_name]:
+ setattr(module, func, getattr(module, f"orig_{func}"))
+ wandb.patched[module_name] = []
+
+
+def unpatch_kfp():
+ unpatch("kfp.components")
+ unpatch("kfp.components._python_op")
+ unpatch("wandb.integration.kfp")
+
+
+def patch_kfp():
+ to_patch = [
+ (
+ "kfp.components",
+ create_component_from_func,
+ ),
+ (
+ "kfp.components._python_op",
+ create_component_from_func,
+ ),
+ (
+ "kfp.components._python_op",
+ _get_function_source_definition,
+ ),
+ ("kfp.components._python_op", strip_type_hints),
+ ]
+
+ successes = []
+ for module_name, func in to_patch:
+ success = patch(module_name, func)
+ successes.append(success)
+ if not all(successes):
+ wandb.termerror(
+ "Failed to patch one or more kfp functions. Patching @wandb_log decorator to no-op."
+ )
+ patch("wandb.integration.kfp", wandb_log)
+
+
+def wandb_log(
+ func=None,
+ # /, # py38 only
+ log_component_file=True,
+):
+ """Wrap a standard python function and log to W&B.
+
+ NOTE: Because patching failed, this decorator is a no-op.
+ """
+ from functools import wraps
+
+ def decorator(func):
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ return func(*args, **kwargs)
+
+ return wrapper
+
+ if func is None:
+ return decorator
+ else:
+ return decorator(func)
+
+
+def _get_function_source_definition(func: Callable) -> str:
+ """Get the source code of a function.
+
+ This function is modified from KFP. The original source is below:
+ https://github.com/kubeflow/pipelines/blob/b6406b02f45cdb195c7b99e2f6d22bf85b12268b/sdk/python/kfp/components/_python_op.py#L300-L319.
+ """
+ func_code = inspect.getsource(func)
+
+ # Function might be defined in some indented scope (e.g. in another
+ # function). We need to handle this and properly dedent the function source
+ # code
+ func_code = textwrap.dedent(func_code)
+ func_code_lines = func_code.split("\n")
+
+ # For wandb, allow decorators (so we can use the @wandb_log decorator)
+ func_code_lines = itertools.dropwhile(
+ lambda x: not (x.startswith("def") or x.startswith("@wandb_log")),
+ func_code_lines,
+ )
+
+ if not func_code_lines:
+ raise ValueError(
+ f'Failed to dedent and clean up the source of function "{func.__name__}". '
+ "It is probably not properly indented."
+ )
+
+ return "\n".join(func_code_lines)
+
+
+def create_component_from_func(
+ func: Callable,
+ output_component_file: Optional[str] = None,
+ base_image: Optional[str] = None,
+ packages_to_install: Optional[List[str]] = None,
+ annotations: Optional[Mapping[str, str]] = None,
+):
+ '''Convert a Python function to a component and returns a task factory.
+
+ The returned task factory accepts arguments and returns a task object.
+
+ This function is modified from KFP. The original source is below:
+ https://github.com/kubeflow/pipelines/blob/b6406b02f45cdb195c7b99e2f6d22bf85b12268b/sdk/python/kfp/components/_python_op.py#L998-L1110.
+
+ Args:
+ func: The python function to convert
+ base_image: Optional. Specify a custom Docker container image to use in the component. For lightweight components, the image needs to have python 3.5+. Default is the python image corresponding to the current python environment.
+ output_component_file: Optional. Write a component definition to a local file. The produced component file can be loaded back by calling :code:`load_component_from_file` or :code:`load_component_from_uri`.
+ packages_to_install: Optional. List of [versioned] python packages to pip install before executing the user function.
+ annotations: Optional. Allows adding arbitrary key-value data to the component specification.
+
+ Returns:
+ A factory function with a strongly-typed signature taken from the python function.
+ Once called with the required arguments, the factory constructs a task instance that can run the original function in a container.
+
+ Examples:
+ The function name and docstring are used as component name and description. Argument and return annotations are used as component input/output types::
+
+ def add(a: float, b: float) -> float:
+ """Return sum of two arguments"""
+ return a + b
+
+ # add_op is a task factory function that creates a task object when given arguments
+ add_op = create_component_from_func(
+ func=add,
+ base_image='python:3.7', # Optional
+ output_component_file='add.component.yaml', # Optional
+ packages_to_install=['pandas==0.24'], # Optional
+ )
+
+ # The component spec can be accessed through the .component_spec attribute:
+ add_op.component_spec.save('add.component.yaml')
+
+ # The component function can be called with arguments to create a task:
+ add_task = add_op(1, 3)
+
+ # The resulting task has output references, corresponding to the component outputs.
+ # When the function only has a single anonymous return value, the output name is "Output":
+ sum_output_ref = add_task.outputs['Output']
+
+ # These task output references can be passed to other component functions, constructing a computation graph:
+ task2 = add_op(sum_output_ref, 5)
+
+
+ :code:`create_component_from_func` function can also be used as decorator::
+
+ @create_component_from_func
+ def add_op(a: float, b: float) -> float:
+ """Return sum of two arguments"""
+ return a + b
+
+ To declare a function with multiple return values, use the :code:`NamedTuple` return annotation syntax::
+
+ from typing import NamedTuple
+
+ def add_multiply_two_numbers(a: float, b: float) -> NamedTuple('Outputs', [('sum', float), ('product', float)]):
+ """Return sum and product of two arguments"""
+ return (a + b, a * b)
+
+ add_multiply_op = create_component_from_func(add_multiply_two_numbers)
+
+ # The component function can be called with arguments to create a task:
+ add_multiply_task = add_multiply_op(1, 3)
+
+ # The resulting task has output references, corresponding to the component outputs:
+ sum_output_ref = add_multiply_task.outputs['sum']
+
+ # These task output references can be passed to other component functions, constructing a computation graph:
+ task2 = add_multiply_op(sum_output_ref, 5)
+
+ Bigger data should be read from files and written to files.
+ Use the :py:class:`kfp.components.InputPath` parameter annotation to tell the system that the function wants to consume the corresponding input data as a file. The system will download the data, write it to a local file and then pass the **path** of that file to the function.
+ Use the :py:class:`kfp.components.OutputPath` parameter annotation to tell the system that the function wants to produce the corresponding output data as a file. The system will prepare and pass the **path** of a file where the function should write the output data. After the function exits, the system will upload the data to the storage system so that it can be passed to downstream components.
+
+ You can specify the type of the consumed/produced data by specifying the type argument to :py:class:`kfp.components.InputPath` and :py:class:`kfp.components.OutputPath`. The type can be a python type or an arbitrary type name string. :code:`OutputPath('CatBoostModel')` means that the function states that the data it has written to a file has type :code:`CatBoostModel`. :code:`InputPath('CatBoostModel')` means that the function states that it expect the data it reads from a file to have type 'CatBoostModel'. When the pipeline author connects inputs to outputs the system checks whether the types match.
+ Every kind of data can be consumed as a file input. Conversely, bigger data should not be consumed by value as all value inputs pass through the command line.
+
+ Example of a component function declaring file input and output::
+
+ def catboost_train_classifier(
+ training_data_path: InputPath('CSV'), # Path to input data file of type "CSV"
+ trained_model_path: OutputPath('CatBoostModel'), # Path to output data file of type "CatBoostModel"
+ number_of_trees: int = 100, # Small output of type "Integer"
+ ) -> NamedTuple('Outputs', [
+ ('Accuracy', float), # Small output of type "Float"
+ ('Precision', float), # Small output of type "Float"
+ ('JobUri', 'URI'), # Small output of type "URI"
+ ]):
+ """Train CatBoost classification model"""
+ ...
+
+ return (accuracy, precision, recall)
+ '''
+ core_packages = ["wandb", "kfp"]
+
+ if not packages_to_install:
+ packages_to_install = core_packages
+ else:
+ packages_to_install += core_packages
+
+ component_spec = _func_to_component_spec(
+ func=func,
+ extra_code=wandb_logging_extras,
+ base_image=base_image,
+ packages_to_install=packages_to_install,
+ )
+ if annotations:
+ component_spec.metadata = structures.MetadataSpec(
+ annotations=annotations,
+ )
+
+ if output_component_file:
+ component_spec.save(output_component_file)
+
+ return _create_task_factory_from_component_spec(component_spec)
+
+
+def strip_type_hints(source_code: str) -> str:
+ """Strip type hints from source code.
+
+ This function is modified from KFP. The original source is below:
+ https://github.com/kubeflow/pipelines/blob/b6406b02f45cdb195c7b99e2f6d22bf85b12268b/sdk/python/kfp/components/_python_op.py#L237-L248.
+ """
+ # For wandb, do not strip type hints
+
+ # try:
+ # return _strip_type_hints_using_lib2to3(source_code)
+ # except Exception as ex:
+ # print('Error when stripping type annotations: ' + str(ex))
+
+ # try:
+ # return _strip_type_hints_using_strip_hints(source_code)
+ # except Exception as ex:
+ # print('Error when stripping type annotations: ' + str(ex))
+
+ return source_code
diff --git a/wandb/integration/kfp/wandb_logging.py b/wandb/integration/kfp/wandb_logging.py
new file mode 100644
index 0000000000000000000000000000000000000000..005c44d6acf4ce02c695be89092cf78f48020afd
--- /dev/null
+++ b/wandb/integration/kfp/wandb_logging.py
@@ -0,0 +1,182 @@
+def wandb_log( # noqa: C901
+ func=None,
+ # /, # py38 only
+ log_component_file=True,
+):
+ """Wrap a standard python function and log to W&B."""
+ import json
+ import os
+ from functools import wraps
+ from inspect import Parameter, signature
+
+ from kfp import components
+ from kfp.components import (
+ InputArtifact,
+ InputBinaryFile,
+ InputPath,
+ InputTextFile,
+ OutputArtifact,
+ OutputBinaryFile,
+ OutputPath,
+ OutputTextFile,
+ )
+
+ import wandb
+ from wandb.sdk.lib import telemetry as wb_telemetry
+
+ output_types = (OutputArtifact, OutputBinaryFile, OutputPath, OutputTextFile)
+ input_types = (InputArtifact, InputBinaryFile, InputPath, InputTextFile)
+
+ def isinstance_namedtuple(x):
+ t = type(x)
+ b = t.__bases__
+ if len(b) != 1 or b[0] != tuple:
+ return False
+ f = getattr(t, "_fields", None)
+ if not isinstance(f, tuple):
+ return False
+ return all(isinstance(n, str) for n in f)
+
+ def get_iframe_html(run):
+ return f'<iframe src="{run.url}?kfp=true" style="border:none;width:100%;height:100%;min-width:900px;min-height:600px;"></iframe>'
+
+ def get_link_back_to_kubeflow():
+ wandb_kubeflow_url = os.getenv("WANDB_KUBEFLOW_URL")
+ return f"{wandb_kubeflow_url}/#/runs/details/{{workflow.uid}}"
+
+ def log_input_scalar(name, data, run=None):
+ run.config[name] = data
+ wandb.termlog(f"Setting config: {name} to {data}")
+
+ def log_input_artifact(name, data, type, run=None):
+ artifact = wandb.Artifact(name, type=type)
+ artifact.add_file(data)
+ run.use_artifact(artifact)
+ wandb.termlog(f"Using artifact: {name}")
+
+ def log_output_scalar(name, data, run=None):
+ if isinstance_namedtuple(data):
+ for k, v in zip(data._fields, data):
+ run.log({f"{func.__name__}.{k}": v})
+ else:
+ run.log({name: data})
+
+ def log_output_artifact(name, data, type, run=None):
+ artifact = wandb.Artifact(name, type=type)
+ artifact.add_file(data)
+ run.log_artifact(artifact)
+ wandb.termlog(f"Logging artifact: {name}")
+
+ def _log_component_file(func, run=None):
+ name = func.__name__
+ output_component_file = f"{name}.yml"
+ components._python_op.func_to_component_file(func, output_component_file)
+ artifact = wandb.Artifact(name, type="kubeflow_component_file")
+ artifact.add_file(output_component_file)
+ run.log_artifact(artifact)
+ wandb.termlog(f"Logging component file: {output_component_file}")
+
+ # Add `mlpipeline_ui_metadata_path` to signature to show W&B run in "ML Visualizations tab"
+ sig = signature(func)
+ no_default = []
+ has_default = []
+
+ for param in sig.parameters.values():
+ if param.default is param.empty:
+ no_default.append(param)
+ else:
+ has_default.append(param)
+
+ new_params = tuple(
+ (
+ *no_default,
+ Parameter(
+ "mlpipeline_ui_metadata_path",
+ annotation=OutputPath(),
+ kind=Parameter.POSITIONAL_OR_KEYWORD,
+ ),
+ *has_default,
+ )
+ )
+ new_sig = sig.replace(parameters=new_params)
+ new_anns = {param.name: param.annotation for param in new_params}
+ if "return" in func.__annotations__:
+ new_anns["return"] = func.__annotations__["return"]
+
+ def decorator(func):
+ input_scalars = {}
+ input_artifacts = {}
+ output_scalars = {}
+ output_artifacts = {}
+
+ for name, ann in func.__annotations__.items():
+ if name == "return":
+ output_scalars[name] = ann
+ elif isinstance(ann, output_types):
+ output_artifacts[name] = ann
+ elif isinstance(ann, input_types):
+ input_artifacts[name] = ann
+ else:
+ input_scalars[name] = ann
+
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ bound = new_sig.bind(*args, **kwargs)
+ bound.apply_defaults()
+
+ mlpipeline_ui_metadata_path = bound.arguments["mlpipeline_ui_metadata_path"]
+ del bound.arguments["mlpipeline_ui_metadata_path"]
+
+ with wandb.init(
+ job_type=func.__name__,
+ group="{{workflow.annotations.pipelines.kubeflow.org/run_name}}",
+ ) as run:
+ # Link back to the kfp UI
+ kubeflow_url = get_link_back_to_kubeflow()
+ run.notes = kubeflow_url
+ run.config["LINK_TO_KUBEFLOW_RUN"] = kubeflow_url
+
+ iframe_html = get_iframe_html(run)
+ metadata = {
+ "outputs": [
+ {
+ "type": "markdown",
+ "storage": "inline",
+ "source": iframe_html,
+ }
+ ]
+ }
+
+ with open(mlpipeline_ui_metadata_path, "w") as metadata_file:
+ json.dump(metadata, metadata_file)
+
+ if log_component_file:
+ _log_component_file(func, run=run)
+
+ for name, _ in input_scalars.items():
+ log_input_scalar(name, kwargs[name], run)
+
+ for name, ann in input_artifacts.items():
+ log_input_artifact(name, kwargs[name], ann.type, run)
+
+ with wb_telemetry.context(run=run) as tel:
+ tel.feature.kfp_wandb_log = True
+
+ result = func(*bound.args, **bound.kwargs)
+
+ for name, _ in output_scalars.items():
+ log_output_scalar(name, result, run)
+
+ for name, ann in output_artifacts.items():
+ log_output_artifact(name, kwargs[name], ann.type, run)
+
+ return result
+
+ wrapper.__signature__ = new_sig
+ wrapper.__annotations__ = new_anns
+ return wrapper
+
+ if func is None:
+ return decorator
+ else:
+ return decorator(func)
diff --git a/wandb/integration/langchain/__init__.py b/wandb/integration/langchain/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..aaec971a312e9369e1740f0b31e31782bb9b9e3f
--- /dev/null
+++ b/wandb/integration/langchain/__init__.py
@@ -0,0 +1,3 @@
+__all__ = ("WandbTracer",)
+
+from .wandb_tracer import WandbTracer
diff --git a/wandb/integration/langchain/wandb_tracer.py b/wandb/integration/langchain/wandb_tracer.py
new file mode 100644
index 0000000000000000000000000000000000000000..16f5c1f8181f1908d220f9696947bdf6946e4a1e
--- /dev/null
+++ b/wandb/integration/langchain/wandb_tracer.py
@@ -0,0 +1,47 @@
+"""This module contains an integration with the LangChain library.
+
+Specifically, it exposes a `WandbTracer` class that can be used to stream
+LangChain activity to W&B. The intended usage pattern is to call
+`tracer = WandbTracer()` at the top of the script/notebook, and call
+`tracer.finish()` at the end of the script/notebook.
+ This will stream all LangChain activity to W&B.
+
+Technical Note:
+LangChain is in very rapid development - meaning their APIs and schemas are actively changing.
+As a matter of precaution, any call to LangChain apis, or use of their returned data is wrapped
+in a try/except block. This is to ensure that if a breaking change is introduced, the W&B
+integration will not break user code. The one exception to the rule is at import time. If
+LangChain is not installed, or the symbols are not in the same place, the appropriate error
+will be raised when importing this module.
+"""
+from packaging import version
+
+import wandb.util
+from wandb.sdk.lib import deprecate
+
+langchain = wandb.util.get_module(
+ name="langchain",
+ required="To use the LangChain WandbTracer you need to have the `langchain` python "
+ "package installed. Please install it with `pip install langchain`.",
+)
+
+if version.parse(langchain.__version__) < version.parse("0.0.188"):
+ raise ValueError(
+ "The Weights & Biases Langchain integration does not support versions 0.0.187 and lower. "
+ "To ensure proper functionality, please use version 0.0.188 or higher."
+ )
+
+# isort: off
+from langchain.callbacks.tracers import WandbTracer # noqa: E402, I001
+
+
+class WandbTracer(WandbTracer):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ deprecate.deprecate(
+ field_name=deprecate.Deprecated.langchain_tracer,
+ warning_message="This feature is deprecated and has been moved to `langchain`. Enable tracing by setting "
+ "LANGCHAIN_WANDB_TRACING=true in your environment. See the documentation at "
+ "https://python.langchain.com/docs/ecosystem/integrations/agent_with_wandb_tracing for guidance. "
+ "Replace your current import with `from langchain.callbacks.tracers import WandbTracer`.",
+ )
diff --git a/wandb/integration/lightgbm/__init__.py b/wandb/integration/lightgbm/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..c75b77e69a7d4a247fb32c51b0166381153d7548
--- /dev/null
+++ b/wandb/integration/lightgbm/__init__.py
@@ -0,0 +1,239 @@
+"""W&B callback for lightgbm.
+
+Really simple callback to get logging for each tree
+
+Example usage:
+
+param_list = [("eta", 0.08), ("max_depth", 6), ("subsample", 0.8), ("colsample_bytree", 0.8), ("alpha", 8), ("num_class", 10)]
+config.update(dict(param_list))
+lgb = lgb.train(param_list, d_train, callbacks=[wandb_callback()])
+"""
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Callable
+
+import lightgbm # type: ignore
+from lightgbm import Booster
+
+import wandb
+from wandb.sdk.lib import telemetry as wb_telemetry
+
+MINIMIZE_METRICS = [
+ "l1",
+ "l2",
+ "rmse",
+ "mape",
+ "huber",
+ "fair",
+ "poisson",
+ "gamma",
+ "binary_logloss",
+]
+
+MAXIMIZE_METRICS = ["map", "auc", "average_precision"]
+
+
+if TYPE_CHECKING:
+ from typing import Any, Dict, List, NamedTuple, Tuple, Union
+
+ # Note: upstream lightgbm has this defined incorrectly
+ _EvalResultTuple = Union[
+ Tuple[str, str, float, bool], Tuple[str, str, float, bool, float]
+ ]
+
+ class CallbackEnv(NamedTuple):
+ model: Any
+ params: Dict
+ iteration: int
+ begin_interation: int
+ end_iteration: int
+ evaluation_result_list: List[_EvalResultTuple]
+
+
+def _define_metric(data: str, metric_name: str) -> None:
+ """Capture model performance at the best step.
+
+ instead of the last step, of training in your `wandb.summary`
+ """
+ if "loss" in str.lower(metric_name):
+ wandb.define_metric(f"{data}_{metric_name}", summary="min")
+ elif str.lower(metric_name) in MINIMIZE_METRICS:
+ wandb.define_metric(f"{data}_{metric_name}", summary="min")
+ elif str.lower(metric_name) in MAXIMIZE_METRICS:
+ wandb.define_metric(f"{data}_{metric_name}", summary="max")
+
+
+def _checkpoint_artifact(
+ model: "Booster", iteration: int, aliases: "List[str]"
+) -> None:
+ """Upload model checkpoint as W&B artifact."""
+ # NOTE: type ignore required because wandb.run is improperly inferred as None type
+ model_name = f"model_{wandb.run.id}" # type: ignore
+ model_path = Path(wandb.run.dir) / f"model_ckpt_{iteration}.txt" # type: ignore
+
+ model.save_model(model_path, num_iteration=iteration)
+
+ model_artifact = wandb.Artifact(name=model_name, type="model")
+ model_artifact.add_file(str(model_path))
+ wandb.log_artifact(model_artifact, aliases=aliases)
+
+
+def _log_feature_importance(model: "Booster") -> None:
+ """Log feature importance."""
+ feat_imps = model.feature_importance()
+ feats = model.feature_name()
+ fi_data = [[feat, feat_imp] for feat, feat_imp in zip(feats, feat_imps)]
+ table = wandb.Table(data=fi_data, columns=["Feature", "Importance"])
+ wandb.log(
+ {
+ "Feature Importance": wandb.plot.bar(
+ table, "Feature", "Importance", title="Feature Importance"
+ )
+ },
+ commit=False,
+ )
+
+
+class _WandbCallback:
+ """Internal class to handle `wandb_callback` logic.
+
+ This callback is adapted form the LightGBM's `_RecordEvaluationCallback`.
+ """
+
+ def __init__(self, log_params: bool = True, define_metric: bool = True) -> None:
+ self.order = 20
+ self.before_iteration = False
+ self.log_params = log_params
+ self.define_metric_bool = define_metric
+
+ def _init(self, env: "CallbackEnv") -> None:
+ with wb_telemetry.context() as tel:
+ tel.feature.lightgbm_wandb_callback = True
+
+ # log the params as W&B config.
+ if self.log_params:
+ wandb.config.update(env.params)
+
+ # use `define_metric` to set the wandb summary to the best metric value.
+ for item in env.evaluation_result_list:
+ if self.define_metric_bool:
+ if len(item) == 4:
+ data_name, eval_name = item[:2]
+ _define_metric(data_name, eval_name)
+ else:
+ data_name, eval_name = item[1].split()
+ _define_metric(data_name, f"{eval_name}-mean")
+ _define_metric(data_name, f"{eval_name}-stdv")
+
+ def __call__(self, env: "CallbackEnv") -> None:
+ if env.iteration == env.begin_iteration: # type: ignore
+ self._init(env)
+
+ for item in env.evaluation_result_list:
+ if len(item) == 4:
+ data_name, eval_name, result = item[:3]
+ wandb.log(
+ {data_name + "_" + eval_name: result},
+ commit=False,
+ )
+ else:
+ data_name, eval_name = item[1].split()
+ res_mean = item[2]
+ res_stdv = item[4]
+ wandb.log(
+ {
+ data_name + "_" + eval_name + "-mean": res_mean,
+ data_name + "_" + eval_name + "-stdv": res_stdv,
+ },
+ commit=False,
+ )
+
+ # call `commit=True` to log the data as a single W&B step.
+ wandb.log({"iteration": env.iteration}, commit=True)
+
+
+def wandb_callback(log_params: bool = True, define_metric: bool = True) -> Callable:
+ """Automatically integrates LightGBM with wandb.
+
+ Arguments:
+ log_params: (boolean) if True (default) logs params passed to lightgbm.train as W&B config
+ define_metric: (boolean) if True (default) capture model performance at the best step, instead of the last step, of training in your `wandb.summary`
+
+ Passing `wandb_callback` to LightGBM will:
+ - log params passed to lightgbm.train as W&B config (default).
+ - log evaluation metrics collected by LightGBM, such as rmse, accuracy etc to Weights & Biases
+ - Capture the best metric in `wandb.summary` when `define_metric=True` (default).
+
+ Use `log_summary` as an extension of this callback.
+
+ Example:
+ ```python
+ params = {
+ "boosting_type": "gbdt",
+ "objective": "regression",
+ }
+ gbm = lgb.train(
+ params,
+ lgb_train,
+ num_boost_round=10,
+ valid_sets=lgb_eval,
+ valid_names=("validation"),
+ callbacks=[wandb_callback()],
+ )
+ ```
+ """
+ return _WandbCallback(define_metric)
+
+
+def log_summary(
+ model: Booster, feature_importance: bool = True, save_model_checkpoint: bool = False
+) -> None:
+ """Log useful metrics about lightgbm model after training is done.
+
+ Arguments:
+ model: (Booster) is an instance of lightgbm.basic.Booster.
+ feature_importance: (boolean) if True (default), logs the feature importance plot.
+ save_model_checkpoint: (boolean) if True saves the best model and upload as W&B artifacts.
+
+ Using this along with `wandb_callback` will:
+
+ - log `best_iteration` and `best_score` as `wandb.summary`.
+ - log feature importance plot.
+ - save and upload your best trained model to Weights & Biases Artifacts (when `save_model_checkpoint = True`)
+
+ Example:
+ ```python
+ params = {
+ "boosting_type": "gbdt",
+ "objective": "regression",
+ }
+ gbm = lgb.train(
+ params,
+ lgb_train,
+ num_boost_round=10,
+ valid_sets=lgb_eval,
+ valid_names=("validation"),
+ callbacks=[wandb_callback()],
+ )
+
+ log_summary(gbm)
+ ```
+ """
+ if wandb.run is None:
+ raise wandb.Error("You must call wandb.init() before WandbCallback()")
+
+ if not isinstance(model, Booster):
+ raise wandb.Error("Model should be an instance of lightgbm.basic.Booster")
+
+ wandb.run.summary["best_iteration"] = model.best_iteration
+ wandb.run.summary["best_score"] = model.best_score
+
+ # Log feature importance
+ if feature_importance:
+ _log_feature_importance(model)
+
+ if save_model_checkpoint:
+ _checkpoint_artifact(model, model.best_iteration, aliases=["best"])
+
+ with wb_telemetry.context() as tel:
+ tel.feature.lightgbm_log_summary = True
diff --git a/wandb/integration/magic.py b/wandb/integration/magic.py
new file mode 100644
index 0000000000000000000000000000000000000000..0bf03dc62d44e0d01185009094b0dc75b92026de
--- /dev/null
+++ b/wandb/integration/magic.py
@@ -0,0 +1,556 @@
+import argparse
+import copy
+import json
+import os
+import re
+import sys
+
+import yaml
+
+import wandb
+from wandb import trigger
+from wandb.util import add_import_hook, get_optional_module
+
+_import_hook = None
+_run_once = False
+_args_argparse = None
+_args_system = None
+_args_absl = None
+_magic_init_seen = False
+_magic_config = {}
+
+
+class ArgumentException(Exception):
+ pass
+
+
+class SafeArgumentParser(argparse.ArgumentParser):
+ def error(self, message):
+ raise ArgumentException()
+
+
+def _merge_dicts(source, destination):
+ for key, value in source.items():
+ if isinstance(value, dict):
+ node = destination.setdefault(key, {})
+ _merge_dicts(value, node)
+ else:
+ destination[key] = value
+ return destination
+
+
+def _dict_from_keyval(k, v, json_parse=True):
+ d = ret = {}
+ keys = k.split(".")
+ for k in keys[:-1]:
+ d = d.setdefault(k, {})
+ if json_parse:
+ try:
+ v = json.loads(v.strip('"'))
+ except ValueError:
+ pass
+ d[keys[-1]] = v
+ return ret
+
+
+def _magic_get_config(k, default):
+ d = _magic_config
+ keys = k.split(".")
+ for k in keys[:-1]:
+ d = d.get(k, {})
+ return d.get(keys[-1], default)
+
+
+_magic_defaults = {
+ "enable": None,
+ #'wandb': {
+ # 'disable': None,
+ # },
+ "keras": {
+ "fit": {
+ "callbacks": {
+ "tensorboard": {
+ "enable": True,
+ "duplicate": False,
+ "overwrite": False,
+ "write_graph": None,
+ "histogram_freq": None,
+ "update_freq": None,
+ "write_grads": None,
+ "write_images": None,
+ "batch_size": None,
+ },
+ "wandb": {
+ "enable": True,
+ "duplicate": False,
+ "overwrite": False,
+ "log_gradients": None,
+ "log_weights": None,
+ "data_type": "auto",
+ "input_type": None,
+ "output_type": None,
+ "log_evaluation": None,
+ "labels": None,
+ "predictions": None,
+ "save_model": None,
+ "save_weights_only": None,
+ "monitor": None,
+ "mode": None,
+ "verbose": None,
+ },
+ "epochs": None,
+ "batch_size": None,
+ }
+ },
+ #'compile': {
+ # 'optimizer': {
+ # 'name': False,
+ # },
+ # 'loss': None,
+ # },
+ },
+ "args": {
+ "absl": None,
+ "argparse": None,
+ "sys": None,
+ },
+}
+
+
+def _parse_magic(val):
+ # attempt to treat string as a json
+ not_set = {}
+ if val is None:
+ return _magic_defaults, not_set
+ if val.startswith("{"):
+ try:
+ val = json.loads(val)
+ except ValueError:
+ wandb.termwarn("Unable to parse magic json", repeat=False)
+ return _magic_defaults, not_set
+ conf = _merge_dicts(_magic_defaults, {})
+ return _merge_dicts(val, conf), val
+ if os.path.isfile(val):
+ try:
+ with open(val) as stream:
+ val = yaml.safe_load(stream)
+ except OSError as e:
+ wandb.termwarn("Unable to read magic config file", repeat=False)
+ return _magic_defaults, not_set
+ except yaml.YAMLError as e:
+ wandb.termwarn("Unable to parse magic yaml file", repeat=False)
+ return _magic_defaults, not_set
+ conf = _merge_dicts(_magic_defaults, {})
+ return _merge_dicts(val, conf), val
+ # parse as a list of key value pairs
+ if val.find("=") > 0:
+ # split on commas but ignore commas inside quotes
+ # Using this re allows env variable parsing like:
+ # WANDB_MAGIC=key1='"["cat","dog","pizza"]"',key2=true
+ items = re.findall(r'(?:[^\s,"]|"(?:\\.|[^"])*")+', val)
+ conf_set = {}
+ for kv in items:
+ kv = kv.split("=")
+ if len(kv) != 2:
+ wandb.termwarn("Unable to parse magic key value pair", repeat=False)
+ continue
+ d = _dict_from_keyval(*kv)
+ _merge_dicts(d, conf_set)
+ conf = _merge_dicts(_magic_defaults, {})
+ return _merge_dicts(conf_set, conf), conf_set
+ wandb.termwarn("Unable to parse magic parameter", repeat=False)
+ return _magic_defaults, not_set
+
+
+def set_entity(value, env=None):
+ if env is None:
+ env = os.environ
+
+
+def _fit_wrapper(self, fn, generator=None, *args, **kwargs):
+ trigger.call("on_fit")
+ keras = sys.modules.get("keras", None)
+ tfkeras = sys.modules.get("tensorflow.python.keras", None)
+ epochs = kwargs.pop("epochs", None)
+ batch_size = kwargs.pop("batch_size", None)
+
+ magic_epochs = _magic_get_config("keras.fit.epochs", None)
+ if magic_epochs is not None:
+ epochs = magic_epochs
+ magic_batch_size = _magic_get_config("keras.fit.batch_size", None)
+ if magic_batch_size is not None:
+ batch_size = magic_batch_size
+ callbacks = kwargs.pop("callbacks", [])
+
+ tb_enabled = _magic_get_config("keras.fit.callbacks.tensorboard.enable", None)
+ if tb_enabled:
+ k = getattr(self, "_keras_or_tfkeras", None)
+ if k:
+ tb_duplicate = _magic_get_config(
+ "keras.fit.callbacks.tensorboard.duplicate", None
+ )
+ tb_overwrite = _magic_get_config(
+ "keras.fit.callbacks.tensorboard.overwrite", None
+ )
+ tb_present = any(
+ [isinstance(cb, k.callbacks.TensorBoard) for cb in callbacks]
+ )
+ if tb_present and tb_overwrite:
+ callbacks = [
+ cb
+ for cb in callbacks
+ if not isinstance(cb, k.callbacks.TensorBoard)
+ ]
+ if tb_overwrite or tb_duplicate or not tb_present:
+ tb_callback_kwargs = {"log_dir": wandb.run.dir}
+ cb_args = (
+ "write_graph",
+ "histogram_freq",
+ "update_freq",
+ "write_grads",
+ "write_images",
+ "batch_size",
+ )
+ for cb_arg in cb_args:
+ v = _magic_get_config(
+ "keras.fit.callbacks.tensorboard." + cb_arg, None
+ )
+ if v is not None:
+ tb_callback_kwargs[cb_arg] = v
+ tb_callback = k.callbacks.TensorBoard(**tb_callback_kwargs)
+ callbacks.append(tb_callback)
+
+ wandb_enabled = _magic_get_config("keras.fit.callbacks.wandb.enable", None)
+ if wandb_enabled:
+ wandb_duplicate = _magic_get_config("keras.fit.callbacks.wandb.duplicate", None)
+ wandb_overwrite = _magic_get_config("keras.fit.callbacks.wandb.overwrite", None)
+ wandb_present = any(
+ [isinstance(cb, wandb.keras.WandbCallback) for cb in callbacks]
+ )
+ if wandb_present and wandb_overwrite:
+ callbacks = [
+ cb for cb in callbacks if not isinstance(cb, wandb.keras.WandbCallback)
+ ]
+ if wandb_overwrite or wandb_duplicate or not wandb_present:
+ wandb_callback_kwargs = {}
+ log_gradients = _magic_get_config(
+ "keras.fit.callbacks.wandb.log_gradients", None
+ )
+ if log_gradients and kwargs.get("x") and kwargs.get("y"):
+ wandb_callback_kwargs["log_gradients"] = log_gradients
+ cb_args = (
+ "predictions",
+ "log_weights",
+ "data_type",
+ "save_model",
+ "save_weights_only",
+ "monitor",
+ "mode",
+ "verbose",
+ "input_type",
+ "output_type",
+ "log_evaluation",
+ "labels",
+ )
+ for cb_arg in cb_args:
+ v = _magic_get_config("keras.fit.callbacks.wandb." + cb_arg, None)
+ if v is not None:
+ wandb_callback_kwargs[cb_arg] = v
+ wandb_callback = wandb.keras.WandbCallback(**wandb_callback_kwargs)
+ callbacks.append(wandb_callback)
+
+ kwargs["callbacks"] = callbacks
+ if epochs is not None:
+ kwargs["epochs"] = epochs
+ if batch_size is not None:
+ kwargs["batch_size"] = batch_size
+ if generator:
+ return fn(generator, *args, **kwargs)
+ return fn(*args, **kwargs)
+
+
+# NOTE(jhr): need to spell out all usable args so that users who inspect can see args
+def _magic_fit(
+ self,
+ x=None,
+ y=None,
+ batch_size=None,
+ epochs=1,
+ # FIXME: there is more
+ # verbose=1,
+ # callbacks=None,
+ # validation_split=0.,
+ # validation_data=None,
+ # shuffle=True,
+ # class_weight=None,
+ # sample_weight=None,
+ # initial_epoch=0,
+ # steps_per_epoch=None,
+ # validation_steps=None,
+ # validation_freq=1,
+ # max_queue_size=10,
+ # workers=1,
+ # use_multiprocessing=False,
+ *args,
+ **kwargs,
+):
+ if hasattr(self, "_wandb_internal_model"):
+ return self._fit(
+ x=x, y=y, batch_size=batch_size, epochs=epochs, *args, **kwargs
+ )
+ return _fit_wrapper(
+ self, self._fit, x=x, y=y, batch_size=batch_size, epochs=epochs, *args, **kwargs
+ )
+
+
+def _magic_fit_generator(
+ self,
+ generator,
+ steps_per_epoch=None,
+ epochs=1,
+ # FIXME: there is more
+ # verbose=1,
+ # verbose=1,
+ # callbacks=None,
+ # validation_data=None,
+ # validation_steps=None,
+ # validation_freq=1,
+ # class_weight=None,
+ # max_queue_size=10,
+ # workers=1,
+ ##use_multiprocessing=False,
+ # shuffle=True,
+ # initial_epoch=0,
+ *args,
+ **kwargs,
+):
+ return _fit_wrapper(
+ self,
+ self._fit_generator,
+ generator=generator,
+ steps_per_epoch=steps_per_epoch,
+ epochs=epochs,
+ *args,
+ **kwargs,
+ )
+
+
+def _monkey_tfkeras():
+ from tensorflow import keras as tfkeras
+
+ from wandb.integration.keras import WandbCallback # add keras import hooks first
+
+ models = getattr(tfkeras, "models", None)
+ if not models:
+ return
+ models.Model._keras_or_tfkeras = tfkeras
+ if models.Model.fit == _magic_fit:
+ return
+ models.Model._fit = models.Model.fit
+ models.Model.fit = _magic_fit
+ models.Model._fit_generator = models.Model.fit_generator
+ models.Model.fit_generator = _magic_fit_generator
+
+
+def _monkey_absl():
+ from absl import app as absl_app
+
+ def _absl_callback():
+ absl_flags = sys.modules.get("absl.flags")
+ if not absl_flags:
+ return
+ _flags = getattr(absl_flags, "FLAGS", None)
+ if not _flags:
+ return
+ _flags_as_dict = getattr(_flags, "flag_values_dict", None)
+ if not _flags_as_dict:
+ return
+ _flags_module = getattr(_flags, "find_module_defining_flag", None)
+ if not _flags_module:
+ return
+ flags_dict = {}
+ for f, v in _flags_as_dict().items():
+ m = _flags_module(f)
+ if not m or m.startswith("absl."):
+ continue
+ flags_dict[f] = v
+ global _args_absl
+ _args_absl = flags_dict
+
+ call_after_init = getattr(absl_app, "call_after_init", None)
+ if not call_after_init:
+ return
+ call_after_init(_absl_callback)
+
+
+def _process_system_args():
+ global _args_system
+ # try using argparse
+ parser = SafeArgumentParser(add_help=False)
+ for num, arg in enumerate(sys.argv):
+ try:
+ next_arg = sys.argv[num + 1]
+ except IndexError:
+ next_arg = ""
+ if arg.startswith(("-", "--")) and not next_arg.startswith(("-", "--")):
+ try:
+ parser.add_argument(arg)
+ except ValueError:
+ pass
+ try:
+ parsed, unknown = parser.parse_known_args()
+ except ArgumentException:
+ pass
+ else:
+ _args_system = vars(parsed)
+
+
+def _monkey_argparse():
+ argparse._ArgumentParser = argparse.ArgumentParser
+
+ def _install():
+ argparse.ArgumentParser = MonitoredArgumentParser
+
+ def _uninstall():
+ argparse.ArgumentParser = argparse._ArgumentParser
+
+ def monitored(self, args, unknown=None):
+ global _args_argparse
+ _args_argparse = copy.deepcopy(vars(args))
+
+ class MonitoredArgumentParser(argparse._ArgumentParser):
+ def __init__(self, *args, **kwargs):
+ _uninstall()
+ super().__init__(*args, **kwargs)
+ _install()
+
+ def parse_args(self, *args, **kwargs):
+ args = super().parse_args(*args, **kwargs)
+ return args
+
+ def parse_known_args(self, *args, **kwargs):
+ args, unknown = super().parse_known_args(*args, **kwargs)
+ if self._callback:
+ self._callback(args, unknown=unknown)
+ return args, unknown
+
+ _install()
+ argparse.ArgumentParser._callback = monitored
+
+
+def _magic_update_config():
+ # if we already have config set, don't add anymore
+ if wandb.run and wandb.run.config:
+ c = wandb.run.config
+ user_config = dict(c.items())
+ # ignore keys set by magic integration when checking
+ # if user added any keys
+ if set(user_config).difference({"magic"}):
+ return
+ if _magic_get_config("args.absl", None) is False:
+ global _args_absl
+ _args_absl = None
+ if _magic_get_config("args.argparse", None) is False:
+ global _args_argparse
+ _args_argparse = None
+ if _magic_get_config("args.sys", None) is False:
+ global _args_system
+ _args_system = None
+ # prefer absl, then argparse values, fallback to parsed system args
+ args = _args_absl or _args_argparse or _args_system
+ if args and wandb.run and wandb.run.config:
+ wandb.run.config.update(args)
+
+
+def _magic_init(**kwargs):
+ magic_arg = kwargs.get("magic", None)
+ if magic_arg is not None and magic_arg is not False:
+ global _magic_init_seen
+ if _magic_init_seen and magic_arg is not True:
+ wandb.termwarn(
+ "wandb.init() magic argument ignored because wandb magic has already been initialized",
+ repeat=False,
+ )
+ _magic_init_seen = True
+ else:
+ wandb.termwarn(
+ "wandb.init() arguments ignored because wandb magic has already been initialized",
+ repeat=False,
+ )
+
+
+def magic_install(init_args=None):
+ if wandb.setup().settings._noop:
+ return
+ global _run_once
+ if _run_once:
+ return
+ _run_once = True
+
+ global _magic_config
+ global _import_hook
+
+ # parse config early, before we have wandb.config overrides
+ _magic_config, magic_set = _parse_magic(wandb.env.get_magic())
+
+ # we are implicitly enabling magic
+ if _magic_config.get("enable") is None:
+ _magic_config["enable"] = True
+ magic_set["enable"] = True
+
+ # allow early config to disable magic
+ if not _magic_config.get("enable"):
+ return
+
+ # process system args
+ _process_system_args()
+ # install argparse wrapper
+ in_jupyter_or_ipython = wandb.wandb_sdk.lib.ipython._get_python_type() != "python"
+ if not in_jupyter_or_ipython:
+ _monkey_argparse()
+
+ # track init calls
+ trigger.register("on_init", _magic_init)
+
+ # if wandb.init has already been called, this call is ignored
+ init_args = init_args or {}
+ init_args["magic"] = True
+ wandb.init(**init_args)
+
+ # parse magic from wandb.config (from flattened to dict)
+ magic_from_config = {}
+ MAGIC_KEY = "wandb_magic"
+ for k in wandb.config.keys():
+ if not k.startswith(MAGIC_KEY + "."):
+ continue
+ d = _dict_from_keyval(k, wandb.config[k], json_parse=False)
+ _merge_dicts(d, magic_from_config)
+ magic_from_config = magic_from_config.get(MAGIC_KEY, {})
+ _merge_dicts(magic_from_config, _magic_config)
+
+ # allow late config to disable magic
+ if not _magic_config.get("enable"):
+ return
+
+ # store magic_set into config
+ if magic_set:
+ wandb.config["magic"] = magic_set
+ wandb.config.persist()
+
+ # Monkey patch tf.keras
+ if get_optional_module("tensorflow"):
+ if "tensorflow.python.keras" in sys.modules or "keras" in sys.modules:
+ _monkey_tfkeras()
+
+ # Always setup import hooks looking for keras or tf.keras
+ add_import_hook(fullname="keras", on_import=_monkey_tfkeras)
+ add_import_hook(fullname="tensorflow.python.keras", on_import=_monkey_tfkeras)
+
+ if "absl.app" in sys.modules:
+ _monkey_absl()
+ else:
+ add_import_hook(fullname="absl.app", on_import=_monkey_absl)
+
+ # update wandb.config on fit or program finish
+ trigger.register("on_fit", _magic_update_config)
+ trigger.register("on_finished", _magic_update_config)
diff --git a/wandb/integration/metaflow/__init__.py b/wandb/integration/metaflow/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..748a71eaebae9f844d881b0842251b102f6b127d
--- /dev/null
+++ b/wandb/integration/metaflow/__init__.py
@@ -0,0 +1,3 @@
+from .metaflow import wandb_log, wandb_track, wandb_use
+
+__all__ = ["wandb_log", "wandb_track", "wandb_use"]
diff --git a/wandb/integration/metaflow/metaflow.py b/wandb/integration/metaflow/metaflow.py
new file mode 100644
index 0000000000000000000000000000000000000000..6747c7e47476d76fc4dbdd97380db81209f3a5ce
--- /dev/null
+++ b/wandb/integration/metaflow/metaflow.py
@@ -0,0 +1,348 @@
+"""W&B Integration for Metaflow.
+
+This integration lets users apply decorators to Metaflow flows and steps to automatically log parameters and artifacts to W&B by type dispatch.
+
+- Decorating a step will enable or disable logging for certain types within that step
+- Decorating the flow is equivalent to decorating all steps with a default
+- Decorating a step after decorating the flow will overwrite the flow decoration
+
+Examples can be found at wandb/wandb/functional_tests/metaflow
+"""
+
+import inspect
+import pickle
+from functools import wraps
+from pathlib import Path
+
+import wandb
+from wandb.sdk.lib import telemetry as wb_telemetry
+
+try:
+ from metaflow import current
+except ImportError as e:
+ raise Exception(
+ "Error: `metaflow` not installed >> This integration requires metaflow! To fix, please `pip install -Uqq metaflow`"
+ ) from e
+
+try:
+ from fastcore.all import typedispatch
+except ImportError as e:
+ raise Exception(
+ "Error: `fastcore` not installed >> This integration requires fastcore! To fix, please `pip install -Uqq fastcore`"
+ ) from e
+
+
+try:
+ import pandas as pd
+
+ @typedispatch # noqa: F811
+ def _wandb_use(name: str, data: pd.DataFrame, datasets=False, run=None, testing=False, *args, **kwargs): # type: ignore
+ if testing:
+ return "datasets" if datasets else None
+
+ if datasets:
+ run.use_artifact(f"{name}:latest")
+ wandb.termlog(f"Using artifact: {name} ({type(data)})")
+
+ @typedispatch # noqa: F811
+ def wandb_track(
+ name: str,
+ data: pd.DataFrame,
+ datasets=False,
+ run=None,
+ testing=False,
+ *args,
+ **kwargs,
+ ):
+ if testing:
+ return "pd.DataFrame" if datasets else None
+
+ if datasets:
+ artifact = wandb.Artifact(name, type="dataset")
+ with artifact.new_file(f"{name}.parquet", "wb") as f:
+ data.to_parquet(f, engine="pyarrow")
+ run.log_artifact(artifact)
+ wandb.termlog(f"Logging artifact: {name} ({type(data)})")
+
+except ImportError:
+ print(
+ "Warning: `pandas` not installed >> @wandb_log(datasets=True) may not auto log your dataset!"
+ )
+
+try:
+ import torch
+ import torch.nn as nn
+
+ @typedispatch # noqa: F811
+ def _wandb_use(name: str, data: nn.Module, models=False, run=None, testing=False, *args, **kwargs): # type: ignore
+ if testing:
+ return "models" if models else None
+
+ if models:
+ run.use_artifact(f"{name}:latest")
+ wandb.termlog(f"Using artifact: {name} ({type(data)})")
+
+ @typedispatch # noqa: F811
+ def wandb_track(
+ name: str,
+ data: nn.Module,
+ models=False,
+ run=None,
+ testing=False,
+ *args,
+ **kwargs,
+ ):
+ if testing:
+ return "nn.Module" if models else None
+
+ if models:
+ artifact = wandb.Artifact(name, type="model")
+ with artifact.new_file(f"{name}.pkl", "wb") as f:
+ torch.save(data, f)
+ run.log_artifact(artifact)
+ wandb.termlog(f"Logging artifact: {name} ({type(data)})")
+
+except ImportError:
+ print(
+ "Warning: `pytorch` not installed >> @wandb_log(models=True) may not auto log your model!"
+ )
+
+try:
+ from sklearn.base import BaseEstimator
+
+ @typedispatch # noqa: F811
+ def _wandb_use(name: str, data: BaseEstimator, models=False, run=None, testing=False, *args, **kwargs): # type: ignore
+ if testing:
+ return "models" if models else None
+
+ if models:
+ run.use_artifact(f"{name}:latest")
+ wandb.termlog(f"Using artifact: {name} ({type(data)})")
+
+ @typedispatch # noqa: F811
+ def wandb_track(
+ name: str,
+ data: BaseEstimator,
+ models=False,
+ run=None,
+ testing=False,
+ *args,
+ **kwargs,
+ ):
+ if testing:
+ return "BaseEstimator" if models else None
+
+ if models:
+ artifact = wandb.Artifact(name, type="model")
+ with artifact.new_file(f"{name}.pkl", "wb") as f:
+ pickle.dump(data, f)
+ run.log_artifact(artifact)
+ wandb.termlog(f"Logging artifact: {name} ({type(data)})")
+
+except ImportError:
+ print(
+ "Warning: `sklearn` not installed >> @wandb_log(models=True) may not auto log your model!"
+ )
+
+
+class ArtifactProxy:
+ def __init__(self, flow):
+ # do this to avoid recursion problem with __setattr__
+ self.__dict__.update(
+ {
+ "flow": flow,
+ "inputs": {},
+ "outputs": {},
+ "base": set(dir(flow)),
+ "params": {p: getattr(flow, p) for p in current.parameter_names},
+ }
+ )
+
+ def __setattr__(self, key, val):
+ self.outputs[key] = val
+ return setattr(self.flow, key, val)
+
+ def __getattr__(self, key):
+ if key not in self.base and key not in self.outputs:
+ self.inputs[key] = getattr(self.flow, key)
+ return getattr(self.flow, key)
+
+
+@typedispatch # noqa: F811
+def wandb_track(name: str, data: (dict, list, set, str, int, float, bool), run=None, testing=False, *args, **kwargs): # type: ignore
+ if testing:
+ return "scalar"
+
+ run.log({name: data})
+
+
+@typedispatch # noqa: F811
+def wandb_track(
+ name: str, data: Path, datasets=False, run=None, testing=False, *args, **kwargs
+):
+ if testing:
+ return "Path" if datasets else None
+
+ if datasets:
+ artifact = wandb.Artifact(name, type="dataset")
+ if data.is_dir():
+ artifact.add_dir(data)
+ elif data.is_file():
+ artifact.add_file(data)
+ run.log_artifact(artifact)
+ wandb.termlog(f"Logging artifact: {name} ({type(data)})")
+
+
+# this is the base case
+@typedispatch # noqa: F811
+def wandb_track(
+ name: str, data, others=False, run=None, testing=False, *args, **kwargs
+):
+ if testing:
+ return "generic" if others else None
+
+ if others:
+ artifact = wandb.Artifact(name, type="other")
+ with artifact.new_file(f"{name}.pkl", "wb") as f:
+ pickle.dump(data, f)
+ run.log_artifact(artifact)
+ wandb.termlog(f"Logging artifact: {name} ({type(data)})")
+
+
+@typedispatch
+def wandb_use(name: str, data, *args, **kwargs):
+ try:
+ return _wandb_use(name, data, *args, **kwargs)
+ except wandb.CommError:
+ print(
+ f"This artifact ({name}, {type(data)}) does not exist in the wandb datastore!"
+ f"If you created an instance inline (e.g. sklearn.ensemble.RandomForestClassifier), then you can safely ignore this"
+ f"Otherwise you may want to check your internet connection!"
+ )
+
+
+@typedispatch # noqa: F811
+def wandb_use(name: str, data: (dict, list, set, str, int, float, bool), *args, **kwargs): # type: ignore
+ pass # do nothing for these types
+
+
+@typedispatch # noqa: F811
+def _wandb_use(name: str, data: Path, datasets=False, run=None, testing=False, *args, **kwargs): # type: ignore
+ if testing:
+ return "datasets" if datasets else None
+
+ if datasets:
+ run.use_artifact(f"{name}:latest")
+ wandb.termlog(f"Using artifact: {name} ({type(data)})")
+
+
+@typedispatch # noqa: F811
+def _wandb_use(name: str, data, others=False, run=None, testing=False, *args, **kwargs): # type: ignore
+ if testing:
+ return "others" if others else None
+
+ if others:
+ run.use_artifact(f"{name}:latest")
+ wandb.termlog(f"Using artifact: {name} ({type(data)})")
+
+
+def coalesce(*arg):
+ return next((a for a in arg if a is not None), None)
+
+
+def wandb_log(
+ func=None,
+ # /, # py38 only
+ datasets=False,
+ models=False,
+ others=False,
+ settings=None,
+):
+ """Automatically log parameters and artifacts to W&B by type dispatch.
+
+ This decorator can be applied to a flow, step, or both.
+ - Decorating a step will enable or disable logging for certain types within that step
+ - Decorating the flow is equivalent to decorating all steps with a default
+ - Decorating a step after decorating the flow will overwrite the flow decoration
+
+ Arguments:
+ func: (`Callable`). The method or class being decorated (if decorating a step or flow respectively).
+ datasets: (`bool`). If `True`, log datasets. Datasets can be a `pd.DataFrame` or `pathlib.Path`. The default value is `False`, so datasets are not logged.
+ models: (`bool`). If `True`, log models. Models can be a `nn.Module` or `sklearn.base.BaseEstimator`. The default value is `False`, so models are not logged.
+ others: (`bool`). If `True`, log anything pickle-able. The default value is `False`, so files are not logged.
+ settings: (`wandb.sdk.wandb_settings.Settings`). Custom settings passed to `wandb.init`. The default value is `None`, and is the same as passing `wandb.Settings()`. If `settings.run_group` is `None`, it will be set to `{flow_name}/{run_id}. If `settings.run_job_type` is `None`, it will be set to `{run_job_type}/{step_name}`
+ """
+
+ @wraps(func)
+ def decorator(func):
+ # If you decorate a class, apply the decoration to all methods in that class
+ if inspect.isclass(func):
+ cls = func
+ for attr in cls.__dict__:
+ if callable(getattr(cls, attr)):
+ if not hasattr(attr, "_base_func"):
+ setattr(cls, attr, decorator(getattr(cls, attr)))
+ return cls
+
+ # prefer the earliest decoration (i.e. method decoration overrides class decoration)
+ if hasattr(func, "_base_func"):
+ return func
+
+ @wraps(func)
+ def wrapper(self, *args, settings=settings, **kwargs):
+ if not isinstance(settings, wandb.sdk.wandb_settings.Settings):
+ settings = wandb.Settings()
+
+ settings.update(
+ run_group=coalesce(
+ settings.run_group, f"{current.flow_name}/{current.run_id}"
+ ),
+ source=wandb.sdk.wandb_settings.Source.INIT,
+ )
+ settings.update(
+ run_job_type=coalesce(settings.run_job_type, current.step_name),
+ source=wandb.sdk.wandb_settings.Source.INIT,
+ )
+
+ with wandb.init(settings=settings) as run:
+ with wb_telemetry.context(run=run) as tel:
+ tel.feature.metaflow = True
+ proxy = ArtifactProxy(self)
+ run.config.update(proxy.params)
+ func(proxy, *args, **kwargs)
+
+ for name, data in proxy.inputs.items():
+ wandb_use(
+ name,
+ data,
+ datasets=datasets,
+ models=models,
+ others=others,
+ run=run,
+ )
+
+ for name, data in proxy.outputs.items():
+ wandb_track(
+ name,
+ data,
+ datasets=datasets,
+ models=models,
+ others=others,
+ run=run,
+ )
+
+ wrapper._base_func = func
+
+ # Add for testing visibility
+ wrapper._kwargs = {
+ "datasets": datasets,
+ "models": models,
+ "others": others,
+ "settings": settings,
+ }
+ return wrapper
+
+ if func is None:
+ return decorator
+ else:
+ return decorator(func)
diff --git a/wandb/integration/openai/__init__.py b/wandb/integration/openai/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..f2c216c5f72a7d0310b5cfab98b4c6ff5e0e75e6
--- /dev/null
+++ b/wandb/integration/openai/__init__.py
@@ -0,0 +1,3 @@
+__all__ = ("autolog", "WandbLogger")
+
+from .openai import autolog
diff --git a/wandb/integration/openai/fine_tuning.py b/wandb/integration/openai/fine_tuning.py
new file mode 100644
index 0000000000000000000000000000000000000000..bc2da320f25b59cf41dd982559f44578192d12c6
--- /dev/null
+++ b/wandb/integration/openai/fine_tuning.py
@@ -0,0 +1,418 @@
+import datetime
+import io
+import json
+import re
+import time
+from typing import Any, Dict, Optional, Tuple
+
+from pkg_resources import parse_version
+
+import wandb
+from wandb import util
+from wandb.data_types import Table
+from wandb.sdk.lib import telemetry
+from wandb.sdk.wandb_run import Run
+
+openai = util.get_module(
+ name="openai",
+ required="`openai` not installed. This integration requires `openai`. To fix, please `pip install openai`",
+ lazy="False",
+)
+
+if parse_version(openai.__version__) < parse_version("1.0.1"):
+ raise wandb.Error(
+ f"This integration requires openai version 1.0.1 and above. Your current version is {openai.__version__} "
+ "To fix, please `pip install -U openai`"
+ )
+
+from openai import OpenAI # noqa: E402
+from openai.types.fine_tuning import FineTuningJob # noqa: E402
+from openai.types.fine_tuning.fine_tuning_job import Hyperparameters # noqa: E402
+
+np = util.get_module(
+ name="numpy",
+ required="`numpy` not installed >> This integration requires numpy! To fix, please `pip install numpy`",
+ lazy="False",
+)
+
+pd = util.get_module(
+ name="pandas",
+ required="`pandas` not installed >> This integration requires pandas! To fix, please `pip install pandas`",
+ lazy="False",
+)
+
+
+class WandbLogger:
+ """Log OpenAI fine-tunes to [Weights & Biases](https://wandb.me/openai-docs)."""
+
+ _wandb_api: wandb.Api = None
+ _logged_in: bool = False
+ openai_client: OpenAI = None
+ _run: Run = None
+
+ @classmethod
+ def sync(
+ cls,
+ fine_tune_job_id: Optional[str] = None,
+ openai_client: Optional[OpenAI] = None,
+ num_fine_tunes: Optional[int] = None,
+ project: str = "OpenAI-Fine-Tune",
+ entity: Optional[str] = None,
+ overwrite: bool = False,
+ wait_for_job_success: bool = True,
+ **kwargs_wandb_init: Dict[str, Any],
+ ) -> str:
+ """Sync fine-tunes to Weights & Biases.
+
+ :param fine_tune_job_id: The id of the fine-tune (optional)
+ :param openai_client: Pass the `OpenAI()` client (optional)
+ :param num_fine_tunes: Number of most recent fine-tunes to log when an fine_tune_job_id is not provided. By default, every fine-tune is synced.
+ :param project: Name of the project where you're sending runs. By default, it is "GPT-3".
+ :param entity: Username or team name where you're sending runs. By default, your default entity is used, which is usually your username.
+ :param overwrite: Forces logging and overwrite existing wandb run of the same fine-tune.
+ :param wait_for_job_success: Waits for the fine-tune to be complete and then log metrics to W&B. By default, it is True.
+ """
+ if openai_client is None:
+ openai_client = OpenAI()
+ cls.openai_client = openai_client
+
+ if fine_tune_job_id:
+ wandb.termlog("Retrieving fine-tune job...")
+ fine_tune = openai_client.fine_tuning.jobs.retrieve(
+ fine_tuning_job_id=fine_tune_job_id
+ )
+ fine_tunes = [fine_tune]
+ else:
+ # get list of fine_tune to log
+ fine_tunes = openai_client.fine_tuning.jobs.list()
+ if not fine_tunes or fine_tunes.data is None:
+ wandb.termwarn("No fine-tune has been retrieved")
+ return
+ # Select the `num_fine_tunes` from the `fine_tunes.data` list.
+ # If `num_fine_tunes` is None, it selects all items in the list (from start to end).
+ # If for example, `num_fine_tunes` is 5, it selects the last 5 items in the list.
+ # Note that the last items in the list are the latest fine-tune jobs.
+ fine_tunes = fine_tunes.data[
+ -num_fine_tunes if num_fine_tunes is not None else None :
+ ]
+
+ # log starting from oldest fine_tune
+ show_individual_warnings = (
+ fine_tune_job_id is not None or num_fine_tunes is not None
+ )
+ fine_tune_logged = []
+ for fine_tune in fine_tunes:
+ fine_tune_id = fine_tune.id
+ # check run with the given `fine_tune_id` has not been logged already
+ run_path = f"{project}/{fine_tune_id}"
+ if entity is not None:
+ run_path = f"{entity}/{run_path}"
+ wandb_run = cls._get_wandb_run(run_path)
+ if wandb_run:
+ wandb_status = wandb_run.summary.get("status")
+ if show_individual_warnings:
+ if wandb_status == "succeeded" and not overwrite:
+ wandb.termwarn(
+ f"Fine-tune {fine_tune_id} has already been logged successfully at {wandb_run.url}. "
+ "Use `overwrite=True` if you want to overwrite previous run"
+ )
+ elif wandb_status != "succeeded" or overwrite:
+ if wandb_status != "succeeded":
+ wandb.termwarn(
+ f"A run for fine-tune {fine_tune_id} was previously created but didn't end successfully"
+ )
+ wandb.termlog(
+ f"A new wandb run will be created for fine-tune {fine_tune_id} and previous run will be overwritten"
+ )
+ overwrite = True
+ if wandb_status == "succeeded" and not overwrite:
+ return
+
+ # check if the user has not created a wandb run externally
+ if wandb.run is None:
+ cls._run = wandb.init(
+ job_type="fine-tune",
+ project=project,
+ entity=entity,
+ name=fine_tune_id,
+ id=fine_tune_id,
+ **kwargs_wandb_init,
+ )
+ else:
+ # if a run exits - created externally
+ cls._run = wandb.run
+
+ if wait_for_job_success:
+ fine_tune = cls._wait_for_job_success(fine_tune)
+
+ cls._log_fine_tune(
+ fine_tune,
+ project,
+ entity,
+ overwrite,
+ show_individual_warnings,
+ **kwargs_wandb_init,
+ )
+
+ if not show_individual_warnings and not any(fine_tune_logged):
+ wandb.termwarn("No new successful fine-tunes were found")
+
+ return "🎉 wandb sync completed successfully"
+
+ @classmethod
+ def _wait_for_job_success(cls, fine_tune: FineTuningJob) -> FineTuningJob:
+ wandb.termlog("Waiting for the OpenAI fine-tuning job to be finished...")
+ while True:
+ if fine_tune.status == "succeeded":
+ wandb.termlog(
+ "Fine-tuning finished, logging metrics, model metadata, and more to W&B"
+ )
+ return fine_tune
+ if fine_tune.status == "failed":
+ wandb.termwarn(
+ f"Fine-tune {fine_tune.id} has failed and will not be logged"
+ )
+ return fine_tune
+ if fine_tune.status == "cancelled":
+ wandb.termwarn(
+ f"Fine-tune {fine_tune.id} was cancelled and will not be logged"
+ )
+ return fine_tune
+ time.sleep(10)
+ fine_tune = cls.openai_client.fine_tuning.jobs.retrieve(
+ fine_tuning_job_id=fine_tune.id
+ )
+
+ @classmethod
+ def _log_fine_tune(
+ cls,
+ fine_tune: FineTuningJob,
+ project: str,
+ entity: Optional[str],
+ overwrite: bool,
+ show_individual_warnings: bool,
+ **kwargs_wandb_init: Dict[str, Any],
+ ):
+ fine_tune_id = fine_tune.id
+ status = fine_tune.status
+
+ with telemetry.context(run=cls._run) as tel:
+ tel.feature.openai_finetuning = True
+
+ # check run completed successfully
+ if status != "succeeded":
+ if show_individual_warnings:
+ wandb.termwarn(
+ f'Fine-tune {fine_tune_id} has the status "{status}" and will not be logged'
+ )
+ return
+
+ # check results are present
+ try:
+ results_id = fine_tune.result_files[0]
+ results = cls.openai_client.files.retrieve_content(file_id=results_id)
+ except openai.NotFoundError:
+ if show_individual_warnings:
+ wandb.termwarn(
+ f"Fine-tune {fine_tune_id} has no results and will not be logged"
+ )
+ return
+
+ # update the config
+ cls._run.config.update(cls._get_config(fine_tune))
+
+ # log results
+ df_results = pd.read_csv(io.StringIO(results))
+ for _, row in df_results.iterrows():
+ metrics = {k: v for k, v in row.items() if not np.isnan(v)}
+ step = metrics.pop("step")
+ if step is not None:
+ step = int(step)
+ cls._run.log(metrics, step=step)
+ fine_tuned_model = fine_tune.fine_tuned_model
+ if fine_tuned_model is not None:
+ cls._run.summary["fine_tuned_model"] = fine_tuned_model
+
+ # training/validation files and fine-tune details
+ cls._log_artifacts(fine_tune, project, entity)
+
+ # mark run as complete
+ cls._run.summary["status"] = "succeeded"
+
+ cls._run.finish()
+ return True
+
+ @classmethod
+ def _ensure_logged_in(cls):
+ if not cls._logged_in:
+ if wandb.login():
+ cls._logged_in = True
+ else:
+ raise Exception(
+ "It appears you are not currently logged in to Weights & Biases. "
+ "Please run `wandb login` in your terminal. "
+ "When prompted, you can obtain your API key by visiting wandb.ai/authorize."
+ )
+
+ @classmethod
+ def _get_wandb_run(cls, run_path: str):
+ cls._ensure_logged_in()
+ try:
+ if cls._wandb_api is None:
+ cls._wandb_api = wandb.Api()
+ return cls._wandb_api.run(run_path)
+ except Exception:
+ return None
+
+ @classmethod
+ def _get_wandb_artifact(cls, artifact_path: str):
+ cls._ensure_logged_in()
+ try:
+ if cls._wandb_api is None:
+ cls._wandb_api = wandb.Api()
+ return cls._wandb_api.artifact(artifact_path)
+ except Exception:
+ return None
+
+ @classmethod
+ def _get_config(cls, fine_tune: FineTuningJob) -> Dict[str, Any]:
+ config = dict(fine_tune)
+ config["result_files"] = config["result_files"][0]
+ if config.get("created_at"):
+ config["created_at"] = datetime.datetime.fromtimestamp(
+ config["created_at"]
+ ).strftime("%Y-%m-%d %H:%M:%S")
+ if config.get("finished_at"):
+ config["finished_at"] = datetime.datetime.fromtimestamp(
+ config["finished_at"]
+ ).strftime("%Y-%m-%d %H:%M:%S")
+ if config.get("hyperparameters"):
+ hyperparameters = config.pop("hyperparameters")
+ hyperparams = cls._unpack_hyperparameters(hyperparameters)
+ if hyperparams is None:
+ # If unpacking fails, log the object which will render as string
+ config["hyperparameters"] = hyperparameters
+ else:
+ # nested rendering on hyperparameters
+ config["hyperparameters"] = hyperparams
+
+ return config
+
+ @classmethod
+ def _unpack_hyperparameters(cls, hyperparameters: Hyperparameters):
+ # `Hyperparameters` object is not unpacking properly using `vars` or `__dict__`,
+ # vars(hyperparameters) return {n_epochs: n} only.
+ hyperparams = {}
+ try:
+ hyperparams["n_epochs"] = hyperparameters.n_epochs
+ hyperparams["batch_size"] = hyperparameters.batch_size
+ hyperparams[
+ "learning_rate_multiplier"
+ ] = hyperparameters.learning_rate_multiplier
+ except Exception:
+ # If unpacking fails, return the object to be logged as config
+ return None
+
+ return hyperparams
+
+ @classmethod
+ def _log_artifacts(
+ cls, fine_tune: FineTuningJob, project: str, entity: Optional[str]
+ ) -> None:
+ # training/validation files
+ training_file = fine_tune.training_file if fine_tune.training_file else None
+ validation_file = (
+ fine_tune.validation_file if fine_tune.validation_file else None
+ )
+ for file, prefix, artifact_type in (
+ (training_file, "train", "training_files"),
+ (validation_file, "valid", "validation_files"),
+ ):
+ if file is not None:
+ cls._log_artifact_inputs(file, prefix, artifact_type, project, entity)
+
+ # fine-tune details
+ fine_tune_id = fine_tune.id
+ artifact = wandb.Artifact(
+ "model_metadata",
+ type="model",
+ metadata=dict(fine_tune),
+ )
+ with artifact.new_file("model_metadata.json", mode="w", encoding="utf-8") as f:
+ dict_fine_tune = dict(fine_tune)
+ dict_fine_tune["hyperparameters"] = dict(dict_fine_tune["hyperparameters"])
+ json.dump(dict_fine_tune, f, indent=2)
+ cls._run.log_artifact(
+ artifact,
+ aliases=["latest", fine_tune_id],
+ )
+
+ @classmethod
+ def _log_artifact_inputs(
+ cls,
+ file_id: Optional[str],
+ prefix: str,
+ artifact_type: str,
+ project: str,
+ entity: Optional[str],
+ ) -> None:
+ # get input artifact
+ artifact_name = f"{prefix}-{file_id}"
+ # sanitize name to valid wandb artifact name
+ artifact_name = re.sub(r"[^a-zA-Z0-9_\-.]", "_", artifact_name)
+ artifact_alias = file_id
+ artifact_path = f"{project}/{artifact_name}:{artifact_alias}"
+ if entity is not None:
+ artifact_path = f"{entity}/{artifact_path}"
+ artifact = cls._get_wandb_artifact(artifact_path)
+
+ # create artifact if file not already logged previously
+ if artifact is None:
+ # get file content
+ try:
+ file_content = cls.openai_client.files.retrieve_content(file_id=file_id)
+ except openai.NotFoundError:
+ wandb.termerror(
+ f"File {file_id} could not be retrieved. Make sure you are allowed to download training/validation files"
+ )
+ return
+
+ artifact = wandb.Artifact(artifact_name, type=artifact_type)
+ with artifact.new_file(file_id, mode="w", encoding="utf-8") as f:
+ f.write(file_content)
+
+ # create a Table
+ try:
+ table, n_items = cls._make_table(file_content)
+ # Add table to the artifact.
+ artifact.add(table, file_id)
+ # Add the same table to the workspace.
+ cls._run.log({f"{prefix}_data": table})
+ # Update the run config and artifact metadata
+ cls._run.config.update({f"n_{prefix}": n_items})
+ artifact.metadata["items"] = n_items
+ except Exception:
+ wandb.termerror(
+ f"File {file_id} could not be read as a valid JSON file"
+ )
+ else:
+ # log number of items
+ cls._run.config.update({f"n_{prefix}": artifact.metadata.get("items")})
+
+ cls._run.use_artifact(artifact, aliases=["latest", artifact_alias])
+
+ @classmethod
+ def _make_table(cls, file_content: str) -> Tuple[Table, int]:
+ table = wandb.Table(columns=["role: system", "role: user", "role: assistant"])
+
+ df = pd.read_json(io.StringIO(file_content), orient="records", lines=True)
+ for _idx, message in df.iterrows():
+ messages = message.messages
+ assert len(messages) == 3
+ table.add_data(
+ messages[0]["content"],
+ messages[1]["content"],
+ messages[2]["content"],
+ )
+
+ return table, len(df)
diff --git a/wandb/integration/openai/openai.py b/wandb/integration/openai/openai.py
new file mode 100644
index 0000000000000000000000000000000000000000..250d437d7239bbcd31cc950c80563519f87e5fde
--- /dev/null
+++ b/wandb/integration/openai/openai.py
@@ -0,0 +1,22 @@
+import logging
+
+from wandb.sdk.integration_utils.auto_logging import AutologAPI
+
+from .resolver import OpenAIRequestResponseResolver
+
+logger = logging.getLogger(__name__)
+
+
+autolog = AutologAPI(
+ name="OpenAI",
+ symbols=(
+ "Edit.create",
+ "Completion.create",
+ "ChatCompletion.create",
+ "Edit.acreate",
+ "Completion.acreate",
+ "ChatCompletion.acreate",
+ ),
+ resolver=OpenAIRequestResponseResolver(),
+ telemetry_feature="openai_autolog",
+)
diff --git a/wandb/integration/openai/resolver.py b/wandb/integration/openai/resolver.py
new file mode 100644
index 0000000000000000000000000000000000000000..500c58ce2f4b33a6a987243169bc998cf387fef4
--- /dev/null
+++ b/wandb/integration/openai/resolver.py
@@ -0,0 +1,240 @@
+import datetime
+import io
+import logging
+from dataclasses import asdict, dataclass
+from typing import Any, Dict, List, Optional, Sequence
+
+import wandb
+from wandb.sdk.data_types import trace_tree
+from wandb.sdk.integration_utils.auto_logging import Response
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class UsageMetrics:
+ elapsed_time: float = None
+ prompt_tokens: int = None
+ completion_tokens: int = None
+ total_tokens: int = None
+
+
+@dataclass
+class Metrics:
+ usage: UsageMetrics = None
+ stats: wandb.Table = None
+ trace: trace_tree.WBTraceTree = None
+
+
+usage_metric_keys = {f"usage/{k}" for k in asdict(UsageMetrics())}
+
+
+class OpenAIRequestResponseResolver:
+ def __init__(self):
+ self.define_metrics_called = False
+
+ def __call__(
+ self,
+ args: Sequence[Any],
+ kwargs: Dict[str, Any],
+ response: Response,
+ start_time: float, # pass to comply with the protocol, but use response["created"] instead
+ time_elapsed: float,
+ ) -> Optional[Dict[str, Any]]:
+ request = kwargs
+
+ if not self.define_metrics_called:
+ # define metrics on first call
+ for key in usage_metric_keys:
+ wandb.define_metric(key, step_metric="_timestamp")
+ self.define_metrics_called = True
+
+ try:
+ if response.get("object") == "edit":
+ return self._resolve_edit(request, response, time_elapsed)
+ elif response.get("object") == "text_completion":
+ return self._resolve_completion(request, response, time_elapsed)
+ elif response.get("object") == "chat.completion":
+ return self._resolve_chat_completion(request, response, time_elapsed)
+ else:
+ # todo: properly treat failed requests
+ logger.info(
+ f"Unsupported OpenAI response object: {response.get('object')}"
+ )
+ except Exception as e:
+ logger.warning(f"Failed to resolve request/response: {e}")
+ return None
+
+ @staticmethod
+ def results_to_trace_tree(
+ request: Dict[str, Any],
+ response: Response,
+ results: List[trace_tree.Result],
+ time_elapsed: float,
+ ) -> trace_tree.WBTraceTree:
+ """Converts the request, response, and results into a trace tree.
+
+ params:
+ request: The request dictionary
+ response: The response object
+ results: A list of results object
+ time_elapsed: The time elapsed in seconds
+ returns:
+ A wandb trace tree object.
+ """
+ start_time_ms = int(round(response["created"] * 1000))
+ end_time_ms = start_time_ms + int(round(time_elapsed * 1000))
+ span = trace_tree.Span(
+ name=f"{response.get('model', 'openai')}_{response['object']}_{response.get('created')}",
+ attributes=dict(response), # type: ignore
+ start_time_ms=start_time_ms,
+ end_time_ms=end_time_ms,
+ span_kind=trace_tree.SpanKind.LLM,
+ results=results,
+ )
+ model_obj = {"request": request, "response": response, "_kind": "openai"}
+ return trace_tree.WBTraceTree(root_span=span, model_dict=model_obj)
+
+ def _resolve_edit(
+ self,
+ request: Dict[str, Any],
+ response: Response,
+ time_elapsed: float,
+ ) -> Dict[str, Any]:
+ """Resolves the request and response objects for `openai.Edit`."""
+ request_str = (
+ f"\n\n**Instruction**: {request['instruction']}\n\n"
+ f"**Input**: {request['input']}\n"
+ )
+ choices = [
+ f"\n\n**Edited**: {choice['text']}\n" for choice in response["choices"]
+ ]
+
+ return self._resolve_metrics(
+ request=request,
+ response=response,
+ request_str=request_str,
+ choices=choices,
+ time_elapsed=time_elapsed,
+ )
+
+ def _resolve_completion(
+ self,
+ request: Dict[str, Any],
+ response: Response,
+ time_elapsed: float,
+ ) -> Dict[str, Any]:
+ """Resolves the request and response objects for `openai.Completion`."""
+ request_str = f"\n\n**Prompt**: {request['prompt']}\n"
+ choices = [
+ f"\n\n**Completion**: {choice['text']}\n" for choice in response["choices"]
+ ]
+
+ return self._resolve_metrics(
+ request=request,
+ response=response,
+ request_str=request_str,
+ choices=choices,
+ time_elapsed=time_elapsed,
+ )
+
+ def _resolve_chat_completion(
+ self,
+ request: Dict[str, Any],
+ response: Response,
+ time_elapsed: float,
+ ) -> Dict[str, Any]:
+ """Resolves the request and response objects for `openai.Completion`."""
+ prompt = io.StringIO()
+ for message in request["messages"]:
+ prompt.write(f"\n\n**{message['role']}**: {message['content']}\n")
+ request_str = prompt.getvalue()
+
+ choices = [
+ f"\n\n**{choice['message']['role']}**: {choice['message']['content']}\n"
+ for choice in response["choices"]
+ ]
+
+ return self._resolve_metrics(
+ request=request,
+ response=response,
+ request_str=request_str,
+ choices=choices,
+ time_elapsed=time_elapsed,
+ )
+
+ def _resolve_metrics(
+ self,
+ request: Dict[str, Any],
+ response: Response,
+ request_str: str,
+ choices: List[str],
+ time_elapsed: float,
+ ) -> Dict[str, Any]:
+ """Resolves the request and response objects for `openai.Completion`."""
+ results = [
+ trace_tree.Result(
+ inputs={"request": request_str},
+ outputs={"response": choice},
+ )
+ for choice in choices
+ ]
+ metrics = self._get_metrics_to_log(request, response, results, time_elapsed)
+ return self._convert_metrics_to_dict(metrics)
+
+ @staticmethod
+ def _get_usage_metrics(response: Response, time_elapsed: float) -> UsageMetrics:
+ """Gets the usage stats from the response object."""
+ if response.get("usage"):
+ usage_stats = UsageMetrics(**response["usage"])
+ else:
+ usage_stats = UsageMetrics()
+ usage_stats.elapsed_time = time_elapsed
+ return usage_stats
+
+ def _get_metrics_to_log(
+ self,
+ request: Dict[str, Any],
+ response: Response,
+ results: List[Any],
+ time_elapsed: float,
+ ) -> Metrics:
+ model = response.get("model") or request.get("model")
+ usage_metrics = self._get_usage_metrics(response, time_elapsed)
+
+ usage = []
+ for result in results:
+ row = {
+ "request": result.inputs["request"],
+ "response": result.outputs["response"],
+ "model": model,
+ "start_time": datetime.datetime.fromtimestamp(response["created"]),
+ "end_time": datetime.datetime.fromtimestamp(
+ response["created"] + time_elapsed
+ ),
+ "request_id": response.get("id", None),
+ "api_type": response.get("api_type", "openai"),
+ "session_id": wandb.run.id,
+ }
+ row.update(asdict(usage_metrics))
+ usage.append(row)
+ usage_table = wandb.Table(
+ columns=list(usage[0].keys()),
+ data=[(item.values()) for item in usage],
+ )
+
+ trace = self.results_to_trace_tree(request, response, results, time_elapsed)
+
+ metrics = Metrics(stats=usage_table, trace=trace, usage=usage_metrics)
+ return metrics
+
+ @staticmethod
+ def _convert_metrics_to_dict(metrics: Metrics) -> Dict[str, Any]:
+ """Converts metrics to a dict."""
+ metrics_dict = {
+ "stats": metrics.stats,
+ "trace": metrics.trace,
+ }
+ usage_stats = {f"usage/{k}": v for k, v in asdict(metrics.usage).items()}
+ metrics_dict.update(usage_stats)
+ return metrics_dict
diff --git a/wandb/integration/prodigy/__init__.py b/wandb/integration/prodigy/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..94ed0ea26bc5f886cf90db26da37dc8f08f9b18e
--- /dev/null
+++ b/wandb/integration/prodigy/__init__.py
@@ -0,0 +1,3 @@
+from .prodigy import upload_dataset
+
+__all__ = ["upload_dataset"]
diff --git a/wandb/integration/prodigy/prodigy.py b/wandb/integration/prodigy/prodigy.py
new file mode 100644
index 0000000000000000000000000000000000000000..457c3b8c3bbc28c2c5ae005024ea3848ba92cbe5
--- /dev/null
+++ b/wandb/integration/prodigy/prodigy.py
@@ -0,0 +1,299 @@
+"""Prodigy integration for W&B.
+
+User can upload Prodigy annotated datasets directly
+from the local database to W&B in Tables format.
+
+Example usage:
+
+```python
+import wandb
+from wandb.integration.prodigy import upload_dataset
+
+run = wandb.init(project="prodigy")
+upload_dataset("name_of_dataset")
+wandb.finish()
+```
+"""
+
+import base64
+import collections.abc
+import io
+import urllib
+from copy import deepcopy
+
+import pandas as pd
+from PIL import Image
+
+import wandb
+from wandb import util
+from wandb.plots.utils import test_missing
+from wandb.sdk.lib import telemetry as wb_telemetry
+
+
+def named_entity(docs):
+ """Create a named entity visualization.
+
+ Taken from https://github.com/wandb/wandb/blob/main/wandb/plots/named_entity.py.
+ """
+ spacy = util.get_module(
+ "spacy",
+ required="part_of_speech requires the spacy library, install with `pip install spacy`",
+ )
+
+ util.get_module(
+ "en_core_web_md",
+ required="part_of_speech requires `en_core_web_md` library, install with `python -m spacy download en_core_web_md`",
+ )
+
+ # Test for required packages and missing & non-integer values in docs data
+ if test_missing(docs=docs):
+ html = spacy.displacy.render(
+ docs, style="ent", page=True, minify=True, jupyter=False
+ )
+ wandb_html = wandb.Html(html)
+ return wandb_html
+
+
+def merge(dict1, dict2):
+ """Return a new dictionary by merging two dictionaries recursively."""
+ result = deepcopy(dict1)
+
+ for key, value in dict2.items():
+ if isinstance(value, collections.abc.Mapping):
+ result[key] = merge(result.get(key, {}), value)
+ else:
+ result[key] = deepcopy(dict2[key])
+
+ return result
+
+
+def get_schema(list_data_dict, struct, array_dict_types):
+ """Get a schema of the dataset's structure and data types."""
+ # Get the structure of the JSON objects in the database
+ # This is similar to getting a JSON schema but with slightly different format
+ for _i, item in enumerate(list_data_dict):
+ # If the list contains dict objects
+ for k, v in item.items():
+ # Check if key already exists in template
+ if k not in struct.keys():
+ if isinstance(v, list):
+ if len(v) > 0 and isinstance(v[0], list):
+ # nested list structure
+ struct[k] = type(v) # type list
+ elif len(v) > 0 and not (
+ isinstance(v[0], list) or isinstance(v[0], dict)
+ ):
+ # list of singular values
+ struct[k] = type(v) # type list
+ else:
+ # list of dicts
+ array_dict_types.append(
+ k
+ ) # keep track of keys that are type list[dict]
+ struct[k] = {}
+ struct[k] = get_schema(v, struct[k], array_dict_types)
+ elif isinstance(v, dict):
+ struct[k] = {}
+ struct[k] = get_schema([v], struct[k], array_dict_types)
+ else:
+ struct[k] = type(v)
+ else:
+ # Get the value of struct[k] which is the current template
+ # Find new keys and then merge the two templates together
+ cur_struct = struct[k]
+ if isinstance(v, list):
+ if len(v) > 0 and isinstance(v[0], list):
+ # nested list coordinate structure
+ # if the value in the item is currently None, then update
+ if v is not None:
+ struct[k] = type(v) # type list
+ elif len(v) > 0 and not (
+ isinstance(v[0], list) or isinstance(v[0], dict)
+ ):
+ # single list with values
+ # if the value in the item is currently None, then update
+ if v is not None:
+ struct[k] = type(v) # type list
+ else:
+ array_dict_types.append(
+ k
+ ) # keep track of keys that are type list[dict]
+ struct[k] = {}
+ struct[k] = get_schema(v, struct[k], array_dict_types)
+ # merge cur_struct and struct[k], remove duplicates
+ struct[k] = merge(struct[k], cur_struct)
+ elif isinstance(v, dict):
+ struct[k] = {}
+ struct[k] = get_schema([v], struct[k], array_dict_types)
+ # merge cur_struct and struct[k], remove duplicates
+ struct[k] = merge(struct[k], cur_struct)
+ else:
+ # if the value in the item is currently None, then update
+ if v is not None:
+ struct[k] = type(v)
+
+ return struct
+
+
+def standardize(item, structure, array_dict_types):
+ """Standardize all rows/entries in dataset to fit the schema.
+
+ Will look for missing values and fill it in so all rows have
+ the same items and structure.
+ """
+ for k, v in structure.items():
+ if k not in item:
+ # If the structure/field does not exist
+ if isinstance(v, dict) and (k not in array_dict_types):
+ # If key k is of type dict, and not not a type list[dict]
+ item[k] = {}
+ standardize(item[k], v, array_dict_types)
+ elif isinstance(v, dict) and (k in array_dict_types):
+ # If key k is of type dict, and is actually of type list[dict],
+ # just treat as a list and set to None by default
+ item[k] = None
+ else:
+ # Assign a default type
+ item[k] = v()
+ else:
+ # If the structure/field already exists and is a list or dict
+ if isinstance(item[k], list):
+ # ignore if item is a nested list structure or list of non-dicts
+ condition = (
+ not (len(item[k]) > 0 and isinstance(item[k][0], list))
+ ) and (
+ not (
+ len(item[k]) > 0
+ and not (
+ isinstance(item[k][0], list) or isinstance(item[k][0], dict)
+ )
+ )
+ )
+ if condition:
+ for sub_item in item[k]:
+ standardize(sub_item, v, array_dict_types)
+ elif isinstance(item[k], dict):
+ standardize(item[k], v, array_dict_types)
+
+
+def create_table(data):
+ """Create a W&B Table.
+
+ - Create/decode images from URL/Base64
+ - Uses spacy to translate NER span data to visualizations.
+ """
+ # create table object from columns
+ table_df = pd.DataFrame(data)
+ columns = list(table_df.columns)
+ if ("spans" in table_df.columns) and ("text" in table_df.columns):
+ columns.append("spans_visual")
+ if "image" in columns:
+ columns.append("image_visual")
+ main_table = wandb.Table(columns=columns)
+
+ # Convert to dictionary format to maintain order during processing
+ matrix = table_df.to_dict(orient="records")
+
+ # Import en_core_web_md if exists
+ en_core_web_md = util.get_module(
+ "en_core_web_md",
+ required="part_of_speech requires `en_core_web_md` library, install with `python -m spacy download en_core_web_md`",
+ )
+ nlp = en_core_web_md.load(disable=["ner"])
+
+ # Go through each individual row
+ for _i, document in enumerate(matrix):
+ # Text NER span visualizations
+ if ("spans_visual" in columns) and ("text" in columns):
+ # Add visuals for spans
+ document["spans_visual"] = None
+ doc = nlp(document["text"])
+ ents = []
+ if ("spans" in document) and (document["spans"] is not None):
+ for span in document["spans"]:
+ if ("start" in span) and ("end" in span) and ("label" in span):
+ charspan = doc.char_span(
+ span["start"], span["end"], span["label"]
+ )
+ ents.append(charspan)
+ doc.ents = ents
+ document["spans_visual"] = named_entity(docs=doc)
+
+ # Convert image link to wandb Image
+ if "image" in columns:
+ # Turn into wandb image
+ document["image_visual"] = None
+ if ("image" in document) and (document["image"] is not None):
+ isurl = urllib.parse.urlparse(document["image"]).scheme in (
+ "http",
+ "https",
+ )
+ isbase64 = ("data:" in document["image"]) and (
+ ";base64" in document["image"]
+ )
+ if isurl:
+ # is url
+ try:
+ im = Image.open(urllib.request.urlopen(document["image"]))
+ document["image_visual"] = wandb.Image(im)
+ except urllib.error.URLError:
+ print(
+ "Warning: Image URL "
+ + str(document["image"])
+ + " is invalid."
+ )
+ document["image_visual"] = None
+ elif isbase64:
+ # is base64 uri
+ imgb64 = document["image"].split("base64,")[1]
+ try:
+ msg = base64.b64decode(imgb64)
+ buf = io.BytesIO(msg)
+ im = Image.open(buf)
+ document["image_visual"] = wandb.Image(im)
+ except base64.binascii.Error:
+ print(
+ "Warning: Base64 string "
+ + str(document["image"])
+ + " is invalid."
+ )
+ document["image_visual"] = None
+ else:
+ # is data path
+ document["image_visual"] = wandb.Image(document["image"])
+
+ # Create row and append to table
+ values_list = list(document.values())
+ main_table.add_data(*values_list)
+ return main_table
+
+
+def upload_dataset(dataset_name):
+ """Upload dataset from local database to Weights & Biases.
+
+ Args:
+ dataset_name: The name of the dataset in the Prodigy database.
+ """
+ # Check if wandb.init has been called
+ if wandb.run is None:
+ raise ValueError("You must call wandb.init() before upload_dataset()")
+
+ with wb_telemetry.context(run=wandb.run) as tel:
+ tel.feature.prodigy = True
+
+ prodigy_db = util.get_module(
+ "prodigy.components.db",
+ required="`prodigy` library is required but not installed. Please see https://prodi.gy/docs/install",
+ )
+ # Retrieve and upload prodigy dataset
+ database = prodigy_db.connect()
+ data = database.get_dataset(dataset_name)
+
+ array_dict_types = []
+ schema = get_schema(data, {}, array_dict_types)
+
+ for i, _d in enumerate(data):
+ standardize(data[i], schema, array_dict_types)
+ table = create_table(data)
+ wandb.log({dataset_name: table})
+ print("Prodigy dataset `" + dataset_name + "` uploaded.")
diff --git a/wandb/integration/sacred/__init__.py b/wandb/integration/sacred/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..69bc1abced937a3681d6ccf3e305f6eceb775985
--- /dev/null
+++ b/wandb/integration/sacred/__init__.py
@@ -0,0 +1,117 @@
+import warnings
+
+import numpy
+from sacred.dependencies import get_digest
+from sacred.observers import RunObserver
+
+import wandb
+
+
+class WandbObserver(RunObserver):
+ """Log sacred experiment data to W&B.
+
+ Arguments:
+ Accepts all the arguments accepted by wandb.init().
+
+ name — A display name for this run, which shows up in the UI and is editable, doesn't have to be unique
+ notes — A multiline string description associated with the run
+ config — a dictionary-like object to set as initial config
+ project — the name of the project to which this run will belong
+ tags — a list of strings to associate with this run as tags
+ dir — the path to a directory where artifacts will be written (default: ./wandb)
+ entity — the team posting this run (default: your username or your default team)
+ job_type — the type of job you are logging, e.g. eval, worker, ps (default: training)
+ save_code — save the main python or notebook file to wandb to enable diffing (default: editable from your settings page)
+ group — a string by which to group other runs; see Grouping
+ reinit — whether to allow multiple calls to wandb.init in the same process (default: False)
+ id — A unique ID for this run primarily used for Resuming. It must be globally unique, and if you delete a run you can't reuse the ID. Use the name field for a descriptive, useful name for the run. The ID cannot contain special characters.
+ resume — if set to True, the run auto resumes; can also be a unique string for manual resuming; see Resuming (default: False)
+ anonymous — can be "allow", "never", or "must". This enables or explicitly disables anonymous logging. (default: never)
+ force — whether to force a user to be logged into wandb when running a script (default: False)
+ magic — (bool, dict, or str, optional): magic configuration as bool, dict, json string, yaml filename. If set to True will attempt to auto-instrument your script. (default: None)
+ sync_tensorboard — A boolean indicating whether or not copy all TensorBoard logs wandb; see Tensorboard (default: False)
+ monitor_gym — A boolean indicating whether or not to log videos generated by OpenAI Gym; see Ray Tune (default: False)
+ allow_val_change — whether to allow wandb.config values to change, by default we throw an exception if config values are overwritten. (default: False)
+
+ Examples:
+ Create sacred experiment::
+ from wandb.sacred import WandbObserver
+ ex.observers.append(WandbObserver(project='sacred_test',
+ name='test1'))
+ @ex.config
+ def cfg():
+ C = 1.0
+ gamma = 0.7
+ @ex.automain
+ def run(C, gamma, _run):
+ iris = datasets.load_iris()
+ per = permutation(iris.target.size)
+ iris.data = iris.data[per]
+ iris.target = iris.target[per]
+ clf = svm.SVC(C, 'rbf', gamma=gamma)
+ clf.fit(iris.data[:90],
+ iris.target[:90])
+ return clf.score(iris.data[90:],
+ iris.target[90:])
+ """
+
+ def __init__(self, **kwargs):
+ self.run = wandb.init(**kwargs)
+ self.resources = {}
+
+ def started_event(
+ self, ex_info, command, host_info, start_time, config, meta_info, _id
+ ):
+ # TODO: add the source code file
+ # TODO: add dependencies and metadata.
+ self.__update_config(config)
+
+ def completed_event(self, stop_time, result):
+ if result:
+ if not isinstance(result, tuple):
+ result = (
+ result,
+ ) # transform single result to tuple so that both single & multiple results use same code
+
+ for i, r in enumerate(result):
+ if isinstance(r, float) or isinstance(r, int):
+ wandb.log({f"result_{i}": float(r)})
+ elif isinstance(r, dict):
+ wandb.log(r)
+ elif isinstance(r, object):
+ artifact = wandb.Artifact(f"result_{i}.pkl", type="result")
+ artifact.add_file(r)
+ self.run.log_artifact(artifact)
+ elif isinstance(r, numpy.ndarray):
+ wandb.log({f"result_{i}": wandb.Image(r)})
+ else:
+ warnings.warn(
+ f"logging results does not support type '{type(r)}' results. Ignoring this result",
+ stacklevel=2,
+ )
+
+ def artifact_event(self, name, filename, metadata=None, content_type=None):
+ if content_type is None:
+ content_type = "file"
+ artifact = wandb.Artifact(name, type=content_type)
+ artifact.add_file(filename)
+ self.run.log_artifact(artifact)
+
+ def resource_event(self, filename):
+ """TODO: Maintain resources list."""
+ if filename not in self.resources:
+ md5 = get_digest(filename)
+ self.resources[filename] = md5
+
+ def log_metrics(self, metrics_by_name, info):
+ for metric_name, metric_ptr in metrics_by_name.items():
+ for _step, value in zip(metric_ptr["steps"], metric_ptr["values"]):
+ if isinstance(value, numpy.ndarray):
+ wandb.log({metric_name: wandb.Image(value)})
+ else:
+ wandb.log({metric_name: value})
+
+ def __update_config(self, config):
+ for k, v in config.items():
+ self.run.config[k] = v
+ self.run.config["resources"] = []
diff --git a/wandb/integration/sagemaker/__init__.py b/wandb/integration/sagemaker/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..07af0c9954d0e6a287cf604731f0b7601b54fa62
--- /dev/null
+++ b/wandb/integration/sagemaker/__init__.py
@@ -0,0 +1,12 @@
+"""wandb integration sagemaker module."""
+
+from .auth import sagemaker_auth
+from .config import parse_sm_config
+from .resources import parse_sm_resources, parse_sm_secrets
+
+__all__ = [
+ "sagemaker_auth",
+ "parse_sm_config",
+ "parse_sm_secrets",
+ "parse_sm_resources",
+]
diff --git a/wandb/integration/sagemaker/auth.py b/wandb/integration/sagemaker/auth.py
new file mode 100644
index 0000000000000000000000000000000000000000..8db3e9341eaf9ea7bf50d0913e945999f1082f2e
--- /dev/null
+++ b/wandb/integration/sagemaker/auth.py
@@ -0,0 +1,28 @@
+import os
+
+import wandb
+from wandb import env
+
+
+def sagemaker_auth(overrides=None, path=".", api_key=None):
+ """Write a secrets.env file with the W&B ApiKey and any additional secrets passed.
+
+ Arguments:
+ overrides (dict, optional): Additional environment variables to write
+ to secrets.env
+ path (str, optional): The path to write the secrets file.
+ """
+ settings = wandb.setup().settings
+ current_api_key = wandb.wandb_lib.apikey.api_key(settings=settings)
+
+ overrides = overrides or dict()
+ api_key = overrides.get(env.API_KEY, api_key or current_api_key)
+ if api_key is None:
+ raise ValueError(
+ "Can't find W&B ApiKey, set the WANDB_API_KEY env variable "
+ "or run `wandb login`"
+ )
+ overrides[env.API_KEY] = api_key
+ with open(os.path.join(path, "secrets.env"), "w") as file:
+ for k, v in overrides.items():
+ file.write(f"{k}={v}\n")
diff --git a/wandb/integration/sagemaker/config.py b/wandb/integration/sagemaker/config.py
new file mode 100644
index 0000000000000000000000000000000000000000..fbc575607bbf8200bac42ce32514864f835100d3
--- /dev/null
+++ b/wandb/integration/sagemaker/config.py
@@ -0,0 +1,49 @@
+import json
+import os
+import re
+import warnings
+from typing import Any, Dict
+
+from . import files as sm_files
+
+
+def parse_sm_config() -> Dict[str, Any]:
+ """Parses SageMaker configuration.
+
+ Returns:
+ A dictionary of SageMaker config keys/values
+ or an empty dict if not found.
+ SM_TRAINING_ENV is a json string of the
+ training environment variables set by SageMaker
+ and is only available when running in SageMaker,
+ but not in local mode.
+ SM_TRAINING_ENV is set by the SageMaker container and
+ contains arguments such as hyperparameters
+ and arguments passed to the training job.
+ """
+ conf = {}
+
+ if os.path.exists(sm_files.SM_PARAM_CONFIG) and os.path.exists(
+ sm_files.SM_RESOURCE_CONFIG
+ ):
+ conf["sagemaker_training_job_name"] = os.getenv("TRAINING_JOB_NAME")
+
+ # Hyperparameter searches quote configs...
+ with open(sm_files.SM_PARAM_CONFIG) as fid:
+ for key, val in json.load(fid).items():
+ cast = val.strip('"')
+ if re.match(r"^-?[\d]+$", cast):
+ cast = int(cast)
+ elif re.match(r"^-?[.\d]+$", cast):
+ cast = float(cast)
+ conf[key] = cast
+
+ if "SM_TRAINING_ENV" in os.environ:
+ try:
+ conf = {**conf, **json.loads(os.environ["SM_TRAINING_ENV"])}
+ except json.JSONDecodeError:
+ warnings.warn(
+ "Failed to parse SM_TRAINING_ENV not valid JSON string", stacklevel=2
+ )
+
+ return conf
diff --git a/wandb/integration/sagemaker/files.py b/wandb/integration/sagemaker/files.py
new file mode 100644
index 0000000000000000000000000000000000000000..4c0b46b0b3a847cac4503fbdcd20d45a5ac8bbcb
--- /dev/null
+++ b/wandb/integration/sagemaker/files.py
@@ -0,0 +1,3 @@
+SM_PARAM_CONFIG = "/opt/ml/input/config/hyperparameters.json"
+SM_RESOURCE_CONFIG = "/opt/ml/input/config/resourceconfig.json"
+SM_SECRETS = "secrets.env"
diff --git a/wandb/integration/sagemaker/resources.py b/wandb/integration/sagemaker/resources.py
new file mode 100644
index 0000000000000000000000000000000000000000..88cfd58bd7db1d2d629ebe0a9f2ab5ce2ea966bd
--- /dev/null
+++ b/wandb/integration/sagemaker/resources.py
@@ -0,0 +1,34 @@
+import os
+import secrets
+import socket
+import string
+from typing import Dict, Tuple
+
+from . import files as sm_files
+
+
+def parse_sm_secrets() -> Dict[str, str]:
+ """We read our api_key from secrets.env in SageMaker."""
+ env_dict = dict()
+ # Set secret variables
+ if os.path.exists(sm_files.SM_SECRETS):
+ for line in open(sm_files.SM_SECRETS):
+ key, val = line.strip().split("=", 1)
+ env_dict[key] = val
+ return env_dict
+
+
+def parse_sm_resources() -> Tuple[Dict[str, str], Dict[str, str]]:
+ run_dict = dict()
+ run_id = os.getenv("TRAINING_JOB_NAME")
+
+ if run_id and os.getenv("WANDB_RUN_ID") is None:
+ suffix = "".join(
+ secrets.choice(string.ascii_lowercase + string.digits) for _ in range(6)
+ )
+ run_dict["run_id"] = "-".join(
+ [run_id, suffix, os.getenv("CURRENT_HOST", socket.gethostname())]
+ )
+ run_dict["run_group"] = os.getenv("TRAINING_JOB_NAME")
+ env_dict = parse_sm_secrets()
+ return run_dict, env_dict
diff --git a/wandb/integration/sb3/__init__.py b/wandb/integration/sb3/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..29dd81a941cb1745cf82fecbcf12a857ce786f45
--- /dev/null
+++ b/wandb/integration/sb3/__init__.py
@@ -0,0 +1,3 @@
+from .sb3 import WandbCallback
+
+__all__ = ["WandbCallback"]
diff --git a/wandb/integration/sb3/sb3.py b/wandb/integration/sb3/sb3.py
new file mode 100644
index 0000000000000000000000000000000000000000..96c7da458e241490d40612537e34919a3eda22f5
--- /dev/null
+++ b/wandb/integration/sb3/sb3.py
@@ -0,0 +1,153 @@
+"""W&B callback for sb3.
+
+Really simple callback to get logging for each tree
+
+Example usage:
+
+```python
+import gym
+from stable_baselines3 import PPO
+from stable_baselines3.common.monitor import Monitor
+from stable_baselines3.common.vec_env import DummyVecEnv, VecVideoRecorder
+import wandb
+from wandb.integration.sb3 import WandbCallback
+
+
+config = {
+ "policy_type": "MlpPolicy",
+ "total_timesteps": 25000,
+ "env_name": "CartPole-v1",
+}
+run = wandb.init(
+ project="sb3",
+ config=config,
+ sync_tensorboard=True, # auto-upload sb3's tensorboard metrics
+ monitor_gym=True, # auto-upload the videos of agents playing the game
+ save_code=True, # optional
+)
+
+
+def make_env():
+ env = gym.make(config["env_name"])
+ env = Monitor(env) # record stats such as returns
+ return env
+
+
+env = DummyVecEnv([make_env])
+env = VecVideoRecorder(
+ env, "videos", record_video_trigger=lambda x: x % 2000 == 0, video_length=200
+)
+model = PPO(config["policy_type"], env, verbose=1, tensorboard_log=f"runs")
+model.learn(
+ total_timesteps=config["total_timesteps"],
+ callback=WandbCallback(
+ model_save_path=f"models/{run.id}",
+ gradient_save_freq=100,
+ log="all",
+ ),
+)
+```
+"""
+
+import logging
+import os
+import sys
+from typing import Optional
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+from stable_baselines3.common.callbacks import BaseCallback # type: ignore
+
+import wandb
+from wandb.sdk.lib import telemetry as wb_telemetry
+
+logger = logging.getLogger(__name__)
+
+
+class WandbCallback(BaseCallback):
+ """Callback for logging experiments to Weights and Biases.
+
+ Log SB3 experiments to Weights and Biases
+ - Added model tracking and uploading
+ - Added complete hyperparameters recording
+ - Added gradient logging
+ - Note that `wandb.init(...)` must be called before the WandbCallback can be used.
+
+ Args:
+ verbose: The verbosity of sb3 output
+ model_save_path: Path to the folder where the model will be saved, The default value is `None` so the model is not logged
+ model_save_freq: Frequency to save the model
+ gradient_save_freq: Frequency to log gradient. The default value is 0 so the gradients are not logged
+ log: What to log. One of "gradients", "parameters", or "all".
+ """
+
+ def __init__(
+ self,
+ verbose: int = 0,
+ model_save_path: Optional[str] = None,
+ model_save_freq: int = 0,
+ gradient_save_freq: int = 0,
+ log: Optional[Literal["gradients", "parameters", "all"]] = "all",
+ ) -> None:
+ super().__init__(verbose)
+ if wandb.run is None:
+ raise wandb.Error("You must call wandb.init() before WandbCallback()")
+ with wb_telemetry.context() as tel:
+ tel.feature.sb3 = True
+ self.model_save_freq = model_save_freq
+ self.model_save_path = model_save_path
+ self.gradient_save_freq = gradient_save_freq
+ if log not in ["gradients", "parameters", "all", None]:
+ wandb.termwarn(
+ "`log` must be one of `None`, 'gradients', 'parameters', or 'all', "
+ "falling back to 'all'"
+ )
+ log = "all"
+ self.log = log
+ # Create folder if needed
+ if self.model_save_path is not None:
+ os.makedirs(self.model_save_path, exist_ok=True)
+ self.path = os.path.join(self.model_save_path, "model.zip")
+ else:
+ assert (
+ self.model_save_freq == 0
+ ), "to use the `model_save_freq` you have to set the `model_save_path` parameter"
+
+ def _init_callback(self) -> None:
+ d = {}
+ if "algo" not in d:
+ d["algo"] = type(self.model).__name__
+ for key in self.model.__dict__:
+ if key in wandb.config:
+ continue
+ if type(self.model.__dict__[key]) in [float, int, str]:
+ d[key] = self.model.__dict__[key]
+ else:
+ d[key] = str(self.model.__dict__[key])
+ if self.gradient_save_freq > 0:
+ wandb.watch(
+ self.model.policy,
+ log_freq=self.gradient_save_freq,
+ log=self.log,
+ )
+ wandb.config.setdefaults(d)
+
+ def _on_step(self) -> bool:
+ if self.model_save_freq > 0:
+ if self.model_save_path is not None:
+ if self.n_calls % self.model_save_freq == 0:
+ self.save_model()
+ return True
+
+ def _on_training_end(self) -> None:
+ if self.model_save_path is not None:
+ self.save_model()
+
+ def save_model(self) -> None:
+ self.model.save(self.path)
+ wandb.save(self.path, base_path=self.model_save_path)
+ if self.verbose > 1:
+ logger.info(f"Saving model checkpoint to {self.path}")
diff --git a/wandb/integration/tensorboard/__init__.py b/wandb/integration/tensorboard/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..980ba7ff090708a50f754eac626d122ca24839ce
--- /dev/null
+++ b/wandb/integration/tensorboard/__init__.py
@@ -0,0 +1,10 @@
+"""wandb integration tensorboard module."""
+
+from .log import _log, log, reset_state, tf_summary_to_dict # noqa: F401
+from .monkeypatch import patch, unpatch
+
+__all__ = [
+ "patch",
+ "unpatch",
+ "log",
+]
diff --git a/wandb/integration/tensorboard/log.py b/wandb/integration/tensorboard/log.py
new file mode 100644
index 0000000000000000000000000000000000000000..f94c6bdccebc0beb32425d9a47ee34be017e6d4f
--- /dev/null
+++ b/wandb/integration/tensorboard/log.py
@@ -0,0 +1,358 @@
+import io
+import re
+import time
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+
+import wandb
+import wandb.util
+from wandb.sdk.lib import telemetry
+from wandb.viz import custom_chart
+
+if TYPE_CHECKING:
+ import numpy as np
+
+ from wandb.sdk.internal.tb_watcher import TBHistory
+
+# We have at least the default namestep and a global step to track
+# TODO: reset this structure on wandb.finish
+STEPS: Dict[str, Dict[str, Any]] = {
+ "": {"step": 0},
+ "global": {"step": 0, "last_log": None},
+}
+# TODO(cling): Set these when tensorboard behavior is configured.
+# We support rate limited logging by setting this to number of seconds,
+# can be a floating point.
+RATE_LIMIT_SECONDS: Optional[Union[float, int]] = None
+IGNORE_KINDS = ["graphs"]
+tensor_util = wandb.util.get_module("tensorboard.util.tensor_util")
+
+
+# prefer tensorboard, fallback to protobuf in tensorflow when tboard isn't available
+pb = wandb.util.get_module(
+ "tensorboard.compat.proto.summary_pb2"
+) or wandb.util.get_module("tensorflow.core.framework.summary_pb2")
+
+Summary = pb.Summary if pb else None
+
+
+def make_ndarray(tensor: Any) -> Optional["np.ndarray"]:
+ if tensor_util:
+ res = tensor_util.make_ndarray(tensor)
+ # Tensorboard can log generic objects, and we don't want to save them
+ if res.dtype == "object":
+ return None
+ else:
+ return res # type: ignore
+ else:
+ wandb.termwarn(
+ "Can't convert tensor summary, upgrade tensorboard with `pip"
+ " install tensorboard --upgrade`"
+ )
+ return None
+
+
+def namespaced_tag(tag: str, namespace: str = "") -> str:
+ if not namespace:
+ return tag
+ elif tag in namespace:
+ # This happens with tensorboardX
+ return namespace
+ else:
+ return namespace + "/" + tag
+
+
+def history_image_key(key: str, namespace: str = "") -> str:
+ """Convert invalid filesystem characters to _ for use in History keys.
+
+ Unfortunately this means currently certain image keys will collide silently. We
+ implement this mapping up here in the TensorFlow stuff rather than in the History
+ stuff so that we don't have to store a mapping anywhere from the original keys to
+ the safe ones.
+ """
+ return namespaced_tag(re.sub(r"[/\\]", "_", key), namespace)
+
+
+def tf_summary_to_dict( # noqa: C901
+ tf_summary_str_or_pb: Any, namespace: str = ""
+) -> Optional[Dict[str, Any]]:
+ """Convert a Tensorboard Summary to a dictionary.
+
+ Accepts a tensorflow.summary.Summary, one encoded as a string,
+ or a list of such encoded as strings.
+ """
+ values = {}
+ if hasattr(tf_summary_str_or_pb, "summary"):
+ summary_pb = tf_summary_str_or_pb.summary
+ values[namespaced_tag("global_step", namespace)] = tf_summary_str_or_pb.step
+ values["_timestamp"] = tf_summary_str_or_pb.wall_time
+ elif isinstance(tf_summary_str_or_pb, (str, bytes, bytearray)):
+ summary_pb = Summary()
+ summary_pb.ParseFromString(tf_summary_str_or_pb)
+ elif hasattr(tf_summary_str_or_pb, "__iter__"):
+ summary_pb = [Summary() for _ in range(len(tf_summary_str_or_pb))]
+ for i, summary in enumerate(tf_summary_str_or_pb):
+ summary_pb[i].ParseFromString(summary)
+ if i > 0:
+ summary_pb[0].MergeFrom(summary_pb[i])
+ summary_pb = summary_pb[0]
+ else:
+ summary_pb = tf_summary_str_or_pb
+
+ if not hasattr(summary_pb, "value") or len(summary_pb.value) == 0:
+ # Ignore these, caller is responsible for handling None
+ return None
+
+ def encode_images(_img_strs: List[bytes], _value: Any) -> None:
+ try:
+ from PIL import Image
+ except ImportError:
+ wandb.termwarn(
+ "Install pillow if you are logging images with Tensorboard. "
+ "To install, run `pip install pillow`.",
+ repeat=False,
+ )
+ return None
+
+ if len(_img_strs) == 0:
+ return None
+
+ images: List[Union[wandb.Video, wandb.Image]] = []
+ for _img_str in _img_strs:
+ # Supports gifs from TensorboardX
+ if _img_str.startswith(b"GIF"):
+ images.append(wandb.Video(io.BytesIO(_img_str), format="gif"))
+ else:
+ images.append(wandb.Image(Image.open(io.BytesIO(_img_str))))
+ tag_idx = _value.tag.rsplit("/", 1)
+ if len(tag_idx) > 1 and tag_idx[1].isdigit():
+ tag, idx = tag_idx
+ values.setdefault(history_image_key(tag, namespace), []).extend(images)
+ else:
+ values[history_image_key(_value.tag, namespace)] = images
+
+ return None
+
+ for value in summary_pb.value:
+ kind = value.WhichOneof("value")
+ if kind in IGNORE_KINDS:
+ continue
+ if kind == "simple_value":
+ values[namespaced_tag(value.tag, namespace)] = value.simple_value
+ elif kind == "tensor":
+ plugin_name = value.metadata.plugin_data.plugin_name
+ if plugin_name == "scalars" or plugin_name == "":
+ values[namespaced_tag(value.tag, namespace)] = make_ndarray(
+ value.tensor
+ )
+ elif plugin_name == "images":
+ img_strs = value.tensor.string_val[2:] # First two items are dims.
+ encode_images(img_strs, value)
+ elif plugin_name == "histograms":
+ # https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/histogram/summary_v2.py#L15-L26
+ ndarray = make_ndarray(value.tensor)
+ if ndarray is None:
+ continue
+ shape = ndarray.shape
+ counts = []
+ bins = []
+ if shape[0] > 1:
+ bins.append(ndarray[0][0]) # Add the left most edge
+ for v in ndarray:
+ counts.append(v[2])
+ bins.append(v[1]) # Add the right most edges
+ elif shape[0] == 1:
+ counts = [ndarray[0][2]]
+ bins = ndarray[0][:2]
+ if len(counts) > 0:
+ try:
+ # TODO: we should just re-bin if there are too many buckets
+ values[namespaced_tag(value.tag, namespace)] = wandb.Histogram(
+ np_histogram=(counts, bins) # type: ignore
+ )
+ except ValueError:
+ wandb.termwarn(
+ 'Not logging key "{}". '
+ "Histograms must have fewer than {} bins".format(
+ namespaced_tag(value.tag, namespace),
+ wandb.Histogram.MAX_LENGTH,
+ ),
+ repeat=False,
+ )
+ elif plugin_name == "pr_curves":
+ pr_curve_data = make_ndarray(value.tensor)
+ if pr_curve_data is None:
+ continue
+ precision = pr_curve_data[-2, :].tolist()
+ recall = pr_curve_data[-1, :].tolist()
+ # TODO: (kdg) implement spec for showing additional info in tool tips
+ # true_pos = pr_curve_data[1,:]
+ # false_pos = pr_curve_data[2,:]
+ # true_neg = pr_curve_data[1,:]
+ # false_neg = pr_curve_data[1,:]
+ # threshold = [1.0 / n for n in range(len(true_pos), 0, -1)]
+ # min of each in case tensorboard ever changes their pr_curve
+ # to allow for different length outputs
+ data = []
+ for i in range(min(len(precision), len(recall))):
+ # drop additional threshold values if they exist
+ if precision[i] != 0 or recall[i] != 0:
+ data.append((recall[i], precision[i]))
+ # sort data so custom chart looks the same as tb generated pr curve
+ # ascending recall, descending precision for the same recall values
+ data = sorted(data, key=lambda x: (x[0], -x[1]))
+ data_table = wandb.Table(data=data, columns=["recall", "precision"])
+ name = namespaced_tag(value.tag, namespace)
+
+ values[name] = custom_chart(
+ "wandb/line/v0",
+ data_table,
+ {"x": "recall", "y": "precision"},
+ {"title": f"{name} Precision v. Recall"},
+ )
+ elif kind == "image":
+ img_str = value.image.encoded_image_string
+ encode_images([img_str], value)
+ # Coming soon...
+ # elif kind == "audio":
+ # audio = wandb.Audio(
+ # six.BytesIO(value.audio.encoded_audio_string),
+ # sample_rate=value.audio.sample_rate,
+ # content_type=value.audio.content_type,
+ # )
+ elif kind == "histo":
+ tag = namespaced_tag(value.tag, namespace)
+ if len(value.histo.bucket_limit) >= 3:
+ first = (
+ value.histo.bucket_limit[0]
+ + value.histo.bucket_limit[0]
+ - value.histo.bucket_limit[1]
+ )
+ last = (
+ value.histo.bucket_limit[-2]
+ + value.histo.bucket_limit[-2]
+ - value.histo.bucket_limit[-3]
+ )
+ np_histogram = (
+ list(value.histo.bucket),
+ [first] + value.histo.bucket_limit[:-1] + [last],
+ )
+ try:
+ # TODO: we should just re-bin if there are too many buckets
+ values[tag] = wandb.Histogram(np_histogram=np_histogram) # type: ignore
+ except ValueError:
+ wandb.termwarn(
+ f"Not logging key {tag!r}. "
+ f"Histograms must have fewer than {wandb.Histogram.MAX_LENGTH} bins",
+ repeat=False,
+ )
+ else:
+ # TODO: is there a case where we can render this?
+ wandb.termwarn(
+ f"Not logging key {tag!r}. Found a histogram with only 2 bins.",
+ repeat=False,
+ )
+ # TODO(jhr): figure out how to share this between userspace and internal process or dont
+ # elif value.tag == "_hparams_/session_start_info":
+ # if wandb.util.get_module("tensorboard.plugins.hparams"):
+ # from tensorboard.plugins.hparams import plugin_data_pb2
+ #
+ # plugin_data = plugin_data_pb2.HParamsPluginData() #
+ # plugin_data.ParseFromString(value.metadata.plugin_data.content)
+ # for key, param in six.iteritems(plugin_data.session_start_info.hparams):
+ # if not wandb.run.config.get(key):
+ # wandb.run.config[key] = (
+ # param.number_value or param.string_value or param.bool_value
+ # )
+ # else:
+ # wandb.termerror(
+ # "Received hparams tf.summary, but could not import "
+ # "the hparams plugin from tensorboard"
+ # )
+ return values
+
+
+def reset_state() -> None:
+ """Internal method for resetting state, called by wandb.finish()."""
+ global STEPS
+ STEPS = {"": {"step": 0}, "global": {"step": 0, "last_log": None}}
+
+
+def _log(
+ tf_summary_str_or_pb: Any,
+ history: Optional["TBHistory"] = None,
+ step: int = 0,
+ namespace: str = "",
+ **kwargs: Any,
+) -> None:
+ """Logs a tfsummary to wandb.
+
+ Can accept a tf summary string or parsed event. Will use wandb.run.history unless a
+ history object is passed. Can optionally namespace events. Results are committed
+ when step increases for this namespace.
+
+ NOTE: This assumes that events being passed in are in chronological order
+ """
+ global STEPS
+ global RATE_LIMIT_SECONDS
+ # To handle multiple global_steps, we keep track of them here instead
+ # of the global log
+ last_step = STEPS.get(namespace, {"step": 0})
+
+ # Commit our existing data if this namespace increased its step
+ commit = False
+ if last_step["step"] < step:
+ commit = True
+
+ log_dict = tf_summary_to_dict(tf_summary_str_or_pb, namespace)
+ if log_dict is None:
+ # not an event, just return
+ return
+
+ # Pass timestamp to history for loading historic data
+ timestamp = log_dict.get("_timestamp", time.time())
+ # Store our initial timestamp
+ if STEPS["global"]["last_log"] is None:
+ STEPS["global"]["last_log"] = timestamp
+ # Rollup events that share the same step across namespaces
+ if commit and step == STEPS["global"]["step"]:
+ commit = False
+ # Always add the biggest global_step key for non-default namespaces
+ if step > STEPS["global"]["step"]:
+ STEPS["global"]["step"] = step
+ if namespace != "":
+ log_dict["global_step"] = STEPS["global"]["step"]
+
+ # Keep internal step counter
+ STEPS[namespace] = {"step": step}
+
+ if commit:
+ # Only commit our data if we're below the rate limit or don't have one
+ if (
+ RATE_LIMIT_SECONDS is None
+ or timestamp - STEPS["global"]["last_log"] >= RATE_LIMIT_SECONDS
+ ):
+ if history is None:
+ if wandb.run is not None:
+ wandb.run._log({})
+ else:
+ history.add({})
+
+ STEPS["global"]["last_log"] = timestamp
+
+ if history is None:
+ if wandb.run is not None:
+ wandb.run._log(log_dict, commit=False)
+ else:
+ history._row_update(log_dict)
+
+
+def log(tf_summary_str_or_pb: Any, step: int = 0, namespace: str = "") -> None:
+ if wandb.run is None:
+ raise wandb.Error(
+ "You must call `wandb.init()` before calling `wandb.tensorflow.log`"
+ )
+
+ with telemetry.context() as tel:
+ tel.feature.tensorboard_log = True
+
+ _log(tf_summary_str_or_pb, namespace=namespace, step=step)
diff --git a/wandb/integration/tensorboard/monkeypatch.py b/wandb/integration/tensorboard/monkeypatch.py
new file mode 100644
index 0000000000000000000000000000000000000000..de3b8c32e68db5a6fa89c83eeda7badd52c3dd1d
--- /dev/null
+++ b/wandb/integration/tensorboard/monkeypatch.py
@@ -0,0 +1,185 @@
+"""monkeypatch: patch code to add tensorboard hooks."""
+
+import os
+import re
+import socket
+from typing import Any, Optional
+
+import wandb
+import wandb.util
+
+TENSORBOARD_C_MODULE = "tensorflow.python.ops.gen_summary_ops"
+TENSORBOARD_X_MODULE = "tensorboardX.writer"
+TENSORFLOW_PY_MODULE = "tensorflow.python.summary.writer.writer"
+TENSORBOARD_WRITER_MODULE = "tensorboard.summary.writer.event_file_writer"
+TENSORBOARD_PYTORCH_MODULE = "torch.utils.tensorboard.writer"
+
+
+def unpatch() -> None:
+ for module, method in wandb.patched["tensorboard"]:
+ writer = wandb.util.get_module(module, lazy=False)
+ setattr(writer, method, getattr(writer, f"orig_{method}"))
+ wandb.patched["tensorboard"] = []
+
+
+def patch(
+ save: bool = True,
+ tensorboard_x: Optional[bool] = None,
+ pytorch: Optional[bool] = None,
+ root_logdir: str = "",
+) -> None:
+ if len(wandb.patched["tensorboard"]) > 0:
+ raise ValueError(
+ "Tensorboard already patched, remove `sync_tensorboard=True` "
+ "from `wandb.init` or only call `wandb.tensorboard.patch` once."
+ )
+
+ # TODO: Some older versions of tensorflow don't require tensorboard to be present.
+ # we may want to lift this requirement, but it's safer to have it for now
+ wandb.util.get_module(
+ "tensorboard", required="Please install tensorboard package", lazy=False
+ )
+ c_writer = wandb.util.get_module(TENSORBOARD_C_MODULE, lazy=False)
+ py_writer = wandb.util.get_module(TENSORFLOW_PY_MODULE, lazy=False)
+ tb_writer = wandb.util.get_module(TENSORBOARD_WRITER_MODULE, lazy=False)
+ pt_writer = wandb.util.get_module(TENSORBOARD_PYTORCH_MODULE, lazy=False)
+ tbx_writer = wandb.util.get_module(TENSORBOARD_X_MODULE, lazy=False)
+
+ if not pytorch and not tensorboard_x and c_writer:
+ _patch_tensorflow2(
+ writer=c_writer,
+ module=TENSORBOARD_C_MODULE,
+ save=save,
+ root_logdir=root_logdir,
+ )
+ # This is for tensorflow <= 1.15 (tf.compat.v1.summary.FileWriter)
+ if py_writer:
+ _patch_file_writer(
+ writer=py_writer,
+ module=TENSORFLOW_PY_MODULE,
+ save=save,
+ root_logdir=root_logdir,
+ )
+ if tb_writer:
+ _patch_file_writer(
+ writer=tb_writer,
+ module=TENSORBOARD_WRITER_MODULE,
+ save=save,
+ root_logdir=root_logdir,
+ )
+ if pt_writer:
+ _patch_file_writer(
+ writer=pt_writer,
+ module=TENSORBOARD_PYTORCH_MODULE,
+ save=save,
+ root_logdir=root_logdir,
+ )
+ if tbx_writer:
+ _patch_file_writer(
+ writer=tbx_writer,
+ module=TENSORBOARD_X_MODULE,
+ save=save,
+ root_logdir=root_logdir,
+ )
+ if not c_writer and not tb_writer and not tb_writer:
+ wandb.termerror("Unsupported tensorboard configuration")
+
+
+def _patch_tensorflow2(
+ writer: Any,
+ module: Any,
+ save: bool = True,
+ root_logdir: str = "",
+) -> None:
+ # This configures TensorFlow 2 style Tensorboard logging
+ old_csfw_func = writer.create_summary_file_writer
+ logdir_hist = []
+
+ def new_csfw_func(*args: Any, **kwargs: Any) -> Any:
+ logdir = (
+ kwargs["logdir"].numpy().decode("utf8")
+ if hasattr(kwargs["logdir"], "numpy")
+ else kwargs["logdir"]
+ )
+ logdir_hist.append(logdir)
+ root_logdir_arg = root_logdir
+
+ if len(set(logdir_hist)) > 1 and root_logdir == "":
+ wandb.termwarn(
+ "When using several event log directories, "
+ 'please call `wandb.tensorboard.patch(root_logdir="...")` before `wandb.init`'
+ )
+ # if the logdir contains the hostname, the writer was not given a logdir.
+ # In this case, the generated logdir
+ # is generated and ends with the hostname, update the root_logdir to match.
+ hostname = socket.gethostname()
+ search = re.search(rf"-\d+_{hostname}", logdir)
+ if search:
+ root_logdir_arg = logdir[: search.span()[1]]
+ elif root_logdir is not None and not os.path.abspath(logdir).startswith(
+ os.path.abspath(root_logdir)
+ ):
+ wandb.termwarn(
+ "Found log directory outside of given root_logdir, "
+ f"dropping given root_logdir for event file in {logdir}"
+ )
+ root_logdir_arg = ""
+
+ _notify_tensorboard_logdir(logdir, save=save, root_logdir=root_logdir_arg)
+ return old_csfw_func(*args, **kwargs)
+
+ writer.orig_create_summary_file_writer = old_csfw_func
+ writer.create_summary_file_writer = new_csfw_func
+ wandb.patched["tensorboard"].append([module, "create_summary_file_writer"])
+
+
+def _patch_file_writer(
+ writer: Any,
+ module: Any,
+ save: bool = True,
+ root_logdir: str = "",
+) -> None:
+ # This configures non-TensorFlow Tensorboard logging, or tensorflow <= 1.15
+ logdir_hist = []
+
+ class TBXEventFileWriter(writer.EventFileWriter):
+ def __init__(self, logdir: str, *args: Any, **kwargs: Any) -> None:
+ logdir_hist.append(logdir)
+ root_logdir_arg = root_logdir
+ if len(set(logdir_hist)) > 1 and root_logdir == "":
+ wandb.termwarn(
+ "When using several event log directories, "
+ 'please call `wandb.tensorboard.patch(root_logdir="...")` before `wandb.init`'
+ )
+
+ # if the logdir contains the hostname, the writer was not given a logdir.
+ # In this case, the logdir is generated and ends with the hostname,
+ # update the root_logdir to match.
+ hostname = socket.gethostname()
+ search = re.search(rf"-\d+_{hostname}", logdir)
+ if search:
+ root_logdir_arg = logdir[: search.span()[1]]
+
+ elif root_logdir is not None and not os.path.abspath(logdir).startswith(
+ os.path.abspath(root_logdir)
+ ):
+ wandb.termwarn(
+ "Found log directory outside of given root_logdir, "
+ f"dropping given root_logdir for event file in {logdir}"
+ )
+ root_logdir_arg = ""
+
+ _notify_tensorboard_logdir(logdir, save=save, root_logdir=root_logdir_arg)
+
+ super().__init__(logdir, *args, **kwargs)
+
+ writer.orig_EventFileWriter = writer.EventFileWriter
+ writer.EventFileWriter = TBXEventFileWriter
+ wandb.patched["tensorboard"].append([module, "EventFileWriter"])
+
+
+def _notify_tensorboard_logdir(
+ logdir: str, save: bool = True, root_logdir: str = ""
+) -> None:
+ if wandb.run is not None:
+ wandb.run._tensorboard_callback(logdir, save=save, root_logdir=root_logdir)
diff --git a/wandb/integration/tensorflow/__init__.py b/wandb/integration/tensorflow/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..ff96a20b590d04e65c51bd900c48271c66ed9666
--- /dev/null
+++ b/wandb/integration/tensorflow/__init__.py
@@ -0,0 +1,5 @@
+"""api."""
+
+from wandb.integration.tensorboard import log # noqa: F401
+
+from .estimator_hook import WandbHook # noqa: F401
diff --git a/wandb/integration/tensorflow/estimator_hook.py b/wandb/integration/tensorflow/estimator_hook.py
new file mode 100644
index 0000000000000000000000000000000000000000..88d58abe3c6ee81759ce01a8e7254e1e324034de
--- /dev/null
+++ b/wandb/integration/tensorflow/estimator_hook.py
@@ -0,0 +1,54 @@
+import tensorflow as tf
+
+import wandb
+from wandb.sdk.lib import telemetry
+
+if hasattr(tf.estimator, "SessionRunHook"):
+ # In tf 1.14 and beyond, SessionRunHook is in the estimator package.
+ SessionRunHook = tf.estimator.SessionRunHook
+ SessionRunArgs = tf.estimator.SessionRunArgs
+else:
+ # In older versions it's in train.
+ SessionRunHook = tf.train.SessionRunHook
+ SessionRunArgs = tf.train.SessionRunArgs
+
+if hasattr(tf.train, "get_global_step"):
+ get_global_step = tf.train.get_global_step
+else:
+ get_global_step = tf.compat.v1.train.get_global_step
+
+if hasattr(tf.summary, "merge_all"):
+ merge_all_summaries = tf.summary.merge_all
+else:
+ merge_all_summaries = tf.compat.v1.summary.merge_all
+
+
+class WandbHook(SessionRunHook):
+ def __init__(self, summary_op=None, steps_per_log=1000, history=None):
+ self._summary_op = summary_op
+ self._steps_per_log = steps_per_log
+ self._history = history
+
+ with telemetry.context() as tel:
+ tel.feature.estimator_hook = True
+
+ def begin(self):
+ if wandb.run is None:
+ raise wandb.Error("You must call `wandb.init()` before calling `WandbHook`")
+ if self._summary_op is None:
+ self._summary_op = merge_all_summaries()
+ self._step = -1
+
+ def before_run(self, run_context):
+ return SessionRunArgs(
+ {"summary": self._summary_op, "global_step": get_global_step()}
+ )
+
+ def after_run(self, run_context, run_values):
+ step = run_values.results["global_step"]
+ if step % self._steps_per_log == 0:
+ wandb.tensorboard._log(
+ run_values.results["summary"],
+ history=self._history,
+ step=step,
+ )
diff --git a/wandb/integration/torch/__init__.py b/wandb/integration/torch/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/integration/ultralytics/__init__.py b/wandb/integration/ultralytics/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..921640dce43382e9513ad29fdc7ca7528e2282e6
--- /dev/null
+++ b/wandb/integration/ultralytics/__init__.py
@@ -0,0 +1,9 @@
+"""Tools for integrating with [`ultralytics`](https://docs.ultralytics.com/).
+
+Ultralytics is a computer vision framework for training and deploying YOLOv8
+models.
+"""
+
+from wandb.integration.ultralytics.callback import add_wandb_callback
+
+__all__ = ("add_wandb_callback",)
diff --git a/wandb/integration/ultralytics/bbox_utils.py b/wandb/integration/ultralytics/bbox_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..9cee9157d23c59f2a047b5a9d6518db0dfc97926
--- /dev/null
+++ b/wandb/integration/ultralytics/bbox_utils.py
@@ -0,0 +1,200 @@
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import torch
+from ultralytics.engine.results import Results
+from ultralytics.models.yolo.detect import DetectionPredictor
+
+try:
+ from ultralytics.yolo.utils import ops
+except ModuleNotFoundError:
+ from ultralytics.utils import ops
+
+import wandb
+
+
+def scale_bounding_box_to_original_image_shape(
+ box: torch.Tensor,
+ resized_image_shape: Tuple,
+ original_image_shape: Tuple,
+ ratio_pad: bool,
+) -> List[int]:
+ """YOLOv8 resizes images during training and the label values are normalized based on this resized shape.
+
+ This function rescales the bounding box labels to the original
+ image shape.
+
+ Reference: https://github.com/ultralytics/ultralytics/blob/main/ultralytics/yolo/utils/callbacks/comet.py#L105
+ """
+ resized_image_height, resized_image_width = resized_image_shape
+ # Convert normalized xywh format predictions to xyxy in resized scale format
+ box = ops.xywhn2xyxy(box, h=resized_image_height, w=resized_image_width)
+ # Scale box predictions from resized image scale back to original image scale
+ box = ops.scale_boxes(resized_image_shape, box, original_image_shape, ratio_pad)
+ # # Convert bounding box format from xyxy to xywh for Comet logging
+ box = ops.xyxy2xywh(box)
+ return box.tolist()
+
+
+def get_ground_truth_bbox_annotations(
+ img_idx: int, image_path: str, batch: Dict, class_name_map: Dict = None
+) -> List[Dict[str, Any]]:
+ """Get ground truth bounding box annotation data in the form required for `wandb.Image` overlay system."""
+ indices = batch["batch_idx"] == img_idx
+ bboxes = batch["bboxes"][indices]
+ cls_labels = batch["cls"][indices].squeeze(1).tolist()
+
+ class_name_map_reverse = {v: k for k, v in class_name_map.items()}
+
+ if len(bboxes) == 0:
+ wandb.termwarn(
+ f"Image: {image_path} has no bounding boxes labels", repeat=False
+ )
+ return None
+
+ cls_labels = batch["cls"][indices].squeeze(1).tolist()
+ if class_name_map:
+ cls_labels = [str(class_name_map[label]) for label in cls_labels]
+
+ original_image_shape = batch["ori_shape"][img_idx]
+ resized_image_shape = batch["resized_shape"][img_idx]
+ ratio_pad = batch["ratio_pad"][img_idx]
+
+ data = []
+ for box, label in zip(bboxes, cls_labels):
+ box = scale_bounding_box_to_original_image_shape(
+ box, resized_image_shape, original_image_shape, ratio_pad
+ )
+ data.append(
+ {
+ "position": {
+ "middle": [int(box[0]), int(box[1])],
+ "width": int(box[2]),
+ "height": int(box[3]),
+ },
+ "domain": "pixel",
+ "class_id": class_name_map_reverse[label],
+ "box_caption": label,
+ }
+ )
+
+ return data
+
+
+def get_mean_confidence_map(
+ classes: List, confidence: List, class_id_to_label: Dict
+) -> Dict[str, float]:
+ """Get Mean-confidence map from the predictions to be logged into a `wandb.Table`."""
+ confidence_map = {v: [] for _, v in class_id_to_label.items()}
+ for class_idx, confidence_value in zip(classes, confidence):
+ confidence_map[class_id_to_label[class_idx]].append(confidence_value)
+ updated_confidence_map = {}
+ for label, confidence_list in confidence_map.items():
+ if len(confidence_list) > 0:
+ updated_confidence_map[label] = sum(confidence_list) / len(confidence_list)
+ else:
+ updated_confidence_map[label] = 0
+ return updated_confidence_map
+
+
+def get_boxes(result: Results) -> Tuple[Dict, Dict]:
+ """Convert an ultralytics prediction result into metadata for the `wandb.Image` overlay system."""
+ boxes = result.boxes.xywh.long().numpy()
+ classes = result.boxes.cls.long().numpy()
+ confidence = result.boxes.conf.numpy()
+ class_id_to_label = {int(k): str(v) for k, v in result.names.items()}
+ mean_confidence_map = get_mean_confidence_map(
+ classes, confidence, class_id_to_label
+ )
+ box_data = []
+ for idx in range(len(boxes)):
+ box_data.append(
+ {
+ "position": {
+ "middle": [int(boxes[idx][0]), int(boxes[idx][1])],
+ "width": int(boxes[idx][2]),
+ "height": int(boxes[idx][3]),
+ },
+ "domain": "pixel",
+ "class_id": int(classes[idx]),
+ "box_caption": class_id_to_label[int(classes[idx])],
+ "scores": {"confidence": float(confidence[idx])},
+ }
+ )
+ boxes = {
+ "predictions": {
+ "box_data": box_data,
+ "class_labels": class_id_to_label,
+ },
+ }
+ return boxes, mean_confidence_map
+
+
+def plot_predictions(
+ result: Results, model_name: str, table: Optional[wandb.Table] = None
+) -> Union[wandb.Table, Tuple[wandb.Image, Dict, Dict]]:
+ """Plot the images with the W&B overlay system. The `wandb.Image` is either added to a `wandb.Table` or returned."""
+ result = result.to("cpu")
+ boxes, mean_confidence_map = get_boxes(result)
+ image = wandb.Image(result.orig_img[:, :, ::-1], boxes=boxes)
+ if table is not None:
+ table.add_data(
+ model_name,
+ image,
+ len(boxes["predictions"]["box_data"]),
+ mean_confidence_map,
+ result.speed,
+ )
+ return table
+ return image, boxes["predictions"], mean_confidence_map
+
+
+def plot_validation_results(
+ dataloader: Any,
+ class_label_map: Dict,
+ model_name: str,
+ predictor: DetectionPredictor,
+ table: wandb.Table,
+ max_validation_batches: int,
+ epoch: Optional[int] = None,
+) -> wandb.Table:
+ """Plot validation results in a table."""
+ data_idx = 0
+ for batch_idx, batch in enumerate(dataloader):
+ for img_idx, image_path in enumerate(batch["im_file"]):
+ prediction_result = predictor(image_path)[0]
+ _, prediction_box_data, mean_confidence_map = plot_predictions(
+ prediction_result, model_name
+ )
+ try:
+ ground_truth_data = get_ground_truth_bbox_annotations(
+ img_idx, image_path, batch, class_label_map
+ )
+ wandb_image = wandb.Image(
+ image_path,
+ boxes={
+ "ground-truth": {
+ "box_data": ground_truth_data,
+ "class_labels": class_label_map,
+ },
+ "predictions": {
+ "box_data": prediction_box_data["box_data"],
+ "class_labels": class_label_map,
+ },
+ },
+ )
+ table_rows = [
+ data_idx,
+ batch_idx,
+ wandb_image,
+ mean_confidence_map,
+ prediction_result.speed,
+ ]
+ table_rows = [epoch] + table_rows if epoch is not None else table_rows
+ table_rows = [model_name] + table_rows
+ table.add_data(*table_rows)
+ data_idx += 1
+ except TypeError:
+ pass
+ if batch_idx + 1 == max_validation_batches:
+ break
+ return table
diff --git a/wandb/integration/ultralytics/callback.py b/wandb/integration/ultralytics/callback.py
new file mode 100644
index 0000000000000000000000000000000000000000..618fc5affc8226b6816b2f90e0e7879ab6df23a9
--- /dev/null
+++ b/wandb/integration/ultralytics/callback.py
@@ -0,0 +1,473 @@
+import copy
+from datetime import datetime
+from typing import Callable, Dict, Optional, Union
+
+from packaging import version
+
+try:
+ import dill as pickle
+except ImportError:
+ import pickle
+
+import wandb
+from wandb.sdk.lib import telemetry
+
+try:
+ import torch
+ import ultralytics
+ from tqdm.auto import tqdm
+
+ if version.parse(ultralytics.__version__) > version.parse("8.0.186"):
+ wandb.termwarn(
+ """This integration is tested and supported for ultralytics v8.0.186 and below.
+ Please report any issues to https://github.com/wandb/wandb/issues with the tag `yolov8`.""",
+ repeat=False,
+ )
+
+ from ultralytics.models import YOLO
+ from ultralytics.models.yolo.classify import (
+ ClassificationPredictor,
+ ClassificationTrainer,
+ ClassificationValidator,
+ )
+ from ultralytics.models.yolo.detect import (
+ DetectionPredictor,
+ DetectionTrainer,
+ DetectionValidator,
+ )
+ from ultralytics.models.yolo.pose import PosePredictor, PoseTrainer, PoseValidator
+ from ultralytics.models.yolo.segment import (
+ SegmentationPredictor,
+ SegmentationTrainer,
+ SegmentationValidator,
+ )
+ from ultralytics.utils.torch_utils import de_parallel
+
+ try:
+ from ultralytics.yolo.utils import RANK, __version__
+ except ModuleNotFoundError:
+ from ultralytics.utils import RANK, __version__
+
+ from wandb.integration.ultralytics.bbox_utils import (
+ plot_predictions,
+ plot_validation_results,
+ )
+ from wandb.integration.ultralytics.classification_utils import (
+ plot_classification_predictions,
+ plot_classification_validation_results,
+ )
+ from wandb.integration.ultralytics.mask_utils import (
+ plot_mask_predictions,
+ plot_mask_validation_results,
+ )
+ from wandb.integration.ultralytics.pose_utils import (
+ plot_pose_predictions,
+ plot_pose_validation_results,
+ )
+except ImportError as e:
+ wandb.Error(e)
+
+
+TRAINER_TYPE = Union[
+ ClassificationTrainer, DetectionTrainer, SegmentationTrainer, PoseTrainer
+]
+VALIDATOR_TYPE = Union[
+ ClassificationValidator, DetectionValidator, SegmentationValidator, PoseValidator
+]
+PREDICTOR_TYPE = Union[
+ ClassificationPredictor, DetectionPredictor, SegmentationPredictor, PosePredictor
+]
+
+
+class WandBUltralyticsCallback:
+ """Stateful callback for logging to W&B.
+
+ In particular, it will log model checkpoints, predictions, and
+ ground-truth annotations with interactive overlays for bounding boxes
+ to Weights & Biases Tables during training, validation and prediction
+ for a `ultratytics` workflow.
+
+ **Usage:**
+
+ ```python
+ from ultralytics.yolo.engine.model import YOLO
+ from wandb.yolov8 import add_wandb_callback
+
+ # initialize YOLO model
+ model = YOLO("yolov8n.pt")
+
+ # add wandb callback
+ add_wandb_callback(model, max_validation_batches=2, enable_model_checkpointing=True)
+
+ # train
+ model.train(data="coco128.yaml", epochs=5, imgsz=640)
+
+ # validate
+ model.val()
+
+ # perform inference
+ model(["img1.jpeg", "img2.jpeg"])
+ ```
+
+ Args:
+ model: YOLO Model of type `:class:ultralytics.yolo.engine.model.YOLO`.
+ max_validation_batches: maximum number of validation batches to log to
+ a table per epoch.
+ enable_model_checkpointing: enable logging model checkpoints as
+ artifacts at the end of eveny epoch if set to `True`.
+ visualize_skeleton: visualize pose skeleton by drawing lines connecting
+ keypoints for human pose.
+ """
+
+ def __init__(
+ self,
+ model: YOLO,
+ max_validation_batches: int = 1,
+ enable_model_checkpointing: bool = False,
+ visualize_skeleton: bool = False,
+ ) -> None:
+ self.max_validation_batches = max_validation_batches
+ self.enable_model_checkpointing = enable_model_checkpointing
+ self.visualize_skeleton = visualize_skeleton
+ self.task = model.task
+ self.task_map = model.task_map
+ self.model_name = model.overrides["model"].split(".")[0]
+ self._make_tables()
+ self._make_predictor(model)
+ self.supported_tasks = ["detect", "segment", "pose", "classify"]
+
+ def _make_tables(self):
+ if self.task in ["detect", "segment"]:
+ validation_columns = [
+ "Data-Index",
+ "Batch-Index",
+ "Image",
+ "Mean-Confidence",
+ "Speed",
+ ]
+ train_columns = ["Epoch"] + validation_columns
+ self.train_validation_table = wandb.Table(
+ columns=["Model-Name"] + train_columns
+ )
+ self.validation_table = wandb.Table(
+ columns=["Model-Name"] + validation_columns
+ )
+ self.prediction_table = wandb.Table(
+ columns=[
+ "Model-Name",
+ "Image",
+ "Num-Objects",
+ "Mean-Confidence",
+ "Speed",
+ ]
+ )
+ elif self.task == "classify":
+ classification_columns = [
+ "Image",
+ "Predicted-Category",
+ "Prediction-Confidence",
+ "Top-5-Prediction-Categories",
+ "Top-5-Prediction-Confindence",
+ "Probabilities",
+ "Speed",
+ ]
+ validation_columns = ["Data-Index", "Batch-Index"] + classification_columns
+ validation_columns.insert(3, "Ground-Truth-Category")
+ self.train_validation_table = wandb.Table(
+ columns=["Model-Name", "Epoch"] + validation_columns
+ )
+ self.validation_table = wandb.Table(
+ columns=["Model-Name"] + validation_columns
+ )
+ self.prediction_table = wandb.Table(
+ columns=["Model-Name"] + classification_columns
+ )
+ elif self.task == "pose":
+ validation_columns = [
+ "Data-Index",
+ "Batch-Index",
+ "Image-Ground-Truth",
+ "Image-Prediction",
+ "Num-Instances",
+ "Mean-Confidence",
+ "Speed",
+ ]
+ train_columns = ["Epoch"] + validation_columns
+ self.train_validation_table = wandb.Table(
+ columns=["Model-Name"] + train_columns
+ )
+ self.validation_table = wandb.Table(
+ columns=["Model-Name"] + validation_columns
+ )
+ self.prediction_table = wandb.Table(
+ columns=[
+ "Model-Name",
+ "Image-Prediction",
+ "Num-Instances",
+ "Mean-Confidence",
+ "Speed",
+ ]
+ )
+
+ def _make_predictor(self, model: YOLO):
+ overrides = copy.deepcopy(model.overrides)
+ overrides["conf"] = 0.1
+ self.predictor = self.task_map[self.task]["predictor"](
+ overrides=overrides, _callbacks=None
+ )
+
+ def _save_model(self, trainer: TRAINER_TYPE):
+ model_checkpoint_artifact = wandb.Artifact(
+ f"run_{wandb.run.id}_model", "model", metadata=vars(trainer.args)
+ )
+ checkpoint_dict = {
+ "epoch": trainer.epoch,
+ "best_fitness": trainer.best_fitness,
+ "model": copy.deepcopy(de_parallel(self.model)).half(),
+ "ema": copy.deepcopy(trainer.ema.ema).half(),
+ "updates": trainer.ema.updates,
+ "optimizer": trainer.optimizer.state_dict(),
+ "train_args": vars(trainer.args),
+ "date": datetime.now().isoformat(),
+ "version": __version__,
+ }
+ checkpoint_path = trainer.wdir / f"epoch{trainer.epoch}.pt"
+ torch.save(checkpoint_dict, checkpoint_path, pickle_module=pickle)
+ model_checkpoint_artifact.add_file(checkpoint_path)
+ wandb.log_artifact(
+ model_checkpoint_artifact, aliases=[f"epoch_{trainer.epoch}"]
+ )
+
+ def on_train_start(self, trainer: TRAINER_TYPE):
+ with telemetry.context(run=wandb.run) as tel:
+ tel.feature.ultralytics_yolov8 = True
+ wandb.config.train = vars(trainer.args)
+
+ def on_fit_epoch_end(self, trainer: TRAINER_TYPE):
+ if self.task in self.supported_tasks:
+ validator = trainer.validator
+ dataloader = validator.dataloader
+ class_label_map = validator.names
+ with torch.no_grad():
+ self.device = next(trainer.model.parameters()).device
+ if isinstance(trainer.model, torch.nn.parallel.DistributedDataParallel):
+ model = trainer.model.module
+ else:
+ model = trainer.model
+ self.model = copy.deepcopy(model).eval().to(self.device)
+ self.predictor.setup_model(model=self.model, verbose=False)
+ if self.task == "pose":
+ self.train_validation_table = plot_pose_validation_results(
+ dataloader=dataloader,
+ class_label_map=class_label_map,
+ model_name=self.model_name,
+ predictor=self.predictor,
+ visualize_skeleton=self.visualize_skeleton,
+ table=self.train_validation_table,
+ max_validation_batches=self.max_validation_batches,
+ epoch=trainer.epoch,
+ )
+ elif self.task == "segment":
+ self.train_validation_table = plot_mask_validation_results(
+ dataloader=dataloader,
+ class_label_map=class_label_map,
+ model_name=self.model_name,
+ predictor=self.predictor,
+ table=self.train_validation_table,
+ max_validation_batches=self.max_validation_batches,
+ epoch=trainer.epoch,
+ )
+ elif self.task == "detect":
+ self.train_validation_table = plot_validation_results(
+ dataloader=dataloader,
+ class_label_map=class_label_map,
+ model_name=self.model_name,
+ predictor=self.predictor,
+ table=self.train_validation_table,
+ max_validation_batches=self.max_validation_batches,
+ epoch=trainer.epoch,
+ )
+ elif self.task == "classify":
+ self.train_validation_table = (
+ plot_classification_validation_results(
+ dataloader=dataloader,
+ model_name=self.model_name,
+ predictor=self.predictor,
+ table=self.train_validation_table,
+ max_validation_batches=self.max_validation_batches,
+ epoch=trainer.epoch,
+ )
+ )
+ if self.enable_model_checkpointing:
+ self._save_model(trainer)
+ self.model.to("cpu")
+ trainer.model.to(self.device)
+
+ def on_train_end(self, trainer: TRAINER_TYPE):
+ if self.task in self.supported_tasks:
+ wandb.log({"Train-Validation-Table": self.train_validation_table})
+
+ def on_val_end(self, trainer: VALIDATOR_TYPE):
+ if self.task in self.supported_tasks:
+ validator = trainer
+ dataloader = validator.dataloader
+ class_label_map = validator.names
+ with torch.no_grad():
+ self.model.to(self.device)
+ self.predictor.setup_model(model=self.model, verbose=False)
+ if self.task == "pose":
+ self.validation_table = plot_pose_validation_results(
+ dataloader=dataloader,
+ class_label_map=class_label_map,
+ model_name=self.model_name,
+ predictor=self.predictor,
+ visualize_skeleton=self.visualize_skeleton,
+ table=self.validation_table,
+ max_validation_batches=self.max_validation_batches,
+ )
+ elif self.task == "segment":
+ self.validation_table = plot_mask_validation_results(
+ dataloader=dataloader,
+ class_label_map=class_label_map,
+ model_name=self.model_name,
+ predictor=self.predictor,
+ table=self.validation_table,
+ max_validation_batches=self.max_validation_batches,
+ )
+ elif self.task == "detect":
+ self.validation_table = plot_validation_results(
+ dataloader=dataloader,
+ class_label_map=class_label_map,
+ model_name=self.model_name,
+ predictor=self.predictor,
+ table=self.validation_table,
+ max_validation_batches=self.max_validation_batches,
+ )
+ elif self.task == "classify":
+ self.validation_table = plot_classification_validation_results(
+ dataloader=dataloader,
+ model_name=self.model_name,
+ predictor=self.predictor,
+ table=self.validation_table,
+ max_validation_batches=self.max_validation_batches,
+ )
+ wandb.log({"Validation-Table": self.validation_table})
+
+ def on_predict_end(self, predictor: PREDICTOR_TYPE):
+ wandb.config.prediction_configs = vars(predictor.args)
+ if self.task in self.supported_tasks:
+ for result in tqdm(predictor.results):
+ if self.task == "pose":
+ self.prediction_table = plot_pose_predictions(
+ result,
+ self.model_name,
+ self.visualize_skeleton,
+ self.prediction_table,
+ )
+ elif self.task == "segment":
+ self.prediction_table = plot_mask_predictions(
+ result, self.model_name, self.prediction_table
+ )
+ elif self.task == "detect":
+ self.prediction_table = plot_predictions(
+ result, self.model_name, self.prediction_table
+ )
+ elif self.task == "classify":
+ self.prediction_table = plot_classification_predictions(
+ result, self.model_name, self.prediction_table
+ )
+
+ wandb.log({"Prediction-Table": self.prediction_table})
+
+ @property
+ def callbacks(self) -> Dict[str, Callable]:
+ """Property contains all the relevant callbacks to add to the YOLO model for the Weights & Biases logging."""
+ return {
+ "on_train_start": self.on_train_start,
+ "on_fit_epoch_end": self.on_fit_epoch_end,
+ "on_train_end": self.on_train_end,
+ "on_val_end": self.on_val_end,
+ "on_predict_end": self.on_predict_end,
+ }
+
+
+def add_wandb_callback(
+ model: YOLO,
+ enable_model_checkpointing: bool = False,
+ enable_train_validation_logging: bool = True,
+ enable_validation_logging: bool = True,
+ enable_prediction_logging: bool = True,
+ max_validation_batches: Optional[int] = 1,
+ visualize_skeleton: Optional[bool] = True,
+):
+ """Function to add the `WandBUltralyticsCallback` callback to the `YOLO` model.
+
+ **Usage:**
+
+ ```python
+ from ultralytics.yolo.engine.model import YOLO
+ from wandb.yolov8 import add_wandb_callback
+
+ # initialize YOLO model
+ model = YOLO("yolov8n.pt")
+
+ # add wandb callback
+ add_wandb_callback(model, max_validation_batches=2, enable_model_checkpointing=True)
+
+ # train
+ model.train(data="coco128.yaml", epochs=5, imgsz=640)
+
+ # validate
+ model.val()
+
+ # perform inference
+ model(["img1.jpeg", "img2.jpeg"])
+ ```
+
+ Args:
+ model: YOLO Model of type `:class:ultralytics.yolo.engine.model.YOLO`.
+ enable_model_checkpointing: enable logging model checkpoints as
+ artifacts at the end of eveny epoch if set to `True`.
+ enable_train_validation_logging: enable logging the predictions and
+ ground-truths as interactive image overlays on the images from
+ the validation dataloader to a `wandb.Table` along with
+ mean-confidence of the predictions per-class at the end of each
+ training epoch.
+ enable_validation_logging: enable logging the predictions and
+ ground-truths as interactive image overlays on the images from the
+ validation dataloader to a `wandb.Table` along with
+ mean-confidence of the predictions per-class at the end of
+ validation.
+ enable_prediction_logging: enable logging the predictions and
+ ground-truths as interactive image overlays on the images from the
+ validation dataloader to a `wandb.Table` along with mean-confidence
+ of the predictions per-class at the end of each prediction.
+ max_validation_batches: maximum number of validation batches to log to
+ a table per epoch.
+ visualize_skeleton: visualize pose skeleton by drawing lines connecting
+ keypoints for human pose.
+ """
+ if RANK in [-1, 0]:
+ wandb_callback = WandBUltralyticsCallback(
+ copy.deepcopy(model),
+ max_validation_batches,
+ enable_model_checkpointing,
+ visualize_skeleton,
+ )
+ callbacks = wandb_callback.callbacks
+ if not enable_train_validation_logging:
+ _ = callbacks.pop("on_fit_epoch_end")
+ _ = callbacks.pop("on_train_end")
+ if not enable_validation_logging:
+ _ = callbacks.pop("on_val_end")
+ if not enable_prediction_logging:
+ _ = callbacks.pop("on_predict_end")
+ for event, callback_fn in callbacks.items():
+ model.add_callback(event, callback_fn)
+ else:
+ wandb.termerror(
+ "The RANK of the process to add the callbacks was neither 0 or "
+ "-1. No Weights & Biases callbacks were added to this instance "
+ "of the YOLO model."
+ )
+ return model
diff --git a/wandb/integration/ultralytics/classification_utils.py b/wandb/integration/ultralytics/classification_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..12938113602d33c2cff75a2918dfd3afe8469ff1
--- /dev/null
+++ b/wandb/integration/ultralytics/classification_utils.py
@@ -0,0 +1,66 @@
+from typing import Any, Optional
+
+import numpy as np
+from ultralytics.engine.results import Results
+from ultralytics.models.yolo.classify import ClassificationPredictor
+
+import wandb
+
+
+def plot_classification_predictions(
+ result: Results, model_name: str, table: Optional[wandb.Table] = None
+):
+ """Plot classification prediction results to a `wandb.Table` if the table is passed otherwise return the data."""
+ result = result.to("cpu")
+ probabilities = result.probs
+ probabilities_list = probabilities.data.numpy().tolist()
+ class_id_to_label = {int(k): str(v) for k, v in result.names.items()}
+ table_row = [
+ model_name,
+ wandb.Image(result.orig_img),
+ class_id_to_label[int(probabilities.top1)],
+ probabilities.top1conf,
+ [class_id_to_label[int(class_idx)] for class_idx in list(probabilities.top5)],
+ [probabilities_list[int(class_idx)] for class_idx in list(probabilities.top5)],
+ {
+ class_id_to_label[int(class_idx)]: probability
+ for class_idx, probability in enumerate(probabilities_list)
+ },
+ result.speed,
+ ]
+ if table is not None:
+ table.add_data(*table_row)
+ return table
+ return class_id_to_label, table_row
+
+
+def plot_classification_validation_results(
+ dataloader: Any,
+ model_name: str,
+ predictor: ClassificationPredictor,
+ table: wandb.Table,
+ max_validation_batches: int,
+ epoch: Optional[int] = None,
+):
+ """Plot classification results to a `wandb.Table`."""
+ data_idx = 0
+ predictor.args.save = False
+ predictor.args.show = False
+ for batch_idx, batch in enumerate(dataloader):
+ image_batch = batch["img"].numpy()
+ ground_truth = batch["cls"].numpy().tolist()
+ for img_idx in range(image_batch.shape[0]):
+ image = np.transpose(image_batch[img_idx], (1, 2, 0))
+ prediction_result = predictor(image, show=False)[0]
+ class_id_to_label, table_row = plot_classification_predictions(
+ prediction_result, model_name
+ )
+ table_row = [data_idx, batch_idx] + table_row[1:]
+ table_row.insert(3, class_id_to_label[ground_truth[img_idx]])
+ table_row = [epoch] + table_row if epoch is not None else table_row
+ table_row = [model_name] + table_row
+ table.add_data(*table_row)
+ data_idx += 1
+ if batch_idx + 1 == max_validation_batches:
+ break
+ return table
diff --git a/wandb/integration/ultralytics/mask_utils.py b/wandb/integration/ultralytics/mask_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..ad7edeff4c198662c058f2e4b822b94be44edc28
--- /dev/null
+++ b/wandb/integration/ultralytics/mask_utils.py
@@ -0,0 +1,141 @@
+from typing import Dict, Optional, Tuple
+
+import numpy as np
+from ultralytics.engine.results import Results
+from ultralytics.models.yolo.segment import SegmentationPredictor
+from ultralytics.utils.ops import scale_image
+
+import wandb
+
+from .bbox_utils import get_ground_truth_bbox_annotations, get_mean_confidence_map
+
+
+def instance_mask_to_semantic_mask(instance_mask, class_indices):
+ height, width, num_instances = instance_mask.shape
+ semantic_mask = np.zeros((height, width), dtype=np.uint8)
+ for i in range(num_instances):
+ instance_map = instance_mask[:, :, i]
+ class_index = class_indices[i]
+ semantic_mask[instance_map == 1] = class_index
+ return semantic_mask
+
+
+def get_boxes_and_masks(result: Results) -> Tuple[Dict, Dict, Dict]:
+ boxes = result.boxes.xywh.long().numpy()
+ classes = result.boxes.cls.long().numpy()
+ confidence = result.boxes.conf.numpy()
+ class_id_to_label = {int(k): str(v) for k, v in result.names.items()}
+ class_id_to_label.update({len(result.names.items()): "background"})
+ mean_confidence_map = get_mean_confidence_map(
+ classes, confidence, class_id_to_label
+ )
+ masks = None
+ if result.masks is not None:
+ scaled_instance_mask = scale_image(
+ np.transpose(result.masks.data.numpy(), (1, 2, 0)),
+ result.orig_img[:, :, ::-1].shape,
+ )
+ scaled_semantic_mask = instance_mask_to_semantic_mask(
+ scaled_instance_mask, classes.tolist()
+ )
+ scaled_semantic_mask[scaled_semantic_mask == 0] = len(result.names.items())
+ masks = {
+ "predictions": {
+ "mask_data": scaled_semantic_mask,
+ "class_labels": class_id_to_label,
+ }
+ }
+ box_data, total_confidence = [], 0.0
+ for idx in range(len(boxes)):
+ box_data.append(
+ {
+ "position": {
+ "middle": [int(boxes[idx][0]), int(boxes[idx][1])],
+ "width": int(boxes[idx][2]),
+ "height": int(boxes[idx][3]),
+ },
+ "domain": "pixel",
+ "class_id": int(classes[idx]),
+ "box_caption": class_id_to_label[int(classes[idx])],
+ "scores": {"confidence": float(confidence[idx])},
+ }
+ )
+ total_confidence += float(confidence[idx])
+
+ boxes = {
+ "predictions": {
+ "box_data": box_data,
+ "class_labels": class_id_to_label,
+ },
+ }
+ return boxes, masks, mean_confidence_map
+
+
+def plot_mask_predictions(
+ result: Results, model_name: str, table: Optional[wandb.Table] = None
+) -> Tuple[wandb.Image, Dict, Dict, Dict]:
+ result = result.to("cpu")
+ boxes, masks, mean_confidence_map = get_boxes_and_masks(result)
+ image = wandb.Image(result.orig_img[:, :, ::-1], boxes=boxes, masks=masks)
+ if table is not None:
+ table.add_data(
+ model_name,
+ image,
+ len(boxes["predictions"]["box_data"]),
+ mean_confidence_map,
+ result.speed,
+ )
+ return table
+ return image, masks, boxes["predictions"], mean_confidence_map
+
+
+def plot_mask_validation_results(
+ dataloader,
+ class_label_map,
+ model_name: str,
+ predictor: SegmentationPredictor,
+ table: wandb.Table,
+ max_validation_batches: int,
+ epoch: Optional[int] = None,
+):
+ data_idx = 0
+ for batch_idx, batch in enumerate(dataloader):
+ for img_idx, image_path in enumerate(batch["im_file"]):
+ prediction_result = predictor(image_path)[0]
+ (
+ _,
+ prediction_mask_data,
+ prediction_box_data,
+ mean_confidence_map,
+ ) = plot_mask_predictions(prediction_result, model_name)
+ try:
+ ground_truth_data = get_ground_truth_bbox_annotations(
+ img_idx, image_path, batch, class_label_map
+ )
+ wandb_image = wandb.Image(
+ image_path,
+ boxes={
+ "ground-truth": {
+ "box_data": ground_truth_data,
+ "class_labels": class_label_map,
+ },
+ "predictions": prediction_box_data,
+ },
+ masks=prediction_mask_data,
+ )
+ table_rows = [
+ data_idx,
+ batch_idx,
+ wandb_image,
+ mean_confidence_map,
+ prediction_result.speed,
+ ]
+ table_rows = [epoch] + table_rows if epoch is not None else table_rows
+ table_rows = [model_name] + table_rows
+ table.add_data(*table_rows)
+ data_idx += 1
+ except TypeError:
+ pass
+ if batch_idx + 1 == max_validation_batches:
+ break
+ return table
diff --git a/wandb/integration/ultralytics/pose_utils.py b/wandb/integration/ultralytics/pose_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..ac05a55418833085a190e583af2330edf5323245
--- /dev/null
+++ b/wandb/integration/ultralytics/pose_utils.py
@@ -0,0 +1,92 @@
+from typing import Any, Optional
+
+import numpy as np
+from PIL import Image
+from ultralytics.engine.results import Results
+from ultralytics.models.yolo.pose import PosePredictor
+from ultralytics.utils.plotting import Annotator
+
+import wandb
+
+from .bbox_utils import get_boxes, get_ground_truth_bbox_annotations
+
+
+def annotate_keypoint_results(result: Results, visualize_skeleton: bool):
+ annotator = Annotator(np.ascontiguousarray(result.orig_img[:, :, ::-1]))
+ key_points = result.keypoints.data.numpy()
+ for idx in range(key_points.shape[0]):
+ annotator.kpts(key_points[idx], kpt_line=visualize_skeleton)
+ return annotator.im
+
+
+def annotate_keypoint_batch(image_path: str, keypoints: Any, visualize_skeleton: bool):
+ original_image = None
+ with Image.open(image_path) as original_image:
+ original_image = np.ascontiguousarray(original_image)
+ annotator = Annotator(original_image)
+ annotator.kpts(keypoints.numpy(), kpt_line=visualize_skeleton)
+ return annotator.im
+
+
+def plot_pose_predictions(
+ result: Results,
+ model_name: str,
+ visualize_skeleton: bool,
+ table: Optional[wandb.Table] = None,
+):
+ result = result.to("cpu")
+ boxes, mean_confidence_map = get_boxes(result)
+ prediction_image = wandb.Image(
+ annotate_keypoint_results(result, visualize_skeleton), boxes=boxes
+ )
+ table_row = [
+ model_name,
+ prediction_image,
+ len(boxes["predictions"]["box_data"]),
+ mean_confidence_map,
+ result.speed,
+ ]
+ if table is not None:
+ table.add_data(*table_row)
+ return table
+ return table_row
+
+
+def plot_pose_validation_results(
+ dataloader,
+ class_label_map,
+ model_name: str,
+ predictor: PosePredictor,
+ visualize_skeleton: bool,
+ table: wandb.Table,
+ max_validation_batches: int,
+ epoch: Optional[int] = None,
+) -> wandb.Table:
+ data_idx = 0
+ for batch_idx, batch in enumerate(dataloader):
+ for img_idx, image_path in enumerate(batch["im_file"]):
+ prediction_result = predictor(image_path)[0].to("cpu")
+ table_row = plot_pose_predictions(
+ prediction_result, model_name, visualize_skeleton
+ )
+ ground_truth_image = wandb.Image(
+ annotate_keypoint_batch(
+ image_path, batch["keypoints"][img_idx], visualize_skeleton
+ ),
+ boxes={
+ "ground-truth": {
+ "box_data": get_ground_truth_bbox_annotations(
+ img_idx, image_path, batch, class_label_map
+ ),
+ "class_labels": class_label_map,
+ },
+ },
+ )
+ table_row = [data_idx, batch_idx, ground_truth_image] + table_row[1:]
+ table_row = [epoch] + table_row if epoch is not None else table_row
+ table_row = [model_name] + table_row
+ table.add_data(*table_row)
+ data_idx += 1
+ if batch_idx + 1 == max_validation_batches:
+ break
+ return table
diff --git a/wandb/integration/xgboost/__init__.py b/wandb/integration/xgboost/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..052083e0cf7034a469a82582422ff7fee62965a5
--- /dev/null
+++ b/wandb/integration/xgboost/__init__.py
@@ -0,0 +1,11 @@
+"""W&B callback for xgboost.
+
+Simple callback to get logging for each tree
+
+Use the `wandb_callback` to add `wandb` logging to any `XGboost` model. However, it will
+be deprecated in favor of WandbCallback. Use it instead for more features.
+"""
+
+from .xgboost import WandbCallback, wandb_callback
+
+__all__ = ["wandb_callback", "WandbCallback"]
diff --git a/wandb/integration/xgboost/xgboost.py b/wandb/integration/xgboost/xgboost.py
new file mode 100644
index 0000000000000000000000000000000000000000..9b5cdbdce1e9afe407a6b60dfd752e73bad82cd6
--- /dev/null
+++ b/wandb/integration/xgboost/xgboost.py
@@ -0,0 +1,189 @@
+"""xgboost init!"""
+
+import json
+import warnings
+from pathlib import Path
+from typing import TYPE_CHECKING, cast
+
+import xgboost as xgb # type: ignore
+from xgboost import Booster
+
+import wandb
+from wandb.sdk.lib import telemetry as wb_telemetry
+
+MINIMIZE_METRICS = [
+ "rmse",
+ "rmsle",
+ "mae",
+ "mape",
+ "mphe",
+ "logloss",
+ "error",
+ "error@t",
+ "merror",
+]
+
+MAXIMIZE_METRICS = ["auc", "aucpr", "ndcg", "map", "ndcg@n", "map@n"]
+
+
+if TYPE_CHECKING:
+ from typing import Callable, List, NamedTuple
+
+ class CallbackEnv(NamedTuple):
+ evaluation_result_list: List
+
+
+def wandb_callback() -> "Callable":
+ """Old style callback that will be deprecated in favor of WandbCallback. Please try the new logger for more features."""
+ warnings.warn(
+ "wandb_callback will be deprecated in favor of WandbCallback. Please use WandbCallback for more features.",
+ UserWarning,
+ stacklevel=2,
+ )
+
+ with wb_telemetry.context() as tel:
+ tel.feature.xgboost_old_wandb_callback = True
+
+ def callback(env: "CallbackEnv") -> None:
+ for k, v in env.evaluation_result_list:
+ wandb.log({k: v}, commit=False)
+ wandb.log({})
+
+ return callback
+
+
+class WandbCallback(xgb.callback.TrainingCallback):
+ """`WandbCallback` automatically integrates XGBoost with wandb.
+
+ Arguments:
+ log_model: (boolean) if True save and upload the model to Weights & Biases Artifacts
+ log_feature_importance: (boolean) if True log a feature importance bar plot
+ importance_type: (str) one of {weight, gain, cover, total_gain, total_cover} for tree model. weight for linear model.
+ define_metric: (boolean) if True (default) capture model performance at the best step, instead of the last step, of training in your `wandb.summary`.
+
+ Passing `WandbCallback` to XGBoost will:
+
+ - log the booster model configuration to Weights & Biases
+ - log evaluation metrics collected by XGBoost, such as rmse, accuracy etc. to Weights & Biases
+ - log training metric collected by XGBoost (if you provide training data to eval_set)
+ - log the best score and the best iteration
+ - save and upload your trained model to Weights & Biases Artifacts (when `log_model = True`)
+ - log feature importance plot when `log_feature_importance=True` (default).
+ - Capture the best eval metric in `wandb.summary` when `define_metric=True` (default).
+
+ Example:
+ ```python
+ bst_params = dict(
+ objective="reg:squarederror",
+ colsample_bytree=0.3,
+ learning_rate=0.1,
+ max_depth=5,
+ alpha=10,
+ n_estimators=10,
+ tree_method="hist",
+ callbacks=[WandbCallback()],
+ )
+
+ xg_reg = xgb.XGBRegressor(**bst_params)
+ xg_reg.fit(
+ X_train,
+ y_train,
+ eval_set=[(X_test, y_test)],
+ )
+ ```
+ """
+
+ def __init__(
+ self,
+ log_model: bool = False,
+ log_feature_importance: bool = True,
+ importance_type: str = "gain",
+ define_metric: bool = True,
+ ):
+ self.log_model: bool = log_model
+ self.log_feature_importance: bool = log_feature_importance
+ self.importance_type: str = importance_type
+ self.define_metric: bool = define_metric
+
+ if wandb.run is None:
+ raise wandb.Error("You must call wandb.init() before WandbCallback()")
+
+ with wb_telemetry.context() as tel:
+ tel.feature.xgboost_wandb_callback = True
+
+ def before_training(self, model: Booster) -> Booster:
+ """Run before training is finished."""
+ # Update W&B config
+ config = model.save_config()
+ wandb.config.update(json.loads(config))
+
+ return model
+
+ def after_training(self, model: Booster) -> Booster:
+ """Run after training is finished."""
+ # Log the booster model as artifacts
+ if self.log_model:
+ self._log_model_as_artifact(model)
+
+ # Plot feature importance
+ if self.log_feature_importance:
+ self._log_feature_importance(model)
+
+ # Log the best score and best iteration
+ if model.attr("best_score") is not None:
+ wandb.log(
+ {
+ "best_score": float(cast(str, model.attr("best_score"))),
+ "best_iteration": int(cast(str, model.attr("best_iteration"))),
+ }
+ )
+
+ return model
+
+ def after_iteration(self, model: Booster, epoch: int, evals_log: dict) -> bool:
+ """Run after each iteration. Return True when training should stop."""
+ # Log metrics
+ for data, metric in evals_log.items():
+ for metric_name, log in metric.items():
+ if self.define_metric:
+ self._define_metric(data, metric_name)
+ wandb.log({f"{data}-{metric_name}": log[-1]}, commit=False)
+ else:
+ wandb.log({f"{data}-{metric_name}": log[-1]}, commit=False)
+
+ wandb.log({"epoch": epoch})
+
+ self.define_metric = False
+
+ return False
+
+ def _log_model_as_artifact(self, model: Booster) -> None:
+ model_name = f"{wandb.run.id}_model.json" # type: ignore
+ model_path = Path(wandb.run.dir) / model_name # type: ignore
+ model.save_model(str(model_path))
+
+ model_artifact = wandb.Artifact(name=model_name, type="model")
+ model_artifact.add_file(str(model_path))
+ wandb.log_artifact(model_artifact)
+
+ def _log_feature_importance(self, model: Booster) -> None:
+ fi = model.get_score(importance_type=self.importance_type)
+ fi_data = [[k, fi[k]] for k in fi]
+ table = wandb.Table(data=fi_data, columns=["Feature", "Importance"])
+ wandb.log(
+ {
+ "Feature Importance": wandb.plot.bar(
+ table, "Feature", "Importance", title="Feature Importance"
+ )
+ }
+ )
+
+ def _define_metric(self, data: str, metric_name: str) -> None:
+ if "loss" in str.lower(metric_name):
+ wandb.define_metric(f"{data}-{metric_name}", summary="min")
+ elif str.lower(metric_name) in MINIMIZE_METRICS:
+ wandb.define_metric(f"{data}-{metric_name}", summary="min")
+ elif str.lower(metric_name) in MAXIMIZE_METRICS:
+ wandb.define_metric(f"{data}-{metric_name}", summary="max")
+ else:
+ pass
diff --git a/wandb/integration/yolov8/__init__.py b/wandb/integration/yolov8/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/integration/yolov8/yolov8.py b/wandb/integration/yolov8/yolov8.py
new file mode 100644
index 0000000000000000000000000000000000000000..1c5dbd0fe8de39f387b6427b4696f28f9c029b44
--- /dev/null
+++ b/wandb/integration/yolov8/yolov8.py
@@ -0,0 +1,284 @@
+from typing import Any, Callable, Dict, List, Optional
+
+from ultralytics.yolo.engine.model import YOLO
+from ultralytics.yolo.engine.trainer import BaseTrainer
+
+try:
+ from ultralytics.yolo.utils import RANK
+ from ultralytics.yolo.utils.torch_utils import get_flops, get_num_params
+except ModuleNotFoundError:
+ from ultralytics.utils import RANK
+ from ultralytics.utils.torch_utils import get_flops, get_num_params
+from ultralytics.yolo.v8.classify.train import ClassificationTrainer
+
+import wandb
+from wandb.sdk.lib import telemetry
+
+
+class WandbCallback:
+ """An internal YOLO model wrapper that tracks metrics, and logs models to Weights & Biases.
+
+ Usage:
+ ```python
+ from wandb.integration.yolov8.yolov8 import WandbCallback
+
+ model = YOLO("yolov8n.pt")
+ wandb_logger = WandbCallback(
+ model,
+ )
+ for event, callback_fn in wandb_logger.callbacks.items():
+ model.add_callback(event, callback_fn)
+ ```
+ """
+
+ def __init__(
+ self,
+ yolo: YOLO,
+ run_name: Optional[str] = None,
+ project: Optional[str] = None,
+ tags: Optional[List[str]] = None,
+ resume: Optional[str] = None,
+ **kwargs: Optional[Any],
+ ) -> None:
+ """A utility class to manage wandb run and various callbacks for the ultralytics YOLOv8 framework.
+
+ Args:
+ yolo: A YOLOv8 model that's inherited from `:class:ultralytics.yolo.engine.model.YOLO`
+ run_name, str: The name of the Weights & Biases run, defaults to an auto generated run_name if `trainer.args.name` is not defined.
+ project, str: The name of the Weights & Biases project, defaults to `"YOLOv8"` if `trainer.args.project` is not defined.
+ tags, List[str]: A list of tags to be added to the Weights & Biases run, defaults to `["YOLOv8"]`.
+ resume, str: Whether to resume a previous run on Weights & Biases, defaults to `None`.
+ **kwargs: Additional arguments to be passed to `wandb.init()`.
+ """
+ self.yolo = yolo
+ self.run_name = run_name
+ self.project = project
+ self.tags = tags
+ self.resume = resume
+ self.kwargs = kwargs
+
+ def on_pretrain_routine_start(self, trainer: BaseTrainer) -> None:
+ """Starts a new wandb run to track the training process and log to Weights & Biases.
+
+ Args:
+ trainer: A task trainer that's inherited from `:class:ultralytics.yolo.engine.trainer.BaseTrainer`
+ that contains the model training and optimization routine.
+ """
+ if wandb.run is None:
+ self.run = wandb.init(
+ name=self.run_name if self.run_name else trainer.args.name,
+ project=self.project
+ if self.project
+ else trainer.args.project or "YOLOv8",
+ tags=self.tags if self.tags else ["YOLOv8"],
+ config=vars(trainer.args),
+ resume=self.resume if self.resume else None,
+ **self.kwargs,
+ )
+ else:
+ self.run = wandb.run
+ assert self.run is not None
+ self.run.define_metric("epoch", hidden=True)
+ self.run.define_metric(
+ "train/*", step_metric="epoch", step_sync=True, summary="min"
+ )
+
+ self.run.define_metric(
+ "val/*", step_metric="epoch", step_sync=True, summary="min"
+ )
+
+ self.run.define_metric(
+ "metrics/*", step_metric="epoch", step_sync=True, summary="max"
+ )
+ self.run.define_metric(
+ "lr/*", step_metric="epoch", step_sync=True, summary="last"
+ )
+
+ with telemetry.context(run=wandb.run) as tel:
+ tel.feature.ultralytics_yolov8 = True
+
+ def on_pretrain_routine_end(self, trainer: BaseTrainer) -> None:
+ assert self.run is not None
+ self.run.summary.update(
+ {
+ "model/parameters": get_num_params(trainer.model),
+ "model/GFLOPs": round(get_flops(trainer.model), 3),
+ }
+ )
+
+ def on_train_epoch_start(self, trainer: BaseTrainer) -> None:
+ """On train epoch start we only log epoch number to the Weights & Biases run."""
+ # We log the epoch number here to commit the previous step,
+ assert self.run is not None
+ self.run.log({"epoch": trainer.epoch + 1})
+
+ def on_train_epoch_end(self, trainer: BaseTrainer) -> None:
+ """On train epoch end we log all the metrics to the Weights & Biases run."""
+ assert self.run is not None
+ self.run.log(
+ {
+ **trainer.metrics,
+ **trainer.label_loss_items(trainer.tloss, prefix="train"),
+ **trainer.lr,
+ },
+ )
+ # Currently only the detection and segmentation trainers save images to the save_dir
+ if not isinstance(trainer, ClassificationTrainer):
+ self.run.log(
+ {
+ "train_batch_images": [
+ wandb.Image(str(image_path), caption=image_path.stem)
+ for image_path in trainer.save_dir.glob("train_batch*.jpg")
+ ]
+ }
+ )
+
+ def on_fit_epoch_end(self, trainer: BaseTrainer) -> None:
+ """On fit epoch end we log all the best metrics and model detail to Weights & Biases run summary."""
+ assert self.run is not None
+ if trainer.epoch == 0:
+ speeds = [
+ trainer.validator.speed.get(
+ key,
+ )
+ for key in (1, "inference")
+ ]
+ speed = speeds[0] if speeds[0] else speeds[1]
+ if speed:
+ self.run.summary.update(
+ {
+ "model/speed(ms/img)": round(speed, 3),
+ }
+ )
+ if trainer.best_fitness == trainer.fitness:
+ self.run.summary.update(
+ {
+ "best/epoch": trainer.epoch + 1,
+ **{f"best/{key}": val for key, val in trainer.metrics.items()},
+ }
+ )
+
+ def on_train_end(self, trainer: BaseTrainer) -> None:
+ """On train end we log all the media, including plots, images and best model artifact to Weights & Biases."""
+ # Currently only the detection and segmentation trainers save images to the save_dir
+ assert self.run is not None
+ if not isinstance(trainer, ClassificationTrainer):
+ assert self.run is not None
+ self.run.log(
+ {
+ "plots": [
+ wandb.Image(str(image_path), caption=image_path.stem)
+ for image_path in trainer.save_dir.glob("*.png")
+ ],
+ "val_images": [
+ wandb.Image(str(image_path), caption=image_path.stem)
+ for image_path in trainer.validator.save_dir.glob("val*.jpg")
+ ],
+ },
+ )
+
+ if trainer.best.exists():
+ assert self.run is not None
+ self.run.log_artifact(
+ str(trainer.best),
+ type="model",
+ name=f"{self.run.name}_{trainer.args.task}.pt",
+ aliases=["best", f"epoch_{trainer.epoch + 1}"],
+ )
+
+ def on_model_save(self, trainer: BaseTrainer) -> None:
+ """On model save we log the model as an artifact to Weights & Biases."""
+ assert self.run is not None
+ self.run.log_artifact(
+ str(trainer.last),
+ type="model",
+ name=f"{self.run.name}_{trainer.args.task}.pt",
+ aliases=["last", f"epoch_{trainer.epoch + 1}"],
+ )
+
+ def teardown(self, _trainer: BaseTrainer) -> None:
+ """On teardown, we finish the Weights & Biases run and set it to None."""
+ assert self.run is not None
+ self.run.finish()
+ self.run = None
+
+ @property
+ def callbacks(
+ self,
+ ) -> Dict[str, Callable]:
+ """Property contains all the relevant callbacks to add to the YOLO model for the Weights & Biases logging."""
+ return {
+ "on_pretrain_routine_start": self.on_pretrain_routine_start,
+ "on_pretrain_routine_end": self.on_pretrain_routine_end,
+ "on_train_epoch_start": self.on_train_epoch_start,
+ "on_train_epoch_end": self.on_train_epoch_end,
+ "on_fit_epoch_end": self.on_fit_epoch_end,
+ "on_train_end": self.on_train_end,
+ "on_model_save": self.on_model_save,
+ "teardown": self.teardown,
+ }
+
+
+def add_callbacks(
+ yolo: YOLO,
+ run_name: Optional[str] = None,
+ project: Optional[str] = None,
+ tags: Optional[List[str]] = None,
+ resume: Optional[str] = None,
+ **kwargs: Optional[Any],
+) -> YOLO:
+ """A YOLO model wrapper that tracks metrics, and logs models to Weights & Biases.
+
+ Args:
+ yolo: A YOLOv8 model that's inherited from `:class:ultralytics.yolo.engine.model.YOLO`
+ run_name, str: The name of the Weights & Biases run, defaults to an auto generated name if `trainer.args.name` is not defined.
+ project, str: The name of the Weights & Biases project, defaults to `"YOLOv8"` if `trainer.args.project` is not defined.
+ tags, List[str]: A list of tags to be added to the Weights & Biases run, defaults to `["YOLOv8"]`.
+ resume, str: Whether to resume a previous run on Weights & Biases, defaults to `None`.
+ **kwargs: Additional arguments to be passed to `wandb.init()`.
+
+ Usage:
+ ```python
+ from wandb.integration.yolov8 import add_callbacks as add_wandb_callbacks
+
+ model = YOLO("yolov8n.pt")
+ add_wandb_callbacks(
+ model,
+ )
+ model.train(
+ data="coco128.yaml",
+ epochs=3,
+ imgsz=640,
+ )
+ ```
+ """
+ wandb.termwarn(
+ """The wandb callback is currently in beta and is subject to change based on updates to `ultralytics yolov8`.
+ The callback is tested and supported for ultralytics v8.0.43 and above.
+ Please report any issues to https://github.com/wandb/wandb/issues with the tag `yolov8`.
+ """,
+ repeat=False,
+ )
+ wandb.termwarn(
+ """This wandb callback is no longer functional and would be deprecated in the near future.
+ We recommend you to use the updated callback using `from wandb.integration.ultralytics import add_wandb_callback`.
+ The updated callback is tested and supported for ultralytics 8.0.167 and above.
+ You can refer to https://docs.wandb.ai/guides/integrations/ultralytics for the updated documentation.
+ Please report any issues to https://github.com/wandb/wandb/issues with the tag `yolov8`.
+ """,
+ repeat=False,
+ )
+
+ if RANK in [-1, 0]:
+ wandb_logger = WandbCallback(
+ yolo, run_name=run_name, project=project, tags=tags, resume=resume, **kwargs
+ )
+ for event, callback_fn in wandb_logger.callbacks.items():
+ yolo.add_callback(event, callback_fn)
+ return yolo
+ else:
+ wandb.termerror(
+ "The RANK of the process to add the callbacks was neither 0 or -1."
+ "No Weights & Biases callbacks were added to this instance of the YOLO model."
+ )
+ return yolo
diff --git a/wandb/jupyter.py b/wandb/jupyter.py
new file mode 100644
index 0000000000000000000000000000000000000000..068e1e7db2aff8914ac1abb9c902dcd7d91af1d5
--- /dev/null
+++ b/wandb/jupyter.py
@@ -0,0 +1,501 @@
+import json
+import logging
+import os
+import re
+import shutil
+import sys
+from base64 import b64encode
+from typing import Dict
+
+import requests
+from requests.compat import urljoin
+
+import wandb
+import wandb.util
+from wandb.sdk.lib import filesystem
+
+try:
+ from IPython.core.getipython import get_ipython
+ from IPython.core.magic import Magics, line_cell_magic, magics_class
+ from IPython.core.magic_arguments import argument, magic_arguments, parse_argstring
+ from IPython.display import display
+except ImportError:
+ wandb.termwarn("ipython is not supported in python 2.7, upgrade to 3.x")
+
+ class Magics:
+ pass
+
+ def magics_class(*args, **kwargs):
+ return lambda *args, **kwargs: None
+
+ def magic_arguments(*args, **kwargs):
+ return lambda *args, **kwargs: None
+
+ def argument(*args, **kwargs):
+ return lambda *args, **kwargs: None
+
+ def line_cell_magic(*args, **kwargs):
+ return lambda *args, **kwargs: None
+
+
+logger = logging.getLogger(__name__)
+
+__IFrame = None
+
+
+def maybe_display():
+ """Display a run if the user added cell magic and we have run."""
+ if __IFrame is not None:
+ return __IFrame.maybe_display()
+ return False
+
+
+def quiet():
+ if __IFrame is not None:
+ return __IFrame.opts.get("quiet")
+ return False
+
+
+class IFrame:
+ def __init__(self, path=None, opts=None):
+ self.path = path
+ self.api = wandb.Api()
+ self.opts = opts or {}
+ self.displayed = False
+ self.height = self.opts.get("height", 420)
+
+ def maybe_display(self) -> bool:
+ if not self.displayed and (self.path or wandb.run):
+ display(self)
+ return self.displayed
+
+ def _repr_html_(self):
+ try:
+ self.displayed = True
+ if self.opts.get("workspace", False):
+ if self.path is None and wandb.run:
+ self.path = wandb.run.path
+ if isinstance(self.path, str):
+ object = self.api.from_path(self.path)
+ else:
+ object = wandb.run
+ if object is None:
+ if wandb.Api().api_key is None:
+ return "You must be logged in to render wandb in jupyter, run `wandb.login()`"
+ else:
+ object = self.api.project(
+ "/".join(
+ [
+ wandb.Api().default_entity,
+ wandb.util.auto_project_name(None),
+ ]
+ )
+ )
+ return object.to_html(self.height, hidden=False)
+ except wandb.Error as e:
+ return f"Can't display wandb interface<br/>{e}"
+
+
+@magics_class
+class WandBMagics(Magics):
+ def __init__(self, shell, require_interaction=False):
+ super().__init__(shell)
+ self.options = {}
+
+ @magic_arguments()
+ @argument(
+ "path",
+ default=None,
+ nargs="?",
+ help="A path to a resource you want to display, defaults to wandb.run.path",
+ )
+ @argument(
+ "-w",
+ "--workspace",
+ default=False,
+ action="store_true",
+ help="Display the entire run project workspace",
+ )
+ @argument(
+ "-q",
+ "--quiet",
+ default=False,
+ action="store_true",
+ help="Display the minimal amount of output",
+ )
+ @argument(
+ "-h",
+ "--height",
+ default=420,
+ type=int,
+ help="The height of the iframe in pixels",
+ )
+ @line_cell_magic
+ def wandb(self, line, cell=None):
+ """Display wandb resources in jupyter. This can be used as cell or line magic.
+
+ %wandb USERNAME/PROJECT/runs/RUN_ID
+ ---
+ %%wandb -h 1024
+ with wandb.init() as run:
+ run.log({"loss": 1})
+ """
+ # Record options
+ args = parse_argstring(self.wandb, line)
+ self.options["height"] = args.height
+ self.options["workspace"] = args.workspace
+ self.options["quiet"] = args.quiet
+ iframe = IFrame(args.path, opts=self.options)
+ displayed = iframe.maybe_display()
+ if cell is not None:
+ if not displayed:
+ # Store the IFrame globally and attempt to display if we have a run
+ cell = (
+ f"wandb.jupyter.__IFrame = wandb.jupyter.IFrame(opts={self.options})\n"
+ + cell
+ + "\nwandb.jupyter.__IFrame = None"
+ )
+ get_ipython().run_cell(cell)
+
+
+def notebook_metadata_from_jupyter_servers_and_kernel_id():
+ servers, kernel_id = jupyter_servers_and_kernel_id()
+ for s in servers:
+ if s.get("password"):
+ raise ValueError("Can't query password protected kernel")
+ res = requests.get(
+ urljoin(s["url"], "api/sessions"), params={"token": s.get("token", "")}
+ ).json()
+ for nn in res:
+ # TODO: wandb/client#400 found a case where res returned an array of
+ # strings...
+ if isinstance(nn, dict) and nn.get("kernel") and "notebook" in nn:
+ if nn["kernel"]["id"] == kernel_id:
+ return {
+ "root": s.get("root_dir", s.get("notebook_dir", os.getcwd())),
+ "path": nn["notebook"]["path"],
+ "name": nn["notebook"]["name"],
+ }
+ return None
+
+
+def notebook_metadata(silent: bool) -> Dict[str, str]:
+ """Attempt to query jupyter for the path and name of the notebook file.
+
+ This can handle different jupyter environments, specifically:
+
+ 1. Colab
+ 2. Kaggle
+ 3. JupyterLab
+ 4. Notebooks
+ 5. Other?
+ """
+ error_message = (
+ "Failed to detect the name of this notebook, you can set it manually with "
+ "the WANDB_NOTEBOOK_NAME environment variable to enable code saving."
+ )
+ try:
+ jupyter_metadata = notebook_metadata_from_jupyter_servers_and_kernel_id()
+
+ # Colab:
+ # request the most recent contents
+ ipynb = attempt_colab_load_ipynb()
+ if ipynb is not None and jupyter_metadata is not None:
+ return {
+ "root": "/content",
+ "path": jupyter_metadata["path"],
+ "name": jupyter_metadata["name"],
+ }
+
+ # Kaggle:
+ if wandb.util._is_kaggle():
+ # request the most recent contents
+ ipynb = attempt_kaggle_load_ipynb()
+ if ipynb:
+ return {
+ "root": "/kaggle/working",
+ "path": ipynb["metadata"]["name"],
+ "name": ipynb["metadata"]["name"],
+ }
+
+ if jupyter_metadata:
+ return jupyter_metadata
+ if not silent:
+ logger.error(error_message)
+ return {}
+ except Exception:
+ # TODO: report this exception
+ # TODO: Fix issue this is not the logger initialized in in wandb.init()
+ # since logger is not attached, outputs to notebook
+ if not silent:
+ logger.error(error_message)
+ return {}
+
+
+def jupyter_servers_and_kernel_id():
+ """Return a list of servers and the current kernel_id.
+
+ Used to query for the name of the notebook.
+ """
+ try:
+ import ipykernel
+
+ kernel_id = re.search(
+ "kernel-(.*).json", ipykernel.connect.get_connection_file()
+ ).group(1)
+ # We're either in jupyterlab or a notebook, lets prefer the newer jupyter_server package
+ serverapp = wandb.util.get_module("jupyter_server.serverapp")
+ notebookapp = wandb.util.get_module("notebook.notebookapp")
+ servers = []
+ if serverapp is not None:
+ servers.extend(list(serverapp.list_running_servers()))
+ if notebookapp is not None:
+ servers.extend(list(notebookapp.list_running_servers()))
+ return servers, kernel_id
+ except (AttributeError, ValueError, ImportError):
+ return [], None
+
+
+def attempt_colab_load_ipynb():
+ colab = wandb.util.get_module("google.colab")
+ if colab:
+ # This isn't thread safe, never call in a thread
+ response = colab._message.blocking_request("get_ipynb", timeout_sec=5)
+ if response:
+ return response["ipynb"]
+
+
+def attempt_kaggle_load_ipynb():
+ kaggle = wandb.util.get_module("kaggle_session")
+ if kaggle:
+ try:
+ client = kaggle.UserSessionClient()
+ parsed = json.loads(client.get_exportable_ipynb()["source"])
+ # TODO: couldn't find a way to get the name of the notebook...
+ parsed["metadata"]["name"] = "kaggle.ipynb"
+ return parsed
+ except Exception:
+ logger.exception("Unable to load kaggle notebook")
+ return None
+
+
+def attempt_colab_login(app_url):
+ """This renders an iframe to wandb in the hopes it posts back an api key."""
+ from google.colab import output
+ from google.colab._message import MessageError
+ from IPython import display
+
+ display.display(
+ display.Javascript(
+ """
+ window._wandbApiKey = new Promise((resolve, reject) => {
+ function loadScript(url) {
+ return new Promise(function(resolve, reject) {
+ let newScript = document.createElement("script");
+ newScript.onerror = reject;
+ newScript.onload = resolve;
+ document.body.appendChild(newScript);
+ newScript.src = url;
+ });
+ }
+ loadScript("https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js").then(() => {
+ const iframe = document.createElement('iframe')
+ iframe.style.cssText = "width:0;height:0;border:none"
+ document.body.appendChild(iframe)
+ const handshake = new Postmate({
+ container: iframe,
+ url: '%s/authorize'
+ });
+ const timeout = setTimeout(() => reject("Couldn't auto authenticate"), 5000)
+ handshake.then(function(child) {
+ child.on('authorize', data => {
+ clearTimeout(timeout)
+ resolve(data)
+ });
+ });
+ })
+ });
+ """
+ % app_url.replace("http:", "https:")
+ )
+ )
+ try:
+ return output.eval_js("_wandbApiKey")
+ except MessageError:
+ return None
+
+
+class Notebook:
+ def __init__(self, settings):
+ self.outputs = {}
+ self.settings = settings
+ self.shell = get_ipython()
+
+ def save_display(self, exc_count, data_with_metadata):
+ self.outputs[exc_count] = self.outputs.get(exc_count, [])
+
+ # byte values such as images need to be encoded in base64
+ # otherwise nbformat.v4.new_output will throw a NotebookValidationError
+ data = data_with_metadata["data"]
+ b64_data = {}
+ for key in data:
+ val = data[key]
+ if isinstance(val, bytes):
+ b64_data[key] = b64encode(val).decode("utf-8")
+ else:
+ b64_data[key] = val
+
+ self.outputs[exc_count].append(
+ {"data": b64_data, "metadata": data_with_metadata["metadata"]}
+ )
+
+ def probe_ipynb(self):
+ """Return notebook as dict or None."""
+ relpath = self.settings._jupyter_path
+ if relpath:
+ if os.path.exists(relpath):
+ with open(relpath) as json_file:
+ data = json.load(json_file)
+ return data
+
+ colab_ipynb = attempt_colab_load_ipynb()
+ if colab_ipynb:
+ return colab_ipynb
+
+ kaggle_ipynb = attempt_kaggle_load_ipynb()
+ if kaggle_ipynb and len(kaggle_ipynb["cells"]) > 0:
+ return kaggle_ipynb
+
+ return
+
+ def save_ipynb(self) -> bool:
+ if not self.settings.save_code:
+ logger.info("not saving jupyter notebook")
+ return False
+ ret = False
+ try:
+ ret = self._save_ipynb()
+ except Exception as e:
+ logger.info(f"Problem saving notebook: {repr(e)}")
+ return ret
+
+ def _save_ipynb(self) -> bool:
+ relpath = self.settings._jupyter_path
+ logger.info("looking for notebook: %s", relpath)
+ if relpath:
+ if os.path.exists(relpath):
+ shutil.copy(
+ relpath,
+ os.path.join(
+ self.settings._tmp_code_dir, os.path.basename(relpath)
+ ),
+ )
+ return True
+
+ # TODO: likely only save if the code has changed
+ colab_ipynb = attempt_colab_load_ipynb()
+ if colab_ipynb:
+ try:
+ jupyter_metadata = (
+ notebook_metadata_from_jupyter_servers_and_kernel_id()
+ )
+ nb_name = jupyter_metadata["name"]
+ except Exception:
+ nb_name = "colab.ipynb"
+ if not nb_name.endswith(".ipynb"):
+ nb_name += ".ipynb"
+ with open(
+ os.path.join(
+ self.settings._tmp_code_dir,
+ nb_name,
+ ),
+ "w",
+ encoding="utf-8",
+ ) as f:
+ f.write(json.dumps(colab_ipynb))
+ return True
+
+ kaggle_ipynb = attempt_kaggle_load_ipynb()
+ if kaggle_ipynb and len(kaggle_ipynb["cells"]) > 0:
+ with open(
+ os.path.join(
+ self.settings._tmp_code_dir, kaggle_ipynb["metadata"]["name"]
+ ),
+ "w",
+ encoding="utf-8",
+ ) as f:
+ f.write(json.dumps(kaggle_ipynb))
+ return True
+
+ return False
+
+ def save_history(self):
+ """This saves all cell executions in the current session as a new notebook."""
+ try:
+ from nbformat import v4, validator, write
+ except ImportError:
+ logger.error("Run pip install nbformat to save notebook history")
+ return
+ # TODO: some tests didn't patch ipython properly?
+ if self.shell is None:
+ return
+ cells = []
+ hist = list(self.shell.history_manager.get_range(output=True))
+ if len(hist) <= 1 or not self.settings.save_code:
+ logger.info("not saving jupyter history")
+ return
+ try:
+ for _, execution_count, exc in hist:
+ if exc[1]:
+ # TODO: capture stderr?
+ outputs = [
+ v4.new_output(output_type="stream", name="stdout", text=exc[1])
+ ]
+ else:
+ outputs = []
+ if self.outputs.get(execution_count):
+ for out in self.outputs[execution_count]:
+ outputs.append(
+ v4.new_output(
+ output_type="display_data",
+ data=out["data"],
+ metadata=out["metadata"] or {},
+ )
+ )
+ cells.append(
+ v4.new_code_cell(
+ execution_count=execution_count, source=exc[0], outputs=outputs
+ )
+ )
+ if hasattr(self.shell, "kernel"):
+ language_info = self.shell.kernel.language_info
+ else:
+ language_info = {"name": "python", "version": sys.version}
+ logger.info("saving %i cells to _session_history.ipynb", len(cells))
+ nb = v4.new_notebook(
+ cells=cells,
+ metadata={
+ "kernelspec": {
+ "display_name": "Python %i" % sys.version_info[0],
+ "name": "python%i" % sys.version_info[0],
+ "language": "python",
+ },
+ "language_info": language_info,
+ },
+ )
+ state_path = os.path.join("code", "_session_history.ipynb")
+ wandb.run._set_config_wandb("session_history", state_path)
+ filesystem.mkdir_exists_ok(os.path.join(wandb.run.dir, "code"))
+ with open(
+ os.path.join(self.settings._tmp_code_dir, "_session_history.ipynb"),
+ "w",
+ encoding="utf-8",
+ ) as f:
+ write(nb, f, version=4)
+ with open(
+ os.path.join(wandb.run.dir, state_path), "w", encoding="utf-8"
+ ) as f:
+ write(nb, f, version=4)
+ except (OSError, validator.NotebookValidationError) as e:
+ logger.error("Unable to save ipython session history:\n%s", e)
+ pass
diff --git a/wandb/keras/__init__.py b/wandb/keras/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..b9f01301740af80028a43e101b6a77dece690ce3
--- /dev/null
+++ b/wandb/keras/__init__.py
@@ -0,0 +1,18 @@
+"""Compatibility keras module.
+
+In the future use e.g.:
+ from wandb.integration.keras import WandbCallback
+"""
+__all__ = (
+ "WandbCallback",
+ "WandbMetricsLogger",
+ "WandbModelCheckpoint",
+ "WandbEvalCallback",
+)
+
+from wandb.integration.keras import (
+ WandbCallback,
+ WandbEvalCallback,
+ WandbMetricsLogger,
+ WandbModelCheckpoint,
+)
diff --git a/wandb/lightgbm/__init__.py b/wandb/lightgbm/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..15395dae0e6b382d14d1643096a38229b0e92f99
--- /dev/null
+++ b/wandb/lightgbm/__init__.py
@@ -0,0 +1,9 @@
+"""Compatibility lightgbm module.
+
+In the future use:
+ from wandb.integration.lightgbm import wandb_callback
+"""
+
+from wandb.integration.lightgbm import log_summary, wandb_callback
+
+__all__ = ["wandb_callback", "log_summary"]
diff --git a/wandb/magic.py b/wandb/magic.py
new file mode 100644
index 0000000000000000000000000000000000000000..0622a4a90913849785cc862b5b1b29491660ea94
--- /dev/null
+++ b/wandb/magic.py
@@ -0,0 +1,3 @@
+from wandb.integration import magic
+
+magic.magic_install()
diff --git a/wandb/mpmain/__init__.py b/wandb/mpmain/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/mpmain/__main__.py b/wandb/mpmain/__main__.py
new file mode 100644
index 0000000000000000000000000000000000000000..02e8a24d9fbce482e31cde126d8407586a61b3fd
--- /dev/null
+++ b/wandb/mpmain/__main__.py
@@ -0,0 +1 @@
+# This module is initialized after multiprocessing spawn
diff --git a/wandb/old/README.md b/wandb/old/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..2b21b99ad761085f1a8da46c72c35fdd5d42b7a0
--- /dev/null
+++ b/wandb/old/README.md
@@ -0,0 +1 @@
+Needs refactor
diff --git a/wandb/old/__init__.py b/wandb/old/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/old/core.py b/wandb/old/core.py
new file mode 100644
index 0000000000000000000000000000000000000000..4956fa7289cb13a9b60a44f887b7d13045f7817f
--- /dev/null
+++ b/wandb/old/core.py
@@ -0,0 +1,131 @@
+"""Core variables, functions, and classes that we want in the wandb
+module but are also used in modules that import the wandb module.
+
+The purpose of this module is to break circular imports.
+"""
+
+import os
+import sys
+import tempfile
+import time
+
+import click
+
+import wandb
+from wandb import env
+
+# We use the hidden version if it already exists, otherwise non-hidden.
+if os.path.exists(os.path.join(env.get_dir(os.getcwd()), ".wandb")):
+ __stage_dir__ = ".wandb" + os.sep
+elif os.path.exists(os.path.join(env.get_dir(os.getcwd()), "wandb")):
+ __stage_dir__ = "wandb" + os.sep
+else:
+ __stage_dir__ = None
+
+SCRIPT_PATH = os.path.abspath(sys.argv[0])
+wandb.START_TIME = time.time()
+LIB_ROOT = os.path.join(os.path.dirname(__file__), "..")
+IS_GIT = os.path.exists(os.path.join(LIB_ROOT, ".git"))
+
+
+def wandb_dir(root_dir=None):
+ if root_dir is None or root_dir == "":
+ try:
+ cwd = os.getcwd()
+ except OSError:
+ termwarn("os.getcwd() no longer exists, using system temp directory")
+ cwd = tempfile.gettempdir()
+ root_dir = env.get_dir(cwd)
+ path = os.path.join(root_dir, __stage_dir__ or ("wandb" + os.sep))
+ if not os.access(root_dir, os.W_OK):
+ termwarn(
+ f"Path {path} wasn't writable, using system temp directory", repeat=False
+ )
+ path = os.path.join(tempfile.gettempdir(), __stage_dir__ or ("wandb" + os.sep))
+ return path
+
+
+def _set_stage_dir(stage_dir):
+ # Used when initing a new project with "wandb init"
+ global __stage_dir__
+ __stage_dir__ = stage_dir
+
+
+class Error(Exception):
+ """Base W&B Error"""
+
+ def __init__(self, message):
+ super().__init__(message)
+ self.message = message
+
+ # For python 2 support
+ def encode(self, encoding):
+ return self.message
+
+
+class WandbWarning(Warning):
+ """Base W&B Warning"""
+
+ pass
+
+
+LOG_STRING = click.style("wandb", fg="blue", bold=True)
+ERROR_STRING = click.style("ERROR", bg="red", fg="green")
+WARN_STRING = click.style("WARNING", fg="yellow")
+PRINTED_MESSAGES = set()
+
+
+# TODO(adrian): if output has been redirected, make this write to the original STDERR
+# so it doesn't get logged to the backend
+def termlog(string="", newline=True, repeat=True):
+ """Log to standard error with formatting.
+
+ Arguments:
+ string (str, optional): The string to print
+ newline (bool, optional): Print a newline at the end of the string
+ repeat (bool, optional): If set to False only prints the string once per process
+ """
+ if string:
+ line = "\n".join([f"{LOG_STRING}: {s}" for s in string.split("\n")])
+ else:
+ line = ""
+ if not repeat and line in PRINTED_MESSAGES:
+ return
+ # Repeated line tracking limited to 1k messages
+ if len(PRINTED_MESSAGES) < 1000:
+ PRINTED_MESSAGES.add(line)
+ if os.getenv(env.SILENT):
+ from wandb import util
+ from wandb.sdk.lib import filesystem
+
+ filesystem.mkdir_exists_ok(os.path.dirname(util.get_log_file_path()))
+ with open(util.get_log_file_path(), "w") as log:
+ click.echo(line, file=log, nl=newline)
+ else:
+ click.echo(line, file=sys.stderr, nl=newline)
+
+
+def termwarn(string, **kwargs):
+ string = "\n".join([f"{WARN_STRING} {s}" for s in string.split("\n")])
+ termlog(string=string, newline=True, **kwargs)
+
+
+def termerror(string, **kwargs):
+ string = "\n".join([f"{ERROR_STRING} {s}" for s in string.split("\n")])
+ termlog(string=string, newline=True, **kwargs)
+
+
+__all__ = [
+ "__stage_dir__",
+ "SCRIPT_PATH",
+ "START_TIME",
+ "wandb_dir",
+ "_set_stage_dir",
+ "Error",
+ "WandbWarning",
+ "LOG_STRING",
+ "ERROR_STRING",
+ "termlog",
+ "termwarn",
+ "termerror",
+]
diff --git a/wandb/old/settings.py b/wandb/old/settings.py
new file mode 100644
index 0000000000000000000000000000000000000000..76629d9349bc235e89e7a53e34419cb4abf6e8da
--- /dev/null
+++ b/wandb/old/settings.py
@@ -0,0 +1,173 @@
+import configparser
+import getpass
+import os
+import tempfile
+from typing import Any, Optional
+
+from wandb import env
+from wandb.old import core
+from wandb.sdk.lib import filesystem
+from wandb.sdk.lib.runid import generate_id
+
+
+class Settings:
+ """Global W&B settings stored under $WANDB_CONFIG_DIR/settings."""
+
+ DEFAULT_SECTION = "default"
+
+ _UNSET = object()
+
+ def __init__(
+ self, load_settings: bool = True, root_dir: Optional[str] = None
+ ) -> None:
+ self._global_settings = Settings._settings()
+ self._local_settings = Settings._settings()
+ self.root_dir = root_dir
+
+ if load_settings:
+ global_path = Settings._global_path()
+ if global_path is not None:
+ self._global_settings.read([global_path])
+ # Only attempt to read if there is a directory existing
+ if os.path.isdir(core.wandb_dir(self.root_dir)):
+ self._local_settings.read([Settings._local_path(self.root_dir)])
+
+ def get(self, section: str, key: str, fallback: Any = _UNSET) -> Any:
+ # Try the local settings first. If we can't find the key, then try the global settings.
+ # If a fallback is provided, return it if we can't find the key in either the local or global
+ # settings.
+ try:
+ return self._local_settings.get(section, key)
+ except configparser.NoOptionError:
+ try:
+ return self._global_settings.get(section, key)
+ except configparser.NoOptionError:
+ if fallback is not Settings._UNSET:
+ return fallback
+ else:
+ raise
+
+ def _persist_settings(self, settings, settings_path) -> None:
+ # write a temp file and then move it to the settings path
+ target_dir = os.path.dirname(settings_path)
+ with tempfile.NamedTemporaryFile(
+ "w+", suffix=".tmp", delete=False, dir=target_dir
+ ) as fp:
+ path = os.path.abspath(fp.name)
+ with open(path, "w+") as f:
+ settings.write(f)
+ try:
+ os.replace(path, settings_path)
+ except AttributeError:
+ os.rename(path, settings_path)
+
+ def set(self, section, key, value, globally=False, persist=False) -> None:
+ """Persist settings to disk if persist = True"""
+
+ def write_setting(settings, settings_path, persist):
+ if not settings.has_section(section):
+ Settings._safe_add_section(settings, Settings.DEFAULT_SECTION)
+ settings.set(section, key, str(value))
+
+ if persist:
+ self._persist_settings(settings, settings_path)
+
+ if globally:
+ global_path = Settings._global_path()
+ if global_path is not None:
+ write_setting(self._global_settings, global_path, persist)
+ else:
+ write_setting(
+ self._local_settings, Settings._local_path(self.root_dir), persist
+ )
+
+ def clear(self, section, key, globally=False, persist=False) -> None:
+ def clear_setting(settings, settings_path, persist):
+ settings.remove_option(section, key)
+ if persist:
+ self._persist_settings(settings, settings_path)
+
+ if globally:
+ global_path = Settings._global_path()
+ if global_path is not None:
+ clear_setting(self._global_settings, global_path, persist)
+ else:
+ clear_setting(
+ self._local_settings, Settings._local_path(self.root_dir), persist
+ )
+
+ def items(self, section=None):
+ section = section if section is not None else Settings.DEFAULT_SECTION
+
+ result = {"section": section}
+
+ try:
+ if section in self._global_settings.sections():
+ for option in self._global_settings.options(section):
+ result[option] = self._global_settings.get(section, option)
+ if section in self._local_settings.sections():
+ for option in self._local_settings.options(section):
+ result[option] = self._local_settings.get(section, option)
+ except configparser.InterpolationSyntaxError:
+ core.termwarn("Unable to parse settings file")
+
+ return result
+
+ @staticmethod
+ def _safe_add_section(settings, section):
+ if not settings.has_section(section):
+ settings.add_section(section)
+
+ @staticmethod
+ def _settings(default_settings={}):
+ settings = configparser.ConfigParser()
+ Settings._safe_add_section(settings, Settings.DEFAULT_SECTION)
+ for key, value in default_settings.items():
+ settings.set(Settings.DEFAULT_SECTION, key, str(value))
+ return settings
+
+ @staticmethod
+ def _global_path() -> Optional[str]:
+ def try_create_dir(path) -> bool:
+ try:
+ os.makedirs(path, exist_ok=True)
+ if os.access(path, os.W_OK):
+ return True
+ except OSError:
+ pass
+ return False
+
+ def get_username() -> str:
+ try:
+ return getpass.getuser()
+ except (ImportError, KeyError):
+ return generate_id()
+
+ try:
+ home_config_dir = os.path.join(os.path.expanduser("~"), ".config", "wandb")
+
+ if not try_create_dir(home_config_dir):
+ temp_config_dir = os.path.join(
+ tempfile.gettempdir(), ".config", "wandb"
+ )
+
+ if not try_create_dir(temp_config_dir):
+ username = get_username()
+ config_dir = os.path.join(
+ tempfile.gettempdir(), username, ".config", "wandb"
+ )
+ try_create_dir(config_dir)
+ else:
+ config_dir = temp_config_dir
+ else:
+ config_dir = home_config_dir
+
+ config_dir = os.environ.get(env.CONFIG_DIR, config_dir)
+ return os.path.join(config_dir, "settings")
+ except Exception:
+ return None
+
+ @staticmethod
+ def _local_path(root_dir=None):
+ filesystem.mkdir_exists_ok(core.wandb_dir(root_dir))
+ return os.path.join(core.wandb_dir(root_dir), "settings")
diff --git a/wandb/old/summary.py b/wandb/old/summary.py
new file mode 100644
index 0000000000000000000000000000000000000000..7eac539ff3f1bf84ac9d38da2597bfd568e17d0d
--- /dev/null
+++ b/wandb/old/summary.py
@@ -0,0 +1,435 @@
+import json
+import os
+import time
+
+from wandb_gql import gql
+
+import wandb
+from wandb import util
+from wandb.apis.internal import Api
+from wandb.sdk import lib as wandb_lib
+from wandb.sdk.data_types.utils import val_to_json
+
+DEEP_SUMMARY_FNAME = "wandb.h5"
+H5_TYPES = ("numpy.ndarray", "tensorflow.Tensor", "torch.Tensor")
+h5py = util.get_module("h5py")
+np = util.get_module("numpy")
+
+
+class SummarySubDict:
+ """Nested dict-like object that proxies read and write operations through a root object.
+
+ This lets us do synchronous serialization and lazy loading of large values.
+ """
+
+ def __init__(self, root=None, path=()):
+ self._path = tuple(path)
+ if root is None:
+ self._root = self
+ self._json_dict = {}
+ else:
+ self._root = root
+ json_dict = root._json_dict
+ for k in path:
+ json_dict = json_dict.get(k, {})
+
+ self._json_dict = json_dict
+ self._dict = {}
+
+ # We use this to track which keys the user has set explicitly
+ # so that we don't automatically overwrite them when we update
+ # the summary from the history.
+ self._locked_keys = set()
+
+ def __setattr__(self, k, v):
+ k = k.strip()
+ if k.startswith("_"):
+ object.__setattr__(self, k, v)
+ else:
+ self[k] = v
+
+ def __getattr__(self, k):
+ k = k.strip()
+ if k.startswith("_"):
+ return object.__getattribute__(self, k)
+ else:
+ return self[k]
+
+ def _root_get(self, path, child_dict):
+ """Load a value at a particular path from the root.
+
+ This should only be implemented by the "_root" child class.
+
+ We pass the child_dict so the item can be set on it or not as
+ appropriate. Returning None for a nonexistant path wouldn't be
+ distinguishable from that path being set to the value None.
+ """
+ raise NotImplementedError
+
+ def _root_set(self, path, new_keys_values):
+ """Set a value at a particular path in the root.
+
+ This should only be implemented by the "_root" child class.
+ """
+ raise NotImplementedError
+
+ def _root_del(self, path):
+ """Delete a value at a particular path in the root.
+
+ This should only be implemented by the "_root" child class.
+ """
+ raise NotImplementedError
+
+ def _write(self, commit=False):
+ # should only be implemented on the root summary
+ raise NotImplementedError
+
+ def keys(self):
+ # _json_dict has the full set of keys, including those for h5 objects
+ # that may not have been loaded yet
+ return self._json_dict.keys()
+
+ def get(self, k, default=None):
+ if isinstance(k, str):
+ k = k.strip()
+ if k not in self._dict:
+ self._root._root_get(self._path + (k,), self._dict)
+ return self._dict.get(k, default)
+
+ def items(self):
+ # not all items may be loaded into self._dict, so we
+ # have to build the sequence of items from scratch
+ for k in self.keys():
+ yield k, self[k]
+
+ def __getitem__(self, k):
+ if isinstance(k, str):
+ k = k.strip()
+
+ self.get(k) # load the value into _dict if it should be there
+ res = self._dict[k]
+
+ return res
+
+ def __contains__(self, k):
+ if isinstance(k, str):
+ k = k.strip()
+
+ return k in self._json_dict
+
+ def __setitem__(self, k, v):
+ if isinstance(k, str):
+ k = k.strip()
+
+ path = self._path
+
+ if isinstance(v, dict):
+ self._dict[k] = SummarySubDict(self._root, path + (k,))
+ self._root._root_set(path, [(k, {})])
+ self._dict[k].update(v)
+ else:
+ self._dict[k] = v
+ self._root._root_set(path, [(k, v)])
+
+ self._locked_keys.add(k)
+
+ self._root._write()
+
+ return v
+
+ def __delitem__(self, k):
+ k = k.strip()
+ del self._dict[k]
+ self._root._root_del(self._path + (k,))
+
+ self._root._write()
+
+ def __repr__(self):
+ # use a copy of _dict, except add placeholders for h5 objects, etc.
+ # that haven't been loaded yet
+ repr_dict = dict(self._dict)
+ for k in self._json_dict:
+ v = self._json_dict[k]
+ if (
+ k not in repr_dict
+ and isinstance(v, dict)
+ and v.get("_type") in H5_TYPES
+ ):
+ # unloaded h5 objects may be very large. use a placeholder for them
+ # if we haven't already loaded them
+ repr_dict[k] = "..."
+ else:
+ repr_dict[k] = self[k]
+
+ return repr(repr_dict)
+
+ def update(self, key_vals=None, overwrite=True):
+ """Locked keys will be overwritten unless overwrite=False.
+
+ Otherwise, written keys will be added to the "locked" list.
+ """
+ if key_vals:
+ write_items = self._update(key_vals, overwrite)
+ self._root._root_set(self._path, write_items)
+ self._root._write(commit=True)
+
+ def _update(self, key_vals, overwrite):
+ if not key_vals:
+ return
+ key_vals = {k.strip(): v for k, v in key_vals.items()}
+ if overwrite:
+ write_items = list(key_vals.items())
+ self._locked_keys.update(key_vals.keys())
+ else:
+ write_keys = set(key_vals.keys()) - self._locked_keys
+ write_items = [(k, key_vals[k]) for k in write_keys]
+
+ for key, value in write_items:
+ if isinstance(value, dict):
+ self._dict[key] = SummarySubDict(self._root, self._path + (key,))
+ self._dict[key]._update(value, overwrite)
+ else:
+ self._dict[key] = value
+
+ return write_items
+
+
+class Summary(SummarySubDict):
+ """Store summary metrics (eg. accuracy) during and after a run.
+
+ You can manipulate this as if it's a Python dictionary but the keys
+ get mangled. .strip() is called on them, so spaces at the beginning
+ and end are removed.
+ """
+
+ def __init__(self, run, summary=None):
+ super().__init__()
+ self._run = run
+ self._h5_path = os.path.join(self._run.dir, DEEP_SUMMARY_FNAME)
+ # Lazy load the h5 file
+ self._h5 = None
+
+ # Mirrored version of self._dict with versions of values that get written
+ # to JSON kept up to date by self._root_set() and self._root_del().
+ self._json_dict = {}
+
+ if summary is not None:
+ self._json_dict = summary
+
+ def _json_get(self, path):
+ pass
+
+ def _root_get(self, path, child_dict):
+ json_dict = self._json_dict
+ for key in path[:-1]:
+ json_dict = json_dict[key]
+
+ key = path[-1]
+ if key in json_dict:
+ child_dict[key] = self._decode(path, json_dict[key])
+
+ def _root_del(self, path):
+ json_dict = self._json_dict
+ for key in path[:-1]:
+ json_dict = json_dict[key]
+
+ val = json_dict[path[-1]]
+ del json_dict[path[-1]]
+ if isinstance(val, dict) and val.get("_type") in H5_TYPES:
+ if not h5py:
+ wandb.termerror("Deleting tensors in summary requires h5py")
+ else:
+ self.open_h5()
+ h5_key = "summary/" + ".".join(path)
+ del self._h5[h5_key]
+ self._h5.flush()
+
+ def _root_set(self, path, new_keys_values):
+ json_dict = self._json_dict
+ for key in path:
+ json_dict = json_dict[key]
+
+ for new_key, new_value in new_keys_values:
+ json_dict[new_key] = self._encode(new_value, path + (new_key,))
+
+ def write_h5(self, path, val):
+ # ensure the file is open
+ self.open_h5()
+
+ if not self._h5:
+ wandb.termerror("Storing tensors in summary requires h5py")
+ else:
+ try:
+ del self._h5["summary/" + ".".join(path)]
+ except KeyError:
+ pass
+ self._h5["summary/" + ".".join(path)] = val
+ self._h5.flush()
+
+ def read_h5(self, path, val=None):
+ # ensure the file is open
+ self.open_h5()
+
+ if not self._h5:
+ wandb.termerror("Reading tensors from summary requires h5py")
+ else:
+ return self._h5.get("summary/" + ".".join(path), val)
+
+ def open_h5(self):
+ if not self._h5 and h5py:
+ self._h5 = h5py.File(self._h5_path, "a", libver="latest")
+
+ def _decode(self, path, json_value):
+ """Decode a `dict` encoded by `Summary._encode()`, loading h5 objects.
+
+ h5 objects may be very large, so we won't have loaded them automatically.
+ """
+ if isinstance(json_value, dict):
+ if json_value.get("_type") in H5_TYPES:
+ return self.read_h5(path, json_value)
+ elif json_value.get("_type") == "data-frame":
+ wandb.termerror(
+ "This data frame was saved via the wandb data API. Contact support@wandb.com for help."
+ )
+ return None
+ # TODO: transform wandb objects and plots
+ else:
+ return SummarySubDict(self, path)
+ else:
+ return json_value
+
+ def _encode(self, value, path_from_root):
+ """Normalize, compress, and encode sub-objects for backend storage.
+
+ value: Object to encode.
+ path_from_root: `tuple` of key strings from the top-level summary to the
+ current `value`.
+
+ Returns:
+ A new tree of dict's with large objects replaced with dictionaries
+ with "_type" entries that say which type the original data was.
+ """
+
+ # Constructs a new `dict` tree in `json_value` that discards and/or
+ # encodes objects that aren't JSON serializable.
+
+ if isinstance(value, dict):
+ json_value = {}
+ for key, value in value.items():
+ json_value[key] = self._encode(value, path_from_root + (key,))
+ return json_value
+ else:
+ path = ".".join(path_from_root)
+ friendly_value, converted = util.json_friendly(
+ val_to_json(self._run, path, value, namespace="summary")
+ )
+ json_value, compressed = util.maybe_compress_summary(
+ friendly_value, util.get_h5_typename(value)
+ )
+ if compressed:
+ self.write_h5(path_from_root, friendly_value)
+
+ return json_value
+
+
+def download_h5(run_id, entity=None, project=None, out_dir=None):
+ api = Api()
+ meta = api.download_url(
+ project or api.settings("project"),
+ DEEP_SUMMARY_FNAME,
+ entity=entity or api.settings("entity"),
+ run=run_id,
+ )
+ if meta and "md5" in meta and meta["md5"] is not None:
+ # TODO: make this non-blocking
+ wandb.termlog("Downloading summary data...")
+ path, res = api.download_write_file(meta, out_dir=out_dir)
+ return path
+
+
+def upload_h5(file, run_id, entity=None, project=None):
+ api = Api()
+ wandb.termlog("Uploading summary data...")
+ with open(file, "rb") as f:
+ api.push(
+ {os.path.basename(file): f}, run=run_id, project=project, entity=entity
+ )
+
+
+class FileSummary(Summary):
+ def __init__(self, run):
+ super().__init__(run)
+ self._fname = os.path.join(run.dir, wandb_lib.filenames.SUMMARY_FNAME)
+ self.load()
+
+ def load(self):
+ try:
+ with open(self._fname) as f:
+ self._json_dict = json.load(f)
+ except (OSError, ValueError):
+ self._json_dict = {}
+
+ def _write(self, commit=False):
+ # TODO: we just ignore commit to ensure backward capability
+ with open(self._fname, "w") as f:
+ f.write(util.json_dumps_safer(self._json_dict))
+ f.write("\n")
+ f.flush()
+ os.fsync(f.fileno())
+ if self._h5:
+ self._h5.close()
+ self._h5 = None
+ if wandb.run and wandb.run._jupyter_agent:
+ wandb.run._jupyter_agent.start()
+
+
+class HTTPSummary(Summary):
+ def __init__(self, run, client, summary=None):
+ super().__init__(run, summary=summary)
+ self._run = run
+ self._client = client
+ self._started = time.time()
+
+ def load(self):
+ pass
+
+ def open_h5(self):
+ if not self._h5 and h5py:
+ download_h5(
+ self._run.id,
+ entity=self._run.entity,
+ project=self._run.project,
+ out_dir=self._run.dir,
+ )
+ super().open_h5()
+
+ def _write(self, commit=False):
+ mutation = gql(
+ """
+ mutation UpsertBucket( $id: String, $summaryMetrics: JSONString) {
+ upsertBucket(input: { id: $id, summaryMetrics: $summaryMetrics}) {
+ bucket { id }
+ }
+ }
+ """
+ )
+ if commit:
+ if self._h5:
+ self._h5.close()
+ self._h5 = None
+ res = self._client.execute(
+ mutation,
+ variable_values={
+ "id": self._run.storage_id,
+ "summaryMetrics": util.json_dumps_safer(self._json_dict),
+ },
+ )
+ assert res["upsertBucket"]["bucket"]["id"]
+ entity, project, run = self._run.path
+ if (
+ os.path.exists(self._h5_path)
+ and os.path.getmtime(self._h5_path) >= self._started
+ ):
+ upload_h5(self._h5_path, run, entity=entity, project=project)
+ else:
+ return False
diff --git a/wandb/plot/__init__.py b/wandb/plot/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..5b6220af7cd0395cafbbbe1088120d241f693260
--- /dev/null
+++ b/wandb/plot/__init__.py
@@ -0,0 +1,19 @@
+from wandb.plot.bar import bar
+from wandb.plot.confusion_matrix import confusion_matrix
+from wandb.plot.histogram import histogram
+from wandb.plot.line import line
+from wandb.plot.line_series import line_series
+from wandb.plot.pr_curve import pr_curve
+from wandb.plot.roc_curve import roc_curve
+from wandb.plot.scatter import scatter
+
+__all__ = [
+ "line",
+ "histogram",
+ "scatter",
+ "bar",
+ "roc_curve",
+ "pr_curve",
+ "confusion_matrix",
+ "line_series",
+]
diff --git a/wandb/plot/bar.py b/wandb/plot/bar.py
new file mode 100644
index 0000000000000000000000000000000000000000..a78bec2aed357bfe5aef5e3d78f9b77e9a0fd27f
--- /dev/null
+++ b/wandb/plot/bar.py
@@ -0,0 +1,42 @@
+from typing import Optional
+
+import wandb
+
+
+def bar(
+ table: wandb.Table,
+ label: str,
+ value: str,
+ title: Optional[str] = None,
+ split_table: Optional[bool] = False,
+):
+ """Construct a bar plot.
+
+ Arguments:
+ table (wandb.Table): Table of data.
+ label (string): Name of column to use as each bar's label.
+ value (string): Name of column to use as each bar's value.
+ title (string): Plot title.
+ split_table (bool): If True, adds "Custom Chart Tables/" to the key of the table so that it's logged in a different section.
+
+ Returns:
+ A plot object, to be passed to wandb.log()
+
+ Example:
+ ```
+ table = wandb.Table(data=[
+ ['car', random.random()],
+ ['bus', random.random()],
+ ['road', random.random()],
+ ['person', random.random()],
+ ], columns=["class", "acc"])
+ wandb.log({'bar-plot1': wandb.plot.bar(table, "class", "acc")})
+ ```
+ """
+ return wandb.plot_table(
+ "wandb/bar/v0",
+ table,
+ {"label": label, "value": value},
+ {"title": title},
+ split_table=split_table,
+ )
diff --git a/wandb/plot/confusion_matrix.py b/wandb/plot/confusion_matrix.py
new file mode 100644
index 0000000000000000000000000000000000000000..ed2b948563f5ae0a367a6287f314fcafc7184ec1
--- /dev/null
+++ b/wandb/plot/confusion_matrix.py
@@ -0,0 +1,99 @@
+from typing import Optional, Sequence
+
+import wandb
+from wandb import util
+
+chart_limit = wandb.Table.MAX_ROWS
+
+
+def confusion_matrix(
+ probs: Optional[Sequence[Sequence]] = None,
+ y_true: Optional[Sequence] = None,
+ preds: Optional[Sequence] = None,
+ class_names: Optional[Sequence[str]] = None,
+ title: Optional[str] = None,
+ split_table: Optional[bool] = False,
+):
+ """Compute a multi-run confusion matrix.
+
+ Arguments:
+ probs (2-d arr): Shape [n_examples, n_classes]
+ y_true (arr): Array of label indices.
+ preds (arr): Array of predicted label indices.
+ class_names (arr): Array of class names.
+ split_table (bool): If True, adds "Custom Chart Tables/" to the key of the table so that it's logged in a different section.
+
+ Returns:
+ Nothing. To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```
+ vals = np.random.uniform(size=(10, 5))
+ probs = np.exp(vals)/np.sum(np.exp(vals), keepdims=True, axis=1)
+ y_true = np.random.randint(0, 5, size=(10))
+ labels = ["Cat", "Dog", "Bird", "Fish", "Horse"]
+ wandb.log({'confusion_matrix': wandb.plot.confusion_matrix(probs, y_true=y_true, class_names=labels)})
+ ```
+ """
+ np = util.get_module(
+ "numpy",
+ required="confusion matrix requires the numpy library, install with `pip install numpy`",
+ )
+ # change warning
+ assert probs is None or len(probs.shape) == 2, (
+ "confusion_matrix has been updated to accept"
+ " probabilities as the default first argument. Use preds=..."
+ )
+
+ assert (probs is None or preds is None) and not (
+ probs is None and preds is None
+ ), "Must provide probabilties or predictions but not both to confusion matrix"
+
+ if probs is not None:
+ preds = np.argmax(probs, axis=1).tolist()
+
+ assert len(preds) == len(
+ y_true
+ ), "Number of predictions and label indices must match"
+
+ if class_names is not None:
+ n_classes = len(class_names)
+ class_inds = [i for i in range(n_classes)]
+ assert max(preds) <= len(
+ class_names
+ ), "Higher predicted index than number of classes"
+ assert max(y_true) <= len(
+ class_names
+ ), "Higher label class index than number of classes"
+ else:
+ class_inds = set(preds).union(set(y_true))
+ n_classes = len(class_inds)
+ class_names = [f"Class_{i}" for i in range(1, n_classes + 1)]
+
+ # get mapping of inds to class index in case user has weird prediction indices
+ class_mapping = {}
+ for i, val in enumerate(sorted(list(class_inds))):
+ class_mapping[val] = i
+ counts = np.zeros((n_classes, n_classes))
+ for i in range(len(preds)):
+ counts[class_mapping[y_true[i]], class_mapping[preds[i]]] += 1
+
+ data = []
+ for i in range(n_classes):
+ for j in range(n_classes):
+ data.append([class_names[i], class_names[j], counts[i, j]])
+
+ fields = {
+ "Actual": "Actual",
+ "Predicted": "Predicted",
+ "nPredictions": "nPredictions",
+ }
+ title = title or ""
+ return wandb.plot_table(
+ "wandb/confusion_matrix/v1",
+ wandb.Table(columns=["Actual", "Predicted", "nPredictions"], data=data),
+ fields,
+ {"title": title},
+ split_table=split_table,
+ )
diff --git a/wandb/plot/histogram.py b/wandb/plot/histogram.py
new file mode 100644
index 0000000000000000000000000000000000000000..6e294fd3b41de940812d84ad20c297eb95c08444
--- /dev/null
+++ b/wandb/plot/histogram.py
@@ -0,0 +1,36 @@
+from typing import Optional
+
+import wandb
+
+
+def histogram(
+ table: wandb.Table,
+ value: str,
+ title: Optional[str] = None,
+ split_table: Optional[bool] = False,
+):
+ """Construct a histogram plot.
+
+ Arguments:
+ table (wandb.Table): Table of data.
+ value (string): Name of column to use as data for bucketing.
+ title (string): Plot title.
+ split_table (bool): If True, adds "Custom Chart Tables/" to the key of the table so that it's logged in a different section.
+
+ Returns:
+ A plot object, to be passed to wandb.log()
+
+ Example:
+ ```
+ data = [[i, random.random() + math.sin(i / 10)] for i in range(100)]
+ table = wandb.Table(data=data, columns=["step", "height"])
+ wandb.log({'histogram-plot1': wandb.plot.histogram(table, "height")})
+ ```
+ """
+ return wandb.plot_table(
+ "wandb/histogram/v0",
+ table,
+ {"value": value},
+ {"title": title},
+ split_table=split_table,
+ )
diff --git a/wandb/plot/line.py b/wandb/plot/line.py
new file mode 100644
index 0000000000000000000000000000000000000000..4fb8f06bd955afa51d7923b90a154e072ec7008e
--- /dev/null
+++ b/wandb/plot/line.py
@@ -0,0 +1,40 @@
+from typing import Optional
+
+import wandb
+
+
+def line(
+ table: wandb.Table,
+ x: str,
+ y: str,
+ stroke: Optional[str] = None,
+ title: Optional[str] = None,
+ split_table: Optional[bool] = False,
+):
+ """Construct a line plot.
+
+ Arguments:
+ table (wandb.Table): Table of data.
+ x (string): Name of column to as for x-axis values.
+ y (string): Name of column to as for y-axis values.
+ stroke (string): Name of column to map to the line stroke scale.
+ title (string): Plot title.
+ split_table (bool): If True, adds "Custom Chart Tables/" to the key of the table so that it's logged in a different section.
+
+ Returns:
+ A plot object, to be passed to wandb.log()
+
+ Example:
+ ```
+ data = [[i, random.random() + math.sin(i / 10)] for i in range(100)]
+ table = wandb.Table(data=data, columns=["step", "height"])
+ wandb.log({'line-plot1': wandb.plot.line(table, "step", "height")})
+ ```
+ """
+ return wandb.plot_table(
+ "wandb/line/v0",
+ table,
+ {"x": x, "y": y, "stroke": stroke},
+ {"title": title},
+ split_table=split_table,
+ )
diff --git a/wandb/plot/line_series.py b/wandb/plot/line_series.py
new file mode 100644
index 0000000000000000000000000000000000000000..02f5a50a7c1b60ff33613a9b90ebcf6d9ce561d5
--- /dev/null
+++ b/wandb/plot/line_series.py
@@ -0,0 +1,88 @@
+import typing as t
+from collections.abc import Iterable
+
+import wandb
+
+
+def line_series(
+ xs: t.Union[t.Iterable, t.Iterable[t.Iterable]],
+ ys: t.Iterable[t.Iterable],
+ keys: t.Optional[t.Iterable] = None,
+ title: t.Optional[str] = None,
+ xname: t.Optional[str] = None,
+ split_table: t.Optional[bool] = False,
+):
+ """Construct a line series plot.
+
+ Arguments:
+ xs (array of arrays, or array): Array of arrays of x values
+ ys (array of arrays): Array of y values
+ keys (array): Array of labels for the line plots
+ title (string): Plot title.
+ xname: Title of x-axis
+ split_table (bool): If True, adds "Custom Chart Tables/" to the key of the table so that it's logged in a different section.
+
+ Returns:
+ A plot object, to be passed to wandb.log()
+
+ Example:
+ When logging a singular array for xs, all ys are plotted against that xs
+ <!--yeadoc-test:plot-line-series-single-->
+ ```python
+ import wandb
+
+ run = wandb.init()
+ xs = [i for i in range(10)]
+ ys = [[i for i in range(10)], [i**2 for i in range(10)]]
+ run.log(
+ {"line-series-plot1": wandb.plot.line_series(xs, ys, title="title", xname="step")}
+ )
+ run.finish()
+ ```
+ xs can also contain an array of arrays for having different steps for each metric
+ <!--yeadoc-test:plot-line-series-double-->
+ ```python
+ import wandb
+
+ run = wandb.init()
+ xs = [[i for i in range(10)], [2 * i for i in range(10)]]
+ ys = [[i for i in range(10)], [i**2 for i in range(10)]]
+ run.log(
+ {"line-series-plot2": wandb.plot.line_series(xs, ys, title="title", xname="step")}
+ )
+ run.finish()
+ ```
+ """
+ if not isinstance(xs, Iterable):
+ raise TypeError(f"Expected xs to be an array instead got {type(xs)}")
+
+ if not isinstance(ys, Iterable):
+ raise TypeError(f"Expected ys to be an array instead got {type(xs)}")
+
+ for y in ys:
+ if not isinstance(y, Iterable):
+ raise TypeError(
+ f"Expected ys to be an array of arrays instead got {type(y)}"
+ )
+
+ if not isinstance(xs[0], Iterable) or isinstance(xs[0], (str, bytes)):
+ xs = [xs for _ in range(len(ys))]
+ assert len(xs) == len(ys), "Number of x-lines and y-lines must match"
+
+ if keys is not None:
+ assert len(keys) == len(ys), "Number of keys and y-lines must match"
+ data = [
+ [x, f"key_{i}" if keys is None else keys[i], y]
+ for i, (xx, yy) in enumerate(zip(xs, ys))
+ for x, y in zip(xx, yy)
+ ]
+
+ table = wandb.Table(data=data, columns=["step", "lineKey", "lineVal"])
+
+ return wandb.plot_table(
+ "wandb/lineseries/v0",
+ table,
+ {"step": "step", "lineKey": "lineKey", "lineVal": "lineVal"},
+ {"title": title, "xname": xname or "x"},
+ split_table=split_table,
+ )
diff --git a/wandb/plot/pr_curve.py b/wandb/plot/pr_curve.py
new file mode 100644
index 0000000000000000000000000000000000000000..4bc832683b7f42c97004c90ba80c43452ff018da
--- /dev/null
+++ b/wandb/plot/pr_curve.py
@@ -0,0 +1,135 @@
+from typing import Optional
+
+import wandb
+from wandb import util
+from wandb.plots.utils import test_missing, test_types
+
+
+def pr_curve(
+ y_true=None,
+ y_probas=None,
+ labels=None,
+ classes_to_plot=None,
+ interp_size=21,
+ title=None,
+ split_table: Optional[bool] = False,
+):
+ """Compute the tradeoff between precision and recall for different thresholds.
+
+ A high area under the curve represents both high recall and high precision, where
+ high precision relates to a low false positive rate, and high recall relates to a
+ low false negative rate. High scores for both show that the classifier is returning
+ accurate results (high precision), and returning a majority of all positive results
+ (high recall). PR curve is useful when the classes are very imbalanced.
+
+ Arguments:
+ y_true (arr): true sparse labels y_probas (arr): Target scores, can either be
+ probability estimates, confidence values, or non-thresholded measure of
+ decisions. shape: (*y_true.shape, num_classes)
+ labels (list): Named labels for target variable (y). Makes plots easier to read
+ by replacing target values with corresponding index. For example labels =
+ ['dog', 'cat', 'owl'] all 0s are replaced by 'dog', 1s by 'cat'.
+ classes_to_plot (list): unique values of y_true to include in the plot
+ interp_size (int): the recall values will be fixed to `interp_size` points
+ uniform on [0, 1] and the precision will be interpolated for these recall
+ values.
+ split_table (bool): If True, adds "Custom Chart Tables/" to the key of the table so that it's logged in a different section.
+
+ Returns:
+ Nothing. To see plots, go to your W&B run page then expand the 'media' tab under
+ 'auto visualizations'.
+
+ Example:
+ ```
+ wandb.log({"pr-curve": wandb.plot.pr_curve(y_true, y_probas, labels)})
+ ```
+ """
+ np = util.get_module(
+ "numpy",
+ required="roc requires the numpy library, install with `pip install numpy`",
+ )
+ pd = util.get_module(
+ "pandas",
+ required="roc requires the pandas library, install with `pip install pandas`",
+ )
+ sklearn_metrics = util.get_module(
+ "sklearn.metrics",
+ "roc requires the scikit library, install with `pip install scikit-learn`",
+ )
+ sklearn_utils = util.get_module(
+ "sklearn.utils",
+ "roc requires the scikit library, install with `pip install scikit-learn`",
+ )
+
+ def _step(x):
+ y = np.array(x)
+ for i in range(1, len(y)):
+ y[i] = max(y[i], y[i - 1])
+ return y
+
+ y_true = np.array(y_true)
+ y_probas = np.array(y_probas)
+
+ if not test_missing(y_true=y_true, y_probas=y_probas):
+ return
+ if not test_types(y_true=y_true, y_probas=y_probas):
+ return
+
+ classes = np.unique(y_true)
+ if classes_to_plot is None:
+ classes_to_plot = classes
+
+ precision = dict()
+ interp_recall = np.linspace(0, 1, interp_size)[::-1]
+ indices_to_plot = np.where(np.isin(classes, classes_to_plot))[0]
+ for i in indices_to_plot:
+ if labels is not None and (
+ isinstance(classes[i], int) or isinstance(classes[0], np.integer)
+ ):
+ class_label = labels[classes[i]]
+ else:
+ class_label = classes[i]
+
+ cur_precision, cur_recall, _ = sklearn_metrics.precision_recall_curve(
+ y_true, y_probas[:, i], pos_label=classes[i]
+ )
+ # smooth the precision (monotonically increasing)
+ cur_precision = _step(cur_precision)
+
+ # reverse order so that recall in ascending
+ cur_precision = cur_precision[::-1]
+ cur_recall = cur_recall[::-1]
+ indices = np.searchsorted(cur_recall, interp_recall, side="left")
+ precision[class_label] = cur_precision[indices]
+
+ df = pd.DataFrame(
+ {
+ "class": np.hstack([[k] * len(v) for k, v in precision.items()]),
+ "precision": np.hstack(list(precision.values())),
+ "recall": np.tile(interp_recall, len(precision)),
+ }
+ )
+ df = df.round(3)
+
+ if len(df) > wandb.Table.MAX_ROWS:
+ wandb.termwarn(
+ "wandb uses only %d data points to create the plots." % wandb.Table.MAX_ROWS
+ )
+ # different sampling could be applied, possibly to ensure endpoints are kept
+ df = sklearn_utils.resample(
+ df,
+ replace=False,
+ n_samples=wandb.Table.MAX_ROWS,
+ random_state=42,
+ stratify=df["class"],
+ ).sort_values(["precision", "recall", "class"])
+
+ table = wandb.Table(dataframe=df)
+ title = title or "Precision v. Recall"
+ return wandb.plot_table(
+ "wandb/area-under-curve/v0",
+ table,
+ {"x": "recall", "y": "precision", "class": "class"},
+ {"title": title},
+ split_table=split_table,
+ )
diff --git a/wandb/plot/roc_curve.py b/wandb/plot/roc_curve.py
new file mode 100644
index 0000000000000000000000000000000000000000..a828aabed698f56d40cc418b150c31bd1b532728
--- /dev/null
+++ b/wandb/plot/roc_curve.py
@@ -0,0 +1,117 @@
+from typing import Optional
+
+import wandb
+from wandb import util
+from wandb.plots.utils import test_missing, test_types
+
+
+def roc_curve(
+ y_true=None,
+ y_probas=None,
+ labels=None,
+ classes_to_plot=None,
+ title=None,
+ split_table: Optional[bool] = False,
+):
+ """Calculate and visualize receiver operating characteristic (ROC) scores.
+
+ Arguments:
+ y_true (arr): true sparse labels
+ y_probas (arr): Target scores, can either be probability estimates, confidence
+ values, or non-thresholded measure of decisions.
+ shape: (*y_true.shape, num_classes)
+ labels (list): Named labels for target variable (y). Makes plots easier to
+ read by replacing target values with corresponding index.
+ For example labels = ['dog', 'cat', 'owl'] all 0s are
+ replaced by 'dog', 1s by 'cat'.
+ classes_to_plot (list): unique values of y_true to include in the plot
+ split_table (bool): If True, adds "Custom Chart Tables/" to the key of the table so that it's logged in a different section.
+
+ Returns:
+ Nothing. To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```
+ wandb.log({'roc-curve': wandb.plot.roc_curve(y_true, y_probas, labels)})
+ ```
+ """
+ np = util.get_module(
+ "numpy",
+ required="roc requires the numpy library, install with `pip install numpy`",
+ )
+ pd = util.get_module(
+ "pandas",
+ required="roc requires the pandas library, install with `pip install pandas`",
+ )
+ sklearn_metrics = util.get_module(
+ "sklearn.metrics",
+ "roc requires the scikit library, install with `pip install scikit-learn`",
+ )
+ sklearn_utils = util.get_module(
+ "sklearn.utils",
+ "roc requires the scikit library, install with `pip install scikit-learn`",
+ )
+
+ y_true = np.array(y_true)
+ y_probas = np.array(y_probas)
+
+ if not test_missing(y_true=y_true, y_probas=y_probas):
+ return
+ if not test_types(y_true=y_true, y_probas=y_probas):
+ return
+
+ classes = np.unique(y_true)
+ if classes_to_plot is None:
+ classes_to_plot = classes
+
+ fpr = dict()
+ tpr = dict()
+ indices_to_plot = np.where(np.isin(classes, classes_to_plot))[0]
+ for i in indices_to_plot:
+ if labels is not None and (
+ isinstance(classes[i], int) or isinstance(classes[0], np.integer)
+ ):
+ class_label = labels[classes[i]]
+ else:
+ class_label = classes[i]
+
+ fpr[class_label], tpr[class_label], _ = sklearn_metrics.roc_curve(
+ y_true, y_probas[..., i], pos_label=classes[i]
+ )
+
+ df = pd.DataFrame(
+ {
+ "class": np.hstack([[k] * len(v) for k, v in fpr.items()]),
+ "fpr": np.hstack(list(fpr.values())),
+ "tpr": np.hstack(list(tpr.values())),
+ }
+ )
+ df = df.round(3)
+
+ if len(df) > wandb.Table.MAX_ROWS:
+ wandb.termwarn(
+ "wandb uses only %d data points to create the plots." % wandb.Table.MAX_ROWS
+ )
+ # different sampling could be applied, possibly to ensure endpoints are kept
+ df = sklearn_utils.resample(
+ df,
+ replace=False,
+ n_samples=wandb.Table.MAX_ROWS,
+ random_state=42,
+ stratify=df["class"],
+ ).sort_values(["fpr", "tpr", "class"])
+
+ table = wandb.Table(dataframe=df)
+ title = title or "ROC"
+ return wandb.plot_table(
+ "wandb/area-under-curve/v0",
+ table,
+ {"x": "fpr", "y": "tpr", "class": "class"},
+ {
+ "title": title,
+ "x-axis-title": "False positive rate",
+ "y-axis-title": "True positive rate",
+ },
+ split_table=split_table,
+ )
diff --git a/wandb/plot/scatter.py b/wandb/plot/scatter.py
new file mode 100644
index 0000000000000000000000000000000000000000..b5b8e775caa712406fe355a6bf61bc20229dc5e6
--- /dev/null
+++ b/wandb/plot/scatter.py
@@ -0,0 +1,32 @@
+from typing import Optional
+
+import wandb
+
+
+def scatter(table, x, y, title=None, split_table: Optional[bool] = False):
+ """Construct a scatter plot.
+
+ Arguments:
+ table (wandb.Table): Table of data.
+ x (string): Name of column to as for x-axis values.
+ y (string): Name of column to as for y-axis values.
+ title (string): Plot title.
+ split_table (bool): If True, adds "Custom Chart Tables/" to the key of the table so that it's logged in a different section.
+
+ Returns:
+ A plot object, to be passed to wandb.log()
+
+ Example:
+ ```
+ data = [[i, random.random() + math.sin(i / 10)] for i in range(100)]
+ table = wandb.Table(data=data, columns=["step", "height"])
+ wandb.log({'scatter-plot1': wandb.plot.scatter(table, "step", "height")})
+ ```
+ """
+ return wandb.plot_table(
+ "wandb/scatter/v0",
+ table,
+ {"x": x, "y": y},
+ {"title": title},
+ split_table=split_table,
+ )
diff --git a/wandb/plots/__init__.py b/wandb/plots/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..c15a4d7d06882efaa890e25d80bd9a34b021f2b0
--- /dev/null
+++ b/wandb/plots/__init__.py
@@ -0,0 +1,6 @@
+from wandb.plots.explain_text import explain_text as ExplainText
+from wandb.plots.heatmap import heatmap as HeatMap
+from wandb.plots.named_entity import named_entity as NER
+from wandb.plots.part_of_speech import part_of_speech as POS
+from wandb.plots.precision_recall import precision_recall
+from wandb.plots.roc import roc as ROC
diff --git a/wandb/plots/explain_text.py b/wandb/plots/explain_text.py
new file mode 100644
index 0000000000000000000000000000000000000000..f8314934c348be57f92a2e098164f8585f5ef30a
--- /dev/null
+++ b/wandb/plots/explain_text.py
@@ -0,0 +1,36 @@
+import wandb
+from wandb import util
+from wandb.plots.utils import (
+ deprecation_notice,
+ encode_labels,
+ test_missing,
+ test_types,
+)
+
+
+def explain_text(text, probas, target_names=None):
+ """
+ ExplainText adds support for eli5's LIME based TextExplainer.
+ Arguments:
+ text (str): Text to explain
+ probas (black-box classification pipeline): A function which
+ takes a list of strings (documents) and returns a matrix
+ of shape (n_samples, n_classes) with probability values,
+ i.e. a row per document and a column per output label.
+ Returns:
+ Nothing. To see plots, go to your W&B run page.
+ Example:
+ wandb.log({'roc': wandb.plots.ExplainText(text, probas)})
+ """
+ deprecation_notice()
+ eli5 = util.get_module(
+ "eli5",
+ required="explain_text requires the eli5 library, install with `pip install eli5`",
+ )
+ if test_missing(text=text, probas=probas):
+ # and test_types(proba=proba)):
+ wandb.termlog("Visualizing TextExplainer.")
+ te = eli5.lime.TextExplainer(random_state=42)
+ te.fit(text, probas)
+ html = te.show_prediction(target_names=target_names)
+ return wandb.Html(html.data)
diff --git a/wandb/plots/heatmap.py b/wandb/plots/heatmap.py
new file mode 100644
index 0000000000000000000000000000000000000000..9bb5fb225714eb1455c0a2bef378b8532a722673
--- /dev/null
+++ b/wandb/plots/heatmap.py
@@ -0,0 +1,81 @@
+import wandb
+from wandb import util
+from wandb.plots.utils import (
+ deprecation_notice,
+ encode_labels,
+ test_missing,
+ test_types,
+)
+
+chart_limit = wandb.Table.MAX_ROWS
+
+
+def heatmap(x_labels, y_labels, matrix_values, show_text=False):
+ """
+ Generates a heatmap.
+
+ Arguments:
+ matrix_values (arr): 2D dataset of shape x_labels * y_labels, containing
+ heatmap values that can be coerced into an ndarray.
+ x_labels (list): Named labels for rows (x_axis).
+ y_labels (list): Named labels for columns (y_axis).
+ show_text (bool): Show text values in heatmap cells.
+
+ Returns:
+ Nothing. To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ wandb.log({'heatmap': wandb.plots.HeatMap(x_labels, y_labels,
+ matrix_values)})
+ """
+ deprecation_notice()
+
+ np = util.get_module(
+ "numpy",
+ required="roc requires the numpy library, install with `pip install numpy`",
+ )
+ scikit = util.get_module(
+ "sklearn",
+ required="roc requires the scikit library, install with `pip install scikit-learn`",
+ )
+
+ if test_missing(
+ x_labels=x_labels, y_labels=y_labels, matrix_values=matrix_values
+ ) and test_types(x_labels=x_labels, y_labels=y_labels, matrix_values=matrix_values):
+ matrix_values = np.array(matrix_values)
+ wandb.termlog("Visualizing heatmap.")
+
+ def heatmap_table(x_labels, y_labels, matrix_values, show_text):
+ x_axis = []
+ y_axis = []
+ values = []
+ count = 0
+ for i, x in enumerate(x_labels):
+ for j, y in enumerate(y_labels):
+ x_axis.append(x)
+ y_axis.append(y)
+ values.append(matrix_values[j][i])
+ count += 1
+ if count >= chart_limit:
+ wandb.termwarn(
+ "wandb uses only the first %d datapoints to create the plots."
+ % wandb.Table.MAX_ROWS
+ )
+ break
+ if show_text:
+ heatmap_key = "wandb/heatmap/v1"
+ else:
+ heatmap_key = "wandb/heatmap_no_text/v1"
+ return wandb.visualize(
+ heatmap_key,
+ wandb.Table(
+ columns=["x_axis", "y_axis", "values"],
+ data=[
+ [x_axis[i], y_axis[i], round(values[i], 2)]
+ for i in range(len(x_axis))
+ ],
+ ),
+ )
+
+ return heatmap_table(x_labels, y_labels, matrix_values, show_text)
diff --git a/wandb/plots/named_entity.py b/wandb/plots/named_entity.py
new file mode 100644
index 0000000000000000000000000000000000000000..e4f81e04fb9a68a8e8c5239ffd7a76fe6dae16f6
--- /dev/null
+++ b/wandb/plots/named_entity.py
@@ -0,0 +1,43 @@
+import wandb
+from wandb import util
+from wandb.plots.utils import (
+ deprecation_notice,
+ encode_labels,
+ test_missing,
+ test_types,
+)
+
+
+def named_entity(docs):
+ """
+ Adds support for spaCy's entity visualizer, which highlights named
+ entities and their labels in a text.
+
+ Arguments:
+ docs (list, Doc, Span): Document(s) to visualize.
+
+ Returns:
+ Nothing. To see plots, go to your W&B run page.
+
+ Example:
+ wandb.log({'NER': wandb.plots.NER(docs=doc)})
+ """
+ deprecation_notice()
+
+ spacy = util.get_module(
+ "spacy",
+ required="part_of_speech requires the spacy library, install with `pip install spacy`",
+ )
+ en_core_web_md = util.get_module(
+ "en_core_web_md",
+ required="part_of_speech requires the en_core_web_md library, install with `python -m spacy download en_core_web_md`",
+ )
+ nlp = en_core_web_md.load()
+
+ if test_missing(docs=docs):
+ # and test_types(docs=docs)):
+ wandb.termlog("Visualizing named entity recognition.")
+ html = spacy.displacy.render(
+ nlp(str(docs)), style="ent", page=True, minify=True
+ )
+ return wandb.Html(html)
diff --git a/wandb/plots/part_of_speech.py b/wandb/plots/part_of_speech.py
new file mode 100644
index 0000000000000000000000000000000000000000..8dab7dc5d820c59f86880f3973724b6254704b0a
--- /dev/null
+++ b/wandb/plots/part_of_speech.py
@@ -0,0 +1,50 @@
+import wandb
+from wandb import util
+from wandb.plots.utils import (
+ deprecation_notice,
+ encode_labels,
+ test_missing,
+ test_types,
+)
+
+
+def part_of_speech(docs):
+ """
+ Adds support for spaCy's dependency visualizer which shows
+ part-of-speech tags and syntactic dependencies.
+
+ Arguments:
+ docs (list, Doc, Span): Document(s) to visualize.
+
+ Returns:
+ Nothing. To see plots, go to your W&B run page.
+
+ Example:
+ wandb.log({'part_of_speech': wandb.plots.POS(docs=doc)})
+ """
+ deprecation_notice()
+
+ spacy = util.get_module(
+ "spacy",
+ required="part_of_speech requires the spacy library, install with `pip install spacy`",
+ )
+ en_core_web_md = util.get_module(
+ "en_core_web_md",
+ required="part_of_speech requires the en_core_web_md library, install with `python -m spacy download en_core_web_md`",
+ )
+ nlp = en_core_web_md.load()
+
+ if test_missing(docs=docs):
+ # and test_types(docs=docs)):
+ wandb.termlog("Visualizing part of speech.")
+ options = {
+ "compact": True,
+ "color": "#1a1c1f",
+ "font": "Source Sans Pro",
+ "collapse_punct": True,
+ "collapse_phrases": True,
+ }
+ html = spacy.displacy.render(
+ nlp(str(docs)), style="dep", minify=True, options=options, page=True
+ )
+ return wandb.Html(html)
diff --git a/wandb/plots/plot_definitions.py b/wandb/plots/plot_definitions.py
new file mode 100644
index 0000000000000000000000000000000000000000..0cdbae3e892d247a78b369529a9cc2b2d1836bc7
--- /dev/null
+++ b/wandb/plots/plot_definitions.py
@@ -0,0 +1,768 @@
+"""
+plot_summary_metrics
+{
+ "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+ "data": {
+ "name": "${history-table:rows:x-axis,key}"
+ },
+ "title": "Summary Metrics",
+ "encoding": {
+ "y": {"field": "metric_name", "type": "nominal"},
+ "x": {"field": "metric_value", "type": "quantitative"},
+ "color": {"field": "metric_name", "type": "nominal",
+ "scale": {
+ "range": ["#AB47BC", "#3498DB", "#5C6BC0", "#3F51B5"]
+ }},
+ "opacity": {"value": 0.8}
+ },
+ "layer": [{
+ "mark": "bar"
+ }, {
+ "mark": {
+ "type": "text",
+ "align": "left",
+ "baseline": "middle",
+ "dx": 3
+ },
+ "encoding": {
+ "text": {"field": "metric_value", "type": "quantitative"}
+ }
+ }]
+}
+
+plot_learning_curve
+l2k2/sklearn_learningcurve
+
+{
+ "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+ "padding": 5,
+ "width": "500",
+ "height": "500",
+ "data":
+ {
+ "name": "${history-table:rows:x-axis,key}"
+ },
+ "title": {
+ "text": "Learning Curve"
+ },
+ "layer": [
+ {
+ "encoding": {
+ "x": {"field": "train_size", "type": "quantitative"},
+ "y": {"field": "score", "type": "quantitative"},
+ "color": {"field": "dataset", "type": "nominal"},
+ "opacity": {"value": 0.7}
+ },
+ "layer": [
+ {"mark": "line"},
+ {
+ "selection": {
+ "label": {
+ "type": "single",
+ "nearest": true,
+ "on": "mouseover",
+ "encodings": ["x"],
+ "empty": "none"
+ }
+ },
+ "mark": "point",
+ "encoding": {
+ "opacity": {
+ "condition": {"selection": "label", "value": 1},
+ "value": 0
+ }
+ }
+ }
+ ]
+ },
+ {
+ "transform": [{"filter": {"selection": "label"}}],
+ "layer": [
+ {
+ "mark": {"type": "rule", "color": "gray"},
+ "encoding": {
+ "x": {"type": "quantitative", "field": "train_size"}
+ }
+ },
+ {
+ "encoding": {
+ "text": {"type": "quantitative", "field": "score"},
+ "x": {"type": "quantitative", "field": "train_size"},
+ "y": {"type": "quantitative", "field": "score"}
+ },
+ "layer": [
+ {
+ "mark": {
+ "type": "text",
+ "stroke": "white",
+ "strokeWidth": 2,
+ "align": "left",
+ "dx": 5,
+ "dy": -5
+ }
+ },
+ {
+ "mark": {"type": "text", "align": "left", "dx": 5, "dy": -5},
+ "encoding": {
+ "color": {
+ "type": "nominal", "field": "dataset", "scale": {
+ "domain": ["train", "test"],
+ "range": ["#3498DB", "#AB47BC"]
+ },
+ "legend": {
+ "title": " "
+ }
+ }
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
+
+plot_roc
+{
+ "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+ "padding": 5,
+ "width": "500",
+ "height": "500",
+ "data":
+ {
+ "name": "${history-table:rows:x-axis,key}"
+ },
+ "title": {
+ "text": "ROC Curve"
+ },"layer": [
+ {
+ "encoding": {
+ "x": {"field": "fpr", "type": "quantitative", "axis": {"title": "False Positive Rate"}},
+ "y": {"field": "tpr", "type": "quantitative", "axis": {"title": "True Positive Rate"}},
+ "color": {"field": "class", "type": "nominal"},
+ "opacity": {"value": 0.7}
+ },
+ "layer": [
+ {"mark": "line"},
+ {
+ "selection": {
+ "label": {
+ "type": "single",
+ "nearest": true,
+ "on": "mouseover",
+ "encodings": ["x"],
+ "empty": "none"
+ }
+ },
+ "mark": "point",
+ "encoding": {
+ "opacity": {
+ "condition": {"selection": "label", "value": 1},
+ "value": 0
+ }
+ }
+ }
+ ]
+ },
+ {
+ "transform": [{"filter": {"selection": "label"}}],
+ "layer": [
+ {
+ "mark": {"type": "rule", "color": "gray"},
+ "encoding": {
+ "x": {"type": "quantitative", "field": "train_size"}
+ }
+ },
+ {
+ "encoding": {
+ "text": {"type": "quantitative", "field": "fpr"},
+ "x": {"type": "quantitative", "field": "fpr"}
+ },
+ "layer": [
+ {
+ "mark": {
+ "type": "text",
+ "stroke": "white",
+ "strokeWidth": 2,
+ "align": "left",
+ "dx": 5,
+ "dy": -5
+ }
+ },
+ {
+ "mark": {"type": "text", "align": "left", "dx": 5, "dy": -5},
+ "encoding": {
+ "color": {
+ "type": "nominal", "field": "class", "scale": {
+ "range": ["#3498DB", "#AB47BC"]
+ },
+ "legend": {
+ "title": " "
+ }
+ }
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
+
+plot_confusion_matrix
+{
+ "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+ "padding": 5,
+ "width": 500,
+ "height": 500,
+ "data":
+ {
+ "name": "${history-table:rows:x-axis,key}"
+ },
+ "title": {
+ "text": "Confusion Matrix"
+ },
+ "mark": "circle",
+ "encoding": {
+ "x": {
+ "field": "Predicted",
+ "type": "nominal",
+ "axis": {
+ "maxExtent": 50,
+ "labelLimit": 40,
+ "labelAngle": -45
+ }
+ },
+ "y": {
+ "field": "Actual",
+ "type": "nominal"
+
+ },
+ "size": {
+ "field": "Count",
+ "type": "quantitative"
+ },
+ "color": {
+ "value": "#3498DB"
+ }
+ }
+}
+
+plot_precision_recall
+{
+ "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+ "padding": 5,
+ "width": 500,
+ "height": 500,
+ "data":
+ {
+ "name": "${history-table:rows:x-axis,key}"
+ },
+ "title": {
+ "text": "Precision Recall"
+ },"layer": [
+ {
+ "encoding": {
+ "x": {"field": "precision", "type": "quantitative"},
+ "y": {"field": "recall", "type": "quantitative"},
+ "color": {"field": "class", "type": "nominal"},
+ "opacity": {"value": 0.7}
+ },
+ "layer": [
+ {"mark": "line"},
+ {
+ "selection": {
+ "label": {
+ "type": "single",
+ "nearest": true,
+ "on": "mouseover",
+ "encodings": ["x"],
+ "empty": "none"
+ }
+ },
+ "mark": "point",
+ "encoding": {
+ "opacity": {
+ "condition": {"selection": "label", "value": 1},
+ "value": 0
+ }
+ }
+ }
+ ]
+ },
+ {
+ "transform": [{"filter": {"selection": "label"}}],
+ "layer": [
+ {
+ "encoding": {
+ "text": {"type": "nominal", "field": "class"},
+ "x": {"type": "quantitative", "field": "precision"},
+ "y": {"type": "quantitative", "field": "recall"}
+ },
+ "layer": [
+ {
+ "mark": {
+ "type": "text",
+ "stroke": "white",
+ "strokeWidth": 2,
+ "align": "left",
+ "dx": 5,
+ "dy": -5
+ }
+ },
+ {
+ "mark": {"type": "text", "align": "left", "dx": 5, "dy": -5},
+ "encoding": {
+ "color": {
+ "type": "nominal", "field": "class", "scale": {
+ "range": ["#3498DB", "#AB47BC", "#55BBBB", "#BB9955"]
+ },
+ "legend": {
+ "title": " "
+ }
+ }
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
+
+plot_feature_importances
+{
+ "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+ "data": {
+ "name": "${history-table:rows:x-axis,key}"
+ },
+ "title": "Feature Importances",
+ "mark": "bar",
+ "encoding": {
+ "y": {"field": "feature_names", "type": "nominal", "axis": {"title":"Features"},"sort": "-x"},
+ "x": {"field": "importances", "type": "quantitative", "axis": {"title":"Importances"}},
+ "color": {"value": "#3498DB"},
+ "opacity": {"value": 0.9}
+ }
+}
+
+plot_elbow_curve
+{
+ "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+ "description": "A dual axis chart, created by setting y's scale resolution to `\"independent\"`",
+ "width": 400, "height": 300,
+ "data": {
+ "name": "${history-table:rows:x-axis,key}"
+ },
+ "title": "Elbow Plot - Errors vs Cluster Size",
+ "encoding": {
+ "x": {
+ "field": "cluster_ranges",
+ "bin": true,
+ "axis": {"title": "Number of Clusters"},
+ "type": "quantitative"
+ }
+ },
+ "layer": [
+ {
+ "mark": {"opacity": 0.5, "type": "line", "color": "#AB47BC"},
+ "encoding": {
+ "y": {
+ "field": "errors",
+ "type": "quantitative",
+ "axis": {"title": "Sum of Squared Errors", "titleColor": "#AB47BC"}
+ }
+ }
+ },
+ {
+ "mark": {"opacity": 0.3, "stroke": "#3498DB", "strokeDash": [6, 4], "type": "line"},
+ "encoding": {
+ "y": {
+ "field": "clustering_time",
+ "type": "quantitative",
+ "axis": {"title": "Clustering Time", "titleColor":"#3498DB"}
+ }
+ }
+ }
+ ],
+ "resolve": {"scale": {"y": "independent"}}
+}
+
+plot_silhouette
+{
+ "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+ "data": {"name": "${history-table:rows:x-axis,key}"},
+ "title": "Silhouette analysis of cluster centers",
+ "hconcat": [
+ {
+ "width": 400,
+ "height": 400,
+ "layer": [
+ {
+ "mark": "area",
+ "encoding": {
+ "x": {
+ "field": "x1",
+ "type": "quantitative",
+ "axis": {"title":"Silhouette Coefficients"}
+ },
+ "x2": {
+ "field": "x2"
+ },
+ "y": {
+ "title": "Cluster Label",
+ "field": "y_sil",
+ "type": "quantitative",
+ "axis": {"title":"Clusters", "labels": false}
+ },
+ "color": {
+ "field": "color_sil",
+ "type": "nominal",
+ "axis": {"title":"Cluster Labels"},
+ "scale": {
+ "range": ["#AB47BC", "#3498DB", "#55BBBB", "#5C6BC0", "#FBC02D", "#3F51B5"]}
+ },
+ "opacity": { "value": 0.7 }
+ }},
+ {
+
+ "mark": {
+ "type":"rule",
+ "strokeDash": [6, 4],
+ "stroke":"#f88c99"},
+ "encoding": {
+ "x": {
+ "field": "silhouette_avg",
+ "type": "quantitative"
+ },
+ "color": {"value": "red"},
+ "size": {"value": 1},
+ "opacity": { "value": 0.5 }
+ }
+ }]
+ },
+ {
+ "width": 400,
+ "height": 400,
+ "layer": [
+ {
+ "mark": "circle",
+ "encoding": {
+ "x": {"field": "x", "type": "quantitative", "scale": {"zero": false}, "axis": {"title":"Feature Space for 1st Feature"}},
+ "y": {"field": "y", "type": "quantitative", "scale": {"zero": false}}, "axis": {"title":"Feature Space for 2nd Feature"},
+ "color": {"field": "colors", "type": "nominal", "axis": {"title":"Cluster Labels"}}
+ }
+ },
+ {
+ "mark": "point",
+ "encoding": {
+ "x": {"field": "centerx", "type": "quantitative", "scale": {"zero": false}, "axis": {"title":"Feature Space for 1st Feature"}},
+ "y": {"field": "centery", "type": "quantitative", "scale": {"zero": false}, "axis": {"title":"Feature Space for 2nd Feature"}},
+ "color": {"field": "colors", "type": "nominal", "axis": {"title":"Cluster Labels"}},
+ "size": {"value": 80}
+ }
+ }
+ ]
+ }
+ ]
+}
+
+plot_class_balance
+{
+ "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+ "width": 500,
+ "height": 500,
+ "title": "Class Proportions in Target Variable",
+ "data": {
+ "name": "${history-table:rows:x-axis,key}"
+ },
+ "selection": {
+ "highlight": {"type": "single", "empty": "none", "on": "mouseover"},
+ "select": {"type": "multi"}
+ },
+ "mark": {
+ "type": "bar",
+ "stroke": "black",
+ "cursor": "pointer"
+ },
+ "encoding": {
+ "x": {"field": "class", "type": "ordinal", "axis": {"title": "Class"}},
+ "y": {"field": "count", "type": "quantitative", "axis": {"title": "Number of instances"}},
+ "fillOpacity": {
+ "condition": {"selection": "select", "value": 1},
+ "value": 0.3
+ },
+ "opacity": {"value": 0.9},
+ "color": {
+ "field": "dataset",
+ "type": "nominal",
+ "scale": {
+ "domain": ["train", "test"],
+ "range": ["#3498DB", "#4DB6AC"]
+ },
+ "legend": {"title": "Dataset"}
+ },
+ "strokeWidth": {
+ "condition": [
+ {
+ "test": {
+ "and": [
+ {"selection": "select"},
+ "length(data(\"select_store\"))"
+ ]
+ },
+ "value": 2
+ },
+ {"selection": "highlight", "value": 1}
+ ],
+ "value": 0
+ }
+ },
+ "config": {
+ "scale": {
+ "bandPaddingInner": 0.2
+ }
+ }
+}
+
+plot_calibration_curve
+{
+ "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+ "padding": 5,
+ "data":
+ {
+ "name": "${history-table:rows:x-axis,key}"
+ },
+ "title": "Calibration Curve",
+ "vconcat": [
+ {
+ "layer": [
+ {
+ "encoding": {
+ "x": {"field": "mean_predicted_value", "type": "quantitative", "axis": {"title": "Mean predicted value"}},
+ "y": {"field": "fraction_of_positives", "type": "quantitative", "axis": {"title": "Fraction of positives"}},
+ "color": {
+ "field": "model",
+ "type": "nominal",
+ "axis": {"title": "Models"},
+ "scale": {
+ "range": ["#3498DB", "#AB47BC", "#55BBBB", "#BB9955", "#FBC02D"]
+ }
+ }
+ },
+ "layer": [
+ {
+ "mark": {
+ "type": "line",
+ "point": {
+ "filled": false,
+ "fill": "white"
+ }
+ }
+ }
+ ]
+ }]
+ },
+ {
+ "mark": {"type": "tick"},
+ "encoding": {
+ "x": {"field": "edge_dict", "type": "quantitative","bin":true, "axis": {"title": "Mean predicted value"}},
+ "y": {"field": "hist_dict", "type": "quantitative", "axis": {"title": "Counts"}},
+ "strokeWidth": {
+ "value": 2
+ },
+ "color": {
+ "field": "model",
+ "type": "nominal",
+ "axis": {"title": "Models"},
+ "scale": {
+ "range": ["#3498DB", "#AB47BC", "#55BBBB", "#BB9955"]
+ }
+ }
+ }
+ }
+ ]
+}
+
+plot_outlier_candidates
+{
+ "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+ "padding": 5,
+ "data":
+ {
+ "name": "${history-table:rows:x-axis,key}"
+ },
+ "title": {
+ "text": "Cook's Distance Outlier Detection"
+ },
+ "layer": [{
+ "mark": "bar",
+ "encoding": {
+ "x": {
+ "field": "instance_indicies",
+ "type": "quantitative",
+ "axis": {"title": "Instances"}
+ },
+ "y": {
+ "field": "distance",
+ "type": "quantitative",
+ "axis": {"title": "Influence (Cook's Distance)"}
+ },
+ "color": {"value": "#3498DB"},
+ "opacity": {"value": 0.4}
+ }
+ },{
+ "mark": {
+ "type":"rule",
+ "strokeDash": [6, 4],
+ "stroke":"#f88c99"},
+ "encoding": {
+ "y": {
+ "field": "influence_threshold",
+ "type": "quantitative"
+ },
+ "color": {"value": "red"},
+ "size": {"value": 1}
+ }
+ }, {
+ "mark": {
+ "type": "text",
+ "align": "left",
+ "baseline": "top",
+ "dx": 0
+ }
+ }]
+}
+
+plot_residuals
+{
+ "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+ "width": "container",
+ "data":
+ {
+ "name": "${history-table:rows:x-axis,key}"
+ },
+ "title": "Residuals Plot",
+ "vconcat": [
+ {
+ "layer": [
+ {
+ "encoding": {
+ "y": {"field": "y_pred", "type": "quantitative", "axis": {"title": "Predicted Value"}},
+ "x": {"field": "residuals", "type": "quantitative", "axis": {"title": "Residuals"}},
+ "color": {
+ "field": "dataset",
+ "type": "nominal",
+ "axis": {"title": "Dataset"}
+ }
+ },
+ "layer": [
+ {
+ "mark": {
+ "type": "point",
+ "opacity": 0.5,
+ "filled" : true
+ }
+ }
+ ]
+ }]
+ },
+ {
+ "mark": {"type": "bar",
+ "opacity": 0.8},
+ "encoding": {
+ "x": {"field": "residuals", "type": "quantitative", "bin": true, "axis": {"title": "Residuals"}},
+ "y": {
+ "aggregate": "count", "field": "residuals", "type": "quantitative", "axis": {"title": "Distribution"}},
+ "strokeWidth": {
+ "value": 1
+ },
+ "color": {
+ "field": "dataset",
+ "type": "nominal",
+ "axis": {"title": "Dataset"},
+ "scale": {
+ "range": ["#AB47BC", "#3498DB"]
+ }
+ }
+ }
+ }
+ ]
+}
+
+plot_decision_boundaries
+{
+ "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+ "data": {"name": "${history-table:rows:x-axis,key}"},
+ "title": "Decision Boundary - Projected Into 2D Space",
+ "width": 300,
+ "height": 200,
+ "layer": [
+ {
+ "mark": {"type" :"point", "opacity": 0.5},
+ "encoding": {
+ "x": {"field": "x", "type": "quantitative", "scale": {"zero": false}, "axis": {"title":"Principle Component Dimension 1"}},
+ "y": {"field": "y", "type": "quantitative", "scale": {"zero": false}, "axis": {"title":"Principle Component Dimension 2"}},
+ "color": {
+ "field": "color",
+ "type": "nominal",
+ "axis": {"title":"Cluster Labels"},
+ "scale": {
+ "range": ["#5C6BC0", "#AB47BC", "#4aa3df", "#3498DB", "#55BBBB"]
+ }
+ }
+ }
+ }
+ ]
+}
+"""
+
+"""
+heatmap/v1
+{
+ "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+ "padding": 5,
+ "width": 500,
+ "height": 500,
+ "data":
+ {
+ "name": "${history-table:rows:x-axis,key}"
+ },
+ "title": {
+ "text": {"value": ""}
+ },
+ "encoding": {
+ "x": {
+ "field": "x_axis",
+ "type": "nominal",
+ "axis": { "title": "" }
+ },
+ "y": {
+ "field": "y_axis",
+ "type": "nominal",
+ "axis": { "title": "" }
+ }
+ },
+ "layer": [
+ {
+ "mark": "rect",
+ "encoding": {
+ "color": {
+ "field": "values",
+ "type": "quantitative",
+ "title": "Values",
+ "scale": {
+ "scheme": "tealblues"
+ }
+ }
+ }
+ },
+ {
+ "mark": "text",
+ "encoding": {
+ "text": {"field": "values", "type": "quantitative"}
+ }
+ }
+ ]
+}
+"""
diff --git a/wandb/plots/precision_recall.py b/wandb/plots/precision_recall.py
new file mode 100644
index 0000000000000000000000000000000000000000..17605025ac452793fc93b21e190ad76cc9f0895c
--- /dev/null
+++ b/wandb/plots/precision_recall.py
@@ -0,0 +1,121 @@
+import wandb
+from wandb import util
+from wandb.plots.utils import (
+ deprecation_notice,
+ encode_labels,
+ test_missing,
+ test_types,
+)
+
+chart_limit = wandb.Table.MAX_ROWS
+
+
+def precision_recall(
+ y_true=None, y_probas=None, labels=None, plot_micro=True, classes_to_plot=None
+):
+ """
+ Computes the tradeoff between precision and recall for different thresholds.
+ A high area under the curve represents both high recall and high precision,
+ where high precision relates to a low false positive rate, and high recall
+ relates to a low false negative rate. High scores for both show that the
+ classifier is returning accurate results (high precision), as well as
+ returning a majority of all positive results (high recall).
+ PR curve is useful when the classes are very imbalanced.
+
+ Arguments:
+ y_true (arr): Test set labels.
+ y_probas (arr): Test set predicted probabilities.
+ labels (list): Named labels for target varible (y). Makes plots easier to
+ read by replacing target values with corresponding index.
+ For example labels= ['dog', 'cat', 'owl'] all 0s are
+ replaced by 'dog', 1s by 'cat'.
+
+ Returns:
+ Nothing. To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ wandb.log({'pr': wandb.plots.precision_recall(y_true, y_probas, labels)})
+ """
+ deprecation_notice()
+
+ np = util.get_module(
+ "numpy",
+ required="roc requires the numpy library, install with `pip install numpy`",
+ )
+ preprocessing = util.get_module(
+ "sklearn.preprocessing",
+ "roc requires the scikit preprocessing submodule, install with `pip install scikit-learn`",
+ )
+ metrics = util.get_module(
+ "sklearn.metrics",
+ "roc requires the scikit metrics submodule, install with `pip install scikit-learn`",
+ )
+
+ y_true = np.array(y_true)
+ y_probas = np.array(y_probas)
+
+ if test_missing(y_true=y_true, y_probas=y_probas) and test_types(
+ y_true=y_true, y_probas=y_probas
+ ):
+ classes = np.unique(y_true)
+ probas = y_probas
+
+ if classes_to_plot is None:
+ classes_to_plot = classes
+
+ binarized_y_true = preprocessing.label_binarize(y_true, classes=classes)
+ if len(classes) == 2:
+ binarized_y_true = np.hstack((1 - binarized_y_true, binarized_y_true))
+
+ pr_curves = {}
+ indices_to_plot = np.in1d(classes, classes_to_plot)
+ for i, to_plot in enumerate(indices_to_plot):
+ if to_plot:
+ average_precision = metrics.average_precision_score(
+ binarized_y_true[:, i], probas[:, i]
+ )
+ precision, recall, _ = metrics.precision_recall_curve(
+ y_true, probas[:, i], pos_label=classes[i]
+ )
+
+ samples = 20
+ sample_precision = []
+ sample_recall = []
+ for k in range(samples):
+ sample_precision.append(
+ precision[int(len(precision) * k / samples)]
+ )
+ sample_recall.append(recall[int(len(recall) * k / samples)])
+
+ pr_curves[classes[i]] = (sample_precision, sample_recall)
+
+ def pr_table(pr_curves):
+ data = []
+ count = 0
+ for i, class_name in enumerate(pr_curves.keys()):
+ precision, recall = pr_curves[class_name]
+ for p, r in zip(precision, recall):
+ # if class_names are ints and labels are set
+ if labels is not None and (
+ isinstance(class_name, int)
+ or isinstance(class_name, np.integer)
+ ):
+ class_name = labels[class_name]
+ # if class_names are ints and labels are not set
+ # or, if class_names have something other than ints
+ # (string, float, date) - user class_names
+ data.append([class_name, round(p, 3), round(r, 3)])
+ count += 1
+ if count >= chart_limit:
+ wandb.termwarn(
+ "wandb uses only the first %d datapoints to create the plots."
+ % wandb.Table.MAX_ROWS
+ )
+ break
+ return wandb.visualize(
+ "wandb/pr_curve/v1",
+ wandb.Table(columns=["class", "precision", "recall"], data=data),
+ )
+
+ return pr_table(pr_curves)
diff --git a/wandb/plots/roc.py b/wandb/plots/roc.py
new file mode 100644
index 0000000000000000000000000000000000000000..b627fc19d688c454823779bce2783533833b15f2
--- /dev/null
+++ b/wandb/plots/roc.py
@@ -0,0 +1,103 @@
+import wandb
+from wandb import util
+from wandb.plots.utils import (
+ deprecation_notice,
+ encode_labels,
+ test_missing,
+ test_types,
+)
+
+chart_limit = wandb.Table.MAX_ROWS
+
+
+def roc(
+ y_true=None,
+ y_probas=None,
+ labels=None,
+ plot_micro=True,
+ plot_macro=True,
+ classes_to_plot=None,
+):
+ """
+ Calculates receiver operating characteristic scores and visualizes them as the
+ ROC curve.
+
+ Arguments:
+ y_true (arr): Test set labels.
+ y_probas (arr): Test set predicted probabilities.
+ labels (list): Named labels for target varible (y). Makes plots easier to
+ read by replacing target values with corresponding index.
+ For example labels= ['dog', 'cat', 'owl'] all 0s are
+ replaced by 'dog', 1s by 'cat'.
+
+ Returns:
+ Nothing. To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ wandb.log({'roc': wandb.plots.ROC(y_true, y_probas, labels)})
+ """
+ deprecation_notice()
+
+ np = util.get_module(
+ "numpy",
+ required="roc requires the numpy library, install with `pip install numpy`",
+ )
+ sklearn = util.get_module(
+ "sklearn",
+ required="roc requires the scikit library, install with `pip install scikit-learn`",
+ )
+ from sklearn.metrics import auc, roc_curve
+
+ if test_missing(y_true=y_true, y_probas=y_probas) and test_types(
+ y_true=y_true, y_probas=y_probas
+ ):
+ y_true = np.array(y_true)
+ y_probas = np.array(y_probas)
+ classes = np.unique(y_true)
+ probas = y_probas
+
+ if classes_to_plot is None:
+ classes_to_plot = classes
+
+ fpr_dict = dict()
+ tpr_dict = dict()
+
+ indices_to_plot = np.in1d(classes, classes_to_plot)
+
+ def roc_table(fpr_dict, tpr_dict, classes, indices_to_plot):
+ data = []
+ count = 0
+
+ for i, to_plot in enumerate(indices_to_plot):
+ fpr_dict[i], tpr_dict[i], _ = roc_curve(
+ y_true, probas[:, i], pos_label=classes[i]
+ )
+ if to_plot:
+ roc_auc = auc(fpr_dict[i], tpr_dict[i])
+ for j in range(len(fpr_dict[i])):
+ if labels is not None and (
+ isinstance(classes[i], int)
+ or isinstance(classes[0], np.integer)
+ ):
+ class_dict = labels[classes[i]]
+ else:
+ class_dict = classes[i]
+ fpr = [
+ class_dict,
+ round(fpr_dict[i][j], 3),
+ round(tpr_dict[i][j], 3),
+ ]
+ data.append(fpr)
+ count += 1
+ if count >= chart_limit:
+ wandb.termwarn(
+ "wandb uses only the first %d datapoints to create the plots."
+ % wandb.Table.MAX_ROWS
+ )
+ break
+ return wandb.visualize(
+ "wandb/roc/v1", wandb.Table(columns=["class", "fpr", "tpr"], data=data)
+ )
+
+ return roc_table(fpr_dict, tpr_dict, classes, indices_to_plot)
diff --git a/wandb/plots/utils.py b/wandb/plots/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..9d8bb0f1cab8118ced4f80335f74f41ebfd85cc3
--- /dev/null
+++ b/wandb/plots/utils.py
@@ -0,0 +1,195 @@
+from collections.abc import Iterable, Sequence
+
+import wandb
+from wandb import util
+from wandb.sdk.lib import deprecate
+
+
+def deprecation_notice() -> None:
+ deprecate.deprecate(
+ field_name=deprecate.Deprecated.plots,
+ warning_message=(
+ "wandb.plots.* functions are deprecated and will be removed in a future release. "
+ "Please use wandb.plot.* instead."
+ ),
+ )
+
+
+# Test assumptions for plotting parameters and datasets
+def test_missing(**kwargs):
+ np = util.get_module("numpy", required="Logging plots requires numpy")
+ pd = util.get_module("pandas", required="Logging dataframes requires pandas")
+ scipy = util.get_module("scipy", required="Logging scipy matrices requires scipy")
+
+ test_passed = True
+ for k, v in kwargs.items():
+ # Missing/empty params/datapoint arrays
+ if v is None:
+ wandb.termerror("%s is None. Please try again." % (k))
+ test_passed = False
+ if (k == "X") or (k == "X_test"):
+ if isinstance(v, scipy.sparse.csr.csr_matrix):
+ v = v.toarray()
+ elif isinstance(v, (pd.DataFrame, pd.Series)):
+ v = v.to_numpy()
+ elif isinstance(v, list):
+ v = np.asarray(v)
+
+ # Warn the user about missing values
+ missing = 0
+ missing = np.count_nonzero(pd.isnull(v))
+ if missing > 0:
+ wandb.termwarn("%s contains %d missing values. " % (k, missing))
+ test_passed = False
+ # Ensure the dataset contains only integers
+ non_nums = 0
+ if v.ndim == 1:
+ non_nums = sum(
+ 1
+ for val in v
+ if (
+ not isinstance(val, (int, float, complex))
+ and not isinstance(val, np.number)
+ )
+ )
+ else:
+ non_nums = sum(
+ 1
+ for sl in v
+ for val in sl
+ if (
+ not isinstance(val, (int, float, complex))
+ and not isinstance(val, np.number)
+ )
+ )
+ if non_nums > 0:
+ wandb.termerror(
+ "%s contains values that are not numbers. Please vectorize, label encode or one hot encode %s and call the plotting function again."
+ % (k, k)
+ )
+ test_passed = False
+ return test_passed
+
+
+def test_fitted(model):
+ np = util.get_module("numpy", required="Logging plots requires numpy")
+ pd = util.get_module("pandas", required="Logging dataframes requires pandas")
+ scipy = util.get_module("scipy", required="Logging scipy matrices requires scipy")
+ scikit_utils = util.get_module(
+ "sklearn.utils",
+ required="roc requires the scikit utils submodule, install with `pip install scikit-learn`",
+ )
+ scikit_exceptions = util.get_module(
+ "sklearn.exceptions",
+ "roc requires the scikit preprocessing submodule, install with `pip install scikit-learn`",
+ )
+
+ try:
+ model.predict(np.zeros((7, 3)))
+ except scikit_exceptions.NotFittedError:
+ wandb.termerror("Please fit the model before passing it in.")
+ return False
+ except AttributeError:
+ # Some clustering models (LDA, PCA, Agglomerative) don't implement ``predict``
+ try:
+ scikit_utils.validation.check_is_fitted(
+ model,
+ [
+ "coef_",
+ "estimator_",
+ "labels_",
+ "n_clusters_",
+ "children_",
+ "components_",
+ "n_components_",
+ "n_iter_",
+ "n_batch_iter_",
+ "explained_variance_",
+ "singular_values_",
+ "mean_",
+ ],
+ all_or_any=any,
+ )
+ return True
+ except scikit_exceptions.NotFittedError:
+ wandb.termerror("Please fit the model before passing it in.")
+ return False
+ except Exception:
+ # Assume it's fitted, since ``NotFittedError`` wasn't raised
+ return True
+
+
+def encode_labels(df):
+ pd = util.get_module("pandas", required="Logging dataframes requires pandas")
+ preprocessing = util.get_module(
+ "sklearn.preprocessing",
+ "roc requires the scikit preprocessing submodule, install with `pip install scikit-learn`",
+ )
+
+ le = preprocessing.LabelEncoder()
+ # apply le on categorical feature columns
+ categorical_cols = df.select_dtypes(
+ exclude=["int", "float", "float64", "float32", "int32", "int64"]
+ ).columns
+ df[categorical_cols] = df[categorical_cols].apply(lambda col: le.fit_transform(col))
+
+
+def test_types(**kwargs):
+ np = util.get_module("numpy", required="Logging plots requires numpy")
+ pd = util.get_module("pandas", required="Logging dataframes requires pandas")
+ scipy = util.get_module("scipy", required="Logging scipy matrices requires scipy")
+
+ base = util.get_module(
+ "sklearn.base",
+ "roc requires the scikit base submodule, install with `pip install scikit-learn`",
+ )
+
+ test_passed = True
+ for k, v in kwargs.items():
+ # check for incorrect types
+ if (
+ (k == "X")
+ or (k == "X_test")
+ or (k == "y")
+ or (k == "y_test")
+ or (k == "y_true")
+ or (k == "y_probas")
+ or (k == "x_labels")
+ or (k == "y_labels")
+ or (k == "matrix_values")
+ ):
+ # FIXME: do this individually
+ if not isinstance(
+ v,
+ (
+ Sequence,
+ Iterable,
+ np.ndarray,
+ np.generic,
+ pd.DataFrame,
+ pd.Series,
+ list,
+ ),
+ ):
+ wandb.termerror("%s is not an array. Please try again." % (k))
+ test_passed = False
+ # check for classifier types
+ if k == "model":
+ if (not base.is_classifier(v)) and (not base.is_regressor(v)):
+ wandb.termerror(
+ "%s is not a classifier or regressor. Please try again." % (k)
+ )
+ test_passed = False
+ elif k == "clf" or k == "binary_clf":
+ if not (base.is_classifier(v)):
+ wandb.termerror("%s is not a classifier. Please try again." % (k))
+ test_passed = False
+ elif k == "regressor":
+ if not base.is_regressor(v):
+ wandb.termerror("%s is not a regressor. Please try again." % (k))
+ test_passed = False
+ elif k == "clusterer":
+ if not (getattr(v, "_estimator_type", None) == "clusterer"):
+ wandb.termerror("%s is not a clusterer. Please try again." % (k))
+ test_passed = False
+ return test_passed
diff --git a/wandb/proto/__init__.py b/wandb/proto/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/proto/v3/__init__.py b/wandb/proto/v3/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/proto/v3/wandb_base_pb2.py b/wandb/proto/v3/wandb_base_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..826b30fe2f045b8511180f94efeceb6c2b15c151
--- /dev/null
+++ b/wandb/proto/v3/wandb_base_pb2.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: wandb/proto/wandb_base.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cwandb/proto/wandb_base.proto\x12\x0ewandb_internal\"6\n\x0b_RecordInfo\x12\x11\n\tstream_id\x18\x01 \x01(\t\x12\x14\n\x0c_tracelog_id\x18\x64 \x01(\t\"!\n\x0c_RequestInfo\x12\x11\n\tstream_id\x18\x01 \x01(\t\"#\n\x0b_ResultInfo\x12\x14\n\x0c_tracelog_id\x18\x64 \x01(\tb\x06proto3')
+
+
+
+__RECORDINFO = DESCRIPTOR.message_types_by_name['_RecordInfo']
+__REQUESTINFO = DESCRIPTOR.message_types_by_name['_RequestInfo']
+__RESULTINFO = DESCRIPTOR.message_types_by_name['_ResultInfo']
+_RecordInfo = _reflection.GeneratedProtocolMessageType('_RecordInfo', (_message.Message,), {
+ 'DESCRIPTOR' : __RECORDINFO,
+ '__module__' : 'wandb.proto.wandb_base_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal._RecordInfo)
+ })
+_sym_db.RegisterMessage(_RecordInfo)
+
+_RequestInfo = _reflection.GeneratedProtocolMessageType('_RequestInfo', (_message.Message,), {
+ 'DESCRIPTOR' : __REQUESTINFO,
+ '__module__' : 'wandb.proto.wandb_base_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal._RequestInfo)
+ })
+_sym_db.RegisterMessage(_RequestInfo)
+
+_ResultInfo = _reflection.GeneratedProtocolMessageType('_ResultInfo', (_message.Message,), {
+ 'DESCRIPTOR' : __RESULTINFO,
+ '__module__' : 'wandb.proto.wandb_base_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal._ResultInfo)
+ })
+_sym_db.RegisterMessage(_ResultInfo)
+
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+ DESCRIPTOR._options = None
+ __RECORDINFO._serialized_start=48
+ __RECORDINFO._serialized_end=102
+ __REQUESTINFO._serialized_start=104
+ __REQUESTINFO._serialized_end=137
+ __RESULTINFO._serialized_start=139
+ __RESULTINFO._serialized_end=174
+# @@protoc_insertion_point(module_scope)
diff --git a/wandb/proto/v3/wandb_base_pb2.pyi b/wandb/proto/v3/wandb_base_pb2.pyi
new file mode 100644
index 0000000000000000000000000000000000000000..503eeb5de83bd981a45433d00850d93367aa4ab0
--- /dev/null
+++ b/wandb/proto/v3/wandb_base_pb2.pyi
@@ -0,0 +1,68 @@
+"""
+@generated by mypy-protobuf. Do not edit manually!
+isort:skip_file
+"""
+import builtins
+import google.protobuf.descriptor
+import google.protobuf.message
+import sys
+
+if sys.version_info >= (3, 8):
+ import typing as typing_extensions
+else:
+ import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+class _RecordInfo(google.protobuf.message.Message):
+ """
+ _RecordInfo, _RequestInfo: extra info for all records and requests
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ STREAM_ID_FIELD_NUMBER: builtins.int
+ _TRACELOG_ID_FIELD_NUMBER: builtins.int
+ stream_id: builtins.str
+ _tracelog_id: builtins.str
+ def __init__(
+ self,
+ *,
+ stream_id: builtins.str = ...,
+ _tracelog_id: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_tracelog_id", b"_tracelog_id", "stream_id", b"stream_id"]) -> None: ...
+
+global____RecordInfo = _RecordInfo
+
+class _RequestInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ STREAM_ID_FIELD_NUMBER: builtins.int
+ stream_id: builtins.str
+ def __init__(
+ self,
+ *,
+ stream_id: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["stream_id", b"stream_id"]) -> None: ...
+
+global____RequestInfo = _RequestInfo
+
+class _ResultInfo(google.protobuf.message.Message):
+ """
+ _ResultInfo: extra info for all results
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _TRACELOG_ID_FIELD_NUMBER: builtins.int
+ _tracelog_id: builtins.str
+ def __init__(
+ self,
+ *,
+ _tracelog_id: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_tracelog_id", b"_tracelog_id"]) -> None: ...
+
+global____ResultInfo = _ResultInfo
diff --git a/wandb/proto/v3/wandb_internal_pb2.py b/wandb/proto/v3/wandb_internal_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..483eccb78db53fd585ca1b2785cdbd6f978fb9b8
--- /dev/null
+++ b/wandb/proto/v3/wandb_internal_pb2.py
@@ -0,0 +1,1534 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: wandb/proto/wandb_internal.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
+from wandb.proto import wandb_base_pb2 as wandb_dot_proto_dot_wandb__base__pb2
+from wandb.proto import wandb_telemetry_pb2 as wandb_dot_proto_dot_wandb__telemetry__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n wandb/proto/wandb_internal.proto\x12\x0ewandb_internal\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1cwandb/proto/wandb_base.proto\x1a!wandb/proto/wandb_telemetry.proto\"\x9c\t\n\x06Record\x12\x0b\n\x03num\x18\x01 \x01(\x03\x12\x30\n\x07history\x18\x02 \x01(\x0b\x32\x1d.wandb_internal.HistoryRecordH\x00\x12\x30\n\x07summary\x18\x03 \x01(\x0b\x32\x1d.wandb_internal.SummaryRecordH\x00\x12.\n\x06output\x18\x04 \x01(\x0b\x32\x1c.wandb_internal.OutputRecordH\x00\x12.\n\x06\x63onfig\x18\x05 \x01(\x0b\x32\x1c.wandb_internal.ConfigRecordH\x00\x12,\n\x05\x66iles\x18\x06 \x01(\x0b\x32\x1b.wandb_internal.FilesRecordH\x00\x12,\n\x05stats\x18\x07 \x01(\x0b\x32\x1b.wandb_internal.StatsRecordH\x00\x12\x32\n\x08\x61rtifact\x18\x08 \x01(\x0b\x32\x1e.wandb_internal.ArtifactRecordH\x00\x12,\n\x08tbrecord\x18\t \x01(\x0b\x32\x18.wandb_internal.TBRecordH\x00\x12,\n\x05\x61lert\x18\n \x01(\x0b\x32\x1b.wandb_internal.AlertRecordH\x00\x12\x34\n\ttelemetry\x18\x0b \x01(\x0b\x32\x1f.wandb_internal.TelemetryRecordH\x00\x12.\n\x06metric\x18\x0c \x01(\x0b\x32\x1c.wandb_internal.MetricRecordH\x00\x12\x35\n\noutput_raw\x18\r \x01(\x0b\x32\x1f.wandb_internal.OutputRawRecordH\x00\x12(\n\x03run\x18\x11 \x01(\x0b\x32\x19.wandb_internal.RunRecordH\x00\x12-\n\x04\x65xit\x18\x12 \x01(\x0b\x32\x1d.wandb_internal.RunExitRecordH\x00\x12,\n\x05\x66inal\x18\x14 \x01(\x0b\x32\x1b.wandb_internal.FinalRecordH\x00\x12.\n\x06header\x18\x15 \x01(\x0b\x32\x1c.wandb_internal.HeaderRecordH\x00\x12.\n\x06\x66ooter\x18\x16 \x01(\x0b\x32\x1c.wandb_internal.FooterRecordH\x00\x12\x39\n\npreempting\x18\x17 \x01(\x0b\x32#.wandb_internal.RunPreemptingRecordH\x00\x12;\n\rlink_artifact\x18\x18 \x01(\x0b\x32\".wandb_internal.LinkArtifactRecordH\x00\x12\x39\n\x0cuse_artifact\x18\x19 \x01(\x0b\x32!.wandb_internal.UseArtifactRecordH\x00\x12*\n\x07request\x18\x64 \x01(\x0b\x32\x17.wandb_internal.RequestH\x00\x12(\n\x07\x63ontrol\x18\x10 \x01(\x0b\x32\x17.wandb_internal.Control\x12\x0c\n\x04uuid\x18\x13 \x01(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfoB\r\n\x0brecord_type\"\xa8\x01\n\x07\x43ontrol\x12\x10\n\x08req_resp\x18\x01 \x01(\x08\x12\r\n\x05local\x18\x02 \x01(\x08\x12\x10\n\x08relay_id\x18\x03 \x01(\t\x12\x14\n\x0cmailbox_slot\x18\x04 \x01(\t\x12\x13\n\x0b\x61lways_send\x18\x05 \x01(\x08\x12\x14\n\x0c\x66low_control\x18\x06 \x01(\x08\x12\x12\n\nend_offset\x18\x07 \x01(\x03\x12\x15\n\rconnection_id\x18\x08 \x01(\t\"\xf3\x03\n\x06Result\x12\x35\n\nrun_result\x18\x11 \x01(\x0b\x32\x1f.wandb_internal.RunUpdateResultH\x00\x12\x34\n\x0b\x65xit_result\x18\x12 \x01(\x0b\x32\x1d.wandb_internal.RunExitResultH\x00\x12\x33\n\nlog_result\x18\x14 \x01(\x0b\x32\x1d.wandb_internal.HistoryResultH\x00\x12\x37\n\x0esummary_result\x18\x15 \x01(\x0b\x32\x1d.wandb_internal.SummaryResultH\x00\x12\x35\n\routput_result\x18\x16 \x01(\x0b\x32\x1c.wandb_internal.OutputResultH\x00\x12\x35\n\rconfig_result\x18\x17 \x01(\x0b\x32\x1c.wandb_internal.ConfigResultH\x00\x12,\n\x08response\x18\x64 \x01(\x0b\x32\x18.wandb_internal.ResponseH\x00\x12(\n\x07\x63ontrol\x18\x10 \x01(\x0b\x32\x17.wandb_internal.Control\x12\x0c\n\x04uuid\x18\x18 \x01(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._ResultInfoB\r\n\x0bresult_type\":\n\x0b\x46inalRecord\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"b\n\x0bVersionInfo\x12\x10\n\x08producer\x18\x01 \x01(\t\x12\x14\n\x0cmin_consumer\x18\x02 \x01(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"n\n\x0cHeaderRecord\x12\x31\n\x0cversion_info\x18\x01 \x01(\x0b\x32\x1b.wandb_internal.VersionInfo\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\";\n\x0c\x46ooterRecord\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\xce\x04\n\tRunRecord\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x0e\n\x06\x65ntity\x18\x02 \x01(\t\x12\x0f\n\x07project\x18\x03 \x01(\t\x12,\n\x06\x63onfig\x18\x04 \x01(\x0b\x32\x1c.wandb_internal.ConfigRecord\x12.\n\x07summary\x18\x05 \x01(\x0b\x32\x1d.wandb_internal.SummaryRecord\x12\x11\n\trun_group\x18\x06 \x01(\t\x12\x10\n\x08job_type\x18\x07 \x01(\t\x12\x14\n\x0c\x64isplay_name\x18\x08 \x01(\t\x12\r\n\x05notes\x18\t \x01(\t\x12\x0c\n\x04tags\x18\n \x03(\t\x12\x30\n\x08settings\x18\x0b \x01(\x0b\x32\x1e.wandb_internal.SettingsRecord\x12\x10\n\x08sweep_id\x18\x0c \x01(\t\x12\x0c\n\x04host\x18\r \x01(\t\x12\x15\n\rstarting_step\x18\x0e \x01(\x03\x12\x12\n\nstorage_id\x18\x10 \x01(\t\x12.\n\nstart_time\x18\x11 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07resumed\x18\x12 \x01(\x08\x12\x32\n\ttelemetry\x18\x13 \x01(\x0b\x32\x1f.wandb_internal.TelemetryRecord\x12\x0f\n\x07runtime\x18\x14 \x01(\x05\x12*\n\x03git\x18\x15 \x01(\x0b\x32\x1d.wandb_internal.GitRepoRecord\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\";\n\rGitRepoRecord\x12\x1a\n\nremote_url\x18\x01 \x01(\tR\x06remote\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\t\"c\n\x0fRunUpdateResult\x12&\n\x03run\x18\x01 \x01(\x0b\x32\x19.wandb_internal.RunRecord\x12(\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x19.wandb_internal.ErrorInfo\"\xac\x01\n\tErrorInfo\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x31\n\x04\x63ode\x18\x02 \x01(\x0e\x32#.wandb_internal.ErrorInfo.ErrorCode\"[\n\tErrorCode\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x11\n\rCOMMUNICATION\x10\x01\x12\x12\n\x0e\x41UTHENTICATION\x10\x02\x12\t\n\x05USAGE\x10\x03\x12\x0f\n\x0bUNSUPPORTED\x10\x04\"`\n\rRunExitRecord\x12\x11\n\texit_code\x18\x01 \x01(\x05\x12\x0f\n\x07runtime\x18\x02 \x01(\x05\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x0f\n\rRunExitResult\"B\n\x13RunPreemptingRecord\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x15\n\x13RunPreemptingResult\"i\n\x0eSettingsRecord\x12*\n\x04item\x18\x01 \x03(\x0b\x32\x1c.wandb_internal.SettingsItem\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"/\n\x0cSettingsItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nvalue_json\x18\x10 \x01(\t\"\x1a\n\x0bHistoryStep\x12\x0b\n\x03num\x18\x01 \x01(\x03\"\x92\x01\n\rHistoryRecord\x12)\n\x04item\x18\x01 \x03(\x0b\x32\x1b.wandb_internal.HistoryItem\x12)\n\x04step\x18\x02 \x01(\x0b\x32\x1b.wandb_internal.HistoryStep\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"B\n\x0bHistoryItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nnested_key\x18\x02 \x03(\t\x12\x12\n\nvalue_json\x18\x10 \x01(\t\"\x0f\n\rHistoryResult\"\xdc\x01\n\x0cOutputRecord\x12<\n\x0boutput_type\x18\x01 \x01(\x0e\x32\'.wandb_internal.OutputRecord.OutputType\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04line\x18\x03 \x01(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"$\n\nOutputType\x12\n\n\x06STDERR\x10\x00\x12\n\n\x06STDOUT\x10\x01\"\x0e\n\x0cOutputResult\"\xe2\x01\n\x0fOutputRawRecord\x12?\n\x0boutput_type\x18\x01 \x01(\x0e\x32*.wandb_internal.OutputRawRecord.OutputType\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04line\x18\x03 \x01(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"$\n\nOutputType\x12\n\n\x06STDERR\x10\x00\x12\n\n\x06STDOUT\x10\x01\"\x11\n\x0fOutputRawResult\"\x98\x03\n\x0cMetricRecord\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\tglob_name\x18\x02 \x01(\t\x12\x13\n\x0bstep_metric\x18\x04 \x01(\t\x12\x19\n\x11step_metric_index\x18\x05 \x01(\x05\x12.\n\x07options\x18\x06 \x01(\x0b\x32\x1d.wandb_internal.MetricOptions\x12.\n\x07summary\x18\x07 \x01(\x0b\x32\x1d.wandb_internal.MetricSummary\x12\x35\n\x04goal\x18\x08 \x01(\x0e\x32\'.wandb_internal.MetricRecord.MetricGoal\x12/\n\x08_control\x18\t \x01(\x0b\x32\x1d.wandb_internal.MetricControl\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"B\n\nMetricGoal\x12\x0e\n\nGOAL_UNSET\x10\x00\x12\x11\n\rGOAL_MINIMIZE\x10\x01\x12\x11\n\rGOAL_MAXIMIZE\x10\x02\"\x0e\n\x0cMetricResult\"C\n\rMetricOptions\x12\x11\n\tstep_sync\x18\x01 \x01(\x08\x12\x0e\n\x06hidden\x18\x02 \x01(\x08\x12\x0f\n\x07\x64\x65\x66ined\x18\x03 \x01(\x08\"\"\n\rMetricControl\x12\x11\n\toverwrite\x18\x01 \x01(\x08\"o\n\rMetricSummary\x12\x0b\n\x03min\x18\x01 \x01(\x08\x12\x0b\n\x03max\x18\x02 \x01(\x08\x12\x0c\n\x04mean\x18\x03 \x01(\x08\x12\x0c\n\x04\x62\x65st\x18\x04 \x01(\x08\x12\x0c\n\x04last\x18\x05 \x01(\x08\x12\x0c\n\x04none\x18\x06 \x01(\x08\x12\x0c\n\x04\x63opy\x18\x07 \x01(\x08\"\x93\x01\n\x0c\x43onfigRecord\x12*\n\x06update\x18\x01 \x03(\x0b\x32\x1a.wandb_internal.ConfigItem\x12*\n\x06remove\x18\x02 \x03(\x0b\x32\x1a.wandb_internal.ConfigItem\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"A\n\nConfigItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nnested_key\x18\x02 \x03(\t\x12\x12\n\nvalue_json\x18\x10 \x01(\t\"\x0e\n\x0c\x43onfigResult\"\x96\x01\n\rSummaryRecord\x12+\n\x06update\x18\x01 \x03(\x0b\x32\x1b.wandb_internal.SummaryItem\x12+\n\x06remove\x18\x02 \x03(\x0b\x32\x1b.wandb_internal.SummaryItem\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"B\n\x0bSummaryItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nnested_key\x18\x02 \x03(\t\x12\x12\n\nvalue_json\x18\x10 \x01(\t\"\x0f\n\rSummaryResult\"d\n\x0b\x46ilesRecord\x12(\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x19.wandb_internal.FilesItem\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\xfd\x01\n\tFilesItem\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x34\n\x06policy\x18\x02 \x01(\x0e\x32$.wandb_internal.FilesItem.PolicyType\x12\x30\n\x04type\x18\x03 \x01(\x0e\x32\".wandb_internal.FilesItem.FileType\x12\x15\n\rexternal_path\x18\x10 \x01(\t\"(\n\nPolicyType\x12\x07\n\x03NOW\x10\x00\x12\x07\n\x03\x45ND\x10\x01\x12\x08\n\x04LIVE\x10\x02\"9\n\x08\x46ileType\x12\t\n\x05OTHER\x10\x00\x12\t\n\x05WANDB\x10\x01\x12\t\n\x05MEDIA\x10\x02\x12\x0c\n\x08\x41RTIFACT\x10\x03\"\r\n\x0b\x46ilesResult\"\xe6\x01\n\x0bStatsRecord\x12\x39\n\nstats_type\x18\x01 \x01(\x0e\x32%.wandb_internal.StatsRecord.StatsType\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\'\n\x04item\x18\x03 \x03(\x0b\x32\x19.wandb_internal.StatsItem\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x17\n\tStatsType\x12\n\n\x06SYSTEM\x10\x00\",\n\tStatsItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nvalue_json\x18\x10 \x01(\t\"\xd9\x03\n\x0e\x41rtifactRecord\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x65ntity\x18\x03 \x01(\t\x12\x0c\n\x04type\x18\x04 \x01(\t\x12\x0c\n\x04name\x18\x05 \x01(\t\x12\x0e\n\x06\x64igest\x18\x06 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x07 \x01(\t\x12\x10\n\x08metadata\x18\x08 \x01(\t\x12\x14\n\x0cuser_created\x18\t \x01(\x08\x12\x18\n\x10use_after_commit\x18\n \x01(\x08\x12\x0f\n\x07\x61liases\x18\x0b \x03(\t\x12\x32\n\x08manifest\x18\x0c \x01(\x0b\x32 .wandb_internal.ArtifactManifest\x12\x16\n\x0e\x64istributed_id\x18\r \x01(\t\x12\x10\n\x08\x66inalize\x18\x0e \x01(\x08\x12\x11\n\tclient_id\x18\x0f \x01(\t\x12\x1a\n\x12sequence_client_id\x18\x10 \x01(\t\x12\x0f\n\x07\x62\x61se_id\x18\x11 \x01(\t\x12\x1c\n\x14ttl_duration_seconds\x18\x12 \x01(\x03\x12\x19\n\x11incremental_beta1\x18\x64 \x01(\x08\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\xbc\x01\n\x10\x41rtifactManifest\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x16\n\x0estorage_policy\x18\x02 \x01(\t\x12\x46\n\x15storage_policy_config\x18\x03 \x03(\x0b\x32\'.wandb_internal.StoragePolicyConfigItem\x12\x37\n\x08\x63ontents\x18\x04 \x03(\x0b\x32%.wandb_internal.ArtifactManifestEntry\"\xbb\x01\n\x15\x41rtifactManifestEntry\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0e\n\x06\x64igest\x18\x02 \x01(\t\x12\x0b\n\x03ref\x18\x03 \x01(\t\x12\x0c\n\x04size\x18\x04 \x01(\x03\x12\x10\n\x08mimetype\x18\x05 \x01(\t\x12\x12\n\nlocal_path\x18\x06 \x01(\t\x12\x19\n\x11\x62irth_artifact_id\x18\x07 \x01(\t\x12(\n\x05\x65xtra\x18\x10 \x03(\x0b\x32\x19.wandb_internal.ExtraItem\",\n\tExtraItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nvalue_json\x18\x02 \x01(\t\":\n\x17StoragePolicyConfigItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nvalue_json\x18\x02 \x01(\t\"\x10\n\x0e\x41rtifactResult\"\x14\n\x12LinkArtifactResult\"\xcf\x01\n\x12LinkArtifactRecord\x12\x11\n\tclient_id\x18\x01 \x01(\t\x12\x11\n\tserver_id\x18\x02 \x01(\t\x12\x16\n\x0eportfolio_name\x18\x03 \x01(\t\x12\x18\n\x10portfolio_entity\x18\x04 \x01(\t\x12\x19\n\x11portfolio_project\x18\x05 \x01(\t\x12\x19\n\x11portfolio_aliases\x18\x06 \x03(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"h\n\x08TBRecord\x12\x0f\n\x07log_dir\x18\x01 \x01(\t\x12\x0c\n\x04save\x18\x02 \x01(\x08\x12\x10\n\x08root_dir\x18\x03 \x01(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\n\n\x08TBResult\"}\n\x0b\x41lertRecord\x12\r\n\x05title\x18\x01 \x01(\t\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\r\n\x05level\x18\x03 \x01(\t\x12\x15\n\rwait_duration\x18\x04 \x01(\x03\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\r\n\x0b\x41lertResult\"\xc9\x0f\n\x07Request\x12\x38\n\x0bstop_status\x18\x01 \x01(\x0b\x32!.wandb_internal.StopStatusRequestH\x00\x12>\n\x0enetwork_status\x18\x02 \x01(\x0b\x32$.wandb_internal.NetworkStatusRequestH\x00\x12-\n\x05\x64\x65\x66\x65r\x18\x03 \x01(\x0b\x32\x1c.wandb_internal.DeferRequestH\x00\x12\x38\n\x0bget_summary\x18\x04 \x01(\x0b\x32!.wandb_internal.GetSummaryRequestH\x00\x12-\n\x05login\x18\x05 \x01(\x0b\x32\x1c.wandb_internal.LoginRequestH\x00\x12-\n\x05pause\x18\x06 \x01(\x0b\x32\x1c.wandb_internal.PauseRequestH\x00\x12/\n\x06resume\x18\x07 \x01(\x0b\x32\x1d.wandb_internal.ResumeRequestH\x00\x12\x34\n\tpoll_exit\x18\x08 \x01(\x0b\x32\x1f.wandb_internal.PollExitRequestH\x00\x12@\n\x0fsampled_history\x18\t \x01(\x0b\x32%.wandb_internal.SampledHistoryRequestH\x00\x12@\n\x0fpartial_history\x18\n \x01(\x0b\x32%.wandb_internal.PartialHistoryRequestH\x00\x12\x34\n\trun_start\x18\x0b \x01(\x0b\x32\x1f.wandb_internal.RunStartRequestH\x00\x12<\n\rcheck_version\x18\x0c \x01(\x0b\x32#.wandb_internal.CheckVersionRequestH\x00\x12:\n\x0clog_artifact\x18\r \x01(\x0b\x32\".wandb_internal.LogArtifactRequestH\x00\x12\x44\n\x11\x64ownload_artifact\x18\x0e \x01(\x0b\x32\'.wandb_internal.DownloadArtifactRequestH\x00\x12\x35\n\tkeepalive\x18\x11 \x01(\x0b\x32 .wandb_internal.KeepaliveRequestH\x00\x12\x36\n\nrun_status\x18\x14 \x01(\x0b\x32 .wandb_internal.RunStatusRequestH\x00\x12/\n\x06\x63\x61ncel\x18\x15 \x01(\x0b\x32\x1d.wandb_internal.CancelRequestH\x00\x12\x33\n\x08metadata\x18\x16 \x01(\x0b\x32\x1f.wandb_internal.MetadataRequestH\x00\x12\x44\n\x11internal_messages\x18\x17 \x01(\x0b\x32\'.wandb_internal.InternalMessagesRequestH\x00\x12@\n\x0fpython_packages\x18\x18 \x01(\x0b\x32%.wandb_internal.PythonPackagesRequestH\x00\x12\x33\n\x08shutdown\x18@ \x01(\x0b\x32\x1f.wandb_internal.ShutdownRequestH\x00\x12/\n\x06\x61ttach\x18\x41 \x01(\x0b\x32\x1d.wandb_internal.AttachRequestH\x00\x12/\n\x06status\x18\x42 \x01(\x0b\x32\x1d.wandb_internal.StatusRequestH\x00\x12\x38\n\x0bserver_info\x18\x43 \x01(\x0b\x32!.wandb_internal.ServerInfoRequestH\x00\x12\x38\n\x0bsender_mark\x18\x44 \x01(\x0b\x32!.wandb_internal.SenderMarkRequestH\x00\x12\x38\n\x0bsender_read\x18\x45 \x01(\x0b\x32!.wandb_internal.SenderReadRequestH\x00\x12<\n\rstatus_report\x18\x46 \x01(\x0b\x32#.wandb_internal.StatusReportRequestH\x00\x12>\n\x0esummary_record\x18G \x01(\x0b\x32$.wandb_internal.SummaryRecordRequestH\x00\x12\x42\n\x10telemetry_record\x18H \x01(\x0b\x32&.wandb_internal.TelemetryRecordRequestH\x00\x12\x32\n\x08job_info\x18I \x01(\x0b\x32\x1e.wandb_internal.JobInfoRequestH\x00\x12\x45\n\x12get_system_metrics\x18J \x01(\x0b\x32\'.wandb_internal.GetSystemMetricsRequestH\x00\x12\x45\n\x12\x66ile_transfer_info\x18K \x01(\x0b\x32\'.wandb_internal.FileTransferInfoRequestH\x00\x12+\n\x04sync\x18L \x01(\x0b\x32\x1b.wandb_internal.SyncRequestH\x00\x12\x39\n\x0btest_inject\x18\xe8\x07 \x01(\x0b\x32!.wandb_internal.TestInjectRequestH\x00\x42\x0e\n\x0crequest_type\"\xe2\x0b\n\x08Response\x12?\n\x12keepalive_response\x18\x12 \x01(\x0b\x32!.wandb_internal.KeepaliveResponseH\x00\x12\x42\n\x14stop_status_response\x18\x13 \x01(\x0b\x32\".wandb_internal.StopStatusResponseH\x00\x12H\n\x17network_status_response\x18\x14 \x01(\x0b\x32%.wandb_internal.NetworkStatusResponseH\x00\x12\x37\n\x0elogin_response\x18\x18 \x01(\x0b\x32\x1d.wandb_internal.LoginResponseH\x00\x12\x42\n\x14get_summary_response\x18\x19 \x01(\x0b\x32\".wandb_internal.GetSummaryResponseH\x00\x12>\n\x12poll_exit_response\x18\x1a \x01(\x0b\x32 .wandb_internal.PollExitResponseH\x00\x12J\n\x18sampled_history_response\x18\x1b \x01(\x0b\x32&.wandb_internal.SampledHistoryResponseH\x00\x12>\n\x12run_start_response\x18\x1c \x01(\x0b\x32 .wandb_internal.RunStartResponseH\x00\x12\x46\n\x16\x63heck_version_response\x18\x1d \x01(\x0b\x32$.wandb_internal.CheckVersionResponseH\x00\x12\x44\n\x15log_artifact_response\x18\x1e \x01(\x0b\x32#.wandb_internal.LogArtifactResponseH\x00\x12N\n\x1a\x64ownload_artifact_response\x18\x1f \x01(\x0b\x32(.wandb_internal.DownloadArtifactResponseH\x00\x12@\n\x13run_status_response\x18# \x01(\x0b\x32!.wandb_internal.RunStatusResponseH\x00\x12\x39\n\x0f\x63\x61ncel_response\x18$ \x01(\x0b\x32\x1e.wandb_internal.CancelResponseH\x00\x12N\n\x1ainternal_messages_response\x18% \x01(\x0b\x32(.wandb_internal.InternalMessagesResponseH\x00\x12=\n\x11shutdown_response\x18@ \x01(\x0b\x32 .wandb_internal.ShutdownResponseH\x00\x12\x39\n\x0f\x61ttach_response\x18\x41 \x01(\x0b\x32\x1e.wandb_internal.AttachResponseH\x00\x12\x39\n\x0fstatus_response\x18\x42 \x01(\x0b\x32\x1e.wandb_internal.StatusResponseH\x00\x12\x42\n\x14server_info_response\x18\x43 \x01(\x0b\x32\".wandb_internal.ServerInfoResponseH\x00\x12<\n\x11job_info_response\x18\x44 \x01(\x0b\x32\x1f.wandb_internal.JobInfoResponseH\x00\x12O\n\x1bget_system_metrics_response\x18\x45 \x01(\x0b\x32(.wandb_internal.GetSystemMetricsResponseH\x00\x12\x35\n\rsync_response\x18\x46 \x01(\x0b\x32\x1c.wandb_internal.SyncResponseH\x00\x12\x43\n\x14test_inject_response\x18\xe8\x07 \x01(\x0b\x32\".wandb_internal.TestInjectResponseH\x00\x42\x0f\n\rresponse_type\"\xc0\x02\n\x0c\x44\x65\x66\x65rRequest\x12\x36\n\x05state\x18\x01 \x01(\x0e\x32\'.wandb_internal.DeferRequest.DeferState\"\xf7\x01\n\nDeferState\x12\t\n\x05\x42\x45GIN\x10\x00\x12\r\n\tFLUSH_RUN\x10\x01\x12\x0f\n\x0b\x46LUSH_STATS\x10\x02\x12\x19\n\x15\x46LUSH_PARTIAL_HISTORY\x10\x03\x12\x0c\n\x08\x46LUSH_TB\x10\x04\x12\r\n\tFLUSH_SUM\x10\x05\x12\x13\n\x0f\x46LUSH_DEBOUNCER\x10\x06\x12\x10\n\x0c\x46LUSH_OUTPUT\x10\x07\x12\r\n\tFLUSH_JOB\x10\x08\x12\r\n\tFLUSH_DIR\x10\t\x12\x0c\n\x08\x46LUSH_FP\x10\n\x12\x0b\n\x07JOIN_FP\x10\x0b\x12\x0c\n\x08\x46LUSH_FS\x10\x0c\x12\x0f\n\x0b\x46LUSH_FINAL\x10\r\x12\x07\n\x03\x45ND\x10\x0e\"<\n\x0cPauseRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x0f\n\rPauseResponse\"=\n\rResumeRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x10\n\x0eResumeResponse\"M\n\x0cLoginRequest\x12\x0f\n\x07\x61pi_key\x18\x01 \x01(\t\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"&\n\rLoginResponse\x12\x15\n\ractive_entity\x18\x01 \x01(\t\"A\n\x11GetSummaryRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"?\n\x12GetSummaryResponse\x12)\n\x04item\x18\x01 \x03(\x0b\x32\x1b.wandb_internal.SummaryItem\"G\n\x17GetSystemMetricsRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"R\n\x12SystemMetricSample\x12-\n\ttimestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\r\n\x05value\x18\x02 \x01(\x02\"I\n\x13SystemMetricsBuffer\x12\x32\n\x06record\x18\x01 \x03(\x0b\x32\".wandb_internal.SystemMetricSample\"\xca\x01\n\x18GetSystemMetricsResponse\x12S\n\x0esystem_metrics\x18\x01 \x03(\x0b\x32;.wandb_internal.GetSystemMetricsResponse.SystemMetricsEntry\x1aY\n\x12SystemMetricsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x32\n\x05value\x18\x02 \x01(\x0b\x32#.wandb_internal.SystemMetricsBuffer:\x02\x38\x01\"=\n\rStatusRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\")\n\x0eStatusResponse\x12\x17\n\x0frun_should_stop\x18\x01 \x01(\x08\"A\n\x11StopStatusRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"-\n\x12StopStatusResponse\x12\x17\n\x0frun_should_stop\x18\x01 \x01(\x08\"D\n\x14NetworkStatusRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"P\n\x15NetworkStatusResponse\x12\x37\n\x11network_responses\x18\x01 \x03(\x0b\x32\x1c.wandb_internal.HttpResponse\"D\n\x0cHttpResponse\x12\x18\n\x10http_status_code\x18\x01 \x01(\x05\x12\x1a\n\x12http_response_text\x18\x02 \x01(\t\"G\n\x17InternalMessagesRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"N\n\x18InternalMessagesResponse\x12\x32\n\x08messages\x18\x01 \x01(\x0b\x32 .wandb_internal.InternalMessages\"#\n\x10InternalMessages\x12\x0f\n\x07warning\x18\x01 \x03(\t\"?\n\x0fPollExitRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\xbc\x01\n\x10PollExitResponse\x12\x0c\n\x04\x64one\x18\x01 \x01(\x08\x12\x32\n\x0b\x65xit_result\x18\x02 \x01(\x0b\x32\x1d.wandb_internal.RunExitResult\x12\x35\n\x0cpusher_stats\x18\x03 \x01(\x0b\x32\x1f.wandb_internal.FilePusherStats\x12/\n\x0b\x66ile_counts\x18\x04 \x01(\x0b\x32\x1a.wandb_internal.FileCounts\"@\n\rSyncOverwrite\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x0e\n\x06\x65ntity\x18\x02 \x01(\t\x12\x0f\n\x07project\x18\x03 \x01(\t\"\x1e\n\x08SyncSkip\x12\x12\n\noutput_raw\x18\x01 \x01(\x08\"\x13\n\x11SenderMarkRequest\"\x93\x01\n\x0bSyncRequest\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03\x12\x14\n\x0c\x66inal_offset\x18\x02 \x01(\x03\x12\x30\n\toverwrite\x18\x03 \x01(\x0b\x32\x1d.wandb_internal.SyncOverwrite\x12&\n\x04skip\x18\x04 \x01(\x0b\x32\x18.wandb_internal.SyncSkip\"E\n\x0cSyncResponse\x12\x0b\n\x03url\x18\x01 \x01(\t\x12(\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x19.wandb_internal.ErrorInfo\"?\n\x11SenderReadRequest\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03\x12\x14\n\x0c\x66inal_offset\x18\x02 \x01(\x03\"m\n\x13StatusReportRequest\x12\x12\n\nrecord_num\x18\x01 \x01(\x03\x12\x13\n\x0bsent_offset\x18\x02 \x01(\x03\x12-\n\tsync_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"F\n\x14SummaryRecordRequest\x12.\n\x07summary\x18\x01 \x01(\x0b\x32\x1d.wandb_internal.SummaryRecord\"L\n\x16TelemetryRecordRequest\x12\x32\n\ttelemetry\x18\x01 \x01(\x0b\x32\x1f.wandb_internal.TelemetryRecord\"A\n\x11ServerInfoRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"|\n\x12ServerInfoResponse\x12-\n\nlocal_info\x18\x01 \x01(\x0b\x32\x19.wandb_internal.LocalInfo\x12\x37\n\x0fserver_messages\x18\x02 \x01(\x0b\x32\x1e.wandb_internal.ServerMessages\"=\n\x0eServerMessages\x12+\n\x04item\x18\x01 \x03(\x0b\x32\x1d.wandb_internal.ServerMessage\"e\n\rServerMessage\x12\x12\n\nplain_text\x18\x01 \x01(\t\x12\x10\n\x08utf_text\x18\x02 \x01(\t\x12\x11\n\thtml_text\x18\x03 \x01(\t\x12\x0c\n\x04type\x18\x04 \x01(\t\x12\r\n\x05level\x18\x05 \x01(\x05\"c\n\nFileCounts\x12\x13\n\x0bwandb_count\x18\x01 \x01(\x05\x12\x13\n\x0bmedia_count\x18\x02 \x01(\x05\x12\x16\n\x0e\x61rtifact_count\x18\x03 \x01(\x05\x12\x13\n\x0bother_count\x18\x04 \x01(\x05\"U\n\x0f\x46ilePusherStats\x12\x16\n\x0euploaded_bytes\x18\x01 \x01(\x03\x12\x13\n\x0btotal_bytes\x18\x02 \x01(\x03\x12\x15\n\rdeduped_bytes\x18\x03 \x01(\x03\"\x1e\n\rFilesUploaded\x12\r\n\x05\x66iles\x18\x01 \x03(\t\"\xf4\x01\n\x17\x46ileTransferInfoRequest\x12\x42\n\x04type\x18\x01 \x01(\x0e\x32\x34.wandb_internal.FileTransferInfoRequest.TransferType\x12\x0c\n\x04path\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12\x0c\n\x04size\x18\x04 \x01(\x03\x12\x11\n\tprocessed\x18\x05 \x01(\x03\x12/\n\x0b\x66ile_counts\x18\x06 \x01(\x0b\x32\x1a.wandb_internal.FileCounts\"(\n\x0cTransferType\x12\n\n\x06Upload\x10\x00\x12\x0c\n\x08\x44ownload\x10\x01\"1\n\tLocalInfo\x12\x0f\n\x07version\x18\x01 \x01(\t\x12\x13\n\x0bout_of_date\x18\x02 \x01(\x08\"?\n\x0fShutdownRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x12\n\x10ShutdownResponse\"P\n\rAttachRequest\x12\x11\n\tattach_id\x18\x14 \x01(\t\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"b\n\x0e\x41ttachResponse\x12&\n\x03run\x18\x01 \x01(\x0b\x32\x19.wandb_internal.RunRecord\x12(\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x19.wandb_internal.ErrorInfo\"\xd5\x02\n\x11TestInjectRequest\x12\x13\n\x0bhandler_exc\x18\x01 \x01(\x08\x12\x14\n\x0chandler_exit\x18\x02 \x01(\x08\x12\x15\n\rhandler_abort\x18\x03 \x01(\x08\x12\x12\n\nsender_exc\x18\x04 \x01(\x08\x12\x13\n\x0bsender_exit\x18\x05 \x01(\x08\x12\x14\n\x0csender_abort\x18\x06 \x01(\x08\x12\x0f\n\x07req_exc\x18\x07 \x01(\x08\x12\x10\n\x08req_exit\x18\x08 \x01(\x08\x12\x11\n\treq_abort\x18\t \x01(\x08\x12\x10\n\x08resp_exc\x18\n \x01(\x08\x12\x11\n\tresp_exit\x18\x0b \x01(\x08\x12\x12\n\nresp_abort\x18\x0c \x01(\x08\x12\x10\n\x08msg_drop\x18\r \x01(\x08\x12\x10\n\x08msg_hang\x18\x0e \x01(\x08\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x14\n\x12TestInjectResponse\"\x1e\n\rHistoryAction\x12\r\n\x05\x66lush\x18\x01 \x01(\x08\"\xca\x01\n\x15PartialHistoryRequest\x12)\n\x04item\x18\x01 \x03(\x0b\x32\x1b.wandb_internal.HistoryItem\x12)\n\x04step\x18\x02 \x01(\x0b\x32\x1b.wandb_internal.HistoryStep\x12-\n\x06\x61\x63tion\x18\x03 \x01(\x0b\x32\x1d.wandb_internal.HistoryAction\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x18\n\x16PartialHistoryResponse\"E\n\x15SampledHistoryRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"_\n\x12SampledHistoryItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nnested_key\x18\x02 \x03(\t\x12\x14\n\x0cvalues_float\x18\x03 \x03(\x02\x12\x12\n\nvalues_int\x18\x04 \x03(\x03\"J\n\x16SampledHistoryResponse\x12\x30\n\x04item\x18\x01 \x03(\x0b\x32\".wandb_internal.SampledHistoryItem\"@\n\x10RunStatusRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"x\n\x11RunStatusResponse\x12\x18\n\x10sync_items_total\x18\x01 \x01(\x03\x12\x1a\n\x12sync_items_pending\x18\x02 \x01(\x03\x12-\n\tsync_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"g\n\x0fRunStartRequest\x12&\n\x03run\x18\x01 \x01(\x0b\x32\x19.wandb_internal.RunRecord\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x12\n\x10RunStartResponse\"\\\n\x13\x43heckVersionRequest\x12\x17\n\x0f\x63urrent_version\x18\x01 \x01(\t\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"]\n\x14\x43heckVersionResponse\x12\x17\n\x0fupgrade_message\x18\x01 \x01(\t\x12\x14\n\x0cyank_message\x18\x02 \x01(\t\x12\x16\n\x0e\x64\x65lete_message\x18\x03 \x01(\t\">\n\x0eJobInfoRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"6\n\x0fJobInfoResponse\x12\x12\n\nsequenceId\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\"\x9f\x01\n\x12LogArtifactRequest\x12\x30\n\x08\x61rtifact\x18\x01 \x01(\x0b\x32\x1e.wandb_internal.ArtifactRecord\x12\x14\n\x0chistory_step\x18\x02 \x01(\x03\x12\x13\n\x0bstaging_dir\x18\x03 \x01(\t\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"A\n\x13LogArtifactResponse\x12\x13\n\x0b\x61rtifact_id\x18\x01 \x01(\t\x12\x15\n\rerror_message\x18\x02 \x01(\t\"\x95\x01\n\x17\x44ownloadArtifactRequest\x12\x13\n\x0b\x61rtifact_id\x18\x01 \x01(\t\x12\x15\n\rdownload_root\x18\x02 \x01(\t\x12 \n\x18\x61llow_missing_references\x18\x04 \x01(\x08\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"1\n\x18\x44ownloadArtifactResponse\x12\x15\n\rerror_message\x18\x01 \x01(\t\"@\n\x10KeepaliveRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x13\n\x11KeepaliveResponse\"F\n\x0c\x41rtifactInfo\x12\x10\n\x08\x61rtifact\x18\x01 \x01(\t\x12\x12\n\nentrypoint\x18\x02 \x03(\t\x12\x10\n\x08notebook\x18\x03 \x01(\x08\")\n\x07GitInfo\x12\x0e\n\x06remote\x18\x01 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\t\"\\\n\tGitSource\x12)\n\x08git_info\x18\x01 \x01(\x0b\x32\x17.wandb_internal.GitInfo\x12\x12\n\nentrypoint\x18\x02 \x03(\t\x12\x10\n\x08notebook\x18\x03 \x01(\x08\"\x1c\n\x0bImageSource\x12\r\n\x05image\x18\x01 \x01(\t\"\x8c\x01\n\x06Source\x12&\n\x03git\x18\x01 \x01(\x0b\x32\x19.wandb_internal.GitSource\x12.\n\x08\x61rtifact\x18\x02 \x01(\x0b\x32\x1c.wandb_internal.ArtifactInfo\x12*\n\x05image\x18\x03 \x01(\x0b\x32\x1b.wandb_internal.ImageSource\"k\n\tJobSource\x12\x10\n\x08_version\x18\x01 \x01(\t\x12\x13\n\x0bsource_type\x18\x02 \x01(\t\x12&\n\x06source\x18\x03 \x01(\x0b\x32\x16.wandb_internal.Source\x12\x0f\n\x07runtime\x18\x04 \x01(\t\"V\n\x12PartialJobArtifact\x12\x10\n\x08job_name\x18\x01 \x01(\t\x12.\n\x0bsource_info\x18\x02 \x01(\x0b\x32\x19.wandb_internal.JobSource\"\x9d\x01\n\x11UseArtifactRecord\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x33\n\x07partial\x18\x04 \x01(\x0b\x32\".wandb_internal.PartialJobArtifact\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x13\n\x11UseArtifactResult\"R\n\rCancelRequest\x12\x13\n\x0b\x63\x61ncel_slot\x18\x01 \x01(\t\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x10\n\x0e\x43\x61ncelResponse\"\'\n\x08\x44iskInfo\x12\r\n\x05total\x18\x01 \x01(\x04\x12\x0c\n\x04used\x18\x02 \x01(\x04\"\x1b\n\nMemoryInfo\x12\r\n\x05total\x18\x01 \x01(\x04\"/\n\x07\x43puInfo\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x15\n\rcount_logical\x18\x02 \x01(\r\">\n\x0cGpuAppleInfo\x12\x0f\n\x07gpuType\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\r\n\x05\x63ores\x18\x03 \x01(\r\"3\n\rGpuNvidiaInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x14\n\x0cmemory_total\x18\x02 \x01(\x04\"\x89\x02\n\nGpuAmdInfo\x12\n\n\x02id\x18\x01 \x01(\t\x12\x11\n\tunique_id\x18\x02 \x01(\t\x12\x15\n\rvbios_version\x18\x03 \x01(\t\x12\x19\n\x11performance_level\x18\x04 \x01(\t\x12\x15\n\rgpu_overdrive\x18\x05 \x01(\t\x12\x1c\n\x14gpu_memory_overdrive\x18\x06 \x01(\t\x12\x11\n\tmax_power\x18\x07 \x01(\t\x12\x0e\n\x06series\x18\x08 \x01(\t\x12\r\n\x05model\x18\t \x01(\t\x12\x0e\n\x06vendor\x18\n \x01(\t\x12\x0b\n\x03sku\x18\x0b \x01(\t\x12\x12\n\nsclk_range\x18\x0c \x01(\t\x12\x12\n\nmclk_range\x18\r \x01(\t\"\x96\x08\n\x0fMetadataRequest\x12\n\n\x02os\x18\x01 \x01(\t\x12\x0e\n\x06python\x18\x02 \x01(\t\x12/\n\x0bheartbeatAt\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12-\n\tstartedAt\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06\x64ocker\x18\x05 \x01(\t\x12\x0c\n\x04\x63uda\x18\x06 \x01(\t\x12\x0c\n\x04\x61rgs\x18\x07 \x03(\t\x12\r\n\x05state\x18\x08 \x01(\t\x12\x0f\n\x07program\x18\t \x01(\t\x12\x1b\n\tcode_path\x18\n \x01(\tR\x08\x63odePath\x12*\n\x03git\x18\x0b \x01(\x0b\x32\x1d.wandb_internal.GitRepoRecord\x12\r\n\x05\x65mail\x18\x0c \x01(\t\x12\x0c\n\x04root\x18\r \x01(\t\x12\x0c\n\x04host\x18\x0e \x01(\t\x12\x10\n\x08username\x18\x0f \x01(\t\x12\x12\n\nexecutable\x18\x10 \x01(\t\x12&\n\x0f\x63ode_path_local\x18\x11 \x01(\tR\rcodePathLocal\x12\r\n\x05\x63olab\x18\x12 \x01(\t\x12\x1c\n\tcpu_count\x18\x13 \x01(\rR\tcpu_count\x12,\n\x11\x63pu_count_logical\x18\x14 \x01(\rR\x11\x63pu_count_logical\x12\x15\n\x08gpu_type\x18\x15 \x01(\tR\x03gpu\x12\x1c\n\tgpu_count\x18\x16 \x01(\rR\tgpu_count\x12\x37\n\x04\x64isk\x18\x17 \x03(\x0b\x32).wandb_internal.MetadataRequest.DiskEntry\x12*\n\x06memory\x18\x18 \x01(\x0b\x32\x1a.wandb_internal.MemoryInfo\x12$\n\x03\x63pu\x18\x19 \x01(\x0b\x32\x17.wandb_internal.CpuInfo\x12\x39\n\tgpu_apple\x18\x1a \x01(\x0b\x32\x1c.wandb_internal.GpuAppleInfoR\x08gpuapple\x12=\n\ngpu_nvidia\x18\x1b \x03(\x0b\x32\x1d.wandb_internal.GpuNvidiaInfoR\ngpu_nvidia\x12\x34\n\x07gpu_amd\x18\x1c \x03(\x0b\x32\x1a.wandb_internal.GpuAmdInfoR\x07gpu_amd\x12\x39\n\x05slurm\x18\x1d \x03(\x0b\x32*.wandb_internal.MetadataRequest.SlurmEntry\x1a\x45\n\tDiskEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b\x32\x18.wandb_internal.DiskInfo:\x02\x38\x01\x1a,\n\nSlurmEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8d\x01\n\x15PythonPackagesRequest\x12\x44\n\x07package\x18\x01 \x03(\x0b\x32\x33.wandb_internal.PythonPackagesRequest.PythonPackage\x1a.\n\rPythonPackage\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\tb\x06proto3')
+
+
+
+_RECORD = DESCRIPTOR.message_types_by_name['Record']
+_CONTROL = DESCRIPTOR.message_types_by_name['Control']
+_RESULT = DESCRIPTOR.message_types_by_name['Result']
+_FINALRECORD = DESCRIPTOR.message_types_by_name['FinalRecord']
+_VERSIONINFO = DESCRIPTOR.message_types_by_name['VersionInfo']
+_HEADERRECORD = DESCRIPTOR.message_types_by_name['HeaderRecord']
+_FOOTERRECORD = DESCRIPTOR.message_types_by_name['FooterRecord']
+_RUNRECORD = DESCRIPTOR.message_types_by_name['RunRecord']
+_GITREPORECORD = DESCRIPTOR.message_types_by_name['GitRepoRecord']
+_RUNUPDATERESULT = DESCRIPTOR.message_types_by_name['RunUpdateResult']
+_ERRORINFO = DESCRIPTOR.message_types_by_name['ErrorInfo']
+_RUNEXITRECORD = DESCRIPTOR.message_types_by_name['RunExitRecord']
+_RUNEXITRESULT = DESCRIPTOR.message_types_by_name['RunExitResult']
+_RUNPREEMPTINGRECORD = DESCRIPTOR.message_types_by_name['RunPreemptingRecord']
+_RUNPREEMPTINGRESULT = DESCRIPTOR.message_types_by_name['RunPreemptingResult']
+_SETTINGSRECORD = DESCRIPTOR.message_types_by_name['SettingsRecord']
+_SETTINGSITEM = DESCRIPTOR.message_types_by_name['SettingsItem']
+_HISTORYSTEP = DESCRIPTOR.message_types_by_name['HistoryStep']
+_HISTORYRECORD = DESCRIPTOR.message_types_by_name['HistoryRecord']
+_HISTORYITEM = DESCRIPTOR.message_types_by_name['HistoryItem']
+_HISTORYRESULT = DESCRIPTOR.message_types_by_name['HistoryResult']
+_OUTPUTRECORD = DESCRIPTOR.message_types_by_name['OutputRecord']
+_OUTPUTRESULT = DESCRIPTOR.message_types_by_name['OutputResult']
+_OUTPUTRAWRECORD = DESCRIPTOR.message_types_by_name['OutputRawRecord']
+_OUTPUTRAWRESULT = DESCRIPTOR.message_types_by_name['OutputRawResult']
+_METRICRECORD = DESCRIPTOR.message_types_by_name['MetricRecord']
+_METRICRESULT = DESCRIPTOR.message_types_by_name['MetricResult']
+_METRICOPTIONS = DESCRIPTOR.message_types_by_name['MetricOptions']
+_METRICCONTROL = DESCRIPTOR.message_types_by_name['MetricControl']
+_METRICSUMMARY = DESCRIPTOR.message_types_by_name['MetricSummary']
+_CONFIGRECORD = DESCRIPTOR.message_types_by_name['ConfigRecord']
+_CONFIGITEM = DESCRIPTOR.message_types_by_name['ConfigItem']
+_CONFIGRESULT = DESCRIPTOR.message_types_by_name['ConfigResult']
+_SUMMARYRECORD = DESCRIPTOR.message_types_by_name['SummaryRecord']
+_SUMMARYITEM = DESCRIPTOR.message_types_by_name['SummaryItem']
+_SUMMARYRESULT = DESCRIPTOR.message_types_by_name['SummaryResult']
+_FILESRECORD = DESCRIPTOR.message_types_by_name['FilesRecord']
+_FILESITEM = DESCRIPTOR.message_types_by_name['FilesItem']
+_FILESRESULT = DESCRIPTOR.message_types_by_name['FilesResult']
+_STATSRECORD = DESCRIPTOR.message_types_by_name['StatsRecord']
+_STATSITEM = DESCRIPTOR.message_types_by_name['StatsItem']
+_ARTIFACTRECORD = DESCRIPTOR.message_types_by_name['ArtifactRecord']
+_ARTIFACTMANIFEST = DESCRIPTOR.message_types_by_name['ArtifactManifest']
+_ARTIFACTMANIFESTENTRY = DESCRIPTOR.message_types_by_name['ArtifactManifestEntry']
+_EXTRAITEM = DESCRIPTOR.message_types_by_name['ExtraItem']
+_STORAGEPOLICYCONFIGITEM = DESCRIPTOR.message_types_by_name['StoragePolicyConfigItem']
+_ARTIFACTRESULT = DESCRIPTOR.message_types_by_name['ArtifactResult']
+_LINKARTIFACTRESULT = DESCRIPTOR.message_types_by_name['LinkArtifactResult']
+_LINKARTIFACTRECORD = DESCRIPTOR.message_types_by_name['LinkArtifactRecord']
+_TBRECORD = DESCRIPTOR.message_types_by_name['TBRecord']
+_TBRESULT = DESCRIPTOR.message_types_by_name['TBResult']
+_ALERTRECORD = DESCRIPTOR.message_types_by_name['AlertRecord']
+_ALERTRESULT = DESCRIPTOR.message_types_by_name['AlertResult']
+_REQUEST = DESCRIPTOR.message_types_by_name['Request']
+_RESPONSE = DESCRIPTOR.message_types_by_name['Response']
+_DEFERREQUEST = DESCRIPTOR.message_types_by_name['DeferRequest']
+_PAUSEREQUEST = DESCRIPTOR.message_types_by_name['PauseRequest']
+_PAUSERESPONSE = DESCRIPTOR.message_types_by_name['PauseResponse']
+_RESUMEREQUEST = DESCRIPTOR.message_types_by_name['ResumeRequest']
+_RESUMERESPONSE = DESCRIPTOR.message_types_by_name['ResumeResponse']
+_LOGINREQUEST = DESCRIPTOR.message_types_by_name['LoginRequest']
+_LOGINRESPONSE = DESCRIPTOR.message_types_by_name['LoginResponse']
+_GETSUMMARYREQUEST = DESCRIPTOR.message_types_by_name['GetSummaryRequest']
+_GETSUMMARYRESPONSE = DESCRIPTOR.message_types_by_name['GetSummaryResponse']
+_GETSYSTEMMETRICSREQUEST = DESCRIPTOR.message_types_by_name['GetSystemMetricsRequest']
+_SYSTEMMETRICSAMPLE = DESCRIPTOR.message_types_by_name['SystemMetricSample']
+_SYSTEMMETRICSBUFFER = DESCRIPTOR.message_types_by_name['SystemMetricsBuffer']
+_GETSYSTEMMETRICSRESPONSE = DESCRIPTOR.message_types_by_name['GetSystemMetricsResponse']
+_GETSYSTEMMETRICSRESPONSE_SYSTEMMETRICSENTRY = _GETSYSTEMMETRICSRESPONSE.nested_types_by_name['SystemMetricsEntry']
+_STATUSREQUEST = DESCRIPTOR.message_types_by_name['StatusRequest']
+_STATUSRESPONSE = DESCRIPTOR.message_types_by_name['StatusResponse']
+_STOPSTATUSREQUEST = DESCRIPTOR.message_types_by_name['StopStatusRequest']
+_STOPSTATUSRESPONSE = DESCRIPTOR.message_types_by_name['StopStatusResponse']
+_NETWORKSTATUSREQUEST = DESCRIPTOR.message_types_by_name['NetworkStatusRequest']
+_NETWORKSTATUSRESPONSE = DESCRIPTOR.message_types_by_name['NetworkStatusResponse']
+_HTTPRESPONSE = DESCRIPTOR.message_types_by_name['HttpResponse']
+_INTERNALMESSAGESREQUEST = DESCRIPTOR.message_types_by_name['InternalMessagesRequest']
+_INTERNALMESSAGESRESPONSE = DESCRIPTOR.message_types_by_name['InternalMessagesResponse']
+_INTERNALMESSAGES = DESCRIPTOR.message_types_by_name['InternalMessages']
+_POLLEXITREQUEST = DESCRIPTOR.message_types_by_name['PollExitRequest']
+_POLLEXITRESPONSE = DESCRIPTOR.message_types_by_name['PollExitResponse']
+_SYNCOVERWRITE = DESCRIPTOR.message_types_by_name['SyncOverwrite']
+_SYNCSKIP = DESCRIPTOR.message_types_by_name['SyncSkip']
+_SENDERMARKREQUEST = DESCRIPTOR.message_types_by_name['SenderMarkRequest']
+_SYNCREQUEST = DESCRIPTOR.message_types_by_name['SyncRequest']
+_SYNCRESPONSE = DESCRIPTOR.message_types_by_name['SyncResponse']
+_SENDERREADREQUEST = DESCRIPTOR.message_types_by_name['SenderReadRequest']
+_STATUSREPORTREQUEST = DESCRIPTOR.message_types_by_name['StatusReportRequest']
+_SUMMARYRECORDREQUEST = DESCRIPTOR.message_types_by_name['SummaryRecordRequest']
+_TELEMETRYRECORDREQUEST = DESCRIPTOR.message_types_by_name['TelemetryRecordRequest']
+_SERVERINFOREQUEST = DESCRIPTOR.message_types_by_name['ServerInfoRequest']
+_SERVERINFORESPONSE = DESCRIPTOR.message_types_by_name['ServerInfoResponse']
+_SERVERMESSAGES = DESCRIPTOR.message_types_by_name['ServerMessages']
+_SERVERMESSAGE = DESCRIPTOR.message_types_by_name['ServerMessage']
+_FILECOUNTS = DESCRIPTOR.message_types_by_name['FileCounts']
+_FILEPUSHERSTATS = DESCRIPTOR.message_types_by_name['FilePusherStats']
+_FILESUPLOADED = DESCRIPTOR.message_types_by_name['FilesUploaded']
+_FILETRANSFERINFOREQUEST = DESCRIPTOR.message_types_by_name['FileTransferInfoRequest']
+_LOCALINFO = DESCRIPTOR.message_types_by_name['LocalInfo']
+_SHUTDOWNREQUEST = DESCRIPTOR.message_types_by_name['ShutdownRequest']
+_SHUTDOWNRESPONSE = DESCRIPTOR.message_types_by_name['ShutdownResponse']
+_ATTACHREQUEST = DESCRIPTOR.message_types_by_name['AttachRequest']
+_ATTACHRESPONSE = DESCRIPTOR.message_types_by_name['AttachResponse']
+_TESTINJECTREQUEST = DESCRIPTOR.message_types_by_name['TestInjectRequest']
+_TESTINJECTRESPONSE = DESCRIPTOR.message_types_by_name['TestInjectResponse']
+_HISTORYACTION = DESCRIPTOR.message_types_by_name['HistoryAction']
+_PARTIALHISTORYREQUEST = DESCRIPTOR.message_types_by_name['PartialHistoryRequest']
+_PARTIALHISTORYRESPONSE = DESCRIPTOR.message_types_by_name['PartialHistoryResponse']
+_SAMPLEDHISTORYREQUEST = DESCRIPTOR.message_types_by_name['SampledHistoryRequest']
+_SAMPLEDHISTORYITEM = DESCRIPTOR.message_types_by_name['SampledHistoryItem']
+_SAMPLEDHISTORYRESPONSE = DESCRIPTOR.message_types_by_name['SampledHistoryResponse']
+_RUNSTATUSREQUEST = DESCRIPTOR.message_types_by_name['RunStatusRequest']
+_RUNSTATUSRESPONSE = DESCRIPTOR.message_types_by_name['RunStatusResponse']
+_RUNSTARTREQUEST = DESCRIPTOR.message_types_by_name['RunStartRequest']
+_RUNSTARTRESPONSE = DESCRIPTOR.message_types_by_name['RunStartResponse']
+_CHECKVERSIONREQUEST = DESCRIPTOR.message_types_by_name['CheckVersionRequest']
+_CHECKVERSIONRESPONSE = DESCRIPTOR.message_types_by_name['CheckVersionResponse']
+_JOBINFOREQUEST = DESCRIPTOR.message_types_by_name['JobInfoRequest']
+_JOBINFORESPONSE = DESCRIPTOR.message_types_by_name['JobInfoResponse']
+_LOGARTIFACTREQUEST = DESCRIPTOR.message_types_by_name['LogArtifactRequest']
+_LOGARTIFACTRESPONSE = DESCRIPTOR.message_types_by_name['LogArtifactResponse']
+_DOWNLOADARTIFACTREQUEST = DESCRIPTOR.message_types_by_name['DownloadArtifactRequest']
+_DOWNLOADARTIFACTRESPONSE = DESCRIPTOR.message_types_by_name['DownloadArtifactResponse']
+_KEEPALIVEREQUEST = DESCRIPTOR.message_types_by_name['KeepaliveRequest']
+_KEEPALIVERESPONSE = DESCRIPTOR.message_types_by_name['KeepaliveResponse']
+_ARTIFACTINFO = DESCRIPTOR.message_types_by_name['ArtifactInfo']
+_GITINFO = DESCRIPTOR.message_types_by_name['GitInfo']
+_GITSOURCE = DESCRIPTOR.message_types_by_name['GitSource']
+_IMAGESOURCE = DESCRIPTOR.message_types_by_name['ImageSource']
+_SOURCE = DESCRIPTOR.message_types_by_name['Source']
+_JOBSOURCE = DESCRIPTOR.message_types_by_name['JobSource']
+_PARTIALJOBARTIFACT = DESCRIPTOR.message_types_by_name['PartialJobArtifact']
+_USEARTIFACTRECORD = DESCRIPTOR.message_types_by_name['UseArtifactRecord']
+_USEARTIFACTRESULT = DESCRIPTOR.message_types_by_name['UseArtifactResult']
+_CANCELREQUEST = DESCRIPTOR.message_types_by_name['CancelRequest']
+_CANCELRESPONSE = DESCRIPTOR.message_types_by_name['CancelResponse']
+_DISKINFO = DESCRIPTOR.message_types_by_name['DiskInfo']
+_MEMORYINFO = DESCRIPTOR.message_types_by_name['MemoryInfo']
+_CPUINFO = DESCRIPTOR.message_types_by_name['CpuInfo']
+_GPUAPPLEINFO = DESCRIPTOR.message_types_by_name['GpuAppleInfo']
+_GPUNVIDIAINFO = DESCRIPTOR.message_types_by_name['GpuNvidiaInfo']
+_GPUAMDINFO = DESCRIPTOR.message_types_by_name['GpuAmdInfo']
+_METADATAREQUEST = DESCRIPTOR.message_types_by_name['MetadataRequest']
+_METADATAREQUEST_DISKENTRY = _METADATAREQUEST.nested_types_by_name['DiskEntry']
+_METADATAREQUEST_SLURMENTRY = _METADATAREQUEST.nested_types_by_name['SlurmEntry']
+_PYTHONPACKAGESREQUEST = DESCRIPTOR.message_types_by_name['PythonPackagesRequest']
+_PYTHONPACKAGESREQUEST_PYTHONPACKAGE = _PYTHONPACKAGESREQUEST.nested_types_by_name['PythonPackage']
+_ERRORINFO_ERRORCODE = _ERRORINFO.enum_types_by_name['ErrorCode']
+_OUTPUTRECORD_OUTPUTTYPE = _OUTPUTRECORD.enum_types_by_name['OutputType']
+_OUTPUTRAWRECORD_OUTPUTTYPE = _OUTPUTRAWRECORD.enum_types_by_name['OutputType']
+_METRICRECORD_METRICGOAL = _METRICRECORD.enum_types_by_name['MetricGoal']
+_FILESITEM_POLICYTYPE = _FILESITEM.enum_types_by_name['PolicyType']
+_FILESITEM_FILETYPE = _FILESITEM.enum_types_by_name['FileType']
+_STATSRECORD_STATSTYPE = _STATSRECORD.enum_types_by_name['StatsType']
+_DEFERREQUEST_DEFERSTATE = _DEFERREQUEST.enum_types_by_name['DeferState']
+_FILETRANSFERINFOREQUEST_TRANSFERTYPE = _FILETRANSFERINFOREQUEST.enum_types_by_name['TransferType']
+Record = _reflection.GeneratedProtocolMessageType('Record', (_message.Message,), {
+ 'DESCRIPTOR' : _RECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.Record)
+ })
+_sym_db.RegisterMessage(Record)
+
+Control = _reflection.GeneratedProtocolMessageType('Control', (_message.Message,), {
+ 'DESCRIPTOR' : _CONTROL,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.Control)
+ })
+_sym_db.RegisterMessage(Control)
+
+Result = _reflection.GeneratedProtocolMessageType('Result', (_message.Message,), {
+ 'DESCRIPTOR' : _RESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.Result)
+ })
+_sym_db.RegisterMessage(Result)
+
+FinalRecord = _reflection.GeneratedProtocolMessageType('FinalRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _FINALRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.FinalRecord)
+ })
+_sym_db.RegisterMessage(FinalRecord)
+
+VersionInfo = _reflection.GeneratedProtocolMessageType('VersionInfo', (_message.Message,), {
+ 'DESCRIPTOR' : _VERSIONINFO,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.VersionInfo)
+ })
+_sym_db.RegisterMessage(VersionInfo)
+
+HeaderRecord = _reflection.GeneratedProtocolMessageType('HeaderRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _HEADERRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.HeaderRecord)
+ })
+_sym_db.RegisterMessage(HeaderRecord)
+
+FooterRecord = _reflection.GeneratedProtocolMessageType('FooterRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _FOOTERRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.FooterRecord)
+ })
+_sym_db.RegisterMessage(FooterRecord)
+
+RunRecord = _reflection.GeneratedProtocolMessageType('RunRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _RUNRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.RunRecord)
+ })
+_sym_db.RegisterMessage(RunRecord)
+
+GitRepoRecord = _reflection.GeneratedProtocolMessageType('GitRepoRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _GITREPORECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.GitRepoRecord)
+ })
+_sym_db.RegisterMessage(GitRepoRecord)
+
+RunUpdateResult = _reflection.GeneratedProtocolMessageType('RunUpdateResult', (_message.Message,), {
+ 'DESCRIPTOR' : _RUNUPDATERESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.RunUpdateResult)
+ })
+_sym_db.RegisterMessage(RunUpdateResult)
+
+ErrorInfo = _reflection.GeneratedProtocolMessageType('ErrorInfo', (_message.Message,), {
+ 'DESCRIPTOR' : _ERRORINFO,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ErrorInfo)
+ })
+_sym_db.RegisterMessage(ErrorInfo)
+
+RunExitRecord = _reflection.GeneratedProtocolMessageType('RunExitRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _RUNEXITRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.RunExitRecord)
+ })
+_sym_db.RegisterMessage(RunExitRecord)
+
+RunExitResult = _reflection.GeneratedProtocolMessageType('RunExitResult', (_message.Message,), {
+ 'DESCRIPTOR' : _RUNEXITRESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.RunExitResult)
+ })
+_sym_db.RegisterMessage(RunExitResult)
+
+RunPreemptingRecord = _reflection.GeneratedProtocolMessageType('RunPreemptingRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _RUNPREEMPTINGRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.RunPreemptingRecord)
+ })
+_sym_db.RegisterMessage(RunPreemptingRecord)
+
+RunPreemptingResult = _reflection.GeneratedProtocolMessageType('RunPreemptingResult', (_message.Message,), {
+ 'DESCRIPTOR' : _RUNPREEMPTINGRESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.RunPreemptingResult)
+ })
+_sym_db.RegisterMessage(RunPreemptingResult)
+
+SettingsRecord = _reflection.GeneratedProtocolMessageType('SettingsRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _SETTINGSRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SettingsRecord)
+ })
+_sym_db.RegisterMessage(SettingsRecord)
+
+SettingsItem = _reflection.GeneratedProtocolMessageType('SettingsItem', (_message.Message,), {
+ 'DESCRIPTOR' : _SETTINGSITEM,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SettingsItem)
+ })
+_sym_db.RegisterMessage(SettingsItem)
+
+HistoryStep = _reflection.GeneratedProtocolMessageType('HistoryStep', (_message.Message,), {
+ 'DESCRIPTOR' : _HISTORYSTEP,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.HistoryStep)
+ })
+_sym_db.RegisterMessage(HistoryStep)
+
+HistoryRecord = _reflection.GeneratedProtocolMessageType('HistoryRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _HISTORYRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.HistoryRecord)
+ })
+_sym_db.RegisterMessage(HistoryRecord)
+
+HistoryItem = _reflection.GeneratedProtocolMessageType('HistoryItem', (_message.Message,), {
+ 'DESCRIPTOR' : _HISTORYITEM,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.HistoryItem)
+ })
+_sym_db.RegisterMessage(HistoryItem)
+
+HistoryResult = _reflection.GeneratedProtocolMessageType('HistoryResult', (_message.Message,), {
+ 'DESCRIPTOR' : _HISTORYRESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.HistoryResult)
+ })
+_sym_db.RegisterMessage(HistoryResult)
+
+OutputRecord = _reflection.GeneratedProtocolMessageType('OutputRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _OUTPUTRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.OutputRecord)
+ })
+_sym_db.RegisterMessage(OutputRecord)
+
+OutputResult = _reflection.GeneratedProtocolMessageType('OutputResult', (_message.Message,), {
+ 'DESCRIPTOR' : _OUTPUTRESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.OutputResult)
+ })
+_sym_db.RegisterMessage(OutputResult)
+
+OutputRawRecord = _reflection.GeneratedProtocolMessageType('OutputRawRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _OUTPUTRAWRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.OutputRawRecord)
+ })
+_sym_db.RegisterMessage(OutputRawRecord)
+
+OutputRawResult = _reflection.GeneratedProtocolMessageType('OutputRawResult', (_message.Message,), {
+ 'DESCRIPTOR' : _OUTPUTRAWRESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.OutputRawResult)
+ })
+_sym_db.RegisterMessage(OutputRawResult)
+
+MetricRecord = _reflection.GeneratedProtocolMessageType('MetricRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _METRICRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.MetricRecord)
+ })
+_sym_db.RegisterMessage(MetricRecord)
+
+MetricResult = _reflection.GeneratedProtocolMessageType('MetricResult', (_message.Message,), {
+ 'DESCRIPTOR' : _METRICRESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.MetricResult)
+ })
+_sym_db.RegisterMessage(MetricResult)
+
+MetricOptions = _reflection.GeneratedProtocolMessageType('MetricOptions', (_message.Message,), {
+ 'DESCRIPTOR' : _METRICOPTIONS,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.MetricOptions)
+ })
+_sym_db.RegisterMessage(MetricOptions)
+
+MetricControl = _reflection.GeneratedProtocolMessageType('MetricControl', (_message.Message,), {
+ 'DESCRIPTOR' : _METRICCONTROL,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.MetricControl)
+ })
+_sym_db.RegisterMessage(MetricControl)
+
+MetricSummary = _reflection.GeneratedProtocolMessageType('MetricSummary', (_message.Message,), {
+ 'DESCRIPTOR' : _METRICSUMMARY,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.MetricSummary)
+ })
+_sym_db.RegisterMessage(MetricSummary)
+
+ConfigRecord = _reflection.GeneratedProtocolMessageType('ConfigRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _CONFIGRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ConfigRecord)
+ })
+_sym_db.RegisterMessage(ConfigRecord)
+
+ConfigItem = _reflection.GeneratedProtocolMessageType('ConfigItem', (_message.Message,), {
+ 'DESCRIPTOR' : _CONFIGITEM,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ConfigItem)
+ })
+_sym_db.RegisterMessage(ConfigItem)
+
+ConfigResult = _reflection.GeneratedProtocolMessageType('ConfigResult', (_message.Message,), {
+ 'DESCRIPTOR' : _CONFIGRESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ConfigResult)
+ })
+_sym_db.RegisterMessage(ConfigResult)
+
+SummaryRecord = _reflection.GeneratedProtocolMessageType('SummaryRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _SUMMARYRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SummaryRecord)
+ })
+_sym_db.RegisterMessage(SummaryRecord)
+
+SummaryItem = _reflection.GeneratedProtocolMessageType('SummaryItem', (_message.Message,), {
+ 'DESCRIPTOR' : _SUMMARYITEM,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SummaryItem)
+ })
+_sym_db.RegisterMessage(SummaryItem)
+
+SummaryResult = _reflection.GeneratedProtocolMessageType('SummaryResult', (_message.Message,), {
+ 'DESCRIPTOR' : _SUMMARYRESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SummaryResult)
+ })
+_sym_db.RegisterMessage(SummaryResult)
+
+FilesRecord = _reflection.GeneratedProtocolMessageType('FilesRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _FILESRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.FilesRecord)
+ })
+_sym_db.RegisterMessage(FilesRecord)
+
+FilesItem = _reflection.GeneratedProtocolMessageType('FilesItem', (_message.Message,), {
+ 'DESCRIPTOR' : _FILESITEM,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.FilesItem)
+ })
+_sym_db.RegisterMessage(FilesItem)
+
+FilesResult = _reflection.GeneratedProtocolMessageType('FilesResult', (_message.Message,), {
+ 'DESCRIPTOR' : _FILESRESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.FilesResult)
+ })
+_sym_db.RegisterMessage(FilesResult)
+
+StatsRecord = _reflection.GeneratedProtocolMessageType('StatsRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _STATSRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.StatsRecord)
+ })
+_sym_db.RegisterMessage(StatsRecord)
+
+StatsItem = _reflection.GeneratedProtocolMessageType('StatsItem', (_message.Message,), {
+ 'DESCRIPTOR' : _STATSITEM,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.StatsItem)
+ })
+_sym_db.RegisterMessage(StatsItem)
+
+ArtifactRecord = _reflection.GeneratedProtocolMessageType('ArtifactRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _ARTIFACTRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ArtifactRecord)
+ })
+_sym_db.RegisterMessage(ArtifactRecord)
+
+ArtifactManifest = _reflection.GeneratedProtocolMessageType('ArtifactManifest', (_message.Message,), {
+ 'DESCRIPTOR' : _ARTIFACTMANIFEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ArtifactManifest)
+ })
+_sym_db.RegisterMessage(ArtifactManifest)
+
+ArtifactManifestEntry = _reflection.GeneratedProtocolMessageType('ArtifactManifestEntry', (_message.Message,), {
+ 'DESCRIPTOR' : _ARTIFACTMANIFESTENTRY,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ArtifactManifestEntry)
+ })
+_sym_db.RegisterMessage(ArtifactManifestEntry)
+
+ExtraItem = _reflection.GeneratedProtocolMessageType('ExtraItem', (_message.Message,), {
+ 'DESCRIPTOR' : _EXTRAITEM,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ExtraItem)
+ })
+_sym_db.RegisterMessage(ExtraItem)
+
+StoragePolicyConfigItem = _reflection.GeneratedProtocolMessageType('StoragePolicyConfigItem', (_message.Message,), {
+ 'DESCRIPTOR' : _STORAGEPOLICYCONFIGITEM,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.StoragePolicyConfigItem)
+ })
+_sym_db.RegisterMessage(StoragePolicyConfigItem)
+
+ArtifactResult = _reflection.GeneratedProtocolMessageType('ArtifactResult', (_message.Message,), {
+ 'DESCRIPTOR' : _ARTIFACTRESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ArtifactResult)
+ })
+_sym_db.RegisterMessage(ArtifactResult)
+
+LinkArtifactResult = _reflection.GeneratedProtocolMessageType('LinkArtifactResult', (_message.Message,), {
+ 'DESCRIPTOR' : _LINKARTIFACTRESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.LinkArtifactResult)
+ })
+_sym_db.RegisterMessage(LinkArtifactResult)
+
+LinkArtifactRecord = _reflection.GeneratedProtocolMessageType('LinkArtifactRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _LINKARTIFACTRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.LinkArtifactRecord)
+ })
+_sym_db.RegisterMessage(LinkArtifactRecord)
+
+TBRecord = _reflection.GeneratedProtocolMessageType('TBRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _TBRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.TBRecord)
+ })
+_sym_db.RegisterMessage(TBRecord)
+
+TBResult = _reflection.GeneratedProtocolMessageType('TBResult', (_message.Message,), {
+ 'DESCRIPTOR' : _TBRESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.TBResult)
+ })
+_sym_db.RegisterMessage(TBResult)
+
+AlertRecord = _reflection.GeneratedProtocolMessageType('AlertRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _ALERTRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.AlertRecord)
+ })
+_sym_db.RegisterMessage(AlertRecord)
+
+AlertResult = _reflection.GeneratedProtocolMessageType('AlertResult', (_message.Message,), {
+ 'DESCRIPTOR' : _ALERTRESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.AlertResult)
+ })
+_sym_db.RegisterMessage(AlertResult)
+
+Request = _reflection.GeneratedProtocolMessageType('Request', (_message.Message,), {
+ 'DESCRIPTOR' : _REQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.Request)
+ })
+_sym_db.RegisterMessage(Request)
+
+Response = _reflection.GeneratedProtocolMessageType('Response', (_message.Message,), {
+ 'DESCRIPTOR' : _RESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.Response)
+ })
+_sym_db.RegisterMessage(Response)
+
+DeferRequest = _reflection.GeneratedProtocolMessageType('DeferRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _DEFERREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.DeferRequest)
+ })
+_sym_db.RegisterMessage(DeferRequest)
+
+PauseRequest = _reflection.GeneratedProtocolMessageType('PauseRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _PAUSEREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.PauseRequest)
+ })
+_sym_db.RegisterMessage(PauseRequest)
+
+PauseResponse = _reflection.GeneratedProtocolMessageType('PauseResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _PAUSERESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.PauseResponse)
+ })
+_sym_db.RegisterMessage(PauseResponse)
+
+ResumeRequest = _reflection.GeneratedProtocolMessageType('ResumeRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _RESUMEREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ResumeRequest)
+ })
+_sym_db.RegisterMessage(ResumeRequest)
+
+ResumeResponse = _reflection.GeneratedProtocolMessageType('ResumeResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _RESUMERESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ResumeResponse)
+ })
+_sym_db.RegisterMessage(ResumeResponse)
+
+LoginRequest = _reflection.GeneratedProtocolMessageType('LoginRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _LOGINREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.LoginRequest)
+ })
+_sym_db.RegisterMessage(LoginRequest)
+
+LoginResponse = _reflection.GeneratedProtocolMessageType('LoginResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _LOGINRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.LoginResponse)
+ })
+_sym_db.RegisterMessage(LoginResponse)
+
+GetSummaryRequest = _reflection.GeneratedProtocolMessageType('GetSummaryRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _GETSUMMARYREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.GetSummaryRequest)
+ })
+_sym_db.RegisterMessage(GetSummaryRequest)
+
+GetSummaryResponse = _reflection.GeneratedProtocolMessageType('GetSummaryResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _GETSUMMARYRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.GetSummaryResponse)
+ })
+_sym_db.RegisterMessage(GetSummaryResponse)
+
+GetSystemMetricsRequest = _reflection.GeneratedProtocolMessageType('GetSystemMetricsRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _GETSYSTEMMETRICSREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.GetSystemMetricsRequest)
+ })
+_sym_db.RegisterMessage(GetSystemMetricsRequest)
+
+SystemMetricSample = _reflection.GeneratedProtocolMessageType('SystemMetricSample', (_message.Message,), {
+ 'DESCRIPTOR' : _SYSTEMMETRICSAMPLE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SystemMetricSample)
+ })
+_sym_db.RegisterMessage(SystemMetricSample)
+
+SystemMetricsBuffer = _reflection.GeneratedProtocolMessageType('SystemMetricsBuffer', (_message.Message,), {
+ 'DESCRIPTOR' : _SYSTEMMETRICSBUFFER,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SystemMetricsBuffer)
+ })
+_sym_db.RegisterMessage(SystemMetricsBuffer)
+
+GetSystemMetricsResponse = _reflection.GeneratedProtocolMessageType('GetSystemMetricsResponse', (_message.Message,), {
+
+ 'SystemMetricsEntry' : _reflection.GeneratedProtocolMessageType('SystemMetricsEntry', (_message.Message,), {
+ 'DESCRIPTOR' : _GETSYSTEMMETRICSRESPONSE_SYSTEMMETRICSENTRY,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.GetSystemMetricsResponse.SystemMetricsEntry)
+ })
+ ,
+ 'DESCRIPTOR' : _GETSYSTEMMETRICSRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.GetSystemMetricsResponse)
+ })
+_sym_db.RegisterMessage(GetSystemMetricsResponse)
+_sym_db.RegisterMessage(GetSystemMetricsResponse.SystemMetricsEntry)
+
+StatusRequest = _reflection.GeneratedProtocolMessageType('StatusRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _STATUSREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.StatusRequest)
+ })
+_sym_db.RegisterMessage(StatusRequest)
+
+StatusResponse = _reflection.GeneratedProtocolMessageType('StatusResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _STATUSRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.StatusResponse)
+ })
+_sym_db.RegisterMessage(StatusResponse)
+
+StopStatusRequest = _reflection.GeneratedProtocolMessageType('StopStatusRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _STOPSTATUSREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.StopStatusRequest)
+ })
+_sym_db.RegisterMessage(StopStatusRequest)
+
+StopStatusResponse = _reflection.GeneratedProtocolMessageType('StopStatusResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _STOPSTATUSRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.StopStatusResponse)
+ })
+_sym_db.RegisterMessage(StopStatusResponse)
+
+NetworkStatusRequest = _reflection.GeneratedProtocolMessageType('NetworkStatusRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _NETWORKSTATUSREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.NetworkStatusRequest)
+ })
+_sym_db.RegisterMessage(NetworkStatusRequest)
+
+NetworkStatusResponse = _reflection.GeneratedProtocolMessageType('NetworkStatusResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _NETWORKSTATUSRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.NetworkStatusResponse)
+ })
+_sym_db.RegisterMessage(NetworkStatusResponse)
+
+HttpResponse = _reflection.GeneratedProtocolMessageType('HttpResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _HTTPRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.HttpResponse)
+ })
+_sym_db.RegisterMessage(HttpResponse)
+
+InternalMessagesRequest = _reflection.GeneratedProtocolMessageType('InternalMessagesRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _INTERNALMESSAGESREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.InternalMessagesRequest)
+ })
+_sym_db.RegisterMessage(InternalMessagesRequest)
+
+InternalMessagesResponse = _reflection.GeneratedProtocolMessageType('InternalMessagesResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _INTERNALMESSAGESRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.InternalMessagesResponse)
+ })
+_sym_db.RegisterMessage(InternalMessagesResponse)
+
+InternalMessages = _reflection.GeneratedProtocolMessageType('InternalMessages', (_message.Message,), {
+ 'DESCRIPTOR' : _INTERNALMESSAGES,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.InternalMessages)
+ })
+_sym_db.RegisterMessage(InternalMessages)
+
+PollExitRequest = _reflection.GeneratedProtocolMessageType('PollExitRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _POLLEXITREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.PollExitRequest)
+ })
+_sym_db.RegisterMessage(PollExitRequest)
+
+PollExitResponse = _reflection.GeneratedProtocolMessageType('PollExitResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _POLLEXITRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.PollExitResponse)
+ })
+_sym_db.RegisterMessage(PollExitResponse)
+
+SyncOverwrite = _reflection.GeneratedProtocolMessageType('SyncOverwrite', (_message.Message,), {
+ 'DESCRIPTOR' : _SYNCOVERWRITE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SyncOverwrite)
+ })
+_sym_db.RegisterMessage(SyncOverwrite)
+
+SyncSkip = _reflection.GeneratedProtocolMessageType('SyncSkip', (_message.Message,), {
+ 'DESCRIPTOR' : _SYNCSKIP,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SyncSkip)
+ })
+_sym_db.RegisterMessage(SyncSkip)
+
+SenderMarkRequest = _reflection.GeneratedProtocolMessageType('SenderMarkRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SENDERMARKREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SenderMarkRequest)
+ })
+_sym_db.RegisterMessage(SenderMarkRequest)
+
+SyncRequest = _reflection.GeneratedProtocolMessageType('SyncRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SYNCREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SyncRequest)
+ })
+_sym_db.RegisterMessage(SyncRequest)
+
+SyncResponse = _reflection.GeneratedProtocolMessageType('SyncResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _SYNCRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SyncResponse)
+ })
+_sym_db.RegisterMessage(SyncResponse)
+
+SenderReadRequest = _reflection.GeneratedProtocolMessageType('SenderReadRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SENDERREADREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SenderReadRequest)
+ })
+_sym_db.RegisterMessage(SenderReadRequest)
+
+StatusReportRequest = _reflection.GeneratedProtocolMessageType('StatusReportRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _STATUSREPORTREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.StatusReportRequest)
+ })
+_sym_db.RegisterMessage(StatusReportRequest)
+
+SummaryRecordRequest = _reflection.GeneratedProtocolMessageType('SummaryRecordRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SUMMARYRECORDREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SummaryRecordRequest)
+ })
+_sym_db.RegisterMessage(SummaryRecordRequest)
+
+TelemetryRecordRequest = _reflection.GeneratedProtocolMessageType('TelemetryRecordRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _TELEMETRYRECORDREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.TelemetryRecordRequest)
+ })
+_sym_db.RegisterMessage(TelemetryRecordRequest)
+
+ServerInfoRequest = _reflection.GeneratedProtocolMessageType('ServerInfoRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERINFOREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerInfoRequest)
+ })
+_sym_db.RegisterMessage(ServerInfoRequest)
+
+ServerInfoResponse = _reflection.GeneratedProtocolMessageType('ServerInfoResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERINFORESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerInfoResponse)
+ })
+_sym_db.RegisterMessage(ServerInfoResponse)
+
+ServerMessages = _reflection.GeneratedProtocolMessageType('ServerMessages', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERMESSAGES,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerMessages)
+ })
+_sym_db.RegisterMessage(ServerMessages)
+
+ServerMessage = _reflection.GeneratedProtocolMessageType('ServerMessage', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERMESSAGE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerMessage)
+ })
+_sym_db.RegisterMessage(ServerMessage)
+
+FileCounts = _reflection.GeneratedProtocolMessageType('FileCounts', (_message.Message,), {
+ 'DESCRIPTOR' : _FILECOUNTS,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.FileCounts)
+ })
+_sym_db.RegisterMessage(FileCounts)
+
+FilePusherStats = _reflection.GeneratedProtocolMessageType('FilePusherStats', (_message.Message,), {
+ 'DESCRIPTOR' : _FILEPUSHERSTATS,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.FilePusherStats)
+ })
+_sym_db.RegisterMessage(FilePusherStats)
+
+FilesUploaded = _reflection.GeneratedProtocolMessageType('FilesUploaded', (_message.Message,), {
+ 'DESCRIPTOR' : _FILESUPLOADED,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.FilesUploaded)
+ })
+_sym_db.RegisterMessage(FilesUploaded)
+
+FileTransferInfoRequest = _reflection.GeneratedProtocolMessageType('FileTransferInfoRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _FILETRANSFERINFOREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.FileTransferInfoRequest)
+ })
+_sym_db.RegisterMessage(FileTransferInfoRequest)
+
+LocalInfo = _reflection.GeneratedProtocolMessageType('LocalInfo', (_message.Message,), {
+ 'DESCRIPTOR' : _LOCALINFO,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.LocalInfo)
+ })
+_sym_db.RegisterMessage(LocalInfo)
+
+ShutdownRequest = _reflection.GeneratedProtocolMessageType('ShutdownRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SHUTDOWNREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ShutdownRequest)
+ })
+_sym_db.RegisterMessage(ShutdownRequest)
+
+ShutdownResponse = _reflection.GeneratedProtocolMessageType('ShutdownResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _SHUTDOWNRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ShutdownResponse)
+ })
+_sym_db.RegisterMessage(ShutdownResponse)
+
+AttachRequest = _reflection.GeneratedProtocolMessageType('AttachRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _ATTACHREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.AttachRequest)
+ })
+_sym_db.RegisterMessage(AttachRequest)
+
+AttachResponse = _reflection.GeneratedProtocolMessageType('AttachResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _ATTACHRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.AttachResponse)
+ })
+_sym_db.RegisterMessage(AttachResponse)
+
+TestInjectRequest = _reflection.GeneratedProtocolMessageType('TestInjectRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _TESTINJECTREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.TestInjectRequest)
+ })
+_sym_db.RegisterMessage(TestInjectRequest)
+
+TestInjectResponse = _reflection.GeneratedProtocolMessageType('TestInjectResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _TESTINJECTRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.TestInjectResponse)
+ })
+_sym_db.RegisterMessage(TestInjectResponse)
+
+HistoryAction = _reflection.GeneratedProtocolMessageType('HistoryAction', (_message.Message,), {
+ 'DESCRIPTOR' : _HISTORYACTION,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.HistoryAction)
+ })
+_sym_db.RegisterMessage(HistoryAction)
+
+PartialHistoryRequest = _reflection.GeneratedProtocolMessageType('PartialHistoryRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _PARTIALHISTORYREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.PartialHistoryRequest)
+ })
+_sym_db.RegisterMessage(PartialHistoryRequest)
+
+PartialHistoryResponse = _reflection.GeneratedProtocolMessageType('PartialHistoryResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _PARTIALHISTORYRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.PartialHistoryResponse)
+ })
+_sym_db.RegisterMessage(PartialHistoryResponse)
+
+SampledHistoryRequest = _reflection.GeneratedProtocolMessageType('SampledHistoryRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SAMPLEDHISTORYREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SampledHistoryRequest)
+ })
+_sym_db.RegisterMessage(SampledHistoryRequest)
+
+SampledHistoryItem = _reflection.GeneratedProtocolMessageType('SampledHistoryItem', (_message.Message,), {
+ 'DESCRIPTOR' : _SAMPLEDHISTORYITEM,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SampledHistoryItem)
+ })
+_sym_db.RegisterMessage(SampledHistoryItem)
+
+SampledHistoryResponse = _reflection.GeneratedProtocolMessageType('SampledHistoryResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _SAMPLEDHISTORYRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.SampledHistoryResponse)
+ })
+_sym_db.RegisterMessage(SampledHistoryResponse)
+
+RunStatusRequest = _reflection.GeneratedProtocolMessageType('RunStatusRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _RUNSTATUSREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.RunStatusRequest)
+ })
+_sym_db.RegisterMessage(RunStatusRequest)
+
+RunStatusResponse = _reflection.GeneratedProtocolMessageType('RunStatusResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _RUNSTATUSRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.RunStatusResponse)
+ })
+_sym_db.RegisterMessage(RunStatusResponse)
+
+RunStartRequest = _reflection.GeneratedProtocolMessageType('RunStartRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _RUNSTARTREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.RunStartRequest)
+ })
+_sym_db.RegisterMessage(RunStartRequest)
+
+RunStartResponse = _reflection.GeneratedProtocolMessageType('RunStartResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _RUNSTARTRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.RunStartResponse)
+ })
+_sym_db.RegisterMessage(RunStartResponse)
+
+CheckVersionRequest = _reflection.GeneratedProtocolMessageType('CheckVersionRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _CHECKVERSIONREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.CheckVersionRequest)
+ })
+_sym_db.RegisterMessage(CheckVersionRequest)
+
+CheckVersionResponse = _reflection.GeneratedProtocolMessageType('CheckVersionResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _CHECKVERSIONRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.CheckVersionResponse)
+ })
+_sym_db.RegisterMessage(CheckVersionResponse)
+
+JobInfoRequest = _reflection.GeneratedProtocolMessageType('JobInfoRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _JOBINFOREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.JobInfoRequest)
+ })
+_sym_db.RegisterMessage(JobInfoRequest)
+
+JobInfoResponse = _reflection.GeneratedProtocolMessageType('JobInfoResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _JOBINFORESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.JobInfoResponse)
+ })
+_sym_db.RegisterMessage(JobInfoResponse)
+
+LogArtifactRequest = _reflection.GeneratedProtocolMessageType('LogArtifactRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _LOGARTIFACTREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.LogArtifactRequest)
+ })
+_sym_db.RegisterMessage(LogArtifactRequest)
+
+LogArtifactResponse = _reflection.GeneratedProtocolMessageType('LogArtifactResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _LOGARTIFACTRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.LogArtifactResponse)
+ })
+_sym_db.RegisterMessage(LogArtifactResponse)
+
+DownloadArtifactRequest = _reflection.GeneratedProtocolMessageType('DownloadArtifactRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _DOWNLOADARTIFACTREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.DownloadArtifactRequest)
+ })
+_sym_db.RegisterMessage(DownloadArtifactRequest)
+
+DownloadArtifactResponse = _reflection.GeneratedProtocolMessageType('DownloadArtifactResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _DOWNLOADARTIFACTRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.DownloadArtifactResponse)
+ })
+_sym_db.RegisterMessage(DownloadArtifactResponse)
+
+KeepaliveRequest = _reflection.GeneratedProtocolMessageType('KeepaliveRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _KEEPALIVEREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.KeepaliveRequest)
+ })
+_sym_db.RegisterMessage(KeepaliveRequest)
+
+KeepaliveResponse = _reflection.GeneratedProtocolMessageType('KeepaliveResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _KEEPALIVERESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.KeepaliveResponse)
+ })
+_sym_db.RegisterMessage(KeepaliveResponse)
+
+ArtifactInfo = _reflection.GeneratedProtocolMessageType('ArtifactInfo', (_message.Message,), {
+ 'DESCRIPTOR' : _ARTIFACTINFO,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ArtifactInfo)
+ })
+_sym_db.RegisterMessage(ArtifactInfo)
+
+GitInfo = _reflection.GeneratedProtocolMessageType('GitInfo', (_message.Message,), {
+ 'DESCRIPTOR' : _GITINFO,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.GitInfo)
+ })
+_sym_db.RegisterMessage(GitInfo)
+
+GitSource = _reflection.GeneratedProtocolMessageType('GitSource', (_message.Message,), {
+ 'DESCRIPTOR' : _GITSOURCE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.GitSource)
+ })
+_sym_db.RegisterMessage(GitSource)
+
+ImageSource = _reflection.GeneratedProtocolMessageType('ImageSource', (_message.Message,), {
+ 'DESCRIPTOR' : _IMAGESOURCE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ImageSource)
+ })
+_sym_db.RegisterMessage(ImageSource)
+
+Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), {
+ 'DESCRIPTOR' : _SOURCE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.Source)
+ })
+_sym_db.RegisterMessage(Source)
+
+JobSource = _reflection.GeneratedProtocolMessageType('JobSource', (_message.Message,), {
+ 'DESCRIPTOR' : _JOBSOURCE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.JobSource)
+ })
+_sym_db.RegisterMessage(JobSource)
+
+PartialJobArtifact = _reflection.GeneratedProtocolMessageType('PartialJobArtifact', (_message.Message,), {
+ 'DESCRIPTOR' : _PARTIALJOBARTIFACT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.PartialJobArtifact)
+ })
+_sym_db.RegisterMessage(PartialJobArtifact)
+
+UseArtifactRecord = _reflection.GeneratedProtocolMessageType('UseArtifactRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _USEARTIFACTRECORD,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.UseArtifactRecord)
+ })
+_sym_db.RegisterMessage(UseArtifactRecord)
+
+UseArtifactResult = _reflection.GeneratedProtocolMessageType('UseArtifactResult', (_message.Message,), {
+ 'DESCRIPTOR' : _USEARTIFACTRESULT,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.UseArtifactResult)
+ })
+_sym_db.RegisterMessage(UseArtifactResult)
+
+CancelRequest = _reflection.GeneratedProtocolMessageType('CancelRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _CANCELREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.CancelRequest)
+ })
+_sym_db.RegisterMessage(CancelRequest)
+
+CancelResponse = _reflection.GeneratedProtocolMessageType('CancelResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _CANCELRESPONSE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.CancelResponse)
+ })
+_sym_db.RegisterMessage(CancelResponse)
+
+DiskInfo = _reflection.GeneratedProtocolMessageType('DiskInfo', (_message.Message,), {
+ 'DESCRIPTOR' : _DISKINFO,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.DiskInfo)
+ })
+_sym_db.RegisterMessage(DiskInfo)
+
+MemoryInfo = _reflection.GeneratedProtocolMessageType('MemoryInfo', (_message.Message,), {
+ 'DESCRIPTOR' : _MEMORYINFO,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.MemoryInfo)
+ })
+_sym_db.RegisterMessage(MemoryInfo)
+
+CpuInfo = _reflection.GeneratedProtocolMessageType('CpuInfo', (_message.Message,), {
+ 'DESCRIPTOR' : _CPUINFO,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.CpuInfo)
+ })
+_sym_db.RegisterMessage(CpuInfo)
+
+GpuAppleInfo = _reflection.GeneratedProtocolMessageType('GpuAppleInfo', (_message.Message,), {
+ 'DESCRIPTOR' : _GPUAPPLEINFO,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.GpuAppleInfo)
+ })
+_sym_db.RegisterMessage(GpuAppleInfo)
+
+GpuNvidiaInfo = _reflection.GeneratedProtocolMessageType('GpuNvidiaInfo', (_message.Message,), {
+ 'DESCRIPTOR' : _GPUNVIDIAINFO,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.GpuNvidiaInfo)
+ })
+_sym_db.RegisterMessage(GpuNvidiaInfo)
+
+GpuAmdInfo = _reflection.GeneratedProtocolMessageType('GpuAmdInfo', (_message.Message,), {
+ 'DESCRIPTOR' : _GPUAMDINFO,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.GpuAmdInfo)
+ })
+_sym_db.RegisterMessage(GpuAmdInfo)
+
+MetadataRequest = _reflection.GeneratedProtocolMessageType('MetadataRequest', (_message.Message,), {
+
+ 'DiskEntry' : _reflection.GeneratedProtocolMessageType('DiskEntry', (_message.Message,), {
+ 'DESCRIPTOR' : _METADATAREQUEST_DISKENTRY,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.MetadataRequest.DiskEntry)
+ })
+ ,
+
+ 'SlurmEntry' : _reflection.GeneratedProtocolMessageType('SlurmEntry', (_message.Message,), {
+ 'DESCRIPTOR' : _METADATAREQUEST_SLURMENTRY,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.MetadataRequest.SlurmEntry)
+ })
+ ,
+ 'DESCRIPTOR' : _METADATAREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.MetadataRequest)
+ })
+_sym_db.RegisterMessage(MetadataRequest)
+_sym_db.RegisterMessage(MetadataRequest.DiskEntry)
+_sym_db.RegisterMessage(MetadataRequest.SlurmEntry)
+
+PythonPackagesRequest = _reflection.GeneratedProtocolMessageType('PythonPackagesRequest', (_message.Message,), {
+
+ 'PythonPackage' : _reflection.GeneratedProtocolMessageType('PythonPackage', (_message.Message,), {
+ 'DESCRIPTOR' : _PYTHONPACKAGESREQUEST_PYTHONPACKAGE,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.PythonPackagesRequest.PythonPackage)
+ })
+ ,
+ 'DESCRIPTOR' : _PYTHONPACKAGESREQUEST,
+ '__module__' : 'wandb.proto.wandb_internal_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.PythonPackagesRequest)
+ })
+_sym_db.RegisterMessage(PythonPackagesRequest)
+_sym_db.RegisterMessage(PythonPackagesRequest.PythonPackage)
+
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+ DESCRIPTOR._options = None
+ _GETSYSTEMMETRICSRESPONSE_SYSTEMMETRICSENTRY._options = None
+ _GETSYSTEMMETRICSRESPONSE_SYSTEMMETRICSENTRY._serialized_options = b'8\001'
+ _METADATAREQUEST_DISKENTRY._options = None
+ _METADATAREQUEST_DISKENTRY._serialized_options = b'8\001'
+ _METADATAREQUEST_SLURMENTRY._options = None
+ _METADATAREQUEST_SLURMENTRY._serialized_options = b'8\001'
+ _RECORD._serialized_start=151
+ _RECORD._serialized_end=1331
+ _CONTROL._serialized_start=1334
+ _CONTROL._serialized_end=1502
+ _RESULT._serialized_start=1505
+ _RESULT._serialized_end=2004
+ _FINALRECORD._serialized_start=2006
+ _FINALRECORD._serialized_end=2064
+ _VERSIONINFO._serialized_start=2066
+ _VERSIONINFO._serialized_end=2164
+ _HEADERRECORD._serialized_start=2166
+ _HEADERRECORD._serialized_end=2276
+ _FOOTERRECORD._serialized_start=2278
+ _FOOTERRECORD._serialized_end=2337
+ _RUNRECORD._serialized_start=2340
+ _RUNRECORD._serialized_end=2930
+ _GITREPORECORD._serialized_start=2932
+ _GITREPORECORD._serialized_end=2991
+ _RUNUPDATERESULT._serialized_start=2993
+ _RUNUPDATERESULT._serialized_end=3092
+ _ERRORINFO._serialized_start=3095
+ _ERRORINFO._serialized_end=3267
+ _ERRORINFO_ERRORCODE._serialized_start=3176
+ _ERRORINFO_ERRORCODE._serialized_end=3267
+ _RUNEXITRECORD._serialized_start=3269
+ _RUNEXITRECORD._serialized_end=3365
+ _RUNEXITRESULT._serialized_start=3367
+ _RUNEXITRESULT._serialized_end=3382
+ _RUNPREEMPTINGRECORD._serialized_start=3384
+ _RUNPREEMPTINGRECORD._serialized_end=3450
+ _RUNPREEMPTINGRESULT._serialized_start=3452
+ _RUNPREEMPTINGRESULT._serialized_end=3473
+ _SETTINGSRECORD._serialized_start=3475
+ _SETTINGSRECORD._serialized_end=3580
+ _SETTINGSITEM._serialized_start=3582
+ _SETTINGSITEM._serialized_end=3629
+ _HISTORYSTEP._serialized_start=3631
+ _HISTORYSTEP._serialized_end=3657
+ _HISTORYRECORD._serialized_start=3660
+ _HISTORYRECORD._serialized_end=3806
+ _HISTORYITEM._serialized_start=3808
+ _HISTORYITEM._serialized_end=3874
+ _HISTORYRESULT._serialized_start=3876
+ _HISTORYRESULT._serialized_end=3891
+ _OUTPUTRECORD._serialized_start=3894
+ _OUTPUTRECORD._serialized_end=4114
+ _OUTPUTRECORD_OUTPUTTYPE._serialized_start=4078
+ _OUTPUTRECORD_OUTPUTTYPE._serialized_end=4114
+ _OUTPUTRESULT._serialized_start=4116
+ _OUTPUTRESULT._serialized_end=4130
+ _OUTPUTRAWRECORD._serialized_start=4133
+ _OUTPUTRAWRECORD._serialized_end=4359
+ _OUTPUTRAWRECORD_OUTPUTTYPE._serialized_start=4078
+ _OUTPUTRAWRECORD_OUTPUTTYPE._serialized_end=4114
+ _OUTPUTRAWRESULT._serialized_start=4361
+ _OUTPUTRAWRESULT._serialized_end=4378
+ _METRICRECORD._serialized_start=4381
+ _METRICRECORD._serialized_end=4789
+ _METRICRECORD_METRICGOAL._serialized_start=4723
+ _METRICRECORD_METRICGOAL._serialized_end=4789
+ _METRICRESULT._serialized_start=4791
+ _METRICRESULT._serialized_end=4805
+ _METRICOPTIONS._serialized_start=4807
+ _METRICOPTIONS._serialized_end=4874
+ _METRICCONTROL._serialized_start=4876
+ _METRICCONTROL._serialized_end=4910
+ _METRICSUMMARY._serialized_start=4912
+ _METRICSUMMARY._serialized_end=5023
+ _CONFIGRECORD._serialized_start=5026
+ _CONFIGRECORD._serialized_end=5173
+ _CONFIGITEM._serialized_start=5175
+ _CONFIGITEM._serialized_end=5240
+ _CONFIGRESULT._serialized_start=5242
+ _CONFIGRESULT._serialized_end=5256
+ _SUMMARYRECORD._serialized_start=5259
+ _SUMMARYRECORD._serialized_end=5409
+ _SUMMARYITEM._serialized_start=5411
+ _SUMMARYITEM._serialized_end=5477
+ _SUMMARYRESULT._serialized_start=5479
+ _SUMMARYRESULT._serialized_end=5494
+ _FILESRECORD._serialized_start=5496
+ _FILESRECORD._serialized_end=5596
+ _FILESITEM._serialized_start=5599
+ _FILESITEM._serialized_end=5852
+ _FILESITEM_POLICYTYPE._serialized_start=5753
+ _FILESITEM_POLICYTYPE._serialized_end=5793
+ _FILESITEM_FILETYPE._serialized_start=5795
+ _FILESITEM_FILETYPE._serialized_end=5852
+ _FILESRESULT._serialized_start=5854
+ _FILESRESULT._serialized_end=5867
+ _STATSRECORD._serialized_start=5870
+ _STATSRECORD._serialized_end=6100
+ _STATSRECORD_STATSTYPE._serialized_start=6077
+ _STATSRECORD_STATSTYPE._serialized_end=6100
+ _STATSITEM._serialized_start=6102
+ _STATSITEM._serialized_end=6146
+ _ARTIFACTRECORD._serialized_start=6149
+ _ARTIFACTRECORD._serialized_end=6622
+ _ARTIFACTMANIFEST._serialized_start=6625
+ _ARTIFACTMANIFEST._serialized_end=6813
+ _ARTIFACTMANIFESTENTRY._serialized_start=6816
+ _ARTIFACTMANIFESTENTRY._serialized_end=7003
+ _EXTRAITEM._serialized_start=7005
+ _EXTRAITEM._serialized_end=7049
+ _STORAGEPOLICYCONFIGITEM._serialized_start=7051
+ _STORAGEPOLICYCONFIGITEM._serialized_end=7109
+ _ARTIFACTRESULT._serialized_start=7111
+ _ARTIFACTRESULT._serialized_end=7127
+ _LINKARTIFACTRESULT._serialized_start=7129
+ _LINKARTIFACTRESULT._serialized_end=7149
+ _LINKARTIFACTRECORD._serialized_start=7152
+ _LINKARTIFACTRECORD._serialized_end=7359
+ _TBRECORD._serialized_start=7361
+ _TBRECORD._serialized_end=7465
+ _TBRESULT._serialized_start=7467
+ _TBRESULT._serialized_end=7477
+ _ALERTRECORD._serialized_start=7479
+ _ALERTRECORD._serialized_end=7604
+ _ALERTRESULT._serialized_start=7606
+ _ALERTRESULT._serialized_end=7619
+ _REQUEST._serialized_start=7622
+ _REQUEST._serialized_end=9615
+ _RESPONSE._serialized_start=9618
+ _RESPONSE._serialized_end=11124
+ _DEFERREQUEST._serialized_start=11127
+ _DEFERREQUEST._serialized_end=11447
+ _DEFERREQUEST_DEFERSTATE._serialized_start=11200
+ _DEFERREQUEST_DEFERSTATE._serialized_end=11447
+ _PAUSEREQUEST._serialized_start=11449
+ _PAUSEREQUEST._serialized_end=11509
+ _PAUSERESPONSE._serialized_start=11511
+ _PAUSERESPONSE._serialized_end=11526
+ _RESUMEREQUEST._serialized_start=11528
+ _RESUMEREQUEST._serialized_end=11589
+ _RESUMERESPONSE._serialized_start=11591
+ _RESUMERESPONSE._serialized_end=11607
+ _LOGINREQUEST._serialized_start=11609
+ _LOGINREQUEST._serialized_end=11686
+ _LOGINRESPONSE._serialized_start=11688
+ _LOGINRESPONSE._serialized_end=11726
+ _GETSUMMARYREQUEST._serialized_start=11728
+ _GETSUMMARYREQUEST._serialized_end=11793
+ _GETSUMMARYRESPONSE._serialized_start=11795
+ _GETSUMMARYRESPONSE._serialized_end=11858
+ _GETSYSTEMMETRICSREQUEST._serialized_start=11860
+ _GETSYSTEMMETRICSREQUEST._serialized_end=11931
+ _SYSTEMMETRICSAMPLE._serialized_start=11933
+ _SYSTEMMETRICSAMPLE._serialized_end=12015
+ _SYSTEMMETRICSBUFFER._serialized_start=12017
+ _SYSTEMMETRICSBUFFER._serialized_end=12090
+ _GETSYSTEMMETRICSRESPONSE._serialized_start=12093
+ _GETSYSTEMMETRICSRESPONSE._serialized_end=12295
+ _GETSYSTEMMETRICSRESPONSE_SYSTEMMETRICSENTRY._serialized_start=12206
+ _GETSYSTEMMETRICSRESPONSE_SYSTEMMETRICSENTRY._serialized_end=12295
+ _STATUSREQUEST._serialized_start=12297
+ _STATUSREQUEST._serialized_end=12358
+ _STATUSRESPONSE._serialized_start=12360
+ _STATUSRESPONSE._serialized_end=12401
+ _STOPSTATUSREQUEST._serialized_start=12403
+ _STOPSTATUSREQUEST._serialized_end=12468
+ _STOPSTATUSRESPONSE._serialized_start=12470
+ _STOPSTATUSRESPONSE._serialized_end=12515
+ _NETWORKSTATUSREQUEST._serialized_start=12517
+ _NETWORKSTATUSREQUEST._serialized_end=12585
+ _NETWORKSTATUSRESPONSE._serialized_start=12587
+ _NETWORKSTATUSRESPONSE._serialized_end=12667
+ _HTTPRESPONSE._serialized_start=12669
+ _HTTPRESPONSE._serialized_end=12737
+ _INTERNALMESSAGESREQUEST._serialized_start=12739
+ _INTERNALMESSAGESREQUEST._serialized_end=12810
+ _INTERNALMESSAGESRESPONSE._serialized_start=12812
+ _INTERNALMESSAGESRESPONSE._serialized_end=12890
+ _INTERNALMESSAGES._serialized_start=12892
+ _INTERNALMESSAGES._serialized_end=12927
+ _POLLEXITREQUEST._serialized_start=12929
+ _POLLEXITREQUEST._serialized_end=12992
+ _POLLEXITRESPONSE._serialized_start=12995
+ _POLLEXITRESPONSE._serialized_end=13183
+ _SYNCOVERWRITE._serialized_start=13185
+ _SYNCOVERWRITE._serialized_end=13249
+ _SYNCSKIP._serialized_start=13251
+ _SYNCSKIP._serialized_end=13281
+ _SENDERMARKREQUEST._serialized_start=13283
+ _SENDERMARKREQUEST._serialized_end=13302
+ _SYNCREQUEST._serialized_start=13305
+ _SYNCREQUEST._serialized_end=13452
+ _SYNCRESPONSE._serialized_start=13454
+ _SYNCRESPONSE._serialized_end=13523
+ _SENDERREADREQUEST._serialized_start=13525
+ _SENDERREADREQUEST._serialized_end=13588
+ _STATUSREPORTREQUEST._serialized_start=13590
+ _STATUSREPORTREQUEST._serialized_end=13699
+ _SUMMARYRECORDREQUEST._serialized_start=13701
+ _SUMMARYRECORDREQUEST._serialized_end=13771
+ _TELEMETRYRECORDREQUEST._serialized_start=13773
+ _TELEMETRYRECORDREQUEST._serialized_end=13849
+ _SERVERINFOREQUEST._serialized_start=13851
+ _SERVERINFOREQUEST._serialized_end=13916
+ _SERVERINFORESPONSE._serialized_start=13918
+ _SERVERINFORESPONSE._serialized_end=14042
+ _SERVERMESSAGES._serialized_start=14044
+ _SERVERMESSAGES._serialized_end=14105
+ _SERVERMESSAGE._serialized_start=14107
+ _SERVERMESSAGE._serialized_end=14208
+ _FILECOUNTS._serialized_start=14210
+ _FILECOUNTS._serialized_end=14309
+ _FILEPUSHERSTATS._serialized_start=14311
+ _FILEPUSHERSTATS._serialized_end=14396
+ _FILESUPLOADED._serialized_start=14398
+ _FILESUPLOADED._serialized_end=14428
+ _FILETRANSFERINFOREQUEST._serialized_start=14431
+ _FILETRANSFERINFOREQUEST._serialized_end=14675
+ _FILETRANSFERINFOREQUEST_TRANSFERTYPE._serialized_start=14635
+ _FILETRANSFERINFOREQUEST_TRANSFERTYPE._serialized_end=14675
+ _LOCALINFO._serialized_start=14677
+ _LOCALINFO._serialized_end=14726
+ _SHUTDOWNREQUEST._serialized_start=14728
+ _SHUTDOWNREQUEST._serialized_end=14791
+ _SHUTDOWNRESPONSE._serialized_start=14793
+ _SHUTDOWNRESPONSE._serialized_end=14811
+ _ATTACHREQUEST._serialized_start=14813
+ _ATTACHREQUEST._serialized_end=14893
+ _ATTACHRESPONSE._serialized_start=14895
+ _ATTACHRESPONSE._serialized_end=14993
+ _TESTINJECTREQUEST._serialized_start=14996
+ _TESTINJECTREQUEST._serialized_end=15337
+ _TESTINJECTRESPONSE._serialized_start=15339
+ _TESTINJECTRESPONSE._serialized_end=15359
+ _HISTORYACTION._serialized_start=15361
+ _HISTORYACTION._serialized_end=15391
+ _PARTIALHISTORYREQUEST._serialized_start=15394
+ _PARTIALHISTORYREQUEST._serialized_end=15596
+ _PARTIALHISTORYRESPONSE._serialized_start=15598
+ _PARTIALHISTORYRESPONSE._serialized_end=15622
+ _SAMPLEDHISTORYREQUEST._serialized_start=15624
+ _SAMPLEDHISTORYREQUEST._serialized_end=15693
+ _SAMPLEDHISTORYITEM._serialized_start=15695
+ _SAMPLEDHISTORYITEM._serialized_end=15790
+ _SAMPLEDHISTORYRESPONSE._serialized_start=15792
+ _SAMPLEDHISTORYRESPONSE._serialized_end=15866
+ _RUNSTATUSREQUEST._serialized_start=15868
+ _RUNSTATUSREQUEST._serialized_end=15932
+ _RUNSTATUSRESPONSE._serialized_start=15934
+ _RUNSTATUSRESPONSE._serialized_end=16054
+ _RUNSTARTREQUEST._serialized_start=16056
+ _RUNSTARTREQUEST._serialized_end=16159
+ _RUNSTARTRESPONSE._serialized_start=16161
+ _RUNSTARTRESPONSE._serialized_end=16179
+ _CHECKVERSIONREQUEST._serialized_start=16181
+ _CHECKVERSIONREQUEST._serialized_end=16273
+ _CHECKVERSIONRESPONSE._serialized_start=16275
+ _CHECKVERSIONRESPONSE._serialized_end=16368
+ _JOBINFOREQUEST._serialized_start=16370
+ _JOBINFOREQUEST._serialized_end=16432
+ _JOBINFORESPONSE._serialized_start=16434
+ _JOBINFORESPONSE._serialized_end=16488
+ _LOGARTIFACTREQUEST._serialized_start=16491
+ _LOGARTIFACTREQUEST._serialized_end=16650
+ _LOGARTIFACTRESPONSE._serialized_start=16652
+ _LOGARTIFACTRESPONSE._serialized_end=16717
+ _DOWNLOADARTIFACTREQUEST._serialized_start=16720
+ _DOWNLOADARTIFACTREQUEST._serialized_end=16869
+ _DOWNLOADARTIFACTRESPONSE._serialized_start=16871
+ _DOWNLOADARTIFACTRESPONSE._serialized_end=16920
+ _KEEPALIVEREQUEST._serialized_start=16922
+ _KEEPALIVEREQUEST._serialized_end=16986
+ _KEEPALIVERESPONSE._serialized_start=16988
+ _KEEPALIVERESPONSE._serialized_end=17007
+ _ARTIFACTINFO._serialized_start=17009
+ _ARTIFACTINFO._serialized_end=17079
+ _GITINFO._serialized_start=17081
+ _GITINFO._serialized_end=17122
+ _GITSOURCE._serialized_start=17124
+ _GITSOURCE._serialized_end=17216
+ _IMAGESOURCE._serialized_start=17218
+ _IMAGESOURCE._serialized_end=17246
+ _SOURCE._serialized_start=17249
+ _SOURCE._serialized_end=17389
+ _JOBSOURCE._serialized_start=17391
+ _JOBSOURCE._serialized_end=17498
+ _PARTIALJOBARTIFACT._serialized_start=17500
+ _PARTIALJOBARTIFACT._serialized_end=17586
+ _USEARTIFACTRECORD._serialized_start=17589
+ _USEARTIFACTRECORD._serialized_end=17746
+ _USEARTIFACTRESULT._serialized_start=17748
+ _USEARTIFACTRESULT._serialized_end=17767
+ _CANCELREQUEST._serialized_start=17769
+ _CANCELREQUEST._serialized_end=17851
+ _CANCELRESPONSE._serialized_start=17853
+ _CANCELRESPONSE._serialized_end=17869
+ _DISKINFO._serialized_start=17871
+ _DISKINFO._serialized_end=17910
+ _MEMORYINFO._serialized_start=17912
+ _MEMORYINFO._serialized_end=17939
+ _CPUINFO._serialized_start=17941
+ _CPUINFO._serialized_end=17988
+ _GPUAPPLEINFO._serialized_start=17990
+ _GPUAPPLEINFO._serialized_end=18052
+ _GPUNVIDIAINFO._serialized_start=18054
+ _GPUNVIDIAINFO._serialized_end=18105
+ _GPUAMDINFO._serialized_start=18108
+ _GPUAMDINFO._serialized_end=18373
+ _METADATAREQUEST._serialized_start=18376
+ _METADATAREQUEST._serialized_end=19422
+ _METADATAREQUEST_DISKENTRY._serialized_start=19307
+ _METADATAREQUEST_DISKENTRY._serialized_end=19376
+ _METADATAREQUEST_SLURMENTRY._serialized_start=19378
+ _METADATAREQUEST_SLURMENTRY._serialized_end=19422
+ _PYTHONPACKAGESREQUEST._serialized_start=19425
+ _PYTHONPACKAGESREQUEST._serialized_end=19566
+ _PYTHONPACKAGESREQUEST_PYTHONPACKAGE._serialized_start=19520
+ _PYTHONPACKAGESREQUEST_PYTHONPACKAGE._serialized_end=19566
+# @@protoc_insertion_point(module_scope)
diff --git a/wandb/proto/v3/wandb_internal_pb2.pyi b/wandb/proto/v3/wandb_internal_pb2.pyi
new file mode 100644
index 0000000000000000000000000000000000000000..39884ff0410005b8fa579f223c689b237fe6e6f7
--- /dev/null
+++ b/wandb/proto/v3/wandb_internal_pb2.pyi
@@ -0,0 +1,3683 @@
+"""
+@generated by mypy-protobuf. Do not edit manually!
+isort:skip_file
+"""
+import builtins
+import collections.abc
+import google.protobuf.descriptor
+import google.protobuf.internal.containers
+import google.protobuf.internal.enum_type_wrapper
+import google.protobuf.message
+import google.protobuf.timestamp_pb2
+import sys
+import typing
+import wandb.proto.wandb_base_pb2
+import wandb.proto.wandb_telemetry_pb2
+
+if sys.version_info >= (3, 10):
+ import typing as typing_extensions
+else:
+ import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+class Record(google.protobuf.message.Message):
+ """***********************
+ Records and Results
+ **********************
+
+
+ Record: joined record for message passing and persistence
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ NUM_FIELD_NUMBER: builtins.int
+ HISTORY_FIELD_NUMBER: builtins.int
+ SUMMARY_FIELD_NUMBER: builtins.int
+ OUTPUT_FIELD_NUMBER: builtins.int
+ CONFIG_FIELD_NUMBER: builtins.int
+ FILES_FIELD_NUMBER: builtins.int
+ STATS_FIELD_NUMBER: builtins.int
+ ARTIFACT_FIELD_NUMBER: builtins.int
+ TBRECORD_FIELD_NUMBER: builtins.int
+ ALERT_FIELD_NUMBER: builtins.int
+ TELEMETRY_FIELD_NUMBER: builtins.int
+ METRIC_FIELD_NUMBER: builtins.int
+ OUTPUT_RAW_FIELD_NUMBER: builtins.int
+ RUN_FIELD_NUMBER: builtins.int
+ EXIT_FIELD_NUMBER: builtins.int
+ FINAL_FIELD_NUMBER: builtins.int
+ HEADER_FIELD_NUMBER: builtins.int
+ FOOTER_FIELD_NUMBER: builtins.int
+ PREEMPTING_FIELD_NUMBER: builtins.int
+ LINK_ARTIFACT_FIELD_NUMBER: builtins.int
+ USE_ARTIFACT_FIELD_NUMBER: builtins.int
+ REQUEST_FIELD_NUMBER: builtins.int
+ CONTROL_FIELD_NUMBER: builtins.int
+ UUID_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ num: builtins.int
+ @property
+ def history(self) -> global___HistoryRecord:
+ """Low numbers for more frequent data"""
+ @property
+ def summary(self) -> global___SummaryRecord: ...
+ @property
+ def output(self) -> global___OutputRecord: ...
+ @property
+ def config(self) -> global___ConfigRecord: ...
+ @property
+ def files(self) -> global___FilesRecord: ...
+ @property
+ def stats(self) -> global___StatsRecord: ...
+ @property
+ def artifact(self) -> global___ArtifactRecord: ...
+ @property
+ def tbrecord(self) -> global___TBRecord: ...
+ @property
+ def alert(self) -> global___AlertRecord: ...
+ @property
+ def telemetry(self) -> wandb.proto.wandb_telemetry_pb2.TelemetryRecord: ...
+ @property
+ def metric(self) -> global___MetricRecord: ...
+ @property
+ def output_raw(self) -> global___OutputRawRecord: ...
+ @property
+ def run(self) -> global___RunRecord:
+ """Higher numbers for less frequent data"""
+ @property
+ def exit(self) -> global___RunExitRecord: ...
+ @property
+ def final(self) -> global___FinalRecord: ...
+ @property
+ def header(self) -> global___HeaderRecord: ...
+ @property
+ def footer(self) -> global___FooterRecord: ...
+ @property
+ def preempting(self) -> global___RunPreemptingRecord: ...
+ @property
+ def link_artifact(self) -> global___LinkArtifactRecord: ...
+ @property
+ def use_artifact(self) -> global___UseArtifactRecord: ...
+ @property
+ def request(self) -> global___Request:
+ """request field does not belong here longterm"""
+ @property
+ def control(self) -> global___Control: ...
+ uuid: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ num: builtins.int = ...,
+ history: global___HistoryRecord | None = ...,
+ summary: global___SummaryRecord | None = ...,
+ output: global___OutputRecord | None = ...,
+ config: global___ConfigRecord | None = ...,
+ files: global___FilesRecord | None = ...,
+ stats: global___StatsRecord | None = ...,
+ artifact: global___ArtifactRecord | None = ...,
+ tbrecord: global___TBRecord | None = ...,
+ alert: global___AlertRecord | None = ...,
+ telemetry: wandb.proto.wandb_telemetry_pb2.TelemetryRecord | None = ...,
+ metric: global___MetricRecord | None = ...,
+ output_raw: global___OutputRawRecord | None = ...,
+ run: global___RunRecord | None = ...,
+ exit: global___RunExitRecord | None = ...,
+ final: global___FinalRecord | None = ...,
+ header: global___HeaderRecord | None = ...,
+ footer: global___FooterRecord | None = ...,
+ preempting: global___RunPreemptingRecord | None = ...,
+ link_artifact: global___LinkArtifactRecord | None = ...,
+ use_artifact: global___UseArtifactRecord | None = ...,
+ request: global___Request | None = ...,
+ control: global___Control | None = ...,
+ uuid: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "alert", b"alert", "artifact", b"artifact", "config", b"config", "control", b"control", "exit", b"exit", "files", b"files", "final", b"final", "footer", b"footer", "header", b"header", "history", b"history", "link_artifact", b"link_artifact", "metric", b"metric", "output", b"output", "output_raw", b"output_raw", "preempting", b"preempting", "record_type", b"record_type", "request", b"request", "run", b"run", "stats", b"stats", "summary", b"summary", "tbrecord", b"tbrecord", "telemetry", b"telemetry", "use_artifact", b"use_artifact"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "alert", b"alert", "artifact", b"artifact", "config", b"config", "control", b"control", "exit", b"exit", "files", b"files", "final", b"final", "footer", b"footer", "header", b"header", "history", b"history", "link_artifact", b"link_artifact", "metric", b"metric", "num", b"num", "output", b"output", "output_raw", b"output_raw", "preempting", b"preempting", "record_type", b"record_type", "request", b"request", "run", b"run", "stats", b"stats", "summary", b"summary", "tbrecord", b"tbrecord", "telemetry", b"telemetry", "use_artifact", b"use_artifact", "uuid", b"uuid"]) -> None: ...
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["record_type", b"record_type"]) -> typing_extensions.Literal["history", "summary", "output", "config", "files", "stats", "artifact", "tbrecord", "alert", "telemetry", "metric", "output_raw", "run", "exit", "final", "header", "footer", "preempting", "link_artifact", "use_artifact", "request"] | None: ...
+
+global___Record = Record
+
+class Control(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ REQ_RESP_FIELD_NUMBER: builtins.int
+ LOCAL_FIELD_NUMBER: builtins.int
+ RELAY_ID_FIELD_NUMBER: builtins.int
+ MAILBOX_SLOT_FIELD_NUMBER: builtins.int
+ ALWAYS_SEND_FIELD_NUMBER: builtins.int
+ FLOW_CONTROL_FIELD_NUMBER: builtins.int
+ END_OFFSET_FIELD_NUMBER: builtins.int
+ CONNECTION_ID_FIELD_NUMBER: builtins.int
+ req_resp: builtins.bool
+ """record is expecting a result"""
+ local: builtins.bool
+ """should not be persisted or synchronized"""
+ relay_id: builtins.str
+ """used by service transport to identify correct stream"""
+ mailbox_slot: builtins.str
+ """mailbox slot"""
+ always_send: builtins.bool
+ """message to sender"""
+ flow_control: builtins.bool
+ """message should be passed to flow control"""
+ end_offset: builtins.int
+ """end of message offset of this written message"""
+ connection_id: builtins.str
+ """connection id"""
+ def __init__(
+ self,
+ *,
+ req_resp: builtins.bool = ...,
+ local: builtins.bool = ...,
+ relay_id: builtins.str = ...,
+ mailbox_slot: builtins.str = ...,
+ always_send: builtins.bool = ...,
+ flow_control: builtins.bool = ...,
+ end_offset: builtins.int = ...,
+ connection_id: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["always_send", b"always_send", "connection_id", b"connection_id", "end_offset", b"end_offset", "flow_control", b"flow_control", "local", b"local", "mailbox_slot", b"mailbox_slot", "relay_id", b"relay_id", "req_resp", b"req_resp"]) -> None: ...
+
+global___Control = Control
+
+class Result(google.protobuf.message.Message):
+ """
+ Result: all results
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_RESULT_FIELD_NUMBER: builtins.int
+ EXIT_RESULT_FIELD_NUMBER: builtins.int
+ LOG_RESULT_FIELD_NUMBER: builtins.int
+ SUMMARY_RESULT_FIELD_NUMBER: builtins.int
+ OUTPUT_RESULT_FIELD_NUMBER: builtins.int
+ CONFIG_RESULT_FIELD_NUMBER: builtins.int
+ RESPONSE_FIELD_NUMBER: builtins.int
+ CONTROL_FIELD_NUMBER: builtins.int
+ UUID_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def run_result(self) -> global___RunUpdateResult: ...
+ @property
+ def exit_result(self) -> global___RunExitResult: ...
+ @property
+ def log_result(self) -> global___HistoryResult: ...
+ @property
+ def summary_result(self) -> global___SummaryResult: ...
+ @property
+ def output_result(self) -> global___OutputResult: ...
+ @property
+ def config_result(self) -> global___ConfigResult: ...
+ @property
+ def response(self) -> global___Response:
+ """response field does not belong here longterm"""
+ @property
+ def control(self) -> global___Control: ...
+ uuid: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._ResultInfo: ...
+ def __init__(
+ self,
+ *,
+ run_result: global___RunUpdateResult | None = ...,
+ exit_result: global___RunExitResult | None = ...,
+ log_result: global___HistoryResult | None = ...,
+ summary_result: global___SummaryResult | None = ...,
+ output_result: global___OutputResult | None = ...,
+ config_result: global___ConfigResult | None = ...,
+ response: global___Response | None = ...,
+ control: global___Control | None = ...,
+ uuid: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._ResultInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "config_result", b"config_result", "control", b"control", "exit_result", b"exit_result", "log_result", b"log_result", "output_result", b"output_result", "response", b"response", "result_type", b"result_type", "run_result", b"run_result", "summary_result", b"summary_result"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "config_result", b"config_result", "control", b"control", "exit_result", b"exit_result", "log_result", b"log_result", "output_result", b"output_result", "response", b"response", "result_type", b"result_type", "run_result", b"run_result", "summary_result", b"summary_result", "uuid", b"uuid"]) -> None: ...
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["result_type", b"result_type"]) -> typing_extensions.Literal["run_result", "exit_result", "log_result", "summary_result", "output_result", "config_result", "response"] | None: ...
+
+global___Result = Result
+
+class FinalRecord(google.protobuf.message.Message):
+ """
+ FinalRecord
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___FinalRecord = FinalRecord
+
+class VersionInfo(google.protobuf.message.Message):
+ """
+ Version definition
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ PRODUCER_FIELD_NUMBER: builtins.int
+ MIN_CONSUMER_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ producer: builtins.str
+ """The version of the SDK backend that produced the data"""
+ min_consumer: builtins.str
+ """Minimum version of the wandb server that can read the data"""
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ producer: builtins.str = ...,
+ min_consumer: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "min_consumer", b"min_consumer", "producer", b"producer"]) -> None: ...
+
+global___VersionInfo = VersionInfo
+
+class HeaderRecord(google.protobuf.message.Message):
+ """
+ HeaderRecord
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ VERSION_INFO_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def version_info(self) -> global___VersionInfo: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ version_info: global___VersionInfo | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "version_info", b"version_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "version_info", b"version_info"]) -> None: ...
+
+global___HeaderRecord = HeaderRecord
+
+class FooterRecord(google.protobuf.message.Message):
+ """
+ FooterRecord
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___FooterRecord = FooterRecord
+
+class RunRecord(google.protobuf.message.Message):
+ """
+ RunRecord: wandb/sdk/wandb_run/Run
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_ID_FIELD_NUMBER: builtins.int
+ ENTITY_FIELD_NUMBER: builtins.int
+ PROJECT_FIELD_NUMBER: builtins.int
+ CONFIG_FIELD_NUMBER: builtins.int
+ SUMMARY_FIELD_NUMBER: builtins.int
+ RUN_GROUP_FIELD_NUMBER: builtins.int
+ JOB_TYPE_FIELD_NUMBER: builtins.int
+ DISPLAY_NAME_FIELD_NUMBER: builtins.int
+ NOTES_FIELD_NUMBER: builtins.int
+ TAGS_FIELD_NUMBER: builtins.int
+ SETTINGS_FIELD_NUMBER: builtins.int
+ SWEEP_ID_FIELD_NUMBER: builtins.int
+ HOST_FIELD_NUMBER: builtins.int
+ STARTING_STEP_FIELD_NUMBER: builtins.int
+ STORAGE_ID_FIELD_NUMBER: builtins.int
+ START_TIME_FIELD_NUMBER: builtins.int
+ RESUMED_FIELD_NUMBER: builtins.int
+ TELEMETRY_FIELD_NUMBER: builtins.int
+ RUNTIME_FIELD_NUMBER: builtins.int
+ GIT_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ run_id: builtins.str
+ entity: builtins.str
+ project: builtins.str
+ @property
+ def config(self) -> global___ConfigRecord: ...
+ @property
+ def summary(self) -> global___SummaryRecord: ...
+ run_group: builtins.str
+ job_type: builtins.str
+ display_name: builtins.str
+ notes: builtins.str
+ @property
+ def tags(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ @property
+ def settings(self) -> global___SettingsRecord: ...
+ sweep_id: builtins.str
+ host: builtins.str
+ starting_step: builtins.int
+ storage_id: builtins.str
+ @property
+ def start_time(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ resumed: builtins.bool
+ @property
+ def telemetry(self) -> wandb.proto.wandb_telemetry_pb2.TelemetryRecord: ...
+ runtime: builtins.int
+ @property
+ def git(self) -> global___GitRepoRecord: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ run_id: builtins.str = ...,
+ entity: builtins.str = ...,
+ project: builtins.str = ...,
+ config: global___ConfigRecord | None = ...,
+ summary: global___SummaryRecord | None = ...,
+ run_group: builtins.str = ...,
+ job_type: builtins.str = ...,
+ display_name: builtins.str = ...,
+ notes: builtins.str = ...,
+ tags: collections.abc.Iterable[builtins.str] | None = ...,
+ settings: global___SettingsRecord | None = ...,
+ sweep_id: builtins.str = ...,
+ host: builtins.str = ...,
+ starting_step: builtins.int = ...,
+ storage_id: builtins.str = ...,
+ start_time: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ resumed: builtins.bool = ...,
+ telemetry: wandb.proto.wandb_telemetry_pb2.TelemetryRecord | None = ...,
+ runtime: builtins.int = ...,
+ git: global___GitRepoRecord | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "config", b"config", "git", b"git", "settings", b"settings", "start_time", b"start_time", "summary", b"summary", "telemetry", b"telemetry"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "config", b"config", "display_name", b"display_name", "entity", b"entity", "git", b"git", "host", b"host", "job_type", b"job_type", "notes", b"notes", "project", b"project", "resumed", b"resumed", "run_group", b"run_group", "run_id", b"run_id", "runtime", b"runtime", "settings", b"settings", "start_time", b"start_time", "starting_step", b"starting_step", "storage_id", b"storage_id", "summary", b"summary", "sweep_id", b"sweep_id", "tags", b"tags", "telemetry", b"telemetry"]) -> None: ...
+
+global___RunRecord = RunRecord
+
+class GitRepoRecord(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ REMOTE_URL_FIELD_NUMBER: builtins.int
+ COMMIT_FIELD_NUMBER: builtins.int
+ remote_url: builtins.str
+ commit: builtins.str
+ def __init__(
+ self,
+ *,
+ remote_url: builtins.str = ...,
+ commit: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["commit", b"commit", "remote_url", b"remote_url"]) -> None: ...
+
+global___GitRepoRecord = GitRepoRecord
+
+class RunUpdateResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_FIELD_NUMBER: builtins.int
+ ERROR_FIELD_NUMBER: builtins.int
+ @property
+ def run(self) -> global___RunRecord: ...
+ @property
+ def error(self) -> global___ErrorInfo: ...
+ def __init__(
+ self,
+ *,
+ run: global___RunRecord | None = ...,
+ error: global___ErrorInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["error", b"error", "run", b"run"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["error", b"error", "run", b"run"]) -> None: ...
+
+global___RunUpdateResult = RunUpdateResult
+
+class ErrorInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _ErrorCode:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _ErrorCodeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[ErrorInfo._ErrorCode.ValueType], builtins.type): # noqa: F821
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ UNKNOWN: ErrorInfo._ErrorCode.ValueType # 0
+ COMMUNICATION: ErrorInfo._ErrorCode.ValueType # 1
+ AUTHENTICATION: ErrorInfo._ErrorCode.ValueType # 2
+ USAGE: ErrorInfo._ErrorCode.ValueType # 3
+ UNSUPPORTED: ErrorInfo._ErrorCode.ValueType # 4
+
+ class ErrorCode(_ErrorCode, metaclass=_ErrorCodeEnumTypeWrapper): ...
+ UNKNOWN: ErrorInfo.ErrorCode.ValueType # 0
+ COMMUNICATION: ErrorInfo.ErrorCode.ValueType # 1
+ AUTHENTICATION: ErrorInfo.ErrorCode.ValueType # 2
+ USAGE: ErrorInfo.ErrorCode.ValueType # 3
+ UNSUPPORTED: ErrorInfo.ErrorCode.ValueType # 4
+
+ MESSAGE_FIELD_NUMBER: builtins.int
+ CODE_FIELD_NUMBER: builtins.int
+ message: builtins.str
+ code: global___ErrorInfo.ErrorCode.ValueType
+ def __init__(
+ self,
+ *,
+ message: builtins.str = ...,
+ code: global___ErrorInfo.ErrorCode.ValueType = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["code", b"code", "message", b"message"]) -> None: ...
+
+global___ErrorInfo = ErrorInfo
+
+class RunExitRecord(google.protobuf.message.Message):
+ """
+ RunExitRecord: exit status of process
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ EXIT_CODE_FIELD_NUMBER: builtins.int
+ RUNTIME_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ exit_code: builtins.int
+ runtime: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ exit_code: builtins.int = ...,
+ runtime: builtins.int = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "exit_code", b"exit_code", "runtime", b"runtime"]) -> None: ...
+
+global___RunExitRecord = RunExitRecord
+
+class RunExitResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___RunExitResult = RunExitResult
+
+class RunPreemptingRecord(google.protobuf.message.Message):
+ """
+ RunPreemptingRecord: run being preempted
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___RunPreemptingRecord = RunPreemptingRecord
+
+class RunPreemptingResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___RunPreemptingResult = RunPreemptingResult
+
+class SettingsRecord(google.protobuf.message.Message):
+ """
+ SettingsRecord: wandb/sdk/wandb_settings/Settings
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ITEM_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def item(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___SettingsItem]: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ item: collections.abc.Iterable[global___SettingsItem] | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "item", b"item"]) -> None: ...
+
+global___SettingsRecord = SettingsRecord
+
+class SettingsItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_JSON_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ value_json: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value_json: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value_json", b"value_json"]) -> None: ...
+
+global___SettingsItem = SettingsItem
+
+class HistoryStep(google.protobuf.message.Message):
+ """
+ HistoryRecord: wandb/sdk/wandb_history/History
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ NUM_FIELD_NUMBER: builtins.int
+ num: builtins.int
+ def __init__(
+ self,
+ *,
+ num: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["num", b"num"]) -> None: ...
+
+global___HistoryStep = HistoryStep
+
+class HistoryRecord(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ITEM_FIELD_NUMBER: builtins.int
+ STEP_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def item(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___HistoryItem]: ...
+ @property
+ def step(self) -> global___HistoryStep: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ item: collections.abc.Iterable[global___HistoryItem] | None = ...,
+ step: global___HistoryStep | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "step", b"step"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "item", b"item", "step", b"step"]) -> None: ...
+
+global___HistoryRecord = HistoryRecord
+
+class HistoryItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ NESTED_KEY_FIELD_NUMBER: builtins.int
+ VALUE_JSON_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ @property
+ def nested_key(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ value_json: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ nested_key: collections.abc.Iterable[builtins.str] | None = ...,
+ value_json: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "nested_key", b"nested_key", "value_json", b"value_json"]) -> None: ...
+
+global___HistoryItem = HistoryItem
+
+class HistoryResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___HistoryResult = HistoryResult
+
+class OutputRecord(google.protobuf.message.Message):
+ """
+ OutputRecord: console output
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _OutputType:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _OutputTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[OutputRecord._OutputType.ValueType], builtins.type): # noqa: F821
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ STDERR: OutputRecord._OutputType.ValueType # 0
+ STDOUT: OutputRecord._OutputType.ValueType # 1
+
+ class OutputType(_OutputType, metaclass=_OutputTypeEnumTypeWrapper): ...
+ STDERR: OutputRecord.OutputType.ValueType # 0
+ STDOUT: OutputRecord.OutputType.ValueType # 1
+
+ OUTPUT_TYPE_FIELD_NUMBER: builtins.int
+ TIMESTAMP_FIELD_NUMBER: builtins.int
+ LINE_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ output_type: global___OutputRecord.OutputType.ValueType
+ @property
+ def timestamp(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ line: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ output_type: global___OutputRecord.OutputType.ValueType = ...,
+ timestamp: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ line: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "timestamp", b"timestamp"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "line", b"line", "output_type", b"output_type", "timestamp", b"timestamp"]) -> None: ...
+
+global___OutputRecord = OutputRecord
+
+class OutputResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___OutputResult = OutputResult
+
+class OutputRawRecord(google.protobuf.message.Message):
+ """
+ OutputRawRecord: raw console output
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _OutputType:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _OutputTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[OutputRawRecord._OutputType.ValueType], builtins.type): # noqa: F821
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ STDERR: OutputRawRecord._OutputType.ValueType # 0
+ STDOUT: OutputRawRecord._OutputType.ValueType # 1
+
+ class OutputType(_OutputType, metaclass=_OutputTypeEnumTypeWrapper): ...
+ STDERR: OutputRawRecord.OutputType.ValueType # 0
+ STDOUT: OutputRawRecord.OutputType.ValueType # 1
+
+ OUTPUT_TYPE_FIELD_NUMBER: builtins.int
+ TIMESTAMP_FIELD_NUMBER: builtins.int
+ LINE_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ output_type: global___OutputRawRecord.OutputType.ValueType
+ @property
+ def timestamp(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ line: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ output_type: global___OutputRawRecord.OutputType.ValueType = ...,
+ timestamp: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ line: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "timestamp", b"timestamp"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "line", b"line", "output_type", b"output_type", "timestamp", b"timestamp"]) -> None: ...
+
+global___OutputRawRecord = OutputRawRecord
+
+class OutputRawResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___OutputRawResult = OutputRawResult
+
+class MetricRecord(google.protobuf.message.Message):
+ """
+ MetricRecord: wandb/sdk/wandb_metric/Metric
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _MetricGoal:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _MetricGoalEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[MetricRecord._MetricGoal.ValueType], builtins.type): # noqa: F821
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ GOAL_UNSET: MetricRecord._MetricGoal.ValueType # 0
+ GOAL_MINIMIZE: MetricRecord._MetricGoal.ValueType # 1
+ GOAL_MAXIMIZE: MetricRecord._MetricGoal.ValueType # 2
+
+ class MetricGoal(_MetricGoal, metaclass=_MetricGoalEnumTypeWrapper): ...
+ GOAL_UNSET: MetricRecord.MetricGoal.ValueType # 0
+ GOAL_MINIMIZE: MetricRecord.MetricGoal.ValueType # 1
+ GOAL_MAXIMIZE: MetricRecord.MetricGoal.ValueType # 2
+
+ NAME_FIELD_NUMBER: builtins.int
+ GLOB_NAME_FIELD_NUMBER: builtins.int
+ STEP_METRIC_FIELD_NUMBER: builtins.int
+ STEP_METRIC_INDEX_FIELD_NUMBER: builtins.int
+ OPTIONS_FIELD_NUMBER: builtins.int
+ SUMMARY_FIELD_NUMBER: builtins.int
+ GOAL_FIELD_NUMBER: builtins.int
+ _CONTROL_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ name: builtins.str
+ """only name or globname is set"""
+ glob_name: builtins.str
+ step_metric: builtins.str
+ """step metric index can be used instead of step_metric when
+ MetricRecord is encoded in a list of MetricRecords
+ """
+ step_metric_index: builtins.int
+ """one-based array index"""
+ @property
+ def options(self) -> global___MetricOptions: ...
+ @property
+ def summary(self) -> global___MetricSummary: ...
+ goal: global___MetricRecord.MetricGoal.ValueType
+ @property
+ def _control(self) -> global___MetricControl: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ name: builtins.str = ...,
+ glob_name: builtins.str = ...,
+ step_metric: builtins.str = ...,
+ step_metric_index: builtins.int = ...,
+ options: global___MetricOptions | None = ...,
+ summary: global___MetricSummary | None = ...,
+ goal: global___MetricRecord.MetricGoal.ValueType = ...,
+ _control: global___MetricControl | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_control", b"_control", "_info", b"_info", "options", b"options", "summary", b"summary"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_control", b"_control", "_info", b"_info", "glob_name", b"glob_name", "goal", b"goal", "name", b"name", "options", b"options", "step_metric", b"step_metric", "step_metric_index", b"step_metric_index", "summary", b"summary"]) -> None: ...
+
+global___MetricRecord = MetricRecord
+
+class MetricResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___MetricResult = MetricResult
+
+class MetricOptions(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ STEP_SYNC_FIELD_NUMBER: builtins.int
+ HIDDEN_FIELD_NUMBER: builtins.int
+ DEFINED_FIELD_NUMBER: builtins.int
+ step_sync: builtins.bool
+ hidden: builtins.bool
+ defined: builtins.bool
+ """metric explicitly defined (not from glob match or step metric)"""
+ def __init__(
+ self,
+ *,
+ step_sync: builtins.bool = ...,
+ hidden: builtins.bool = ...,
+ defined: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["defined", b"defined", "hidden", b"hidden", "step_sync", b"step_sync"]) -> None: ...
+
+global___MetricOptions = MetricOptions
+
+class MetricControl(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ OVERWRITE_FIELD_NUMBER: builtins.int
+ overwrite: builtins.bool
+ def __init__(
+ self,
+ *,
+ overwrite: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["overwrite", b"overwrite"]) -> None: ...
+
+global___MetricControl = MetricControl
+
+class MetricSummary(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ MIN_FIELD_NUMBER: builtins.int
+ MAX_FIELD_NUMBER: builtins.int
+ MEAN_FIELD_NUMBER: builtins.int
+ BEST_FIELD_NUMBER: builtins.int
+ LAST_FIELD_NUMBER: builtins.int
+ NONE_FIELD_NUMBER: builtins.int
+ COPY_FIELD_NUMBER: builtins.int
+ min: builtins.bool
+ max: builtins.bool
+ mean: builtins.bool
+ best: builtins.bool
+ last: builtins.bool
+ none: builtins.bool
+ copy: builtins.bool
+ def __init__(
+ self,
+ *,
+ min: builtins.bool = ...,
+ max: builtins.bool = ...,
+ mean: builtins.bool = ...,
+ best: builtins.bool = ...,
+ last: builtins.bool = ...,
+ none: builtins.bool = ...,
+ copy: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["best", b"best", "copy", b"copy", "last", b"last", "max", b"max", "mean", b"mean", "min", b"min", "none", b"none"]) -> None: ...
+
+global___MetricSummary = MetricSummary
+
+class ConfigRecord(google.protobuf.message.Message):
+ """
+ ConfigRecord: wandb/sdk/wandb_config/Config
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ UPDATE_FIELD_NUMBER: builtins.int
+ REMOVE_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def update(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ConfigItem]: ...
+ @property
+ def remove(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ConfigItem]: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ update: collections.abc.Iterable[global___ConfigItem] | None = ...,
+ remove: collections.abc.Iterable[global___ConfigItem] | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "remove", b"remove", "update", b"update"]) -> None: ...
+
+global___ConfigRecord = ConfigRecord
+
+class ConfigItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ NESTED_KEY_FIELD_NUMBER: builtins.int
+ VALUE_JSON_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ @property
+ def nested_key(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ value_json: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ nested_key: collections.abc.Iterable[builtins.str] | None = ...,
+ value_json: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "nested_key", b"nested_key", "value_json", b"value_json"]) -> None: ...
+
+global___ConfigItem = ConfigItem
+
+class ConfigResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ConfigResult = ConfigResult
+
+class SummaryRecord(google.protobuf.message.Message):
+ """
+ SummaryRecord: wandb/sdk/wandb_summary/Summary
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ UPDATE_FIELD_NUMBER: builtins.int
+ REMOVE_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def update(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___SummaryItem]: ...
+ @property
+ def remove(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___SummaryItem]: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ update: collections.abc.Iterable[global___SummaryItem] | None = ...,
+ remove: collections.abc.Iterable[global___SummaryItem] | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "remove", b"remove", "update", b"update"]) -> None: ...
+
+global___SummaryRecord = SummaryRecord
+
+class SummaryItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ NESTED_KEY_FIELD_NUMBER: builtins.int
+ VALUE_JSON_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ @property
+ def nested_key(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ value_json: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ nested_key: collections.abc.Iterable[builtins.str] | None = ...,
+ value_json: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "nested_key", b"nested_key", "value_json", b"value_json"]) -> None: ...
+
+global___SummaryItem = SummaryItem
+
+class SummaryResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___SummaryResult = SummaryResult
+
+class FilesRecord(google.protobuf.message.Message):
+ """
+ FilesRecord: files added to run
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ FILES_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def files(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___FilesItem]: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ files: collections.abc.Iterable[global___FilesItem] | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "files", b"files"]) -> None: ...
+
+global___FilesRecord = FilesRecord
+
+class FilesItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _PolicyType:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _PolicyTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[FilesItem._PolicyType.ValueType], builtins.type): # noqa: F821
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ NOW: FilesItem._PolicyType.ValueType # 0
+ END: FilesItem._PolicyType.ValueType # 1
+ LIVE: FilesItem._PolicyType.ValueType # 2
+
+ class PolicyType(_PolicyType, metaclass=_PolicyTypeEnumTypeWrapper): ...
+ NOW: FilesItem.PolicyType.ValueType # 0
+ END: FilesItem.PolicyType.ValueType # 1
+ LIVE: FilesItem.PolicyType.ValueType # 2
+
+ class _FileType:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _FileTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[FilesItem._FileType.ValueType], builtins.type): # noqa: F821
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ OTHER: FilesItem._FileType.ValueType # 0
+ WANDB: FilesItem._FileType.ValueType # 1
+ MEDIA: FilesItem._FileType.ValueType # 2
+ ARTIFACT: FilesItem._FileType.ValueType # 3
+
+ class FileType(_FileType, metaclass=_FileTypeEnumTypeWrapper): ...
+ OTHER: FilesItem.FileType.ValueType # 0
+ WANDB: FilesItem.FileType.ValueType # 1
+ MEDIA: FilesItem.FileType.ValueType # 2
+ ARTIFACT: FilesItem.FileType.ValueType # 3
+
+ PATH_FIELD_NUMBER: builtins.int
+ POLICY_FIELD_NUMBER: builtins.int
+ TYPE_FIELD_NUMBER: builtins.int
+ EXTERNAL_PATH_FIELD_NUMBER: builtins.int
+ path: builtins.str
+ policy: global___FilesItem.PolicyType.ValueType
+ type: global___FilesItem.FileType.ValueType
+ external_path: builtins.str
+ def __init__(
+ self,
+ *,
+ path: builtins.str = ...,
+ policy: global___FilesItem.PolicyType.ValueType = ...,
+ type: global___FilesItem.FileType.ValueType = ...,
+ external_path: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["external_path", b"external_path", "path", b"path", "policy", b"policy", "type", b"type"]) -> None: ...
+
+global___FilesItem = FilesItem
+
+class FilesResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___FilesResult = FilesResult
+
+class StatsRecord(google.protobuf.message.Message):
+ """
+ StatsRecord: system metrics
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _StatsType:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _StatsTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[StatsRecord._StatsType.ValueType], builtins.type): # noqa: F821
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ SYSTEM: StatsRecord._StatsType.ValueType # 0
+
+ class StatsType(_StatsType, metaclass=_StatsTypeEnumTypeWrapper): ...
+ SYSTEM: StatsRecord.StatsType.ValueType # 0
+
+ STATS_TYPE_FIELD_NUMBER: builtins.int
+ TIMESTAMP_FIELD_NUMBER: builtins.int
+ ITEM_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ stats_type: global___StatsRecord.StatsType.ValueType
+ @property
+ def timestamp(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ @property
+ def item(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___StatsItem]: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ stats_type: global___StatsRecord.StatsType.ValueType = ...,
+ timestamp: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ item: collections.abc.Iterable[global___StatsItem] | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "timestamp", b"timestamp"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "item", b"item", "stats_type", b"stats_type", "timestamp", b"timestamp"]) -> None: ...
+
+global___StatsRecord = StatsRecord
+
+class StatsItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_JSON_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ value_json: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value_json: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value_json", b"value_json"]) -> None: ...
+
+global___StatsItem = StatsItem
+
+class ArtifactRecord(google.protobuf.message.Message):
+ """
+ ArtifactRecord: track artifacts
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_ID_FIELD_NUMBER: builtins.int
+ PROJECT_FIELD_NUMBER: builtins.int
+ ENTITY_FIELD_NUMBER: builtins.int
+ TYPE_FIELD_NUMBER: builtins.int
+ NAME_FIELD_NUMBER: builtins.int
+ DIGEST_FIELD_NUMBER: builtins.int
+ DESCRIPTION_FIELD_NUMBER: builtins.int
+ METADATA_FIELD_NUMBER: builtins.int
+ USER_CREATED_FIELD_NUMBER: builtins.int
+ USE_AFTER_COMMIT_FIELD_NUMBER: builtins.int
+ ALIASES_FIELD_NUMBER: builtins.int
+ MANIFEST_FIELD_NUMBER: builtins.int
+ DISTRIBUTED_ID_FIELD_NUMBER: builtins.int
+ FINALIZE_FIELD_NUMBER: builtins.int
+ CLIENT_ID_FIELD_NUMBER: builtins.int
+ SEQUENCE_CLIENT_ID_FIELD_NUMBER: builtins.int
+ BASE_ID_FIELD_NUMBER: builtins.int
+ TTL_DURATION_SECONDS_FIELD_NUMBER: builtins.int
+ INCREMENTAL_BETA1_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ run_id: builtins.str
+ project: builtins.str
+ entity: builtins.str
+ type: builtins.str
+ name: builtins.str
+ digest: builtins.str
+ description: builtins.str
+ metadata: builtins.str
+ user_created: builtins.bool
+ use_after_commit: builtins.bool
+ @property
+ def aliases(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ @property
+ def manifest(self) -> global___ArtifactManifest: ...
+ distributed_id: builtins.str
+ finalize: builtins.bool
+ client_id: builtins.str
+ sequence_client_id: builtins.str
+ base_id: builtins.str
+ ttl_duration_seconds: builtins.int
+ incremental_beta1: builtins.bool
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ run_id: builtins.str = ...,
+ project: builtins.str = ...,
+ entity: builtins.str = ...,
+ type: builtins.str = ...,
+ name: builtins.str = ...,
+ digest: builtins.str = ...,
+ description: builtins.str = ...,
+ metadata: builtins.str = ...,
+ user_created: builtins.bool = ...,
+ use_after_commit: builtins.bool = ...,
+ aliases: collections.abc.Iterable[builtins.str] | None = ...,
+ manifest: global___ArtifactManifest | None = ...,
+ distributed_id: builtins.str = ...,
+ finalize: builtins.bool = ...,
+ client_id: builtins.str = ...,
+ sequence_client_id: builtins.str = ...,
+ base_id: builtins.str = ...,
+ ttl_duration_seconds: builtins.int = ...,
+ incremental_beta1: builtins.bool = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "manifest", b"manifest"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "aliases", b"aliases", "base_id", b"base_id", "client_id", b"client_id", "description", b"description", "digest", b"digest", "distributed_id", b"distributed_id", "entity", b"entity", "finalize", b"finalize", "incremental_beta1", b"incremental_beta1", "manifest", b"manifest", "metadata", b"metadata", "name", b"name", "project", b"project", "run_id", b"run_id", "sequence_client_id", b"sequence_client_id", "ttl_duration_seconds", b"ttl_duration_seconds", "type", b"type", "use_after_commit", b"use_after_commit", "user_created", b"user_created"]) -> None: ...
+
+global___ArtifactRecord = ArtifactRecord
+
+class ArtifactManifest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ VERSION_FIELD_NUMBER: builtins.int
+ STORAGE_POLICY_FIELD_NUMBER: builtins.int
+ STORAGE_POLICY_CONFIG_FIELD_NUMBER: builtins.int
+ CONTENTS_FIELD_NUMBER: builtins.int
+ version: builtins.int
+ storage_policy: builtins.str
+ @property
+ def storage_policy_config(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___StoragePolicyConfigItem]: ...
+ @property
+ def contents(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ArtifactManifestEntry]: ...
+ def __init__(
+ self,
+ *,
+ version: builtins.int = ...,
+ storage_policy: builtins.str = ...,
+ storage_policy_config: collections.abc.Iterable[global___StoragePolicyConfigItem] | None = ...,
+ contents: collections.abc.Iterable[global___ArtifactManifestEntry] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["contents", b"contents", "storage_policy", b"storage_policy", "storage_policy_config", b"storage_policy_config", "version", b"version"]) -> None: ...
+
+global___ArtifactManifest = ArtifactManifest
+
+class ArtifactManifestEntry(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ PATH_FIELD_NUMBER: builtins.int
+ DIGEST_FIELD_NUMBER: builtins.int
+ REF_FIELD_NUMBER: builtins.int
+ SIZE_FIELD_NUMBER: builtins.int
+ MIMETYPE_FIELD_NUMBER: builtins.int
+ LOCAL_PATH_FIELD_NUMBER: builtins.int
+ BIRTH_ARTIFACT_ID_FIELD_NUMBER: builtins.int
+ EXTRA_FIELD_NUMBER: builtins.int
+ path: builtins.str
+ digest: builtins.str
+ ref: builtins.str
+ size: builtins.int
+ mimetype: builtins.str
+ local_path: builtins.str
+ birth_artifact_id: builtins.str
+ @property
+ def extra(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ExtraItem]: ...
+ def __init__(
+ self,
+ *,
+ path: builtins.str = ...,
+ digest: builtins.str = ...,
+ ref: builtins.str = ...,
+ size: builtins.int = ...,
+ mimetype: builtins.str = ...,
+ local_path: builtins.str = ...,
+ birth_artifact_id: builtins.str = ...,
+ extra: collections.abc.Iterable[global___ExtraItem] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["birth_artifact_id", b"birth_artifact_id", "digest", b"digest", "extra", b"extra", "local_path", b"local_path", "mimetype", b"mimetype", "path", b"path", "ref", b"ref", "size", b"size"]) -> None: ...
+
+global___ArtifactManifestEntry = ArtifactManifestEntry
+
+class ExtraItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_JSON_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ value_json: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value_json: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value_json", b"value_json"]) -> None: ...
+
+global___ExtraItem = ExtraItem
+
+class StoragePolicyConfigItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_JSON_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ value_json: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value_json: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value_json", b"value_json"]) -> None: ...
+
+global___StoragePolicyConfigItem = StoragePolicyConfigItem
+
+class ArtifactResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ArtifactResult = ArtifactResult
+
+class LinkArtifactResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___LinkArtifactResult = LinkArtifactResult
+
+class LinkArtifactRecord(google.protobuf.message.Message):
+ """
+ LinkArtifactRecord: link artifact to portfolio
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ CLIENT_ID_FIELD_NUMBER: builtins.int
+ SERVER_ID_FIELD_NUMBER: builtins.int
+ PORTFOLIO_NAME_FIELD_NUMBER: builtins.int
+ PORTFOLIO_ENTITY_FIELD_NUMBER: builtins.int
+ PORTFOLIO_PROJECT_FIELD_NUMBER: builtins.int
+ PORTFOLIO_ALIASES_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ client_id: builtins.str
+ server_id: builtins.str
+ portfolio_name: builtins.str
+ portfolio_entity: builtins.str
+ portfolio_project: builtins.str
+ @property
+ def portfolio_aliases(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ client_id: builtins.str = ...,
+ server_id: builtins.str = ...,
+ portfolio_name: builtins.str = ...,
+ portfolio_entity: builtins.str = ...,
+ portfolio_project: builtins.str = ...,
+ portfolio_aliases: collections.abc.Iterable[builtins.str] | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "client_id", b"client_id", "portfolio_aliases", b"portfolio_aliases", "portfolio_entity", b"portfolio_entity", "portfolio_name", b"portfolio_name", "portfolio_project", b"portfolio_project", "server_id", b"server_id"]) -> None: ...
+
+global___LinkArtifactRecord = LinkArtifactRecord
+
+class TBRecord(google.protobuf.message.Message):
+ """
+ TBRecord: store tb locations
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ LOG_DIR_FIELD_NUMBER: builtins.int
+ SAVE_FIELD_NUMBER: builtins.int
+ ROOT_DIR_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ log_dir: builtins.str
+ save: builtins.bool
+ root_dir: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ log_dir: builtins.str = ...,
+ save: builtins.bool = ...,
+ root_dir: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "log_dir", b"log_dir", "root_dir", b"root_dir", "save", b"save"]) -> None: ...
+
+global___TBRecord = TBRecord
+
+class TBResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___TBResult = TBResult
+
+class AlertRecord(google.protobuf.message.Message):
+ """
+ AlertRecord: store alert notifications
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ TITLE_FIELD_NUMBER: builtins.int
+ TEXT_FIELD_NUMBER: builtins.int
+ LEVEL_FIELD_NUMBER: builtins.int
+ WAIT_DURATION_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ title: builtins.str
+ text: builtins.str
+ level: builtins.str
+ wait_duration: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ title: builtins.str = ...,
+ text: builtins.str = ...,
+ level: builtins.str = ...,
+ wait_duration: builtins.int = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "level", b"level", "text", b"text", "title", b"title", "wait_duration", b"wait_duration"]) -> None: ...
+
+global___AlertRecord = AlertRecord
+
+class AlertResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___AlertResult = AlertResult
+
+class Request(google.protobuf.message.Message):
+ """***********************
+ Requests and Responses
+ **********************
+
+
+ Request: all non persistent messages
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ STOP_STATUS_FIELD_NUMBER: builtins.int
+ NETWORK_STATUS_FIELD_NUMBER: builtins.int
+ DEFER_FIELD_NUMBER: builtins.int
+ GET_SUMMARY_FIELD_NUMBER: builtins.int
+ LOGIN_FIELD_NUMBER: builtins.int
+ PAUSE_FIELD_NUMBER: builtins.int
+ RESUME_FIELD_NUMBER: builtins.int
+ POLL_EXIT_FIELD_NUMBER: builtins.int
+ SAMPLED_HISTORY_FIELD_NUMBER: builtins.int
+ PARTIAL_HISTORY_FIELD_NUMBER: builtins.int
+ RUN_START_FIELD_NUMBER: builtins.int
+ CHECK_VERSION_FIELD_NUMBER: builtins.int
+ LOG_ARTIFACT_FIELD_NUMBER: builtins.int
+ DOWNLOAD_ARTIFACT_FIELD_NUMBER: builtins.int
+ KEEPALIVE_FIELD_NUMBER: builtins.int
+ RUN_STATUS_FIELD_NUMBER: builtins.int
+ CANCEL_FIELD_NUMBER: builtins.int
+ METADATA_FIELD_NUMBER: builtins.int
+ INTERNAL_MESSAGES_FIELD_NUMBER: builtins.int
+ PYTHON_PACKAGES_FIELD_NUMBER: builtins.int
+ SHUTDOWN_FIELD_NUMBER: builtins.int
+ ATTACH_FIELD_NUMBER: builtins.int
+ STATUS_FIELD_NUMBER: builtins.int
+ SERVER_INFO_FIELD_NUMBER: builtins.int
+ SENDER_MARK_FIELD_NUMBER: builtins.int
+ SENDER_READ_FIELD_NUMBER: builtins.int
+ STATUS_REPORT_FIELD_NUMBER: builtins.int
+ SUMMARY_RECORD_FIELD_NUMBER: builtins.int
+ TELEMETRY_RECORD_FIELD_NUMBER: builtins.int
+ JOB_INFO_FIELD_NUMBER: builtins.int
+ GET_SYSTEM_METRICS_FIELD_NUMBER: builtins.int
+ FILE_TRANSFER_INFO_FIELD_NUMBER: builtins.int
+ SYNC_FIELD_NUMBER: builtins.int
+ TEST_INJECT_FIELD_NUMBER: builtins.int
+ @property
+ def stop_status(self) -> global___StopStatusRequest: ...
+ @property
+ def network_status(self) -> global___NetworkStatusRequest: ...
+ @property
+ def defer(self) -> global___DeferRequest: ...
+ @property
+ def get_summary(self) -> global___GetSummaryRequest: ...
+ @property
+ def login(self) -> global___LoginRequest: ...
+ @property
+ def pause(self) -> global___PauseRequest: ...
+ @property
+ def resume(self) -> global___ResumeRequest: ...
+ @property
+ def poll_exit(self) -> global___PollExitRequest: ...
+ @property
+ def sampled_history(self) -> global___SampledHistoryRequest: ...
+ @property
+ def partial_history(self) -> global___PartialHistoryRequest: ...
+ @property
+ def run_start(self) -> global___RunStartRequest: ...
+ @property
+ def check_version(self) -> global___CheckVersionRequest: ...
+ @property
+ def log_artifact(self) -> global___LogArtifactRequest: ...
+ @property
+ def download_artifact(self) -> global___DownloadArtifactRequest: ...
+ @property
+ def keepalive(self) -> global___KeepaliveRequest: ...
+ @property
+ def run_status(self) -> global___RunStatusRequest: ...
+ @property
+ def cancel(self) -> global___CancelRequest: ...
+ @property
+ def metadata(self) -> global___MetadataRequest: ...
+ @property
+ def internal_messages(self) -> global___InternalMessagesRequest: ...
+ @property
+ def python_packages(self) -> global___PythonPackagesRequest: ...
+ @property
+ def shutdown(self) -> global___ShutdownRequest: ...
+ @property
+ def attach(self) -> global___AttachRequest: ...
+ @property
+ def status(self) -> global___StatusRequest: ...
+ @property
+ def server_info(self) -> global___ServerInfoRequest: ...
+ @property
+ def sender_mark(self) -> global___SenderMarkRequest: ...
+ @property
+ def sender_read(self) -> global___SenderReadRequest: ...
+ @property
+ def status_report(self) -> global___StatusReportRequest: ...
+ @property
+ def summary_record(self) -> global___SummaryRecordRequest: ...
+ @property
+ def telemetry_record(self) -> global___TelemetryRecordRequest: ...
+ @property
+ def job_info(self) -> global___JobInfoRequest: ...
+ @property
+ def get_system_metrics(self) -> global___GetSystemMetricsRequest: ...
+ @property
+ def file_transfer_info(self) -> global___FileTransferInfoRequest: ...
+ @property
+ def sync(self) -> global___SyncRequest: ...
+ @property
+ def test_inject(self) -> global___TestInjectRequest: ...
+ def __init__(
+ self,
+ *,
+ stop_status: global___StopStatusRequest | None = ...,
+ network_status: global___NetworkStatusRequest | None = ...,
+ defer: global___DeferRequest | None = ...,
+ get_summary: global___GetSummaryRequest | None = ...,
+ login: global___LoginRequest | None = ...,
+ pause: global___PauseRequest | None = ...,
+ resume: global___ResumeRequest | None = ...,
+ poll_exit: global___PollExitRequest | None = ...,
+ sampled_history: global___SampledHistoryRequest | None = ...,
+ partial_history: global___PartialHistoryRequest | None = ...,
+ run_start: global___RunStartRequest | None = ...,
+ check_version: global___CheckVersionRequest | None = ...,
+ log_artifact: global___LogArtifactRequest | None = ...,
+ download_artifact: global___DownloadArtifactRequest | None = ...,
+ keepalive: global___KeepaliveRequest | None = ...,
+ run_status: global___RunStatusRequest | None = ...,
+ cancel: global___CancelRequest | None = ...,
+ metadata: global___MetadataRequest | None = ...,
+ internal_messages: global___InternalMessagesRequest | None = ...,
+ python_packages: global___PythonPackagesRequest | None = ...,
+ shutdown: global___ShutdownRequest | None = ...,
+ attach: global___AttachRequest | None = ...,
+ status: global___StatusRequest | None = ...,
+ server_info: global___ServerInfoRequest | None = ...,
+ sender_mark: global___SenderMarkRequest | None = ...,
+ sender_read: global___SenderReadRequest | None = ...,
+ status_report: global___StatusReportRequest | None = ...,
+ summary_record: global___SummaryRecordRequest | None = ...,
+ telemetry_record: global___TelemetryRecordRequest | None = ...,
+ job_info: global___JobInfoRequest | None = ...,
+ get_system_metrics: global___GetSystemMetricsRequest | None = ...,
+ file_transfer_info: global___FileTransferInfoRequest | None = ...,
+ sync: global___SyncRequest | None = ...,
+ test_inject: global___TestInjectRequest | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["attach", b"attach", "cancel", b"cancel", "check_version", b"check_version", "defer", b"defer", "download_artifact", b"download_artifact", "file_transfer_info", b"file_transfer_info", "get_summary", b"get_summary", "get_system_metrics", b"get_system_metrics", "internal_messages", b"internal_messages", "job_info", b"job_info", "keepalive", b"keepalive", "log_artifact", b"log_artifact", "login", b"login", "metadata", b"metadata", "network_status", b"network_status", "partial_history", b"partial_history", "pause", b"pause", "poll_exit", b"poll_exit", "python_packages", b"python_packages", "request_type", b"request_type", "resume", b"resume", "run_start", b"run_start", "run_status", b"run_status", "sampled_history", b"sampled_history", "sender_mark", b"sender_mark", "sender_read", b"sender_read", "server_info", b"server_info", "shutdown", b"shutdown", "status", b"status", "status_report", b"status_report", "stop_status", b"stop_status", "summary_record", b"summary_record", "sync", b"sync", "telemetry_record", b"telemetry_record", "test_inject", b"test_inject"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["attach", b"attach", "cancel", b"cancel", "check_version", b"check_version", "defer", b"defer", "download_artifact", b"download_artifact", "file_transfer_info", b"file_transfer_info", "get_summary", b"get_summary", "get_system_metrics", b"get_system_metrics", "internal_messages", b"internal_messages", "job_info", b"job_info", "keepalive", b"keepalive", "log_artifact", b"log_artifact", "login", b"login", "metadata", b"metadata", "network_status", b"network_status", "partial_history", b"partial_history", "pause", b"pause", "poll_exit", b"poll_exit", "python_packages", b"python_packages", "request_type", b"request_type", "resume", b"resume", "run_start", b"run_start", "run_status", b"run_status", "sampled_history", b"sampled_history", "sender_mark", b"sender_mark", "sender_read", b"sender_read", "server_info", b"server_info", "shutdown", b"shutdown", "status", b"status", "status_report", b"status_report", "stop_status", b"stop_status", "summary_record", b"summary_record", "sync", b"sync", "telemetry_record", b"telemetry_record", "test_inject", b"test_inject"]) -> None: ...
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["request_type", b"request_type"]) -> typing_extensions.Literal["stop_status", "network_status", "defer", "get_summary", "login", "pause", "resume", "poll_exit", "sampled_history", "partial_history", "run_start", "check_version", "log_artifact", "download_artifact", "keepalive", "run_status", "cancel", "metadata", "internal_messages", "python_packages", "shutdown", "attach", "status", "server_info", "sender_mark", "sender_read", "status_report", "summary_record", "telemetry_record", "job_info", "get_system_metrics", "file_transfer_info", "sync", "test_inject"] | None: ...
+
+global___Request = Request
+
+class Response(google.protobuf.message.Message):
+ """
+ Response: all non persistent responses to Requests
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEEPALIVE_RESPONSE_FIELD_NUMBER: builtins.int
+ STOP_STATUS_RESPONSE_FIELD_NUMBER: builtins.int
+ NETWORK_STATUS_RESPONSE_FIELD_NUMBER: builtins.int
+ LOGIN_RESPONSE_FIELD_NUMBER: builtins.int
+ GET_SUMMARY_RESPONSE_FIELD_NUMBER: builtins.int
+ POLL_EXIT_RESPONSE_FIELD_NUMBER: builtins.int
+ SAMPLED_HISTORY_RESPONSE_FIELD_NUMBER: builtins.int
+ RUN_START_RESPONSE_FIELD_NUMBER: builtins.int
+ CHECK_VERSION_RESPONSE_FIELD_NUMBER: builtins.int
+ LOG_ARTIFACT_RESPONSE_FIELD_NUMBER: builtins.int
+ DOWNLOAD_ARTIFACT_RESPONSE_FIELD_NUMBER: builtins.int
+ RUN_STATUS_RESPONSE_FIELD_NUMBER: builtins.int
+ CANCEL_RESPONSE_FIELD_NUMBER: builtins.int
+ INTERNAL_MESSAGES_RESPONSE_FIELD_NUMBER: builtins.int
+ SHUTDOWN_RESPONSE_FIELD_NUMBER: builtins.int
+ ATTACH_RESPONSE_FIELD_NUMBER: builtins.int
+ STATUS_RESPONSE_FIELD_NUMBER: builtins.int
+ SERVER_INFO_RESPONSE_FIELD_NUMBER: builtins.int
+ JOB_INFO_RESPONSE_FIELD_NUMBER: builtins.int
+ GET_SYSTEM_METRICS_RESPONSE_FIELD_NUMBER: builtins.int
+ SYNC_RESPONSE_FIELD_NUMBER: builtins.int
+ TEST_INJECT_RESPONSE_FIELD_NUMBER: builtins.int
+ @property
+ def keepalive_response(self) -> global___KeepaliveResponse: ...
+ @property
+ def stop_status_response(self) -> global___StopStatusResponse: ...
+ @property
+ def network_status_response(self) -> global___NetworkStatusResponse: ...
+ @property
+ def login_response(self) -> global___LoginResponse: ...
+ @property
+ def get_summary_response(self) -> global___GetSummaryResponse: ...
+ @property
+ def poll_exit_response(self) -> global___PollExitResponse: ...
+ @property
+ def sampled_history_response(self) -> global___SampledHistoryResponse: ...
+ @property
+ def run_start_response(self) -> global___RunStartResponse: ...
+ @property
+ def check_version_response(self) -> global___CheckVersionResponse: ...
+ @property
+ def log_artifact_response(self) -> global___LogArtifactResponse: ...
+ @property
+ def download_artifact_response(self) -> global___DownloadArtifactResponse: ...
+ @property
+ def run_status_response(self) -> global___RunStatusResponse: ...
+ @property
+ def cancel_response(self) -> global___CancelResponse: ...
+ @property
+ def internal_messages_response(self) -> global___InternalMessagesResponse: ...
+ @property
+ def shutdown_response(self) -> global___ShutdownResponse: ...
+ @property
+ def attach_response(self) -> global___AttachResponse: ...
+ @property
+ def status_response(self) -> global___StatusResponse: ...
+ @property
+ def server_info_response(self) -> global___ServerInfoResponse: ...
+ @property
+ def job_info_response(self) -> global___JobInfoResponse: ...
+ @property
+ def get_system_metrics_response(self) -> global___GetSystemMetricsResponse: ...
+ @property
+ def sync_response(self) -> global___SyncResponse: ...
+ @property
+ def test_inject_response(self) -> global___TestInjectResponse: ...
+ def __init__(
+ self,
+ *,
+ keepalive_response: global___KeepaliveResponse | None = ...,
+ stop_status_response: global___StopStatusResponse | None = ...,
+ network_status_response: global___NetworkStatusResponse | None = ...,
+ login_response: global___LoginResponse | None = ...,
+ get_summary_response: global___GetSummaryResponse | None = ...,
+ poll_exit_response: global___PollExitResponse | None = ...,
+ sampled_history_response: global___SampledHistoryResponse | None = ...,
+ run_start_response: global___RunStartResponse | None = ...,
+ check_version_response: global___CheckVersionResponse | None = ...,
+ log_artifact_response: global___LogArtifactResponse | None = ...,
+ download_artifact_response: global___DownloadArtifactResponse | None = ...,
+ run_status_response: global___RunStatusResponse | None = ...,
+ cancel_response: global___CancelResponse | None = ...,
+ internal_messages_response: global___InternalMessagesResponse | None = ...,
+ shutdown_response: global___ShutdownResponse | None = ...,
+ attach_response: global___AttachResponse | None = ...,
+ status_response: global___StatusResponse | None = ...,
+ server_info_response: global___ServerInfoResponse | None = ...,
+ job_info_response: global___JobInfoResponse | None = ...,
+ get_system_metrics_response: global___GetSystemMetricsResponse | None = ...,
+ sync_response: global___SyncResponse | None = ...,
+ test_inject_response: global___TestInjectResponse | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["attach_response", b"attach_response", "cancel_response", b"cancel_response", "check_version_response", b"check_version_response", "download_artifact_response", b"download_artifact_response", "get_summary_response", b"get_summary_response", "get_system_metrics_response", b"get_system_metrics_response", "internal_messages_response", b"internal_messages_response", "job_info_response", b"job_info_response", "keepalive_response", b"keepalive_response", "log_artifact_response", b"log_artifact_response", "login_response", b"login_response", "network_status_response", b"network_status_response", "poll_exit_response", b"poll_exit_response", "response_type", b"response_type", "run_start_response", b"run_start_response", "run_status_response", b"run_status_response", "sampled_history_response", b"sampled_history_response", "server_info_response", b"server_info_response", "shutdown_response", b"shutdown_response", "status_response", b"status_response", "stop_status_response", b"stop_status_response", "sync_response", b"sync_response", "test_inject_response", b"test_inject_response"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["attach_response", b"attach_response", "cancel_response", b"cancel_response", "check_version_response", b"check_version_response", "download_artifact_response", b"download_artifact_response", "get_summary_response", b"get_summary_response", "get_system_metrics_response", b"get_system_metrics_response", "internal_messages_response", b"internal_messages_response", "job_info_response", b"job_info_response", "keepalive_response", b"keepalive_response", "log_artifact_response", b"log_artifact_response", "login_response", b"login_response", "network_status_response", b"network_status_response", "poll_exit_response", b"poll_exit_response", "response_type", b"response_type", "run_start_response", b"run_start_response", "run_status_response", b"run_status_response", "sampled_history_response", b"sampled_history_response", "server_info_response", b"server_info_response", "shutdown_response", b"shutdown_response", "status_response", b"status_response", "stop_status_response", b"stop_status_response", "sync_response", b"sync_response", "test_inject_response", b"test_inject_response"]) -> None: ...
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["response_type", b"response_type"]) -> typing_extensions.Literal["keepalive_response", "stop_status_response", "network_status_response", "login_response", "get_summary_response", "poll_exit_response", "sampled_history_response", "run_start_response", "check_version_response", "log_artifact_response", "download_artifact_response", "run_status_response", "cancel_response", "internal_messages_response", "shutdown_response", "attach_response", "status_response", "server_info_response", "job_info_response", "get_system_metrics_response", "sync_response", "test_inject_response"] | None: ...
+
+global___Response = Response
+
+class DeferRequest(google.protobuf.message.Message):
+ """
+ DeferRequest: internal message to defer work
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _DeferState:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _DeferStateEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[DeferRequest._DeferState.ValueType], builtins.type): # noqa: F821
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ BEGIN: DeferRequest._DeferState.ValueType # 0
+ FLUSH_RUN: DeferRequest._DeferState.ValueType # 1
+ FLUSH_STATS: DeferRequest._DeferState.ValueType # 2
+ FLUSH_PARTIAL_HISTORY: DeferRequest._DeferState.ValueType # 3
+ FLUSH_TB: DeferRequest._DeferState.ValueType # 4
+ FLUSH_SUM: DeferRequest._DeferState.ValueType # 5
+ FLUSH_DEBOUNCER: DeferRequest._DeferState.ValueType # 6
+ FLUSH_OUTPUT: DeferRequest._DeferState.ValueType # 7
+ FLUSH_JOB: DeferRequest._DeferState.ValueType # 8
+ FLUSH_DIR: DeferRequest._DeferState.ValueType # 9
+ FLUSH_FP: DeferRequest._DeferState.ValueType # 10
+ JOIN_FP: DeferRequest._DeferState.ValueType # 11
+ FLUSH_FS: DeferRequest._DeferState.ValueType # 12
+ FLUSH_FINAL: DeferRequest._DeferState.ValueType # 13
+ END: DeferRequest._DeferState.ValueType # 14
+
+ class DeferState(_DeferState, metaclass=_DeferStateEnumTypeWrapper): ...
+ BEGIN: DeferRequest.DeferState.ValueType # 0
+ FLUSH_RUN: DeferRequest.DeferState.ValueType # 1
+ FLUSH_STATS: DeferRequest.DeferState.ValueType # 2
+ FLUSH_PARTIAL_HISTORY: DeferRequest.DeferState.ValueType # 3
+ FLUSH_TB: DeferRequest.DeferState.ValueType # 4
+ FLUSH_SUM: DeferRequest.DeferState.ValueType # 5
+ FLUSH_DEBOUNCER: DeferRequest.DeferState.ValueType # 6
+ FLUSH_OUTPUT: DeferRequest.DeferState.ValueType # 7
+ FLUSH_JOB: DeferRequest.DeferState.ValueType # 8
+ FLUSH_DIR: DeferRequest.DeferState.ValueType # 9
+ FLUSH_FP: DeferRequest.DeferState.ValueType # 10
+ JOIN_FP: DeferRequest.DeferState.ValueType # 11
+ FLUSH_FS: DeferRequest.DeferState.ValueType # 12
+ FLUSH_FINAL: DeferRequest.DeferState.ValueType # 13
+ END: DeferRequest.DeferState.ValueType # 14
+
+ STATE_FIELD_NUMBER: builtins.int
+ state: global___DeferRequest.DeferState.ValueType
+ """Internal message, no _info field needed"""
+ def __init__(
+ self,
+ *,
+ state: global___DeferRequest.DeferState.ValueType = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["state", b"state"]) -> None: ...
+
+global___DeferRequest = DeferRequest
+
+class PauseRequest(google.protobuf.message.Message):
+ """
+ PauseRequest: internal message to pause the heartbeat
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___PauseRequest = PauseRequest
+
+class PauseResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___PauseResponse = PauseResponse
+
+class ResumeRequest(google.protobuf.message.Message):
+ """
+ ResumeRequest: internal message to resume the heartbeat
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ResumeRequest = ResumeRequest
+
+class ResumeResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ResumeResponse = ResumeResponse
+
+class LoginRequest(google.protobuf.message.Message):
+ """
+ LoginRequest: wandb/sdk/wandb_login
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ API_KEY_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ api_key: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ api_key: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "api_key", b"api_key"]) -> None: ...
+
+global___LoginRequest = LoginRequest
+
+class LoginResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ACTIVE_ENTITY_FIELD_NUMBER: builtins.int
+ active_entity: builtins.str
+ def __init__(
+ self,
+ *,
+ active_entity: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["active_entity", b"active_entity"]) -> None: ...
+
+global___LoginResponse = LoginResponse
+
+class GetSummaryRequest(google.protobuf.message.Message):
+ """
+ GetSummaryRequest: request consolidated summary
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___GetSummaryRequest = GetSummaryRequest
+
+class GetSummaryResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ITEM_FIELD_NUMBER: builtins.int
+ @property
+ def item(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___SummaryItem]: ...
+ def __init__(
+ self,
+ *,
+ item: collections.abc.Iterable[global___SummaryItem] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["item", b"item"]) -> None: ...
+
+global___GetSummaryResponse = GetSummaryResponse
+
+class GetSystemMetricsRequest(google.protobuf.message.Message):
+ """
+ GetSystemMetrics: request system metrics
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___GetSystemMetricsRequest = GetSystemMetricsRequest
+
+class SystemMetricSample(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ TIMESTAMP_FIELD_NUMBER: builtins.int
+ VALUE_FIELD_NUMBER: builtins.int
+ @property
+ def timestamp(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ value: builtins.float
+ def __init__(
+ self,
+ *,
+ timestamp: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ value: builtins.float = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["timestamp", b"timestamp"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["timestamp", b"timestamp", "value", b"value"]) -> None: ...
+
+global___SystemMetricSample = SystemMetricSample
+
+class SystemMetricsBuffer(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RECORD_FIELD_NUMBER: builtins.int
+ @property
+ def record(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___SystemMetricSample]: ...
+ def __init__(
+ self,
+ *,
+ record: collections.abc.Iterable[global___SystemMetricSample] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["record", b"record"]) -> None: ...
+
+global___SystemMetricsBuffer = SystemMetricsBuffer
+
+class GetSystemMetricsResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class SystemMetricsEntry(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ @property
+ def value(self) -> global___SystemMetricsBuffer: ...
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value: global___SystemMetricsBuffer | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["value", b"value"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ...
+
+ SYSTEM_METRICS_FIELD_NUMBER: builtins.int
+ @property
+ def system_metrics(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___SystemMetricsBuffer]: ...
+ def __init__(
+ self,
+ *,
+ system_metrics: collections.abc.Mapping[builtins.str, global___SystemMetricsBuffer] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["system_metrics", b"system_metrics"]) -> None: ...
+
+global___GetSystemMetricsResponse = GetSystemMetricsResponse
+
+class StatusRequest(google.protobuf.message.Message):
+ """
+ StatusRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___StatusRequest = StatusRequest
+
+class StatusResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_SHOULD_STOP_FIELD_NUMBER: builtins.int
+ run_should_stop: builtins.bool
+ def __init__(
+ self,
+ *,
+ run_should_stop: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["run_should_stop", b"run_should_stop"]) -> None: ...
+
+global___StatusResponse = StatusResponse
+
+class StopStatusRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___StopStatusRequest = StopStatusRequest
+
+class StopStatusResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_SHOULD_STOP_FIELD_NUMBER: builtins.int
+ run_should_stop: builtins.bool
+ def __init__(
+ self,
+ *,
+ run_should_stop: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["run_should_stop", b"run_should_stop"]) -> None: ...
+
+global___StopStatusResponse = StopStatusResponse
+
+class NetworkStatusRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___NetworkStatusRequest = NetworkStatusRequest
+
+class NetworkStatusResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ NETWORK_RESPONSES_FIELD_NUMBER: builtins.int
+ @property
+ def network_responses(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___HttpResponse]: ...
+ def __init__(
+ self,
+ *,
+ network_responses: collections.abc.Iterable[global___HttpResponse] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["network_responses", b"network_responses"]) -> None: ...
+
+global___NetworkStatusResponse = NetworkStatusResponse
+
+class HttpResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ HTTP_STATUS_CODE_FIELD_NUMBER: builtins.int
+ HTTP_RESPONSE_TEXT_FIELD_NUMBER: builtins.int
+ http_status_code: builtins.int
+ http_response_text: builtins.str
+ def __init__(
+ self,
+ *,
+ http_status_code: builtins.int = ...,
+ http_response_text: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["http_response_text", b"http_response_text", "http_status_code", b"http_status_code"]) -> None: ...
+
+global___HttpResponse = HttpResponse
+
+class InternalMessagesRequest(google.protobuf.message.Message):
+ """
+ InternalMessagesRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___InternalMessagesRequest = InternalMessagesRequest
+
+class InternalMessagesResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ MESSAGES_FIELD_NUMBER: builtins.int
+ @property
+ def messages(self) -> global___InternalMessages: ...
+ def __init__(
+ self,
+ *,
+ messages: global___InternalMessages | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["messages", b"messages"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["messages", b"messages"]) -> None: ...
+
+global___InternalMessagesResponse = InternalMessagesResponse
+
+class InternalMessages(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ WARNING_FIELD_NUMBER: builtins.int
+ @property
+ def warning(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ def __init__(
+ self,
+ *,
+ warning: collections.abc.Iterable[builtins.str] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["warning", b"warning"]) -> None: ...
+
+global___InternalMessages = InternalMessages
+
+class PollExitRequest(google.protobuf.message.Message):
+ """
+ PollExitRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___PollExitRequest = PollExitRequest
+
+class PollExitResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ DONE_FIELD_NUMBER: builtins.int
+ EXIT_RESULT_FIELD_NUMBER: builtins.int
+ PUSHER_STATS_FIELD_NUMBER: builtins.int
+ FILE_COUNTS_FIELD_NUMBER: builtins.int
+ done: builtins.bool
+ @property
+ def exit_result(self) -> global___RunExitResult: ...
+ @property
+ def pusher_stats(self) -> global___FilePusherStats: ...
+ @property
+ def file_counts(self) -> global___FileCounts: ...
+ def __init__(
+ self,
+ *,
+ done: builtins.bool = ...,
+ exit_result: global___RunExitResult | None = ...,
+ pusher_stats: global___FilePusherStats | None = ...,
+ file_counts: global___FileCounts | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["exit_result", b"exit_result", "file_counts", b"file_counts", "pusher_stats", b"pusher_stats"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["done", b"done", "exit_result", b"exit_result", "file_counts", b"file_counts", "pusher_stats", b"pusher_stats"]) -> None: ...
+
+global___PollExitResponse = PollExitResponse
+
+class SyncOverwrite(google.protobuf.message.Message):
+ """
+ Sender requests
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_ID_FIELD_NUMBER: builtins.int
+ ENTITY_FIELD_NUMBER: builtins.int
+ PROJECT_FIELD_NUMBER: builtins.int
+ run_id: builtins.str
+ entity: builtins.str
+ project: builtins.str
+ def __init__(
+ self,
+ *,
+ run_id: builtins.str = ...,
+ entity: builtins.str = ...,
+ project: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["entity", b"entity", "project", b"project", "run_id", b"run_id"]) -> None: ...
+
+global___SyncOverwrite = SyncOverwrite
+
+class SyncSkip(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ OUTPUT_RAW_FIELD_NUMBER: builtins.int
+ output_raw: builtins.bool
+ def __init__(
+ self,
+ *,
+ output_raw: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["output_raw", b"output_raw"]) -> None: ...
+
+global___SyncSkip = SyncSkip
+
+class SenderMarkRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___SenderMarkRequest = SenderMarkRequest
+
+class SyncRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ START_OFFSET_FIELD_NUMBER: builtins.int
+ FINAL_OFFSET_FIELD_NUMBER: builtins.int
+ OVERWRITE_FIELD_NUMBER: builtins.int
+ SKIP_FIELD_NUMBER: builtins.int
+ start_offset: builtins.int
+ final_offset: builtins.int
+ @property
+ def overwrite(self) -> global___SyncOverwrite: ...
+ @property
+ def skip(self) -> global___SyncSkip: ...
+ def __init__(
+ self,
+ *,
+ start_offset: builtins.int = ...,
+ final_offset: builtins.int = ...,
+ overwrite: global___SyncOverwrite | None = ...,
+ skip: global___SyncSkip | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["overwrite", b"overwrite", "skip", b"skip"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["final_offset", b"final_offset", "overwrite", b"overwrite", "skip", b"skip", "start_offset", b"start_offset"]) -> None: ...
+
+global___SyncRequest = SyncRequest
+
+class SyncResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ URL_FIELD_NUMBER: builtins.int
+ ERROR_FIELD_NUMBER: builtins.int
+ url: builtins.str
+ @property
+ def error(self) -> global___ErrorInfo: ...
+ def __init__(
+ self,
+ *,
+ url: builtins.str = ...,
+ error: global___ErrorInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["error", b"error"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["error", b"error", "url", b"url"]) -> None: ...
+
+global___SyncResponse = SyncResponse
+
+class SenderReadRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ START_OFFSET_FIELD_NUMBER: builtins.int
+ FINAL_OFFSET_FIELD_NUMBER: builtins.int
+ start_offset: builtins.int
+ final_offset: builtins.int
+ """TODO: implement cancel for paused ops
+ repeated string cancel_list = 3;
+ """
+ def __init__(
+ self,
+ *,
+ start_offset: builtins.int = ...,
+ final_offset: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["final_offset", b"final_offset", "start_offset", b"start_offset"]) -> None: ...
+
+global___SenderReadRequest = SenderReadRequest
+
+class StatusReportRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RECORD_NUM_FIELD_NUMBER: builtins.int
+ SENT_OFFSET_FIELD_NUMBER: builtins.int
+ SYNC_TIME_FIELD_NUMBER: builtins.int
+ record_num: builtins.int
+ sent_offset: builtins.int
+ @property
+ def sync_time(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ def __init__(
+ self,
+ *,
+ record_num: builtins.int = ...,
+ sent_offset: builtins.int = ...,
+ sync_time: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["sync_time", b"sync_time"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["record_num", b"record_num", "sent_offset", b"sent_offset", "sync_time", b"sync_time"]) -> None: ...
+
+global___StatusReportRequest = StatusReportRequest
+
+class SummaryRecordRequest(google.protobuf.message.Message):
+ """
+ Requests wrapping Records
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SUMMARY_FIELD_NUMBER: builtins.int
+ @property
+ def summary(self) -> global___SummaryRecord: ...
+ def __init__(
+ self,
+ *,
+ summary: global___SummaryRecord | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["summary", b"summary"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["summary", b"summary"]) -> None: ...
+
+global___SummaryRecordRequest = SummaryRecordRequest
+
+class TelemetryRecordRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ TELEMETRY_FIELD_NUMBER: builtins.int
+ @property
+ def telemetry(self) -> wandb.proto.wandb_telemetry_pb2.TelemetryRecord: ...
+ def __init__(
+ self,
+ *,
+ telemetry: wandb.proto.wandb_telemetry_pb2.TelemetryRecord | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["telemetry", b"telemetry"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["telemetry", b"telemetry"]) -> None: ...
+
+global___TelemetryRecordRequest = TelemetryRecordRequest
+
+class ServerInfoRequest(google.protobuf.message.Message):
+ """
+ ServerInfoRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ServerInfoRequest = ServerInfoRequest
+
+class ServerInfoResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ LOCAL_INFO_FIELD_NUMBER: builtins.int
+ SERVER_MESSAGES_FIELD_NUMBER: builtins.int
+ @property
+ def local_info(self) -> global___LocalInfo: ...
+ @property
+ def server_messages(self) -> global___ServerMessages: ...
+ def __init__(
+ self,
+ *,
+ local_info: global___LocalInfo | None = ...,
+ server_messages: global___ServerMessages | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["local_info", b"local_info", "server_messages", b"server_messages"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["local_info", b"local_info", "server_messages", b"server_messages"]) -> None: ...
+
+global___ServerInfoResponse = ServerInfoResponse
+
+class ServerMessages(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ITEM_FIELD_NUMBER: builtins.int
+ @property
+ def item(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ServerMessage]: ...
+ def __init__(
+ self,
+ *,
+ item: collections.abc.Iterable[global___ServerMessage] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["item", b"item"]) -> None: ...
+
+global___ServerMessages = ServerMessages
+
+class ServerMessage(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ PLAIN_TEXT_FIELD_NUMBER: builtins.int
+ UTF_TEXT_FIELD_NUMBER: builtins.int
+ HTML_TEXT_FIELD_NUMBER: builtins.int
+ TYPE_FIELD_NUMBER: builtins.int
+ LEVEL_FIELD_NUMBER: builtins.int
+ plain_text: builtins.str
+ utf_text: builtins.str
+ html_text: builtins.str
+ type: builtins.str
+ level: builtins.int
+ def __init__(
+ self,
+ *,
+ plain_text: builtins.str = ...,
+ utf_text: builtins.str = ...,
+ html_text: builtins.str = ...,
+ type: builtins.str = ...,
+ level: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["html_text", b"html_text", "level", b"level", "plain_text", b"plain_text", "type", b"type", "utf_text", b"utf_text"]) -> None: ...
+
+global___ServerMessage = ServerMessage
+
+class FileCounts(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ WANDB_COUNT_FIELD_NUMBER: builtins.int
+ MEDIA_COUNT_FIELD_NUMBER: builtins.int
+ ARTIFACT_COUNT_FIELD_NUMBER: builtins.int
+ OTHER_COUNT_FIELD_NUMBER: builtins.int
+ wandb_count: builtins.int
+ media_count: builtins.int
+ artifact_count: builtins.int
+ other_count: builtins.int
+ def __init__(
+ self,
+ *,
+ wandb_count: builtins.int = ...,
+ media_count: builtins.int = ...,
+ artifact_count: builtins.int = ...,
+ other_count: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["artifact_count", b"artifact_count", "media_count", b"media_count", "other_count", b"other_count", "wandb_count", b"wandb_count"]) -> None: ...
+
+global___FileCounts = FileCounts
+
+class FilePusherStats(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ UPLOADED_BYTES_FIELD_NUMBER: builtins.int
+ TOTAL_BYTES_FIELD_NUMBER: builtins.int
+ DEDUPED_BYTES_FIELD_NUMBER: builtins.int
+ uploaded_bytes: builtins.int
+ total_bytes: builtins.int
+ deduped_bytes: builtins.int
+ def __init__(
+ self,
+ *,
+ uploaded_bytes: builtins.int = ...,
+ total_bytes: builtins.int = ...,
+ deduped_bytes: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["deduped_bytes", b"deduped_bytes", "total_bytes", b"total_bytes", "uploaded_bytes", b"uploaded_bytes"]) -> None: ...
+
+global___FilePusherStats = FilePusherStats
+
+class FilesUploaded(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ FILES_FIELD_NUMBER: builtins.int
+ @property
+ def files(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ def __init__(
+ self,
+ *,
+ files: collections.abc.Iterable[builtins.str] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["files", b"files"]) -> None: ...
+
+global___FilesUploaded = FilesUploaded
+
+class FileTransferInfoRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _TransferType:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _TransferTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[FileTransferInfoRequest._TransferType.ValueType], builtins.type): # noqa: F821
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ Upload: FileTransferInfoRequest._TransferType.ValueType # 0
+ Download: FileTransferInfoRequest._TransferType.ValueType # 1
+
+ class TransferType(_TransferType, metaclass=_TransferTypeEnumTypeWrapper): ...
+ Upload: FileTransferInfoRequest.TransferType.ValueType # 0
+ Download: FileTransferInfoRequest.TransferType.ValueType # 1
+
+ TYPE_FIELD_NUMBER: builtins.int
+ PATH_FIELD_NUMBER: builtins.int
+ URL_FIELD_NUMBER: builtins.int
+ SIZE_FIELD_NUMBER: builtins.int
+ PROCESSED_FIELD_NUMBER: builtins.int
+ FILE_COUNTS_FIELD_NUMBER: builtins.int
+ type: global___FileTransferInfoRequest.TransferType.ValueType
+ path: builtins.str
+ url: builtins.str
+ size: builtins.int
+ processed: builtins.int
+ @property
+ def file_counts(self) -> global___FileCounts: ...
+ def __init__(
+ self,
+ *,
+ type: global___FileTransferInfoRequest.TransferType.ValueType = ...,
+ path: builtins.str = ...,
+ url: builtins.str = ...,
+ size: builtins.int = ...,
+ processed: builtins.int = ...,
+ file_counts: global___FileCounts | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["file_counts", b"file_counts"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["file_counts", b"file_counts", "path", b"path", "processed", b"processed", "size", b"size", "type", b"type", "url", b"url"]) -> None: ...
+
+global___FileTransferInfoRequest = FileTransferInfoRequest
+
+class LocalInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ VERSION_FIELD_NUMBER: builtins.int
+ OUT_OF_DATE_FIELD_NUMBER: builtins.int
+ version: builtins.str
+ out_of_date: builtins.bool
+ def __init__(
+ self,
+ *,
+ version: builtins.str = ...,
+ out_of_date: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["out_of_date", b"out_of_date", "version", b"version"]) -> None: ...
+
+global___LocalInfo = LocalInfo
+
+class ShutdownRequest(google.protobuf.message.Message):
+ """
+ ShutdownRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ShutdownRequest = ShutdownRequest
+
+class ShutdownResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ShutdownResponse = ShutdownResponse
+
+class AttachRequest(google.protobuf.message.Message):
+ """
+ AttachRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ATTACH_ID_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ attach_id: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ attach_id: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "attach_id", b"attach_id"]) -> None: ...
+
+global___AttachRequest = AttachRequest
+
+class AttachResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_FIELD_NUMBER: builtins.int
+ ERROR_FIELD_NUMBER: builtins.int
+ @property
+ def run(self) -> global___RunRecord: ...
+ @property
+ def error(self) -> global___ErrorInfo: ...
+ def __init__(
+ self,
+ *,
+ run: global___RunRecord | None = ...,
+ error: global___ErrorInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["error", b"error", "run", b"run"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["error", b"error", "run", b"run"]) -> None: ...
+
+global___AttachResponse = AttachResponse
+
+class TestInjectRequest(google.protobuf.message.Message):
+ """
+ TestInjectRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ HANDLER_EXC_FIELD_NUMBER: builtins.int
+ HANDLER_EXIT_FIELD_NUMBER: builtins.int
+ HANDLER_ABORT_FIELD_NUMBER: builtins.int
+ SENDER_EXC_FIELD_NUMBER: builtins.int
+ SENDER_EXIT_FIELD_NUMBER: builtins.int
+ SENDER_ABORT_FIELD_NUMBER: builtins.int
+ REQ_EXC_FIELD_NUMBER: builtins.int
+ REQ_EXIT_FIELD_NUMBER: builtins.int
+ REQ_ABORT_FIELD_NUMBER: builtins.int
+ RESP_EXC_FIELD_NUMBER: builtins.int
+ RESP_EXIT_FIELD_NUMBER: builtins.int
+ RESP_ABORT_FIELD_NUMBER: builtins.int
+ MSG_DROP_FIELD_NUMBER: builtins.int
+ MSG_HANG_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ handler_exc: builtins.bool
+ handler_exit: builtins.bool
+ handler_abort: builtins.bool
+ sender_exc: builtins.bool
+ sender_exit: builtins.bool
+ sender_abort: builtins.bool
+ req_exc: builtins.bool
+ req_exit: builtins.bool
+ req_abort: builtins.bool
+ resp_exc: builtins.bool
+ resp_exit: builtins.bool
+ resp_abort: builtins.bool
+ msg_drop: builtins.bool
+ msg_hang: builtins.bool
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ handler_exc: builtins.bool = ...,
+ handler_exit: builtins.bool = ...,
+ handler_abort: builtins.bool = ...,
+ sender_exc: builtins.bool = ...,
+ sender_exit: builtins.bool = ...,
+ sender_abort: builtins.bool = ...,
+ req_exc: builtins.bool = ...,
+ req_exit: builtins.bool = ...,
+ req_abort: builtins.bool = ...,
+ resp_exc: builtins.bool = ...,
+ resp_exit: builtins.bool = ...,
+ resp_abort: builtins.bool = ...,
+ msg_drop: builtins.bool = ...,
+ msg_hang: builtins.bool = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "handler_abort", b"handler_abort", "handler_exc", b"handler_exc", "handler_exit", b"handler_exit", "msg_drop", b"msg_drop", "msg_hang", b"msg_hang", "req_abort", b"req_abort", "req_exc", b"req_exc", "req_exit", b"req_exit", "resp_abort", b"resp_abort", "resp_exc", b"resp_exc", "resp_exit", b"resp_exit", "sender_abort", b"sender_abort", "sender_exc", b"sender_exc", "sender_exit", b"sender_exit"]) -> None: ...
+
+global___TestInjectRequest = TestInjectRequest
+
+class TestInjectResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___TestInjectResponse = TestInjectResponse
+
+class HistoryAction(google.protobuf.message.Message):
+ """
+ PartialHistoryRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ FLUSH_FIELD_NUMBER: builtins.int
+ flush: builtins.bool
+ def __init__(
+ self,
+ *,
+ flush: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["flush", b"flush"]) -> None: ...
+
+global___HistoryAction = HistoryAction
+
+class PartialHistoryRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ITEM_FIELD_NUMBER: builtins.int
+ STEP_FIELD_NUMBER: builtins.int
+ ACTION_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def item(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___HistoryItem]: ...
+ @property
+ def step(self) -> global___HistoryStep: ...
+ @property
+ def action(self) -> global___HistoryAction: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ item: collections.abc.Iterable[global___HistoryItem] | None = ...,
+ step: global___HistoryStep | None = ...,
+ action: global___HistoryAction | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "action", b"action", "step", b"step"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "action", b"action", "item", b"item", "step", b"step"]) -> None: ...
+
+global___PartialHistoryRequest = PartialHistoryRequest
+
+class PartialHistoryResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___PartialHistoryResponse = PartialHistoryResponse
+
+class SampledHistoryRequest(google.protobuf.message.Message):
+ """
+ SampledHistoryRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___SampledHistoryRequest = SampledHistoryRequest
+
+class SampledHistoryItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ NESTED_KEY_FIELD_NUMBER: builtins.int
+ VALUES_FLOAT_FIELD_NUMBER: builtins.int
+ VALUES_INT_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ @property
+ def nested_key(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ @property
+ def values_float(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.float]: ...
+ @property
+ def values_int(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ...
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ nested_key: collections.abc.Iterable[builtins.str] | None = ...,
+ values_float: collections.abc.Iterable[builtins.float] | None = ...,
+ values_int: collections.abc.Iterable[builtins.int] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "nested_key", b"nested_key", "values_float", b"values_float", "values_int", b"values_int"]) -> None: ...
+
+global___SampledHistoryItem = SampledHistoryItem
+
+class SampledHistoryResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ITEM_FIELD_NUMBER: builtins.int
+ @property
+ def item(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___SampledHistoryItem]: ...
+ def __init__(
+ self,
+ *,
+ item: collections.abc.Iterable[global___SampledHistoryItem] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["item", b"item"]) -> None: ...
+
+global___SampledHistoryResponse = SampledHistoryResponse
+
+class RunStatusRequest(google.protobuf.message.Message):
+ """
+ RunStatusRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___RunStatusRequest = RunStatusRequest
+
+class RunStatusResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SYNC_ITEMS_TOTAL_FIELD_NUMBER: builtins.int
+ SYNC_ITEMS_PENDING_FIELD_NUMBER: builtins.int
+ SYNC_TIME_FIELD_NUMBER: builtins.int
+ sync_items_total: builtins.int
+ sync_items_pending: builtins.int
+ @property
+ def sync_time(self) -> google.protobuf.timestamp_pb2.Timestamp:
+ """TODO(flowcontrol): can we give the user an indication of step position
+ int64 sync_history_step = 3;
+ google.protobuf.Timestamp sync_history_time = 4;
+ """
+ def __init__(
+ self,
+ *,
+ sync_items_total: builtins.int = ...,
+ sync_items_pending: builtins.int = ...,
+ sync_time: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["sync_time", b"sync_time"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["sync_items_pending", b"sync_items_pending", "sync_items_total", b"sync_items_total", "sync_time", b"sync_time"]) -> None: ...
+
+global___RunStatusResponse = RunStatusResponse
+
+class RunStartRequest(google.protobuf.message.Message):
+ """
+ RunStartRequest: start the run
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def run(self) -> global___RunRecord: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ run: global___RunRecord | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "run", b"run"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "run", b"run"]) -> None: ...
+
+global___RunStartRequest = RunStartRequest
+
+class RunStartResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___RunStartResponse = RunStartResponse
+
+class CheckVersionRequest(google.protobuf.message.Message):
+ """
+ CheckVersion:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ CURRENT_VERSION_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ current_version: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ current_version: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "current_version", b"current_version"]) -> None: ...
+
+global___CheckVersionRequest = CheckVersionRequest
+
+class CheckVersionResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ UPGRADE_MESSAGE_FIELD_NUMBER: builtins.int
+ YANK_MESSAGE_FIELD_NUMBER: builtins.int
+ DELETE_MESSAGE_FIELD_NUMBER: builtins.int
+ upgrade_message: builtins.str
+ yank_message: builtins.str
+ delete_message: builtins.str
+ def __init__(
+ self,
+ *,
+ upgrade_message: builtins.str = ...,
+ yank_message: builtins.str = ...,
+ delete_message: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["delete_message", b"delete_message", "upgrade_message", b"upgrade_message", "yank_message", b"yank_message"]) -> None: ...
+
+global___CheckVersionResponse = CheckVersionResponse
+
+class JobInfoRequest(google.protobuf.message.Message):
+ """
+ JobInfo:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___JobInfoRequest = JobInfoRequest
+
+class JobInfoResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SEQUENCEID_FIELD_NUMBER: builtins.int
+ VERSION_FIELD_NUMBER: builtins.int
+ sequenceId: builtins.str
+ version: builtins.str
+ def __init__(
+ self,
+ *,
+ sequenceId: builtins.str = ...,
+ version: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["sequenceId", b"sequenceId", "version", b"version"]) -> None: ...
+
+global___JobInfoResponse = JobInfoResponse
+
+class LogArtifactRequest(google.protobuf.message.Message):
+ """
+ LogArtifact:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ARTIFACT_FIELD_NUMBER: builtins.int
+ HISTORY_STEP_FIELD_NUMBER: builtins.int
+ STAGING_DIR_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def artifact(self) -> global___ArtifactRecord: ...
+ history_step: builtins.int
+ staging_dir: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ artifact: global___ArtifactRecord | None = ...,
+ history_step: builtins.int = ...,
+ staging_dir: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "artifact", b"artifact"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "artifact", b"artifact", "history_step", b"history_step", "staging_dir", b"staging_dir"]) -> None: ...
+
+global___LogArtifactRequest = LogArtifactRequest
+
+class LogArtifactResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ARTIFACT_ID_FIELD_NUMBER: builtins.int
+ ERROR_MESSAGE_FIELD_NUMBER: builtins.int
+ artifact_id: builtins.str
+ error_message: builtins.str
+ def __init__(
+ self,
+ *,
+ artifact_id: builtins.str = ...,
+ error_message: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["artifact_id", b"artifact_id", "error_message", b"error_message"]) -> None: ...
+
+global___LogArtifactResponse = LogArtifactResponse
+
+class DownloadArtifactRequest(google.protobuf.message.Message):
+ """
+ DownloadArtifact:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ARTIFACT_ID_FIELD_NUMBER: builtins.int
+ DOWNLOAD_ROOT_FIELD_NUMBER: builtins.int
+ ALLOW_MISSING_REFERENCES_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ artifact_id: builtins.str
+ download_root: builtins.str
+ allow_missing_references: builtins.bool
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ artifact_id: builtins.str = ...,
+ download_root: builtins.str = ...,
+ allow_missing_references: builtins.bool = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "allow_missing_references", b"allow_missing_references", "artifact_id", b"artifact_id", "download_root", b"download_root"]) -> None: ...
+
+global___DownloadArtifactRequest = DownloadArtifactRequest
+
+class DownloadArtifactResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ERROR_MESSAGE_FIELD_NUMBER: builtins.int
+ error_message: builtins.str
+ def __init__(
+ self,
+ *,
+ error_message: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["error_message", b"error_message"]) -> None: ...
+
+global___DownloadArtifactResponse = DownloadArtifactResponse
+
+class KeepaliveRequest(google.protobuf.message.Message):
+ """
+ Keepalive:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___KeepaliveRequest = KeepaliveRequest
+
+class KeepaliveResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___KeepaliveResponse = KeepaliveResponse
+
+class ArtifactInfo(google.protobuf.message.Message):
+ """
+ Job info specific for Partial -> Job upgrade
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ARTIFACT_FIELD_NUMBER: builtins.int
+ ENTRYPOINT_FIELD_NUMBER: builtins.int
+ NOTEBOOK_FIELD_NUMBER: builtins.int
+ artifact: builtins.str
+ @property
+ def entrypoint(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ notebook: builtins.bool
+ def __init__(
+ self,
+ *,
+ artifact: builtins.str = ...,
+ entrypoint: collections.abc.Iterable[builtins.str] | None = ...,
+ notebook: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["artifact", b"artifact", "entrypoint", b"entrypoint", "notebook", b"notebook"]) -> None: ...
+
+global___ArtifactInfo = ArtifactInfo
+
+class GitInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ REMOTE_FIELD_NUMBER: builtins.int
+ COMMIT_FIELD_NUMBER: builtins.int
+ remote: builtins.str
+ commit: builtins.str
+ def __init__(
+ self,
+ *,
+ remote: builtins.str = ...,
+ commit: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["commit", b"commit", "remote", b"remote"]) -> None: ...
+
+global___GitInfo = GitInfo
+
+class GitSource(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ GIT_INFO_FIELD_NUMBER: builtins.int
+ ENTRYPOINT_FIELD_NUMBER: builtins.int
+ NOTEBOOK_FIELD_NUMBER: builtins.int
+ @property
+ def git_info(self) -> global___GitInfo: ...
+ @property
+ def entrypoint(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ notebook: builtins.bool
+ def __init__(
+ self,
+ *,
+ git_info: global___GitInfo | None = ...,
+ entrypoint: collections.abc.Iterable[builtins.str] | None = ...,
+ notebook: builtins.bool = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["git_info", b"git_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["entrypoint", b"entrypoint", "git_info", b"git_info", "notebook", b"notebook"]) -> None: ...
+
+global___GitSource = GitSource
+
+class ImageSource(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ IMAGE_FIELD_NUMBER: builtins.int
+ image: builtins.str
+ def __init__(
+ self,
+ *,
+ image: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["image", b"image"]) -> None: ...
+
+global___ImageSource = ImageSource
+
+class Source(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ GIT_FIELD_NUMBER: builtins.int
+ ARTIFACT_FIELD_NUMBER: builtins.int
+ IMAGE_FIELD_NUMBER: builtins.int
+ @property
+ def git(self) -> global___GitSource: ...
+ @property
+ def artifact(self) -> global___ArtifactInfo: ...
+ @property
+ def image(self) -> global___ImageSource: ...
+ def __init__(
+ self,
+ *,
+ git: global___GitSource | None = ...,
+ artifact: global___ArtifactInfo | None = ...,
+ image: global___ImageSource | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["artifact", b"artifact", "git", b"git", "image", b"image"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["artifact", b"artifact", "git", b"git", "image", b"image"]) -> None: ...
+
+global___Source = Source
+
+class JobSource(google.protobuf.message.Message):
+ """
+ Mirrors JobSourceDict:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _VERSION_FIELD_NUMBER: builtins.int
+ SOURCE_TYPE_FIELD_NUMBER: builtins.int
+ SOURCE_FIELD_NUMBER: builtins.int
+ RUNTIME_FIELD_NUMBER: builtins.int
+ _version: builtins.str
+ source_type: builtins.str
+ @property
+ def source(self) -> global___Source: ...
+ runtime: builtins.str
+ def __init__(
+ self,
+ *,
+ _version: builtins.str = ...,
+ source_type: builtins.str = ...,
+ source: global___Source | None = ...,
+ runtime: builtins.str = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["source", b"source"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_version", b"_version", "runtime", b"runtime", "source", b"source", "source_type", b"source_type"]) -> None: ...
+
+global___JobSource = JobSource
+
+class PartialJobArtifact(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ JOB_NAME_FIELD_NUMBER: builtins.int
+ SOURCE_INFO_FIELD_NUMBER: builtins.int
+ job_name: builtins.str
+ @property
+ def source_info(self) -> global___JobSource: ...
+ def __init__(
+ self,
+ *,
+ job_name: builtins.str = ...,
+ source_info: global___JobSource | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["source_info", b"source_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["job_name", b"job_name", "source_info", b"source_info"]) -> None: ...
+
+global___PartialJobArtifact = PartialJobArtifact
+
+class UseArtifactRecord(google.protobuf.message.Message):
+ """
+ UseArtifact:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ID_FIELD_NUMBER: builtins.int
+ TYPE_FIELD_NUMBER: builtins.int
+ NAME_FIELD_NUMBER: builtins.int
+ PARTIAL_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ id: builtins.str
+ type: builtins.str
+ name: builtins.str
+ @property
+ def partial(self) -> global___PartialJobArtifact: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ id: builtins.str = ...,
+ type: builtins.str = ...,
+ name: builtins.str = ...,
+ partial: global___PartialJobArtifact | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "partial", b"partial"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "id", b"id", "name", b"name", "partial", b"partial", "type", b"type"]) -> None: ...
+
+global___UseArtifactRecord = UseArtifactRecord
+
+class UseArtifactResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___UseArtifactResult = UseArtifactResult
+
+class CancelRequest(google.protobuf.message.Message):
+ """
+ Cancel:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ CANCEL_SLOT_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ cancel_slot: builtins.str
+ """mailbox slot"""
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ cancel_slot: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "cancel_slot", b"cancel_slot"]) -> None: ...
+
+global___CancelRequest = CancelRequest
+
+class CancelResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___CancelResponse = CancelResponse
+
+class DiskInfo(google.protobuf.message.Message):
+ """
+ MetadataRequest
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ TOTAL_FIELD_NUMBER: builtins.int
+ USED_FIELD_NUMBER: builtins.int
+ total: builtins.int
+ used: builtins.int
+ def __init__(
+ self,
+ *,
+ total: builtins.int = ...,
+ used: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["total", b"total", "used", b"used"]) -> None: ...
+
+global___DiskInfo = DiskInfo
+
+class MemoryInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ TOTAL_FIELD_NUMBER: builtins.int
+ total: builtins.int
+ def __init__(
+ self,
+ *,
+ total: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["total", b"total"]) -> None: ...
+
+global___MemoryInfo = MemoryInfo
+
+class CpuInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ COUNT_FIELD_NUMBER: builtins.int
+ COUNT_LOGICAL_FIELD_NUMBER: builtins.int
+ count: builtins.int
+ count_logical: builtins.int
+ def __init__(
+ self,
+ *,
+ count: builtins.int = ...,
+ count_logical: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["count", b"count", "count_logical", b"count_logical"]) -> None: ...
+
+global___CpuInfo = CpuInfo
+
+class GpuAppleInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ GPUTYPE_FIELD_NUMBER: builtins.int
+ VENDOR_FIELD_NUMBER: builtins.int
+ CORES_FIELD_NUMBER: builtins.int
+ gpuType: builtins.str
+ vendor: builtins.str
+ cores: builtins.int
+ def __init__(
+ self,
+ *,
+ gpuType: builtins.str = ...,
+ vendor: builtins.str = ...,
+ cores: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["cores", b"cores", "gpuType", b"gpuType", "vendor", b"vendor"]) -> None: ...
+
+global___GpuAppleInfo = GpuAppleInfo
+
+class GpuNvidiaInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ NAME_FIELD_NUMBER: builtins.int
+ MEMORY_TOTAL_FIELD_NUMBER: builtins.int
+ name: builtins.str
+ memory_total: builtins.int
+ def __init__(
+ self,
+ *,
+ name: builtins.str = ...,
+ memory_total: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["memory_total", b"memory_total", "name", b"name"]) -> None: ...
+
+global___GpuNvidiaInfo = GpuNvidiaInfo
+
+class GpuAmdInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ID_FIELD_NUMBER: builtins.int
+ UNIQUE_ID_FIELD_NUMBER: builtins.int
+ VBIOS_VERSION_FIELD_NUMBER: builtins.int
+ PERFORMANCE_LEVEL_FIELD_NUMBER: builtins.int
+ GPU_OVERDRIVE_FIELD_NUMBER: builtins.int
+ GPU_MEMORY_OVERDRIVE_FIELD_NUMBER: builtins.int
+ MAX_POWER_FIELD_NUMBER: builtins.int
+ SERIES_FIELD_NUMBER: builtins.int
+ MODEL_FIELD_NUMBER: builtins.int
+ VENDOR_FIELD_NUMBER: builtins.int
+ SKU_FIELD_NUMBER: builtins.int
+ SCLK_RANGE_FIELD_NUMBER: builtins.int
+ MCLK_RANGE_FIELD_NUMBER: builtins.int
+ id: builtins.str
+ unique_id: builtins.str
+ vbios_version: builtins.str
+ performance_level: builtins.str
+ gpu_overdrive: builtins.str
+ gpu_memory_overdrive: builtins.str
+ max_power: builtins.str
+ series: builtins.str
+ model: builtins.str
+ vendor: builtins.str
+ sku: builtins.str
+ sclk_range: builtins.str
+ mclk_range: builtins.str
+ def __init__(
+ self,
+ *,
+ id: builtins.str = ...,
+ unique_id: builtins.str = ...,
+ vbios_version: builtins.str = ...,
+ performance_level: builtins.str = ...,
+ gpu_overdrive: builtins.str = ...,
+ gpu_memory_overdrive: builtins.str = ...,
+ max_power: builtins.str = ...,
+ series: builtins.str = ...,
+ model: builtins.str = ...,
+ vendor: builtins.str = ...,
+ sku: builtins.str = ...,
+ sclk_range: builtins.str = ...,
+ mclk_range: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["gpu_memory_overdrive", b"gpu_memory_overdrive", "gpu_overdrive", b"gpu_overdrive", "id", b"id", "max_power", b"max_power", "mclk_range", b"mclk_range", "model", b"model", "performance_level", b"performance_level", "sclk_range", b"sclk_range", "series", b"series", "sku", b"sku", "unique_id", b"unique_id", "vbios_version", b"vbios_version", "vendor", b"vendor"]) -> None: ...
+
+global___GpuAmdInfo = GpuAmdInfo
+
+class MetadataRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class DiskEntry(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ @property
+ def value(self) -> global___DiskInfo: ...
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value: global___DiskInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["value", b"value"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ...
+
+ class SlurmEntry(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ value: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ...
+
+ OS_FIELD_NUMBER: builtins.int
+ PYTHON_FIELD_NUMBER: builtins.int
+ HEARTBEATAT_FIELD_NUMBER: builtins.int
+ STARTEDAT_FIELD_NUMBER: builtins.int
+ DOCKER_FIELD_NUMBER: builtins.int
+ CUDA_FIELD_NUMBER: builtins.int
+ ARGS_FIELD_NUMBER: builtins.int
+ STATE_FIELD_NUMBER: builtins.int
+ PROGRAM_FIELD_NUMBER: builtins.int
+ CODE_PATH_FIELD_NUMBER: builtins.int
+ GIT_FIELD_NUMBER: builtins.int
+ EMAIL_FIELD_NUMBER: builtins.int
+ ROOT_FIELD_NUMBER: builtins.int
+ HOST_FIELD_NUMBER: builtins.int
+ USERNAME_FIELD_NUMBER: builtins.int
+ EXECUTABLE_FIELD_NUMBER: builtins.int
+ CODE_PATH_LOCAL_FIELD_NUMBER: builtins.int
+ COLAB_FIELD_NUMBER: builtins.int
+ CPU_COUNT_FIELD_NUMBER: builtins.int
+ CPU_COUNT_LOGICAL_FIELD_NUMBER: builtins.int
+ GPU_TYPE_FIELD_NUMBER: builtins.int
+ GPU_COUNT_FIELD_NUMBER: builtins.int
+ DISK_FIELD_NUMBER: builtins.int
+ MEMORY_FIELD_NUMBER: builtins.int
+ CPU_FIELD_NUMBER: builtins.int
+ GPU_APPLE_FIELD_NUMBER: builtins.int
+ GPU_NVIDIA_FIELD_NUMBER: builtins.int
+ GPU_AMD_FIELD_NUMBER: builtins.int
+ SLURM_FIELD_NUMBER: builtins.int
+ os: builtins.str
+ python: builtins.str
+ @property
+ def heartbeatAt(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ @property
+ def startedAt(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ docker: builtins.str
+ cuda: builtins.str
+ @property
+ def args(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ state: builtins.str
+ program: builtins.str
+ code_path: builtins.str
+ @property
+ def git(self) -> global___GitRepoRecord: ...
+ email: builtins.str
+ root: builtins.str
+ host: builtins.str
+ username: builtins.str
+ executable: builtins.str
+ code_path_local: builtins.str
+ colab: builtins.str
+ cpu_count: builtins.int
+ cpu_count_logical: builtins.int
+ gpu_type: builtins.str
+ gpu_count: builtins.int
+ @property
+ def disk(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___DiskInfo]: ...
+ @property
+ def memory(self) -> global___MemoryInfo: ...
+ @property
+ def cpu(self) -> global___CpuInfo: ...
+ @property
+ def gpu_apple(self) -> global___GpuAppleInfo: ...
+ @property
+ def gpu_nvidia(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___GpuNvidiaInfo]: ...
+ @property
+ def gpu_amd(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___GpuAmdInfo]: ...
+ @property
+ def slurm(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ...
+ def __init__(
+ self,
+ *,
+ os: builtins.str = ...,
+ python: builtins.str = ...,
+ heartbeatAt: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ startedAt: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ docker: builtins.str = ...,
+ cuda: builtins.str = ...,
+ args: collections.abc.Iterable[builtins.str] | None = ...,
+ state: builtins.str = ...,
+ program: builtins.str = ...,
+ code_path: builtins.str = ...,
+ git: global___GitRepoRecord | None = ...,
+ email: builtins.str = ...,
+ root: builtins.str = ...,
+ host: builtins.str = ...,
+ username: builtins.str = ...,
+ executable: builtins.str = ...,
+ code_path_local: builtins.str = ...,
+ colab: builtins.str = ...,
+ cpu_count: builtins.int = ...,
+ cpu_count_logical: builtins.int = ...,
+ gpu_type: builtins.str = ...,
+ gpu_count: builtins.int = ...,
+ disk: collections.abc.Mapping[builtins.str, global___DiskInfo] | None = ...,
+ memory: global___MemoryInfo | None = ...,
+ cpu: global___CpuInfo | None = ...,
+ gpu_apple: global___GpuAppleInfo | None = ...,
+ gpu_nvidia: collections.abc.Iterable[global___GpuNvidiaInfo] | None = ...,
+ gpu_amd: collections.abc.Iterable[global___GpuAmdInfo] | None = ...,
+ slurm: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["cpu", b"cpu", "git", b"git", "gpu_apple", b"gpu_apple", "heartbeatAt", b"heartbeatAt", "memory", b"memory", "startedAt", b"startedAt"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["args", b"args", "code_path", b"code_path", "code_path_local", b"code_path_local", "colab", b"colab", "cpu", b"cpu", "cpu_count", b"cpu_count", "cpu_count_logical", b"cpu_count_logical", "cuda", b"cuda", "disk", b"disk", "docker", b"docker", "email", b"email", "executable", b"executable", "git", b"git", "gpu_amd", b"gpu_amd", "gpu_apple", b"gpu_apple", "gpu_count", b"gpu_count", "gpu_nvidia", b"gpu_nvidia", "gpu_type", b"gpu_type", "heartbeatAt", b"heartbeatAt", "host", b"host", "memory", b"memory", "os", b"os", "program", b"program", "python", b"python", "root", b"root", "slurm", b"slurm", "startedAt", b"startedAt", "state", b"state", "username", b"username"]) -> None: ...
+
+global___MetadataRequest = MetadataRequest
+
+class PythonPackagesRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class PythonPackage(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ NAME_FIELD_NUMBER: builtins.int
+ VERSION_FIELD_NUMBER: builtins.int
+ name: builtins.str
+ version: builtins.str
+ def __init__(
+ self,
+ *,
+ name: builtins.str = ...,
+ version: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["name", b"name", "version", b"version"]) -> None: ...
+
+ PACKAGE_FIELD_NUMBER: builtins.int
+ @property
+ def package(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___PythonPackagesRequest.PythonPackage]: ...
+ def __init__(
+ self,
+ *,
+ package: collections.abc.Iterable[global___PythonPackagesRequest.PythonPackage] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["package", b"package"]) -> None: ...
+
+global___PythonPackagesRequest = PythonPackagesRequest
diff --git a/wandb/proto/v3/wandb_server_pb2.py b/wandb/proto/v3/wandb_server_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..4aed9c9eec090018e38eb3f3e9c8f480e98aae84
--- /dev/null
+++ b/wandb/proto/v3/wandb_server_pb2.py
@@ -0,0 +1,207 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: wandb/proto/wandb_server.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from wandb.proto import wandb_base_pb2 as wandb_dot_proto_dot_wandb__base__pb2
+from wandb.proto import wandb_internal_pb2 as wandb_dot_proto_dot_wandb__internal__pb2
+from wandb.proto import wandb_settings_pb2 as wandb_dot_proto_dot_wandb__settings__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1ewandb/proto/wandb_server.proto\x12\x0ewandb_internal\x1a\x1cwandb/proto/wandb_base.proto\x1a wandb/proto/wandb_internal.proto\x1a wandb/proto/wandb_settings.proto\"D\n\x15ServerShutdownRequest\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x18\n\x16ServerShutdownResponse\"B\n\x13ServerStatusRequest\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x16\n\x14ServerStatusResponse\"r\n\x17ServerInformInitRequest\x12*\n\x08settings\x18\x01 \x01(\x0b\x32\x18.wandb_internal.Settings\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x1a\n\x18ServerInformInitResponse\"s\n\x18ServerInformStartRequest\x12*\n\x08settings\x18\x01 \x01(\x0b\x32\x18.wandb_internal.Settings\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x1b\n\x19ServerInformStartResponse\"H\n\x19ServerInformFinishRequest\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x1c\n\x1aServerInformFinishResponse\"H\n\x19ServerInformAttachRequest\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"u\n\x1aServerInformAttachResponse\x12*\n\x08settings\x18\x01 \x01(\x0b\x32\x18.wandb_internal.Settings\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"H\n\x19ServerInformDetachRequest\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x1c\n\x1aServerInformDetachResponse\"]\n\x1bServerInformTeardownRequest\x12\x11\n\texit_code\x18\x01 \x01(\x05\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x1e\n\x1cServerInformTeardownResponse\"\xa4\x04\n\rServerRequest\x12\x30\n\x0erecord_publish\x18\x01 \x01(\x0b\x32\x16.wandb_internal.RecordH\x00\x12\x34\n\x12record_communicate\x18\x02 \x01(\x0b\x32\x16.wandb_internal.RecordH\x00\x12>\n\x0binform_init\x18\x03 \x01(\x0b\x32\'.wandb_internal.ServerInformInitRequestH\x00\x12\x42\n\rinform_finish\x18\x04 \x01(\x0b\x32).wandb_internal.ServerInformFinishRequestH\x00\x12\x42\n\rinform_attach\x18\x05 \x01(\x0b\x32).wandb_internal.ServerInformAttachRequestH\x00\x12\x42\n\rinform_detach\x18\x06 \x01(\x0b\x32).wandb_internal.ServerInformDetachRequestH\x00\x12\x46\n\x0finform_teardown\x18\x07 \x01(\x0b\x32+.wandb_internal.ServerInformTeardownRequestH\x00\x12@\n\x0cinform_start\x18\x08 \x01(\x0b\x32(.wandb_internal.ServerInformStartRequestH\x00\x42\x15\n\x13server_request_type\"\xb0\x04\n\x0eServerResponse\x12\x34\n\x12result_communicate\x18\x02 \x01(\x0b\x32\x16.wandb_internal.ResultH\x00\x12H\n\x14inform_init_response\x18\x03 \x01(\x0b\x32(.wandb_internal.ServerInformInitResponseH\x00\x12L\n\x16inform_finish_response\x18\x04 \x01(\x0b\x32*.wandb_internal.ServerInformFinishResponseH\x00\x12L\n\x16inform_attach_response\x18\x05 \x01(\x0b\x32*.wandb_internal.ServerInformAttachResponseH\x00\x12L\n\x16inform_detach_response\x18\x06 \x01(\x0b\x32*.wandb_internal.ServerInformDetachResponseH\x00\x12P\n\x18inform_teardown_response\x18\x07 \x01(\x0b\x32,.wandb_internal.ServerInformTeardownResponseH\x00\x12J\n\x15inform_start_response\x18\x08 \x01(\x0b\x32).wandb_internal.ServerInformStartResponseH\x00\x42\x16\n\x14server_response_typeb\x06proto3')
+
+
+
+_SERVERSHUTDOWNREQUEST = DESCRIPTOR.message_types_by_name['ServerShutdownRequest']
+_SERVERSHUTDOWNRESPONSE = DESCRIPTOR.message_types_by_name['ServerShutdownResponse']
+_SERVERSTATUSREQUEST = DESCRIPTOR.message_types_by_name['ServerStatusRequest']
+_SERVERSTATUSRESPONSE = DESCRIPTOR.message_types_by_name['ServerStatusResponse']
+_SERVERINFORMINITREQUEST = DESCRIPTOR.message_types_by_name['ServerInformInitRequest']
+_SERVERINFORMINITRESPONSE = DESCRIPTOR.message_types_by_name['ServerInformInitResponse']
+_SERVERINFORMSTARTREQUEST = DESCRIPTOR.message_types_by_name['ServerInformStartRequest']
+_SERVERINFORMSTARTRESPONSE = DESCRIPTOR.message_types_by_name['ServerInformStartResponse']
+_SERVERINFORMFINISHREQUEST = DESCRIPTOR.message_types_by_name['ServerInformFinishRequest']
+_SERVERINFORMFINISHRESPONSE = DESCRIPTOR.message_types_by_name['ServerInformFinishResponse']
+_SERVERINFORMATTACHREQUEST = DESCRIPTOR.message_types_by_name['ServerInformAttachRequest']
+_SERVERINFORMATTACHRESPONSE = DESCRIPTOR.message_types_by_name['ServerInformAttachResponse']
+_SERVERINFORMDETACHREQUEST = DESCRIPTOR.message_types_by_name['ServerInformDetachRequest']
+_SERVERINFORMDETACHRESPONSE = DESCRIPTOR.message_types_by_name['ServerInformDetachResponse']
+_SERVERINFORMTEARDOWNREQUEST = DESCRIPTOR.message_types_by_name['ServerInformTeardownRequest']
+_SERVERINFORMTEARDOWNRESPONSE = DESCRIPTOR.message_types_by_name['ServerInformTeardownResponse']
+_SERVERREQUEST = DESCRIPTOR.message_types_by_name['ServerRequest']
+_SERVERRESPONSE = DESCRIPTOR.message_types_by_name['ServerResponse']
+ServerShutdownRequest = _reflection.GeneratedProtocolMessageType('ServerShutdownRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERSHUTDOWNREQUEST,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerShutdownRequest)
+ })
+_sym_db.RegisterMessage(ServerShutdownRequest)
+
+ServerShutdownResponse = _reflection.GeneratedProtocolMessageType('ServerShutdownResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERSHUTDOWNRESPONSE,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerShutdownResponse)
+ })
+_sym_db.RegisterMessage(ServerShutdownResponse)
+
+ServerStatusRequest = _reflection.GeneratedProtocolMessageType('ServerStatusRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERSTATUSREQUEST,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerStatusRequest)
+ })
+_sym_db.RegisterMessage(ServerStatusRequest)
+
+ServerStatusResponse = _reflection.GeneratedProtocolMessageType('ServerStatusResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERSTATUSRESPONSE,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerStatusResponse)
+ })
+_sym_db.RegisterMessage(ServerStatusResponse)
+
+ServerInformInitRequest = _reflection.GeneratedProtocolMessageType('ServerInformInitRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERINFORMINITREQUEST,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerInformInitRequest)
+ })
+_sym_db.RegisterMessage(ServerInformInitRequest)
+
+ServerInformInitResponse = _reflection.GeneratedProtocolMessageType('ServerInformInitResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERINFORMINITRESPONSE,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerInformInitResponse)
+ })
+_sym_db.RegisterMessage(ServerInformInitResponse)
+
+ServerInformStartRequest = _reflection.GeneratedProtocolMessageType('ServerInformStartRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERINFORMSTARTREQUEST,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerInformStartRequest)
+ })
+_sym_db.RegisterMessage(ServerInformStartRequest)
+
+ServerInformStartResponse = _reflection.GeneratedProtocolMessageType('ServerInformStartResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERINFORMSTARTRESPONSE,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerInformStartResponse)
+ })
+_sym_db.RegisterMessage(ServerInformStartResponse)
+
+ServerInformFinishRequest = _reflection.GeneratedProtocolMessageType('ServerInformFinishRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERINFORMFINISHREQUEST,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerInformFinishRequest)
+ })
+_sym_db.RegisterMessage(ServerInformFinishRequest)
+
+ServerInformFinishResponse = _reflection.GeneratedProtocolMessageType('ServerInformFinishResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERINFORMFINISHRESPONSE,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerInformFinishResponse)
+ })
+_sym_db.RegisterMessage(ServerInformFinishResponse)
+
+ServerInformAttachRequest = _reflection.GeneratedProtocolMessageType('ServerInformAttachRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERINFORMATTACHREQUEST,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerInformAttachRequest)
+ })
+_sym_db.RegisterMessage(ServerInformAttachRequest)
+
+ServerInformAttachResponse = _reflection.GeneratedProtocolMessageType('ServerInformAttachResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERINFORMATTACHRESPONSE,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerInformAttachResponse)
+ })
+_sym_db.RegisterMessage(ServerInformAttachResponse)
+
+ServerInformDetachRequest = _reflection.GeneratedProtocolMessageType('ServerInformDetachRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERINFORMDETACHREQUEST,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerInformDetachRequest)
+ })
+_sym_db.RegisterMessage(ServerInformDetachRequest)
+
+ServerInformDetachResponse = _reflection.GeneratedProtocolMessageType('ServerInformDetachResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERINFORMDETACHRESPONSE,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerInformDetachResponse)
+ })
+_sym_db.RegisterMessage(ServerInformDetachResponse)
+
+ServerInformTeardownRequest = _reflection.GeneratedProtocolMessageType('ServerInformTeardownRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERINFORMTEARDOWNREQUEST,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerInformTeardownRequest)
+ })
+_sym_db.RegisterMessage(ServerInformTeardownRequest)
+
+ServerInformTeardownResponse = _reflection.GeneratedProtocolMessageType('ServerInformTeardownResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERINFORMTEARDOWNRESPONSE,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerInformTeardownResponse)
+ })
+_sym_db.RegisterMessage(ServerInformTeardownResponse)
+
+ServerRequest = _reflection.GeneratedProtocolMessageType('ServerRequest', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERREQUEST,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerRequest)
+ })
+_sym_db.RegisterMessage(ServerRequest)
+
+ServerResponse = _reflection.GeneratedProtocolMessageType('ServerResponse', (_message.Message,), {
+ 'DESCRIPTOR' : _SERVERRESPONSE,
+ '__module__' : 'wandb.proto.wandb_server_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ServerResponse)
+ })
+_sym_db.RegisterMessage(ServerResponse)
+
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+ DESCRIPTOR._options = None
+ _SERVERSHUTDOWNREQUEST._serialized_start=148
+ _SERVERSHUTDOWNREQUEST._serialized_end=216
+ _SERVERSHUTDOWNRESPONSE._serialized_start=218
+ _SERVERSHUTDOWNRESPONSE._serialized_end=242
+ _SERVERSTATUSREQUEST._serialized_start=244
+ _SERVERSTATUSREQUEST._serialized_end=310
+ _SERVERSTATUSRESPONSE._serialized_start=312
+ _SERVERSTATUSRESPONSE._serialized_end=334
+ _SERVERINFORMINITREQUEST._serialized_start=336
+ _SERVERINFORMINITREQUEST._serialized_end=450
+ _SERVERINFORMINITRESPONSE._serialized_start=452
+ _SERVERINFORMINITRESPONSE._serialized_end=478
+ _SERVERINFORMSTARTREQUEST._serialized_start=480
+ _SERVERINFORMSTARTREQUEST._serialized_end=595
+ _SERVERINFORMSTARTRESPONSE._serialized_start=597
+ _SERVERINFORMSTARTRESPONSE._serialized_end=624
+ _SERVERINFORMFINISHREQUEST._serialized_start=626
+ _SERVERINFORMFINISHREQUEST._serialized_end=698
+ _SERVERINFORMFINISHRESPONSE._serialized_start=700
+ _SERVERINFORMFINISHRESPONSE._serialized_end=728
+ _SERVERINFORMATTACHREQUEST._serialized_start=730
+ _SERVERINFORMATTACHREQUEST._serialized_end=802
+ _SERVERINFORMATTACHRESPONSE._serialized_start=804
+ _SERVERINFORMATTACHRESPONSE._serialized_end=921
+ _SERVERINFORMDETACHREQUEST._serialized_start=923
+ _SERVERINFORMDETACHREQUEST._serialized_end=995
+ _SERVERINFORMDETACHRESPONSE._serialized_start=997
+ _SERVERINFORMDETACHRESPONSE._serialized_end=1025
+ _SERVERINFORMTEARDOWNREQUEST._serialized_start=1027
+ _SERVERINFORMTEARDOWNREQUEST._serialized_end=1120
+ _SERVERINFORMTEARDOWNRESPONSE._serialized_start=1122
+ _SERVERINFORMTEARDOWNRESPONSE._serialized_end=1152
+ _SERVERREQUEST._serialized_start=1155
+ _SERVERREQUEST._serialized_end=1703
+ _SERVERRESPONSE._serialized_start=1706
+ _SERVERRESPONSE._serialized_end=2266
+# @@protoc_insertion_point(module_scope)
diff --git a/wandb/proto/v3/wandb_server_pb2.pyi b/wandb/proto/v3/wandb_server_pb2.pyi
new file mode 100644
index 0000000000000000000000000000000000000000..fdb90345997a547c370b7f90036ee23233390359
--- /dev/null
+++ b/wandb/proto/v3/wandb_server_pb2.pyi
@@ -0,0 +1,330 @@
+"""
+@generated by mypy-protobuf. Do not edit manually!
+isort:skip_file
+"""
+import builtins
+import google.protobuf.descriptor
+import google.protobuf.message
+import sys
+import wandb.proto.wandb_base_pb2
+import wandb.proto.wandb_internal_pb2
+import wandb.proto.wandb_settings_pb2
+
+if sys.version_info >= (3, 8):
+ import typing as typing_extensions
+else:
+ import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+class ServerShutdownRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ServerShutdownRequest = ServerShutdownRequest
+
+class ServerShutdownResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ServerShutdownResponse = ServerShutdownResponse
+
+class ServerStatusRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ServerStatusRequest = ServerStatusRequest
+
+class ServerStatusResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ServerStatusResponse = ServerStatusResponse
+
+class ServerInformInitRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SETTINGS_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def settings(self) -> wandb.proto.wandb_settings_pb2.Settings: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ settings: wandb.proto.wandb_settings_pb2.Settings | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "settings", b"settings"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "settings", b"settings"]) -> None: ...
+
+global___ServerInformInitRequest = ServerInformInitRequest
+
+class ServerInformInitResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ServerInformInitResponse = ServerInformInitResponse
+
+class ServerInformStartRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SETTINGS_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def settings(self) -> wandb.proto.wandb_settings_pb2.Settings: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ settings: wandb.proto.wandb_settings_pb2.Settings | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "settings", b"settings"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "settings", b"settings"]) -> None: ...
+
+global___ServerInformStartRequest = ServerInformStartRequest
+
+class ServerInformStartResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ServerInformStartResponse = ServerInformStartResponse
+
+class ServerInformFinishRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ServerInformFinishRequest = ServerInformFinishRequest
+
+class ServerInformFinishResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ServerInformFinishResponse = ServerInformFinishResponse
+
+class ServerInformAttachRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ServerInformAttachRequest = ServerInformAttachRequest
+
+class ServerInformAttachResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SETTINGS_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def settings(self) -> wandb.proto.wandb_settings_pb2.Settings: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ settings: wandb.proto.wandb_settings_pb2.Settings | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "settings", b"settings"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "settings", b"settings"]) -> None: ...
+
+global___ServerInformAttachResponse = ServerInformAttachResponse
+
+class ServerInformDetachRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ServerInformDetachRequest = ServerInformDetachRequest
+
+class ServerInformDetachResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ServerInformDetachResponse = ServerInformDetachResponse
+
+class ServerInformTeardownRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ EXIT_CODE_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ exit_code: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ exit_code: builtins.int = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "exit_code", b"exit_code"]) -> None: ...
+
+global___ServerInformTeardownRequest = ServerInformTeardownRequest
+
+class ServerInformTeardownResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ServerInformTeardownResponse = ServerInformTeardownResponse
+
+class ServerRequest(google.protobuf.message.Message):
+ """
+ ServerRequest, ServerResponse: used in sock server
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RECORD_PUBLISH_FIELD_NUMBER: builtins.int
+ RECORD_COMMUNICATE_FIELD_NUMBER: builtins.int
+ INFORM_INIT_FIELD_NUMBER: builtins.int
+ INFORM_FINISH_FIELD_NUMBER: builtins.int
+ INFORM_ATTACH_FIELD_NUMBER: builtins.int
+ INFORM_DETACH_FIELD_NUMBER: builtins.int
+ INFORM_TEARDOWN_FIELD_NUMBER: builtins.int
+ INFORM_START_FIELD_NUMBER: builtins.int
+ @property
+ def record_publish(self) -> wandb.proto.wandb_internal_pb2.Record: ...
+ @property
+ def record_communicate(self) -> wandb.proto.wandb_internal_pb2.Record: ...
+ @property
+ def inform_init(self) -> global___ServerInformInitRequest: ...
+ @property
+ def inform_finish(self) -> global___ServerInformFinishRequest: ...
+ @property
+ def inform_attach(self) -> global___ServerInformAttachRequest: ...
+ @property
+ def inform_detach(self) -> global___ServerInformDetachRequest: ...
+ @property
+ def inform_teardown(self) -> global___ServerInformTeardownRequest: ...
+ @property
+ def inform_start(self) -> global___ServerInformStartRequest: ...
+ def __init__(
+ self,
+ *,
+ record_publish: wandb.proto.wandb_internal_pb2.Record | None = ...,
+ record_communicate: wandb.proto.wandb_internal_pb2.Record | None = ...,
+ inform_init: global___ServerInformInitRequest | None = ...,
+ inform_finish: global___ServerInformFinishRequest | None = ...,
+ inform_attach: global___ServerInformAttachRequest | None = ...,
+ inform_detach: global___ServerInformDetachRequest | None = ...,
+ inform_teardown: global___ServerInformTeardownRequest | None = ...,
+ inform_start: global___ServerInformStartRequest | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["inform_attach", b"inform_attach", "inform_detach", b"inform_detach", "inform_finish", b"inform_finish", "inform_init", b"inform_init", "inform_start", b"inform_start", "inform_teardown", b"inform_teardown", "record_communicate", b"record_communicate", "record_publish", b"record_publish", "server_request_type", b"server_request_type"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["inform_attach", b"inform_attach", "inform_detach", b"inform_detach", "inform_finish", b"inform_finish", "inform_init", b"inform_init", "inform_start", b"inform_start", "inform_teardown", b"inform_teardown", "record_communicate", b"record_communicate", "record_publish", b"record_publish", "server_request_type", b"server_request_type"]) -> None: ...
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["server_request_type", b"server_request_type"]) -> typing_extensions.Literal["record_publish", "record_communicate", "inform_init", "inform_finish", "inform_attach", "inform_detach", "inform_teardown", "inform_start"] | None: ...
+
+global___ServerRequest = ServerRequest
+
+class ServerResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RESULT_COMMUNICATE_FIELD_NUMBER: builtins.int
+ INFORM_INIT_RESPONSE_FIELD_NUMBER: builtins.int
+ INFORM_FINISH_RESPONSE_FIELD_NUMBER: builtins.int
+ INFORM_ATTACH_RESPONSE_FIELD_NUMBER: builtins.int
+ INFORM_DETACH_RESPONSE_FIELD_NUMBER: builtins.int
+ INFORM_TEARDOWN_RESPONSE_FIELD_NUMBER: builtins.int
+ INFORM_START_RESPONSE_FIELD_NUMBER: builtins.int
+ @property
+ def result_communicate(self) -> wandb.proto.wandb_internal_pb2.Result: ...
+ @property
+ def inform_init_response(self) -> global___ServerInformInitResponse: ...
+ @property
+ def inform_finish_response(self) -> global___ServerInformFinishResponse: ...
+ @property
+ def inform_attach_response(self) -> global___ServerInformAttachResponse: ...
+ @property
+ def inform_detach_response(self) -> global___ServerInformDetachResponse: ...
+ @property
+ def inform_teardown_response(self) -> global___ServerInformTeardownResponse: ...
+ @property
+ def inform_start_response(self) -> global___ServerInformStartResponse: ...
+ def __init__(
+ self,
+ *,
+ result_communicate: wandb.proto.wandb_internal_pb2.Result | None = ...,
+ inform_init_response: global___ServerInformInitResponse | None = ...,
+ inform_finish_response: global___ServerInformFinishResponse | None = ...,
+ inform_attach_response: global___ServerInformAttachResponse | None = ...,
+ inform_detach_response: global___ServerInformDetachResponse | None = ...,
+ inform_teardown_response: global___ServerInformTeardownResponse | None = ...,
+ inform_start_response: global___ServerInformStartResponse | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["inform_attach_response", b"inform_attach_response", "inform_detach_response", b"inform_detach_response", "inform_finish_response", b"inform_finish_response", "inform_init_response", b"inform_init_response", "inform_start_response", b"inform_start_response", "inform_teardown_response", b"inform_teardown_response", "result_communicate", b"result_communicate", "server_response_type", b"server_response_type"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["inform_attach_response", b"inform_attach_response", "inform_detach_response", b"inform_detach_response", "inform_finish_response", b"inform_finish_response", "inform_init_response", b"inform_init_response", "inform_start_response", b"inform_start_response", "inform_teardown_response", b"inform_teardown_response", "result_communicate", b"result_communicate", "server_response_type", b"server_response_type"]) -> None: ...
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["server_response_type", b"server_response_type"]) -> typing_extensions.Literal["result_communicate", "inform_init_response", "inform_finish_response", "inform_attach_response", "inform_detach_response", "inform_teardown_response", "inform_start_response"] | None: ...
+
+global___ServerResponse = ServerResponse
diff --git a/wandb/proto/v3/wandb_settings_pb2.py b/wandb/proto/v3/wandb_settings_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..beaa4c5761e33ca9af07c608e601654d0d4660b3
--- /dev/null
+++ b/wandb/proto/v3/wandb_settings_pb2.py
@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: wandb/proto/wandb_settings.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n wandb/proto/wandb_settings.proto\x12\x0ewandb_internal\x1a\x1egoogle/protobuf/wrappers.proto\" \n\x0fListStringValue\x12\r\n\x05value\x18\x01 \x03(\t\"\x8a\x01\n\x17MapStringKeyStringValue\x12\x41\n\x05value\x18\x01 \x03(\x0b\x32\x32.wandb_internal.MapStringKeyStringValue.ValueEntry\x1a,\n\nValueEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xcb\x01\n#MapStringKeyMapStringKeyStringValue\x12M\n\x05value\x18\x01 \x03(\x0b\x32>.wandb_internal.MapStringKeyMapStringKeyStringValue.ValueEntry\x1aU\n\nValueEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x36\n\x05value\x18\x02 \x01(\x0b\x32\'.wandb_internal.MapStringKeyStringValue:\x02\x38\x01\"\x9a\x01\n\x12OpenMetricsFilters\x12\x33\n\x08sequence\x18\x01 \x01(\x0b\x32\x1f.wandb_internal.ListStringValueH\x00\x12\x46\n\x07mapping\x18\x02 \x01(\x0b\x32\x33.wandb_internal.MapStringKeyMapStringKeyStringValueH\x00\x42\x07\n\x05value\"\xe1\x43\n\x08Settings\x12.\n\x05_args\x18\x01 \x01(\x0b\x32\x1f.wandb_internal.ListStringValue\x12/\n\x0b_aws_lambda\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x44\n\x1f_async_upload_concurrency_limit\x18\x03 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12\x32\n\x0e_cli_only_mode\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12*\n\x06_colab\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12+\n\x05_cuda\x18\x06 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x31\n\r_disable_meta\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x34\n\x10_disable_service\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x39\n\x15_disable_setproctitle\x18\t \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x32\n\x0e_disable_stats\x18\n \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x33\n\x0f_disable_viewer\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x30\n\x0c_except_exit\x18\x0c \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x31\n\x0b_executable\x18\r \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x44\n\x13_extra_http_headers\x18\x0e \x01(\x0b\x32\'.wandb_internal.MapStringKeyStringValue\x12\x42\n\x1c_file_stream_timeout_seconds\x18\x0f \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x38\n\x14_flow_control_custom\x18\x10 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12:\n\x16_flow_control_disabled\x18\x11 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12=\n\x17_internal_check_process\x18\x12 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12=\n\x17_internal_queue_timeout\x18\x13 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12,\n\x08_ipython\x18\x14 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12,\n\x08_jupyter\x18\x15 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x33\n\r_jupyter_root\x18\x16 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12+\n\x07_kaggle\x18\x17 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12<\n\x17_live_policy_rate_limit\x18\x18 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12;\n\x16_live_policy_wait_time\x18\x19 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12/\n\n_log_level\x18\x1a \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12\x34\n\x0f_network_buffer\x18\x1b \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12)\n\x05_noop\x18\x1c \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12-\n\t_notebook\x18\x1d \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12,\n\x08_offline\x18\x1e \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12)\n\x05_sync\x18\x1f \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12)\n\x03_os\x18 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\t_platform\x18! \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x07_python\x18\" \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x37\n\x11_runqueue_item_id\x18# \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x31\n\r_require_core\x18$ \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x36\n\x12_save_requirements\x18% \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x38\n\x12_service_transport\x18& \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x33\n\r_service_wait\x18\' \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x35\n\x0f_start_datetime\x18( \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x31\n\x0b_start_time\x18) \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12/\n\n_stats_pid\x18* \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12@\n\x1a_stats_sample_rate_seconds\x18+ \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12>\n\x19_stats_samples_to_average\x18, \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12\x36\n\x12_stats_join_assets\x18- \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12G\n!_stats_neuron_monitor_config_path\x18. \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12N\n\x1d_stats_open_metrics_endpoints\x18/ \x01(\x0b\x32\'.wandb_internal.MapStringKeyStringValue\x12G\n\x1b_stats_open_metrics_filters\x18\x30 \x01(\x0b\x32\".wandb_internal.OpenMetricsFilters\x12\x33\n\r_tmp_code_dir\x18\x31 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\t_tracelog\x18\x32 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x36\n\r_unsaved_keys\x18\x33 \x01(\x0b\x32\x1f.wandb_internal.ListStringValue\x12,\n\x08_windows\x18\x34 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x34\n\x10\x61llow_val_change\x18\x35 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12/\n\tanonymous\x18\x36 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x07\x61pi_key\x18\x37 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12P\n\x1f\x61zure_account_url_to_access_key\x18\x38 \x01(\x0b\x32\'.wandb_internal.MapStringKeyStringValue\x12.\n\x08\x62\x61se_url\x18\x39 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12.\n\x08\x63ode_dir\x18: \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x35\n\x0c\x63onfig_paths\x18; \x01(\x0b\x32\x1f.wandb_internal.ListStringValue\x12-\n\x07\x63onsole\x18< \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x30\n\ndeployment\x18= \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x30\n\x0c\x64isable_code\x18> \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12/\n\x0b\x64isable_git\x18? \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x31\n\rdisable_hints\x18@ \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x38\n\x14\x64isable_job_creation\x18\x41 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12,\n\x08\x64isabled\x18\x42 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12,\n\x06\x64ocker\x18\x43 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12+\n\x05\x65mail\x18\x44 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12,\n\x06\x65ntity\x18\x45 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\tfiles_dir\x18\x46 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12)\n\x05\x66orce\x18G \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x30\n\ngit_commit\x18H \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x30\n\ngit_remote\x18I \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x34\n\x0egit_remote_url\x18J \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12.\n\x08git_root\x18K \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x36\n\x11heartbeat_seconds\x18L \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12*\n\x04host\x18M \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x35\n\x0cignore_globs\x18N \x01(\x0b\x32\x1f.wandb_internal.ListStringValue\x12\x32\n\x0cinit_timeout\x18O \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12,\n\x08is_local\x18P \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x30\n\njob_source\x18Q \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x31\n\rlabel_disable\x18R \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12*\n\x06launch\x18S \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x38\n\x12launch_config_path\x18T \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x07log_dir\x18U \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x32\n\x0clog_internal\x18V \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12:\n\x14log_symlink_internal\x18W \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x36\n\x10log_symlink_user\x18X \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12.\n\x08log_user\x18Y \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x33\n\rlogin_timeout\x18Z \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12*\n\x04mode\x18\\ \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x33\n\rnotebook_name\x18] \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x07problem\x18^ \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x07program\x18_ \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x35\n\x0fprogram_relpath\x18` \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x07project\x18\x61 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x31\n\x0bproject_url\x18\x62 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12)\n\x05quiet\x18\x63 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12*\n\x06reinit\x18\x64 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12+\n\x07relogin\x18\x65 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12,\n\x06resume\x18\x66 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x32\n\x0cresume_fname\x18g \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12+\n\x07resumed\x18h \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12.\n\x08root_dir\x18i \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\trun_group\x18j \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12,\n\x06run_id\x18k \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x32\n\x0crun_job_type\x18l \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12.\n\x08run_mode\x18m \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12.\n\x08run_name\x18n \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\trun_notes\x18o \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x31\n\x08run_tags\x18p \x01(\x0b\x32\x1f.wandb_internal.ListStringValue\x12-\n\x07run_url\x18q \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x35\n\x11sagemaker_disable\x18r \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12-\n\tsave_code\x18s \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x35\n\x0fsettings_system\x18t \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x38\n\x12settings_workspace\x18u \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\x0bshow_colors\x18v \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12.\n\nshow_emoji\x18w \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12/\n\x0bshow_errors\x18x \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12-\n\tshow_info\x18y \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x31\n\rshow_warnings\x18z \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12*\n\x06silent\x18{ \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x32\n\x0cstart_method\x18| \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12*\n\x06strict\x18} \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x33\n\x0esummary_errors\x18~ \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12\x34\n\x0fsummary_timeout\x18\x7f \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12\x36\n\x10summary_warnings\x18\x80\x01 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12/\n\x08sweep_id\x18\x81\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x37\n\x10sweep_param_path\x18\x82\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x30\n\tsweep_url\x18\x83\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12,\n\x07symlink\x18\x84\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12/\n\x08sync_dir\x18\x85\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x30\n\tsync_file\x18\x86\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12:\n\x13sync_symlink_latest\x18\x87\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x33\n\rsystem_sample\x18\x88\x01 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12;\n\x15system_sample_seconds\x18\x89\x01 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12J\n%table_raise_on_max_row_limit_exceeded\x18\x8a\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12/\n\x08timespec\x18\x8b\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12.\n\x07tmp_dir\x18\x8c\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\x08username\x18\x8d\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x30\n\twandb_dir\x18\x8e\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x34\n\r_jupyter_name\x18\x8f\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x34\n\r_jupyter_path\x18\x90\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\x08job_name\x18\x91\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12;\n\x11_stats_disk_paths\x18\x92\x01 \x01(\x0b\x32\x1f.wandb_internal.ListStringValue\x12<\n\x16_file_stream_retry_max\x18\x93\x01 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12J\n#_file_stream_retry_wait_min_seconds\x18\x94\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12J\n#_file_stream_retry_wait_max_seconds\x18\x95\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12>\n\x18_file_transfer_retry_max\x18\x96\x01 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12L\n%_file_transfer_retry_wait_min_seconds\x18\x97\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12L\n%_file_transfer_retry_wait_max_seconds\x18\x98\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x45\n\x1e_file_transfer_timeout_seconds\x18\x99\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x38\n\x12_graphql_retry_max\x18\x9a\x01 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12\x46\n\x1f_graphql_retry_wait_min_seconds\x18\x9b\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x46\n\x1f_graphql_retry_wait_max_seconds\x18\x9c\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12?\n\x18_graphql_timeout_seconds\x18\x9d\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12:\n\x15_disable_machine_info\x18\x9e\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x36\n\x0fprogram_abspath\x18\x9f\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x30\n\tcolab_url\x18\xa0\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x38\n\x12_stats_buffer_size\x18\xa1\x01 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12,\n\x07_shared\x18\xa2\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12:\n\x08_proxies\x18\xc8\x01 \x01(\x0b\x32\'.wandb_internal.MapStringKeyStringValueb\x06proto3')
+
+
+
+_LISTSTRINGVALUE = DESCRIPTOR.message_types_by_name['ListStringValue']
+_MAPSTRINGKEYSTRINGVALUE = DESCRIPTOR.message_types_by_name['MapStringKeyStringValue']
+_MAPSTRINGKEYSTRINGVALUE_VALUEENTRY = _MAPSTRINGKEYSTRINGVALUE.nested_types_by_name['ValueEntry']
+_MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE = DESCRIPTOR.message_types_by_name['MapStringKeyMapStringKeyStringValue']
+_MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE_VALUEENTRY = _MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE.nested_types_by_name['ValueEntry']
+_OPENMETRICSFILTERS = DESCRIPTOR.message_types_by_name['OpenMetricsFilters']
+_SETTINGS = DESCRIPTOR.message_types_by_name['Settings']
+ListStringValue = _reflection.GeneratedProtocolMessageType('ListStringValue', (_message.Message,), {
+ 'DESCRIPTOR' : _LISTSTRINGVALUE,
+ '__module__' : 'wandb.proto.wandb_settings_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.ListStringValue)
+ })
+_sym_db.RegisterMessage(ListStringValue)
+
+MapStringKeyStringValue = _reflection.GeneratedProtocolMessageType('MapStringKeyStringValue', (_message.Message,), {
+
+ 'ValueEntry' : _reflection.GeneratedProtocolMessageType('ValueEntry', (_message.Message,), {
+ 'DESCRIPTOR' : _MAPSTRINGKEYSTRINGVALUE_VALUEENTRY,
+ '__module__' : 'wandb.proto.wandb_settings_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.MapStringKeyStringValue.ValueEntry)
+ })
+ ,
+ 'DESCRIPTOR' : _MAPSTRINGKEYSTRINGVALUE,
+ '__module__' : 'wandb.proto.wandb_settings_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.MapStringKeyStringValue)
+ })
+_sym_db.RegisterMessage(MapStringKeyStringValue)
+_sym_db.RegisterMessage(MapStringKeyStringValue.ValueEntry)
+
+MapStringKeyMapStringKeyStringValue = _reflection.GeneratedProtocolMessageType('MapStringKeyMapStringKeyStringValue', (_message.Message,), {
+
+ 'ValueEntry' : _reflection.GeneratedProtocolMessageType('ValueEntry', (_message.Message,), {
+ 'DESCRIPTOR' : _MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE_VALUEENTRY,
+ '__module__' : 'wandb.proto.wandb_settings_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.MapStringKeyMapStringKeyStringValue.ValueEntry)
+ })
+ ,
+ 'DESCRIPTOR' : _MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE,
+ '__module__' : 'wandb.proto.wandb_settings_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.MapStringKeyMapStringKeyStringValue)
+ })
+_sym_db.RegisterMessage(MapStringKeyMapStringKeyStringValue)
+_sym_db.RegisterMessage(MapStringKeyMapStringKeyStringValue.ValueEntry)
+
+OpenMetricsFilters = _reflection.GeneratedProtocolMessageType('OpenMetricsFilters', (_message.Message,), {
+ 'DESCRIPTOR' : _OPENMETRICSFILTERS,
+ '__module__' : 'wandb.proto.wandb_settings_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.OpenMetricsFilters)
+ })
+_sym_db.RegisterMessage(OpenMetricsFilters)
+
+Settings = _reflection.GeneratedProtocolMessageType('Settings', (_message.Message,), {
+ 'DESCRIPTOR' : _SETTINGS,
+ '__module__' : 'wandb.proto.wandb_settings_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.Settings)
+ })
+_sym_db.RegisterMessage(Settings)
+
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+ DESCRIPTOR._options = None
+ _MAPSTRINGKEYSTRINGVALUE_VALUEENTRY._options = None
+ _MAPSTRINGKEYSTRINGVALUE_VALUEENTRY._serialized_options = b'8\001'
+ _MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE_VALUEENTRY._options = None
+ _MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE_VALUEENTRY._serialized_options = b'8\001'
+ _LISTSTRINGVALUE._serialized_start=84
+ _LISTSTRINGVALUE._serialized_end=116
+ _MAPSTRINGKEYSTRINGVALUE._serialized_start=119
+ _MAPSTRINGKEYSTRINGVALUE._serialized_end=257
+ _MAPSTRINGKEYSTRINGVALUE_VALUEENTRY._serialized_start=213
+ _MAPSTRINGKEYSTRINGVALUE_VALUEENTRY._serialized_end=257
+ _MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE._serialized_start=260
+ _MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE._serialized_end=463
+ _MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE_VALUEENTRY._serialized_start=378
+ _MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE_VALUEENTRY._serialized_end=463
+ _OPENMETRICSFILTERS._serialized_start=466
+ _OPENMETRICSFILTERS._serialized_end=620
+ _SETTINGS._serialized_start=623
+ _SETTINGS._serialized_end=9296
+# @@protoc_insertion_point(module_scope)
diff --git a/wandb/proto/v3/wandb_settings_pb2.pyi b/wandb/proto/v3/wandb_settings_pb2.pyi
new file mode 100644
index 0000000000000000000000000000000000000000..445497db6525e85b8bc5bdda8be6c4ff4da2ecec
--- /dev/null
+++ b/wandb/proto/v3/wandb_settings_pb2.pyi
@@ -0,0 +1,776 @@
+"""
+@generated by mypy-protobuf. Do not edit manually!
+isort:skip_file
+"""
+import builtins
+import collections.abc
+import google.protobuf.descriptor
+import google.protobuf.internal.containers
+import google.protobuf.message
+import google.protobuf.wrappers_pb2
+import sys
+
+if sys.version_info >= (3, 8):
+ import typing as typing_extensions
+else:
+ import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+class ListStringValue(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ VALUE_FIELD_NUMBER: builtins.int
+ @property
+ def value(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ def __init__(
+ self,
+ *,
+ value: collections.abc.Iterable[builtins.str] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["value", b"value"]) -> None: ...
+
+global___ListStringValue = ListStringValue
+
+class MapStringKeyStringValue(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class ValueEntry(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ value: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ...
+
+ VALUE_FIELD_NUMBER: builtins.int
+ @property
+ def value(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ...
+ def __init__(
+ self,
+ *,
+ value: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["value", b"value"]) -> None: ...
+
+global___MapStringKeyStringValue = MapStringKeyStringValue
+
+class MapStringKeyMapStringKeyStringValue(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class ValueEntry(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ @property
+ def value(self) -> global___MapStringKeyStringValue: ...
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value: global___MapStringKeyStringValue | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["value", b"value"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ...
+
+ VALUE_FIELD_NUMBER: builtins.int
+ @property
+ def value(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___MapStringKeyStringValue]: ...
+ def __init__(
+ self,
+ *,
+ value: collections.abc.Mapping[builtins.str, global___MapStringKeyStringValue] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["value", b"value"]) -> None: ...
+
+global___MapStringKeyMapStringKeyStringValue = MapStringKeyMapStringKeyStringValue
+
+class OpenMetricsFilters(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SEQUENCE_FIELD_NUMBER: builtins.int
+ MAPPING_FIELD_NUMBER: builtins.int
+ @property
+ def sequence(self) -> global___ListStringValue: ...
+ @property
+ def mapping(self) -> global___MapStringKeyMapStringKeyStringValue: ...
+ def __init__(
+ self,
+ *,
+ sequence: global___ListStringValue | None = ...,
+ mapping: global___MapStringKeyMapStringKeyStringValue | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["mapping", b"mapping", "sequence", b"sequence", "value", b"value"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["mapping", b"mapping", "sequence", b"sequence", "value", b"value"]) -> None: ...
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["value", b"value"]) -> typing_extensions.Literal["sequence", "mapping"] | None: ...
+
+global___OpenMetricsFilters = OpenMetricsFilters
+
+class Settings(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _ARGS_FIELD_NUMBER: builtins.int
+ _AWS_LAMBDA_FIELD_NUMBER: builtins.int
+ _ASYNC_UPLOAD_CONCURRENCY_LIMIT_FIELD_NUMBER: builtins.int
+ _CLI_ONLY_MODE_FIELD_NUMBER: builtins.int
+ _COLAB_FIELD_NUMBER: builtins.int
+ _CUDA_FIELD_NUMBER: builtins.int
+ _DISABLE_META_FIELD_NUMBER: builtins.int
+ _DISABLE_SERVICE_FIELD_NUMBER: builtins.int
+ _DISABLE_SETPROCTITLE_FIELD_NUMBER: builtins.int
+ _DISABLE_STATS_FIELD_NUMBER: builtins.int
+ _DISABLE_VIEWER_FIELD_NUMBER: builtins.int
+ _EXCEPT_EXIT_FIELD_NUMBER: builtins.int
+ _EXECUTABLE_FIELD_NUMBER: builtins.int
+ _EXTRA_HTTP_HEADERS_FIELD_NUMBER: builtins.int
+ _FILE_STREAM_TIMEOUT_SECONDS_FIELD_NUMBER: builtins.int
+ _FLOW_CONTROL_CUSTOM_FIELD_NUMBER: builtins.int
+ _FLOW_CONTROL_DISABLED_FIELD_NUMBER: builtins.int
+ _INTERNAL_CHECK_PROCESS_FIELD_NUMBER: builtins.int
+ _INTERNAL_QUEUE_TIMEOUT_FIELD_NUMBER: builtins.int
+ _IPYTHON_FIELD_NUMBER: builtins.int
+ _JUPYTER_FIELD_NUMBER: builtins.int
+ _JUPYTER_ROOT_FIELD_NUMBER: builtins.int
+ _KAGGLE_FIELD_NUMBER: builtins.int
+ _LIVE_POLICY_RATE_LIMIT_FIELD_NUMBER: builtins.int
+ _LIVE_POLICY_WAIT_TIME_FIELD_NUMBER: builtins.int
+ _LOG_LEVEL_FIELD_NUMBER: builtins.int
+ _NETWORK_BUFFER_FIELD_NUMBER: builtins.int
+ _NOOP_FIELD_NUMBER: builtins.int
+ _NOTEBOOK_FIELD_NUMBER: builtins.int
+ _OFFLINE_FIELD_NUMBER: builtins.int
+ _SYNC_FIELD_NUMBER: builtins.int
+ _OS_FIELD_NUMBER: builtins.int
+ _PLATFORM_FIELD_NUMBER: builtins.int
+ _PYTHON_FIELD_NUMBER: builtins.int
+ _RUNQUEUE_ITEM_ID_FIELD_NUMBER: builtins.int
+ _REQUIRE_CORE_FIELD_NUMBER: builtins.int
+ _SAVE_REQUIREMENTS_FIELD_NUMBER: builtins.int
+ _SERVICE_TRANSPORT_FIELD_NUMBER: builtins.int
+ _SERVICE_WAIT_FIELD_NUMBER: builtins.int
+ _START_DATETIME_FIELD_NUMBER: builtins.int
+ _START_TIME_FIELD_NUMBER: builtins.int
+ _STATS_PID_FIELD_NUMBER: builtins.int
+ _STATS_SAMPLE_RATE_SECONDS_FIELD_NUMBER: builtins.int
+ _STATS_SAMPLES_TO_AVERAGE_FIELD_NUMBER: builtins.int
+ _STATS_JOIN_ASSETS_FIELD_NUMBER: builtins.int
+ _STATS_NEURON_MONITOR_CONFIG_PATH_FIELD_NUMBER: builtins.int
+ _STATS_OPEN_METRICS_ENDPOINTS_FIELD_NUMBER: builtins.int
+ _STATS_OPEN_METRICS_FILTERS_FIELD_NUMBER: builtins.int
+ _TMP_CODE_DIR_FIELD_NUMBER: builtins.int
+ _TRACELOG_FIELD_NUMBER: builtins.int
+ _UNSAVED_KEYS_FIELD_NUMBER: builtins.int
+ _WINDOWS_FIELD_NUMBER: builtins.int
+ ALLOW_VAL_CHANGE_FIELD_NUMBER: builtins.int
+ ANONYMOUS_FIELD_NUMBER: builtins.int
+ API_KEY_FIELD_NUMBER: builtins.int
+ AZURE_ACCOUNT_URL_TO_ACCESS_KEY_FIELD_NUMBER: builtins.int
+ BASE_URL_FIELD_NUMBER: builtins.int
+ CODE_DIR_FIELD_NUMBER: builtins.int
+ CONFIG_PATHS_FIELD_NUMBER: builtins.int
+ CONSOLE_FIELD_NUMBER: builtins.int
+ DEPLOYMENT_FIELD_NUMBER: builtins.int
+ DISABLE_CODE_FIELD_NUMBER: builtins.int
+ DISABLE_GIT_FIELD_NUMBER: builtins.int
+ DISABLE_HINTS_FIELD_NUMBER: builtins.int
+ DISABLE_JOB_CREATION_FIELD_NUMBER: builtins.int
+ DISABLED_FIELD_NUMBER: builtins.int
+ DOCKER_FIELD_NUMBER: builtins.int
+ EMAIL_FIELD_NUMBER: builtins.int
+ ENTITY_FIELD_NUMBER: builtins.int
+ FILES_DIR_FIELD_NUMBER: builtins.int
+ FORCE_FIELD_NUMBER: builtins.int
+ GIT_COMMIT_FIELD_NUMBER: builtins.int
+ GIT_REMOTE_FIELD_NUMBER: builtins.int
+ GIT_REMOTE_URL_FIELD_NUMBER: builtins.int
+ GIT_ROOT_FIELD_NUMBER: builtins.int
+ HEARTBEAT_SECONDS_FIELD_NUMBER: builtins.int
+ HOST_FIELD_NUMBER: builtins.int
+ IGNORE_GLOBS_FIELD_NUMBER: builtins.int
+ INIT_TIMEOUT_FIELD_NUMBER: builtins.int
+ IS_LOCAL_FIELD_NUMBER: builtins.int
+ JOB_SOURCE_FIELD_NUMBER: builtins.int
+ LABEL_DISABLE_FIELD_NUMBER: builtins.int
+ LAUNCH_FIELD_NUMBER: builtins.int
+ LAUNCH_CONFIG_PATH_FIELD_NUMBER: builtins.int
+ LOG_DIR_FIELD_NUMBER: builtins.int
+ LOG_INTERNAL_FIELD_NUMBER: builtins.int
+ LOG_SYMLINK_INTERNAL_FIELD_NUMBER: builtins.int
+ LOG_SYMLINK_USER_FIELD_NUMBER: builtins.int
+ LOG_USER_FIELD_NUMBER: builtins.int
+ LOGIN_TIMEOUT_FIELD_NUMBER: builtins.int
+ MODE_FIELD_NUMBER: builtins.int
+ NOTEBOOK_NAME_FIELD_NUMBER: builtins.int
+ PROBLEM_FIELD_NUMBER: builtins.int
+ PROGRAM_FIELD_NUMBER: builtins.int
+ PROGRAM_RELPATH_FIELD_NUMBER: builtins.int
+ PROJECT_FIELD_NUMBER: builtins.int
+ PROJECT_URL_FIELD_NUMBER: builtins.int
+ QUIET_FIELD_NUMBER: builtins.int
+ REINIT_FIELD_NUMBER: builtins.int
+ RELOGIN_FIELD_NUMBER: builtins.int
+ RESUME_FIELD_NUMBER: builtins.int
+ RESUME_FNAME_FIELD_NUMBER: builtins.int
+ RESUMED_FIELD_NUMBER: builtins.int
+ ROOT_DIR_FIELD_NUMBER: builtins.int
+ RUN_GROUP_FIELD_NUMBER: builtins.int
+ RUN_ID_FIELD_NUMBER: builtins.int
+ RUN_JOB_TYPE_FIELD_NUMBER: builtins.int
+ RUN_MODE_FIELD_NUMBER: builtins.int
+ RUN_NAME_FIELD_NUMBER: builtins.int
+ RUN_NOTES_FIELD_NUMBER: builtins.int
+ RUN_TAGS_FIELD_NUMBER: builtins.int
+ RUN_URL_FIELD_NUMBER: builtins.int
+ SAGEMAKER_DISABLE_FIELD_NUMBER: builtins.int
+ SAVE_CODE_FIELD_NUMBER: builtins.int
+ SETTINGS_SYSTEM_FIELD_NUMBER: builtins.int
+ SETTINGS_WORKSPACE_FIELD_NUMBER: builtins.int
+ SHOW_COLORS_FIELD_NUMBER: builtins.int
+ SHOW_EMOJI_FIELD_NUMBER: builtins.int
+ SHOW_ERRORS_FIELD_NUMBER: builtins.int
+ SHOW_INFO_FIELD_NUMBER: builtins.int
+ SHOW_WARNINGS_FIELD_NUMBER: builtins.int
+ SILENT_FIELD_NUMBER: builtins.int
+ START_METHOD_FIELD_NUMBER: builtins.int
+ STRICT_FIELD_NUMBER: builtins.int
+ SUMMARY_ERRORS_FIELD_NUMBER: builtins.int
+ SUMMARY_TIMEOUT_FIELD_NUMBER: builtins.int
+ SUMMARY_WARNINGS_FIELD_NUMBER: builtins.int
+ SWEEP_ID_FIELD_NUMBER: builtins.int
+ SWEEP_PARAM_PATH_FIELD_NUMBER: builtins.int
+ SWEEP_URL_FIELD_NUMBER: builtins.int
+ SYMLINK_FIELD_NUMBER: builtins.int
+ SYNC_DIR_FIELD_NUMBER: builtins.int
+ SYNC_FILE_FIELD_NUMBER: builtins.int
+ SYNC_SYMLINK_LATEST_FIELD_NUMBER: builtins.int
+ SYSTEM_SAMPLE_FIELD_NUMBER: builtins.int
+ SYSTEM_SAMPLE_SECONDS_FIELD_NUMBER: builtins.int
+ TABLE_RAISE_ON_MAX_ROW_LIMIT_EXCEEDED_FIELD_NUMBER: builtins.int
+ TIMESPEC_FIELD_NUMBER: builtins.int
+ TMP_DIR_FIELD_NUMBER: builtins.int
+ USERNAME_FIELD_NUMBER: builtins.int
+ WANDB_DIR_FIELD_NUMBER: builtins.int
+ _JUPYTER_NAME_FIELD_NUMBER: builtins.int
+ _JUPYTER_PATH_FIELD_NUMBER: builtins.int
+ JOB_NAME_FIELD_NUMBER: builtins.int
+ _STATS_DISK_PATHS_FIELD_NUMBER: builtins.int
+ _FILE_STREAM_RETRY_MAX_FIELD_NUMBER: builtins.int
+ _FILE_STREAM_RETRY_WAIT_MIN_SECONDS_FIELD_NUMBER: builtins.int
+ _FILE_STREAM_RETRY_WAIT_MAX_SECONDS_FIELD_NUMBER: builtins.int
+ _FILE_TRANSFER_RETRY_MAX_FIELD_NUMBER: builtins.int
+ _FILE_TRANSFER_RETRY_WAIT_MIN_SECONDS_FIELD_NUMBER: builtins.int
+ _FILE_TRANSFER_RETRY_WAIT_MAX_SECONDS_FIELD_NUMBER: builtins.int
+ _FILE_TRANSFER_TIMEOUT_SECONDS_FIELD_NUMBER: builtins.int
+ _GRAPHQL_RETRY_MAX_FIELD_NUMBER: builtins.int
+ _GRAPHQL_RETRY_WAIT_MIN_SECONDS_FIELD_NUMBER: builtins.int
+ _GRAPHQL_RETRY_WAIT_MAX_SECONDS_FIELD_NUMBER: builtins.int
+ _GRAPHQL_TIMEOUT_SECONDS_FIELD_NUMBER: builtins.int
+ _DISABLE_MACHINE_INFO_FIELD_NUMBER: builtins.int
+ PROGRAM_ABSPATH_FIELD_NUMBER: builtins.int
+ COLAB_URL_FIELD_NUMBER: builtins.int
+ _STATS_BUFFER_SIZE_FIELD_NUMBER: builtins.int
+ _SHARED_FIELD_NUMBER: builtins.int
+ _PROXIES_FIELD_NUMBER: builtins.int
+ @property
+ def _args(self) -> global___ListStringValue: ...
+ @property
+ def _aws_lambda(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _async_upload_concurrency_limit(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _cli_only_mode(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _colab(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _cuda(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _disable_meta(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _disable_service(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _disable_setproctitle(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _disable_stats(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _disable_viewer(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _except_exit(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _executable(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _extra_http_headers(self) -> global___MapStringKeyStringValue: ...
+ @property
+ def _file_stream_timeout_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _flow_control_custom(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _flow_control_disabled(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _internal_check_process(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _internal_queue_timeout(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _ipython(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _jupyter(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _jupyter_root(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _kaggle(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _live_policy_rate_limit(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _live_policy_wait_time(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _log_level(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _network_buffer(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _noop(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _notebook(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _offline(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _sync(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _os(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _platform(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _python(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _runqueue_item_id(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _require_core(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _save_requirements(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _service_transport(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _service_wait(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _start_datetime(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _start_time(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _stats_pid(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _stats_sample_rate_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _stats_samples_to_average(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _stats_join_assets(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _stats_neuron_monitor_config_path(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _stats_open_metrics_endpoints(self) -> global___MapStringKeyStringValue: ...
+ @property
+ def _stats_open_metrics_filters(self) -> global___OpenMetricsFilters: ...
+ @property
+ def _tmp_code_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _tracelog(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _unsaved_keys(self) -> global___ListStringValue: ...
+ @property
+ def _windows(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def allow_val_change(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def anonymous(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def api_key(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def azure_account_url_to_access_key(self) -> global___MapStringKeyStringValue: ...
+ @property
+ def base_url(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def code_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def config_paths(self) -> global___ListStringValue: ...
+ @property
+ def console(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def deployment(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def disable_code(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def disable_git(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def disable_hints(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def disable_job_creation(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def disabled(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def docker(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def email(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def entity(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def files_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def force(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def git_commit(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def git_remote(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def git_remote_url(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def git_root(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def heartbeat_seconds(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def host(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def ignore_globs(self) -> global___ListStringValue: ...
+ @property
+ def init_timeout(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def is_local(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def job_source(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def label_disable(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def launch(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def launch_config_path(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def log_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def log_internal(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def log_symlink_internal(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def log_symlink_user(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def log_user(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def login_timeout(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def mode(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def notebook_name(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def problem(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def program(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def program_relpath(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def project(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def project_url(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def quiet(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def reinit(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def relogin(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def resume(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def resume_fname(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def resumed(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def root_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def run_group(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def run_id(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def run_job_type(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def run_mode(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def run_name(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def run_notes(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def run_tags(self) -> global___ListStringValue: ...
+ @property
+ def run_url(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def sagemaker_disable(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def save_code(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def settings_system(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def settings_workspace(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def show_colors(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def show_emoji(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def show_errors(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def show_info(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def show_warnings(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def silent(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def start_method(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def strict(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def summary_errors(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def summary_timeout(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def summary_warnings(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def sweep_id(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def sweep_param_path(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def sweep_url(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def symlink(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def sync_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def sync_file(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def sync_symlink_latest(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def system_sample(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def system_sample_seconds(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def table_raise_on_max_row_limit_exceeded(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def timespec(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def tmp_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def username(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def wandb_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _jupyter_name(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _jupyter_path(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def job_name(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _stats_disk_paths(self) -> global___ListStringValue: ...
+ @property
+ def _file_stream_retry_max(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _file_stream_retry_wait_min_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _file_stream_retry_wait_max_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _file_transfer_retry_max(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _file_transfer_retry_wait_min_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _file_transfer_retry_wait_max_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _file_transfer_timeout_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _graphql_retry_max(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _graphql_retry_wait_min_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _graphql_retry_wait_max_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _graphql_timeout_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _disable_machine_info(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def program_abspath(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def colab_url(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _stats_buffer_size(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _shared(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _proxies(self) -> global___MapStringKeyStringValue: ...
+ def __init__(
+ self,
+ *,
+ _args: global___ListStringValue | None = ...,
+ _aws_lambda: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _async_upload_concurrency_limit: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _cli_only_mode: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _colab: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _cuda: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _disable_meta: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _disable_service: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _disable_setproctitle: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _disable_stats: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _disable_viewer: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _except_exit: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _executable: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _extra_http_headers: global___MapStringKeyStringValue | None = ...,
+ _file_stream_timeout_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _flow_control_custom: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _flow_control_disabled: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _internal_check_process: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _internal_queue_timeout: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _ipython: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _jupyter: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _jupyter_root: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _kaggle: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _live_policy_rate_limit: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _live_policy_wait_time: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _log_level: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _network_buffer: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _noop: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _notebook: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _offline: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _sync: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _os: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _platform: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _python: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _runqueue_item_id: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _require_core: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _save_requirements: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _service_transport: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _service_wait: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _start_datetime: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _start_time: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _stats_pid: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _stats_sample_rate_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _stats_samples_to_average: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _stats_join_assets: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _stats_neuron_monitor_config_path: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _stats_open_metrics_endpoints: global___MapStringKeyStringValue | None = ...,
+ _stats_open_metrics_filters: global___OpenMetricsFilters | None = ...,
+ _tmp_code_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _tracelog: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _unsaved_keys: global___ListStringValue | None = ...,
+ _windows: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ allow_val_change: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ anonymous: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ api_key: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ azure_account_url_to_access_key: global___MapStringKeyStringValue | None = ...,
+ base_url: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ code_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ config_paths: global___ListStringValue | None = ...,
+ console: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ deployment: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ disable_code: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ disable_git: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ disable_hints: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ disable_job_creation: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ disabled: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ docker: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ email: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ entity: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ files_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ force: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ git_commit: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ git_remote: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ git_remote_url: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ git_root: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ heartbeat_seconds: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ host: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ ignore_globs: global___ListStringValue | None = ...,
+ init_timeout: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ is_local: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ job_source: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ label_disable: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ launch: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ launch_config_path: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ log_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ log_internal: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ log_symlink_internal: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ log_symlink_user: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ log_user: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ login_timeout: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ mode: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ notebook_name: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ problem: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ program: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ program_relpath: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ project: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ project_url: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ quiet: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ reinit: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ relogin: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ resume: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ resume_fname: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ resumed: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ root_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ run_group: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ run_id: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ run_job_type: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ run_mode: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ run_name: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ run_notes: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ run_tags: global___ListStringValue | None = ...,
+ run_url: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ sagemaker_disable: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ save_code: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ settings_system: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ settings_workspace: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ show_colors: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ show_emoji: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ show_errors: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ show_info: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ show_warnings: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ silent: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ start_method: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ strict: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ summary_errors: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ summary_timeout: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ summary_warnings: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ sweep_id: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ sweep_param_path: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ sweep_url: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ symlink: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ sync_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ sync_file: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ sync_symlink_latest: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ system_sample: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ system_sample_seconds: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ table_raise_on_max_row_limit_exceeded: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ timespec: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ tmp_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ username: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ wandb_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _jupyter_name: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _jupyter_path: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ job_name: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _stats_disk_paths: global___ListStringValue | None = ...,
+ _file_stream_retry_max: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _file_stream_retry_wait_min_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _file_stream_retry_wait_max_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _file_transfer_retry_max: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _file_transfer_retry_wait_min_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _file_transfer_retry_wait_max_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _file_transfer_timeout_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _graphql_retry_max: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _graphql_retry_wait_min_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _graphql_retry_wait_max_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _graphql_timeout_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _disable_machine_info: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ program_abspath: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ colab_url: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _stats_buffer_size: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _shared: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _proxies: global___MapStringKeyStringValue | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_args", b"_args", "_async_upload_concurrency_limit", b"_async_upload_concurrency_limit", "_aws_lambda", b"_aws_lambda", "_cli_only_mode", b"_cli_only_mode", "_colab", b"_colab", "_cuda", b"_cuda", "_disable_machine_info", b"_disable_machine_info", "_disable_meta", b"_disable_meta", "_disable_service", b"_disable_service", "_disable_setproctitle", b"_disable_setproctitle", "_disable_stats", b"_disable_stats", "_disable_viewer", b"_disable_viewer", "_except_exit", b"_except_exit", "_executable", b"_executable", "_extra_http_headers", b"_extra_http_headers", "_file_stream_retry_max", b"_file_stream_retry_max", "_file_stream_retry_wait_max_seconds", b"_file_stream_retry_wait_max_seconds", "_file_stream_retry_wait_min_seconds", b"_file_stream_retry_wait_min_seconds", "_file_stream_timeout_seconds", b"_file_stream_timeout_seconds", "_file_transfer_retry_max", b"_file_transfer_retry_max", "_file_transfer_retry_wait_max_seconds", b"_file_transfer_retry_wait_max_seconds", "_file_transfer_retry_wait_min_seconds", b"_file_transfer_retry_wait_min_seconds", "_file_transfer_timeout_seconds", b"_file_transfer_timeout_seconds", "_flow_control_custom", b"_flow_control_custom", "_flow_control_disabled", b"_flow_control_disabled", "_graphql_retry_max", b"_graphql_retry_max", "_graphql_retry_wait_max_seconds", b"_graphql_retry_wait_max_seconds", "_graphql_retry_wait_min_seconds", b"_graphql_retry_wait_min_seconds", "_graphql_timeout_seconds", b"_graphql_timeout_seconds", "_internal_check_process", b"_internal_check_process", "_internal_queue_timeout", b"_internal_queue_timeout", "_ipython", b"_ipython", "_jupyter", b"_jupyter", "_jupyter_name", b"_jupyter_name", "_jupyter_path", b"_jupyter_path", "_jupyter_root", b"_jupyter_root", "_kaggle", b"_kaggle", "_live_policy_rate_limit", b"_live_policy_rate_limit", "_live_policy_wait_time", b"_live_policy_wait_time", "_log_level", b"_log_level", "_network_buffer", b"_network_buffer", "_noop", b"_noop", "_notebook", b"_notebook", "_offline", b"_offline", "_os", b"_os", "_platform", b"_platform", "_proxies", b"_proxies", "_python", b"_python", "_require_core", b"_require_core", "_runqueue_item_id", b"_runqueue_item_id", "_save_requirements", b"_save_requirements", "_service_transport", b"_service_transport", "_service_wait", b"_service_wait", "_shared", b"_shared", "_start_datetime", b"_start_datetime", "_start_time", b"_start_time", "_stats_buffer_size", b"_stats_buffer_size", "_stats_disk_paths", b"_stats_disk_paths", "_stats_join_assets", b"_stats_join_assets", "_stats_neuron_monitor_config_path", b"_stats_neuron_monitor_config_path", "_stats_open_metrics_endpoints", b"_stats_open_metrics_endpoints", "_stats_open_metrics_filters", b"_stats_open_metrics_filters", "_stats_pid", b"_stats_pid", "_stats_sample_rate_seconds", b"_stats_sample_rate_seconds", "_stats_samples_to_average", b"_stats_samples_to_average", "_sync", b"_sync", "_tmp_code_dir", b"_tmp_code_dir", "_tracelog", b"_tracelog", "_unsaved_keys", b"_unsaved_keys", "_windows", b"_windows", "allow_val_change", b"allow_val_change", "anonymous", b"anonymous", "api_key", b"api_key", "azure_account_url_to_access_key", b"azure_account_url_to_access_key", "base_url", b"base_url", "code_dir", b"code_dir", "colab_url", b"colab_url", "config_paths", b"config_paths", "console", b"console", "deployment", b"deployment", "disable_code", b"disable_code", "disable_git", b"disable_git", "disable_hints", b"disable_hints", "disable_job_creation", b"disable_job_creation", "disabled", b"disabled", "docker", b"docker", "email", b"email", "entity", b"entity", "files_dir", b"files_dir", "force", b"force", "git_commit", b"git_commit", "git_remote", b"git_remote", "git_remote_url", b"git_remote_url", "git_root", b"git_root", "heartbeat_seconds", b"heartbeat_seconds", "host", b"host", "ignore_globs", b"ignore_globs", "init_timeout", b"init_timeout", "is_local", b"is_local", "job_name", b"job_name", "job_source", b"job_source", "label_disable", b"label_disable", "launch", b"launch", "launch_config_path", b"launch_config_path", "log_dir", b"log_dir", "log_internal", b"log_internal", "log_symlink_internal", b"log_symlink_internal", "log_symlink_user", b"log_symlink_user", "log_user", b"log_user", "login_timeout", b"login_timeout", "mode", b"mode", "notebook_name", b"notebook_name", "problem", b"problem", "program", b"program", "program_abspath", b"program_abspath", "program_relpath", b"program_relpath", "project", b"project", "project_url", b"project_url", "quiet", b"quiet", "reinit", b"reinit", "relogin", b"relogin", "resume", b"resume", "resume_fname", b"resume_fname", "resumed", b"resumed", "root_dir", b"root_dir", "run_group", b"run_group", "run_id", b"run_id", "run_job_type", b"run_job_type", "run_mode", b"run_mode", "run_name", b"run_name", "run_notes", b"run_notes", "run_tags", b"run_tags", "run_url", b"run_url", "sagemaker_disable", b"sagemaker_disable", "save_code", b"save_code", "settings_system", b"settings_system", "settings_workspace", b"settings_workspace", "show_colors", b"show_colors", "show_emoji", b"show_emoji", "show_errors", b"show_errors", "show_info", b"show_info", "show_warnings", b"show_warnings", "silent", b"silent", "start_method", b"start_method", "strict", b"strict", "summary_errors", b"summary_errors", "summary_timeout", b"summary_timeout", "summary_warnings", b"summary_warnings", "sweep_id", b"sweep_id", "sweep_param_path", b"sweep_param_path", "sweep_url", b"sweep_url", "symlink", b"symlink", "sync_dir", b"sync_dir", "sync_file", b"sync_file", "sync_symlink_latest", b"sync_symlink_latest", "system_sample", b"system_sample", "system_sample_seconds", b"system_sample_seconds", "table_raise_on_max_row_limit_exceeded", b"table_raise_on_max_row_limit_exceeded", "timespec", b"timespec", "tmp_dir", b"tmp_dir", "username", b"username", "wandb_dir", b"wandb_dir"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_args", b"_args", "_async_upload_concurrency_limit", b"_async_upload_concurrency_limit", "_aws_lambda", b"_aws_lambda", "_cli_only_mode", b"_cli_only_mode", "_colab", b"_colab", "_cuda", b"_cuda", "_disable_machine_info", b"_disable_machine_info", "_disable_meta", b"_disable_meta", "_disable_service", b"_disable_service", "_disable_setproctitle", b"_disable_setproctitle", "_disable_stats", b"_disable_stats", "_disable_viewer", b"_disable_viewer", "_except_exit", b"_except_exit", "_executable", b"_executable", "_extra_http_headers", b"_extra_http_headers", "_file_stream_retry_max", b"_file_stream_retry_max", "_file_stream_retry_wait_max_seconds", b"_file_stream_retry_wait_max_seconds", "_file_stream_retry_wait_min_seconds", b"_file_stream_retry_wait_min_seconds", "_file_stream_timeout_seconds", b"_file_stream_timeout_seconds", "_file_transfer_retry_max", b"_file_transfer_retry_max", "_file_transfer_retry_wait_max_seconds", b"_file_transfer_retry_wait_max_seconds", "_file_transfer_retry_wait_min_seconds", b"_file_transfer_retry_wait_min_seconds", "_file_transfer_timeout_seconds", b"_file_transfer_timeout_seconds", "_flow_control_custom", b"_flow_control_custom", "_flow_control_disabled", b"_flow_control_disabled", "_graphql_retry_max", b"_graphql_retry_max", "_graphql_retry_wait_max_seconds", b"_graphql_retry_wait_max_seconds", "_graphql_retry_wait_min_seconds", b"_graphql_retry_wait_min_seconds", "_graphql_timeout_seconds", b"_graphql_timeout_seconds", "_internal_check_process", b"_internal_check_process", "_internal_queue_timeout", b"_internal_queue_timeout", "_ipython", b"_ipython", "_jupyter", b"_jupyter", "_jupyter_name", b"_jupyter_name", "_jupyter_path", b"_jupyter_path", "_jupyter_root", b"_jupyter_root", "_kaggle", b"_kaggle", "_live_policy_rate_limit", b"_live_policy_rate_limit", "_live_policy_wait_time", b"_live_policy_wait_time", "_log_level", b"_log_level", "_network_buffer", b"_network_buffer", "_noop", b"_noop", "_notebook", b"_notebook", "_offline", b"_offline", "_os", b"_os", "_platform", b"_platform", "_proxies", b"_proxies", "_python", b"_python", "_require_core", b"_require_core", "_runqueue_item_id", b"_runqueue_item_id", "_save_requirements", b"_save_requirements", "_service_transport", b"_service_transport", "_service_wait", b"_service_wait", "_shared", b"_shared", "_start_datetime", b"_start_datetime", "_start_time", b"_start_time", "_stats_buffer_size", b"_stats_buffer_size", "_stats_disk_paths", b"_stats_disk_paths", "_stats_join_assets", b"_stats_join_assets", "_stats_neuron_monitor_config_path", b"_stats_neuron_monitor_config_path", "_stats_open_metrics_endpoints", b"_stats_open_metrics_endpoints", "_stats_open_metrics_filters", b"_stats_open_metrics_filters", "_stats_pid", b"_stats_pid", "_stats_sample_rate_seconds", b"_stats_sample_rate_seconds", "_stats_samples_to_average", b"_stats_samples_to_average", "_sync", b"_sync", "_tmp_code_dir", b"_tmp_code_dir", "_tracelog", b"_tracelog", "_unsaved_keys", b"_unsaved_keys", "_windows", b"_windows", "allow_val_change", b"allow_val_change", "anonymous", b"anonymous", "api_key", b"api_key", "azure_account_url_to_access_key", b"azure_account_url_to_access_key", "base_url", b"base_url", "code_dir", b"code_dir", "colab_url", b"colab_url", "config_paths", b"config_paths", "console", b"console", "deployment", b"deployment", "disable_code", b"disable_code", "disable_git", b"disable_git", "disable_hints", b"disable_hints", "disable_job_creation", b"disable_job_creation", "disabled", b"disabled", "docker", b"docker", "email", b"email", "entity", b"entity", "files_dir", b"files_dir", "force", b"force", "git_commit", b"git_commit", "git_remote", b"git_remote", "git_remote_url", b"git_remote_url", "git_root", b"git_root", "heartbeat_seconds", b"heartbeat_seconds", "host", b"host", "ignore_globs", b"ignore_globs", "init_timeout", b"init_timeout", "is_local", b"is_local", "job_name", b"job_name", "job_source", b"job_source", "label_disable", b"label_disable", "launch", b"launch", "launch_config_path", b"launch_config_path", "log_dir", b"log_dir", "log_internal", b"log_internal", "log_symlink_internal", b"log_symlink_internal", "log_symlink_user", b"log_symlink_user", "log_user", b"log_user", "login_timeout", b"login_timeout", "mode", b"mode", "notebook_name", b"notebook_name", "problem", b"problem", "program", b"program", "program_abspath", b"program_abspath", "program_relpath", b"program_relpath", "project", b"project", "project_url", b"project_url", "quiet", b"quiet", "reinit", b"reinit", "relogin", b"relogin", "resume", b"resume", "resume_fname", b"resume_fname", "resumed", b"resumed", "root_dir", b"root_dir", "run_group", b"run_group", "run_id", b"run_id", "run_job_type", b"run_job_type", "run_mode", b"run_mode", "run_name", b"run_name", "run_notes", b"run_notes", "run_tags", b"run_tags", "run_url", b"run_url", "sagemaker_disable", b"sagemaker_disable", "save_code", b"save_code", "settings_system", b"settings_system", "settings_workspace", b"settings_workspace", "show_colors", b"show_colors", "show_emoji", b"show_emoji", "show_errors", b"show_errors", "show_info", b"show_info", "show_warnings", b"show_warnings", "silent", b"silent", "start_method", b"start_method", "strict", b"strict", "summary_errors", b"summary_errors", "summary_timeout", b"summary_timeout", "summary_warnings", b"summary_warnings", "sweep_id", b"sweep_id", "sweep_param_path", b"sweep_param_path", "sweep_url", b"sweep_url", "symlink", b"symlink", "sync_dir", b"sync_dir", "sync_file", b"sync_file", "sync_symlink_latest", b"sync_symlink_latest", "system_sample", b"system_sample", "system_sample_seconds", b"system_sample_seconds", "table_raise_on_max_row_limit_exceeded", b"table_raise_on_max_row_limit_exceeded", "timespec", b"timespec", "tmp_dir", b"tmp_dir", "username", b"username", "wandb_dir", b"wandb_dir"]) -> None: ...
+
+global___Settings = Settings
diff --git a/wandb/proto/v3/wandb_telemetry_pb2.py b/wandb/proto/v3/wandb_telemetry_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..c932db321502d718a7e5e0f270aa2ededb00a646
--- /dev/null
+++ b/wandb/proto/v3/wandb_telemetry_pb2.py
@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: wandb/proto/wandb_telemetry.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from wandb.proto import wandb_base_pb2 as wandb_dot_proto_dot_wandb__base__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n!wandb/proto/wandb_telemetry.proto\x12\x0ewandb_internal\x1a\x1cwandb/proto/wandb_base.proto\"\xdb\x03\n\x0fTelemetryRecord\x12-\n\x0cimports_init\x18\x01 \x01(\x0b\x32\x17.wandb_internal.Imports\x12/\n\x0eimports_finish\x18\x02 \x01(\x0b\x32\x17.wandb_internal.Imports\x12(\n\x07\x66\x65\x61ture\x18\x03 \x01(\x0b\x32\x17.wandb_internal.Feature\x12\x16\n\x0epython_version\x18\x04 \x01(\t\x12\x13\n\x0b\x63li_version\x18\x05 \x01(\t\x12\x1b\n\x13huggingface_version\x18\x06 \x01(\t\x12 \n\x03\x65nv\x18\x08 \x01(\x0b\x32\x13.wandb_internal.Env\x12%\n\x05label\x18\t \x01(\x0b\x32\x16.wandb_internal.Labels\x12.\n\ndeprecated\x18\n \x01(\x0b\x32\x1a.wandb_internal.Deprecated\x12&\n\x06issues\x18\x0b \x01(\x0b\x32\x16.wandb_internal.Issues\x12\x14\n\x0c\x63ore_version\x18\x0c \x01(\t\x12\x10\n\x08platform\x18\r \x01(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x11\n\x0fTelemetryResult\"\xe2\r\n\x07Imports\x12\r\n\x05torch\x18\x01 \x01(\x08\x12\r\n\x05keras\x18\x02 \x01(\x08\x12\x12\n\ntensorflow\x18\x03 \x01(\x08\x12\x0e\n\x06\x66\x61stai\x18\x04 \x01(\x08\x12\x0f\n\x07sklearn\x18\x05 \x01(\x08\x12\x0f\n\x07xgboost\x18\x06 \x01(\x08\x12\x10\n\x08\x63\x61tboost\x18\x07 \x01(\x08\x12\x10\n\x08lightgbm\x18\x08 \x01(\x08\x12\x19\n\x11pytorch_lightning\x18\t \x01(\x08\x12\x0e\n\x06ignite\x18\n \x01(\x08\x12\x14\n\x0ctransformers\x18\x0b \x01(\x08\x12\x0b\n\x03jax\x18\x0c \x01(\x08\x12\x10\n\x08metaflow\x18\r \x01(\x08\x12\x10\n\x08\x61llennlp\x18\x0e \x01(\x08\x12\x11\n\tautogluon\x18\x0f \x01(\x08\x12\x11\n\tautokeras\x18\x10 \x01(\x08\x12\x10\n\x08\x63\x61talyst\x18\x12 \x01(\x08\x12\x10\n\x08\x64\x65\x65pchem\x18\x15 \x01(\x08\x12\x0f\n\x07\x64\x65\x65pctr\x18\x16 \x01(\x08\x12\x0f\n\x07pycaret\x18\x1c \x01(\x08\x12\x14\n\x0cpytorchvideo\x18\x1d \x01(\x08\x12\x0b\n\x03ray\x18\x1e \x01(\x08\x12\x1a\n\x12simpletransformers\x18\x1f \x01(\x08\x12\x0e\n\x06skorch\x18 \x01(\x08\x12\r\n\x05spacy\x18! \x01(\x08\x12\r\n\x05\x66lash\x18\" \x01(\x08\x12\x0e\n\x06optuna\x18# \x01(\x08\x12\x0f\n\x07recbole\x18$ \x01(\x08\x12\x0c\n\x04mmcv\x18% \x01(\x08\x12\r\n\x05mmdet\x18& \x01(\x08\x12\x11\n\ttorchdrug\x18\' \x01(\x08\x12\x11\n\ttorchtext\x18( \x01(\x08\x12\x13\n\x0btorchvision\x18) \x01(\x08\x12\r\n\x05\x65legy\x18* \x01(\x08\x12\x12\n\ndetectron2\x18+ \x01(\x08\x12\r\n\x05\x66lair\x18, \x01(\x08\x12\x0c\n\x04\x66lax\x18- \x01(\x08\x12\x0c\n\x04syft\x18. \x01(\x08\x12\x0b\n\x03TTS\x18/ \x01(\x08\x12\r\n\x05monai\x18\x30 \x01(\x08\x12\x17\n\x0fhuggingface_hub\x18\x31 \x01(\x08\x12\r\n\x05hydra\x18\x32 \x01(\x08\x12\x10\n\x08\x64\x61tasets\x18\x33 \x01(\x08\x12\x0e\n\x06sacred\x18\x34 \x01(\x08\x12\x0e\n\x06joblib\x18\x35 \x01(\x08\x12\x0c\n\x04\x64\x61sk\x18\x36 \x01(\x08\x12\x0f\n\x07\x61syncio\x18\x37 \x01(\x08\x12\x11\n\tpaddleocr\x18\x38 \x01(\x08\x12\r\n\x05ppdet\x18\x39 \x01(\x08\x12\x11\n\tpaddleseg\x18: \x01(\x08\x12\x11\n\tpaddlenlp\x18; \x01(\x08\x12\r\n\x05mmseg\x18< \x01(\x08\x12\r\n\x05mmocr\x18= \x01(\x08\x12\r\n\x05mmcls\x18> \x01(\x08\x12\x0c\n\x04timm\x18? \x01(\x08\x12\x0f\n\x07\x66\x61irseq\x18@ \x01(\x08\x12\x12\n\ndeepchecks\x18\x41 \x01(\x08\x12\x10\n\x08\x63omposer\x18\x42 \x01(\x08\x12\x10\n\x08sparseml\x18\x43 \x01(\x08\x12\x10\n\x08\x61nomalib\x18\x44 \x01(\x08\x12\r\n\x05zenml\x18\x45 \x01(\x08\x12\x12\n\ncolossalai\x18\x46 \x01(\x08\x12\x12\n\naccelerate\x18G \x01(\x08\x12\x0e\n\x06merlin\x18H \x01(\x08\x12\x0f\n\x07nanodet\x18I \x01(\x08\x12#\n\x1bsegmentation_models_pytorch\x18J \x01(\x08\x12\x1d\n\x15sentence_transformers\x18K \x01(\x08\x12\x0b\n\x03\x64gl\x18L \x01(\x08\x12\x17\n\x0ftorch_geometric\x18M \x01(\x08\x12\x0c\n\x04jina\x18N \x01(\x08\x12\x0e\n\x06kornia\x18O \x01(\x08\x12\x16\n\x0e\x61lbumentations\x18P \x01(\x08\x12\x10\n\x08keras_cv\x18Q \x01(\x08\x12\x10\n\x08mmengine\x18R \x01(\x08\x12\x11\n\tdiffusers\x18S \x01(\x08\x12\x0b\n\x03trl\x18T \x01(\x08\x12\x0c\n\x04trlx\x18U \x01(\x08\x12\x11\n\tlangchain\x18V \x01(\x08\x12\x13\n\x0bllama_index\x18W \x01(\x08\x12\x15\n\rstability_sdk\x18X \x01(\x08\x12\x0f\n\x07prefect\x18Y \x01(\x08\x12\x13\n\x0bprefect_ray\x18Z \x01(\x08\x12\x10\n\x08pinecone\x18[ \x01(\x08\x12\x10\n\x08\x63hromadb\x18\\ \x01(\x08\x12\x10\n\x08weaviate\x18] \x01(\x08\x12\x13\n\x0bpromptlayer\x18^ \x01(\x08\x12\x0e\n\x06openai\x18_ \x01(\x08\x12\x0e\n\x06\x63ohere\x18` \x01(\x08\x12\x11\n\tanthropic\x18\x61 \x01(\x08\x12\x0c\n\x04peft\x18\x62 \x01(\x08\x12\x0f\n\x07optimum\x18\x63 \x01(\x08\x12\x10\n\x08\x65valuate\x18\x64 \x01(\x08\x12\x10\n\x08langflow\x18\x65 \x01(\x08\x12\x12\n\nkeras_core\x18\x66 \x01(\x08\x12\x18\n\x10lightning_fabric\x18g \x01(\x08\x12\x1c\n\x14\x63urated_transformers\x18h \x01(\x08\x12\x0e\n\x06orjson\x18i \x01(\x08\"\xf3\n\n\x07\x46\x65\x61ture\x12\r\n\x05watch\x18\x01 \x01(\x08\x12\x0e\n\x06\x66inish\x18\x02 \x01(\x08\x12\x0c\n\x04save\x18\x03 \x01(\x08\x12\x0f\n\x07offline\x18\x04 \x01(\x08\x12\x0f\n\x07resumed\x18\x05 \x01(\x08\x12\x0c\n\x04grpc\x18\x06 \x01(\x08\x12\x0e\n\x06metric\x18\x07 \x01(\x08\x12\r\n\x05keras\x18\x08 \x01(\x08\x12\x11\n\tsagemaker\x18\t \x01(\x08\x12\x1c\n\x14\x61rtifact_incremental\x18\n \x01(\x08\x12\x10\n\x08metaflow\x18\x0b \x01(\x08\x12\x0f\n\x07prodigy\x18\x0c \x01(\x08\x12\x15\n\rset_init_name\x18\r \x01(\x08\x12\x13\n\x0bset_init_id\x18\x0e \x01(\x08\x12\x15\n\rset_init_tags\x18\x0f \x01(\x08\x12\x17\n\x0fset_init_config\x18\x10 \x01(\x08\x12\x14\n\x0cset_run_name\x18\x11 \x01(\x08\x12\x14\n\x0cset_run_tags\x18\x12 \x01(\x08\x12\x17\n\x0fset_config_item\x18\x13 \x01(\x08\x12\x0e\n\x06launch\x18\x14 \x01(\x08\x12\x1c\n\x14torch_profiler_trace\x18\x15 \x01(\x08\x12\x0b\n\x03sb3\x18\x16 \x01(\x08\x12\x0f\n\x07service\x18\x17 \x01(\x08\x12\x17\n\x0finit_return_run\x18\x18 \x01(\x08\x12\x1f\n\x17lightgbm_wandb_callback\x18\x19 \x01(\x08\x12\x1c\n\x14lightgbm_log_summary\x18\x1a \x01(\x08\x12\x1f\n\x17\x63\x61tboost_wandb_callback\x18\x1b \x01(\x08\x12\x1c\n\x14\x63\x61tboost_log_summary\x18\x1c \x01(\x08\x12\x17\n\x0ftensorboard_log\x18\x1d \x01(\x08\x12\x16\n\x0e\x65stimator_hook\x18\x1e \x01(\x08\x12\x1e\n\x16xgboost_wandb_callback\x18\x1f \x01(\x08\x12\"\n\x1axgboost_old_wandb_callback\x18 \x01(\x08\x12\x0e\n\x06\x61ttach\x18! \x01(\x08\x12\x19\n\x11tensorboard_patch\x18\" \x01(\x08\x12\x18\n\x10tensorboard_sync\x18# \x01(\x08\x12\x15\n\rkfp_wandb_log\x18$ \x01(\x08\x12\x1b\n\x13maybe_run_overwrite\x18% \x01(\x08\x12\x1c\n\x14keras_metrics_logger\x18& \x01(\x08\x12\x1e\n\x16keras_model_checkpoint\x18\' \x01(\x08\x12!\n\x19keras_wandb_eval_callback\x18( \x01(\x08\x12\x1d\n\x15\x66low_control_overflow\x18) \x01(\x08\x12\x0c\n\x04sync\x18* \x01(\x08\x12\x1d\n\x15\x66low_control_disabled\x18+ \x01(\x08\x12\x1b\n\x13\x66low_control_custom\x18, \x01(\x08\x12\x18\n\x10service_disabled\x18- \x01(\x08\x12\x14\n\x0copen_metrics\x18. \x01(\x08\x12\x1a\n\x12ultralytics_yolov8\x18/ \x01(\x08\x12\x17\n\x0fimporter_mlflow\x18\x30 \x01(\x08\x12\x15\n\rsync_tfevents\x18\x31 \x01(\x08\x12\x15\n\rasync_uploads\x18\x32 \x01(\x08\x12\x16\n\x0eopenai_autolog\x18\x33 \x01(\x08\x12\x18\n\x10langchain_tracer\x18\x34 \x01(\x08\x12\x16\n\x0e\x63ohere_autolog\x18\x35 \x01(\x08\x12\x1b\n\x13hf_pipeline_autolog\x18\x36 \x01(\x08\x12\x0c\n\x04\x63ore\x18\x37 \x01(\x08\x12\r\n\x05lib_c\x18\x38 \x01(\x08\x12\x0f\n\x07lib_cpp\x18\x39 \x01(\x08\x12\x19\n\x11openai_finetuning\x18: \x01(\x08\x12\x19\n\x11\x64iffusers_autolog\x18; \x01(\x08\"\x96\x02\n\x03\x45nv\x12\x0f\n\x07jupyter\x18\x01 \x01(\x08\x12\x0e\n\x06kaggle\x18\x02 \x01(\x08\x12\x0f\n\x07windows\x18\x03 \x01(\x08\x12\x0e\n\x06m1_gpu\x18\x04 \x01(\x08\x12\x13\n\x0bstart_spawn\x18\x05 \x01(\x08\x12\x12\n\nstart_fork\x18\x06 \x01(\x08\x12\x18\n\x10start_forkserver\x18\x07 \x01(\x08\x12\x14\n\x0cstart_thread\x18\x08 \x01(\x08\x12\x10\n\x08maybe_mp\x18\t \x01(\x08\x12\x10\n\x08trainium\x18\n \x01(\x08\x12\x0b\n\x03pex\x18\x0b \x01(\x08\x12\r\n\x05\x63olab\x18\x0c \x01(\x08\x12\x0f\n\x07ipython\x18\r \x01(\x08\x12\x12\n\naws_lambda\x18\x0e \x01(\x08\x12\x0f\n\x07\x61md_gpu\x18\x0f \x01(\x08\"H\n\x06Labels\x12\x13\n\x0b\x63ode_string\x18\x01 \x01(\t\x12\x13\n\x0brepo_string\x18\x02 \x01(\t\x12\x14\n\x0c\x63ode_version\x18\x03 \x01(\t\"\x9a\x02\n\nDeprecated\x12!\n\x19keras_callback__data_type\x18\x01 \x01(\x08\x12\x11\n\trun__mode\x18\x02 \x01(\x08\x12\x19\n\x11run__save_no_args\x18\x03 \x01(\x08\x12\x11\n\trun__join\x18\x04 \x01(\x08\x12\r\n\x05plots\x18\x05 \x01(\x08\x12\x15\n\rrun__log_sync\x18\x06 \x01(\x08\x12!\n\x19init__config_include_keys\x18\x07 \x01(\x08\x12!\n\x19init__config_exclude_keys\x18\x08 \x01(\x08\x12\"\n\x1akeras_callback__save_model\x18\t \x01(\x08\x12\x18\n\x10langchain_tracer\x18\n \x01(\x08\"|\n\x06Issues\x12%\n\x1dsettings__validation_warnings\x18\x01 \x01(\x08\x12!\n\x19settings__unexpected_args\x18\x02 \x01(\x08\x12(\n settings__preprocessing_warnings\x18\x03 \x01(\x08\x62\x06proto3')
+
+
+
+_TELEMETRYRECORD = DESCRIPTOR.message_types_by_name['TelemetryRecord']
+_TELEMETRYRESULT = DESCRIPTOR.message_types_by_name['TelemetryResult']
+_IMPORTS = DESCRIPTOR.message_types_by_name['Imports']
+_FEATURE = DESCRIPTOR.message_types_by_name['Feature']
+_ENV = DESCRIPTOR.message_types_by_name['Env']
+_LABELS = DESCRIPTOR.message_types_by_name['Labels']
+_DEPRECATED = DESCRIPTOR.message_types_by_name['Deprecated']
+_ISSUES = DESCRIPTOR.message_types_by_name['Issues']
+TelemetryRecord = _reflection.GeneratedProtocolMessageType('TelemetryRecord', (_message.Message,), {
+ 'DESCRIPTOR' : _TELEMETRYRECORD,
+ '__module__' : 'wandb.proto.wandb_telemetry_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.TelemetryRecord)
+ })
+_sym_db.RegisterMessage(TelemetryRecord)
+
+TelemetryResult = _reflection.GeneratedProtocolMessageType('TelemetryResult', (_message.Message,), {
+ 'DESCRIPTOR' : _TELEMETRYRESULT,
+ '__module__' : 'wandb.proto.wandb_telemetry_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.TelemetryResult)
+ })
+_sym_db.RegisterMessage(TelemetryResult)
+
+Imports = _reflection.GeneratedProtocolMessageType('Imports', (_message.Message,), {
+ 'DESCRIPTOR' : _IMPORTS,
+ '__module__' : 'wandb.proto.wandb_telemetry_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.Imports)
+ })
+_sym_db.RegisterMessage(Imports)
+
+Feature = _reflection.GeneratedProtocolMessageType('Feature', (_message.Message,), {
+ 'DESCRIPTOR' : _FEATURE,
+ '__module__' : 'wandb.proto.wandb_telemetry_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.Feature)
+ })
+_sym_db.RegisterMessage(Feature)
+
+Env = _reflection.GeneratedProtocolMessageType('Env', (_message.Message,), {
+ 'DESCRIPTOR' : _ENV,
+ '__module__' : 'wandb.proto.wandb_telemetry_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.Env)
+ })
+_sym_db.RegisterMessage(Env)
+
+Labels = _reflection.GeneratedProtocolMessageType('Labels', (_message.Message,), {
+ 'DESCRIPTOR' : _LABELS,
+ '__module__' : 'wandb.proto.wandb_telemetry_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.Labels)
+ })
+_sym_db.RegisterMessage(Labels)
+
+Deprecated = _reflection.GeneratedProtocolMessageType('Deprecated', (_message.Message,), {
+ 'DESCRIPTOR' : _DEPRECATED,
+ '__module__' : 'wandb.proto.wandb_telemetry_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.Deprecated)
+ })
+_sym_db.RegisterMessage(Deprecated)
+
+Issues = _reflection.GeneratedProtocolMessageType('Issues', (_message.Message,), {
+ 'DESCRIPTOR' : _ISSUES,
+ '__module__' : 'wandb.proto.wandb_telemetry_pb2'
+ # @@protoc_insertion_point(class_scope:wandb_internal.Issues)
+ })
+_sym_db.RegisterMessage(Issues)
+
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+ DESCRIPTOR._options = None
+ _TELEMETRYRECORD._serialized_start=84
+ _TELEMETRYRECORD._serialized_end=559
+ _TELEMETRYRESULT._serialized_start=561
+ _TELEMETRYRESULT._serialized_end=578
+ _IMPORTS._serialized_start=581
+ _IMPORTS._serialized_end=2343
+ _FEATURE._serialized_start=2346
+ _FEATURE._serialized_end=3741
+ _ENV._serialized_start=3744
+ _ENV._serialized_end=4022
+ _LABELS._serialized_start=4024
+ _LABELS._serialized_end=4096
+ _DEPRECATED._serialized_start=4099
+ _DEPRECATED._serialized_end=4381
+ _ISSUES._serialized_start=4383
+ _ISSUES._serialized_end=4507
+# @@protoc_insertion_point(module_scope)
diff --git a/wandb/proto/v3/wandb_telemetry_pb2.pyi b/wandb/proto/v3/wandb_telemetry_pb2.pyi
new file mode 100644
index 0000000000000000000000000000000000000000..04296219b64347a4aae827bf36ecd8960ae4d45f
--- /dev/null
+++ b/wandb/proto/v3/wandb_telemetry_pb2.pyi
@@ -0,0 +1,823 @@
+"""
+@generated by mypy-protobuf. Do not edit manually!
+isort:skip_file
+"""
+import builtins
+import google.protobuf.descriptor
+import google.protobuf.message
+import sys
+import wandb.proto.wandb_base_pb2
+
+if sys.version_info >= (3, 8):
+ import typing as typing_extensions
+else:
+ import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+class TelemetryRecord(google.protobuf.message.Message):
+ """
+ Telemetry
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ IMPORTS_INIT_FIELD_NUMBER: builtins.int
+ IMPORTS_FINISH_FIELD_NUMBER: builtins.int
+ FEATURE_FIELD_NUMBER: builtins.int
+ PYTHON_VERSION_FIELD_NUMBER: builtins.int
+ CLI_VERSION_FIELD_NUMBER: builtins.int
+ HUGGINGFACE_VERSION_FIELD_NUMBER: builtins.int
+ ENV_FIELD_NUMBER: builtins.int
+ LABEL_FIELD_NUMBER: builtins.int
+ DEPRECATED_FIELD_NUMBER: builtins.int
+ ISSUES_FIELD_NUMBER: builtins.int
+ CORE_VERSION_FIELD_NUMBER: builtins.int
+ PLATFORM_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def imports_init(self) -> global___Imports: ...
+ @property
+ def imports_finish(self) -> global___Imports: ...
+ @property
+ def feature(self) -> global___Feature: ...
+ python_version: builtins.str
+ cli_version: builtins.str
+ huggingface_version: builtins.str
+ @property
+ def env(self) -> global___Env:
+ """string framework = 7;"""
+ @property
+ def label(self) -> global___Labels: ...
+ @property
+ def deprecated(self) -> global___Deprecated: ...
+ @property
+ def issues(self) -> global___Issues: ...
+ core_version: builtins.str
+ platform: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ imports_init: global___Imports | None = ...,
+ imports_finish: global___Imports | None = ...,
+ feature: global___Feature | None = ...,
+ python_version: builtins.str = ...,
+ cli_version: builtins.str = ...,
+ huggingface_version: builtins.str = ...,
+ env: global___Env | None = ...,
+ label: global___Labels | None = ...,
+ deprecated: global___Deprecated | None = ...,
+ issues: global___Issues | None = ...,
+ core_version: builtins.str = ...,
+ platform: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "deprecated", b"deprecated", "env", b"env", "feature", b"feature", "imports_finish", b"imports_finish", "imports_init", b"imports_init", "issues", b"issues", "label", b"label"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "cli_version", b"cli_version", "core_version", b"core_version", "deprecated", b"deprecated", "env", b"env", "feature", b"feature", "huggingface_version", b"huggingface_version", "imports_finish", b"imports_finish", "imports_init", b"imports_init", "issues", b"issues", "label", b"label", "platform", b"platform", "python_version", b"python_version"]) -> None: ...
+
+global___TelemetryRecord = TelemetryRecord
+
+class TelemetryResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___TelemetryResult = TelemetryResult
+
+class Imports(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ TORCH_FIELD_NUMBER: builtins.int
+ KERAS_FIELD_NUMBER: builtins.int
+ TENSORFLOW_FIELD_NUMBER: builtins.int
+ FASTAI_FIELD_NUMBER: builtins.int
+ SKLEARN_FIELD_NUMBER: builtins.int
+ XGBOOST_FIELD_NUMBER: builtins.int
+ CATBOOST_FIELD_NUMBER: builtins.int
+ LIGHTGBM_FIELD_NUMBER: builtins.int
+ PYTORCH_LIGHTNING_FIELD_NUMBER: builtins.int
+ IGNITE_FIELD_NUMBER: builtins.int
+ TRANSFORMERS_FIELD_NUMBER: builtins.int
+ JAX_FIELD_NUMBER: builtins.int
+ METAFLOW_FIELD_NUMBER: builtins.int
+ ALLENNLP_FIELD_NUMBER: builtins.int
+ AUTOGLUON_FIELD_NUMBER: builtins.int
+ AUTOKERAS_FIELD_NUMBER: builtins.int
+ CATALYST_FIELD_NUMBER: builtins.int
+ DEEPCHEM_FIELD_NUMBER: builtins.int
+ DEEPCTR_FIELD_NUMBER: builtins.int
+ PYCARET_FIELD_NUMBER: builtins.int
+ PYTORCHVIDEO_FIELD_NUMBER: builtins.int
+ RAY_FIELD_NUMBER: builtins.int
+ SIMPLETRANSFORMERS_FIELD_NUMBER: builtins.int
+ SKORCH_FIELD_NUMBER: builtins.int
+ SPACY_FIELD_NUMBER: builtins.int
+ FLASH_FIELD_NUMBER: builtins.int
+ OPTUNA_FIELD_NUMBER: builtins.int
+ RECBOLE_FIELD_NUMBER: builtins.int
+ MMCV_FIELD_NUMBER: builtins.int
+ MMDET_FIELD_NUMBER: builtins.int
+ TORCHDRUG_FIELD_NUMBER: builtins.int
+ TORCHTEXT_FIELD_NUMBER: builtins.int
+ TORCHVISION_FIELD_NUMBER: builtins.int
+ ELEGY_FIELD_NUMBER: builtins.int
+ DETECTRON2_FIELD_NUMBER: builtins.int
+ FLAIR_FIELD_NUMBER: builtins.int
+ FLAX_FIELD_NUMBER: builtins.int
+ SYFT_FIELD_NUMBER: builtins.int
+ TTS_FIELD_NUMBER: builtins.int
+ MONAI_FIELD_NUMBER: builtins.int
+ HUGGINGFACE_HUB_FIELD_NUMBER: builtins.int
+ HYDRA_FIELD_NUMBER: builtins.int
+ DATASETS_FIELD_NUMBER: builtins.int
+ SACRED_FIELD_NUMBER: builtins.int
+ JOBLIB_FIELD_NUMBER: builtins.int
+ DASK_FIELD_NUMBER: builtins.int
+ ASYNCIO_FIELD_NUMBER: builtins.int
+ PADDLEOCR_FIELD_NUMBER: builtins.int
+ PPDET_FIELD_NUMBER: builtins.int
+ PADDLESEG_FIELD_NUMBER: builtins.int
+ PADDLENLP_FIELD_NUMBER: builtins.int
+ MMSEG_FIELD_NUMBER: builtins.int
+ MMOCR_FIELD_NUMBER: builtins.int
+ MMCLS_FIELD_NUMBER: builtins.int
+ TIMM_FIELD_NUMBER: builtins.int
+ FAIRSEQ_FIELD_NUMBER: builtins.int
+ DEEPCHECKS_FIELD_NUMBER: builtins.int
+ COMPOSER_FIELD_NUMBER: builtins.int
+ SPARSEML_FIELD_NUMBER: builtins.int
+ ANOMALIB_FIELD_NUMBER: builtins.int
+ ZENML_FIELD_NUMBER: builtins.int
+ COLOSSALAI_FIELD_NUMBER: builtins.int
+ ACCELERATE_FIELD_NUMBER: builtins.int
+ MERLIN_FIELD_NUMBER: builtins.int
+ NANODET_FIELD_NUMBER: builtins.int
+ SEGMENTATION_MODELS_PYTORCH_FIELD_NUMBER: builtins.int
+ SENTENCE_TRANSFORMERS_FIELD_NUMBER: builtins.int
+ DGL_FIELD_NUMBER: builtins.int
+ TORCH_GEOMETRIC_FIELD_NUMBER: builtins.int
+ JINA_FIELD_NUMBER: builtins.int
+ KORNIA_FIELD_NUMBER: builtins.int
+ ALBUMENTATIONS_FIELD_NUMBER: builtins.int
+ KERAS_CV_FIELD_NUMBER: builtins.int
+ MMENGINE_FIELD_NUMBER: builtins.int
+ DIFFUSERS_FIELD_NUMBER: builtins.int
+ TRL_FIELD_NUMBER: builtins.int
+ TRLX_FIELD_NUMBER: builtins.int
+ LANGCHAIN_FIELD_NUMBER: builtins.int
+ LLAMA_INDEX_FIELD_NUMBER: builtins.int
+ STABILITY_SDK_FIELD_NUMBER: builtins.int
+ PREFECT_FIELD_NUMBER: builtins.int
+ PREFECT_RAY_FIELD_NUMBER: builtins.int
+ PINECONE_FIELD_NUMBER: builtins.int
+ CHROMADB_FIELD_NUMBER: builtins.int
+ WEAVIATE_FIELD_NUMBER: builtins.int
+ PROMPTLAYER_FIELD_NUMBER: builtins.int
+ OPENAI_FIELD_NUMBER: builtins.int
+ COHERE_FIELD_NUMBER: builtins.int
+ ANTHROPIC_FIELD_NUMBER: builtins.int
+ PEFT_FIELD_NUMBER: builtins.int
+ OPTIMUM_FIELD_NUMBER: builtins.int
+ EVALUATE_FIELD_NUMBER: builtins.int
+ LANGFLOW_FIELD_NUMBER: builtins.int
+ KERAS_CORE_FIELD_NUMBER: builtins.int
+ LIGHTNING_FABRIC_FIELD_NUMBER: builtins.int
+ CURATED_TRANSFORMERS_FIELD_NUMBER: builtins.int
+ ORJSON_FIELD_NUMBER: builtins.int
+ torch: builtins.bool
+ keras: builtins.bool
+ tensorflow: builtins.bool
+ fastai: builtins.bool
+ sklearn: builtins.bool
+ xgboost: builtins.bool
+ catboost: builtins.bool
+ lightgbm: builtins.bool
+ pytorch_lightning: builtins.bool
+ ignite: builtins.bool
+ transformers: builtins.bool
+ jax: builtins.bool
+ metaflow: builtins.bool
+ allennlp: builtins.bool
+ autogluon: builtins.bool
+ autokeras: builtins.bool
+ catalyst: builtins.bool
+ """bool avalanche = 17;"""
+ deepchem: builtins.bool
+ """bool dalle_pytorch = 19;
+ bool datasets = 20;
+ """
+ deepctr: builtins.bool
+ pycaret: builtins.bool
+ """bool deeppavlov = 23;
+ bool detectron = 24;
+ bool paddle = 25;
+ bool parlai = 26;
+ bool prophet = 27;
+ """
+ pytorchvideo: builtins.bool
+ ray: builtins.bool
+ simpletransformers: builtins.bool
+ skorch: builtins.bool
+ spacy: builtins.bool
+ flash: builtins.bool
+ optuna: builtins.bool
+ recbole: builtins.bool
+ mmcv: builtins.bool
+ mmdet: builtins.bool
+ torchdrug: builtins.bool
+ torchtext: builtins.bool
+ torchvision: builtins.bool
+ elegy: builtins.bool
+ detectron2: builtins.bool
+ flair: builtins.bool
+ flax: builtins.bool
+ syft: builtins.bool
+ TTS: builtins.bool
+ monai: builtins.bool
+ huggingface_hub: builtins.bool
+ hydra: builtins.bool
+ datasets: builtins.bool
+ sacred: builtins.bool
+ joblib: builtins.bool
+ dask: builtins.bool
+ asyncio: builtins.bool
+ paddleocr: builtins.bool
+ ppdet: builtins.bool
+ paddleseg: builtins.bool
+ paddlenlp: builtins.bool
+ mmseg: builtins.bool
+ mmocr: builtins.bool
+ mmcls: builtins.bool
+ timm: builtins.bool
+ fairseq: builtins.bool
+ deepchecks: builtins.bool
+ composer: builtins.bool
+ sparseml: builtins.bool
+ anomalib: builtins.bool
+ zenml: builtins.bool
+ colossalai: builtins.bool
+ accelerate: builtins.bool
+ merlin: builtins.bool
+ nanodet: builtins.bool
+ segmentation_models_pytorch: builtins.bool
+ sentence_transformers: builtins.bool
+ dgl: builtins.bool
+ torch_geometric: builtins.bool
+ jina: builtins.bool
+ kornia: builtins.bool
+ albumentations: builtins.bool
+ keras_cv: builtins.bool
+ mmengine: builtins.bool
+ diffusers: builtins.bool
+ trl: builtins.bool
+ trlx: builtins.bool
+ langchain: builtins.bool
+ llama_index: builtins.bool
+ stability_sdk: builtins.bool
+ prefect: builtins.bool
+ prefect_ray: builtins.bool
+ pinecone: builtins.bool
+ """pinecone-client"""
+ chromadb: builtins.bool
+ weaviate: builtins.bool
+ """weaviate-client"""
+ promptlayer: builtins.bool
+ openai: builtins.bool
+ cohere: builtins.bool
+ anthropic: builtins.bool
+ peft: builtins.bool
+ optimum: builtins.bool
+ evaluate: builtins.bool
+ langflow: builtins.bool
+ keras_core: builtins.bool
+ """keras-core"""
+ lightning_fabric: builtins.bool
+ """lightning-fabric"""
+ curated_transformers: builtins.bool
+ """curated-transformers"""
+ orjson: builtins.bool
+ def __init__(
+ self,
+ *,
+ torch: builtins.bool = ...,
+ keras: builtins.bool = ...,
+ tensorflow: builtins.bool = ...,
+ fastai: builtins.bool = ...,
+ sklearn: builtins.bool = ...,
+ xgboost: builtins.bool = ...,
+ catboost: builtins.bool = ...,
+ lightgbm: builtins.bool = ...,
+ pytorch_lightning: builtins.bool = ...,
+ ignite: builtins.bool = ...,
+ transformers: builtins.bool = ...,
+ jax: builtins.bool = ...,
+ metaflow: builtins.bool = ...,
+ allennlp: builtins.bool = ...,
+ autogluon: builtins.bool = ...,
+ autokeras: builtins.bool = ...,
+ catalyst: builtins.bool = ...,
+ deepchem: builtins.bool = ...,
+ deepctr: builtins.bool = ...,
+ pycaret: builtins.bool = ...,
+ pytorchvideo: builtins.bool = ...,
+ ray: builtins.bool = ...,
+ simpletransformers: builtins.bool = ...,
+ skorch: builtins.bool = ...,
+ spacy: builtins.bool = ...,
+ flash: builtins.bool = ...,
+ optuna: builtins.bool = ...,
+ recbole: builtins.bool = ...,
+ mmcv: builtins.bool = ...,
+ mmdet: builtins.bool = ...,
+ torchdrug: builtins.bool = ...,
+ torchtext: builtins.bool = ...,
+ torchvision: builtins.bool = ...,
+ elegy: builtins.bool = ...,
+ detectron2: builtins.bool = ...,
+ flair: builtins.bool = ...,
+ flax: builtins.bool = ...,
+ syft: builtins.bool = ...,
+ TTS: builtins.bool = ...,
+ monai: builtins.bool = ...,
+ huggingface_hub: builtins.bool = ...,
+ hydra: builtins.bool = ...,
+ datasets: builtins.bool = ...,
+ sacred: builtins.bool = ...,
+ joblib: builtins.bool = ...,
+ dask: builtins.bool = ...,
+ asyncio: builtins.bool = ...,
+ paddleocr: builtins.bool = ...,
+ ppdet: builtins.bool = ...,
+ paddleseg: builtins.bool = ...,
+ paddlenlp: builtins.bool = ...,
+ mmseg: builtins.bool = ...,
+ mmocr: builtins.bool = ...,
+ mmcls: builtins.bool = ...,
+ timm: builtins.bool = ...,
+ fairseq: builtins.bool = ...,
+ deepchecks: builtins.bool = ...,
+ composer: builtins.bool = ...,
+ sparseml: builtins.bool = ...,
+ anomalib: builtins.bool = ...,
+ zenml: builtins.bool = ...,
+ colossalai: builtins.bool = ...,
+ accelerate: builtins.bool = ...,
+ merlin: builtins.bool = ...,
+ nanodet: builtins.bool = ...,
+ segmentation_models_pytorch: builtins.bool = ...,
+ sentence_transformers: builtins.bool = ...,
+ dgl: builtins.bool = ...,
+ torch_geometric: builtins.bool = ...,
+ jina: builtins.bool = ...,
+ kornia: builtins.bool = ...,
+ albumentations: builtins.bool = ...,
+ keras_cv: builtins.bool = ...,
+ mmengine: builtins.bool = ...,
+ diffusers: builtins.bool = ...,
+ trl: builtins.bool = ...,
+ trlx: builtins.bool = ...,
+ langchain: builtins.bool = ...,
+ llama_index: builtins.bool = ...,
+ stability_sdk: builtins.bool = ...,
+ prefect: builtins.bool = ...,
+ prefect_ray: builtins.bool = ...,
+ pinecone: builtins.bool = ...,
+ chromadb: builtins.bool = ...,
+ weaviate: builtins.bool = ...,
+ promptlayer: builtins.bool = ...,
+ openai: builtins.bool = ...,
+ cohere: builtins.bool = ...,
+ anthropic: builtins.bool = ...,
+ peft: builtins.bool = ...,
+ optimum: builtins.bool = ...,
+ evaluate: builtins.bool = ...,
+ langflow: builtins.bool = ...,
+ keras_core: builtins.bool = ...,
+ lightning_fabric: builtins.bool = ...,
+ curated_transformers: builtins.bool = ...,
+ orjson: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["TTS", b"TTS", "accelerate", b"accelerate", "albumentations", b"albumentations", "allennlp", b"allennlp", "anomalib", b"anomalib", "anthropic", b"anthropic", "asyncio", b"asyncio", "autogluon", b"autogluon", "autokeras", b"autokeras", "catalyst", b"catalyst", "catboost", b"catboost", "chromadb", b"chromadb", "cohere", b"cohere", "colossalai", b"colossalai", "composer", b"composer", "curated_transformers", b"curated_transformers", "dask", b"dask", "datasets", b"datasets", "deepchecks", b"deepchecks", "deepchem", b"deepchem", "deepctr", b"deepctr", "detectron2", b"detectron2", "dgl", b"dgl", "diffusers", b"diffusers", "elegy", b"elegy", "evaluate", b"evaluate", "fairseq", b"fairseq", "fastai", b"fastai", "flair", b"flair", "flash", b"flash", "flax", b"flax", "huggingface_hub", b"huggingface_hub", "hydra", b"hydra", "ignite", b"ignite", "jax", b"jax", "jina", b"jina", "joblib", b"joblib", "keras", b"keras", "keras_core", b"keras_core", "keras_cv", b"keras_cv", "kornia", b"kornia", "langchain", b"langchain", "langflow", b"langflow", "lightgbm", b"lightgbm", "lightning_fabric", b"lightning_fabric", "llama_index", b"llama_index", "merlin", b"merlin", "metaflow", b"metaflow", "mmcls", b"mmcls", "mmcv", b"mmcv", "mmdet", b"mmdet", "mmengine", b"mmengine", "mmocr", b"mmocr", "mmseg", b"mmseg", "monai", b"monai", "nanodet", b"nanodet", "openai", b"openai", "optimum", b"optimum", "optuna", b"optuna", "orjson", b"orjson", "paddlenlp", b"paddlenlp", "paddleocr", b"paddleocr", "paddleseg", b"paddleseg", "peft", b"peft", "pinecone", b"pinecone", "ppdet", b"ppdet", "prefect", b"prefect", "prefect_ray", b"prefect_ray", "promptlayer", b"promptlayer", "pycaret", b"pycaret", "pytorch_lightning", b"pytorch_lightning", "pytorchvideo", b"pytorchvideo", "ray", b"ray", "recbole", b"recbole", "sacred", b"sacred", "segmentation_models_pytorch", b"segmentation_models_pytorch", "sentence_transformers", b"sentence_transformers", "simpletransformers", b"simpletransformers", "sklearn", b"sklearn", "skorch", b"skorch", "spacy", b"spacy", "sparseml", b"sparseml", "stability_sdk", b"stability_sdk", "syft", b"syft", "tensorflow", b"tensorflow", "timm", b"timm", "torch", b"torch", "torch_geometric", b"torch_geometric", "torchdrug", b"torchdrug", "torchtext", b"torchtext", "torchvision", b"torchvision", "transformers", b"transformers", "trl", b"trl", "trlx", b"trlx", "weaviate", b"weaviate", "xgboost", b"xgboost", "zenml", b"zenml"]) -> None: ...
+
+global___Imports = Imports
+
+class Feature(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ WATCH_FIELD_NUMBER: builtins.int
+ FINISH_FIELD_NUMBER: builtins.int
+ SAVE_FIELD_NUMBER: builtins.int
+ OFFLINE_FIELD_NUMBER: builtins.int
+ RESUMED_FIELD_NUMBER: builtins.int
+ GRPC_FIELD_NUMBER: builtins.int
+ METRIC_FIELD_NUMBER: builtins.int
+ KERAS_FIELD_NUMBER: builtins.int
+ SAGEMAKER_FIELD_NUMBER: builtins.int
+ ARTIFACT_INCREMENTAL_FIELD_NUMBER: builtins.int
+ METAFLOW_FIELD_NUMBER: builtins.int
+ PRODIGY_FIELD_NUMBER: builtins.int
+ SET_INIT_NAME_FIELD_NUMBER: builtins.int
+ SET_INIT_ID_FIELD_NUMBER: builtins.int
+ SET_INIT_TAGS_FIELD_NUMBER: builtins.int
+ SET_INIT_CONFIG_FIELD_NUMBER: builtins.int
+ SET_RUN_NAME_FIELD_NUMBER: builtins.int
+ SET_RUN_TAGS_FIELD_NUMBER: builtins.int
+ SET_CONFIG_ITEM_FIELD_NUMBER: builtins.int
+ LAUNCH_FIELD_NUMBER: builtins.int
+ TORCH_PROFILER_TRACE_FIELD_NUMBER: builtins.int
+ SB3_FIELD_NUMBER: builtins.int
+ SERVICE_FIELD_NUMBER: builtins.int
+ INIT_RETURN_RUN_FIELD_NUMBER: builtins.int
+ LIGHTGBM_WANDB_CALLBACK_FIELD_NUMBER: builtins.int
+ LIGHTGBM_LOG_SUMMARY_FIELD_NUMBER: builtins.int
+ CATBOOST_WANDB_CALLBACK_FIELD_NUMBER: builtins.int
+ CATBOOST_LOG_SUMMARY_FIELD_NUMBER: builtins.int
+ TENSORBOARD_LOG_FIELD_NUMBER: builtins.int
+ ESTIMATOR_HOOK_FIELD_NUMBER: builtins.int
+ XGBOOST_WANDB_CALLBACK_FIELD_NUMBER: builtins.int
+ XGBOOST_OLD_WANDB_CALLBACK_FIELD_NUMBER: builtins.int
+ ATTACH_FIELD_NUMBER: builtins.int
+ TENSORBOARD_PATCH_FIELD_NUMBER: builtins.int
+ TENSORBOARD_SYNC_FIELD_NUMBER: builtins.int
+ KFP_WANDB_LOG_FIELD_NUMBER: builtins.int
+ MAYBE_RUN_OVERWRITE_FIELD_NUMBER: builtins.int
+ KERAS_METRICS_LOGGER_FIELD_NUMBER: builtins.int
+ KERAS_MODEL_CHECKPOINT_FIELD_NUMBER: builtins.int
+ KERAS_WANDB_EVAL_CALLBACK_FIELD_NUMBER: builtins.int
+ FLOW_CONTROL_OVERFLOW_FIELD_NUMBER: builtins.int
+ SYNC_FIELD_NUMBER: builtins.int
+ FLOW_CONTROL_DISABLED_FIELD_NUMBER: builtins.int
+ FLOW_CONTROL_CUSTOM_FIELD_NUMBER: builtins.int
+ SERVICE_DISABLED_FIELD_NUMBER: builtins.int
+ OPEN_METRICS_FIELD_NUMBER: builtins.int
+ ULTRALYTICS_YOLOV8_FIELD_NUMBER: builtins.int
+ IMPORTER_MLFLOW_FIELD_NUMBER: builtins.int
+ SYNC_TFEVENTS_FIELD_NUMBER: builtins.int
+ ASYNC_UPLOADS_FIELD_NUMBER: builtins.int
+ OPENAI_AUTOLOG_FIELD_NUMBER: builtins.int
+ LANGCHAIN_TRACER_FIELD_NUMBER: builtins.int
+ COHERE_AUTOLOG_FIELD_NUMBER: builtins.int
+ HF_PIPELINE_AUTOLOG_FIELD_NUMBER: builtins.int
+ CORE_FIELD_NUMBER: builtins.int
+ LIB_C_FIELD_NUMBER: builtins.int
+ LIB_CPP_FIELD_NUMBER: builtins.int
+ OPENAI_FINETUNING_FIELD_NUMBER: builtins.int
+ DIFFUSERS_AUTOLOG_FIELD_NUMBER: builtins.int
+ watch: builtins.bool
+ """wandb.watch() called"""
+ finish: builtins.bool
+ """wandb.finish() called"""
+ save: builtins.bool
+ """wandb.save() called"""
+ offline: builtins.bool
+ """offline run was synced"""
+ resumed: builtins.bool
+ """run was resumed"""
+ grpc: builtins.bool
+ """grpc-server (java integration)"""
+ metric: builtins.bool
+ """define_metric() called"""
+ keras: builtins.bool
+ """Keras WandbCallback used"""
+ sagemaker: builtins.bool
+ """User is using sagemaker"""
+ artifact_incremental: builtins.bool
+ """Artifact(incremental=True) used"""
+ metaflow: builtins.bool
+ """Using metaflow integration"""
+ prodigy: builtins.bool
+ """Using prodigy integration"""
+ set_init_name: builtins.bool
+ """users set run name from wandb.init"""
+ set_init_id: builtins.bool
+ """users set run id from wandb.init"""
+ set_init_tags: builtins.bool
+ """users set tags within wandb.init"""
+ set_init_config: builtins.bool
+ """users set run config in wandb.init"""
+ set_run_name: builtins.bool
+ """user sets run name via wandb.run.name = ..."""
+ set_run_tags: builtins.bool
+ """user sets run name via wandb.run.tags = ..."""
+ set_config_item: builtins.bool
+ """users set key in run config via run.config.key"""
+ launch: builtins.bool
+ """or run.config["key"]
+ run is created through wandb launch
+ """
+ torch_profiler_trace: builtins.bool
+ """wandb.profiler.torch_trace_handler() called"""
+ sb3: builtins.bool
+ """Using stable_baselines3 integration"""
+ service: builtins.bool
+ """Using wandb service internal process"""
+ init_return_run: builtins.bool
+ """wandb.init() called in the same process returning previous run"""
+ lightgbm_wandb_callback: builtins.bool
+ """lightgbm callback used"""
+ lightgbm_log_summary: builtins.bool
+ """lightgbm log summary used"""
+ catboost_wandb_callback: builtins.bool
+ """catboost callback used"""
+ catboost_log_summary: builtins.bool
+ """catboost log summary used"""
+ tensorboard_log: builtins.bool
+ """wandb.tensorflow.log or wandb.tensorboard.log used"""
+ estimator_hook: builtins.bool
+ """wandb.tensorflow.WandbHook used"""
+ xgboost_wandb_callback: builtins.bool
+ """xgboost callback used"""
+ xgboost_old_wandb_callback: builtins.bool
+ """xgboost old callback used (to be depreciated)"""
+ attach: builtins.bool
+ """attach to a run in another process"""
+ tensorboard_patch: builtins.bool
+ """wandb.tensorboard.patch(...)"""
+ tensorboard_sync: builtins.bool
+ """wandb.init(sync_tensorboard=True)"""
+ kfp_wandb_log: builtins.bool
+ """wandb.integration.kfp.wandb_log"""
+ maybe_run_overwrite: builtins.bool
+ """Run might have been overwritten"""
+ keras_metrics_logger: builtins.bool
+ """Keras WandbMetricsLogger used"""
+ keras_model_checkpoint: builtins.bool
+ """Keras WandbModelCheckpoint used"""
+ keras_wandb_eval_callback: builtins.bool
+ """Keras WandbEvalCallback used"""
+ flow_control_overflow: builtins.bool
+ """Hit flow control threshold"""
+ sync: builtins.bool
+ """Run was synced with wandb sync"""
+ flow_control_disabled: builtins.bool
+ """Flow control disabled by user"""
+ flow_control_custom: builtins.bool
+ """Flow control customized by user"""
+ service_disabled: builtins.bool
+ """Service disabled by user"""
+ open_metrics: builtins.bool
+ """Consuming metrics from an OpenMetrics endpoint"""
+ ultralytics_yolov8: builtins.bool
+ """Ultralytics YOLOv8 integration callbacks used"""
+ importer_mlflow: builtins.bool
+ """Using Import API for MLFlow"""
+ sync_tfevents: builtins.bool
+ """Using wandb sync for tfevent files"""
+ async_uploads: builtins.bool
+ """Async file uploads enabled by user"""
+ openai_autolog: builtins.bool
+ """OpenAI autolog used"""
+ langchain_tracer: builtins.bool
+ """Langchain wandb tracer callback used"""
+ cohere_autolog: builtins.bool
+ """Cohere autolog used"""
+ hf_pipeline_autolog: builtins.bool
+ """HuggingFace Autologging"""
+ core: builtins.bool
+ """Using wandb core internal process"""
+ lib_c: builtins.bool
+ """Using c wandb library"""
+ lib_cpp: builtins.bool
+ """Using cpp wandb library"""
+ openai_finetuning: builtins.bool
+ """Using openai finetuning WandbLogger"""
+ diffusers_autolog: builtins.bool
+ """Using Diffusers autologger"""
+ def __init__(
+ self,
+ *,
+ watch: builtins.bool = ...,
+ finish: builtins.bool = ...,
+ save: builtins.bool = ...,
+ offline: builtins.bool = ...,
+ resumed: builtins.bool = ...,
+ grpc: builtins.bool = ...,
+ metric: builtins.bool = ...,
+ keras: builtins.bool = ...,
+ sagemaker: builtins.bool = ...,
+ artifact_incremental: builtins.bool = ...,
+ metaflow: builtins.bool = ...,
+ prodigy: builtins.bool = ...,
+ set_init_name: builtins.bool = ...,
+ set_init_id: builtins.bool = ...,
+ set_init_tags: builtins.bool = ...,
+ set_init_config: builtins.bool = ...,
+ set_run_name: builtins.bool = ...,
+ set_run_tags: builtins.bool = ...,
+ set_config_item: builtins.bool = ...,
+ launch: builtins.bool = ...,
+ torch_profiler_trace: builtins.bool = ...,
+ sb3: builtins.bool = ...,
+ service: builtins.bool = ...,
+ init_return_run: builtins.bool = ...,
+ lightgbm_wandb_callback: builtins.bool = ...,
+ lightgbm_log_summary: builtins.bool = ...,
+ catboost_wandb_callback: builtins.bool = ...,
+ catboost_log_summary: builtins.bool = ...,
+ tensorboard_log: builtins.bool = ...,
+ estimator_hook: builtins.bool = ...,
+ xgboost_wandb_callback: builtins.bool = ...,
+ xgboost_old_wandb_callback: builtins.bool = ...,
+ attach: builtins.bool = ...,
+ tensorboard_patch: builtins.bool = ...,
+ tensorboard_sync: builtins.bool = ...,
+ kfp_wandb_log: builtins.bool = ...,
+ maybe_run_overwrite: builtins.bool = ...,
+ keras_metrics_logger: builtins.bool = ...,
+ keras_model_checkpoint: builtins.bool = ...,
+ keras_wandb_eval_callback: builtins.bool = ...,
+ flow_control_overflow: builtins.bool = ...,
+ sync: builtins.bool = ...,
+ flow_control_disabled: builtins.bool = ...,
+ flow_control_custom: builtins.bool = ...,
+ service_disabled: builtins.bool = ...,
+ open_metrics: builtins.bool = ...,
+ ultralytics_yolov8: builtins.bool = ...,
+ importer_mlflow: builtins.bool = ...,
+ sync_tfevents: builtins.bool = ...,
+ async_uploads: builtins.bool = ...,
+ openai_autolog: builtins.bool = ...,
+ langchain_tracer: builtins.bool = ...,
+ cohere_autolog: builtins.bool = ...,
+ hf_pipeline_autolog: builtins.bool = ...,
+ core: builtins.bool = ...,
+ lib_c: builtins.bool = ...,
+ lib_cpp: builtins.bool = ...,
+ openai_finetuning: builtins.bool = ...,
+ diffusers_autolog: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["artifact_incremental", b"artifact_incremental", "async_uploads", b"async_uploads", "attach", b"attach", "catboost_log_summary", b"catboost_log_summary", "catboost_wandb_callback", b"catboost_wandb_callback", "cohere_autolog", b"cohere_autolog", "core", b"core", "diffusers_autolog", b"diffusers_autolog", "estimator_hook", b"estimator_hook", "finish", b"finish", "flow_control_custom", b"flow_control_custom", "flow_control_disabled", b"flow_control_disabled", "flow_control_overflow", b"flow_control_overflow", "grpc", b"grpc", "hf_pipeline_autolog", b"hf_pipeline_autolog", "importer_mlflow", b"importer_mlflow", "init_return_run", b"init_return_run", "keras", b"keras", "keras_metrics_logger", b"keras_metrics_logger", "keras_model_checkpoint", b"keras_model_checkpoint", "keras_wandb_eval_callback", b"keras_wandb_eval_callback", "kfp_wandb_log", b"kfp_wandb_log", "langchain_tracer", b"langchain_tracer", "launch", b"launch", "lib_c", b"lib_c", "lib_cpp", b"lib_cpp", "lightgbm_log_summary", b"lightgbm_log_summary", "lightgbm_wandb_callback", b"lightgbm_wandb_callback", "maybe_run_overwrite", b"maybe_run_overwrite", "metaflow", b"metaflow", "metric", b"metric", "offline", b"offline", "open_metrics", b"open_metrics", "openai_autolog", b"openai_autolog", "openai_finetuning", b"openai_finetuning", "prodigy", b"prodigy", "resumed", b"resumed", "sagemaker", b"sagemaker", "save", b"save", "sb3", b"sb3", "service", b"service", "service_disabled", b"service_disabled", "set_config_item", b"set_config_item", "set_init_config", b"set_init_config", "set_init_id", b"set_init_id", "set_init_name", b"set_init_name", "set_init_tags", b"set_init_tags", "set_run_name", b"set_run_name", "set_run_tags", b"set_run_tags", "sync", b"sync", "sync_tfevents", b"sync_tfevents", "tensorboard_log", b"tensorboard_log", "tensorboard_patch", b"tensorboard_patch", "tensorboard_sync", b"tensorboard_sync", "torch_profiler_trace", b"torch_profiler_trace", "ultralytics_yolov8", b"ultralytics_yolov8", "watch", b"watch", "xgboost_old_wandb_callback", b"xgboost_old_wandb_callback", "xgboost_wandb_callback", b"xgboost_wandb_callback"]) -> None: ...
+
+global___Feature = Feature
+
+class Env(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ JUPYTER_FIELD_NUMBER: builtins.int
+ KAGGLE_FIELD_NUMBER: builtins.int
+ WINDOWS_FIELD_NUMBER: builtins.int
+ M1_GPU_FIELD_NUMBER: builtins.int
+ START_SPAWN_FIELD_NUMBER: builtins.int
+ START_FORK_FIELD_NUMBER: builtins.int
+ START_FORKSERVER_FIELD_NUMBER: builtins.int
+ START_THREAD_FIELD_NUMBER: builtins.int
+ MAYBE_MP_FIELD_NUMBER: builtins.int
+ TRAINIUM_FIELD_NUMBER: builtins.int
+ PEX_FIELD_NUMBER: builtins.int
+ COLAB_FIELD_NUMBER: builtins.int
+ IPYTHON_FIELD_NUMBER: builtins.int
+ AWS_LAMBDA_FIELD_NUMBER: builtins.int
+ AMD_GPU_FIELD_NUMBER: builtins.int
+ jupyter: builtins.bool
+ """jupyter env detected"""
+ kaggle: builtins.bool
+ """kaggle env detected"""
+ windows: builtins.bool
+ """windows detected"""
+ m1_gpu: builtins.bool
+ """apple silicon M1 gpu found"""
+ start_spawn: builtins.bool
+ """multiprocessing spawn"""
+ start_fork: builtins.bool
+ """multiprocessing fork"""
+ start_forkserver: builtins.bool
+ """multiprocessing forkserver"""
+ start_thread: builtins.bool
+ """thread start method"""
+ maybe_mp: builtins.bool
+ """maybe user running multiprocessing"""
+ trainium: builtins.bool
+ """AWS Trainium env detected"""
+ pex: builtins.bool
+ """pex env detected"""
+ colab: builtins.bool
+ """colab env detected"""
+ ipython: builtins.bool
+ """ipython env detected"""
+ aws_lambda: builtins.bool
+ """running in AWS Lambda"""
+ amd_gpu: builtins.bool
+ """AMD GPU detected"""
+ def __init__(
+ self,
+ *,
+ jupyter: builtins.bool = ...,
+ kaggle: builtins.bool = ...,
+ windows: builtins.bool = ...,
+ m1_gpu: builtins.bool = ...,
+ start_spawn: builtins.bool = ...,
+ start_fork: builtins.bool = ...,
+ start_forkserver: builtins.bool = ...,
+ start_thread: builtins.bool = ...,
+ maybe_mp: builtins.bool = ...,
+ trainium: builtins.bool = ...,
+ pex: builtins.bool = ...,
+ colab: builtins.bool = ...,
+ ipython: builtins.bool = ...,
+ aws_lambda: builtins.bool = ...,
+ amd_gpu: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["amd_gpu", b"amd_gpu", "aws_lambda", b"aws_lambda", "colab", b"colab", "ipython", b"ipython", "jupyter", b"jupyter", "kaggle", b"kaggle", "m1_gpu", b"m1_gpu", "maybe_mp", b"maybe_mp", "pex", b"pex", "start_fork", b"start_fork", "start_forkserver", b"start_forkserver", "start_spawn", b"start_spawn", "start_thread", b"start_thread", "trainium", b"trainium", "windows", b"windows"]) -> None: ...
+
+global___Env = Env
+
+class Labels(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ CODE_STRING_FIELD_NUMBER: builtins.int
+ REPO_STRING_FIELD_NUMBER: builtins.int
+ CODE_VERSION_FIELD_NUMBER: builtins.int
+ code_string: builtins.str
+ """code identification"""
+ repo_string: builtins.str
+ """repo identification"""
+ code_version: builtins.str
+ """code version"""
+ def __init__(
+ self,
+ *,
+ code_string: builtins.str = ...,
+ repo_string: builtins.str = ...,
+ code_version: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["code_string", b"code_string", "code_version", b"code_version", "repo_string", b"repo_string"]) -> None: ...
+
+global___Labels = Labels
+
+class Deprecated(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KERAS_CALLBACK__DATA_TYPE_FIELD_NUMBER: builtins.int
+ RUN__MODE_FIELD_NUMBER: builtins.int
+ RUN__SAVE_NO_ARGS_FIELD_NUMBER: builtins.int
+ RUN__JOIN_FIELD_NUMBER: builtins.int
+ PLOTS_FIELD_NUMBER: builtins.int
+ RUN__LOG_SYNC_FIELD_NUMBER: builtins.int
+ INIT__CONFIG_INCLUDE_KEYS_FIELD_NUMBER: builtins.int
+ INIT__CONFIG_EXCLUDE_KEYS_FIELD_NUMBER: builtins.int
+ KERAS_CALLBACK__SAVE_MODEL_FIELD_NUMBER: builtins.int
+ LANGCHAIN_TRACER_FIELD_NUMBER: builtins.int
+ keras_callback__data_type: builtins.bool
+ """wandb.keras.WandbCallback(data_type=...) called"""
+ run__mode: builtins.bool
+ """wandb.run.mode called"""
+ run__save_no_args: builtins.bool
+ """wandb.run.save() called without arguments"""
+ run__join: builtins.bool
+ """wandb.run.join() called"""
+ plots: builtins.bool
+ """wandb.plots.* called"""
+ run__log_sync: builtins.bool
+ """wandb.run.log(sync=...) called"""
+ init__config_include_keys: builtins.bool
+ """wandb.init(config_include_keys=...) called"""
+ init__config_exclude_keys: builtins.bool
+ """wandb.init(config_exclude_keys=...) called"""
+ keras_callback__save_model: builtins.bool
+ """wandb.keras.WandbCallback(save_model=True) called"""
+ langchain_tracer: builtins.bool
+ """wandb.integration.langchain.WandbTracer called"""
+ def __init__(
+ self,
+ *,
+ keras_callback__data_type: builtins.bool = ...,
+ run__mode: builtins.bool = ...,
+ run__save_no_args: builtins.bool = ...,
+ run__join: builtins.bool = ...,
+ plots: builtins.bool = ...,
+ run__log_sync: builtins.bool = ...,
+ init__config_include_keys: builtins.bool = ...,
+ init__config_exclude_keys: builtins.bool = ...,
+ keras_callback__save_model: builtins.bool = ...,
+ langchain_tracer: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["init__config_exclude_keys", b"init__config_exclude_keys", "init__config_include_keys", b"init__config_include_keys", "keras_callback__data_type", b"keras_callback__data_type", "keras_callback__save_model", b"keras_callback__save_model", "langchain_tracer", b"langchain_tracer", "plots", b"plots", "run__join", b"run__join", "run__log_sync", b"run__log_sync", "run__mode", b"run__mode", "run__save_no_args", b"run__save_no_args"]) -> None: ...
+
+global___Deprecated = Deprecated
+
+class Issues(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SETTINGS__VALIDATION_WARNINGS_FIELD_NUMBER: builtins.int
+ SETTINGS__UNEXPECTED_ARGS_FIELD_NUMBER: builtins.int
+ SETTINGS__PREPROCESSING_WARNINGS_FIELD_NUMBER: builtins.int
+ settings__validation_warnings: builtins.bool
+ """validation warnings for settings"""
+ settings__unexpected_args: builtins.bool
+ """unexpected settings init args"""
+ settings__preprocessing_warnings: builtins.bool
+ """settings preprocessing warnings"""
+ def __init__(
+ self,
+ *,
+ settings__validation_warnings: builtins.bool = ...,
+ settings__unexpected_args: builtins.bool = ...,
+ settings__preprocessing_warnings: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["settings__preprocessing_warnings", b"settings__preprocessing_warnings", "settings__unexpected_args", b"settings__unexpected_args", "settings__validation_warnings", b"settings__validation_warnings"]) -> None: ...
+
+global___Issues = Issues
diff --git a/wandb/proto/v4/__init__.py b/wandb/proto/v4/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/proto/v4/wandb_base_pb2.py b/wandb/proto/v4/wandb_base_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..daff9f904e24fb6e668d1afe7cce1b50164ae119
--- /dev/null
+++ b/wandb/proto/v4/wandb_base_pb2.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: wandb/proto/wandb_base.proto
+"""Generated protocol buffer code."""
+from google.protobuf.internal import builder as _builder
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cwandb/proto/wandb_base.proto\x12\x0ewandb_internal\"6\n\x0b_RecordInfo\x12\x11\n\tstream_id\x18\x01 \x01(\t\x12\x14\n\x0c_tracelog_id\x18\x64 \x01(\t\"!\n\x0c_RequestInfo\x12\x11\n\tstream_id\x18\x01 \x01(\t\"#\n\x0b_ResultInfo\x12\x14\n\x0c_tracelog_id\x18\x64 \x01(\tb\x06proto3')
+
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'wandb.proto.wandb_base_pb2', globals())
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+ DESCRIPTOR._options = None
+ __RECORDINFO._serialized_start=48
+ __RECORDINFO._serialized_end=102
+ __REQUESTINFO._serialized_start=104
+ __REQUESTINFO._serialized_end=137
+ __RESULTINFO._serialized_start=139
+ __RESULTINFO._serialized_end=174
+# @@protoc_insertion_point(module_scope)
diff --git a/wandb/proto/v4/wandb_base_pb2.pyi b/wandb/proto/v4/wandb_base_pb2.pyi
new file mode 100644
index 0000000000000000000000000000000000000000..c4664b9c904170c0dfd67f1ef8faf01f66bcba5e
--- /dev/null
+++ b/wandb/proto/v4/wandb_base_pb2.pyi
@@ -0,0 +1,71 @@
+"""
+@generated by mypy-protobuf. Do not edit manually!
+isort:skip_file
+"""
+import builtins
+import google.protobuf.descriptor
+import google.protobuf.message
+import sys
+
+if sys.version_info >= (3, 8):
+ import typing as typing_extensions
+else:
+ import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+@typing_extensions.final
+class _RecordInfo(google.protobuf.message.Message):
+ """
+ _RecordInfo, _RequestInfo: extra info for all records and requests
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ STREAM_ID_FIELD_NUMBER: builtins.int
+ _TRACELOG_ID_FIELD_NUMBER: builtins.int
+ stream_id: builtins.str
+ _tracelog_id: builtins.str
+ def __init__(
+ self,
+ *,
+ stream_id: builtins.str = ...,
+ _tracelog_id: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_tracelog_id", b"_tracelog_id", "stream_id", b"stream_id"]) -> None: ...
+
+global____RecordInfo = _RecordInfo
+
+@typing_extensions.final
+class _RequestInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ STREAM_ID_FIELD_NUMBER: builtins.int
+ stream_id: builtins.str
+ def __init__(
+ self,
+ *,
+ stream_id: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["stream_id", b"stream_id"]) -> None: ...
+
+global____RequestInfo = _RequestInfo
+
+@typing_extensions.final
+class _ResultInfo(google.protobuf.message.Message):
+ """
+ _ResultInfo: extra info for all results
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _TRACELOG_ID_FIELD_NUMBER: builtins.int
+ _tracelog_id: builtins.str
+ def __init__(
+ self,
+ *,
+ _tracelog_id: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_tracelog_id", b"_tracelog_id"]) -> None: ...
+
+global____ResultInfo = _ResultInfo
diff --git a/wandb/proto/v4/wandb_internal_pb2.py b/wandb/proto/v4/wandb_internal_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..ff194ba2aa669416332c1ed167534aac5e18ebef
--- /dev/null
+++ b/wandb/proto/v4/wandb_internal_pb2.py
@@ -0,0 +1,344 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: wandb/proto/wandb_internal.proto
+"""Generated protocol buffer code."""
+from google.protobuf.internal import builder as _builder
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
+from wandb.proto import wandb_base_pb2 as wandb_dot_proto_dot_wandb__base__pb2
+from wandb.proto import wandb_telemetry_pb2 as wandb_dot_proto_dot_wandb__telemetry__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n wandb/proto/wandb_internal.proto\x12\x0ewandb_internal\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1cwandb/proto/wandb_base.proto\x1a!wandb/proto/wandb_telemetry.proto\"\x9c\t\n\x06Record\x12\x0b\n\x03num\x18\x01 \x01(\x03\x12\x30\n\x07history\x18\x02 \x01(\x0b\x32\x1d.wandb_internal.HistoryRecordH\x00\x12\x30\n\x07summary\x18\x03 \x01(\x0b\x32\x1d.wandb_internal.SummaryRecordH\x00\x12.\n\x06output\x18\x04 \x01(\x0b\x32\x1c.wandb_internal.OutputRecordH\x00\x12.\n\x06\x63onfig\x18\x05 \x01(\x0b\x32\x1c.wandb_internal.ConfigRecordH\x00\x12,\n\x05\x66iles\x18\x06 \x01(\x0b\x32\x1b.wandb_internal.FilesRecordH\x00\x12,\n\x05stats\x18\x07 \x01(\x0b\x32\x1b.wandb_internal.StatsRecordH\x00\x12\x32\n\x08\x61rtifact\x18\x08 \x01(\x0b\x32\x1e.wandb_internal.ArtifactRecordH\x00\x12,\n\x08tbrecord\x18\t \x01(\x0b\x32\x18.wandb_internal.TBRecordH\x00\x12,\n\x05\x61lert\x18\n \x01(\x0b\x32\x1b.wandb_internal.AlertRecordH\x00\x12\x34\n\ttelemetry\x18\x0b \x01(\x0b\x32\x1f.wandb_internal.TelemetryRecordH\x00\x12.\n\x06metric\x18\x0c \x01(\x0b\x32\x1c.wandb_internal.MetricRecordH\x00\x12\x35\n\noutput_raw\x18\r \x01(\x0b\x32\x1f.wandb_internal.OutputRawRecordH\x00\x12(\n\x03run\x18\x11 \x01(\x0b\x32\x19.wandb_internal.RunRecordH\x00\x12-\n\x04\x65xit\x18\x12 \x01(\x0b\x32\x1d.wandb_internal.RunExitRecordH\x00\x12,\n\x05\x66inal\x18\x14 \x01(\x0b\x32\x1b.wandb_internal.FinalRecordH\x00\x12.\n\x06header\x18\x15 \x01(\x0b\x32\x1c.wandb_internal.HeaderRecordH\x00\x12.\n\x06\x66ooter\x18\x16 \x01(\x0b\x32\x1c.wandb_internal.FooterRecordH\x00\x12\x39\n\npreempting\x18\x17 \x01(\x0b\x32#.wandb_internal.RunPreemptingRecordH\x00\x12;\n\rlink_artifact\x18\x18 \x01(\x0b\x32\".wandb_internal.LinkArtifactRecordH\x00\x12\x39\n\x0cuse_artifact\x18\x19 \x01(\x0b\x32!.wandb_internal.UseArtifactRecordH\x00\x12*\n\x07request\x18\x64 \x01(\x0b\x32\x17.wandb_internal.RequestH\x00\x12(\n\x07\x63ontrol\x18\x10 \x01(\x0b\x32\x17.wandb_internal.Control\x12\x0c\n\x04uuid\x18\x13 \x01(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfoB\r\n\x0brecord_type\"\xa8\x01\n\x07\x43ontrol\x12\x10\n\x08req_resp\x18\x01 \x01(\x08\x12\r\n\x05local\x18\x02 \x01(\x08\x12\x10\n\x08relay_id\x18\x03 \x01(\t\x12\x14\n\x0cmailbox_slot\x18\x04 \x01(\t\x12\x13\n\x0b\x61lways_send\x18\x05 \x01(\x08\x12\x14\n\x0c\x66low_control\x18\x06 \x01(\x08\x12\x12\n\nend_offset\x18\x07 \x01(\x03\x12\x15\n\rconnection_id\x18\x08 \x01(\t\"\xf3\x03\n\x06Result\x12\x35\n\nrun_result\x18\x11 \x01(\x0b\x32\x1f.wandb_internal.RunUpdateResultH\x00\x12\x34\n\x0b\x65xit_result\x18\x12 \x01(\x0b\x32\x1d.wandb_internal.RunExitResultH\x00\x12\x33\n\nlog_result\x18\x14 \x01(\x0b\x32\x1d.wandb_internal.HistoryResultH\x00\x12\x37\n\x0esummary_result\x18\x15 \x01(\x0b\x32\x1d.wandb_internal.SummaryResultH\x00\x12\x35\n\routput_result\x18\x16 \x01(\x0b\x32\x1c.wandb_internal.OutputResultH\x00\x12\x35\n\rconfig_result\x18\x17 \x01(\x0b\x32\x1c.wandb_internal.ConfigResultH\x00\x12,\n\x08response\x18\x64 \x01(\x0b\x32\x18.wandb_internal.ResponseH\x00\x12(\n\x07\x63ontrol\x18\x10 \x01(\x0b\x32\x17.wandb_internal.Control\x12\x0c\n\x04uuid\x18\x18 \x01(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._ResultInfoB\r\n\x0bresult_type\":\n\x0b\x46inalRecord\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"b\n\x0bVersionInfo\x12\x10\n\x08producer\x18\x01 \x01(\t\x12\x14\n\x0cmin_consumer\x18\x02 \x01(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"n\n\x0cHeaderRecord\x12\x31\n\x0cversion_info\x18\x01 \x01(\x0b\x32\x1b.wandb_internal.VersionInfo\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\";\n\x0c\x46ooterRecord\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\xce\x04\n\tRunRecord\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x0e\n\x06\x65ntity\x18\x02 \x01(\t\x12\x0f\n\x07project\x18\x03 \x01(\t\x12,\n\x06\x63onfig\x18\x04 \x01(\x0b\x32\x1c.wandb_internal.ConfigRecord\x12.\n\x07summary\x18\x05 \x01(\x0b\x32\x1d.wandb_internal.SummaryRecord\x12\x11\n\trun_group\x18\x06 \x01(\t\x12\x10\n\x08job_type\x18\x07 \x01(\t\x12\x14\n\x0c\x64isplay_name\x18\x08 \x01(\t\x12\r\n\x05notes\x18\t \x01(\t\x12\x0c\n\x04tags\x18\n \x03(\t\x12\x30\n\x08settings\x18\x0b \x01(\x0b\x32\x1e.wandb_internal.SettingsRecord\x12\x10\n\x08sweep_id\x18\x0c \x01(\t\x12\x0c\n\x04host\x18\r \x01(\t\x12\x15\n\rstarting_step\x18\x0e \x01(\x03\x12\x12\n\nstorage_id\x18\x10 \x01(\t\x12.\n\nstart_time\x18\x11 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07resumed\x18\x12 \x01(\x08\x12\x32\n\ttelemetry\x18\x13 \x01(\x0b\x32\x1f.wandb_internal.TelemetryRecord\x12\x0f\n\x07runtime\x18\x14 \x01(\x05\x12*\n\x03git\x18\x15 \x01(\x0b\x32\x1d.wandb_internal.GitRepoRecord\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\";\n\rGitRepoRecord\x12\x1a\n\nremote_url\x18\x01 \x01(\tR\x06remote\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\t\"c\n\x0fRunUpdateResult\x12&\n\x03run\x18\x01 \x01(\x0b\x32\x19.wandb_internal.RunRecord\x12(\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x19.wandb_internal.ErrorInfo\"\xac\x01\n\tErrorInfo\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x31\n\x04\x63ode\x18\x02 \x01(\x0e\x32#.wandb_internal.ErrorInfo.ErrorCode\"[\n\tErrorCode\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x11\n\rCOMMUNICATION\x10\x01\x12\x12\n\x0e\x41UTHENTICATION\x10\x02\x12\t\n\x05USAGE\x10\x03\x12\x0f\n\x0bUNSUPPORTED\x10\x04\"`\n\rRunExitRecord\x12\x11\n\texit_code\x18\x01 \x01(\x05\x12\x0f\n\x07runtime\x18\x02 \x01(\x05\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x0f\n\rRunExitResult\"B\n\x13RunPreemptingRecord\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x15\n\x13RunPreemptingResult\"i\n\x0eSettingsRecord\x12*\n\x04item\x18\x01 \x03(\x0b\x32\x1c.wandb_internal.SettingsItem\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"/\n\x0cSettingsItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nvalue_json\x18\x10 \x01(\t\"\x1a\n\x0bHistoryStep\x12\x0b\n\x03num\x18\x01 \x01(\x03\"\x92\x01\n\rHistoryRecord\x12)\n\x04item\x18\x01 \x03(\x0b\x32\x1b.wandb_internal.HistoryItem\x12)\n\x04step\x18\x02 \x01(\x0b\x32\x1b.wandb_internal.HistoryStep\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"B\n\x0bHistoryItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nnested_key\x18\x02 \x03(\t\x12\x12\n\nvalue_json\x18\x10 \x01(\t\"\x0f\n\rHistoryResult\"\xdc\x01\n\x0cOutputRecord\x12<\n\x0boutput_type\x18\x01 \x01(\x0e\x32\'.wandb_internal.OutputRecord.OutputType\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04line\x18\x03 \x01(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"$\n\nOutputType\x12\n\n\x06STDERR\x10\x00\x12\n\n\x06STDOUT\x10\x01\"\x0e\n\x0cOutputResult\"\xe2\x01\n\x0fOutputRawRecord\x12?\n\x0boutput_type\x18\x01 \x01(\x0e\x32*.wandb_internal.OutputRawRecord.OutputType\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04line\x18\x03 \x01(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"$\n\nOutputType\x12\n\n\x06STDERR\x10\x00\x12\n\n\x06STDOUT\x10\x01\"\x11\n\x0fOutputRawResult\"\x98\x03\n\x0cMetricRecord\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\tglob_name\x18\x02 \x01(\t\x12\x13\n\x0bstep_metric\x18\x04 \x01(\t\x12\x19\n\x11step_metric_index\x18\x05 \x01(\x05\x12.\n\x07options\x18\x06 \x01(\x0b\x32\x1d.wandb_internal.MetricOptions\x12.\n\x07summary\x18\x07 \x01(\x0b\x32\x1d.wandb_internal.MetricSummary\x12\x35\n\x04goal\x18\x08 \x01(\x0e\x32\'.wandb_internal.MetricRecord.MetricGoal\x12/\n\x08_control\x18\t \x01(\x0b\x32\x1d.wandb_internal.MetricControl\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"B\n\nMetricGoal\x12\x0e\n\nGOAL_UNSET\x10\x00\x12\x11\n\rGOAL_MINIMIZE\x10\x01\x12\x11\n\rGOAL_MAXIMIZE\x10\x02\"\x0e\n\x0cMetricResult\"C\n\rMetricOptions\x12\x11\n\tstep_sync\x18\x01 \x01(\x08\x12\x0e\n\x06hidden\x18\x02 \x01(\x08\x12\x0f\n\x07\x64\x65\x66ined\x18\x03 \x01(\x08\"\"\n\rMetricControl\x12\x11\n\toverwrite\x18\x01 \x01(\x08\"o\n\rMetricSummary\x12\x0b\n\x03min\x18\x01 \x01(\x08\x12\x0b\n\x03max\x18\x02 \x01(\x08\x12\x0c\n\x04mean\x18\x03 \x01(\x08\x12\x0c\n\x04\x62\x65st\x18\x04 \x01(\x08\x12\x0c\n\x04last\x18\x05 \x01(\x08\x12\x0c\n\x04none\x18\x06 \x01(\x08\x12\x0c\n\x04\x63opy\x18\x07 \x01(\x08\"\x93\x01\n\x0c\x43onfigRecord\x12*\n\x06update\x18\x01 \x03(\x0b\x32\x1a.wandb_internal.ConfigItem\x12*\n\x06remove\x18\x02 \x03(\x0b\x32\x1a.wandb_internal.ConfigItem\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"A\n\nConfigItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nnested_key\x18\x02 \x03(\t\x12\x12\n\nvalue_json\x18\x10 \x01(\t\"\x0e\n\x0c\x43onfigResult\"\x96\x01\n\rSummaryRecord\x12+\n\x06update\x18\x01 \x03(\x0b\x32\x1b.wandb_internal.SummaryItem\x12+\n\x06remove\x18\x02 \x03(\x0b\x32\x1b.wandb_internal.SummaryItem\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"B\n\x0bSummaryItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nnested_key\x18\x02 \x03(\t\x12\x12\n\nvalue_json\x18\x10 \x01(\t\"\x0f\n\rSummaryResult\"d\n\x0b\x46ilesRecord\x12(\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x19.wandb_internal.FilesItem\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\xfd\x01\n\tFilesItem\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x34\n\x06policy\x18\x02 \x01(\x0e\x32$.wandb_internal.FilesItem.PolicyType\x12\x30\n\x04type\x18\x03 \x01(\x0e\x32\".wandb_internal.FilesItem.FileType\x12\x15\n\rexternal_path\x18\x10 \x01(\t\"(\n\nPolicyType\x12\x07\n\x03NOW\x10\x00\x12\x07\n\x03\x45ND\x10\x01\x12\x08\n\x04LIVE\x10\x02\"9\n\x08\x46ileType\x12\t\n\x05OTHER\x10\x00\x12\t\n\x05WANDB\x10\x01\x12\t\n\x05MEDIA\x10\x02\x12\x0c\n\x08\x41RTIFACT\x10\x03\"\r\n\x0b\x46ilesResult\"\xe6\x01\n\x0bStatsRecord\x12\x39\n\nstats_type\x18\x01 \x01(\x0e\x32%.wandb_internal.StatsRecord.StatsType\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\'\n\x04item\x18\x03 \x03(\x0b\x32\x19.wandb_internal.StatsItem\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x17\n\tStatsType\x12\n\n\x06SYSTEM\x10\x00\",\n\tStatsItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nvalue_json\x18\x10 \x01(\t\"\xd9\x03\n\x0e\x41rtifactRecord\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x65ntity\x18\x03 \x01(\t\x12\x0c\n\x04type\x18\x04 \x01(\t\x12\x0c\n\x04name\x18\x05 \x01(\t\x12\x0e\n\x06\x64igest\x18\x06 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x07 \x01(\t\x12\x10\n\x08metadata\x18\x08 \x01(\t\x12\x14\n\x0cuser_created\x18\t \x01(\x08\x12\x18\n\x10use_after_commit\x18\n \x01(\x08\x12\x0f\n\x07\x61liases\x18\x0b \x03(\t\x12\x32\n\x08manifest\x18\x0c \x01(\x0b\x32 .wandb_internal.ArtifactManifest\x12\x16\n\x0e\x64istributed_id\x18\r \x01(\t\x12\x10\n\x08\x66inalize\x18\x0e \x01(\x08\x12\x11\n\tclient_id\x18\x0f \x01(\t\x12\x1a\n\x12sequence_client_id\x18\x10 \x01(\t\x12\x0f\n\x07\x62\x61se_id\x18\x11 \x01(\t\x12\x1c\n\x14ttl_duration_seconds\x18\x12 \x01(\x03\x12\x19\n\x11incremental_beta1\x18\x64 \x01(\x08\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\xbc\x01\n\x10\x41rtifactManifest\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x16\n\x0estorage_policy\x18\x02 \x01(\t\x12\x46\n\x15storage_policy_config\x18\x03 \x03(\x0b\x32\'.wandb_internal.StoragePolicyConfigItem\x12\x37\n\x08\x63ontents\x18\x04 \x03(\x0b\x32%.wandb_internal.ArtifactManifestEntry\"\xbb\x01\n\x15\x41rtifactManifestEntry\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0e\n\x06\x64igest\x18\x02 \x01(\t\x12\x0b\n\x03ref\x18\x03 \x01(\t\x12\x0c\n\x04size\x18\x04 \x01(\x03\x12\x10\n\x08mimetype\x18\x05 \x01(\t\x12\x12\n\nlocal_path\x18\x06 \x01(\t\x12\x19\n\x11\x62irth_artifact_id\x18\x07 \x01(\t\x12(\n\x05\x65xtra\x18\x10 \x03(\x0b\x32\x19.wandb_internal.ExtraItem\",\n\tExtraItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nvalue_json\x18\x02 \x01(\t\":\n\x17StoragePolicyConfigItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nvalue_json\x18\x02 \x01(\t\"\x10\n\x0e\x41rtifactResult\"\x14\n\x12LinkArtifactResult\"\xcf\x01\n\x12LinkArtifactRecord\x12\x11\n\tclient_id\x18\x01 \x01(\t\x12\x11\n\tserver_id\x18\x02 \x01(\t\x12\x16\n\x0eportfolio_name\x18\x03 \x01(\t\x12\x18\n\x10portfolio_entity\x18\x04 \x01(\t\x12\x19\n\x11portfolio_project\x18\x05 \x01(\t\x12\x19\n\x11portfolio_aliases\x18\x06 \x03(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"h\n\x08TBRecord\x12\x0f\n\x07log_dir\x18\x01 \x01(\t\x12\x0c\n\x04save\x18\x02 \x01(\x08\x12\x10\n\x08root_dir\x18\x03 \x01(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\n\n\x08TBResult\"}\n\x0b\x41lertRecord\x12\r\n\x05title\x18\x01 \x01(\t\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\r\n\x05level\x18\x03 \x01(\t\x12\x15\n\rwait_duration\x18\x04 \x01(\x03\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\r\n\x0b\x41lertResult\"\xc9\x0f\n\x07Request\x12\x38\n\x0bstop_status\x18\x01 \x01(\x0b\x32!.wandb_internal.StopStatusRequestH\x00\x12>\n\x0enetwork_status\x18\x02 \x01(\x0b\x32$.wandb_internal.NetworkStatusRequestH\x00\x12-\n\x05\x64\x65\x66\x65r\x18\x03 \x01(\x0b\x32\x1c.wandb_internal.DeferRequestH\x00\x12\x38\n\x0bget_summary\x18\x04 \x01(\x0b\x32!.wandb_internal.GetSummaryRequestH\x00\x12-\n\x05login\x18\x05 \x01(\x0b\x32\x1c.wandb_internal.LoginRequestH\x00\x12-\n\x05pause\x18\x06 \x01(\x0b\x32\x1c.wandb_internal.PauseRequestH\x00\x12/\n\x06resume\x18\x07 \x01(\x0b\x32\x1d.wandb_internal.ResumeRequestH\x00\x12\x34\n\tpoll_exit\x18\x08 \x01(\x0b\x32\x1f.wandb_internal.PollExitRequestH\x00\x12@\n\x0fsampled_history\x18\t \x01(\x0b\x32%.wandb_internal.SampledHistoryRequestH\x00\x12@\n\x0fpartial_history\x18\n \x01(\x0b\x32%.wandb_internal.PartialHistoryRequestH\x00\x12\x34\n\trun_start\x18\x0b \x01(\x0b\x32\x1f.wandb_internal.RunStartRequestH\x00\x12<\n\rcheck_version\x18\x0c \x01(\x0b\x32#.wandb_internal.CheckVersionRequestH\x00\x12:\n\x0clog_artifact\x18\r \x01(\x0b\x32\".wandb_internal.LogArtifactRequestH\x00\x12\x44\n\x11\x64ownload_artifact\x18\x0e \x01(\x0b\x32\'.wandb_internal.DownloadArtifactRequestH\x00\x12\x35\n\tkeepalive\x18\x11 \x01(\x0b\x32 .wandb_internal.KeepaliveRequestH\x00\x12\x36\n\nrun_status\x18\x14 \x01(\x0b\x32 .wandb_internal.RunStatusRequestH\x00\x12/\n\x06\x63\x61ncel\x18\x15 \x01(\x0b\x32\x1d.wandb_internal.CancelRequestH\x00\x12\x33\n\x08metadata\x18\x16 \x01(\x0b\x32\x1f.wandb_internal.MetadataRequestH\x00\x12\x44\n\x11internal_messages\x18\x17 \x01(\x0b\x32\'.wandb_internal.InternalMessagesRequestH\x00\x12@\n\x0fpython_packages\x18\x18 \x01(\x0b\x32%.wandb_internal.PythonPackagesRequestH\x00\x12\x33\n\x08shutdown\x18@ \x01(\x0b\x32\x1f.wandb_internal.ShutdownRequestH\x00\x12/\n\x06\x61ttach\x18\x41 \x01(\x0b\x32\x1d.wandb_internal.AttachRequestH\x00\x12/\n\x06status\x18\x42 \x01(\x0b\x32\x1d.wandb_internal.StatusRequestH\x00\x12\x38\n\x0bserver_info\x18\x43 \x01(\x0b\x32!.wandb_internal.ServerInfoRequestH\x00\x12\x38\n\x0bsender_mark\x18\x44 \x01(\x0b\x32!.wandb_internal.SenderMarkRequestH\x00\x12\x38\n\x0bsender_read\x18\x45 \x01(\x0b\x32!.wandb_internal.SenderReadRequestH\x00\x12<\n\rstatus_report\x18\x46 \x01(\x0b\x32#.wandb_internal.StatusReportRequestH\x00\x12>\n\x0esummary_record\x18G \x01(\x0b\x32$.wandb_internal.SummaryRecordRequestH\x00\x12\x42\n\x10telemetry_record\x18H \x01(\x0b\x32&.wandb_internal.TelemetryRecordRequestH\x00\x12\x32\n\x08job_info\x18I \x01(\x0b\x32\x1e.wandb_internal.JobInfoRequestH\x00\x12\x45\n\x12get_system_metrics\x18J \x01(\x0b\x32\'.wandb_internal.GetSystemMetricsRequestH\x00\x12\x45\n\x12\x66ile_transfer_info\x18K \x01(\x0b\x32\'.wandb_internal.FileTransferInfoRequestH\x00\x12+\n\x04sync\x18L \x01(\x0b\x32\x1b.wandb_internal.SyncRequestH\x00\x12\x39\n\x0btest_inject\x18\xe8\x07 \x01(\x0b\x32!.wandb_internal.TestInjectRequestH\x00\x42\x0e\n\x0crequest_type\"\xe2\x0b\n\x08Response\x12?\n\x12keepalive_response\x18\x12 \x01(\x0b\x32!.wandb_internal.KeepaliveResponseH\x00\x12\x42\n\x14stop_status_response\x18\x13 \x01(\x0b\x32\".wandb_internal.StopStatusResponseH\x00\x12H\n\x17network_status_response\x18\x14 \x01(\x0b\x32%.wandb_internal.NetworkStatusResponseH\x00\x12\x37\n\x0elogin_response\x18\x18 \x01(\x0b\x32\x1d.wandb_internal.LoginResponseH\x00\x12\x42\n\x14get_summary_response\x18\x19 \x01(\x0b\x32\".wandb_internal.GetSummaryResponseH\x00\x12>\n\x12poll_exit_response\x18\x1a \x01(\x0b\x32 .wandb_internal.PollExitResponseH\x00\x12J\n\x18sampled_history_response\x18\x1b \x01(\x0b\x32&.wandb_internal.SampledHistoryResponseH\x00\x12>\n\x12run_start_response\x18\x1c \x01(\x0b\x32 .wandb_internal.RunStartResponseH\x00\x12\x46\n\x16\x63heck_version_response\x18\x1d \x01(\x0b\x32$.wandb_internal.CheckVersionResponseH\x00\x12\x44\n\x15log_artifact_response\x18\x1e \x01(\x0b\x32#.wandb_internal.LogArtifactResponseH\x00\x12N\n\x1a\x64ownload_artifact_response\x18\x1f \x01(\x0b\x32(.wandb_internal.DownloadArtifactResponseH\x00\x12@\n\x13run_status_response\x18# \x01(\x0b\x32!.wandb_internal.RunStatusResponseH\x00\x12\x39\n\x0f\x63\x61ncel_response\x18$ \x01(\x0b\x32\x1e.wandb_internal.CancelResponseH\x00\x12N\n\x1ainternal_messages_response\x18% \x01(\x0b\x32(.wandb_internal.InternalMessagesResponseH\x00\x12=\n\x11shutdown_response\x18@ \x01(\x0b\x32 .wandb_internal.ShutdownResponseH\x00\x12\x39\n\x0f\x61ttach_response\x18\x41 \x01(\x0b\x32\x1e.wandb_internal.AttachResponseH\x00\x12\x39\n\x0fstatus_response\x18\x42 \x01(\x0b\x32\x1e.wandb_internal.StatusResponseH\x00\x12\x42\n\x14server_info_response\x18\x43 \x01(\x0b\x32\".wandb_internal.ServerInfoResponseH\x00\x12<\n\x11job_info_response\x18\x44 \x01(\x0b\x32\x1f.wandb_internal.JobInfoResponseH\x00\x12O\n\x1bget_system_metrics_response\x18\x45 \x01(\x0b\x32(.wandb_internal.GetSystemMetricsResponseH\x00\x12\x35\n\rsync_response\x18\x46 \x01(\x0b\x32\x1c.wandb_internal.SyncResponseH\x00\x12\x43\n\x14test_inject_response\x18\xe8\x07 \x01(\x0b\x32\".wandb_internal.TestInjectResponseH\x00\x42\x0f\n\rresponse_type\"\xc0\x02\n\x0c\x44\x65\x66\x65rRequest\x12\x36\n\x05state\x18\x01 \x01(\x0e\x32\'.wandb_internal.DeferRequest.DeferState\"\xf7\x01\n\nDeferState\x12\t\n\x05\x42\x45GIN\x10\x00\x12\r\n\tFLUSH_RUN\x10\x01\x12\x0f\n\x0b\x46LUSH_STATS\x10\x02\x12\x19\n\x15\x46LUSH_PARTIAL_HISTORY\x10\x03\x12\x0c\n\x08\x46LUSH_TB\x10\x04\x12\r\n\tFLUSH_SUM\x10\x05\x12\x13\n\x0f\x46LUSH_DEBOUNCER\x10\x06\x12\x10\n\x0c\x46LUSH_OUTPUT\x10\x07\x12\r\n\tFLUSH_JOB\x10\x08\x12\r\n\tFLUSH_DIR\x10\t\x12\x0c\n\x08\x46LUSH_FP\x10\n\x12\x0b\n\x07JOIN_FP\x10\x0b\x12\x0c\n\x08\x46LUSH_FS\x10\x0c\x12\x0f\n\x0b\x46LUSH_FINAL\x10\r\x12\x07\n\x03\x45ND\x10\x0e\"<\n\x0cPauseRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x0f\n\rPauseResponse\"=\n\rResumeRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x10\n\x0eResumeResponse\"M\n\x0cLoginRequest\x12\x0f\n\x07\x61pi_key\x18\x01 \x01(\t\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"&\n\rLoginResponse\x12\x15\n\ractive_entity\x18\x01 \x01(\t\"A\n\x11GetSummaryRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"?\n\x12GetSummaryResponse\x12)\n\x04item\x18\x01 \x03(\x0b\x32\x1b.wandb_internal.SummaryItem\"G\n\x17GetSystemMetricsRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"R\n\x12SystemMetricSample\x12-\n\ttimestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\r\n\x05value\x18\x02 \x01(\x02\"I\n\x13SystemMetricsBuffer\x12\x32\n\x06record\x18\x01 \x03(\x0b\x32\".wandb_internal.SystemMetricSample\"\xca\x01\n\x18GetSystemMetricsResponse\x12S\n\x0esystem_metrics\x18\x01 \x03(\x0b\x32;.wandb_internal.GetSystemMetricsResponse.SystemMetricsEntry\x1aY\n\x12SystemMetricsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x32\n\x05value\x18\x02 \x01(\x0b\x32#.wandb_internal.SystemMetricsBuffer:\x02\x38\x01\"=\n\rStatusRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\")\n\x0eStatusResponse\x12\x17\n\x0frun_should_stop\x18\x01 \x01(\x08\"A\n\x11StopStatusRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"-\n\x12StopStatusResponse\x12\x17\n\x0frun_should_stop\x18\x01 \x01(\x08\"D\n\x14NetworkStatusRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"P\n\x15NetworkStatusResponse\x12\x37\n\x11network_responses\x18\x01 \x03(\x0b\x32\x1c.wandb_internal.HttpResponse\"D\n\x0cHttpResponse\x12\x18\n\x10http_status_code\x18\x01 \x01(\x05\x12\x1a\n\x12http_response_text\x18\x02 \x01(\t\"G\n\x17InternalMessagesRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"N\n\x18InternalMessagesResponse\x12\x32\n\x08messages\x18\x01 \x01(\x0b\x32 .wandb_internal.InternalMessages\"#\n\x10InternalMessages\x12\x0f\n\x07warning\x18\x01 \x03(\t\"?\n\x0fPollExitRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\xbc\x01\n\x10PollExitResponse\x12\x0c\n\x04\x64one\x18\x01 \x01(\x08\x12\x32\n\x0b\x65xit_result\x18\x02 \x01(\x0b\x32\x1d.wandb_internal.RunExitResult\x12\x35\n\x0cpusher_stats\x18\x03 \x01(\x0b\x32\x1f.wandb_internal.FilePusherStats\x12/\n\x0b\x66ile_counts\x18\x04 \x01(\x0b\x32\x1a.wandb_internal.FileCounts\"@\n\rSyncOverwrite\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x0e\n\x06\x65ntity\x18\x02 \x01(\t\x12\x0f\n\x07project\x18\x03 \x01(\t\"\x1e\n\x08SyncSkip\x12\x12\n\noutput_raw\x18\x01 \x01(\x08\"\x13\n\x11SenderMarkRequest\"\x93\x01\n\x0bSyncRequest\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03\x12\x14\n\x0c\x66inal_offset\x18\x02 \x01(\x03\x12\x30\n\toverwrite\x18\x03 \x01(\x0b\x32\x1d.wandb_internal.SyncOverwrite\x12&\n\x04skip\x18\x04 \x01(\x0b\x32\x18.wandb_internal.SyncSkip\"E\n\x0cSyncResponse\x12\x0b\n\x03url\x18\x01 \x01(\t\x12(\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x19.wandb_internal.ErrorInfo\"?\n\x11SenderReadRequest\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03\x12\x14\n\x0c\x66inal_offset\x18\x02 \x01(\x03\"m\n\x13StatusReportRequest\x12\x12\n\nrecord_num\x18\x01 \x01(\x03\x12\x13\n\x0bsent_offset\x18\x02 \x01(\x03\x12-\n\tsync_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"F\n\x14SummaryRecordRequest\x12.\n\x07summary\x18\x01 \x01(\x0b\x32\x1d.wandb_internal.SummaryRecord\"L\n\x16TelemetryRecordRequest\x12\x32\n\ttelemetry\x18\x01 \x01(\x0b\x32\x1f.wandb_internal.TelemetryRecord\"A\n\x11ServerInfoRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"|\n\x12ServerInfoResponse\x12-\n\nlocal_info\x18\x01 \x01(\x0b\x32\x19.wandb_internal.LocalInfo\x12\x37\n\x0fserver_messages\x18\x02 \x01(\x0b\x32\x1e.wandb_internal.ServerMessages\"=\n\x0eServerMessages\x12+\n\x04item\x18\x01 \x03(\x0b\x32\x1d.wandb_internal.ServerMessage\"e\n\rServerMessage\x12\x12\n\nplain_text\x18\x01 \x01(\t\x12\x10\n\x08utf_text\x18\x02 \x01(\t\x12\x11\n\thtml_text\x18\x03 \x01(\t\x12\x0c\n\x04type\x18\x04 \x01(\t\x12\r\n\x05level\x18\x05 \x01(\x05\"c\n\nFileCounts\x12\x13\n\x0bwandb_count\x18\x01 \x01(\x05\x12\x13\n\x0bmedia_count\x18\x02 \x01(\x05\x12\x16\n\x0e\x61rtifact_count\x18\x03 \x01(\x05\x12\x13\n\x0bother_count\x18\x04 \x01(\x05\"U\n\x0f\x46ilePusherStats\x12\x16\n\x0euploaded_bytes\x18\x01 \x01(\x03\x12\x13\n\x0btotal_bytes\x18\x02 \x01(\x03\x12\x15\n\rdeduped_bytes\x18\x03 \x01(\x03\"\x1e\n\rFilesUploaded\x12\r\n\x05\x66iles\x18\x01 \x03(\t\"\xf4\x01\n\x17\x46ileTransferInfoRequest\x12\x42\n\x04type\x18\x01 \x01(\x0e\x32\x34.wandb_internal.FileTransferInfoRequest.TransferType\x12\x0c\n\x04path\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12\x0c\n\x04size\x18\x04 \x01(\x03\x12\x11\n\tprocessed\x18\x05 \x01(\x03\x12/\n\x0b\x66ile_counts\x18\x06 \x01(\x0b\x32\x1a.wandb_internal.FileCounts\"(\n\x0cTransferType\x12\n\n\x06Upload\x10\x00\x12\x0c\n\x08\x44ownload\x10\x01\"1\n\tLocalInfo\x12\x0f\n\x07version\x18\x01 \x01(\t\x12\x13\n\x0bout_of_date\x18\x02 \x01(\x08\"?\n\x0fShutdownRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x12\n\x10ShutdownResponse\"P\n\rAttachRequest\x12\x11\n\tattach_id\x18\x14 \x01(\t\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"b\n\x0e\x41ttachResponse\x12&\n\x03run\x18\x01 \x01(\x0b\x32\x19.wandb_internal.RunRecord\x12(\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x19.wandb_internal.ErrorInfo\"\xd5\x02\n\x11TestInjectRequest\x12\x13\n\x0bhandler_exc\x18\x01 \x01(\x08\x12\x14\n\x0chandler_exit\x18\x02 \x01(\x08\x12\x15\n\rhandler_abort\x18\x03 \x01(\x08\x12\x12\n\nsender_exc\x18\x04 \x01(\x08\x12\x13\n\x0bsender_exit\x18\x05 \x01(\x08\x12\x14\n\x0csender_abort\x18\x06 \x01(\x08\x12\x0f\n\x07req_exc\x18\x07 \x01(\x08\x12\x10\n\x08req_exit\x18\x08 \x01(\x08\x12\x11\n\treq_abort\x18\t \x01(\x08\x12\x10\n\x08resp_exc\x18\n \x01(\x08\x12\x11\n\tresp_exit\x18\x0b \x01(\x08\x12\x12\n\nresp_abort\x18\x0c \x01(\x08\x12\x10\n\x08msg_drop\x18\r \x01(\x08\x12\x10\n\x08msg_hang\x18\x0e \x01(\x08\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x14\n\x12TestInjectResponse\"\x1e\n\rHistoryAction\x12\r\n\x05\x66lush\x18\x01 \x01(\x08\"\xca\x01\n\x15PartialHistoryRequest\x12)\n\x04item\x18\x01 \x03(\x0b\x32\x1b.wandb_internal.HistoryItem\x12)\n\x04step\x18\x02 \x01(\x0b\x32\x1b.wandb_internal.HistoryStep\x12-\n\x06\x61\x63tion\x18\x03 \x01(\x0b\x32\x1d.wandb_internal.HistoryAction\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x18\n\x16PartialHistoryResponse\"E\n\x15SampledHistoryRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"_\n\x12SampledHistoryItem\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\nnested_key\x18\x02 \x03(\t\x12\x14\n\x0cvalues_float\x18\x03 \x03(\x02\x12\x12\n\nvalues_int\x18\x04 \x03(\x03\"J\n\x16SampledHistoryResponse\x12\x30\n\x04item\x18\x01 \x03(\x0b\x32\".wandb_internal.SampledHistoryItem\"@\n\x10RunStatusRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"x\n\x11RunStatusResponse\x12\x18\n\x10sync_items_total\x18\x01 \x01(\x03\x12\x1a\n\x12sync_items_pending\x18\x02 \x01(\x03\x12-\n\tsync_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"g\n\x0fRunStartRequest\x12&\n\x03run\x18\x01 \x01(\x0b\x32\x19.wandb_internal.RunRecord\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x12\n\x10RunStartResponse\"\\\n\x13\x43heckVersionRequest\x12\x17\n\x0f\x63urrent_version\x18\x01 \x01(\t\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"]\n\x14\x43heckVersionResponse\x12\x17\n\x0fupgrade_message\x18\x01 \x01(\t\x12\x14\n\x0cyank_message\x18\x02 \x01(\t\x12\x16\n\x0e\x64\x65lete_message\x18\x03 \x01(\t\">\n\x0eJobInfoRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"6\n\x0fJobInfoResponse\x12\x12\n\nsequenceId\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\"\x9f\x01\n\x12LogArtifactRequest\x12\x30\n\x08\x61rtifact\x18\x01 \x01(\x0b\x32\x1e.wandb_internal.ArtifactRecord\x12\x14\n\x0chistory_step\x18\x02 \x01(\x03\x12\x13\n\x0bstaging_dir\x18\x03 \x01(\t\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"A\n\x13LogArtifactResponse\x12\x13\n\x0b\x61rtifact_id\x18\x01 \x01(\t\x12\x15\n\rerror_message\x18\x02 \x01(\t\"\x95\x01\n\x17\x44ownloadArtifactRequest\x12\x13\n\x0b\x61rtifact_id\x18\x01 \x01(\t\x12\x15\n\rdownload_root\x18\x02 \x01(\t\x12 \n\x18\x61llow_missing_references\x18\x04 \x01(\x08\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"1\n\x18\x44ownloadArtifactResponse\x12\x15\n\rerror_message\x18\x01 \x01(\t\"@\n\x10KeepaliveRequest\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x13\n\x11KeepaliveResponse\"F\n\x0c\x41rtifactInfo\x12\x10\n\x08\x61rtifact\x18\x01 \x01(\t\x12\x12\n\nentrypoint\x18\x02 \x03(\t\x12\x10\n\x08notebook\x18\x03 \x01(\x08\")\n\x07GitInfo\x12\x0e\n\x06remote\x18\x01 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\t\"\\\n\tGitSource\x12)\n\x08git_info\x18\x01 \x01(\x0b\x32\x17.wandb_internal.GitInfo\x12\x12\n\nentrypoint\x18\x02 \x03(\t\x12\x10\n\x08notebook\x18\x03 \x01(\x08\"\x1c\n\x0bImageSource\x12\r\n\x05image\x18\x01 \x01(\t\"\x8c\x01\n\x06Source\x12&\n\x03git\x18\x01 \x01(\x0b\x32\x19.wandb_internal.GitSource\x12.\n\x08\x61rtifact\x18\x02 \x01(\x0b\x32\x1c.wandb_internal.ArtifactInfo\x12*\n\x05image\x18\x03 \x01(\x0b\x32\x1b.wandb_internal.ImageSource\"k\n\tJobSource\x12\x10\n\x08_version\x18\x01 \x01(\t\x12\x13\n\x0bsource_type\x18\x02 \x01(\t\x12&\n\x06source\x18\x03 \x01(\x0b\x32\x16.wandb_internal.Source\x12\x0f\n\x07runtime\x18\x04 \x01(\t\"V\n\x12PartialJobArtifact\x12\x10\n\x08job_name\x18\x01 \x01(\t\x12.\n\x0bsource_info\x18\x02 \x01(\x0b\x32\x19.wandb_internal.JobSource\"\x9d\x01\n\x11UseArtifactRecord\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x33\n\x07partial\x18\x04 \x01(\x0b\x32\".wandb_internal.PartialJobArtifact\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x13\n\x11UseArtifactResult\"R\n\rCancelRequest\x12\x13\n\x0b\x63\x61ncel_slot\x18\x01 \x01(\t\x12,\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1c.wandb_internal._RequestInfo\"\x10\n\x0e\x43\x61ncelResponse\"\'\n\x08\x44iskInfo\x12\r\n\x05total\x18\x01 \x01(\x04\x12\x0c\n\x04used\x18\x02 \x01(\x04\"\x1b\n\nMemoryInfo\x12\r\n\x05total\x18\x01 \x01(\x04\"/\n\x07\x43puInfo\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x15\n\rcount_logical\x18\x02 \x01(\r\">\n\x0cGpuAppleInfo\x12\x0f\n\x07gpuType\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\r\n\x05\x63ores\x18\x03 \x01(\r\"3\n\rGpuNvidiaInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x14\n\x0cmemory_total\x18\x02 \x01(\x04\"\x89\x02\n\nGpuAmdInfo\x12\n\n\x02id\x18\x01 \x01(\t\x12\x11\n\tunique_id\x18\x02 \x01(\t\x12\x15\n\rvbios_version\x18\x03 \x01(\t\x12\x19\n\x11performance_level\x18\x04 \x01(\t\x12\x15\n\rgpu_overdrive\x18\x05 \x01(\t\x12\x1c\n\x14gpu_memory_overdrive\x18\x06 \x01(\t\x12\x11\n\tmax_power\x18\x07 \x01(\t\x12\x0e\n\x06series\x18\x08 \x01(\t\x12\r\n\x05model\x18\t \x01(\t\x12\x0e\n\x06vendor\x18\n \x01(\t\x12\x0b\n\x03sku\x18\x0b \x01(\t\x12\x12\n\nsclk_range\x18\x0c \x01(\t\x12\x12\n\nmclk_range\x18\r \x01(\t\"\x96\x08\n\x0fMetadataRequest\x12\n\n\x02os\x18\x01 \x01(\t\x12\x0e\n\x06python\x18\x02 \x01(\t\x12/\n\x0bheartbeatAt\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12-\n\tstartedAt\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06\x64ocker\x18\x05 \x01(\t\x12\x0c\n\x04\x63uda\x18\x06 \x01(\t\x12\x0c\n\x04\x61rgs\x18\x07 \x03(\t\x12\r\n\x05state\x18\x08 \x01(\t\x12\x0f\n\x07program\x18\t \x01(\t\x12\x1b\n\tcode_path\x18\n \x01(\tR\x08\x63odePath\x12*\n\x03git\x18\x0b \x01(\x0b\x32\x1d.wandb_internal.GitRepoRecord\x12\r\n\x05\x65mail\x18\x0c \x01(\t\x12\x0c\n\x04root\x18\r \x01(\t\x12\x0c\n\x04host\x18\x0e \x01(\t\x12\x10\n\x08username\x18\x0f \x01(\t\x12\x12\n\nexecutable\x18\x10 \x01(\t\x12&\n\x0f\x63ode_path_local\x18\x11 \x01(\tR\rcodePathLocal\x12\r\n\x05\x63olab\x18\x12 \x01(\t\x12\x1c\n\tcpu_count\x18\x13 \x01(\rR\tcpu_count\x12,\n\x11\x63pu_count_logical\x18\x14 \x01(\rR\x11\x63pu_count_logical\x12\x15\n\x08gpu_type\x18\x15 \x01(\tR\x03gpu\x12\x1c\n\tgpu_count\x18\x16 \x01(\rR\tgpu_count\x12\x37\n\x04\x64isk\x18\x17 \x03(\x0b\x32).wandb_internal.MetadataRequest.DiskEntry\x12*\n\x06memory\x18\x18 \x01(\x0b\x32\x1a.wandb_internal.MemoryInfo\x12$\n\x03\x63pu\x18\x19 \x01(\x0b\x32\x17.wandb_internal.CpuInfo\x12\x39\n\tgpu_apple\x18\x1a \x01(\x0b\x32\x1c.wandb_internal.GpuAppleInfoR\x08gpuapple\x12=\n\ngpu_nvidia\x18\x1b \x03(\x0b\x32\x1d.wandb_internal.GpuNvidiaInfoR\ngpu_nvidia\x12\x34\n\x07gpu_amd\x18\x1c \x03(\x0b\x32\x1a.wandb_internal.GpuAmdInfoR\x07gpu_amd\x12\x39\n\x05slurm\x18\x1d \x03(\x0b\x32*.wandb_internal.MetadataRequest.SlurmEntry\x1a\x45\n\tDiskEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b\x32\x18.wandb_internal.DiskInfo:\x02\x38\x01\x1a,\n\nSlurmEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8d\x01\n\x15PythonPackagesRequest\x12\x44\n\x07package\x18\x01 \x03(\x0b\x32\x33.wandb_internal.PythonPackagesRequest.PythonPackage\x1a.\n\rPythonPackage\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\tb\x06proto3')
+
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'wandb.proto.wandb_internal_pb2', globals())
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+ DESCRIPTOR._options = None
+ _GETSYSTEMMETRICSRESPONSE_SYSTEMMETRICSENTRY._options = None
+ _GETSYSTEMMETRICSRESPONSE_SYSTEMMETRICSENTRY._serialized_options = b'8\001'
+ _METADATAREQUEST_DISKENTRY._options = None
+ _METADATAREQUEST_DISKENTRY._serialized_options = b'8\001'
+ _METADATAREQUEST_SLURMENTRY._options = None
+ _METADATAREQUEST_SLURMENTRY._serialized_options = b'8\001'
+ _RECORD._serialized_start=151
+ _RECORD._serialized_end=1331
+ _CONTROL._serialized_start=1334
+ _CONTROL._serialized_end=1502
+ _RESULT._serialized_start=1505
+ _RESULT._serialized_end=2004
+ _FINALRECORD._serialized_start=2006
+ _FINALRECORD._serialized_end=2064
+ _VERSIONINFO._serialized_start=2066
+ _VERSIONINFO._serialized_end=2164
+ _HEADERRECORD._serialized_start=2166
+ _HEADERRECORD._serialized_end=2276
+ _FOOTERRECORD._serialized_start=2278
+ _FOOTERRECORD._serialized_end=2337
+ _RUNRECORD._serialized_start=2340
+ _RUNRECORD._serialized_end=2930
+ _GITREPORECORD._serialized_start=2932
+ _GITREPORECORD._serialized_end=2991
+ _RUNUPDATERESULT._serialized_start=2993
+ _RUNUPDATERESULT._serialized_end=3092
+ _ERRORINFO._serialized_start=3095
+ _ERRORINFO._serialized_end=3267
+ _ERRORINFO_ERRORCODE._serialized_start=3176
+ _ERRORINFO_ERRORCODE._serialized_end=3267
+ _RUNEXITRECORD._serialized_start=3269
+ _RUNEXITRECORD._serialized_end=3365
+ _RUNEXITRESULT._serialized_start=3367
+ _RUNEXITRESULT._serialized_end=3382
+ _RUNPREEMPTINGRECORD._serialized_start=3384
+ _RUNPREEMPTINGRECORD._serialized_end=3450
+ _RUNPREEMPTINGRESULT._serialized_start=3452
+ _RUNPREEMPTINGRESULT._serialized_end=3473
+ _SETTINGSRECORD._serialized_start=3475
+ _SETTINGSRECORD._serialized_end=3580
+ _SETTINGSITEM._serialized_start=3582
+ _SETTINGSITEM._serialized_end=3629
+ _HISTORYSTEP._serialized_start=3631
+ _HISTORYSTEP._serialized_end=3657
+ _HISTORYRECORD._serialized_start=3660
+ _HISTORYRECORD._serialized_end=3806
+ _HISTORYITEM._serialized_start=3808
+ _HISTORYITEM._serialized_end=3874
+ _HISTORYRESULT._serialized_start=3876
+ _HISTORYRESULT._serialized_end=3891
+ _OUTPUTRECORD._serialized_start=3894
+ _OUTPUTRECORD._serialized_end=4114
+ _OUTPUTRECORD_OUTPUTTYPE._serialized_start=4078
+ _OUTPUTRECORD_OUTPUTTYPE._serialized_end=4114
+ _OUTPUTRESULT._serialized_start=4116
+ _OUTPUTRESULT._serialized_end=4130
+ _OUTPUTRAWRECORD._serialized_start=4133
+ _OUTPUTRAWRECORD._serialized_end=4359
+ _OUTPUTRAWRECORD_OUTPUTTYPE._serialized_start=4078
+ _OUTPUTRAWRECORD_OUTPUTTYPE._serialized_end=4114
+ _OUTPUTRAWRESULT._serialized_start=4361
+ _OUTPUTRAWRESULT._serialized_end=4378
+ _METRICRECORD._serialized_start=4381
+ _METRICRECORD._serialized_end=4789
+ _METRICRECORD_METRICGOAL._serialized_start=4723
+ _METRICRECORD_METRICGOAL._serialized_end=4789
+ _METRICRESULT._serialized_start=4791
+ _METRICRESULT._serialized_end=4805
+ _METRICOPTIONS._serialized_start=4807
+ _METRICOPTIONS._serialized_end=4874
+ _METRICCONTROL._serialized_start=4876
+ _METRICCONTROL._serialized_end=4910
+ _METRICSUMMARY._serialized_start=4912
+ _METRICSUMMARY._serialized_end=5023
+ _CONFIGRECORD._serialized_start=5026
+ _CONFIGRECORD._serialized_end=5173
+ _CONFIGITEM._serialized_start=5175
+ _CONFIGITEM._serialized_end=5240
+ _CONFIGRESULT._serialized_start=5242
+ _CONFIGRESULT._serialized_end=5256
+ _SUMMARYRECORD._serialized_start=5259
+ _SUMMARYRECORD._serialized_end=5409
+ _SUMMARYITEM._serialized_start=5411
+ _SUMMARYITEM._serialized_end=5477
+ _SUMMARYRESULT._serialized_start=5479
+ _SUMMARYRESULT._serialized_end=5494
+ _FILESRECORD._serialized_start=5496
+ _FILESRECORD._serialized_end=5596
+ _FILESITEM._serialized_start=5599
+ _FILESITEM._serialized_end=5852
+ _FILESITEM_POLICYTYPE._serialized_start=5753
+ _FILESITEM_POLICYTYPE._serialized_end=5793
+ _FILESITEM_FILETYPE._serialized_start=5795
+ _FILESITEM_FILETYPE._serialized_end=5852
+ _FILESRESULT._serialized_start=5854
+ _FILESRESULT._serialized_end=5867
+ _STATSRECORD._serialized_start=5870
+ _STATSRECORD._serialized_end=6100
+ _STATSRECORD_STATSTYPE._serialized_start=6077
+ _STATSRECORD_STATSTYPE._serialized_end=6100
+ _STATSITEM._serialized_start=6102
+ _STATSITEM._serialized_end=6146
+ _ARTIFACTRECORD._serialized_start=6149
+ _ARTIFACTRECORD._serialized_end=6622
+ _ARTIFACTMANIFEST._serialized_start=6625
+ _ARTIFACTMANIFEST._serialized_end=6813
+ _ARTIFACTMANIFESTENTRY._serialized_start=6816
+ _ARTIFACTMANIFESTENTRY._serialized_end=7003
+ _EXTRAITEM._serialized_start=7005
+ _EXTRAITEM._serialized_end=7049
+ _STORAGEPOLICYCONFIGITEM._serialized_start=7051
+ _STORAGEPOLICYCONFIGITEM._serialized_end=7109
+ _ARTIFACTRESULT._serialized_start=7111
+ _ARTIFACTRESULT._serialized_end=7127
+ _LINKARTIFACTRESULT._serialized_start=7129
+ _LINKARTIFACTRESULT._serialized_end=7149
+ _LINKARTIFACTRECORD._serialized_start=7152
+ _LINKARTIFACTRECORD._serialized_end=7359
+ _TBRECORD._serialized_start=7361
+ _TBRECORD._serialized_end=7465
+ _TBRESULT._serialized_start=7467
+ _TBRESULT._serialized_end=7477
+ _ALERTRECORD._serialized_start=7479
+ _ALERTRECORD._serialized_end=7604
+ _ALERTRESULT._serialized_start=7606
+ _ALERTRESULT._serialized_end=7619
+ _REQUEST._serialized_start=7622
+ _REQUEST._serialized_end=9615
+ _RESPONSE._serialized_start=9618
+ _RESPONSE._serialized_end=11124
+ _DEFERREQUEST._serialized_start=11127
+ _DEFERREQUEST._serialized_end=11447
+ _DEFERREQUEST_DEFERSTATE._serialized_start=11200
+ _DEFERREQUEST_DEFERSTATE._serialized_end=11447
+ _PAUSEREQUEST._serialized_start=11449
+ _PAUSEREQUEST._serialized_end=11509
+ _PAUSERESPONSE._serialized_start=11511
+ _PAUSERESPONSE._serialized_end=11526
+ _RESUMEREQUEST._serialized_start=11528
+ _RESUMEREQUEST._serialized_end=11589
+ _RESUMERESPONSE._serialized_start=11591
+ _RESUMERESPONSE._serialized_end=11607
+ _LOGINREQUEST._serialized_start=11609
+ _LOGINREQUEST._serialized_end=11686
+ _LOGINRESPONSE._serialized_start=11688
+ _LOGINRESPONSE._serialized_end=11726
+ _GETSUMMARYREQUEST._serialized_start=11728
+ _GETSUMMARYREQUEST._serialized_end=11793
+ _GETSUMMARYRESPONSE._serialized_start=11795
+ _GETSUMMARYRESPONSE._serialized_end=11858
+ _GETSYSTEMMETRICSREQUEST._serialized_start=11860
+ _GETSYSTEMMETRICSREQUEST._serialized_end=11931
+ _SYSTEMMETRICSAMPLE._serialized_start=11933
+ _SYSTEMMETRICSAMPLE._serialized_end=12015
+ _SYSTEMMETRICSBUFFER._serialized_start=12017
+ _SYSTEMMETRICSBUFFER._serialized_end=12090
+ _GETSYSTEMMETRICSRESPONSE._serialized_start=12093
+ _GETSYSTEMMETRICSRESPONSE._serialized_end=12295
+ _GETSYSTEMMETRICSRESPONSE_SYSTEMMETRICSENTRY._serialized_start=12206
+ _GETSYSTEMMETRICSRESPONSE_SYSTEMMETRICSENTRY._serialized_end=12295
+ _STATUSREQUEST._serialized_start=12297
+ _STATUSREQUEST._serialized_end=12358
+ _STATUSRESPONSE._serialized_start=12360
+ _STATUSRESPONSE._serialized_end=12401
+ _STOPSTATUSREQUEST._serialized_start=12403
+ _STOPSTATUSREQUEST._serialized_end=12468
+ _STOPSTATUSRESPONSE._serialized_start=12470
+ _STOPSTATUSRESPONSE._serialized_end=12515
+ _NETWORKSTATUSREQUEST._serialized_start=12517
+ _NETWORKSTATUSREQUEST._serialized_end=12585
+ _NETWORKSTATUSRESPONSE._serialized_start=12587
+ _NETWORKSTATUSRESPONSE._serialized_end=12667
+ _HTTPRESPONSE._serialized_start=12669
+ _HTTPRESPONSE._serialized_end=12737
+ _INTERNALMESSAGESREQUEST._serialized_start=12739
+ _INTERNALMESSAGESREQUEST._serialized_end=12810
+ _INTERNALMESSAGESRESPONSE._serialized_start=12812
+ _INTERNALMESSAGESRESPONSE._serialized_end=12890
+ _INTERNALMESSAGES._serialized_start=12892
+ _INTERNALMESSAGES._serialized_end=12927
+ _POLLEXITREQUEST._serialized_start=12929
+ _POLLEXITREQUEST._serialized_end=12992
+ _POLLEXITRESPONSE._serialized_start=12995
+ _POLLEXITRESPONSE._serialized_end=13183
+ _SYNCOVERWRITE._serialized_start=13185
+ _SYNCOVERWRITE._serialized_end=13249
+ _SYNCSKIP._serialized_start=13251
+ _SYNCSKIP._serialized_end=13281
+ _SENDERMARKREQUEST._serialized_start=13283
+ _SENDERMARKREQUEST._serialized_end=13302
+ _SYNCREQUEST._serialized_start=13305
+ _SYNCREQUEST._serialized_end=13452
+ _SYNCRESPONSE._serialized_start=13454
+ _SYNCRESPONSE._serialized_end=13523
+ _SENDERREADREQUEST._serialized_start=13525
+ _SENDERREADREQUEST._serialized_end=13588
+ _STATUSREPORTREQUEST._serialized_start=13590
+ _STATUSREPORTREQUEST._serialized_end=13699
+ _SUMMARYRECORDREQUEST._serialized_start=13701
+ _SUMMARYRECORDREQUEST._serialized_end=13771
+ _TELEMETRYRECORDREQUEST._serialized_start=13773
+ _TELEMETRYRECORDREQUEST._serialized_end=13849
+ _SERVERINFOREQUEST._serialized_start=13851
+ _SERVERINFOREQUEST._serialized_end=13916
+ _SERVERINFORESPONSE._serialized_start=13918
+ _SERVERINFORESPONSE._serialized_end=14042
+ _SERVERMESSAGES._serialized_start=14044
+ _SERVERMESSAGES._serialized_end=14105
+ _SERVERMESSAGE._serialized_start=14107
+ _SERVERMESSAGE._serialized_end=14208
+ _FILECOUNTS._serialized_start=14210
+ _FILECOUNTS._serialized_end=14309
+ _FILEPUSHERSTATS._serialized_start=14311
+ _FILEPUSHERSTATS._serialized_end=14396
+ _FILESUPLOADED._serialized_start=14398
+ _FILESUPLOADED._serialized_end=14428
+ _FILETRANSFERINFOREQUEST._serialized_start=14431
+ _FILETRANSFERINFOREQUEST._serialized_end=14675
+ _FILETRANSFERINFOREQUEST_TRANSFERTYPE._serialized_start=14635
+ _FILETRANSFERINFOREQUEST_TRANSFERTYPE._serialized_end=14675
+ _LOCALINFO._serialized_start=14677
+ _LOCALINFO._serialized_end=14726
+ _SHUTDOWNREQUEST._serialized_start=14728
+ _SHUTDOWNREQUEST._serialized_end=14791
+ _SHUTDOWNRESPONSE._serialized_start=14793
+ _SHUTDOWNRESPONSE._serialized_end=14811
+ _ATTACHREQUEST._serialized_start=14813
+ _ATTACHREQUEST._serialized_end=14893
+ _ATTACHRESPONSE._serialized_start=14895
+ _ATTACHRESPONSE._serialized_end=14993
+ _TESTINJECTREQUEST._serialized_start=14996
+ _TESTINJECTREQUEST._serialized_end=15337
+ _TESTINJECTRESPONSE._serialized_start=15339
+ _TESTINJECTRESPONSE._serialized_end=15359
+ _HISTORYACTION._serialized_start=15361
+ _HISTORYACTION._serialized_end=15391
+ _PARTIALHISTORYREQUEST._serialized_start=15394
+ _PARTIALHISTORYREQUEST._serialized_end=15596
+ _PARTIALHISTORYRESPONSE._serialized_start=15598
+ _PARTIALHISTORYRESPONSE._serialized_end=15622
+ _SAMPLEDHISTORYREQUEST._serialized_start=15624
+ _SAMPLEDHISTORYREQUEST._serialized_end=15693
+ _SAMPLEDHISTORYITEM._serialized_start=15695
+ _SAMPLEDHISTORYITEM._serialized_end=15790
+ _SAMPLEDHISTORYRESPONSE._serialized_start=15792
+ _SAMPLEDHISTORYRESPONSE._serialized_end=15866
+ _RUNSTATUSREQUEST._serialized_start=15868
+ _RUNSTATUSREQUEST._serialized_end=15932
+ _RUNSTATUSRESPONSE._serialized_start=15934
+ _RUNSTATUSRESPONSE._serialized_end=16054
+ _RUNSTARTREQUEST._serialized_start=16056
+ _RUNSTARTREQUEST._serialized_end=16159
+ _RUNSTARTRESPONSE._serialized_start=16161
+ _RUNSTARTRESPONSE._serialized_end=16179
+ _CHECKVERSIONREQUEST._serialized_start=16181
+ _CHECKVERSIONREQUEST._serialized_end=16273
+ _CHECKVERSIONRESPONSE._serialized_start=16275
+ _CHECKVERSIONRESPONSE._serialized_end=16368
+ _JOBINFOREQUEST._serialized_start=16370
+ _JOBINFOREQUEST._serialized_end=16432
+ _JOBINFORESPONSE._serialized_start=16434
+ _JOBINFORESPONSE._serialized_end=16488
+ _LOGARTIFACTREQUEST._serialized_start=16491
+ _LOGARTIFACTREQUEST._serialized_end=16650
+ _LOGARTIFACTRESPONSE._serialized_start=16652
+ _LOGARTIFACTRESPONSE._serialized_end=16717
+ _DOWNLOADARTIFACTREQUEST._serialized_start=16720
+ _DOWNLOADARTIFACTREQUEST._serialized_end=16869
+ _DOWNLOADARTIFACTRESPONSE._serialized_start=16871
+ _DOWNLOADARTIFACTRESPONSE._serialized_end=16920
+ _KEEPALIVEREQUEST._serialized_start=16922
+ _KEEPALIVEREQUEST._serialized_end=16986
+ _KEEPALIVERESPONSE._serialized_start=16988
+ _KEEPALIVERESPONSE._serialized_end=17007
+ _ARTIFACTINFO._serialized_start=17009
+ _ARTIFACTINFO._serialized_end=17079
+ _GITINFO._serialized_start=17081
+ _GITINFO._serialized_end=17122
+ _GITSOURCE._serialized_start=17124
+ _GITSOURCE._serialized_end=17216
+ _IMAGESOURCE._serialized_start=17218
+ _IMAGESOURCE._serialized_end=17246
+ _SOURCE._serialized_start=17249
+ _SOURCE._serialized_end=17389
+ _JOBSOURCE._serialized_start=17391
+ _JOBSOURCE._serialized_end=17498
+ _PARTIALJOBARTIFACT._serialized_start=17500
+ _PARTIALJOBARTIFACT._serialized_end=17586
+ _USEARTIFACTRECORD._serialized_start=17589
+ _USEARTIFACTRECORD._serialized_end=17746
+ _USEARTIFACTRESULT._serialized_start=17748
+ _USEARTIFACTRESULT._serialized_end=17767
+ _CANCELREQUEST._serialized_start=17769
+ _CANCELREQUEST._serialized_end=17851
+ _CANCELRESPONSE._serialized_start=17853
+ _CANCELRESPONSE._serialized_end=17869
+ _DISKINFO._serialized_start=17871
+ _DISKINFO._serialized_end=17910
+ _MEMORYINFO._serialized_start=17912
+ _MEMORYINFO._serialized_end=17939
+ _CPUINFO._serialized_start=17941
+ _CPUINFO._serialized_end=17988
+ _GPUAPPLEINFO._serialized_start=17990
+ _GPUAPPLEINFO._serialized_end=18052
+ _GPUNVIDIAINFO._serialized_start=18054
+ _GPUNVIDIAINFO._serialized_end=18105
+ _GPUAMDINFO._serialized_start=18108
+ _GPUAMDINFO._serialized_end=18373
+ _METADATAREQUEST._serialized_start=18376
+ _METADATAREQUEST._serialized_end=19422
+ _METADATAREQUEST_DISKENTRY._serialized_start=19307
+ _METADATAREQUEST_DISKENTRY._serialized_end=19376
+ _METADATAREQUEST_SLURMENTRY._serialized_start=19378
+ _METADATAREQUEST_SLURMENTRY._serialized_end=19422
+ _PYTHONPACKAGESREQUEST._serialized_start=19425
+ _PYTHONPACKAGESREQUEST._serialized_end=19566
+ _PYTHONPACKAGESREQUEST_PYTHONPACKAGE._serialized_start=19520
+ _PYTHONPACKAGESREQUEST_PYTHONPACKAGE._serialized_end=19566
+# @@protoc_insertion_point(module_scope)
diff --git a/wandb/proto/v4/wandb_internal_pb2.pyi b/wandb/proto/v4/wandb_internal_pb2.pyi
new file mode 100644
index 0000000000000000000000000000000000000000..8ecec7263b9a17b93e555afa395dcb6a90b52b42
--- /dev/null
+++ b/wandb/proto/v4/wandb_internal_pb2.pyi
@@ -0,0 +1,3830 @@
+"""
+@generated by mypy-protobuf. Do not edit manually!
+isort:skip_file
+"""
+import builtins
+import collections.abc
+import google.protobuf.descriptor
+import google.protobuf.internal.containers
+import google.protobuf.internal.enum_type_wrapper
+import google.protobuf.message
+import google.protobuf.timestamp_pb2
+import sys
+import typing
+import wandb.proto.wandb_base_pb2
+import wandb.proto.wandb_telemetry_pb2
+
+if sys.version_info >= (3, 10):
+ import typing as typing_extensions
+else:
+ import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+@typing_extensions.final
+class Record(google.protobuf.message.Message):
+ """***********************
+ Records and Results
+ **********************
+
+
+ Record: joined record for message passing and persistence
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ NUM_FIELD_NUMBER: builtins.int
+ HISTORY_FIELD_NUMBER: builtins.int
+ SUMMARY_FIELD_NUMBER: builtins.int
+ OUTPUT_FIELD_NUMBER: builtins.int
+ CONFIG_FIELD_NUMBER: builtins.int
+ FILES_FIELD_NUMBER: builtins.int
+ STATS_FIELD_NUMBER: builtins.int
+ ARTIFACT_FIELD_NUMBER: builtins.int
+ TBRECORD_FIELD_NUMBER: builtins.int
+ ALERT_FIELD_NUMBER: builtins.int
+ TELEMETRY_FIELD_NUMBER: builtins.int
+ METRIC_FIELD_NUMBER: builtins.int
+ OUTPUT_RAW_FIELD_NUMBER: builtins.int
+ RUN_FIELD_NUMBER: builtins.int
+ EXIT_FIELD_NUMBER: builtins.int
+ FINAL_FIELD_NUMBER: builtins.int
+ HEADER_FIELD_NUMBER: builtins.int
+ FOOTER_FIELD_NUMBER: builtins.int
+ PREEMPTING_FIELD_NUMBER: builtins.int
+ LINK_ARTIFACT_FIELD_NUMBER: builtins.int
+ USE_ARTIFACT_FIELD_NUMBER: builtins.int
+ REQUEST_FIELD_NUMBER: builtins.int
+ CONTROL_FIELD_NUMBER: builtins.int
+ UUID_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ num: builtins.int
+ @property
+ def history(self) -> global___HistoryRecord:
+ """Low numbers for more frequent data"""
+ @property
+ def summary(self) -> global___SummaryRecord: ...
+ @property
+ def output(self) -> global___OutputRecord: ...
+ @property
+ def config(self) -> global___ConfigRecord: ...
+ @property
+ def files(self) -> global___FilesRecord: ...
+ @property
+ def stats(self) -> global___StatsRecord: ...
+ @property
+ def artifact(self) -> global___ArtifactRecord: ...
+ @property
+ def tbrecord(self) -> global___TBRecord: ...
+ @property
+ def alert(self) -> global___AlertRecord: ...
+ @property
+ def telemetry(self) -> wandb.proto.wandb_telemetry_pb2.TelemetryRecord: ...
+ @property
+ def metric(self) -> global___MetricRecord: ...
+ @property
+ def output_raw(self) -> global___OutputRawRecord: ...
+ @property
+ def run(self) -> global___RunRecord:
+ """Higher numbers for less frequent data"""
+ @property
+ def exit(self) -> global___RunExitRecord: ...
+ @property
+ def final(self) -> global___FinalRecord: ...
+ @property
+ def header(self) -> global___HeaderRecord: ...
+ @property
+ def footer(self) -> global___FooterRecord: ...
+ @property
+ def preempting(self) -> global___RunPreemptingRecord: ...
+ @property
+ def link_artifact(self) -> global___LinkArtifactRecord: ...
+ @property
+ def use_artifact(self) -> global___UseArtifactRecord: ...
+ @property
+ def request(self) -> global___Request:
+ """request field does not belong here longterm"""
+ @property
+ def control(self) -> global___Control: ...
+ uuid: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ num: builtins.int = ...,
+ history: global___HistoryRecord | None = ...,
+ summary: global___SummaryRecord | None = ...,
+ output: global___OutputRecord | None = ...,
+ config: global___ConfigRecord | None = ...,
+ files: global___FilesRecord | None = ...,
+ stats: global___StatsRecord | None = ...,
+ artifact: global___ArtifactRecord | None = ...,
+ tbrecord: global___TBRecord | None = ...,
+ alert: global___AlertRecord | None = ...,
+ telemetry: wandb.proto.wandb_telemetry_pb2.TelemetryRecord | None = ...,
+ metric: global___MetricRecord | None = ...,
+ output_raw: global___OutputRawRecord | None = ...,
+ run: global___RunRecord | None = ...,
+ exit: global___RunExitRecord | None = ...,
+ final: global___FinalRecord | None = ...,
+ header: global___HeaderRecord | None = ...,
+ footer: global___FooterRecord | None = ...,
+ preempting: global___RunPreemptingRecord | None = ...,
+ link_artifact: global___LinkArtifactRecord | None = ...,
+ use_artifact: global___UseArtifactRecord | None = ...,
+ request: global___Request | None = ...,
+ control: global___Control | None = ...,
+ uuid: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "alert", b"alert", "artifact", b"artifact", "config", b"config", "control", b"control", "exit", b"exit", "files", b"files", "final", b"final", "footer", b"footer", "header", b"header", "history", b"history", "link_artifact", b"link_artifact", "metric", b"metric", "output", b"output", "output_raw", b"output_raw", "preempting", b"preempting", "record_type", b"record_type", "request", b"request", "run", b"run", "stats", b"stats", "summary", b"summary", "tbrecord", b"tbrecord", "telemetry", b"telemetry", "use_artifact", b"use_artifact"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "alert", b"alert", "artifact", b"artifact", "config", b"config", "control", b"control", "exit", b"exit", "files", b"files", "final", b"final", "footer", b"footer", "header", b"header", "history", b"history", "link_artifact", b"link_artifact", "metric", b"metric", "num", b"num", "output", b"output", "output_raw", b"output_raw", "preempting", b"preempting", "record_type", b"record_type", "request", b"request", "run", b"run", "stats", b"stats", "summary", b"summary", "tbrecord", b"tbrecord", "telemetry", b"telemetry", "use_artifact", b"use_artifact", "uuid", b"uuid"]) -> None: ...
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["record_type", b"record_type"]) -> typing_extensions.Literal["history", "summary", "output", "config", "files", "stats", "artifact", "tbrecord", "alert", "telemetry", "metric", "output_raw", "run", "exit", "final", "header", "footer", "preempting", "link_artifact", "use_artifact", "request"] | None: ...
+
+global___Record = Record
+
+@typing_extensions.final
+class Control(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ REQ_RESP_FIELD_NUMBER: builtins.int
+ LOCAL_FIELD_NUMBER: builtins.int
+ RELAY_ID_FIELD_NUMBER: builtins.int
+ MAILBOX_SLOT_FIELD_NUMBER: builtins.int
+ ALWAYS_SEND_FIELD_NUMBER: builtins.int
+ FLOW_CONTROL_FIELD_NUMBER: builtins.int
+ END_OFFSET_FIELD_NUMBER: builtins.int
+ CONNECTION_ID_FIELD_NUMBER: builtins.int
+ req_resp: builtins.bool
+ """record is expecting a result"""
+ local: builtins.bool
+ """should not be persisted or synchronized"""
+ relay_id: builtins.str
+ """used by service transport to identify correct stream"""
+ mailbox_slot: builtins.str
+ """mailbox slot"""
+ always_send: builtins.bool
+ """message to sender"""
+ flow_control: builtins.bool
+ """message should be passed to flow control"""
+ end_offset: builtins.int
+ """end of message offset of this written message"""
+ connection_id: builtins.str
+ """connection id"""
+ def __init__(
+ self,
+ *,
+ req_resp: builtins.bool = ...,
+ local: builtins.bool = ...,
+ relay_id: builtins.str = ...,
+ mailbox_slot: builtins.str = ...,
+ always_send: builtins.bool = ...,
+ flow_control: builtins.bool = ...,
+ end_offset: builtins.int = ...,
+ connection_id: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["always_send", b"always_send", "connection_id", b"connection_id", "end_offset", b"end_offset", "flow_control", b"flow_control", "local", b"local", "mailbox_slot", b"mailbox_slot", "relay_id", b"relay_id", "req_resp", b"req_resp"]) -> None: ...
+
+global___Control = Control
+
+@typing_extensions.final
+class Result(google.protobuf.message.Message):
+ """
+ Result: all results
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_RESULT_FIELD_NUMBER: builtins.int
+ EXIT_RESULT_FIELD_NUMBER: builtins.int
+ LOG_RESULT_FIELD_NUMBER: builtins.int
+ SUMMARY_RESULT_FIELD_NUMBER: builtins.int
+ OUTPUT_RESULT_FIELD_NUMBER: builtins.int
+ CONFIG_RESULT_FIELD_NUMBER: builtins.int
+ RESPONSE_FIELD_NUMBER: builtins.int
+ CONTROL_FIELD_NUMBER: builtins.int
+ UUID_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def run_result(self) -> global___RunUpdateResult: ...
+ @property
+ def exit_result(self) -> global___RunExitResult: ...
+ @property
+ def log_result(self) -> global___HistoryResult: ...
+ @property
+ def summary_result(self) -> global___SummaryResult: ...
+ @property
+ def output_result(self) -> global___OutputResult: ...
+ @property
+ def config_result(self) -> global___ConfigResult: ...
+ @property
+ def response(self) -> global___Response:
+ """response field does not belong here longterm"""
+ @property
+ def control(self) -> global___Control: ...
+ uuid: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._ResultInfo: ...
+ def __init__(
+ self,
+ *,
+ run_result: global___RunUpdateResult | None = ...,
+ exit_result: global___RunExitResult | None = ...,
+ log_result: global___HistoryResult | None = ...,
+ summary_result: global___SummaryResult | None = ...,
+ output_result: global___OutputResult | None = ...,
+ config_result: global___ConfigResult | None = ...,
+ response: global___Response | None = ...,
+ control: global___Control | None = ...,
+ uuid: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._ResultInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "config_result", b"config_result", "control", b"control", "exit_result", b"exit_result", "log_result", b"log_result", "output_result", b"output_result", "response", b"response", "result_type", b"result_type", "run_result", b"run_result", "summary_result", b"summary_result"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "config_result", b"config_result", "control", b"control", "exit_result", b"exit_result", "log_result", b"log_result", "output_result", b"output_result", "response", b"response", "result_type", b"result_type", "run_result", b"run_result", "summary_result", b"summary_result", "uuid", b"uuid"]) -> None: ...
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["result_type", b"result_type"]) -> typing_extensions.Literal["run_result", "exit_result", "log_result", "summary_result", "output_result", "config_result", "response"] | None: ...
+
+global___Result = Result
+
+@typing_extensions.final
+class FinalRecord(google.protobuf.message.Message):
+ """
+ FinalRecord
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___FinalRecord = FinalRecord
+
+@typing_extensions.final
+class VersionInfo(google.protobuf.message.Message):
+ """
+ Version definition
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ PRODUCER_FIELD_NUMBER: builtins.int
+ MIN_CONSUMER_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ producer: builtins.str
+ """The version of the SDK backend that produced the data"""
+ min_consumer: builtins.str
+ """Minimum version of the wandb server that can read the data"""
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ producer: builtins.str = ...,
+ min_consumer: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "min_consumer", b"min_consumer", "producer", b"producer"]) -> None: ...
+
+global___VersionInfo = VersionInfo
+
+@typing_extensions.final
+class HeaderRecord(google.protobuf.message.Message):
+ """
+ HeaderRecord
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ VERSION_INFO_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def version_info(self) -> global___VersionInfo: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ version_info: global___VersionInfo | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "version_info", b"version_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "version_info", b"version_info"]) -> None: ...
+
+global___HeaderRecord = HeaderRecord
+
+@typing_extensions.final
+class FooterRecord(google.protobuf.message.Message):
+ """
+ FooterRecord
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___FooterRecord = FooterRecord
+
+@typing_extensions.final
+class RunRecord(google.protobuf.message.Message):
+ """
+ RunRecord: wandb/sdk/wandb_run/Run
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_ID_FIELD_NUMBER: builtins.int
+ ENTITY_FIELD_NUMBER: builtins.int
+ PROJECT_FIELD_NUMBER: builtins.int
+ CONFIG_FIELD_NUMBER: builtins.int
+ SUMMARY_FIELD_NUMBER: builtins.int
+ RUN_GROUP_FIELD_NUMBER: builtins.int
+ JOB_TYPE_FIELD_NUMBER: builtins.int
+ DISPLAY_NAME_FIELD_NUMBER: builtins.int
+ NOTES_FIELD_NUMBER: builtins.int
+ TAGS_FIELD_NUMBER: builtins.int
+ SETTINGS_FIELD_NUMBER: builtins.int
+ SWEEP_ID_FIELD_NUMBER: builtins.int
+ HOST_FIELD_NUMBER: builtins.int
+ STARTING_STEP_FIELD_NUMBER: builtins.int
+ STORAGE_ID_FIELD_NUMBER: builtins.int
+ START_TIME_FIELD_NUMBER: builtins.int
+ RESUMED_FIELD_NUMBER: builtins.int
+ TELEMETRY_FIELD_NUMBER: builtins.int
+ RUNTIME_FIELD_NUMBER: builtins.int
+ GIT_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ run_id: builtins.str
+ entity: builtins.str
+ project: builtins.str
+ @property
+ def config(self) -> global___ConfigRecord: ...
+ @property
+ def summary(self) -> global___SummaryRecord: ...
+ run_group: builtins.str
+ job_type: builtins.str
+ display_name: builtins.str
+ notes: builtins.str
+ @property
+ def tags(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ @property
+ def settings(self) -> global___SettingsRecord: ...
+ sweep_id: builtins.str
+ host: builtins.str
+ starting_step: builtins.int
+ storage_id: builtins.str
+ @property
+ def start_time(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ resumed: builtins.bool
+ @property
+ def telemetry(self) -> wandb.proto.wandb_telemetry_pb2.TelemetryRecord: ...
+ runtime: builtins.int
+ @property
+ def git(self) -> global___GitRepoRecord: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ run_id: builtins.str = ...,
+ entity: builtins.str = ...,
+ project: builtins.str = ...,
+ config: global___ConfigRecord | None = ...,
+ summary: global___SummaryRecord | None = ...,
+ run_group: builtins.str = ...,
+ job_type: builtins.str = ...,
+ display_name: builtins.str = ...,
+ notes: builtins.str = ...,
+ tags: collections.abc.Iterable[builtins.str] | None = ...,
+ settings: global___SettingsRecord | None = ...,
+ sweep_id: builtins.str = ...,
+ host: builtins.str = ...,
+ starting_step: builtins.int = ...,
+ storage_id: builtins.str = ...,
+ start_time: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ resumed: builtins.bool = ...,
+ telemetry: wandb.proto.wandb_telemetry_pb2.TelemetryRecord | None = ...,
+ runtime: builtins.int = ...,
+ git: global___GitRepoRecord | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "config", b"config", "git", b"git", "settings", b"settings", "start_time", b"start_time", "summary", b"summary", "telemetry", b"telemetry"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "config", b"config", "display_name", b"display_name", "entity", b"entity", "git", b"git", "host", b"host", "job_type", b"job_type", "notes", b"notes", "project", b"project", "resumed", b"resumed", "run_group", b"run_group", "run_id", b"run_id", "runtime", b"runtime", "settings", b"settings", "start_time", b"start_time", "starting_step", b"starting_step", "storage_id", b"storage_id", "summary", b"summary", "sweep_id", b"sweep_id", "tags", b"tags", "telemetry", b"telemetry"]) -> None: ...
+
+global___RunRecord = RunRecord
+
+@typing_extensions.final
+class GitRepoRecord(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ REMOTE_URL_FIELD_NUMBER: builtins.int
+ COMMIT_FIELD_NUMBER: builtins.int
+ remote_url: builtins.str
+ commit: builtins.str
+ def __init__(
+ self,
+ *,
+ remote_url: builtins.str = ...,
+ commit: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["commit", b"commit", "remote_url", b"remote_url"]) -> None: ...
+
+global___GitRepoRecord = GitRepoRecord
+
+@typing_extensions.final
+class RunUpdateResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_FIELD_NUMBER: builtins.int
+ ERROR_FIELD_NUMBER: builtins.int
+ @property
+ def run(self) -> global___RunRecord: ...
+ @property
+ def error(self) -> global___ErrorInfo: ...
+ def __init__(
+ self,
+ *,
+ run: global___RunRecord | None = ...,
+ error: global___ErrorInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["error", b"error", "run", b"run"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["error", b"error", "run", b"run"]) -> None: ...
+
+global___RunUpdateResult = RunUpdateResult
+
+@typing_extensions.final
+class ErrorInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _ErrorCode:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _ErrorCodeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[ErrorInfo._ErrorCode.ValueType], builtins.type):
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ UNKNOWN: ErrorInfo._ErrorCode.ValueType # 0
+ COMMUNICATION: ErrorInfo._ErrorCode.ValueType # 1
+ AUTHENTICATION: ErrorInfo._ErrorCode.ValueType # 2
+ USAGE: ErrorInfo._ErrorCode.ValueType # 3
+ UNSUPPORTED: ErrorInfo._ErrorCode.ValueType # 4
+
+ class ErrorCode(_ErrorCode, metaclass=_ErrorCodeEnumTypeWrapper): ...
+ UNKNOWN: ErrorInfo.ErrorCode.ValueType # 0
+ COMMUNICATION: ErrorInfo.ErrorCode.ValueType # 1
+ AUTHENTICATION: ErrorInfo.ErrorCode.ValueType # 2
+ USAGE: ErrorInfo.ErrorCode.ValueType # 3
+ UNSUPPORTED: ErrorInfo.ErrorCode.ValueType # 4
+
+ MESSAGE_FIELD_NUMBER: builtins.int
+ CODE_FIELD_NUMBER: builtins.int
+ message: builtins.str
+ code: global___ErrorInfo.ErrorCode.ValueType
+ def __init__(
+ self,
+ *,
+ message: builtins.str = ...,
+ code: global___ErrorInfo.ErrorCode.ValueType = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["code", b"code", "message", b"message"]) -> None: ...
+
+global___ErrorInfo = ErrorInfo
+
+@typing_extensions.final
+class RunExitRecord(google.protobuf.message.Message):
+ """
+ RunExitRecord: exit status of process
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ EXIT_CODE_FIELD_NUMBER: builtins.int
+ RUNTIME_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ exit_code: builtins.int
+ runtime: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ exit_code: builtins.int = ...,
+ runtime: builtins.int = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "exit_code", b"exit_code", "runtime", b"runtime"]) -> None: ...
+
+global___RunExitRecord = RunExitRecord
+
+@typing_extensions.final
+class RunExitResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___RunExitResult = RunExitResult
+
+@typing_extensions.final
+class RunPreemptingRecord(google.protobuf.message.Message):
+ """
+ RunPreemptingRecord: run being preempted
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___RunPreemptingRecord = RunPreemptingRecord
+
+@typing_extensions.final
+class RunPreemptingResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___RunPreemptingResult = RunPreemptingResult
+
+@typing_extensions.final
+class SettingsRecord(google.protobuf.message.Message):
+ """
+ SettingsRecord: wandb/sdk/wandb_settings/Settings
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ITEM_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def item(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___SettingsItem]: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ item: collections.abc.Iterable[global___SettingsItem] | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "item", b"item"]) -> None: ...
+
+global___SettingsRecord = SettingsRecord
+
+@typing_extensions.final
+class SettingsItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_JSON_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ value_json: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value_json: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value_json", b"value_json"]) -> None: ...
+
+global___SettingsItem = SettingsItem
+
+@typing_extensions.final
+class HistoryStep(google.protobuf.message.Message):
+ """
+ HistoryRecord: wandb/sdk/wandb_history/History
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ NUM_FIELD_NUMBER: builtins.int
+ num: builtins.int
+ def __init__(
+ self,
+ *,
+ num: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["num", b"num"]) -> None: ...
+
+global___HistoryStep = HistoryStep
+
+@typing_extensions.final
+class HistoryRecord(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ITEM_FIELD_NUMBER: builtins.int
+ STEP_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def item(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___HistoryItem]: ...
+ @property
+ def step(self) -> global___HistoryStep: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ item: collections.abc.Iterable[global___HistoryItem] | None = ...,
+ step: global___HistoryStep | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "step", b"step"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "item", b"item", "step", b"step"]) -> None: ...
+
+global___HistoryRecord = HistoryRecord
+
+@typing_extensions.final
+class HistoryItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ NESTED_KEY_FIELD_NUMBER: builtins.int
+ VALUE_JSON_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ @property
+ def nested_key(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ value_json: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ nested_key: collections.abc.Iterable[builtins.str] | None = ...,
+ value_json: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "nested_key", b"nested_key", "value_json", b"value_json"]) -> None: ...
+
+global___HistoryItem = HistoryItem
+
+@typing_extensions.final
+class HistoryResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___HistoryResult = HistoryResult
+
+@typing_extensions.final
+class OutputRecord(google.protobuf.message.Message):
+ """
+ OutputRecord: console output
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _OutputType:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _OutputTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[OutputRecord._OutputType.ValueType], builtins.type):
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ STDERR: OutputRecord._OutputType.ValueType # 0
+ STDOUT: OutputRecord._OutputType.ValueType # 1
+
+ class OutputType(_OutputType, metaclass=_OutputTypeEnumTypeWrapper): ...
+ STDERR: OutputRecord.OutputType.ValueType # 0
+ STDOUT: OutputRecord.OutputType.ValueType # 1
+
+ OUTPUT_TYPE_FIELD_NUMBER: builtins.int
+ TIMESTAMP_FIELD_NUMBER: builtins.int
+ LINE_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ output_type: global___OutputRecord.OutputType.ValueType
+ @property
+ def timestamp(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ line: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ output_type: global___OutputRecord.OutputType.ValueType = ...,
+ timestamp: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ line: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "timestamp", b"timestamp"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "line", b"line", "output_type", b"output_type", "timestamp", b"timestamp"]) -> None: ...
+
+global___OutputRecord = OutputRecord
+
+@typing_extensions.final
+class OutputResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___OutputResult = OutputResult
+
+@typing_extensions.final
+class OutputRawRecord(google.protobuf.message.Message):
+ """
+ OutputRawRecord: raw console output
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _OutputType:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _OutputTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[OutputRawRecord._OutputType.ValueType], builtins.type):
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ STDERR: OutputRawRecord._OutputType.ValueType # 0
+ STDOUT: OutputRawRecord._OutputType.ValueType # 1
+
+ class OutputType(_OutputType, metaclass=_OutputTypeEnumTypeWrapper): ...
+ STDERR: OutputRawRecord.OutputType.ValueType # 0
+ STDOUT: OutputRawRecord.OutputType.ValueType # 1
+
+ OUTPUT_TYPE_FIELD_NUMBER: builtins.int
+ TIMESTAMP_FIELD_NUMBER: builtins.int
+ LINE_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ output_type: global___OutputRawRecord.OutputType.ValueType
+ @property
+ def timestamp(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ line: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ output_type: global___OutputRawRecord.OutputType.ValueType = ...,
+ timestamp: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ line: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "timestamp", b"timestamp"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "line", b"line", "output_type", b"output_type", "timestamp", b"timestamp"]) -> None: ...
+
+global___OutputRawRecord = OutputRawRecord
+
+@typing_extensions.final
+class OutputRawResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___OutputRawResult = OutputRawResult
+
+@typing_extensions.final
+class MetricRecord(google.protobuf.message.Message):
+ """
+ MetricRecord: wandb/sdk/wandb_metric/Metric
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _MetricGoal:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _MetricGoalEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[MetricRecord._MetricGoal.ValueType], builtins.type):
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ GOAL_UNSET: MetricRecord._MetricGoal.ValueType # 0
+ GOAL_MINIMIZE: MetricRecord._MetricGoal.ValueType # 1
+ GOAL_MAXIMIZE: MetricRecord._MetricGoal.ValueType # 2
+
+ class MetricGoal(_MetricGoal, metaclass=_MetricGoalEnumTypeWrapper): ...
+ GOAL_UNSET: MetricRecord.MetricGoal.ValueType # 0
+ GOAL_MINIMIZE: MetricRecord.MetricGoal.ValueType # 1
+ GOAL_MAXIMIZE: MetricRecord.MetricGoal.ValueType # 2
+
+ NAME_FIELD_NUMBER: builtins.int
+ GLOB_NAME_FIELD_NUMBER: builtins.int
+ STEP_METRIC_FIELD_NUMBER: builtins.int
+ STEP_METRIC_INDEX_FIELD_NUMBER: builtins.int
+ OPTIONS_FIELD_NUMBER: builtins.int
+ SUMMARY_FIELD_NUMBER: builtins.int
+ GOAL_FIELD_NUMBER: builtins.int
+ _CONTROL_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ name: builtins.str
+ """only name or globname is set"""
+ glob_name: builtins.str
+ step_metric: builtins.str
+ """step metric index can be used instead of step_metric when
+ MetricRecord is encoded in a list of MetricRecords
+ """
+ step_metric_index: builtins.int
+ """one-based array index"""
+ @property
+ def options(self) -> global___MetricOptions: ...
+ @property
+ def summary(self) -> global___MetricSummary: ...
+ goal: global___MetricRecord.MetricGoal.ValueType
+ @property
+ def _control(self) -> global___MetricControl: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ name: builtins.str = ...,
+ glob_name: builtins.str = ...,
+ step_metric: builtins.str = ...,
+ step_metric_index: builtins.int = ...,
+ options: global___MetricOptions | None = ...,
+ summary: global___MetricSummary | None = ...,
+ goal: global___MetricRecord.MetricGoal.ValueType = ...,
+ _control: global___MetricControl | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_control", b"_control", "_info", b"_info", "options", b"options", "summary", b"summary"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_control", b"_control", "_info", b"_info", "glob_name", b"glob_name", "goal", b"goal", "name", b"name", "options", b"options", "step_metric", b"step_metric", "step_metric_index", b"step_metric_index", "summary", b"summary"]) -> None: ...
+
+global___MetricRecord = MetricRecord
+
+@typing_extensions.final
+class MetricResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___MetricResult = MetricResult
+
+@typing_extensions.final
+class MetricOptions(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ STEP_SYNC_FIELD_NUMBER: builtins.int
+ HIDDEN_FIELD_NUMBER: builtins.int
+ DEFINED_FIELD_NUMBER: builtins.int
+ step_sync: builtins.bool
+ hidden: builtins.bool
+ defined: builtins.bool
+ """metric explicitly defined (not from glob match or step metric)"""
+ def __init__(
+ self,
+ *,
+ step_sync: builtins.bool = ...,
+ hidden: builtins.bool = ...,
+ defined: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["defined", b"defined", "hidden", b"hidden", "step_sync", b"step_sync"]) -> None: ...
+
+global___MetricOptions = MetricOptions
+
+@typing_extensions.final
+class MetricControl(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ OVERWRITE_FIELD_NUMBER: builtins.int
+ overwrite: builtins.bool
+ def __init__(
+ self,
+ *,
+ overwrite: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["overwrite", b"overwrite"]) -> None: ...
+
+global___MetricControl = MetricControl
+
+@typing_extensions.final
+class MetricSummary(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ MIN_FIELD_NUMBER: builtins.int
+ MAX_FIELD_NUMBER: builtins.int
+ MEAN_FIELD_NUMBER: builtins.int
+ BEST_FIELD_NUMBER: builtins.int
+ LAST_FIELD_NUMBER: builtins.int
+ NONE_FIELD_NUMBER: builtins.int
+ COPY_FIELD_NUMBER: builtins.int
+ min: builtins.bool
+ max: builtins.bool
+ mean: builtins.bool
+ best: builtins.bool
+ last: builtins.bool
+ none: builtins.bool
+ copy: builtins.bool
+ def __init__(
+ self,
+ *,
+ min: builtins.bool = ...,
+ max: builtins.bool = ...,
+ mean: builtins.bool = ...,
+ best: builtins.bool = ...,
+ last: builtins.bool = ...,
+ none: builtins.bool = ...,
+ copy: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["best", b"best", "copy", b"copy", "last", b"last", "max", b"max", "mean", b"mean", "min", b"min", "none", b"none"]) -> None: ...
+
+global___MetricSummary = MetricSummary
+
+@typing_extensions.final
+class ConfigRecord(google.protobuf.message.Message):
+ """
+ ConfigRecord: wandb/sdk/wandb_config/Config
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ UPDATE_FIELD_NUMBER: builtins.int
+ REMOVE_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def update(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ConfigItem]: ...
+ @property
+ def remove(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ConfigItem]: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ update: collections.abc.Iterable[global___ConfigItem] | None = ...,
+ remove: collections.abc.Iterable[global___ConfigItem] | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "remove", b"remove", "update", b"update"]) -> None: ...
+
+global___ConfigRecord = ConfigRecord
+
+@typing_extensions.final
+class ConfigItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ NESTED_KEY_FIELD_NUMBER: builtins.int
+ VALUE_JSON_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ @property
+ def nested_key(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ value_json: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ nested_key: collections.abc.Iterable[builtins.str] | None = ...,
+ value_json: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "nested_key", b"nested_key", "value_json", b"value_json"]) -> None: ...
+
+global___ConfigItem = ConfigItem
+
+@typing_extensions.final
+class ConfigResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ConfigResult = ConfigResult
+
+@typing_extensions.final
+class SummaryRecord(google.protobuf.message.Message):
+ """
+ SummaryRecord: wandb/sdk/wandb_summary/Summary
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ UPDATE_FIELD_NUMBER: builtins.int
+ REMOVE_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def update(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___SummaryItem]: ...
+ @property
+ def remove(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___SummaryItem]: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ update: collections.abc.Iterable[global___SummaryItem] | None = ...,
+ remove: collections.abc.Iterable[global___SummaryItem] | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "remove", b"remove", "update", b"update"]) -> None: ...
+
+global___SummaryRecord = SummaryRecord
+
+@typing_extensions.final
+class SummaryItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ NESTED_KEY_FIELD_NUMBER: builtins.int
+ VALUE_JSON_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ @property
+ def nested_key(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ value_json: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ nested_key: collections.abc.Iterable[builtins.str] | None = ...,
+ value_json: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "nested_key", b"nested_key", "value_json", b"value_json"]) -> None: ...
+
+global___SummaryItem = SummaryItem
+
+@typing_extensions.final
+class SummaryResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___SummaryResult = SummaryResult
+
+@typing_extensions.final
+class FilesRecord(google.protobuf.message.Message):
+ """
+ FilesRecord: files added to run
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ FILES_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def files(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___FilesItem]: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ files: collections.abc.Iterable[global___FilesItem] | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "files", b"files"]) -> None: ...
+
+global___FilesRecord = FilesRecord
+
+@typing_extensions.final
+class FilesItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _PolicyType:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _PolicyTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[FilesItem._PolicyType.ValueType], builtins.type):
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ NOW: FilesItem._PolicyType.ValueType # 0
+ END: FilesItem._PolicyType.ValueType # 1
+ LIVE: FilesItem._PolicyType.ValueType # 2
+
+ class PolicyType(_PolicyType, metaclass=_PolicyTypeEnumTypeWrapper): ...
+ NOW: FilesItem.PolicyType.ValueType # 0
+ END: FilesItem.PolicyType.ValueType # 1
+ LIVE: FilesItem.PolicyType.ValueType # 2
+
+ class _FileType:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _FileTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[FilesItem._FileType.ValueType], builtins.type):
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ OTHER: FilesItem._FileType.ValueType # 0
+ WANDB: FilesItem._FileType.ValueType # 1
+ MEDIA: FilesItem._FileType.ValueType # 2
+ ARTIFACT: FilesItem._FileType.ValueType # 3
+
+ class FileType(_FileType, metaclass=_FileTypeEnumTypeWrapper): ...
+ OTHER: FilesItem.FileType.ValueType # 0
+ WANDB: FilesItem.FileType.ValueType # 1
+ MEDIA: FilesItem.FileType.ValueType # 2
+ ARTIFACT: FilesItem.FileType.ValueType # 3
+
+ PATH_FIELD_NUMBER: builtins.int
+ POLICY_FIELD_NUMBER: builtins.int
+ TYPE_FIELD_NUMBER: builtins.int
+ EXTERNAL_PATH_FIELD_NUMBER: builtins.int
+ path: builtins.str
+ policy: global___FilesItem.PolicyType.ValueType
+ type: global___FilesItem.FileType.ValueType
+ external_path: builtins.str
+ def __init__(
+ self,
+ *,
+ path: builtins.str = ...,
+ policy: global___FilesItem.PolicyType.ValueType = ...,
+ type: global___FilesItem.FileType.ValueType = ...,
+ external_path: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["external_path", b"external_path", "path", b"path", "policy", b"policy", "type", b"type"]) -> None: ...
+
+global___FilesItem = FilesItem
+
+@typing_extensions.final
+class FilesResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___FilesResult = FilesResult
+
+@typing_extensions.final
+class StatsRecord(google.protobuf.message.Message):
+ """
+ StatsRecord: system metrics
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _StatsType:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _StatsTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[StatsRecord._StatsType.ValueType], builtins.type):
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ SYSTEM: StatsRecord._StatsType.ValueType # 0
+
+ class StatsType(_StatsType, metaclass=_StatsTypeEnumTypeWrapper): ...
+ SYSTEM: StatsRecord.StatsType.ValueType # 0
+
+ STATS_TYPE_FIELD_NUMBER: builtins.int
+ TIMESTAMP_FIELD_NUMBER: builtins.int
+ ITEM_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ stats_type: global___StatsRecord.StatsType.ValueType
+ @property
+ def timestamp(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ @property
+ def item(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___StatsItem]: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ stats_type: global___StatsRecord.StatsType.ValueType = ...,
+ timestamp: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ item: collections.abc.Iterable[global___StatsItem] | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "timestamp", b"timestamp"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "item", b"item", "stats_type", b"stats_type", "timestamp", b"timestamp"]) -> None: ...
+
+global___StatsRecord = StatsRecord
+
+@typing_extensions.final
+class StatsItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_JSON_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ value_json: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value_json: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value_json", b"value_json"]) -> None: ...
+
+global___StatsItem = StatsItem
+
+@typing_extensions.final
+class ArtifactRecord(google.protobuf.message.Message):
+ """
+ ArtifactRecord: track artifacts
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_ID_FIELD_NUMBER: builtins.int
+ PROJECT_FIELD_NUMBER: builtins.int
+ ENTITY_FIELD_NUMBER: builtins.int
+ TYPE_FIELD_NUMBER: builtins.int
+ NAME_FIELD_NUMBER: builtins.int
+ DIGEST_FIELD_NUMBER: builtins.int
+ DESCRIPTION_FIELD_NUMBER: builtins.int
+ METADATA_FIELD_NUMBER: builtins.int
+ USER_CREATED_FIELD_NUMBER: builtins.int
+ USE_AFTER_COMMIT_FIELD_NUMBER: builtins.int
+ ALIASES_FIELD_NUMBER: builtins.int
+ MANIFEST_FIELD_NUMBER: builtins.int
+ DISTRIBUTED_ID_FIELD_NUMBER: builtins.int
+ FINALIZE_FIELD_NUMBER: builtins.int
+ CLIENT_ID_FIELD_NUMBER: builtins.int
+ SEQUENCE_CLIENT_ID_FIELD_NUMBER: builtins.int
+ BASE_ID_FIELD_NUMBER: builtins.int
+ TTL_DURATION_SECONDS_FIELD_NUMBER: builtins.int
+ INCREMENTAL_BETA1_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ run_id: builtins.str
+ project: builtins.str
+ entity: builtins.str
+ type: builtins.str
+ name: builtins.str
+ digest: builtins.str
+ description: builtins.str
+ metadata: builtins.str
+ user_created: builtins.bool
+ use_after_commit: builtins.bool
+ @property
+ def aliases(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ @property
+ def manifest(self) -> global___ArtifactManifest: ...
+ distributed_id: builtins.str
+ finalize: builtins.bool
+ client_id: builtins.str
+ sequence_client_id: builtins.str
+ base_id: builtins.str
+ ttl_duration_seconds: builtins.int
+ incremental_beta1: builtins.bool
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ run_id: builtins.str = ...,
+ project: builtins.str = ...,
+ entity: builtins.str = ...,
+ type: builtins.str = ...,
+ name: builtins.str = ...,
+ digest: builtins.str = ...,
+ description: builtins.str = ...,
+ metadata: builtins.str = ...,
+ user_created: builtins.bool = ...,
+ use_after_commit: builtins.bool = ...,
+ aliases: collections.abc.Iterable[builtins.str] | None = ...,
+ manifest: global___ArtifactManifest | None = ...,
+ distributed_id: builtins.str = ...,
+ finalize: builtins.bool = ...,
+ client_id: builtins.str = ...,
+ sequence_client_id: builtins.str = ...,
+ base_id: builtins.str = ...,
+ ttl_duration_seconds: builtins.int = ...,
+ incremental_beta1: builtins.bool = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "manifest", b"manifest"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "aliases", b"aliases", "base_id", b"base_id", "client_id", b"client_id", "description", b"description", "digest", b"digest", "distributed_id", b"distributed_id", "entity", b"entity", "finalize", b"finalize", "incremental_beta1", b"incremental_beta1", "manifest", b"manifest", "metadata", b"metadata", "name", b"name", "project", b"project", "run_id", b"run_id", "sequence_client_id", b"sequence_client_id", "ttl_duration_seconds", b"ttl_duration_seconds", "type", b"type", "use_after_commit", b"use_after_commit", "user_created", b"user_created"]) -> None: ...
+
+global___ArtifactRecord = ArtifactRecord
+
+@typing_extensions.final
+class ArtifactManifest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ VERSION_FIELD_NUMBER: builtins.int
+ STORAGE_POLICY_FIELD_NUMBER: builtins.int
+ STORAGE_POLICY_CONFIG_FIELD_NUMBER: builtins.int
+ CONTENTS_FIELD_NUMBER: builtins.int
+ version: builtins.int
+ storage_policy: builtins.str
+ @property
+ def storage_policy_config(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___StoragePolicyConfigItem]: ...
+ @property
+ def contents(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ArtifactManifestEntry]: ...
+ def __init__(
+ self,
+ *,
+ version: builtins.int = ...,
+ storage_policy: builtins.str = ...,
+ storage_policy_config: collections.abc.Iterable[global___StoragePolicyConfigItem] | None = ...,
+ contents: collections.abc.Iterable[global___ArtifactManifestEntry] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["contents", b"contents", "storage_policy", b"storage_policy", "storage_policy_config", b"storage_policy_config", "version", b"version"]) -> None: ...
+
+global___ArtifactManifest = ArtifactManifest
+
+@typing_extensions.final
+class ArtifactManifestEntry(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ PATH_FIELD_NUMBER: builtins.int
+ DIGEST_FIELD_NUMBER: builtins.int
+ REF_FIELD_NUMBER: builtins.int
+ SIZE_FIELD_NUMBER: builtins.int
+ MIMETYPE_FIELD_NUMBER: builtins.int
+ LOCAL_PATH_FIELD_NUMBER: builtins.int
+ BIRTH_ARTIFACT_ID_FIELD_NUMBER: builtins.int
+ EXTRA_FIELD_NUMBER: builtins.int
+ path: builtins.str
+ digest: builtins.str
+ ref: builtins.str
+ size: builtins.int
+ mimetype: builtins.str
+ local_path: builtins.str
+ birth_artifact_id: builtins.str
+ @property
+ def extra(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ExtraItem]: ...
+ def __init__(
+ self,
+ *,
+ path: builtins.str = ...,
+ digest: builtins.str = ...,
+ ref: builtins.str = ...,
+ size: builtins.int = ...,
+ mimetype: builtins.str = ...,
+ local_path: builtins.str = ...,
+ birth_artifact_id: builtins.str = ...,
+ extra: collections.abc.Iterable[global___ExtraItem] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["birth_artifact_id", b"birth_artifact_id", "digest", b"digest", "extra", b"extra", "local_path", b"local_path", "mimetype", b"mimetype", "path", b"path", "ref", b"ref", "size", b"size"]) -> None: ...
+
+global___ArtifactManifestEntry = ArtifactManifestEntry
+
+@typing_extensions.final
+class ExtraItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_JSON_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ value_json: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value_json: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value_json", b"value_json"]) -> None: ...
+
+global___ExtraItem = ExtraItem
+
+@typing_extensions.final
+class StoragePolicyConfigItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_JSON_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ value_json: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value_json: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value_json", b"value_json"]) -> None: ...
+
+global___StoragePolicyConfigItem = StoragePolicyConfigItem
+
+@typing_extensions.final
+class ArtifactResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ArtifactResult = ArtifactResult
+
+@typing_extensions.final
+class LinkArtifactResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___LinkArtifactResult = LinkArtifactResult
+
+@typing_extensions.final
+class LinkArtifactRecord(google.protobuf.message.Message):
+ """
+ LinkArtifactRecord: link artifact to portfolio
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ CLIENT_ID_FIELD_NUMBER: builtins.int
+ SERVER_ID_FIELD_NUMBER: builtins.int
+ PORTFOLIO_NAME_FIELD_NUMBER: builtins.int
+ PORTFOLIO_ENTITY_FIELD_NUMBER: builtins.int
+ PORTFOLIO_PROJECT_FIELD_NUMBER: builtins.int
+ PORTFOLIO_ALIASES_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ client_id: builtins.str
+ server_id: builtins.str
+ portfolio_name: builtins.str
+ portfolio_entity: builtins.str
+ portfolio_project: builtins.str
+ @property
+ def portfolio_aliases(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ client_id: builtins.str = ...,
+ server_id: builtins.str = ...,
+ portfolio_name: builtins.str = ...,
+ portfolio_entity: builtins.str = ...,
+ portfolio_project: builtins.str = ...,
+ portfolio_aliases: collections.abc.Iterable[builtins.str] | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "client_id", b"client_id", "portfolio_aliases", b"portfolio_aliases", "portfolio_entity", b"portfolio_entity", "portfolio_name", b"portfolio_name", "portfolio_project", b"portfolio_project", "server_id", b"server_id"]) -> None: ...
+
+global___LinkArtifactRecord = LinkArtifactRecord
+
+@typing_extensions.final
+class TBRecord(google.protobuf.message.Message):
+ """
+ TBRecord: store tb locations
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ LOG_DIR_FIELD_NUMBER: builtins.int
+ SAVE_FIELD_NUMBER: builtins.int
+ ROOT_DIR_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ log_dir: builtins.str
+ save: builtins.bool
+ root_dir: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ log_dir: builtins.str = ...,
+ save: builtins.bool = ...,
+ root_dir: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "log_dir", b"log_dir", "root_dir", b"root_dir", "save", b"save"]) -> None: ...
+
+global___TBRecord = TBRecord
+
+@typing_extensions.final
+class TBResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___TBResult = TBResult
+
+@typing_extensions.final
+class AlertRecord(google.protobuf.message.Message):
+ """
+ AlertRecord: store alert notifications
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ TITLE_FIELD_NUMBER: builtins.int
+ TEXT_FIELD_NUMBER: builtins.int
+ LEVEL_FIELD_NUMBER: builtins.int
+ WAIT_DURATION_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ title: builtins.str
+ text: builtins.str
+ level: builtins.str
+ wait_duration: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ title: builtins.str = ...,
+ text: builtins.str = ...,
+ level: builtins.str = ...,
+ wait_duration: builtins.int = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "level", b"level", "text", b"text", "title", b"title", "wait_duration", b"wait_duration"]) -> None: ...
+
+global___AlertRecord = AlertRecord
+
+@typing_extensions.final
+class AlertResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___AlertResult = AlertResult
+
+@typing_extensions.final
+class Request(google.protobuf.message.Message):
+ """***********************
+ Requests and Responses
+ **********************
+
+
+ Request: all non persistent messages
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ STOP_STATUS_FIELD_NUMBER: builtins.int
+ NETWORK_STATUS_FIELD_NUMBER: builtins.int
+ DEFER_FIELD_NUMBER: builtins.int
+ GET_SUMMARY_FIELD_NUMBER: builtins.int
+ LOGIN_FIELD_NUMBER: builtins.int
+ PAUSE_FIELD_NUMBER: builtins.int
+ RESUME_FIELD_NUMBER: builtins.int
+ POLL_EXIT_FIELD_NUMBER: builtins.int
+ SAMPLED_HISTORY_FIELD_NUMBER: builtins.int
+ PARTIAL_HISTORY_FIELD_NUMBER: builtins.int
+ RUN_START_FIELD_NUMBER: builtins.int
+ CHECK_VERSION_FIELD_NUMBER: builtins.int
+ LOG_ARTIFACT_FIELD_NUMBER: builtins.int
+ DOWNLOAD_ARTIFACT_FIELD_NUMBER: builtins.int
+ KEEPALIVE_FIELD_NUMBER: builtins.int
+ RUN_STATUS_FIELD_NUMBER: builtins.int
+ CANCEL_FIELD_NUMBER: builtins.int
+ METADATA_FIELD_NUMBER: builtins.int
+ INTERNAL_MESSAGES_FIELD_NUMBER: builtins.int
+ PYTHON_PACKAGES_FIELD_NUMBER: builtins.int
+ SHUTDOWN_FIELD_NUMBER: builtins.int
+ ATTACH_FIELD_NUMBER: builtins.int
+ STATUS_FIELD_NUMBER: builtins.int
+ SERVER_INFO_FIELD_NUMBER: builtins.int
+ SENDER_MARK_FIELD_NUMBER: builtins.int
+ SENDER_READ_FIELD_NUMBER: builtins.int
+ STATUS_REPORT_FIELD_NUMBER: builtins.int
+ SUMMARY_RECORD_FIELD_NUMBER: builtins.int
+ TELEMETRY_RECORD_FIELD_NUMBER: builtins.int
+ JOB_INFO_FIELD_NUMBER: builtins.int
+ GET_SYSTEM_METRICS_FIELD_NUMBER: builtins.int
+ FILE_TRANSFER_INFO_FIELD_NUMBER: builtins.int
+ SYNC_FIELD_NUMBER: builtins.int
+ TEST_INJECT_FIELD_NUMBER: builtins.int
+ @property
+ def stop_status(self) -> global___StopStatusRequest: ...
+ @property
+ def network_status(self) -> global___NetworkStatusRequest: ...
+ @property
+ def defer(self) -> global___DeferRequest: ...
+ @property
+ def get_summary(self) -> global___GetSummaryRequest: ...
+ @property
+ def login(self) -> global___LoginRequest: ...
+ @property
+ def pause(self) -> global___PauseRequest: ...
+ @property
+ def resume(self) -> global___ResumeRequest: ...
+ @property
+ def poll_exit(self) -> global___PollExitRequest: ...
+ @property
+ def sampled_history(self) -> global___SampledHistoryRequest: ...
+ @property
+ def partial_history(self) -> global___PartialHistoryRequest: ...
+ @property
+ def run_start(self) -> global___RunStartRequest: ...
+ @property
+ def check_version(self) -> global___CheckVersionRequest: ...
+ @property
+ def log_artifact(self) -> global___LogArtifactRequest: ...
+ @property
+ def download_artifact(self) -> global___DownloadArtifactRequest: ...
+ @property
+ def keepalive(self) -> global___KeepaliveRequest: ...
+ @property
+ def run_status(self) -> global___RunStatusRequest: ...
+ @property
+ def cancel(self) -> global___CancelRequest: ...
+ @property
+ def metadata(self) -> global___MetadataRequest: ...
+ @property
+ def internal_messages(self) -> global___InternalMessagesRequest: ...
+ @property
+ def python_packages(self) -> global___PythonPackagesRequest: ...
+ @property
+ def shutdown(self) -> global___ShutdownRequest: ...
+ @property
+ def attach(self) -> global___AttachRequest: ...
+ @property
+ def status(self) -> global___StatusRequest: ...
+ @property
+ def server_info(self) -> global___ServerInfoRequest: ...
+ @property
+ def sender_mark(self) -> global___SenderMarkRequest: ...
+ @property
+ def sender_read(self) -> global___SenderReadRequest: ...
+ @property
+ def status_report(self) -> global___StatusReportRequest: ...
+ @property
+ def summary_record(self) -> global___SummaryRecordRequest: ...
+ @property
+ def telemetry_record(self) -> global___TelemetryRecordRequest: ...
+ @property
+ def job_info(self) -> global___JobInfoRequest: ...
+ @property
+ def get_system_metrics(self) -> global___GetSystemMetricsRequest: ...
+ @property
+ def file_transfer_info(self) -> global___FileTransferInfoRequest: ...
+ @property
+ def sync(self) -> global___SyncRequest: ...
+ @property
+ def test_inject(self) -> global___TestInjectRequest: ...
+ def __init__(
+ self,
+ *,
+ stop_status: global___StopStatusRequest | None = ...,
+ network_status: global___NetworkStatusRequest | None = ...,
+ defer: global___DeferRequest | None = ...,
+ get_summary: global___GetSummaryRequest | None = ...,
+ login: global___LoginRequest | None = ...,
+ pause: global___PauseRequest | None = ...,
+ resume: global___ResumeRequest | None = ...,
+ poll_exit: global___PollExitRequest | None = ...,
+ sampled_history: global___SampledHistoryRequest | None = ...,
+ partial_history: global___PartialHistoryRequest | None = ...,
+ run_start: global___RunStartRequest | None = ...,
+ check_version: global___CheckVersionRequest | None = ...,
+ log_artifact: global___LogArtifactRequest | None = ...,
+ download_artifact: global___DownloadArtifactRequest | None = ...,
+ keepalive: global___KeepaliveRequest | None = ...,
+ run_status: global___RunStatusRequest | None = ...,
+ cancel: global___CancelRequest | None = ...,
+ metadata: global___MetadataRequest | None = ...,
+ internal_messages: global___InternalMessagesRequest | None = ...,
+ python_packages: global___PythonPackagesRequest | None = ...,
+ shutdown: global___ShutdownRequest | None = ...,
+ attach: global___AttachRequest | None = ...,
+ status: global___StatusRequest | None = ...,
+ server_info: global___ServerInfoRequest | None = ...,
+ sender_mark: global___SenderMarkRequest | None = ...,
+ sender_read: global___SenderReadRequest | None = ...,
+ status_report: global___StatusReportRequest | None = ...,
+ summary_record: global___SummaryRecordRequest | None = ...,
+ telemetry_record: global___TelemetryRecordRequest | None = ...,
+ job_info: global___JobInfoRequest | None = ...,
+ get_system_metrics: global___GetSystemMetricsRequest | None = ...,
+ file_transfer_info: global___FileTransferInfoRequest | None = ...,
+ sync: global___SyncRequest | None = ...,
+ test_inject: global___TestInjectRequest | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["attach", b"attach", "cancel", b"cancel", "check_version", b"check_version", "defer", b"defer", "download_artifact", b"download_artifact", "file_transfer_info", b"file_transfer_info", "get_summary", b"get_summary", "get_system_metrics", b"get_system_metrics", "internal_messages", b"internal_messages", "job_info", b"job_info", "keepalive", b"keepalive", "log_artifact", b"log_artifact", "login", b"login", "metadata", b"metadata", "network_status", b"network_status", "partial_history", b"partial_history", "pause", b"pause", "poll_exit", b"poll_exit", "python_packages", b"python_packages", "request_type", b"request_type", "resume", b"resume", "run_start", b"run_start", "run_status", b"run_status", "sampled_history", b"sampled_history", "sender_mark", b"sender_mark", "sender_read", b"sender_read", "server_info", b"server_info", "shutdown", b"shutdown", "status", b"status", "status_report", b"status_report", "stop_status", b"stop_status", "summary_record", b"summary_record", "sync", b"sync", "telemetry_record", b"telemetry_record", "test_inject", b"test_inject"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["attach", b"attach", "cancel", b"cancel", "check_version", b"check_version", "defer", b"defer", "download_artifact", b"download_artifact", "file_transfer_info", b"file_transfer_info", "get_summary", b"get_summary", "get_system_metrics", b"get_system_metrics", "internal_messages", b"internal_messages", "job_info", b"job_info", "keepalive", b"keepalive", "log_artifact", b"log_artifact", "login", b"login", "metadata", b"metadata", "network_status", b"network_status", "partial_history", b"partial_history", "pause", b"pause", "poll_exit", b"poll_exit", "python_packages", b"python_packages", "request_type", b"request_type", "resume", b"resume", "run_start", b"run_start", "run_status", b"run_status", "sampled_history", b"sampled_history", "sender_mark", b"sender_mark", "sender_read", b"sender_read", "server_info", b"server_info", "shutdown", b"shutdown", "status", b"status", "status_report", b"status_report", "stop_status", b"stop_status", "summary_record", b"summary_record", "sync", b"sync", "telemetry_record", b"telemetry_record", "test_inject", b"test_inject"]) -> None: ...
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["request_type", b"request_type"]) -> typing_extensions.Literal["stop_status", "network_status", "defer", "get_summary", "login", "pause", "resume", "poll_exit", "sampled_history", "partial_history", "run_start", "check_version", "log_artifact", "download_artifact", "keepalive", "run_status", "cancel", "metadata", "internal_messages", "python_packages", "shutdown", "attach", "status", "server_info", "sender_mark", "sender_read", "status_report", "summary_record", "telemetry_record", "job_info", "get_system_metrics", "file_transfer_info", "sync", "test_inject"] | None: ...
+
+global___Request = Request
+
+@typing_extensions.final
+class Response(google.protobuf.message.Message):
+ """
+ Response: all non persistent responses to Requests
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEEPALIVE_RESPONSE_FIELD_NUMBER: builtins.int
+ STOP_STATUS_RESPONSE_FIELD_NUMBER: builtins.int
+ NETWORK_STATUS_RESPONSE_FIELD_NUMBER: builtins.int
+ LOGIN_RESPONSE_FIELD_NUMBER: builtins.int
+ GET_SUMMARY_RESPONSE_FIELD_NUMBER: builtins.int
+ POLL_EXIT_RESPONSE_FIELD_NUMBER: builtins.int
+ SAMPLED_HISTORY_RESPONSE_FIELD_NUMBER: builtins.int
+ RUN_START_RESPONSE_FIELD_NUMBER: builtins.int
+ CHECK_VERSION_RESPONSE_FIELD_NUMBER: builtins.int
+ LOG_ARTIFACT_RESPONSE_FIELD_NUMBER: builtins.int
+ DOWNLOAD_ARTIFACT_RESPONSE_FIELD_NUMBER: builtins.int
+ RUN_STATUS_RESPONSE_FIELD_NUMBER: builtins.int
+ CANCEL_RESPONSE_FIELD_NUMBER: builtins.int
+ INTERNAL_MESSAGES_RESPONSE_FIELD_NUMBER: builtins.int
+ SHUTDOWN_RESPONSE_FIELD_NUMBER: builtins.int
+ ATTACH_RESPONSE_FIELD_NUMBER: builtins.int
+ STATUS_RESPONSE_FIELD_NUMBER: builtins.int
+ SERVER_INFO_RESPONSE_FIELD_NUMBER: builtins.int
+ JOB_INFO_RESPONSE_FIELD_NUMBER: builtins.int
+ GET_SYSTEM_METRICS_RESPONSE_FIELD_NUMBER: builtins.int
+ SYNC_RESPONSE_FIELD_NUMBER: builtins.int
+ TEST_INJECT_RESPONSE_FIELD_NUMBER: builtins.int
+ @property
+ def keepalive_response(self) -> global___KeepaliveResponse: ...
+ @property
+ def stop_status_response(self) -> global___StopStatusResponse: ...
+ @property
+ def network_status_response(self) -> global___NetworkStatusResponse: ...
+ @property
+ def login_response(self) -> global___LoginResponse: ...
+ @property
+ def get_summary_response(self) -> global___GetSummaryResponse: ...
+ @property
+ def poll_exit_response(self) -> global___PollExitResponse: ...
+ @property
+ def sampled_history_response(self) -> global___SampledHistoryResponse: ...
+ @property
+ def run_start_response(self) -> global___RunStartResponse: ...
+ @property
+ def check_version_response(self) -> global___CheckVersionResponse: ...
+ @property
+ def log_artifact_response(self) -> global___LogArtifactResponse: ...
+ @property
+ def download_artifact_response(self) -> global___DownloadArtifactResponse: ...
+ @property
+ def run_status_response(self) -> global___RunStatusResponse: ...
+ @property
+ def cancel_response(self) -> global___CancelResponse: ...
+ @property
+ def internal_messages_response(self) -> global___InternalMessagesResponse: ...
+ @property
+ def shutdown_response(self) -> global___ShutdownResponse: ...
+ @property
+ def attach_response(self) -> global___AttachResponse: ...
+ @property
+ def status_response(self) -> global___StatusResponse: ...
+ @property
+ def server_info_response(self) -> global___ServerInfoResponse: ...
+ @property
+ def job_info_response(self) -> global___JobInfoResponse: ...
+ @property
+ def get_system_metrics_response(self) -> global___GetSystemMetricsResponse: ...
+ @property
+ def sync_response(self) -> global___SyncResponse: ...
+ @property
+ def test_inject_response(self) -> global___TestInjectResponse: ...
+ def __init__(
+ self,
+ *,
+ keepalive_response: global___KeepaliveResponse | None = ...,
+ stop_status_response: global___StopStatusResponse | None = ...,
+ network_status_response: global___NetworkStatusResponse | None = ...,
+ login_response: global___LoginResponse | None = ...,
+ get_summary_response: global___GetSummaryResponse | None = ...,
+ poll_exit_response: global___PollExitResponse | None = ...,
+ sampled_history_response: global___SampledHistoryResponse | None = ...,
+ run_start_response: global___RunStartResponse | None = ...,
+ check_version_response: global___CheckVersionResponse | None = ...,
+ log_artifact_response: global___LogArtifactResponse | None = ...,
+ download_artifact_response: global___DownloadArtifactResponse | None = ...,
+ run_status_response: global___RunStatusResponse | None = ...,
+ cancel_response: global___CancelResponse | None = ...,
+ internal_messages_response: global___InternalMessagesResponse | None = ...,
+ shutdown_response: global___ShutdownResponse | None = ...,
+ attach_response: global___AttachResponse | None = ...,
+ status_response: global___StatusResponse | None = ...,
+ server_info_response: global___ServerInfoResponse | None = ...,
+ job_info_response: global___JobInfoResponse | None = ...,
+ get_system_metrics_response: global___GetSystemMetricsResponse | None = ...,
+ sync_response: global___SyncResponse | None = ...,
+ test_inject_response: global___TestInjectResponse | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["attach_response", b"attach_response", "cancel_response", b"cancel_response", "check_version_response", b"check_version_response", "download_artifact_response", b"download_artifact_response", "get_summary_response", b"get_summary_response", "get_system_metrics_response", b"get_system_metrics_response", "internal_messages_response", b"internal_messages_response", "job_info_response", b"job_info_response", "keepalive_response", b"keepalive_response", "log_artifact_response", b"log_artifact_response", "login_response", b"login_response", "network_status_response", b"network_status_response", "poll_exit_response", b"poll_exit_response", "response_type", b"response_type", "run_start_response", b"run_start_response", "run_status_response", b"run_status_response", "sampled_history_response", b"sampled_history_response", "server_info_response", b"server_info_response", "shutdown_response", b"shutdown_response", "status_response", b"status_response", "stop_status_response", b"stop_status_response", "sync_response", b"sync_response", "test_inject_response", b"test_inject_response"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["attach_response", b"attach_response", "cancel_response", b"cancel_response", "check_version_response", b"check_version_response", "download_artifact_response", b"download_artifact_response", "get_summary_response", b"get_summary_response", "get_system_metrics_response", b"get_system_metrics_response", "internal_messages_response", b"internal_messages_response", "job_info_response", b"job_info_response", "keepalive_response", b"keepalive_response", "log_artifact_response", b"log_artifact_response", "login_response", b"login_response", "network_status_response", b"network_status_response", "poll_exit_response", b"poll_exit_response", "response_type", b"response_type", "run_start_response", b"run_start_response", "run_status_response", b"run_status_response", "sampled_history_response", b"sampled_history_response", "server_info_response", b"server_info_response", "shutdown_response", b"shutdown_response", "status_response", b"status_response", "stop_status_response", b"stop_status_response", "sync_response", b"sync_response", "test_inject_response", b"test_inject_response"]) -> None: ...
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["response_type", b"response_type"]) -> typing_extensions.Literal["keepalive_response", "stop_status_response", "network_status_response", "login_response", "get_summary_response", "poll_exit_response", "sampled_history_response", "run_start_response", "check_version_response", "log_artifact_response", "download_artifact_response", "run_status_response", "cancel_response", "internal_messages_response", "shutdown_response", "attach_response", "status_response", "server_info_response", "job_info_response", "get_system_metrics_response", "sync_response", "test_inject_response"] | None: ...
+
+global___Response = Response
+
+@typing_extensions.final
+class DeferRequest(google.protobuf.message.Message):
+ """
+ DeferRequest: internal message to defer work
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _DeferState:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _DeferStateEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[DeferRequest._DeferState.ValueType], builtins.type):
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ BEGIN: DeferRequest._DeferState.ValueType # 0
+ FLUSH_RUN: DeferRequest._DeferState.ValueType # 1
+ FLUSH_STATS: DeferRequest._DeferState.ValueType # 2
+ FLUSH_PARTIAL_HISTORY: DeferRequest._DeferState.ValueType # 3
+ FLUSH_TB: DeferRequest._DeferState.ValueType # 4
+ FLUSH_SUM: DeferRequest._DeferState.ValueType # 5
+ FLUSH_DEBOUNCER: DeferRequest._DeferState.ValueType # 6
+ FLUSH_OUTPUT: DeferRequest._DeferState.ValueType # 7
+ FLUSH_JOB: DeferRequest._DeferState.ValueType # 8
+ FLUSH_DIR: DeferRequest._DeferState.ValueType # 9
+ FLUSH_FP: DeferRequest._DeferState.ValueType # 10
+ JOIN_FP: DeferRequest._DeferState.ValueType # 11
+ FLUSH_FS: DeferRequest._DeferState.ValueType # 12
+ FLUSH_FINAL: DeferRequest._DeferState.ValueType # 13
+ END: DeferRequest._DeferState.ValueType # 14
+
+ class DeferState(_DeferState, metaclass=_DeferStateEnumTypeWrapper): ...
+ BEGIN: DeferRequest.DeferState.ValueType # 0
+ FLUSH_RUN: DeferRequest.DeferState.ValueType # 1
+ FLUSH_STATS: DeferRequest.DeferState.ValueType # 2
+ FLUSH_PARTIAL_HISTORY: DeferRequest.DeferState.ValueType # 3
+ FLUSH_TB: DeferRequest.DeferState.ValueType # 4
+ FLUSH_SUM: DeferRequest.DeferState.ValueType # 5
+ FLUSH_DEBOUNCER: DeferRequest.DeferState.ValueType # 6
+ FLUSH_OUTPUT: DeferRequest.DeferState.ValueType # 7
+ FLUSH_JOB: DeferRequest.DeferState.ValueType # 8
+ FLUSH_DIR: DeferRequest.DeferState.ValueType # 9
+ FLUSH_FP: DeferRequest.DeferState.ValueType # 10
+ JOIN_FP: DeferRequest.DeferState.ValueType # 11
+ FLUSH_FS: DeferRequest.DeferState.ValueType # 12
+ FLUSH_FINAL: DeferRequest.DeferState.ValueType # 13
+ END: DeferRequest.DeferState.ValueType # 14
+
+ STATE_FIELD_NUMBER: builtins.int
+ state: global___DeferRequest.DeferState.ValueType
+ """Internal message, no _info field needed"""
+ def __init__(
+ self,
+ *,
+ state: global___DeferRequest.DeferState.ValueType = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["state", b"state"]) -> None: ...
+
+global___DeferRequest = DeferRequest
+
+@typing_extensions.final
+class PauseRequest(google.protobuf.message.Message):
+ """
+ PauseRequest: internal message to pause the heartbeat
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___PauseRequest = PauseRequest
+
+@typing_extensions.final
+class PauseResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___PauseResponse = PauseResponse
+
+@typing_extensions.final
+class ResumeRequest(google.protobuf.message.Message):
+ """
+ ResumeRequest: internal message to resume the heartbeat
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ResumeRequest = ResumeRequest
+
+@typing_extensions.final
+class ResumeResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ResumeResponse = ResumeResponse
+
+@typing_extensions.final
+class LoginRequest(google.protobuf.message.Message):
+ """
+ LoginRequest: wandb/sdk/wandb_login
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ API_KEY_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ api_key: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ api_key: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "api_key", b"api_key"]) -> None: ...
+
+global___LoginRequest = LoginRequest
+
+@typing_extensions.final
+class LoginResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ACTIVE_ENTITY_FIELD_NUMBER: builtins.int
+ active_entity: builtins.str
+ def __init__(
+ self,
+ *,
+ active_entity: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["active_entity", b"active_entity"]) -> None: ...
+
+global___LoginResponse = LoginResponse
+
+@typing_extensions.final
+class GetSummaryRequest(google.protobuf.message.Message):
+ """
+ GetSummaryRequest: request consolidated summary
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___GetSummaryRequest = GetSummaryRequest
+
+@typing_extensions.final
+class GetSummaryResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ITEM_FIELD_NUMBER: builtins.int
+ @property
+ def item(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___SummaryItem]: ...
+ def __init__(
+ self,
+ *,
+ item: collections.abc.Iterable[global___SummaryItem] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["item", b"item"]) -> None: ...
+
+global___GetSummaryResponse = GetSummaryResponse
+
+@typing_extensions.final
+class GetSystemMetricsRequest(google.protobuf.message.Message):
+ """
+ GetSystemMetrics: request system metrics
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___GetSystemMetricsRequest = GetSystemMetricsRequest
+
+@typing_extensions.final
+class SystemMetricSample(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ TIMESTAMP_FIELD_NUMBER: builtins.int
+ VALUE_FIELD_NUMBER: builtins.int
+ @property
+ def timestamp(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ value: builtins.float
+ def __init__(
+ self,
+ *,
+ timestamp: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ value: builtins.float = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["timestamp", b"timestamp"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["timestamp", b"timestamp", "value", b"value"]) -> None: ...
+
+global___SystemMetricSample = SystemMetricSample
+
+@typing_extensions.final
+class SystemMetricsBuffer(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RECORD_FIELD_NUMBER: builtins.int
+ @property
+ def record(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___SystemMetricSample]: ...
+ def __init__(
+ self,
+ *,
+ record: collections.abc.Iterable[global___SystemMetricSample] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["record", b"record"]) -> None: ...
+
+global___SystemMetricsBuffer = SystemMetricsBuffer
+
+@typing_extensions.final
+class GetSystemMetricsResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ @typing_extensions.final
+ class SystemMetricsEntry(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ @property
+ def value(self) -> global___SystemMetricsBuffer: ...
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value: global___SystemMetricsBuffer | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["value", b"value"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ...
+
+ SYSTEM_METRICS_FIELD_NUMBER: builtins.int
+ @property
+ def system_metrics(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___SystemMetricsBuffer]: ...
+ def __init__(
+ self,
+ *,
+ system_metrics: collections.abc.Mapping[builtins.str, global___SystemMetricsBuffer] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["system_metrics", b"system_metrics"]) -> None: ...
+
+global___GetSystemMetricsResponse = GetSystemMetricsResponse
+
+@typing_extensions.final
+class StatusRequest(google.protobuf.message.Message):
+ """
+ StatusRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___StatusRequest = StatusRequest
+
+@typing_extensions.final
+class StatusResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_SHOULD_STOP_FIELD_NUMBER: builtins.int
+ run_should_stop: builtins.bool
+ def __init__(
+ self,
+ *,
+ run_should_stop: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["run_should_stop", b"run_should_stop"]) -> None: ...
+
+global___StatusResponse = StatusResponse
+
+@typing_extensions.final
+class StopStatusRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___StopStatusRequest = StopStatusRequest
+
+@typing_extensions.final
+class StopStatusResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_SHOULD_STOP_FIELD_NUMBER: builtins.int
+ run_should_stop: builtins.bool
+ def __init__(
+ self,
+ *,
+ run_should_stop: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["run_should_stop", b"run_should_stop"]) -> None: ...
+
+global___StopStatusResponse = StopStatusResponse
+
+@typing_extensions.final
+class NetworkStatusRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___NetworkStatusRequest = NetworkStatusRequest
+
+@typing_extensions.final
+class NetworkStatusResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ NETWORK_RESPONSES_FIELD_NUMBER: builtins.int
+ @property
+ def network_responses(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___HttpResponse]: ...
+ def __init__(
+ self,
+ *,
+ network_responses: collections.abc.Iterable[global___HttpResponse] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["network_responses", b"network_responses"]) -> None: ...
+
+global___NetworkStatusResponse = NetworkStatusResponse
+
+@typing_extensions.final
+class HttpResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ HTTP_STATUS_CODE_FIELD_NUMBER: builtins.int
+ HTTP_RESPONSE_TEXT_FIELD_NUMBER: builtins.int
+ http_status_code: builtins.int
+ http_response_text: builtins.str
+ def __init__(
+ self,
+ *,
+ http_status_code: builtins.int = ...,
+ http_response_text: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["http_response_text", b"http_response_text", "http_status_code", b"http_status_code"]) -> None: ...
+
+global___HttpResponse = HttpResponse
+
+@typing_extensions.final
+class InternalMessagesRequest(google.protobuf.message.Message):
+ """
+ InternalMessagesRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___InternalMessagesRequest = InternalMessagesRequest
+
+@typing_extensions.final
+class InternalMessagesResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ MESSAGES_FIELD_NUMBER: builtins.int
+ @property
+ def messages(self) -> global___InternalMessages: ...
+ def __init__(
+ self,
+ *,
+ messages: global___InternalMessages | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["messages", b"messages"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["messages", b"messages"]) -> None: ...
+
+global___InternalMessagesResponse = InternalMessagesResponse
+
+@typing_extensions.final
+class InternalMessages(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ WARNING_FIELD_NUMBER: builtins.int
+ @property
+ def warning(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ def __init__(
+ self,
+ *,
+ warning: collections.abc.Iterable[builtins.str] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["warning", b"warning"]) -> None: ...
+
+global___InternalMessages = InternalMessages
+
+@typing_extensions.final
+class PollExitRequest(google.protobuf.message.Message):
+ """
+ PollExitRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___PollExitRequest = PollExitRequest
+
+@typing_extensions.final
+class PollExitResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ DONE_FIELD_NUMBER: builtins.int
+ EXIT_RESULT_FIELD_NUMBER: builtins.int
+ PUSHER_STATS_FIELD_NUMBER: builtins.int
+ FILE_COUNTS_FIELD_NUMBER: builtins.int
+ done: builtins.bool
+ @property
+ def exit_result(self) -> global___RunExitResult: ...
+ @property
+ def pusher_stats(self) -> global___FilePusherStats: ...
+ @property
+ def file_counts(self) -> global___FileCounts: ...
+ def __init__(
+ self,
+ *,
+ done: builtins.bool = ...,
+ exit_result: global___RunExitResult | None = ...,
+ pusher_stats: global___FilePusherStats | None = ...,
+ file_counts: global___FileCounts | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["exit_result", b"exit_result", "file_counts", b"file_counts", "pusher_stats", b"pusher_stats"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["done", b"done", "exit_result", b"exit_result", "file_counts", b"file_counts", "pusher_stats", b"pusher_stats"]) -> None: ...
+
+global___PollExitResponse = PollExitResponse
+
+@typing_extensions.final
+class SyncOverwrite(google.protobuf.message.Message):
+ """
+ Sender requests
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_ID_FIELD_NUMBER: builtins.int
+ ENTITY_FIELD_NUMBER: builtins.int
+ PROJECT_FIELD_NUMBER: builtins.int
+ run_id: builtins.str
+ entity: builtins.str
+ project: builtins.str
+ def __init__(
+ self,
+ *,
+ run_id: builtins.str = ...,
+ entity: builtins.str = ...,
+ project: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["entity", b"entity", "project", b"project", "run_id", b"run_id"]) -> None: ...
+
+global___SyncOverwrite = SyncOverwrite
+
+@typing_extensions.final
+class SyncSkip(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ OUTPUT_RAW_FIELD_NUMBER: builtins.int
+ output_raw: builtins.bool
+ def __init__(
+ self,
+ *,
+ output_raw: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["output_raw", b"output_raw"]) -> None: ...
+
+global___SyncSkip = SyncSkip
+
+@typing_extensions.final
+class SenderMarkRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___SenderMarkRequest = SenderMarkRequest
+
+@typing_extensions.final
+class SyncRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ START_OFFSET_FIELD_NUMBER: builtins.int
+ FINAL_OFFSET_FIELD_NUMBER: builtins.int
+ OVERWRITE_FIELD_NUMBER: builtins.int
+ SKIP_FIELD_NUMBER: builtins.int
+ start_offset: builtins.int
+ final_offset: builtins.int
+ @property
+ def overwrite(self) -> global___SyncOverwrite: ...
+ @property
+ def skip(self) -> global___SyncSkip: ...
+ def __init__(
+ self,
+ *,
+ start_offset: builtins.int = ...,
+ final_offset: builtins.int = ...,
+ overwrite: global___SyncOverwrite | None = ...,
+ skip: global___SyncSkip | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["overwrite", b"overwrite", "skip", b"skip"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["final_offset", b"final_offset", "overwrite", b"overwrite", "skip", b"skip", "start_offset", b"start_offset"]) -> None: ...
+
+global___SyncRequest = SyncRequest
+
+@typing_extensions.final
+class SyncResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ URL_FIELD_NUMBER: builtins.int
+ ERROR_FIELD_NUMBER: builtins.int
+ url: builtins.str
+ @property
+ def error(self) -> global___ErrorInfo: ...
+ def __init__(
+ self,
+ *,
+ url: builtins.str = ...,
+ error: global___ErrorInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["error", b"error"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["error", b"error", "url", b"url"]) -> None: ...
+
+global___SyncResponse = SyncResponse
+
+@typing_extensions.final
+class SenderReadRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ START_OFFSET_FIELD_NUMBER: builtins.int
+ FINAL_OFFSET_FIELD_NUMBER: builtins.int
+ start_offset: builtins.int
+ final_offset: builtins.int
+ """TODO: implement cancel for paused ops
+ repeated string cancel_list = 3;
+ """
+ def __init__(
+ self,
+ *,
+ start_offset: builtins.int = ...,
+ final_offset: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["final_offset", b"final_offset", "start_offset", b"start_offset"]) -> None: ...
+
+global___SenderReadRequest = SenderReadRequest
+
+@typing_extensions.final
+class StatusReportRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RECORD_NUM_FIELD_NUMBER: builtins.int
+ SENT_OFFSET_FIELD_NUMBER: builtins.int
+ SYNC_TIME_FIELD_NUMBER: builtins.int
+ record_num: builtins.int
+ sent_offset: builtins.int
+ @property
+ def sync_time(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ def __init__(
+ self,
+ *,
+ record_num: builtins.int = ...,
+ sent_offset: builtins.int = ...,
+ sync_time: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["sync_time", b"sync_time"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["record_num", b"record_num", "sent_offset", b"sent_offset", "sync_time", b"sync_time"]) -> None: ...
+
+global___StatusReportRequest = StatusReportRequest
+
+@typing_extensions.final
+class SummaryRecordRequest(google.protobuf.message.Message):
+ """
+ Requests wrapping Records
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SUMMARY_FIELD_NUMBER: builtins.int
+ @property
+ def summary(self) -> global___SummaryRecord: ...
+ def __init__(
+ self,
+ *,
+ summary: global___SummaryRecord | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["summary", b"summary"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["summary", b"summary"]) -> None: ...
+
+global___SummaryRecordRequest = SummaryRecordRequest
+
+@typing_extensions.final
+class TelemetryRecordRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ TELEMETRY_FIELD_NUMBER: builtins.int
+ @property
+ def telemetry(self) -> wandb.proto.wandb_telemetry_pb2.TelemetryRecord: ...
+ def __init__(
+ self,
+ *,
+ telemetry: wandb.proto.wandb_telemetry_pb2.TelemetryRecord | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["telemetry", b"telemetry"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["telemetry", b"telemetry"]) -> None: ...
+
+global___TelemetryRecordRequest = TelemetryRecordRequest
+
+@typing_extensions.final
+class ServerInfoRequest(google.protobuf.message.Message):
+ """
+ ServerInfoRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ServerInfoRequest = ServerInfoRequest
+
+@typing_extensions.final
+class ServerInfoResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ LOCAL_INFO_FIELD_NUMBER: builtins.int
+ SERVER_MESSAGES_FIELD_NUMBER: builtins.int
+ @property
+ def local_info(self) -> global___LocalInfo: ...
+ @property
+ def server_messages(self) -> global___ServerMessages: ...
+ def __init__(
+ self,
+ *,
+ local_info: global___LocalInfo | None = ...,
+ server_messages: global___ServerMessages | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["local_info", b"local_info", "server_messages", b"server_messages"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["local_info", b"local_info", "server_messages", b"server_messages"]) -> None: ...
+
+global___ServerInfoResponse = ServerInfoResponse
+
+@typing_extensions.final
+class ServerMessages(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ITEM_FIELD_NUMBER: builtins.int
+ @property
+ def item(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ServerMessage]: ...
+ def __init__(
+ self,
+ *,
+ item: collections.abc.Iterable[global___ServerMessage] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["item", b"item"]) -> None: ...
+
+global___ServerMessages = ServerMessages
+
+@typing_extensions.final
+class ServerMessage(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ PLAIN_TEXT_FIELD_NUMBER: builtins.int
+ UTF_TEXT_FIELD_NUMBER: builtins.int
+ HTML_TEXT_FIELD_NUMBER: builtins.int
+ TYPE_FIELD_NUMBER: builtins.int
+ LEVEL_FIELD_NUMBER: builtins.int
+ plain_text: builtins.str
+ utf_text: builtins.str
+ html_text: builtins.str
+ type: builtins.str
+ level: builtins.int
+ def __init__(
+ self,
+ *,
+ plain_text: builtins.str = ...,
+ utf_text: builtins.str = ...,
+ html_text: builtins.str = ...,
+ type: builtins.str = ...,
+ level: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["html_text", b"html_text", "level", b"level", "plain_text", b"plain_text", "type", b"type", "utf_text", b"utf_text"]) -> None: ...
+
+global___ServerMessage = ServerMessage
+
+@typing_extensions.final
+class FileCounts(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ WANDB_COUNT_FIELD_NUMBER: builtins.int
+ MEDIA_COUNT_FIELD_NUMBER: builtins.int
+ ARTIFACT_COUNT_FIELD_NUMBER: builtins.int
+ OTHER_COUNT_FIELD_NUMBER: builtins.int
+ wandb_count: builtins.int
+ media_count: builtins.int
+ artifact_count: builtins.int
+ other_count: builtins.int
+ def __init__(
+ self,
+ *,
+ wandb_count: builtins.int = ...,
+ media_count: builtins.int = ...,
+ artifact_count: builtins.int = ...,
+ other_count: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["artifact_count", b"artifact_count", "media_count", b"media_count", "other_count", b"other_count", "wandb_count", b"wandb_count"]) -> None: ...
+
+global___FileCounts = FileCounts
+
+@typing_extensions.final
+class FilePusherStats(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ UPLOADED_BYTES_FIELD_NUMBER: builtins.int
+ TOTAL_BYTES_FIELD_NUMBER: builtins.int
+ DEDUPED_BYTES_FIELD_NUMBER: builtins.int
+ uploaded_bytes: builtins.int
+ total_bytes: builtins.int
+ deduped_bytes: builtins.int
+ def __init__(
+ self,
+ *,
+ uploaded_bytes: builtins.int = ...,
+ total_bytes: builtins.int = ...,
+ deduped_bytes: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["deduped_bytes", b"deduped_bytes", "total_bytes", b"total_bytes", "uploaded_bytes", b"uploaded_bytes"]) -> None: ...
+
+global___FilePusherStats = FilePusherStats
+
+@typing_extensions.final
+class FilesUploaded(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ FILES_FIELD_NUMBER: builtins.int
+ @property
+ def files(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ def __init__(
+ self,
+ *,
+ files: collections.abc.Iterable[builtins.str] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["files", b"files"]) -> None: ...
+
+global___FilesUploaded = FilesUploaded
+
+@typing_extensions.final
+class FileTransferInfoRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ class _TransferType:
+ ValueType = typing.NewType("ValueType", builtins.int)
+ V: typing_extensions.TypeAlias = ValueType
+
+ class _TransferTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[FileTransferInfoRequest._TransferType.ValueType], builtins.type):
+ DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+ Upload: FileTransferInfoRequest._TransferType.ValueType # 0
+ Download: FileTransferInfoRequest._TransferType.ValueType # 1
+
+ class TransferType(_TransferType, metaclass=_TransferTypeEnumTypeWrapper): ...
+ Upload: FileTransferInfoRequest.TransferType.ValueType # 0
+ Download: FileTransferInfoRequest.TransferType.ValueType # 1
+
+ TYPE_FIELD_NUMBER: builtins.int
+ PATH_FIELD_NUMBER: builtins.int
+ URL_FIELD_NUMBER: builtins.int
+ SIZE_FIELD_NUMBER: builtins.int
+ PROCESSED_FIELD_NUMBER: builtins.int
+ FILE_COUNTS_FIELD_NUMBER: builtins.int
+ type: global___FileTransferInfoRequest.TransferType.ValueType
+ path: builtins.str
+ url: builtins.str
+ size: builtins.int
+ processed: builtins.int
+ @property
+ def file_counts(self) -> global___FileCounts: ...
+ def __init__(
+ self,
+ *,
+ type: global___FileTransferInfoRequest.TransferType.ValueType = ...,
+ path: builtins.str = ...,
+ url: builtins.str = ...,
+ size: builtins.int = ...,
+ processed: builtins.int = ...,
+ file_counts: global___FileCounts | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["file_counts", b"file_counts"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["file_counts", b"file_counts", "path", b"path", "processed", b"processed", "size", b"size", "type", b"type", "url", b"url"]) -> None: ...
+
+global___FileTransferInfoRequest = FileTransferInfoRequest
+
+@typing_extensions.final
+class LocalInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ VERSION_FIELD_NUMBER: builtins.int
+ OUT_OF_DATE_FIELD_NUMBER: builtins.int
+ version: builtins.str
+ out_of_date: builtins.bool
+ def __init__(
+ self,
+ *,
+ version: builtins.str = ...,
+ out_of_date: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["out_of_date", b"out_of_date", "version", b"version"]) -> None: ...
+
+global___LocalInfo = LocalInfo
+
+@typing_extensions.final
+class ShutdownRequest(google.protobuf.message.Message):
+ """
+ ShutdownRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ShutdownRequest = ShutdownRequest
+
+@typing_extensions.final
+class ShutdownResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ShutdownResponse = ShutdownResponse
+
+@typing_extensions.final
+class AttachRequest(google.protobuf.message.Message):
+ """
+ AttachRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ATTACH_ID_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ attach_id: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ attach_id: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "attach_id", b"attach_id"]) -> None: ...
+
+global___AttachRequest = AttachRequest
+
+@typing_extensions.final
+class AttachResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_FIELD_NUMBER: builtins.int
+ ERROR_FIELD_NUMBER: builtins.int
+ @property
+ def run(self) -> global___RunRecord: ...
+ @property
+ def error(self) -> global___ErrorInfo: ...
+ def __init__(
+ self,
+ *,
+ run: global___RunRecord | None = ...,
+ error: global___ErrorInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["error", b"error", "run", b"run"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["error", b"error", "run", b"run"]) -> None: ...
+
+global___AttachResponse = AttachResponse
+
+@typing_extensions.final
+class TestInjectRequest(google.protobuf.message.Message):
+ """
+ TestInjectRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ HANDLER_EXC_FIELD_NUMBER: builtins.int
+ HANDLER_EXIT_FIELD_NUMBER: builtins.int
+ HANDLER_ABORT_FIELD_NUMBER: builtins.int
+ SENDER_EXC_FIELD_NUMBER: builtins.int
+ SENDER_EXIT_FIELD_NUMBER: builtins.int
+ SENDER_ABORT_FIELD_NUMBER: builtins.int
+ REQ_EXC_FIELD_NUMBER: builtins.int
+ REQ_EXIT_FIELD_NUMBER: builtins.int
+ REQ_ABORT_FIELD_NUMBER: builtins.int
+ RESP_EXC_FIELD_NUMBER: builtins.int
+ RESP_EXIT_FIELD_NUMBER: builtins.int
+ RESP_ABORT_FIELD_NUMBER: builtins.int
+ MSG_DROP_FIELD_NUMBER: builtins.int
+ MSG_HANG_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ handler_exc: builtins.bool
+ handler_exit: builtins.bool
+ handler_abort: builtins.bool
+ sender_exc: builtins.bool
+ sender_exit: builtins.bool
+ sender_abort: builtins.bool
+ req_exc: builtins.bool
+ req_exit: builtins.bool
+ req_abort: builtins.bool
+ resp_exc: builtins.bool
+ resp_exit: builtins.bool
+ resp_abort: builtins.bool
+ msg_drop: builtins.bool
+ msg_hang: builtins.bool
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ handler_exc: builtins.bool = ...,
+ handler_exit: builtins.bool = ...,
+ handler_abort: builtins.bool = ...,
+ sender_exc: builtins.bool = ...,
+ sender_exit: builtins.bool = ...,
+ sender_abort: builtins.bool = ...,
+ req_exc: builtins.bool = ...,
+ req_exit: builtins.bool = ...,
+ req_abort: builtins.bool = ...,
+ resp_exc: builtins.bool = ...,
+ resp_exit: builtins.bool = ...,
+ resp_abort: builtins.bool = ...,
+ msg_drop: builtins.bool = ...,
+ msg_hang: builtins.bool = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "handler_abort", b"handler_abort", "handler_exc", b"handler_exc", "handler_exit", b"handler_exit", "msg_drop", b"msg_drop", "msg_hang", b"msg_hang", "req_abort", b"req_abort", "req_exc", b"req_exc", "req_exit", b"req_exit", "resp_abort", b"resp_abort", "resp_exc", b"resp_exc", "resp_exit", b"resp_exit", "sender_abort", b"sender_abort", "sender_exc", b"sender_exc", "sender_exit", b"sender_exit"]) -> None: ...
+
+global___TestInjectRequest = TestInjectRequest
+
+@typing_extensions.final
+class TestInjectResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___TestInjectResponse = TestInjectResponse
+
+@typing_extensions.final
+class HistoryAction(google.protobuf.message.Message):
+ """
+ PartialHistoryRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ FLUSH_FIELD_NUMBER: builtins.int
+ flush: builtins.bool
+ def __init__(
+ self,
+ *,
+ flush: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["flush", b"flush"]) -> None: ...
+
+global___HistoryAction = HistoryAction
+
+@typing_extensions.final
+class PartialHistoryRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ITEM_FIELD_NUMBER: builtins.int
+ STEP_FIELD_NUMBER: builtins.int
+ ACTION_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def item(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___HistoryItem]: ...
+ @property
+ def step(self) -> global___HistoryStep: ...
+ @property
+ def action(self) -> global___HistoryAction: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ item: collections.abc.Iterable[global___HistoryItem] | None = ...,
+ step: global___HistoryStep | None = ...,
+ action: global___HistoryAction | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "action", b"action", "step", b"step"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "action", b"action", "item", b"item", "step", b"step"]) -> None: ...
+
+global___PartialHistoryRequest = PartialHistoryRequest
+
+@typing_extensions.final
+class PartialHistoryResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___PartialHistoryResponse = PartialHistoryResponse
+
+@typing_extensions.final
+class SampledHistoryRequest(google.protobuf.message.Message):
+ """
+ SampledHistoryRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___SampledHistoryRequest = SampledHistoryRequest
+
+@typing_extensions.final
+class SampledHistoryItem(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ NESTED_KEY_FIELD_NUMBER: builtins.int
+ VALUES_FLOAT_FIELD_NUMBER: builtins.int
+ VALUES_INT_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ @property
+ def nested_key(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ @property
+ def values_float(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.float]: ...
+ @property
+ def values_int(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ...
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ nested_key: collections.abc.Iterable[builtins.str] | None = ...,
+ values_float: collections.abc.Iterable[builtins.float] | None = ...,
+ values_int: collections.abc.Iterable[builtins.int] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "nested_key", b"nested_key", "values_float", b"values_float", "values_int", b"values_int"]) -> None: ...
+
+global___SampledHistoryItem = SampledHistoryItem
+
+@typing_extensions.final
+class SampledHistoryResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ITEM_FIELD_NUMBER: builtins.int
+ @property
+ def item(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___SampledHistoryItem]: ...
+ def __init__(
+ self,
+ *,
+ item: collections.abc.Iterable[global___SampledHistoryItem] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["item", b"item"]) -> None: ...
+
+global___SampledHistoryResponse = SampledHistoryResponse
+
+@typing_extensions.final
+class RunStatusRequest(google.protobuf.message.Message):
+ """
+ RunStatusRequest:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___RunStatusRequest = RunStatusRequest
+
+@typing_extensions.final
+class RunStatusResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SYNC_ITEMS_TOTAL_FIELD_NUMBER: builtins.int
+ SYNC_ITEMS_PENDING_FIELD_NUMBER: builtins.int
+ SYNC_TIME_FIELD_NUMBER: builtins.int
+ sync_items_total: builtins.int
+ sync_items_pending: builtins.int
+ @property
+ def sync_time(self) -> google.protobuf.timestamp_pb2.Timestamp:
+ """TODO(flowcontrol): can we give the user an indication of step position
+ int64 sync_history_step = 3;
+ google.protobuf.Timestamp sync_history_time = 4;
+ """
+ def __init__(
+ self,
+ *,
+ sync_items_total: builtins.int = ...,
+ sync_items_pending: builtins.int = ...,
+ sync_time: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["sync_time", b"sync_time"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["sync_items_pending", b"sync_items_pending", "sync_items_total", b"sync_items_total", "sync_time", b"sync_time"]) -> None: ...
+
+global___RunStatusResponse = RunStatusResponse
+
+@typing_extensions.final
+class RunStartRequest(google.protobuf.message.Message):
+ """
+ RunStartRequest: start the run
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RUN_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def run(self) -> global___RunRecord: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ run: global___RunRecord | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "run", b"run"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "run", b"run"]) -> None: ...
+
+global___RunStartRequest = RunStartRequest
+
+@typing_extensions.final
+class RunStartResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___RunStartResponse = RunStartResponse
+
+@typing_extensions.final
+class CheckVersionRequest(google.protobuf.message.Message):
+ """
+ CheckVersion:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ CURRENT_VERSION_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ current_version: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ current_version: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "current_version", b"current_version"]) -> None: ...
+
+global___CheckVersionRequest = CheckVersionRequest
+
+@typing_extensions.final
+class CheckVersionResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ UPGRADE_MESSAGE_FIELD_NUMBER: builtins.int
+ YANK_MESSAGE_FIELD_NUMBER: builtins.int
+ DELETE_MESSAGE_FIELD_NUMBER: builtins.int
+ upgrade_message: builtins.str
+ yank_message: builtins.str
+ delete_message: builtins.str
+ def __init__(
+ self,
+ *,
+ upgrade_message: builtins.str = ...,
+ yank_message: builtins.str = ...,
+ delete_message: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["delete_message", b"delete_message", "upgrade_message", b"upgrade_message", "yank_message", b"yank_message"]) -> None: ...
+
+global___CheckVersionResponse = CheckVersionResponse
+
+@typing_extensions.final
+class JobInfoRequest(google.protobuf.message.Message):
+ """
+ JobInfo:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___JobInfoRequest = JobInfoRequest
+
+@typing_extensions.final
+class JobInfoResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SEQUENCEID_FIELD_NUMBER: builtins.int
+ VERSION_FIELD_NUMBER: builtins.int
+ sequenceId: builtins.str
+ version: builtins.str
+ def __init__(
+ self,
+ *,
+ sequenceId: builtins.str = ...,
+ version: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["sequenceId", b"sequenceId", "version", b"version"]) -> None: ...
+
+global___JobInfoResponse = JobInfoResponse
+
+@typing_extensions.final
+class LogArtifactRequest(google.protobuf.message.Message):
+ """
+ LogArtifact:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ARTIFACT_FIELD_NUMBER: builtins.int
+ HISTORY_STEP_FIELD_NUMBER: builtins.int
+ STAGING_DIR_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def artifact(self) -> global___ArtifactRecord: ...
+ history_step: builtins.int
+ staging_dir: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ artifact: global___ArtifactRecord | None = ...,
+ history_step: builtins.int = ...,
+ staging_dir: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "artifact", b"artifact"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "artifact", b"artifact", "history_step", b"history_step", "staging_dir", b"staging_dir"]) -> None: ...
+
+global___LogArtifactRequest = LogArtifactRequest
+
+@typing_extensions.final
+class LogArtifactResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ARTIFACT_ID_FIELD_NUMBER: builtins.int
+ ERROR_MESSAGE_FIELD_NUMBER: builtins.int
+ artifact_id: builtins.str
+ error_message: builtins.str
+ def __init__(
+ self,
+ *,
+ artifact_id: builtins.str = ...,
+ error_message: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["artifact_id", b"artifact_id", "error_message", b"error_message"]) -> None: ...
+
+global___LogArtifactResponse = LogArtifactResponse
+
+@typing_extensions.final
+class DownloadArtifactRequest(google.protobuf.message.Message):
+ """
+ DownloadArtifact:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ARTIFACT_ID_FIELD_NUMBER: builtins.int
+ DOWNLOAD_ROOT_FIELD_NUMBER: builtins.int
+ ALLOW_MISSING_REFERENCES_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ artifact_id: builtins.str
+ download_root: builtins.str
+ allow_missing_references: builtins.bool
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ artifact_id: builtins.str = ...,
+ download_root: builtins.str = ...,
+ allow_missing_references: builtins.bool = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "allow_missing_references", b"allow_missing_references", "artifact_id", b"artifact_id", "download_root", b"download_root"]) -> None: ...
+
+global___DownloadArtifactRequest = DownloadArtifactRequest
+
+@typing_extensions.final
+class DownloadArtifactResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ERROR_MESSAGE_FIELD_NUMBER: builtins.int
+ error_message: builtins.str
+ def __init__(
+ self,
+ *,
+ error_message: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["error_message", b"error_message"]) -> None: ...
+
+global___DownloadArtifactResponse = DownloadArtifactResponse
+
+@typing_extensions.final
+class KeepaliveRequest(google.protobuf.message.Message):
+ """
+ Keepalive:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___KeepaliveRequest = KeepaliveRequest
+
+@typing_extensions.final
+class KeepaliveResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___KeepaliveResponse = KeepaliveResponse
+
+@typing_extensions.final
+class ArtifactInfo(google.protobuf.message.Message):
+ """
+ Job info specific for Partial -> Job upgrade
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ARTIFACT_FIELD_NUMBER: builtins.int
+ ENTRYPOINT_FIELD_NUMBER: builtins.int
+ NOTEBOOK_FIELD_NUMBER: builtins.int
+ artifact: builtins.str
+ @property
+ def entrypoint(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ notebook: builtins.bool
+ def __init__(
+ self,
+ *,
+ artifact: builtins.str = ...,
+ entrypoint: collections.abc.Iterable[builtins.str] | None = ...,
+ notebook: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["artifact", b"artifact", "entrypoint", b"entrypoint", "notebook", b"notebook"]) -> None: ...
+
+global___ArtifactInfo = ArtifactInfo
+
+@typing_extensions.final
+class GitInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ REMOTE_FIELD_NUMBER: builtins.int
+ COMMIT_FIELD_NUMBER: builtins.int
+ remote: builtins.str
+ commit: builtins.str
+ def __init__(
+ self,
+ *,
+ remote: builtins.str = ...,
+ commit: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["commit", b"commit", "remote", b"remote"]) -> None: ...
+
+global___GitInfo = GitInfo
+
+@typing_extensions.final
+class GitSource(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ GIT_INFO_FIELD_NUMBER: builtins.int
+ ENTRYPOINT_FIELD_NUMBER: builtins.int
+ NOTEBOOK_FIELD_NUMBER: builtins.int
+ @property
+ def git_info(self) -> global___GitInfo: ...
+ @property
+ def entrypoint(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ notebook: builtins.bool
+ def __init__(
+ self,
+ *,
+ git_info: global___GitInfo | None = ...,
+ entrypoint: collections.abc.Iterable[builtins.str] | None = ...,
+ notebook: builtins.bool = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["git_info", b"git_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["entrypoint", b"entrypoint", "git_info", b"git_info", "notebook", b"notebook"]) -> None: ...
+
+global___GitSource = GitSource
+
+@typing_extensions.final
+class ImageSource(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ IMAGE_FIELD_NUMBER: builtins.int
+ image: builtins.str
+ def __init__(
+ self,
+ *,
+ image: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["image", b"image"]) -> None: ...
+
+global___ImageSource = ImageSource
+
+@typing_extensions.final
+class Source(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ GIT_FIELD_NUMBER: builtins.int
+ ARTIFACT_FIELD_NUMBER: builtins.int
+ IMAGE_FIELD_NUMBER: builtins.int
+ @property
+ def git(self) -> global___GitSource: ...
+ @property
+ def artifact(self) -> global___ArtifactInfo: ...
+ @property
+ def image(self) -> global___ImageSource: ...
+ def __init__(
+ self,
+ *,
+ git: global___GitSource | None = ...,
+ artifact: global___ArtifactInfo | None = ...,
+ image: global___ImageSource | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["artifact", b"artifact", "git", b"git", "image", b"image"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["artifact", b"artifact", "git", b"git", "image", b"image"]) -> None: ...
+
+global___Source = Source
+
+@typing_extensions.final
+class JobSource(google.protobuf.message.Message):
+ """
+ Mirrors JobSourceDict:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _VERSION_FIELD_NUMBER: builtins.int
+ SOURCE_TYPE_FIELD_NUMBER: builtins.int
+ SOURCE_FIELD_NUMBER: builtins.int
+ RUNTIME_FIELD_NUMBER: builtins.int
+ _version: builtins.str
+ source_type: builtins.str
+ @property
+ def source(self) -> global___Source: ...
+ runtime: builtins.str
+ def __init__(
+ self,
+ *,
+ _version: builtins.str = ...,
+ source_type: builtins.str = ...,
+ source: global___Source | None = ...,
+ runtime: builtins.str = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["source", b"source"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_version", b"_version", "runtime", b"runtime", "source", b"source", "source_type", b"source_type"]) -> None: ...
+
+global___JobSource = JobSource
+
+@typing_extensions.final
+class PartialJobArtifact(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ JOB_NAME_FIELD_NUMBER: builtins.int
+ SOURCE_INFO_FIELD_NUMBER: builtins.int
+ job_name: builtins.str
+ @property
+ def source_info(self) -> global___JobSource: ...
+ def __init__(
+ self,
+ *,
+ job_name: builtins.str = ...,
+ source_info: global___JobSource | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["source_info", b"source_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["job_name", b"job_name", "source_info", b"source_info"]) -> None: ...
+
+global___PartialJobArtifact = PartialJobArtifact
+
+@typing_extensions.final
+class UseArtifactRecord(google.protobuf.message.Message):
+ """
+ UseArtifact:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ID_FIELD_NUMBER: builtins.int
+ TYPE_FIELD_NUMBER: builtins.int
+ NAME_FIELD_NUMBER: builtins.int
+ PARTIAL_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ id: builtins.str
+ type: builtins.str
+ name: builtins.str
+ @property
+ def partial(self) -> global___PartialJobArtifact: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ id: builtins.str = ...,
+ type: builtins.str = ...,
+ name: builtins.str = ...,
+ partial: global___PartialJobArtifact | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "partial", b"partial"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "id", b"id", "name", b"name", "partial", b"partial", "type", b"type"]) -> None: ...
+
+global___UseArtifactRecord = UseArtifactRecord
+
+@typing_extensions.final
+class UseArtifactResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___UseArtifactResult = UseArtifactResult
+
+@typing_extensions.final
+class CancelRequest(google.protobuf.message.Message):
+ """
+ Cancel:
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ CANCEL_SLOT_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ cancel_slot: builtins.str
+ """mailbox slot"""
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RequestInfo: ...
+ def __init__(
+ self,
+ *,
+ cancel_slot: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RequestInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "cancel_slot", b"cancel_slot"]) -> None: ...
+
+global___CancelRequest = CancelRequest
+
+@typing_extensions.final
+class CancelResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___CancelResponse = CancelResponse
+
+@typing_extensions.final
+class DiskInfo(google.protobuf.message.Message):
+ """
+ MetadataRequest
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ TOTAL_FIELD_NUMBER: builtins.int
+ USED_FIELD_NUMBER: builtins.int
+ total: builtins.int
+ used: builtins.int
+ def __init__(
+ self,
+ *,
+ total: builtins.int = ...,
+ used: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["total", b"total", "used", b"used"]) -> None: ...
+
+global___DiskInfo = DiskInfo
+
+@typing_extensions.final
+class MemoryInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ TOTAL_FIELD_NUMBER: builtins.int
+ total: builtins.int
+ def __init__(
+ self,
+ *,
+ total: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["total", b"total"]) -> None: ...
+
+global___MemoryInfo = MemoryInfo
+
+@typing_extensions.final
+class CpuInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ COUNT_FIELD_NUMBER: builtins.int
+ COUNT_LOGICAL_FIELD_NUMBER: builtins.int
+ count: builtins.int
+ count_logical: builtins.int
+ def __init__(
+ self,
+ *,
+ count: builtins.int = ...,
+ count_logical: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["count", b"count", "count_logical", b"count_logical"]) -> None: ...
+
+global___CpuInfo = CpuInfo
+
+@typing_extensions.final
+class GpuAppleInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ GPUTYPE_FIELD_NUMBER: builtins.int
+ VENDOR_FIELD_NUMBER: builtins.int
+ CORES_FIELD_NUMBER: builtins.int
+ gpuType: builtins.str
+ vendor: builtins.str
+ cores: builtins.int
+ def __init__(
+ self,
+ *,
+ gpuType: builtins.str = ...,
+ vendor: builtins.str = ...,
+ cores: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["cores", b"cores", "gpuType", b"gpuType", "vendor", b"vendor"]) -> None: ...
+
+global___GpuAppleInfo = GpuAppleInfo
+
+@typing_extensions.final
+class GpuNvidiaInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ NAME_FIELD_NUMBER: builtins.int
+ MEMORY_TOTAL_FIELD_NUMBER: builtins.int
+ name: builtins.str
+ memory_total: builtins.int
+ def __init__(
+ self,
+ *,
+ name: builtins.str = ...,
+ memory_total: builtins.int = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["memory_total", b"memory_total", "name", b"name"]) -> None: ...
+
+global___GpuNvidiaInfo = GpuNvidiaInfo
+
+@typing_extensions.final
+class GpuAmdInfo(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ ID_FIELD_NUMBER: builtins.int
+ UNIQUE_ID_FIELD_NUMBER: builtins.int
+ VBIOS_VERSION_FIELD_NUMBER: builtins.int
+ PERFORMANCE_LEVEL_FIELD_NUMBER: builtins.int
+ GPU_OVERDRIVE_FIELD_NUMBER: builtins.int
+ GPU_MEMORY_OVERDRIVE_FIELD_NUMBER: builtins.int
+ MAX_POWER_FIELD_NUMBER: builtins.int
+ SERIES_FIELD_NUMBER: builtins.int
+ MODEL_FIELD_NUMBER: builtins.int
+ VENDOR_FIELD_NUMBER: builtins.int
+ SKU_FIELD_NUMBER: builtins.int
+ SCLK_RANGE_FIELD_NUMBER: builtins.int
+ MCLK_RANGE_FIELD_NUMBER: builtins.int
+ id: builtins.str
+ unique_id: builtins.str
+ vbios_version: builtins.str
+ performance_level: builtins.str
+ gpu_overdrive: builtins.str
+ gpu_memory_overdrive: builtins.str
+ max_power: builtins.str
+ series: builtins.str
+ model: builtins.str
+ vendor: builtins.str
+ sku: builtins.str
+ sclk_range: builtins.str
+ mclk_range: builtins.str
+ def __init__(
+ self,
+ *,
+ id: builtins.str = ...,
+ unique_id: builtins.str = ...,
+ vbios_version: builtins.str = ...,
+ performance_level: builtins.str = ...,
+ gpu_overdrive: builtins.str = ...,
+ gpu_memory_overdrive: builtins.str = ...,
+ max_power: builtins.str = ...,
+ series: builtins.str = ...,
+ model: builtins.str = ...,
+ vendor: builtins.str = ...,
+ sku: builtins.str = ...,
+ sclk_range: builtins.str = ...,
+ mclk_range: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["gpu_memory_overdrive", b"gpu_memory_overdrive", "gpu_overdrive", b"gpu_overdrive", "id", b"id", "max_power", b"max_power", "mclk_range", b"mclk_range", "model", b"model", "performance_level", b"performance_level", "sclk_range", b"sclk_range", "series", b"series", "sku", b"sku", "unique_id", b"unique_id", "vbios_version", b"vbios_version", "vendor", b"vendor"]) -> None: ...
+
+global___GpuAmdInfo = GpuAmdInfo
+
+@typing_extensions.final
+class MetadataRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ @typing_extensions.final
+ class DiskEntry(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ @property
+ def value(self) -> global___DiskInfo: ...
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value: global___DiskInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["value", b"value"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ...
+
+ @typing_extensions.final
+ class SlurmEntry(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ value: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ...
+
+ OS_FIELD_NUMBER: builtins.int
+ PYTHON_FIELD_NUMBER: builtins.int
+ HEARTBEATAT_FIELD_NUMBER: builtins.int
+ STARTEDAT_FIELD_NUMBER: builtins.int
+ DOCKER_FIELD_NUMBER: builtins.int
+ CUDA_FIELD_NUMBER: builtins.int
+ ARGS_FIELD_NUMBER: builtins.int
+ STATE_FIELD_NUMBER: builtins.int
+ PROGRAM_FIELD_NUMBER: builtins.int
+ CODE_PATH_FIELD_NUMBER: builtins.int
+ GIT_FIELD_NUMBER: builtins.int
+ EMAIL_FIELD_NUMBER: builtins.int
+ ROOT_FIELD_NUMBER: builtins.int
+ HOST_FIELD_NUMBER: builtins.int
+ USERNAME_FIELD_NUMBER: builtins.int
+ EXECUTABLE_FIELD_NUMBER: builtins.int
+ CODE_PATH_LOCAL_FIELD_NUMBER: builtins.int
+ COLAB_FIELD_NUMBER: builtins.int
+ CPU_COUNT_FIELD_NUMBER: builtins.int
+ CPU_COUNT_LOGICAL_FIELD_NUMBER: builtins.int
+ GPU_TYPE_FIELD_NUMBER: builtins.int
+ GPU_COUNT_FIELD_NUMBER: builtins.int
+ DISK_FIELD_NUMBER: builtins.int
+ MEMORY_FIELD_NUMBER: builtins.int
+ CPU_FIELD_NUMBER: builtins.int
+ GPU_APPLE_FIELD_NUMBER: builtins.int
+ GPU_NVIDIA_FIELD_NUMBER: builtins.int
+ GPU_AMD_FIELD_NUMBER: builtins.int
+ SLURM_FIELD_NUMBER: builtins.int
+ os: builtins.str
+ python: builtins.str
+ @property
+ def heartbeatAt(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ @property
+ def startedAt(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
+ docker: builtins.str
+ cuda: builtins.str
+ @property
+ def args(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ state: builtins.str
+ program: builtins.str
+ code_path: builtins.str
+ @property
+ def git(self) -> global___GitRepoRecord: ...
+ email: builtins.str
+ root: builtins.str
+ host: builtins.str
+ username: builtins.str
+ executable: builtins.str
+ code_path_local: builtins.str
+ colab: builtins.str
+ cpu_count: builtins.int
+ cpu_count_logical: builtins.int
+ gpu_type: builtins.str
+ gpu_count: builtins.int
+ @property
+ def disk(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___DiskInfo]: ...
+ @property
+ def memory(self) -> global___MemoryInfo: ...
+ @property
+ def cpu(self) -> global___CpuInfo: ...
+ @property
+ def gpu_apple(self) -> global___GpuAppleInfo: ...
+ @property
+ def gpu_nvidia(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___GpuNvidiaInfo]: ...
+ @property
+ def gpu_amd(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___GpuAmdInfo]: ...
+ @property
+ def slurm(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ...
+ def __init__(
+ self,
+ *,
+ os: builtins.str = ...,
+ python: builtins.str = ...,
+ heartbeatAt: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ startedAt: google.protobuf.timestamp_pb2.Timestamp | None = ...,
+ docker: builtins.str = ...,
+ cuda: builtins.str = ...,
+ args: collections.abc.Iterable[builtins.str] | None = ...,
+ state: builtins.str = ...,
+ program: builtins.str = ...,
+ code_path: builtins.str = ...,
+ git: global___GitRepoRecord | None = ...,
+ email: builtins.str = ...,
+ root: builtins.str = ...,
+ host: builtins.str = ...,
+ username: builtins.str = ...,
+ executable: builtins.str = ...,
+ code_path_local: builtins.str = ...,
+ colab: builtins.str = ...,
+ cpu_count: builtins.int = ...,
+ cpu_count_logical: builtins.int = ...,
+ gpu_type: builtins.str = ...,
+ gpu_count: builtins.int = ...,
+ disk: collections.abc.Mapping[builtins.str, global___DiskInfo] | None = ...,
+ memory: global___MemoryInfo | None = ...,
+ cpu: global___CpuInfo | None = ...,
+ gpu_apple: global___GpuAppleInfo | None = ...,
+ gpu_nvidia: collections.abc.Iterable[global___GpuNvidiaInfo] | None = ...,
+ gpu_amd: collections.abc.Iterable[global___GpuAmdInfo] | None = ...,
+ slurm: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["cpu", b"cpu", "git", b"git", "gpu_apple", b"gpu_apple", "heartbeatAt", b"heartbeatAt", "memory", b"memory", "startedAt", b"startedAt"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["args", b"args", "code_path", b"code_path", "code_path_local", b"code_path_local", "colab", b"colab", "cpu", b"cpu", "cpu_count", b"cpu_count", "cpu_count_logical", b"cpu_count_logical", "cuda", b"cuda", "disk", b"disk", "docker", b"docker", "email", b"email", "executable", b"executable", "git", b"git", "gpu_amd", b"gpu_amd", "gpu_apple", b"gpu_apple", "gpu_count", b"gpu_count", "gpu_nvidia", b"gpu_nvidia", "gpu_type", b"gpu_type", "heartbeatAt", b"heartbeatAt", "host", b"host", "memory", b"memory", "os", b"os", "program", b"program", "python", b"python", "root", b"root", "slurm", b"slurm", "startedAt", b"startedAt", "state", b"state", "username", b"username"]) -> None: ...
+
+global___MetadataRequest = MetadataRequest
+
+@typing_extensions.final
+class PythonPackagesRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ @typing_extensions.final
+ class PythonPackage(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ NAME_FIELD_NUMBER: builtins.int
+ VERSION_FIELD_NUMBER: builtins.int
+ name: builtins.str
+ version: builtins.str
+ def __init__(
+ self,
+ *,
+ name: builtins.str = ...,
+ version: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["name", b"name", "version", b"version"]) -> None: ...
+
+ PACKAGE_FIELD_NUMBER: builtins.int
+ @property
+ def package(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___PythonPackagesRequest.PythonPackage]: ...
+ def __init__(
+ self,
+ *,
+ package: collections.abc.Iterable[global___PythonPackagesRequest.PythonPackage] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["package", b"package"]) -> None: ...
+
+global___PythonPackagesRequest = PythonPackagesRequest
diff --git a/wandb/proto/v4/wandb_server_pb2.py b/wandb/proto/v4/wandb_server_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..0fc31368d57f73c4c5cde8ea3898414bc859ed74
--- /dev/null
+++ b/wandb/proto/v4/wandb_server_pb2.py
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: wandb/proto/wandb_server.proto
+"""Generated protocol buffer code."""
+from google.protobuf.internal import builder as _builder
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from wandb.proto import wandb_base_pb2 as wandb_dot_proto_dot_wandb__base__pb2
+from wandb.proto import wandb_internal_pb2 as wandb_dot_proto_dot_wandb__internal__pb2
+from wandb.proto import wandb_settings_pb2 as wandb_dot_proto_dot_wandb__settings__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1ewandb/proto/wandb_server.proto\x12\x0ewandb_internal\x1a\x1cwandb/proto/wandb_base.proto\x1a wandb/proto/wandb_internal.proto\x1a wandb/proto/wandb_settings.proto\"D\n\x15ServerShutdownRequest\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x18\n\x16ServerShutdownResponse\"B\n\x13ServerStatusRequest\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x16\n\x14ServerStatusResponse\"r\n\x17ServerInformInitRequest\x12*\n\x08settings\x18\x01 \x01(\x0b\x32\x18.wandb_internal.Settings\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x1a\n\x18ServerInformInitResponse\"s\n\x18ServerInformStartRequest\x12*\n\x08settings\x18\x01 \x01(\x0b\x32\x18.wandb_internal.Settings\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x1b\n\x19ServerInformStartResponse\"H\n\x19ServerInformFinishRequest\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x1c\n\x1aServerInformFinishResponse\"H\n\x19ServerInformAttachRequest\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"u\n\x1aServerInformAttachResponse\x12*\n\x08settings\x18\x01 \x01(\x0b\x32\x18.wandb_internal.Settings\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"H\n\x19ServerInformDetachRequest\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x1c\n\x1aServerInformDetachResponse\"]\n\x1bServerInformTeardownRequest\x12\x11\n\texit_code\x18\x01 \x01(\x05\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x1e\n\x1cServerInformTeardownResponse\"\xa4\x04\n\rServerRequest\x12\x30\n\x0erecord_publish\x18\x01 \x01(\x0b\x32\x16.wandb_internal.RecordH\x00\x12\x34\n\x12record_communicate\x18\x02 \x01(\x0b\x32\x16.wandb_internal.RecordH\x00\x12>\n\x0binform_init\x18\x03 \x01(\x0b\x32\'.wandb_internal.ServerInformInitRequestH\x00\x12\x42\n\rinform_finish\x18\x04 \x01(\x0b\x32).wandb_internal.ServerInformFinishRequestH\x00\x12\x42\n\rinform_attach\x18\x05 \x01(\x0b\x32).wandb_internal.ServerInformAttachRequestH\x00\x12\x42\n\rinform_detach\x18\x06 \x01(\x0b\x32).wandb_internal.ServerInformDetachRequestH\x00\x12\x46\n\x0finform_teardown\x18\x07 \x01(\x0b\x32+.wandb_internal.ServerInformTeardownRequestH\x00\x12@\n\x0cinform_start\x18\x08 \x01(\x0b\x32(.wandb_internal.ServerInformStartRequestH\x00\x42\x15\n\x13server_request_type\"\xb0\x04\n\x0eServerResponse\x12\x34\n\x12result_communicate\x18\x02 \x01(\x0b\x32\x16.wandb_internal.ResultH\x00\x12H\n\x14inform_init_response\x18\x03 \x01(\x0b\x32(.wandb_internal.ServerInformInitResponseH\x00\x12L\n\x16inform_finish_response\x18\x04 \x01(\x0b\x32*.wandb_internal.ServerInformFinishResponseH\x00\x12L\n\x16inform_attach_response\x18\x05 \x01(\x0b\x32*.wandb_internal.ServerInformAttachResponseH\x00\x12L\n\x16inform_detach_response\x18\x06 \x01(\x0b\x32*.wandb_internal.ServerInformDetachResponseH\x00\x12P\n\x18inform_teardown_response\x18\x07 \x01(\x0b\x32,.wandb_internal.ServerInformTeardownResponseH\x00\x12J\n\x15inform_start_response\x18\x08 \x01(\x0b\x32).wandb_internal.ServerInformStartResponseH\x00\x42\x16\n\x14server_response_typeb\x06proto3')
+
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'wandb.proto.wandb_server_pb2', globals())
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+ DESCRIPTOR._options = None
+ _SERVERSHUTDOWNREQUEST._serialized_start=148
+ _SERVERSHUTDOWNREQUEST._serialized_end=216
+ _SERVERSHUTDOWNRESPONSE._serialized_start=218
+ _SERVERSHUTDOWNRESPONSE._serialized_end=242
+ _SERVERSTATUSREQUEST._serialized_start=244
+ _SERVERSTATUSREQUEST._serialized_end=310
+ _SERVERSTATUSRESPONSE._serialized_start=312
+ _SERVERSTATUSRESPONSE._serialized_end=334
+ _SERVERINFORMINITREQUEST._serialized_start=336
+ _SERVERINFORMINITREQUEST._serialized_end=450
+ _SERVERINFORMINITRESPONSE._serialized_start=452
+ _SERVERINFORMINITRESPONSE._serialized_end=478
+ _SERVERINFORMSTARTREQUEST._serialized_start=480
+ _SERVERINFORMSTARTREQUEST._serialized_end=595
+ _SERVERINFORMSTARTRESPONSE._serialized_start=597
+ _SERVERINFORMSTARTRESPONSE._serialized_end=624
+ _SERVERINFORMFINISHREQUEST._serialized_start=626
+ _SERVERINFORMFINISHREQUEST._serialized_end=698
+ _SERVERINFORMFINISHRESPONSE._serialized_start=700
+ _SERVERINFORMFINISHRESPONSE._serialized_end=728
+ _SERVERINFORMATTACHREQUEST._serialized_start=730
+ _SERVERINFORMATTACHREQUEST._serialized_end=802
+ _SERVERINFORMATTACHRESPONSE._serialized_start=804
+ _SERVERINFORMATTACHRESPONSE._serialized_end=921
+ _SERVERINFORMDETACHREQUEST._serialized_start=923
+ _SERVERINFORMDETACHREQUEST._serialized_end=995
+ _SERVERINFORMDETACHRESPONSE._serialized_start=997
+ _SERVERINFORMDETACHRESPONSE._serialized_end=1025
+ _SERVERINFORMTEARDOWNREQUEST._serialized_start=1027
+ _SERVERINFORMTEARDOWNREQUEST._serialized_end=1120
+ _SERVERINFORMTEARDOWNRESPONSE._serialized_start=1122
+ _SERVERINFORMTEARDOWNRESPONSE._serialized_end=1152
+ _SERVERREQUEST._serialized_start=1155
+ _SERVERREQUEST._serialized_end=1703
+ _SERVERRESPONSE._serialized_start=1706
+ _SERVERRESPONSE._serialized_end=2266
+# @@protoc_insertion_point(module_scope)
diff --git a/wandb/proto/v4/wandb_server_pb2.pyi b/wandb/proto/v4/wandb_server_pb2.pyi
new file mode 100644
index 0000000000000000000000000000000000000000..01f52f9454dc65cd681d1eb312952e8d90c12087
--- /dev/null
+++ b/wandb/proto/v4/wandb_server_pb2.pyi
@@ -0,0 +1,348 @@
+"""
+@generated by mypy-protobuf. Do not edit manually!
+isort:skip_file
+"""
+import builtins
+import google.protobuf.descriptor
+import google.protobuf.message
+import sys
+import wandb.proto.wandb_base_pb2
+import wandb.proto.wandb_internal_pb2
+import wandb.proto.wandb_settings_pb2
+
+if sys.version_info >= (3, 8):
+ import typing as typing_extensions
+else:
+ import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+@typing_extensions.final
+class ServerShutdownRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ServerShutdownRequest = ServerShutdownRequest
+
+@typing_extensions.final
+class ServerShutdownResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ServerShutdownResponse = ServerShutdownResponse
+
+@typing_extensions.final
+class ServerStatusRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ServerStatusRequest = ServerStatusRequest
+
+@typing_extensions.final
+class ServerStatusResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ServerStatusResponse = ServerStatusResponse
+
+@typing_extensions.final
+class ServerInformInitRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SETTINGS_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def settings(self) -> wandb.proto.wandb_settings_pb2.Settings: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ settings: wandb.proto.wandb_settings_pb2.Settings | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "settings", b"settings"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "settings", b"settings"]) -> None: ...
+
+global___ServerInformInitRequest = ServerInformInitRequest
+
+@typing_extensions.final
+class ServerInformInitResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ServerInformInitResponse = ServerInformInitResponse
+
+@typing_extensions.final
+class ServerInformStartRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SETTINGS_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def settings(self) -> wandb.proto.wandb_settings_pb2.Settings: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ settings: wandb.proto.wandb_settings_pb2.Settings | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "settings", b"settings"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "settings", b"settings"]) -> None: ...
+
+global___ServerInformStartRequest = ServerInformStartRequest
+
+@typing_extensions.final
+class ServerInformStartResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ServerInformStartResponse = ServerInformStartResponse
+
+@typing_extensions.final
+class ServerInformFinishRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ServerInformFinishRequest = ServerInformFinishRequest
+
+@typing_extensions.final
+class ServerInformFinishResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ServerInformFinishResponse = ServerInformFinishResponse
+
+@typing_extensions.final
+class ServerInformAttachRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ServerInformAttachRequest = ServerInformAttachRequest
+
+@typing_extensions.final
+class ServerInformAttachResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SETTINGS_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def settings(self) -> wandb.proto.wandb_settings_pb2.Settings: ...
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ settings: wandb.proto.wandb_settings_pb2.Settings | None = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "settings", b"settings"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "settings", b"settings"]) -> None: ...
+
+global___ServerInformAttachResponse = ServerInformAttachResponse
+
+@typing_extensions.final
+class ServerInformDetachRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> None: ...
+
+global___ServerInformDetachRequest = ServerInformDetachRequest
+
+@typing_extensions.final
+class ServerInformDetachResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ServerInformDetachResponse = ServerInformDetachResponse
+
+@typing_extensions.final
+class ServerInformTeardownRequest(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ EXIT_CODE_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ exit_code: builtins.int
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ exit_code: builtins.int = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "exit_code", b"exit_code"]) -> None: ...
+
+global___ServerInformTeardownRequest = ServerInformTeardownRequest
+
+@typing_extensions.final
+class ServerInformTeardownResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___ServerInformTeardownResponse = ServerInformTeardownResponse
+
+@typing_extensions.final
+class ServerRequest(google.protobuf.message.Message):
+ """
+ ServerRequest, ServerResponse: used in sock server
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RECORD_PUBLISH_FIELD_NUMBER: builtins.int
+ RECORD_COMMUNICATE_FIELD_NUMBER: builtins.int
+ INFORM_INIT_FIELD_NUMBER: builtins.int
+ INFORM_FINISH_FIELD_NUMBER: builtins.int
+ INFORM_ATTACH_FIELD_NUMBER: builtins.int
+ INFORM_DETACH_FIELD_NUMBER: builtins.int
+ INFORM_TEARDOWN_FIELD_NUMBER: builtins.int
+ INFORM_START_FIELD_NUMBER: builtins.int
+ @property
+ def record_publish(self) -> wandb.proto.wandb_internal_pb2.Record: ...
+ @property
+ def record_communicate(self) -> wandb.proto.wandb_internal_pb2.Record: ...
+ @property
+ def inform_init(self) -> global___ServerInformInitRequest: ...
+ @property
+ def inform_finish(self) -> global___ServerInformFinishRequest: ...
+ @property
+ def inform_attach(self) -> global___ServerInformAttachRequest: ...
+ @property
+ def inform_detach(self) -> global___ServerInformDetachRequest: ...
+ @property
+ def inform_teardown(self) -> global___ServerInformTeardownRequest: ...
+ @property
+ def inform_start(self) -> global___ServerInformStartRequest: ...
+ def __init__(
+ self,
+ *,
+ record_publish: wandb.proto.wandb_internal_pb2.Record | None = ...,
+ record_communicate: wandb.proto.wandb_internal_pb2.Record | None = ...,
+ inform_init: global___ServerInformInitRequest | None = ...,
+ inform_finish: global___ServerInformFinishRequest | None = ...,
+ inform_attach: global___ServerInformAttachRequest | None = ...,
+ inform_detach: global___ServerInformDetachRequest | None = ...,
+ inform_teardown: global___ServerInformTeardownRequest | None = ...,
+ inform_start: global___ServerInformStartRequest | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["inform_attach", b"inform_attach", "inform_detach", b"inform_detach", "inform_finish", b"inform_finish", "inform_init", b"inform_init", "inform_start", b"inform_start", "inform_teardown", b"inform_teardown", "record_communicate", b"record_communicate", "record_publish", b"record_publish", "server_request_type", b"server_request_type"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["inform_attach", b"inform_attach", "inform_detach", b"inform_detach", "inform_finish", b"inform_finish", "inform_init", b"inform_init", "inform_start", b"inform_start", "inform_teardown", b"inform_teardown", "record_communicate", b"record_communicate", "record_publish", b"record_publish", "server_request_type", b"server_request_type"]) -> None: ...
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["server_request_type", b"server_request_type"]) -> typing_extensions.Literal["record_publish", "record_communicate", "inform_init", "inform_finish", "inform_attach", "inform_detach", "inform_teardown", "inform_start"] | None: ...
+
+global___ServerRequest = ServerRequest
+
+@typing_extensions.final
+class ServerResponse(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ RESULT_COMMUNICATE_FIELD_NUMBER: builtins.int
+ INFORM_INIT_RESPONSE_FIELD_NUMBER: builtins.int
+ INFORM_FINISH_RESPONSE_FIELD_NUMBER: builtins.int
+ INFORM_ATTACH_RESPONSE_FIELD_NUMBER: builtins.int
+ INFORM_DETACH_RESPONSE_FIELD_NUMBER: builtins.int
+ INFORM_TEARDOWN_RESPONSE_FIELD_NUMBER: builtins.int
+ INFORM_START_RESPONSE_FIELD_NUMBER: builtins.int
+ @property
+ def result_communicate(self) -> wandb.proto.wandb_internal_pb2.Result: ...
+ @property
+ def inform_init_response(self) -> global___ServerInformInitResponse: ...
+ @property
+ def inform_finish_response(self) -> global___ServerInformFinishResponse: ...
+ @property
+ def inform_attach_response(self) -> global___ServerInformAttachResponse: ...
+ @property
+ def inform_detach_response(self) -> global___ServerInformDetachResponse: ...
+ @property
+ def inform_teardown_response(self) -> global___ServerInformTeardownResponse: ...
+ @property
+ def inform_start_response(self) -> global___ServerInformStartResponse: ...
+ def __init__(
+ self,
+ *,
+ result_communicate: wandb.proto.wandb_internal_pb2.Result | None = ...,
+ inform_init_response: global___ServerInformInitResponse | None = ...,
+ inform_finish_response: global___ServerInformFinishResponse | None = ...,
+ inform_attach_response: global___ServerInformAttachResponse | None = ...,
+ inform_detach_response: global___ServerInformDetachResponse | None = ...,
+ inform_teardown_response: global___ServerInformTeardownResponse | None = ...,
+ inform_start_response: global___ServerInformStartResponse | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["inform_attach_response", b"inform_attach_response", "inform_detach_response", b"inform_detach_response", "inform_finish_response", b"inform_finish_response", "inform_init_response", b"inform_init_response", "inform_start_response", b"inform_start_response", "inform_teardown_response", b"inform_teardown_response", "result_communicate", b"result_communicate", "server_response_type", b"server_response_type"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["inform_attach_response", b"inform_attach_response", "inform_detach_response", b"inform_detach_response", "inform_finish_response", b"inform_finish_response", "inform_init_response", b"inform_init_response", "inform_start_response", b"inform_start_response", "inform_teardown_response", b"inform_teardown_response", "result_communicate", b"result_communicate", "server_response_type", b"server_response_type"]) -> None: ...
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["server_response_type", b"server_response_type"]) -> typing_extensions.Literal["result_communicate", "inform_init_response", "inform_finish_response", "inform_attach_response", "inform_detach_response", "inform_teardown_response", "inform_start_response"] | None: ...
+
+global___ServerResponse = ServerResponse
diff --git a/wandb/proto/v4/wandb_settings_pb2.py b/wandb/proto/v4/wandb_settings_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..5b09a5d4cb599a3823406ba8e99f96f2dc4dad69
--- /dev/null
+++ b/wandb/proto/v4/wandb_settings_pb2.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: wandb/proto/wandb_settings.proto
+"""Generated protocol buffer code."""
+from google.protobuf.internal import builder as _builder
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n wandb/proto/wandb_settings.proto\x12\x0ewandb_internal\x1a\x1egoogle/protobuf/wrappers.proto\" \n\x0fListStringValue\x12\r\n\x05value\x18\x01 \x03(\t\"\x8a\x01\n\x17MapStringKeyStringValue\x12\x41\n\x05value\x18\x01 \x03(\x0b\x32\x32.wandb_internal.MapStringKeyStringValue.ValueEntry\x1a,\n\nValueEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xcb\x01\n#MapStringKeyMapStringKeyStringValue\x12M\n\x05value\x18\x01 \x03(\x0b\x32>.wandb_internal.MapStringKeyMapStringKeyStringValue.ValueEntry\x1aU\n\nValueEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x36\n\x05value\x18\x02 \x01(\x0b\x32\'.wandb_internal.MapStringKeyStringValue:\x02\x38\x01\"\x9a\x01\n\x12OpenMetricsFilters\x12\x33\n\x08sequence\x18\x01 \x01(\x0b\x32\x1f.wandb_internal.ListStringValueH\x00\x12\x46\n\x07mapping\x18\x02 \x01(\x0b\x32\x33.wandb_internal.MapStringKeyMapStringKeyStringValueH\x00\x42\x07\n\x05value\"\xe1\x43\n\x08Settings\x12.\n\x05_args\x18\x01 \x01(\x0b\x32\x1f.wandb_internal.ListStringValue\x12/\n\x0b_aws_lambda\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x44\n\x1f_async_upload_concurrency_limit\x18\x03 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12\x32\n\x0e_cli_only_mode\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12*\n\x06_colab\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12+\n\x05_cuda\x18\x06 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x31\n\r_disable_meta\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x34\n\x10_disable_service\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x39\n\x15_disable_setproctitle\x18\t \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x32\n\x0e_disable_stats\x18\n \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x33\n\x0f_disable_viewer\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x30\n\x0c_except_exit\x18\x0c \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x31\n\x0b_executable\x18\r \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x44\n\x13_extra_http_headers\x18\x0e \x01(\x0b\x32\'.wandb_internal.MapStringKeyStringValue\x12\x42\n\x1c_file_stream_timeout_seconds\x18\x0f \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x38\n\x14_flow_control_custom\x18\x10 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12:\n\x16_flow_control_disabled\x18\x11 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12=\n\x17_internal_check_process\x18\x12 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12=\n\x17_internal_queue_timeout\x18\x13 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12,\n\x08_ipython\x18\x14 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12,\n\x08_jupyter\x18\x15 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x33\n\r_jupyter_root\x18\x16 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12+\n\x07_kaggle\x18\x17 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12<\n\x17_live_policy_rate_limit\x18\x18 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12;\n\x16_live_policy_wait_time\x18\x19 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12/\n\n_log_level\x18\x1a \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12\x34\n\x0f_network_buffer\x18\x1b \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12)\n\x05_noop\x18\x1c \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12-\n\t_notebook\x18\x1d \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12,\n\x08_offline\x18\x1e \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12)\n\x05_sync\x18\x1f \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12)\n\x03_os\x18 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\t_platform\x18! \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x07_python\x18\" \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x37\n\x11_runqueue_item_id\x18# \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x31\n\r_require_core\x18$ \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x36\n\x12_save_requirements\x18% \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x38\n\x12_service_transport\x18& \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x33\n\r_service_wait\x18\' \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x35\n\x0f_start_datetime\x18( \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x31\n\x0b_start_time\x18) \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12/\n\n_stats_pid\x18* \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12@\n\x1a_stats_sample_rate_seconds\x18+ \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12>\n\x19_stats_samples_to_average\x18, \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12\x36\n\x12_stats_join_assets\x18- \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12G\n!_stats_neuron_monitor_config_path\x18. \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12N\n\x1d_stats_open_metrics_endpoints\x18/ \x01(\x0b\x32\'.wandb_internal.MapStringKeyStringValue\x12G\n\x1b_stats_open_metrics_filters\x18\x30 \x01(\x0b\x32\".wandb_internal.OpenMetricsFilters\x12\x33\n\r_tmp_code_dir\x18\x31 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\t_tracelog\x18\x32 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x36\n\r_unsaved_keys\x18\x33 \x01(\x0b\x32\x1f.wandb_internal.ListStringValue\x12,\n\x08_windows\x18\x34 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x34\n\x10\x61llow_val_change\x18\x35 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12/\n\tanonymous\x18\x36 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x07\x61pi_key\x18\x37 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12P\n\x1f\x61zure_account_url_to_access_key\x18\x38 \x01(\x0b\x32\'.wandb_internal.MapStringKeyStringValue\x12.\n\x08\x62\x61se_url\x18\x39 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12.\n\x08\x63ode_dir\x18: \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x35\n\x0c\x63onfig_paths\x18; \x01(\x0b\x32\x1f.wandb_internal.ListStringValue\x12-\n\x07\x63onsole\x18< \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x30\n\ndeployment\x18= \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x30\n\x0c\x64isable_code\x18> \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12/\n\x0b\x64isable_git\x18? \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x31\n\rdisable_hints\x18@ \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x38\n\x14\x64isable_job_creation\x18\x41 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12,\n\x08\x64isabled\x18\x42 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12,\n\x06\x64ocker\x18\x43 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12+\n\x05\x65mail\x18\x44 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12,\n\x06\x65ntity\x18\x45 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\tfiles_dir\x18\x46 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12)\n\x05\x66orce\x18G \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x30\n\ngit_commit\x18H \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x30\n\ngit_remote\x18I \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x34\n\x0egit_remote_url\x18J \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12.\n\x08git_root\x18K \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x36\n\x11heartbeat_seconds\x18L \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12*\n\x04host\x18M \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x35\n\x0cignore_globs\x18N \x01(\x0b\x32\x1f.wandb_internal.ListStringValue\x12\x32\n\x0cinit_timeout\x18O \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12,\n\x08is_local\x18P \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x30\n\njob_source\x18Q \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x31\n\rlabel_disable\x18R \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12*\n\x06launch\x18S \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x38\n\x12launch_config_path\x18T \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x07log_dir\x18U \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x32\n\x0clog_internal\x18V \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12:\n\x14log_symlink_internal\x18W \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x36\n\x10log_symlink_user\x18X \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12.\n\x08log_user\x18Y \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x33\n\rlogin_timeout\x18Z \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12*\n\x04mode\x18\\ \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x33\n\rnotebook_name\x18] \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x07problem\x18^ \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x07program\x18_ \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x35\n\x0fprogram_relpath\x18` \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x07project\x18\x61 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x31\n\x0bproject_url\x18\x62 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12)\n\x05quiet\x18\x63 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12*\n\x06reinit\x18\x64 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12+\n\x07relogin\x18\x65 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12,\n\x06resume\x18\x66 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x32\n\x0cresume_fname\x18g \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12+\n\x07resumed\x18h \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12.\n\x08root_dir\x18i \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\trun_group\x18j \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12,\n\x06run_id\x18k \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x32\n\x0crun_job_type\x18l \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12.\n\x08run_mode\x18m \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12.\n\x08run_name\x18n \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\trun_notes\x18o \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x31\n\x08run_tags\x18p \x01(\x0b\x32\x1f.wandb_internal.ListStringValue\x12-\n\x07run_url\x18q \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x35\n\x11sagemaker_disable\x18r \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12-\n\tsave_code\x18s \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x35\n\x0fsettings_system\x18t \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x38\n\x12settings_workspace\x18u \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\x0bshow_colors\x18v \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12.\n\nshow_emoji\x18w \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12/\n\x0bshow_errors\x18x \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12-\n\tshow_info\x18y \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x31\n\rshow_warnings\x18z \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12*\n\x06silent\x18{ \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x32\n\x0cstart_method\x18| \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12*\n\x06strict\x18} \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x33\n\x0esummary_errors\x18~ \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12\x34\n\x0fsummary_timeout\x18\x7f \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12\x36\n\x10summary_warnings\x18\x80\x01 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12/\n\x08sweep_id\x18\x81\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x37\n\x10sweep_param_path\x18\x82\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x30\n\tsweep_url\x18\x83\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12,\n\x07symlink\x18\x84\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12/\n\x08sync_dir\x18\x85\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x30\n\tsync_file\x18\x86\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12:\n\x13sync_symlink_latest\x18\x87\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x33\n\rsystem_sample\x18\x88\x01 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12;\n\x15system_sample_seconds\x18\x89\x01 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12J\n%table_raise_on_max_row_limit_exceeded\x18\x8a\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12/\n\x08timespec\x18\x8b\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12.\n\x07tmp_dir\x18\x8c\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\x08username\x18\x8d\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x30\n\twandb_dir\x18\x8e\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x34\n\r_jupyter_name\x18\x8f\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x34\n\r_jupyter_path\x18\x90\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\x08job_name\x18\x91\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12;\n\x11_stats_disk_paths\x18\x92\x01 \x01(\x0b\x32\x1f.wandb_internal.ListStringValue\x12<\n\x16_file_stream_retry_max\x18\x93\x01 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12J\n#_file_stream_retry_wait_min_seconds\x18\x94\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12J\n#_file_stream_retry_wait_max_seconds\x18\x95\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12>\n\x18_file_transfer_retry_max\x18\x96\x01 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12L\n%_file_transfer_retry_wait_min_seconds\x18\x97\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12L\n%_file_transfer_retry_wait_max_seconds\x18\x98\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x45\n\x1e_file_transfer_timeout_seconds\x18\x99\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x38\n\x12_graphql_retry_max\x18\x9a\x01 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12\x46\n\x1f_graphql_retry_wait_min_seconds\x18\x9b\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x46\n\x1f_graphql_retry_wait_max_seconds\x18\x9c\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12?\n\x18_graphql_timeout_seconds\x18\x9d\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12:\n\x15_disable_machine_info\x18\x9e\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x36\n\x0fprogram_abspath\x18\x9f\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x30\n\tcolab_url\x18\xa0\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x38\n\x12_stats_buffer_size\x18\xa1\x01 \x01(\x0b\x32\x1b.google.protobuf.Int32Value\x12,\n\x07_shared\x18\xa2\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12:\n\x08_proxies\x18\xc8\x01 \x01(\x0b\x32\'.wandb_internal.MapStringKeyStringValueb\x06proto3')
+
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'wandb.proto.wandb_settings_pb2', globals())
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+ DESCRIPTOR._options = None
+ _MAPSTRINGKEYSTRINGVALUE_VALUEENTRY._options = None
+ _MAPSTRINGKEYSTRINGVALUE_VALUEENTRY._serialized_options = b'8\001'
+ _MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE_VALUEENTRY._options = None
+ _MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE_VALUEENTRY._serialized_options = b'8\001'
+ _LISTSTRINGVALUE._serialized_start=84
+ _LISTSTRINGVALUE._serialized_end=116
+ _MAPSTRINGKEYSTRINGVALUE._serialized_start=119
+ _MAPSTRINGKEYSTRINGVALUE._serialized_end=257
+ _MAPSTRINGKEYSTRINGVALUE_VALUEENTRY._serialized_start=213
+ _MAPSTRINGKEYSTRINGVALUE_VALUEENTRY._serialized_end=257
+ _MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE._serialized_start=260
+ _MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE._serialized_end=463
+ _MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE_VALUEENTRY._serialized_start=378
+ _MAPSTRINGKEYMAPSTRINGKEYSTRINGVALUE_VALUEENTRY._serialized_end=463
+ _OPENMETRICSFILTERS._serialized_start=466
+ _OPENMETRICSFILTERS._serialized_end=620
+ _SETTINGS._serialized_start=623
+ _SETTINGS._serialized_end=9296
+# @@protoc_insertion_point(module_scope)
diff --git a/wandb/proto/v4/wandb_settings_pb2.pyi b/wandb/proto/v4/wandb_settings_pb2.pyi
new file mode 100644
index 0000000000000000000000000000000000000000..231061ca3e286d32397f1faa5e4f08a13de85630
--- /dev/null
+++ b/wandb/proto/v4/wandb_settings_pb2.pyi
@@ -0,0 +1,783 @@
+"""
+@generated by mypy-protobuf. Do not edit manually!
+isort:skip_file
+"""
+import builtins
+import collections.abc
+import google.protobuf.descriptor
+import google.protobuf.internal.containers
+import google.protobuf.message
+import google.protobuf.wrappers_pb2
+import sys
+
+if sys.version_info >= (3, 8):
+ import typing as typing_extensions
+else:
+ import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+@typing_extensions.final
+class ListStringValue(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ VALUE_FIELD_NUMBER: builtins.int
+ @property
+ def value(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ def __init__(
+ self,
+ *,
+ value: collections.abc.Iterable[builtins.str] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["value", b"value"]) -> None: ...
+
+global___ListStringValue = ListStringValue
+
+@typing_extensions.final
+class MapStringKeyStringValue(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ @typing_extensions.final
+ class ValueEntry(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ value: builtins.str
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ...
+
+ VALUE_FIELD_NUMBER: builtins.int
+ @property
+ def value(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ...
+ def __init__(
+ self,
+ *,
+ value: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["value", b"value"]) -> None: ...
+
+global___MapStringKeyStringValue = MapStringKeyStringValue
+
+@typing_extensions.final
+class MapStringKeyMapStringKeyStringValue(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ @typing_extensions.final
+ class ValueEntry(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KEY_FIELD_NUMBER: builtins.int
+ VALUE_FIELD_NUMBER: builtins.int
+ key: builtins.str
+ @property
+ def value(self) -> global___MapStringKeyStringValue: ...
+ def __init__(
+ self,
+ *,
+ key: builtins.str = ...,
+ value: global___MapStringKeyStringValue | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["value", b"value"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ...
+
+ VALUE_FIELD_NUMBER: builtins.int
+ @property
+ def value(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___MapStringKeyStringValue]: ...
+ def __init__(
+ self,
+ *,
+ value: collections.abc.Mapping[builtins.str, global___MapStringKeyStringValue] | None = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["value", b"value"]) -> None: ...
+
+global___MapStringKeyMapStringKeyStringValue = MapStringKeyMapStringKeyStringValue
+
+@typing_extensions.final
+class OpenMetricsFilters(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SEQUENCE_FIELD_NUMBER: builtins.int
+ MAPPING_FIELD_NUMBER: builtins.int
+ @property
+ def sequence(self) -> global___ListStringValue: ...
+ @property
+ def mapping(self) -> global___MapStringKeyMapStringKeyStringValue: ...
+ def __init__(
+ self,
+ *,
+ sequence: global___ListStringValue | None = ...,
+ mapping: global___MapStringKeyMapStringKeyStringValue | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["mapping", b"mapping", "sequence", b"sequence", "value", b"value"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["mapping", b"mapping", "sequence", b"sequence", "value", b"value"]) -> None: ...
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["value", b"value"]) -> typing_extensions.Literal["sequence", "mapping"] | None: ...
+
+global___OpenMetricsFilters = OpenMetricsFilters
+
+@typing_extensions.final
+class Settings(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ _ARGS_FIELD_NUMBER: builtins.int
+ _AWS_LAMBDA_FIELD_NUMBER: builtins.int
+ _ASYNC_UPLOAD_CONCURRENCY_LIMIT_FIELD_NUMBER: builtins.int
+ _CLI_ONLY_MODE_FIELD_NUMBER: builtins.int
+ _COLAB_FIELD_NUMBER: builtins.int
+ _CUDA_FIELD_NUMBER: builtins.int
+ _DISABLE_META_FIELD_NUMBER: builtins.int
+ _DISABLE_SERVICE_FIELD_NUMBER: builtins.int
+ _DISABLE_SETPROCTITLE_FIELD_NUMBER: builtins.int
+ _DISABLE_STATS_FIELD_NUMBER: builtins.int
+ _DISABLE_VIEWER_FIELD_NUMBER: builtins.int
+ _EXCEPT_EXIT_FIELD_NUMBER: builtins.int
+ _EXECUTABLE_FIELD_NUMBER: builtins.int
+ _EXTRA_HTTP_HEADERS_FIELD_NUMBER: builtins.int
+ _FILE_STREAM_TIMEOUT_SECONDS_FIELD_NUMBER: builtins.int
+ _FLOW_CONTROL_CUSTOM_FIELD_NUMBER: builtins.int
+ _FLOW_CONTROL_DISABLED_FIELD_NUMBER: builtins.int
+ _INTERNAL_CHECK_PROCESS_FIELD_NUMBER: builtins.int
+ _INTERNAL_QUEUE_TIMEOUT_FIELD_NUMBER: builtins.int
+ _IPYTHON_FIELD_NUMBER: builtins.int
+ _JUPYTER_FIELD_NUMBER: builtins.int
+ _JUPYTER_ROOT_FIELD_NUMBER: builtins.int
+ _KAGGLE_FIELD_NUMBER: builtins.int
+ _LIVE_POLICY_RATE_LIMIT_FIELD_NUMBER: builtins.int
+ _LIVE_POLICY_WAIT_TIME_FIELD_NUMBER: builtins.int
+ _LOG_LEVEL_FIELD_NUMBER: builtins.int
+ _NETWORK_BUFFER_FIELD_NUMBER: builtins.int
+ _NOOP_FIELD_NUMBER: builtins.int
+ _NOTEBOOK_FIELD_NUMBER: builtins.int
+ _OFFLINE_FIELD_NUMBER: builtins.int
+ _SYNC_FIELD_NUMBER: builtins.int
+ _OS_FIELD_NUMBER: builtins.int
+ _PLATFORM_FIELD_NUMBER: builtins.int
+ _PYTHON_FIELD_NUMBER: builtins.int
+ _RUNQUEUE_ITEM_ID_FIELD_NUMBER: builtins.int
+ _REQUIRE_CORE_FIELD_NUMBER: builtins.int
+ _SAVE_REQUIREMENTS_FIELD_NUMBER: builtins.int
+ _SERVICE_TRANSPORT_FIELD_NUMBER: builtins.int
+ _SERVICE_WAIT_FIELD_NUMBER: builtins.int
+ _START_DATETIME_FIELD_NUMBER: builtins.int
+ _START_TIME_FIELD_NUMBER: builtins.int
+ _STATS_PID_FIELD_NUMBER: builtins.int
+ _STATS_SAMPLE_RATE_SECONDS_FIELD_NUMBER: builtins.int
+ _STATS_SAMPLES_TO_AVERAGE_FIELD_NUMBER: builtins.int
+ _STATS_JOIN_ASSETS_FIELD_NUMBER: builtins.int
+ _STATS_NEURON_MONITOR_CONFIG_PATH_FIELD_NUMBER: builtins.int
+ _STATS_OPEN_METRICS_ENDPOINTS_FIELD_NUMBER: builtins.int
+ _STATS_OPEN_METRICS_FILTERS_FIELD_NUMBER: builtins.int
+ _TMP_CODE_DIR_FIELD_NUMBER: builtins.int
+ _TRACELOG_FIELD_NUMBER: builtins.int
+ _UNSAVED_KEYS_FIELD_NUMBER: builtins.int
+ _WINDOWS_FIELD_NUMBER: builtins.int
+ ALLOW_VAL_CHANGE_FIELD_NUMBER: builtins.int
+ ANONYMOUS_FIELD_NUMBER: builtins.int
+ API_KEY_FIELD_NUMBER: builtins.int
+ AZURE_ACCOUNT_URL_TO_ACCESS_KEY_FIELD_NUMBER: builtins.int
+ BASE_URL_FIELD_NUMBER: builtins.int
+ CODE_DIR_FIELD_NUMBER: builtins.int
+ CONFIG_PATHS_FIELD_NUMBER: builtins.int
+ CONSOLE_FIELD_NUMBER: builtins.int
+ DEPLOYMENT_FIELD_NUMBER: builtins.int
+ DISABLE_CODE_FIELD_NUMBER: builtins.int
+ DISABLE_GIT_FIELD_NUMBER: builtins.int
+ DISABLE_HINTS_FIELD_NUMBER: builtins.int
+ DISABLE_JOB_CREATION_FIELD_NUMBER: builtins.int
+ DISABLED_FIELD_NUMBER: builtins.int
+ DOCKER_FIELD_NUMBER: builtins.int
+ EMAIL_FIELD_NUMBER: builtins.int
+ ENTITY_FIELD_NUMBER: builtins.int
+ FILES_DIR_FIELD_NUMBER: builtins.int
+ FORCE_FIELD_NUMBER: builtins.int
+ GIT_COMMIT_FIELD_NUMBER: builtins.int
+ GIT_REMOTE_FIELD_NUMBER: builtins.int
+ GIT_REMOTE_URL_FIELD_NUMBER: builtins.int
+ GIT_ROOT_FIELD_NUMBER: builtins.int
+ HEARTBEAT_SECONDS_FIELD_NUMBER: builtins.int
+ HOST_FIELD_NUMBER: builtins.int
+ IGNORE_GLOBS_FIELD_NUMBER: builtins.int
+ INIT_TIMEOUT_FIELD_NUMBER: builtins.int
+ IS_LOCAL_FIELD_NUMBER: builtins.int
+ JOB_SOURCE_FIELD_NUMBER: builtins.int
+ LABEL_DISABLE_FIELD_NUMBER: builtins.int
+ LAUNCH_FIELD_NUMBER: builtins.int
+ LAUNCH_CONFIG_PATH_FIELD_NUMBER: builtins.int
+ LOG_DIR_FIELD_NUMBER: builtins.int
+ LOG_INTERNAL_FIELD_NUMBER: builtins.int
+ LOG_SYMLINK_INTERNAL_FIELD_NUMBER: builtins.int
+ LOG_SYMLINK_USER_FIELD_NUMBER: builtins.int
+ LOG_USER_FIELD_NUMBER: builtins.int
+ LOGIN_TIMEOUT_FIELD_NUMBER: builtins.int
+ MODE_FIELD_NUMBER: builtins.int
+ NOTEBOOK_NAME_FIELD_NUMBER: builtins.int
+ PROBLEM_FIELD_NUMBER: builtins.int
+ PROGRAM_FIELD_NUMBER: builtins.int
+ PROGRAM_RELPATH_FIELD_NUMBER: builtins.int
+ PROJECT_FIELD_NUMBER: builtins.int
+ PROJECT_URL_FIELD_NUMBER: builtins.int
+ QUIET_FIELD_NUMBER: builtins.int
+ REINIT_FIELD_NUMBER: builtins.int
+ RELOGIN_FIELD_NUMBER: builtins.int
+ RESUME_FIELD_NUMBER: builtins.int
+ RESUME_FNAME_FIELD_NUMBER: builtins.int
+ RESUMED_FIELD_NUMBER: builtins.int
+ ROOT_DIR_FIELD_NUMBER: builtins.int
+ RUN_GROUP_FIELD_NUMBER: builtins.int
+ RUN_ID_FIELD_NUMBER: builtins.int
+ RUN_JOB_TYPE_FIELD_NUMBER: builtins.int
+ RUN_MODE_FIELD_NUMBER: builtins.int
+ RUN_NAME_FIELD_NUMBER: builtins.int
+ RUN_NOTES_FIELD_NUMBER: builtins.int
+ RUN_TAGS_FIELD_NUMBER: builtins.int
+ RUN_URL_FIELD_NUMBER: builtins.int
+ SAGEMAKER_DISABLE_FIELD_NUMBER: builtins.int
+ SAVE_CODE_FIELD_NUMBER: builtins.int
+ SETTINGS_SYSTEM_FIELD_NUMBER: builtins.int
+ SETTINGS_WORKSPACE_FIELD_NUMBER: builtins.int
+ SHOW_COLORS_FIELD_NUMBER: builtins.int
+ SHOW_EMOJI_FIELD_NUMBER: builtins.int
+ SHOW_ERRORS_FIELD_NUMBER: builtins.int
+ SHOW_INFO_FIELD_NUMBER: builtins.int
+ SHOW_WARNINGS_FIELD_NUMBER: builtins.int
+ SILENT_FIELD_NUMBER: builtins.int
+ START_METHOD_FIELD_NUMBER: builtins.int
+ STRICT_FIELD_NUMBER: builtins.int
+ SUMMARY_ERRORS_FIELD_NUMBER: builtins.int
+ SUMMARY_TIMEOUT_FIELD_NUMBER: builtins.int
+ SUMMARY_WARNINGS_FIELD_NUMBER: builtins.int
+ SWEEP_ID_FIELD_NUMBER: builtins.int
+ SWEEP_PARAM_PATH_FIELD_NUMBER: builtins.int
+ SWEEP_URL_FIELD_NUMBER: builtins.int
+ SYMLINK_FIELD_NUMBER: builtins.int
+ SYNC_DIR_FIELD_NUMBER: builtins.int
+ SYNC_FILE_FIELD_NUMBER: builtins.int
+ SYNC_SYMLINK_LATEST_FIELD_NUMBER: builtins.int
+ SYSTEM_SAMPLE_FIELD_NUMBER: builtins.int
+ SYSTEM_SAMPLE_SECONDS_FIELD_NUMBER: builtins.int
+ TABLE_RAISE_ON_MAX_ROW_LIMIT_EXCEEDED_FIELD_NUMBER: builtins.int
+ TIMESPEC_FIELD_NUMBER: builtins.int
+ TMP_DIR_FIELD_NUMBER: builtins.int
+ USERNAME_FIELD_NUMBER: builtins.int
+ WANDB_DIR_FIELD_NUMBER: builtins.int
+ _JUPYTER_NAME_FIELD_NUMBER: builtins.int
+ _JUPYTER_PATH_FIELD_NUMBER: builtins.int
+ JOB_NAME_FIELD_NUMBER: builtins.int
+ _STATS_DISK_PATHS_FIELD_NUMBER: builtins.int
+ _FILE_STREAM_RETRY_MAX_FIELD_NUMBER: builtins.int
+ _FILE_STREAM_RETRY_WAIT_MIN_SECONDS_FIELD_NUMBER: builtins.int
+ _FILE_STREAM_RETRY_WAIT_MAX_SECONDS_FIELD_NUMBER: builtins.int
+ _FILE_TRANSFER_RETRY_MAX_FIELD_NUMBER: builtins.int
+ _FILE_TRANSFER_RETRY_WAIT_MIN_SECONDS_FIELD_NUMBER: builtins.int
+ _FILE_TRANSFER_RETRY_WAIT_MAX_SECONDS_FIELD_NUMBER: builtins.int
+ _FILE_TRANSFER_TIMEOUT_SECONDS_FIELD_NUMBER: builtins.int
+ _GRAPHQL_RETRY_MAX_FIELD_NUMBER: builtins.int
+ _GRAPHQL_RETRY_WAIT_MIN_SECONDS_FIELD_NUMBER: builtins.int
+ _GRAPHQL_RETRY_WAIT_MAX_SECONDS_FIELD_NUMBER: builtins.int
+ _GRAPHQL_TIMEOUT_SECONDS_FIELD_NUMBER: builtins.int
+ _DISABLE_MACHINE_INFO_FIELD_NUMBER: builtins.int
+ PROGRAM_ABSPATH_FIELD_NUMBER: builtins.int
+ COLAB_URL_FIELD_NUMBER: builtins.int
+ _STATS_BUFFER_SIZE_FIELD_NUMBER: builtins.int
+ _SHARED_FIELD_NUMBER: builtins.int
+ _PROXIES_FIELD_NUMBER: builtins.int
+ @property
+ def _args(self) -> global___ListStringValue: ...
+ @property
+ def _aws_lambda(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _async_upload_concurrency_limit(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _cli_only_mode(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _colab(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _cuda(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _disable_meta(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _disable_service(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _disable_setproctitle(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _disable_stats(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _disable_viewer(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _except_exit(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _executable(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _extra_http_headers(self) -> global___MapStringKeyStringValue: ...
+ @property
+ def _file_stream_timeout_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _flow_control_custom(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _flow_control_disabled(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _internal_check_process(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _internal_queue_timeout(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _ipython(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _jupyter(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _jupyter_root(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _kaggle(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _live_policy_rate_limit(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _live_policy_wait_time(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _log_level(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _network_buffer(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _noop(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _notebook(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _offline(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _sync(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _os(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _platform(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _python(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _runqueue_item_id(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _require_core(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _save_requirements(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _service_transport(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _service_wait(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _start_datetime(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _start_time(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _stats_pid(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _stats_sample_rate_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _stats_samples_to_average(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _stats_join_assets(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _stats_neuron_monitor_config_path(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _stats_open_metrics_endpoints(self) -> global___MapStringKeyStringValue: ...
+ @property
+ def _stats_open_metrics_filters(self) -> global___OpenMetricsFilters: ...
+ @property
+ def _tmp_code_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _tracelog(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _unsaved_keys(self) -> global___ListStringValue: ...
+ @property
+ def _windows(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def allow_val_change(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def anonymous(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def api_key(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def azure_account_url_to_access_key(self) -> global___MapStringKeyStringValue: ...
+ @property
+ def base_url(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def code_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def config_paths(self) -> global___ListStringValue: ...
+ @property
+ def console(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def deployment(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def disable_code(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def disable_git(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def disable_hints(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def disable_job_creation(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def disabled(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def docker(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def email(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def entity(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def files_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def force(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def git_commit(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def git_remote(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def git_remote_url(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def git_root(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def heartbeat_seconds(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def host(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def ignore_globs(self) -> global___ListStringValue: ...
+ @property
+ def init_timeout(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def is_local(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def job_source(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def label_disable(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def launch(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def launch_config_path(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def log_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def log_internal(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def log_symlink_internal(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def log_symlink_user(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def log_user(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def login_timeout(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def mode(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def notebook_name(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def problem(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def program(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def program_relpath(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def project(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def project_url(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def quiet(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def reinit(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def relogin(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def resume(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def resume_fname(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def resumed(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def root_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def run_group(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def run_id(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def run_job_type(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def run_mode(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def run_name(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def run_notes(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def run_tags(self) -> global___ListStringValue: ...
+ @property
+ def run_url(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def sagemaker_disable(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def save_code(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def settings_system(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def settings_workspace(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def show_colors(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def show_emoji(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def show_errors(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def show_info(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def show_warnings(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def silent(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def start_method(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def strict(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def summary_errors(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def summary_timeout(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def summary_warnings(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def sweep_id(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def sweep_param_path(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def sweep_url(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def symlink(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def sync_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def sync_file(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def sync_symlink_latest(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def system_sample(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def system_sample_seconds(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def table_raise_on_max_row_limit_exceeded(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def timespec(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def tmp_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def username(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def wandb_dir(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _jupyter_name(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _jupyter_path(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def job_name(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _stats_disk_paths(self) -> global___ListStringValue: ...
+ @property
+ def _file_stream_retry_max(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _file_stream_retry_wait_min_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _file_stream_retry_wait_max_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _file_transfer_retry_max(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _file_transfer_retry_wait_min_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _file_transfer_retry_wait_max_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _file_transfer_timeout_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _graphql_retry_max(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _graphql_retry_wait_min_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _graphql_retry_wait_max_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _graphql_timeout_seconds(self) -> google.protobuf.wrappers_pb2.DoubleValue: ...
+ @property
+ def _disable_machine_info(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def program_abspath(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def colab_url(self) -> google.protobuf.wrappers_pb2.StringValue: ...
+ @property
+ def _stats_buffer_size(self) -> google.protobuf.wrappers_pb2.Int32Value: ...
+ @property
+ def _shared(self) -> google.protobuf.wrappers_pb2.BoolValue: ...
+ @property
+ def _proxies(self) -> global___MapStringKeyStringValue: ...
+ def __init__(
+ self,
+ *,
+ _args: global___ListStringValue | None = ...,
+ _aws_lambda: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _async_upload_concurrency_limit: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _cli_only_mode: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _colab: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _cuda: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _disable_meta: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _disable_service: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _disable_setproctitle: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _disable_stats: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _disable_viewer: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _except_exit: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _executable: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _extra_http_headers: global___MapStringKeyStringValue | None = ...,
+ _file_stream_timeout_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _flow_control_custom: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _flow_control_disabled: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _internal_check_process: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _internal_queue_timeout: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _ipython: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _jupyter: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _jupyter_root: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _kaggle: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _live_policy_rate_limit: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _live_policy_wait_time: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _log_level: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _network_buffer: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _noop: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _notebook: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _offline: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _sync: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _os: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _platform: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _python: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _runqueue_item_id: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _require_core: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _save_requirements: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _service_transport: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _service_wait: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _start_datetime: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _start_time: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _stats_pid: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _stats_sample_rate_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _stats_samples_to_average: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _stats_join_assets: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _stats_neuron_monitor_config_path: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _stats_open_metrics_endpoints: global___MapStringKeyStringValue | None = ...,
+ _stats_open_metrics_filters: global___OpenMetricsFilters | None = ...,
+ _tmp_code_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _tracelog: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _unsaved_keys: global___ListStringValue | None = ...,
+ _windows: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ allow_val_change: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ anonymous: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ api_key: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ azure_account_url_to_access_key: global___MapStringKeyStringValue | None = ...,
+ base_url: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ code_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ config_paths: global___ListStringValue | None = ...,
+ console: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ deployment: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ disable_code: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ disable_git: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ disable_hints: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ disable_job_creation: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ disabled: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ docker: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ email: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ entity: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ files_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ force: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ git_commit: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ git_remote: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ git_remote_url: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ git_root: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ heartbeat_seconds: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ host: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ ignore_globs: global___ListStringValue | None = ...,
+ init_timeout: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ is_local: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ job_source: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ label_disable: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ launch: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ launch_config_path: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ log_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ log_internal: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ log_symlink_internal: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ log_symlink_user: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ log_user: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ login_timeout: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ mode: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ notebook_name: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ problem: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ program: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ program_relpath: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ project: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ project_url: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ quiet: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ reinit: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ relogin: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ resume: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ resume_fname: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ resumed: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ root_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ run_group: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ run_id: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ run_job_type: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ run_mode: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ run_name: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ run_notes: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ run_tags: global___ListStringValue | None = ...,
+ run_url: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ sagemaker_disable: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ save_code: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ settings_system: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ settings_workspace: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ show_colors: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ show_emoji: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ show_errors: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ show_info: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ show_warnings: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ silent: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ start_method: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ strict: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ summary_errors: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ summary_timeout: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ summary_warnings: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ sweep_id: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ sweep_param_path: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ sweep_url: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ symlink: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ sync_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ sync_file: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ sync_symlink_latest: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ system_sample: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ system_sample_seconds: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ table_raise_on_max_row_limit_exceeded: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ timespec: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ tmp_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ username: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ wandb_dir: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _jupyter_name: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _jupyter_path: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ job_name: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _stats_disk_paths: global___ListStringValue | None = ...,
+ _file_stream_retry_max: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _file_stream_retry_wait_min_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _file_stream_retry_wait_max_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _file_transfer_retry_max: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _file_transfer_retry_wait_min_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _file_transfer_retry_wait_max_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _file_transfer_timeout_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _graphql_retry_max: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _graphql_retry_wait_min_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _graphql_retry_wait_max_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _graphql_timeout_seconds: google.protobuf.wrappers_pb2.DoubleValue | None = ...,
+ _disable_machine_info: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ program_abspath: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ colab_url: google.protobuf.wrappers_pb2.StringValue | None = ...,
+ _stats_buffer_size: google.protobuf.wrappers_pb2.Int32Value | None = ...,
+ _shared: google.protobuf.wrappers_pb2.BoolValue | None = ...,
+ _proxies: global___MapStringKeyStringValue | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_args", b"_args", "_async_upload_concurrency_limit", b"_async_upload_concurrency_limit", "_aws_lambda", b"_aws_lambda", "_cli_only_mode", b"_cli_only_mode", "_colab", b"_colab", "_cuda", b"_cuda", "_disable_machine_info", b"_disable_machine_info", "_disable_meta", b"_disable_meta", "_disable_service", b"_disable_service", "_disable_setproctitle", b"_disable_setproctitle", "_disable_stats", b"_disable_stats", "_disable_viewer", b"_disable_viewer", "_except_exit", b"_except_exit", "_executable", b"_executable", "_extra_http_headers", b"_extra_http_headers", "_file_stream_retry_max", b"_file_stream_retry_max", "_file_stream_retry_wait_max_seconds", b"_file_stream_retry_wait_max_seconds", "_file_stream_retry_wait_min_seconds", b"_file_stream_retry_wait_min_seconds", "_file_stream_timeout_seconds", b"_file_stream_timeout_seconds", "_file_transfer_retry_max", b"_file_transfer_retry_max", "_file_transfer_retry_wait_max_seconds", b"_file_transfer_retry_wait_max_seconds", "_file_transfer_retry_wait_min_seconds", b"_file_transfer_retry_wait_min_seconds", "_file_transfer_timeout_seconds", b"_file_transfer_timeout_seconds", "_flow_control_custom", b"_flow_control_custom", "_flow_control_disabled", b"_flow_control_disabled", "_graphql_retry_max", b"_graphql_retry_max", "_graphql_retry_wait_max_seconds", b"_graphql_retry_wait_max_seconds", "_graphql_retry_wait_min_seconds", b"_graphql_retry_wait_min_seconds", "_graphql_timeout_seconds", b"_graphql_timeout_seconds", "_internal_check_process", b"_internal_check_process", "_internal_queue_timeout", b"_internal_queue_timeout", "_ipython", b"_ipython", "_jupyter", b"_jupyter", "_jupyter_name", b"_jupyter_name", "_jupyter_path", b"_jupyter_path", "_jupyter_root", b"_jupyter_root", "_kaggle", b"_kaggle", "_live_policy_rate_limit", b"_live_policy_rate_limit", "_live_policy_wait_time", b"_live_policy_wait_time", "_log_level", b"_log_level", "_network_buffer", b"_network_buffer", "_noop", b"_noop", "_notebook", b"_notebook", "_offline", b"_offline", "_os", b"_os", "_platform", b"_platform", "_proxies", b"_proxies", "_python", b"_python", "_require_core", b"_require_core", "_runqueue_item_id", b"_runqueue_item_id", "_save_requirements", b"_save_requirements", "_service_transport", b"_service_transport", "_service_wait", b"_service_wait", "_shared", b"_shared", "_start_datetime", b"_start_datetime", "_start_time", b"_start_time", "_stats_buffer_size", b"_stats_buffer_size", "_stats_disk_paths", b"_stats_disk_paths", "_stats_join_assets", b"_stats_join_assets", "_stats_neuron_monitor_config_path", b"_stats_neuron_monitor_config_path", "_stats_open_metrics_endpoints", b"_stats_open_metrics_endpoints", "_stats_open_metrics_filters", b"_stats_open_metrics_filters", "_stats_pid", b"_stats_pid", "_stats_sample_rate_seconds", b"_stats_sample_rate_seconds", "_stats_samples_to_average", b"_stats_samples_to_average", "_sync", b"_sync", "_tmp_code_dir", b"_tmp_code_dir", "_tracelog", b"_tracelog", "_unsaved_keys", b"_unsaved_keys", "_windows", b"_windows", "allow_val_change", b"allow_val_change", "anonymous", b"anonymous", "api_key", b"api_key", "azure_account_url_to_access_key", b"azure_account_url_to_access_key", "base_url", b"base_url", "code_dir", b"code_dir", "colab_url", b"colab_url", "config_paths", b"config_paths", "console", b"console", "deployment", b"deployment", "disable_code", b"disable_code", "disable_git", b"disable_git", "disable_hints", b"disable_hints", "disable_job_creation", b"disable_job_creation", "disabled", b"disabled", "docker", b"docker", "email", b"email", "entity", b"entity", "files_dir", b"files_dir", "force", b"force", "git_commit", b"git_commit", "git_remote", b"git_remote", "git_remote_url", b"git_remote_url", "git_root", b"git_root", "heartbeat_seconds", b"heartbeat_seconds", "host", b"host", "ignore_globs", b"ignore_globs", "init_timeout", b"init_timeout", "is_local", b"is_local", "job_name", b"job_name", "job_source", b"job_source", "label_disable", b"label_disable", "launch", b"launch", "launch_config_path", b"launch_config_path", "log_dir", b"log_dir", "log_internal", b"log_internal", "log_symlink_internal", b"log_symlink_internal", "log_symlink_user", b"log_symlink_user", "log_user", b"log_user", "login_timeout", b"login_timeout", "mode", b"mode", "notebook_name", b"notebook_name", "problem", b"problem", "program", b"program", "program_abspath", b"program_abspath", "program_relpath", b"program_relpath", "project", b"project", "project_url", b"project_url", "quiet", b"quiet", "reinit", b"reinit", "relogin", b"relogin", "resume", b"resume", "resume_fname", b"resume_fname", "resumed", b"resumed", "root_dir", b"root_dir", "run_group", b"run_group", "run_id", b"run_id", "run_job_type", b"run_job_type", "run_mode", b"run_mode", "run_name", b"run_name", "run_notes", b"run_notes", "run_tags", b"run_tags", "run_url", b"run_url", "sagemaker_disable", b"sagemaker_disable", "save_code", b"save_code", "settings_system", b"settings_system", "settings_workspace", b"settings_workspace", "show_colors", b"show_colors", "show_emoji", b"show_emoji", "show_errors", b"show_errors", "show_info", b"show_info", "show_warnings", b"show_warnings", "silent", b"silent", "start_method", b"start_method", "strict", b"strict", "summary_errors", b"summary_errors", "summary_timeout", b"summary_timeout", "summary_warnings", b"summary_warnings", "sweep_id", b"sweep_id", "sweep_param_path", b"sweep_param_path", "sweep_url", b"sweep_url", "symlink", b"symlink", "sync_dir", b"sync_dir", "sync_file", b"sync_file", "sync_symlink_latest", b"sync_symlink_latest", "system_sample", b"system_sample", "system_sample_seconds", b"system_sample_seconds", "table_raise_on_max_row_limit_exceeded", b"table_raise_on_max_row_limit_exceeded", "timespec", b"timespec", "tmp_dir", b"tmp_dir", "username", b"username", "wandb_dir", b"wandb_dir"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_args", b"_args", "_async_upload_concurrency_limit", b"_async_upload_concurrency_limit", "_aws_lambda", b"_aws_lambda", "_cli_only_mode", b"_cli_only_mode", "_colab", b"_colab", "_cuda", b"_cuda", "_disable_machine_info", b"_disable_machine_info", "_disable_meta", b"_disable_meta", "_disable_service", b"_disable_service", "_disable_setproctitle", b"_disable_setproctitle", "_disable_stats", b"_disable_stats", "_disable_viewer", b"_disable_viewer", "_except_exit", b"_except_exit", "_executable", b"_executable", "_extra_http_headers", b"_extra_http_headers", "_file_stream_retry_max", b"_file_stream_retry_max", "_file_stream_retry_wait_max_seconds", b"_file_stream_retry_wait_max_seconds", "_file_stream_retry_wait_min_seconds", b"_file_stream_retry_wait_min_seconds", "_file_stream_timeout_seconds", b"_file_stream_timeout_seconds", "_file_transfer_retry_max", b"_file_transfer_retry_max", "_file_transfer_retry_wait_max_seconds", b"_file_transfer_retry_wait_max_seconds", "_file_transfer_retry_wait_min_seconds", b"_file_transfer_retry_wait_min_seconds", "_file_transfer_timeout_seconds", b"_file_transfer_timeout_seconds", "_flow_control_custom", b"_flow_control_custom", "_flow_control_disabled", b"_flow_control_disabled", "_graphql_retry_max", b"_graphql_retry_max", "_graphql_retry_wait_max_seconds", b"_graphql_retry_wait_max_seconds", "_graphql_retry_wait_min_seconds", b"_graphql_retry_wait_min_seconds", "_graphql_timeout_seconds", b"_graphql_timeout_seconds", "_internal_check_process", b"_internal_check_process", "_internal_queue_timeout", b"_internal_queue_timeout", "_ipython", b"_ipython", "_jupyter", b"_jupyter", "_jupyter_name", b"_jupyter_name", "_jupyter_path", b"_jupyter_path", "_jupyter_root", b"_jupyter_root", "_kaggle", b"_kaggle", "_live_policy_rate_limit", b"_live_policy_rate_limit", "_live_policy_wait_time", b"_live_policy_wait_time", "_log_level", b"_log_level", "_network_buffer", b"_network_buffer", "_noop", b"_noop", "_notebook", b"_notebook", "_offline", b"_offline", "_os", b"_os", "_platform", b"_platform", "_proxies", b"_proxies", "_python", b"_python", "_require_core", b"_require_core", "_runqueue_item_id", b"_runqueue_item_id", "_save_requirements", b"_save_requirements", "_service_transport", b"_service_transport", "_service_wait", b"_service_wait", "_shared", b"_shared", "_start_datetime", b"_start_datetime", "_start_time", b"_start_time", "_stats_buffer_size", b"_stats_buffer_size", "_stats_disk_paths", b"_stats_disk_paths", "_stats_join_assets", b"_stats_join_assets", "_stats_neuron_monitor_config_path", b"_stats_neuron_monitor_config_path", "_stats_open_metrics_endpoints", b"_stats_open_metrics_endpoints", "_stats_open_metrics_filters", b"_stats_open_metrics_filters", "_stats_pid", b"_stats_pid", "_stats_sample_rate_seconds", b"_stats_sample_rate_seconds", "_stats_samples_to_average", b"_stats_samples_to_average", "_sync", b"_sync", "_tmp_code_dir", b"_tmp_code_dir", "_tracelog", b"_tracelog", "_unsaved_keys", b"_unsaved_keys", "_windows", b"_windows", "allow_val_change", b"allow_val_change", "anonymous", b"anonymous", "api_key", b"api_key", "azure_account_url_to_access_key", b"azure_account_url_to_access_key", "base_url", b"base_url", "code_dir", b"code_dir", "colab_url", b"colab_url", "config_paths", b"config_paths", "console", b"console", "deployment", b"deployment", "disable_code", b"disable_code", "disable_git", b"disable_git", "disable_hints", b"disable_hints", "disable_job_creation", b"disable_job_creation", "disabled", b"disabled", "docker", b"docker", "email", b"email", "entity", b"entity", "files_dir", b"files_dir", "force", b"force", "git_commit", b"git_commit", "git_remote", b"git_remote", "git_remote_url", b"git_remote_url", "git_root", b"git_root", "heartbeat_seconds", b"heartbeat_seconds", "host", b"host", "ignore_globs", b"ignore_globs", "init_timeout", b"init_timeout", "is_local", b"is_local", "job_name", b"job_name", "job_source", b"job_source", "label_disable", b"label_disable", "launch", b"launch", "launch_config_path", b"launch_config_path", "log_dir", b"log_dir", "log_internal", b"log_internal", "log_symlink_internal", b"log_symlink_internal", "log_symlink_user", b"log_symlink_user", "log_user", b"log_user", "login_timeout", b"login_timeout", "mode", b"mode", "notebook_name", b"notebook_name", "problem", b"problem", "program", b"program", "program_abspath", b"program_abspath", "program_relpath", b"program_relpath", "project", b"project", "project_url", b"project_url", "quiet", b"quiet", "reinit", b"reinit", "relogin", b"relogin", "resume", b"resume", "resume_fname", b"resume_fname", "resumed", b"resumed", "root_dir", b"root_dir", "run_group", b"run_group", "run_id", b"run_id", "run_job_type", b"run_job_type", "run_mode", b"run_mode", "run_name", b"run_name", "run_notes", b"run_notes", "run_tags", b"run_tags", "run_url", b"run_url", "sagemaker_disable", b"sagemaker_disable", "save_code", b"save_code", "settings_system", b"settings_system", "settings_workspace", b"settings_workspace", "show_colors", b"show_colors", "show_emoji", b"show_emoji", "show_errors", b"show_errors", "show_info", b"show_info", "show_warnings", b"show_warnings", "silent", b"silent", "start_method", b"start_method", "strict", b"strict", "summary_errors", b"summary_errors", "summary_timeout", b"summary_timeout", "summary_warnings", b"summary_warnings", "sweep_id", b"sweep_id", "sweep_param_path", b"sweep_param_path", "sweep_url", b"sweep_url", "symlink", b"symlink", "sync_dir", b"sync_dir", "sync_file", b"sync_file", "sync_symlink_latest", b"sync_symlink_latest", "system_sample", b"system_sample", "system_sample_seconds", b"system_sample_seconds", "table_raise_on_max_row_limit_exceeded", b"table_raise_on_max_row_limit_exceeded", "timespec", b"timespec", "tmp_dir", b"tmp_dir", "username", b"username", "wandb_dir", b"wandb_dir"]) -> None: ...
+
+global___Settings = Settings
diff --git a/wandb/proto/v4/wandb_telemetry_pb2.py b/wandb/proto/v4/wandb_telemetry_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..3922c8c3c67f2e8379df736f726fb4890a16b37f
--- /dev/null
+++ b/wandb/proto/v4/wandb_telemetry_pb2.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: wandb/proto/wandb_telemetry.proto
+"""Generated protocol buffer code."""
+from google.protobuf.internal import builder as _builder
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from wandb.proto import wandb_base_pb2 as wandb_dot_proto_dot_wandb__base__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n!wandb/proto/wandb_telemetry.proto\x12\x0ewandb_internal\x1a\x1cwandb/proto/wandb_base.proto\"\xdb\x03\n\x0fTelemetryRecord\x12-\n\x0cimports_init\x18\x01 \x01(\x0b\x32\x17.wandb_internal.Imports\x12/\n\x0eimports_finish\x18\x02 \x01(\x0b\x32\x17.wandb_internal.Imports\x12(\n\x07\x66\x65\x61ture\x18\x03 \x01(\x0b\x32\x17.wandb_internal.Feature\x12\x16\n\x0epython_version\x18\x04 \x01(\t\x12\x13\n\x0b\x63li_version\x18\x05 \x01(\t\x12\x1b\n\x13huggingface_version\x18\x06 \x01(\t\x12 \n\x03\x65nv\x18\x08 \x01(\x0b\x32\x13.wandb_internal.Env\x12%\n\x05label\x18\t \x01(\x0b\x32\x16.wandb_internal.Labels\x12.\n\ndeprecated\x18\n \x01(\x0b\x32\x1a.wandb_internal.Deprecated\x12&\n\x06issues\x18\x0b \x01(\x0b\x32\x16.wandb_internal.Issues\x12\x14\n\x0c\x63ore_version\x18\x0c \x01(\t\x12\x10\n\x08platform\x18\r \x01(\t\x12+\n\x05_info\x18\xc8\x01 \x01(\x0b\x32\x1b.wandb_internal._RecordInfo\"\x11\n\x0fTelemetryResult\"\xe2\r\n\x07Imports\x12\r\n\x05torch\x18\x01 \x01(\x08\x12\r\n\x05keras\x18\x02 \x01(\x08\x12\x12\n\ntensorflow\x18\x03 \x01(\x08\x12\x0e\n\x06\x66\x61stai\x18\x04 \x01(\x08\x12\x0f\n\x07sklearn\x18\x05 \x01(\x08\x12\x0f\n\x07xgboost\x18\x06 \x01(\x08\x12\x10\n\x08\x63\x61tboost\x18\x07 \x01(\x08\x12\x10\n\x08lightgbm\x18\x08 \x01(\x08\x12\x19\n\x11pytorch_lightning\x18\t \x01(\x08\x12\x0e\n\x06ignite\x18\n \x01(\x08\x12\x14\n\x0ctransformers\x18\x0b \x01(\x08\x12\x0b\n\x03jax\x18\x0c \x01(\x08\x12\x10\n\x08metaflow\x18\r \x01(\x08\x12\x10\n\x08\x61llennlp\x18\x0e \x01(\x08\x12\x11\n\tautogluon\x18\x0f \x01(\x08\x12\x11\n\tautokeras\x18\x10 \x01(\x08\x12\x10\n\x08\x63\x61talyst\x18\x12 \x01(\x08\x12\x10\n\x08\x64\x65\x65pchem\x18\x15 \x01(\x08\x12\x0f\n\x07\x64\x65\x65pctr\x18\x16 \x01(\x08\x12\x0f\n\x07pycaret\x18\x1c \x01(\x08\x12\x14\n\x0cpytorchvideo\x18\x1d \x01(\x08\x12\x0b\n\x03ray\x18\x1e \x01(\x08\x12\x1a\n\x12simpletransformers\x18\x1f \x01(\x08\x12\x0e\n\x06skorch\x18 \x01(\x08\x12\r\n\x05spacy\x18! \x01(\x08\x12\r\n\x05\x66lash\x18\" \x01(\x08\x12\x0e\n\x06optuna\x18# \x01(\x08\x12\x0f\n\x07recbole\x18$ \x01(\x08\x12\x0c\n\x04mmcv\x18% \x01(\x08\x12\r\n\x05mmdet\x18& \x01(\x08\x12\x11\n\ttorchdrug\x18\' \x01(\x08\x12\x11\n\ttorchtext\x18( \x01(\x08\x12\x13\n\x0btorchvision\x18) \x01(\x08\x12\r\n\x05\x65legy\x18* \x01(\x08\x12\x12\n\ndetectron2\x18+ \x01(\x08\x12\r\n\x05\x66lair\x18, \x01(\x08\x12\x0c\n\x04\x66lax\x18- \x01(\x08\x12\x0c\n\x04syft\x18. \x01(\x08\x12\x0b\n\x03TTS\x18/ \x01(\x08\x12\r\n\x05monai\x18\x30 \x01(\x08\x12\x17\n\x0fhuggingface_hub\x18\x31 \x01(\x08\x12\r\n\x05hydra\x18\x32 \x01(\x08\x12\x10\n\x08\x64\x61tasets\x18\x33 \x01(\x08\x12\x0e\n\x06sacred\x18\x34 \x01(\x08\x12\x0e\n\x06joblib\x18\x35 \x01(\x08\x12\x0c\n\x04\x64\x61sk\x18\x36 \x01(\x08\x12\x0f\n\x07\x61syncio\x18\x37 \x01(\x08\x12\x11\n\tpaddleocr\x18\x38 \x01(\x08\x12\r\n\x05ppdet\x18\x39 \x01(\x08\x12\x11\n\tpaddleseg\x18: \x01(\x08\x12\x11\n\tpaddlenlp\x18; \x01(\x08\x12\r\n\x05mmseg\x18< \x01(\x08\x12\r\n\x05mmocr\x18= \x01(\x08\x12\r\n\x05mmcls\x18> \x01(\x08\x12\x0c\n\x04timm\x18? \x01(\x08\x12\x0f\n\x07\x66\x61irseq\x18@ \x01(\x08\x12\x12\n\ndeepchecks\x18\x41 \x01(\x08\x12\x10\n\x08\x63omposer\x18\x42 \x01(\x08\x12\x10\n\x08sparseml\x18\x43 \x01(\x08\x12\x10\n\x08\x61nomalib\x18\x44 \x01(\x08\x12\r\n\x05zenml\x18\x45 \x01(\x08\x12\x12\n\ncolossalai\x18\x46 \x01(\x08\x12\x12\n\naccelerate\x18G \x01(\x08\x12\x0e\n\x06merlin\x18H \x01(\x08\x12\x0f\n\x07nanodet\x18I \x01(\x08\x12#\n\x1bsegmentation_models_pytorch\x18J \x01(\x08\x12\x1d\n\x15sentence_transformers\x18K \x01(\x08\x12\x0b\n\x03\x64gl\x18L \x01(\x08\x12\x17\n\x0ftorch_geometric\x18M \x01(\x08\x12\x0c\n\x04jina\x18N \x01(\x08\x12\x0e\n\x06kornia\x18O \x01(\x08\x12\x16\n\x0e\x61lbumentations\x18P \x01(\x08\x12\x10\n\x08keras_cv\x18Q \x01(\x08\x12\x10\n\x08mmengine\x18R \x01(\x08\x12\x11\n\tdiffusers\x18S \x01(\x08\x12\x0b\n\x03trl\x18T \x01(\x08\x12\x0c\n\x04trlx\x18U \x01(\x08\x12\x11\n\tlangchain\x18V \x01(\x08\x12\x13\n\x0bllama_index\x18W \x01(\x08\x12\x15\n\rstability_sdk\x18X \x01(\x08\x12\x0f\n\x07prefect\x18Y \x01(\x08\x12\x13\n\x0bprefect_ray\x18Z \x01(\x08\x12\x10\n\x08pinecone\x18[ \x01(\x08\x12\x10\n\x08\x63hromadb\x18\\ \x01(\x08\x12\x10\n\x08weaviate\x18] \x01(\x08\x12\x13\n\x0bpromptlayer\x18^ \x01(\x08\x12\x0e\n\x06openai\x18_ \x01(\x08\x12\x0e\n\x06\x63ohere\x18` \x01(\x08\x12\x11\n\tanthropic\x18\x61 \x01(\x08\x12\x0c\n\x04peft\x18\x62 \x01(\x08\x12\x0f\n\x07optimum\x18\x63 \x01(\x08\x12\x10\n\x08\x65valuate\x18\x64 \x01(\x08\x12\x10\n\x08langflow\x18\x65 \x01(\x08\x12\x12\n\nkeras_core\x18\x66 \x01(\x08\x12\x18\n\x10lightning_fabric\x18g \x01(\x08\x12\x1c\n\x14\x63urated_transformers\x18h \x01(\x08\x12\x0e\n\x06orjson\x18i \x01(\x08\"\xf3\n\n\x07\x46\x65\x61ture\x12\r\n\x05watch\x18\x01 \x01(\x08\x12\x0e\n\x06\x66inish\x18\x02 \x01(\x08\x12\x0c\n\x04save\x18\x03 \x01(\x08\x12\x0f\n\x07offline\x18\x04 \x01(\x08\x12\x0f\n\x07resumed\x18\x05 \x01(\x08\x12\x0c\n\x04grpc\x18\x06 \x01(\x08\x12\x0e\n\x06metric\x18\x07 \x01(\x08\x12\r\n\x05keras\x18\x08 \x01(\x08\x12\x11\n\tsagemaker\x18\t \x01(\x08\x12\x1c\n\x14\x61rtifact_incremental\x18\n \x01(\x08\x12\x10\n\x08metaflow\x18\x0b \x01(\x08\x12\x0f\n\x07prodigy\x18\x0c \x01(\x08\x12\x15\n\rset_init_name\x18\r \x01(\x08\x12\x13\n\x0bset_init_id\x18\x0e \x01(\x08\x12\x15\n\rset_init_tags\x18\x0f \x01(\x08\x12\x17\n\x0fset_init_config\x18\x10 \x01(\x08\x12\x14\n\x0cset_run_name\x18\x11 \x01(\x08\x12\x14\n\x0cset_run_tags\x18\x12 \x01(\x08\x12\x17\n\x0fset_config_item\x18\x13 \x01(\x08\x12\x0e\n\x06launch\x18\x14 \x01(\x08\x12\x1c\n\x14torch_profiler_trace\x18\x15 \x01(\x08\x12\x0b\n\x03sb3\x18\x16 \x01(\x08\x12\x0f\n\x07service\x18\x17 \x01(\x08\x12\x17\n\x0finit_return_run\x18\x18 \x01(\x08\x12\x1f\n\x17lightgbm_wandb_callback\x18\x19 \x01(\x08\x12\x1c\n\x14lightgbm_log_summary\x18\x1a \x01(\x08\x12\x1f\n\x17\x63\x61tboost_wandb_callback\x18\x1b \x01(\x08\x12\x1c\n\x14\x63\x61tboost_log_summary\x18\x1c \x01(\x08\x12\x17\n\x0ftensorboard_log\x18\x1d \x01(\x08\x12\x16\n\x0e\x65stimator_hook\x18\x1e \x01(\x08\x12\x1e\n\x16xgboost_wandb_callback\x18\x1f \x01(\x08\x12\"\n\x1axgboost_old_wandb_callback\x18 \x01(\x08\x12\x0e\n\x06\x61ttach\x18! \x01(\x08\x12\x19\n\x11tensorboard_patch\x18\" \x01(\x08\x12\x18\n\x10tensorboard_sync\x18# \x01(\x08\x12\x15\n\rkfp_wandb_log\x18$ \x01(\x08\x12\x1b\n\x13maybe_run_overwrite\x18% \x01(\x08\x12\x1c\n\x14keras_metrics_logger\x18& \x01(\x08\x12\x1e\n\x16keras_model_checkpoint\x18\' \x01(\x08\x12!\n\x19keras_wandb_eval_callback\x18( \x01(\x08\x12\x1d\n\x15\x66low_control_overflow\x18) \x01(\x08\x12\x0c\n\x04sync\x18* \x01(\x08\x12\x1d\n\x15\x66low_control_disabled\x18+ \x01(\x08\x12\x1b\n\x13\x66low_control_custom\x18, \x01(\x08\x12\x18\n\x10service_disabled\x18- \x01(\x08\x12\x14\n\x0copen_metrics\x18. \x01(\x08\x12\x1a\n\x12ultralytics_yolov8\x18/ \x01(\x08\x12\x17\n\x0fimporter_mlflow\x18\x30 \x01(\x08\x12\x15\n\rsync_tfevents\x18\x31 \x01(\x08\x12\x15\n\rasync_uploads\x18\x32 \x01(\x08\x12\x16\n\x0eopenai_autolog\x18\x33 \x01(\x08\x12\x18\n\x10langchain_tracer\x18\x34 \x01(\x08\x12\x16\n\x0e\x63ohere_autolog\x18\x35 \x01(\x08\x12\x1b\n\x13hf_pipeline_autolog\x18\x36 \x01(\x08\x12\x0c\n\x04\x63ore\x18\x37 \x01(\x08\x12\r\n\x05lib_c\x18\x38 \x01(\x08\x12\x0f\n\x07lib_cpp\x18\x39 \x01(\x08\x12\x19\n\x11openai_finetuning\x18: \x01(\x08\x12\x19\n\x11\x64iffusers_autolog\x18; \x01(\x08\"\x96\x02\n\x03\x45nv\x12\x0f\n\x07jupyter\x18\x01 \x01(\x08\x12\x0e\n\x06kaggle\x18\x02 \x01(\x08\x12\x0f\n\x07windows\x18\x03 \x01(\x08\x12\x0e\n\x06m1_gpu\x18\x04 \x01(\x08\x12\x13\n\x0bstart_spawn\x18\x05 \x01(\x08\x12\x12\n\nstart_fork\x18\x06 \x01(\x08\x12\x18\n\x10start_forkserver\x18\x07 \x01(\x08\x12\x14\n\x0cstart_thread\x18\x08 \x01(\x08\x12\x10\n\x08maybe_mp\x18\t \x01(\x08\x12\x10\n\x08trainium\x18\n \x01(\x08\x12\x0b\n\x03pex\x18\x0b \x01(\x08\x12\r\n\x05\x63olab\x18\x0c \x01(\x08\x12\x0f\n\x07ipython\x18\r \x01(\x08\x12\x12\n\naws_lambda\x18\x0e \x01(\x08\x12\x0f\n\x07\x61md_gpu\x18\x0f \x01(\x08\"H\n\x06Labels\x12\x13\n\x0b\x63ode_string\x18\x01 \x01(\t\x12\x13\n\x0brepo_string\x18\x02 \x01(\t\x12\x14\n\x0c\x63ode_version\x18\x03 \x01(\t\"\x9a\x02\n\nDeprecated\x12!\n\x19keras_callback__data_type\x18\x01 \x01(\x08\x12\x11\n\trun__mode\x18\x02 \x01(\x08\x12\x19\n\x11run__save_no_args\x18\x03 \x01(\x08\x12\x11\n\trun__join\x18\x04 \x01(\x08\x12\r\n\x05plots\x18\x05 \x01(\x08\x12\x15\n\rrun__log_sync\x18\x06 \x01(\x08\x12!\n\x19init__config_include_keys\x18\x07 \x01(\x08\x12!\n\x19init__config_exclude_keys\x18\x08 \x01(\x08\x12\"\n\x1akeras_callback__save_model\x18\t \x01(\x08\x12\x18\n\x10langchain_tracer\x18\n \x01(\x08\"|\n\x06Issues\x12%\n\x1dsettings__validation_warnings\x18\x01 \x01(\x08\x12!\n\x19settings__unexpected_args\x18\x02 \x01(\x08\x12(\n settings__preprocessing_warnings\x18\x03 \x01(\x08\x62\x06proto3')
+
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'wandb.proto.wandb_telemetry_pb2', globals())
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+ DESCRIPTOR._options = None
+ _TELEMETRYRECORD._serialized_start=84
+ _TELEMETRYRECORD._serialized_end=559
+ _TELEMETRYRESULT._serialized_start=561
+ _TELEMETRYRESULT._serialized_end=578
+ _IMPORTS._serialized_start=581
+ _IMPORTS._serialized_end=2343
+ _FEATURE._serialized_start=2346
+ _FEATURE._serialized_end=3741
+ _ENV._serialized_start=3744
+ _ENV._serialized_end=4022
+ _LABELS._serialized_start=4024
+ _LABELS._serialized_end=4096
+ _DEPRECATED._serialized_start=4099
+ _DEPRECATED._serialized_end=4381
+ _ISSUES._serialized_start=4383
+ _ISSUES._serialized_end=4507
+# @@protoc_insertion_point(module_scope)
diff --git a/wandb/proto/v4/wandb_telemetry_pb2.pyi b/wandb/proto/v4/wandb_telemetry_pb2.pyi
new file mode 100644
index 0000000000000000000000000000000000000000..6eb2e3ee2380b49422ca4ae2a261bf2c0823080c
--- /dev/null
+++ b/wandb/proto/v4/wandb_telemetry_pb2.pyi
@@ -0,0 +1,831 @@
+"""
+@generated by mypy-protobuf. Do not edit manually!
+isort:skip_file
+"""
+import builtins
+import google.protobuf.descriptor
+import google.protobuf.message
+import sys
+import wandb.proto.wandb_base_pb2
+
+if sys.version_info >= (3, 8):
+ import typing as typing_extensions
+else:
+ import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+@typing_extensions.final
+class TelemetryRecord(google.protobuf.message.Message):
+ """
+ Telemetry
+ """
+
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ IMPORTS_INIT_FIELD_NUMBER: builtins.int
+ IMPORTS_FINISH_FIELD_NUMBER: builtins.int
+ FEATURE_FIELD_NUMBER: builtins.int
+ PYTHON_VERSION_FIELD_NUMBER: builtins.int
+ CLI_VERSION_FIELD_NUMBER: builtins.int
+ HUGGINGFACE_VERSION_FIELD_NUMBER: builtins.int
+ ENV_FIELD_NUMBER: builtins.int
+ LABEL_FIELD_NUMBER: builtins.int
+ DEPRECATED_FIELD_NUMBER: builtins.int
+ ISSUES_FIELD_NUMBER: builtins.int
+ CORE_VERSION_FIELD_NUMBER: builtins.int
+ PLATFORM_FIELD_NUMBER: builtins.int
+ _INFO_FIELD_NUMBER: builtins.int
+ @property
+ def imports_init(self) -> global___Imports: ...
+ @property
+ def imports_finish(self) -> global___Imports: ...
+ @property
+ def feature(self) -> global___Feature: ...
+ python_version: builtins.str
+ cli_version: builtins.str
+ huggingface_version: builtins.str
+ @property
+ def env(self) -> global___Env:
+ """string framework = 7;"""
+ @property
+ def label(self) -> global___Labels: ...
+ @property
+ def deprecated(self) -> global___Deprecated: ...
+ @property
+ def issues(self) -> global___Issues: ...
+ core_version: builtins.str
+ platform: builtins.str
+ @property
+ def _info(self) -> wandb.proto.wandb_base_pb2._RecordInfo: ...
+ def __init__(
+ self,
+ *,
+ imports_init: global___Imports | None = ...,
+ imports_finish: global___Imports | None = ...,
+ feature: global___Feature | None = ...,
+ python_version: builtins.str = ...,
+ cli_version: builtins.str = ...,
+ huggingface_version: builtins.str = ...,
+ env: global___Env | None = ...,
+ label: global___Labels | None = ...,
+ deprecated: global___Deprecated | None = ...,
+ issues: global___Issues | None = ...,
+ core_version: builtins.str = ...,
+ platform: builtins.str = ...,
+ _info: wandb.proto.wandb_base_pb2._RecordInfo | None = ...,
+ ) -> None: ...
+ def HasField(self, field_name: typing_extensions.Literal["_info", b"_info", "deprecated", b"deprecated", "env", b"env", "feature", b"feature", "imports_finish", b"imports_finish", "imports_init", b"imports_init", "issues", b"issues", "label", b"label"]) -> builtins.bool: ...
+ def ClearField(self, field_name: typing_extensions.Literal["_info", b"_info", "cli_version", b"cli_version", "core_version", b"core_version", "deprecated", b"deprecated", "env", b"env", "feature", b"feature", "huggingface_version", b"huggingface_version", "imports_finish", b"imports_finish", "imports_init", b"imports_init", "issues", b"issues", "label", b"label", "platform", b"platform", "python_version", b"python_version"]) -> None: ...
+
+global___TelemetryRecord = TelemetryRecord
+
+@typing_extensions.final
+class TelemetryResult(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ def __init__(
+ self,
+ ) -> None: ...
+
+global___TelemetryResult = TelemetryResult
+
+@typing_extensions.final
+class Imports(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ TORCH_FIELD_NUMBER: builtins.int
+ KERAS_FIELD_NUMBER: builtins.int
+ TENSORFLOW_FIELD_NUMBER: builtins.int
+ FASTAI_FIELD_NUMBER: builtins.int
+ SKLEARN_FIELD_NUMBER: builtins.int
+ XGBOOST_FIELD_NUMBER: builtins.int
+ CATBOOST_FIELD_NUMBER: builtins.int
+ LIGHTGBM_FIELD_NUMBER: builtins.int
+ PYTORCH_LIGHTNING_FIELD_NUMBER: builtins.int
+ IGNITE_FIELD_NUMBER: builtins.int
+ TRANSFORMERS_FIELD_NUMBER: builtins.int
+ JAX_FIELD_NUMBER: builtins.int
+ METAFLOW_FIELD_NUMBER: builtins.int
+ ALLENNLP_FIELD_NUMBER: builtins.int
+ AUTOGLUON_FIELD_NUMBER: builtins.int
+ AUTOKERAS_FIELD_NUMBER: builtins.int
+ CATALYST_FIELD_NUMBER: builtins.int
+ DEEPCHEM_FIELD_NUMBER: builtins.int
+ DEEPCTR_FIELD_NUMBER: builtins.int
+ PYCARET_FIELD_NUMBER: builtins.int
+ PYTORCHVIDEO_FIELD_NUMBER: builtins.int
+ RAY_FIELD_NUMBER: builtins.int
+ SIMPLETRANSFORMERS_FIELD_NUMBER: builtins.int
+ SKORCH_FIELD_NUMBER: builtins.int
+ SPACY_FIELD_NUMBER: builtins.int
+ FLASH_FIELD_NUMBER: builtins.int
+ OPTUNA_FIELD_NUMBER: builtins.int
+ RECBOLE_FIELD_NUMBER: builtins.int
+ MMCV_FIELD_NUMBER: builtins.int
+ MMDET_FIELD_NUMBER: builtins.int
+ TORCHDRUG_FIELD_NUMBER: builtins.int
+ TORCHTEXT_FIELD_NUMBER: builtins.int
+ TORCHVISION_FIELD_NUMBER: builtins.int
+ ELEGY_FIELD_NUMBER: builtins.int
+ DETECTRON2_FIELD_NUMBER: builtins.int
+ FLAIR_FIELD_NUMBER: builtins.int
+ FLAX_FIELD_NUMBER: builtins.int
+ SYFT_FIELD_NUMBER: builtins.int
+ TTS_FIELD_NUMBER: builtins.int
+ MONAI_FIELD_NUMBER: builtins.int
+ HUGGINGFACE_HUB_FIELD_NUMBER: builtins.int
+ HYDRA_FIELD_NUMBER: builtins.int
+ DATASETS_FIELD_NUMBER: builtins.int
+ SACRED_FIELD_NUMBER: builtins.int
+ JOBLIB_FIELD_NUMBER: builtins.int
+ DASK_FIELD_NUMBER: builtins.int
+ ASYNCIO_FIELD_NUMBER: builtins.int
+ PADDLEOCR_FIELD_NUMBER: builtins.int
+ PPDET_FIELD_NUMBER: builtins.int
+ PADDLESEG_FIELD_NUMBER: builtins.int
+ PADDLENLP_FIELD_NUMBER: builtins.int
+ MMSEG_FIELD_NUMBER: builtins.int
+ MMOCR_FIELD_NUMBER: builtins.int
+ MMCLS_FIELD_NUMBER: builtins.int
+ TIMM_FIELD_NUMBER: builtins.int
+ FAIRSEQ_FIELD_NUMBER: builtins.int
+ DEEPCHECKS_FIELD_NUMBER: builtins.int
+ COMPOSER_FIELD_NUMBER: builtins.int
+ SPARSEML_FIELD_NUMBER: builtins.int
+ ANOMALIB_FIELD_NUMBER: builtins.int
+ ZENML_FIELD_NUMBER: builtins.int
+ COLOSSALAI_FIELD_NUMBER: builtins.int
+ ACCELERATE_FIELD_NUMBER: builtins.int
+ MERLIN_FIELD_NUMBER: builtins.int
+ NANODET_FIELD_NUMBER: builtins.int
+ SEGMENTATION_MODELS_PYTORCH_FIELD_NUMBER: builtins.int
+ SENTENCE_TRANSFORMERS_FIELD_NUMBER: builtins.int
+ DGL_FIELD_NUMBER: builtins.int
+ TORCH_GEOMETRIC_FIELD_NUMBER: builtins.int
+ JINA_FIELD_NUMBER: builtins.int
+ KORNIA_FIELD_NUMBER: builtins.int
+ ALBUMENTATIONS_FIELD_NUMBER: builtins.int
+ KERAS_CV_FIELD_NUMBER: builtins.int
+ MMENGINE_FIELD_NUMBER: builtins.int
+ DIFFUSERS_FIELD_NUMBER: builtins.int
+ TRL_FIELD_NUMBER: builtins.int
+ TRLX_FIELD_NUMBER: builtins.int
+ LANGCHAIN_FIELD_NUMBER: builtins.int
+ LLAMA_INDEX_FIELD_NUMBER: builtins.int
+ STABILITY_SDK_FIELD_NUMBER: builtins.int
+ PREFECT_FIELD_NUMBER: builtins.int
+ PREFECT_RAY_FIELD_NUMBER: builtins.int
+ PINECONE_FIELD_NUMBER: builtins.int
+ CHROMADB_FIELD_NUMBER: builtins.int
+ WEAVIATE_FIELD_NUMBER: builtins.int
+ PROMPTLAYER_FIELD_NUMBER: builtins.int
+ OPENAI_FIELD_NUMBER: builtins.int
+ COHERE_FIELD_NUMBER: builtins.int
+ ANTHROPIC_FIELD_NUMBER: builtins.int
+ PEFT_FIELD_NUMBER: builtins.int
+ OPTIMUM_FIELD_NUMBER: builtins.int
+ EVALUATE_FIELD_NUMBER: builtins.int
+ LANGFLOW_FIELD_NUMBER: builtins.int
+ KERAS_CORE_FIELD_NUMBER: builtins.int
+ LIGHTNING_FABRIC_FIELD_NUMBER: builtins.int
+ CURATED_TRANSFORMERS_FIELD_NUMBER: builtins.int
+ ORJSON_FIELD_NUMBER: builtins.int
+ torch: builtins.bool
+ keras: builtins.bool
+ tensorflow: builtins.bool
+ fastai: builtins.bool
+ sklearn: builtins.bool
+ xgboost: builtins.bool
+ catboost: builtins.bool
+ lightgbm: builtins.bool
+ pytorch_lightning: builtins.bool
+ ignite: builtins.bool
+ transformers: builtins.bool
+ jax: builtins.bool
+ metaflow: builtins.bool
+ allennlp: builtins.bool
+ autogluon: builtins.bool
+ autokeras: builtins.bool
+ catalyst: builtins.bool
+ """bool avalanche = 17;"""
+ deepchem: builtins.bool
+ """bool dalle_pytorch = 19;
+ bool datasets = 20;
+ """
+ deepctr: builtins.bool
+ pycaret: builtins.bool
+ """bool deeppavlov = 23;
+ bool detectron = 24;
+ bool paddle = 25;
+ bool parlai = 26;
+ bool prophet = 27;
+ """
+ pytorchvideo: builtins.bool
+ ray: builtins.bool
+ simpletransformers: builtins.bool
+ skorch: builtins.bool
+ spacy: builtins.bool
+ flash: builtins.bool
+ optuna: builtins.bool
+ recbole: builtins.bool
+ mmcv: builtins.bool
+ mmdet: builtins.bool
+ torchdrug: builtins.bool
+ torchtext: builtins.bool
+ torchvision: builtins.bool
+ elegy: builtins.bool
+ detectron2: builtins.bool
+ flair: builtins.bool
+ flax: builtins.bool
+ syft: builtins.bool
+ TTS: builtins.bool
+ monai: builtins.bool
+ huggingface_hub: builtins.bool
+ hydra: builtins.bool
+ datasets: builtins.bool
+ sacred: builtins.bool
+ joblib: builtins.bool
+ dask: builtins.bool
+ asyncio: builtins.bool
+ paddleocr: builtins.bool
+ ppdet: builtins.bool
+ paddleseg: builtins.bool
+ paddlenlp: builtins.bool
+ mmseg: builtins.bool
+ mmocr: builtins.bool
+ mmcls: builtins.bool
+ timm: builtins.bool
+ fairseq: builtins.bool
+ deepchecks: builtins.bool
+ composer: builtins.bool
+ sparseml: builtins.bool
+ anomalib: builtins.bool
+ zenml: builtins.bool
+ colossalai: builtins.bool
+ accelerate: builtins.bool
+ merlin: builtins.bool
+ nanodet: builtins.bool
+ segmentation_models_pytorch: builtins.bool
+ sentence_transformers: builtins.bool
+ dgl: builtins.bool
+ torch_geometric: builtins.bool
+ jina: builtins.bool
+ kornia: builtins.bool
+ albumentations: builtins.bool
+ keras_cv: builtins.bool
+ mmengine: builtins.bool
+ diffusers: builtins.bool
+ trl: builtins.bool
+ trlx: builtins.bool
+ langchain: builtins.bool
+ llama_index: builtins.bool
+ stability_sdk: builtins.bool
+ prefect: builtins.bool
+ prefect_ray: builtins.bool
+ pinecone: builtins.bool
+ """pinecone-client"""
+ chromadb: builtins.bool
+ weaviate: builtins.bool
+ """weaviate-client"""
+ promptlayer: builtins.bool
+ openai: builtins.bool
+ cohere: builtins.bool
+ anthropic: builtins.bool
+ peft: builtins.bool
+ optimum: builtins.bool
+ evaluate: builtins.bool
+ langflow: builtins.bool
+ keras_core: builtins.bool
+ """keras-core"""
+ lightning_fabric: builtins.bool
+ """lightning-fabric"""
+ curated_transformers: builtins.bool
+ """curated-transformers"""
+ orjson: builtins.bool
+ def __init__(
+ self,
+ *,
+ torch: builtins.bool = ...,
+ keras: builtins.bool = ...,
+ tensorflow: builtins.bool = ...,
+ fastai: builtins.bool = ...,
+ sklearn: builtins.bool = ...,
+ xgboost: builtins.bool = ...,
+ catboost: builtins.bool = ...,
+ lightgbm: builtins.bool = ...,
+ pytorch_lightning: builtins.bool = ...,
+ ignite: builtins.bool = ...,
+ transformers: builtins.bool = ...,
+ jax: builtins.bool = ...,
+ metaflow: builtins.bool = ...,
+ allennlp: builtins.bool = ...,
+ autogluon: builtins.bool = ...,
+ autokeras: builtins.bool = ...,
+ catalyst: builtins.bool = ...,
+ deepchem: builtins.bool = ...,
+ deepctr: builtins.bool = ...,
+ pycaret: builtins.bool = ...,
+ pytorchvideo: builtins.bool = ...,
+ ray: builtins.bool = ...,
+ simpletransformers: builtins.bool = ...,
+ skorch: builtins.bool = ...,
+ spacy: builtins.bool = ...,
+ flash: builtins.bool = ...,
+ optuna: builtins.bool = ...,
+ recbole: builtins.bool = ...,
+ mmcv: builtins.bool = ...,
+ mmdet: builtins.bool = ...,
+ torchdrug: builtins.bool = ...,
+ torchtext: builtins.bool = ...,
+ torchvision: builtins.bool = ...,
+ elegy: builtins.bool = ...,
+ detectron2: builtins.bool = ...,
+ flair: builtins.bool = ...,
+ flax: builtins.bool = ...,
+ syft: builtins.bool = ...,
+ TTS: builtins.bool = ...,
+ monai: builtins.bool = ...,
+ huggingface_hub: builtins.bool = ...,
+ hydra: builtins.bool = ...,
+ datasets: builtins.bool = ...,
+ sacred: builtins.bool = ...,
+ joblib: builtins.bool = ...,
+ dask: builtins.bool = ...,
+ asyncio: builtins.bool = ...,
+ paddleocr: builtins.bool = ...,
+ ppdet: builtins.bool = ...,
+ paddleseg: builtins.bool = ...,
+ paddlenlp: builtins.bool = ...,
+ mmseg: builtins.bool = ...,
+ mmocr: builtins.bool = ...,
+ mmcls: builtins.bool = ...,
+ timm: builtins.bool = ...,
+ fairseq: builtins.bool = ...,
+ deepchecks: builtins.bool = ...,
+ composer: builtins.bool = ...,
+ sparseml: builtins.bool = ...,
+ anomalib: builtins.bool = ...,
+ zenml: builtins.bool = ...,
+ colossalai: builtins.bool = ...,
+ accelerate: builtins.bool = ...,
+ merlin: builtins.bool = ...,
+ nanodet: builtins.bool = ...,
+ segmentation_models_pytorch: builtins.bool = ...,
+ sentence_transformers: builtins.bool = ...,
+ dgl: builtins.bool = ...,
+ torch_geometric: builtins.bool = ...,
+ jina: builtins.bool = ...,
+ kornia: builtins.bool = ...,
+ albumentations: builtins.bool = ...,
+ keras_cv: builtins.bool = ...,
+ mmengine: builtins.bool = ...,
+ diffusers: builtins.bool = ...,
+ trl: builtins.bool = ...,
+ trlx: builtins.bool = ...,
+ langchain: builtins.bool = ...,
+ llama_index: builtins.bool = ...,
+ stability_sdk: builtins.bool = ...,
+ prefect: builtins.bool = ...,
+ prefect_ray: builtins.bool = ...,
+ pinecone: builtins.bool = ...,
+ chromadb: builtins.bool = ...,
+ weaviate: builtins.bool = ...,
+ promptlayer: builtins.bool = ...,
+ openai: builtins.bool = ...,
+ cohere: builtins.bool = ...,
+ anthropic: builtins.bool = ...,
+ peft: builtins.bool = ...,
+ optimum: builtins.bool = ...,
+ evaluate: builtins.bool = ...,
+ langflow: builtins.bool = ...,
+ keras_core: builtins.bool = ...,
+ lightning_fabric: builtins.bool = ...,
+ curated_transformers: builtins.bool = ...,
+ orjson: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["TTS", b"TTS", "accelerate", b"accelerate", "albumentations", b"albumentations", "allennlp", b"allennlp", "anomalib", b"anomalib", "anthropic", b"anthropic", "asyncio", b"asyncio", "autogluon", b"autogluon", "autokeras", b"autokeras", "catalyst", b"catalyst", "catboost", b"catboost", "chromadb", b"chromadb", "cohere", b"cohere", "colossalai", b"colossalai", "composer", b"composer", "curated_transformers", b"curated_transformers", "dask", b"dask", "datasets", b"datasets", "deepchecks", b"deepchecks", "deepchem", b"deepchem", "deepctr", b"deepctr", "detectron2", b"detectron2", "dgl", b"dgl", "diffusers", b"diffusers", "elegy", b"elegy", "evaluate", b"evaluate", "fairseq", b"fairseq", "fastai", b"fastai", "flair", b"flair", "flash", b"flash", "flax", b"flax", "huggingface_hub", b"huggingface_hub", "hydra", b"hydra", "ignite", b"ignite", "jax", b"jax", "jina", b"jina", "joblib", b"joblib", "keras", b"keras", "keras_core", b"keras_core", "keras_cv", b"keras_cv", "kornia", b"kornia", "langchain", b"langchain", "langflow", b"langflow", "lightgbm", b"lightgbm", "lightning_fabric", b"lightning_fabric", "llama_index", b"llama_index", "merlin", b"merlin", "metaflow", b"metaflow", "mmcls", b"mmcls", "mmcv", b"mmcv", "mmdet", b"mmdet", "mmengine", b"mmengine", "mmocr", b"mmocr", "mmseg", b"mmseg", "monai", b"monai", "nanodet", b"nanodet", "openai", b"openai", "optimum", b"optimum", "optuna", b"optuna", "orjson", b"orjson", "paddlenlp", b"paddlenlp", "paddleocr", b"paddleocr", "paddleseg", b"paddleseg", "peft", b"peft", "pinecone", b"pinecone", "ppdet", b"ppdet", "prefect", b"prefect", "prefect_ray", b"prefect_ray", "promptlayer", b"promptlayer", "pycaret", b"pycaret", "pytorch_lightning", b"pytorch_lightning", "pytorchvideo", b"pytorchvideo", "ray", b"ray", "recbole", b"recbole", "sacred", b"sacred", "segmentation_models_pytorch", b"segmentation_models_pytorch", "sentence_transformers", b"sentence_transformers", "simpletransformers", b"simpletransformers", "sklearn", b"sklearn", "skorch", b"skorch", "spacy", b"spacy", "sparseml", b"sparseml", "stability_sdk", b"stability_sdk", "syft", b"syft", "tensorflow", b"tensorflow", "timm", b"timm", "torch", b"torch", "torch_geometric", b"torch_geometric", "torchdrug", b"torchdrug", "torchtext", b"torchtext", "torchvision", b"torchvision", "transformers", b"transformers", "trl", b"trl", "trlx", b"trlx", "weaviate", b"weaviate", "xgboost", b"xgboost", "zenml", b"zenml"]) -> None: ...
+
+global___Imports = Imports
+
+@typing_extensions.final
+class Feature(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ WATCH_FIELD_NUMBER: builtins.int
+ FINISH_FIELD_NUMBER: builtins.int
+ SAVE_FIELD_NUMBER: builtins.int
+ OFFLINE_FIELD_NUMBER: builtins.int
+ RESUMED_FIELD_NUMBER: builtins.int
+ GRPC_FIELD_NUMBER: builtins.int
+ METRIC_FIELD_NUMBER: builtins.int
+ KERAS_FIELD_NUMBER: builtins.int
+ SAGEMAKER_FIELD_NUMBER: builtins.int
+ ARTIFACT_INCREMENTAL_FIELD_NUMBER: builtins.int
+ METAFLOW_FIELD_NUMBER: builtins.int
+ PRODIGY_FIELD_NUMBER: builtins.int
+ SET_INIT_NAME_FIELD_NUMBER: builtins.int
+ SET_INIT_ID_FIELD_NUMBER: builtins.int
+ SET_INIT_TAGS_FIELD_NUMBER: builtins.int
+ SET_INIT_CONFIG_FIELD_NUMBER: builtins.int
+ SET_RUN_NAME_FIELD_NUMBER: builtins.int
+ SET_RUN_TAGS_FIELD_NUMBER: builtins.int
+ SET_CONFIG_ITEM_FIELD_NUMBER: builtins.int
+ LAUNCH_FIELD_NUMBER: builtins.int
+ TORCH_PROFILER_TRACE_FIELD_NUMBER: builtins.int
+ SB3_FIELD_NUMBER: builtins.int
+ SERVICE_FIELD_NUMBER: builtins.int
+ INIT_RETURN_RUN_FIELD_NUMBER: builtins.int
+ LIGHTGBM_WANDB_CALLBACK_FIELD_NUMBER: builtins.int
+ LIGHTGBM_LOG_SUMMARY_FIELD_NUMBER: builtins.int
+ CATBOOST_WANDB_CALLBACK_FIELD_NUMBER: builtins.int
+ CATBOOST_LOG_SUMMARY_FIELD_NUMBER: builtins.int
+ TENSORBOARD_LOG_FIELD_NUMBER: builtins.int
+ ESTIMATOR_HOOK_FIELD_NUMBER: builtins.int
+ XGBOOST_WANDB_CALLBACK_FIELD_NUMBER: builtins.int
+ XGBOOST_OLD_WANDB_CALLBACK_FIELD_NUMBER: builtins.int
+ ATTACH_FIELD_NUMBER: builtins.int
+ TENSORBOARD_PATCH_FIELD_NUMBER: builtins.int
+ TENSORBOARD_SYNC_FIELD_NUMBER: builtins.int
+ KFP_WANDB_LOG_FIELD_NUMBER: builtins.int
+ MAYBE_RUN_OVERWRITE_FIELD_NUMBER: builtins.int
+ KERAS_METRICS_LOGGER_FIELD_NUMBER: builtins.int
+ KERAS_MODEL_CHECKPOINT_FIELD_NUMBER: builtins.int
+ KERAS_WANDB_EVAL_CALLBACK_FIELD_NUMBER: builtins.int
+ FLOW_CONTROL_OVERFLOW_FIELD_NUMBER: builtins.int
+ SYNC_FIELD_NUMBER: builtins.int
+ FLOW_CONTROL_DISABLED_FIELD_NUMBER: builtins.int
+ FLOW_CONTROL_CUSTOM_FIELD_NUMBER: builtins.int
+ SERVICE_DISABLED_FIELD_NUMBER: builtins.int
+ OPEN_METRICS_FIELD_NUMBER: builtins.int
+ ULTRALYTICS_YOLOV8_FIELD_NUMBER: builtins.int
+ IMPORTER_MLFLOW_FIELD_NUMBER: builtins.int
+ SYNC_TFEVENTS_FIELD_NUMBER: builtins.int
+ ASYNC_UPLOADS_FIELD_NUMBER: builtins.int
+ OPENAI_AUTOLOG_FIELD_NUMBER: builtins.int
+ LANGCHAIN_TRACER_FIELD_NUMBER: builtins.int
+ COHERE_AUTOLOG_FIELD_NUMBER: builtins.int
+ HF_PIPELINE_AUTOLOG_FIELD_NUMBER: builtins.int
+ CORE_FIELD_NUMBER: builtins.int
+ LIB_C_FIELD_NUMBER: builtins.int
+ LIB_CPP_FIELD_NUMBER: builtins.int
+ OPENAI_FINETUNING_FIELD_NUMBER: builtins.int
+ DIFFUSERS_AUTOLOG_FIELD_NUMBER: builtins.int
+ watch: builtins.bool
+ """wandb.watch() called"""
+ finish: builtins.bool
+ """wandb.finish() called"""
+ save: builtins.bool
+ """wandb.save() called"""
+ offline: builtins.bool
+ """offline run was synced"""
+ resumed: builtins.bool
+ """run was resumed"""
+ grpc: builtins.bool
+ """grpc-server (java integration)"""
+ metric: builtins.bool
+ """define_metric() called"""
+ keras: builtins.bool
+ """Keras WandbCallback used"""
+ sagemaker: builtins.bool
+ """User is using sagemaker"""
+ artifact_incremental: builtins.bool
+ """Artifact(incremental=True) used"""
+ metaflow: builtins.bool
+ """Using metaflow integration"""
+ prodigy: builtins.bool
+ """Using prodigy integration"""
+ set_init_name: builtins.bool
+ """users set run name from wandb.init"""
+ set_init_id: builtins.bool
+ """users set run id from wandb.init"""
+ set_init_tags: builtins.bool
+ """users set tags within wandb.init"""
+ set_init_config: builtins.bool
+ """users set run config in wandb.init"""
+ set_run_name: builtins.bool
+ """user sets run name via wandb.run.name = ..."""
+ set_run_tags: builtins.bool
+ """user sets run name via wandb.run.tags = ..."""
+ set_config_item: builtins.bool
+ """users set key in run config via run.config.key"""
+ launch: builtins.bool
+ """or run.config["key"]
+ run is created through wandb launch
+ """
+ torch_profiler_trace: builtins.bool
+ """wandb.profiler.torch_trace_handler() called"""
+ sb3: builtins.bool
+ """Using stable_baselines3 integration"""
+ service: builtins.bool
+ """Using wandb service internal process"""
+ init_return_run: builtins.bool
+ """wandb.init() called in the same process returning previous run"""
+ lightgbm_wandb_callback: builtins.bool
+ """lightgbm callback used"""
+ lightgbm_log_summary: builtins.bool
+ """lightgbm log summary used"""
+ catboost_wandb_callback: builtins.bool
+ """catboost callback used"""
+ catboost_log_summary: builtins.bool
+ """catboost log summary used"""
+ tensorboard_log: builtins.bool
+ """wandb.tensorflow.log or wandb.tensorboard.log used"""
+ estimator_hook: builtins.bool
+ """wandb.tensorflow.WandbHook used"""
+ xgboost_wandb_callback: builtins.bool
+ """xgboost callback used"""
+ xgboost_old_wandb_callback: builtins.bool
+ """xgboost old callback used (to be depreciated)"""
+ attach: builtins.bool
+ """attach to a run in another process"""
+ tensorboard_patch: builtins.bool
+ """wandb.tensorboard.patch(...)"""
+ tensorboard_sync: builtins.bool
+ """wandb.init(sync_tensorboard=True)"""
+ kfp_wandb_log: builtins.bool
+ """wandb.integration.kfp.wandb_log"""
+ maybe_run_overwrite: builtins.bool
+ """Run might have been overwritten"""
+ keras_metrics_logger: builtins.bool
+ """Keras WandbMetricsLogger used"""
+ keras_model_checkpoint: builtins.bool
+ """Keras WandbModelCheckpoint used"""
+ keras_wandb_eval_callback: builtins.bool
+ """Keras WandbEvalCallback used"""
+ flow_control_overflow: builtins.bool
+ """Hit flow control threshold"""
+ sync: builtins.bool
+ """Run was synced with wandb sync"""
+ flow_control_disabled: builtins.bool
+ """Flow control disabled by user"""
+ flow_control_custom: builtins.bool
+ """Flow control customized by user"""
+ service_disabled: builtins.bool
+ """Service disabled by user"""
+ open_metrics: builtins.bool
+ """Consuming metrics from an OpenMetrics endpoint"""
+ ultralytics_yolov8: builtins.bool
+ """Ultralytics YOLOv8 integration callbacks used"""
+ importer_mlflow: builtins.bool
+ """Using Import API for MLFlow"""
+ sync_tfevents: builtins.bool
+ """Using wandb sync for tfevent files"""
+ async_uploads: builtins.bool
+ """Async file uploads enabled by user"""
+ openai_autolog: builtins.bool
+ """OpenAI autolog used"""
+ langchain_tracer: builtins.bool
+ """Langchain wandb tracer callback used"""
+ cohere_autolog: builtins.bool
+ """Cohere autolog used"""
+ hf_pipeline_autolog: builtins.bool
+ """HuggingFace Autologging"""
+ core: builtins.bool
+ """Using wandb core internal process"""
+ lib_c: builtins.bool
+ """Using c wandb library"""
+ lib_cpp: builtins.bool
+ """Using cpp wandb library"""
+ openai_finetuning: builtins.bool
+ """Using openai finetuning WandbLogger"""
+ diffusers_autolog: builtins.bool
+ """Using Diffusers autologger"""
+ def __init__(
+ self,
+ *,
+ watch: builtins.bool = ...,
+ finish: builtins.bool = ...,
+ save: builtins.bool = ...,
+ offline: builtins.bool = ...,
+ resumed: builtins.bool = ...,
+ grpc: builtins.bool = ...,
+ metric: builtins.bool = ...,
+ keras: builtins.bool = ...,
+ sagemaker: builtins.bool = ...,
+ artifact_incremental: builtins.bool = ...,
+ metaflow: builtins.bool = ...,
+ prodigy: builtins.bool = ...,
+ set_init_name: builtins.bool = ...,
+ set_init_id: builtins.bool = ...,
+ set_init_tags: builtins.bool = ...,
+ set_init_config: builtins.bool = ...,
+ set_run_name: builtins.bool = ...,
+ set_run_tags: builtins.bool = ...,
+ set_config_item: builtins.bool = ...,
+ launch: builtins.bool = ...,
+ torch_profiler_trace: builtins.bool = ...,
+ sb3: builtins.bool = ...,
+ service: builtins.bool = ...,
+ init_return_run: builtins.bool = ...,
+ lightgbm_wandb_callback: builtins.bool = ...,
+ lightgbm_log_summary: builtins.bool = ...,
+ catboost_wandb_callback: builtins.bool = ...,
+ catboost_log_summary: builtins.bool = ...,
+ tensorboard_log: builtins.bool = ...,
+ estimator_hook: builtins.bool = ...,
+ xgboost_wandb_callback: builtins.bool = ...,
+ xgboost_old_wandb_callback: builtins.bool = ...,
+ attach: builtins.bool = ...,
+ tensorboard_patch: builtins.bool = ...,
+ tensorboard_sync: builtins.bool = ...,
+ kfp_wandb_log: builtins.bool = ...,
+ maybe_run_overwrite: builtins.bool = ...,
+ keras_metrics_logger: builtins.bool = ...,
+ keras_model_checkpoint: builtins.bool = ...,
+ keras_wandb_eval_callback: builtins.bool = ...,
+ flow_control_overflow: builtins.bool = ...,
+ sync: builtins.bool = ...,
+ flow_control_disabled: builtins.bool = ...,
+ flow_control_custom: builtins.bool = ...,
+ service_disabled: builtins.bool = ...,
+ open_metrics: builtins.bool = ...,
+ ultralytics_yolov8: builtins.bool = ...,
+ importer_mlflow: builtins.bool = ...,
+ sync_tfevents: builtins.bool = ...,
+ async_uploads: builtins.bool = ...,
+ openai_autolog: builtins.bool = ...,
+ langchain_tracer: builtins.bool = ...,
+ cohere_autolog: builtins.bool = ...,
+ hf_pipeline_autolog: builtins.bool = ...,
+ core: builtins.bool = ...,
+ lib_c: builtins.bool = ...,
+ lib_cpp: builtins.bool = ...,
+ openai_finetuning: builtins.bool = ...,
+ diffusers_autolog: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["artifact_incremental", b"artifact_incremental", "async_uploads", b"async_uploads", "attach", b"attach", "catboost_log_summary", b"catboost_log_summary", "catboost_wandb_callback", b"catboost_wandb_callback", "cohere_autolog", b"cohere_autolog", "core", b"core", "diffusers_autolog", b"diffusers_autolog", "estimator_hook", b"estimator_hook", "finish", b"finish", "flow_control_custom", b"flow_control_custom", "flow_control_disabled", b"flow_control_disabled", "flow_control_overflow", b"flow_control_overflow", "grpc", b"grpc", "hf_pipeline_autolog", b"hf_pipeline_autolog", "importer_mlflow", b"importer_mlflow", "init_return_run", b"init_return_run", "keras", b"keras", "keras_metrics_logger", b"keras_metrics_logger", "keras_model_checkpoint", b"keras_model_checkpoint", "keras_wandb_eval_callback", b"keras_wandb_eval_callback", "kfp_wandb_log", b"kfp_wandb_log", "langchain_tracer", b"langchain_tracer", "launch", b"launch", "lib_c", b"lib_c", "lib_cpp", b"lib_cpp", "lightgbm_log_summary", b"lightgbm_log_summary", "lightgbm_wandb_callback", b"lightgbm_wandb_callback", "maybe_run_overwrite", b"maybe_run_overwrite", "metaflow", b"metaflow", "metric", b"metric", "offline", b"offline", "open_metrics", b"open_metrics", "openai_autolog", b"openai_autolog", "openai_finetuning", b"openai_finetuning", "prodigy", b"prodigy", "resumed", b"resumed", "sagemaker", b"sagemaker", "save", b"save", "sb3", b"sb3", "service", b"service", "service_disabled", b"service_disabled", "set_config_item", b"set_config_item", "set_init_config", b"set_init_config", "set_init_id", b"set_init_id", "set_init_name", b"set_init_name", "set_init_tags", b"set_init_tags", "set_run_name", b"set_run_name", "set_run_tags", b"set_run_tags", "sync", b"sync", "sync_tfevents", b"sync_tfevents", "tensorboard_log", b"tensorboard_log", "tensorboard_patch", b"tensorboard_patch", "tensorboard_sync", b"tensorboard_sync", "torch_profiler_trace", b"torch_profiler_trace", "ultralytics_yolov8", b"ultralytics_yolov8", "watch", b"watch", "xgboost_old_wandb_callback", b"xgboost_old_wandb_callback", "xgboost_wandb_callback", b"xgboost_wandb_callback"]) -> None: ...
+
+global___Feature = Feature
+
+@typing_extensions.final
+class Env(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ JUPYTER_FIELD_NUMBER: builtins.int
+ KAGGLE_FIELD_NUMBER: builtins.int
+ WINDOWS_FIELD_NUMBER: builtins.int
+ M1_GPU_FIELD_NUMBER: builtins.int
+ START_SPAWN_FIELD_NUMBER: builtins.int
+ START_FORK_FIELD_NUMBER: builtins.int
+ START_FORKSERVER_FIELD_NUMBER: builtins.int
+ START_THREAD_FIELD_NUMBER: builtins.int
+ MAYBE_MP_FIELD_NUMBER: builtins.int
+ TRAINIUM_FIELD_NUMBER: builtins.int
+ PEX_FIELD_NUMBER: builtins.int
+ COLAB_FIELD_NUMBER: builtins.int
+ IPYTHON_FIELD_NUMBER: builtins.int
+ AWS_LAMBDA_FIELD_NUMBER: builtins.int
+ AMD_GPU_FIELD_NUMBER: builtins.int
+ jupyter: builtins.bool
+ """jupyter env detected"""
+ kaggle: builtins.bool
+ """kaggle env detected"""
+ windows: builtins.bool
+ """windows detected"""
+ m1_gpu: builtins.bool
+ """apple silicon M1 gpu found"""
+ start_spawn: builtins.bool
+ """multiprocessing spawn"""
+ start_fork: builtins.bool
+ """multiprocessing fork"""
+ start_forkserver: builtins.bool
+ """multiprocessing forkserver"""
+ start_thread: builtins.bool
+ """thread start method"""
+ maybe_mp: builtins.bool
+ """maybe user running multiprocessing"""
+ trainium: builtins.bool
+ """AWS Trainium env detected"""
+ pex: builtins.bool
+ """pex env detected"""
+ colab: builtins.bool
+ """colab env detected"""
+ ipython: builtins.bool
+ """ipython env detected"""
+ aws_lambda: builtins.bool
+ """running in AWS Lambda"""
+ amd_gpu: builtins.bool
+ """AMD GPU detected"""
+ def __init__(
+ self,
+ *,
+ jupyter: builtins.bool = ...,
+ kaggle: builtins.bool = ...,
+ windows: builtins.bool = ...,
+ m1_gpu: builtins.bool = ...,
+ start_spawn: builtins.bool = ...,
+ start_fork: builtins.bool = ...,
+ start_forkserver: builtins.bool = ...,
+ start_thread: builtins.bool = ...,
+ maybe_mp: builtins.bool = ...,
+ trainium: builtins.bool = ...,
+ pex: builtins.bool = ...,
+ colab: builtins.bool = ...,
+ ipython: builtins.bool = ...,
+ aws_lambda: builtins.bool = ...,
+ amd_gpu: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["amd_gpu", b"amd_gpu", "aws_lambda", b"aws_lambda", "colab", b"colab", "ipython", b"ipython", "jupyter", b"jupyter", "kaggle", b"kaggle", "m1_gpu", b"m1_gpu", "maybe_mp", b"maybe_mp", "pex", b"pex", "start_fork", b"start_fork", "start_forkserver", b"start_forkserver", "start_spawn", b"start_spawn", "start_thread", b"start_thread", "trainium", b"trainium", "windows", b"windows"]) -> None: ...
+
+global___Env = Env
+
+@typing_extensions.final
+class Labels(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ CODE_STRING_FIELD_NUMBER: builtins.int
+ REPO_STRING_FIELD_NUMBER: builtins.int
+ CODE_VERSION_FIELD_NUMBER: builtins.int
+ code_string: builtins.str
+ """code identification"""
+ repo_string: builtins.str
+ """repo identification"""
+ code_version: builtins.str
+ """code version"""
+ def __init__(
+ self,
+ *,
+ code_string: builtins.str = ...,
+ repo_string: builtins.str = ...,
+ code_version: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["code_string", b"code_string", "code_version", b"code_version", "repo_string", b"repo_string"]) -> None: ...
+
+global___Labels = Labels
+
+@typing_extensions.final
+class Deprecated(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ KERAS_CALLBACK__DATA_TYPE_FIELD_NUMBER: builtins.int
+ RUN__MODE_FIELD_NUMBER: builtins.int
+ RUN__SAVE_NO_ARGS_FIELD_NUMBER: builtins.int
+ RUN__JOIN_FIELD_NUMBER: builtins.int
+ PLOTS_FIELD_NUMBER: builtins.int
+ RUN__LOG_SYNC_FIELD_NUMBER: builtins.int
+ INIT__CONFIG_INCLUDE_KEYS_FIELD_NUMBER: builtins.int
+ INIT__CONFIG_EXCLUDE_KEYS_FIELD_NUMBER: builtins.int
+ KERAS_CALLBACK__SAVE_MODEL_FIELD_NUMBER: builtins.int
+ LANGCHAIN_TRACER_FIELD_NUMBER: builtins.int
+ keras_callback__data_type: builtins.bool
+ """wandb.keras.WandbCallback(data_type=...) called"""
+ run__mode: builtins.bool
+ """wandb.run.mode called"""
+ run__save_no_args: builtins.bool
+ """wandb.run.save() called without arguments"""
+ run__join: builtins.bool
+ """wandb.run.join() called"""
+ plots: builtins.bool
+ """wandb.plots.* called"""
+ run__log_sync: builtins.bool
+ """wandb.run.log(sync=...) called"""
+ init__config_include_keys: builtins.bool
+ """wandb.init(config_include_keys=...) called"""
+ init__config_exclude_keys: builtins.bool
+ """wandb.init(config_exclude_keys=...) called"""
+ keras_callback__save_model: builtins.bool
+ """wandb.keras.WandbCallback(save_model=True) called"""
+ langchain_tracer: builtins.bool
+ """wandb.integration.langchain.WandbTracer called"""
+ def __init__(
+ self,
+ *,
+ keras_callback__data_type: builtins.bool = ...,
+ run__mode: builtins.bool = ...,
+ run__save_no_args: builtins.bool = ...,
+ run__join: builtins.bool = ...,
+ plots: builtins.bool = ...,
+ run__log_sync: builtins.bool = ...,
+ init__config_include_keys: builtins.bool = ...,
+ init__config_exclude_keys: builtins.bool = ...,
+ keras_callback__save_model: builtins.bool = ...,
+ langchain_tracer: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["init__config_exclude_keys", b"init__config_exclude_keys", "init__config_include_keys", b"init__config_include_keys", "keras_callback__data_type", b"keras_callback__data_type", "keras_callback__save_model", b"keras_callback__save_model", "langchain_tracer", b"langchain_tracer", "plots", b"plots", "run__join", b"run__join", "run__log_sync", b"run__log_sync", "run__mode", b"run__mode", "run__save_no_args", b"run__save_no_args"]) -> None: ...
+
+global___Deprecated = Deprecated
+
+@typing_extensions.final
+class Issues(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ SETTINGS__VALIDATION_WARNINGS_FIELD_NUMBER: builtins.int
+ SETTINGS__UNEXPECTED_ARGS_FIELD_NUMBER: builtins.int
+ SETTINGS__PREPROCESSING_WARNINGS_FIELD_NUMBER: builtins.int
+ settings__validation_warnings: builtins.bool
+ """validation warnings for settings"""
+ settings__unexpected_args: builtins.bool
+ """unexpected settings init args"""
+ settings__preprocessing_warnings: builtins.bool
+ """settings preprocessing warnings"""
+ def __init__(
+ self,
+ *,
+ settings__validation_warnings: builtins.bool = ...,
+ settings__unexpected_args: builtins.bool = ...,
+ settings__preprocessing_warnings: builtins.bool = ...,
+ ) -> None: ...
+ def ClearField(self, field_name: typing_extensions.Literal["settings__preprocessing_warnings", b"settings__preprocessing_warnings", "settings__unexpected_args", b"settings__unexpected_args", "settings__validation_warnings", b"settings__validation_warnings"]) -> None: ...
+
+global___Issues = Issues
diff --git a/wandb/proto/wandb_base.proto b/wandb/proto/wandb_base.proto
new file mode 100644
index 0000000000000000000000000000000000000000..a0ffea703f26e05c8745dc92117aa67a58724577
--- /dev/null
+++ b/wandb/proto/wandb_base.proto
@@ -0,0 +1,22 @@
+syntax = "proto3";
+
+package wandb_internal;
+
+/*
+ * _RecordInfo, _RequestInfo: extra info for all records and requests
+ */
+message _RecordInfo {
+ string stream_id = 1;
+ string _tracelog_id = 100;
+}
+
+message _RequestInfo {
+ string stream_id = 1;
+}
+
+/*
+ * _ResultInfo: extra info for all results
+ */
+message _ResultInfo {
+ string _tracelog_id = 100;
+}
diff --git a/wandb/proto/wandb_base_pb2.py b/wandb/proto/wandb_base_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..1589e30e52c4388a053aa0275bd850801e66f1ba
--- /dev/null
+++ b/wandb/proto/wandb_base_pb2.py
@@ -0,0 +1,8 @@
+import google.protobuf
+
+protobuf_version = google.protobuf.__version__[0]
+
+if protobuf_version == "3":
+ from wandb.proto.v3.wandb_base_pb2 import *
+elif protobuf_version == "4":
+ from wandb.proto.v4.wandb_base_pb2 import *
diff --git a/wandb/proto/wandb_deprecated.py b/wandb/proto/wandb_deprecated.py
new file mode 100644
index 0000000000000000000000000000000000000000..37b389c0702c4420a8c5b7fe8a80df3dba9f9795
--- /dev/null
+++ b/wandb/proto/wandb_deprecated.py
@@ -0,0 +1,37 @@
+# Generated by wandb/proto/wandb_internal_codegen.py. DO NOT EDIT!
+
+
+import sys
+
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+
+DEPRECATED_FEATURES = Literal[
+ "keras_callback__data_type",
+ "run__mode",
+ "run__save_no_args",
+ "run__join",
+ "plots",
+ "run__log_sync",
+ "init__config_include_keys",
+ "init__config_exclude_keys",
+ "keras_callback__save_model",
+ "langchain_tracer"
+]
+
+
+class Deprecated:
+ keras_callback__data_type: DEPRECATED_FEATURES = "keras_callback__data_type"
+ run__mode: DEPRECATED_FEATURES = "run__mode"
+ run__save_no_args: DEPRECATED_FEATURES = "run__save_no_args"
+ run__join: DEPRECATED_FEATURES = "run__join"
+ plots: DEPRECATED_FEATURES = "plots"
+ run__log_sync: DEPRECATED_FEATURES = "run__log_sync"
+ init__config_include_keys: DEPRECATED_FEATURES = "init__config_include_keys"
+ init__config_exclude_keys: DEPRECATED_FEATURES = "init__config_exclude_keys"
+ keras_callback__save_model: DEPRECATED_FEATURES = "keras_callback__save_model"
+ langchain_tracer: DEPRECATED_FEATURES = "langchain_tracer"
diff --git a/wandb/proto/wandb_internal.proto b/wandb/proto/wandb_internal.proto
new file mode 100644
index 0000000000000000000000000000000000000000..5bf6d5122f8ec65867d1b3451b2a454ff725d02f
--- /dev/null
+++ b/wandb/proto/wandb_internal.proto
@@ -0,0 +1,1133 @@
+syntax = "proto3";
+
+package wandb_internal;
+
+import "google/protobuf/timestamp.proto";
+import "wandb/proto/wandb_base.proto";
+import "wandb/proto/wandb_telemetry.proto";
+
+/*
+ * Record: Persistent on disk data (BE CAREFUL)
+ * Result: responses from Record requests
+ *
+ * Request: Communication requests between processes
+ * Response: Responses from Request messages
+ */
+
+/************************
+ * Records and Results
+ ************************/
+
+/*
+ * Record: joined record for message passing and persistence
+ */
+message Record {
+ int64 num = 1;
+ oneof record_type {
+ // Low numbers for more frequent data
+ HistoryRecord history = 2;
+ SummaryRecord summary = 3;
+ OutputRecord output = 4;
+ ConfigRecord config = 5;
+ FilesRecord files = 6;
+ StatsRecord stats = 7;
+ ArtifactRecord artifact = 8;
+ TBRecord tbrecord = 9;
+ AlertRecord alert = 10;
+ TelemetryRecord telemetry = 11;
+ MetricRecord metric = 12;
+ OutputRawRecord output_raw = 13;
+ // Higher numbers for less frequent data
+ RunRecord run = 17;
+ RunExitRecord exit = 18;
+ FinalRecord final = 20;
+ HeaderRecord header = 21;
+ FooterRecord footer = 22;
+ RunPreemptingRecord preempting = 23;
+ LinkArtifactRecord link_artifact = 24;
+ UseArtifactRecord use_artifact = 25;
+ // request field does not belong here longterm
+ Request request = 100;
+ }
+ Control control = 16;
+ string uuid = 19;
+ _RecordInfo _info = 200;
+}
+
+message Control {
+ bool req_resp = 1; // record is expecting a result
+ bool local = 2; // should not be persisted or synchronized
+ string relay_id = 3; // used by service transport to identify correct stream
+ string mailbox_slot = 4; // mailbox slot
+ bool always_send = 5; // message to sender
+ bool flow_control = 6; // message should be passed to flow control
+ int64 end_offset = 7; // end of message offset of this written message
+ string connection_id = 8; // connection id
+}
+
+/*
+ * Result: all results
+ */
+message Result {
+ oneof result_type {
+ RunUpdateResult run_result = 17;
+ RunExitResult exit_result = 18;
+ HistoryResult log_result = 20;
+ SummaryResult summary_result = 21;
+ OutputResult output_result = 22;
+ ConfigResult config_result = 23;
+ /* response field does not belong here longterm */
+ Response response = 100;
+ }
+ Control control = 16;
+ string uuid = 24;
+ _ResultInfo _info = 200;
+}
+
+/*
+ * FinalRecord
+ */
+message FinalRecord {
+ _RecordInfo _info = 200;
+}
+
+/*
+ * Version definition
+ */
+message VersionInfo {
+ // The version of the SDK backend that produced the data
+ string producer = 1;
+ // Minimum version of the wandb server that can read the data
+ string min_consumer = 2;
+ _RecordInfo _info = 200;
+}
+
+/*
+ * HeaderRecord
+ */
+message HeaderRecord {
+ VersionInfo version_info = 1;
+ _RecordInfo _info = 200;
+}
+
+/*
+ * FooterRecord
+ */
+message FooterRecord {
+ _RecordInfo _info = 200;
+}
+
+/*
+ * RunRecord: wandb/sdk/wandb_run/Run
+ */
+message RunRecord {
+ string run_id = 1;
+ string entity = 2;
+ string project = 3;
+ ConfigRecord config = 4;
+ SummaryRecord summary = 5;
+ string run_group = 6;
+ string job_type = 7;
+ string display_name = 8;
+ string notes = 9;
+ repeated string tags = 10;
+ SettingsRecord settings = 11;
+ string sweep_id = 12;
+ string host = 13;
+ int64 starting_step = 14;
+
+ string storage_id = 16;
+ google.protobuf.Timestamp start_time = 17;
+ bool resumed = 18;
+ TelemetryRecord telemetry = 19;
+ int32 runtime = 20;
+ GitRepoRecord git = 21;
+ _RecordInfo _info = 200;
+}
+
+message GitRepoRecord {
+ string remote_url = 1 [json_name = "remote"];
+ string commit = 2;
+}
+
+message RunUpdateResult {
+ RunRecord run = 1;
+ ErrorInfo error = 2;
+}
+
+message ErrorInfo {
+ enum ErrorCode {
+ UNKNOWN = 0;
+ COMMUNICATION = 1;
+ AUTHENTICATION = 2;
+ USAGE = 3;
+ UNSUPPORTED = 4;
+ }
+ string message = 1;
+ ErrorCode code = 2;
+}
+
+/*
+ * RunExitRecord: exit status of process
+ */
+message RunExitRecord {
+ int32 exit_code = 1;
+ int32 runtime = 2;
+ _RecordInfo _info = 200;
+}
+
+message RunExitResult {}
+
+/*
+ * RunPreemptingRecord: run being preempted
+ */
+message RunPreemptingRecord {
+ _RecordInfo _info = 200;
+}
+
+message RunPreemptingResult {}
+
+/*
+ * SettingsRecord: wandb/sdk/wandb_settings/Settings
+ */
+message SettingsRecord {
+ repeated SettingsItem item = 1;
+ _RecordInfo _info = 200;
+}
+
+message SettingsItem {
+ string key = 1;
+ string value_json = 16;
+}
+
+/*
+ * HistoryRecord: wandb/sdk/wandb_history/History
+ */
+message HistoryStep {
+ int64 num = 1;
+}
+
+message HistoryRecord {
+ repeated HistoryItem item = 1;
+ HistoryStep step = 2;
+ _RecordInfo _info = 200;
+}
+
+message HistoryItem {
+ string key = 1;
+ repeated string nested_key = 2;
+ string value_json = 16;
+}
+
+message HistoryResult {}
+
+/*
+ * OutputRecord: console output
+ */
+message OutputRecord {
+ enum OutputType {
+ STDERR = 0;
+ STDOUT = 1;
+ }
+ OutputType output_type = 1;
+ google.protobuf.Timestamp timestamp = 2;
+ string line = 3;
+ _RecordInfo _info = 200;
+}
+
+message OutputResult {}
+
+/*
+ * OutputRawRecord: raw console output
+ */
+message OutputRawRecord {
+ enum OutputType {
+ STDERR = 0;
+ STDOUT = 1;
+ }
+ OutputType output_type = 1;
+ google.protobuf.Timestamp timestamp = 2;
+ string line = 3;
+ _RecordInfo _info = 200;
+}
+
+message OutputRawResult {}
+
+/*
+ * MetricRecord: wandb/sdk/wandb_metric/Metric
+ */
+message MetricRecord {
+ // only name or globname is set
+ string name = 1;
+ string glob_name = 2;
+
+ // step metric index can be used instead of step_metric when
+ // MetricRecord is encoded in a list of MetricRecords
+ string step_metric = 4;
+ int32 step_metric_index = 5; // one-based array index
+
+ MetricOptions options = 6;
+ MetricSummary summary = 7;
+ MetricGoal goal = 8;
+ MetricControl _control = 9;
+
+ enum MetricGoal {
+ GOAL_UNSET = 0;
+ GOAL_MINIMIZE = 1;
+ GOAL_MAXIMIZE = 2;
+ }
+ _RecordInfo _info = 200;
+}
+
+message MetricResult {}
+
+message MetricOptions {
+ bool step_sync = 1;
+ bool hidden = 2;
+ bool defined = 3; // metric explicitly defined (not from glob match or step metric)
+}
+
+message MetricControl {
+ bool overwrite = 1;
+}
+
+message MetricSummary {
+ bool min = 1;
+ bool max = 2;
+ bool mean = 3;
+ bool best = 4;
+ bool last = 5;
+ bool none = 6;
+ bool copy = 7;
+}
+
+/*
+ * ConfigRecord: wandb/sdk/wandb_config/Config
+ */
+message ConfigRecord {
+ repeated ConfigItem update = 1;
+ repeated ConfigItem remove = 2;
+ _RecordInfo _info = 200;
+}
+
+message ConfigItem {
+ string key = 1;
+ repeated string nested_key = 2;
+ string value_json = 16;
+}
+
+message ConfigResult {}
+
+/*
+ * SummaryRecord: wandb/sdk/wandb_summary/Summary
+ */
+message SummaryRecord {
+ repeated SummaryItem update = 1;
+ repeated SummaryItem remove = 2;
+ _RecordInfo _info = 200;
+}
+
+message SummaryItem {
+ string key = 1;
+ repeated string nested_key = 2;
+ string value_json = 16;
+}
+
+message SummaryResult {}
+
+/*
+ * FilesRecord: files added to run
+ */
+message FilesRecord {
+ repeated FilesItem files = 1;
+ _RecordInfo _info = 200;
+}
+
+message FilesItem {
+ enum PolicyType {
+ NOW = 0;
+ END = 1;
+ LIVE = 2;
+ }
+ enum FileType {
+ OTHER = 0;
+ WANDB = 1;
+ MEDIA = 2;
+ ARTIFACT = 3;
+ }
+ string path = 1;
+ PolicyType policy = 2;
+ FileType type = 3;
+ string external_path = 16;
+}
+
+message FilesResult {}
+
+/*
+ * StatsRecord: system metrics
+ */
+message StatsRecord {
+ enum StatsType {
+ SYSTEM = 0;
+ }
+ StatsType stats_type = 1;
+ google.protobuf.Timestamp timestamp = 2;
+ repeated StatsItem item = 3;
+ _RecordInfo _info = 200;
+}
+
+message StatsItem {
+ string key = 1;
+ string value_json = 16;
+}
+
+/*
+ * ArtifactRecord: track artifacts
+ */
+message ArtifactRecord {
+ string run_id = 1;
+ string project = 2;
+ string entity = 3;
+ string type = 4;
+ string name = 5;
+ string digest = 6;
+ string description = 7;
+ string metadata = 8;
+ bool user_created = 9;
+ bool use_after_commit = 10;
+ repeated string aliases = 11;
+ ArtifactManifest manifest = 12;
+ string distributed_id = 13;
+ bool finalize = 14;
+ string client_id = 15;
+ string sequence_client_id = 16;
+ string base_id = 17;
+ int64 ttl_duration_seconds = 18;
+ bool incremental_beta1 = 100;
+ _RecordInfo _info = 200;
+}
+
+message ArtifactManifest {
+ int32 version = 1;
+ string storage_policy = 2;
+ repeated StoragePolicyConfigItem storage_policy_config = 3;
+ repeated ArtifactManifestEntry contents = 4;
+}
+
+message ArtifactManifestEntry {
+ string path = 1;
+ string digest = 2;
+ string ref = 3;
+ int64 size = 4;
+ string mimetype = 5;
+ string local_path = 6;
+ string birth_artifact_id = 7;
+ repeated ExtraItem extra = 16;
+}
+
+message ExtraItem {
+ string key = 1;
+ string value_json = 2;
+}
+
+message StoragePolicyConfigItem {
+ string key = 1;
+ string value_json = 2;
+}
+
+message ArtifactResult {}
+
+message LinkArtifactResult {}
+
+/*
+ * LinkArtifactRecord: link artifact to portfolio
+ */
+message LinkArtifactRecord {
+ string client_id = 1;
+ string server_id = 2;
+ string portfolio_name = 3;
+ string portfolio_entity = 4;
+ string portfolio_project = 5;
+ repeated string portfolio_aliases = 6;
+ _RecordInfo _info = 200;
+}
+
+/*
+ * TBRecord: store tb locations
+ */
+message TBRecord {
+ string log_dir = 1;
+ bool save = 2;
+ string root_dir = 3;
+ _RecordInfo _info = 200;
+}
+
+message TBResult {}
+
+/*
+ * AlertRecord: store alert notifications
+ */
+message AlertRecord {
+ string title = 1;
+ string text = 2;
+ string level = 3;
+ int64 wait_duration = 4;
+ _RecordInfo _info = 200;
+}
+
+message AlertResult {}
+
+/************************
+ * Requests and Responses
+ ************************/
+
+/*
+ * Request: all non persistent messages
+ */
+message Request {
+ oneof request_type {
+ StopStatusRequest stop_status = 1;
+ NetworkStatusRequest network_status = 2;
+ DeferRequest defer = 3;
+ GetSummaryRequest get_summary = 4;
+ LoginRequest login = 5;
+ PauseRequest pause = 6;
+ ResumeRequest resume = 7;
+ PollExitRequest poll_exit = 8;
+ SampledHistoryRequest sampled_history = 9;
+ PartialHistoryRequest partial_history = 10;
+ RunStartRequest run_start = 11;
+ CheckVersionRequest check_version = 12;
+ LogArtifactRequest log_artifact = 13;
+ DownloadArtifactRequest download_artifact = 14;
+ KeepaliveRequest keepalive = 17;
+ RunStatusRequest run_status = 20;
+ CancelRequest cancel = 21;
+ MetadataRequest metadata = 22;
+ InternalMessagesRequest internal_messages = 23;
+ PythonPackagesRequest python_packages = 24;
+ ShutdownRequest shutdown = 64;
+ AttachRequest attach = 65;
+ StatusRequest status = 66;
+ ServerInfoRequest server_info = 67;
+ SenderMarkRequest sender_mark = 68;
+ SenderReadRequest sender_read = 69;
+ StatusReportRequest status_report = 70;
+ SummaryRecordRequest summary_record = 71;
+ TelemetryRecordRequest telemetry_record = 72;
+ JobInfoRequest job_info = 73;
+ GetSystemMetricsRequest get_system_metrics = 74;
+ FileTransferInfoRequest file_transfer_info = 75;
+ SyncRequest sync = 76;
+ TestInjectRequest test_inject = 1000;
+ }
+}
+
+/*
+ * Response: all non persistent responses to Requests
+ */
+message Response {
+ oneof response_type {
+ KeepaliveResponse keepalive_response = 18;
+ StopStatusResponse stop_status_response = 19;
+ NetworkStatusResponse network_status_response = 20;
+ LoginResponse login_response = 24;
+ GetSummaryResponse get_summary_response = 25;
+ PollExitResponse poll_exit_response = 26;
+ SampledHistoryResponse sampled_history_response = 27;
+ RunStartResponse run_start_response = 28;
+ CheckVersionResponse check_version_response = 29;
+ LogArtifactResponse log_artifact_response = 30;
+ DownloadArtifactResponse download_artifact_response = 31;
+ RunStatusResponse run_status_response = 35;
+ CancelResponse cancel_response = 36;
+ InternalMessagesResponse internal_messages_response = 37;
+ ShutdownResponse shutdown_response = 64;
+ AttachResponse attach_response = 65;
+ StatusResponse status_response = 66;
+ ServerInfoResponse server_info_response = 67;
+ JobInfoResponse job_info_response = 68;
+ GetSystemMetricsResponse get_system_metrics_response = 69;
+ SyncResponse sync_response = 70;
+ TestInjectResponse test_inject_response = 1000;
+ }
+}
+
+/*
+ * DeferRequest: internal message to defer work
+ */
+message DeferRequest {
+ enum DeferState {
+ BEGIN = 0;
+ FLUSH_RUN = 1;
+ FLUSH_STATS = 2;
+ FLUSH_PARTIAL_HISTORY = 3;
+ FLUSH_TB = 4;
+ FLUSH_SUM = 5;
+ FLUSH_DEBOUNCER = 6;
+ FLUSH_OUTPUT = 7;
+ FLUSH_JOB = 8;
+ FLUSH_DIR = 9;
+ FLUSH_FP = 10;
+ JOIN_FP = 11;
+ FLUSH_FS = 12;
+ FLUSH_FINAL = 13;
+ END = 14;
+ }
+ DeferState state = 1;
+ // Internal message, no _info field needed
+}
+
+/*
+ * PauseRequest: internal message to pause the heartbeat
+ */
+message PauseRequest {
+ _RequestInfo _info = 200;
+}
+
+message PauseResponse {}
+
+/*
+ * ResumeRequest: internal message to resume the heartbeat
+ */
+message ResumeRequest {
+ _RequestInfo _info = 200;
+}
+
+message ResumeResponse {}
+
+/*
+ * LoginRequest: wandb/sdk/wandb_login
+ */
+message LoginRequest {
+ string api_key = 1;
+ _RequestInfo _info = 200;
+}
+
+message LoginResponse {
+ string active_entity = 1;
+}
+
+/*
+ * GetSummaryRequest: request consolidated summary
+ */
+message GetSummaryRequest {
+ _RequestInfo _info = 200;
+}
+
+message GetSummaryResponse {
+ repeated SummaryItem item = 1;
+}
+
+/*
+ * GetSystemMetrics: request system metrics
+ */
+message GetSystemMetricsRequest {
+ _RequestInfo _info = 200;
+}
+
+message SystemMetricSample {
+ google.protobuf.Timestamp timestamp = 1;
+ float value = 2;
+}
+
+message SystemMetricsBuffer {
+ repeated SystemMetricSample record = 1;
+}
+
+message GetSystemMetricsResponse {
+ map<string, SystemMetricsBuffer> system_metrics = 1;
+}
+
+/*
+ * StatusRequest:
+ */
+message StatusRequest {
+ _RequestInfo _info = 200;
+}
+
+message StatusResponse {
+ bool run_should_stop = 1;
+}
+
+message StopStatusRequest {
+ _RequestInfo _info = 200;
+}
+
+message StopStatusResponse {
+ bool run_should_stop = 1;
+}
+
+message NetworkStatusRequest {
+ _RequestInfo _info = 200;
+}
+
+message NetworkStatusResponse {
+ repeated HttpResponse network_responses = 1;
+}
+
+message HttpResponse {
+ int32 http_status_code = 1;
+ string http_response_text = 2;
+}
+
+/*
+ * InternalMessagesRequest:
+ */
+message InternalMessagesRequest {
+ _RequestInfo _info = 200;
+}
+
+message InternalMessagesResponse {
+ InternalMessages messages = 1;
+}
+
+message InternalMessages {
+ repeated string warning = 1;
+}
+
+/*
+ * PollExitRequest:
+ */
+message PollExitRequest {
+ _RequestInfo _info = 200;
+}
+
+message PollExitResponse {
+ bool done = 1;
+ RunExitResult exit_result = 2;
+ FilePusherStats pusher_stats = 3;
+ FileCounts file_counts = 4;
+}
+
+/*
+ * Sender requests
+ */
+message SyncOverwrite {
+ string run_id = 1;
+ string entity = 2;
+ string project = 3;
+}
+
+message SyncSkip {
+ bool output_raw = 1;
+}
+
+message SenderMarkRequest {}
+
+message SyncRequest {
+ int64 start_offset = 1;
+ int64 final_offset = 2;
+ SyncOverwrite overwrite = 3;
+ SyncSkip skip = 4;
+}
+
+message SyncResponse {
+ string url = 1;
+ ErrorInfo error = 2;
+}
+
+message SenderReadRequest {
+ int64 start_offset = 1;
+ int64 final_offset = 2;
+ // TODO: implement cancel for paused ops
+ // repeated string cancel_list = 3;
+}
+
+message StatusReportRequest {
+ int64 record_num = 1;
+ int64 sent_offset = 2;
+ google.protobuf.Timestamp sync_time = 3;
+}
+
+/*
+ * Requests wrapping Records
+ */
+message SummaryRecordRequest {
+ SummaryRecord summary = 1;
+}
+
+message TelemetryRecordRequest {
+ TelemetryRecord telemetry = 1;
+}
+
+/*
+ * ServerInfoRequest:
+ */
+message ServerInfoRequest {
+ _RequestInfo _info = 200;
+}
+
+message ServerInfoResponse {
+ LocalInfo local_info = 1;
+ ServerMessages server_messages = 2;
+}
+
+message ServerMessages {
+ repeated ServerMessage item = 1;
+}
+
+message ServerMessage {
+ string plain_text = 1;
+ string utf_text = 2;
+ string html_text = 3;
+ string type = 4;
+ int32 level = 5;
+}
+
+message FileCounts {
+ int32 wandb_count = 1;
+ int32 media_count = 2;
+ int32 artifact_count = 3;
+ int32 other_count = 4;
+}
+
+message FilePusherStats {
+ int64 uploaded_bytes = 1;
+ int64 total_bytes = 2;
+ int64 deduped_bytes = 3;
+}
+
+message FilesUploaded {
+ repeated string files = 1;
+}
+
+message FileTransferInfoRequest {
+ enum TransferType {
+ Upload = 0;
+ Download = 1;
+ }
+ TransferType type = 1;
+ string path = 2;
+ string url = 3;
+ int64 size = 4;
+ int64 processed = 5;
+ FileCounts file_counts = 6;
+}
+
+message LocalInfo {
+ string version = 1;
+ bool out_of_date = 2;
+}
+
+/*
+ * ShutdownRequest:
+ */
+message ShutdownRequest {
+ _RequestInfo _info = 200;
+}
+
+message ShutdownResponse {}
+
+/*
+ * AttachRequest:
+ */
+message AttachRequest {
+ string attach_id = 20;
+ _RequestInfo _info = 200;
+}
+
+message AttachResponse {
+ RunRecord run = 1;
+ ErrorInfo error = 2;
+}
+
+/*
+ * TestInjectRequest:
+ */
+message TestInjectRequest {
+ bool handler_exc = 1;
+ bool handler_exit = 2;
+ bool handler_abort = 3;
+ bool sender_exc = 4;
+ bool sender_exit = 5;
+ bool sender_abort = 6;
+ bool req_exc = 7;
+ bool req_exit = 8;
+ bool req_abort = 9;
+ bool resp_exc = 10;
+ bool resp_exit = 11;
+ bool resp_abort = 12;
+ bool msg_drop = 13;
+ bool msg_hang = 14;
+ _RequestInfo _info = 200;
+}
+
+message TestInjectResponse {}
+
+/*
+ * PartialHistoryRequest:
+ */
+message HistoryAction {
+ bool flush = 1;
+}
+message PartialHistoryRequest {
+ repeated HistoryItem item = 1;
+ HistoryStep step = 2;
+ HistoryAction action = 3;
+ _RequestInfo _info = 200;
+}
+
+message PartialHistoryResponse {}
+
+/*
+ * SampledHistoryRequest:
+ */
+message SampledHistoryRequest {
+ _RequestInfo _info = 200;
+}
+
+message SampledHistoryItem {
+ string key = 1;
+ repeated string nested_key = 2;
+ repeated float values_float = 3;
+ repeated int64 values_int = 4;
+}
+
+message SampledHistoryResponse {
+ repeated SampledHistoryItem item = 1;
+}
+
+/*
+ * RunStatusRequest:
+ */
+message RunStatusRequest {
+ _RequestInfo _info = 200;
+}
+
+message RunStatusResponse {
+ int64 sync_items_total = 1;
+ int64 sync_items_pending = 2;
+ google.protobuf.Timestamp sync_time = 3;
+ // TODO(flowcontrol): can we give the user an indication of step position
+ // int64 sync_history_step = 3;
+ // google.protobuf.Timestamp sync_history_time = 4;
+}
+
+/*
+ * RunStartRequest: start the run
+ */
+message RunStartRequest {
+ RunRecord run = 1;
+ _RequestInfo _info = 200;
+}
+
+message RunStartResponse {}
+
+/*
+ * CheckVersion:
+ */
+message CheckVersionRequest {
+ string current_version = 1;
+ _RequestInfo _info = 200;
+}
+
+message CheckVersionResponse {
+ string upgrade_message = 1;
+ string yank_message = 2;
+ string delete_message = 3;
+}
+
+/*
+ * JobInfo:
+ */
+message JobInfoRequest {
+ _RequestInfo _info = 200;
+}
+
+message JobInfoResponse {
+ string sequenceId = 1;
+ string version = 2;
+}
+
+/*
+ * LogArtifact:
+ */
+message LogArtifactRequest {
+ ArtifactRecord artifact = 1;
+ int64 history_step = 2;
+ string staging_dir = 3;
+ _RequestInfo _info = 200;
+}
+
+message LogArtifactResponse {
+ string artifact_id = 1;
+ string error_message = 2;
+}
+
+/*
+ * DownloadArtifact:
+ */
+message DownloadArtifactRequest {
+ string artifact_id = 1;
+ string download_root = 2;
+ bool allow_missing_references = 4;
+ _RequestInfo _info = 200;
+}
+
+message DownloadArtifactResponse {
+ string error_message = 1;
+}
+
+/*
+ * Keepalive:
+ */
+message KeepaliveRequest {
+ _RequestInfo _info = 200;
+}
+
+message KeepaliveResponse {}
+
+/*
+ * Job info specific for Partial -> Job upgrade
+ */
+message ArtifactInfo {
+ string artifact = 1;
+ repeated string entrypoint = 2;
+ bool notebook = 3;
+}
+
+message GitInfo {
+ string remote = 1;
+ string commit = 2;
+}
+
+message GitSource {
+ GitInfo git_info = 1;
+ repeated string entrypoint = 2;
+ bool notebook = 3;
+}
+
+message ImageSource {
+ string image = 1;
+}
+
+message Source {
+ GitSource git = 1;
+ ArtifactInfo artifact = 2;
+ ImageSource image = 3;
+}
+
+/*
+ * Mirrors JobSourceDict:
+ */
+message JobSource {
+ string _version = 1;
+ string source_type = 2;
+ Source source = 3;
+ string runtime = 4;
+}
+
+message PartialJobArtifact {
+ string job_name = 1;
+ JobSource source_info = 2;
+}
+
+/*
+ * UseArtifact:
+ */
+message UseArtifactRecord {
+ string id = 1;
+ string type = 2;
+ string name = 3;
+
+ PartialJobArtifact partial = 4;
+
+ _RecordInfo _info = 200;
+}
+
+message UseArtifactResult {}
+
+/*
+ * Cancel:
+ */
+message CancelRequest {
+ string cancel_slot = 1; // mailbox slot
+ _RequestInfo _info = 200;
+}
+
+message CancelResponse {}
+
+/*
+ * MetadataRequest
+ */
+message DiskInfo {
+ uint64 total = 1;
+ uint64 used = 2;
+}
+
+message MemoryInfo {
+ uint64 total = 1;
+}
+
+message CpuInfo {
+ uint32 count = 1;
+ uint32 count_logical = 2;
+}
+
+message GpuAppleInfo {
+ string gpuType = 1;
+ string vendor = 2;
+ uint32 cores = 3;
+}
+
+message GpuNvidiaInfo {
+ string name = 1;
+ uint64 memory_total = 2;
+}
+
+message GpuAmdInfo {
+ string id = 1;
+ string unique_id = 2;
+ string vbios_version = 3;
+ string performance_level = 4;
+ string gpu_overdrive = 5;
+ string gpu_memory_overdrive = 6;
+ string max_power = 7;
+ string series = 8;
+ string model = 9;
+ string vendor = 10;
+ string sku = 11;
+ string sclk_range = 12;
+ string mclk_range = 13;
+}
+
+message MetadataRequest {
+ string os = 1;
+ string python = 2;
+ google.protobuf.Timestamp heartbeatAt = 3;
+ google.protobuf.Timestamp startedAt = 4;
+ string docker = 5;
+ string cuda = 6;
+ repeated string args = 7;
+ string state = 8;
+ string program = 9;
+ string code_path = 10 [json_name = "codePath"];
+ GitRepoRecord git = 11;
+ string email = 12;
+ string root = 13;
+ string host = 14;
+ string username = 15;
+ string executable = 16;
+ string code_path_local = 17 [json_name = "codePathLocal"];
+ string colab = 18;
+ uint32 cpu_count = 19 [json_name = "cpu_count"];
+ uint32 cpu_count_logical = 20 [json_name = "cpu_count_logical"];
+ string gpu_type = 21 [json_name = "gpu"];
+ uint32 gpu_count = 22 [json_name = "gpu_count"];
+ map<string, DiskInfo> disk = 23;
+ MemoryInfo memory = 24;
+ CpuInfo cpu = 25;
+ GpuAppleInfo gpu_apple = 26 [json_name = "gpuapple"];
+ repeated GpuNvidiaInfo gpu_nvidia = 27 [json_name = "gpu_nvidia"];
+ repeated GpuAmdInfo gpu_amd = 28 [json_name = "gpu_amd"];
+ map<string, string> slurm = 29;
+}
+
+message PythonPackagesRequest {
+ message PythonPackage {
+ string name = 1;
+ string version = 2;
+ }
+ repeated PythonPackage package = 1;
+}
diff --git a/wandb/proto/wandb_internal_codegen.py b/wandb/proto/wandb_internal_codegen.py
new file mode 100755
index 0000000000000000000000000000000000000000..e58c4b91242d3e815c4ca74d4bbb54767966055c
--- /dev/null
+++ b/wandb/proto/wandb_internal_codegen.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+
+import os
+import pathlib
+
+import grpc_tools # type: ignore
+from grpc_tools import protoc # type: ignore
+import importlib.metadata
+from packaging import version
+
+
+def generate_deprecated_class_definition() -> None:
+ """
+ Generate a class definition listing the deprecated features.
+ This is to allow static checks to ensure that proper field names are used.
+ """
+ from wandb.proto.wandb_telemetry_pb2 import Deprecated # type: ignore[import]
+
+ deprecated_features = Deprecated.DESCRIPTOR.fields_by_name.keys()
+
+ code: str = (
+ "# Generated by wandb/proto/wandb_internal_codegen.py. DO NOT EDIT!\n\n\n"
+ "import sys\n\n\n"
+ "if sys.version_info >= (3, 8):\n"
+ " from typing import Literal\n"
+ "else:\n"
+ " from typing_extensions import Literal\n\n\n"
+ "DEPRECATED_FEATURES = Literal[\n"
+ + ",\n".join(f' "{feature}"' for feature in deprecated_features)
+ + "\n"
+ + "]\n\n\n"
+ "class Deprecated:\n"
+ + "".join(
+ [
+ f' {feature}: DEPRECATED_FEATURES = "{feature}"\n'
+ for feature in deprecated_features
+ ]
+ )
+ )
+ with open("wandb/proto/wandb_deprecated.py", "w") as f:
+ f.write(code)
+
+
+def get_pip_package_version(package_name: str) -> str:
+ try:
+ return importlib.metadata.version(package_name)
+ except importlib.metadata.PackageNotFoundError:
+ raise ValueError(f"Package `{package_name}` not found")
+
+
+def get_min_required_version(requirements_file_name: str, package_name: str) -> str:
+ with open(requirements_file_name) as f:
+ lines = f.readlines()
+ for line in lines:
+ tokens = line.strip().split(">=")
+ if tokens[0] == package_name:
+ if len(tokens) == 2:
+ return tokens[1]
+ else:
+ raise ValueError(
+ f"Minimum version not specified for package `{package_name}`"
+ )
+ raise ValueError(f"Package `{package_name}` not found in requirements file")
+
+
+package: str = "grpcio-tools"
+package_version = get_pip_package_version(package)
+requirements_file: str = "../../requirements_build.txt"
+requirements_min_version = get_min_required_version(requirements_file, package)
+# check that the installed version of the package is at least the required version
+assert version.Version(package_version) >= version.Version(
+ requirements_min_version
+), f"Package {package} found={package_version} required>={requirements_min_version}"
+
+protobuf_version = version.Version(get_pip_package_version("protobuf"))
+
+proto_root = os.path.join(os.path.dirname(grpc_tools.__file__), "_proto")
+tmp_out: pathlib.Path = pathlib.Path(f"wandb/proto/v{protobuf_version.major}/")
+
+os.chdir("../..")
+for proto_file in [
+ "wandb_base.proto",
+ "wandb_internal.proto",
+ "wandb_settings.proto",
+ "wandb_telemetry.proto",
+ "wandb_server.proto",
+]:
+ ret = protoc.main(
+ (
+ "",
+ "-I",
+ proto_root,
+ "-I",
+ ".",
+ f"--python_out={tmp_out}",
+ f"--mypy_out={tmp_out}",
+ f"wandb/proto/{proto_file}",
+ )
+ )
+ assert not ret
+
+# clean up tmp dirs
+for p in (tmp_out / "wandb" / "proto").glob("*pb2*"):
+ p.rename(tmp_out / p.name)
+os.rmdir(tmp_out / "wandb" / "proto")
+os.rmdir(tmp_out / "wandb")
+
+generate_deprecated_class_definition()
diff --git a/wandb/proto/wandb_internal_pb2.py b/wandb/proto/wandb_internal_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..74ed8e94f889a84edda019b16f3a573a96b5c329
--- /dev/null
+++ b/wandb/proto/wandb_internal_pb2.py
@@ -0,0 +1,8 @@
+import google.protobuf
+
+protobuf_version = google.protobuf.__version__[0]
+
+if protobuf_version == "3":
+ from wandb.proto.v3.wandb_internal_pb2 import *
+elif protobuf_version == "4":
+ from wandb.proto.v4.wandb_internal_pb2 import *
diff --git a/wandb/proto/wandb_server.proto b/wandb/proto/wandb_server.proto
new file mode 100644
index 0000000000000000000000000000000000000000..000e4c0d70b5dcb9a2d8ad93a6e1297e5b759551
--- /dev/null
+++ b/wandb/proto/wandb_server.proto
@@ -0,0 +1,90 @@
+syntax = "proto3";
+
+package wandb_internal;
+
+import "wandb/proto/wandb_base.proto";
+import "wandb/proto/wandb_internal.proto";
+import "wandb/proto/wandb_settings.proto";
+
+message ServerShutdownRequest {
+ _RecordInfo _info = 200;
+}
+
+message ServerShutdownResponse {}
+
+message ServerStatusRequest {
+ _RecordInfo _info = 200;
+}
+
+message ServerStatusResponse {}
+
+message ServerInformInitRequest {
+ Settings settings = 1;
+ _RecordInfo _info = 200;
+}
+
+message ServerInformInitResponse {}
+
+message ServerInformStartRequest {
+ Settings settings = 1;
+ _RecordInfo _info = 200;
+}
+
+message ServerInformStartResponse {}
+
+message ServerInformFinishRequest {
+ _RecordInfo _info = 200;
+}
+
+message ServerInformFinishResponse {}
+
+message ServerInformAttachRequest {
+ _RecordInfo _info = 200;
+}
+
+message ServerInformAttachResponse {
+ Settings settings = 1;
+ _RecordInfo _info = 200;
+}
+
+message ServerInformDetachRequest {
+ _RecordInfo _info = 200;
+}
+
+message ServerInformDetachResponse {}
+
+message ServerInformTeardownRequest {
+ int32 exit_code = 1;
+ _RecordInfo _info = 200;
+}
+
+message ServerInformTeardownResponse {}
+
+/*
+ * ServerRequest, ServerResponse: used in sock server
+ */
+
+message ServerRequest {
+ oneof server_request_type {
+ Record record_publish = 1;
+ Record record_communicate = 2;
+ ServerInformInitRequest inform_init = 3;
+ ServerInformFinishRequest inform_finish = 4;
+ ServerInformAttachRequest inform_attach = 5;
+ ServerInformDetachRequest inform_detach = 6;
+ ServerInformTeardownRequest inform_teardown = 7;
+ ServerInformStartRequest inform_start = 8;
+ }
+}
+
+message ServerResponse {
+ oneof server_response_type {
+ Result result_communicate = 2;
+ ServerInformInitResponse inform_init_response = 3;
+ ServerInformFinishResponse inform_finish_response = 4;
+ ServerInformAttachResponse inform_attach_response = 5;
+ ServerInformDetachResponse inform_detach_response = 6;
+ ServerInformTeardownResponse inform_teardown_response = 7;
+ ServerInformStartResponse inform_start_response = 8;
+ }
+}
diff --git a/wandb/proto/wandb_server_pb2.py b/wandb/proto/wandb_server_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..8167564b3d92756906ee39c5d81f4d60fd714f67
--- /dev/null
+++ b/wandb/proto/wandb_server_pb2.py
@@ -0,0 +1,8 @@
+import google.protobuf
+
+protobuf_version = google.protobuf.__version__[0]
+
+if protobuf_version == "3":
+ from wandb.proto.v3.wandb_server_pb2 import *
+elif protobuf_version == "4":
+ from wandb.proto.v4.wandb_server_pb2 import *
diff --git a/wandb/proto/wandb_settings.proto b/wandb/proto/wandb_settings.proto
new file mode 100644
index 0000000000000000000000000000000000000000..a3b306d9e3e4d582c92527f1b37f850297f7da86
--- /dev/null
+++ b/wandb/proto/wandb_settings.proto
@@ -0,0 +1,194 @@
+syntax = "proto3";
+
+package wandb_internal;
+
+import "google/protobuf/wrappers.proto";
+
+message ListStringValue {
+ repeated string value = 1;
+}
+
+message MapStringKeyStringValue {
+ map<string, string> value = 1;
+}
+
+message MapStringKeyMapStringKeyStringValue {
+ map<string, MapStringKeyStringValue> value = 1;
+}
+
+message OpenMetricsFilters {
+ oneof value {
+ ListStringValue sequence = 1;
+ MapStringKeyMapStringKeyStringValue mapping = 2;
+ }
+}
+
+message Settings {
+ ListStringValue _args = 1;
+ google.protobuf.BoolValue _aws_lambda = 2;
+ google.protobuf.Int32Value _async_upload_concurrency_limit = 3;
+ google.protobuf.BoolValue _cli_only_mode = 4;
+ google.protobuf.BoolValue _colab = 5;
+ google.protobuf.StringValue _cuda = 6;
+ google.protobuf.BoolValue _disable_meta = 7;
+ google.protobuf.BoolValue _disable_service = 8;
+ google.protobuf.BoolValue _disable_setproctitle = 9;
+ google.protobuf.BoolValue _disable_stats = 10;
+ google.protobuf.BoolValue _disable_viewer = 11;
+ google.protobuf.BoolValue _except_exit = 12;
+ google.protobuf.StringValue _executable = 13;
+ MapStringKeyStringValue _extra_http_headers = 14;
+ google.protobuf.DoubleValue _file_stream_timeout_seconds = 15;
+ google.protobuf.BoolValue _flow_control_custom = 16;
+ google.protobuf.BoolValue _flow_control_disabled = 17;
+ google.protobuf.DoubleValue _internal_check_process = 18;
+ google.protobuf.DoubleValue _internal_queue_timeout = 19;
+ google.protobuf.BoolValue _ipython = 20;
+ google.protobuf.BoolValue _jupyter = 21;
+ google.protobuf.StringValue _jupyter_root = 22;
+ google.protobuf.BoolValue _kaggle = 23;
+ google.protobuf.Int32Value _live_policy_rate_limit = 24;
+ google.protobuf.Int32Value _live_policy_wait_time = 25;
+ google.protobuf.Int32Value _log_level = 26;
+ google.protobuf.Int32Value _network_buffer = 27;
+ google.protobuf.BoolValue _noop = 28;
+ google.protobuf.BoolValue _notebook = 29;
+ google.protobuf.BoolValue _offline = 30;
+ google.protobuf.BoolValue _sync = 31;
+ google.protobuf.StringValue _os = 32;
+ google.protobuf.StringValue _platform = 33;
+ google.protobuf.StringValue _python = 34;
+ google.protobuf.StringValue _runqueue_item_id = 35;
+ google.protobuf.BoolValue _require_core = 36;
+ google.protobuf.BoolValue _save_requirements = 37;
+ google.protobuf.StringValue _service_transport = 38;
+ google.protobuf.DoubleValue _service_wait = 39;
+ google.protobuf.StringValue _start_datetime = 40;
+ google.protobuf.DoubleValue _start_time = 41;
+ google.protobuf.Int32Value _stats_pid = 42;
+ google.protobuf.DoubleValue _stats_sample_rate_seconds = 43;
+ google.protobuf.Int32Value _stats_samples_to_average = 44;
+ google.protobuf.BoolValue _stats_join_assets = 45;
+ google.protobuf.StringValue _stats_neuron_monitor_config_path = 46;
+ MapStringKeyStringValue _stats_open_metrics_endpoints = 47;
+ OpenMetricsFilters _stats_open_metrics_filters = 48;
+ google.protobuf.StringValue _tmp_code_dir = 49;
+ google.protobuf.StringValue _tracelog = 50;
+ ListStringValue _unsaved_keys = 51;
+ google.protobuf.BoolValue _windows = 52;
+ google.protobuf.BoolValue allow_val_change = 53;
+ google.protobuf.StringValue anonymous = 54;
+ google.protobuf.StringValue api_key = 55;
+ MapStringKeyStringValue azure_account_url_to_access_key = 56;
+ google.protobuf.StringValue base_url = 57;
+ google.protobuf.StringValue code_dir = 58;
+ ListStringValue config_paths = 59;
+ google.protobuf.StringValue console = 60;
+ google.protobuf.StringValue deployment = 61;
+ google.protobuf.BoolValue disable_code = 62;
+ google.protobuf.BoolValue disable_git = 63;
+ google.protobuf.BoolValue disable_hints = 64;
+ google.protobuf.BoolValue disable_job_creation = 65;
+ google.protobuf.BoolValue disabled = 66;
+ google.protobuf.StringValue docker = 67;
+ google.protobuf.StringValue email = 68;
+ google.protobuf.StringValue entity = 69;
+ google.protobuf.StringValue files_dir = 70;
+ google.protobuf.BoolValue force = 71;
+ google.protobuf.StringValue git_commit = 72;
+ google.protobuf.StringValue git_remote = 73;
+ google.protobuf.StringValue git_remote_url = 74;
+ google.protobuf.StringValue git_root = 75;
+ google.protobuf.Int32Value heartbeat_seconds = 76;
+ google.protobuf.StringValue host = 77;
+ ListStringValue ignore_globs = 78;
+ google.protobuf.DoubleValue init_timeout = 79;
+ google.protobuf.BoolValue is_local = 80;
+ google.protobuf.StringValue job_source = 81;
+ google.protobuf.BoolValue label_disable = 82;
+ google.protobuf.BoolValue launch = 83;
+ google.protobuf.StringValue launch_config_path = 84;
+ google.protobuf.StringValue log_dir = 85;
+ google.protobuf.StringValue log_internal = 86;
+ google.protobuf.StringValue log_symlink_internal = 87;
+ google.protobuf.StringValue log_symlink_user = 88;
+ google.protobuf.StringValue log_user = 89;
+ google.protobuf.DoubleValue login_timeout = 90;
+ google.protobuf.StringValue mode = 92;
+ google.protobuf.StringValue notebook_name = 93;
+ google.protobuf.StringValue problem = 94;
+ google.protobuf.StringValue program = 95;
+ google.protobuf.StringValue program_relpath = 96;
+ google.protobuf.StringValue project = 97;
+ google.protobuf.StringValue project_url = 98;
+ google.protobuf.BoolValue quiet = 99;
+ google.protobuf.BoolValue reinit = 100;
+ google.protobuf.BoolValue relogin = 101;
+ google.protobuf.StringValue resume = 102;
+ google.protobuf.StringValue resume_fname = 103;
+ google.protobuf.BoolValue resumed = 104;
+ google.protobuf.StringValue root_dir = 105;
+ google.protobuf.StringValue run_group = 106;
+ google.protobuf.StringValue run_id = 107;
+ google.protobuf.StringValue run_job_type = 108;
+ google.protobuf.StringValue run_mode = 109;
+ google.protobuf.StringValue run_name = 110;
+ google.protobuf.StringValue run_notes = 111;
+ ListStringValue run_tags = 112;
+ google.protobuf.StringValue run_url = 113;
+ google.protobuf.BoolValue sagemaker_disable = 114;
+ google.protobuf.BoolValue save_code = 115;
+ google.protobuf.StringValue settings_system = 116;
+ google.protobuf.StringValue settings_workspace = 117;
+ google.protobuf.BoolValue show_colors = 118;
+ google.protobuf.BoolValue show_emoji = 119;
+ google.protobuf.BoolValue show_errors = 120;
+ google.protobuf.BoolValue show_info = 121;
+ google.protobuf.BoolValue show_warnings = 122;
+ google.protobuf.BoolValue silent = 123;
+ google.protobuf.StringValue start_method = 124;
+ google.protobuf.BoolValue strict = 125;
+ google.protobuf.Int32Value summary_errors = 126;
+ google.protobuf.Int32Value summary_timeout = 127;
+ google.protobuf.Int32Value summary_warnings = 128;
+ google.protobuf.StringValue sweep_id = 129;
+ google.protobuf.StringValue sweep_param_path = 130;
+ google.protobuf.StringValue sweep_url = 131;
+ google.protobuf.BoolValue symlink = 132;
+ google.protobuf.StringValue sync_dir = 133;
+ google.protobuf.StringValue sync_file = 134;
+ google.protobuf.StringValue sync_symlink_latest = 135;
+ google.protobuf.Int32Value system_sample = 136;
+ google.protobuf.Int32Value system_sample_seconds = 137;
+ google.protobuf.BoolValue table_raise_on_max_row_limit_exceeded = 138;
+ google.protobuf.StringValue timespec = 139;
+ google.protobuf.StringValue tmp_dir = 140;
+ google.protobuf.StringValue username = 141;
+ google.protobuf.StringValue wandb_dir = 142;
+ google.protobuf.StringValue _jupyter_name = 143;
+ google.protobuf.StringValue _jupyter_path = 144;
+ google.protobuf.StringValue job_name = 145;
+ ListStringValue _stats_disk_paths = 146;
+ google.protobuf.Int32Value _file_stream_retry_max = 147;
+ google.protobuf.DoubleValue _file_stream_retry_wait_min_seconds = 148;
+ google.protobuf.DoubleValue _file_stream_retry_wait_max_seconds = 149;
+ google.protobuf.Int32Value _file_transfer_retry_max = 150;
+ google.protobuf.DoubleValue _file_transfer_retry_wait_min_seconds = 151;
+ google.protobuf.DoubleValue _file_transfer_retry_wait_max_seconds = 152;
+ google.protobuf.DoubleValue _file_transfer_timeout_seconds = 153;
+ google.protobuf.Int32Value _graphql_retry_max = 154;
+ google.protobuf.DoubleValue _graphql_retry_wait_min_seconds = 155;
+ google.protobuf.DoubleValue _graphql_retry_wait_max_seconds = 156;
+ google.protobuf.DoubleValue _graphql_timeout_seconds = 157;
+ google.protobuf.BoolValue _disable_machine_info = 158;
+ google.protobuf.StringValue program_abspath = 159;
+ google.protobuf.StringValue colab_url = 160;
+ google.protobuf.Int32Value _stats_buffer_size = 161;
+ google.protobuf.BoolValue _shared = 162;
+
+ MapStringKeyStringValue _proxies = 200;
+
+ // todo?
+ // map<google.protobuf.StringValue, google.protobuf.Int32Value> _sources =
+ // 500;
+}
diff --git a/wandb/proto/wandb_settings_pb2.py b/wandb/proto/wandb_settings_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..794ff4165881f29874c074b662b6e939935ce45a
--- /dev/null
+++ b/wandb/proto/wandb_settings_pb2.py
@@ -0,0 +1,8 @@
+import google.protobuf
+
+protobuf_version = google.protobuf.__version__[0]
+
+if protobuf_version == "3":
+ from wandb.proto.v3.wandb_settings_pb2 import *
+elif protobuf_version == "4":
+ from wandb.proto.v4.wandb_settings_pb2 import *
diff --git a/wandb/proto/wandb_telemetry.proto b/wandb/proto/wandb_telemetry.proto
new file mode 100644
index 0000000000000000000000000000000000000000..5a4de79d5719e0a08a0e69b6369bf0fb66977fa3
--- /dev/null
+++ b/wandb/proto/wandb_telemetry.proto
@@ -0,0 +1,241 @@
+syntax = "proto3";
+
+package wandb_internal;
+
+import "wandb/proto/wandb_base.proto";
+
+/*
+ * Telemetry
+ */
+message TelemetryRecord {
+ Imports imports_init = 1;
+ Imports imports_finish = 2;
+ Feature feature = 3;
+ string python_version = 4;
+ string cli_version = 5;
+ string huggingface_version = 6;
+ // string framework = 7;
+ Env env = 8;
+ Labels label = 9;
+ Deprecated deprecated = 10;
+ Issues issues = 11;
+ string core_version = 12;
+ string platform = 13;
+ _RecordInfo _info = 200;
+}
+
+message TelemetryResult {}
+
+message Imports {
+ bool torch = 1;
+ bool keras = 2;
+ bool tensorflow = 3;
+ bool fastai = 4;
+ bool sklearn = 5;
+ bool xgboost = 6;
+ bool catboost = 7;
+ bool lightgbm = 8;
+ bool pytorch_lightning = 9;
+ bool ignite = 10;
+ bool transformers = 11;
+ bool jax = 12;
+ bool metaflow = 13;
+ bool allennlp = 14;
+ bool autogluon = 15;
+ bool autokeras = 16;
+ // bool avalanche = 17;
+ bool catalyst = 18;
+ // bool dalle_pytorch = 19;
+ // bool datasets = 20;
+ bool deepchem = 21;
+ bool deepctr = 22;
+ // bool deeppavlov = 23;
+ // bool detectron = 24;
+ // bool paddle = 25;
+ // bool parlai = 26;
+ // bool prophet = 27;
+ bool pycaret = 28;
+ bool pytorchvideo = 29;
+ bool ray = 30;
+ bool simpletransformers = 31;
+ bool skorch = 32;
+ bool spacy = 33;
+ bool flash = 34;
+ bool optuna = 35;
+ bool recbole = 36;
+ bool mmcv = 37;
+ bool mmdet = 38;
+ bool torchdrug = 39;
+ bool torchtext = 40;
+ bool torchvision = 41;
+ bool elegy = 42;
+ bool detectron2 = 43;
+ bool flair = 44;
+ bool flax = 45;
+ bool syft = 46;
+ bool TTS = 47;
+ bool monai = 48;
+ bool huggingface_hub = 49;
+ bool hydra = 50;
+ bool datasets = 51;
+ bool sacred = 52;
+ bool joblib = 53;
+ bool dask = 54;
+ bool asyncio = 55;
+ bool paddleocr = 56;
+ bool ppdet = 57;
+ bool paddleseg = 58;
+ bool paddlenlp = 59;
+ bool mmseg = 60;
+ bool mmocr = 61;
+ bool mmcls = 62;
+ bool timm = 63;
+ bool fairseq = 64;
+ bool deepchecks = 65;
+ bool composer = 66;
+ bool sparseml = 67;
+ bool anomalib = 68;
+ bool zenml = 69;
+ bool colossalai = 70;
+ bool accelerate = 71;
+ bool merlin = 72;
+ bool nanodet = 73;
+ bool segmentation_models_pytorch = 74;
+ bool sentence_transformers = 75;
+ bool dgl = 76;
+ bool torch_geometric = 77;
+ bool jina = 78;
+ bool kornia = 79;
+ bool albumentations = 80;
+ bool keras_cv = 81;
+ bool mmengine = 82;
+ bool diffusers = 83;
+ bool trl = 84;
+ bool trlx = 85;
+ bool langchain = 86;
+ bool llama_index = 87;
+ bool stability_sdk = 88;
+ bool prefect = 89;
+ bool prefect_ray = 90;
+ bool pinecone = 91; // pinecone-client
+ bool chromadb = 92;
+ bool weaviate = 93; // weaviate-client
+ bool promptlayer = 94;
+ bool openai = 95;
+ bool cohere = 96;
+ bool anthropic = 97;
+ bool peft = 98;
+ bool optimum = 99;
+ bool evaluate = 100;
+ bool langflow = 101;
+ bool keras_core = 102; // keras-core
+ bool lightning_fabric = 103; // lightning-fabric
+ bool curated_transformers = 104; // curated-transformers
+ bool orjson = 105;
+}
+
+message Feature {
+ bool watch = 1; // wandb.watch() called
+ bool finish = 2; // wandb.finish() called
+ bool save = 3; // wandb.save() called
+ bool offline = 4; // offline run was synced
+ bool resumed = 5; // run was resumed
+ bool grpc = 6; // grpc-server (java integration)
+ bool metric = 7; // define_metric() called
+ bool keras = 8; // Keras WandbCallback used
+ bool sagemaker = 9; // User is using sagemaker
+ bool artifact_incremental = 10; // Artifact(incremental=True) used
+ bool metaflow = 11; // Using metaflow integration
+ bool prodigy = 12; // Using prodigy integration
+ bool set_init_name = 13; // users set run name from wandb.init
+ bool set_init_id = 14; // users set run id from wandb.init
+ bool set_init_tags = 15; // users set tags within wandb.init
+ bool set_init_config = 16; // users set run config in wandb.init
+ bool set_run_name = 17; // user sets run name via wandb.run.name = ...
+ bool set_run_tags = 18; // user sets run name via wandb.run.tags = ...
+ bool set_config_item = 19; // users set key in run config via run.config.key
+ // or run.config["key"]
+ bool launch = 20; // run is created through wandb launch
+ bool torch_profiler_trace = 21; // wandb.profiler.torch_trace_handler() called
+ bool sb3 = 22; // Using stable_baselines3 integration
+ bool service = 23; // Using wandb service internal process
+ bool init_return_run = 24; // wandb.init() called in the same process returning previous run
+ bool lightgbm_wandb_callback = 25; // lightgbm callback used
+ bool lightgbm_log_summary = 26; // lightgbm log summary used
+ bool catboost_wandb_callback = 27; // catboost callback used
+ bool catboost_log_summary = 28; // catboost log summary used
+ bool tensorboard_log = 29; // wandb.tensorflow.log or wandb.tensorboard.log used
+ bool estimator_hook = 30; // wandb.tensorflow.WandbHook used
+ bool xgboost_wandb_callback = 31; // xgboost callback used
+ bool xgboost_old_wandb_callback = 32; // xgboost old callback used (to be depreciated)
+ bool attach = 33; // attach to a run in another process
+ bool tensorboard_patch = 34; // wandb.tensorboard.patch(...)
+ bool tensorboard_sync = 35; // wandb.init(sync_tensorboard=True)
+ bool kfp_wandb_log = 36; // wandb.integration.kfp.wandb_log
+ bool maybe_run_overwrite = 37; // Run might have been overwritten
+ bool keras_metrics_logger = 38; // Keras WandbMetricsLogger used
+ bool keras_model_checkpoint = 39; // Keras WandbModelCheckpoint used
+ bool keras_wandb_eval_callback = 40; // Keras WandbEvalCallback used
+ bool flow_control_overflow = 41; // Hit flow control threshold
+ bool sync = 42; // Run was synced with wandb sync
+ bool flow_control_disabled = 43; // Flow control disabled by user
+ bool flow_control_custom = 44; // Flow control customized by user
+ bool service_disabled = 45; // Service disabled by user
+ bool open_metrics = 46; // Consuming metrics from an OpenMetrics endpoint
+ bool ultralytics_yolov8 = 47; // Ultralytics YOLOv8 integration callbacks used
+ bool importer_mlflow = 48; // Using Import API for MLFlow
+ bool sync_tfevents = 49; // Using wandb sync for tfevent files
+ bool async_uploads = 50; // Async file uploads enabled by user
+ bool openai_autolog = 51; // OpenAI autolog used
+ bool langchain_tracer = 52; // Langchain wandb tracer callback used
+ bool cohere_autolog = 53; // Cohere autolog used
+ bool hf_pipeline_autolog = 54; // HuggingFace Autologging
+ bool core = 55; // Using wandb core internal process
+ bool lib_c = 56; // Using c wandb library
+ bool lib_cpp = 57; // Using cpp wandb library
+ bool openai_finetuning = 58; // Using openai finetuning WandbLogger
+ bool diffusers_autolog = 59; // Using Diffusers autologger
+}
+
+message Env {
+ bool jupyter = 1; // jupyter env detected
+ bool kaggle = 2; // kaggle env detected
+ bool windows = 3; // windows detected
+ bool m1_gpu = 4; // apple silicon M1 gpu found
+ bool start_spawn = 5; // multiprocessing spawn
+ bool start_fork = 6; // multiprocessing fork
+ bool start_forkserver = 7; // multiprocessing forkserver
+ bool start_thread = 8; // thread start method
+ bool maybe_mp = 9; // maybe user running multiprocessing
+ bool trainium = 10; // AWS Trainium env detected
+ bool pex = 11; // pex env detected
+ bool colab = 12; // colab env detected
+ bool ipython = 13; // ipython env detected
+ bool aws_lambda = 14; // running in AWS Lambda
+ bool amd_gpu = 15; // AMD GPU detected
+}
+
+message Labels {
+ string code_string = 1; // code identification
+ string repo_string = 2; // repo identification
+ string code_version = 3; // code version
+}
+
+message Deprecated {
+ bool keras_callback__data_type = 1; // wandb.keras.WandbCallback(data_type=...) called
+ bool run__mode = 2; // wandb.run.mode called
+ bool run__save_no_args = 3; // wandb.run.save() called without arguments
+ bool run__join = 4; // wandb.run.join() called
+ bool plots = 5; // wandb.plots.* called
+ bool run__log_sync = 6; // wandb.run.log(sync=...) called
+ bool init__config_include_keys = 7; // wandb.init(config_include_keys=...) called
+ bool init__config_exclude_keys = 8; // wandb.init(config_exclude_keys=...) called
+ bool keras_callback__save_model = 9; // wandb.keras.WandbCallback(save_model=True) called
+ bool langchain_tracer = 10; // wandb.integration.langchain.WandbTracer called
+}
+
+message Issues {
+ bool settings__validation_warnings = 1; // validation warnings for settings
+ bool settings__unexpected_args = 2; // unexpected settings init args
+ bool settings__preprocessing_warnings = 3; // settings preprocessing warnings
+}
diff --git a/wandb/proto/wandb_telemetry_pb2.py b/wandb/proto/wandb_telemetry_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..74454ce91aaa4c5dd6719031a6828ac9552b61b8
--- /dev/null
+++ b/wandb/proto/wandb_telemetry_pb2.py
@@ -0,0 +1,8 @@
+import google.protobuf
+
+protobuf_version = google.protobuf.__version__[0]
+
+if protobuf_version == "3":
+ from wandb.proto.v3.wandb_telemetry_pb2 import *
+elif protobuf_version == "4":
+ from wandb.proto.v4.wandb_telemetry_pb2 import *
diff --git a/wandb/py.typed b/wandb/py.typed
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sacred/__init__.py b/wandb/sacred/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..a558e407357eabd96cd169dafb6642f7586a32ab
--- /dev/null
+++ b/wandb/sacred/__init__.py
@@ -0,0 +1,3 @@
+from wandb.integration.sacred import WandbObserver
+
+__all__ = ["WandbObserver"]
diff --git a/wandb/sdk/__init__.py b/wandb/sdk/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..da11ef33dfb9175f959359904c8d08036cdf928f
--- /dev/null
+++ b/wandb/sdk/__init__.py
@@ -0,0 +1,37 @@
+"""module sdk."""
+
+__all__ = (
+ "Config",
+ "Settings",
+ "Summary",
+ "Artifact",
+ "AlertLevel",
+ "init",
+ "setup",
+ "_attach",
+ "_sync",
+ "login",
+ "require",
+ "finish",
+ "teardown",
+ "watch",
+ "unwatch",
+ "sweep",
+ "controller",
+ "helper",
+)
+
+from . import wandb_helper as helper
+from .artifacts.artifact import Artifact
+from .wandb_alerts import AlertLevel
+from .wandb_config import Config
+from .wandb_init import _attach, init
+from .wandb_login import login
+from .wandb_require import require
+from .wandb_run import finish
+from .wandb_settings import Settings
+from .wandb_setup import setup, teardown
+from .wandb_summary import Summary
+from .wandb_sweep import controller, sweep
+from .wandb_sync import _sync
+from .wandb_watch import unwatch, watch
diff --git a/wandb/sdk/artifacts/__init__.py b/wandb/sdk/artifacts/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sdk/artifacts/artifact.py b/wandb/sdk/artifacts/artifact.py
new file mode 100644
index 0000000000000000000000000000000000000000..8ffa19a68090b208ef4219146a30ae669a41b118
--- /dev/null
+++ b/wandb/sdk/artifacts/artifact.py
@@ -0,0 +1,2336 @@
+"""Artifact class."""
+import atexit
+import concurrent.futures
+import contextlib
+import json
+import multiprocessing.dummy
+import os
+import re
+import shutil
+import tempfile
+import time
+from copy import copy
+from datetime import datetime, timedelta
+from functools import partial
+from pathlib import PurePosixPath
+from typing import (
+ IO,
+ TYPE_CHECKING,
+ Any,
+ Dict,
+ Generator,
+ List,
+ Optional,
+ Sequence,
+ Set,
+ Tuple,
+ Type,
+ Union,
+ cast,
+)
+from urllib.parse import urlparse
+
+import requests
+
+import wandb
+from wandb import data_types, env, util
+from wandb.apis.normalize import normalize_exceptions
+from wandb.apis.public import ArtifactCollection, ArtifactFiles, RetryingClient, Run
+from wandb.data_types import WBValue
+from wandb.errors.term import termerror, termlog, termwarn
+from wandb.sdk.artifacts.artifact_download_logger import ArtifactDownloadLogger
+from wandb.sdk.artifacts.artifact_instance_cache import artifact_instance_cache
+from wandb.sdk.artifacts.artifact_manifest import ArtifactManifest
+from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+from wandb.sdk.artifacts.artifact_manifests.artifact_manifest_v1 import (
+ ArtifactManifestV1,
+)
+from wandb.sdk.artifacts.artifact_state import ArtifactState
+from wandb.sdk.artifacts.artifact_ttl import ArtifactTTL
+from wandb.sdk.artifacts.exceptions import (
+ ArtifactFinalizedError,
+ ArtifactNotLoggedError,
+ WaitTimeoutError,
+)
+from wandb.sdk.artifacts.staging import get_staging_dir
+from wandb.sdk.artifacts.storage_layout import StorageLayout
+from wandb.sdk.artifacts.storage_policies import WANDB_STORAGE_POLICY
+from wandb.sdk.artifacts.storage_policy import StoragePolicy
+from wandb.sdk.data_types._dtypes import Type as WBType
+from wandb.sdk.data_types._dtypes import TypeRegistry
+from wandb.sdk.internal.internal_api import Api as InternalApi
+from wandb.sdk.internal.thread_local_settings import _thread_local_api_settings
+from wandb.sdk.lib import filesystem, retry, runid, telemetry
+from wandb.sdk.lib.hashutil import B64MD5, b64_to_hex_id, md5_file_b64
+from wandb.sdk.lib.mailbox import Mailbox
+from wandb.sdk.lib.paths import FilePathStr, LogicalPath, StrPath, URIStr
+from wandb.sdk.lib.runid import generate_id
+from wandb.util import get_core_path
+
+reset_path = util.vendor_setup()
+
+from wandb_gql import gql # noqa: E402
+
+reset_path()
+
+if TYPE_CHECKING:
+ from wandb.sdk.lib.mailbox import MailboxHandle
+
+
+class Artifact:
+ """Flexible and lightweight building block for dataset and model versioning.
+
+ Constructs an empty artifact whose contents can be populated using its `add` family
+ of functions. Once the artifact has all the desired files, you can call
+ `wandb.log_artifact()` to log it.
+
+ Arguments:
+ name: A human-readable name for this artifact, which is how you can identify
+ this artifact in the UI or reference it in `use_artifact` calls. Names can
+ contain letters, numbers, underscores, hyphens, and dots. The name must be
+ unique across a project.
+ type: The type of the artifact, which is used to organize and differentiate
+ artifacts. Common types include `dataset` or `model`, but you can use any
+ string containing letters, numbers, underscores, hyphens, and dots.
+ description: Free text that offers a description of the artifact. The
+ description is markdown rendered in the UI, so this is a good place to place
+ tables, links, etc.
+ metadata: Structured data associated with the artifact, for example class
+ distribution of a dataset. This will eventually be queryable and plottable
+ in the UI. There is a hard limit of 100 total keys.
+
+ Returns:
+ An `Artifact` object.
+
+ Examples:
+ Basic usage:
+ ```python
+ wandb.init()
+
+ artifact = wandb.Artifact("mnist", type="dataset")
+ artifact.add_dir("mnist/")
+ wandb.log_artifact(artifact)
+ ```
+ """
+
+ _TMP_DIR = tempfile.TemporaryDirectory("wandb-artifacts")
+ atexit.register(_TMP_DIR.cleanup)
+
+ def __init__(
+ self,
+ name: str,
+ type: str,
+ description: Optional[str] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ incremental: bool = False,
+ use_as: Optional[str] = None,
+ ) -> None:
+ if not re.match(r"^[a-zA-Z0-9_\-.]+$", name):
+ raise ValueError(
+ f"Artifact name may only contain alphanumeric characters, dashes, "
+ f"underscores, and dots. Invalid name: {name}"
+ )
+ if type == "job" or type.startswith("wandb-"):
+ raise ValueError(
+ "Artifact types 'job' and 'wandb-*' are reserved for internal use. "
+ "Please use a different type."
+ )
+ if incremental:
+ termwarn("Using experimental arg `incremental`")
+
+ # Internal.
+ self._client: Optional[RetryingClient] = None
+
+ storage_policy_cls = StoragePolicy.lookup_by_name(WANDB_STORAGE_POLICY)
+ layout = StorageLayout.V1 if env.get_use_v1_artifacts() else StorageLayout.V2
+ policy_config = {"storageLayout": layout}
+ self._storage_policy = storage_policy_cls.from_config(config=policy_config)
+
+ self._tmp_dir: Optional[tempfile.TemporaryDirectory] = None
+ self._added_objs: Dict[
+ int, Tuple[data_types.WBValue, ArtifactManifestEntry]
+ ] = {}
+ self._added_local_paths: Dict[str, ArtifactManifestEntry] = {}
+ self._save_handle: Optional["MailboxHandle"] = None
+ self._download_roots: Set[str] = set()
+ # Set by new_draft(), otherwise the latest artifact will be used as the base.
+ self._base_id: Optional[str] = None
+ # Properties.
+ self._id: Optional[str] = None
+ self._client_id: str = runid.generate_id(128)
+ self._sequence_client_id: str = runid.generate_id(128)
+ self._entity: Optional[str] = None
+ self._project: Optional[str] = None
+ self._name: str = name # includes version after saving
+ self._version: Optional[str] = None
+ self._source_entity: Optional[str] = None
+ self._source_project: Optional[str] = None
+ self._source_name: str = name # includes version after saving
+ self._source_version: Optional[str] = None
+ self._type: str = type
+ self._description: Optional[str] = description
+ self._metadata: dict = self._normalize_metadata(metadata)
+ self._ttl_duration_seconds: Optional[int] = None
+ self._ttl_is_inherited: bool = True
+ self._ttl_changed: bool = False
+ self._aliases: List[str] = []
+ self._saved_aliases: List[str] = []
+ self._distributed_id: Optional[str] = None
+ self._incremental: bool = incremental
+ self._use_as: Optional[str] = use_as
+ self._state: ArtifactState = ArtifactState.PENDING
+ self._manifest: Optional[ArtifactManifest] = ArtifactManifestV1(
+ self._storage_policy
+ )
+ self._commit_hash: Optional[str] = None
+ self._file_count: Optional[int] = None
+ self._created_at: Optional[str] = None
+ self._updated_at: Optional[str] = None
+ self._final: bool = False
+
+ # Cache.
+ artifact_instance_cache[self._client_id] = self
+
+ def __repr__(self) -> str:
+ return f"<Artifact {self.id or self.name}>"
+
+ @classmethod
+ def _from_id(cls, artifact_id: str, client: RetryingClient) -> Optional["Artifact"]:
+ artifact = artifact_instance_cache.get(artifact_id)
+ if artifact is not None:
+ return artifact
+
+ query = gql(
+ """
+ query ArtifactByID($id: ID!) {
+ artifact(id: $id) {
+ ...ArtifactFragment
+ currentManifest {
+ file {
+ directUrl
+ }
+ }
+ }
+ }
+ """
+ + cls._get_gql_artifact_fragment()
+ )
+ response = client.execute(
+ query,
+ variable_values={"id": artifact_id},
+ )
+ attrs = response.get("artifact")
+ if attrs is None:
+ return None
+ entity = attrs["artifactSequence"]["project"]["entityName"]
+ project = attrs["artifactSequence"]["project"]["name"]
+ name = "{}:v{}".format(attrs["artifactSequence"]["name"], attrs["versionIndex"])
+ return cls._from_attrs(entity, project, name, attrs, client)
+
+ @classmethod
+ def _from_name(
+ cls, entity: str, project: str, name: str, client: RetryingClient
+ ) -> "Artifact":
+ query = gql(
+ """
+ query ArtifactByName(
+ $entityName: String!,
+ $projectName: String!,
+ $name: String!
+ ) {
+ project(name: $projectName, entityName: $entityName) {
+ artifact(name: $name) {
+ ...ArtifactFragment
+ }
+ }
+ }
+ """
+ + cls._get_gql_artifact_fragment()
+ )
+ response = client.execute(
+ query,
+ variable_values={
+ "entityName": entity,
+ "projectName": project,
+ "name": name,
+ },
+ )
+ attrs = response.get("project", {}).get("artifact")
+ if attrs is None:
+ raise ValueError(
+ f"Unable to fetch artifact with name {entity}/{project}/{name}"
+ )
+ return cls._from_attrs(entity, project, name, attrs, client)
+
+ @classmethod
+ def _from_attrs(
+ cls,
+ entity: str,
+ project: str,
+ name: str,
+ attrs: Dict[str, Any],
+ client: RetryingClient,
+ ) -> "Artifact":
+ # Placeholder is required to skip validation.
+ artifact = cls("placeholder", type="placeholder")
+ artifact._client = client
+ artifact._id = attrs["id"]
+ artifact._entity = entity
+ artifact._project = project
+ artifact._name = name
+ aliases = [
+ alias["alias"]
+ for alias in attrs["aliases"]
+ if alias["artifactCollection"]["project"]["entityName"] == entity
+ and alias["artifactCollection"]["project"]["name"] == project
+ and alias["artifactCollection"]["name"] == name.split(":")[0]
+ ]
+ version_aliases = [
+ alias for alias in aliases if util.alias_is_version_index(alias)
+ ]
+ assert len(version_aliases) == 1
+ artifact._version = version_aliases[0]
+ artifact._source_entity = attrs["artifactSequence"]["project"]["entityName"]
+ artifact._source_project = attrs["artifactSequence"]["project"]["name"]
+ artifact._source_name = "{}:v{}".format(
+ attrs["artifactSequence"]["name"], attrs["versionIndex"]
+ )
+ artifact._source_version = "v{}".format(attrs["versionIndex"])
+ artifact._type = attrs["artifactType"]["name"]
+ artifact._description = attrs["description"]
+ artifact.metadata = cls._normalize_metadata(
+ json.loads(attrs["metadata"] or "{}")
+ )
+ artifact._ttl_duration_seconds = artifact._ttl_duration_seconds_from_gql(
+ attrs.get("ttlDurationSeconds")
+ )
+ artifact._ttl_is_inherited = (
+ True if attrs.get("ttlIsInherited") is None else attrs["ttlIsInherited"]
+ )
+ artifact._aliases = [
+ alias for alias in aliases if not util.alias_is_version_index(alias)
+ ]
+ artifact._saved_aliases = copy(artifact._aliases)
+ artifact._state = ArtifactState(attrs["state"])
+ if "currentManifest" in attrs:
+ artifact._load_manifest(attrs["currentManifest"]["file"]["directUrl"])
+ else:
+ artifact._manifest = None
+ artifact._commit_hash = attrs["commitHash"]
+ artifact._file_count = attrs["fileCount"]
+ artifact._created_at = attrs["createdAt"]
+ artifact._updated_at = attrs["updatedAt"]
+ artifact._final = True
+ # Cache.
+
+ assert artifact.id is not None
+ artifact_instance_cache[artifact.id] = artifact
+ return artifact
+
+ def new_draft(self) -> "Artifact":
+ """Create a new draft artifact with the same content as this committed artifact.
+
+ The artifact returned can be extended or modified and logged as a new version.
+
+ Raises:
+ ArtifactNotLoggedError: if the artifact has not been logged
+ """
+ self._ensure_logged("new_draft")
+
+ # Name, _entity and _project are set to the *source* name/entity/project:
+ # if this artifact is saved it must be saved to the source sequence.
+ artifact = Artifact(self.source_name.split(":")[0], self.type)
+ artifact._entity = self._source_entity
+ artifact._project = self._source_project
+ artifact._source_entity = self._source_entity
+ artifact._source_project = self._source_project
+
+ # This artifact's parent is the one we are making a draft from.
+ artifact._base_id = self.id
+
+ # We can reuse the client, and copy over all the attributes that aren't
+ # version-dependent and don't depend on having been logged.
+ artifact._client = self._client
+ artifact._description = self.description
+ artifact._metadata = self.metadata
+ artifact._manifest = ArtifactManifest.from_manifest_json(
+ self.manifest.to_manifest_json()
+ )
+ return artifact
+
+ # Properties.
+
+ @property
+ def id(self) -> Optional[str]:
+ """The artifact's ID."""
+ if self.is_draft():
+ return None
+ assert self._id is not None
+ return self._id
+
+ @property
+ def entity(self) -> str:
+ """The name of the entity of the secondary (portfolio) artifact collection."""
+ self._ensure_logged("entity")
+ assert self._entity is not None
+ return self._entity
+
+ @property
+ def project(self) -> str:
+ """The name of the project of the secondary (portfolio) artifact collection."""
+ self._ensure_logged("project")
+ assert self._project is not None
+ return self._project
+
+ @property
+ def name(self) -> str:
+ """The artifact name and version in its secondary (portfolio) collection.
+
+ A string with the format {collection}:{alias}. Before the artifact is saved,
+ contains only the name since the version is not yet known.
+ """
+ return self._name
+
+ @property
+ def qualified_name(self) -> str:
+ """The entity/project/name of the secondary (portfolio) collection."""
+ return f"{self.entity}/{self.project}/{self.name}"
+
+ @property
+ def version(self) -> str:
+ """The artifact's version in its secondary (portfolio) collection."""
+ self._ensure_logged("version")
+ assert self._version is not None
+ return self._version
+
+ @property
+ def collection(self) -> ArtifactCollection:
+ """The collection this artifact was retrieved from.
+
+ If this artifact was retrieved from a portfolio / linked collection, that
+ collection will be returned rather than the source sequence.
+ """
+ self._ensure_logged("collection")
+ base_name = self.name.split(":")[0]
+ return ArtifactCollection(
+ self._client, self.entity, self.project, base_name, self.type
+ )
+
+ @property
+ def source_entity(self) -> str:
+ """The name of the entity of the primary (sequence) artifact collection."""
+ self._ensure_logged("source_entity")
+ assert self._source_entity is not None
+ return self._source_entity
+
+ @property
+ def source_project(self) -> str:
+ """The name of the project of the primary (sequence) artifact collection."""
+ self._ensure_logged("source_project")
+ assert self._source_project is not None
+ return self._source_project
+
+ @property
+ def source_name(self) -> str:
+ """The artifact name and version in its primary (sequence) collection.
+
+ A string with the format {collection}:{alias}. Before the artifact is saved,
+ contains only the name since the version is not yet known.
+ """
+ return self._source_name
+
+ @property
+ def source_qualified_name(self) -> str:
+ """The entity/project/name of the primary (sequence) collection."""
+ return f"{self.source_entity}/{self.source_project}/{self.source_name}"
+
+ @property
+ def source_version(self) -> str:
+ """The artifact's version in its primary (sequence) collection.
+
+ A string with the format "v{number}".
+ """
+ self._ensure_logged("source_version")
+ assert self._source_version is not None
+ return self._source_version
+
+ @property
+ def source_collection(self) -> ArtifactCollection:
+ """The artifact's primary (sequence) collection."""
+ self._ensure_logged("source_collection")
+ base_name = self.source_name.split(":")[0]
+ return ArtifactCollection(
+ self._client, self.source_entity, self.source_project, base_name, self.type
+ )
+
+ @property
+ def type(self) -> str:
+ """The artifact's type."""
+ return self._type
+
+ @property
+ def description(self) -> Optional[str]:
+ """The artifact description.
+
+ Free text that offers a user-set description of the artifact.
+ """
+ return self._description
+
+ @description.setter
+ def description(self, description: Optional[str]) -> None:
+ """Set the description of the artifact.
+
+ The description is markdown rendered in the UI, so this is a good place to put
+ links, etc.
+
+ Arguments:
+ desc: Free text that offers a description of the artifact.
+ """
+ self._description = description
+
+ @property
+ def metadata(self) -> dict:
+ """User-defined artifact metadata.
+
+ Structured data associated with the artifact.
+ """
+ return self._metadata
+
+ @metadata.setter
+ def metadata(self, metadata: dict) -> None:
+ """User-defined artifact metadata.
+
+ Metadata set this way will eventually be queryable and plottable in the UI; e.g.
+ the class distribution of a dataset.
+
+ Note: There is currently a limit of 100 total keys.
+
+ Arguments:
+ metadata: Structured data associated with the artifact.
+ """
+ self._metadata = self._normalize_metadata(metadata)
+
+ @property
+ def ttl(self) -> Union[timedelta, None]:
+ """Time To Live (TTL).
+
+ The artifact will be deleted shortly after TTL since its creation.
+ None means the artifact will never expire.
+ If TTL is not set on an artifact, it will inherit the default for its collection.
+
+ Raises:
+ ArtifactNotLoggedError: Unable to fetch inherited TTL if the artifact has not been logged or saved
+ """
+ if self._ttl_is_inherited and (self.is_draft() or self._ttl_changed):
+ raise ArtifactNotLoggedError(self, "ttl")
+ if self._ttl_duration_seconds is None:
+ return None
+ return timedelta(seconds=self._ttl_duration_seconds)
+
+ @ttl.setter
+ def ttl(self, ttl: Union[timedelta, ArtifactTTL, None]) -> None:
+ """Time To Live (TTL).
+
+ The artifact will be deleted shortly after TTL since its creation. None means the artifact will never expire.
+ If TTL is not set on an artifact, it will inherit the default TTL rules for its collection.
+
+ Arguments:
+ ttl: How long the artifact will remain active from its creation.
+ - Timedelta must be positive.
+ - `None` means the artifact will never expire.
+ - wandb.ArtifactTTL.INHERIT will set the TTL to go back to the default and inherit from collection rules.
+ """
+ if self.type == "wandb-history":
+ raise ValueError("Cannot set artifact TTL for type wandb-history")
+
+ self._ttl_changed = True
+ if isinstance(ttl, ArtifactTTL):
+ if ttl == ArtifactTTL.INHERIT:
+ self._ttl_is_inherited = True
+ else:
+ raise ValueError(f"Unhandled ArtifactTTL enum {ttl}")
+ else:
+ self._ttl_is_inherited = False
+ if ttl is None:
+ self._ttl_duration_seconds = None
+ else:
+ if ttl.total_seconds() <= 0:
+ raise ValueError(
+ f"Artifact TTL Duration has to be positive. ttl: {ttl.total_seconds()}"
+ )
+ self._ttl_duration_seconds = int(ttl.total_seconds())
+
+ @property
+ def aliases(self) -> List[str]:
+ """The aliases associated with this artifact.
+
+ The list is mutable and calling `save()` will persist all alias changes.
+ """
+ self._ensure_logged("aliases")
+ return self._aliases
+
+ @aliases.setter
+ def aliases(self, aliases: List[str]) -> None:
+ """Set the aliases associated with this artifact."""
+ self._ensure_logged("aliases")
+
+ if any(char in alias for alias in aliases for char in ["/", ":"]):
+ raise ValueError(
+ "Aliases must not contain any of the following characters: /, :"
+ )
+ self._aliases = aliases
+
+ @property
+ def distributed_id(self) -> Optional[str]:
+ return self._distributed_id
+
+ @distributed_id.setter
+ def distributed_id(self, distributed_id: Optional[str]) -> None:
+ self._distributed_id = distributed_id
+
+ @property
+ def incremental(self) -> bool:
+ return self._incremental
+
+ @property
+ def use_as(self) -> Optional[str]:
+ return self._use_as
+
+ @property
+ def state(self) -> str:
+ """The status of the artifact. One of: "PENDING", "COMMITTED", or "DELETED"."""
+ return self._state.value
+
+ @property
+ def manifest(self) -> ArtifactManifest:
+ """The artifact's manifest.
+
+ The manifest lists all of its contents, and can't be changed once the artifact
+ has been logged.
+ """
+ if self._manifest is None:
+ query = gql(
+ """
+ query ArtifactManifest(
+ $entityName: String!,
+ $projectName: String!,
+ $name: String!
+ ) {
+ project(entityName: $entityName, name: $projectName) {
+ artifact(name: $name) {
+ currentManifest {
+ file {
+ directUrl
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+ assert self._client is not None
+ response = self._client.execute(
+ query,
+ variable_values={
+ "entityName": self._entity,
+ "projectName": self._project,
+ "name": self._name,
+ },
+ )
+ attrs = response["project"]["artifact"]
+ self._load_manifest(attrs["currentManifest"]["file"]["directUrl"])
+ assert self._manifest is not None
+ return self._manifest
+
+ @property
+ def digest(self) -> str:
+ """The logical digest of the artifact.
+
+ The digest is the checksum of the artifact's contents. If an artifact has the
+ same digest as the current `latest` version, then `log_artifact` is a no-op.
+ """
+ return self.manifest.digest()
+
+ @property
+ def size(self) -> int:
+ """The total size of the artifact in bytes.
+
+ Includes any references tracked by this artifact.
+ """
+ total_size: int = 0
+ for entry in self.manifest.entries.values():
+ if entry.size is not None:
+ total_size += entry.size
+ return total_size
+
+ @property
+ def commit_hash(self) -> str:
+ """The hash returned when this artifact was committed."""
+ self._ensure_logged("commit_hash")
+ assert self._commit_hash is not None
+ return self._commit_hash
+
+ @property
+ def file_count(self) -> int:
+ """The number of files (including references)."""
+ self._ensure_logged("file_count")
+ assert self._file_count is not None
+ return self._file_count
+
+ @property
+ def created_at(self) -> str:
+ """The time at which the artifact was created."""
+ self._ensure_logged("created_at")
+ assert self._created_at is not None
+ return self._created_at
+
+ @property
+ def updated_at(self) -> str:
+ """The time at which the artifact was last updated."""
+ self._ensure_logged("updated_at")
+ assert self._created_at is not None
+ return self._updated_at or self._created_at
+
+ # State management.
+
+ def finalize(self) -> None:
+ """Mark this artifact as final, disallowing further modifications.
+
+ This happens automatically when calling `log_artifact`.
+ """
+ self._final = True
+
+ def _ensure_can_add(self) -> None:
+ if self._final:
+ raise ArtifactFinalizedError(artifact=self)
+
+ def _ensure_logged(self, attr: Optional[str] = None) -> None:
+ if self.is_draft():
+ raise ArtifactNotLoggedError(self, attr)
+
+ def is_draft(self) -> bool:
+ """Whether the artifact is a draft, i.e. it hasn't been saved yet."""
+ return self._state == ArtifactState.PENDING
+
+ def _is_draft_save_started(self) -> bool:
+ return self._save_handle is not None
+
+ def save(
+ self,
+ project: Optional[str] = None,
+ settings: Optional["wandb.wandb_sdk.wandb_settings.Settings"] = None,
+ ) -> None:
+ """Persist any changes made to the artifact.
+
+ If currently in a run, that run will log this artifact. If not currently in a
+ run, a run of type "auto" will be created to track this artifact.
+
+ Arguments:
+ project: A project to use for the artifact in the case that a run is not
+ already in context
+ settings: A settings object to use when initializing an automatic run. Most
+ commonly used in testing harness.
+ """
+ if self._state != ArtifactState.PENDING:
+ return self._update()
+
+ if self._incremental:
+ with telemetry.context() as tel:
+ tel.feature.artifact_incremental = True
+
+ if wandb.run is None:
+ if settings is None:
+ settings = wandb.Settings(silent="true")
+ with wandb.init( # type: ignore
+ entity=self._source_entity,
+ project=project or self._source_project,
+ job_type="auto",
+ settings=settings,
+ ) as run:
+ # redoing this here because in this branch we know we didn't
+ # have the run at the beginning of the method
+ if self._incremental:
+ with telemetry.context(run=run) as tel:
+ tel.feature.artifact_incremental = True
+ run.log_artifact(self)
+ else:
+ wandb.run.log_artifact(self)
+
+ def _set_save_handle(
+ self, save_handle: "MailboxHandle", client: RetryingClient
+ ) -> None:
+ self._save_handle = save_handle
+ self._client = client
+
+ def wait(self, timeout: Optional[int] = None) -> "Artifact":
+ """Wait for this artifact to finish logging, if needed.
+
+ Arguments:
+ timeout: Wait up to this long.
+ """
+ if self.is_draft():
+ if self._save_handle is None:
+ raise ArtifactNotLoggedError(self, "wait")
+
+ if timeout is None:
+ timeout = -1
+ termlog(f"Waiting for artifact {self.name} to be committed...")
+ result = self._save_handle.wait(timeout=timeout)
+ if not result:
+ raise WaitTimeoutError(
+ "Artifact upload wait timed out, failed to fetch Artifact response"
+ )
+ response = result.response.log_artifact_response
+ if response.error_message:
+ raise ValueError(response.error_message)
+ self._populate_after_save(response.artifact_id)
+ termlog(prefix=False, newline=True)
+ termlog(f"Committed artifact {self.qualified_name}")
+ return self
+
+ def _populate_after_save(self, artifact_id: str) -> None:
+ query_template = """
+ query ArtifactByIDShort($id: ID!) {
+ artifact(id: $id) {
+ artifactSequence {
+ project {
+ entityName
+ name
+ }
+ name
+ }
+ versionIndex
+ ttlDurationSeconds
+ ttlIsInherited
+ aliases {
+ artifactCollection {
+ project {
+ entityName
+ name
+ }
+ name
+ }
+ alias
+ }
+ state
+ currentManifest {
+ file {
+ directUrl
+ }
+ }
+ commitHash
+ fileCount
+ createdAt
+ updatedAt
+ }
+ }
+ """
+
+ fields = InternalApi().server_artifact_introspection()
+ if "ttlIsInherited" not in fields:
+ query_template = query_template.replace("ttlDurationSeconds", "").replace(
+ "ttlIsInherited",
+ "",
+ )
+ query = gql(query_template)
+
+ assert self._client is not None
+ response = self._client.execute(
+ query,
+ variable_values={"id": artifact_id},
+ )
+ attrs = response.get("artifact")
+ if attrs is None:
+ raise ValueError(f"Unable to fetch artifact with id {artifact_id}")
+ self._id = artifact_id
+ self._entity = attrs["artifactSequence"]["project"]["entityName"]
+ self._project = attrs["artifactSequence"]["project"]["name"]
+ self._name = "{}:v{}".format(
+ attrs["artifactSequence"]["name"], attrs["versionIndex"]
+ )
+ self._version = "v{}".format(attrs["versionIndex"])
+ self._source_entity = self._entity
+ self._source_project = self._project
+ self._source_name = self._name
+ self._source_version = self._version
+ self._ttl_duration_seconds = self._ttl_duration_seconds_from_gql(
+ attrs.get("ttlDurationSeconds")
+ )
+ self._ttl_is_inherited = (
+ True if attrs.get("ttlIsInherited") is None else attrs["ttlIsInherited"]
+ )
+ self._ttl_changed = False # Reset after saving artifact
+ self._aliases = [
+ alias["alias"]
+ for alias in attrs["aliases"]
+ if alias["artifactCollection"]["project"]["entityName"] == self._entity
+ and alias["artifactCollection"]["project"]["name"] == self._project
+ and alias["artifactCollection"]["name"] == self._name.split(":")[0]
+ and not util.alias_is_version_index(alias["alias"])
+ ]
+ self._state = ArtifactState(attrs["state"])
+ self._load_manifest(attrs["currentManifest"]["file"]["directUrl"])
+ self._commit_hash = attrs["commitHash"]
+ self._file_count = attrs["fileCount"]
+ self._created_at = attrs["createdAt"]
+ self._updated_at = attrs["updatedAt"]
+
+ @normalize_exceptions
+ def _update(self) -> None:
+ """Persists artifact changes to the wandb backend."""
+ aliases = None
+ introspect_query = gql(
+ """
+ query ProbeServerAddAliasesInput {
+ AddAliasesInputInfoType: __type(name: "AddAliasesInput") {
+ name
+ inputFields {
+ name
+ }
+ }
+ }
+ """
+ )
+ assert self._client is not None
+ response = self._client.execute(introspect_query)
+ if response.get("AddAliasesInputInfoType"): # wandb backend version >= 0.13.0
+ aliases_to_add = set(self._aliases) - set(self._saved_aliases)
+ aliases_to_delete = set(self._saved_aliases) - set(self._aliases)
+ if len(aliases_to_add) > 0:
+ add_mutation = gql(
+ """
+ mutation addAliases(
+ $artifactID: ID!,
+ $aliases: [ArtifactCollectionAliasInput!]!,
+ ) {
+ addAliases(
+ input: {artifactID: $artifactID, aliases: $aliases}
+ ) {
+ success
+ }
+ }
+ """
+ )
+ assert self._client is not None
+ self._client.execute(
+ add_mutation,
+ variable_values={
+ "artifactID": self.id,
+ "aliases": [
+ {
+ "entityName": self._entity,
+ "projectName": self._project,
+ "artifactCollectionName": self._name.split(":")[0],
+ "alias": alias,
+ }
+ for alias in aliases_to_add
+ ],
+ },
+ )
+ if len(aliases_to_delete) > 0:
+ delete_mutation = gql(
+ """
+ mutation deleteAliases(
+ $artifactID: ID!,
+ $aliases: [ArtifactCollectionAliasInput!]!,
+ ) {
+ deleteAliases(
+ input: {artifactID: $artifactID, aliases: $aliases}
+ ) {
+ success
+ }
+ }
+ """
+ )
+ assert self._client is not None
+ self._client.execute(
+ delete_mutation,
+ variable_values={
+ "artifactID": self.id,
+ "aliases": [
+ {
+ "entityName": self._entity,
+ "projectName": self._project,
+ "artifactCollectionName": self._name.split(":")[0],
+ "alias": alias,
+ }
+ for alias in aliases_to_delete
+ ],
+ },
+ )
+ self._saved_aliases = copy(self._aliases)
+ else: # wandb backend version < 0.13.0
+ aliases = [
+ {
+ "artifactCollectionName": self._name.split(":")[0],
+ "alias": alias,
+ }
+ for alias in self._aliases
+ ]
+
+ mutation_template = """
+ mutation updateArtifact(
+ $artifactID: ID!,
+ $description: String,
+ $metadata: JSONString,
+ _TTL_DURATION_SECONDS_TYPE_
+ $aliases: [ArtifactAliasInput!]
+ ) {
+ updateArtifact(
+ input: {
+ artifactID: $artifactID,
+ description: $description,
+ metadata: $metadata,
+ _TTL_DURATION_SECONDS_VALUE_
+ aliases: $aliases
+ }
+ ) {
+ artifact {
+ id
+ _TTL_DURATION_SECONDS_FIELDS_
+ }
+ }
+ }
+ """
+ fields = InternalApi().server_artifact_introspection()
+ if "ttlIsInherited" in fields:
+ mutation_template = (
+ mutation_template.replace(
+ "_TTL_DURATION_SECONDS_TYPE_", "$ttlDurationSeconds: Int64,"
+ )
+ .replace(
+ "_TTL_DURATION_SECONDS_VALUE_",
+ "ttlDurationSeconds: $ttlDurationSeconds,",
+ )
+ .replace(
+ "_TTL_DURATION_SECONDS_FIELDS_", "ttlDurationSeconds ttlIsInherited"
+ )
+ )
+ else:
+ if self._ttl_changed:
+ termwarn(
+ "Server not compatible with setting Artifact TTLs, please upgrade the server to use Artifact TTL"
+ )
+ mutation_template = (
+ mutation_template.replace("_TTL_DURATION_SECONDS_TYPE_", "")
+ .replace(
+ "_TTL_DURATION_SECONDS_VALUE_",
+ "",
+ )
+ .replace("_TTL_DURATION_SECONDS_FIELDS_", "")
+ )
+ mutation = gql(mutation_template)
+ assert self._client is not None
+
+ ttl_duration_input = self._ttl_duration_seconds_to_gql()
+ response = self._client.execute(
+ mutation,
+ variable_values={
+ "artifactID": self.id,
+ "description": self.description,
+ "metadata": util.json_dumps_safer(self.metadata),
+ "ttlDurationSeconds": ttl_duration_input,
+ "aliases": aliases,
+ },
+ )
+ attrs = response["updateArtifact"]["artifact"]
+
+ # Update ttl_duration_seconds based on updateArtifact
+ self._ttl_duration_seconds = self._ttl_duration_seconds_from_gql(
+ attrs.get("ttlDurationSeconds")
+ )
+ self._ttl_is_inherited = (
+ True if attrs.get("ttlIsInherited") is None else attrs["ttlIsInherited"]
+ )
+ self._ttl_changed = False # Reset after updating artifact
+
+ # Adding, removing, getting entries.
+
+ def __getitem__(self, name: str) -> Optional[data_types.WBValue]:
+ """Get the WBValue object located at the artifact relative `name`.
+
+ Arguments:
+ name: The artifact relative name to get
+
+ Raises:
+ ArtifactNotLoggedError: if the artifact isn't logged or the run is offline
+
+ Examples:
+ Basic usage:
+ ```python
+ artifact = wandb.Artifact("my_table", type="dataset")
+ table = wandb.Table(
+ columns=["a", "b", "c"], data=[(i, i * 2, 2**i) for i in range(10)]
+ )
+ artifact["my_table"] = table
+
+ wandb.log_artifact(artifact)
+ ```
+
+ Retrieving an object:
+ ```python
+ artifact = wandb.use_artifact("my_table:latest")
+ table = artifact["my_table"]
+ ```
+ """
+ return self.get(name)
+
+ def __setitem__(self, name: str, item: data_types.WBValue) -> ArtifactManifestEntry:
+ """Add `item` to the artifact at path `name`.
+
+ Arguments:
+ name: The path within the artifact to add the object.
+ item: The object to add.
+
+ Returns:
+ The added manifest entry
+
+ Raises:
+ ArtifactFinalizedError: if the artifact has already been finalized.
+
+ Examples:
+ Basic usage:
+ ```python
+ artifact = wandb.Artifact("my_table", type="dataset")
+ table = wandb.Table(
+ columns=["a", "b", "c"], data=[(i, i * 2, 2**i) for i in range(10)]
+ )
+ artifact["my_table"] = table
+
+ wandb.log_artifact(artifact)
+ ```
+
+ Retrieving an object:
+ ```python
+ artifact = wandb.use_artifact("my_table:latest")
+ table = artifact["my_table"]
+ ```
+ """
+ return self.add(item, name)
+
+ @contextlib.contextmanager
+ def new_file(
+ self, name: str, mode: str = "w", encoding: Optional[str] = None
+ ) -> Generator[IO, None, None]:
+ """Open a new temporary file that will be automatically added to the artifact.
+
+ Arguments:
+ name: The name of the new file being added to the artifact.
+ mode: The mode in which to open the new file.
+ encoding: The encoding in which to open the new file.
+
+ Returns:
+ A new file object that can be written to. Upon closing, the file will be
+ automatically added to the artifact.
+
+ Raises:
+ ArtifactFinalizedError: if the artifact has already been finalized.
+
+ Examples:
+ ```python
+ artifact = wandb.Artifact("my_data", type="dataset")
+ with artifact.new_file("hello.txt") as f:
+ f.write("hello!")
+ wandb.log_artifact(artifact)
+ ```
+ """
+ self._ensure_can_add()
+ if self._tmp_dir is None:
+ self._tmp_dir = tempfile.TemporaryDirectory()
+ path = os.path.join(self._tmp_dir.name, name.lstrip("/"))
+ if os.path.exists(path):
+ raise ValueError(f"File with name {name!r} already exists at {path!r}")
+
+ filesystem.mkdir_exists_ok(os.path.dirname(path))
+ try:
+ with util.fsync_open(path, mode, encoding) as f:
+ yield f
+ except UnicodeEncodeError as e:
+ termerror(
+ f"Failed to open the provided file (UnicodeEncodeError: {e}). Please "
+ f"provide the proper encoding."
+ )
+ raise e
+
+ self.add_file(path, name=name)
+
+ def add_file(
+ self,
+ local_path: str,
+ name: Optional[str] = None,
+ is_tmp: Optional[bool] = False,
+ ) -> ArtifactManifestEntry:
+ """Add a local file to the artifact.
+
+ Arguments:
+ local_path: The path to the file being added.
+ name: The path within the artifact to use for the file being added. Defaults
+ to the basename of the file.
+ is_tmp: If true, then the file is renamed deterministically to avoid
+ collisions.
+
+ Returns:
+ The added manifest entry
+
+ Raises:
+ ArtifactFinalizedError: if the artifact has already been finalized
+
+ Examples:
+ Add a file without an explicit name:
+ ```python
+ # Add as `file.txt'
+ artifact.add_file("path/to/file.txt")
+ ```
+
+ Add a file with an explicit name:
+ ```python
+ # Add as 'new/path/file.txt'
+ artifact.add_file("path/to/file.txt", name="new/path/file.txt")
+ ```
+ """
+ self._ensure_can_add()
+ if not os.path.isfile(local_path):
+ raise ValueError("Path is not a file: %s" % local_path)
+
+ name = LogicalPath(name or os.path.basename(local_path))
+ digest = md5_file_b64(local_path)
+
+ if is_tmp:
+ file_path, file_name = os.path.split(name)
+ file_name_parts = file_name.split(".")
+ file_name_parts[0] = b64_to_hex_id(digest)[:20]
+ name = os.path.join(file_path, ".".join(file_name_parts))
+
+ return self._add_local_file(name, local_path, digest=digest)
+
+ def add_dir(self, local_path: str, name: Optional[str] = None) -> None:
+ """Add a local directory to the artifact.
+
+ Arguments:
+ local_path: The path to the directory being added.
+ name: The path within the artifact to use for the directory being added.
+ Defaults to the root of the artifact.
+
+ Raises:
+ ArtifactFinalizedError: if the artifact has already been finalized
+
+ Examples:
+ Add a directory without an explicit name:
+ ```python
+ # All files in `my_dir/` are added at the root of the artifact.
+ artifact.add_dir("my_dir/")
+ ```
+
+ Add a directory and name it explicitly:
+ ```python
+ # All files in `my_dir/` are added under `destination/`.
+ artifact.add_dir("my_dir/", name="destination")
+ ```
+ """
+ self._ensure_can_add()
+ if not os.path.isdir(local_path):
+ raise ValueError("Path is not a directory: %s" % local_path)
+
+ termlog(
+ "Adding directory to artifact (%s)... "
+ % os.path.join(".", os.path.normpath(local_path)),
+ newline=False,
+ )
+ start_time = time.time()
+
+ paths = []
+ for dirpath, _, filenames in os.walk(local_path, followlinks=True):
+ for fname in filenames:
+ physical_path = os.path.join(dirpath, fname)
+ logical_path = os.path.relpath(physical_path, start=local_path)
+ if name is not None:
+ logical_path = os.path.join(name, logical_path)
+ paths.append((logical_path, physical_path))
+
+ def add_manifest_file(log_phy_path: Tuple[str, str]) -> None:
+ logical_path, physical_path = log_phy_path
+ self._add_local_file(logical_path, physical_path)
+
+ num_threads = 8
+ pool = multiprocessing.dummy.Pool(num_threads)
+ pool.map(add_manifest_file, paths)
+ pool.close()
+ pool.join()
+
+ termlog("Done. %.1fs" % (time.time() - start_time), prefix=False)
+
+ def add_reference(
+ self,
+ uri: Union[ArtifactManifestEntry, str],
+ name: Optional[StrPath] = None,
+ checksum: bool = True,
+ max_objects: Optional[int] = None,
+ ) -> Sequence[ArtifactManifestEntry]:
+ """Add a reference denoted by a URI to the artifact.
+
+ Unlike adding files or directories, references are NOT uploaded to W&B. However,
+ artifact methods such as `download()` can be used regardless of whether the
+ artifact contains references or uploaded files.
+
+ By default, W&B offers special handling for the following schemes:
+
+ - http(s): The size and digest of the file will be inferred by the
+ `Content-Length` and the `ETag` response headers returned by the server.
+ - s3: The checksum and size will be pulled from the object metadata. If bucket
+ versioning is enabled, then the version ID is also tracked.
+ - gs: The checksum and size will be pulled from the object metadata. If bucket
+ versioning is enabled, then the version ID is also tracked.
+ - https, domain matching *.blob.core.windows.net (Azure): The checksum and size
+ will be pulled from the blob metadata. If storage account versioning is
+ enabled, then the version ID is also tracked.
+ - file: The checksum and size will be pulled from the file system. This scheme
+ is useful if you have an NFS share or other externally mounted volume
+ containing files you wish to track but not necessarily upload.
+
+ For any other scheme, the digest is just a hash of the URI and the size is left
+ blank.
+
+ Arguments:
+ uri: The URI path of the reference to add. Can be an object returned from
+ Artifact.get_entry to store a reference to another artifact's entry.
+ name: The path within the artifact to place the contents of this reference
+ checksum: Whether or not to checksum the resource(s) located at the
+ reference URI. Checksumming is strongly recommended as it enables
+ automatic integrity validation, however it can be disabled to speed up
+ artifact creation. (default: True)
+ max_objects: The maximum number of objects to consider when adding a
+ reference that points to directory or bucket store prefix. For S3 and
+ GCS, this limit is 10,000 by default but is uncapped for other URI
+ schemes. (default: None)
+
+ Returns:
+ The added manifest entries.
+
+ Raises:
+ ArtifactFinalizedError: if the artifact has already been finalized.
+
+ Examples:
+ Add an HTTP link:
+ ```python
+ # Adds `file.txt` to the root of the artifact as a reference.
+ artifact.add_reference("http://myserver.com/file.txt")
+ ```
+
+ Add an S3 prefix without an explicit name:
+ ```python
+ # All objects under `prefix/` will be added at the root of the artifact.
+ artifact.add_reference("s3://mybucket/prefix")
+ ```
+
+ Add a GCS prefix with an explicit name:
+ ```python
+ # All objects under `prefix/` will be added under `path/` at the artifact
+ # root.
+ artifact.add_reference("gs://mybucket/prefix", name="path")
+ ```
+ """
+ self._ensure_can_add()
+ if name is not None:
+ name = LogicalPath(name)
+
+ # This is a bit of a hack, we want to check if the uri is a of the type
+ # ArtifactManifestEntry. If so, then recover the reference URL.
+ if isinstance(uri, ArtifactManifestEntry):
+ uri_str = uri.ref_url()
+ elif isinstance(uri, str):
+ uri_str = uri
+ url = urlparse(str(uri_str))
+ if not url.scheme:
+ raise ValueError(
+ "References must be URIs. To reference a local file, use file://"
+ )
+
+ manifest_entries = self._storage_policy.store_reference(
+ self,
+ URIStr(uri_str),
+ name=name,
+ checksum=checksum,
+ max_objects=max_objects,
+ )
+ for entry in manifest_entries:
+ self.manifest.add_entry(entry)
+
+ return manifest_entries
+
+ def add(self, obj: data_types.WBValue, name: StrPath) -> ArtifactManifestEntry:
+ """Add wandb.WBValue `obj` to the artifact.
+
+ Arguments:
+ obj: The object to add. Currently support one of Bokeh, JoinedTable,
+ PartitionedTable, Table, Classes, ImageMask, BoundingBoxes2D, Audio,
+ Image, Video, Html, Object3D
+ name: The path within the artifact to add the object.
+
+ Returns:
+ The added manifest entry
+
+ Raises:
+ ArtifactFinalizedError: if the artifact has already been finalized
+
+ Examples:
+ Basic usage:
+ ```python
+ artifact = wandb.Artifact("my_table", type="dataset")
+ table = wandb.Table(
+ columns=["a", "b", "c"], data=[(i, i * 2, 2**i) for i in range(10)]
+ )
+ artifact.add(table, "my_table")
+
+ wandb.log_artifact(artifact)
+ ```
+
+ Retrieve an object:
+ ```python
+ artifact = wandb.use_artifact("my_table:latest")
+ table = artifact.get("my_table")
+ ```
+ """
+ self._ensure_can_add()
+ name = LogicalPath(name)
+
+ # This is a "hack" to automatically rename tables added to
+ # the wandb /media/tables directory to their sha-based name.
+ # TODO: figure out a more appropriate convention.
+ is_tmp_name = name.startswith("media/tables")
+
+ # Validate that the object is one of the correct wandb.Media types
+ # TODO: move this to checking subclass of wandb.Media once all are
+ # generally supported
+ allowed_types = [
+ data_types.Bokeh,
+ data_types.JoinedTable,
+ data_types.PartitionedTable,
+ data_types.Table,
+ data_types.Classes,
+ data_types.ImageMask,
+ data_types.BoundingBoxes2D,
+ data_types.Audio,
+ data_types.Image,
+ data_types.Video,
+ data_types.Html,
+ data_types.Object3D,
+ data_types.Molecule,
+ data_types._SavedModel,
+ ]
+
+ if not any(isinstance(obj, t) for t in allowed_types):
+ raise ValueError(
+ "Found object of type {}, expected one of {}.".format(
+ obj.__class__, allowed_types
+ )
+ )
+
+ obj_id = id(obj)
+ if obj_id in self._added_objs:
+ return self._added_objs[obj_id][1]
+
+ # If the object is coming from another artifact, save it as a reference
+ ref_path = obj._get_artifact_entry_ref_url()
+ if ref_path is not None:
+ return self.add_reference(ref_path, type(obj).with_suffix(name))[0]
+
+ val = obj.to_json(self)
+ name = obj.with_suffix(name)
+ entry = self.manifest.get_entry_by_path(name)
+ if entry is not None:
+ return entry
+
+ def do_write(f: IO) -> None:
+ import json
+
+ # TODO: Do we need to open with utf-8 codec?
+ f.write(json.dumps(val, sort_keys=True))
+
+ if is_tmp_name:
+ file_path = os.path.join(self._TMP_DIR.name, str(id(self)), name)
+ folder_path, _ = os.path.split(file_path)
+ if not os.path.exists(folder_path):
+ os.makedirs(folder_path)
+ with open(file_path, "w") as tmp_f:
+ do_write(tmp_f)
+ else:
+ with self.new_file(name) as f:
+ file_path = f.name
+ do_write(f)
+
+ # Note, we add the file from our temp directory.
+ # It will be added again later on finalize, but succeed since
+ # the checksum should match
+ entry = self.add_file(file_path, name, is_tmp_name)
+ # We store a reference to the obj so that its id doesn't get reused.
+ self._added_objs[obj_id] = (obj, entry)
+ if obj._artifact_target is None:
+ obj._set_artifact_target(self, entry.path)
+
+ if is_tmp_name:
+ if os.path.exists(file_path):
+ os.remove(file_path)
+
+ return entry
+
+ def _add_local_file(
+ self, name: StrPath, path: StrPath, digest: Optional[B64MD5] = None
+ ) -> ArtifactManifestEntry:
+ with tempfile.NamedTemporaryFile(dir=get_staging_dir(), delete=False) as f:
+ staging_path = f.name
+ shutil.copyfile(path, staging_path)
+ os.chmod(staging_path, 0o400)
+
+ entry = ArtifactManifestEntry(
+ path=name,
+ digest=digest or md5_file_b64(staging_path),
+ size=os.path.getsize(staging_path),
+ local_path=staging_path,
+ )
+
+ self.manifest.add_entry(entry)
+ self._added_local_paths[os.fspath(path)] = entry
+ return entry
+
+ def remove(self, item: Union[StrPath, "ArtifactManifestEntry"]) -> None:
+ """Remove an item from the artifact.
+
+ Arguments:
+ item: the item to remove. Can be a specific manifest entry or the name of an
+ artifact-relative path. If the item matches a directory all items in
+ that directory will be removed.
+
+ Raises:
+ ArtifactFinalizedError: if the artifact has already been finalized.
+ FileNotFoundError: if the item isn't found in the artifact.
+ """
+ self._ensure_can_add()
+
+ if isinstance(item, ArtifactManifestEntry):
+ self.manifest.remove_entry(item)
+ return
+
+ path = str(PurePosixPath(item))
+ entry = self.manifest.get_entry_by_path(path)
+ if entry:
+ self.manifest.remove_entry(entry)
+ return
+
+ entries = self.manifest.get_entries_in_directory(path)
+ if not entries:
+ raise FileNotFoundError(f"No such file or directory: {path}")
+ for entry in entries:
+ self.manifest.remove_entry(entry)
+
+ def get_path(self, name: StrPath) -> ArtifactManifestEntry:
+ """Deprecated, use get_entry(name) instead."""
+ termwarn(
+ "Artifact.get_path(name) is deprecated, use Artifact.get_entry(name) instead."
+ )
+ return self.get_entry(name)
+
+ def get_entry(self, name: StrPath) -> ArtifactManifestEntry:
+ """Get the entry with the given name.
+
+ Arguments:
+ name: The artifact relative name to get
+
+ Raises:
+ ArtifactNotLoggedError: if the artifact isn't logged or the run is offline
+ KeyError: if the artifact doesn't contain an entry with the given name
+
+ Examples:
+ Basic usage:
+ ```python
+ # Run logging the artifact
+ with wandb.init() as r:
+ artifact = wandb.Artifact("my_dataset", type="dataset")
+ artifact.add_file("path/to/file.txt")
+ wandb.log_artifact(artifact)
+
+ # Run using the artifact
+ with wandb.init() as r:
+ artifact = r.use_artifact("my_dataset:latest")
+ entry = artifact.get_entry("file.txt")
+
+ # Can now download 'file.txt' directly:
+ entry.download()
+ ```
+ """
+ self._ensure_logged("get_entry")
+
+ name = LogicalPath(name)
+ entry = self.manifest.entries.get(name) or self._get_obj_entry(name)[0]
+ if entry is None:
+ raise KeyError("Path not contained in artifact: %s" % name)
+ entry._parent_artifact = self
+ return entry
+
+ def get(self, name: str) -> Optional[data_types.WBValue]:
+ """Get the WBValue object located at the artifact relative `name`.
+
+ Arguments:
+ name: The artifact relative name to get
+
+ Raises:
+ ArtifactNotLoggedError: if the artifact isn't logged or the run is offline
+
+ Examples:
+ Basic usage:
+ ```python
+ # Run logging the artifact
+ with wandb.init() as r:
+ artifact = wandb.Artifact("my_dataset", type="dataset")
+ table = wandb.Table(
+ columns=["a", "b", "c"], data=[(i, i * 2, 2**i) for i in range(10)]
+ )
+ artifact.add(table, "my_table")
+ wandb.log_artifact(artifact)
+
+ # Run using the artifact
+ with wandb.init() as r:
+ artifact = r.use_artifact("my_dataset:latest")
+ table = artifact.get("my_table")
+ ```
+ """
+ self._ensure_logged("get")
+
+ entry, wb_class = self._get_obj_entry(name)
+ if entry is None or wb_class is None:
+ return None
+
+ # If the entry is a reference from another artifact, then get it directly from
+ # that artifact.
+ referenced_id = entry._referenced_artifact_id()
+ if referenced_id:
+ assert self._client is not None
+ artifact = self._from_id(referenced_id, client=self._client)
+ assert artifact is not None
+ return artifact.get(util.uri_from_path(entry.ref))
+
+ # Special case for wandb.Table. This is intended to be a short term
+ # optimization. Since tables are likely to download many other assets in
+ # artifact(s), we eagerly download the artifact using the parallelized
+ # `artifact.download`. In the future, we should refactor the deserialization
+ # pattern such that this special case is not needed.
+ if wb_class == wandb.Table:
+ self.download()
+
+ # Get the ArtifactManifestEntry
+ item = self.get_entry(entry.path)
+ item_path = item.download()
+
+ # Load the object from the JSON blob
+ result = None
+ json_obj = {}
+ with open(item_path) as file:
+ json_obj = json.load(file)
+ result = wb_class.from_json(json_obj, self)
+ result._set_artifact_source(self, name)
+ return result
+
+ def get_added_local_path_name(self, local_path: str) -> Optional[str]:
+ """Get the artifact relative name of a file added by a local filesystem path.
+
+ Arguments:
+ local_path: The local path to resolve into an artifact relative name.
+
+ Returns:
+ The artifact relative name.
+
+ Examples:
+ Basic usage:
+ ```python
+ artifact = wandb.Artifact("my_dataset", type="dataset")
+ artifact.add_file("path/to/file.txt", name="artifact/path/file.txt")
+
+ # Returns `artifact/path/file.txt`:
+ name = artifact.get_added_local_path_name("path/to/file.txt")
+ ```
+ """
+ entry = self._added_local_paths.get(local_path, None)
+ if entry is None:
+ return None
+ return entry.path
+
+ def _get_obj_entry(
+ self, name: str
+ ) -> Tuple[Optional["ArtifactManifestEntry"], Optional[Type[WBValue]]]:
+ """Return an object entry by name, handling any type suffixes.
+
+ When objects are added with `.add(obj, name)`, the name is typically changed to
+ include the suffix of the object type when serializing to JSON. So we need to be
+ able to resolve a name, without tasking the user with appending .THING.json.
+ This method returns an entry if it exists by a suffixed name.
+
+ Arguments:
+ name: name used when adding
+ """
+ for wb_class in WBValue.type_mapping().values():
+ wandb_file_name = wb_class.with_suffix(name)
+ entry = self.manifest.entries.get(wandb_file_name)
+ if entry is not None:
+ return entry, wb_class
+ return None, None
+
+ # Downloading.
+
+ def download(
+ self,
+ root: Optional[str] = None,
+ allow_missing_references: bool = False,
+ skip_cache: Optional[bool] = None,
+ ) -> FilePathStr:
+ """Download the contents of the artifact to the specified root directory.
+
+ NOTE: Any existing files at `root` are left untouched. Explicitly delete
+ root before calling `download` if you want the contents of `root` to exactly
+ match the artifact.
+
+ Arguments:
+ root: The directory in which to download this artifact's files.
+
+ Returns:
+ The path to the downloaded contents.
+
+ Raises:
+ ArtifactNotLoggedError: if the artifact has not been logged
+ """
+ self._ensure_logged("download")
+
+ root = root or self._default_root()
+ self._add_download_root(root)
+
+ if get_core_path() != "":
+ return self._download_using_core(
+ root=root,
+ allow_missing_references=allow_missing_references,
+ )
+ return self._download(
+ root=root,
+ allow_missing_references=allow_missing_references,
+ skip_cache=skip_cache,
+ )
+
+ def _download_using_core(
+ self,
+ root: str,
+ allow_missing_references: bool = False,
+ ) -> FilePathStr:
+ import pathlib
+
+ from wandb.sdk.backend.backend import Backend
+
+ if wandb.run is None:
+ # ensure wandb-core is up and running
+ wl = wandb.sdk.wandb_setup.setup()
+ assert wl is not None
+
+ stream_id = generate_id()
+
+ settings = wl.settings.to_proto()
+ # TODO: remove this
+ tmp_dir = pathlib.Path(tempfile.mkdtemp())
+ settings.sync_dir.value = str(tmp_dir)
+ settings.sync_file.value = str(tmp_dir / f"{stream_id}.wandb")
+ settings.files_dir.value = str(tmp_dir / "files")
+ settings.run_id.value = stream_id
+
+ manager = wl._get_manager()
+ manager._inform_init(settings=settings, run_id=stream_id)
+
+ mailbox = Mailbox()
+ backend = Backend(settings=wl.settings, manager=manager, mailbox=mailbox)
+ backend.ensure_launched()
+
+ assert backend.interface
+ backend.interface._stream_id = stream_id # type: ignore
+
+ mailbox.enable_keepalive()
+ else:
+ assert wandb.run._backend
+ backend = wandb.run._backend
+
+ assert backend.interface
+ handle = backend.interface.deliver_download_artifact(
+ self.id, # type: ignore
+ root,
+ allow_missing_references,
+ )
+ # TODO: Start the download process in the user process too, to handle reference downloads
+ self._download(
+ root=root,
+ allow_missing_references=allow_missing_references,
+ )
+ result = handle.wait(timeout=-1)
+
+ if result is None:
+ handle.abandon()
+ assert result is not None
+ response = result.response.download_artifact_response
+ if response.error_message:
+ raise ValueError(f"Error downloading artifact: {response.error_message}")
+
+ if wandb.run is None:
+ backend.cleanup()
+ # TODO: remove this
+ shutil.rmtree(tmp_dir)
+
+ return FilePathStr(root)
+
+ def _download(
+ self,
+ root: str,
+ allow_missing_references: bool = False,
+ skip_cache: Optional[bool] = None,
+ ) -> FilePathStr:
+ # todo: remove once artifact reference downloads are supported in core
+ require_core = get_core_path() != ""
+
+ nfiles = len(self.manifest.entries)
+ size = sum(e.size or 0 for e in self.manifest.entries.values())
+ log = False
+ if nfiles > 5000 or size > 50 * 1024 * 1024:
+ log = True
+ termlog(
+ "Downloading large artifact {}, {:.2f}MB. {} files... ".format(
+ self.name, size / (1024 * 1024), nfiles
+ ),
+ )
+ start_time = datetime.now()
+ download_logger = ArtifactDownloadLogger(nfiles=nfiles)
+
+ def _download_entry(
+ entry: ArtifactManifestEntry,
+ api_key: Optional[str],
+ cookies: Optional[Dict],
+ headers: Optional[Dict],
+ ) -> None:
+ _thread_local_api_settings.api_key = api_key
+ _thread_local_api_settings.cookies = cookies
+ _thread_local_api_settings.headers = headers
+
+ try:
+ entry.download(root, skip_cache=skip_cache)
+ except FileNotFoundError as e:
+ if allow_missing_references:
+ wandb.termwarn(str(e))
+ return
+ raise
+ download_logger.notify_downloaded()
+
+ download_entry = partial(
+ _download_entry,
+ api_key=_thread_local_api_settings.api_key,
+ cookies=_thread_local_api_settings.cookies,
+ headers=_thread_local_api_settings.headers,
+ )
+
+ with concurrent.futures.ThreadPoolExecutor(64) as executor:
+ active_futures = set()
+ has_next_page = True
+ cursor = None
+ while has_next_page:
+ fetch_url_batch_size = env.get_artifact_fetch_file_url_batch_size()
+ attrs = self._fetch_file_urls(cursor, fetch_url_batch_size)
+ has_next_page = attrs["pageInfo"]["hasNextPage"]
+ cursor = attrs["pageInfo"]["endCursor"]
+ for edge in attrs["edges"]:
+ entry = self.get_entry(edge["node"]["name"])
+ if require_core and entry.ref is None:
+ # Handled by core
+ continue
+ entry._download_url = edge["node"]["directUrl"]
+ active_futures.add(executor.submit(download_entry, entry))
+ # Wait for download threads to catch up.
+ max_backlog = fetch_url_batch_size
+ if len(active_futures) > max_backlog:
+ for future in concurrent.futures.as_completed(active_futures):
+ future.result() # check for errors
+ active_futures.remove(future)
+ if len(active_futures) <= max_backlog:
+ break
+ # Check for errors.
+ for future in concurrent.futures.as_completed(active_futures):
+ future.result()
+
+ if log:
+ now = datetime.now()
+ delta = abs((now - start_time).total_seconds())
+ hours = int(delta // 3600)
+ minutes = int((delta - hours * 3600) // 60)
+ seconds = delta - hours * 3600 - minutes * 60
+ termlog(
+ f"Done. {hours}:{minutes}:{seconds:.1f}",
+ prefix=False,
+ )
+ return FilePathStr(root)
+
+ @retry.retriable(
+ retry_timedelta=timedelta(minutes=3),
+ retryable_exceptions=(requests.RequestException),
+ )
+ def _fetch_file_urls(
+ self, cursor: Optional[str], per_page: Optional[int] = 5000
+ ) -> Any:
+ query = gql(
+ """
+ query ArtifactFileURLs($id: ID!, $cursor: String, $perPage: Int) {
+ artifact(id: $id) {
+ files(after: $cursor, first: $perPage) {
+ pageInfo {
+ hasNextPage
+ endCursor
+ }
+ edges {
+ node {
+ name
+ directUrl
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+ assert self._client is not None
+ response = self._client.execute(
+ query,
+ variable_values={"id": self.id, "cursor": cursor, "perPage": per_page},
+ timeout=60,
+ )
+ return response["artifact"]["files"]
+
+ def checkout(self, root: Optional[str] = None) -> str:
+ """Replace the specified root directory with the contents of the artifact.
+
+ WARNING: This will DELETE all files in `root` that are not included in the
+ artifact.
+
+ Arguments:
+ root: The directory to replace with this artifact's files.
+
+ Returns:
+ The path to the checked out contents.
+
+ Raises:
+ ArtifactNotLoggedError: if the artifact has not been logged
+ """
+ self._ensure_logged("checkout")
+
+ root = root or self._default_root(include_version=False)
+
+ for dirpath, _, files in os.walk(root):
+ for file in files:
+ full_path = os.path.join(dirpath, file)
+ artifact_path = os.path.relpath(full_path, start=root)
+ try:
+ self.get_entry(artifact_path)
+ except KeyError:
+ # File is not part of the artifact, remove it.
+ os.remove(full_path)
+
+ return self.download(root=root)
+
+ def verify(self, root: Optional[str] = None) -> None:
+ """Verify that the actual contents of an artifact match the manifest.
+
+ All files in the directory are checksummed and the checksums are then
+ cross-referenced against the artifact's manifest.
+
+ NOTE: References are not verified.
+
+ Arguments:
+ root: The directory to verify. If None artifact will be downloaded to
+ './artifacts/self.name/'
+
+ Raises:
+ ArtifactNotLoggedError: if the artifact has not been logged
+ ValueError: If the verification fails.
+ """
+ self._ensure_logged("verify")
+
+ root = root or self._default_root()
+
+ for dirpath, _, files in os.walk(root):
+ for file in files:
+ full_path = os.path.join(dirpath, file)
+ artifact_path = os.path.relpath(full_path, start=root)
+ try:
+ self.get_entry(artifact_path)
+ except KeyError:
+ raise ValueError(
+ "Found file {} which is not a member of artifact {}".format(
+ full_path, self.name
+ )
+ )
+
+ ref_count = 0
+ for entry in self.manifest.entries.values():
+ if entry.ref is None:
+ if md5_file_b64(os.path.join(root, entry.path)) != entry.digest:
+ raise ValueError("Digest mismatch for file: %s" % entry.path)
+ else:
+ ref_count += 1
+ if ref_count > 0:
+ print("Warning: skipped verification of %s refs" % ref_count)
+
+ def file(self, root: Optional[str] = None) -> StrPath:
+ """Download a single file artifact to dir specified by the root.
+
+ Arguments:
+ root: The root directory in which to place the file. Defaults to
+ './artifacts/self.name/'.
+
+ Returns:
+ The full path of the downloaded file.
+
+ Raises:
+ ArtifactNotLoggedError: if the artifact has not been logged
+ ValueError: if the artifact contains more than one file
+ """
+ self._ensure_logged("file")
+
+ if root is None:
+ root = os.path.join(".", "artifacts", self.name)
+
+ if len(self.manifest.entries) > 1:
+ raise ValueError(
+ "This artifact contains more than one file, call `.download()` to get "
+ 'all files or call .get_entry("filename").download()'
+ )
+
+ return self.get_entry(list(self.manifest.entries)[0]).download(root)
+
+ def files(
+ self, names: Optional[List[str]] = None, per_page: int = 50
+ ) -> ArtifactFiles:
+ """Iterate over all files stored in this artifact.
+
+ Arguments:
+ names: The filename paths relative to the root of the artifact you wish to
+ list.
+ per_page: The number of files to return per request
+
+ Returns:
+ An iterator containing `File` objects
+
+ Raises:
+ ArtifactNotLoggedError: if the artifact has not been logged
+ """
+ self._ensure_logged("files")
+ return ArtifactFiles(self._client, self, names, per_page)
+
+ def _default_root(self, include_version: bool = True) -> FilePathStr:
+ name = self.source_name if include_version else self.source_name.split(":")[0]
+ root = os.path.join(env.get_artifact_dir(), name)
+ # In case we're on a system where the artifact dir has a name corresponding to
+ # an unexpected filesystem, we'll check for alternate roots. If one exists we'll
+ # use that, otherwise we'll fall back to the system-preferred path.
+ path = filesystem.check_exists(root) or filesystem.system_preferred_path(root)
+ return FilePathStr(str(path))
+
+ def _add_download_root(self, dir_path: str) -> None:
+ self._download_roots.add(os.path.abspath(dir_path))
+
+ def _local_path_to_name(self, file_path: str) -> Optional[str]:
+ """Convert a local file path to a path entry in the artifact."""
+ abs_file_path = os.path.abspath(file_path)
+ abs_file_parts = abs_file_path.split(os.sep)
+ for i in range(len(abs_file_parts) + 1):
+ if os.path.join(os.sep, *abs_file_parts[:i]) in self._download_roots:
+ return os.path.join(*abs_file_parts[i:])
+ return None
+
+ # Others.
+
+ def delete(self, delete_aliases: bool = False) -> None:
+ """Delete an artifact and its files.
+
+ Arguments:
+ delete_aliases: If true, deletes all aliases associated with the artifact.
+ Otherwise, this raises an exception if the artifact has existing
+ aliases.
+
+ Raises:
+ ArtifactNotLoggedError: if the artifact has not been logged
+
+ Examples:
+ Delete all the "model" artifacts a run has logged:
+ ```python
+ runs = api.runs(path="my_entity/my_project")
+ for run in runs:
+ for artifact in run.logged_artifacts():
+ if artifact.type == "model":
+ artifact.delete(delete_aliases=True)
+ ```
+ """
+ self._ensure_logged("delete")
+ self._delete(delete_aliases)
+
+ @normalize_exceptions
+ def _delete(self, delete_aliases: bool = False) -> None:
+ mutation = gql(
+ """
+ mutation DeleteArtifact($artifactID: ID!, $deleteAliases: Boolean) {
+ deleteArtifact(input: {
+ artifactID: $artifactID
+ deleteAliases: $deleteAliases
+ }) {
+ artifact {
+ id
+ }
+ }
+ }
+ """
+ )
+ assert self._client is not None
+ self._client.execute(
+ mutation,
+ variable_values={
+ "artifactID": self.id,
+ "deleteAliases": delete_aliases,
+ },
+ )
+
+ @normalize_exceptions
+ def link(self, target_path: str, aliases: Optional[List[str]] = None) -> None:
+ """Link this artifact to a portfolio (a promoted collection of artifacts).
+
+ Arguments:
+ target_path: The path to the portfolio. It must take the form {portfolio},
+ {project}/{portfolio} or {entity}/{project}/{portfolio}.
+ aliases: A list of strings which uniquely identifies the artifact inside the
+ specified portfolio.
+
+ Raises:
+ ArtifactNotLoggedError: if the artifact has not been logged
+ """
+ if wandb.run is None:
+ with wandb.init( # type: ignore
+ entity=self._source_entity,
+ project=self._source_project,
+ job_type="auto",
+ settings=wandb.Settings(silent="true"),
+ ) as run:
+ run.link_artifact(self, target_path, aliases)
+ else:
+ wandb.run.link_artifact(self, target_path, aliases)
+
+ def used_by(self) -> List[Run]:
+ """Get a list of the runs that have used this artifact.
+
+ Raises:
+ ArtifactNotLoggedError: if the artifact has not been logged
+ """
+ self._ensure_logged("used_by")
+
+ query = gql(
+ """
+ query ArtifactUsedBy(
+ $id: ID!,
+ ) {
+ artifact(id: $id) {
+ usedBy {
+ edges {
+ node {
+ name
+ project {
+ name
+ entityName
+ }
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+ assert self._client is not None
+ response = self._client.execute(
+ query,
+ variable_values={"id": self.id},
+ )
+ return [
+ Run(
+ self._client,
+ edge["node"]["project"]["entityName"],
+ edge["node"]["project"]["name"],
+ edge["node"]["name"],
+ )
+ for edge in response.get("artifact", {}).get("usedBy", {}).get("edges", [])
+ ]
+
+ def logged_by(self) -> Optional[Run]:
+ """Get the run that first logged this artifact.
+
+ Raises:
+ ArtifactNotLoggedError: if the artifact has not been logged
+ """
+ self._ensure_logged("logged_by")
+
+ query = gql(
+ """
+ query ArtifactCreatedBy(
+ $id: ID!
+ ) {
+ artifact(id: $id) {
+ createdBy {
+ ... on Run {
+ name
+ project {
+ name
+ entityName
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+ assert self._client is not None
+ response = self._client.execute(
+ query,
+ variable_values={"id": self.id},
+ )
+ creator = response.get("artifact", {}).get("createdBy", {})
+ if creator.get("name") is None:
+ return None
+ return Run(
+ self._client,
+ creator["project"]["entityName"],
+ creator["project"]["name"],
+ creator["name"],
+ )
+
+ def json_encode(self) -> Dict[str, Any]:
+ self._ensure_logged("json_encode")
+ return util.artifact_to_json(self)
+
+ @staticmethod
+ def _expected_type(
+ entity_name: str, project_name: str, name: str, client: RetryingClient
+ ) -> Optional[str]:
+ """Returns the expected type for a given artifact name and project."""
+ query = gql(
+ """
+ query ArtifactType(
+ $entityName: String,
+ $projectName: String,
+ $name: String!
+ ) {
+ project(name: $projectName, entityName: $entityName) {
+ artifact(name: $name) {
+ artifactType {
+ name
+ }
+ }
+ }
+ }
+ """
+ )
+ if ":" not in name:
+ name += ":latest"
+ response = client.execute(
+ query,
+ variable_values={
+ "entityName": entity_name,
+ "projectName": project_name,
+ "name": name,
+ },
+ )
+ return (
+ ((response.get("project") or {}).get("artifact") or {}).get("artifactType")
+ or {}
+ ).get("name")
+
+ @staticmethod
+ def _normalize_metadata(metadata: Optional[Dict[str, Any]]) -> Dict[str, Any]:
+ if metadata is None:
+ return {}
+ if not isinstance(metadata, dict):
+ raise TypeError(f"metadata must be dict, not {type(metadata)}")
+ return cast(
+ Dict[str, Any], json.loads(json.dumps(util.json_friendly_val(metadata)))
+ )
+
+ def _load_manifest(self, url: str) -> None:
+ with requests.get(url) as request:
+ request.raise_for_status()
+ self._manifest = ArtifactManifest.from_manifest_json(
+ json.loads(util.ensure_text(request.content))
+ )
+
+ @staticmethod
+ def _get_gql_artifact_fragment() -> str:
+ fields = InternalApi().server_artifact_introspection()
+ fragment = """
+ fragment ArtifactFragment on Artifact {
+ id
+ artifactSequence {
+ project {
+ entityName
+ name
+ }
+ name
+ }
+ versionIndex
+ artifactType {
+ name
+ }
+ description
+ metadata
+ ttlDurationSeconds
+ ttlIsInherited
+ aliases {
+ artifactCollection {
+ project {
+ entityName
+ name
+ }
+ name
+ }
+ alias
+ }
+ state
+ commitHash
+ fileCount
+ createdAt
+ updatedAt
+ }
+ """
+ if "ttlIsInherited" not in fields:
+ return fragment.replace("ttlDurationSeconds", "").replace(
+ "ttlIsInherited", ""
+ )
+ return fragment
+
+ def _ttl_duration_seconds_to_gql(self) -> Optional[int]:
+ # Set artifact ttl value to ttl_duration_seconds if the user set a value
+ # otherwise use ttl_status to indicate the backend INHERIT(-1) or DISABLED(-2) when the TTL is None
+ # When ttl_change = None its a no op since nothing changed
+ INHERIT = -1 # noqa: N806
+ DISABLED = -2 # noqa: N806
+
+ if not self._ttl_changed:
+ return None
+ if self._ttl_is_inherited:
+ return INHERIT
+ return self._ttl_duration_seconds or DISABLED
+
+ def _ttl_duration_seconds_from_gql(
+ self, gql_ttl_duration_seconds: Optional[int]
+ ) -> Optional[int]:
+ # If gql_ttl_duration_seconds is not positive, its indicating that TTL is DISABLED(-2)
+ # gql_ttl_duration_seconds only returns None if the server is not compatible with setting Artifact TTLs
+ if gql_ttl_duration_seconds and gql_ttl_duration_seconds > 0:
+ return gql_ttl_duration_seconds
+ return None
+
+
+class _ArtifactVersionType(WBType):
+ name = "artifactVersion"
+ types = [Artifact]
+
+
+TypeRegistry.add(_ArtifactVersionType)
diff --git a/wandb/sdk/artifacts/artifact_download_logger.py b/wandb/sdk/artifacts/artifact_download_logger.py
new file mode 100644
index 0000000000000000000000000000000000000000..f800505affbd36a488d2e9b1aa88aba7339bcb4b
--- /dev/null
+++ b/wandb/sdk/artifacts/artifact_download_logger.py
@@ -0,0 +1,42 @@
+"""Artifact download logger."""
+import multiprocessing.dummy
+import time
+from typing import Callable
+
+from wandb.errors.term import termlog
+
+
+class ArtifactDownloadLogger:
+ def __init__(
+ self,
+ nfiles: int,
+ clock_for_testing: Callable[[], float] = time.monotonic,
+ termlog_for_testing: Callable[..., None] = termlog,
+ ) -> None:
+ self._nfiles = nfiles
+ self._clock = clock_for_testing
+ self._termlog = termlog_for_testing
+
+ self._n_files_downloaded = 0
+ self._spinner_index = 0
+ self._last_log_time = self._clock()
+ self._lock = multiprocessing.dummy.Lock()
+
+ def notify_downloaded(self) -> None:
+ with self._lock:
+ self._n_files_downloaded += 1
+ if self._n_files_downloaded == self._nfiles:
+ self._termlog(
+ f" {self._nfiles} of {self._nfiles} files downloaded. ",
+ # ^ trailing spaces to wipe out ellipsis from previous logs
+ newline=True,
+ )
+ self._last_log_time = self._clock()
+ elif self._clock() - self._last_log_time > 0.1:
+ self._spinner_index += 1
+ spinner = r"-\|/"[self._spinner_index % 4]
+ self._termlog(
+ f"{spinner} {self._n_files_downloaded} of {self._nfiles} files downloaded...\r",
+ newline=False,
+ )
+ self._last_log_time = self._clock()
diff --git a/wandb/sdk/artifacts/artifact_file_cache.py b/wandb/sdk/artifacts/artifact_file_cache.py
new file mode 100644
index 0000000000000000000000000000000000000000..3f0dc7ea39e37c1ec6f38e75530140acb46784ca
--- /dev/null
+++ b/wandb/sdk/artifacts/artifact_file_cache.py
@@ -0,0 +1,215 @@
+"""Artifact cache."""
+import contextlib
+import errno
+import hashlib
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+from typing import IO, TYPE_CHECKING, ContextManager, Generator, Optional, Tuple
+
+import wandb
+from wandb import env, util
+from wandb.errors import term
+from wandb.sdk.lib.filesystem import files_in
+from wandb.sdk.lib.hashutil import B64MD5, ETag, b64_to_hex_id
+from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
+
+if TYPE_CHECKING:
+ if sys.version_info >= (3, 8):
+ from typing import Protocol
+ else:
+ from typing_extensions import Protocol
+
+ class Opener(Protocol):
+ def __call__(self, mode: str = ...) -> ContextManager[IO]:
+ pass
+
+
+class ArtifactFileCache:
+ def __init__(self, cache_dir: StrPath) -> None:
+ self._cache_dir = Path(cache_dir)
+ self._obj_dir = self._cache_dir / "obj"
+ self._temp_dir = self._cache_dir / "tmp"
+ self._temp_dir.mkdir(parents=True, exist_ok=True)
+
+ # NamedTemporaryFile sets the file mode to 600 [1], we reset to the default.
+ # [1] https://stackoverflow.com/questions/10541760/can-i-set-the-umask-for-tempfile-namedtemporaryfile-in-python
+ umask_cmd = (sys.executable, "-c", "import os; print(os.umask(22))")
+ umask = int(subprocess.check_output(umask_cmd))
+ self._sys_umask = umask
+
+ self._override_cache_path: Optional[StrPath] = None
+
+ def check_md5_obj_path(
+ self, b64_md5: B64MD5, size: int
+ ) -> Tuple[FilePathStr, bool, "Opener"]:
+ if self._override_cache_path is not None:
+ path = Path(self._override_cache_path)
+ else:
+ hex_md5 = b64_to_hex_id(b64_md5)
+ path = self._obj_dir / "md5" / hex_md5[:2] / hex_md5[2:]
+ return self._check_or_create(path, size)
+
+ # TODO(spencerpearson): this method at least needs its signature changed.
+ # An ETag is not (necessarily) a checksum.
+ def check_etag_obj_path(
+ self,
+ url: URIStr,
+ etag: ETag,
+ size: int,
+ ) -> Tuple[FilePathStr, bool, "Opener"]:
+ if self._override_cache_path is not None:
+ path = Path(self._override_cache_path)
+ else:
+ hexhash = hashlib.sha256(
+ hashlib.sha256(url.encode("utf-8")).digest()
+ + hashlib.sha256(etag.encode("utf-8")).digest()
+ ).hexdigest()
+ path = self._obj_dir / "etag" / hexhash[:2] / hexhash[2:]
+ return self._check_or_create(path, size)
+
+ def _check_or_create(
+ self, path: Path, size: int
+ ) -> Tuple[FilePathStr, bool, "Opener"]:
+ opener = self._cache_opener(path, size)
+ hit = path.is_file() and path.stat().st_size == size
+ return FilePathStr(str(path)), hit, opener
+
+ def cleanup(
+ self,
+ target_size: Optional[int] = None,
+ remove_temp: bool = False,
+ target_fraction: Optional[float] = None,
+ ) -> int:
+ """Clean up the cache, removing the least recently used files first.
+
+ Args:
+ target_size: The target size of the cache in bytes. If the cache is larger
+ than this, we will remove the least recently used files until the cache
+ is smaller than this size.
+ remove_temp: Whether to remove temporary files. Temporary files are files
+ that are currently being written to the cache. If remove_temp is True,
+ all temp files will be removed, regardless of the target_size or
+ target_fraction.
+ target_fraction: The target fraction of the cache to reclaim. If the cache
+ is larger than this, we will remove the least recently used files until
+ the cache is smaller than this fraction of its current size. It is an
+ error to specify both target_size and target_fraction.
+
+ Returns:
+ The number of bytes reclaimed.
+ """
+ if target_size is None and target_fraction is None:
+ # Default to clearing the entire cache.
+ target_size = 0
+ if target_size is not None and target_fraction is not None:
+ raise ValueError("Cannot specify both target_size and target_fraction")
+ if target_size and target_size < 0:
+ raise ValueError("target_size must be non-negative")
+ if target_fraction and (target_fraction < 0 or target_fraction > 1):
+ raise ValueError("target_fraction must be between 0 and 1")
+
+ bytes_reclaimed = 0
+ total_size = 0
+ temp_size = 0
+
+ # Remove all temporary files if requested. Otherwise sum their size.
+ for entry in files_in(self._temp_dir):
+ size = entry.stat().st_size
+ total_size += size
+ if remove_temp:
+ try:
+ os.remove(entry.path)
+ bytes_reclaimed += size
+ except OSError:
+ pass
+ else:
+ temp_size += size
+ if temp_size:
+ wandb.termwarn(
+ f"Cache contains {util.to_human_size(temp_size)} of temporary files. "
+ "Run `wandb artifact cleanup --remove-temp` to remove them."
+ )
+
+ entries = []
+ for file_entry in files_in(self._obj_dir):
+ total_size += file_entry.stat().st_size
+ entries.append(file_entry)
+
+ if target_fraction is not None:
+ target_size = int(total_size * target_fraction)
+ assert target_size is not None
+
+ for entry in sorted(entries, key=lambda x: x.stat().st_atime):
+ if total_size <= target_size:
+ return bytes_reclaimed
+ try:
+ os.remove(entry.path)
+ except OSError:
+ pass
+ total_size -= entry.stat().st_size
+ bytes_reclaimed += entry.stat().st_size
+
+ if total_size > target_size:
+ wandb.termerror(
+ f"Failed to reclaim enough space in {self._cache_dir}. Try running"
+ " `wandb artifact cleanup --remove-temp` to remove temporary files."
+ )
+
+ return bytes_reclaimed
+
+ def _free_space(self) -> int:
+ """Return the number of bytes of free space in the cache directory."""
+ return shutil.disk_usage(self._cache_dir)[2]
+
+ def _reserve_space(self, size: int) -> None:
+ """If a `size` write would exceed disk space, remove cached items to make space.
+
+ Raises:
+ OSError: If there is not enough space to write `size` bytes, even after
+ removing cached items.
+ """
+ if size <= self._free_space():
+ return
+
+ term.termwarn("Cache size exceeded. Attempting to reclaim space...")
+ self.cleanup(target_fraction=0.5)
+ if size <= self._free_space():
+ return
+
+ self.cleanup(target_size=0)
+ if size > self._free_space():
+ raise OSError(errno.ENOSPC, f"Insufficient free space in {self._cache_dir}")
+
+ def _cache_opener(self, path: Path, size: int) -> "Opener":
+ @contextlib.contextmanager
+ def helper(mode: str = "w") -> Generator[IO, None, None]:
+ if "a" in mode:
+ raise ValueError("Appending to cache files is not supported")
+
+ self._reserve_space(size)
+ temp_file = NamedTemporaryFile(dir=self._temp_dir, mode=mode, delete=False)
+ try:
+ yield temp_file
+ temp_file.close()
+ os.chmod(temp_file.name, 0o666 & ~self._sys_umask)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ os.replace(temp_file.name, path)
+ except Exception:
+ os.remove(temp_file.name)
+ raise
+
+ return helper
+
+
+_artifact_file_cache = None
+
+
+def get_artifact_file_cache() -> ArtifactFileCache:
+ global _artifact_file_cache
+ if _artifact_file_cache is None:
+ _artifact_file_cache = ArtifactFileCache(env.get_cache_dir() / "artifacts")
+ return _artifact_file_cache
diff --git a/wandb/sdk/artifacts/artifact_instance_cache.py b/wandb/sdk/artifacts/artifact_instance_cache.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2cb183fca2337b90a5d3c4aeee2c138b9ba1a7b
--- /dev/null
+++ b/wandb/sdk/artifacts/artifact_instance_cache.py
@@ -0,0 +1,14 @@
+"""Recent Artifact storage.
+
+Artifacts are registered in the cache to ensure they won't be immediately garbage
+collected and can be retrieved by their ID.
+"""
+from typing import TYPE_CHECKING, Dict
+
+from wandb.sdk.lib.capped_dict import CappedDict
+
+if TYPE_CHECKING:
+ from wandb.sdk.artifacts.artifact import Artifact
+
+# There is nothing special about the artifact cache, it's just a global capped dict.
+artifact_instance_cache: Dict[str, "Artifact"] = CappedDict(100)
diff --git a/wandb/sdk/artifacts/artifact_manifest.py b/wandb/sdk/artifacts/artifact_manifest.py
new file mode 100644
index 0000000000000000000000000000000000000000..fb694dbedc7a5578ae32302e7882e4738214baec
--- /dev/null
+++ b/wandb/sdk/artifacts/artifact_manifest.py
@@ -0,0 +1,68 @@
+"""Artifact manifest."""
+from typing import TYPE_CHECKING, Dict, List, Mapping, Optional
+
+from wandb.sdk.lib.hashutil import HexMD5
+
+if TYPE_CHECKING:
+ from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+ from wandb.sdk.artifacts.storage_policy import StoragePolicy
+
+
+class ArtifactManifest:
+ entries: Dict[str, "ArtifactManifestEntry"]
+
+ @classmethod
+ def from_manifest_json(cls, manifest_json: Dict) -> "ArtifactManifest":
+ if "version" not in manifest_json:
+ raise ValueError("Invalid manifest format. Must contain version field.")
+ version = manifest_json["version"]
+ for sub in cls.__subclasses__():
+ if sub.version() == version:
+ return sub.from_manifest_json(manifest_json)
+ raise ValueError("Invalid manifest version.")
+
+ @classmethod
+ def version(cls) -> int:
+ raise NotImplementedError
+
+ def __init__(
+ self,
+ storage_policy: "StoragePolicy",
+ entries: Optional[Mapping[str, "ArtifactManifestEntry"]] = None,
+ ) -> None:
+ self.storage_policy = storage_policy
+ self.entries = dict(entries) if entries else {}
+
+ def __len__(self) -> int:
+ return len(self.entries)
+
+ def to_manifest_json(self) -> Dict:
+ raise NotImplementedError
+
+ def digest(self) -> HexMD5:
+ raise NotImplementedError
+
+ def add_entry(self, entry: "ArtifactManifestEntry") -> None:
+ if (
+ entry.path in self.entries
+ and entry.digest != self.entries[entry.path].digest
+ ):
+ raise ValueError("Cannot add the same path twice: %s" % entry.path)
+ self.entries[entry.path] = entry
+
+ def remove_entry(self, entry: "ArtifactManifestEntry") -> None:
+ if entry.path not in self.entries:
+ raise FileNotFoundError(f"Cannot remove missing entry: '{entry.path}'")
+ del self.entries[entry.path]
+
+ def get_entry_by_path(self, path: str) -> Optional["ArtifactManifestEntry"]:
+ return self.entries.get(path)
+
+ def get_entries_in_directory(self, directory: str) -> List["ArtifactManifestEntry"]:
+ return [
+ self.entries[entry_key]
+ for entry_key in self.entries
+ if entry_key.startswith(
+ directory + "/"
+ ) # entries use forward slash even for windows
+ ]
diff --git a/wandb/sdk/artifacts/artifact_manifest_entry.py b/wandb/sdk/artifacts/artifact_manifest_entry.py
new file mode 100644
index 0000000000000000000000000000000000000000..1df81c93e0ec862bbfd6033dcd110cd2883db83b
--- /dev/null
+++ b/wandb/sdk/artifacts/artifact_manifest_entry.py
@@ -0,0 +1,198 @@
+"""Artifact manifest entry."""
+import json
+import os
+from pathlib import Path
+from typing import TYPE_CHECKING, Dict, Optional, Union
+from urllib.parse import urlparse
+
+from wandb.errors.term import termwarn
+from wandb.sdk.lib import filesystem
+from wandb.sdk.lib.hashutil import (
+ B64MD5,
+ ETag,
+ b64_to_hex_id,
+ hex_to_b64_id,
+ md5_file_b64,
+)
+from wandb.sdk.lib.paths import FilePathStr, LogicalPath, StrPath, URIStr
+
+if TYPE_CHECKING:
+ from wandb.sdk.artifacts.artifact import Artifact
+
+
+class ArtifactManifestEntry:
+ """A single entry in an artifact manifest."""
+
+ path: LogicalPath
+ digest: Union[B64MD5, URIStr, FilePathStr, ETag]
+ ref: Optional[Union[FilePathStr, URIStr]]
+ birth_artifact_id: Optional[str]
+ size: Optional[int]
+ extra: Dict
+ local_path: Optional[str]
+
+ _parent_artifact: Optional["Artifact"] = None
+ _download_url: Optional[str] = None
+
+ def __init__(
+ self,
+ path: StrPath,
+ digest: Union[B64MD5, URIStr, FilePathStr, ETag],
+ ref: Optional[Union[FilePathStr, URIStr]] = None,
+ birth_artifact_id: Optional[str] = None,
+ size: Optional[int] = None,
+ extra: Optional[Dict] = None,
+ local_path: Optional[StrPath] = None,
+ ) -> None:
+ self.path = LogicalPath(path)
+ self.digest = digest
+ self.ref = ref
+ self.birth_artifact_id = birth_artifact_id
+ self.size = size
+ self.extra = extra or {}
+ self.local_path = str(local_path) if local_path else None
+ if self.local_path and self.size is None:
+ self.size = Path(self.local_path).stat().st_size
+
+ def __repr__(self) -> str:
+ cls = self.__class__.__name__
+ ref = f", ref={self.ref!r}" if self.ref is not None else ""
+ birth_artifact_id = (
+ f", birth_artifact_id={self.birth_artifact_id!r}"
+ if self.birth_artifact_id is not None
+ else ""
+ )
+ size = f", size={self.size}" if self.size is not None else ""
+ extra = f", extra={json.dumps(self.extra)}" if self.extra else ""
+ local_path = f", local_path={self.local_path!r}" if self.local_path else ""
+ others = ref + birth_artifact_id + size + extra + local_path
+ return f"{cls}(path={self.path!r}, digest={self.digest!r}{others})"
+
+ def __eq__(self, other: object) -> bool:
+ """Strict equality, comparing all public fields.
+
+ ArtifactManifestEntries for the same file may not compare equal if they were
+ added in different ways or created for different parent artifacts.
+ """
+ if not isinstance(other, ArtifactManifestEntry):
+ return False
+ return (
+ self.path == other.path
+ and self.digest == other.digest
+ and self.ref == other.ref
+ and self.birth_artifact_id == other.birth_artifact_id
+ and self.size == other.size
+ and self.extra == other.extra
+ and self.local_path == other.local_path
+ )
+
+ @property
+ def name(self) -> LogicalPath:
+ # TODO(hugh): add telemetry to see if anyone is still using this.
+ termwarn("ArtifactManifestEntry.name is deprecated, use .path instead")
+ return self.path
+
+ def parent_artifact(self) -> "Artifact":
+ """Get the artifact to which this artifact entry belongs.
+
+ Returns:
+ (PublicArtifact): The parent artifact
+ """
+ if self._parent_artifact is None:
+ raise NotImplementedError
+ return self._parent_artifact
+
+ def download(
+ self, root: Optional[str] = None, skip_cache: Optional[bool] = None
+ ) -> FilePathStr:
+ """Download this artifact entry to the specified root path.
+
+ Arguments:
+ root: (str, optional) The root path in which to download this
+ artifact entry. Defaults to the artifact's root.
+
+ Returns:
+ (str): The path of the downloaded artifact entry.
+ """
+ if self._parent_artifact is None:
+ raise NotImplementedError
+
+ root = root or self._parent_artifact._default_root()
+ self._parent_artifact._add_download_root(root)
+ dest_path = os.path.join(root, self.path)
+
+ if skip_cache:
+ override_cache_path = dest_path
+ else:
+ override_cache_path = None
+
+ # Skip checking the cache (and possibly downloading) if the file already exists
+ # and has the digest we're expecting.
+ if os.path.exists(dest_path) and self.digest == md5_file_b64(dest_path):
+ return FilePathStr(dest_path)
+
+ if self.ref is not None:
+ cache_path = self._parent_artifact.manifest.storage_policy.load_reference(
+ self, local=True, dest_path=override_cache_path
+ )
+ else:
+ cache_path = self._parent_artifact.manifest.storage_policy.load_file(
+ self._parent_artifact, self, dest_path=override_cache_path
+ )
+
+ if skip_cache:
+ return FilePathStr(dest_path)
+ else:
+ return FilePathStr(
+ str(filesystem.copy_or_overwrite_changed(cache_path, dest_path))
+ )
+
+ def ref_target(self) -> Union[FilePathStr, URIStr]:
+ """Get the reference URL that is targeted by this artifact entry.
+
+ Returns:
+ (str): The reference URL of this artifact entry.
+
+ Raises:
+ ValueError: If this artifact entry was not a reference.
+ """
+ if self.ref is None:
+ raise ValueError("Only reference entries support ref_target().")
+ if self._parent_artifact is None:
+ return self.ref
+ return self._parent_artifact.manifest.storage_policy.load_reference(
+ self._parent_artifact.manifest.entries[self.path], local=False
+ )
+
+ def ref_url(self) -> str:
+ """Get a URL to this artifact entry.
+
+ These URLs can be referenced by another artifact.
+
+ Returns:
+ (str): A URL representing this artifact entry.
+
+ Examples:
+ Basic usage
+ ```
+ ref_url = source_artifact.get_entry('file.txt').ref_url()
+ derived_artifact.add_reference(ref_url)
+ ```
+ """
+ if self._parent_artifact is None:
+ raise NotImplementedError
+ assert self._parent_artifact.id is not None
+ return (
+ "wandb-artifact://"
+ + b64_to_hex_id(B64MD5(self._parent_artifact.id))
+ + "/"
+ + self.path
+ )
+
+ def _is_artifact_reference(self) -> bool:
+ return self.ref is not None and urlparse(self.ref).scheme == "wandb-artifact"
+
+ def _referenced_artifact_id(self) -> Optional[str]:
+ if not self._is_artifact_reference():
+ return None
+ return hex_to_b64_id(urlparse(self.ref).netloc)
diff --git a/wandb/sdk/artifacts/artifact_manifests/__init__.py b/wandb/sdk/artifacts/artifact_manifests/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py b/wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py
new file mode 100644
index 0000000000000000000000000000000000000000..2e391549c5974b8a813b1f7b404be79d925388aa
--- /dev/null
+++ b/wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py
@@ -0,0 +1,83 @@
+"""Artifact manifest v1."""
+from typing import Any, Dict, Mapping, Optional
+
+from wandb.sdk.artifacts.artifact_manifest import ArtifactManifest
+from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+from wandb.sdk.artifacts.storage_policy import StoragePolicy
+from wandb.sdk.lib.hashutil import HexMD5, _md5
+
+
+class ArtifactManifestV1(ArtifactManifest):
+ @classmethod
+ def version(cls) -> int:
+ return 1
+
+ @classmethod
+ def from_manifest_json(cls, manifest_json: Dict) -> "ArtifactManifestV1":
+ if manifest_json["version"] != cls.version():
+ raise ValueError(
+ "Expected manifest version 1, got %s" % manifest_json["version"]
+ )
+
+ storage_policy_name = manifest_json["storagePolicy"]
+ storage_policy_config = manifest_json.get("storagePolicyConfig", {})
+ storage_policy_cls = StoragePolicy.lookup_by_name(storage_policy_name)
+
+ entries: Mapping[str, ArtifactManifestEntry]
+ entries = {
+ name: ArtifactManifestEntry(
+ path=name,
+ digest=val["digest"],
+ birth_artifact_id=val.get("birthArtifactID"),
+ ref=val.get("ref"),
+ size=val.get("size"),
+ extra=val.get("extra"),
+ local_path=val.get("local_path"),
+ )
+ for name, val in manifest_json["contents"].items()
+ }
+
+ return cls(storage_policy_cls.from_config(storage_policy_config), entries)
+
+ def __init__(
+ self,
+ storage_policy: "StoragePolicy",
+ entries: Optional[Mapping[str, ArtifactManifestEntry]] = None,
+ ) -> None:
+ super().__init__(storage_policy, entries=entries)
+
+ def to_manifest_json(self) -> Dict:
+ """This is the JSON that's stored in wandb_manifest.json.
+
+ If include_local is True we also include the local paths to files. This is
+ used to represent an artifact that's waiting to be saved on the current
+ system. We don't need to include the local paths in the artifact manifest
+ contents.
+ """
+ contents = {}
+ for entry in sorted(self.entries.values(), key=lambda k: k.path):
+ json_entry: Dict[str, Any] = {
+ "digest": entry.digest,
+ }
+ if entry.birth_artifact_id:
+ json_entry["birthArtifactID"] = entry.birth_artifact_id
+ if entry.ref:
+ json_entry["ref"] = entry.ref
+ if entry.extra:
+ json_entry["extra"] = entry.extra
+ if entry.size is not None:
+ json_entry["size"] = entry.size
+ contents[entry.path] = json_entry
+ return {
+ "version": self.__class__.version(),
+ "storagePolicy": self.storage_policy.name(),
+ "storagePolicyConfig": self.storage_policy.config() or {},
+ "contents": contents,
+ }
+
+ def digest(self) -> HexMD5:
+ hasher = _md5()
+ hasher.update(b"wandb-artifact-manifest-v1\n")
+ for name, entry in sorted(self.entries.items(), key=lambda kv: kv[0]):
+ hasher.update(f"{name}:{entry.digest}\n".encode())
+ return HexMD5(hasher.hexdigest())
diff --git a/wandb/sdk/artifacts/artifact_saver.py b/wandb/sdk/artifacts/artifact_saver.py
new file mode 100644
index 0000000000000000000000000000000000000000..807c699b7d326bc4c2b62e0986902c3fba7985d9
--- /dev/null
+++ b/wandb/sdk/artifacts/artifact_saver.py
@@ -0,0 +1,298 @@
+"""Artifact saver."""
+import concurrent.futures
+import json
+import os
+import sys
+import tempfile
+from typing import TYPE_CHECKING, Awaitable, Dict, Optional, Sequence, Tuple
+
+import wandb
+import wandb.filesync.step_prepare
+from wandb import util
+from wandb.sdk.artifacts.artifact_manifest import ArtifactManifest
+from wandb.sdk.artifacts.staging import get_staging_dir
+from wandb.sdk.lib.hashutil import B64MD5, b64_to_hex_id, md5_file_b64
+from wandb.sdk.lib.paths import URIStr
+
+if TYPE_CHECKING:
+ from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+ from wandb.sdk.internal.file_pusher import FilePusher
+ from wandb.sdk.internal.internal_api import Api as InternalApi
+ from wandb.sdk.internal.progress import ProgressFn
+
+ if sys.version_info >= (3, 8):
+ from typing import Protocol
+ else:
+ from typing_extensions import Protocol
+
+ class SaveFn(Protocol):
+ def __call__(
+ self, entry: "ArtifactManifestEntry", progress_callback: "ProgressFn"
+ ) -> bool:
+ pass
+
+ class SaveFnAsync(Protocol):
+ def __call__(
+ self, entry: "ArtifactManifestEntry", progress_callback: "ProgressFn"
+ ) -> Awaitable[bool]:
+ pass
+
+
+class ArtifactSaver:
+ _server_artifact: Optional[Dict] # TODO better define this dict
+
+ def __init__(
+ self,
+ api: "InternalApi",
+ digest: str,
+ manifest_json: Dict,
+ file_pusher: "FilePusher",
+ is_user_created: bool = False,
+ ) -> None:
+ self._api = api
+ self._file_pusher = file_pusher
+ self._digest = digest
+ self._manifest = ArtifactManifest.from_manifest_json(manifest_json)
+ self._is_user_created = is_user_created
+ self._server_artifact = None
+
+ def save(
+ self,
+ type: str,
+ name: str,
+ client_id: str,
+ sequence_client_id: str,
+ distributed_id: Optional[str] = None,
+ finalize: bool = True,
+ metadata: Optional[Dict] = None,
+ ttl_duration_seconds: Optional[int] = None,
+ description: Optional[str] = None,
+ aliases: Optional[Sequence[str]] = None,
+ use_after_commit: bool = False,
+ incremental: bool = False,
+ history_step: Optional[int] = None,
+ base_id: Optional[str] = None,
+ ) -> Tuple[Dict, Optional[concurrent.futures.Future]]:
+ server_artifact = None
+ commit_fut = None
+ try:
+ server_artifact, commit_fut = self._save_internal(
+ type,
+ name,
+ client_id,
+ sequence_client_id,
+ distributed_id,
+ finalize,
+ metadata,
+ ttl_duration_seconds,
+ description,
+ aliases,
+ use_after_commit,
+ incremental,
+ history_step,
+ base_id,
+ )
+ return server_artifact, commit_fut
+ finally:
+ if commit_fut is not None:
+ commit_fut.add_done_callback(lambda _: self._cleanup_staged_entries())
+ else:
+ self._cleanup_staged_entries()
+
+ def _save_internal(
+ self,
+ type: str,
+ name: str,
+ client_id: str,
+ sequence_client_id: str,
+ distributed_id: Optional[str] = None,
+ finalize: bool = True,
+ metadata: Optional[Dict] = None,
+ ttl_duration_seconds: Optional[int] = None,
+ description: Optional[str] = None,
+ aliases: Optional[Sequence[str]] = None,
+ use_after_commit: bool = False,
+ incremental: bool = False,
+ history_step: Optional[int] = None,
+ base_id: Optional[str] = None,
+ ) -> Tuple[Dict, Optional[concurrent.futures.Future]]:
+ alias_specs = []
+ for alias in aliases or []:
+ alias_specs.append({"artifactCollectionName": name, "alias": alias})
+
+ """Returns the server artifact."""
+ self._server_artifact, latest = self._api.create_artifact(
+ type,
+ name,
+ self._digest,
+ metadata=metadata,
+ ttl_duration_seconds=ttl_duration_seconds,
+ aliases=alias_specs,
+ description=description,
+ is_user_created=self._is_user_created,
+ distributed_id=distributed_id,
+ client_id=client_id,
+ sequence_client_id=sequence_client_id,
+ history_step=history_step,
+ )
+
+ assert self._server_artifact is not None # mypy optionality unwrapper
+ artifact_id = self._server_artifact["id"]
+ if base_id is None and latest:
+ base_id = latest["id"]
+ if self._server_artifact["state"] == "COMMITTED":
+ if use_after_commit:
+ self._api.use_artifact(artifact_id)
+ return self._server_artifact, None
+ if (
+ self._server_artifact["state"] != "PENDING"
+ # For old servers, see https://github.com/wandb/wandb/pull/6190
+ and self._server_artifact["state"] != "DELETED"
+ ):
+ raise Exception(
+ 'Unknown artifact state "{}"'.format(self._server_artifact["state"])
+ )
+
+ manifest_type = "FULL"
+ manifest_filename = "wandb_manifest.json"
+ if incremental:
+ manifest_type = "INCREMENTAL"
+ manifest_filename = "wandb_manifest.incremental.json"
+ elif distributed_id:
+ manifest_type = "PATCH"
+ manifest_filename = "wandb_manifest.patch.json"
+ artifact_manifest_id, _ = self._api.create_artifact_manifest(
+ manifest_filename,
+ "",
+ artifact_id,
+ base_artifact_id=base_id,
+ include_upload=False,
+ type=manifest_type,
+ )
+
+ step_prepare = wandb.filesync.step_prepare.StepPrepare(
+ self._api, 0.1, 0.01, 1000
+ ) # TODO: params
+ step_prepare.start()
+
+ # Upload Artifact "L1" files, the actual artifact contents
+ self._file_pusher.store_manifest_files(
+ self._manifest,
+ artifact_id,
+ lambda entry, progress_callback: self._manifest.storage_policy.store_file_sync(
+ artifact_id,
+ artifact_manifest_id,
+ entry,
+ step_prepare,
+ progress_callback=progress_callback,
+ ),
+ lambda entry, progress_callback: self._manifest.storage_policy.store_file_async(
+ artifact_id,
+ artifact_manifest_id,
+ entry,
+ step_prepare,
+ progress_callback=progress_callback,
+ ),
+ )
+
+ def before_commit() -> None:
+ self._resolve_client_id_manifest_references()
+ with tempfile.NamedTemporaryFile("w+", suffix=".json", delete=False) as fp:
+ path = os.path.abspath(fp.name)
+ json.dump(self._manifest.to_manifest_json(), fp, indent=4)
+ digest = md5_file_b64(path)
+ if distributed_id or incremental:
+ # If we're in the distributed flow, we want to update the
+ # patch manifest we created with our finalized digest.
+ _, resp = self._api.update_artifact_manifest(
+ artifact_manifest_id,
+ digest=digest,
+ )
+ else:
+ # In the regular flow, we can recreate the full manifest with the
+ # updated digest.
+ #
+ # NOTE: We do this for backwards compatibility with older backends
+ # that don't support the 'updateArtifactManifest' API.
+ _, resp = self._api.create_artifact_manifest(
+ manifest_filename,
+ digest,
+ artifact_id,
+ base_artifact_id=base_id,
+ )
+
+ # We're duplicating the file upload logic a little, which isn't great.
+ upload_url = resp["uploadUrl"]
+ upload_headers = resp["uploadHeaders"]
+ extra_headers = {}
+ for upload_header in upload_headers:
+ key, val = upload_header.split(":", 1)
+ extra_headers[key] = val
+ with open(path, "rb") as fp2:
+ self._api.upload_file_retry(
+ upload_url,
+ fp2,
+ extra_headers=extra_headers,
+ )
+
+ commit_result: concurrent.futures.Future[None] = concurrent.futures.Future()
+
+ # This will queue the commit. It will only happen after all the file uploads are done
+ self._file_pusher.commit_artifact(
+ artifact_id,
+ finalize=finalize,
+ before_commit=before_commit,
+ result_future=commit_result,
+ )
+
+ saver_future: concurrent.futures.Future[None] = concurrent.futures.Future()
+
+ # do not wait for the commit to finish. Instead, once the commit is finished, set the result in the
+ # a future returned by this function. This allows the caller to decide if they should
+ # synchronously wait for the result of the saver or let it run async in the background
+ def on_commit_result(fut: concurrent.futures.Future) -> None:
+ try:
+ res = fut.result()
+ if finalize and use_after_commit:
+ self._api.use_artifact(artifact_id)
+ saver_future.set_result(res)
+ except Exception as e:
+ saver_future.set_exception(e)
+ finally:
+ step_prepare.shutdown()
+
+ commit_result.add_done_callback(on_commit_result)
+
+ return self._server_artifact, saver_future
+
+ def _resolve_client_id_manifest_references(self) -> None:
+ for entry_path in self._manifest.entries:
+ entry = self._manifest.entries[entry_path]
+ if entry.ref is not None:
+ if entry.ref.startswith("wandb-client-artifact:"):
+ client_id = util.host_from_path(entry.ref)
+ artifact_file_path = util.uri_from_path(entry.ref)
+ artifact_id = self._api._resolve_client_id(client_id)
+ if artifact_id is None:
+ raise RuntimeError(f"Could not resolve client id {client_id}")
+ entry.ref = URIStr(
+ "wandb-artifact://{}/{}".format(
+ b64_to_hex_id(B64MD5(artifact_id)), artifact_file_path
+ )
+ )
+
+ def _cleanup_staged_entries(self) -> None:
+ """Remove all staging copies of local files.
+
+ We made a staging copy of each local file to freeze it at "add" time.
+ We need to delete them once we've uploaded the file or confirmed we
+ already have a committed copy.
+ """
+ staging_dir = get_staging_dir()
+ for entry in self._manifest.entries.values():
+ if entry.local_path and entry.local_path.startswith(staging_dir):
+ try:
+ os.chmod(entry.local_path, 0o600)
+ os.remove(entry.local_path)
+ except OSError:
+ pass
diff --git a/wandb/sdk/artifacts/artifact_state.py b/wandb/sdk/artifacts/artifact_state.py
new file mode 100644
index 0000000000000000000000000000000000000000..1f5f832571255604c46536d4d8357f974931bb51
--- /dev/null
+++ b/wandb/sdk/artifacts/artifact_state.py
@@ -0,0 +1,10 @@
+"""Artifact state."""
+from enum import Enum
+
+
+class ArtifactState(Enum):
+ PENDING = "PENDING"
+ COMMITTED = "COMMITTED"
+ DELETED = "DELETED"
+ GARBAGE_COLLECTED = "GARBAGE_COLLECTED"
+ PENDING_DELETION = "PENDING_DELETION"
diff --git a/wandb/sdk/artifacts/artifact_ttl.py b/wandb/sdk/artifacts/artifact_ttl.py
new file mode 100644
index 0000000000000000000000000000000000000000..b95d645d25cdb442e938eef1a15d9f03f63af34e
--- /dev/null
+++ b/wandb/sdk/artifacts/artifact_ttl.py
@@ -0,0 +1,6 @@
+"""Artifact TTL."""
+from enum import Enum
+
+
+class ArtifactTTL(Enum):
+ INHERIT = 0
diff --git a/wandb/sdk/artifacts/exceptions.py b/wandb/sdk/artifacts/exceptions.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d9caad1b26fe208b1620be86172fe431b764c6d
--- /dev/null
+++ b/wandb/sdk/artifacts/exceptions.py
@@ -0,0 +1,55 @@
+"""Artifact exceptions."""
+from typing import TYPE_CHECKING, Optional
+
+from wandb import errors
+
+if TYPE_CHECKING:
+ from wandb.sdk.artifacts.artifact import Artifact
+
+
+class ArtifactStatusError(AttributeError):
+ """Raised when an artifact is in an invalid state for the requested operation."""
+
+ def __init__(
+ self,
+ artifact: Optional["Artifact"] = None,
+ attr: Optional[str] = None,
+ msg: str = "Artifact is in an invalid state for the requested operation.",
+ ):
+ object_name = artifact.__class__.__name__ if artifact else "Artifact"
+ method_id = f"{object_name}.{attr}" if attr else object_name
+ super().__init__(msg.format(artifact=artifact, attr=attr, method_id=method_id))
+ # Follow the same pattern as AttributeError.
+ self.obj = artifact
+ self.name = attr or ""
+
+
+class ArtifactNotLoggedError(ArtifactStatusError):
+ """Raised for Artifact methods or attributes only available after logging."""
+
+ def __init__(
+ self, artifact: Optional["Artifact"] = None, attr: Optional[str] = None
+ ):
+ super().__init__(
+ artifact,
+ attr,
+ "'{method_id}' used prior to logging artifact or while in offline mode. "
+ "Call wait() before accessing logged artifact properties.",
+ )
+
+
+class ArtifactFinalizedError(ArtifactStatusError):
+ """Raised for Artifact methods or attributes that can't be changed after logging."""
+
+ def __init__(
+ self, artifact: Optional["Artifact"] = None, attr: Optional[str] = None
+ ):
+ super().__init__(
+ artifact,
+ attr,
+ "'{method_id}' used on logged artifact. Can't modify finalized artifact.",
+ )
+
+
+class WaitTimeoutError(errors.Error):
+ """Raised when wait() timeout occurs before process is finished."""
diff --git a/wandb/sdk/artifacts/staging.py b/wandb/sdk/artifacts/staging.py
new file mode 100644
index 0000000000000000000000000000000000000000..d970d71b02f381fe7985e1cc8aed9b9d551ec890
--- /dev/null
+++ b/wandb/sdk/artifacts/staging.py
@@ -0,0 +1,25 @@
+"""Manages artifact file staging.
+
+Artifact files are copied to the staging area as soon as they are added to an artifact
+in order to avoid file changes corrupting the artifact. Once the upload is complete, the
+file should be moved to the artifact cache.
+"""
+
+import os
+
+from wandb import env
+from wandb.sdk.lib.filesystem import mkdir_exists_ok
+from wandb.sdk.lib.paths import FilePathStr
+
+
+def get_staging_dir() -> FilePathStr:
+ path = os.path.join(env.get_data_dir(), "artifacts", "staging")
+ try:
+ mkdir_exists_ok(path)
+ except OSError as e:
+ raise PermissionError(
+ f"Unable to write staging files to {path}. To fix this problem, please set "
+ f"{env.DATA_DIR} to a directory where you have the necessary write access."
+ ) from e
+
+ return FilePathStr(os.path.abspath(os.path.expanduser(path)))
diff --git a/wandb/sdk/artifacts/storage_handler.py b/wandb/sdk/artifacts/storage_handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..cf736fc4e97740280b60538cd1ba793f02850ffa
--- /dev/null
+++ b/wandb/sdk/artifacts/storage_handler.py
@@ -0,0 +1,59 @@
+"""Storage handler."""
+from typing import TYPE_CHECKING, Optional, Sequence, Union
+
+from wandb.sdk.lib.paths import FilePathStr, URIStr
+
+if TYPE_CHECKING:
+ from urllib.parse import ParseResult
+
+ from wandb.sdk.artifacts.artifact import Artifact
+ from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+
+DEFAULT_MAX_OBJECTS = 10000
+
+
+class StorageHandler:
+ def can_handle(self, parsed_url: "ParseResult") -> bool:
+ """Checks whether this handler can handle the given url.
+
+ Returns:
+ Whether this handler can handle the given url.
+ """
+ raise NotImplementedError
+
+ def load_path(
+ self,
+ manifest_entry: "ArtifactManifestEntry",
+ local: bool = False,
+ ) -> Union[URIStr, FilePathStr]:
+ """Load a file or directory given the corresponding index entry.
+
+ Args:
+ manifest_entry: The index entry to load
+ local: Whether to load the file locally or not
+
+ Returns:
+ A path to the file represented by `index_entry`
+ """
+ raise NotImplementedError
+
+ def store_path(
+ self,
+ artifact: "Artifact",
+ path: Union[URIStr, FilePathStr],
+ name: Optional[str] = None,
+ checksum: bool = True,
+ max_objects: Optional[int] = None,
+ ) -> Sequence["ArtifactManifestEntry"]:
+ """Store the file or directory at the given path to the specified artifact.
+
+ Args:
+ path: The path to store
+ name: If specified, the logical name that should map to `path`
+ checksum: Whether to compute the checksum of the file
+ max_objects: The maximum number of objects to store
+
+ Returns:
+ A list of manifest entries to store within the artifact
+ """
+ raise NotImplementedError
diff --git a/wandb/sdk/artifacts/storage_handlers/__init__.py b/wandb/sdk/artifacts/storage_handlers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sdk/artifacts/storage_handlers/azure_handler.py b/wandb/sdk/artifacts/storage_handlers/azure_handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..83ead0c7c7ffe8de2064e7c940e4964e2bf22e4f
--- /dev/null
+++ b/wandb/sdk/artifacts/storage_handlers/azure_handler.py
@@ -0,0 +1,193 @@
+"""Azure storage handler."""
+from pathlib import PurePosixPath
+from types import ModuleType
+from typing import TYPE_CHECKING, Dict, Optional, Sequence, Tuple, Union
+from urllib.parse import ParseResult, parse_qsl, urlparse
+
+import wandb
+from wandb import util
+from wandb.sdk.artifacts.artifact_file_cache import get_artifact_file_cache
+from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+from wandb.sdk.artifacts.storage_handler import DEFAULT_MAX_OBJECTS, StorageHandler
+from wandb.sdk.lib.hashutil import ETag
+from wandb.sdk.lib.paths import FilePathStr, LogicalPath, StrPath, URIStr
+
+if TYPE_CHECKING:
+ import azure.identity # type: ignore
+ import azure.storage.blob # type: ignore
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+
+class AzureHandler(StorageHandler):
+ def can_handle(self, parsed_url: "ParseResult") -> bool:
+ return parsed_url.scheme == "https" and parsed_url.netloc.endswith(
+ ".blob.core.windows.net"
+ )
+
+ def load_path(
+ self,
+ manifest_entry: "ArtifactManifestEntry",
+ local: bool = False,
+ ) -> Union[URIStr, FilePathStr]:
+ assert manifest_entry.ref is not None
+ if not local:
+ return manifest_entry.ref
+
+ path, hit, cache_open = get_artifact_file_cache().check_etag_obj_path(
+ URIStr(manifest_entry.ref),
+ ETag(manifest_entry.digest),
+ manifest_entry.size or 0,
+ )
+ if hit:
+ return path
+
+ account_url, container_name, blob_name, query = self._parse_uri(
+ manifest_entry.ref
+ )
+ version_id = manifest_entry.extra.get("versionID")
+ blob_service_client = self._get_module("azure.storage.blob").BlobServiceClient(
+ account_url, credential=self._get_credential(account_url)
+ )
+ blob_client = blob_service_client.get_blob_client(
+ container=container_name, blob=blob_name
+ )
+ if version_id is None:
+ # Try current version, then all versions.
+ try:
+ downloader = blob_client.download_blob(
+ etag=manifest_entry.digest,
+ match_condition=self._get_module(
+ "azure.core"
+ ).MatchConditions.IfNotModified,
+ )
+ except self._get_module("azure.core.exceptions").ResourceModifiedError:
+ container_client = blob_service_client.get_container_client(
+ container_name
+ )
+ for blob_properties in container_client.walk_blobs(
+ name_starts_with=blob_name, include=["versions"]
+ ):
+ if (
+ blob_properties.name == blob_name
+ and blob_properties.etag == manifest_entry.digest
+ and blob_properties.version_id is not None
+ ):
+ downloader = blob_client.download_blob(
+ version_id=blob_properties.version_id
+ )
+ break
+ else: # didn't break
+ raise ValueError(
+ f"Couldn't find blob version for {manifest_entry.ref} matching "
+ f"etag {manifest_entry.digest}."
+ )
+ else:
+ downloader = blob_client.download_blob(version_id=version_id)
+ with cache_open(mode="wb") as f:
+ downloader.readinto(f)
+ return path
+
+ def store_path(
+ self,
+ artifact: "Artifact",
+ path: Union[URIStr, FilePathStr],
+ name: Optional[StrPath] = None,
+ checksum: bool = True,
+ max_objects: Optional[int] = None,
+ ) -> Sequence["ArtifactManifestEntry"]:
+ account_url, container_name, blob_name, query = self._parse_uri(path)
+ path = URIStr(f"{account_url}/{container_name}/{blob_name}")
+
+ if not checksum:
+ return [
+ ArtifactManifestEntry(path=name or blob_name, digest=path, ref=path)
+ ]
+
+ blob_service_client = self._get_module("azure.storage.blob").BlobServiceClient(
+ account_url, credential=self._get_credential(account_url)
+ )
+ blob_client = blob_service_client.get_blob_client(
+ container=container_name, blob=blob_name
+ )
+ if blob_client.exists(version_id=query.get("versionId")):
+ blob_properties = blob_client.get_blob_properties(
+ version_id=query.get("versionId")
+ )
+ return [
+ self._create_entry(
+ blob_properties,
+ path=name or PurePosixPath(blob_name).name,
+ ref=URIStr(
+ f"{account_url}/{container_name}/{blob_properties.name}"
+ ),
+ )
+ ]
+
+ entries = []
+ container_client = blob_service_client.get_container_client(container_name)
+ max_objects = max_objects or DEFAULT_MAX_OBJECTS
+ for i, blob_properties in enumerate(
+ container_client.list_blobs(name_starts_with=f"{blob_name}/")
+ ):
+ if i >= max_objects:
+ raise ValueError(
+ f"Exceeded {max_objects} objects tracked, pass max_objects to "
+ f"add_reference"
+ )
+ suffix = PurePosixPath(blob_properties.name).relative_to(blob_name)
+ entries.append(
+ self._create_entry(
+ blob_properties,
+ path=LogicalPath(name) / suffix if name else suffix,
+ ref=URIStr(
+ f"{account_url}/{container_name}/{blob_properties.name}"
+ ),
+ )
+ )
+ return entries
+
+ def _get_module(self, name: str) -> ModuleType:
+ module = util.get_module(
+ name,
+ lazy=False,
+ required="Azure references require the azure library, run "
+ "pip install wandb[azure]",
+ )
+ assert isinstance(module, ModuleType)
+ return module
+
+ def _get_credential(
+ self, account_url: str
+ ) -> Union["azure.identity.DefaultAzureCredential", str]:
+ if (
+ wandb.run
+ and wandb.run.settings.azure_account_url_to_access_key is not None
+ and account_url in wandb.run.settings.azure_account_url_to_access_key
+ ):
+ return wandb.run.settings.azure_account_url_to_access_key[account_url]
+ return self._get_module("azure.identity").DefaultAzureCredential()
+
+ def _parse_uri(self, uri: str) -> Tuple[str, str, str, Dict[str, str]]:
+ parsed_url = urlparse(uri)
+ query = dict(parse_qsl(parsed_url.query))
+ account_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
+ _, container_name, blob_name = parsed_url.path.split("/", 2)
+ return account_url, container_name, blob_name, query
+
+ def _create_entry(
+ self,
+ blob_properties: "azure.storage.blob.BlobProperties",
+ path: StrPath,
+ ref: URIStr,
+ ) -> ArtifactManifestEntry:
+ extra = {"etag": blob_properties.etag.strip('"')}
+ if blob_properties.version_id:
+ extra["versionID"] = blob_properties.version_id
+ return ArtifactManifestEntry(
+ path=path,
+ ref=ref,
+ digest=blob_properties.etag.strip('"'),
+ size=blob_properties.size,
+ extra=extra,
+ )
diff --git a/wandb/sdk/artifacts/storage_handlers/gcs_handler.py b/wandb/sdk/artifacts/storage_handlers/gcs_handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..df8abcecb963f6562cb417d8f5bb345edc8c3e20
--- /dev/null
+++ b/wandb/sdk/artifacts/storage_handlers/gcs_handler.py
@@ -0,0 +1,200 @@
+"""GCS storage handler."""
+import time
+from pathlib import PurePosixPath
+from typing import TYPE_CHECKING, Dict, Optional, Sequence, Tuple, Union
+from urllib.parse import ParseResult, urlparse
+
+from wandb import util
+from wandb.errors.term import termlog
+from wandb.sdk.artifacts.artifact_file_cache import get_artifact_file_cache
+from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+from wandb.sdk.artifacts.storage_handler import DEFAULT_MAX_OBJECTS, StorageHandler
+from wandb.sdk.lib.hashutil import B64MD5
+from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
+
+if TYPE_CHECKING:
+ import google.cloud.storage as gcs_module # type: ignore
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+
+class GCSHandler(StorageHandler):
+ _client: Optional["gcs_module.client.Client"]
+
+ def __init__(self, scheme: Optional[str] = None) -> None:
+ self._scheme = scheme or "gs"
+ self._client = None
+ self._cache = get_artifact_file_cache()
+
+ def can_handle(self, parsed_url: "ParseResult") -> bool:
+ return parsed_url.scheme == self._scheme
+
+ def init_gcs(self) -> "gcs_module.client.Client":
+ if self._client is not None:
+ return self._client
+ storage = util.get_module(
+ "google.cloud.storage",
+ required="gs:// references requires the google-cloud-storage library, run pip install wandb[gcp]",
+ )
+ self._client = storage.Client()
+ return self._client
+
+ def _parse_uri(self, uri: str) -> Tuple[str, str, Optional[str]]:
+ url = urlparse(uri)
+ bucket = url.netloc
+ key = url.path[1:]
+ version = url.fragment if url.fragment else None
+ return bucket, key, version
+
+ def load_path(
+ self,
+ manifest_entry: ArtifactManifestEntry,
+ local: bool = False,
+ ) -> Union[URIStr, FilePathStr]:
+ if not local:
+ assert manifest_entry.ref is not None
+ return manifest_entry.ref
+
+ path, hit, cache_open = self._cache.check_md5_obj_path(
+ B64MD5(manifest_entry.digest), # TODO(spencerpearson): unsafe cast
+ manifest_entry.size if manifest_entry.size is not None else 0,
+ )
+ if hit:
+ return path
+
+ self.init_gcs()
+ assert self._client is not None # mypy: unwraps optionality
+ assert manifest_entry.ref is not None
+ bucket, key, _ = self._parse_uri(manifest_entry.ref)
+ version = manifest_entry.extra.get("versionID")
+
+ obj = None
+ # First attempt to get the generation specified, this will return None if versioning is not enabled
+ if version is not None:
+ obj = self._client.bucket(bucket).get_blob(key, generation=version)
+
+ if obj is None:
+ # Object versioning is disabled on the bucket, so just get
+ # the latest version and make sure the MD5 matches.
+ obj = self._client.bucket(bucket).get_blob(key)
+ if obj is None:
+ raise ValueError(
+ f"Unable to download object {manifest_entry.ref} with generation {version}"
+ )
+ md5 = obj.md5_hash
+ if md5 != manifest_entry.digest:
+ raise ValueError(
+ f"Digest mismatch for object {manifest_entry.ref}: expected {manifest_entry.digest} but found {md5}"
+ )
+
+ with cache_open(mode="wb") as f:
+ obj.download_to_file(f)
+ return path
+
+ def store_path(
+ self,
+ artifact: "Artifact",
+ path: Union[URIStr, FilePathStr],
+ name: Optional[StrPath] = None,
+ checksum: bool = True,
+ max_objects: Optional[int] = None,
+ ) -> Sequence[ArtifactManifestEntry]:
+ self.init_gcs()
+ assert self._client is not None # mypy: unwraps optionality
+
+ # After parsing any query params / fragments for additional context,
+ # such as version identifiers, pare down the path to just the bucket
+ # and key.
+ bucket, key, version = self._parse_uri(path)
+ path = URIStr(f"{self._scheme}://{bucket}/{key}")
+ max_objects = max_objects or DEFAULT_MAX_OBJECTS
+
+ if not checksum:
+ return [ArtifactManifestEntry(path=name or key, ref=path, digest=path)]
+
+ start_time = None
+ obj = self._client.bucket(bucket).get_blob(key, generation=version)
+ if obj is None and version is not None:
+ raise ValueError(f"Object does not exist: {path}#{version}")
+ multi = obj is None
+ if multi:
+ start_time = time.time()
+ termlog(
+ 'Generating checksum for up to %i objects with prefix "%s"... '
+ % (max_objects, key),
+ newline=False,
+ )
+ objects = self._client.bucket(bucket).list_blobs(
+ prefix=key, max_results=max_objects
+ )
+ else:
+ objects = [obj]
+
+ entries = [
+ self._entry_from_obj(obj, path, name, prefix=key, multi=multi)
+ for obj in objects
+ ]
+ if start_time is not None:
+ termlog("Done. %.1fs" % (time.time() - start_time), prefix=False)
+ if len(entries) > max_objects:
+ raise ValueError(
+ "Exceeded %i objects tracked, pass max_objects to add_reference"
+ % max_objects
+ )
+ return entries
+
+ def _entry_from_obj(
+ self,
+ obj: "gcs_module.blob.Blob",
+ path: str,
+ name: Optional[StrPath] = None,
+ prefix: str = "",
+ multi: bool = False,
+ ) -> ArtifactManifestEntry:
+ """Create an ArtifactManifestEntry from a GCS object.
+
+ Arguments:
+ obj: The GCS object
+ path: The GCS-style path (e.g.: "gs://bucket/file.txt")
+ name: The user assigned name, or None if not specified
+ prefix: The prefix to add (will be the same as `path` for directories)
+ multi: Whether or not this is a multi-object add.
+ """
+ bucket, key, _ = self._parse_uri(path)
+
+ # Always use posix paths, since that's what S3 uses.
+ posix_key = PurePosixPath(obj.name) # the bucket key
+ posix_path = PurePosixPath(bucket) / PurePosixPath(
+ key
+ ) # the path, with the scheme stripped
+ posix_prefix = PurePosixPath(prefix) # the prefix, if adding a prefix
+ posix_name = PurePosixPath(name or "")
+ posix_ref = posix_path
+
+ if name is None:
+ # We're adding a directory (prefix), so calculate a relative path.
+ if str(posix_prefix) in str(posix_key) and posix_prefix != posix_key:
+ posix_name = posix_key.relative_to(posix_prefix)
+ posix_ref = posix_path / posix_name
+ else:
+ posix_name = PurePosixPath(posix_key.name)
+ posix_ref = posix_path
+ elif multi:
+ # We're adding a directory with a name override.
+ relpath = posix_key.relative_to(posix_prefix)
+ posix_name = posix_name / relpath
+ posix_ref = posix_path / relpath
+ return ArtifactManifestEntry(
+ path=posix_name,
+ ref=URIStr(f"{self._scheme}://{str(posix_ref)}"),
+ digest=obj.md5_hash,
+ size=obj.size,
+ extra=self._extra_from_obj(obj),
+ )
+
+ @staticmethod
+ def _extra_from_obj(obj: "gcs_module.blob.Blob") -> Dict[str, str]:
+ return {
+ "etag": obj.etag,
+ "versionID": obj.generation,
+ }
diff --git a/wandb/sdk/artifacts/storage_handlers/http_handler.py b/wandb/sdk/artifacts/storage_handlers/http_handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..6c3bdb326adb56b9f4110c04649fa50df5bae623
--- /dev/null
+++ b/wandb/sdk/artifacts/storage_handlers/http_handler.py
@@ -0,0 +1,112 @@
+"""HTTP storage handler."""
+import os
+from typing import TYPE_CHECKING, Dict, Optional, Sequence, Tuple, Union
+from urllib.parse import ParseResult
+
+from wandb.sdk.artifacts.artifact_file_cache import get_artifact_file_cache
+from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+from wandb.sdk.artifacts.storage_handler import StorageHandler
+from wandb.sdk.internal.thread_local_settings import _thread_local_api_settings
+from wandb.sdk.lib.hashutil import ETag
+from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
+
+if TYPE_CHECKING:
+ import requests
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+
+class HTTPHandler(StorageHandler):
+ def __init__(
+ self, session: "requests.Session", scheme: Optional[str] = None
+ ) -> None:
+ self._scheme = scheme or "http"
+ self._cache = get_artifact_file_cache()
+ self._session = session
+
+ def can_handle(self, parsed_url: "ParseResult") -> bool:
+ return parsed_url.scheme == self._scheme
+
+ def load_path(
+ self,
+ manifest_entry: ArtifactManifestEntry,
+ local: bool = False,
+ ) -> Union[URIStr, FilePathStr]:
+ if not local:
+ assert manifest_entry.ref is not None
+ return manifest_entry.ref
+
+ assert manifest_entry.ref is not None
+
+ path, hit, cache_open = self._cache.check_etag_obj_path(
+ URIStr(manifest_entry.ref),
+ ETag(manifest_entry.digest), # TODO(spencerpearson): unsafe cast
+ manifest_entry.size if manifest_entry.size is not None else 0,
+ )
+ if hit:
+ return path
+
+ response = self._session.get(
+ manifest_entry.ref,
+ stream=True,
+ cookies=_thread_local_api_settings.cookies,
+ headers=_thread_local_api_settings.headers,
+ )
+ response.raise_for_status()
+
+ digest: Optional[Union[ETag, FilePathStr, URIStr]]
+ digest, size, extra = self._entry_from_headers(response.headers)
+ digest = digest or manifest_entry.ref
+ if manifest_entry.digest != digest:
+ raise ValueError(
+ f"Digest mismatch for url {manifest_entry.ref}: expected {manifest_entry.digest} but found {digest}"
+ )
+
+ with cache_open(mode="wb") as file:
+ for data in response.iter_content(chunk_size=16 * 1024):
+ file.write(data)
+ return path
+
+ def store_path(
+ self,
+ artifact: "Artifact",
+ path: Union[URIStr, FilePathStr],
+ name: Optional[StrPath] = None,
+ checksum: bool = True,
+ max_objects: Optional[int] = None,
+ ) -> Sequence[ArtifactManifestEntry]:
+ name = name or os.path.basename(path)
+ if not checksum:
+ return [ArtifactManifestEntry(path=name, ref=path, digest=path)]
+
+ with self._session.get(
+ path,
+ stream=True,
+ cookies=_thread_local_api_settings.cookies,
+ headers=_thread_local_api_settings.headers,
+ ) as response:
+ response.raise_for_status()
+ digest: Optional[Union[ETag, FilePathStr, URIStr]]
+ digest, size, extra = self._entry_from_headers(response.headers)
+ digest = digest or path
+ return [
+ ArtifactManifestEntry(
+ path=name, ref=path, digest=digest, size=size, extra=extra
+ )
+ ]
+
+ def _entry_from_headers(
+ self, headers: "requests.structures.CaseInsensitiveDict"
+ ) -> Tuple[Optional[ETag], Optional[int], Dict[str, str]]:
+ response_headers = {k.lower(): v for k, v in headers.items()}
+ size = None
+ if response_headers.get("content-length", None):
+ size = int(response_headers["content-length"])
+
+ digest = response_headers.get("etag", None)
+ extra = {}
+ if digest:
+ extra["etag"] = digest
+ if digest and digest[:1] == '"' and digest[-1:] == '"':
+ digest = digest[1:-1] # trim leading and trailing quotes around etag
+ return digest, size, extra
diff --git a/wandb/sdk/artifacts/storage_handlers/local_file_handler.py b/wandb/sdk/artifacts/storage_handlers/local_file_handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..a55c11ae2c409a1e7eedffd3c4badad7c365b965
--- /dev/null
+++ b/wandb/sdk/artifacts/storage_handlers/local_file_handler.py
@@ -0,0 +1,134 @@
+"""Local file storage handler."""
+import os
+import shutil
+import time
+from typing import TYPE_CHECKING, Optional, Sequence, Union
+from urllib.parse import ParseResult
+
+from wandb import util
+from wandb.errors.term import termlog
+from wandb.sdk.artifacts.artifact_file_cache import get_artifact_file_cache
+from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+from wandb.sdk.artifacts.storage_handler import DEFAULT_MAX_OBJECTS, StorageHandler
+from wandb.sdk.lib import filesystem
+from wandb.sdk.lib.hashutil import B64MD5, md5_file_b64, md5_string
+from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
+
+if TYPE_CHECKING:
+ from wandb.sdk.artifacts.artifact import Artifact
+
+
+class LocalFileHandler(StorageHandler):
+ """Handles file:// references."""
+
+ def __init__(self, scheme: Optional[str] = None) -> None:
+ """Track files or directories on a local filesystem.
+
+ Expand directories to create an entry for each file contained.
+ """
+ self._scheme = scheme or "file"
+ self._cache = get_artifact_file_cache()
+
+ def can_handle(self, parsed_url: "ParseResult") -> bool:
+ return parsed_url.scheme == self._scheme
+
+ def load_path(
+ self,
+ manifest_entry: ArtifactManifestEntry,
+ local: bool = False,
+ ) -> Union[URIStr, FilePathStr]:
+ if manifest_entry.ref is None:
+ raise ValueError(f"Cannot add path with no ref: {manifest_entry.path}")
+ local_path = util.local_file_uri_to_path(str(manifest_entry.ref))
+ if not os.path.exists(local_path):
+ raise ValueError(
+ "Local file reference: Failed to find file at path %s" % local_path
+ )
+
+ path, hit, cache_open = self._cache.check_md5_obj_path(
+ B64MD5(manifest_entry.digest), # TODO(spencerpearson): unsafe cast
+ manifest_entry.size if manifest_entry.size is not None else 0,
+ )
+ if hit:
+ return path
+
+ md5 = md5_file_b64(local_path)
+ if md5 != manifest_entry.digest:
+ raise ValueError(
+ f"Local file reference: Digest mismatch for path {local_path}: expected {manifest_entry.digest} but found {md5}"
+ )
+
+ filesystem.mkdir_exists_ok(os.path.dirname(path))
+
+ with cache_open() as f:
+ shutil.copy(local_path, f.name)
+ return path
+
+ def store_path(
+ self,
+ artifact: "Artifact",
+ path: Union[URIStr, FilePathStr],
+ name: Optional[StrPath] = None,
+ checksum: bool = True,
+ max_objects: Optional[int] = None,
+ ) -> Sequence[ArtifactManifestEntry]:
+ local_path = util.local_file_uri_to_path(path)
+ max_objects = max_objects or DEFAULT_MAX_OBJECTS
+ # We have a single file or directory
+ # Note, we follow symlinks for files contained within the directory
+ entries = []
+
+ def md5(path: str) -> B64MD5:
+ return (
+ md5_file_b64(path)
+ if checksum
+ else md5_string(str(os.stat(path).st_size))
+ )
+
+ if os.path.isdir(local_path):
+ i = 0
+ start_time = time.time()
+ if checksum:
+ termlog(
+ 'Generating checksum for up to %i files in "%s"...\n'
+ % (max_objects, local_path),
+ newline=False,
+ )
+ for root, _, files in os.walk(local_path):
+ for sub_path in files:
+ i += 1
+ if i > max_objects:
+ raise ValueError(
+ "Exceeded %i objects tracked, pass max_objects to add_reference"
+ % max_objects
+ )
+ physical_path = os.path.join(root, sub_path)
+ # TODO(spencerpearson): this is not a "logical path" in the sense that
+ # `LogicalPath` returns a "logical path"; it's a relative path
+ # **on the local filesystem**.
+ logical_path = os.path.relpath(physical_path, start=local_path)
+ if name is not None:
+ logical_path = os.path.join(name, logical_path)
+
+ entry = ArtifactManifestEntry(
+ path=logical_path,
+ ref=FilePathStr(os.path.join(path, logical_path)),
+ size=os.path.getsize(physical_path),
+ digest=md5(physical_path),
+ )
+ entries.append(entry)
+ if checksum:
+ termlog("Done. %.1fs" % (time.time() - start_time), prefix=False)
+ elif os.path.isfile(local_path):
+ name = name or os.path.basename(local_path)
+ entry = ArtifactManifestEntry(
+ path=name,
+ ref=path,
+ size=os.path.getsize(local_path),
+ digest=md5(local_path),
+ )
+ entries.append(entry)
+ else:
+ # TODO: update error message if we don't allow directories.
+ raise ValueError('Path "%s" must be a valid file or directory path' % path)
+ return entries
diff --git a/wandb/sdk/artifacts/storage_handlers/multi_handler.py b/wandb/sdk/artifacts/storage_handlers/multi_handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..58516b0c28c4b35ee48299529f996a7deba535c2
--- /dev/null
+++ b/wandb/sdk/artifacts/storage_handlers/multi_handler.py
@@ -0,0 +1,53 @@
+"""Multi storage handler."""
+from typing import TYPE_CHECKING, List, Optional, Sequence, Union
+from urllib.parse import urlparse
+
+from wandb.sdk.artifacts.storage_handler import StorageHandler
+from wandb.sdk.lib.paths import FilePathStr, URIStr
+
+if TYPE_CHECKING:
+ from wandb.sdk.artifacts.artifact import Artifact
+ from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+
+
+class MultiHandler(StorageHandler):
+ _handlers: List[StorageHandler]
+
+ def __init__(
+ self,
+ handlers: Optional[List[StorageHandler]] = None,
+ default_handler: Optional[StorageHandler] = None,
+ ) -> None:
+ self._handlers = handlers or []
+ self._default_handler = default_handler
+
+ def _get_handler(self, url: Union[FilePathStr, URIStr]) -> StorageHandler:
+ parsed_url = urlparse(url)
+ for handler in self._handlers:
+ if handler.can_handle(parsed_url):
+ return handler
+ if self._default_handler is not None:
+ return self._default_handler
+ raise ValueError('No storage handler registered for url "%s"' % str(url))
+
+ def load_path(
+ self,
+ manifest_entry: "ArtifactManifestEntry",
+ local: bool = False,
+ ) -> Union[URIStr, FilePathStr]:
+ assert manifest_entry.ref is not None
+ handler = self._get_handler(manifest_entry.ref)
+ return handler.load_path(manifest_entry, local=local)
+
+ def store_path(
+ self,
+ artifact: "Artifact",
+ path: Union[URIStr, FilePathStr],
+ name: Optional[str] = None,
+ checksum: bool = True,
+ max_objects: Optional[int] = None,
+ ) -> Sequence["ArtifactManifestEntry"]:
+ handler = self._get_handler(path)
+ return handler.store_path(
+ artifact, path, name=name, checksum=checksum, max_objects=max_objects
+ )
diff --git a/wandb/sdk/artifacts/storage_handlers/s3_handler.py b/wandb/sdk/artifacts/storage_handlers/s3_handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..8ce962b4947553a291f15622fa4397d51b162403
--- /dev/null
+++ b/wandb/sdk/artifacts/storage_handlers/s3_handler.py
@@ -0,0 +1,298 @@
+"""S3 storage handler."""
+import os
+import time
+from pathlib import PurePosixPath
+from typing import TYPE_CHECKING, Dict, Optional, Sequence, Tuple, Union
+from urllib.parse import parse_qsl, urlparse
+
+from wandb import util
+from wandb.errors import CommError
+from wandb.errors.term import termlog
+from wandb.sdk.artifacts.artifact_file_cache import get_artifact_file_cache
+from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+from wandb.sdk.artifacts.storage_handler import DEFAULT_MAX_OBJECTS, StorageHandler
+from wandb.sdk.lib.hashutil import ETag
+from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
+
+if TYPE_CHECKING:
+ from urllib.parse import ParseResult
+
+ # We could probably use https://pypi.org/project/boto3-stubs/ or something
+ # instead of `type:ignore`ing these boto imports, but it's nontrivial:
+ # for some reason, despite being actively maintained as of 2022-09-30,
+ # the latest release of boto3-stubs doesn't include all the features we use.
+ import boto3 # type: ignore
+ import boto3.resources.base # type: ignore
+ import boto3.s3 # type: ignore
+ import boto3.session # type: ignore
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+
+class S3Handler(StorageHandler):
+ _s3: Optional["boto3.resources.base.ServiceResource"]
+ _scheme: str
+
+ def __init__(self, scheme: Optional[str] = None) -> None:
+ self._scheme = scheme or "s3"
+ self._s3 = None
+ self._cache = get_artifact_file_cache()
+
+ def can_handle(self, parsed_url: "ParseResult") -> bool:
+ return parsed_url.scheme == self._scheme
+
+ def init_boto(self) -> "boto3.resources.base.ServiceResource":
+ if self._s3 is not None:
+ return self._s3
+ boto: boto3 = util.get_module(
+ "boto3",
+ required="s3:// references requires the boto3 library, run pip install wandb[aws]",
+ lazy=False,
+ )
+ self._s3 = boto.session.Session().resource(
+ "s3",
+ endpoint_url=os.getenv("AWS_S3_ENDPOINT_URL"),
+ region_name=os.getenv("AWS_REGION"),
+ )
+ self._botocore = util.get_module("botocore")
+ return self._s3
+
+ def _parse_uri(self, uri: str) -> Tuple[str, str, Optional[str]]:
+ url = urlparse(uri)
+ query = dict(parse_qsl(url.query))
+
+ bucket = url.netloc
+ key = url.path[1:] # strip leading slash
+ version = query.get("versionId")
+
+ return bucket, key, version
+
+ def load_path(
+ self,
+ manifest_entry: ArtifactManifestEntry,
+ local: bool = False,
+ ) -> Union[URIStr, FilePathStr]:
+ if not local:
+ assert manifest_entry.ref is not None
+ return manifest_entry.ref
+
+ assert manifest_entry.ref is not None
+
+ path, hit, cache_open = self._cache.check_etag_obj_path(
+ URIStr(manifest_entry.ref),
+ ETag(manifest_entry.digest), # TODO(spencerpearson): unsafe cast
+ manifest_entry.size if manifest_entry.size is not None else 0,
+ )
+ if hit:
+ return path
+
+ self.init_boto()
+ assert self._s3 is not None # mypy: unwraps optionality
+ bucket, key, _ = self._parse_uri(manifest_entry.ref)
+ version = manifest_entry.extra.get("versionID")
+
+ extra_args = {}
+ if version:
+ obj_version = self._s3.ObjectVersion(bucket, key, version)
+ extra_args["VersionId"] = version
+ obj = obj_version.Object()
+ else:
+ obj = self._s3.Object(bucket, key)
+
+ try:
+ etag = (
+ obj_version.head()["ETag"][1:-1] # escape leading and trailing
+ if version
+ else self._etag_from_obj(obj)
+ )
+ except self._botocore.exceptions.ClientError as e:
+ if e.response["Error"]["Code"] == "404":
+ raise FileNotFoundError(
+ f"Unable to find {manifest_entry.path} at s3://{bucket}/{key}"
+ ) from e
+ raise
+
+ if etag != manifest_entry.digest:
+ # Try to match the etag with some other version.
+ if version:
+ raise ValueError(
+ f"Digest mismatch for object {manifest_entry.ref} with version {version}: expected {manifest_entry.digest} but found {etag}"
+ )
+ obj = None
+ object_versions = self._s3.Bucket(bucket).object_versions.filter(Prefix=key)
+ for object_version in object_versions:
+ if manifest_entry.extra.get("etag") == self._etag_from_obj(
+ object_version
+ ):
+ obj = object_version.Object()
+ extra_args["VersionId"] = object_version.version_id
+ break
+ if obj is None:
+ raise FileNotFoundError(
+ "Couldn't find object version for {}/{} matching etag {}".format(
+ bucket, key, manifest_entry.extra.get("etag")
+ )
+ )
+
+ with cache_open(mode="wb") as f:
+ obj.download_fileobj(f, ExtraArgs=extra_args)
+ return path
+
+ def store_path(
+ self,
+ artifact: "Artifact",
+ path: Union[URIStr, FilePathStr],
+ name: Optional[StrPath] = None,
+ checksum: bool = True,
+ max_objects: Optional[int] = None,
+ ) -> Sequence[ArtifactManifestEntry]:
+ self.init_boto()
+ assert self._s3 is not None # mypy: unwraps optionality
+
+ # The passed in path might have query string parameters.
+ # We only need to care about a subset, like version, when
+ # parsing. Once we have that, we can store the rest of the
+ # metadata in the artifact entry itself.
+ bucket, key, version = self._parse_uri(path)
+ path = URIStr(f"{self._scheme}://{bucket}/{key}")
+
+ max_objects = max_objects or DEFAULT_MAX_OBJECTS
+ if not checksum:
+ entry_path = name or (key if key != "" else bucket)
+ return [ArtifactManifestEntry(path=entry_path, ref=path, digest=path)]
+
+ # If an explicit version is specified, use that. Otherwise, use the head version.
+ objs = (
+ [self._s3.ObjectVersion(bucket, key, version).Object()]
+ if version
+ else [self._s3.Object(bucket, key)]
+ )
+ start_time = None
+ multi = False
+ if key != "":
+ try:
+ objs[0].load()
+ # S3 doesn't have real folders, however there are cases where the folder key has a valid file which will not
+ # trigger a recursive upload.
+ # we should check the object's metadata says it is a directory and do a multi file upload if it is
+ if "x-directory" in objs[0].content_type:
+ multi = True
+ except self._botocore.exceptions.ClientError as e:
+ if e.response["Error"]["Code"] == "404":
+ multi = True
+ else:
+ raise CommError(
+ "Unable to connect to S3 ({}): {}".format(
+ e.response["Error"]["Code"], e.response["Error"]["Message"]
+ )
+ )
+ else:
+ multi = True
+
+ if multi:
+ start_time = time.time()
+ termlog(
+ 'Generating checksum for up to %i objects in "%s/%s"... '
+ % (max_objects, bucket, key),
+ newline=False,
+ )
+ if key != "":
+ objs = (
+ self._s3.Bucket(bucket)
+ .objects.filter(Prefix=key)
+ .limit(max_objects)
+ )
+ else:
+ objs = self._s3.Bucket(bucket).objects.limit(max_objects)
+ # Weird iterator scoping makes us assign this to a local function
+ size = self._size_from_obj
+ entries = [
+ self._entry_from_obj(obj, path, name, prefix=key, multi=multi)
+ for obj in objs
+ if size(obj) > 0
+ ]
+ if start_time is not None:
+ termlog("Done. %.1fs" % (time.time() - start_time), prefix=False)
+ if len(entries) > max_objects:
+ raise ValueError(
+ "Exceeded %i objects tracked, pass max_objects to add_reference"
+ % max_objects
+ )
+ return entries
+
+ def _size_from_obj(
+ self, obj: Union["boto3.s3.Object", "boto3.s3.ObjectSummary"]
+ ) -> int:
+ # ObjectSummary has size, Object has content_length
+ size: int
+ if hasattr(obj, "size"):
+ size = obj.size
+ else:
+ size = obj.content_length
+ return size
+
+ def _entry_from_obj(
+ self,
+ obj: Union["boto3.s3.Object", "boto3.s3.ObjectSummary"],
+ path: str,
+ name: Optional[StrPath] = None,
+ prefix: str = "",
+ multi: bool = False,
+ ) -> ArtifactManifestEntry:
+ """Create an ArtifactManifestEntry from an S3 object.
+
+ Arguments:
+ obj: The S3 object
+ path: The S3-style path (e.g.: "s3://bucket/file.txt")
+ name: The user assigned name, or None if not specified
+ prefix: The prefix to add (will be the same as `path` for directories)
+ multi: Whether or not this is a multi-object add.
+ """
+ bucket, key, _ = self._parse_uri(path)
+
+ # Always use posix paths, since that's what S3 uses.
+ posix_key = PurePosixPath(obj.key) # the bucket key
+ posix_path = PurePosixPath(bucket) / key # the path, with the scheme stripped
+ posix_prefix = PurePosixPath(prefix) # the prefix, if adding a prefix
+ posix_name = PurePosixPath(name or "")
+ posix_ref = posix_path
+
+ if name is None:
+ # We're adding a directory (prefix), so calculate a relative path.
+ if str(posix_prefix) in str(posix_key) and posix_prefix != posix_key:
+ posix_name = posix_key.relative_to(posix_prefix)
+ posix_ref = posix_path / posix_name
+ else:
+ posix_name = PurePosixPath(posix_key.name)
+ posix_ref = posix_path
+ elif multi:
+ # We're adding a directory with a name override.
+ relpath = posix_key.relative_to(posix_prefix)
+ posix_name = posix_name / relpath
+ posix_ref = posix_path / relpath
+ return ArtifactManifestEntry(
+ path=posix_name,
+ ref=URIStr(f"{self._scheme}://{str(posix_ref)}"),
+ digest=ETag(self._etag_from_obj(obj)),
+ size=self._size_from_obj(obj),
+ extra=self._extra_from_obj(obj),
+ )
+
+ @staticmethod
+ def _etag_from_obj(obj: Union["boto3.s3.Object", "boto3.s3.ObjectSummary"]) -> ETag:
+ etag: ETag
+ etag = obj.e_tag[1:-1] # escape leading and trailing quote
+ return etag
+
+ def _extra_from_obj(
+ self, obj: Union["boto3.s3.Object", "boto3.s3.ObjectSummary"]
+ ) -> Dict[str, str]:
+ extra = {
+ "etag": obj.e_tag[1:-1], # escape leading and trailing quote
+ }
+ if not hasattr(obj, "version_id"):
+ # Convert ObjectSummary to Object to get the version_id.
+ obj = self._s3.Object(obj.bucket_name, obj.key) # type: ignore[union-attr]
+ if hasattr(obj, "version_id") and obj.version_id and obj.version_id != "null":
+ extra["versionID"] = obj.version_id
+ return extra
diff --git a/wandb/sdk/artifacts/storage_handlers/tracking_handler.py b/wandb/sdk/artifacts/storage_handlers/tracking_handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..496f0206d1a5dc657965ed4c9a4f539e27f5e92f
--- /dev/null
+++ b/wandb/sdk/artifacts/storage_handlers/tracking_handler.py
@@ -0,0 +1,67 @@
+"""Tracking storage handler."""
+from typing import TYPE_CHECKING, Optional, Sequence, Union
+from urllib.parse import urlparse
+
+from wandb.errors.term import termwarn
+from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+from wandb.sdk.artifacts.storage_handler import StorageHandler
+from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
+
+if TYPE_CHECKING:
+ from urllib.parse import ParseResult
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+
+class TrackingHandler(StorageHandler):
+ def __init__(self, scheme: Optional[str] = None) -> None:
+ """Track paths with no modification or special processing.
+
+ Useful when paths being tracked are on file systems mounted at a standardized
+ location.
+
+ For example, if the data to track is located on an NFS share mounted on
+ `/data`, then it is sufficient to just track the paths.
+ """
+ self._scheme = scheme or ""
+
+ def can_handle(self, parsed_url: "ParseResult") -> bool:
+ return parsed_url.scheme == self._scheme
+
+ def load_path(
+ self,
+ manifest_entry: ArtifactManifestEntry,
+ local: bool = False,
+ ) -> Union[URIStr, FilePathStr]:
+ if local:
+ # Likely a user error. The tracking handler is
+ # oblivious to the underlying paths, so it has
+ # no way of actually loading it.
+ url = urlparse(manifest_entry.ref)
+ raise ValueError(
+ f"Cannot download file at path {str(manifest_entry.ref)}, scheme {str(url.scheme)} not recognized"
+ )
+ # TODO(spencerpearson): should this go through util.to_native_slash_path
+ # instead of just getting typecast?
+ return FilePathStr(manifest_entry.path)
+
+ def store_path(
+ self,
+ artifact: "Artifact",
+ path: Union[URIStr, FilePathStr],
+ name: Optional[StrPath] = None,
+ checksum: bool = True,
+ max_objects: Optional[int] = None,
+ ) -> Sequence[ArtifactManifestEntry]:
+ url = urlparse(path)
+ if name is None:
+ raise ValueError(
+ 'You must pass name="<entry_name>" when tracking references with unknown schemes. ref: %s'
+ % path
+ )
+ termwarn(
+ "Artifact references with unsupported schemes cannot be checksummed: %s"
+ % path
+ )
+ name = name or url.path[1:] # strip leading slash
+ return [ArtifactManifestEntry(path=name, ref=path, digest=path)]
diff --git a/wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py b/wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..173c145d9834aba8e9d20d3c14cfefc5547b6040
--- /dev/null
+++ b/wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py
@@ -0,0 +1,132 @@
+"""WB artifact storage handler."""
+import os
+from typing import TYPE_CHECKING, Optional, Sequence, Union
+from urllib.parse import urlparse
+
+import wandb
+from wandb import util
+from wandb.apis import PublicApi
+from wandb.sdk.artifacts.artifact_file_cache import get_artifact_file_cache
+from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+from wandb.sdk.artifacts.storage_handler import StorageHandler
+from wandb.sdk.lib.hashutil import B64MD5, b64_to_hex_id, hex_to_b64_id
+from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
+
+if TYPE_CHECKING:
+ from urllib.parse import ParseResult
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+
+class WBArtifactHandler(StorageHandler):
+ """Handles loading and storing Artifact reference-type files."""
+
+ _client: Optional[PublicApi]
+
+ def __init__(self) -> None:
+ self._scheme = "wandb-artifact"
+ self._cache = get_artifact_file_cache()
+ self._client = None
+
+ def can_handle(self, parsed_url: "ParseResult") -> bool:
+ return parsed_url.scheme == self._scheme
+
+ @property
+ def client(self) -> PublicApi:
+ if self._client is None:
+ self._client = PublicApi()
+ return self._client
+
+ def load_path(
+ self,
+ manifest_entry: ArtifactManifestEntry,
+ local: bool = False,
+ ) -> Union[URIStr, FilePathStr]:
+ """Load the file in the specified artifact given its corresponding entry.
+
+ Download the referenced artifact; create and return a new symlink to the caller.
+
+ Arguments:
+ manifest_entry (ArtifactManifestEntry): The index entry to load
+
+ Returns:
+ (os.PathLike): A path to the file represented by `index_entry`
+ """
+ # We don't check for cache hits here. Since we have 0 for size (since this
+ # is a cross-artifact reference which and we've made the choice to store 0
+ # in the size field), we can't confirm if the file is complete. So we just
+ # rely on the dep_artifact entry's download() method to do its own cache
+ # check.
+
+ # Parse the reference path and download the artifact if needed
+ artifact_id = util.host_from_path(manifest_entry.ref)
+ artifact_file_path = util.uri_from_path(manifest_entry.ref)
+
+ dep_artifact = wandb.Artifact._from_id(
+ hex_to_b64_id(artifact_id), self.client.client
+ )
+ assert dep_artifact is not None
+ link_target_path: Union[URIStr, FilePathStr]
+ if local:
+ link_target_path = dep_artifact.get_entry(artifact_file_path).download()
+ else:
+ link_target_path = dep_artifact.get_entry(artifact_file_path).ref_target()
+
+ return link_target_path
+
+ def store_path(
+ self,
+ artifact: "Artifact",
+ path: Union[URIStr, FilePathStr],
+ name: Optional[StrPath] = None,
+ checksum: bool = True,
+ max_objects: Optional[int] = None,
+ ) -> Sequence[ArtifactManifestEntry]:
+ """Store the file or directory at the given path into the specified artifact.
+
+ Recursively resolves the reference until the result is a concrete asset.
+
+ Arguments:
+ artifact: The artifact doing the storing path (str): The path to store name
+ (str): If specified, the logical name that should map to `path`
+
+ Returns:
+ (list[ArtifactManifestEntry]): A list of manifest entries to store within
+ the artifact
+ """
+ # Recursively resolve the reference until a concrete asset is found
+ # TODO: Consider resolving server-side for performance improvements.
+ iter_path: Union[URIStr, FilePathStr, None] = path
+ while iter_path is not None and urlparse(iter_path).scheme == self._scheme:
+ artifact_id = util.host_from_path(iter_path)
+ artifact_file_path = util.uri_from_path(iter_path)
+ target_artifact = wandb.Artifact._from_id(
+ hex_to_b64_id(artifact_id), self.client.client
+ )
+ assert target_artifact is not None
+
+ entry = target_artifact.manifest.get_entry_by_path(artifact_file_path)
+ assert entry is not None
+ iter_path = entry.ref
+
+ # Create the path reference
+ assert target_artifact is not None
+ assert target_artifact.id is not None
+ path = URIStr(
+ "{}://{}/{}".format(
+ self._scheme,
+ b64_to_hex_id(B64MD5(target_artifact.id)),
+ artifact_file_path,
+ )
+ )
+
+ # Return the new entry
+ assert entry is not None
+ return [
+ ArtifactManifestEntry(
+ path=name or os.path.basename(path),
+ ref=path,
+ size=0,
+ digest=entry.digest,
+ )
+ ]
diff --git a/wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py b/wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..dc92eb5e687df6a906af3a134a6d1a2736d01c39
--- /dev/null
+++ b/wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py
@@ -0,0 +1,71 @@
+"""WB local artifact storage handler."""
+import os
+from typing import TYPE_CHECKING, Optional, Sequence, Union
+
+import wandb
+from wandb import util
+from wandb.sdk.artifacts.artifact_instance_cache import artifact_instance_cache
+from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+from wandb.sdk.artifacts.storage_handler import StorageHandler
+from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
+
+if TYPE_CHECKING:
+ from urllib.parse import ParseResult
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+
+class WBLocalArtifactHandler(StorageHandler):
+ """Handles loading and storing Artifact reference-type files."""
+
+ def __init__(self) -> None:
+ self._scheme = "wandb-client-artifact"
+
+ def can_handle(self, parsed_url: "ParseResult") -> bool:
+ return parsed_url.scheme == self._scheme
+
+ def load_path(
+ self,
+ manifest_entry: ArtifactManifestEntry,
+ local: bool = False,
+ ) -> Union[URIStr, FilePathStr]:
+ raise NotImplementedError(
+ "Should not be loading a path for an artifact entry with unresolved client id."
+ )
+
+ def store_path(
+ self,
+ artifact: "Artifact",
+ path: Union[URIStr, FilePathStr],
+ name: Optional[StrPath] = None,
+ checksum: bool = True,
+ max_objects: Optional[int] = None,
+ ) -> Sequence[ArtifactManifestEntry]:
+ """Store the file or directory at the given path within the specified artifact.
+
+ Arguments:
+ artifact: The artifact doing the storing
+ path (str): The path to store
+ name (str): If specified, the logical name that should map to `path`
+
+ Returns:
+ (list[ArtifactManifestEntry]): A list of manifest entries to store within the artifact
+ """
+ client_id = util.host_from_path(path)
+ target_path = util.uri_from_path(path)
+ target_artifact = artifact_instance_cache.get(client_id)
+ if not isinstance(target_artifact, wandb.Artifact):
+ raise RuntimeError("Local Artifact not found - invalid reference")
+ target_entry = target_artifact._manifest.entries[target_path] # type: ignore
+ if target_entry is None:
+ raise RuntimeError("Local entry not found - invalid reference")
+
+ # Return the new entry
+ return [
+ ArtifactManifestEntry(
+ path=name or os.path.basename(path),
+ ref=path,
+ size=0,
+ digest=target_entry.digest,
+ )
+ ]
diff --git a/wandb/sdk/artifacts/storage_layout.py b/wandb/sdk/artifacts/storage_layout.py
new file mode 100644
index 0000000000000000000000000000000000000000..b07cbd8e50fdc507e6f2787611514f09f653b851
--- /dev/null
+++ b/wandb/sdk/artifacts/storage_layout.py
@@ -0,0 +1,6 @@
+"""Storage layout."""
+
+
+class StorageLayout:
+ V1 = "V1"
+ V2 = "V2"
diff --git a/wandb/sdk/artifacts/storage_policies/__init__.py b/wandb/sdk/artifacts/storage_policies/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4d409bcf913dfde9b84f6cc53d3c980a2cbe02e2
--- /dev/null
+++ b/wandb/sdk/artifacts/storage_policies/__init__.py
@@ -0,0 +1,4 @@
+from wandb.sdk.artifacts.storage_policies.register import WANDB_STORAGE_POLICY
+from wandb.sdk.artifacts.storage_policies.wandb_storage_policy import WandbStoragePolicy
+
+__all__ = ["WANDB_STORAGE_POLICY", "WandbStoragePolicy"]
diff --git a/wandb/sdk/artifacts/storage_policies/register.py b/wandb/sdk/artifacts/storage_policies/register.py
new file mode 100644
index 0000000000000000000000000000000000000000..84542d8a70fca03538985fa8aa9c6ef7ea86c2b5
--- /dev/null
+++ b/wandb/sdk/artifacts/storage_policies/register.py
@@ -0,0 +1 @@
+WANDB_STORAGE_POLICY = "wandb-storage-policy-v1"
diff --git a/wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py b/wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py
new file mode 100644
index 0000000000000000000000000000000000000000..5003f7593a4340f8bae43c18b015a038e6f384a8
--- /dev/null
+++ b/wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py
@@ -0,0 +1,397 @@
+"""WandB storage policy."""
+import hashlib
+import math
+import shutil
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union
+from urllib.parse import quote
+
+import requests
+import urllib3
+
+from wandb.apis import InternalApi
+from wandb.errors.term import termwarn
+from wandb.sdk.artifacts.artifact_file_cache import (
+ ArtifactFileCache,
+ get_artifact_file_cache,
+)
+from wandb.sdk.artifacts.storage_handlers.azure_handler import AzureHandler
+from wandb.sdk.artifacts.storage_handlers.gcs_handler import GCSHandler
+from wandb.sdk.artifacts.storage_handlers.http_handler import HTTPHandler
+from wandb.sdk.artifacts.storage_handlers.local_file_handler import LocalFileHandler
+from wandb.sdk.artifacts.storage_handlers.multi_handler import MultiHandler
+from wandb.sdk.artifacts.storage_handlers.s3_handler import S3Handler
+from wandb.sdk.artifacts.storage_handlers.tracking_handler import TrackingHandler
+from wandb.sdk.artifacts.storage_handlers.wb_artifact_handler import WBArtifactHandler
+from wandb.sdk.artifacts.storage_handlers.wb_local_artifact_handler import (
+ WBLocalArtifactHandler,
+)
+from wandb.sdk.artifacts.storage_layout import StorageLayout
+from wandb.sdk.artifacts.storage_policies.register import WANDB_STORAGE_POLICY
+from wandb.sdk.artifacts.storage_policy import StoragePolicy
+from wandb.sdk.internal.thread_local_settings import _thread_local_api_settings
+from wandb.sdk.lib.hashutil import B64MD5, b64_to_hex_id, hex_to_b64_id
+from wandb.sdk.lib.paths import FilePathStr, URIStr
+
+if TYPE_CHECKING:
+ from wandb.filesync.step_prepare import StepPrepare
+ from wandb.sdk.artifacts.artifact import Artifact
+ from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+ from wandb.sdk.internal import progress
+
+# Sleep length: 0, 2, 4, 8, 16, 32, 64, 120, 120, 120, 120, 120, 120, 120, 120, 120
+# seconds, i.e. a total of 20min 6s.
+_REQUEST_RETRY_STRATEGY = urllib3.util.retry.Retry(
+ backoff_factor=1,
+ total=16,
+ status_forcelist=(308, 408, 409, 429, 500, 502, 503, 504),
+)
+_REQUEST_POOL_CONNECTIONS = 64
+_REQUEST_POOL_MAXSIZE = 64
+
+# AWS S3 max upload parts without having to make additional requests for extra parts
+S3_MAX_PART_NUMBERS = 1000
+S3_MIN_MULTI_UPLOAD_SIZE = 2 * 1024**3
+S3_MAX_MULTI_UPLOAD_SIZE = 5 * 1024**4
+
+
+class WandbStoragePolicy(StoragePolicy):
+ @classmethod
+ def name(cls) -> str:
+ return WANDB_STORAGE_POLICY
+
+ @classmethod
+ def from_config(cls, config: Dict) -> "WandbStoragePolicy":
+ return cls(config=config)
+
+ def __init__(
+ self,
+ config: Optional[Dict] = None,
+ cache: Optional[ArtifactFileCache] = None,
+ api: Optional[InternalApi] = None,
+ ) -> None:
+ self._cache = cache or get_artifact_file_cache()
+ self._config = config or {}
+ self._session = requests.Session()
+ adapter = requests.adapters.HTTPAdapter(
+ max_retries=_REQUEST_RETRY_STRATEGY,
+ pool_connections=_REQUEST_POOL_CONNECTIONS,
+ pool_maxsize=_REQUEST_POOL_MAXSIZE,
+ )
+ self._session.mount("http://", adapter)
+ self._session.mount("https://", adapter)
+
+ s3 = S3Handler()
+ gcs = GCSHandler()
+ azure = AzureHandler()
+ http = HTTPHandler(self._session)
+ https = HTTPHandler(self._session, scheme="https")
+ artifact = WBArtifactHandler()
+ local_artifact = WBLocalArtifactHandler()
+ file_handler = LocalFileHandler()
+
+ self._api = api or InternalApi()
+ self._handler = MultiHandler(
+ handlers=[
+ s3,
+ gcs,
+ azure,
+ http,
+ https,
+ artifact,
+ local_artifact,
+ file_handler,
+ ],
+ default_handler=TrackingHandler(),
+ )
+
+ def config(self) -> Dict:
+ return self._config
+
+ def load_file(
+ self,
+ artifact: "Artifact",
+ manifest_entry: "ArtifactManifestEntry",
+ dest_path: Optional[str] = None,
+ ) -> FilePathStr:
+ self._cache._override_cache_path = dest_path
+ path, hit, cache_open = self._cache.check_md5_obj_path(
+ B64MD5(manifest_entry.digest), # TODO(spencerpearson): unsafe cast
+ manifest_entry.size if manifest_entry.size is not None else 0,
+ )
+ if hit:
+ return path
+
+ if manifest_entry._download_url is not None:
+ response = self._session.get(manifest_entry._download_url, stream=True)
+ try:
+ response.raise_for_status()
+ except Exception:
+ # Signed URL might have expired, fall back to fetching it one by one.
+ manifest_entry._download_url = None
+ if manifest_entry._download_url is None:
+ auth = None
+ if not _thread_local_api_settings.cookies:
+ auth = ("api", self._api.api_key)
+ response = self._session.get(
+ self._file_url(self._api, artifact.entity, manifest_entry),
+ auth=auth,
+ cookies=_thread_local_api_settings.cookies,
+ headers=_thread_local_api_settings.headers,
+ stream=True,
+ )
+ response.raise_for_status()
+
+ with cache_open(mode="wb") as file:
+ for data in response.iter_content(chunk_size=16 * 1024):
+ file.write(data)
+ return path
+
+ def store_reference(
+ self,
+ artifact: "Artifact",
+ path: Union[URIStr, FilePathStr],
+ name: Optional[str] = None,
+ checksum: bool = True,
+ max_objects: Optional[int] = None,
+ ) -> Sequence["ArtifactManifestEntry"]:
+ return self._handler.store_path(
+ artifact, path, name=name, checksum=checksum, max_objects=max_objects
+ )
+
+ def load_reference(
+ self,
+ manifest_entry: "ArtifactManifestEntry",
+ local: bool = False,
+ dest_path: Optional[str] = None,
+ ) -> Union[FilePathStr, URIStr]:
+ assert manifest_entry.ref is not None
+ used_handler = self._handler._get_handler(manifest_entry.ref)
+ if hasattr(used_handler, "_cache"):
+ used_handler._cache._override_cache_path = dest_path
+ return self._handler.load_path(manifest_entry, local)
+
+ def _file_url(
+ self,
+ api: InternalApi,
+ entity_name: str,
+ manifest_entry: "ArtifactManifestEntry",
+ ) -> str:
+ storage_layout = self._config.get("storageLayout", StorageLayout.V1)
+ storage_region = self._config.get("storageRegion", "default")
+ md5_hex = b64_to_hex_id(B64MD5(manifest_entry.digest))
+
+ if storage_layout == StorageLayout.V1:
+ return "{}/artifacts/{}/{}".format(
+ api.settings("base_url"), entity_name, md5_hex
+ )
+ elif storage_layout == StorageLayout.V2:
+ return "{}/artifactsV2/{}/{}/{}/{}".format(
+ api.settings("base_url"),
+ storage_region,
+ entity_name,
+ quote(
+ manifest_entry.birth_artifact_id
+ if manifest_entry.birth_artifact_id is not None
+ else ""
+ ),
+ md5_hex,
+ )
+ else:
+ raise Exception(f"unrecognized storage layout: {storage_layout}")
+
+ def s3_multipart_file_upload(
+ self,
+ file_path: str,
+ chunk_size: int,
+ hex_digests: Dict[int, str],
+ multipart_urls: Dict[int, str],
+ extra_headers: Dict[str, str],
+ ) -> List[Dict[str, Any]]:
+ etags = []
+ part_number = 1
+
+ with open(file_path, "rb") as f:
+ while True:
+ data = f.read(chunk_size)
+ if not data:
+ break
+ md5_b64_str = str(hex_to_b64_id(hex_digests[part_number]))
+ upload_resp = self._api.upload_multipart_file_chunk_retry(
+ multipart_urls[part_number],
+ data,
+ extra_headers={
+ "content-md5": md5_b64_str,
+ "content-length": str(len(data)),
+ "content-type": extra_headers.get("Content-Type"),
+ },
+ )
+ etags.append(
+ {"partNumber": part_number, "hexMD5": upload_resp.headers["ETag"]}
+ )
+ part_number += 1
+ return etags
+
+ def default_file_upload(
+ self,
+ upload_url: str,
+ file_path: str,
+ extra_headers: Dict[str, Any],
+ progress_callback: Optional["progress.ProgressFn"] = None,
+ ) -> None:
+ """Upload a file to the artifact store and write to cache."""
+ with open(file_path, "rb") as file:
+ # This fails if we don't send the first byte before the signed URL expires.
+ self._api.upload_file_retry(
+ upload_url,
+ file,
+ progress_callback,
+ extra_headers=extra_headers,
+ )
+
+ def calc_chunk_size(self, file_size: int) -> int:
+ # Default to chunk size of 100MiB. S3 has cap of 10,000 upload parts.
+ # If file size exceeds the default chunk size, recalculate chunk size.
+ default_chunk_size = 100 * 1024**2
+ if default_chunk_size * S3_MAX_PART_NUMBERS < file_size:
+ return math.ceil(file_size / S3_MAX_PART_NUMBERS)
+ return default_chunk_size
+
+ def store_file_sync(
+ self,
+ artifact_id: str,
+ artifact_manifest_id: str,
+ entry: "ArtifactManifestEntry",
+ preparer: "StepPrepare",
+ progress_callback: Optional["progress.ProgressFn"] = None,
+ ) -> bool:
+ """Upload a file to the artifact store.
+
+ Returns:
+ True if the file was a duplicate (did not need to be uploaded),
+ False if it needed to be uploaded or was a reference (nothing to dedupe).
+ """
+ file_size = entry.size if entry.size is not None else 0
+ chunk_size = self.calc_chunk_size(file_size)
+ upload_parts = []
+ hex_digests = {}
+ file_path = entry.local_path if entry.local_path is not None else ""
+ # Logic for AWS s3 multipart upload.
+ # Only chunk files if larger than 2 GiB. Currently can only support up to 5TiB.
+ if (
+ file_size >= S3_MIN_MULTI_UPLOAD_SIZE
+ and file_size <= S3_MAX_MULTI_UPLOAD_SIZE
+ ):
+ part_number = 1
+ with open(file_path, "rb") as f:
+ while True:
+ data = f.read(chunk_size)
+ if not data:
+ break
+ hex_digest = hashlib.md5(data).hexdigest()
+ upload_parts.append(
+ {"hexMD5": hex_digest, "partNumber": part_number}
+ )
+ hex_digests[part_number] = hex_digest
+ part_number += 1
+
+ resp = preparer.prepare_sync(
+ {
+ "artifactID": artifact_id,
+ "artifactManifestID": artifact_manifest_id,
+ "name": entry.path,
+ "md5": entry.digest,
+ "uploadPartsInput": upload_parts,
+ }
+ ).get()
+
+ entry.birth_artifact_id = resp.birth_artifact_id
+
+ multipart_urls = resp.multipart_upload_urls
+ if resp.upload_url is None:
+ return True
+ if entry.local_path is None:
+ return False
+
+ extra_headers = {
+ header.split(":", 1)[0]: header.split(":", 1)[1]
+ for header in (resp.upload_headers or {})
+ }
+
+ # This multipart upload isn't available, do a regular single url upload
+ if multipart_urls is None and resp.upload_url:
+ self.default_file_upload(
+ resp.upload_url, file_path, extra_headers, progress_callback
+ )
+ else:
+ if multipart_urls is None:
+ raise ValueError(f"No multipart urls to upload for file: {file_path}")
+ # Upload files using s3 multipart upload urls
+ etags = self.s3_multipart_file_upload(
+ file_path,
+ chunk_size,
+ hex_digests,
+ multipart_urls,
+ extra_headers,
+ )
+ self._api.complete_multipart_upload_artifact(
+ artifact_id, resp.storage_path, etags, resp.upload_id
+ )
+ self._write_cache(entry)
+
+ return False
+
+ async def store_file_async(
+ self,
+ artifact_id: str,
+ artifact_manifest_id: str,
+ entry: "ArtifactManifestEntry",
+ preparer: "StepPrepare",
+ progress_callback: Optional["progress.ProgressFn"] = None,
+ ) -> bool:
+ """Async equivalent to `store_file_sync`."""
+ resp = await preparer.prepare_async(
+ {
+ "artifactID": artifact_id,
+ "artifactManifestID": artifact_manifest_id,
+ "name": entry.path,
+ "md5": entry.digest,
+ }
+ )
+
+ entry.birth_artifact_id = resp.birth_artifact_id
+ if resp.upload_url is None:
+ return True
+ if entry.local_path is None:
+ return False
+
+ with open(entry.local_path, "rb") as file:
+ # This fails if we don't send the first byte before the signed URL expires.
+ await self._api.upload_file_retry_async(
+ resp.upload_url,
+ file,
+ progress_callback,
+ extra_headers={
+ header.split(":", 1)[0]: header.split(":", 1)[1]
+ for header in (resp.upload_headers or {})
+ },
+ )
+
+ self._write_cache(entry)
+
+ return False
+
+ def _write_cache(self, entry: "ArtifactManifestEntry") -> None:
+ if entry.local_path is None:
+ return
+
+ # Cache upon successful upload.
+ _, hit, cache_open = self._cache.check_md5_obj_path(
+ B64MD5(entry.digest),
+ entry.size if entry.size is not None else 0,
+ )
+ if not hit:
+ try:
+ with cache_open("wb") as f, open(entry.local_path, "rb") as src:
+ shutil.copyfileobj(src, f)
+ except OSError as e:
+ termwarn(f"Failed to cache {entry.local_path}, ignoring {e}")
diff --git a/wandb/sdk/artifacts/storage_policy.py b/wandb/sdk/artifacts/storage_policy.py
new file mode 100644
index 0000000000000000000000000000000000000000..fe9cadb35be7c7cca5abdbae4a953ba2399c0905
--- /dev/null
+++ b/wandb/sdk/artifacts/storage_policy.py
@@ -0,0 +1,79 @@
+"""Storage policy."""
+from typing import TYPE_CHECKING, Dict, Optional, Sequence, Type, Union
+
+from wandb.sdk.lib.paths import FilePathStr, URIStr
+
+if TYPE_CHECKING:
+ from wandb.filesync.step_prepare import StepPrepare
+ from wandb.sdk.artifacts.artifact import Artifact
+ from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
+ from wandb.sdk.internal.progress import ProgressFn
+
+
+class StoragePolicy:
+ @classmethod
+ def lookup_by_name(cls, name: str) -> Type["StoragePolicy"]:
+ import wandb.sdk.artifacts.storage_policies # noqa: F401
+
+ for sub in cls.__subclasses__():
+ if sub.name() == name:
+ return sub
+ raise NotImplementedError(f"Failed to find storage policy '{name}'")
+
+ @classmethod
+ def name(cls) -> str:
+ raise NotImplementedError
+
+ @classmethod
+ def from_config(cls, config: Dict) -> "StoragePolicy":
+ raise NotImplementedError
+
+ def config(self) -> Dict:
+ raise NotImplementedError
+
+ def load_file(
+ self,
+ artifact: "Artifact",
+ manifest_entry: "ArtifactManifestEntry",
+ dest_path: Optional[str] = None,
+ ) -> FilePathStr:
+ raise NotImplementedError
+
+ def store_file_sync(
+ self,
+ artifact_id: str,
+ artifact_manifest_id: str,
+ entry: "ArtifactManifestEntry",
+ preparer: "StepPrepare",
+ progress_callback: Optional["ProgressFn"] = None,
+ ) -> bool:
+ raise NotImplementedError
+
+ async def store_file_async(
+ self,
+ artifact_id: str,
+ artifact_manifest_id: str,
+ entry: "ArtifactManifestEntry",
+ preparer: "StepPrepare",
+ progress_callback: Optional["ProgressFn"] = None,
+ ) -> bool:
+ """Async equivalent to `store_file_sync`."""
+ raise NotImplementedError
+
+ def store_reference(
+ self,
+ artifact: "Artifact",
+ path: Union[URIStr, FilePathStr],
+ name: Optional[str] = None,
+ checksum: bool = True,
+ max_objects: Optional[int] = None,
+ ) -> Sequence["ArtifactManifestEntry"]:
+ raise NotImplementedError
+
+ def load_reference(
+ self,
+ manifest_entry: "ArtifactManifestEntry",
+ local: bool = False,
+ dest_path: Optional[str] = None,
+ ) -> Union[FilePathStr, URIStr]:
+ raise NotImplementedError
diff --git a/wandb/sdk/backend/__init__.py b/wandb/sdk/backend/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sdk/backend/backend.py b/wandb/sdk/backend/backend.py
new file mode 100644
index 0000000000000000000000000000000000000000..4082736a1cc77692e882b6cbf94b4fa34983aeb8
--- /dev/null
+++ b/wandb/sdk/backend/backend.py
@@ -0,0 +1,240 @@
+"""Backend - Send to internal process.
+
+Manage backend.
+
+"""
+
+import importlib.machinery
+import logging
+import multiprocessing
+import os
+import queue
+import sys
+import threading
+from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Union, cast
+
+import wandb
+from wandb.sdk.interface.interface import InterfaceBase
+from wandb.sdk.interface.interface_queue import InterfaceQueue
+from wandb.sdk.internal.internal import wandb_internal
+from wandb.sdk.internal.settings_static import SettingsStatic
+from wandb.sdk.lib.mailbox import Mailbox
+from wandb.sdk.wandb_manager import _Manager
+from wandb.sdk.wandb_settings import Settings
+
+if TYPE_CHECKING:
+ from wandb.proto.wandb_internal_pb2 import Record, Result
+
+ from ..service.service_sock import ServiceSockInterface
+ from ..wandb_run import Run
+
+ RecordQueue = Union[queue.Queue[Record], multiprocessing.Queue[Record]]
+ ResultQueue = Union[queue.Queue[Result], multiprocessing.Queue[Result]]
+
+logger = logging.getLogger("wandb")
+
+
+class BackendThread(threading.Thread):
+ """Class to running internal process as a thread."""
+
+ def __init__(self, target: Callable, kwargs: Dict[str, Any]) -> None:
+ threading.Thread.__init__(self)
+ self.name = "BackendThr"
+ self._target = target
+ self._kwargs = kwargs
+ self.daemon = True
+ self.pid = 0
+
+ def run(self) -> None:
+ self._target(**self._kwargs)
+
+
+class Backend:
+ # multiprocessing context or module
+ _multiprocessing: multiprocessing.context.BaseContext
+ interface: Optional[InterfaceBase]
+ _internal_pid: Optional[int]
+ wandb_process: Optional[multiprocessing.process.BaseProcess]
+ _settings: Optional[Settings]
+ record_q: Optional["RecordQueue"]
+ result_q: Optional["ResultQueue"]
+ _mailbox: Mailbox
+
+ def __init__(
+ self,
+ mailbox: Mailbox,
+ settings: Optional[Settings] = None,
+ log_level: Optional[int] = None,
+ manager: Optional[_Manager] = None,
+ ) -> None:
+ self._done = False
+ self.record_q = None
+ self.result_q = None
+ self.wandb_process = None
+ self.interface = None
+ self._internal_pid = None
+ self._settings = settings
+ self._log_level = log_level
+ self._manager = manager
+ self._mailbox = mailbox
+
+ self._multiprocessing = multiprocessing # type: ignore
+ self._multiprocessing_setup()
+
+ # for _module_main_* methods
+ self._save_mod_path: Optional[str] = None
+ self._save_mod_spec = None
+
+ def _hack_set_run(self, run: "Run") -> None:
+ assert self.interface
+ self.interface._hack_set_run(run)
+
+ def _multiprocessing_setup(self) -> None:
+ assert self._settings
+ if self._settings.start_method == "thread":
+ return
+
+ # defaulting to spawn for now, fork needs more testing
+ start_method = self._settings.start_method or "spawn"
+
+ # TODO: use fork context if unix and frozen?
+ # if py34+, else fall back
+ if not hasattr(multiprocessing, "get_context"):
+ return
+ all_methods = multiprocessing.get_all_start_methods()
+ logger.info(
+ "multiprocessing start_methods={}, using: {}".format(
+ ",".join(all_methods), start_method
+ )
+ )
+ ctx = multiprocessing.get_context(start_method)
+ self._multiprocessing = ctx
+
+ def _module_main_install(self) -> None:
+ # Support running code without a: __name__ == "__main__"
+ main_module = sys.modules["__main__"]
+ main_mod_spec = getattr(main_module, "__spec__", None)
+ main_mod_path = getattr(main_module, "__file__", None)
+ if main_mod_spec is None: # hack for pdb
+ # Note: typing has trouble with BuiltinImporter
+ loader: Loader = importlib.machinery.BuiltinImporter # type: ignore # noqa: F821
+ main_mod_spec = importlib.machinery.ModuleSpec(
+ name="wandb.mpmain", loader=loader
+ )
+ main_module.__spec__ = main_mod_spec
+ else:
+ self._save_mod_spec = main_mod_spec
+
+ if main_mod_path is not None:
+ self._save_mod_path = main_module.__file__
+ fname = os.path.join(
+ os.path.dirname(wandb.__file__), "mpmain", "__main__.py"
+ )
+ main_module.__file__ = fname
+
+ def _module_main_uninstall(self) -> None:
+ main_module = sys.modules["__main__"]
+ # Undo temporary changes from: __name__ == "__main__"
+ main_module.__spec__ = self._save_mod_spec
+ if self._save_mod_path:
+ main_module.__file__ = self._save_mod_path
+
+ def _ensure_launched_manager(self) -> None:
+ assert self._manager
+ svc = self._manager._get_service()
+ assert svc
+ svc_iface = svc.service_interface
+
+ svc_transport = svc_iface.get_transport()
+ if svc_transport == "tcp":
+ from ..interface.interface_sock import InterfaceSock
+
+ svc_iface_sock = cast("ServiceSockInterface", svc_iface)
+ sock_client = svc_iface_sock._get_sock_client()
+ sock_interface = InterfaceSock(sock_client, mailbox=self._mailbox)
+ self.interface = sock_interface
+ else:
+ raise AssertionError(f"Unsupported service transport: {svc_transport}")
+
+ def ensure_launched(self) -> None:
+ """Launch backend worker if not running."""
+ if self._manager:
+ self._ensure_launched_manager()
+ return
+
+ assert self._settings
+ settings = self._settings.copy()
+ settings.update(_log_level=self._log_level or logging.DEBUG)
+
+ start_method = settings.start_method
+
+ settings_static = SettingsStatic(settings.to_proto())
+ user_pid = os.getpid()
+
+ if start_method == "thread":
+ self.record_q = queue.Queue()
+ self.result_q = queue.Queue()
+ wandb._set_internal_process(disable=True) # type: ignore
+ wandb_thread = BackendThread(
+ target=wandb_internal,
+ kwargs=dict(
+ settings=settings_static,
+ record_q=self.record_q,
+ result_q=self.result_q,
+ user_pid=user_pid,
+ ),
+ )
+ # TODO: risky cast, assumes BackendThread Process duck typing
+ self.wandb_process = wandb_thread # type: ignore
+ else:
+ self.record_q = self._multiprocessing.Queue()
+ self.result_q = self._multiprocessing.Queue()
+ self.wandb_process = self._multiprocessing.Process( # type: ignore
+ target=wandb_internal,
+ kwargs=dict(
+ settings=settings_static,
+ record_q=self.record_q,
+ result_q=self.result_q,
+ user_pid=user_pid,
+ ),
+ )
+ assert self.wandb_process
+ self.wandb_process.name = "wandb_internal"
+
+ self._module_main_install()
+
+ logger.info("starting backend process...")
+ # Start the process with __name__ == "__main__" workarounds
+ assert self.wandb_process
+ self.wandb_process.start()
+ self._internal_pid = self.wandb_process.pid
+ logger.info(f"started backend process with pid: {self.wandb_process.pid}")
+
+ self._module_main_uninstall()
+
+ self.interface = InterfaceQueue(
+ process=self.wandb_process,
+ record_q=self.record_q, # type: ignore
+ result_q=self.result_q, # type: ignore
+ mailbox=self._mailbox,
+ )
+
+ def server_status(self) -> None:
+ """Report server status."""
+ pass
+
+ def cleanup(self) -> None:
+ # TODO: make _done atomic
+ if self._done:
+ return
+ self._done = True
+ if self.interface:
+ self.interface.join()
+ if self.wandb_process:
+ self.wandb_process.join()
+
+ if self.record_q and hasattr(self.record_q, "close"):
+ self.record_q.close()
+ if self.result_q and hasattr(self.result_q, "close"):
+ self.result_q.close()
+ # No printing allowed from here until redirect restore!!!
diff --git a/wandb/sdk/data_types/__init__.py b/wandb/sdk/data_types/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sdk/data_types/_dtypes.py b/wandb/sdk/data_types/_dtypes.py
new file mode 100644
index 0000000000000000000000000000000000000000..5d8483f00577474e1866b09e42f7d9fb0fb92fbc
--- /dev/null
+++ b/wandb/sdk/data_types/_dtypes.py
@@ -0,0 +1,911 @@
+import datetime
+import math
+import typing as t
+
+from wandb.util import (
+ _is_artifact_string,
+ _is_artifact_version_weave_dict,
+ get_module,
+ is_numpy_array,
+)
+
+np = get_module("numpy") # intentionally not required
+
+if t.TYPE_CHECKING:
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ConvertableToType = t.Union["Type", t.Type["Type"], type, t.Any]
+
+
+class TypeRegistry:
+ """A resolver for python objects that can deserialize JSON dicts.
+
+ Additional types can be registered via the .add call.
+ """
+
+ _types_by_name = None
+ _types_by_class = None
+
+ @staticmethod
+ def types_by_name():
+ if TypeRegistry._types_by_name is None:
+ TypeRegistry._types_by_name = {}
+ return TypeRegistry._types_by_name
+
+ @staticmethod
+ def types_by_class():
+ if TypeRegistry._types_by_class is None:
+ TypeRegistry._types_by_class = {}
+ return TypeRegistry._types_by_class
+
+ @staticmethod
+ def add(wb_type: t.Type["Type"]) -> None:
+ assert issubclass(wb_type, Type)
+ TypeRegistry.types_by_name().update({wb_type.name: wb_type})
+ for name in wb_type.legacy_names:
+ TypeRegistry.types_by_name().update({name: wb_type})
+ TypeRegistry.types_by_class().update(
+ {_type: wb_type for _type in wb_type.types}
+ )
+
+ @staticmethod
+ def type_of(py_obj: t.Optional[t.Any]) -> "Type":
+ # Special case handler for common case of np.nans. np.nan
+ # is of type 'float', but should be treated as a None. This is
+ # because np.nan can co-exist with other types in dataframes,
+ # but will be ultimately treated as a None. Ignoring type since
+ # mypy does not trust that py_obj is a float by the time it is
+ # passed to isnan.
+ if py_obj.__class__ == float and math.isnan(py_obj): # type: ignore
+ return NoneType()
+
+ # TODO: generalize this to handle other config input types
+ if _is_artifact_string(py_obj) or _is_artifact_version_weave_dict(py_obj):
+ return TypeRegistry.types_by_name().get("artifactVersion")()
+
+ class_handler = TypeRegistry.types_by_class().get(py_obj.__class__)
+ _type = None
+ if class_handler:
+ _type = class_handler.from_obj(py_obj)
+ else:
+ _type = PythonObjectType.from_obj(py_obj)
+ return _type
+
+ @staticmethod
+ def type_from_dict(
+ json_dict: t.Dict[str, t.Any], artifact: t.Optional["Artifact"] = None
+ ) -> "Type":
+ wb_type = json_dict.get("wb_type")
+ if wb_type is None:
+ TypeError("json_dict must contain `wb_type` key")
+ _type = TypeRegistry.types_by_name().get(wb_type)
+ if _type is None:
+ TypeError(f"missing type handler for {wb_type}")
+ return _type.from_json(json_dict, artifact)
+
+ @staticmethod
+ def type_from_dtype(dtype: ConvertableToType) -> "Type":
+ # The dtype is already an instance of Type
+ if isinstance(dtype, Type):
+ wbtype: Type = dtype
+
+ # The dtype is a subclass of Type
+ elif isinstance(dtype, type) and issubclass(dtype, Type):
+ wbtype = dtype()
+
+ # The dtype is a subclass of generic python type
+ elif isinstance(dtype, type):
+ handler = TypeRegistry.types_by_class().get(dtype)
+
+ # and we have a registered handler
+ if handler:
+ wbtype = handler()
+
+ # else, fallback to object type
+ else:
+ wbtype = PythonObjectType.from_obj(dtype)
+
+ # The dtype is a list, then we resolve the list notation
+ elif isinstance(dtype, list):
+ if len(dtype) == 0:
+ wbtype = ListType()
+ elif len(dtype) == 1:
+ wbtype = ListType(TypeRegistry.type_from_dtype(dtype[0]))
+
+ # lists of more than 1 are treated as unions
+ else:
+ wbtype = UnionType([TypeRegistry.type_from_dtype(dt) for dt in dtype])
+
+ # The dtype is a dict, then we resolve the dict notation
+ elif isinstance(dtype, dict):
+ wbtype = TypedDictType(
+ {key: TypeRegistry.type_from_dtype(dtype[key]) for key in dtype}
+ )
+
+ # The dtype is a concrete instance, which we will treat as a constant
+ else:
+ wbtype = ConstType(dtype)
+
+ return wbtype
+
+
+def _params_obj_to_json_obj(
+ params_obj: t.Any,
+ artifact: t.Optional["Artifact"] = None,
+) -> t.Any:
+ """Helper method."""
+ if params_obj.__class__ == dict:
+ return {
+ key: _params_obj_to_json_obj(params_obj[key], artifact)
+ for key in params_obj
+ }
+ elif params_obj.__class__ in [list, set, tuple, frozenset]:
+ return [_params_obj_to_json_obj(item, artifact) for item in list(params_obj)]
+ elif isinstance(params_obj, Type):
+ return params_obj.to_json(artifact)
+ else:
+ return params_obj
+
+
+def _json_obj_to_params_obj(
+ json_obj: t.Any, artifact: t.Optional["Artifact"] = None
+) -> t.Any:
+ """Helper method."""
+ if json_obj.__class__ == dict:
+ if "wb_type" in json_obj:
+ return TypeRegistry.type_from_dict(json_obj, artifact)
+ else:
+ return {
+ key: _json_obj_to_params_obj(json_obj[key], artifact)
+ for key in json_obj
+ }
+ elif json_obj.__class__ == list:
+ return [_json_obj_to_params_obj(item, artifact) for item in json_obj]
+ else:
+ return json_obj
+
+
+class Type:
+ """The most generic type that all types subclass.
+
+ It provides simple serialization and deserialization as well as equality checks.
+ A name class-level property must be uniquely set by subclasses.
+ """
+
+ # Subclasses must override with a unique name. This is used to identify the
+ # class during serializations and deserializations
+ name: t.ClassVar[str] = ""
+
+ # List of names by which this class can deserialize
+ legacy_names: t.ClassVar[t.List[str]] = []
+
+ # Subclasses may override with a list of `types` which this Type is capable
+ # of being initialized. This is used by the Type Registry when calling `TypeRegistry.type_of`.
+ # Some types will have an empty list - for example `Union`. There is no raw python type which
+ # inherently maps to a Union and therefore the list should be empty.
+ types: t.ClassVar[t.List[type]] = []
+
+ # Contains the further specification of the Type
+ _params: t.Dict[str, t.Any]
+
+ def __init__(*args, **kwargs):
+ pass
+
+ @property
+ def params(self):
+ if not hasattr(self, "_params") or self._params is None:
+ self._params = {}
+ return self._params
+
+ def assign(self, py_obj: t.Optional[t.Any] = None) -> "Type":
+ """Assign a python object to the type.
+
+ May to be overridden by subclasses
+
+ Args:
+ py_obj (any, optional): Any python object which the user wishes to assign to
+ this type
+
+ Returns:
+ Type: a new type representing the result of the assignment.
+ """
+ return self.assign_type(TypeRegistry.type_of(py_obj))
+
+ def assign_type(self, wb_type: "Type") -> "Type":
+ # Default - should be overridden
+ if isinstance(wb_type, self.__class__) and self.params == wb_type.params:
+ return self
+ else:
+ return InvalidType()
+
+ def to_json(self, artifact: t.Optional["Artifact"] = None) -> t.Dict[str, t.Any]:
+ """Generate a jsonable dictionary serialization the type.
+
+ If overridden by subclass, ensure that `from_json` is equivalently overridden.
+
+ Args:
+ artifact (wandb.Artifact, optional): If the serialization is being performed
+ for a particular artifact, pass that artifact. Defaults to None.
+
+ Returns:
+ dict: Representation of the type
+ """
+ res = {
+ "wb_type": self.name,
+ "params": _params_obj_to_json_obj(self.params, artifact),
+ }
+ if res["params"] is None or res["params"] == {}:
+ del res["params"]
+
+ return res
+
+ @classmethod
+ def from_json(
+ cls,
+ json_dict: t.Dict[str, t.Any],
+ artifact: t.Optional["Artifact"] = None,
+ ) -> "Type":
+ """Construct a new instance of the type using a JSON dictionary.
+
+ The mirror function of `to_json`. If overridden by subclass, ensure that
+ `to_json` is equivalently overridden.
+
+ Returns:
+ _Type: an instance of a subclass of the _Type class.
+ """
+ return cls(**_json_obj_to_params_obj(json_dict.get("params", {}), artifact))
+
+ @classmethod
+ def from_obj(cls, py_obj: t.Optional[t.Any] = None) -> "Type":
+ return cls()
+
+ def explain(self, other: t.Any, depth=0) -> str:
+ """Explain why an item is not assignable to a type.
+
+ Assumes that the caller has already validated that the assignment fails.
+
+ Args:
+ other (any): Any object depth (int, optional): depth of the type checking.
+ Defaults to 0.
+
+ Returns:
+ str: human-readable explanation
+ """
+ wbtype = TypeRegistry.type_of(other)
+ gap = "".join(["\t"] * depth)
+ if depth > 0:
+ return f"{gap}{wbtype} not assignable to {self}"
+ else:
+ return f"{gap}{other} of type {wbtype} is not assignable to {self}"
+
+ def __repr__(self):
+ rep = self.name.capitalize()
+ if len(self.params.keys()) > 0:
+ rep += "("
+ for ndx, key in enumerate(self.params.keys()):
+ if ndx > 0:
+ rep += ", "
+ rep += key + ":" + str(self.params[key])
+ rep += ")"
+ return rep
+
+ def __eq__(self, other):
+ return self is other or (
+ isinstance(self, Type)
+ and isinstance(other, Type)
+ and self.name == other.name
+ and self.params.keys() == other.params.keys()
+ and all([self.params[k] == other.params[k] for k in self.params])
+ )
+
+
+class InvalidType(Type):
+ """A disallowed type.
+
+ Assignments to a InvalidType result in a Never Type. InvalidType is basically the
+ invalid case.
+ """
+
+ name = "invalid"
+ types: t.ClassVar[t.List[type]] = []
+
+ def assign_type(self, wb_type: "Type") -> "InvalidType":
+ return self
+
+
+class AnyType(Type):
+ """An object that can be any type.
+
+ Assignments to an AnyType result in the AnyType except None which results in an
+ InvalidType.
+ """
+
+ name = "any"
+ types: t.ClassVar[t.List[type]] = []
+
+ def assign_type(self, wb_type: "Type") -> t.Union["AnyType", InvalidType]:
+ return (
+ self
+ if not (isinstance(wb_type, NoneType) or isinstance(wb_type, InvalidType))
+ else InvalidType()
+ )
+
+
+class UnknownType(Type):
+ """An object with an unknown type.
+
+ All assignments to an UnknownType result in the type of the assigned object except
+ `None` which results in a InvalidType.
+ """
+
+ name = "unknown"
+ types: t.ClassVar[t.List[type]] = []
+
+ def assign_type(self, wb_type: "Type") -> "Type":
+ return wb_type if not isinstance(wb_type, NoneType) else InvalidType()
+
+
+class NoneType(Type):
+ name = "none"
+ types: t.ClassVar[t.List[type]] = [None.__class__]
+
+
+class StringType(Type):
+ name = "string"
+ types: t.ClassVar[t.List[type]] = [str]
+
+
+class NumberType(Type):
+ name = "number"
+ types: t.ClassVar[t.List[type]] = [int, float]
+
+
+if np:
+ NumberType.types.append(np.byte)
+ NumberType.types.append(np.short)
+ NumberType.types.append(np.ushort)
+ NumberType.types.append(np.intc)
+ NumberType.types.append(np.uintc)
+ NumberType.types.append(np.int_)
+ NumberType.types.append(np.uint)
+ NumberType.types.append(np.longlong)
+ NumberType.types.append(np.ulonglong)
+ NumberType.types.append(np.half)
+ NumberType.types.append(np.float16)
+ NumberType.types.append(np.single)
+ NumberType.types.append(np.double)
+ NumberType.types.append(np.longdouble)
+ NumberType.types.append(np.csingle)
+ NumberType.types.append(np.cdouble)
+ NumberType.types.append(np.clongdouble)
+ NumberType.types.append(np.int8)
+ NumberType.types.append(np.int16)
+ NumberType.types.append(np.int32)
+ NumberType.types.append(np.int64)
+ NumberType.types.append(np.uint8)
+ NumberType.types.append(np.uint16)
+ NumberType.types.append(np.uint32)
+ NumberType.types.append(np.uint64)
+ NumberType.types.append(np.intp)
+ NumberType.types.append(np.uintp)
+ NumberType.types.append(np.float32)
+ NumberType.types.append(np.float64)
+ NumberType.types.append(np.float_)
+ NumberType.types.append(np.complex64)
+ NumberType.types.append(np.complex128)
+ NumberType.types.append(np.complex_)
+
+
+class TimestampType(Type):
+ name = "timestamp"
+ types: t.ClassVar[t.List[type]] = [datetime.datetime, datetime.date]
+
+
+if np:
+ TimestampType.types.append(np.datetime64)
+
+
+class BooleanType(Type):
+ name = "boolean"
+ types: t.ClassVar[t.List[type]] = [bool]
+
+
+if np:
+ BooleanType.types.append(np.bool_)
+
+
+class PythonObjectType(Type):
+ """A backup type that keeps track of the python object name."""
+
+ name = "pythonObject"
+ legacy_names = ["object"]
+ types: t.ClassVar[t.List[type]] = []
+
+ def __init__(self, class_name: str):
+ self.params.update({"class_name": class_name})
+
+ @classmethod
+ def from_obj(cls, py_obj: t.Optional[t.Any] = None) -> "PythonObjectType":
+ return cls(py_obj.__class__.__name__)
+
+
+class ConstType(Type):
+ """A constant value (currently only primitives supported)."""
+
+ name = "const"
+ types: t.ClassVar[t.List[type]] = []
+
+ def __init__(self, val: t.Optional[t.Any] = None, is_set: t.Optional[bool] = False):
+ if val.__class__ not in [str, int, float, bool, set, list, None.__class__]:
+ TypeError(
+ f"ConstType only supports str, int, float, bool, set, list, and None types. Found {val}"
+ )
+ if is_set or isinstance(val, set):
+ is_set = True
+ assert isinstance(val, set) or isinstance(val, list)
+ val = set(val)
+
+ self.params.update({"val": val, "is_set": is_set})
+
+ def assign(self, py_obj: t.Optional[t.Any] = None) -> "Type":
+ return self.assign_type(ConstType(py_obj))
+
+ @classmethod
+ def from_obj(cls, py_obj: t.Optional[t.Any] = None) -> "ConstType":
+ return cls(py_obj)
+
+ def __repr__(self):
+ return str(self.params["val"])
+
+
+def _flatten_union_types(wb_types: t.List[Type]) -> t.List[Type]:
+ final_types = []
+ for allowed_type in wb_types:
+ if isinstance(allowed_type, UnionType):
+ internal_types = _flatten_union_types(allowed_type.params["allowed_types"])
+ for internal_type in internal_types:
+ final_types.append(internal_type)
+ else:
+ final_types.append(allowed_type)
+ return final_types
+
+
+def _union_assigner(
+ allowed_types: t.List[Type],
+ obj_or_type: t.Union[Type, t.Optional[t.Any]],
+ type_mode=False,
+) -> t.Union[t.List[Type], InvalidType]:
+ resolved_types = []
+ valid = False
+ unknown_count = 0
+
+ for allowed_type in allowed_types:
+ if valid:
+ resolved_types.append(allowed_type)
+ else:
+ if isinstance(allowed_type, UnknownType):
+ unknown_count += 1
+ else:
+ if type_mode:
+ assert isinstance(obj_or_type, Type)
+ assigned_type = allowed_type.assign_type(obj_or_type)
+ else:
+ assigned_type = allowed_type.assign(obj_or_type)
+ if isinstance(assigned_type, InvalidType):
+ resolved_types.append(allowed_type)
+ else:
+ resolved_types.append(assigned_type)
+ valid = True
+
+ if not valid:
+ if unknown_count == 0:
+ return InvalidType()
+ else:
+ if type_mode:
+ assert isinstance(obj_or_type, Type)
+ new_type = obj_or_type
+ else:
+ new_type = UnknownType().assign(obj_or_type)
+ if isinstance(new_type, InvalidType):
+ return InvalidType()
+ else:
+ resolved_types.append(new_type)
+ unknown_count -= 1
+
+ for _ in range(unknown_count):
+ resolved_types.append(UnknownType())
+
+ resolved_types = _flatten_union_types(resolved_types)
+ resolved_types.sort(key=str)
+ return resolved_types
+
+
+class UnionType(Type):
+ """An "or" of types."""
+
+ name = "union"
+ types: t.ClassVar[t.List[type]] = []
+
+ def __init__(
+ self,
+ allowed_types: t.Optional[t.Sequence[ConvertableToType]] = None,
+ ):
+ assert allowed_types is None or (allowed_types.__class__ == list)
+ if allowed_types is None:
+ wb_types = []
+ else:
+ wb_types = [TypeRegistry.type_from_dtype(dt) for dt in allowed_types]
+
+ wb_types = _flatten_union_types(wb_types)
+ wb_types.sort(key=str)
+ self.params.update({"allowed_types": wb_types})
+
+ def assign(
+ self, py_obj: t.Optional[t.Any] = None
+ ) -> t.Union["UnionType", InvalidType]:
+ resolved_types = _union_assigner(
+ self.params["allowed_types"], py_obj, type_mode=False
+ )
+ if isinstance(resolved_types, InvalidType):
+ return InvalidType()
+ return self.__class__(resolved_types)
+
+ def assign_type(self, wb_type: "Type") -> t.Union["UnionType", InvalidType]:
+ if isinstance(wb_type, UnionType):
+ assignees = wb_type.params["allowed_types"]
+ else:
+ assignees = [wb_type]
+
+ resolved_types = self.params["allowed_types"]
+ for assignee in assignees:
+ resolved_types = _union_assigner(resolved_types, assignee, type_mode=True)
+ if isinstance(resolved_types, InvalidType):
+ return InvalidType()
+
+ return self.__class__(resolved_types)
+
+ def explain(self, other: t.Any, depth=0) -> str:
+ exp = super().explain(other, depth)
+ for ndx, subtype in enumerate(self.params["allowed_types"]):
+ if ndx > 0:
+ exp += "\n{}and".format("".join(["\t"] * depth))
+ exp += "\n" + subtype.explain(other, depth=depth + 1)
+ return exp
+
+ def __repr__(self):
+ return "{}".format(" or ".join([str(t) for t in self.params["allowed_types"]]))
+
+
+def OptionalType(dtype: ConvertableToType) -> UnionType: # noqa: N802
+ """Function that mimics the Type class API for constructing an "Optional Type".
+
+ This is just a Union[wb_type, NoneType].
+
+ Args:
+ dtype (Type): type to be optional
+
+ Returns:
+ Type: Optional version of the type.
+ """
+ return UnionType([TypeRegistry.type_from_dtype(dtype), NoneType()])
+
+
+class ListType(Type):
+ """A list of homogenous types."""
+
+ name = "list"
+ types: t.ClassVar[t.List[type]] = [list, tuple, set, frozenset]
+
+ def __init__(
+ self,
+ element_type: t.Optional[ConvertableToType] = None,
+ length: t.Optional[int] = None,
+ ):
+ if element_type is None:
+ wb_type: Type = UnknownType()
+ else:
+ wb_type = TypeRegistry.type_from_dtype(element_type)
+
+ self.params.update({"element_type": wb_type, "length": length})
+
+ @classmethod
+ def from_obj(cls, py_obj: t.Optional[t.Any] = None) -> "ListType":
+ if py_obj is None or not hasattr(py_obj, "__iter__"):
+ raise TypeError("ListType.from_obj expects py_obj to by list-like")
+ else:
+ if hasattr(py_obj, "tolist"):
+ py_list = py_obj.tolist()
+ else:
+ py_list = list(py_obj)
+ elm_type = (
+ UnknownType() if None not in py_list else OptionalType(UnknownType())
+ )
+ for item in py_list:
+ _elm_type = elm_type.assign(item)
+ # Commenting this out since we don't want to crash user code at this point, but rather
+ # retain an invalid internal list type.
+ # if isinstance(_elm_type, InvalidType):
+ # raise TypeError(
+ # "List contained incompatible types. Item at index {}: \n{}".format(
+ # ndx, elm_type.explain(item, 1)
+ # )
+ # )
+
+ elm_type = _elm_type
+
+ return cls(elm_type, len(py_list))
+
+ def assign_type(self, wb_type: "Type") -> t.Union["ListType", InvalidType]:
+ if isinstance(wb_type, ListType):
+ assigned_type = self.params["element_type"].assign_type(
+ wb_type.params["element_type"]
+ )
+ if not isinstance(assigned_type, InvalidType):
+ return ListType(
+ assigned_type,
+ None
+ if self.params["length"] != wb_type.params["length"]
+ else self.params["length"],
+ )
+
+ return InvalidType()
+
+ def assign(
+ self, py_obj: t.Optional[t.Any] = None
+ ) -> t.Union["ListType", InvalidType]:
+ if hasattr(py_obj, "__iter__"):
+ new_element_type = self.params["element_type"]
+ # The following ignore is needed since the above hasattr(py_obj, "__iter__") enforces iteration
+ # error: Argument 1 to "list" has incompatible type "Optional[Any]"; expected "Iterable[Any]"
+ py_list = list(py_obj) # type: ignore
+ for obj in py_list:
+ new_element_type = new_element_type.assign(obj)
+ if isinstance(new_element_type, InvalidType):
+ return InvalidType()
+ return ListType(new_element_type, len(py_list))
+
+ return InvalidType()
+
+ def explain(self, other: t.Any, depth=0) -> str:
+ exp = super().explain(other, depth)
+ gap = "".join(["\t"] * depth)
+ if ( # yes, this is a bit verbose, but the mypy typechecker likes it this way
+ isinstance(other, list)
+ or isinstance(other, tuple)
+ or isinstance(other, set)
+ or isinstance(other, frozenset)
+ ):
+ new_element_type = self.params["element_type"]
+ for ndx, obj in enumerate(list(other)):
+ _new_element_type = new_element_type.assign(obj)
+ if isinstance(_new_element_type, InvalidType):
+ exp += "\n{}Index {}:\n{}".format(
+ gap, ndx, new_element_type.explain(obj, depth + 1)
+ )
+ break
+ new_element_type = _new_element_type
+ return exp
+
+ def __repr__(self):
+ return "{}[]".format(self.params["element_type"])
+
+
+class NDArrayType(Type):
+ """Represents a list of homogenous types."""
+
+ name = "ndarray"
+ types: t.ClassVar[t.List[type]] = [] # will manually add type if np is available
+ _serialization_path: t.Optional[t.Dict[str, str]]
+
+ def __init__(
+ self,
+ shape: t.Sequence[int],
+ serialization_path: t.Optional[t.Dict[str, str]] = None,
+ ):
+ self.params.update({"shape": list(shape)})
+ self._serialization_path = serialization_path
+
+ @classmethod
+ def from_obj(cls, py_obj: t.Optional[t.Any] = None) -> "NDArrayType":
+ if is_numpy_array(py_obj):
+ return cls(py_obj.shape) # type: ignore
+ elif isinstance(py_obj, list):
+ shape = []
+ target = py_obj
+ while isinstance(target, list):
+ dim = len(target)
+ shape.append(dim)
+ if dim > 0:
+ target = target[0]
+ return cls(shape)
+ else:
+ raise TypeError(
+ "NDArrayType.from_obj expects py_obj to be ndarray or list, found {}".format(
+ py_obj.__class__
+ )
+ )
+
+ def assign_type(self, wb_type: "Type") -> t.Union["NDArrayType", InvalidType]:
+ if (
+ isinstance(wb_type, NDArrayType)
+ and self.params["shape"] == wb_type.params["shape"]
+ ):
+ return self
+ elif isinstance(wb_type, ListType):
+ # Should we return error here?
+ return self
+
+ return InvalidType()
+
+ def assign(
+ self, py_obj: t.Optional[t.Any] = None
+ ) -> t.Union["NDArrayType", InvalidType]:
+ if is_numpy_array(py_obj) or isinstance(py_obj, list):
+ py_type = self.from_obj(py_obj)
+ return self.assign_type(py_type)
+
+ return InvalidType()
+
+ def to_json(self, artifact: t.Optional["Artifact"] = None) -> t.Dict[str, t.Any]:
+ # custom override to support serialization path outside of params internal dict
+ res = {
+ "wb_type": self.name,
+ "params": {
+ "shape": self.params["shape"],
+ "serialization_path": self._serialization_path,
+ },
+ }
+
+ return res
+
+ def _get_serialization_path(self) -> t.Optional[t.Dict[str, str]]:
+ return self._serialization_path
+
+ def _set_serialization_path(self, path: str, key: str) -> None:
+ self._serialization_path = {"path": path, "key": key}
+
+ def _clear_serialization_path(self) -> None:
+ self._serialization_path = None
+
+
+if np:
+ NDArrayType.types.append(np.ndarray)
+
+# class KeyPolicy:
+# EXACT = "E" # require exact key match
+# SUBSET = "S" # all known keys are optional and unknown keys are disallowed
+# UNRESTRICTED = "U" # all known keys are optional and unknown keys are Unknown
+
+
+class TypedDictType(Type):
+ """Represents a dictionary object where each key can have a type."""
+
+ name = "typedDict"
+ legacy_names = ["dictionary"]
+ types: t.ClassVar[t.List[type]] = [dict]
+
+ def __init__(
+ self,
+ type_map: t.Optional[t.Dict[str, ConvertableToType]] = None,
+ ):
+ if type_map is None:
+ type_map = {}
+ self.params.update(
+ {
+ "type_map": {
+ key: TypeRegistry.type_from_dtype(type_map[key]) for key in type_map
+ }
+ }
+ )
+
+ @classmethod
+ def from_obj(cls, py_obj: t.Optional[t.Any] = None) -> "TypedDictType":
+ if not isinstance(py_obj, dict):
+ TypeError("TypedDictType.from_obj expects a dictionary")
+
+ assert isinstance(py_obj, dict) # helps mypy type checker
+ return cls({key: TypeRegistry.type_of(py_obj[key]) for key in py_obj})
+
+ def assign_type(self, wb_type: "Type") -> t.Union["TypedDictType", InvalidType]:
+ if (
+ isinstance(wb_type, TypedDictType)
+ and len(
+ set(wb_type.params["type_map"].keys())
+ - set(self.params["type_map"].keys())
+ )
+ == 0
+ ):
+ type_map = {}
+ for key in self.params["type_map"]:
+ type_map[key] = self.params["type_map"][key].assign_type(
+ wb_type.params["type_map"].get(key, UnknownType())
+ )
+ if isinstance(type_map[key], InvalidType):
+ return InvalidType()
+ return TypedDictType(type_map)
+
+ return InvalidType()
+
+ def assign(
+ self, py_obj: t.Optional[t.Any] = None
+ ) -> t.Union["TypedDictType", InvalidType]:
+ if (
+ isinstance(py_obj, dict)
+ and len(set(py_obj.keys()) - set(self.params["type_map"].keys())) == 0
+ ):
+ type_map = {}
+ for key in self.params["type_map"]:
+ type_map[key] = self.params["type_map"][key].assign(
+ py_obj.get(key, None)
+ )
+ if isinstance(type_map[key], InvalidType):
+ return InvalidType()
+ return TypedDictType(type_map)
+
+ return InvalidType()
+
+ def explain(self, other: t.Any, depth=0) -> str:
+ exp = super().explain(other, depth)
+ gap = "".join(["\t"] * depth)
+ if isinstance(other, dict):
+ extra_keys = set(other.keys()) - set(self.params["type_map"].keys())
+ if len(extra_keys) > 0:
+ exp += "\n{}Found extra keys: {}".format(
+ gap, ",".join(list(extra_keys))
+ )
+
+ for key in self.params["type_map"]:
+ val = other.get(key, None)
+ if isinstance(self.params["type_map"][key].assign(val), InvalidType):
+ exp += "\n{}Key '{}':\n{}".format(
+ gap,
+ key,
+ self.params["type_map"][key].explain(val, depth=depth + 1),
+ )
+ return exp
+
+ def __repr__(self):
+ return "{}".format(self.params["type_map"])
+
+
+# Special Types
+TypeRegistry.add(InvalidType)
+TypeRegistry.add(AnyType)
+TypeRegistry.add(UnknownType)
+
+# Types with default type mappings
+TypeRegistry.add(NoneType)
+TypeRegistry.add(StringType)
+TypeRegistry.add(TimestampType)
+TypeRegistry.add(NumberType)
+TypeRegistry.add(BooleanType)
+TypeRegistry.add(ListType)
+TypeRegistry.add(TypedDictType)
+
+# Types without default type mappings
+TypeRegistry.add(UnionType)
+TypeRegistry.add(PythonObjectType)
+TypeRegistry.add(ConstType)
+
+# Common Industry Types
+TypeRegistry.add(NDArrayType)
+
+__all__ = [
+ "TypeRegistry",
+ "InvalidType",
+ "UnknownType",
+ "AnyType",
+ "NoneType",
+ "StringType",
+ "NumberType",
+ "TimestampType",
+ "BooleanType",
+ "ListType",
+ "TypedDictType",
+ "UnionType",
+ "PythonObjectType",
+ "ConstType",
+ "OptionalType",
+ "Type",
+ "NDArrayType",
+]
diff --git a/wandb/sdk/data_types/_private.py b/wandb/sdk/data_types/_private.py
new file mode 100644
index 0000000000000000000000000000000000000000..887bc8a231014d06dbe65590663064ec792eeb84
--- /dev/null
+++ b/wandb/sdk/data_types/_private.py
@@ -0,0 +1,10 @@
+import atexit
+import tempfile
+
+# Staging directory, so we can encode raw data into files, then hash them before
+# we put them into the Run directory to be uploaded.
+MEDIA_TMP = tempfile.TemporaryDirectory("wandb-media")
+
+
+def _cleanup_media_tmp_dir() -> None:
+ atexit.register(MEDIA_TMP.cleanup)
diff --git a/wandb/sdk/data_types/base_types/__init__.py b/wandb/sdk/data_types/base_types/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sdk/data_types/base_types/json_metadata.py b/wandb/sdk/data_types/base_types/json_metadata.py
new file mode 100644
index 0000000000000000000000000000000000000000..b644584abcf1a640c35a1c98e7b7083999f3b446
--- /dev/null
+++ b/wandb/sdk/data_types/base_types/json_metadata.py
@@ -0,0 +1,55 @@
+import codecs
+import os
+from typing import TYPE_CHECKING, Type, Union
+
+from wandb import util
+from wandb.sdk.lib import runid
+
+from .._private import MEDIA_TMP
+from .media import Media
+
+if TYPE_CHECKING: # pragma: no cover
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ from ...wandb_run import Run as LocalRun
+
+
+# Allows encoding of arbitrary JSON structures
+# as a file
+#
+# This class should be used as an abstract class
+# extended to have validation methods
+
+
+class JSONMetadata(Media):
+ """JSONMetadata is a type for encoding arbitrary metadata as files."""
+
+ def __init__(self, val: dict) -> None:
+ super().__init__()
+
+ self.validate(val)
+ self._val = val
+
+ ext = "." + self.type_name() + ".json"
+ tmp_path = os.path.join(MEDIA_TMP.name, runid.generate_id() + ext)
+ with codecs.open(tmp_path, "w", encoding="utf-8") as fp:
+ util.json_dump_uncompressed(self._val, fp)
+ self._set_file(tmp_path, is_tmp=True, extension=ext)
+
+ @classmethod
+ def get_media_subdir(cls: Type["JSONMetadata"]) -> str:
+ return os.path.join("media", "metadata", cls.type_name())
+
+ def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
+ json_dict = super().to_json(run_or_artifact)
+ json_dict["_type"] = self.type_name()
+
+ return json_dict
+
+ # These methods should be overridden in the child class
+ @classmethod
+ def type_name(cls) -> str:
+ return "metadata"
+
+ def validate(self, val: dict) -> bool:
+ return True
diff --git a/wandb/sdk/data_types/base_types/media.py b/wandb/sdk/data_types/base_types/media.py
new file mode 100644
index 0000000000000000000000000000000000000000..d7a6e939a2e882a0f9bbd4c982ab0eb591b357af
--- /dev/null
+++ b/wandb/sdk/data_types/base_types/media.py
@@ -0,0 +1,316 @@
+import hashlib
+import os
+import platform
+import re
+import shutil
+from typing import TYPE_CHECKING, Optional, Sequence, Type, Union, cast
+
+import wandb
+from wandb import util
+from wandb._globals import _datatypes_callback
+from wandb.sdk.lib import filesystem
+from wandb.sdk.lib.paths import LogicalPath
+
+from .wb_value import WBValue
+
+if TYPE_CHECKING: # pragma: no cover
+ import numpy as np
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ from ...wandb_run import Run as LocalRun
+
+
+SYS_PLATFORM = platform.system()
+
+
+def _wb_filename(
+ key: Union[str, int], step: Union[str, int], id: Union[str, int], extension: str
+) -> str:
+ return f"{str(key)}_{str(step)}_{str(id)}{extension}"
+
+
+class Media(WBValue):
+ """A WBValue stored as a file outside JSON that can be rendered in a media panel.
+
+ If necessary, we move or copy the file into the Run's media directory so that it
+ gets uploaded.
+ """
+
+ _path: Optional[str]
+ _run: Optional["LocalRun"]
+ _caption: Optional[str]
+ _is_tmp: Optional[bool]
+ _extension: Optional[str]
+ _sha256: Optional[str]
+ _size: Optional[int]
+
+ def __init__(self, caption: Optional[str] = None) -> None:
+ super().__init__()
+ self._path = None
+ # The run under which this object is bound, if any.
+ self._run = None
+ self._caption = caption
+
+ def _set_file(
+ self, path: str, is_tmp: bool = False, extension: Optional[str] = None
+ ) -> None:
+ self._path = path
+ self._is_tmp = is_tmp
+ self._extension = extension
+ assert extension is None or path.endswith(
+ extension
+ ), f'Media file extension "{extension}" must occur at the end of path "{path}".'
+
+ with open(self._path, "rb") as f:
+ self._sha256 = hashlib.sha256(f.read()).hexdigest()
+ self._size = os.path.getsize(self._path)
+
+ @classmethod
+ def get_media_subdir(cls: Type["Media"]) -> str:
+ raise NotImplementedError
+
+ @staticmethod
+ def captions(
+ media_items: Sequence["Media"],
+ ) -> Union[bool, Sequence[Optional[str]]]:
+ if media_items[0]._caption is not None:
+ return [m._caption for m in media_items]
+ else:
+ return False
+
+ def is_bound(self) -> bool:
+ return self._run is not None
+
+ def file_is_set(self) -> bool:
+ return self._path is not None and self._sha256 is not None
+
+ def bind_to_run(
+ self,
+ run: "LocalRun",
+ key: Union[int, str],
+ step: Union[int, str],
+ id_: Optional[Union[int, str]] = None,
+ ignore_copy_err: Optional[bool] = None,
+ ) -> None:
+ """Bind this object to a particular Run.
+
+ Calling this function is necessary so that we have somewhere specific to put the
+ file associated with this object, from which other Runs can refer to it.
+ """
+ assert self.file_is_set(), "bind_to_run called before _set_file"
+
+ if SYS_PLATFORM == "Windows" and not util.check_windows_valid_filename(key):
+ raise ValueError(
+ f"Media {key} is invalid. Please remove invalid filename characters"
+ )
+
+ # The following two assertions are guaranteed to pass
+ # by definition file_is_set, but are needed for
+ # mypy to understand that these are strings below.
+ assert isinstance(self._path, str)
+ assert isinstance(self._sha256, str)
+
+ assert run is not None, 'Argument "run" must not be None.'
+ self._run = run
+
+ if self._extension is None:
+ _, extension = os.path.splitext(os.path.basename(self._path))
+ else:
+ extension = self._extension
+
+ if id_ is None:
+ id_ = self._sha256[:20]
+
+ file_path = _wb_filename(key, step, id_, extension)
+ media_path = os.path.join(self.get_media_subdir(), file_path)
+ new_path = os.path.join(self._run.dir, media_path)
+ filesystem.mkdir_exists_ok(os.path.dirname(new_path))
+
+ if self._is_tmp:
+ shutil.move(self._path, new_path)
+ self._path = new_path
+ self._is_tmp = False
+ _datatypes_callback(media_path)
+ else:
+ try:
+ shutil.copy(self._path, new_path)
+ except shutil.SameFileError as e:
+ if not ignore_copy_err:
+ raise e
+ self._path = new_path
+ _datatypes_callback(media_path)
+
+ def to_json(self, run: Union["LocalRun", "Artifact"]) -> dict:
+ """Serialize the object into a JSON blob.
+
+ Uses run or artifact to store additional data. If `run_or_artifact` is a
+ wandb.Run then `self.bind_to_run()` must have been previously been called.
+
+ Args:
+ run_or_artifact (wandb.Run | wandb.Artifact): the Run or Artifact for which
+ this object should be generating JSON for - this is useful to store
+ additional data if needed.
+
+ Returns:
+ dict: JSON representation
+ """
+ # NOTE: uses of Audio in this class are a temporary hack -- when Ref support moves up
+ # into Media itself we should get rid of them
+ from wandb import Image
+ from wandb.data_types import Audio
+
+ json_obj = {}
+ if isinstance(run, wandb.wandb_sdk.wandb_run.Run):
+ json_obj.update(
+ {
+ "_type": "file", # TODO(adrian): This isn't (yet) a real media type we support on the frontend.
+ "sha256": self._sha256,
+ "size": self._size,
+ }
+ )
+ artifact_entry_url = self._get_artifact_entry_ref_url()
+ if artifact_entry_url is not None:
+ json_obj["artifact_path"] = artifact_entry_url
+ artifact_entry_latest_url = self._get_artifact_entry_latest_ref_url()
+ if artifact_entry_latest_url is not None:
+ json_obj["_latest_artifact_path"] = artifact_entry_latest_url
+
+ if artifact_entry_url is None or self.is_bound():
+ assert (
+ self.is_bound()
+ ), "Value of type {} must be bound to a run with bind_to_run() before being serialized to JSON.".format(
+ type(self).__name__
+ )
+
+ assert (
+ self._run is run
+ ), "We don't support referring to media files across runs."
+
+ # The following two assertions are guaranteed to pass
+ # by definition is_bound, but are needed for
+ # mypy to understand that these are strings below.
+ assert isinstance(self._path, str)
+ json_obj["path"] = LogicalPath(
+ os.path.relpath(self._path, self._run.dir)
+ )
+
+ elif isinstance(run, wandb.Artifact):
+ if self.file_is_set():
+ # The following two assertions are guaranteed to pass
+ # by definition of the call above, but are needed for
+ # mypy to understand that these are strings below.
+ assert isinstance(self._path, str)
+ assert isinstance(self._sha256, str)
+ artifact = run # Checks if the concrete image has already been added to this artifact
+ name = artifact.get_added_local_path_name(self._path)
+ if name is None:
+ if self._is_tmp:
+ name = os.path.join(
+ self.get_media_subdir(), os.path.basename(self._path)
+ )
+ else:
+ # If the files is not temporary, include the first 8 characters of the file's SHA256 to
+ # avoid name collisions. This way, if there are two images `dir1/img.png` and `dir2/img.png`
+ # we end up with a unique path for each.
+ name = os.path.join(
+ self.get_media_subdir(),
+ self._sha256[:20],
+ os.path.basename(self._path),
+ )
+
+ # if not, check to see if there is a source artifact for this object
+ if (
+ self._artifact_source
+ is not None
+ # and self._artifact_source.artifact != artifact
+ ):
+ default_root = self._artifact_source.artifact._default_root()
+ # if there is, get the name of the entry (this might make sense to move to a helper off artifact)
+ if self._path.startswith(default_root):
+ name = self._path[len(default_root) :]
+ name = name.lstrip(os.sep)
+
+ # Add this image as a reference
+ path = self._artifact_source.artifact.get_entry(name)
+ artifact.add_reference(path.ref_url(), name=name)
+ elif (
+ isinstance(self, Audio) or isinstance(self, Image)
+ ) and self.path_is_reference(self._path):
+ artifact.add_reference(self._path, name=name)
+ else:
+ entry = artifact.add_file(
+ self._path, name=name, is_tmp=self._is_tmp
+ )
+ name = entry.path
+
+ json_obj["path"] = name
+ json_obj["sha256"] = self._sha256
+ json_obj["_type"] = self._log_type
+ return json_obj
+
+ @classmethod
+ def from_json(
+ cls: Type["Media"], json_obj: dict, source_artifact: "Artifact"
+ ) -> "Media":
+ """Likely will need to override for any more complicated media objects."""
+ return cls(source_artifact.get_entry(json_obj["path"]).download())
+
+ def __eq__(self, other: object) -> bool:
+ """Likely will need to override for any more complicated media objects."""
+ return (
+ isinstance(other, self.__class__)
+ and hasattr(self, "_sha256")
+ and hasattr(other, "_sha256")
+ and self._sha256 == other._sha256
+ )
+
+ @staticmethod
+ def path_is_reference(path: Optional[str]) -> bool:
+ return bool(path and re.match(r"^(gs|s3|https?)://", path))
+
+
+class BatchableMedia(Media):
+ """Media that is treated in batches.
+
+ E.g. images and thumbnails. Apart from images, we just use these batches to help
+ organize files by name in the media directory.
+ """
+
+ def __init__(self) -> None:
+ super().__init__()
+
+ @classmethod
+ def seq_to_json(
+ cls: Type["BatchableMedia"],
+ seq: Sequence["BatchableMedia"],
+ run: "LocalRun",
+ key: str,
+ step: Union[int, str],
+ ) -> dict:
+ raise NotImplementedError
+
+
+def _numpy_arrays_to_lists(
+ payload: Union[dict, Sequence, "np.ndarray"]
+) -> Union[Sequence, dict, str, int, float, bool]:
+ # Casts all numpy arrays to lists so we don't convert them to histograms, primarily for Plotly
+
+ if isinstance(payload, dict):
+ res = {}
+ for key, val in payload.items():
+ res[key] = _numpy_arrays_to_lists(val)
+ return res
+ elif isinstance(payload, Sequence) and not isinstance(payload, str):
+ return [_numpy_arrays_to_lists(v) for v in payload]
+ elif util.is_numpy_array(payload):
+ if TYPE_CHECKING:
+ payload = cast("np.ndarray", payload)
+ return [
+ _numpy_arrays_to_lists(v)
+ for v in (payload.tolist() if payload.ndim > 0 else [payload.tolist()])
+ ]
+ # Protects against logging non serializable objects
+ elif isinstance(payload, Media):
+ return str(payload.__class__.__name__)
+ return payload # type: ignore
diff --git a/wandb/sdk/data_types/base_types/wb_value.py b/wandb/sdk/data_types/base_types/wb_value.py
new file mode 100644
index 0000000000000000000000000000000000000000..3dfb472c6910e4d79ffc2644d686cca760a88397
--- /dev/null
+++ b/wandb/sdk/data_types/base_types/wb_value.py
@@ -0,0 +1,274 @@
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Type, Union
+
+from wandb import util
+
+if TYPE_CHECKING: # pragma: no cover
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ from ...wandb_run import Run as LocalRun
+
+ TypeMappingType = Dict[str, Type["WBValue"]]
+
+
+def _server_accepts_client_ids() -> bool:
+ from pkg_resources import parse_version
+
+ # First, if we are offline, assume the backend server cannot
+ # accept client IDs. Unfortunately, this is the best we can do
+ # until we are sure that all local versions are > "0.11.0" max_cli_version.
+ # The practical implication is that tables logged in offline mode
+ # will not show up in the workspace (but will still show up in artifacts). This
+ # means we never lose data, and we can still view using weave. If we decided
+ # to use client ids in offline mode, then the manifests and artifact data
+ # would never be resolvable and would lead to failed uploads. Our position
+ # is to never lose data - and instead take the tradeoff in the UI.
+ if util._is_offline():
+ return False
+
+ # If the script is online, request the max_cli_version and ensure the server
+ # is of a high enough version.
+ max_cli_version = util._get_max_cli_version()
+ if max_cli_version is None:
+ return False
+ accepts_client_ids: bool = parse_version("0.11.0") <= parse_version(max_cli_version)
+ return accepts_client_ids
+
+
+class _WBValueArtifactSource:
+ artifact: "Artifact"
+ name: Optional[str]
+
+ def __init__(self, artifact: "Artifact", name: Optional[str] = None) -> None:
+ self.artifact = artifact
+ self.name = name
+
+
+class _WBValueArtifactTarget:
+ artifact: "Artifact"
+ name: Optional[str]
+
+ def __init__(self, artifact: "Artifact", name: Optional[str] = None) -> None:
+ self.artifact = artifact
+ self.name = name
+
+
+class WBValue:
+ """Typed objects that can be logged with `wandb.log()` and visualized by wandb.
+
+ The objects will be serialized as JSON and always have a _type attribute that
+ indicates how to interpret the other fields.
+ """
+
+ # Class Attributes
+ _type_mapping: ClassVar[Optional["TypeMappingType"]] = None
+ # override _log_type to indicate the type which the subclass deserializes
+ _log_type: ClassVar[Optional[str]] = None
+
+ # Instance Attributes
+ _artifact_source: Optional[_WBValueArtifactSource]
+ _artifact_target: Optional[_WBValueArtifactTarget]
+
+ def __init__(self) -> None:
+ self._artifact_source = None
+ self._artifact_target = None
+
+ def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
+ """Serialize the object into a JSON blob.
+
+ Uses current run or artifact to store additional data.
+
+ Args:
+ run_or_artifact (wandb.Run | wandb.Artifact): the Run or Artifact for which
+ this object should be generating JSON for - this is useful to to store
+ additional data if needed.
+
+ Returns:
+ dict: JSON representation
+ """
+ raise NotImplementedError
+
+ @classmethod
+ def from_json(
+ cls: Type["WBValue"], json_obj: dict, source_artifact: "Artifact"
+ ) -> "WBValue":
+ """Deserialize a `json_obj` into it's class representation.
+
+ If additional resources were stored in the `run_or_artifact` artifact during the
+ `to_json` call, then those resources should be in the `source_artifact`.
+
+ Args:
+ json_obj (dict): A JSON dictionary to deserialize source_artifact
+ (wandb.Artifact): An artifact which will hold any additional
+ resources which were stored during the `to_json` function.
+ """
+ raise NotImplementedError
+
+ @classmethod
+ def with_suffix(cls: Type["WBValue"], name: str, filetype: str = "json") -> str:
+ """Get the name with the appropriate suffix.
+
+ Args:
+ name (str): the name of the file
+ filetype (str, optional): the filetype to use. Defaults to "json".
+
+ Returns:
+ str: a filename which is suffixed with it's `_log_type` followed by the
+ filetype.
+ """
+ if cls._log_type is not None:
+ suffix = cls._log_type + "." + filetype
+ else:
+ suffix = filetype
+ if not name.endswith(suffix):
+ return name + "." + suffix
+ return name
+
+ @staticmethod
+ def init_from_json(
+ json_obj: dict, source_artifact: "Artifact"
+ ) -> Optional["WBValue"]:
+ """Initialize a `WBValue` from a JSON blob based on the class that creatd it.
+
+ Looks through all subclasses and tries to match the json obj with the class
+ which created it. It will then call that subclass' `from_json` method.
+ Importantly, this function will set the return object's `source_artifact`
+ attribute to the passed in source artifact. This is critical for artifact
+ bookkeeping. If you choose to create a wandb.Value via it's `from_json` method,
+ make sure to properly set this `artifact_source` to avoid data duplication.
+
+ Args:
+ json_obj (dict): A JSON dictionary to deserialize. It must contain a `_type`
+ key. This is used to lookup the correct subclass to use.
+ source_artifact (wandb.Artifact): An artifact which will hold any additional
+ resources which were stored during the `to_json` function.
+
+ Returns:
+ wandb.Value: a newly created instance of a subclass of wandb.Value
+ """
+ class_option = WBValue.type_mapping().get(json_obj["_type"])
+ if class_option is not None:
+ obj = class_option.from_json(json_obj, source_artifact)
+ obj._set_artifact_source(source_artifact)
+ return obj
+
+ return None
+
+ @staticmethod
+ def type_mapping() -> "TypeMappingType":
+ """Return a map from `_log_type` to subclass. Used to lookup correct types for deserialization.
+
+ Returns:
+ dict: dictionary of str:class
+ """
+ if WBValue._type_mapping is None:
+ WBValue._type_mapping = {}
+ frontier = [WBValue]
+ explored = set()
+ while len(frontier) > 0:
+ class_option = frontier.pop()
+ explored.add(class_option)
+ if class_option._log_type is not None:
+ WBValue._type_mapping[class_option._log_type] = class_option
+ for subclass in class_option.__subclasses__():
+ if subclass not in explored:
+ frontier.append(subclass)
+ return WBValue._type_mapping
+
+ def __eq__(self, other: object) -> bool:
+ return id(self) == id(other)
+
+ def __ne__(self, other: object) -> bool:
+ return not self.__eq__(other)
+
+ def to_data_array(self) -> List[Any]:
+ """Convert the object to a list of primitives representing the underlying data."""
+ raise NotImplementedError
+
+ def _set_artifact_source(
+ self, artifact: "Artifact", name: Optional[str] = None
+ ) -> None:
+ assert (
+ self._artifact_source is None
+ ), "Cannot update artifact_source. Existing source: {}/{}".format(
+ self._artifact_source.artifact, self._artifact_source.name
+ )
+ self._artifact_source = _WBValueArtifactSource(artifact, name)
+
+ def _set_artifact_target(
+ self, artifact: "Artifact", name: Optional[str] = None
+ ) -> None:
+ assert (
+ self._artifact_target is None
+ ), "Cannot update artifact_target. Existing target: {}/{}".format(
+ self._artifact_target.artifact, self._artifact_target.name
+ )
+ self._artifact_target = _WBValueArtifactTarget(artifact, name)
+
+ def _get_artifact_entry_ref_url(self) -> Optional[str]:
+ # If the object is coming from another artifact
+ if self._artifact_source and self._artifact_source.name:
+ ref_entry = self._artifact_source.artifact.get_entry(
+ type(self).with_suffix(self._artifact_source.name)
+ )
+ return str(ref_entry.ref_url())
+ # Else, if the object is destined for another artifact and we support client IDs
+ elif (
+ self._artifact_target
+ and self._artifact_target.name
+ and self._artifact_target.artifact._client_id is not None
+ and self._artifact_target.artifact._final
+ and _server_accepts_client_ids()
+ ):
+ return "wandb-client-artifact://{}/{}".format(
+ self._artifact_target.artifact._client_id,
+ type(self).with_suffix(self._artifact_target.name),
+ )
+ # Else if we do not support client IDs, but online, then block on upload
+ # Note: this is old behavior just to stay backwards compatible
+ # with older server versions. This code path should be removed
+ # once those versions are no longer supported. This path uses a .wait
+ # which blocks the user process on artifact upload.
+ elif (
+ self._artifact_target
+ and self._artifact_target.name
+ and self._artifact_target.artifact._is_draft_save_started()
+ and not util._is_offline()
+ and not _server_accepts_client_ids()
+ ):
+ self._artifact_target.artifact.wait()
+ ref_entry = self._artifact_target.artifact.get_entry(
+ type(self).with_suffix(self._artifact_target.name)
+ )
+ return str(ref_entry.ref_url())
+ return None
+
+ def _get_artifact_entry_latest_ref_url(self) -> Optional[str]:
+ if (
+ self._artifact_target
+ and self._artifact_target.name
+ and self._artifact_target.artifact._client_id is not None
+ and self._artifact_target.artifact._final
+ and _server_accepts_client_ids()
+ ):
+ return "wandb-client-artifact://{}:latest/{}".format(
+ self._artifact_target.artifact._sequence_client_id,
+ type(self).with_suffix(self._artifact_target.name),
+ )
+ # Else if we do not support client IDs, then block on upload
+ # Note: this is old behavior just to stay backwards compatible
+ # with older server versions. This code path should be removed
+ # once those versions are no longer supported. This path uses a .wait
+ # which blocks the user process on artifact upload.
+ elif (
+ self._artifact_target
+ and self._artifact_target.name
+ and self._artifact_target.artifact._is_draft_save_started()
+ and not util._is_offline()
+ and not _server_accepts_client_ids()
+ ):
+ self._artifact_target.artifact.wait()
+ ref_entry = self._artifact_target.artifact.get_entry(
+ type(self).with_suffix(self._artifact_target.name)
+ )
+ return str(ref_entry.ref_url())
+ return None
diff --git a/wandb/sdk/data_types/helper_types/__init__.py b/wandb/sdk/data_types/helper_types/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sdk/data_types/helper_types/bounding_boxes_2d.py b/wandb/sdk/data_types/helper_types/bounding_boxes_2d.py
new file mode 100644
index 0000000000000000000000000000000000000000..369410044f5384b3d64d13f42f46e836c97d1def
--- /dev/null
+++ b/wandb/sdk/data_types/helper_types/bounding_boxes_2d.py
@@ -0,0 +1,291 @@
+import numbers
+from typing import TYPE_CHECKING, Optional, Type, Union
+
+import wandb
+from wandb import util
+from wandb.util import has_num
+
+from ..base_types.json_metadata import JSONMetadata
+
+if TYPE_CHECKING: # pragma: no cover
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ from ...wandb_run import Run as LocalRun
+
+
+class BoundingBoxes2D(JSONMetadata):
+ """Format images with 2D bounding box overlays for logging to W&B.
+
+ Arguments:
+ val: (dictionary) A dictionary of the following form:
+ box_data: (list of dictionaries) One dictionary for each bounding box, containing:
+ position: (dictionary) the position and size of the bounding box, in one of two formats
+ Note that boxes need not all use the same format.
+ {"minX", "minY", "maxX", "maxY"}: (dictionary) A set of coordinates defining
+ the upper and lower bounds of the box (the bottom left and top right corners)
+ {"middle", "width", "height"}: (dictionary) A set of coordinates defining the
+ center and dimensions of the box, with "middle" as a list [x, y] for the
+ center point and "width" and "height" as numbers
+ domain: (string) One of two options for the bounding box coordinate domain
+ null: By default, or if no argument is passed, the coordinate domain
+ is assumed to be relative to the original image, expressing this box as a fraction
+ or percentage of the original image. This means all coordinates and dimensions
+ passed into the "position" argument are floating point numbers between 0 and 1.
+ "pixel": (string literal) The coordinate domain is set to the pixel space. This means all
+ coordinates and dimensions passed into "position" are integers within the bounds
+ of the image dimensions.
+ class_id: (integer) The class label id for this box
+ scores: (dictionary of string to number, optional) A mapping of named fields
+ to numerical values (float or int), can be used for filtering boxes in the UI
+ based on a range of values for the corresponding field
+ box_caption: (string, optional) A string to be displayed as the label text above this
+ box in the UI, often composed of the class label, class name, and/or scores
+
+ class_labels: (dictionary, optional) A map of integer class labels to their readable class names
+
+ key: (string)
+ The readable name or id for this set of bounding boxes (e.g. predictions, ground_truth)
+
+ Examples:
+ ### Log bounding boxes for a single image
+ <!--yeadoc-test:boundingbox-2d-->
+ ```python
+ import numpy as np
+ import wandb
+
+ wandb.init()
+ image = np.random.randint(low=0, high=256, size=(200, 300, 3))
+
+ class_labels = {0: "person", 1: "car", 2: "road", 3: "building"}
+
+ img = wandb.Image(
+ image,
+ boxes={
+ "predictions": {
+ "box_data": [
+ {
+ # one box expressed in the default relative/fractional domain
+ "position": {"minX": 0.1, "maxX": 0.2, "minY": 0.3, "maxY": 0.4},
+ "class_id": 1,
+ "box_caption": class_labels[1],
+ "scores": {"acc": 0.2, "loss": 1.2},
+ },
+ {
+ # another box expressed in the pixel domain
+ "position": {"middle": [150, 20], "width": 68, "height": 112},
+ "domain": "pixel",
+ "class_id": 3,
+ "box_caption": "a building",
+ "scores": {"acc": 0.5, "loss": 0.7},
+ },
+ # Log as many boxes an as needed
+ ],
+ "class_labels": class_labels,
+ }
+ },
+ )
+
+ wandb.log({"driving_scene": img})
+ ```
+
+ ### Log a bounding box overlay to a Table
+ <!--yeadoc-test:bb2d-image-with-labels-->
+ ```python
+ import numpy as np
+ import wandb
+
+ wandb.init()
+ image = np.random.randint(low=0, high=256, size=(200, 300, 3))
+
+ class_labels = {0: "person", 1: "car", 2: "road", 3: "building"}
+
+ class_set = wandb.Classes(
+ [
+ {"name": "person", "id": 0},
+ {"name": "car", "id": 1},
+ {"name": "road", "id": 2},
+ {"name": "building", "id": 3},
+ ]
+ )
+
+ img = wandb.Image(
+ image,
+ boxes={
+ "predictions": {
+ "box_data": [
+ {
+ # one box expressed in the default relative/fractional domain
+ "position": {"minX": 0.1, "maxX": 0.2, "minY": 0.3, "maxY": 0.4},
+ "class_id": 1,
+ "box_caption": class_labels[1],
+ "scores": {"acc": 0.2, "loss": 1.2},
+ },
+ {
+ # another box expressed in the pixel domain
+ "position": {"middle": [150, 20], "width": 68, "height": 112},
+ "domain": "pixel",
+ "class_id": 3,
+ "box_caption": "a building",
+ "scores": {"acc": 0.5, "loss": 0.7},
+ },
+ # Log as many boxes an as needed
+ ],
+ "class_labels": class_labels,
+ }
+ },
+ classes=class_set,
+ )
+
+ table = wandb.Table(columns=["image"])
+ table.add_data(img)
+ wandb.log({"driving_scene": table})
+ ```
+ """
+
+ _log_type = "bounding-boxes"
+ # TODO: when the change is made to have this produce a dict with a _type, define
+ # it here as _log_type, associate it in to_json
+
+ def __init__(self, val: dict, key: str) -> None:
+ """Initialize a BoundingBoxes object.
+
+ The input dictionary `val` should contain the keys:
+ box_data: a list of dictionaries, each of which describes a bounding box.
+ class_labels: (optional) A map of integer class labels to their readable
+ class names.
+
+ Each bounding box dictionary should contain the following keys:
+ position: (dictionary) the position and size of the bounding box.
+ domain: (string) One of two options for the bounding box coordinate domain.
+ class_id: (integer) The class label id for this box.
+ scores: (dictionary of string to number, optional) A mapping of named fields
+ to numerical values (float or int).
+ box_caption: (optional) The label text, often composed of the class label,
+ class name, and/or scores.
+
+ The position dictionary should be in one of two formats:
+ {"minX", "minY", "maxX", "maxY"}: (dictionary) A set of coordinates defining
+ the upper and lower bounds of the box (the bottom left and top right
+ corners).
+ {"middle", "width", "height"}: (dictionary) A set of coordinates defining
+ the center and dimensions of the box, with "middle" as a list [x, y] for
+ the center point and "width" and "height" as numbers.
+ Note that boxes need not all use the same format.
+
+ Args:
+ val: (dictionary) A dictionary containing the bounding box data.
+ key: (string) The readable name or id for this set of bounding boxes (e.g.
+ predictions, ground_truth)
+ """
+ super().__init__(val)
+ self._val = val["box_data"]
+ self._key = key
+ # Add default class mapping
+ if "class_labels" not in val:
+ np = util.get_module(
+ "numpy", required="Bounding box support requires numpy"
+ )
+ classes = (
+ np.unique(list(box["class_id"] for box in val["box_data"]))
+ .astype(np.int32)
+ .tolist()
+ )
+ class_labels = {c: "class_" + str(c) for c in classes}
+ self._class_labels = class_labels
+ else:
+ self._class_labels = val["class_labels"]
+
+ def bind_to_run(
+ self,
+ run: "LocalRun",
+ key: Union[int, str],
+ step: Union[int, str],
+ id_: Optional[Union[int, str]] = None,
+ ignore_copy_err: Optional[bool] = None,
+ ) -> None:
+ # bind_to_run key argument is the Image parent key
+ # the self._key value is the mask's sub key
+ super().bind_to_run(run, key, step, id_=id_, ignore_copy_err=ignore_copy_err)
+ run._add_singleton(
+ "bounding_box/class_labels",
+ str(key) + "_wandb_delimeter_" + self._key,
+ self._class_labels,
+ )
+
+ @classmethod
+ def type_name(cls) -> str:
+ return "boxes2D"
+
+ def validate(self, val: dict) -> bool:
+ # Optional argument
+ if "class_labels" in val:
+ for k, v in list(val["class_labels"].items()):
+ if (not isinstance(k, numbers.Number)) or (not isinstance(v, str)):
+ raise TypeError(
+ "Class labels must be a dictionary of numbers to string"
+ )
+
+ boxes = val["box_data"]
+ if not isinstance(boxes, list):
+ raise TypeError("Boxes must be a list")
+
+ for box in boxes:
+ # Required arguments
+ error_str = "Each box must contain a position with: middle, width, and height or \
+ \nminX, maxX, minY, maxY."
+ if "position" not in box:
+ raise TypeError(error_str)
+ else:
+ valid = False
+ if (
+ "middle" in box["position"]
+ and len(box["position"]["middle"]) == 2
+ and has_num(box["position"], "width")
+ and has_num(box["position"], "height")
+ ):
+ valid = True
+ elif (
+ has_num(box["position"], "minX")
+ and has_num(box["position"], "maxX")
+ and has_num(box["position"], "minY")
+ and has_num(box["position"], "maxY")
+ ):
+ valid = True
+
+ if not valid:
+ raise TypeError(error_str)
+
+ # Optional arguments
+ if ("scores" in box) and not isinstance(box["scores"], dict):
+ raise TypeError("Box scores must be a dictionary")
+ elif "scores" in box:
+ for k, v in list(box["scores"].items()):
+ if not isinstance(k, str):
+ raise TypeError("A score key must be a string")
+ if not isinstance(v, numbers.Number):
+ raise TypeError("A score value must be a number")
+
+ if ("class_id" in box) and not isinstance(box["class_id"], int):
+ raise TypeError("A box's class_id must be an integer")
+
+ # Optional
+ if ("box_caption" in box) and not isinstance(box["box_caption"], str):
+ raise TypeError("A box's caption must be a string")
+ return True
+
+ def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
+ if isinstance(run_or_artifact, wandb.wandb_sdk.wandb_run.Run):
+ return super().to_json(run_or_artifact)
+ elif isinstance(run_or_artifact, wandb.Artifact):
+ # TODO (tim): I would like to log out a proper dictionary representing this object, but don't
+ # want to mess with the visualizations that are currently available in the UI. This really should output
+ # an object with a _type key. Will need to push this change to the UI first to ensure backwards compat
+ return self._val
+ else:
+ raise ValueError("to_json accepts wandb_run.Run or wandb.Artifact")
+
+ @classmethod
+ def from_json(
+ cls: Type["BoundingBoxes2D"], json_obj: dict, source_artifact: "Artifact"
+ ) -> "BoundingBoxes2D":
+ return cls({"box_data": json_obj}, "")
diff --git a/wandb/sdk/data_types/helper_types/classes.py b/wandb/sdk/data_types/helper_types/classes.py
new file mode 100644
index 0000000000000000000000000000000000000000..54a22054aa44ad58de5b46a72b1123d73bc8836d
--- /dev/null
+++ b/wandb/sdk/data_types/helper_types/classes.py
@@ -0,0 +1,159 @@
+import os
+from typing import TYPE_CHECKING, Any, Dict, Optional, Sequence, Type, Union
+
+from .. import _dtypes
+from ..base_types.media import Media
+
+if TYPE_CHECKING: # pragma: no cover
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ from ...wandb_run import Run as LocalRun
+
+
+class Classes(Media):
+ _log_type = "classes"
+
+ _class_set: Sequence[dict]
+
+ def __init__(self, class_set: Sequence[dict]) -> None:
+ """Classes is holds class metadata intended to be used in concert with other objects when visualizing artifacts.
+
+ Args:
+ class_set (list): list of dicts in the form of {"id":int|str, "name":str}
+ """
+ super().__init__()
+ for class_obj in class_set:
+ assert "id" in class_obj and "name" in class_obj
+ self._class_set = class_set
+
+ @classmethod
+ def from_json(
+ cls: Type["Classes"],
+ json_obj: dict,
+ source_artifact: Optional["Artifact"],
+ ) -> "Classes":
+ return cls(json_obj.get("class_set")) # type: ignore
+
+ def to_json(self, run_or_artifact: Optional[Union["LocalRun", "Artifact"]]) -> dict:
+ json_obj = {}
+ # This is a bit of a hack to allow _ClassesIdType to
+ # be able to operate fully without an artifact in play.
+ # In all other cases, artifact should be a true artifact.
+ if run_or_artifact is not None:
+ json_obj = super().to_json(run_or_artifact)
+ json_obj["_type"] = Classes._log_type
+ json_obj["class_set"] = self._class_set
+ return json_obj
+
+ def get_type(self) -> "_ClassesIdType":
+ return _ClassesIdType(self)
+
+ def __ne__(self, other: object) -> bool:
+ return not self.__eq__(other)
+
+ def __eq__(self, other: object) -> bool:
+ if isinstance(other, Classes):
+ return self._class_set == other._class_set
+ else:
+ return False
+
+
+class _ClassesIdType(_dtypes.Type):
+ name = "classesId"
+ legacy_names = ["wandb.Classes_id"]
+ types = [Classes]
+
+ def __init__(
+ self,
+ classes_obj: Optional[Classes] = None,
+ valid_ids: Optional["_dtypes.UnionType"] = None,
+ ):
+ if valid_ids is None:
+ valid_ids = _dtypes.UnionType()
+ elif isinstance(valid_ids, list):
+ valid_ids = _dtypes.UnionType(
+ [_dtypes.ConstType(item) for item in valid_ids]
+ )
+ elif isinstance(valid_ids, _dtypes.UnionType):
+ valid_ids = valid_ids
+ else:
+ raise TypeError("valid_ids must be None, list, or UnionType")
+
+ if classes_obj is None:
+ classes_obj = Classes(
+ [
+ {"id": _id.params["val"], "name": str(_id.params["val"])}
+ for _id in valid_ids.params["allowed_types"]
+ ]
+ )
+ elif not isinstance(classes_obj, Classes):
+ raise TypeError("valid_ids must be None, or instance of Classes")
+ else:
+ valid_ids = _dtypes.UnionType(
+ [
+ _dtypes.ConstType(class_obj["id"])
+ for class_obj in classes_obj._class_set
+ ]
+ )
+
+ self.wb_classes_obj_ref = classes_obj
+ self.params.update({"valid_ids": valid_ids})
+
+ def assign(self, py_obj: Optional[Any] = None) -> "_dtypes.Type":
+ return self.assign_type(_dtypes.ConstType(py_obj))
+
+ def assign_type(self, wb_type: "_dtypes.Type") -> "_dtypes.Type":
+ valid_ids = self.params["valid_ids"].assign_type(wb_type)
+ if not isinstance(valid_ids, _dtypes.InvalidType):
+ return self
+
+ return _dtypes.InvalidType()
+
+ @classmethod
+ def from_obj(cls, py_obj: Optional[Any] = None) -> "_dtypes.Type":
+ return cls(py_obj)
+
+ def to_json(self, artifact: Optional["Artifact"] = None) -> Dict[str, Any]:
+ cl_dict = super().to_json(artifact)
+ # TODO (tss): Refactor this block with the similar one in wandb.Image.
+ # This is a bit of a smell that the classes object does not follow
+ # the same file-pattern as other media types.
+ if artifact is not None:
+ class_name = os.path.join("media", "cls")
+ classes_entry = artifact.add(self.wb_classes_obj_ref, class_name)
+ cl_dict["params"]["classes_obj"] = {
+ "type": "classes-file",
+ "path": classes_entry.path,
+ "digest": classes_entry.digest, # is this needed really?
+ }
+ else:
+ cl_dict["params"]["classes_obj"] = self.wb_classes_obj_ref.to_json(artifact)
+ return cl_dict
+
+ @classmethod
+ def from_json(
+ cls,
+ json_dict: Dict[str, Any],
+ artifact: Optional["Artifact"] = None,
+ ) -> "_dtypes.Type":
+ classes_obj = None
+ if (
+ json_dict.get("params", {}).get("classes_obj", {}).get("type")
+ == "classes-file"
+ ):
+ if artifact is not None:
+ classes_obj = artifact.get(
+ json_dict.get("params", {}).get("classes_obj", {}).get("path")
+ )
+ assert classes_obj is None or isinstance(classes_obj, Classes)
+ else:
+ raise RuntimeError("Expected artifact to be non-null.")
+ else:
+ classes_obj = Classes.from_json(
+ json_dict["params"]["classes_obj"], artifact
+ )
+
+ return cls(classes_obj)
+
+
+_dtypes.TypeRegistry.add(_ClassesIdType)
diff --git a/wandb/sdk/data_types/helper_types/image_mask.py b/wandb/sdk/data_types/helper_types/image_mask.py
new file mode 100644
index 0000000000000000000000000000000000000000..0f56f5bc5b1a5096979d9646939358d62547cb47
--- /dev/null
+++ b/wandb/sdk/data_types/helper_types/image_mask.py
@@ -0,0 +1,233 @@
+import numbers
+import os
+from typing import TYPE_CHECKING, Optional, Type, Union
+
+import wandb
+from wandb import util
+from wandb.sdk.lib import runid
+
+from .._private import MEDIA_TMP
+from ..base_types.media import Media
+
+if TYPE_CHECKING: # pragma: no cover
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ from ...wandb_run import Run as LocalRun
+
+
+class ImageMask(Media):
+ """Format image masks or overlays for logging to W&B.
+
+ Arguments:
+ val: (dictionary)
+ One of these two keys to represent the image:
+ mask_data : (2D numpy array) The mask containing an integer class label
+ for each pixel in the image
+ path : (string) The path to a saved image file of the mask
+ class_labels : (dictionary of integers to strings, optional) A mapping of the
+ integer class labels in the mask to readable class names. These will default
+ to class_0, class_1, class_2, etc.
+
+ key: (string)
+ The readable name or id for this mask type (e.g. predictions, ground_truth)
+
+ Examples:
+ ### Logging a single masked image
+ <!--yeadoc-test:log-image-mask-->
+ ```python
+ import numpy as np
+ import wandb
+
+ wandb.init()
+ image = np.random.randint(low=0, high=256, size=(100, 100, 3), dtype=np.uint8)
+ predicted_mask = np.empty((100, 100), dtype=np.uint8)
+ ground_truth_mask = np.empty((100, 100), dtype=np.uint8)
+
+ predicted_mask[:50, :50] = 0
+ predicted_mask[50:, :50] = 1
+ predicted_mask[:50, 50:] = 2
+ predicted_mask[50:, 50:] = 3
+
+ ground_truth_mask[:25, :25] = 0
+ ground_truth_mask[25:, :25] = 1
+ ground_truth_mask[:25, 25:] = 2
+ ground_truth_mask[25:, 25:] = 3
+
+ class_labels = {0: "person", 1: "tree", 2: "car", 3: "road"}
+
+ masked_image = wandb.Image(
+ image,
+ masks={
+ "predictions": {"mask_data": predicted_mask, "class_labels": class_labels},
+ "ground_truth": {"mask_data": ground_truth_mask, "class_labels": class_labels},
+ },
+ )
+ wandb.log({"img_with_masks": masked_image})
+ ```
+
+ ### Log a masked image inside a Table
+ <!--yeadoc-test:log-image-mask-table-->
+ ```python
+ import numpy as np
+ import wandb
+
+ wandb.init()
+ image = np.random.randint(low=0, high=256, size=(100, 100, 3), dtype=np.uint8)
+ predicted_mask = np.empty((100, 100), dtype=np.uint8)
+ ground_truth_mask = np.empty((100, 100), dtype=np.uint8)
+
+ predicted_mask[:50, :50] = 0
+ predicted_mask[50:, :50] = 1
+ predicted_mask[:50, 50:] = 2
+ predicted_mask[50:, 50:] = 3
+
+ ground_truth_mask[:25, :25] = 0
+ ground_truth_mask[25:, :25] = 1
+ ground_truth_mask[:25, 25:] = 2
+ ground_truth_mask[25:, 25:] = 3
+
+ class_labels = {0: "person", 1: "tree", 2: "car", 3: "road"}
+
+ class_set = wandb.Classes(
+ [
+ {"name": "person", "id": 0},
+ {"name": "tree", "id": 1},
+ {"name": "car", "id": 2},
+ {"name": "road", "id": 3},
+ ]
+ )
+
+ masked_image = wandb.Image(
+ image,
+ masks={
+ "predictions": {"mask_data": predicted_mask, "class_labels": class_labels},
+ "ground_truth": {"mask_data": ground_truth_mask, "class_labels": class_labels},
+ },
+ classes=class_set,
+ )
+
+ table = wandb.Table(columns=["image"])
+ table.add_data(masked_image)
+ wandb.log({"random_field": table})
+ ```
+ """
+
+ _log_type = "mask"
+
+ def __init__(self, val: dict, key: str) -> None:
+ """Initialize an ImageMask object.
+
+ Arguments:
+ val: (dictionary) One of these two keys to represent the image:
+ mask_data : (2D numpy array) The mask containing an integer class label
+ for each pixel in the image
+ path : (string) The path to a saved image file of the mask
+ class_labels : (dictionary of integers to strings, optional) A mapping
+ of the integer class labels in the mask to readable class names.
+ These will default to class_0, class_1, class_2, etc.
+
+ key: (string)
+ The readable name or id for this mask type (e.g. predictions, ground_truth)
+ """
+ super().__init__()
+
+ if "path" in val:
+ self._set_file(val["path"])
+ else:
+ np = util.get_module("numpy", required="Image mask support requires numpy")
+ # Add default class mapping
+ if "class_labels" not in val:
+ classes = np.unique(val["mask_data"]).astype(np.int32).tolist()
+ class_labels = {c: "class_" + str(c) for c in classes}
+ val["class_labels"] = class_labels
+
+ self.validate(val)
+ self._val = val
+ self._key = key
+
+ ext = "." + self.type_name() + ".png"
+ tmp_path = os.path.join(MEDIA_TMP.name, runid.generate_id() + ext)
+
+ pil_image = util.get_module(
+ "PIL.Image",
+ required='wandb.Image needs the PIL package. To get it, run "pip install pillow".',
+ )
+ image = pil_image.fromarray(val["mask_data"].astype(np.int8), mode="L")
+
+ image.save(tmp_path, transparency=None)
+ self._set_file(tmp_path, is_tmp=True, extension=ext)
+
+ def bind_to_run(
+ self,
+ run: "LocalRun",
+ key: Union[int, str],
+ step: Union[int, str],
+ id_: Optional[Union[int, str]] = None,
+ ignore_copy_err: Optional[bool] = None,
+ ) -> None:
+ # bind_to_run key argument is the Image parent key
+ # the self._key value is the mask's sub key
+ super().bind_to_run(run, key, step, id_=id_, ignore_copy_err=ignore_copy_err)
+ if hasattr(self, "_val") and "class_labels" in self._val:
+ class_labels = self._val["class_labels"]
+
+ run._add_singleton(
+ "mask/class_labels",
+ str(key) + "_wandb_delimeter_" + self._key,
+ class_labels,
+ )
+
+ @classmethod
+ def get_media_subdir(cls: Type["ImageMask"]) -> str:
+ return os.path.join("media", "images", cls.type_name())
+
+ @classmethod
+ def from_json(
+ cls: Type["ImageMask"], json_obj: dict, source_artifact: "Artifact"
+ ) -> "ImageMask":
+ return cls(
+ {"path": source_artifact.get_entry(json_obj["path"]).download()},
+ key="",
+ )
+
+ def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
+ json_dict = super().to_json(run_or_artifact)
+
+ if isinstance(run_or_artifact, wandb.wandb_sdk.wandb_run.Run):
+ json_dict["_type"] = self.type_name()
+ return json_dict
+ elif isinstance(run_or_artifact, wandb.Artifact):
+ # Nothing special to add (used to add "digest", but no longer used.)
+ return json_dict
+ else:
+ raise ValueError("to_json accepts wandb_run.Run or wandb.Artifact")
+
+ @classmethod
+ def type_name(cls: Type["ImageMask"]) -> str:
+ return cls._log_type
+
+ def validate(self, val: dict) -> bool:
+ np = util.get_module("numpy", required="Image mask support requires numpy")
+ # 2D Make this work with all tensor(like) types
+ if "mask_data" not in val:
+ raise TypeError(
+ 'Missing key "mask_data": An image mask requires mask data: a 2D array representing the predictions'
+ )
+ else:
+ error_str = "mask_data must be a 2D array"
+ shape = val["mask_data"].shape
+ if len(shape) != 2:
+ raise TypeError(error_str)
+ if not (
+ (val["mask_data"] >= 0).all() and (val["mask_data"] <= 255).all()
+ ) and issubclass(val["mask_data"].dtype.type, np.integer):
+ raise TypeError("Mask data must be integers between 0 and 255")
+
+ # Optional argument
+ if "class_labels" in val:
+ for k, v in list(val["class_labels"].items()):
+ if (not isinstance(k, numbers.Number)) or (not isinstance(v, str)):
+ raise TypeError(
+ "Class labels must be a dictionary of numbers to strings"
+ )
+ return True
diff --git a/wandb/sdk/data_types/histogram.py b/wandb/sdk/data_types/histogram.py
new file mode 100644
index 0000000000000000000000000000000000000000..9c980c32a54736a4e97760e4e009d5e9c7d620e8
--- /dev/null
+++ b/wandb/sdk/data_types/histogram.py
@@ -0,0 +1,96 @@
+import sys
+from typing import TYPE_CHECKING, Optional, Sequence, Tuple, Union
+
+from wandb import util
+
+from .base_types.wb_value import WBValue
+
+if TYPE_CHECKING: # pragma: no cover
+ import numpy as np
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ from ..wandb_run import Run as LocalRun
+
+ NumpyHistogram = Tuple[np.ndarray, np.ndarray]
+
+
+class Histogram(WBValue):
+ """wandb class for histograms.
+
+ This object works just like numpy's histogram function
+ https://docs.scipy.org/doc/numpy/reference/generated/numpy.histogram.html
+
+ Examples:
+ Generate histogram from a sequence
+ ```python
+ wandb.Histogram([1, 2, 3])
+ ```
+
+ Efficiently initialize from np.histogram.
+ ```python
+ hist = np.histogram(data)
+ wandb.Histogram(np_histogram=hist)
+ ```
+
+ Arguments:
+ sequence: (array_like) input data for histogram
+ np_histogram: (numpy histogram) alternative input of a precomputed histogram
+ num_bins: (int) Number of bins for the histogram. The default number of bins
+ is 64. The maximum number of bins is 512
+
+ Attributes:
+ bins: ([float]) edges of bins
+ histogram: ([int]) number of elements falling in each bin
+ """
+
+ MAX_LENGTH: int = 512
+ _log_type = "histogram"
+
+ def __init__(
+ self,
+ sequence: Optional[Sequence] = None,
+ np_histogram: Optional["NumpyHistogram"] = None,
+ num_bins: int = 64,
+ ) -> None:
+ if np_histogram:
+ if len(np_histogram) == 2:
+ self.histogram = (
+ np_histogram[0].tolist()
+ if hasattr(np_histogram[0], "tolist")
+ else np_histogram[0]
+ )
+ self.bins = (
+ np_histogram[1].tolist()
+ if hasattr(np_histogram[1], "tolist")
+ else np_histogram[1]
+ )
+ else:
+ raise ValueError(
+ "Expected np_histogram to be a tuple of (values, bin_edges) or sequence to be specified"
+ )
+ else:
+ np = util.get_module(
+ "numpy", required="Auto creation of histograms requires numpy"
+ )
+
+ self.histogram, self.bins = np.histogram(sequence, bins=num_bins)
+ self.histogram = self.histogram.tolist()
+ self.bins = self.bins.tolist()
+ if len(self.histogram) > self.MAX_LENGTH:
+ raise ValueError(
+ "The maximum length of a histogram is %i" % self.MAX_LENGTH
+ )
+ if len(self.histogram) + 1 != len(self.bins):
+ raise ValueError("len(bins) must be len(histogram) + 1")
+
+ def to_json(self, run: Optional[Union["LocalRun", "Artifact"]] = None) -> dict:
+ return {"_type": self._log_type, "values": self.histogram, "bins": self.bins}
+
+ def __sizeof__(self) -> int:
+ """Estimated size in bytes.
+
+ Currently the factor of 1.7 is used to account for the JSON encoding. We use
+ this in tb_watcher.TBHistory.
+ """
+ return int((sys.getsizeof(self.histogram) + sys.getsizeof(self.bins)) * 1.7)
diff --git a/wandb/sdk/data_types/html.py b/wandb/sdk/data_types/html.py
new file mode 100644
index 0000000000000000000000000000000000000000..62159200b508ca5e703b3d0e08c0cff49d6c1f98
--- /dev/null
+++ b/wandb/sdk/data_types/html.py
@@ -0,0 +1,115 @@
+import os
+from typing import TYPE_CHECKING, Sequence, Type, Union
+
+from wandb.sdk.lib import filesystem, runid
+
+from . import _dtypes
+from ._private import MEDIA_TMP
+from .base_types.media import BatchableMedia
+
+if TYPE_CHECKING: # pragma: no cover
+ from typing import TextIO
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ from ..wandb_run import Run as LocalRun
+
+
+class Html(BatchableMedia):
+ """Wandb class for arbitrary html.
+
+ Arguments:
+ data: (string or io object) HTML to display in wandb
+ inject: (boolean) Add a stylesheet to the HTML object. If set
+ to False the HTML will pass through unchanged.
+ """
+
+ _log_type = "html-file"
+
+ def __init__(self, data: Union[str, "TextIO"], inject: bool = True) -> None:
+ super().__init__()
+ data_is_path = isinstance(data, str) and os.path.exists(data)
+ data_path = ""
+ if data_is_path:
+ assert isinstance(data, str)
+ data_path = data
+ with open(data_path) as file:
+ self.html = file.read()
+ elif isinstance(data, str):
+ self.html = data
+ elif hasattr(data, "read"):
+ if hasattr(data, "seek"):
+ data.seek(0)
+ self.html = data.read()
+ else:
+ raise ValueError("data must be a string or an io object")
+
+ if inject:
+ self.inject_head()
+
+ if inject or not data_is_path:
+ tmp_path = os.path.join(MEDIA_TMP.name, runid.generate_id() + ".html")
+ with open(tmp_path, "w", encoding="utf-8") as out:
+ out.write(self.html)
+
+ self._set_file(tmp_path, is_tmp=True)
+ else:
+ self._set_file(data_path, is_tmp=False)
+
+ def inject_head(self) -> None:
+ join = ""
+ if "<head>" in self.html:
+ parts = self.html.split("<head>", 1)
+ parts[0] = parts[0] + "<head>"
+ elif "<html>" in self.html:
+ parts = self.html.split("<html>", 1)
+ parts[0] = parts[0] + "<html><head>"
+ parts[1] = "</head>" + parts[1]
+ else:
+ parts = ["", self.html]
+ parts.insert(
+ 1,
+ '<base target="_blank"><link rel="stylesheet" type="text/css" href="https://app.wandb.ai/normalize.css" />',
+ )
+ self.html = join.join(parts).strip()
+
+ @classmethod
+ def get_media_subdir(cls: Type["Html"]) -> str:
+ return os.path.join("media", "html")
+
+ def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
+ json_dict = super().to_json(run_or_artifact)
+ json_dict["_type"] = self._log_type
+ return json_dict
+
+ @classmethod
+ def from_json(
+ cls: Type["Html"], json_obj: dict, source_artifact: "Artifact"
+ ) -> "Html":
+ return cls(source_artifact.get_entry(json_obj["path"]).download(), inject=False)
+
+ @classmethod
+ def seq_to_json(
+ cls: Type["Html"],
+ seq: Sequence["BatchableMedia"],
+ run: "LocalRun",
+ key: str,
+ step: Union[int, str],
+ ) -> dict:
+ base_path = os.path.join(run.dir, cls.get_media_subdir())
+ filesystem.mkdir_exists_ok(base_path)
+
+ meta = {
+ "_type": "html",
+ "count": len(seq),
+ "html": [h.to_json(run) for h in seq],
+ }
+ return meta
+
+
+class _HtmlFileType(_dtypes.Type):
+ name = "html-file"
+ types = [Html]
+
+
+_dtypes.TypeRegistry.add(_HtmlFileType)
diff --git a/wandb/sdk/data_types/image.py b/wandb/sdk/data_types/image.py
new file mode 100644
index 0000000000000000000000000000000000000000..62f3939d29e14d621f25e03078239094842e2356
--- /dev/null
+++ b/wandb/sdk/data_types/image.py
@@ -0,0 +1,687 @@
+import hashlib
+import logging
+import os
+from io import BytesIO
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Type, Union, cast
+from urllib import parse
+
+import wandb
+from wandb import util
+from wandb.sdk.lib import hashutil, runid
+from wandb.sdk.lib.paths import LogicalPath
+
+from ._private import MEDIA_TMP
+from .base_types.media import BatchableMedia, Media
+from .helper_types.bounding_boxes_2d import BoundingBoxes2D
+from .helper_types.classes import Classes
+from .helper_types.image_mask import ImageMask
+
+if TYPE_CHECKING: # pragma: no cover
+ import matplotlib # type: ignore
+ import numpy as np
+ import torch # type: ignore
+ from PIL.Image import Image as PILImage
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ from ..wandb_run import Run as LocalRun
+
+ ImageDataType = Union[
+ "matplotlib.artist.Artist", "PILImage", "TorchTensorType", "np.ndarray"
+ ]
+ ImageDataOrPathType = Union[str, "Image", ImageDataType]
+ TorchTensorType = Union["torch.Tensor", "torch.Variable"]
+
+
+def _server_accepts_image_filenames() -> bool:
+ if util._is_offline():
+ return True
+
+ # Newer versions of wandb accept large image filenames arrays
+ # but older versions would have issues with this.
+ max_cli_version = util._get_max_cli_version()
+ if max_cli_version is None:
+ return False
+ from pkg_resources import parse_version
+
+ accepts_image_filenames: bool = parse_version("0.12.10") <= parse_version(
+ max_cli_version
+ )
+ return accepts_image_filenames
+
+
+def _server_accepts_artifact_path() -> bool:
+ from pkg_resources import parse_version
+
+ target_version = "0.12.14"
+ max_cli_version = util._get_max_cli_version() if not util._is_offline() else None
+ accepts_artifact_path: bool = max_cli_version is not None and parse_version(
+ target_version
+ ) <= parse_version(max_cli_version)
+ return accepts_artifact_path
+
+
+class Image(BatchableMedia):
+ """Format images for logging to W&B.
+
+ Arguments:
+ data_or_path: (numpy array, string, io) Accepts numpy array of
+ image data, or a PIL image. The class attempts to infer
+ the data format and converts it.
+ mode: (string) The PIL mode for an image. Most common are "L", "RGB",
+ "RGBA". Full explanation at https://pillow.readthedocs.io/en/stable/handbook/concepts.html#modes
+ caption: (string) Label for display of image.
+
+ Note : When logging a `torch.Tensor` as a `wandb.Image`, images are normalized. If you do not want to normalize your images, please convert your tensors to a PIL Image.
+
+ Examples:
+ ### Create a wandb.Image from a numpy array
+ <!--yeadoc-test:log-image-numpy-->
+ ```python
+ import numpy as np
+ import wandb
+
+ with wandb.init() as run:
+ examples = []
+ for i in range(3):
+ pixels = np.random.randint(low=0, high=256, size=(100, 100, 3))
+ image = wandb.Image(pixels, caption=f"random field {i}")
+ examples.append(image)
+ run.log({"examples": examples})
+ ```
+
+ ### Create a wandb.Image from a PILImage
+ <!--yeadoc-test:log-image-pillow-->
+ ```python
+ import numpy as np
+ from PIL import Image as PILImage
+ import wandb
+
+ with wandb.init() as run:
+ examples = []
+ for i in range(3):
+ pixels = np.random.randint(low=0, high=256, size=(100, 100, 3), dtype=np.uint8)
+ pil_image = PILImage.fromarray(pixels, mode="RGB")
+ image = wandb.Image(pil_image, caption=f"random field {i}")
+ examples.append(image)
+ run.log({"examples": examples})
+ ```
+
+ ### log .jpg rather than .png (default)
+ <!--yeadoc-test:log-image-format-->
+ ```python
+ import numpy as np
+ import wandb
+
+ with wandb.init() as run:
+ examples = []
+ for i in range(3):
+ pixels = np.random.randint(low=0, high=256, size=(100, 100, 3))
+ image = wandb.Image(pixels, caption=f"random field {i}", file_type="jpg")
+ examples.append(image)
+ run.log({"examples": examples})
+ ```
+ """
+
+ MAX_ITEMS = 108
+
+ # PIL limit
+ MAX_DIMENSION = 65500
+
+ _log_type = "image-file"
+
+ format: Optional[str]
+ _grouping: Optional[int]
+ _caption: Optional[str]
+ _width: Optional[int]
+ _height: Optional[int]
+ _image: Optional["PILImage"]
+ _classes: Optional["Classes"]
+ _boxes: Optional[Dict[str, "BoundingBoxes2D"]]
+ _masks: Optional[Dict[str, "ImageMask"]]
+ _file_type: Optional[str]
+
+ def __init__(
+ self,
+ data_or_path: "ImageDataOrPathType",
+ mode: Optional[str] = None,
+ caption: Optional[str] = None,
+ grouping: Optional[int] = None,
+ classes: Optional[Union["Classes", Sequence[dict]]] = None,
+ boxes: Optional[Union[Dict[str, "BoundingBoxes2D"], Dict[str, dict]]] = None,
+ masks: Optional[Union[Dict[str, "ImageMask"], Dict[str, dict]]] = None,
+ file_type: Optional[str] = None,
+ ) -> None:
+ super().__init__()
+ # TODO: We should remove grouping, it's a terrible name and I don't
+ # think anyone uses it.
+
+ self._grouping = None
+ self._caption = None
+ self._width = None
+ self._height = None
+ self._image = None
+ self._classes = None
+ self._boxes = None
+ self._masks = None
+ self._file_type = None
+
+ # Allows the user to pass an Image object as the first parameter and have a perfect copy,
+ # only overriding additional metdata passed in. If this pattern is compelling, we can generalize.
+ if isinstance(data_or_path, Image):
+ self._initialize_from_wbimage(data_or_path)
+ elif isinstance(data_or_path, str):
+ if self.path_is_reference(data_or_path):
+ self._initialize_from_reference(data_or_path)
+ else:
+ self._initialize_from_path(data_or_path)
+ else:
+ self._initialize_from_data(data_or_path, mode, file_type)
+ self._set_initialization_meta(
+ grouping, caption, classes, boxes, masks, file_type
+ )
+
+ def _set_initialization_meta(
+ self,
+ grouping: Optional[int] = None,
+ caption: Optional[str] = None,
+ classes: Optional[Union["Classes", Sequence[dict]]] = None,
+ boxes: Optional[Union[Dict[str, "BoundingBoxes2D"], Dict[str, dict]]] = None,
+ masks: Optional[Union[Dict[str, "ImageMask"], Dict[str, dict]]] = None,
+ file_type: Optional[str] = None,
+ ) -> None:
+ if grouping is not None:
+ self._grouping = grouping
+
+ if caption is not None:
+ self._caption = caption
+
+ total_classes = {}
+
+ if boxes:
+ if not isinstance(boxes, dict):
+ raise ValueError('Images "boxes" argument must be a dictionary')
+ boxes_final: Dict[str, BoundingBoxes2D] = {}
+ for key in boxes:
+ box_item = boxes[key]
+ if isinstance(box_item, BoundingBoxes2D):
+ boxes_final[key] = box_item
+ elif isinstance(box_item, dict):
+ # TODO: Consider injecting top-level classes if user-provided is empty
+ boxes_final[key] = BoundingBoxes2D(box_item, key)
+ total_classes.update(boxes_final[key]._class_labels)
+ self._boxes = boxes_final
+
+ if masks:
+ if not isinstance(masks, dict):
+ raise ValueError('Images "masks" argument must be a dictionary')
+ masks_final: Dict[str, ImageMask] = {}
+ for key in masks:
+ mask_item = masks[key]
+ if isinstance(mask_item, ImageMask):
+ masks_final[key] = mask_item
+ elif isinstance(mask_item, dict):
+ # TODO: Consider injecting top-level classes if user-provided is empty
+ masks_final[key] = ImageMask(mask_item, key)
+ if hasattr(masks_final[key], "_val"):
+ total_classes.update(masks_final[key]._val["class_labels"])
+ self._masks = masks_final
+
+ if classes is not None:
+ if isinstance(classes, Classes):
+ total_classes.update(
+ {val["id"]: val["name"] for val in classes._class_set}
+ )
+ else:
+ total_classes.update({val["id"]: val["name"] for val in classes})
+
+ if len(total_classes.keys()) > 0:
+ self._classes = Classes(
+ [
+ {"id": key, "name": total_classes[key]}
+ for key in total_classes.keys()
+ ]
+ )
+ if self.image is not None:
+ self._width, self._height = self.image.size
+ self._free_ram()
+
+ def _initialize_from_wbimage(self, wbimage: "Image") -> None:
+ self._grouping = wbimage._grouping
+ self._caption = wbimage._caption
+ self._width = wbimage._width
+ self._height = wbimage._height
+ self._image = wbimage._image
+ self._classes = wbimage._classes
+ self._path = wbimage._path
+ self._is_tmp = wbimage._is_tmp
+ self._extension = wbimage._extension
+ self._sha256 = wbimage._sha256
+ self._size = wbimage._size
+ self.format = wbimage.format
+ self._file_type = wbimage._file_type
+ self._artifact_source = wbimage._artifact_source
+ self._artifact_target = wbimage._artifact_target
+
+ # We do not want to implicitly copy boxes or masks, just the image-related data.
+ # self._boxes = wbimage._boxes
+ # self._masks = wbimage._masks
+
+ def _initialize_from_path(self, path: str) -> None:
+ pil_image = util.get_module(
+ "PIL.Image",
+ required='wandb.Image needs the PIL package. To get it, run "pip install pillow".',
+ )
+ self._set_file(path, is_tmp=False)
+ self._image = pil_image.open(path)
+ assert self._image is not None
+ self._image.load()
+ ext = os.path.splitext(path)[1][1:]
+ self.format = ext
+
+ def _initialize_from_reference(self, path: str) -> None:
+ self._path = path
+ self._is_tmp = False
+ self._sha256 = hashlib.sha256(path.encode("utf-8")).hexdigest()
+ path = parse.urlparse(path).path
+ ext = path.split("/")[-1].split(".")[-1]
+ self.format = ext
+
+ def _initialize_from_data(
+ self,
+ data: "ImageDataType",
+ mode: Optional[str] = None,
+ file_type: Optional[str] = None,
+ ) -> None:
+ pil_image = util.get_module(
+ "PIL.Image",
+ required='wandb.Image needs the PIL package. To get it, run "pip install pillow".',
+ )
+ if util.is_matplotlib_typename(util.get_full_typename(data)):
+ buf = BytesIO()
+ util.ensure_matplotlib_figure(data).savefig(buf, format="png")
+ self._image = pil_image.open(buf, formats=["PNG"])
+ elif isinstance(data, pil_image.Image):
+ self._image = data
+ elif util.is_pytorch_tensor_typename(util.get_full_typename(data)):
+ vis_util = util.get_module(
+ "torchvision.utils", "torchvision is required to render images"
+ )
+ if hasattr(data, "requires_grad") and data.requires_grad:
+ data = data.detach() # type: ignore
+ if hasattr(data, "dtype") and str(data.dtype) == "torch.uint8":
+ data = data.to(float)
+ data = vis_util.make_grid(data, normalize=True)
+ self._image = pil_image.fromarray(
+ data.mul(255).clamp(0, 255).byte().permute(1, 2, 0).cpu().numpy()
+ )
+ else:
+ if hasattr(data, "numpy"): # TF data eager tensors
+ data = data.numpy()
+ if data.ndim > 2:
+ data = data.squeeze() # get rid of trivial dimensions as a convenience
+ self._image = pil_image.fromarray(
+ self.to_uint8(data), mode=mode or self.guess_mode(data)
+ )
+ accepted_formats = ["png", "jpg", "jpeg", "bmp"]
+ if file_type is None:
+ self.format = "png"
+ else:
+ self.format = file_type
+ assert (
+ self.format in accepted_formats
+ ), f"file_type must be one of {accepted_formats}"
+ tmp_path = os.path.join(MEDIA_TMP.name, runid.generate_id() + "." + self.format)
+ assert self._image is not None
+ self._image.save(tmp_path, transparency=None)
+ self._set_file(tmp_path, is_tmp=True)
+
+ @classmethod
+ def from_json(
+ cls: Type["Image"], json_obj: dict, source_artifact: "Artifact"
+ ) -> "Image":
+ classes: Optional[Classes] = None
+ if json_obj.get("classes") is not None:
+ value = source_artifact.get(json_obj["classes"]["path"])
+ assert isinstance(value, (type(None), Classes))
+ classes = value
+
+ masks = json_obj.get("masks")
+ _masks: Optional[Dict[str, ImageMask]] = None
+ if masks:
+ _masks = {}
+ for key in masks:
+ _masks[key] = ImageMask.from_json(masks[key], source_artifact)
+ _masks[key]._set_artifact_source(source_artifact)
+ _masks[key]._key = key
+
+ boxes = json_obj.get("boxes")
+ _boxes: Optional[Dict[str, BoundingBoxes2D]] = None
+ if boxes:
+ _boxes = {}
+ for key in boxes:
+ _boxes[key] = BoundingBoxes2D.from_json(boxes[key], source_artifact)
+ _boxes[key]._key = key
+
+ return cls(
+ source_artifact.get_entry(json_obj["path"]).download(),
+ caption=json_obj.get("caption"),
+ grouping=json_obj.get("grouping"),
+ classes=classes,
+ boxes=_boxes,
+ masks=_masks,
+ )
+
+ @classmethod
+ def get_media_subdir(cls: Type["Image"]) -> str:
+ return os.path.join("media", "images")
+
+ def bind_to_run(
+ self,
+ run: "LocalRun",
+ key: Union[int, str],
+ step: Union[int, str],
+ id_: Optional[Union[int, str]] = None,
+ ignore_copy_err: Optional[bool] = None,
+ ) -> None:
+ # For Images, we are going to avoid copying the image file to the run.
+ # We should make this common functionality for all media types, but that
+ # requires a broader UI refactor. This model can easily be moved to the
+ # higher level Media class, but that will require every UI surface area
+ # that depends on the `path` to be able to instead consume
+ # `artifact_path`. I (Tim) think the media panel makes up most of this
+ # space, but there are also custom charts, and maybe others. Let's
+ # commit to getting all that fixed up before moving this to the top
+ # level Media class.
+ if self.path_is_reference(self._path):
+ raise ValueError(
+ "Image media created by a reference to external storage cannot currently be added to a run"
+ )
+
+ if (
+ not _server_accepts_artifact_path()
+ or self._get_artifact_entry_ref_url() is None
+ ):
+ super().bind_to_run(run, key, step, id_, ignore_copy_err=ignore_copy_err)
+ if self._boxes is not None:
+ for i, k in enumerate(self._boxes):
+ id_ = f"{id_}{i}" if id_ is not None else None
+ self._boxes[k].bind_to_run(
+ run, key, step, id_, ignore_copy_err=ignore_copy_err
+ )
+
+ if self._masks is not None:
+ for i, k in enumerate(self._masks):
+ id_ = f"{id_}{i}" if id_ is not None else None
+ self._masks[k].bind_to_run(
+ run, key, step, id_, ignore_copy_err=ignore_copy_err
+ )
+
+ def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
+ json_dict = super().to_json(run_or_artifact)
+ json_dict["_type"] = Image._log_type
+ json_dict["format"] = self.format
+
+ if self._width is not None:
+ json_dict["width"] = self._width
+ if self._height is not None:
+ json_dict["height"] = self._height
+ if self._grouping:
+ json_dict["grouping"] = self._grouping
+ if self._caption:
+ json_dict["caption"] = self._caption
+
+ if isinstance(run_or_artifact, wandb.Artifact):
+ artifact = run_or_artifact
+ if (
+ self._masks is not None or self._boxes is not None
+ ) and self._classes is None:
+ raise ValueError(
+ "classes must be passed to wandb.Image which have masks or bounding boxes when adding to artifacts"
+ )
+
+ if self._classes is not None:
+ class_id = hashutil._md5(
+ str(self._classes._class_set).encode("utf-8")
+ ).hexdigest()
+ class_name = os.path.join(
+ "media",
+ "classes",
+ class_id + "_cls",
+ )
+ classes_entry = artifact.add(self._classes, class_name)
+ json_dict["classes"] = {
+ "type": "classes-file",
+ "path": classes_entry.path,
+ "digest": classes_entry.digest,
+ }
+
+ elif not isinstance(run_or_artifact, wandb.wandb_sdk.wandb_run.Run):
+ raise ValueError("to_json accepts wandb_run.Run or wandb_artifact.Artifact")
+
+ if self._boxes:
+ json_dict["boxes"] = {
+ k: box.to_json(run_or_artifact) for (k, box) in self._boxes.items()
+ }
+ if self._masks:
+ json_dict["masks"] = {
+ k: mask.to_json(run_or_artifact) for (k, mask) in self._masks.items()
+ }
+ return json_dict
+
+ def guess_mode(self, data: "np.ndarray") -> str:
+ """Guess what type of image the np.array is representing."""
+ # TODO: do we want to support dimensions being at the beginning of the array?
+ if data.ndim == 2:
+ return "L"
+ elif data.shape[-1] == 3:
+ return "RGB"
+ elif data.shape[-1] == 4:
+ return "RGBA"
+ else:
+ raise ValueError(
+ "Un-supported shape for image conversion %s" % list(data.shape)
+ )
+
+ @classmethod
+ def to_uint8(cls, data: "np.ndarray") -> "np.ndarray":
+ """Convert image data to uint8.
+
+ Convert floating point image on the range [0,1] and integer images on the range
+ [0,255] to uint8, clipping if necessary.
+ """
+ np = util.get_module(
+ "numpy",
+ required="wandb.Image requires numpy if not supplying PIL Images: pip install numpy",
+ )
+
+ # I think it's better to check the image range vs the data type, since many
+ # image libraries will return floats between 0 and 255
+
+ # some images have range -1...1 or 0-1
+ dmin = np.min(data)
+ if dmin < 0:
+ data = (data - np.min(data)) / np.ptp(data)
+ if np.max(data) <= 1.0:
+ data = (data * 255).astype(np.int32)
+
+ # assert issubclass(data.dtype.type, np.integer), 'Illegal image format.'
+ return data.clip(0, 255).astype(np.uint8)
+
+ @classmethod
+ def seq_to_json(
+ cls: Type["Image"],
+ seq: Sequence["BatchableMedia"],
+ run: "LocalRun",
+ key: str,
+ step: Union[int, str],
+ ) -> dict:
+ """Combine a list of images into a meta dictionary object describing the child images."""
+ if TYPE_CHECKING:
+ seq = cast(Sequence["Image"], seq)
+
+ jsons = [obj.to_json(run) for obj in seq]
+
+ media_dir = cls.get_media_subdir()
+
+ for obj in jsons:
+ expected = LogicalPath(media_dir)
+ if "path" in obj and not obj["path"].startswith(expected):
+ raise ValueError(
+ "Files in an array of Image's must be in the {} directory, not {}".format(
+ cls.get_media_subdir(), obj["path"]
+ )
+ )
+
+ num_images_to_log = len(seq)
+ width, height = seq[0].image.size # type: ignore
+ format = jsons[0]["format"]
+
+ def size_equals_image(image: "Image") -> bool:
+ img_width, img_height = image.image.size # type: ignore
+ return img_width == width and img_height == height
+
+ sizes_match = all(size_equals_image(img) for img in seq)
+ if not sizes_match:
+ logging.warning(
+ "Images sizes do not match. This will causes images to be display incorrectly in the UI."
+ )
+
+ meta = {
+ "_type": "images/separated",
+ "width": width,
+ "height": height,
+ "format": format,
+ "count": num_images_to_log,
+ }
+ if _server_accepts_image_filenames():
+ meta["filenames"] = [
+ obj.get("path", obj.get("artifact_path")) for obj in jsons
+ ]
+ else:
+ wandb.termwarn(
+ "Unable to log image array filenames. In some cases, this can prevent images from being "
+ "viewed in the UI. Please upgrade your wandb server",
+ repeat=False,
+ )
+
+ captions = Image.all_captions(seq)
+
+ if captions:
+ meta["captions"] = captions
+
+ all_masks = Image.all_masks(seq, run, key, step)
+
+ if all_masks:
+ meta["all_masks"] = all_masks
+
+ all_boxes = Image.all_boxes(seq, run, key, step)
+
+ if all_boxes:
+ meta["all_boxes"] = all_boxes
+
+ return meta
+
+ @classmethod
+ def all_masks(
+ cls: Type["Image"],
+ images: Sequence["Image"],
+ run: "LocalRun",
+ run_key: str,
+ step: Union[int, str],
+ ) -> Union[List[Optional[dict]], bool]:
+ all_mask_groups: List[Optional[dict]] = []
+ for image in images:
+ if image._masks:
+ mask_group = {}
+ for k in image._masks:
+ mask = image._masks[k]
+ mask_group[k] = mask.to_json(run)
+ all_mask_groups.append(mask_group)
+ else:
+ all_mask_groups.append(None)
+ if all_mask_groups and not all(x is None for x in all_mask_groups):
+ return all_mask_groups
+ else:
+ return False
+
+ @classmethod
+ def all_boxes(
+ cls: Type["Image"],
+ images: Sequence["Image"],
+ run: "LocalRun",
+ run_key: str,
+ step: Union[int, str],
+ ) -> Union[List[Optional[dict]], bool]:
+ all_box_groups: List[Optional[dict]] = []
+ for image in images:
+ if image._boxes:
+ box_group = {}
+ for k in image._boxes:
+ box = image._boxes[k]
+ box_group[k] = box.to_json(run)
+ all_box_groups.append(box_group)
+ else:
+ all_box_groups.append(None)
+ if all_box_groups and not all(x is None for x in all_box_groups):
+ return all_box_groups
+ else:
+ return False
+
+ @classmethod
+ def all_captions(
+ cls: Type["Image"], images: Sequence["Media"]
+ ) -> Union[bool, Sequence[Optional[str]]]:
+ return cls.captions(images)
+
+ def __ne__(self, other: object) -> bool:
+ return not self.__eq__(other)
+
+ def __eq__(self, other: object) -> bool:
+ if not isinstance(other, Image):
+ return False
+ else:
+ if self.path_is_reference(self._path) and self.path_is_reference(
+ other._path
+ ):
+ return self._path == other._path
+ self_image = self.image
+ other_image = other.image
+ if self_image is not None:
+ self_image = list(self_image.getdata()) # type: ignore
+ if other_image is not None:
+ other_image = list(other_image.getdata()) # type: ignore
+
+ return (
+ self._grouping == other._grouping
+ and self._caption == other._caption
+ and self._width == other._width
+ and self._height == other._height
+ and self_image == other_image
+ and self._classes == other._classes
+ )
+
+ def to_data_array(self) -> List[Any]:
+ res = []
+ if self.image is not None:
+ data = list(self.image.getdata())
+ for i in range(self.image.height):
+ res.append(data[i * self.image.width : (i + 1) * self.image.width])
+ self._free_ram()
+ return res
+
+ def _free_ram(self) -> None:
+ if self._path is not None:
+ self._image = None
+
+ @property
+ def image(self) -> Optional["PILImage"]:
+ if self._image is None:
+ if self._path is not None and not self.path_is_reference(self._path):
+ pil_image = util.get_module(
+ "PIL.Image",
+ required='wandb.Image needs the PIL package. To get it, run "pip install pillow".',
+ )
+ self._image = pil_image.open(self._path)
+ self._image.load()
+ return self._image
diff --git a/wandb/sdk/data_types/molecule.py b/wandb/sdk/data_types/molecule.py
new file mode 100644
index 0000000000000000000000000000000000000000..6931ee3aaeeace5ffd504e881e3db124d2f32f28
--- /dev/null
+++ b/wandb/sdk/data_types/molecule.py
@@ -0,0 +1,241 @@
+import io
+import os
+import pathlib
+from typing import TYPE_CHECKING, Optional, Sequence, Type, Union
+
+from wandb import util
+from wandb.sdk.lib import runid
+from wandb.sdk.lib.paths import LogicalPath
+
+from ._private import MEDIA_TMP
+from .base_types.media import BatchableMedia, Media
+
+if TYPE_CHECKING: # pragma: no cover
+ from typing import TextIO
+
+ import rdkit.Chem # type: ignore
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ from ..wandb_run import Run as LocalRun
+
+ RDKitDataType = Union[str, "rdkit.Chem.rdchem.Mol"]
+
+
+class Molecule(BatchableMedia):
+ """Wandb class for 3D Molecular data.
+
+ Arguments:
+ data_or_path: (string, io)
+ Molecule can be initialized from a file name or an io object.
+ caption: (string)
+ Caption associated with the molecule for display.
+ """
+
+ SUPPORTED_TYPES = {
+ "pdb",
+ "pqr",
+ "mmcif",
+ "mcif",
+ "cif",
+ "sdf",
+ "sd",
+ "gro",
+ "mol2",
+ "mmtf",
+ }
+ SUPPORTED_RDKIT_TYPES = {"mol", "sdf"}
+ _log_type = "molecule-file"
+
+ def __init__(
+ self,
+ data_or_path: Union[str, "TextIO"],
+ caption: Optional[str] = None,
+ **kwargs: str,
+ ) -> None:
+ super().__init__()
+
+ self._caption = caption
+
+ if hasattr(data_or_path, "name"):
+ # if the file has a path, we just detect the type and copy it from there
+ data_or_path = data_or_path.name
+
+ if hasattr(data_or_path, "read"):
+ if hasattr(data_or_path, "seek"):
+ data_or_path.seek(0)
+ molecule = data_or_path.read()
+
+ extension = kwargs.pop("file_type", None)
+ if extension is None:
+ raise ValueError(
+ "Must pass file_type keyword argument when using io objects."
+ )
+ if extension not in Molecule.SUPPORTED_TYPES:
+ raise ValueError(
+ "Molecule 3D only supports files of the type: "
+ + ", ".join(Molecule.SUPPORTED_TYPES)
+ )
+
+ tmp_path = os.path.join(
+ MEDIA_TMP.name, runid.generate_id() + "." + extension
+ )
+ with open(tmp_path, "w") as f:
+ f.write(molecule)
+
+ self._set_file(tmp_path, is_tmp=True)
+ elif isinstance(data_or_path, str):
+ extension = os.path.splitext(data_or_path)[1][1:]
+ if extension not in Molecule.SUPPORTED_TYPES:
+ raise ValueError(
+ "Molecule only supports files of the type: "
+ + ", ".join(Molecule.SUPPORTED_TYPES)
+ )
+
+ self._set_file(data_or_path, is_tmp=False)
+ else:
+ raise ValueError("Data must be file name or a file object")
+
+ @classmethod
+ def from_rdkit(
+ cls,
+ data_or_path: "RDKitDataType",
+ caption: Optional[str] = None,
+ convert_to_3d_and_optimize: bool = True,
+ mmff_optimize_molecule_max_iterations: int = 200,
+ ) -> "Molecule":
+ """Convert RDKit-supported file/object types to wandb.Molecule.
+
+ Arguments:
+ data_or_path: (string, rdkit.Chem.rdchem.Mol)
+ Molecule can be initialized from a file name or an rdkit.Chem.rdchem.Mol object.
+ caption: (string)
+ Caption associated with the molecule for display.
+ convert_to_3d_and_optimize: (bool)
+ Convert to rdkit.Chem.rdchem.Mol with 3D coordinates.
+ This is an expensive operation that may take a long time for complicated molecules.
+ mmff_optimize_molecule_max_iterations: (int)
+ Number of iterations to use in rdkit.Chem.AllChem.MMFFOptimizeMolecule
+ """
+ rdkit_chem = util.get_module(
+ "rdkit.Chem",
+ required='wandb.Molecule needs the rdkit-pypi package. To get it, run "pip install rdkit-pypi".',
+ )
+ rdkit_chem_all_chem = util.get_module(
+ "rdkit.Chem.AllChem",
+ required='wandb.Molecule needs the rdkit-pypi package. To get it, run "pip install rdkit-pypi".',
+ )
+
+ if isinstance(data_or_path, str):
+ # path to a file?
+ path = pathlib.Path(data_or_path)
+ extension = path.suffix.split(".")[-1]
+ if extension not in Molecule.SUPPORTED_RDKIT_TYPES:
+ raise ValueError(
+ "Molecule.from_rdkit only supports files of the type: "
+ + ", ".join(Molecule.SUPPORTED_RDKIT_TYPES)
+ )
+ # use the appropriate method
+ if extension == "sdf":
+ with rdkit_chem.SDMolSupplier(data_or_path) as supplier:
+ molecule = next(supplier) # get only the first molecule
+ else:
+ molecule = getattr(rdkit_chem, f"MolFrom{extension.capitalize()}File")(
+ data_or_path
+ )
+ elif isinstance(data_or_path, rdkit_chem.rdchem.Mol):
+ molecule = data_or_path
+ else:
+ raise ValueError(
+ "Data must be file name or an rdkit.Chem.rdchem.Mol object"
+ )
+
+ if convert_to_3d_and_optimize:
+ molecule = rdkit_chem.AddHs(molecule)
+ rdkit_chem_all_chem.EmbedMolecule(molecule)
+ rdkit_chem_all_chem.MMFFOptimizeMolecule(
+ molecule,
+ maxIters=mmff_optimize_molecule_max_iterations,
+ )
+ # convert to the pdb format supported by Molecule
+ pdb_block = rdkit_chem.rdmolfiles.MolToPDBBlock(molecule)
+
+ return cls(io.StringIO(pdb_block), caption=caption, file_type="pdb")
+
+ @classmethod
+ def from_smiles(
+ cls,
+ data: str,
+ caption: Optional[str] = None,
+ sanitize: bool = True,
+ convert_to_3d_and_optimize: bool = True,
+ mmff_optimize_molecule_max_iterations: int = 200,
+ ) -> "Molecule":
+ """Convert SMILES string to wandb.Molecule.
+
+ Arguments:
+ data: (string)
+ SMILES string.
+ caption: (string)
+ Caption associated with the molecule for display
+ sanitize: (bool)
+ Check if the molecule is chemically reasonable by the RDKit's definition.
+ convert_to_3d_and_optimize: (bool)
+ Convert to rdkit.Chem.rdchem.Mol with 3D coordinates.
+ This is an expensive operation that may take a long time for complicated molecules.
+ mmff_optimize_molecule_max_iterations: (int)
+ Number of iterations to use in rdkit.Chem.AllChem.MMFFOptimizeMolecule
+ """
+ rdkit_chem = util.get_module(
+ "rdkit.Chem",
+ required='wandb.Molecule needs the rdkit-pypi package. To get it, run "pip install rdkit-pypi".',
+ )
+ molecule = rdkit_chem.MolFromSmiles(data, sanitize=sanitize)
+ if molecule is None:
+ raise ValueError("Unable to parse the SMILES string.")
+
+ return cls.from_rdkit(
+ data_or_path=molecule,
+ caption=caption,
+ convert_to_3d_and_optimize=convert_to_3d_and_optimize,
+ mmff_optimize_molecule_max_iterations=mmff_optimize_molecule_max_iterations,
+ )
+
+ @classmethod
+ def get_media_subdir(cls: Type["Molecule"]) -> str:
+ return os.path.join("media", "molecule")
+
+ def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
+ json_dict = super().to_json(run_or_artifact)
+ json_dict["_type"] = self._log_type
+ if self._caption:
+ json_dict["caption"] = self._caption
+ return json_dict
+
+ @classmethod
+ def seq_to_json(
+ cls: Type["Molecule"],
+ seq: Sequence["BatchableMedia"],
+ run: "LocalRun",
+ key: str,
+ step: Union[int, str],
+ ) -> dict:
+ seq = list(seq)
+
+ jsons = [obj.to_json(run) for obj in seq]
+
+ for obj in jsons:
+ expected = LogicalPath(cls.get_media_subdir())
+ if not obj["path"].startswith(expected):
+ raise ValueError(
+ "Files in an array of Molecule's must be in the {} directory, not {}".format(
+ cls.get_media_subdir(), obj["path"]
+ )
+ )
+
+ return {
+ "_type": "molecule",
+ "filenames": [obj["path"] for obj in jsons],
+ "count": len(jsons),
+ "captions": Media.captions(seq),
+ }
diff --git a/wandb/sdk/data_types/object_3d.py b/wandb/sdk/data_types/object_3d.py
new file mode 100644
index 0000000000000000000000000000000000000000..5b43aefc275eadd181ad6119449783ba7b3f0e4a
--- /dev/null
+++ b/wandb/sdk/data_types/object_3d.py
@@ -0,0 +1,363 @@
+import codecs
+import json
+import os
+import sys
+from typing import (
+ TYPE_CHECKING,
+ ClassVar,
+ Optional,
+ Sequence,
+ Set,
+ TextIO,
+ Tuple,
+ Type,
+ Union,
+)
+
+if sys.version_info >= (3, 8):
+ from typing import Literal, TypedDict
+else:
+ from typing_extensions import Literal, TypedDict
+
+
+import wandb
+from wandb import util
+from wandb.sdk.lib import runid
+from wandb.sdk.lib.paths import LogicalPath
+
+from . import _dtypes
+from ._private import MEDIA_TMP
+from .base_types.media import BatchableMedia
+
+if TYPE_CHECKING: # pragma: no cover
+ import numpy as np
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ from ..wandb_run import Run as LocalRun
+
+ numeric = Union[int, float, np.integer, np.float_]
+ FileFormat3D = Literal[
+ "obj",
+ "gltf",
+ "glb",
+ "babylon",
+ "stl",
+ "pts.json",
+ ]
+ Point3D = Tuple[numeric, numeric, numeric]
+ Point3DWithCategory = Tuple[numeric, numeric, numeric, numeric]
+ Point3DWithColors = Tuple[numeric, numeric, numeric, numeric, numeric, numeric]
+ Point = Union[Point3D, Point3DWithCategory, Point3DWithColors]
+ PointCloudType = Literal["lidar/beta"]
+ RGBColor = Tuple[int, int, int]
+
+ class Box3D(TypedDict):
+ corners: Tuple[
+ Point3D,
+ Point3D,
+ Point3D,
+ Point3D,
+ Point3D,
+ Point3D,
+ Point3D,
+ Point3D,
+ ]
+ label: Optional[str]
+ color: RGBColor
+ score: Optional[numeric]
+
+ class Vector3D(TypedDict):
+ start: Sequence[Point3D]
+ end: Sequence[Point3D]
+
+ class Camera(TypedDict):
+ viewpoint: Sequence[Point3D]
+ target: Sequence[Point3D]
+
+
+class Object3D(BatchableMedia):
+ """Wandb class for 3D point clouds.
+
+ Arguments:
+ data_or_path: (numpy array, string, io)
+ Object3D can be initialized from a file or a numpy array.
+
+ You can pass a path to a file or an io object and a file_type
+ which must be one of SUPPORTED_TYPES
+
+ The shape of the numpy array must be one of either:
+ ```
+ [[x y z], ...] nx3
+ [[x y z c], ...] nx4 where c is a category with supported range [1, 14]
+ [[x y z r g b], ...] nx6 where is rgb is color
+ ```
+ """
+
+ SUPPORTED_TYPES: ClassVar[Set[str]] = {
+ "obj",
+ "gltf",
+ "glb",
+ "babylon",
+ "stl",
+ "pts.json",
+ }
+ SUPPORTED_POINT_CLOUD_TYPES: ClassVar[Set[str]] = {"lidar/beta"}
+ _log_type: ClassVar[str] = "object3D-file"
+
+ def __init__(
+ self,
+ data_or_path: Union["np.ndarray", str, "TextIO", dict],
+ **kwargs: Optional[Union[str, "FileFormat3D"]],
+ ) -> None:
+ super().__init__()
+
+ if hasattr(data_or_path, "name"):
+ # if the file has a path, we just detect the type and copy it from there
+ data_or_path = data_or_path.name
+
+ if hasattr(data_or_path, "read"):
+ if hasattr(data_or_path, "seek"):
+ data_or_path.seek(0)
+ object_3d = data_or_path.read()
+
+ extension = kwargs.pop("file_type", None)
+ if extension is None:
+ raise ValueError(
+ "Must pass file type keyword argument when using io objects."
+ )
+ if extension not in Object3D.SUPPORTED_TYPES:
+ raise ValueError(
+ "Object 3D only supports numpy arrays or files of the type: "
+ + ", ".join(Object3D.SUPPORTED_TYPES)
+ )
+
+ extension = "." + extension
+
+ tmp_path = os.path.join(MEDIA_TMP.name, runid.generate_id() + extension)
+ with open(tmp_path, "w") as f:
+ f.write(object_3d)
+
+ self._set_file(tmp_path, is_tmp=True, extension=extension)
+ elif isinstance(data_or_path, str):
+ path = data_or_path
+ extension = None
+ for supported_type in Object3D.SUPPORTED_TYPES:
+ if path.endswith(supported_type):
+ extension = "." + supported_type
+ break
+
+ if not extension:
+ raise ValueError(
+ "File '"
+ + path
+ + "' is not compatible with Object3D: supported types are: "
+ + ", ".join(Object3D.SUPPORTED_TYPES)
+ )
+
+ self._set_file(data_or_path, is_tmp=False, extension=extension)
+ # Supported different types and scene for 3D scenes
+ elif isinstance(data_or_path, dict) and "type" in data_or_path:
+ if data_or_path["type"] == "lidar/beta":
+ data = {
+ "type": data_or_path["type"],
+ "vectors": data_or_path["vectors"].tolist()
+ if "vectors" in data_or_path
+ else [],
+ "points": data_or_path["points"].tolist()
+ if "points" in data_or_path
+ else [],
+ "boxes": data_or_path["boxes"].tolist()
+ if "boxes" in data_or_path
+ else [],
+ }
+ else:
+ raise ValueError(
+ "Type not supported, only 'lidar/beta' is currently supported"
+ )
+
+ tmp_path = os.path.join(MEDIA_TMP.name, runid.generate_id() + ".pts.json")
+ with codecs.open(tmp_path, "w", encoding="utf-8") as fp:
+ json.dump(
+ data,
+ fp,
+ separators=(",", ":"),
+ sort_keys=True,
+ indent=4,
+ )
+ self._set_file(tmp_path, is_tmp=True, extension=".pts.json")
+ elif util.is_numpy_array(data_or_path):
+ np_data = data_or_path
+
+ # The following assertion is required for numpy to trust that
+ # np_data is numpy array. The reason it is behind a False
+ # guard is to ensure that this line does not run at runtime,
+ # which would cause a runtime error if the user's machine did
+ # not have numpy installed.
+
+ if TYPE_CHECKING:
+ assert isinstance(np_data, np.ndarray)
+
+ if len(np_data.shape) != 2 or np_data.shape[1] not in {3, 4, 6}:
+ raise ValueError(
+ """
+ The shape of the numpy array must be one of either
+ [[x y z], ...] nx3
+ [x y z c], ...] nx4 where c is a category with supported range [1, 14]
+ [x y z r g b], ...] nx6 where rgb is color
+ """
+ )
+
+ list_data = np_data.tolist()
+ tmp_path = os.path.join(MEDIA_TMP.name, runid.generate_id() + ".pts.json")
+ with codecs.open(tmp_path, "w", encoding="utf-8") as fp:
+ json.dump(
+ list_data,
+ fp,
+ separators=(",", ":"),
+ sort_keys=True,
+ indent=4,
+ )
+ self._set_file(tmp_path, is_tmp=True, extension=".pts.json")
+ else:
+ raise ValueError("data must be a numpy array, dict or a file object")
+
+ @classmethod
+ def from_file(
+ cls,
+ data_or_path: Union["TextIO", str],
+ file_type: Optional["FileFormat3D"] = None,
+ ) -> "Object3D":
+ """Initializes Object3D from a file or stream.
+
+ Arguments:
+ data_or_path (Union["TextIO", str]): A path to a file or a `TextIO` stream.
+ file_type (str): Specifies the data format passed to `data_or_path`. Required when `data_or_path` is a
+ `TextIO` stream. This parameter is ignored if a file path is provided. The type is taken from the file extension.
+ """
+ # if file_type is not None and file_type not in cls.SUPPORTED_TYPES:
+ # raise ValueError(
+ # f"Unsupported file type: {file_type}. Supported types are: {cls.SUPPORTED_TYPES}"
+ # )
+ return cls(data_or_path, file_type=file_type)
+
+ @classmethod
+ def from_numpy(cls, data: "np.ndarray") -> "Object3D":
+ """Initializes Object3D from a numpy array.
+
+ Arguments:
+ data (numpy array): Each entry in the array will
+ represent one point in the point cloud.
+
+
+ The shape of the numpy array must be one of either:
+ ```
+ [[x y z], ...] # nx3.
+ [[x y z c], ...] # nx4 where c is a category with supported range [1, 14].
+ [[x y z r g b], ...] # nx6 where is rgb is color.
+ ```
+ """
+ if not util.is_numpy_array(data):
+ raise ValueError("`data` must be a numpy array")
+
+ if len(data.shape) != 2 or data.shape[1] not in {3, 4, 6}:
+ raise ValueError(
+ """
+ The shape of the numpy array must be one of either:
+ [[x y z], ...] nx3
+ [x y z c], ...] nx4 where c is a category with supported range [1, 14]
+ [x y z r g b], ...] nx6 where rgb is color
+ """
+ )
+
+ return cls(data)
+
+ @classmethod
+ def from_point_cloud(
+ cls,
+ points: Sequence["Point"],
+ boxes: Sequence["Box3D"],
+ vectors: Optional[Sequence["Vector3D"]] = None,
+ point_cloud_type: "PointCloudType" = "lidar/beta",
+ # camera: Optional[Camera] = None,
+ ) -> "Object3D":
+ """Initializes Object3D from a python object.
+
+ Arguments:
+ points (Sequence["Point"]): The points in the point cloud.
+ boxes (Sequence["Box3D"]): 3D bounding boxes for labeling the point cloud. Boxes
+ are displayed in point cloud visualizations.
+ vectors (Optional[Sequence["Vector3D"]]): Each vector is displayed in the point cloud
+ visualization. Can be used to indicate directionality of bounding boxes. Defaults to None.
+ point_cloud_type ("lidar/beta"): At this time, only the "lidar/beta" type is supported. Defaults to "lidar/beta".
+ """
+ if point_cloud_type not in cls.SUPPORTED_POINT_CLOUD_TYPES:
+ raise ValueError("Point cloud type not supported")
+
+ numpy = wandb.util.get_module(
+ "numpy",
+ required="wandb.Object3D.from_point_cloud requires numpy. Install with `pip install numpy`",
+ )
+
+ data = {
+ "type": point_cloud_type,
+ "points": numpy.array(points),
+ "boxes": numpy.array(boxes),
+ "vectors": numpy.array(vectors) if vectors is not None else numpy.array([]),
+ }
+
+ return cls(data)
+
+ @classmethod
+ def get_media_subdir(cls: Type["Object3D"]) -> str:
+ return os.path.join("media", "object3D")
+
+ def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
+ json_dict = super().to_json(run_or_artifact)
+ json_dict["_type"] = Object3D._log_type
+
+ if isinstance(run_or_artifact, wandb.Artifact):
+ if self._path is None or not self._path.endswith(".pts.json"):
+ raise ValueError(
+ "Non-point cloud 3D objects are not yet supported with Artifacts"
+ )
+
+ return json_dict
+
+ @classmethod
+ def seq_to_json(
+ cls: Type["Object3D"],
+ seq: Sequence["BatchableMedia"],
+ run: "LocalRun",
+ key: str,
+ step: Union[int, str],
+ ) -> dict:
+ seq = list(seq)
+
+ jsons = [obj.to_json(run) for obj in seq]
+
+ for obj in jsons:
+ expected = LogicalPath(cls.get_media_subdir())
+ if not obj["path"].startswith(expected):
+ raise ValueError(
+ "Files in an array of Object3D's must be in the {} directory, not {}".format(
+ expected, obj["path"]
+ )
+ )
+
+ return {
+ "_type": "object3D",
+ "filenames": [
+ os.path.relpath(j["path"], cls.get_media_subdir()) for j in jsons
+ ],
+ "count": len(jsons),
+ "objects": jsons,
+ }
+
+
+class _Object3DFileType(_dtypes.Type):
+ name = "object3D-file"
+ types = [Object3D]
+
+
+_dtypes.TypeRegistry.add(_Object3DFileType)
diff --git a/wandb/sdk/data_types/plotly.py b/wandb/sdk/data_types/plotly.py
new file mode 100644
index 0000000000000000000000000000000000000000..05162771b113986c603caa06d86bd6afaa6e9938
--- /dev/null
+++ b/wandb/sdk/data_types/plotly.py
@@ -0,0 +1,82 @@
+import codecs
+import os
+from typing import TYPE_CHECKING, Sequence, Type, Union
+
+from wandb import util
+from wandb.sdk.lib import runid
+
+from ._private import MEDIA_TMP
+from .base_types.media import Media, _numpy_arrays_to_lists
+from .base_types.wb_value import WBValue
+from .image import Image
+
+if TYPE_CHECKING: # pragma: no cover
+ import matplotlib # type: ignore
+ import pandas as pd
+ import plotly # type: ignore
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ from ..wandb_run import Run as LocalRun
+
+ ValToJsonType = Union[
+ dict,
+ "WBValue",
+ Sequence["WBValue"],
+ "plotly.Figure",
+ "matplotlib.artist.Artist",
+ "pd.DataFrame",
+ object,
+ ]
+
+
+class Plotly(Media):
+ """Wandb class for plotly plots.
+
+ Arguments:
+ val: matplotlib or plotly figure
+ """
+
+ _log_type = "plotly-file"
+
+ @classmethod
+ def make_plot_media(
+ cls: Type["Plotly"], val: Union["plotly.Figure", "matplotlib.artist.Artist"]
+ ) -> Union[Image, "Plotly"]:
+ if util.is_matplotlib_typename(util.get_full_typename(val)):
+ if util.matplotlib_contains_images(val):
+ return Image(val)
+ val = util.matplotlib_to_plotly(val)
+ return cls(val)
+
+ def __init__(self, val: Union["plotly.Figure", "matplotlib.artist.Artist"]):
+ super().__init__()
+ # First, check to see if the incoming `val` object is a plotfly figure
+ if not util.is_plotly_figure_typename(util.get_full_typename(val)):
+ # If it is not, but it is a matplotlib figure, then attempt to convert it to plotly
+ if util.is_matplotlib_typename(util.get_full_typename(val)):
+ if util.matplotlib_contains_images(val):
+ raise ValueError(
+ "Plotly does not currently support converting matplotlib figures containing images. \
+ You can convert the plot to a static image with `wandb.Image(plt)` "
+ )
+ val = util.matplotlib_to_plotly(val)
+ else:
+ raise ValueError(
+ "Logged plots must be plotly figures, or matplotlib plots convertible to plotly via mpl_to_plotly"
+ )
+
+ tmp_path = os.path.join(MEDIA_TMP.name, runid.generate_id() + ".plotly.json")
+ val = _numpy_arrays_to_lists(val.to_plotly_json())
+ with codecs.open(tmp_path, "w", encoding="utf-8") as fp:
+ util.json_dump_safer(val, fp)
+ self._set_file(tmp_path, is_tmp=True, extension=".plotly.json")
+
+ @classmethod
+ def get_media_subdir(cls: Type["Plotly"]) -> str:
+ return os.path.join("media", "plotly")
+
+ def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
+ json_dict = super().to_json(run_or_artifact)
+ json_dict["_type"] = self._log_type
+ return json_dict
diff --git a/wandb/sdk/data_types/saved_model.py b/wandb/sdk/data_types/saved_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..07b8e36182e88e5bc16663f0cdcb3f48956845f9
--- /dev/null
+++ b/wandb/sdk/data_types/saved_model.py
@@ -0,0 +1,444 @@
+import os
+import shutil
+import sys
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ ClassVar,
+ Generic,
+ List,
+ Optional,
+ Type,
+ TypeVar,
+ Union,
+ cast,
+)
+
+import wandb
+from wandb import util
+from wandb.sdk.lib import runid
+from wandb.sdk.lib.hashutil import md5_file_hex
+from wandb.sdk.lib.paths import LogicalPath
+
+from ._private import MEDIA_TMP
+from .base_types.wb_value import WBValue
+
+if TYPE_CHECKING: # pragma: no cover
+ import cloudpickle # type: ignore
+ import sklearn # type: ignore
+ import tensorflow # type: ignore
+ import torch # type: ignore
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ from ..wandb_run import Run as LocalRun
+
+
+DEBUG_MODE = False
+
+
+def _add_deterministic_dir_to_artifact(
+ artifact: "Artifact", dir_name: str, target_dir_root: str
+) -> str:
+ file_paths = []
+ for dirpath, _, filenames in os.walk(dir_name, topdown=True):
+ for fn in filenames:
+ file_paths.append(os.path.join(dirpath, fn))
+ dirname = md5_file_hex(*file_paths)[:20]
+ target_path = LogicalPath(os.path.join(target_dir_root, dirname))
+ artifact.add_dir(dir_name, target_path)
+ return target_path
+
+
+def _load_dir_from_artifact(source_artifact: "Artifact", path: str) -> str:
+ dl_path = None
+
+ # Look through the entire manifest to find all of the files in the directory.
+ # Construct the directory path by inspecting the target download location.
+ for p, _ in source_artifact.manifest.entries.items():
+ if p.startswith(path):
+ example_path = source_artifact.get_entry(p).download()
+ if dl_path is None:
+ root = example_path[: -len(p)]
+ dl_path = os.path.join(root, path)
+
+ assert dl_path is not None, f"Could not find directory {path} in artifact"
+
+ return dl_path
+
+
+SavedModelObjType = TypeVar("SavedModelObjType")
+
+
+class _SavedModel(WBValue, Generic[SavedModelObjType]):
+ """Internal W&B Artifact model storage.
+
+ _model_type_id: (str) The id of the SavedModel subclass used to serialize the model.
+ """
+
+ _log_type: ClassVar[str]
+ _path_extension: ClassVar[str]
+
+ _model_obj: Optional["SavedModelObjType"]
+ _path: Optional[str]
+ _input_obj_or_path: Union[SavedModelObjType, str]
+
+ # Public Methods
+ def __init__(
+ self, obj_or_path: Union[SavedModelObjType, str], **kwargs: Any
+ ) -> None:
+ super().__init__()
+ if self.__class__ == _SavedModel:
+ raise TypeError(
+ "Cannot instantiate abstract SavedModel class - please use SavedModel.init(...) instead."
+ )
+ self._model_obj = None
+ self._path = None
+ self._input_obj_or_path = obj_or_path
+ input_is_path = isinstance(obj_or_path, str) and os.path.exists(obj_or_path)
+ if input_is_path:
+ assert isinstance(obj_or_path, str) # mypy
+ self._set_obj(self._deserialize(obj_or_path))
+ else:
+ self._set_obj(obj_or_path)
+
+ self._copy_to_disk()
+ # At this point, the model will be saved to a temp path,
+ # and self._path will be set to such temp path. If the model
+ # provided was a path, then both self._path and self._model_obj
+ # are copies of the user-provided data. However, if the input
+ # was a model object, then we want to clear the model object. The first
+ # accessing of the model object (via .model_obj()) will load the model
+ # from the temp path.
+
+ if not input_is_path:
+ self._unset_obj()
+
+ @staticmethod
+ def init(obj_or_path: Any, **kwargs: Any) -> "_SavedModel":
+ maybe_instance = _SavedModel._maybe_init(obj_or_path, **kwargs)
+ if maybe_instance is None:
+ raise ValueError(
+ f"No suitable SavedModel subclass constructor found for obj_or_path: {obj_or_path}"
+ )
+ return maybe_instance
+
+ @classmethod
+ def from_json(
+ cls: Type["_SavedModel"], json_obj: dict, source_artifact: "Artifact"
+ ) -> "_SavedModel":
+ path = json_obj["path"]
+
+ # First, if the entry is a file, the download it.
+ entry = source_artifact.manifest.entries.get(path)
+ if entry is not None:
+ dl_path = str(source_artifact.get_entry(path).download())
+ else:
+ # If not, assume it is directory.
+ # FUTURE: Add this functionality to the artifact loader
+ # (would be nice to parallelize)
+ dl_path = _load_dir_from_artifact(source_artifact, path)
+
+ # Return the SavedModel object instantiated with the downloaded path
+ # and specified adapter.
+ return cls(dl_path)
+
+ def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
+ # Unlike other data types, we do not allow adding to a Run directly. There is a
+ # bit of tech debt in the other data types which requires the input to `to_json`
+ # to accept a Run or Artifact. However, Run additions should be deprecated in the future.
+ # This check helps ensure we do not add to the debt.
+ if isinstance(run_or_artifact, wandb.wandb_sdk.wandb_run.Run):
+ raise ValueError("SavedModel cannot be added to run - must use artifact")
+ artifact = run_or_artifact
+ json_obj = {
+ "type": self._log_type,
+ }
+ assert self._path is not None, "Cannot add SavedModel to Artifact without path"
+ if os.path.isfile(self._path):
+ # If the path is a file, then we can just add it to the artifact,
+ # First checking to see if the artifact already has the file (use the cache)
+ # Else, add it directly, allowing the artifact adder to rename the file deterministically.
+ already_added_path = artifact.get_added_local_path_name(self._path)
+ if already_added_path is not None:
+ json_obj["path"] = already_added_path
+ else:
+ target_path = os.path.join(
+ ".wb_data", "saved_models", os.path.basename(self._path)
+ )
+ json_obj["path"] = artifact.add_file(self._path, target_path, True).path
+ elif os.path.isdir(self._path):
+ # If the path is a directory, then we need to add all of the files
+ # The directory must be named deterministically based on the contents of the directory,
+ # but the files themselves need to have their name preserved.
+ # FUTURE: Add this functionality to the artifact adder itself
+ json_obj["path"] = _add_deterministic_dir_to_artifact(
+ artifact, self._path, os.path.join(".wb_data", "saved_models")
+ )
+ else:
+ raise ValueError(
+ f"Expected a path to a file or directory, got {self._path}"
+ )
+
+ return json_obj
+
+ def model_obj(self) -> SavedModelObjType:
+ """Return the model object."""
+ if self._model_obj is None:
+ assert self._path is not None, "Cannot load model object without path"
+ self._set_obj(self._deserialize(self._path))
+
+ assert self._model_obj is not None, "Model object is None"
+ return self._model_obj
+
+ # Methods to be implemented by subclasses
+ @staticmethod
+ def _deserialize(path: str) -> SavedModelObjType:
+ """Return the model object from a path. Allowed to throw errors."""
+ raise NotImplementedError
+
+ @staticmethod
+ def _validate_obj(obj: Any) -> bool:
+ """Validate the model object. Allowed to throw errors."""
+ raise NotImplementedError
+
+ @staticmethod
+ def _serialize(obj: SavedModelObjType, dir_or_file_path: str) -> None:
+ """Save the model to disk.
+
+ The method will receive a directory path which all files needed for
+ deserialization should be saved. A directory will always be passed if
+ _path_extension is an empty string, else a single file will be passed. Allowed
+ to throw errors.
+ """
+ raise NotImplementedError
+
+ # Private Class Methods
+ @classmethod
+ def _maybe_init(
+ cls: Type["_SavedModel"], obj_or_path: Any, **kwargs: Any
+ ) -> Optional["_SavedModel"]:
+ # _maybe_init is an exception-safe method that will return an instance of this class
+ # (or any subclass of this class - recursively) OR None if no subclass constructor is found.
+ # We first try the current class, then recursively call this method on children classes. This pattern
+ # conforms to the new "Weave-type" pattern developed by Shawn. This way, we can for example have a
+ # pytorch subclass that can itself have two subclasses: one for a TorchScript model, and one for a PyTorch model.
+ # The children subclasses will know how to serialize/deserialize their respective payloads, but the pytorch
+ # parent class can know how to execute inference on the model - regardless of serialization strategy.
+ try:
+ return cls(obj_or_path, **kwargs)
+ except Exception as e:
+ if DEBUG_MODE:
+ print(f"{cls}._maybe_init({obj_or_path}) failed: {e}")
+ pass
+
+ for child_cls in cls.__subclasses__():
+ maybe_instance = child_cls._maybe_init(obj_or_path, **kwargs)
+ if maybe_instance is not None:
+ return maybe_instance
+
+ return None
+
+ @classmethod
+ def _tmp_path(cls: Type["_SavedModel"]) -> str:
+ # Generates a tmp path under our MEDIA_TMP directory which confirms to the file
+ # or folder preferences of the class.
+ assert isinstance(cls._path_extension, str), "_path_extension must be a string"
+ tmp_path = os.path.abspath(os.path.join(MEDIA_TMP.name, runid.generate_id()))
+ if cls._path_extension != "":
+ tmp_path += "." + cls._path_extension
+ return tmp_path
+
+ # Private Instance Methods
+ def _copy_to_disk(self) -> None:
+ # Creates a temporary path and writes a fresh copy of the
+ # model to disk - updating the _path appropriately.
+ tmp_path = self._tmp_path()
+ self._dump(tmp_path)
+ self._path = tmp_path
+
+ def _unset_obj(self) -> None:
+ assert self._path is not None, "Cannot unset object if path is None"
+ self._model_obj = None
+
+ def _set_obj(self, model_obj: Any) -> None:
+ assert model_obj is not None and self._validate_obj(
+ model_obj
+ ), f"Invalid model object {model_obj}"
+ self._model_obj = model_obj
+
+ def _dump(self, target_path: str) -> None:
+ assert self._model_obj is not None, "Cannot dump if model object is None"
+ self._serialize(self._model_obj, target_path)
+
+
+def _get_cloudpickle() -> "cloudpickle":
+ return cast(
+ "cloudpickle",
+ util.get_module("cloudpickle", "ModelAdapter requires `cloudpickle`"),
+ )
+
+
+# TODO: Add pip deps
+# TODO: potentially move this up to the saved model class
+PicklingSavedModelObjType = TypeVar("PicklingSavedModelObjType")
+
+
+class _PicklingSavedModel(_SavedModel[SavedModelObjType]):
+ _dep_py_files: Optional[List[str]] = None
+ _dep_py_files_path: Optional[str] = None
+
+ def __init__(
+ self,
+ obj_or_path: Union[SavedModelObjType, str],
+ dep_py_files: Optional[List[str]] = None,
+ ):
+ super().__init__(obj_or_path)
+ if self.__class__ == _PicklingSavedModel:
+ raise TypeError(
+ "Cannot instantiate abstract _PicklingSavedModel class - please use SavedModel.init(...) instead."
+ )
+ if dep_py_files is not None and len(dep_py_files) > 0:
+ self._dep_py_files = dep_py_files
+ self._dep_py_files_path = os.path.abspath(
+ os.path.join(MEDIA_TMP.name, runid.generate_id())
+ )
+ os.makedirs(self._dep_py_files_path, exist_ok=True)
+ for extra_file in self._dep_py_files:
+ if os.path.isfile(extra_file):
+ shutil.copy(extra_file, self._dep_py_files_path)
+ elif os.path.isdir(extra_file):
+ shutil.copytree(
+ extra_file,
+ os.path.join(
+ self._dep_py_files_path, os.path.basename(extra_file)
+ ),
+ )
+ else:
+ raise ValueError(f"Invalid dependency file: {extra_file}")
+
+ @classmethod
+ def from_json(
+ cls: Type["_SavedModel"], json_obj: dict, source_artifact: "Artifact"
+ ) -> "_PicklingSavedModel":
+ backup_path = [p for p in sys.path]
+ if (
+ "dep_py_files_path" in json_obj
+ and json_obj["dep_py_files_path"] is not None
+ ):
+ dl_path = _load_dir_from_artifact(
+ source_artifact, json_obj["dep_py_files_path"]
+ )
+ assert dl_path is not None
+ sys.path.append(dl_path)
+ inst = super().from_json(json_obj, source_artifact) # type: ignore
+ sys.path = backup_path
+
+ return inst # type: ignore
+
+ def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
+ json_obj = super().to_json(run_or_artifact)
+ assert isinstance(run_or_artifact, wandb.Artifact)
+ if self._dep_py_files_path is not None:
+ json_obj["dep_py_files_path"] = _add_deterministic_dir_to_artifact(
+ run_or_artifact,
+ self._dep_py_files_path,
+ os.path.join(".wb_data", "extra_files"),
+ )
+ return json_obj
+
+
+def _get_torch() -> "torch":
+ return cast(
+ "torch",
+ util.get_module("torch", "ModelAdapter requires `torch`"),
+ )
+
+
+class _PytorchSavedModel(_PicklingSavedModel["torch.nn.Module"]):
+ _log_type = "pytorch-model-file"
+ _path_extension = "pt"
+
+ @staticmethod
+ def _deserialize(dir_or_file_path: str) -> "torch.nn.Module":
+ return _get_torch().load(dir_or_file_path)
+
+ @staticmethod
+ def _validate_obj(obj: Any) -> bool:
+ return isinstance(obj, _get_torch().nn.Module)
+
+ @staticmethod
+ def _serialize(model_obj: "torch.nn.Module", dir_or_file_path: str) -> None:
+ _get_torch().save(
+ model_obj,
+ dir_or_file_path,
+ pickle_module=_get_cloudpickle(),
+ )
+
+
+def _get_sklearn() -> "sklearn":
+ return cast(
+ "sklearn",
+ util.get_module("sklearn", "ModelAdapter requires `sklearn`"),
+ )
+
+
+class _SklearnSavedModel(_PicklingSavedModel["sklearn.base.BaseEstimator"]):
+ _log_type = "sklearn-model-file"
+ _path_extension = "pkl"
+
+ @staticmethod
+ def _deserialize(
+ dir_or_file_path: str,
+ ) -> "sklearn.base.BaseEstimator":
+ with open(dir_or_file_path, "rb") as file:
+ model = _get_cloudpickle().load(file)
+ return model
+
+ @staticmethod
+ def _validate_obj(obj: Any) -> bool:
+ dynamic_sklearn = _get_sklearn()
+ return cast(
+ bool,
+ (
+ dynamic_sklearn.base.is_classifier(obj)
+ or dynamic_sklearn.base.is_outlier_detector(obj)
+ or dynamic_sklearn.base.is_regressor(obj)
+ ),
+ )
+
+ @staticmethod
+ def _serialize(
+ model_obj: "sklearn.base.BaseEstimator", dir_or_file_path: str
+ ) -> None:
+ dynamic_cloudpickle = _get_cloudpickle()
+ with open(dir_or_file_path, "wb") as file:
+ dynamic_cloudpickle.dump(model_obj, file)
+
+
+def _get_tf_keras() -> "tensorflow.keras":
+ return cast(
+ "tensorflow",
+ util.get_module("tensorflow", "ModelAdapter requires `tensorflow`"),
+ ).keras
+
+
+class _TensorflowKerasSavedModel(_SavedModel["tensorflow.keras.Model"]):
+ _log_type = "tfkeras-model-file"
+ _path_extension = ""
+
+ @staticmethod
+ def _deserialize(
+ dir_or_file_path: str,
+ ) -> "tensorflow.keras.Model":
+ return _get_tf_keras().models.load_model(dir_or_file_path)
+
+ @staticmethod
+ def _validate_obj(obj: Any) -> bool:
+ return isinstance(obj, _get_tf_keras().models.Model)
+
+ @staticmethod
+ def _serialize(model_obj: "tensorflow.keras.Model", dir_or_file_path: str) -> None:
+ _get_tf_keras().models.save_model(
+ model_obj, dir_or_file_path, include_optimizer=True
+ )
diff --git a/wandb/sdk/data_types/trace_tree.py b/wandb/sdk/data_types/trace_tree.py
new file mode 100644
index 0000000000000000000000000000000000000000..213402294c2f07d167759f988deb43511e05b052
--- /dev/null
+++ b/wandb/sdk/data_types/trace_tree.py
@@ -0,0 +1,438 @@
+"""This module contains the `WBTraceTree` media type, and the supporting dataclasses.
+
+A `WBTraceTree` is a media object containing a root span and an
+arbitrary model dump as a serializable dictionary. Logging such media type will
+result in a W&B Trace Debugger panel being created in the workspace UI.
+"""
+
+import dataclasses
+import hashlib
+import json
+import typing
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+
+import wandb
+import wandb.data_types
+from wandb.sdk.data_types import _dtypes
+from wandb.sdk.data_types.base_types.media import Media
+
+if TYPE_CHECKING: # pragma: no cover
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ from ..wandb_run import Run as LocalRun
+
+
+class StatusCode(str, Enum):
+ SUCCESS = "SUCCESS"
+ ERROR = "ERROR"
+
+ def __str__(self) -> str:
+ return str(self.value)
+
+
+class SpanKind(str, Enum):
+ LLM = "LLM"
+ CHAIN = "CHAIN"
+ AGENT = "AGENT"
+ TOOL = "TOOL"
+
+ def __str__(self) -> str:
+ return str(self.value)
+
+
+@dataclass()
+class Result:
+ inputs: Optional[Dict[str, Any]] = field(default=None)
+ outputs: Optional[Dict[str, Any]] = field(default=None)
+
+
+@dataclass()
+class Span:
+ span_id: Optional[str] = field(default=None)
+ name: Optional[str] = field(default=None)
+ start_time_ms: Optional[int] = field(default=None)
+ end_time_ms: Optional[int] = field(default=None)
+ status_code: Optional[StatusCode] = field(default=None)
+ status_message: Optional[str] = field(default=None)
+ attributes: Optional[Dict[str, Any]] = field(default=None)
+ results: Optional[List[Result]] = field(default=None)
+ child_spans: Optional[List["Span"]] = field(default=None)
+ span_kind: Optional[SpanKind] = field(default=None)
+
+ def add_attribute(self, key: str, value: Any) -> None:
+ if self.attributes is None:
+ self.attributes = {}
+ self.attributes[key] = value
+
+ def add_named_result(self, inputs: Dict[str, Any], outputs: Dict[str, Any]) -> None:
+ if self.results is None:
+ self.results = []
+ self.results.append(Result(inputs, outputs))
+
+ def add_child_span(self, span: "Span") -> None:
+ if self.child_spans is None:
+ self.child_spans = []
+ self.child_spans.append(span)
+
+
+class WBTraceTree(Media):
+ """Media object for trace tree data.
+
+ Arguments:
+ root_span (Span): The root span of the trace tree.
+ model_dict (dict, optional): A dictionary containing the model dump.
+ NOTE: model_dict is a completely-user-defined dict. The UI will render
+ a JSON viewer for this dict, giving special treatment to dictionaries
+ with a `_kind` key. This is because model vendors have such different
+ serialization formats that we need to be flexible here.
+ """
+
+ _log_type = "wb_trace_tree"
+
+ def __init__(
+ self,
+ root_span: Span,
+ model_dict: typing.Optional[dict] = None,
+ ):
+ super().__init__()
+ self._root_span = root_span
+ self._model_dict = model_dict
+
+ @classmethod
+ def get_media_subdir(cls) -> str:
+ return "media/wb_trace_tree"
+
+ def to_json(self, run: Optional[Union["LocalRun", "Artifact"]]) -> dict:
+ res = {"_type": self._log_type}
+ # Here we use `dumps` to put things into string format. This is because
+ # the complex data structures create problems for gorilla history to parquet.
+ if self._model_dict is not None:
+ model_dump_str = _safe_serialize(self._model_dict)
+ res["model_hash"] = _hash_id(model_dump_str)
+ res["model_dict_dumps"] = model_dump_str
+ res["root_span_dumps"] = _safe_serialize(dataclasses.asdict(self._root_span))
+ return res
+
+ def is_bound(self) -> bool:
+ return True
+
+
+class _WBTraceTreeFileType(_dtypes.Type):
+ name = "wb_trace_tree"
+ types = [WBTraceTree]
+
+
+_dtypes.TypeRegistry.add(_WBTraceTreeFileType)
+
+
+# generate a deterministic 16 character id based on input string
+def _hash_id(s: str) -> str:
+ return hashlib.md5(s.encode("utf-8")).hexdigest()[:16]
+
+
+def _fallback_serialize(obj: Any) -> str:
+ try:
+ return f"<<non-serializable: {type(obj).__qualname__}>>"
+ except Exception:
+ return "<<non-serializable>>"
+
+
+def _safe_serialize(obj: dict) -> str:
+ try:
+ return json.dumps(
+ wandb.data_types._json_helper(obj, None),
+ skipkeys=True,
+ default=_fallback_serialize,
+ )
+ except Exception:
+ return "{}"
+
+
+class TraceAttribute:
+ """Descriptor for accessing and setting attributes of the `Trace` class."""
+
+ def __set_name__(self, owner: type, name: str) -> None:
+ self.name = name
+
+ def __get__(self, instance: "Trace", owner: type) -> Any:
+ return getattr(instance._span, self.name)
+
+ def __set__(self, instance: "Trace", value: Any) -> None:
+ setattr(instance._span, self.name, value)
+
+
+class Trace:
+ """A simplification of WBTraceTree and Span to manage a trace - a collection of spans, their metadata and hierarchy.
+
+ Args:
+ name: (str) The name of the root span.
+ kind: (str, optional) The kind of the root span.
+ status_code: (str, optional) The status of the root span, either "error" or "success".
+ status_message: (str, optional) Any status message associated with the root span.
+ metadata: (dict, optional) Any additional metadata for the root span.
+ start_time_ms: (int, optional) The start time of the root span in milliseconds.
+ end_time_ms: (int, optional) The end time of the root span in milliseconds.
+ inputs: (dict, optional) The named inputs of the root span.
+ outputs: (dict, optional) The named outputs of the root span.
+ model_dict: (dict, optional) A json serializable dictionary containing the model architecture details.
+
+ Example:
+ .. code-block:: python
+ ```
+ trace = Trace(
+ name="My awesome Model",
+ kind="LLM",
+ status_code= "SUCCESS",
+ metadata={"attr_1": 1, "attr_2": 2,},
+ start_time_ms=int(round(time.time() * 1000)),
+ end_time_ms=int(round(time.time() * 1000))+1000,
+ inputs={"user": "How old is google?"},
+ outputs={"assistant": "25 years old"},
+ model_dict={"_kind": "openai", "api_type": "azure"}
+ )
+ run = wandb.init(project=<my_awesome_project>,)
+ trace.log("my_trace")
+ wandb.finish()
+ ```
+ """
+
+ name = TraceAttribute()
+ status_code = TraceAttribute()
+ status_message = TraceAttribute()
+ start_time_ms = TraceAttribute()
+ end_time_ms = TraceAttribute()
+
+ def __init__(
+ self,
+ name: str,
+ kind: Optional[str] = None,
+ status_code: Optional[str] = None,
+ status_message: Optional[str] = None,
+ metadata: Optional[dict] = None,
+ start_time_ms: Optional[int] = None,
+ end_time_ms: Optional[int] = None,
+ inputs: Optional[dict] = None,
+ outputs: Optional[dict] = None,
+ model_dict: Optional[dict] = None,
+ ):
+ self._span = self._assert_and_create_span(
+ name=name,
+ kind=kind,
+ status_code=status_code,
+ status_message=status_message,
+ metadata=metadata,
+ start_time_ms=start_time_ms,
+ end_time_ms=end_time_ms,
+ inputs=inputs,
+ outputs=outputs,
+ )
+ if model_dict is not None:
+ assert isinstance(model_dict, dict), "Model dict must be a dictionary"
+ self._model_dict = model_dict
+
+ def _assert_and_create_span(
+ self,
+ name: str,
+ kind: Optional[str] = None,
+ status_code: Optional[str] = None,
+ status_message: Optional[str] = None,
+ metadata: Optional[dict] = None,
+ start_time_ms: Optional[int] = None,
+ end_time_ms: Optional[int] = None,
+ inputs: Optional[dict] = None,
+ outputs: Optional[dict] = None,
+ ) -> Span:
+ """Utility to assert the validity of the span parameters and create a span object.
+
+ Args:
+ name: The name of the span.
+ kind: The kind of the span.
+ status_code: The status code of the span.
+ status_message: The status message of the span.
+ metadata: Dictionary of metadata to be logged with the span.
+ start_time_ms: Start time of the span in milliseconds.
+ end_time_ms: End time of the span in milliseconds.
+ inputs: Dictionary of inputs to be logged with the span.
+ outputs: Dictionary of outputs to be logged with the span.
+
+ Returns:
+ A Span object.
+ """
+ if kind is not None:
+ assert (
+ kind.upper() in SpanKind.__members__
+ ), "Invalid span kind, can be one of 'LLM', 'AGENT', 'CHAIN', 'TOOL'"
+ kind = SpanKind(kind.upper())
+ if status_code is not None:
+ assert (
+ status_code.upper() in StatusCode.__members__
+ ), "Invalid status code, can be one of 'SUCCESS' or 'ERROR'"
+ status_code = StatusCode(status_code.upper())
+ if inputs is not None:
+ assert isinstance(inputs, dict), "Inputs must be a dictionary"
+ if outputs is not None:
+ assert isinstance(outputs, dict), "Outputs must be a dictionary"
+ if inputs or outputs:
+ result = Result(inputs=inputs, outputs=outputs)
+ else:
+ result = None
+
+ if metadata is not None:
+ assert isinstance(metadata, dict), "Metadata must be a dictionary"
+
+ return Span(
+ name=name,
+ span_kind=kind,
+ status_code=status_code,
+ status_message=status_message,
+ attributes=metadata,
+ start_time_ms=start_time_ms,
+ end_time_ms=end_time_ms,
+ results=[result] if result else None,
+ )
+
+ def add_child(
+ self,
+ child: "Trace",
+ ) -> "Trace":
+ """Utility to add a child span to the current span of the trace.
+
+ Args:
+ child: The child span to be added to the current span of the trace.
+
+ Returns:
+ The current trace object with the child span added to it.
+ """
+ self._span.add_child_span(child._span)
+ if self._model_dict is not None and child._model_dict is not None:
+ self._model_dict.update({child._span.name: child._model_dict})
+ return self
+
+ def add_inputs_and_outputs(self, inputs: dict, outputs: dict) -> "Trace":
+ """Add a result to the span of the current trace.
+
+ Args:
+ inputs: Dictionary of inputs to be logged with the span.
+ outputs: Dictionary of outputs to be logged with the span.
+
+ Returns:
+ The current trace object with the result added to it.
+ """
+ if self._span.results is None:
+ result = Result(inputs=inputs, outputs=outputs)
+ self._span.results = [result]
+ else:
+ result = Result(inputs=inputs, outputs=outputs)
+ self._span.results.append(result)
+ return self
+
+ def add_metadata(self, metadata: dict) -> "Trace":
+ """Add metadata to the span of the current trace."""
+ if self._span.attributes is None:
+ self._span.attributes = metadata
+ else:
+ self._span.attributes.update(metadata)
+ return self
+
+ @property
+ def metadata(self) -> Optional[Dict[str, str]]:
+ """Get the metadata of the trace.
+
+ Returns:
+ Dictionary of metadata.
+ """
+ return self._span.attributes
+
+ @metadata.setter
+ def metadata(self, value: Dict[str, str]) -> None:
+ """Set the metadata of the trace.
+
+ Args:
+ value: Dictionary of metadata to be set.
+ """
+ if self._span.attributes is None:
+ self._span.attributes = value
+ else:
+ self._span.attributes.update(value)
+
+ @property
+ def inputs(self) -> Optional[Dict[str, str]]:
+ """Get the inputs of the trace.
+
+ Returns:
+ Dictionary of inputs.
+ """
+ return self._span.results[-1].inputs if self._span.results else None
+
+ @inputs.setter
+ def inputs(self, value: Dict[str, str]) -> None:
+ """Set the inputs of the trace.
+
+ Args:
+ value: Dictionary of inputs to be set.
+ """
+ if self._span.results is None:
+ result = Result(inputs=value, outputs={})
+ self._span.results = [result]
+ else:
+ result = Result(inputs=value, outputs=self._span.results[-1].outputs)
+ self._span.results.append(result)
+
+ @property
+ def outputs(self) -> Optional[Dict[str, str]]:
+ """Get the outputs of the trace.
+
+ Returns:
+ Dictionary of outputs.
+ """
+ return self._span.results[-1].outputs if self._span.results else None
+
+ @outputs.setter
+ def outputs(self, value: Dict[str, str]) -> None:
+ """Set the outputs of the trace.
+
+ Args:
+ value: Dictionary of outputs to be set.
+ """
+ if self._span.results is None:
+ result = Result(inputs={}, outputs=value)
+ self._span.results = [result]
+ else:
+ result = Result(inputs=self._span.results[-1].inputs, outputs=value)
+ self._span.results.append(result)
+
+ @property
+ def kind(self) -> Optional[str]:
+ """Get the kind of the trace.
+
+ Returns:
+ The kind of the trace.
+ """
+ return self._span.span_kind.value if self._span.span_kind else None
+
+ @kind.setter
+ def kind(self, value: str) -> None:
+ """Set the kind of the trace.
+
+ Args:
+ value: The kind of the trace to be set.
+ """
+ assert (
+ value.upper() in SpanKind.__members__
+ ), "Invalid span kind, can be one of 'LLM', 'AGENT', 'CHAIN', 'TOOL'"
+ self._span.span_kind = SpanKind(value.upper())
+
+ def log(self, name: str) -> None:
+ """Log the trace to a wandb run.
+
+ Args:
+ name: The name of the trace to be logged
+ """
+ trace_tree = WBTraceTree(self._span, self._model_dict)
+ assert (
+ wandb.run is not None
+ ), "You must call wandb.init() before logging a trace"
+ assert len(name.strip()) > 0, "You must provide a valid name to log the trace"
+ wandb.run.log({name: trace_tree})
diff --git a/wandb/sdk/data_types/utils.py b/wandb/sdk/data_types/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..db6ecdeee4a5e2fbaee99c86d624e7d4601f823b
--- /dev/null
+++ b/wandb/sdk/data_types/utils.py
@@ -0,0 +1,180 @@
+import logging
+import os
+import re
+from typing import TYPE_CHECKING, Optional, Sequence, Union, cast
+
+import wandb
+from wandb import util
+
+from .base_types.media import BatchableMedia, Media
+from .base_types.wb_value import WBValue
+from .image import _server_accepts_image_filenames
+from .plotly import Plotly
+
+if TYPE_CHECKING: # pragma: no cover
+ import matplotlib # type: ignore
+ import pandas as pd
+ import plotly # type: ignore
+
+ from ..wandb_run import Run as LocalRun
+
+ ValToJsonType = Union[
+ dict,
+ "WBValue",
+ Sequence["WBValue"],
+ "plotly.Figure",
+ "matplotlib.artist.Artist",
+ "pd.DataFrame",
+ object,
+ ]
+
+
+def history_dict_to_json(
+ run: Optional["LocalRun"],
+ payload: dict,
+ step: Optional[int] = None,
+ ignore_copy_err: Optional[bool] = None,
+) -> dict:
+ # Converts a History row dict's elements so they're friendly for JSON serialization.
+
+ if step is None:
+ # We should be at the top level of the History row; assume this key is set.
+ step = payload["_step"]
+
+ # We use list here because we were still seeing cases of RuntimeError dict changed size
+ for key in list(payload):
+ val = payload[key]
+ if isinstance(val, dict):
+ payload[key] = history_dict_to_json(
+ run, val, step=step, ignore_copy_err=ignore_copy_err
+ )
+ else:
+ payload[key] = val_to_json(
+ run, key, val, namespace=step, ignore_copy_err=ignore_copy_err
+ )
+
+ return payload
+
+
+# TODO: refine this
+def val_to_json(
+ run: Optional["LocalRun"],
+ key: str,
+ val: "ValToJsonType",
+ namespace: Optional[Union[str, int]] = None,
+ ignore_copy_err: Optional[bool] = None,
+) -> Union[Sequence, dict]:
+ # Converts a wandb datatype to its JSON representation.
+ if namespace is None:
+ raise ValueError(
+ "val_to_json must be called with a namespace(a step number, or 'summary') argument"
+ )
+
+ converted = val
+
+ if isinstance(val, (int, float, str, bool)):
+ # These are already JSON-serializable,
+ # no need to do the expensive checks below.
+ return converted # type: ignore[return-value]
+
+ typename = util.get_full_typename(val)
+
+ if util.is_pandas_data_frame(val):
+ val = wandb.Table(dataframe=val)
+
+ elif util.is_matplotlib_typename(typename) or util.is_plotly_typename(typename):
+ val = Plotly.make_plot_media(val)
+ elif isinstance(val, (list, tuple, range)) and all(
+ isinstance(v, WBValue) for v in val
+ ):
+ assert run
+ # This check will break down if Image/Audio/... have child classes.
+ if (
+ len(val)
+ and isinstance(val[0], BatchableMedia)
+ and all(isinstance(v, type(val[0])) for v in val)
+ ):
+ if TYPE_CHECKING:
+ val = cast(Sequence["BatchableMedia"], val)
+
+ items = _prune_max_seq(val)
+
+ if _server_accepts_image_filenames():
+ for item in items:
+ item.bind_to_run(
+ run=run,
+ key=key,
+ step=namespace,
+ ignore_copy_err=ignore_copy_err,
+ )
+ else:
+ for i, item in enumerate(items):
+ item.bind_to_run(
+ run=run,
+ key=key,
+ step=namespace,
+ id_=i,
+ ignore_copy_err=ignore_copy_err,
+ )
+ if run._attach_id and run._init_pid != os.getpid():
+ wandb.termwarn(
+ f"Attempting to log a sequence of {items[0].__class__.__name__} objects from multiple processes might result in data loss. Please upgrade your wandb server",
+ repeat=False,
+ )
+
+ return items[0].seq_to_json(items, run, key, namespace)
+ else:
+ # TODO(adrian): Good idea to pass on the same key here? Maybe include
+ # the array index?
+ # There is a bug here: if this array contains two arrays of the same type of
+ # anonymous media objects, their eventual names will collide.
+ # This used to happen. The frontend doesn't handle heterogeneous arrays
+ # raise ValueError(
+ # "Mixed media types in the same list aren't supported")
+ return [
+ val_to_json(
+ run, key, v, namespace=namespace, ignore_copy_err=ignore_copy_err
+ )
+ for v in val
+ ]
+
+ if isinstance(val, WBValue):
+ assert run
+ if isinstance(val, Media) and not val.is_bound():
+ if hasattr(val, "_log_type") and val._log_type in [
+ "table",
+ "partitioned-table",
+ "joined-table",
+ ]:
+ # Special conditional to log tables as artifact entries as well.
+ # I suspect we will generalize this as we transition to storing all
+ # files in an artifact
+ # we sanitize the key to meet the constraints
+ # in this case, leaving only alphanumerics or underscores.
+ sanitized_key = re.sub(r"[^a-zA-Z0-9_]+", "", key)
+ art = wandb.Artifact(f"run-{run.id}-{sanitized_key}", "run_table")
+ art.add(val, key)
+ run.log_artifact(art)
+
+ # Partitioned tables and joined tables do not support being bound to runs.
+ if not (
+ hasattr(val, "_log_type")
+ and val._log_type in ["partitioned-table", "joined-table"]
+ ):
+ val.bind_to_run(run, key, namespace)
+
+ return val.to_json(run)
+
+ return converted # type: ignore
+
+
+def _prune_max_seq(seq: Sequence["BatchableMedia"]) -> Sequence["BatchableMedia"]:
+ # If media type has a max respect it
+ items = seq
+ if hasattr(seq[0], "MAX_ITEMS") and seq[0].MAX_ITEMS < len(seq):
+ logging.warning(
+ "Only %i %s will be uploaded."
+ % (seq[0].MAX_ITEMS, seq[0].__class__.__name__)
+ )
+ items = seq[: seq[0].MAX_ITEMS]
+ return items
diff --git a/wandb/sdk/data_types/video.py b/wandb/sdk/data_types/video.py
new file mode 100644
index 0000000000000000000000000000000000000000..f660913b721e94900e7e257e24d50f47c792c371
--- /dev/null
+++ b/wandb/sdk/data_types/video.py
@@ -0,0 +1,245 @@
+import logging
+import os
+from io import BytesIO
+from typing import TYPE_CHECKING, Any, Dict, Optional, Sequence, Type, Union
+
+from wandb import util
+from wandb.sdk.lib import filesystem, runid
+
+from . import _dtypes
+from ._private import MEDIA_TMP
+from .base_types.media import BatchableMedia
+
+if TYPE_CHECKING: # pragma: no cover
+ from typing import TextIO
+
+ import numpy as np
+
+ from wandb.sdk.artifacts.artifact import Artifact
+
+ from ..wandb_run import Run as LocalRun
+
+
+# This helper function is a workaround for the issue discussed here:
+# https://github.com/wandb/wandb/issues/3472
+#
+# Essentially, the issue is that moviepy's write_gif function fails to close
+# the open write / file descripter returned from `imageio.save`. The following
+# function is a simplified copy of the function in the moviepy source code.
+# See https://github.com/Zulko/moviepy/blob/7e3e8bb1b739eb6d1c0784b0cb2594b587b93b39/moviepy/video/io/gif_writers.py#L428
+#
+# Except, we close the writer!
+def write_gif_with_image_io(
+ clip: Any, filename: str, fps: Optional[int] = None
+) -> None:
+ imageio = util.get_module(
+ "imageio",
+ required='wandb.Video requires imageio when passing raw data. Install with "pip install imageio"',
+ )
+
+ writer = imageio.save(filename, fps=clip.fps, quantizer=0, palettesize=256, loop=0)
+
+ for frame in clip.iter_frames(fps=fps, dtype="uint8"):
+ writer.append_data(frame)
+
+ writer.close()
+
+
+class Video(BatchableMedia):
+ """Format a video for logging to W&B.
+
+ Arguments:
+ data_or_path: (numpy array, string, io)
+ Video can be initialized with a path to a file or an io object.
+ The format must be "gif", "mp4", "webm" or "ogg".
+ The format must be specified with the format argument.
+ Video can be initialized with a numpy tensor.
+ The numpy tensor must be either 4 dimensional or 5 dimensional.
+ Channels should be (time, channel, height, width) or
+ (batch, time, channel, height width)
+ caption: (string) caption associated with the video for display
+ fps: (int) frames per second for video. Default is 4.
+ format: (string) format of video, necessary if initializing with path or io object.
+
+ Examples:
+ ### Log a numpy array as a video
+ <!--yeadoc-test:log-video-numpy-->
+ ```python
+ import numpy as np
+ import wandb
+
+ wandb.init()
+ # axes are (time, channel, height, width)
+ frames = np.random.randint(low=0, high=256, size=(10, 3, 100, 100), dtype=np.uint8)
+ wandb.log({"video": wandb.Video(frames, fps=4)})
+ ```
+ """
+
+ _log_type = "video-file"
+ EXTS = ("gif", "mp4", "webm", "ogg")
+ _width: Optional[int]
+ _height: Optional[int]
+
+ def __init__(
+ self,
+ data_or_path: Union["np.ndarray", str, "TextIO", "BytesIO"],
+ caption: Optional[str] = None,
+ fps: int = 4,
+ format: Optional[str] = None,
+ ):
+ super().__init__()
+
+ self._fps = fps
+ self._format = format or "gif"
+ self._width = None
+ self._height = None
+ self._channels = None
+ self._caption = caption
+ if self._format not in Video.EXTS:
+ raise ValueError("wandb.Video accepts %s formats" % ", ".join(Video.EXTS))
+
+ if isinstance(data_or_path, BytesIO):
+ filename = os.path.join(
+ MEDIA_TMP.name, runid.generate_id() + "." + self._format
+ )
+ with open(filename, "wb") as f:
+ f.write(data_or_path.read())
+ self._set_file(filename, is_tmp=True)
+ elif isinstance(data_or_path, str):
+ _, ext = os.path.splitext(data_or_path)
+ ext = ext[1:].lower()
+ if ext not in Video.EXTS:
+ raise ValueError(
+ "wandb.Video accepts %s formats" % ", ".join(Video.EXTS)
+ )
+ self._set_file(data_or_path, is_tmp=False)
+ # ffprobe -v error -select_streams v:0 -show_entries stream=width,height -of csv=p=0 data_or_path
+ else:
+ if hasattr(data_or_path, "numpy"): # TF data eager tensors
+ self.data = data_or_path.numpy()
+ elif util.is_numpy_array(data_or_path):
+ self.data = data_or_path
+ else:
+ raise ValueError(
+ "wandb.Video accepts a file path or numpy like data as input"
+ )
+ self.encode()
+
+ def encode(self) -> None:
+ mpy = util.get_module(
+ "moviepy.editor",
+ required='wandb.Video requires moviepy and imageio when passing raw data. Install with "pip install moviepy imageio"',
+ )
+ tensor = self._prepare_video(self.data)
+ _, self._height, self._width, self._channels = tensor.shape # type: ignore
+
+ # encode sequence of images into gif string
+ clip = mpy.ImageSequenceClip(list(tensor), fps=self._fps)
+
+ filename = os.path.join(
+ MEDIA_TMP.name, runid.generate_id() + "." + self._format
+ )
+ if TYPE_CHECKING:
+ kwargs: Dict[str, Optional[bool]] = {}
+ try: # older versions of moviepy do not support logger argument
+ kwargs = {"logger": None}
+ if self._format == "gif":
+ write_gif_with_image_io(clip, filename)
+ else:
+ clip.write_videofile(filename, **kwargs)
+ except TypeError:
+ try: # even older versions of moviepy do not support progress_bar argument
+ kwargs = {"verbose": False, "progress_bar": False}
+ if self._format == "gif":
+ clip.write_gif(filename, **kwargs)
+ else:
+ clip.write_videofile(filename, **kwargs)
+ except TypeError:
+ kwargs = {
+ "verbose": False,
+ }
+ if self._format == "gif":
+ clip.write_gif(filename, **kwargs)
+ else:
+ clip.write_videofile(filename, **kwargs)
+ self._set_file(filename, is_tmp=True)
+
+ @classmethod
+ def get_media_subdir(cls: Type["Video"]) -> str:
+ return os.path.join("media", "videos")
+
+ def to_json(self, run_or_artifact: Union["LocalRun", "Artifact"]) -> dict:
+ json_dict = super().to_json(run_or_artifact)
+ json_dict["_type"] = self._log_type
+
+ if self._width is not None:
+ json_dict["width"] = self._width
+ if self._height is not None:
+ json_dict["height"] = self._height
+ if self._caption:
+ json_dict["caption"] = self._caption
+
+ return json_dict
+
+ def _prepare_video(self, video: "np.ndarray") -> "np.ndarray":
+ """This logic was mostly taken from tensorboardX."""
+ np = util.get_module(
+ "numpy",
+ required='wandb.Video requires numpy when passing raw data. To get it, run "pip install numpy".',
+ )
+ if video.ndim < 4:
+ raise ValueError(
+ "Video must be atleast 4 dimensions: time, channels, height, width"
+ )
+ if video.ndim == 4:
+ video = video.reshape(1, *video.shape)
+ b, t, c, h, w = video.shape
+
+ if video.dtype != np.uint8:
+ logging.warning("Converting video data to uint8")
+ video = video.astype(np.uint8)
+
+ def is_power2(num: int) -> bool:
+ return num != 0 and ((num & (num - 1)) == 0)
+
+ # pad to nearest power of 2, all at once
+ if not is_power2(video.shape[0]):
+ len_addition = int(2 ** video.shape[0].bit_length() - video.shape[0])
+ video = np.concatenate(
+ (video, np.zeros(shape=(len_addition, t, c, h, w))), axis=0
+ )
+
+ n_rows = 2 ** ((b.bit_length() - 1) // 2)
+ n_cols = video.shape[0] // n_rows
+
+ video = np.reshape(video, newshape=(n_rows, n_cols, t, c, h, w))
+ video = np.transpose(video, axes=(2, 0, 4, 1, 5, 3))
+ video = np.reshape(video, newshape=(t, n_rows * h, n_cols * w, c))
+ return video
+
+ @classmethod
+ def seq_to_json(
+ cls: Type["Video"],
+ seq: Sequence["BatchableMedia"],
+ run: "LocalRun",
+ key: str,
+ step: Union[int, str],
+ ) -> dict:
+ base_path = os.path.join(run.dir, cls.get_media_subdir())
+ filesystem.mkdir_exists_ok(base_path)
+
+ meta = {
+ "_type": "videos",
+ "count": len(seq),
+ "videos": [v.to_json(run) for v in seq],
+ "captions": Video.captions(seq),
+ }
+ return meta
+
+
+class _VideoFileType(_dtypes.Type):
+ name = "video-file"
+ types = [Video]
+
+
+_dtypes.TypeRegistry.add(_VideoFileType)
diff --git a/wandb/sdk/integration_utils/__init__.py b/wandb/sdk/integration_utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sdk/integration_utils/auto_logging.py b/wandb/sdk/integration_utils/auto_logging.py
new file mode 100644
index 0000000000000000000000000000000000000000..19042f9ac69b4d9b04e700597e20a6ebb8c575cd
--- /dev/null
+++ b/wandb/sdk/integration_utils/auto_logging.py
@@ -0,0 +1,240 @@
+import asyncio
+import functools
+import inspect
+import logging
+import sys
+from typing import Any, Dict, Optional, Sequence, TypeVar
+
+import wandb.sdk
+import wandb.util
+from wandb.sdk.lib import telemetry as wb_telemetry
+from wandb.sdk.lib.timer import Timer
+
+if sys.version_info >= (3, 8):
+ from typing import Protocol
+else:
+ from typing_extensions import Protocol
+
+
+logger = logging.getLogger(__name__)
+
+
+AutologInitArgs = Optional[Dict[str, Any]]
+
+
+K = TypeVar("K", bound=str)
+V = TypeVar("V")
+
+
+class Response(Protocol[K, V]):
+ def __getitem__(self, key: K) -> V:
+ ... # pragma: no cover
+
+ def get(self, key: K, default: Optional[V] = None) -> Optional[V]:
+ ... # pragma: no cover
+
+
+class ArgumentResponseResolver(Protocol):
+ def __call__(
+ self,
+ args: Sequence[Any],
+ kwargs: Dict[str, Any],
+ response: Response,
+ start_time: float,
+ time_elapsed: float,
+ ) -> Optional[Dict[str, Any]]:
+ ... # pragma: no cover
+
+
+class PatchAPI:
+ def __init__(
+ self,
+ name: str,
+ symbols: Sequence[str],
+ resolver: ArgumentResponseResolver,
+ ) -> None:
+ """Patches the API to log wandb Media or metrics."""
+ # name of the LLM provider, e.g. "Cohere" or "OpenAI" or package name like "Transformers"
+ self.name = name
+ # api library name, e.g. "cohere" or "openai" or "transformers"
+ self._api = None
+ # dictionary of original methods
+ self.original_methods: Dict[str, Any] = {}
+ # list of symbols to patch, e.g. ["Client.generate", "Edit.create"] or ["Pipeline.__call__"]
+ self.symbols = symbols
+ # resolver callable to convert args/response into a dictionary of wandb media objects or metrics
+ self.resolver = resolver
+
+ @property
+ def set_api(self) -> Any:
+ """Returns the API module."""
+ lib_name = self.name.lower()
+ if self._api is None:
+ self._api = wandb.util.get_module(
+ name=lib_name,
+ required=f"To use the W&B {self.name} Autolog, "
+ f"you need to have the `{lib_name}` python "
+ f"package installed. Please install it with `pip install {lib_name}`.",
+ lazy=False,
+ )
+ return self._api
+
+ def patch(self, run: "wandb.sdk.wandb_run.Run") -> None:
+ """Patches the API to log media or metrics to W&B."""
+ for symbol in self.symbols:
+ # split on dots, e.g. "Client.generate" -> ["Client", "generate"]
+ symbol_parts = symbol.split(".")
+ # and get the attribute from the module
+ original = functools.reduce(getattr, symbol_parts, self.set_api)
+
+ def method_factory(original_method: Any):
+ async def async_method(*args, **kwargs):
+ future = asyncio.Future()
+
+ async def callback(coro):
+ try:
+ result = await coro
+ loggable_dict = self.resolver(
+ args, kwargs, result, timer.start_time, timer.elapsed
+ )
+ if loggable_dict is not None:
+ run.log(loggable_dict)
+ future.set_result(result)
+ except Exception as e:
+ logger.warning(e)
+
+ with Timer() as timer:
+ coro = original_method(*args, **kwargs)
+ asyncio.ensure_future(callback(coro))
+
+ return await future
+
+ def sync_method(*args, **kwargs):
+ with Timer() as timer:
+ result = original_method(*args, **kwargs)
+ try:
+ loggable_dict = self.resolver(
+ args, kwargs, result, timer.start_time, timer.elapsed
+ )
+ if loggable_dict is not None:
+ run.log(loggable_dict)
+ except Exception as e:
+ logger.warning(e)
+ return result
+
+ if inspect.iscoroutinefunction(original_method):
+ return functools.wraps(original_method)(async_method)
+ else:
+ return functools.wraps(original_method)(sync_method)
+
+ # save original method
+ self.original_methods[symbol] = original
+ # monkey patch the method
+ if len(symbol_parts) == 1:
+ setattr(self.set_api, symbol_parts[0], method_factory(original))
+ else:
+ setattr(
+ functools.reduce(getattr, symbol_parts[:-1], self.set_api),
+ symbol_parts[-1],
+ method_factory(original),
+ )
+
+ def unpatch(self) -> None:
+ """Unpatches the API."""
+ for symbol, original in self.original_methods.items():
+ # split on dots, e.g. "Client.generate" -> ["Client", "generate"]
+ symbol_parts = symbol.split(".")
+ # unpatch the method
+ if len(symbol_parts) == 1:
+ setattr(self.set_api, symbol_parts[0], original)
+ else:
+ setattr(
+ functools.reduce(getattr, symbol_parts[:-1], self.set_api),
+ symbol_parts[-1],
+ original,
+ )
+
+
+class AutologAPI:
+ def __init__(
+ self,
+ name: str,
+ symbols: Sequence[str],
+ resolver: ArgumentResponseResolver,
+ telemetry_feature: Optional[str] = None,
+ ) -> None:
+ """Autolog API calls to W&B."""
+ self._telemetry_feature = telemetry_feature
+ self._patch_api = PatchAPI(
+ name=name,
+ symbols=symbols,
+ resolver=resolver,
+ )
+ self._name = self._patch_api.name
+ self._run: Optional[wandb.sdk.wandb_run.Run] = None
+ self.__run_created_by_autolog: bool = False
+
+ @property
+ def _is_enabled(self) -> bool:
+ """Returns whether autologging is enabled."""
+ return self._run is not None
+
+ def __call__(self, init: AutologInitArgs = None) -> None:
+ """Enable autologging."""
+ self.enable(init=init)
+
+ def _run_init(self, init: AutologInitArgs = None) -> None:
+ """Handle wandb run initialization."""
+ # - autolog(init: dict = {...}) calls wandb.init(**{...})
+ # regardless of whether there is a wandb.run or not,
+ # we only track if the run was created by autolog
+ # - todo: autolog(init: dict | run = run) would use the user-provided run
+ # - autolog() uses the wandb.run if there is one, otherwise it calls wandb.init()
+ if init:
+ _wandb_run = wandb.run
+ # we delegate dealing with the init dict to wandb.init()
+ self._run = wandb.init(**init)
+ if _wandb_run != self._run:
+ self.__run_created_by_autolog = True
+ elif wandb.run is None:
+ self._run = wandb.init()
+ self.__run_created_by_autolog = True
+ else:
+ self._run = wandb.run
+
+ def enable(self, init: AutologInitArgs = None) -> None:
+ """Enable autologging.
+
+ Args:
+ init: Optional dictionary of arguments to pass to wandb.init().
+
+ """
+ if self._is_enabled:
+ logger.info(
+ f"{self._name} autologging is already enabled, disabling and re-enabling."
+ )
+ self.disable()
+
+ logger.info(f"Enabling {self._name} autologging.")
+ self._run_init(init=init)
+
+ self._patch_api.patch(self._run)
+
+ if self._telemetry_feature:
+ with wb_telemetry.context(self._run) as tel:
+ setattr(tel.feature, self._telemetry_feature, True)
+
+ def disable(self) -> None:
+ """Disable autologging."""
+ if self._run is None:
+ return
+
+ logger.info(f"Disabling {self._name} autologging.")
+
+ if self.__run_created_by_autolog:
+ self._run.finish()
+ self.__run_created_by_autolog = False
+
+ self._run = None
+
+ self._patch_api.unpatch()
diff --git a/wandb/sdk/integration_utils/data_logging.py b/wandb/sdk/integration_utils/data_logging.py
new file mode 100644
index 0000000000000000000000000000000000000000..d3bd027165998dd5012d5fd1cb1c07b54b87dcb6
--- /dev/null
+++ b/wandb/sdk/integration_utils/data_logging.py
@@ -0,0 +1,471 @@
+# wandb.integrations.data_logging.py
+#
+# Contains common utility functions that enable
+# logging datasets and predictions to wandb.
+import sys
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
+
+import wandb
+
+if TYPE_CHECKING:
+ from wandb.data_types import _TableIndex
+
+CAN_INFER_IMAGE_AND_VIDEO = sys.version_info.major == 3 and sys.version_info.minor >= 5
+
+
+class ValidationDataLogger:
+ """Logs validation data as a wandb.Table.
+
+ ValidationDataLogger is intended to be used inside of library integrations
+ in order to facilitate the process of optionally building a validation dataset
+ and logging periodic predictions against such validation data using WandB best
+ practices.
+ """
+
+ validation_inputs: Union[Sequence, Dict[str, Sequence]]
+ validation_targets: Optional[Union[Sequence, Dict[str, Sequence]]]
+ validation_indexes: List["_TableIndex"]
+ prediction_row_processor: Optional[Callable]
+ class_labels_table: Optional["wandb.Table"]
+ infer_missing_processors: bool
+
+ def __init__(
+ self,
+ inputs: Union[Sequence, Dict[str, Sequence]],
+ targets: Optional[Union[Sequence, Dict[str, Sequence]]] = None,
+ indexes: Optional[List["_TableIndex"]] = None,
+ validation_row_processor: Optional[Callable] = None,
+ prediction_row_processor: Optional[Callable] = None,
+ input_col_name: str = "input",
+ target_col_name: str = "target",
+ table_name: str = "wb_validation_data",
+ artifact_type: str = "validation_dataset",
+ class_labels: Optional[List[str]] = None,
+ infer_missing_processors: bool = True,
+ ) -> None:
+ """Initialize a new ValidationDataLogger.
+
+ Args:
+ inputs: A list of input vectors or dictionary of lists of input vectors
+ (used if the model has multiple named inputs)
+ targets: A list of target vectors or dictionary of lists of target vectors
+ (used if the model has multiple named targets/putputs). Defaults to `None`.
+ `targets` and `indexes` cannot both be `None`.
+ indexes: An ordered list of `wandb.data_types._TableIndex` mapping the
+ input items to their source table. This is most commonly retrieved by using
+ `indexes = my_data_table.get_index()`. Defaults to `None`. `targets`
+ and `indexes` cannot both be `None`.
+ validation_row_processor: A function to apply to the validation data,
+ commonly used to visualize the data. The function will receive an `ndx` (`int`)
+ and a `row` (`dict`). If `inputs` is a list, then `row["input"]` will be the input
+ data for the row. Else, it will be keyed based on the name of the input slot
+ (corresponding to `inputs`). If `targets` is a list, then
+ `row["target"]` will be the target data for the row. Else, it will
+ be keyed based on `targets`. For example, if your input data is a
+ single ndarray, but you wish to visualize the data as an image,
+ then you can provide `lambda ndx, row: {"img": wandb.Image(row["input"])}`
+ as the processor. If `None`, we will try to guess the appropriate processor.
+ Ignored if `log_evaluation` is `False` or `val_keys` are present. Defaults to `None`.
+ prediction_row_processor: Same as validation_row_processor, but applied to the
+ model's output. `row["output"]` will contain the results of the model output.
+ Defaults to `None`.
+ input_col_name: The name to use for the input column.
+ Defaults to `"input"`.
+ target_col_name: The name to use for the target column.
+ Defaults to `"target"`.
+ table_name: The name to use for the validation table.
+ Defaults to `"wb_validation_data"`.
+ artifact_type: The artifact type to use for the validation data.
+ Defaults to `"validation_dataset"`.
+ class_labels: Optional list of lables to use in the inferred
+ processors. If the model's `target` or `output` is inferred to be a class,
+ we will attempt to map the class to these labels. Defaults to `None`.
+ infer_missing_processors: Determines if processors are inferred if
+ they are missing. Defaults to True.
+ """
+ class_labels_table: Optional[wandb.Table]
+ if isinstance(class_labels, list) and len(class_labels) > 0:
+ class_labels_table = wandb.Table(
+ columns=["label"], data=[[label] for label in class_labels]
+ )
+ else:
+ class_labels_table = None
+
+ if indexes is None:
+ assert targets is not None
+ local_validation_table = wandb.Table(columns=[], data=[])
+
+ if isinstance(targets, dict):
+ for col_name in targets:
+ local_validation_table.add_column(col_name, targets[col_name])
+ else:
+ local_validation_table.add_column(target_col_name, targets)
+
+ if isinstance(inputs, dict):
+ for col_name in inputs:
+ local_validation_table.add_column(col_name, inputs[col_name])
+ else:
+ local_validation_table.add_column(input_col_name, inputs)
+
+ if validation_row_processor is None and infer_missing_processors:
+ example_input = _make_example(inputs)
+ example_target = _make_example(targets)
+ if example_input is not None and example_target is not None:
+ validation_row_processor = _infer_validation_row_processor(
+ example_input,
+ example_target,
+ class_labels_table,
+ input_col_name,
+ target_col_name,
+ )
+
+ if validation_row_processor is not None:
+ local_validation_table.add_computed_columns(validation_row_processor)
+
+ local_validation_artifact = wandb.Artifact(table_name, artifact_type)
+ local_validation_artifact.add(local_validation_table, "validation_data")
+ if wandb.run:
+ wandb.run.use_artifact(local_validation_artifact)
+ indexes = local_validation_table.get_index()
+ else:
+ local_validation_artifact = None
+
+ self.class_labels_table = class_labels_table
+ self.validation_inputs = inputs
+ self.validation_targets = targets
+ self.validation_indexes = indexes
+ self.prediction_row_processor = prediction_row_processor
+ self.infer_missing_processors = infer_missing_processors
+ self.local_validation_artifact = local_validation_artifact
+ self.input_col_name = input_col_name
+
+ def make_predictions(
+ self, predict_fn: Callable
+ ) -> Union[Sequence, Dict[str, Sequence]]:
+ """Produce predictions by passing `validation_inputs` to `predict_fn`.
+
+ Args:
+ predict_fn (Callable): Any function which can accept `validation_inputs` and produce
+ a list of vectors or dictionary of lists of vectors
+
+ Returns:
+ (Sequence | Dict[str, Sequence]): The returned value of predict_fn
+ """
+ return predict_fn(self.validation_inputs)
+
+ def log_predictions(
+ self,
+ predictions: Union[Sequence, Dict[str, Sequence]],
+ prediction_col_name: str = "output",
+ val_ndx_col_name: str = "val_row",
+ table_name: str = "validation_predictions",
+ commit: bool = True,
+ ) -> wandb.data_types.Table:
+ """Log a set of predictions.
+
+ Intended usage:
+
+ vl.log_predictions(vl.make_predictions(self.model.predict))
+
+ Args:
+ predictions (Sequence | Dict[str, Sequence]): A list of prediction vectors or dictionary
+ of lists of prediction vectors
+ prediction_col_name (str, optional): the name of the prediction column. Defaults to "output".
+ val_ndx_col_name (str, optional): The name of the column linking prediction table
+ to the validation ata table. Defaults to "val_row".
+ table_name (str, optional): name of the prediction table. Defaults to "validation_predictions".
+ commit (bool, optional): determines if commit should be called on the logged data. Defaults to False.
+ """
+ pred_table = wandb.Table(columns=[], data=[])
+ if isinstance(predictions, dict):
+ for col_name in predictions:
+ pred_table.add_column(col_name, predictions[col_name])
+ else:
+ pred_table.add_column(prediction_col_name, predictions)
+ pred_table.add_column(val_ndx_col_name, self.validation_indexes)
+
+ if self.prediction_row_processor is None and self.infer_missing_processors:
+ example_prediction = _make_example(predictions)
+ example_input = _make_example(self.validation_inputs)
+ if example_prediction is not None and example_input is not None:
+ self.prediction_row_processor = _infer_prediction_row_processor(
+ example_prediction,
+ example_input,
+ self.class_labels_table,
+ self.input_col_name,
+ prediction_col_name,
+ )
+
+ if self.prediction_row_processor is not None:
+ pred_table.add_computed_columns(self.prediction_row_processor)
+
+ wandb.log({table_name: pred_table}, commit=commit)
+ return pred_table
+
+
+def _make_example(data: Any) -> Optional[Union[Dict, Sequence, Any]]:
+ """Used to make an example input, target, or output."""
+ example: Optional[Union[Dict, Sequence, Any]]
+
+ if isinstance(data, dict):
+ example = {}
+ for key in data:
+ example[key] = data[key][0]
+ elif hasattr(data, "__len__"):
+ example = data[0]
+ else:
+ example = None
+
+ return example
+
+
+def _get_example_shape(example: Union[Sequence, Any]):
+ """Get the shape of an object if applicable."""
+ shape = []
+ if not isinstance(example, str) and hasattr(example, "__len__"):
+ length = len(example)
+ shape = [length]
+ if length > 0:
+ shape += _get_example_shape(example[0])
+ return shape
+
+
+def _bind(lambda_fn: Callable, **closure_kwargs: Any) -> Callable:
+ """Create a closure around a lambda function by binding `closure_kwargs` to the function."""
+
+ def closure(*args: Any, **kwargs: Any) -> Any:
+ _k = {}
+ _k.update(kwargs)
+ _k.update(closure_kwargs)
+ return lambda_fn(*args, **_k)
+
+ return closure
+
+
+def _infer_single_example_keyed_processor(
+ example: Union[Sequence, Any],
+ class_labels_table: Optional["wandb.Table"] = None,
+ possible_base_example: Optional[Union[Sequence, Any]] = None,
+) -> Dict[str, Callable]:
+ """Infers a processor from a single example.
+
+ Infers a processor from a single example with optional class_labels_table
+ and base_example. Base example is useful for cases such as segmentation masks
+ """
+ shape = _get_example_shape(example)
+ processors: Dict[str, Callable] = {}
+ if (
+ class_labels_table is not None
+ and len(shape) == 1
+ and shape[0] == len(class_labels_table.data)
+ ):
+ np = wandb.util.get_module(
+ "numpy",
+ required="Infering processors require numpy",
+ )
+ # Assume these are logits
+ class_names = class_labels_table.get_column("label")
+
+ processors["max_class"] = lambda n, d, p: class_labels_table.index_ref( # type: ignore
+ np.argmax(d)
+ )
+ # TODO: Consider adding back if users ask
+ # processors["min_class"] = lambda n, d, p: class_labels_table.index_ref( # type: ignore
+ # np.argmin(d)
+ # )
+
+ values = np.unique(example)
+ is_one_hot = len(values) == 2 and set(values) == {0, 1}
+ if not is_one_hot:
+ processors["score"] = lambda n, d, p: {
+ class_names[i]: d[i] for i in range(shape[0])
+ }
+ elif (
+ len(shape) == 1
+ and shape[0] == 1
+ and (
+ isinstance(example[0], int)
+ or (hasattr(example, "tolist") and isinstance(example.tolist()[0], int)) # type: ignore
+ )
+ ):
+ # assume this is a class
+ if class_labels_table is not None:
+ processors["class"] = lambda n, d, p: class_labels_table.index_ref(d[0]) if d[0] < len(class_labels_table.data) else d[0] # type: ignore
+ else:
+ processors["val"] = lambda n, d, p: d[0]
+ elif len(shape) == 1:
+ np = wandb.util.get_module(
+ "numpy",
+ required="Infering processors require numpy",
+ )
+ # This could be anything
+ if shape[0] <= 10:
+ # if less than 10, fan out the results
+ # processors["node"] = lambda n, d, p: {i: d[i] for i in range(shape[0])}
+ processors["node"] = lambda n, d, p: [
+ d[i].tolist() if hasattr(d[i], "tolist") else d[i]
+ for i in range(shape[0])
+ ]
+ # just report the argmax and argmin
+ processors["argmax"] = lambda n, d, p: np.argmax(d)
+
+ values = np.unique(example)
+ is_one_hot = len(values) == 2 and set(values) == {0, 1}
+ if not is_one_hot:
+ processors["argmin"] = lambda n, d, p: np.argmin(d)
+ elif len(shape) == 2 and CAN_INFER_IMAGE_AND_VIDEO:
+ if (
+ class_labels_table is not None
+ and possible_base_example is not None
+ and shape == _get_example_shape(possible_base_example)
+ ):
+ # consider this a segmentation mask
+ processors["image"] = lambda n, d, p: wandb.Image(
+ p,
+ masks={
+ "masks": {
+ "mask_data": d,
+ "class_labels": class_labels_table.get_column("label"), # type: ignore
+ }
+ },
+ )
+ else:
+ # consider this a 2d image
+ processors["image"] = lambda n, d, p: wandb.Image(d)
+ elif len(shape) == 3 and CAN_INFER_IMAGE_AND_VIDEO:
+ # consider this an image
+ processors["image"] = lambda n, d, p: wandb.Image(d)
+ elif len(shape) == 4 and CAN_INFER_IMAGE_AND_VIDEO:
+ # consider this a video
+ processors["video"] = lambda n, d, p: wandb.Video(d)
+
+ return processors
+
+
+def _infer_validation_row_processor(
+ example_input: Union[Dict, Sequence],
+ example_target: Union[Dict, Sequence, Any],
+ class_labels_table: Optional["wandb.Table"] = None,
+ input_col_name: str = "input",
+ target_col_name: str = "target",
+) -> Callable:
+ """Infers the composit processor for the validation data."""
+ single_processors = {}
+ if isinstance(example_input, dict):
+ for key in example_input:
+ key_processors = _infer_single_example_keyed_processor(example_input[key])
+ for p_key in key_processors:
+ single_processors[f"{key}:{p_key}"] = _bind(
+ lambda ndx, row, key_processor, key: key_processor(
+ ndx,
+ row[key],
+ None,
+ ),
+ key_processor=key_processors[p_key],
+ key=key,
+ )
+ else:
+ key = input_col_name
+ key_processors = _infer_single_example_keyed_processor(example_input)
+ for p_key in key_processors:
+ single_processors[f"{key}:{p_key}"] = _bind(
+ lambda ndx, row, key_processor, key: key_processor(
+ ndx,
+ row[key],
+ None,
+ ),
+ key_processor=key_processors[p_key],
+ key=key,
+ )
+
+ if isinstance(example_target, dict):
+ for key in example_target:
+ key_processors = _infer_single_example_keyed_processor(
+ example_target[key], class_labels_table
+ )
+ for p_key in key_processors:
+ single_processors[f"{key}:{p_key}"] = _bind(
+ lambda ndx, row, key_processor, key: key_processor(
+ ndx,
+ row[key],
+ None,
+ ),
+ key_processor=key_processors[p_key],
+ key=key,
+ )
+ else:
+ key = target_col_name
+ key_processors = _infer_single_example_keyed_processor(
+ example_target,
+ class_labels_table,
+ example_input if not isinstance(example_input, dict) else None,
+ )
+ for p_key in key_processors:
+ single_processors[f"{key}:{p_key}"] = _bind(
+ lambda ndx, row, key_processor, key: key_processor(
+ ndx,
+ row[key],
+ row[input_col_name]
+ if not isinstance(example_input, dict)
+ else None,
+ ),
+ key_processor=key_processors[p_key],
+ key=key,
+ )
+
+ def processor(ndx, row):
+ return {key: single_processors[key](ndx, row) for key in single_processors}
+
+ return processor
+
+
+def _infer_prediction_row_processor(
+ example_prediction: Union[Dict, Sequence],
+ example_input: Union[Dict, Sequence],
+ class_labels_table: Optional["wandb.Table"] = None,
+ input_col_name: str = "input",
+ output_col_name: str = "output",
+) -> Callable:
+ """Infers the composit processor for the prediction output data."""
+ single_processors = {}
+
+ if isinstance(example_prediction, dict):
+ for key in example_prediction:
+ key_processors = _infer_single_example_keyed_processor(
+ example_prediction[key], class_labels_table
+ )
+ for p_key in key_processors:
+ single_processors[f"{key}:{p_key}"] = _bind(
+ lambda ndx, row, key_processor, key: key_processor(
+ ndx,
+ row[key],
+ None,
+ ),
+ key_processor=key_processors[p_key],
+ key=key,
+ )
+ else:
+ key = output_col_name
+ key_processors = _infer_single_example_keyed_processor(
+ example_prediction,
+ class_labels_table,
+ example_input if not isinstance(example_input, dict) else None,
+ )
+ for p_key in key_processors:
+ single_processors[f"{key}:{p_key}"] = _bind(
+ lambda ndx, row, key_processor, key: key_processor(
+ ndx,
+ row[key],
+ ndx.get_row().get("val_row").get_row().get(input_col_name)
+ if not isinstance(example_input, dict)
+ else None,
+ ),
+ key_processor=key_processors[p_key],
+ key=key,
+ )
+
+ def processor(ndx, row):
+ return {key: single_processors[key](ndx, row) for key in single_processors}
+
+ return processor
diff --git a/wandb/sdk/interface/__init__.py b/wandb/sdk/interface/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sdk/interface/constants.py b/wandb/sdk/interface/constants.py
new file mode 100644
index 0000000000000000000000000000000000000000..09fe1e26f0ce72e23df8b627ff15e332f6ebc4e6
--- /dev/null
+++ b/wandb/sdk/interface/constants.py
@@ -0,0 +1,4 @@
+#
+NOTIFY_PROCESS = 1
+NOTIFY_SHUTDOWN = 2
+NOTIFY_REQUEST = 3
diff --git a/wandb/sdk/interface/interface.py b/wandb/sdk/interface/interface.py
new file mode 100644
index 0000000000000000000000000000000000000000..da8278c98899421592acb9b736dd2662449ed0cf
--- /dev/null
+++ b/wandb/sdk/interface/interface.py
@@ -0,0 +1,878 @@
+"""Interface base class - Used to send messages to the internal process.
+
+InterfaceBase: The abstract class
+InterfaceShared: Common routines for socket and queue based implementations
+InterfaceQueue: Use multiprocessing queues to send and receive messages
+InterfaceSock: Use socket to send and receive messages
+InterfaceRelay: Responses are routed to a relay queue (not matching uuids)
+
+"""
+
+import logging
+import os
+import sys
+import time
+from abc import abstractmethod
+from typing import TYPE_CHECKING, Any, Dict, Iterable, NewType, Optional, Tuple, Union
+
+from wandb.proto import wandb_internal_pb2 as pb
+from wandb.proto import wandb_telemetry_pb2 as tpb
+from wandb.sdk.artifacts.artifact import Artifact
+from wandb.sdk.artifacts.artifact_manifest import ArtifactManifest
+from wandb.sdk.artifacts.staging import get_staging_dir
+from wandb.sdk.lib import json_util as json
+from wandb.util import (
+ WandBJSONEncoderOld,
+ get_h5_typename,
+ json_dumps_safer,
+ json_dumps_safer_history,
+ json_friendly,
+ json_friendly_val,
+ maybe_compress_summary,
+)
+
+from ..data_types.utils import history_dict_to_json, val_to_json
+from ..lib.mailbox import MailboxHandle
+from . import summary_record as sr
+
+GlobStr = NewType("GlobStr", str)
+
+if TYPE_CHECKING:
+ from ..wandb_run import Run
+
+ if sys.version_info >= (3, 8):
+ from typing import Literal, TypedDict
+ else:
+ from typing_extensions import Literal, TypedDict
+
+ PolicyName = Literal["now", "live", "end"]
+
+ class FilesDict(TypedDict):
+ files: Iterable[Tuple[GlobStr, PolicyName]]
+
+
+logger = logging.getLogger("wandb")
+
+
+def file_policy_to_enum(policy: "PolicyName") -> "pb.FilesItem.PolicyType.V":
+ if policy == "now":
+ enum = pb.FilesItem.PolicyType.NOW
+ elif policy == "end":
+ enum = pb.FilesItem.PolicyType.END
+ elif policy == "live":
+ enum = pb.FilesItem.PolicyType.LIVE
+ return enum
+
+
+def file_enum_to_policy(enum: "pb.FilesItem.PolicyType.V") -> "PolicyName":
+ if enum == pb.FilesItem.PolicyType.NOW:
+ policy: PolicyName = "now"
+ elif enum == pb.FilesItem.PolicyType.END:
+ policy = "end"
+ elif enum == pb.FilesItem.PolicyType.LIVE:
+ policy = "live"
+ return policy
+
+
+class InterfaceBase:
+ _run: Optional["Run"]
+ _drop: bool
+
+ def __init__(self) -> None:
+ self._run = None
+ self._drop = False
+
+ def _hack_set_run(self, run: "Run") -> None:
+ self._run = run
+ current_pid = os.getpid()
+ self._run._set_iface_pid(current_pid)
+
+ def publish_header(self) -> None:
+ header = pb.HeaderRecord()
+ self._publish_header(header)
+
+ @abstractmethod
+ def _publish_header(self, header: pb.HeaderRecord) -> None:
+ raise NotImplementedError
+
+ def communicate_status(self) -> Optional[pb.StatusResponse]:
+ status = pb.StatusRequest()
+ resp = self._communicate_status(status)
+ return resp
+
+ @abstractmethod
+ def _communicate_status(
+ self, status: pb.StatusRequest
+ ) -> Optional[pb.StatusResponse]:
+ raise NotImplementedError
+
+ def _make_config(
+ self,
+ data: Optional[dict] = None,
+ key: Optional[Union[Tuple[str, ...], str]] = None,
+ val: Optional[Any] = None,
+ obj: Optional[pb.ConfigRecord] = None,
+ ) -> pb.ConfigRecord:
+ config = obj or pb.ConfigRecord()
+ if data:
+ for k, v in data.items():
+ update = config.update.add()
+ update.key = k
+ update.value_json = json_dumps_safer(json_friendly(v)[0])
+ if key:
+ update = config.update.add()
+ if isinstance(key, tuple):
+ for k in key:
+ update.nested_key.append(k)
+ else:
+ update.key = key
+ update.value_json = json_dumps_safer(json_friendly(val)[0])
+ return config
+
+ def _make_run(self, run: "Run") -> pb.RunRecord:
+ proto_run = pb.RunRecord()
+ run._make_proto_run(proto_run)
+ if run._settings.host:
+ proto_run.host = run._settings.host
+ if run._config is not None:
+ config_dict = run._config._as_dict() # type: ignore
+ self._make_config(data=config_dict, obj=proto_run.config)
+ if run._telemetry_obj:
+ proto_run.telemetry.MergeFrom(run._telemetry_obj)
+ return proto_run
+
+ def publish_run(self, run: "Run") -> None:
+ run_record = self._make_run(run)
+ self._publish_run(run_record)
+
+ @abstractmethod
+ def _publish_run(self, run: pb.RunRecord) -> None:
+ raise NotImplementedError
+
+ def publish_cancel(self, cancel_slot: str) -> None:
+ cancel = pb.CancelRequest(cancel_slot=cancel_slot)
+ self._publish_cancel(cancel)
+
+ @abstractmethod
+ def _publish_cancel(self, cancel: pb.CancelRequest) -> None:
+ raise NotImplementedError
+
+ def publish_config(
+ self,
+ data: Optional[dict] = None,
+ key: Optional[Union[Tuple[str, ...], str]] = None,
+ val: Optional[Any] = None,
+ ) -> None:
+ cfg = self._make_config(data=data, key=key, val=val)
+
+ self._publish_config(cfg)
+
+ @abstractmethod
+ def _publish_config(self, cfg: pb.ConfigRecord) -> None:
+ raise NotImplementedError
+
+ @abstractmethod
+ def _publish_metric(self, metric: pb.MetricRecord) -> None:
+ raise NotImplementedError
+
+ def _make_summary_from_dict(self, summary_dict: dict) -> pb.SummaryRecord:
+ summary = pb.SummaryRecord()
+ for k, v in summary_dict.items():
+ update = summary.update.add()
+ update.key = k
+ update.value_json = json.dumps(v)
+ return summary
+
+ def _summary_encode(self, value: Any, path_from_root: str) -> dict:
+ """Normalize, compress, and encode sub-objects for backend storage.
+
+ value: Object to encode.
+ path_from_root: `str` dot separated string from the top-level summary to the
+ current `value`.
+
+ Returns:
+ A new tree of dict's with large objects replaced with dictionaries
+ with "_type" entries that say which type the original data was.
+ """
+ # Constructs a new `dict` tree in `json_value` that discards and/or
+ # encodes objects that aren't JSON serializable.
+
+ if isinstance(value, dict):
+ json_value = {}
+ for key, value in value.items(): # noqa: B020
+ json_value[key] = self._summary_encode(
+ value, path_from_root + "." + key
+ )
+ return json_value
+ else:
+ friendly_value, converted = json_friendly(
+ val_to_json(self._run, path_from_root, value, namespace="summary")
+ )
+ json_value, compressed = maybe_compress_summary(
+ friendly_value, get_h5_typename(value)
+ )
+ if compressed:
+ # TODO(jhr): impleement me
+ pass
+ # self.write_h5(path_from_root, friendly_value)
+
+ return json_value
+
+ def _make_summary(self, summary_record: sr.SummaryRecord) -> pb.SummaryRecord:
+ pb_summary_record = pb.SummaryRecord()
+
+ for item in summary_record.update:
+ pb_summary_item = pb_summary_record.update.add()
+ key_length = len(item.key)
+
+ assert key_length > 0
+
+ if key_length > 1:
+ pb_summary_item.nested_key.extend(item.key)
+ else:
+ pb_summary_item.key = item.key[0]
+
+ path_from_root = ".".join(item.key)
+ json_value = self._summary_encode(item.value, path_from_root)
+ json_value, _ = json_friendly(json_value) # type: ignore
+
+ pb_summary_item.value_json = json.dumps(
+ json_value,
+ cls=WandBJSONEncoderOld,
+ )
+
+ for item in summary_record.remove:
+ pb_summary_item = pb_summary_record.remove.add()
+ key_length = len(item.key)
+
+ assert key_length > 0
+
+ if key_length > 1:
+ pb_summary_item.nested_key.extend(item.key)
+ else:
+ pb_summary_item.key = item.key[0]
+
+ return pb_summary_record
+
+ def publish_summary(self, summary_record: sr.SummaryRecord) -> None:
+ pb_summary_record = self._make_summary(summary_record)
+ self._publish_summary(pb_summary_record)
+
+ @abstractmethod
+ def _publish_summary(self, summary: pb.SummaryRecord) -> None:
+ raise NotImplementedError
+
+ def _make_files(self, files_dict: "FilesDict") -> pb.FilesRecord:
+ files = pb.FilesRecord()
+ for path, policy in files_dict["files"]:
+ f = files.files.add()
+ f.path = path
+ f.policy = file_policy_to_enum(policy)
+ return files
+
+ def publish_files(self, files_dict: "FilesDict") -> None:
+ files = self._make_files(files_dict)
+ self._publish_files(files)
+
+ @abstractmethod
+ def _publish_files(self, files: pb.FilesRecord) -> None:
+ raise NotImplementedError
+
+ def publish_python_packages(self, working_set) -> None:
+ python_packages = pb.PythonPackagesRequest()
+ for pkg in working_set:
+ python_packages.package.add(name=pkg.key, version=pkg.version)
+ self._publish_python_packages(python_packages)
+
+ @abstractmethod
+ def _publish_python_packages(
+ self, python_packages: pb.PythonPackagesRequest
+ ) -> None:
+ raise NotImplementedError
+
+ def _make_artifact(self, artifact: "Artifact") -> pb.ArtifactRecord:
+ proto_artifact = pb.ArtifactRecord()
+ proto_artifact.type = artifact.type
+ proto_artifact.name = artifact.name
+ proto_artifact.client_id = artifact._client_id
+ proto_artifact.sequence_client_id = artifact._sequence_client_id
+ proto_artifact.digest = artifact.digest
+ if artifact.distributed_id:
+ proto_artifact.distributed_id = artifact.distributed_id
+ if artifact.description:
+ proto_artifact.description = artifact.description
+ if artifact.metadata:
+ proto_artifact.metadata = json.dumps(json_friendly_val(artifact.metadata))
+ if artifact._base_id:
+ proto_artifact.base_id = artifact._base_id
+
+ ttl_duration_input = artifact._ttl_duration_seconds_to_gql()
+ if ttl_duration_input:
+ proto_artifact.ttl_duration_seconds = ttl_duration_input
+ proto_artifact.incremental_beta1 = artifact.incremental
+ self._make_artifact_manifest(artifact.manifest, obj=proto_artifact.manifest)
+ return proto_artifact
+
+ def _make_artifact_manifest(
+ self,
+ artifact_manifest: ArtifactManifest,
+ obj: Optional[pb.ArtifactManifest] = None,
+ ) -> pb.ArtifactManifest:
+ proto_manifest = obj or pb.ArtifactManifest()
+ proto_manifest.version = artifact_manifest.version()
+ proto_manifest.storage_policy = artifact_manifest.storage_policy.name()
+
+ for k, v in artifact_manifest.storage_policy.config().items() or {}.items():
+ cfg = proto_manifest.storage_policy_config.add()
+ cfg.key = k
+ cfg.value_json = json.dumps(v)
+
+ for entry in sorted(artifact_manifest.entries.values(), key=lambda k: k.path):
+ proto_entry = proto_manifest.contents.add()
+ proto_entry.path = entry.path
+ proto_entry.digest = entry.digest
+ if entry.size:
+ proto_entry.size = entry.size
+ if entry.birth_artifact_id:
+ proto_entry.birth_artifact_id = entry.birth_artifact_id
+ if entry.ref:
+ proto_entry.ref = entry.ref
+ if entry.local_path:
+ proto_entry.local_path = entry.local_path
+ for k, v in entry.extra.items():
+ proto_extra = proto_entry.extra.add()
+ proto_extra.key = k
+ proto_extra.value_json = json.dumps(v)
+ return proto_manifest
+
+ def publish_link_artifact(
+ self,
+ run: "Run",
+ artifact: "Artifact",
+ portfolio_name: str,
+ aliases: Iterable[str],
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+ ) -> None:
+ link_artifact = pb.LinkArtifactRecord()
+ if artifact.is_draft():
+ link_artifact.client_id = artifact._client_id
+ else:
+ link_artifact.server_id = artifact.id if artifact.id else ""
+ link_artifact.portfolio_name = portfolio_name
+ link_artifact.portfolio_entity = entity or run.entity
+ link_artifact.portfolio_project = project or run.project
+ link_artifact.portfolio_aliases.extend(aliases)
+
+ self._publish_link_artifact(link_artifact)
+
+ @abstractmethod
+ def _publish_link_artifact(self, link_artifact: pb.LinkArtifactRecord) -> None:
+ raise NotImplementedError
+
+ @staticmethod
+ def _make_partial_source_str(
+ source: Any, job_info: Dict[str, Any], metadata: Dict[str, Any]
+ ) -> str:
+ """Construct use_artifact.partial.source_info.sourc as str."""
+ source_type = job_info.get("source_type", "").strip()
+ if source_type == "artifact":
+ info_source = job_info.get("source", {})
+ source.artifact.artifact = info_source.get("artifact", "")
+ source.artifact.entrypoint.extend(info_source.get("entrypoint", []))
+ source.artifact.notebook = info_source.get("notebook", False)
+ elif source_type == "repo":
+ source.git.git_info.remote = metadata.get("git", {}).get("remote", "")
+ source.git.git_info.commit = metadata.get("git", {}).get("commit", "")
+ source.git.entrypoint.extend(metadata.get("entrypoint", []))
+ source.git.notebook = metadata.get("notebook", False)
+ elif source_type == "image":
+ source.image.image = metadata.get("docker", "")
+ else:
+ raise ValueError("Invalid source type")
+
+ source_str: str = source.SerializeToString()
+ return source_str
+
+ def _make_proto_use_artifact(
+ self,
+ use_artifact: pb.UseArtifactRecord,
+ job_name: str,
+ job_info: Dict[str, Any],
+ metadata: Dict[str, Any],
+ ) -> pb.UseArtifactRecord:
+ use_artifact.partial.job_name = job_name
+ use_artifact.partial.source_info._version = job_info.get("_version", "")
+ use_artifact.partial.source_info.source_type = job_info.get("source_type", "")
+ use_artifact.partial.source_info.runtime = job_info.get("runtime", "")
+
+ src_str = self._make_partial_source_str(
+ source=use_artifact.partial.source_info.source,
+ job_info=job_info,
+ metadata=metadata,
+ )
+ use_artifact.partial.source_info.source.ParseFromString(src_str)
+
+ return use_artifact
+
+ def publish_use_artifact(
+ self,
+ artifact: "Artifact",
+ ) -> None:
+ assert artifact.id is not None, "Artifact must have an id"
+
+ use_artifact = pb.UseArtifactRecord(
+ id=artifact.id,
+ type=artifact.type,
+ name=artifact.name,
+ )
+
+ # TODO(gst): move to internal process
+ if "_partial" in artifact.metadata:
+ # Download source info from logged partial job artifact
+ job_info = {}
+ try:
+ path = artifact.get_entry("wandb-job.json").download()
+ with open(path) as f:
+ job_info = json.load(f)
+ except Exception as e:
+ logger.warning(
+ f"Failed to download partial job info from artifact {artifact}, : {e}"
+ )
+ use_artifact = self._make_proto_use_artifact(
+ use_artifact=use_artifact,
+ job_name=artifact.name,
+ job_info=job_info,
+ metadata=artifact.metadata,
+ )
+
+ self._publish_use_artifact(use_artifact)
+
+ @abstractmethod
+ def _publish_use_artifact(self, proto_artifact: pb.UseArtifactRecord) -> None:
+ raise NotImplementedError
+
+ def deliver_artifact(
+ self,
+ run: "Run",
+ artifact: "Artifact",
+ aliases: Iterable[str],
+ history_step: Optional[int] = None,
+ is_user_created: bool = False,
+ use_after_commit: bool = False,
+ finalize: bool = True,
+ ) -> MailboxHandle:
+ proto_run = self._make_run(run)
+ proto_artifact = self._make_artifact(artifact)
+ proto_artifact.run_id = proto_run.run_id
+ proto_artifact.project = proto_run.project
+ proto_artifact.entity = proto_run.entity
+ proto_artifact.user_created = is_user_created
+ proto_artifact.use_after_commit = use_after_commit
+ proto_artifact.finalize = finalize
+ for alias in aliases:
+ proto_artifact.aliases.append(alias)
+
+ log_artifact = pb.LogArtifactRequest()
+ log_artifact.artifact.CopyFrom(proto_artifact)
+ if history_step is not None:
+ log_artifact.history_step = history_step
+ log_artifact.staging_dir = get_staging_dir()
+ resp = self._deliver_artifact(log_artifact)
+ return resp
+
+ @abstractmethod
+ def _deliver_artifact(self, log_artifact: pb.LogArtifactRequest) -> MailboxHandle:
+ raise NotImplementedError
+
+ def deliver_download_artifact(
+ self,
+ artifact_id: str,
+ download_root: str,
+ allow_missing_references: bool,
+ ) -> MailboxHandle:
+ download_artifact = pb.DownloadArtifactRequest()
+ download_artifact.artifact_id = artifact_id
+ download_artifact.download_root = download_root
+ download_artifact.allow_missing_references = allow_missing_references
+ resp = self._deliver_download_artifact(download_artifact)
+ return resp
+
+ @abstractmethod
+ def _deliver_download_artifact(
+ self, download_artifact: pb.DownloadArtifactRequest
+ ) -> MailboxHandle:
+ raise NotImplementedError
+
+ def publish_artifact(
+ self,
+ run: "Run",
+ artifact: "Artifact",
+ aliases: Iterable[str],
+ is_user_created: bool = False,
+ use_after_commit: bool = False,
+ finalize: bool = True,
+ ) -> None:
+ proto_run = self._make_run(run)
+ proto_artifact = self._make_artifact(artifact)
+ proto_artifact.run_id = proto_run.run_id
+ proto_artifact.project = proto_run.project
+ proto_artifact.entity = proto_run.entity
+ proto_artifact.user_created = is_user_created
+ proto_artifact.use_after_commit = use_after_commit
+ proto_artifact.finalize = finalize
+ for alias in aliases:
+ proto_artifact.aliases.append(alias)
+ self._publish_artifact(proto_artifact)
+
+ @abstractmethod
+ def _publish_artifact(self, proto_artifact: pb.ArtifactRecord) -> None:
+ raise NotImplementedError
+
+ def publish_tbdata(self, log_dir: str, save: bool, root_logdir: str = "") -> None:
+ tbrecord = pb.TBRecord()
+ tbrecord.log_dir = log_dir
+ tbrecord.save = save
+ tbrecord.root_dir = root_logdir
+ self._publish_tbdata(tbrecord)
+
+ @abstractmethod
+ def _publish_tbdata(self, tbrecord: pb.TBRecord) -> None:
+ raise NotImplementedError
+
+ @abstractmethod
+ def _publish_telemetry(self, telem: tpb.TelemetryRecord) -> None:
+ raise NotImplementedError
+
+ def publish_partial_history(
+ self,
+ data: dict,
+ user_step: int,
+ step: Optional[int] = None,
+ flush: Optional[bool] = None,
+ publish_step: bool = True,
+ run: Optional["Run"] = None,
+ ) -> None:
+ run = run or self._run
+
+ data = history_dict_to_json(run, data, step=user_step, ignore_copy_err=True)
+ data.pop("_step", None)
+
+ # add timestamp to the history request, if not already present
+ # the timestamp might come from the tensorboard log logic
+ if "_timestamp" not in data:
+ data["_timestamp"] = time.time()
+
+ partial_history = pb.PartialHistoryRequest()
+ for k, v in data.items():
+ item = partial_history.item.add()
+ item.key = k
+ item.value_json = json_dumps_safer_history(v)
+
+ if publish_step and step is not None:
+ partial_history.step.num = step
+ if flush is not None:
+ partial_history.action.flush = flush
+ self._publish_partial_history(partial_history)
+
+ @abstractmethod
+ def _publish_partial_history(self, history: pb.PartialHistoryRequest) -> None:
+ raise NotImplementedError
+
+ def publish_history(
+ self,
+ data: dict,
+ step: Optional[int] = None,
+ run: Optional["Run"] = None,
+ publish_step: bool = True,
+ ) -> None:
+ run = run or self._run
+ data = history_dict_to_json(run, data, step=step)
+ history = pb.HistoryRecord()
+ if publish_step:
+ assert step is not None
+ history.step.num = step
+ data.pop("_step", None)
+ for k, v in data.items():
+ item = history.item.add()
+ item.key = k
+ item.value_json = json_dumps_safer_history(v)
+ self._publish_history(history)
+
+ @abstractmethod
+ def _publish_history(self, history: pb.HistoryRecord) -> None:
+ raise NotImplementedError
+
+ def publish_preempting(self) -> None:
+ preempt_rec = pb.RunPreemptingRecord()
+ self._publish_preempting(preempt_rec)
+
+ @abstractmethod
+ def _publish_preempting(self, preempt_rec: pb.RunPreemptingRecord) -> None:
+ raise NotImplementedError
+
+ def publish_output(self, name: str, data: str) -> None:
+ # from vendor.protobuf import google3.protobuf.timestamp
+ # ts = timestamp.Timestamp()
+ # ts.GetCurrentTime()
+ # now = datetime.now()
+ if name == "stdout":
+ otype = pb.OutputRecord.OutputType.STDOUT
+ elif name == "stderr":
+ otype = pb.OutputRecord.OutputType.STDERR
+ else:
+ # TODO(jhr): throw error?
+ print("unknown type")
+ o = pb.OutputRecord(output_type=otype, line=data)
+ o.timestamp.GetCurrentTime()
+ self._publish_output(o)
+
+ @abstractmethod
+ def _publish_output(self, outdata: pb.OutputRecord) -> None:
+ raise NotImplementedError
+
+ def publish_output_raw(self, name: str, data: str) -> None:
+ # from vendor.protobuf import google3.protobuf.timestamp
+ # ts = timestamp.Timestamp()
+ # ts.GetCurrentTime()
+ # now = datetime.now()
+ if name == "stdout":
+ otype = pb.OutputRawRecord.OutputType.STDOUT
+ elif name == "stderr":
+ otype = pb.OutputRawRecord.OutputType.STDERR
+ else:
+ # TODO(jhr): throw error?
+ print("unknown type")
+ o = pb.OutputRawRecord(output_type=otype, line=data)
+ o.timestamp.GetCurrentTime()
+ self._publish_output_raw(o)
+
+ @abstractmethod
+ def _publish_output_raw(self, outdata: pb.OutputRawRecord) -> None:
+ raise NotImplementedError
+
+ def publish_pause(self) -> None:
+ pause = pb.PauseRequest()
+ self._publish_pause(pause)
+
+ @abstractmethod
+ def _publish_pause(self, pause: pb.PauseRequest) -> None:
+ raise NotImplementedError
+
+ def publish_resume(self) -> None:
+ resume = pb.ResumeRequest()
+ self._publish_resume(resume)
+
+ @abstractmethod
+ def _publish_resume(self, resume: pb.ResumeRequest) -> None:
+ raise NotImplementedError
+
+ def publish_alert(
+ self, title: str, text: str, level: str, wait_duration: int
+ ) -> None:
+ proto_alert = pb.AlertRecord()
+ proto_alert.title = title
+ proto_alert.text = text
+ proto_alert.level = level
+ proto_alert.wait_duration = wait_duration
+ self._publish_alert(proto_alert)
+
+ @abstractmethod
+ def _publish_alert(self, alert: pb.AlertRecord) -> None:
+ raise NotImplementedError
+
+ def _make_exit(self, exit_code: Optional[int]) -> pb.RunExitRecord:
+ exit = pb.RunExitRecord()
+ if exit_code is not None:
+ exit.exit_code = exit_code
+ return exit
+
+ def publish_exit(self, exit_code: Optional[int]) -> None:
+ exit_data = self._make_exit(exit_code)
+ self._publish_exit(exit_data)
+
+ @abstractmethod
+ def _publish_exit(self, exit_data: pb.RunExitRecord) -> None:
+ raise NotImplementedError
+
+ def publish_keepalive(self) -> None:
+ keepalive = pb.KeepaliveRequest()
+ self._publish_keepalive(keepalive)
+
+ @abstractmethod
+ def _publish_keepalive(self, keepalive: pb.KeepaliveRequest) -> None:
+ raise NotImplementedError
+
+ def join(self) -> None:
+ # Drop indicates that the internal process has already been shutdown
+ if self._drop:
+ return
+ _ = self._communicate_shutdown()
+
+ @abstractmethod
+ def _communicate_shutdown(self) -> None:
+ raise NotImplementedError
+
+ def deliver_run(self, run: "Run") -> MailboxHandle:
+ run_record = self._make_run(run)
+ return self._deliver_run(run_record)
+
+ def deliver_sync(
+ self,
+ start_offset: int,
+ final_offset: int,
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+ run_id: Optional[str] = None,
+ skip_output_raw: Optional[bool] = None,
+ ) -> MailboxHandle:
+ sync = pb.SyncRequest(
+ start_offset=start_offset,
+ final_offset=final_offset,
+ )
+ if entity:
+ sync.overwrite.entity = entity
+ if project:
+ sync.overwrite.project = project
+ if run_id:
+ sync.overwrite.run_id = run_id
+ if skip_output_raw:
+ sync.skip.output_raw = skip_output_raw
+ return self._deliver_sync(sync)
+
+ @abstractmethod
+ def _deliver_sync(self, sync: pb.SyncRequest) -> MailboxHandle:
+ raise NotImplementedError
+
+ @abstractmethod
+ def _deliver_run(self, run: pb.RunRecord) -> MailboxHandle:
+ raise NotImplementedError
+
+ def deliver_run_start(self, run_pb: pb.RunRecord) -> MailboxHandle:
+ run_start = pb.RunStartRequest()
+ run_start.run.CopyFrom(run_pb)
+ return self._deliver_run_start(run_start)
+
+ @abstractmethod
+ def _deliver_run_start(self, run_start: pb.RunStartRequest) -> MailboxHandle:
+ raise NotImplementedError
+
+ def deliver_attach(self, attach_id: str) -> MailboxHandle:
+ attach = pb.AttachRequest(attach_id=attach_id)
+ return self._deliver_attach(attach)
+
+ @abstractmethod
+ def _deliver_attach(self, status: pb.AttachRequest) -> MailboxHandle:
+ raise NotImplementedError
+
+ def deliver_check_version(
+ self, current_version: Optional[str] = None
+ ) -> MailboxHandle:
+ check_version = pb.CheckVersionRequest()
+ if current_version:
+ check_version.current_version = current_version
+ return self._deliver_check_version(check_version)
+
+ @abstractmethod
+ def _deliver_check_version(
+ self, check_version: pb.CheckVersionRequest
+ ) -> MailboxHandle:
+ raise NotImplementedError
+
+ def deliver_stop_status(self) -> MailboxHandle:
+ status = pb.StopStatusRequest()
+ return self._deliver_stop_status(status)
+
+ @abstractmethod
+ def _deliver_stop_status(self, status: pb.StopStatusRequest) -> MailboxHandle:
+ raise NotImplementedError
+
+ def deliver_network_status(self) -> MailboxHandle:
+ status = pb.NetworkStatusRequest()
+ return self._deliver_network_status(status)
+
+ @abstractmethod
+ def _deliver_network_status(self, status: pb.NetworkStatusRequest) -> MailboxHandle:
+ raise NotImplementedError
+
+ def deliver_internal_messages(self) -> MailboxHandle:
+ internal_message = pb.InternalMessagesRequest()
+ return self._deliver_internal_messages(internal_message)
+
+ @abstractmethod
+ def _deliver_internal_messages(
+ self, internal_message: pb.InternalMessagesRequest
+ ) -> MailboxHandle:
+ raise NotImplementedError
+
+ def deliver_get_summary(self) -> MailboxHandle:
+ get_summary = pb.GetSummaryRequest()
+ return self._deliver_get_summary(get_summary)
+
+ @abstractmethod
+ def _deliver_get_summary(self, get_summary: pb.GetSummaryRequest) -> MailboxHandle:
+ raise NotImplementedError
+
+ def deliver_get_system_metrics(self) -> MailboxHandle:
+ get_summary = pb.GetSystemMetricsRequest()
+ return self._deliver_get_system_metrics(get_summary)
+
+ @abstractmethod
+ def _deliver_get_system_metrics(
+ self, get_summary: pb.GetSystemMetricsRequest
+ ) -> MailboxHandle:
+ raise NotImplementedError
+
+ def deliver_exit(self, exit_code: Optional[int]) -> MailboxHandle:
+ exit_data = self._make_exit(exit_code)
+ return self._deliver_exit(exit_data)
+
+ @abstractmethod
+ def _deliver_exit(self, exit_data: pb.RunExitRecord) -> MailboxHandle:
+ raise NotImplementedError
+
+ def deliver_poll_exit(self) -> MailboxHandle:
+ poll_exit = pb.PollExitRequest()
+ return self._deliver_poll_exit(poll_exit)
+
+ @abstractmethod
+ def _deliver_poll_exit(self, poll_exit: pb.PollExitRequest) -> MailboxHandle:
+ raise NotImplementedError
+
+ def deliver_request_server_info(self) -> MailboxHandle:
+ server_info = pb.ServerInfoRequest()
+ return self._deliver_request_server_info(server_info)
+
+ @abstractmethod
+ def _deliver_request_server_info(
+ self, server_info: pb.ServerInfoRequest
+ ) -> MailboxHandle:
+ raise NotImplementedError
+
+ def deliver_request_sampled_history(self) -> MailboxHandle:
+ sampled_history = pb.SampledHistoryRequest()
+ return self._deliver_request_sampled_history(sampled_history)
+
+ @abstractmethod
+ def _deliver_request_sampled_history(
+ self, sampled_history: pb.SampledHistoryRequest
+ ) -> MailboxHandle:
+ raise NotImplementedError
+
+ def deliver_request_run_status(self) -> MailboxHandle:
+ run_status = pb.RunStatusRequest()
+ return self._deliver_request_run_status(run_status)
+
+ @abstractmethod
+ def _deliver_request_run_status(
+ self, run_status: pb.RunStatusRequest
+ ) -> MailboxHandle:
+ raise NotImplementedError
+
+ def deliver_request_job_info(self) -> MailboxHandle:
+ job_info = pb.JobInfoRequest()
+ return self._deliver_request_job_info(job_info)
+
+ @abstractmethod
+ def _deliver_request_job_info(self, job_info: pb.JobInfoRequest) -> MailboxHandle:
+ raise NotImplementedError
diff --git a/wandb/sdk/interface/interface_queue.py b/wandb/sdk/interface/interface_queue.py
new file mode 100644
index 0000000000000000000000000000000000000000..d1392801ebe98ed45aca843cef0be77bcc898fbb
--- /dev/null
+++ b/wandb/sdk/interface/interface_queue.py
@@ -0,0 +1,59 @@
+"""InterfaceQueue - Derived from InterfaceShared using queues to send to internal thread.
+
+See interface.py for how interface classes relate to each other.
+
+"""
+
+import logging
+from multiprocessing.process import BaseProcess
+from typing import TYPE_CHECKING, Optional
+
+from ..lib import tracelog
+from ..lib.mailbox import Mailbox
+from .interface_shared import InterfaceShared
+from .router_queue import MessageQueueRouter
+
+if TYPE_CHECKING:
+ from queue import Queue
+
+ from wandb.proto import wandb_internal_pb2 as pb
+
+
+logger = logging.getLogger("wandb")
+
+
+class InterfaceQueue(InterfaceShared):
+ record_q: Optional["Queue[pb.Record]"]
+ result_q: Optional["Queue[pb.Result]"]
+ _mailbox: Optional[Mailbox]
+
+ def __init__(
+ self,
+ record_q: Optional["Queue[pb.Record]"] = None,
+ result_q: Optional["Queue[pb.Result]"] = None,
+ process: Optional[BaseProcess] = None,
+ process_check: bool = True,
+ mailbox: Optional[Mailbox] = None,
+ ) -> None:
+ self.record_q = record_q
+ self.result_q = result_q
+ if self.record_q:
+ tracelog.annotate_queue(self.record_q, "record_q")
+ if self.result_q:
+ tracelog.annotate_queue(self.result_q, "result_q")
+ super().__init__(process=process, process_check=process_check, mailbox=mailbox)
+
+ def _init_router(self) -> None:
+ if self.record_q and self.result_q:
+ self._router = MessageQueueRouter(
+ self.record_q, self.result_q, mailbox=self._mailbox
+ )
+
+ def _publish(self, record: "pb.Record", local: Optional[bool] = None) -> None:
+ if self._process_check and self._process and not self._process.is_alive():
+ raise Exception("The wandb backend process has shutdown")
+ if local:
+ record.control.local = local
+ if self.record_q:
+ tracelog.log_message_queue(record, self.record_q)
+ self.record_q.put(record)
diff --git a/wandb/sdk/interface/interface_relay.py b/wandb/sdk/interface/interface_relay.py
new file mode 100644
index 0000000000000000000000000000000000000000..1e0f6d6bb3bc554bb1b513c043db252301d7e701
--- /dev/null
+++ b/wandb/sdk/interface/interface_relay.py
@@ -0,0 +1,53 @@
+"""InterfaceRelay - Derived from InterfaceQueue using RelayRouter to preserve uuid req/resp.
+
+See interface.py for how interface classes relate to each other.
+
+"""
+
+import logging
+from multiprocessing.process import BaseProcess
+from typing import TYPE_CHECKING, Optional
+
+from wandb.proto import wandb_internal_pb2 as pb
+
+from ..lib.mailbox import Mailbox
+from .interface_queue import InterfaceQueue
+from .router_relay import MessageRelayRouter
+
+if TYPE_CHECKING:
+ from queue import Queue
+
+
+logger = logging.getLogger("wandb")
+
+
+class InterfaceRelay(InterfaceQueue):
+ _mailbox: Mailbox
+ relay_q: Optional["Queue[pb.Result]"]
+
+ def __init__(
+ self,
+ mailbox: Mailbox,
+ record_q: Optional["Queue[pb.Record]"] = None,
+ result_q: Optional["Queue[pb.Result]"] = None,
+ relay_q: Optional["Queue[pb.Result]"] = None,
+ process: Optional[BaseProcess] = None,
+ process_check: bool = True,
+ ) -> None:
+ self.relay_q = relay_q
+ super().__init__(
+ record_q=record_q,
+ result_q=result_q,
+ process=process,
+ process_check=process_check,
+ mailbox=mailbox,
+ )
+
+ def _init_router(self) -> None:
+ if self.record_q and self.result_q and self.relay_q:
+ self._router = MessageRelayRouter(
+ request_queue=self.record_q,
+ response_queue=self.result_q,
+ relay_queue=self.relay_q,
+ mailbox=self._mailbox,
+ )
diff --git a/wandb/sdk/interface/interface_shared.py b/wandb/sdk/interface/interface_shared.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d2be2eebcdffbae6c1a25769c9cc9d7ac6bc5d5
--- /dev/null
+++ b/wandb/sdk/interface/interface_shared.py
@@ -0,0 +1,550 @@
+"""InterfaceShared - Derived from InterfaceBase - shared with InterfaceQueue and InterfaceSock.
+
+See interface.py for how interface classes relate to each other.
+
+"""
+
+import logging
+import time
+from abc import abstractmethod
+from multiprocessing.process import BaseProcess
+from typing import Any, Optional, cast
+
+import wandb
+from wandb.proto import wandb_internal_pb2 as pb
+from wandb.proto import wandb_telemetry_pb2 as tpb
+from wandb.util import json_dumps_safer, json_friendly
+
+from ..lib.mailbox import Mailbox, MailboxHandle
+from .interface import InterfaceBase
+from .message_future import MessageFuture
+from .router import MessageRouter
+
+logger = logging.getLogger("wandb")
+
+
+class InterfaceShared(InterfaceBase):
+ process: Optional[BaseProcess]
+ _process_check: bool
+ _router: Optional[MessageRouter]
+ _mailbox: Optional[Mailbox]
+ _transport_success_timestamp: float
+ _transport_failed: bool
+
+ def __init__(
+ self,
+ process: Optional[BaseProcess] = None,
+ process_check: bool = True,
+ mailbox: Optional[Any] = None,
+ ) -> None:
+ super().__init__()
+ self._transport_success_timestamp = time.monotonic()
+ self._transport_failed = False
+ self._process = process
+ self._router = None
+ self._process_check = process_check
+ self._mailbox = mailbox
+ self._init_router()
+
+ @abstractmethod
+ def _init_router(self) -> None:
+ raise NotImplementedError
+
+ @property
+ def transport_failed(self) -> bool:
+ return self._transport_failed
+
+ @property
+ def transport_success_timestamp(self) -> float:
+ return self._transport_success_timestamp
+
+ def _transport_mark_failed(self) -> None:
+ self._transport_failed = True
+
+ def _transport_mark_success(self) -> None:
+ self._transport_success_timestamp = time.monotonic()
+
+ def _publish_output(self, outdata: pb.OutputRecord) -> None:
+ rec = pb.Record()
+ rec.output.CopyFrom(outdata)
+ self._publish(rec)
+
+ def _publish_cancel(self, cancel: pb.CancelRequest) -> None:
+ rec = self._make_request(cancel=cancel)
+ self._publish(rec)
+
+ def _publish_output_raw(self, outdata: pb.OutputRawRecord) -> None:
+ rec = pb.Record()
+ rec.output_raw.CopyFrom(outdata)
+ self._publish(rec)
+
+ def _publish_tbdata(self, tbrecord: pb.TBRecord) -> None:
+ rec = self._make_record(tbrecord=tbrecord)
+ self._publish(rec)
+
+ def _publish_partial_history(
+ self, partial_history: pb.PartialHistoryRequest
+ ) -> None:
+ rec = self._make_request(partial_history=partial_history)
+ self._publish(rec)
+
+ def _publish_history(self, history: pb.HistoryRecord) -> None:
+ rec = self._make_record(history=history)
+ self._publish(rec)
+
+ def _publish_preempting(self, preempt_rec: pb.RunPreemptingRecord) -> None:
+ rec = self._make_record(preempting=preempt_rec)
+ self._publish(rec)
+
+ def _publish_telemetry(self, telem: tpb.TelemetryRecord) -> None:
+ rec = self._make_record(telemetry=telem)
+ self._publish(rec)
+
+ def _make_stats(self, stats_dict: dict) -> pb.StatsRecord:
+ stats = pb.StatsRecord()
+ stats.stats_type = pb.StatsRecord.StatsType.SYSTEM
+ stats.timestamp.GetCurrentTime() # todo: fix this, this is wrong :)
+ for k, v in stats_dict.items():
+ item = stats.item.add()
+ item.key = k
+ item.value_json = json_dumps_safer(json_friendly(v)[0])
+ return stats
+
+ def _make_login(self, api_key: Optional[str] = None) -> pb.LoginRequest:
+ login = pb.LoginRequest()
+ if api_key:
+ login.api_key = api_key
+ return login
+
+ def _make_request( # noqa: C901
+ self,
+ login: Optional[pb.LoginRequest] = None,
+ get_summary: Optional[pb.GetSummaryRequest] = None,
+ pause: Optional[pb.PauseRequest] = None,
+ resume: Optional[pb.ResumeRequest] = None,
+ status: Optional[pb.StatusRequest] = None,
+ stop_status: Optional[pb.StopStatusRequest] = None,
+ internal_messages: Optional[pb.InternalMessagesRequest] = None,
+ network_status: Optional[pb.NetworkStatusRequest] = None,
+ poll_exit: Optional[pb.PollExitRequest] = None,
+ partial_history: Optional[pb.PartialHistoryRequest] = None,
+ sampled_history: Optional[pb.SampledHistoryRequest] = None,
+ run_start: Optional[pb.RunStartRequest] = None,
+ check_version: Optional[pb.CheckVersionRequest] = None,
+ log_artifact: Optional[pb.LogArtifactRequest] = None,
+ download_artifact: Optional[pb.DownloadArtifactRequest] = None,
+ defer: Optional[pb.DeferRequest] = None,
+ attach: Optional[pb.AttachRequest] = None,
+ server_info: Optional[pb.ServerInfoRequest] = None,
+ keepalive: Optional[pb.KeepaliveRequest] = None,
+ run_status: Optional[pb.RunStatusRequest] = None,
+ sender_mark: Optional[pb.SenderMarkRequest] = None,
+ sender_read: Optional[pb.SenderReadRequest] = None,
+ sync: Optional[pb.SyncRequest] = None,
+ status_report: Optional[pb.StatusReportRequest] = None,
+ cancel: Optional[pb.CancelRequest] = None,
+ summary_record: Optional[pb.SummaryRecordRequest] = None,
+ telemetry_record: Optional[pb.TelemetryRecordRequest] = None,
+ job_info: Optional[pb.JobInfoRequest] = None,
+ get_system_metrics: Optional[pb.GetSystemMetricsRequest] = None,
+ python_packages: Optional[pb.PythonPackagesRequest] = None,
+ ) -> pb.Record:
+ request = pb.Request()
+ if login:
+ request.login.CopyFrom(login)
+ elif get_summary:
+ request.get_summary.CopyFrom(get_summary)
+ elif pause:
+ request.pause.CopyFrom(pause)
+ elif resume:
+ request.resume.CopyFrom(resume)
+ elif status:
+ request.status.CopyFrom(status)
+ elif stop_status:
+ request.stop_status.CopyFrom(stop_status)
+ elif internal_messages:
+ request.internal_messages.CopyFrom(internal_messages)
+ elif network_status:
+ request.network_status.CopyFrom(network_status)
+ elif poll_exit:
+ request.poll_exit.CopyFrom(poll_exit)
+ elif partial_history:
+ request.partial_history.CopyFrom(partial_history)
+ elif sampled_history:
+ request.sampled_history.CopyFrom(sampled_history)
+ elif run_start:
+ request.run_start.CopyFrom(run_start)
+ elif check_version:
+ request.check_version.CopyFrom(check_version)
+ elif log_artifact:
+ request.log_artifact.CopyFrom(log_artifact)
+ elif download_artifact:
+ request.download_artifact.CopyFrom(download_artifact)
+ elif defer:
+ request.defer.CopyFrom(defer)
+ elif attach:
+ request.attach.CopyFrom(attach)
+ elif server_info:
+ request.server_info.CopyFrom(server_info)
+ elif keepalive:
+ request.keepalive.CopyFrom(keepalive)
+ elif run_status:
+ request.run_status.CopyFrom(run_status)
+ elif sender_mark:
+ request.sender_mark.CopyFrom(sender_mark)
+ elif sender_read:
+ request.sender_read.CopyFrom(sender_read)
+ elif cancel:
+ request.cancel.CopyFrom(cancel)
+ elif status_report:
+ request.status_report.CopyFrom(status_report)
+ elif summary_record:
+ request.summary_record.CopyFrom(summary_record)
+ elif telemetry_record:
+ request.telemetry_record.CopyFrom(telemetry_record)
+ elif job_info:
+ request.job_info.CopyFrom(job_info)
+ elif get_system_metrics:
+ request.get_system_metrics.CopyFrom(get_system_metrics)
+ elif sync:
+ request.sync.CopyFrom(sync)
+ elif python_packages:
+ request.python_packages.CopyFrom(python_packages)
+ else:
+ raise Exception("Invalid request")
+ record = self._make_record(request=request)
+ # All requests do not get persisted
+ record.control.local = True
+ if status_report:
+ record.control.flow_control = True
+ return record
+
+ def _make_record( # noqa: C901
+ self,
+ run: Optional[pb.RunRecord] = None,
+ config: Optional[pb.ConfigRecord] = None,
+ files: Optional[pb.FilesRecord] = None,
+ summary: Optional[pb.SummaryRecord] = None,
+ history: Optional[pb.HistoryRecord] = None,
+ stats: Optional[pb.StatsRecord] = None,
+ exit: Optional[pb.RunExitRecord] = None,
+ artifact: Optional[pb.ArtifactRecord] = None,
+ tbrecord: Optional[pb.TBRecord] = None,
+ alert: Optional[pb.AlertRecord] = None,
+ final: Optional[pb.FinalRecord] = None,
+ metric: Optional[pb.MetricRecord] = None,
+ header: Optional[pb.HeaderRecord] = None,
+ footer: Optional[pb.FooterRecord] = None,
+ request: Optional[pb.Request] = None,
+ telemetry: Optional[tpb.TelemetryRecord] = None,
+ preempting: Optional[pb.RunPreemptingRecord] = None,
+ link_artifact: Optional[pb.LinkArtifactRecord] = None,
+ use_artifact: Optional[pb.UseArtifactRecord] = None,
+ output: Optional[pb.OutputRecord] = None,
+ output_raw: Optional[pb.OutputRawRecord] = None,
+ ) -> pb.Record:
+ record = pb.Record()
+ if run:
+ record.run.CopyFrom(run)
+ elif config:
+ record.config.CopyFrom(config)
+ elif summary:
+ record.summary.CopyFrom(summary)
+ elif history:
+ record.history.CopyFrom(history)
+ elif files:
+ record.files.CopyFrom(files)
+ elif stats:
+ record.stats.CopyFrom(stats)
+ elif exit:
+ record.exit.CopyFrom(exit)
+ elif artifact:
+ record.artifact.CopyFrom(artifact)
+ elif tbrecord:
+ record.tbrecord.CopyFrom(tbrecord)
+ elif alert:
+ record.alert.CopyFrom(alert)
+ elif final:
+ record.final.CopyFrom(final)
+ elif header:
+ record.header.CopyFrom(header)
+ elif footer:
+ record.footer.CopyFrom(footer)
+ elif request:
+ record.request.CopyFrom(request)
+ elif telemetry:
+ record.telemetry.CopyFrom(telemetry)
+ elif metric:
+ record.metric.CopyFrom(metric)
+ elif preempting:
+ record.preempting.CopyFrom(preempting)
+ elif link_artifact:
+ record.link_artifact.CopyFrom(link_artifact)
+ elif use_artifact:
+ record.use_artifact.CopyFrom(use_artifact)
+ elif output:
+ record.output.CopyFrom(output)
+ elif output_raw:
+ record.output_raw.CopyFrom(output_raw)
+ else:
+ raise Exception("Invalid record")
+ return record
+
+ @abstractmethod
+ def _publish(self, record: pb.Record, local: Optional[bool] = None) -> None:
+ raise NotImplementedError
+
+ def _communicate(
+ self, rec: pb.Record, timeout: Optional[int] = 5, local: Optional[bool] = None
+ ) -> Optional[pb.Result]:
+ return self._communicate_async(rec, local=local).get(timeout=timeout)
+
+ def _communicate_async(
+ self, rec: pb.Record, local: Optional[bool] = None
+ ) -> MessageFuture:
+ assert self._router
+ if self._process_check and self._process and not self._process.is_alive():
+ raise Exception("The wandb backend process has shutdown")
+ future = self._router.send_and_receive(rec, local=local)
+ return future
+
+ def communicate_login(
+ self, api_key: Optional[str] = None, timeout: Optional[int] = 15
+ ) -> pb.LoginResponse:
+ login = self._make_login(api_key)
+ rec = self._make_request(login=login)
+ result = self._communicate(rec, timeout=timeout)
+ if result is None:
+ # TODO: friendlier error message here
+ raise wandb.Error(
+ "Couldn't communicate with backend after %s seconds" % timeout
+ )
+ login_response = result.response.login_response
+ assert login_response
+ return login_response
+
+ def _publish_defer(self, state: "pb.DeferRequest.DeferState.V") -> None:
+ defer = pb.DeferRequest(state=state)
+ rec = self._make_request(defer=defer)
+ self._publish(rec, local=True)
+
+ def publish_defer(self, state: int = 0) -> None:
+ self._publish_defer(cast("pb.DeferRequest.DeferState.V", state))
+
+ def _publish_header(self, header: pb.HeaderRecord) -> None:
+ rec = self._make_record(header=header)
+ self._publish(rec)
+
+ def publish_footer(self) -> None:
+ footer = pb.FooterRecord()
+ rec = self._make_record(footer=footer)
+ self._publish(rec)
+
+ def publish_final(self) -> None:
+ final = pb.FinalRecord()
+ rec = self._make_record(final=final)
+ self._publish(rec)
+
+ def publish_login(self, api_key: Optional[str] = None) -> None:
+ login = self._make_login(api_key)
+ rec = self._make_request(login=login)
+ self._publish(rec)
+
+ def _publish_pause(self, pause: pb.PauseRequest) -> None:
+ rec = self._make_request(pause=pause)
+ self._publish(rec)
+
+ def _publish_resume(self, resume: pb.ResumeRequest) -> None:
+ rec = self._make_request(resume=resume)
+ self._publish(rec)
+
+ def _publish_run(self, run: pb.RunRecord) -> None:
+ rec = self._make_record(run=run)
+ self._publish(rec)
+
+ def _publish_config(self, cfg: pb.ConfigRecord) -> None:
+ rec = self._make_record(config=cfg)
+ self._publish(rec)
+
+ def _publish_summary(self, summary: pb.SummaryRecord) -> None:
+ rec = self._make_record(summary=summary)
+ self._publish(rec)
+
+ def _publish_metric(self, metric: pb.MetricRecord) -> None:
+ rec = self._make_record(metric=metric)
+ self._publish(rec)
+
+ def publish_stats(self, stats_dict: dict) -> None:
+ stats = self._make_stats(stats_dict)
+ rec = self._make_record(stats=stats)
+ self._publish(rec)
+
+ def _publish_python_packages(
+ self, python_packages: pb.PythonPackagesRequest
+ ) -> None:
+ rec = self._make_request(python_packages=python_packages)
+ self._publish(rec)
+
+ def _publish_files(self, files: pb.FilesRecord) -> None:
+ rec = self._make_record(files=files)
+ self._publish(rec)
+
+ def _publish_link_artifact(self, link_artifact: pb.LinkArtifactRecord) -> None:
+ rec = self._make_record(link_artifact=link_artifact)
+ self._publish(rec)
+
+ def _publish_use_artifact(self, use_artifact: pb.UseArtifactRecord) -> None:
+ rec = self._make_record(use_artifact=use_artifact)
+ self._publish(rec)
+
+ def _deliver_artifact(self, log_artifact: pb.LogArtifactRequest) -> MailboxHandle:
+ rec = self._make_request(log_artifact=log_artifact)
+ return self._deliver_record(rec)
+
+ def _deliver_download_artifact(
+ self, download_artifact: pb.DownloadArtifactRequest
+ ) -> MailboxHandle:
+ rec = self._make_request(download_artifact=download_artifact)
+ return self._deliver_record(rec)
+
+ def _publish_artifact(self, proto_artifact: pb.ArtifactRecord) -> None:
+ rec = self._make_record(artifact=proto_artifact)
+ self._publish(rec)
+
+ def _publish_alert(self, proto_alert: pb.AlertRecord) -> None:
+ rec = self._make_record(alert=proto_alert)
+ self._publish(rec)
+
+ def _communicate_status(
+ self, status: pb.StatusRequest
+ ) -> Optional[pb.StatusResponse]:
+ req = self._make_request(status=status)
+ resp = self._communicate(req, local=True)
+ if resp is None:
+ return None
+ assert resp.response.status_response
+ return resp.response.status_response
+
+ def _publish_exit(self, exit_data: pb.RunExitRecord) -> None:
+ rec = self._make_record(exit=exit_data)
+ self._publish(rec)
+
+ def _publish_keepalive(self, keepalive: pb.KeepaliveRequest) -> None:
+ record = self._make_request(keepalive=keepalive)
+ self._publish(record)
+
+ def _communicate_shutdown(self) -> None:
+ # shutdown
+ request = pb.Request(shutdown=pb.ShutdownRequest())
+ record = self._make_record(request=request)
+ _ = self._communicate(record)
+
+ def _get_mailbox(self) -> Mailbox:
+ mailbox = self._mailbox
+ assert mailbox
+ return mailbox
+
+ def _deliver_record(self, record: pb.Record) -> MailboxHandle:
+ mailbox = self._get_mailbox()
+ handle = mailbox._deliver_record(record, interface=self)
+ return handle
+
+ def _deliver_run(self, run: pb.RunRecord) -> MailboxHandle:
+ record = self._make_record(run=run)
+ return self._deliver_record(record)
+
+ def _deliver_sync(self, sync: pb.SyncRequest) -> MailboxHandle:
+ record = self._make_request(sync=sync)
+ return self._deliver_record(record)
+
+ def _deliver_run_start(self, run_start: pb.RunStartRequest) -> MailboxHandle:
+ record = self._make_request(run_start=run_start)
+ return self._deliver_record(record)
+
+ def _deliver_get_summary(self, get_summary: pb.GetSummaryRequest) -> MailboxHandle:
+ record = self._make_request(get_summary=get_summary)
+ return self._deliver_record(record)
+
+ def _deliver_get_system_metrics(
+ self, get_system_metrics: pb.GetSystemMetricsRequest
+ ) -> MailboxHandle:
+ record = self._make_request(get_system_metrics=get_system_metrics)
+ return self._deliver_record(record)
+
+ def _deliver_exit(self, exit_data: pb.RunExitRecord) -> MailboxHandle:
+ record = self._make_record(exit=exit_data)
+ return self._deliver_record(record)
+
+ def _deliver_poll_exit(self, poll_exit: pb.PollExitRequest) -> MailboxHandle:
+ record = self._make_request(poll_exit=poll_exit)
+ return self._deliver_record(record)
+
+ def _deliver_stop_status(self, stop_status: pb.StopStatusRequest) -> MailboxHandle:
+ record = self._make_request(stop_status=stop_status)
+ return self._deliver_record(record)
+
+ def _deliver_attach(self, attach: pb.AttachRequest) -> MailboxHandle:
+ record = self._make_request(attach=attach)
+ return self._deliver_record(record)
+
+ def _deliver_check_version(
+ self, check_version: pb.CheckVersionRequest
+ ) -> MailboxHandle:
+ record = self._make_request(check_version=check_version)
+ return self._deliver_record(record)
+
+ def _deliver_network_status(
+ self, network_status: pb.NetworkStatusRequest
+ ) -> MailboxHandle:
+ record = self._make_request(network_status=network_status)
+ return self._deliver_record(record)
+
+ def _deliver_internal_messages(
+ self, internal_message: pb.InternalMessagesRequest
+ ) -> MailboxHandle:
+ record = self._make_request(internal_messages=internal_message)
+ return self._deliver_record(record)
+
+ def _deliver_request_server_info(
+ self, server_info: pb.ServerInfoRequest
+ ) -> MailboxHandle:
+ record = self._make_request(server_info=server_info)
+ return self._deliver_record(record)
+
+ def _deliver_request_sampled_history(
+ self, sampled_history: pb.SampledHistoryRequest
+ ) -> MailboxHandle:
+ record = self._make_request(sampled_history=sampled_history)
+ return self._deliver_record(record)
+
+ def _deliver_request_run_status(
+ self, run_status: pb.RunStatusRequest
+ ) -> MailboxHandle:
+ record = self._make_request(run_status=run_status)
+ return self._deliver_record(record)
+
+ def _deliver_request_job_info(self, job_info: pb.JobInfoRequest) -> MailboxHandle:
+ record = self._make_request(job_info=job_info)
+ return self._deliver_record(record)
+
+ def _transport_keepalive_failed(self, keepalive_interval: int = 5) -> bool:
+ if self._transport_failed:
+ return True
+
+ now = time.monotonic()
+ if now < self._transport_success_timestamp + keepalive_interval:
+ return False
+
+ try:
+ self.publish_keepalive()
+ except Exception:
+ self._transport_mark_failed()
+ else:
+ self._transport_mark_success()
+ return self._transport_failed
+
+ def join(self) -> None:
+ super().join()
+
+ if self._router:
+ self._router.join()
diff --git a/wandb/sdk/interface/interface_sock.py b/wandb/sdk/interface/interface_sock.py
new file mode 100644
index 0000000000000000000000000000000000000000..3ba47c4bdece1ae341dcd685ceba7de164036435
--- /dev/null
+++ b/wandb/sdk/interface/interface_sock.py
@@ -0,0 +1,61 @@
+"""InterfaceSock - Derived from InterfaceShared using a socket to send to internal thread.
+
+See interface.py for how interface classes relate to each other.
+
+"""
+
+import logging
+from typing import TYPE_CHECKING, Any, Optional
+
+from ..lib.mailbox import Mailbox
+from ..lib.sock_client import SockClient
+from .interface_shared import InterfaceShared
+from .message_future import MessageFuture
+from .router_sock import MessageSockRouter
+
+if TYPE_CHECKING:
+ from wandb.proto import wandb_internal_pb2 as pb
+
+ from ..wandb_run import Run
+
+
+logger = logging.getLogger("wandb")
+
+
+class InterfaceSock(InterfaceShared):
+ _stream_id: Optional[str]
+ _sock_client: SockClient
+ _mailbox: Mailbox
+
+ def __init__(self, sock_client: SockClient, mailbox: Mailbox) -> None:
+ # _sock_client is used when abstract method _init_router() is called by constructor
+ self._sock_client = sock_client
+ super().__init__(mailbox=mailbox)
+ self._process_check = False
+ self._stream_id = None
+
+ def _init_router(self) -> None:
+ self._router = MessageSockRouter(self._sock_client, mailbox=self._mailbox)
+
+ def _hack_set_run(self, run: "Run") -> None:
+ super()._hack_set_run(run)
+ assert run._run_id
+ self._stream_id = run._run_id
+
+ def _assign(self, record: Any) -> None:
+ assert self._stream_id
+ record._info.stream_id = self._stream_id
+
+ def _publish(self, record: "pb.Record", local: Optional[bool] = None) -> None:
+ self._assign(record)
+ self._sock_client.send_record_publish(record)
+
+ def _communicate_async(
+ self, rec: "pb.Record", local: Optional[bool] = None
+ ) -> MessageFuture:
+ self._assign(rec)
+ assert self._router
+ if self._process_check and self._process and not self._process.is_alive():
+ raise Exception("The wandb backend process has shutdown")
+ future = self._router.send_and_receive(rec, local=local)
+ return future
diff --git a/wandb/sdk/interface/message_future.py b/wandb/sdk/interface/message_future.py
new file mode 100644
index 0000000000000000000000000000000000000000..3cc3860e40db9180e1f1101de4e100b14f3f1298
--- /dev/null
+++ b/wandb/sdk/interface/message_future.py
@@ -0,0 +1,27 @@
+"""MessageFuture - represents a message result of an asynchronous operation.
+
+Base class MessageFuture for MessageFutureObject and MessageFuturePoll
+
+"""
+
+import threading
+from abc import abstractmethod
+from typing import Optional
+
+from wandb.proto import wandb_internal_pb2 as pb
+
+
+class MessageFuture:
+ _object: Optional[pb.Result]
+
+ def __init__(self) -> None:
+ self._object = None
+ self._object_ready = threading.Event()
+
+ def _set_object(self, obj: pb.Result) -> None:
+ self._object = obj
+ self._object_ready.set()
+
+ @abstractmethod
+ def get(self, timeout: Optional[int] = None) -> Optional[pb.Result]:
+ raise NotImplementedError
diff --git a/wandb/sdk/interface/message_future_poll.py b/wandb/sdk/interface/message_future_poll.py
new file mode 100644
index 0000000000000000000000000000000000000000..2c41b381b31d6b6c13791f268f2c37b68b8fe2c3
--- /dev/null
+++ b/wandb/sdk/interface/message_future_poll.py
@@ -0,0 +1,50 @@
+"""MessageFuturePoll - Derived from MessageFuture but implementing polling loop.
+
+MessageFuture represents a message result of an asynchronous operation.
+
+MessageFuturePoll implements a polling loop to periodically query for a
+completed async operation.
+
+"""
+
+import time
+from typing import Any, Optional
+
+from wandb.proto import wandb_internal_pb2 as pb
+
+from .message_future import MessageFuture
+
+
+class MessageFuturePoll(MessageFuture):
+ _fn: Any
+ _xid: str
+
+ def __init__(self, fn: Any, xid: str) -> None:
+ super().__init__()
+ self._fn = fn
+ self._xid = xid
+
+ def get(self, timeout: Optional[int] = None) -> Optional[pb.Result]:
+ self._poll(timeout=timeout)
+ if self._object_ready.is_set():
+ return self._object
+ return None
+
+ def _poll(self, timeout: Optional[int] = None) -> None:
+ if self._object_ready.is_set():
+ return
+ done = False
+ start_time = time.time()
+ sleep_time = 0.5
+ while not done:
+ result = self._fn(xid=self._xid)
+ if result:
+ self._set_object(result)
+ done = True
+ continue
+ now_time = time.time()
+ if timeout and start_time - now_time > timeout:
+ done = True
+ continue
+ time.sleep(sleep_time)
+ sleep_time = min(sleep_time * 2, 5)
diff --git a/wandb/sdk/interface/router.py b/wandb/sdk/interface/router.py
new file mode 100644
index 0000000000000000000000000000000000000000..d73b5049b4d30ebca1ce9ea4fb0dded26c41fa49
--- /dev/null
+++ b/wandb/sdk/interface/router.py
@@ -0,0 +1,118 @@
+"""Router - handle message router (base class).
+
+Router to manage responses.
+
+"""
+
+import logging
+import threading
+import uuid
+from abc import abstractmethod
+from typing import TYPE_CHECKING, Dict, Optional
+
+from ..lib import mailbox, tracelog
+from .message_future import MessageFuture
+
+if TYPE_CHECKING:
+ from queue import Queue
+
+ from wandb.proto import wandb_internal_pb2 as pb
+
+
+logger = logging.getLogger("wandb")
+
+
+class MessageRouterClosedError(Exception):
+ """Router has been closed."""
+
+ pass
+
+
+class MessageFutureObject(MessageFuture):
+ def __init__(self) -> None:
+ super().__init__()
+
+ def get(self, timeout: Optional[int] = None) -> Optional["pb.Result"]:
+ is_set = self._object_ready.wait(timeout)
+ if is_set and self._object:
+ return self._object
+ return None
+
+
+class MessageRouter:
+ _pending_reqs: Dict[str, MessageFutureObject]
+ _request_queue: "Queue[pb.Record]"
+ _response_queue: "Queue[pb.Result]"
+ _mailbox: Optional[mailbox.Mailbox]
+
+ def __init__(self, mailbox: Optional[mailbox.Mailbox] = None) -> None:
+ self._mailbox = mailbox
+ self._pending_reqs = {}
+ self._lock = threading.Lock()
+
+ self._join_event = threading.Event()
+ self._thread = threading.Thread(target=self.message_loop)
+ self._thread.name = "MsgRouterThr"
+ self._thread.daemon = True
+ self._thread.start()
+
+ @abstractmethod
+ def _read_message(self) -> Optional["pb.Result"]:
+ raise NotImplementedError
+
+ @abstractmethod
+ def _send_message(self, record: "pb.Record") -> None:
+ raise NotImplementedError
+
+ def message_loop(self) -> None:
+ while not self._join_event.is_set():
+ try:
+ msg = self._read_message()
+ except EOFError:
+ # On abnormal shutdown the queue will be destroyed underneath
+ # resulting in EOFError. message_loop needs to exit..
+ logger.warning("EOFError seen in message_loop")
+ break
+ except MessageRouterClosedError:
+ logger.warning("message_loop has been closed")
+ break
+ if not msg:
+ continue
+ self._handle_msg_rcv(msg)
+
+ def send_and_receive(
+ self, rec: "pb.Record", local: Optional[bool] = None
+ ) -> MessageFuture:
+ rec.control.req_resp = True
+ if local:
+ rec.control.local = local
+ rec.uuid = uuid.uuid4().hex
+ future = MessageFutureObject()
+ with self._lock:
+ self._pending_reqs[rec.uuid] = future
+
+ self._send_message(rec)
+
+ return future
+
+ def join(self) -> None:
+ self._join_event.set()
+ self._thread.join()
+
+ def _handle_msg_rcv(self, msg: "pb.Result") -> None:
+ # deliver mailbox addressed messages to mailbox
+ if self._mailbox and msg.control.mailbox_slot:
+ self._mailbox.deliver(msg)
+ return
+ with self._lock:
+ future = self._pending_reqs.pop(msg.uuid, None)
+ if future is None:
+ # TODO (cvp): saw this in tests, seemed benign enough to ignore, but
+ # could point to other issues.
+ if msg.uuid != "":
+ tracelog.log_message_assert(msg)
+ logger.warning(
+ "No listener found for msg with uuid %s (%s)", msg.uuid, msg
+ )
+ return
+ future._set_object(msg)
diff --git a/wandb/sdk/interface/router_queue.py b/wandb/sdk/interface/router_queue.py
new file mode 100644
index 0000000000000000000000000000000000000000..509d46afa081aa8631d9f67701e76ce479cb966d
--- /dev/null
+++ b/wandb/sdk/interface/router_queue.py
@@ -0,0 +1,44 @@
+"""Router - handle message router (queue).
+
+Router to manage responses from a queue.
+
+"""
+
+import queue
+from typing import TYPE_CHECKING, Optional
+
+from ..lib import tracelog
+from ..lib.mailbox import Mailbox
+from .router import MessageRouter
+
+if TYPE_CHECKING:
+ from queue import Queue
+
+ from wandb.proto import wandb_internal_pb2 as pb
+
+
+class MessageQueueRouter(MessageRouter):
+ _request_queue: "Queue[pb.Record]"
+ _response_queue: "Queue[pb.Result]"
+
+ def __init__(
+ self,
+ request_queue: "Queue[pb.Record]",
+ response_queue: "Queue[pb.Result]",
+ mailbox: Optional[Mailbox] = None,
+ ) -> None:
+ self._request_queue = request_queue
+ self._response_queue = response_queue
+ super().__init__(mailbox=mailbox)
+
+ def _read_message(self) -> Optional["pb.Result"]:
+ try:
+ msg = self._response_queue.get(timeout=1)
+ except queue.Empty:
+ return None
+ tracelog.log_message_dequeue(msg, self._response_queue)
+ return msg
+
+ def _send_message(self, record: "pb.Record") -> None:
+ tracelog.log_message_queue(record, self._request_queue)
+ self._request_queue.put(record)
diff --git a/wandb/sdk/interface/router_relay.py b/wandb/sdk/interface/router_relay.py
new file mode 100644
index 0000000000000000000000000000000000000000..e26022f4cea79555cbf406b8e5c0a9a8fdbe9bec
--- /dev/null
+++ b/wandb/sdk/interface/router_relay.py
@@ -0,0 +1,39 @@
+"""Router - handle message router (relay).
+
+Router to manage responses from a queue with relay.
+
+"""
+
+from typing import TYPE_CHECKING
+
+from ..lib import tracelog
+from ..lib.mailbox import Mailbox
+from .router_queue import MessageQueueRouter
+
+if TYPE_CHECKING:
+ from queue import Queue
+
+ from wandb.proto import wandb_internal_pb2 as pb
+
+
+class MessageRelayRouter(MessageQueueRouter):
+ _relay_queue: "Queue[pb.Result]"
+
+ def __init__(
+ self,
+ request_queue: "Queue[pb.Record]",
+ response_queue: "Queue[pb.Result]",
+ relay_queue: "Queue[pb.Result]",
+ mailbox: Mailbox,
+ ) -> None:
+ self._relay_queue = relay_queue
+ super().__init__(
+ request_queue=request_queue, response_queue=response_queue, mailbox=mailbox
+ )
+
+ def _handle_msg_rcv(self, msg: "pb.Result") -> None:
+ if msg.control.relay_id:
+ tracelog.log_message_queue(msg, self._relay_queue)
+ self._relay_queue.put(msg)
+ return
+ super()._handle_msg_rcv(msg)
diff --git a/wandb/sdk/interface/router_sock.py b/wandb/sdk/interface/router_sock.py
new file mode 100644
index 0000000000000000000000000000000000000000..aabbaf31d3b8f00e9ff8937fe55c5af5bfdb21fb
--- /dev/null
+++ b/wandb/sdk/interface/router_sock.py
@@ -0,0 +1,36 @@
+"""Router - handle message router (sock).
+
+Router to manage responses from a socket client.
+
+"""
+
+from typing import TYPE_CHECKING, Optional
+
+from ..lib.mailbox import Mailbox
+from ..lib.sock_client import SockClient, SockClientClosedError
+from .router import MessageRouter, MessageRouterClosedError
+
+if TYPE_CHECKING:
+ from wandb.proto import wandb_internal_pb2 as pb
+
+
+class MessageSockRouter(MessageRouter):
+ _sock_client: SockClient
+ _mailbox: Mailbox
+
+ def __init__(self, sock_client: SockClient, mailbox: Mailbox) -> None:
+ self._sock_client = sock_client
+ super().__init__(mailbox=mailbox)
+
+ def _read_message(self) -> Optional["pb.Result"]:
+ try:
+ resp = self._sock_client.read_server_response(timeout=1)
+ except SockClientClosedError:
+ raise MessageRouterClosedError
+ if not resp:
+ return None
+ msg = resp.result_communicate
+ return msg
+
+ def _send_message(self, record: "pb.Record") -> None:
+ self._sock_client.send_record_communicate(record)
diff --git a/wandb/sdk/interface/summary_record.py b/wandb/sdk/interface/summary_record.py
new file mode 100644
index 0000000000000000000000000000000000000000..2050a39080759ef12c54364aaf76b1dee6ab7e1d
--- /dev/null
+++ b/wandb/sdk/interface/summary_record.py
@@ -0,0 +1,67 @@
+"""Summary Record.
+
+This module implements a summary record as an intermediate format before being converted
+to a protocol buffer.
+"""
+
+import typing as t
+
+
+class SummaryRecord:
+ """Encodes a diff -- analogous to the SummaryRecord protobuf message."""
+
+ update: t.List["SummaryItem"]
+ remove: t.List["SummaryItem"]
+
+ def __init__(self):
+ self.update = []
+ self.remove = []
+
+ def __str__(self):
+ s = "SummaryRecord:\n Update:\n "
+ s += "\n ".join([str(item) for item in self.update])
+ s += "\n Remove:\n "
+ s += "\n ".join([str(item) for item in self.remove])
+ s += "\n"
+ return s
+
+ __repr__ = __str__
+
+ def _add_next_parent(self, parent_key):
+ with_next_parent = SummaryRecord()
+ with_next_parent.update = [
+ item._add_next_parent(parent_key) for item in self.update
+ ]
+ with_next_parent.remove = [
+ item._add_next_parent(parent_key) for item in self.remove
+ ]
+
+ return with_next_parent
+
+
+class SummaryItem:
+ """Analogous to the SummaryItem protobuf message."""
+
+ key: t.Tuple[str]
+ value: t.Any
+
+ def __init__(self):
+ self.key = tuple()
+ self.value = None
+
+ def __str__(self):
+ return "SummaryItem: key: " + str(self.key) + " value: " + str(self.value)
+
+ __repr__ = __str__
+
+ def _add_next_parent(self, parent_key):
+ with_next_parent = SummaryItem()
+
+ key = self.key
+ if not isinstance(key, tuple):
+ key = (key,)
+
+ with_next_parent.key = (parent_key,) + self.key
+ with_next_parent.value = self.value
+
+ return with_next_parent
diff --git a/wandb/sdk/internal/__init__.py b/wandb/sdk/internal/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sdk/internal/context.py b/wandb/sdk/internal/context.py
new file mode 100644
index 0000000000000000000000000000000000000000..1ad49ee15a1e15b76c1cfacdbc2158fe8cfa9ad7
--- /dev/null
+++ b/wandb/sdk/internal/context.py
@@ -0,0 +1,89 @@
+"""Context Keeper."""
+
+import logging
+import threading
+from typing import Dict, Optional
+
+from wandb.proto.wandb_internal_pb2 import Record, Result
+
+logger = logging.getLogger(__name__)
+
+
+class Context:
+ _cancel_event: threading.Event
+ # TODO(debug_context) add debug setting to enable this
+ # _debug_record: Optional[Record]
+
+ def __init__(self) -> None:
+ self._cancel_event = threading.Event()
+ # TODO(debug_context) see above
+ # self._debug_record = None
+
+ def cancel(self) -> None:
+ self._cancel_event.set()
+
+ @property
+ def cancel_event(self) -> threading.Event:
+ return self._cancel_event
+
+
+def context_id_from_record(record: Record) -> str:
+ context_id = record.control.mailbox_slot
+ return context_id
+
+
+def context_id_from_result(result: Result) -> str:
+ context_id = result.control.mailbox_slot
+ return context_id
+
+
+class ContextKeeper:
+ _active_items: Dict[str, Context]
+
+ def __init__(self) -> None:
+ self._active_items = {}
+
+ def add_from_record(self, record: Record) -> Optional[Context]:
+ context_id = context_id_from_record(record)
+ if not context_id:
+ return None
+ context_obj = self.add(context_id)
+
+ # TODO(debug_context) see above
+ # context_obj._debug_record = record
+
+ return context_obj
+
+ def add(self, context_id: str) -> Context:
+ assert context_id
+ context_obj = Context()
+ self._active_items[context_id] = context_obj
+ return context_obj
+
+ def get(self, context_id: str) -> Optional[Context]:
+ item = self._active_items.get(context_id)
+ return item
+
+ def release(self, context_id: str) -> None:
+ if not context_id:
+ return
+ _ = self._active_items.pop(context_id, None)
+
+ def cancel(self, context_id: str) -> bool:
+ item = self.get(context_id)
+ if item:
+ item.cancel()
+ return True
+ return False
+
+ # TODO(debug_context) see above
+ # def _debug_print_orphans(self, print_to_stdout: bool) -> None:
+ # for context_id, context in self._active_items.items():
+ # record = context._debug_record
+ # record_type = record.WhichOneof("record_type") if record else "unknown"
+ # message = (
+ # f"Context: {context_id} {context.cancel_event.is_set()} {record_type}"
+ # )
+ # logger.warning(message)
+ # if print_to_stdout:
+ # print(message)
diff --git a/wandb/sdk/internal/datastore.py b/wandb/sdk/internal/datastore.py
new file mode 100644
index 0000000000000000000000000000000000000000..afa36c724d0fdc319cdce3da09d9999a4dc38fa6
--- /dev/null
+++ b/wandb/sdk/internal/datastore.py
@@ -0,0 +1,297 @@
+"""leveldb log datastore.
+
+Format is described at:
+ https://github.com/google/leveldb/blob/master/doc/log_format.md
+
+block := record* trailer?
+record :=
+ checksum: uint32 // crc32c of type and data[] ; little-endian
+ length: uint16 // little-endian
+ type: uint8 // One of FULL, FIRST, MIDDLE, LAST
+ data: uint8[length]
+
+header :=
+ ident: char[4]
+ magic: uint16
+ version: uint8
+"""
+
+# TODO: possibly restructure code by porting the C++ or go implementation
+
+import logging
+import os
+import struct
+import zlib
+from typing import TYPE_CHECKING, Optional, Tuple
+
+import wandb
+
+if TYPE_CHECKING:
+ from typing import IO, Any
+
+ from wandb.proto.wandb_internal_pb2 import Record
+
+logger = logging.getLogger(__name__)
+
+LEVELDBLOG_HEADER_LEN = 7
+LEVELDBLOG_BLOCK_LEN = 32768
+LEVELDBLOG_DATA_LEN = LEVELDBLOG_BLOCK_LEN - LEVELDBLOG_HEADER_LEN
+
+LEVELDBLOG_FULL = 1
+LEVELDBLOG_FIRST = 2
+LEVELDBLOG_MIDDLE = 3
+LEVELDBLOG_LAST = 4
+
+LEVELDBLOG_HEADER_IDENT = ":W&B"
+LEVELDBLOG_HEADER_MAGIC = (
+ 0xBEE1 # zlib.crc32(bytes("Weights & Biases", 'iso8859-1')) & 0xffff
+)
+LEVELDBLOG_HEADER_VERSION = 0
+
+try:
+ bytes("", "ascii")
+
+ def strtobytes(x):
+ """strtobytes."""
+ return bytes(x, "iso8859-1")
+
+ # def bytestostr(x):
+ # return str(x, 'iso8859-1')
+
+except Exception:
+ strtobytes = str
+ # bytestostr = str
+
+
+class DataStore:
+ _index: int
+ _flush_offset: int
+
+ def __init__(self) -> None:
+ self._opened_for_scan = False
+ self._fp: Optional["IO[Any]"] = None
+ self._index = 0
+ self._flush_offset = 0
+ self._size_bytes = 0
+
+ self._crc = [0] * (LEVELDBLOG_LAST + 1)
+ for x in range(1, LEVELDBLOG_LAST + 1):
+ self._crc[x] = zlib.crc32(strtobytes(chr(x))) & 0xFFFFFFFF
+
+ assert (
+ wandb._assert_is_internal_process # type: ignore
+ ), "DataStore can only be used in the internal process"
+
+ def open_for_write(self, fname: str) -> None:
+ self._fname = fname
+ logger.info("open: %s", fname)
+ open_flags = "xb"
+ self._fp = open(fname, open_flags)
+ self._write_header()
+
+ def open_for_append(self, fname):
+ # TODO: implement
+ self._fname = fname
+ logger.info("open: %s", fname)
+ self._fp = open(fname, "wb")
+ # do something with _index
+
+ def open_for_scan(self, fname):
+ self._fname = fname
+ logger.info("open for scan: %s", fname)
+ self._fp = open(fname, "r+b")
+ self._index = 0
+ self._size_bytes = os.stat(fname).st_size
+ self._opened_for_scan = True
+ self._read_header()
+
+ def seek(self, offset: int) -> None:
+ self._fp.seek(offset) # type: ignore
+ self._index = offset
+
+ def get_offset(self) -> int:
+ offset = self._fp.tell() # type: ignore
+ return offset
+
+ def in_last_block(self):
+ """Determine if we're in the last block to handle in-progress writes."""
+ return self._index > self._size_bytes - LEVELDBLOG_DATA_LEN
+
+ def scan_record(self):
+ assert self._opened_for_scan, "file not open for scanning"
+ # TODO(jhr): handle some assertions as file corruption issues
+ # assume we have enough room to read header, checked by caller?
+ header = self._fp.read(LEVELDBLOG_HEADER_LEN)
+ if len(header) == 0:
+ return None
+ assert (
+ len(header) == LEVELDBLOG_HEADER_LEN
+ ), "record header is {} bytes instead of the expected {}".format(
+ len(header), LEVELDBLOG_HEADER_LEN
+ )
+ fields = struct.unpack("<IHB", header)
+ checksum, dlength, dtype = fields
+ # check len, better fit in the block
+ self._index += LEVELDBLOG_HEADER_LEN
+ data = self._fp.read(dlength)
+ checksum_computed = zlib.crc32(data, self._crc[dtype]) & 0xFFFFFFFF
+ assert (
+ checksum == checksum_computed
+ ), "record checksum is invalid, data may be corrupt"
+ self._index += dlength
+ return dtype, data
+
+ def scan_data(self):
+ # TODO(jhr): handle some assertions as file corruption issues
+ # how much left in the block. if less than header len, read as pad,
+ offset = self._index % LEVELDBLOG_BLOCK_LEN
+ space_left = LEVELDBLOG_BLOCK_LEN - offset
+ if space_left < LEVELDBLOG_HEADER_LEN:
+ pad_check = strtobytes("\x00" * space_left)
+ pad = self._fp.read(space_left)
+ # verify they are zero
+ assert pad == pad_check, "invalid padding"
+ self._index += space_left
+
+ record = self.scan_record()
+ if record is None: # eof
+ return None
+ dtype, data = record
+ if dtype == LEVELDBLOG_FULL:
+ return data
+
+ assert (
+ dtype == LEVELDBLOG_FIRST
+ ), f"expected record to be type {LEVELDBLOG_FIRST} but found {dtype}"
+ while True:
+ offset = self._index % LEVELDBLOG_BLOCK_LEN
+ record = self.scan_record()
+ if record is None: # eof
+ return None
+ dtype, new_data = record
+ if dtype == LEVELDBLOG_LAST:
+ data += new_data
+ break
+ assert (
+ dtype == LEVELDBLOG_MIDDLE
+ ), f"expected record to be type {LEVELDBLOG_MIDDLE} but found {dtype}"
+ data += new_data
+ return data
+
+ def _write_header(self):
+ data = struct.pack(
+ "<4sHB",
+ strtobytes(LEVELDBLOG_HEADER_IDENT),
+ LEVELDBLOG_HEADER_MAGIC,
+ LEVELDBLOG_HEADER_VERSION,
+ )
+ assert (
+ len(data) == LEVELDBLOG_HEADER_LEN
+ ), f"header size is {len(data)} bytes, expected {LEVELDBLOG_HEADER_LEN}"
+ self._fp.write(data)
+ self._index += len(data)
+
+ def _read_header(self):
+ header = self._fp.read(LEVELDBLOG_HEADER_LEN)
+ assert (
+ len(header) == LEVELDBLOG_HEADER_LEN
+ ), "header is {} bytes instead of the expected {}".format(
+ len(header), LEVELDBLOG_HEADER_LEN
+ )
+ ident, magic, version = struct.unpack("<4sHB", header)
+ if ident != strtobytes(LEVELDBLOG_HEADER_IDENT):
+ raise Exception("Invalid header")
+ if magic != LEVELDBLOG_HEADER_MAGIC:
+ raise Exception("Invalid header")
+ if version != LEVELDBLOG_HEADER_VERSION:
+ raise Exception("Invalid header")
+ self._index += len(header)
+
+ def _write_record(self, s, dtype=None):
+ """Write record that must fit into a block."""
+ # double check that there is enough space
+ # (this is a precondition to calling this method)
+ assert len(s) + LEVELDBLOG_HEADER_LEN <= (
+ LEVELDBLOG_BLOCK_LEN - self._index % LEVELDBLOG_BLOCK_LEN
+ ), "not enough space to write new records"
+
+ dlength = len(s)
+ dtype = dtype or LEVELDBLOG_FULL
+ # print("record: length={} type={}".format(dlength, dtype))
+ checksum = zlib.crc32(s, self._crc[dtype]) & 0xFFFFFFFF
+ # logger.info("write_record: index=%d len=%d dtype=%d",
+ # self._index, dlength, dtype)
+ self._fp.write(struct.pack("<IHB", checksum, dlength, dtype))
+ if dlength:
+ self._fp.write(s)
+ self._index += LEVELDBLOG_HEADER_LEN + len(s)
+
+ def _write_data(self, s):
+ start_offset = self._index
+
+ offset = self._index % LEVELDBLOG_BLOCK_LEN
+ space_left = LEVELDBLOG_BLOCK_LEN - offset
+ data_used = 0
+ data_left = len(s)
+ # logger.info("write_data: index=%d offset=%d len=%d",
+ # self._index, offset, data_left)
+ if space_left < LEVELDBLOG_HEADER_LEN:
+ pad = "\x00" * space_left
+ self._fp.write(strtobytes(pad))
+ self._index += space_left
+ offset = 0
+ space_left = LEVELDBLOG_BLOCK_LEN
+
+ # does it fit in first (possibly partial) block?
+ if data_left + LEVELDBLOG_HEADER_LEN <= space_left:
+ self._write_record(s)
+ else:
+ # write first record (we could still be in the middle of a block,
+ # but this write will end on a block boundary)
+ data_room = space_left - LEVELDBLOG_HEADER_LEN
+ self._write_record(s[:data_room], LEVELDBLOG_FIRST)
+ data_used += data_room
+ data_left -= data_room
+ assert data_left, "data_left should be non-zero"
+
+ # write middles (if any)
+ while data_left > LEVELDBLOG_DATA_LEN:
+ self._write_record(
+ s[data_used : data_used + LEVELDBLOG_DATA_LEN],
+ LEVELDBLOG_MIDDLE,
+ )
+ data_used += LEVELDBLOG_DATA_LEN
+ data_left -= LEVELDBLOG_DATA_LEN
+
+ # write last and flush the entire block to disk
+ self._write_record(s[data_used:], LEVELDBLOG_LAST)
+ self._fp.flush()
+ os.fsync(self._fp.fileno())
+ self._flush_offset = self._index
+
+ return start_offset, self._index, self._flush_offset
+
+ def ensure_flushed(self, off: int) -> None:
+ self._fp.flush() # type: ignore
+
+ def write(self, obj: "Record") -> Tuple[int, int, int]:
+ """Write a protocol buffer.
+
+ Arguments:
+ obj: Protocol buffer to write.
+
+ Returns:
+ (start_offset, end_offset, flush_offset) if successful,
+ None otherwise
+
+ """
+ raw_size = obj.ByteSize()
+ s = obj.SerializeToString()
+ assert len(s) == raw_size, "invalid serialization"
+ ret = self._write_data(s)
+ return ret
+
+ def close(self) -> None:
+ if self._fp is not None:
+ logger.info("close: %s", self._fname)
+ self._fp.close()
diff --git a/wandb/sdk/internal/file_pusher.py b/wandb/sdk/internal/file_pusher.py
new file mode 100644
index 0000000000000000000000000000000000000000..32b9a4bec431e12ce68332e581f332b2b55e0e76
--- /dev/null
+++ b/wandb/sdk/internal/file_pusher.py
@@ -0,0 +1,184 @@
+import concurrent.futures
+import logging
+import os
+import queue
+import tempfile
+import threading
+import time
+from typing import TYPE_CHECKING, Optional, Tuple
+
+import wandb
+import wandb.util
+from wandb.filesync import stats, step_checksum, step_upload
+from wandb.sdk.lib.paths import LogicalPath
+
+if TYPE_CHECKING:
+ from wandb.sdk.artifacts.artifact_manifest import ArtifactManifest
+ from wandb.sdk.artifacts.artifact_saver import SaveFn, SaveFnAsync
+ from wandb.sdk.internal import file_stream, internal_api
+ from wandb.sdk.internal.settings_static import SettingsStatic
+
+
+logger = logging.getLogger(__name__)
+
+
+class FilePusher:
+ """Parallel file upload class.
+
+ This manages uploading multiple files in parallel. It will restart a given file's
+ upload job if it receives a notification that that file has been modified. The
+ finish() method will block until all events have been processed and all uploads are
+ complete.
+ """
+
+ MAX_UPLOAD_JOBS = 64
+
+ def __init__(
+ self,
+ api: "internal_api.Api",
+ file_stream: "file_stream.FileStreamApi",
+ settings: Optional["SettingsStatic"] = None,
+ ) -> None:
+ self._api = api
+
+ # Temporary directory for copies we make of some file types to
+ # reduce the probability that the file gets changed while we're
+ # uploading it.
+ self._tempdir = tempfile.TemporaryDirectory("wandb")
+
+ self._stats = stats.Stats()
+
+ self._incoming_queue: queue.Queue[step_checksum.Event] = queue.Queue()
+ self._event_queue: queue.Queue[step_upload.Event] = queue.Queue()
+
+ self._step_checksum = step_checksum.StepChecksum(
+ self._api,
+ self._tempdir,
+ self._incoming_queue,
+ self._event_queue,
+ self._stats,
+ )
+ self._step_checksum.start()
+
+ self._step_upload = step_upload.StepUpload(
+ self._api,
+ self._stats,
+ self._event_queue,
+ self.MAX_UPLOAD_JOBS,
+ file_stream=file_stream,
+ settings=settings,
+ )
+ self._step_upload.start()
+
+ self._stats_thread_stop = threading.Event()
+ if os.environ.get("WANDB_DEBUG"):
+ # debug thread to monitor and report file pusher stats
+ self._stats_thread = threading.Thread(
+ target=self._file_pusher_stats,
+ daemon=True,
+ name="FPStatsThread",
+ )
+ self._stats_thread.start()
+
+ def _file_pusher_stats(self) -> None:
+ while not self._stats_thread_stop.is_set():
+ logger.info(f"FilePusher stats: {self._stats._stats}")
+ time.sleep(1)
+
+ def get_status(self) -> Tuple[bool, stats.Summary]:
+ running = self.is_alive()
+ summary = self._stats.summary()
+ return running, summary
+
+ def print_status(self, prefix: bool = True) -> None:
+ step = 0
+ spinner_states = ["-", "\\", "|", "/"]
+ stop = False
+ while True:
+ if not self.is_alive():
+ stop = True
+ summary = self._stats.summary()
+ line = " {:.2f}MB of {:.2f}MB uploaded ({:.2f}MB deduped)\r".format(
+ summary.uploaded_bytes / 1048576.0,
+ summary.total_bytes / 1048576.0,
+ summary.deduped_bytes / 1048576.0,
+ )
+ line = spinner_states[step % 4] + line
+ step += 1
+ wandb.termlog(line, newline=False, prefix=prefix)
+ if stop:
+ break
+ time.sleep(0.25)
+ dedupe_fraction = (
+ summary.deduped_bytes / float(summary.total_bytes)
+ if summary.total_bytes > 0
+ else 0
+ )
+ if dedupe_fraction > 0.01:
+ wandb.termlog(
+ "W&B sync reduced upload amount by %.1f%% "
+ % (dedupe_fraction * 100),
+ prefix=prefix,
+ )
+ # clear progress line.
+ wandb.termlog(" " * 79, prefix=prefix)
+
+ def file_counts_by_category(self) -> stats.FileCountsByCategory:
+ return self._stats.file_counts_by_category()
+
+ def file_changed(self, save_name: LogicalPath, path: str, copy: bool = True):
+ """Tell the file pusher that a file's changed and should be uploaded.
+
+ Arguments:
+ save_name: string logical location of the file relative to the run
+ directory.
+ path: actual string path of the file to upload on the filesystem.
+ """
+ # Tests in linux were failing because wandb-events.jsonl didn't exist
+ if not os.path.exists(path) or not os.path.isfile(path):
+ return
+ if os.path.getsize(path) == 0:
+ return
+
+ event = step_checksum.RequestUpload(path, save_name, copy)
+ self._incoming_queue.put(event)
+
+ def store_manifest_files(
+ self,
+ manifest: "ArtifactManifest",
+ artifact_id: str,
+ save_fn: "SaveFn",
+ save_fn_async: "SaveFnAsync",
+ ) -> None:
+ event = step_checksum.RequestStoreManifestFiles(
+ manifest, artifact_id, save_fn, save_fn_async
+ )
+ self._incoming_queue.put(event)
+
+ def commit_artifact(
+ self,
+ artifact_id: str,
+ *,
+ finalize: bool = True,
+ before_commit: step_upload.PreCommitFn,
+ result_future: "concurrent.futures.Future[None]",
+ ):
+ event = step_checksum.RequestCommitArtifact(
+ artifact_id, finalize, before_commit, result_future
+ )
+ self._incoming_queue.put(event)
+
+ def finish(self, callback: Optional[step_upload.OnRequestFinishFn] = None):
+ logger.info("shutting down file pusher")
+ self._incoming_queue.put(step_checksum.RequestFinish(callback))
+ self._stats_thread_stop.set()
+
+ def join(self) -> None:
+ # NOTE: must have called finish before join
+ logger.info("waiting for file pusher")
+ while self.is_alive():
+ time.sleep(0.5)
+ self._tempdir.cleanup()
+
+ def is_alive(self) -> bool:
+ return self._step_checksum.is_alive() or self._step_upload.is_alive()
diff --git a/wandb/sdk/internal/file_stream.py b/wandb/sdk/internal/file_stream.py
new file mode 100644
index 0000000000000000000000000000000000000000..649a87bfafd8c7d325d22a16ca36343fea6e60bf
--- /dev/null
+++ b/wandb/sdk/internal/file_stream.py
@@ -0,0 +1,689 @@
+import base64
+import functools
+import itertools
+import logging
+import os
+import queue
+import random
+import sys
+import threading
+import time
+from types import TracebackType
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Callable,
+ Dict,
+ List,
+ NamedTuple,
+ Optional,
+ Set,
+ Tuple,
+ Type,
+ Union,
+)
+
+if TYPE_CHECKING:
+ if sys.version_info >= (3, 8):
+ from typing import TypedDict
+ else:
+ from typing_extensions import TypedDict
+
+ class ProcessedChunk(TypedDict):
+ offset: int
+ content: List[str]
+
+ class ProcessedBinaryChunk(TypedDict):
+ offset: int
+ content: str
+ encoding: str
+
+
+import requests
+
+import wandb
+from wandb import util
+from wandb.sdk.internal import internal_api
+
+from ..lib import file_stream_utils
+
+logger = logging.getLogger(__name__)
+
+
+class Chunk(NamedTuple):
+ filename: str
+ data: Any
+
+
+class DefaultFilePolicy:
+ def __init__(self, start_chunk_id: int = 0) -> None:
+ self._chunk_id = start_chunk_id
+
+ def process_chunks(
+ self, chunks: List[Chunk]
+ ) -> Union[bool, "ProcessedChunk", "ProcessedBinaryChunk", List["ProcessedChunk"]]:
+ chunk_id = self._chunk_id
+ self._chunk_id += len(chunks)
+ return {"offset": chunk_id, "content": [c.data for c in chunks]}
+
+
+class JsonlFilePolicy(DefaultFilePolicy):
+ def process_chunks(self, chunks: List[Chunk]) -> "ProcessedChunk":
+ chunk_id = self._chunk_id
+ # TODO: chunk_id is getting reset on each request...
+ self._chunk_id += len(chunks)
+ chunk_data = []
+ for chunk in chunks:
+ if len(chunk.data) > util.MAX_LINE_BYTES:
+ msg = "Metric data exceeds maximum size of {} ({})".format(
+ util.to_human_size(util.MAX_LINE_BYTES),
+ util.to_human_size(len(chunk.data)),
+ )
+ wandb.termerror(msg, repeat=False)
+ wandb._sentry.message(msg, repeat=False)
+ else:
+ chunk_data.append(chunk.data)
+
+ return {
+ "offset": chunk_id,
+ "content": chunk_data,
+ }
+
+
+class SummaryFilePolicy(DefaultFilePolicy):
+ def process_chunks(self, chunks: List[Chunk]) -> Union[bool, "ProcessedChunk"]:
+ data = chunks[-1].data
+ if len(data) > util.MAX_LINE_BYTES:
+ msg = "Summary data exceeds maximum size of {}. Dropping it.".format(
+ util.to_human_size(util.MAX_LINE_BYTES)
+ )
+ wandb.termerror(msg, repeat=False)
+ wandb._sentry.message(msg, repeat=False)
+ return False
+ return {"offset": 0, "content": [data]}
+
+
+class StreamCRState:
+ r"""Stream state that tracks carriage returns.
+
+ There are two streams: stdout and stderr. We create two instances for each stream.
+ An instance holds state about:
+ found_cr: if a carriage return has been found in this stream.
+ cr: most recent offset (line number) where we found \r.
+ We update this offset with every progress bar update.
+ last_normal: most recent offset without a \r in this stream.
+ i.e. the most recent "normal" line.
+ """
+
+ found_cr: bool
+ cr: Optional[int]
+ last_normal: Optional[int]
+
+ def __init__(self) -> None:
+ self.found_cr = False
+ self.cr = None
+ self.last_normal = None
+
+
+class CRDedupeFilePolicy(DefaultFilePolicy):
+ r"""File stream policy for removing carriage-return erased characters.
+
+ This is what a terminal does. We use it for console output to reduce the amount of
+ data we need to send over the network (eg. for progress bars), while preserving the
+ output's appearance in the web app.
+
+ CR stands for "carriage return", for the character \r. It tells the terminal to move
+ the cursor back to the start of the current line. Progress bars (like tqdm) use \r
+ repeatedly to overwrite a line with newer updates. This gives the illusion of the
+ progress bar filling up in real-time.
+ """
+
+ def __init__(self, start_chunk_id: int = 0) -> None:
+ super().__init__(start_chunk_id=start_chunk_id)
+ self._prev_chunk = None
+
+ self.global_offset = 0
+ # cr refers to carriage return \r
+ self.stderr = StreamCRState()
+ self.stdout = StreamCRState()
+
+ @staticmethod
+ def get_consecutive_offsets(console: Dict[int, str]) -> List[List[int]]:
+ """Compress consecutive line numbers into an interval.
+
+ Args:
+ console: Dict[int, str] which maps offsets (line numbers) to lines of text.
+ It represents a mini version of our console dashboard on the UI.
+
+ Returns:
+ A list of intervals (we compress consecutive line numbers into an interval).
+
+ Example:
+ >>> console = {2: "", 3: "", 4: "", 5: "", 10: "", 11: "", 20: ""}
+ >>> get_consecutive_offsets(console)
+ [(2, 5), (10, 11), (20, 20)]
+ """
+ offsets = sorted(list(console.keys()))
+ intervals: List = []
+ for i, num in enumerate(offsets):
+ if i == 0:
+ intervals.append([num, num])
+ continue
+ largest = intervals[-1][1]
+ if num == largest + 1:
+ intervals[-1][1] = num
+ else:
+ intervals.append([num, num])
+ return intervals
+
+ @staticmethod
+ def split_chunk(chunk: Chunk) -> Tuple[str, str]:
+ r"""Split chunks.
+
+ Args:
+ chunk: object with two fields: filename (str) & data (str)
+ `chunk.data` is a str containing the lines we want. It usually contains \n or \r or both.
+ `chunk.data` has two possible formats (for the two streams - stdout and stderr):
+ - "2020-08-25T20:38:36.895321 this is my line of text\nsecond line\n"
+ - "ERROR 2020-08-25T20:38:36.895321 this is my line of text\nsecond line\nthird\n".
+
+ Here's another example with a carriage return \r.
+ - "ERROR 2020-08-25T20:38:36.895321 \r progress bar\n"
+
+ Returns:
+ A 2-tuple of strings.
+ First str is prefix, either "ERROR {timestamp} " or "{timestamp} ".
+ Second str is the rest of the string.
+
+ Example:
+ >>> chunk = Chunk(filename="output.log", data="ERROR 2020-08-25T20:38 this is my line of text\n")
+ >>> split_chunk(chunk)
+ ("ERROR 2020-08-25T20:38 ", "this is my line of text\n")
+ """
+ prefix = ""
+ token, rest = chunk.data.split(" ", 1)
+ if token == "ERROR":
+ prefix += token + " "
+ token, rest = rest.split(" ", 1)
+ prefix += token + " "
+ return prefix, rest
+
+ def process_chunks(self, chunks: List) -> List["ProcessedChunk"]:
+ r"""Process chunks.
+
+ Args:
+ chunks: List of Chunk objects. See description of chunk above in `split_chunk(...)`.
+
+ Returns:
+ List[Dict]. Each dict in the list contains two keys: an `offset` which holds the line number
+ and `content` which maps to a list of consecutive lines starting from that offset.
+ `offset` here means global line number in our console on the UI.
+
+ Example:
+ >>> chunks = [
+ Chunk("output.log", "ERROR 2020-08-25T20:38 this is my line of text\nboom\n"),
+ Chunk("output.log", "2020-08-25T20:38 this is test\n"),
+ ]
+ >>> process_chunks(chunks)
+ [
+ {"offset": 0, "content": [
+ "ERROR 2020-08-25T20:38 this is my line of text\n",
+ "ERROR 2020-08-25T20:38 boom\n",
+ "2020-08-25T20:38 this is test\n"
+ ]
+ }
+ ]
+ """
+ # Dict[int->str], each offset (line number) mapped to a line.
+ # Represents a mini-version of our console pane on the UI.
+ console = {}
+ sep = os.linesep
+
+ for c in chunks:
+ prefix, logs_str = self.split_chunk(c)
+ logs = logs_str.split(sep)
+
+ for line in logs:
+ stream = self.stderr if prefix.startswith("ERROR ") else self.stdout
+ if line.startswith("\r"):
+ # line starting with \r will always overwrite a previous offset.
+ offset: int = (
+ stream.cr
+ if (stream.found_cr and stream.cr is not None)
+ else (stream.last_normal or 0)
+ )
+ stream.cr = offset
+ stream.found_cr = True
+ console[offset] = prefix + line[1:] + "\n"
+
+ # Usually logs_str = "\r progress bar\n" for progress bar updates.
+ # If instead logs_str = "\r progress bar\n text\n text\n",
+ # treat this as the end of a progress bar and reset accordingly.
+ if (
+ logs_str.count(sep) > 1
+ and logs_str.replace(sep, "").count("\r") == 1
+ ):
+ stream.found_cr = False
+
+ elif line:
+ console[self.global_offset] = prefix + line + "\n"
+ stream.last_normal = self.global_offset
+ self.global_offset += 1
+
+ intervals = self.get_consecutive_offsets(console)
+ ret = []
+ for a, b in intervals:
+ processed_chunk: ProcessedChunk = {
+ "offset": a,
+ "content": [console[i] for i in range(a, b + 1)],
+ }
+ ret.append(processed_chunk)
+ return ret
+
+
+class BinaryFilePolicy(DefaultFilePolicy):
+ def __init__(self) -> None:
+ super().__init__()
+ self._offset: int = 0
+
+ def process_chunks(self, chunks: List[Chunk]) -> "ProcessedBinaryChunk":
+ data = b"".join([c.data for c in chunks])
+ enc = base64.b64encode(data).decode("ascii")
+ self._offset += len(data)
+ return {"offset": self._offset, "content": enc, "encoding": "base64"}
+
+
+class FileStreamApi:
+ """Pushes chunks of files to our streaming endpoint.
+
+ This class is used as a singleton. It has a thread that serializes access to
+ the streaming endpoint and performs rate-limiting and batching.
+
+ TODO: Differentiate between binary/text encoding.
+ """
+
+ class Finish(NamedTuple):
+ exitcode: int
+
+ class Preempting(NamedTuple):
+ pass
+
+ class PushSuccess(NamedTuple):
+ artifact_id: str
+ save_name: str
+
+ MAX_ITEMS_PER_PUSH = 10000
+
+ def __init__(
+ self,
+ api: "internal_api.Api",
+ run_id: str,
+ start_time: float,
+ timeout: float = 0,
+ settings: Optional[dict] = None,
+ ) -> None:
+ settings = settings or dict()
+ # NOTE: exc_info is set in thread_except_body context and readable by calling threads
+ self._exc_info: Optional[
+ Union[
+ Tuple[Type[BaseException], BaseException, TracebackType],
+ Tuple[None, None, None],
+ ]
+ ] = None
+ self._settings = settings
+ self._api = api
+ self._run_id = run_id
+ self._start_time = start_time
+ self._client = requests.Session()
+ timeout = timeout or 0
+ if timeout > 0:
+ self._client.post = functools.partial(self._client.post, timeout=timeout) # type: ignore[method-assign]
+ self._client.auth = api.client.transport.session.auth
+ self._client.headers.update(api.client.transport.headers or {})
+ self._client.cookies.update(api.client.transport.cookies or {}) # type: ignore[no-untyped-call]
+ self._client.proxies.update(api.client.transport.session.proxies or {})
+ self._file_policies: Dict[str, DefaultFilePolicy] = {}
+ self._dropped_chunks: int = 0
+ self._queue: queue.Queue = queue.Queue()
+ self._thread = threading.Thread(target=self._thread_except_body)
+ # It seems we need to make this a daemon thread to get sync.py's atexit handler to run, which
+ # cleans this thread up.
+ self._thread.name = "FileStreamThread"
+ self._thread.daemon = True
+ self._init_endpoint()
+
+ def _init_endpoint(self) -> None:
+ settings = self._api.settings()
+ settings.update(self._settings)
+ self._endpoint = "{base}/files/{entity}/{project}/{run}/file_stream".format(
+ base=settings["base_url"],
+ entity=settings["entity"],
+ project=settings["project"],
+ run=self._run_id,
+ )
+
+ def start(self) -> None:
+ self._init_endpoint()
+ self._thread.start()
+
+ def set_default_file_policy(
+ self, filename: str, file_policy: "DefaultFilePolicy"
+ ) -> None:
+ """Set an upload policy for a file unless one has already been set."""
+ if filename not in self._file_policies:
+ self._file_policies[filename] = file_policy
+
+ def set_file_policy(self, filename: str, file_policy: "DefaultFilePolicy") -> None:
+ self._file_policies[filename] = file_policy
+
+ @property
+ def heartbeat_seconds(self) -> Union[int, float]:
+ # Defaults to 30
+ heartbeat_seconds: Union[int, float] = self._api.dynamic_settings[
+ "heartbeat_seconds"
+ ]
+ return heartbeat_seconds
+
+ def rate_limit_seconds(self) -> Union[int, float]:
+ run_time = time.time() - self._start_time
+ if run_time < 60:
+ return max(1.0, self.heartbeat_seconds / 15)
+ elif run_time < 300:
+ return max(2.5, self.heartbeat_seconds / 3)
+ else:
+ return max(5.0, self.heartbeat_seconds)
+
+ def _read_queue(self) -> List:
+ # called from the push thread (_thread_body), this does an initial read
+ # that'll block for up to rate_limit_seconds. Then it tries to read
+ # as much out of the queue as it can. We do this because the http post
+ # to the server happens within _thread_body, and can take longer than
+ # our rate limit. So next time we get a chance to read the queue we want
+ # read all the stuff that queue'd up since last time.
+ #
+ # If we have more than MAX_ITEMS_PER_PUSH in the queue then the push thread
+ # will get behind and data will buffer up in the queue.
+ return util.read_many_from_queue(
+ self._queue, self.MAX_ITEMS_PER_PUSH, self.rate_limit_seconds()
+ )
+
+ def _thread_body(self) -> None:
+ posted_data_time = time.time()
+ posted_anything_time = time.time()
+ ready_chunks = []
+ uploaded: Set[str] = set()
+ finished: Optional[FileStreamApi.Finish] = None
+ while finished is None:
+ items = self._read_queue()
+ for item in items:
+ if isinstance(item, self.Finish):
+ finished = item
+ elif isinstance(item, self.Preempting):
+ request_with_retry(
+ self._client.post,
+ self._endpoint,
+ json={
+ "complete": False,
+ "preempting": True,
+ "dropped": self._dropped_chunks,
+ "uploaded": list(uploaded),
+ },
+ )
+ uploaded = set()
+ elif isinstance(item, self.PushSuccess):
+ uploaded.add(item.save_name)
+ else:
+ # item is Chunk
+ ready_chunks.append(item)
+
+ cur_time = time.time()
+
+ if ready_chunks and (
+ finished or cur_time - posted_data_time > self.rate_limit_seconds()
+ ):
+ posted_data_time = cur_time
+ posted_anything_time = cur_time
+ success = self._send(ready_chunks, uploaded=uploaded)
+ ready_chunks = []
+ if success:
+ uploaded = set()
+
+ # If there aren't ready chunks or uploaded files, we still want to
+ # send regular heartbeats so the backend doesn't erroneously mark this
+ # run as crashed.
+ if cur_time - posted_anything_time > self.heartbeat_seconds:
+ posted_anything_time = cur_time
+
+ # If we encountered an error trying to publish the
+ # list of uploaded files, don't reset the `uploaded`
+ # list. Retry publishing the list on the next attempt.
+ if not isinstance(
+ request_with_retry(
+ self._client.post,
+ self._endpoint,
+ json={
+ "complete": False,
+ "failed": False,
+ "dropped": self._dropped_chunks,
+ "uploaded": list(uploaded),
+ },
+ ),
+ Exception,
+ ):
+ uploaded = set()
+
+ # post the final close message. (item is self.Finish instance now)
+ request_with_retry(
+ self._client.post,
+ self._endpoint,
+ json={
+ "complete": True,
+ "exitcode": int(finished.exitcode),
+ "dropped": self._dropped_chunks,
+ "uploaded": list(uploaded),
+ },
+ )
+
+ def _thread_except_body(self) -> None:
+ # TODO: Consolidate with internal_util.ExceptionThread
+ try:
+ self._thread_body()
+ except Exception as e:
+ exc_info = sys.exc_info()
+ self._exc_info = exc_info
+ logger.exception("generic exception in filestream thread")
+ wandb._sentry.exception(exc_info)
+ raise e
+
+ def _handle_response(self, response: Union[Exception, "requests.Response"]) -> None:
+ """Log dropped chunks and updates dynamic settings."""
+ if isinstance(response, Exception):
+ wandb.termerror(
+ "Dropped streaming file chunk (see wandb/debug-internal.log)"
+ )
+ logger.exception("dropped chunk %s" % response)
+ self._dropped_chunks += 1
+ else:
+ parsed: Optional[dict] = None
+ try:
+ parsed = response.json()
+ except Exception:
+ pass
+ if isinstance(parsed, dict):
+ limits = parsed.get("limits")
+ if isinstance(limits, dict):
+ self._api.dynamic_settings.update(limits)
+
+ def _send(self, chunks: List[Chunk], uploaded: Optional[Set[str]] = None) -> bool:
+ uploaded_list = list(uploaded or [])
+ # create files dict. dict of <filename: chunks> pairs where chunks are a list of
+ # [chunk_id, chunk_data] tuples (as lists since this will be json).
+ files = {}
+ # Groupby needs group keys to be consecutive, so sort first.
+ chunks.sort(key=lambda c: c.filename)
+ for filename, file_chunks in itertools.groupby(chunks, lambda c: c.filename):
+ file_chunks_list = list(file_chunks) # groupby returns iterator
+ # Specific file policies are set by internal/sender.py
+ self.set_default_file_policy(filename, DefaultFilePolicy())
+ files[filename] = self._file_policies[filename].process_chunks(
+ file_chunks_list
+ )
+ if not files[filename]:
+ del files[filename]
+
+ for fs in file_stream_utils.split_files(files, max_bytes=util.MAX_LINE_BYTES):
+ self._handle_response(
+ request_with_retry(
+ self._client.post,
+ self._endpoint,
+ json={"files": fs, "dropped": self._dropped_chunks},
+ retry_callback=self._api.retry_callback,
+ )
+ )
+
+ if uploaded_list:
+ if isinstance(
+ request_with_retry(
+ self._client.post,
+ self._endpoint,
+ json={
+ "complete": False,
+ "failed": False,
+ "dropped": self._dropped_chunks,
+ "uploaded": uploaded_list,
+ },
+ ),
+ Exception,
+ ):
+ return False
+ return True
+
+ def stream_file(self, path: str) -> None:
+ name = path.split("/")[-1]
+ with open(path) as f:
+ self._send([Chunk(name, line) for line in f])
+
+ def enqueue_preempting(self) -> None:
+ self._queue.put(self.Preempting())
+
+ def push(self, filename: str, data: Any) -> None:
+ """Push a chunk of a file to the streaming endpoint.
+
+ Arguments:
+ filename: Name of file that this is a chunk of.
+ data: File data.
+ """
+ self._queue.put(Chunk(filename, data))
+
+ def push_success(self, artifact_id: str, save_name: str) -> None:
+ """Notification that a file upload has been successfully completed.
+
+ Arguments:
+ artifact_id: ID of artifact
+ save_name: saved name of the uploaded file
+ """
+ self._queue.put(self.PushSuccess(artifact_id, save_name))
+
+ def finish(self, exitcode: int) -> None:
+ """Clean up.
+
+ Anything pushed after finish will be dropped.
+
+ Arguments:
+ exitcode: The exitcode of the watched process.
+ """
+ logger.info("file stream finish called")
+ self._queue.put(self.Finish(exitcode))
+ # TODO(jhr): join on a thread which exited with an exception is a noop, clean up this path
+ self._thread.join()
+ logger.info("file stream finish is done")
+ if self._exc_info:
+ logger.error("FileStream exception", exc_info=self._exc_info)
+ # re-raising the original exception, will get re-caught in internal.py for the sender thread
+ if self._exc_info[1] is not None:
+ raise self._exc_info[1].with_traceback(self._exc_info[2])
+
+
+MAX_SLEEP_SECONDS = 60 * 5
+
+
+def request_with_retry(
+ func: Callable,
+ *args: Any,
+ **kwargs: Any,
+) -> Union["requests.Response", "requests.RequestException"]:
+ """Perform a requests http call, retrying with exponential backoff.
+
+ Arguments:
+ func: An http-requesting function to call, like requests.post
+ max_retries: Maximum retries before giving up.
+ By default, we retry 30 times in ~2 hours before dropping the chunk
+ *args: passed through to func
+ **kwargs: passed through to func
+ """
+ max_retries: int = kwargs.pop("max_retries", 30)
+ retry_callback: Optional[Callable] = kwargs.pop("retry_callback", None)
+ sleep = 2
+ retry_count = 0
+ while True:
+ try:
+ response: requests.Response = func(*args, **kwargs)
+ response.raise_for_status()
+ return response
+ except (
+ requests.exceptions.ConnectionError,
+ requests.exceptions.HTTPError,
+ requests.exceptions.Timeout,
+ ) as e:
+ if isinstance(e, requests.exceptions.HTTPError):
+ # Non-retriable HTTP errors.
+ #
+ # We retry 500s just to be cautious, and because the back end
+ # returns them when there are infrastructure issues. If retrying
+ # some request winds up being problematic, we'll change the
+ # back end to indicate that it shouldn't be retried.
+ if e.response is not None and e.response.status_code in {
+ 400,
+ 403,
+ 404,
+ 409,
+ }:
+ return e
+
+ if retry_count == max_retries:
+ return e
+ retry_count += 1
+ delay = sleep + random.random() * 0.25 * sleep
+ if isinstance(e, requests.exceptions.HTTPError) and (
+ e.response is not None and e.response.status_code == 429
+ ):
+ err_str = (
+ "Filestream rate limit exceeded, "
+ f"retrying in {delay:.1f} seconds. "
+ )
+ if retry_callback:
+ retry_callback(e.response.status_code, err_str)
+ logger.info(err_str)
+ else:
+ logger.warning(
+ "requests_with_retry encountered retryable exception: %s. func: %s, args: %s, kwargs: %s",
+ e,
+ func,
+ args,
+ kwargs,
+ )
+ time.sleep(delay)
+ sleep *= 2
+ if sleep > MAX_SLEEP_SECONDS:
+ sleep = MAX_SLEEP_SECONDS
+ except requests.exceptions.RequestException as e:
+ error_message = "unknown error"
+ try:
+ error_message = response.json()["error"] # todo: clean this up
+ except Exception:
+ pass
+ logger.error(f"requests_with_retry error: {error_message}")
+ logger.exception(
+ "requests_with_retry encountered unretryable exception: %s", e
+ )
+ return e
diff --git a/wandb/sdk/internal/flow_control.py b/wandb/sdk/internal/flow_control.py
new file mode 100644
index 0000000000000000000000000000000000000000..e72e88552b9c4660f9b74dec7964a669f85bfd1b
--- /dev/null
+++ b/wandb/sdk/internal/flow_control.py
@@ -0,0 +1,263 @@
+"""Flow Control.
+
+States:
+ FORWARDING
+ PAUSING
+
+New messages:
+ pb.SenderMarkRequest writer -> sender (empty message)
+ pb.StatusReportRequest sender -> writer (reports current sender progress)
+ pb.SenderReadRequest writer -> sender (requests read of transaction log)
+
+Thresholds:
+ Threshold_High_MaxOutstandingData - When above this, stop sending requests to sender
+ Threshold_Mid_StartSendingReadRequests - When below this, start sending read requests
+ Threshold_Low_RestartSendingData - When below this, start sending normal records
+
+State machine:
+ FORWARDING
+ -> PAUSED if should_pause
+ There is too much work outstanding to the sender thread, after the current request
+ lets stop sending data.
+ PAUSING
+ -> FORWARDING if should_unpause
+ -> PAUSING if should_recover
+ -> PAUSING if should_quiesce
+
+"""
+
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Callable, Optional
+
+from wandb.proto import wandb_internal_pb2 as pb
+from wandb.sdk.lib import fsm
+
+from .settings_static import SettingsStatic
+
+if TYPE_CHECKING:
+ from wandb.proto.wandb_internal_pb2 import Record
+
+logger = logging.getLogger(__name__)
+
+# By default we will allow 400 MiB of requests in the sender queue
+# before falling back to the transaction log.
+DEFAULT_THRESHOLD = 128 * 1024 * 1024 # 128 MiB
+
+
+def _get_request_type(record: "Record") -> Optional[str]:
+ record_type = record.WhichOneof("record_type")
+ if record_type != "request":
+ return None
+ request_type = record.request.WhichOneof("request_type")
+ return request_type
+
+
+def _is_control_record(record: "Record") -> bool:
+ return record.control.flow_control
+
+
+def _is_local_non_control_record(record: "Record") -> bool:
+ return record.control.local and not record.control.flow_control
+
+
+@dataclass
+class StateContext:
+ last_forwarded_offset: int = 0
+ last_sent_offset: int = 0
+ last_written_offset: int = 0
+
+
+class FlowControl:
+ _fsm: fsm.FsmWithContext["Record", StateContext]
+
+ def __init__(
+ self,
+ settings: SettingsStatic,
+ forward_record: Callable[["Record"], None],
+ write_record: Callable[["Record"], int],
+ pause_marker: Callable[[], None],
+ recover_records: Callable[[int, int], None],
+ _threshold_bytes_high: int = 0,
+ _threshold_bytes_mid: int = 0,
+ _threshold_bytes_low: int = 0,
+ ) -> None:
+ # thresholds to define when to PAUSE, RESTART, FORWARDING
+ if (
+ _threshold_bytes_high == 0
+ or _threshold_bytes_mid == 0
+ or _threshold_bytes_low == 0
+ ):
+ threshold = settings._network_buffer or DEFAULT_THRESHOLD
+ _threshold_bytes_high = threshold
+ _threshold_bytes_mid = threshold // 2
+ _threshold_bytes_low = threshold // 4
+ assert _threshold_bytes_high > _threshold_bytes_mid > _threshold_bytes_low
+
+ # FSM definition
+ state_forwarding = StateForwarding(
+ forward_record=forward_record,
+ pause_marker=pause_marker,
+ threshold_pause=_threshold_bytes_high,
+ )
+ state_pausing = StatePausing(
+ forward_record=forward_record,
+ recover_records=recover_records,
+ threshold_recover=_threshold_bytes_mid,
+ threshold_forward=_threshold_bytes_low,
+ )
+ self._fsm = fsm.FsmWithContext(
+ states=[state_forwarding, state_pausing],
+ table={
+ StateForwarding: [
+ fsm.FsmEntry(
+ state_forwarding._should_pause,
+ StatePausing,
+ state_forwarding._pause,
+ ),
+ ],
+ StatePausing: [
+ fsm.FsmEntry(
+ state_pausing._should_unpause,
+ StateForwarding,
+ state_pausing._unpause,
+ ),
+ fsm.FsmEntry(
+ state_pausing._should_recover,
+ StatePausing,
+ state_pausing._recover,
+ ),
+ fsm.FsmEntry(
+ state_pausing._should_quiesce,
+ StatePausing,
+ state_pausing._quiesce,
+ ),
+ ],
+ },
+ )
+
+ def flush(self) -> None:
+ # TODO(mempressure): what do we do here, how do we make sure we dont have work in pause state
+ pass
+
+ def flow(self, record: "Record") -> None:
+ self._fsm.input(record)
+
+
+class StateShared:
+ _context: StateContext
+
+ def __init__(self) -> None:
+ self._context = StateContext()
+
+ def _update_written_offset(self, record: "Record") -> None:
+ end_offset = record.control.end_offset
+ if end_offset:
+ self._context.last_written_offset = end_offset
+
+ def _update_forwarded_offset(self) -> None:
+ self._context.last_forwarded_offset = self._context.last_written_offset
+
+ def _process(self, record: "Record") -> None:
+ request_type = _get_request_type(record)
+ if not request_type:
+ return
+ process_str = f"_process_{request_type}"
+ process_handler: Optional[Callable[[pb.Record], None]] = getattr(
+ self, process_str, None
+ )
+ if not process_handler:
+ return
+ process_handler(record)
+
+ def _process_status_report(self, record: "Record") -> None:
+ sent_offset = record.request.status_report.sent_offset
+ self._context.last_sent_offset = sent_offset
+
+ def on_exit(self, record: "Record") -> StateContext:
+ return self._context
+
+ def on_enter(self, record: "Record", context: StateContext) -> None:
+ self._context = context
+
+ @property
+ def _behind_bytes(self) -> int:
+ return self._context.last_forwarded_offset - self._context.last_sent_offset
+
+
+class StateForwarding(StateShared):
+ _forward_record: Callable[["Record"], None]
+ _pause_marker: Callable[[], None]
+ _threshold_pause: int
+
+ def __init__(
+ self,
+ forward_record: Callable[["Record"], None],
+ pause_marker: Callable[[], None],
+ threshold_pause: int,
+ ) -> None:
+ super().__init__()
+ self._forward_record = forward_record
+ self._pause_marker = pause_marker
+ self._threshold_pause = threshold_pause
+
+ def _should_pause(self, record: "Record") -> bool:
+ return self._behind_bytes >= self._threshold_pause
+
+ def _pause(self, record: "Record") -> None:
+ self._pause_marker()
+
+ def on_check(self, record: "Record") -> None:
+ self._update_written_offset(record)
+ self._process(record)
+ if not _is_control_record(record):
+ self._forward_record(record)
+ self._update_forwarded_offset()
+
+
+class StatePausing(StateShared):
+ _forward_record: Callable[["Record"], None]
+ _recover_records: Callable[[int, int], None]
+ _threshold_recover: int
+ _threshold_forward: int
+
+ def __init__(
+ self,
+ forward_record: Callable[["Record"], None],
+ recover_records: Callable[[int, int], None],
+ threshold_recover: int,
+ threshold_forward: int,
+ ) -> None:
+ super().__init__()
+ self._forward_record = forward_record
+ self._recover_records = recover_records
+ self._threshold_recover = threshold_recover
+ self._threshold_forward = threshold_forward
+
+ def _should_unpause(self, record: "Record") -> bool:
+ return self._behind_bytes < self._threshold_forward
+
+ def _unpause(self, record: "Record") -> None:
+ self._quiesce(record)
+
+ def _should_recover(self, record: "Record") -> bool:
+ return self._behind_bytes < self._threshold_recover
+
+ def _recover(self, record: "Record") -> None:
+ self._quiesce(record)
+
+ def _should_quiesce(self, record: "Record") -> bool:
+ return _is_local_non_control_record(record)
+
+ def _quiesce(self, record: "Record") -> None:
+ start = self._context.last_forwarded_offset
+ end = self._context.last_written_offset
+ if start != end:
+ self._recover_records(start, end)
+ if _is_local_non_control_record(record):
+ self._forward_record(record)
+ self._update_forwarded_offset()
+
+ def on_check(self, record: "Record") -> None:
+ self._update_written_offset(record)
+ self._process(record)
diff --git a/wandb/sdk/internal/handler.py b/wandb/sdk/internal/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..3426864dcbfaa5437352fd9e3942eeea35f7801e
--- /dev/null
+++ b/wandb/sdk/internal/handler.py
@@ -0,0 +1,896 @@
+"""Handle Manager."""
+
+import json
+import logging
+import math
+import numbers
+import time
+from collections import defaultdict
+from queue import Queue
+from threading import Event
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Callable,
+ Dict,
+ Iterable,
+ List,
+ Optional,
+ Sequence,
+ Tuple,
+ cast,
+)
+
+from wandb.proto.wandb_internal_pb2 import (
+ HistoryRecord,
+ InternalMessages,
+ MetricRecord,
+ Record,
+ Result,
+ RunRecord,
+ SampledHistoryItem,
+ SummaryItem,
+ SummaryRecord,
+ SummaryRecordRequest,
+ SystemMetricSample,
+ SystemMetricsBuffer,
+)
+
+from ..interface.interface_queue import InterfaceQueue
+from ..lib import handler_util, proto_util, tracelog, wburls
+from . import context, sample, tb_watcher
+from .settings_static import SettingsStatic
+from .system.system_monitor import SystemMonitor
+
+if TYPE_CHECKING:
+ from wandb.proto.wandb_internal_pb2 import MetricSummary
+
+
+SummaryDict = Dict[str, Any]
+
+logger = logging.getLogger(__name__)
+
+
+def _dict_nested_set(target: Dict[str, Any], key_list: Sequence[str], v: Any) -> None:
+ # recurse down the dictionary structure:
+
+ for k in key_list[:-1]:
+ target.setdefault(k, {})
+ new_target = target.get(k)
+ if TYPE_CHECKING:
+ new_target = cast(Dict[str, Any], new_target)
+ target = new_target
+ # use the last element of the key to write the leaf:
+ target[key_list[-1]] = v
+
+
+class HandleManager:
+ _consolidated_summary: SummaryDict
+ _sampled_history: Dict[str, sample.UniformSampleAccumulator]
+ _partial_history: Dict[str, Any]
+ _run_proto: Optional[RunRecord]
+ _settings: SettingsStatic
+ _record_q: "Queue[Record]"
+ _result_q: "Queue[Result]"
+ _stopped: Event
+ _writer_q: "Queue[Record]"
+ _interface: InterfaceQueue
+ _system_monitor: Optional[SystemMonitor]
+ _tb_watcher: Optional[tb_watcher.TBWatcher]
+ _metric_defines: Dict[str, MetricRecord]
+ _metric_globs: Dict[str, MetricRecord]
+ _metric_track: Dict[Tuple[str, ...], float]
+ _metric_copy: Dict[Tuple[str, ...], Any]
+ _track_time: Optional[float]
+ _accumulate_time: float
+ _run_start_time: Optional[float]
+ _context_keeper: context.ContextKeeper
+
+ def __init__(
+ self,
+ settings: SettingsStatic,
+ record_q: "Queue[Record]",
+ result_q: "Queue[Result]",
+ stopped: Event,
+ writer_q: "Queue[Record]",
+ interface: InterfaceQueue,
+ context_keeper: context.ContextKeeper,
+ ) -> None:
+ self._settings = settings
+ self._record_q = record_q
+ self._result_q = result_q
+ self._stopped = stopped
+ self._writer_q = writer_q
+ self._interface = interface
+ self._context_keeper = context_keeper
+
+ self._tb_watcher = None
+ self._system_monitor = None
+ self._step = 0
+
+ self._track_time = None
+ self._accumulate_time = 0
+ self._run_start_time = None
+
+ # keep track of summary from key/val updates
+ self._consolidated_summary = dict()
+ self._sampled_history = defaultdict(sample.UniformSampleAccumulator)
+ self._run_proto = None
+ self._partial_history = dict()
+ self._metric_defines = defaultdict(MetricRecord)
+ self._metric_globs = defaultdict(MetricRecord)
+ self._metric_track = dict()
+ self._metric_copy = dict()
+ self._internal_messages = InternalMessages()
+
+ self._dropped_history = False
+
+ def __len__(self) -> int:
+ return self._record_q.qsize()
+
+ def handle(self, record: Record) -> None:
+ self._context_keeper.add_from_record(record)
+ record_type = record.WhichOneof("record_type")
+ assert record_type
+ handler_str = "handle_" + record_type
+ handler: Callable[[Record], None] = getattr(self, handler_str, None) # type: ignore
+ assert handler, f"unknown handle: {handler_str}" # type: ignore
+ handler(record)
+
+ def handle_request(self, record: Record) -> None:
+ request_type = record.request.WhichOneof("request_type")
+ assert request_type
+ handler_str = "handle_request_" + request_type
+ handler: Callable[[Record], None] = getattr(self, handler_str, None) # type: ignore
+ if request_type != "network_status":
+ logger.debug(f"handle_request: {request_type}")
+ assert handler, f"unknown handle: {handler_str}" # type: ignore
+ handler(record)
+
+ def _dispatch_record(self, record: Record, always_send: bool = False) -> None:
+ if always_send:
+ record.control.always_send = True
+ tracelog.log_message_queue(record, self._writer_q)
+ self._writer_q.put(record)
+
+ def _respond_result(self, result: Result) -> None:
+ tracelog.log_message_queue(result, self._result_q)
+ context_id = context.context_id_from_result(result)
+ self._context_keeper.release(context_id)
+ self._result_q.put(result)
+
+ def debounce(self) -> None:
+ pass
+
+ def handle_request_cancel(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_request_defer(self, record: Record) -> None:
+ defer = record.request.defer
+ state = defer.state
+
+ logger.info(f"handle defer: {state}")
+ # only handle flush tb (sender handles the rest)
+ if state == defer.FLUSH_STATS:
+ # TODO(jhr): this could block so we dont really want to call shutdown
+ # from handler thread
+ if self._system_monitor is not None:
+ self._system_monitor.finish()
+ elif state == defer.FLUSH_TB:
+ if self._tb_watcher:
+ # shutdown tensorboard workers so we get all metrics flushed
+ self._tb_watcher.finish()
+ self._tb_watcher = None
+ elif state == defer.FLUSH_PARTIAL_HISTORY:
+ self._flush_partial_history()
+ elif state == defer.FLUSH_SUM:
+ self._save_summary(self._consolidated_summary, flush=True)
+
+ # defer is used to drive the sender finish state machine
+ self._dispatch_record(record, always_send=True)
+
+ def handle_request_login(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_request_python_packages(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_run(self, record: Record) -> None:
+ if self._settings._offline:
+ self._run_proto = record.run
+ result = proto_util._result_from_record(record)
+ result.run_result.run.CopyFrom(record.run)
+ self._respond_result(result)
+ self._dispatch_record(record)
+
+ def handle_stats(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_config(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_output(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_output_raw(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_files(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_link_artifact(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_use_artifact(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_artifact(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_alert(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def _save_summary(self, summary_dict: SummaryDict, flush: bool = False) -> None:
+ summary = SummaryRecord()
+ for k, v in summary_dict.items():
+ update = summary.update.add()
+ update.key = k
+ update.value_json = json.dumps(v)
+ if flush:
+ record = Record(summary=summary)
+ self._dispatch_record(record)
+ elif not self._settings._offline:
+ # Send this summary update as a request since we aren't persisting every update
+ summary_record = SummaryRecordRequest(summary=summary)
+ request_record = self._interface._make_request(
+ summary_record=summary_record
+ )
+ self._dispatch_record(request_record)
+
+ def _save_history(
+ self,
+ history: HistoryRecord,
+ ) -> None:
+ for item in history.item:
+ # TODO(jhr) save nested keys?
+ k = item.key
+ v = json.loads(item.value_json)
+ if isinstance(v, numbers.Real):
+ self._sampled_history[k].add(v)
+
+ def _update_summary_metrics(
+ self,
+ s: "MetricSummary",
+ kl: List[str],
+ v: "numbers.Real",
+ float_v: float,
+ goal_max: Optional[bool],
+ ) -> bool:
+ updated = False
+ best_key: Optional[Tuple[str, ...]] = None
+ if s.none:
+ return False
+ if s.copy:
+ # non-key list copy already done in _update_summary
+ if len(kl) > 1:
+ _dict_nested_set(self._consolidated_summary, kl, v)
+ return True
+ if s.last:
+ last_key = tuple(kl + ["last"])
+ old_last = self._metric_track.get(last_key)
+ if old_last is None or float_v != old_last:
+ self._metric_track[last_key] = float_v
+ _dict_nested_set(self._consolidated_summary, last_key, v)
+ updated = True
+ if s.best:
+ best_key = tuple(kl + ["best"])
+ if s.max or best_key and goal_max:
+ max_key = tuple(kl + ["max"])
+ old_max = self._metric_track.get(max_key)
+ if old_max is None or float_v > old_max:
+ self._metric_track[max_key] = float_v
+ if s.max:
+ _dict_nested_set(self._consolidated_summary, max_key, v)
+ updated = True
+ if best_key:
+ _dict_nested_set(self._consolidated_summary, best_key, v)
+ updated = True
+ # defaulting to minimize if goal is not specified
+ if s.min or best_key and not goal_max:
+ min_key = tuple(kl + ["min"])
+ old_min = self._metric_track.get(min_key)
+ if old_min is None or float_v < old_min:
+ self._metric_track[min_key] = float_v
+ if s.min:
+ _dict_nested_set(self._consolidated_summary, min_key, v)
+ updated = True
+ if best_key:
+ _dict_nested_set(self._consolidated_summary, best_key, v)
+ updated = True
+ if s.mean:
+ tot_key = tuple(kl + ["tot"])
+ num_key = tuple(kl + ["num"])
+ avg_key = tuple(kl + ["mean"])
+ tot = self._metric_track.get(tot_key, 0.0)
+ num = self._metric_track.get(num_key, 0)
+ tot += float_v
+ num += 1
+ self._metric_track[tot_key] = tot
+ self._metric_track[num_key] = num
+ _dict_nested_set(self._consolidated_summary, avg_key, tot / num)
+ updated = True
+ return updated
+
+ def _update_summary_leaf(
+ self,
+ kl: List[str],
+ v: Any,
+ d: Optional[MetricRecord] = None,
+ ) -> bool:
+ has_summary = d and d.HasField("summary")
+ if len(kl) == 1:
+ copy_key = tuple(kl)
+ old_copy = self._metric_copy.get(copy_key)
+ if old_copy is None or v != old_copy:
+ self._metric_copy[copy_key] = v
+ # Store copy metric if not specified, or copy behavior
+ if not has_summary or (d and d.summary.copy):
+ self._consolidated_summary[kl[0]] = v
+ return True
+ if not d:
+ return False
+ if not has_summary:
+ return False
+ if not isinstance(v, numbers.Real):
+ return False
+ if math.isnan(v):
+ return False
+ float_v = float(v)
+ goal_max = None
+ if d.goal:
+ goal_max = d.goal == d.GOAL_MAXIMIZE
+ if self._update_summary_metrics(
+ d.summary, kl=kl, v=v, float_v=float_v, goal_max=goal_max
+ ):
+ return True
+ return False
+
+ def _update_summary_list(
+ self,
+ kl: List[str],
+ v: Any,
+ d: Optional[MetricRecord] = None,
+ ) -> bool:
+ metric_key = ".".join([k.replace(".", "\\.") for k in kl])
+ d = self._metric_defines.get(metric_key, d)
+ # if the dict has _type key, it's a wandb table object
+ if isinstance(v, dict) and not handler_util.metric_is_wandb_dict(v):
+ updated = False
+ for nk, nv in v.items():
+ if self._update_summary_list(kl=kl[:] + [nk], v=nv, d=d):
+ updated = True
+ return updated
+ # If the dict is a media object, update the pointer to the latest alias
+ elif isinstance(v, dict) and handler_util.metric_is_wandb_dict(v):
+ if "_latest_artifact_path" in v and "artifact_path" in v:
+ # TODO: Make non-destructive?
+ v["artifact_path"] = v["_latest_artifact_path"]
+ updated = self._update_summary_leaf(kl=kl, v=v, d=d)
+ return updated
+
+ def _update_summary_media_objects(self, v: Dict[str, Any]) -> Dict[str, Any]:
+ # For now, non-recursive - just top level
+ for nk, nv in v.items():
+ if (
+ isinstance(nv, dict)
+ and handler_util.metric_is_wandb_dict(nv)
+ and "_latest_artifact_path" in nv
+ and "artifact_path" in nv
+ ):
+ # TODO: Make non-destructive?
+ nv["artifact_path"] = nv["_latest_artifact_path"]
+ v[nk] = nv
+ return v
+
+ def _update_summary(self, history_dict: Dict[str, Any]) -> List[str]:
+ # keep old behavior fast path if no define metrics have been used
+ if not self._metric_defines:
+ history_dict = self._update_summary_media_objects(history_dict)
+ self._consolidated_summary.update(history_dict)
+ return list(history_dict.keys())
+ updated_keys = []
+ for k, v in history_dict.items():
+ if self._update_summary_list(kl=[k], v=v):
+ updated_keys.append(k)
+ return updated_keys
+
+ def _history_assign_step(
+ self,
+ history: HistoryRecord,
+ history_dict: Dict[str, Any],
+ ) -> None:
+ has_step = history.HasField("step")
+ item = history.item.add()
+ item.key = "_step"
+ if has_step:
+ step = history.step.num
+ history_dict["_step"] = step
+ item.value_json = json.dumps(step)
+ self._step = step + 1
+ else:
+ history_dict["_step"] = self._step
+ item.value_json = json.dumps(self._step)
+ self._step += 1
+
+ def _history_define_metric(self, hkey: str) -> Optional[MetricRecord]:
+ """Check for hkey match in glob metrics and return the defined metric."""
+ # Dont define metric for internal metrics
+ if hkey.startswith("_"):
+ return None
+ for k, mglob in self._metric_globs.items():
+ if k.endswith("*"):
+ if hkey.startswith(k[:-1]):
+ m = MetricRecord()
+ m.CopyFrom(mglob)
+ m.ClearField("glob_name")
+ m.options.defined = False
+ m.name = hkey
+ return m
+ return None
+
+ def _history_update_leaf(
+ self,
+ kl: List[str],
+ v: Any,
+ history_dict: Dict[str, Any],
+ update_history: Dict[str, Any],
+ ) -> None:
+ hkey = ".".join([k.replace(".", "\\.") for k in kl])
+ m = self._metric_defines.get(hkey)
+ if not m:
+ m = self._history_define_metric(hkey)
+ if not m:
+ return
+ mr = Record()
+ mr.metric.CopyFrom(m)
+ mr.control.local = True # Dont store this, just send it
+ self._handle_defined_metric(mr)
+
+ if m.options.step_sync and m.step_metric:
+ if m.step_metric not in history_dict:
+ copy_key = tuple([m.step_metric])
+ step = self._metric_copy.get(copy_key)
+ if step is not None:
+ update_history[m.step_metric] = step
+
+ def _history_update_list(
+ self,
+ kl: List[str],
+ v: Any,
+ history_dict: Dict[str, Any],
+ update_history: Dict[str, Any],
+ ) -> None:
+ if isinstance(v, dict):
+ for nk, nv in v.items():
+ self._history_update_list(
+ kl=kl[:] + [nk],
+ v=nv,
+ history_dict=history_dict,
+ update_history=update_history,
+ )
+ return
+ self._history_update_leaf(
+ kl=kl, v=v, history_dict=history_dict, update_history=update_history
+ )
+
+ def _history_update(
+ self,
+ history: HistoryRecord,
+ history_dict: Dict[str, Any],
+ ) -> None:
+ # if syncing an old run, we can skip this logic
+ if history_dict.get("_step") is None:
+ self._history_assign_step(history, history_dict)
+
+ update_history: Dict[str, Any] = {}
+ # Look for metric matches
+ if self._metric_defines or self._metric_globs:
+ for hkey, hval in history_dict.items():
+ self._history_update_list([hkey], hval, history_dict, update_history)
+
+ if update_history:
+ history_dict.update(update_history)
+ for k, v in update_history.items():
+ item = history.item.add()
+ item.key = k
+ item.value_json = json.dumps(v)
+
+ def handle_history(self, record: Record) -> None:
+ history_dict = proto_util.dict_from_proto_list(record.history.item)
+
+ # Inject _runtime if it is not present
+ if history_dict is not None:
+ if "_runtime" not in history_dict:
+ self._history_assign_runtime(record.history, history_dict)
+
+ self._history_update(record.history, history_dict)
+ self._dispatch_record(record)
+ self._save_history(record.history)
+ # update summary from history
+ updated_keys = self._update_summary(history_dict)
+ if updated_keys:
+ updated_items = {k: self._consolidated_summary[k] for k in updated_keys}
+ self._save_summary(updated_items)
+
+ def _flush_partial_history(
+ self,
+ step: Optional[int] = None,
+ ) -> None:
+ if not self._partial_history:
+ return
+
+ history = HistoryRecord()
+ for k, v in self._partial_history.items():
+ item = history.item.add()
+ item.key = k
+ item.value_json = json.dumps(v)
+ if step is not None:
+ history.step.num = step
+ self.handle_history(Record(history=history))
+ self._partial_history = {}
+
+ def handle_request_sender_mark_report(self, record: Record) -> None:
+ self._dispatch_record(record, always_send=True)
+
+ def handle_request_status_report(self, record: Record) -> None:
+ self._dispatch_record(record, always_send=True)
+
+ def handle_request_partial_history(self, record: Record) -> None:
+ partial_history = record.request.partial_history
+
+ flush = None
+ if partial_history.HasField("action"):
+ flush = partial_history.action.flush
+
+ step = None
+ if partial_history.HasField("step"):
+ step = partial_history.step.num
+
+ history_dict = proto_util.dict_from_proto_list(partial_history.item)
+ if step is not None:
+ if step < self._step:
+ if not self._dropped_history:
+ message = (
+ "Step only supports monotonically increasing values, use define_metric to set a custom x "
+ f"axis. For details see: {wburls.wburls.get('wandb_define_metric')}"
+ )
+ self._internal_messages.warning.append(message)
+ self._dropped_history = True
+ message = (
+ f"(User provided step: {step} is less than current step: {self._step}. "
+ f"Dropping entry: {history_dict})."
+ )
+ self._internal_messages.warning.append(message)
+ return
+ elif step > self._step:
+ self._flush_partial_history()
+ self._step = step
+ elif flush is None:
+ flush = True
+
+ self._partial_history.update(history_dict)
+
+ if flush:
+ self._flush_partial_history(self._step)
+
+ def handle_summary(self, record: Record) -> None:
+ summary = record.summary
+ for item in summary.update:
+ if len(item.nested_key) > 0:
+ # we use either key or nested_key -- not both
+ assert item.key == ""
+ key = tuple(item.nested_key)
+ else:
+ # no counter-assertion here, because technically
+ # summary[""] is valid
+ key = (item.key,)
+
+ target = self._consolidated_summary
+
+ # recurse down the dictionary structure:
+ for prop in key[:-1]:
+ target = target[prop]
+
+ # use the last element of the key to write the leaf:
+ target[key[-1]] = json.loads(item.value_json)
+
+ for item in summary.remove:
+ if len(item.nested_key) > 0:
+ # we use either key or nested_key -- not both
+ assert item.key == ""
+ key = tuple(item.nested_key)
+ else:
+ # no counter-assertion here, because technically
+ # summary[""] is valid
+ key = (item.key,)
+
+ target = self._consolidated_summary
+
+ # recurse down the dictionary structure:
+ for prop in key[:-1]:
+ target = target[prop]
+
+ # use the last element of the key to erase the leaf:
+ del target[key[-1]]
+
+ self._save_summary(self._consolidated_summary)
+
+ def handle_exit(self, record: Record) -> None:
+ if self._track_time is not None:
+ self._accumulate_time += time.time() - self._track_time
+ record.exit.runtime = int(self._accumulate_time)
+ self._dispatch_record(record, always_send=True)
+
+ def handle_final(self, record: Record) -> None:
+ self._dispatch_record(record, always_send=True)
+
+ def handle_preempting(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_header(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_footer(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_request_check_version(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_request_attach(self, record: Record) -> None:
+ result = proto_util._result_from_record(record)
+ attach_id = record.request.attach.attach_id
+ assert attach_id
+ assert self._run_proto
+ result.response.attach_response.run.CopyFrom(self._run_proto)
+ self._respond_result(result)
+
+ def handle_request_log_artifact(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_telemetry(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_request_run_start(self, record: Record) -> None:
+ run_start = record.request.run_start
+ assert run_start
+ assert run_start.run
+
+ self._run_proto = run_start.run
+
+ self._run_start_time = run_start.run.start_time.ToMicroseconds() / 1e6
+
+ self._track_time = time.time()
+ if run_start.run.resumed and run_start.run.runtime:
+ self._accumulate_time = run_start.run.runtime
+ else:
+ self._accumulate_time = 0
+
+ # system monitor
+ self._system_monitor = SystemMonitor(
+ self._settings,
+ self._interface,
+ )
+ if not self._settings._disable_stats:
+ self._system_monitor.start()
+ if not self._settings._disable_meta and not run_start.run.resumed:
+ self._system_monitor.probe(publish=True)
+
+ self._tb_watcher = tb_watcher.TBWatcher(
+ self._settings, interface=self._interface, run_proto=run_start.run
+ )
+
+ if run_start.run.resumed:
+ self._step = run_start.run.starting_step
+ result = proto_util._result_from_record(record)
+ self._respond_result(result)
+
+ def handle_request_resume(self, record: Record) -> None:
+ if self._system_monitor is not None:
+ logger.info("starting system metrics thread")
+ self._system_monitor.start()
+
+ if self._track_time is not None:
+ self._accumulate_time += time.time() - self._track_time
+ self._track_time = time.time()
+
+ def handle_request_pause(self, record: Record) -> None:
+ if self._system_monitor is not None:
+ logger.info("stopping system metrics thread")
+ self._system_monitor.finish()
+ if self._track_time is not None:
+ self._accumulate_time += time.time() - self._track_time
+ self._track_time = None
+
+ def handle_request_poll_exit(self, record: Record) -> None:
+ self._dispatch_record(record, always_send=True)
+
+ def handle_request_stop_status(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_request_network_status(self, record: Record) -> None:
+ self._dispatch_record(record)
+
+ def handle_request_internal_messages(self, record: Record) -> None:
+ result = proto_util._result_from_record(record)
+ result.response.internal_messages_response.messages.CopyFrom(
+ self._internal_messages
+ )
+ self._internal_messages.Clear()
+ self._respond_result(result)
+
+ def handle_request_status(self, record: Record) -> None:
+ # TODO(mempressure): do something better?
+ assert record.control.req_resp
+ result = proto_util._result_from_record(record)
+ self._respond_result(result)
+
+ def handle_request_get_summary(self, record: Record) -> None:
+ result = proto_util._result_from_record(record)
+ for key, value in self._consolidated_summary.items():
+ item = SummaryItem()
+ item.key = key
+ item.value_json = json.dumps(value)
+ result.response.get_summary_response.item.append(item)
+ self._respond_result(result)
+
+ def handle_request_get_system_metrics(self, record: Record) -> None:
+ result = proto_util._result_from_record(record)
+ if self._system_monitor is None:
+ return
+
+ buffer = self._system_monitor.buffer
+ for key, samples in buffer.items():
+ buff = []
+ for s in samples:
+ sms = SystemMetricSample()
+ sms.timestamp.FromMicroseconds(int(s[0] * 1e6))
+ sms.value = s[1]
+ buff.append(sms)
+
+ result.response.get_system_metrics_response.system_metrics[key].CopyFrom(
+ SystemMetricsBuffer(record=buff)
+ )
+
+ self._respond_result(result)
+
+ def handle_tbrecord(self, record: Record) -> None:
+ logger.info("handling tbrecord: %s", record)
+ if self._tb_watcher:
+ tbrecord = record.tbrecord
+ self._tb_watcher.add(tbrecord.log_dir, tbrecord.save, tbrecord.root_dir)
+ self._dispatch_record(record)
+
+ def _handle_defined_metric(self, record: Record) -> None:
+ metric = record.metric
+ if metric._control.overwrite:
+ self._metric_defines[metric.name].CopyFrom(metric)
+ else:
+ self._metric_defines[metric.name].MergeFrom(metric)
+
+ # before dispatching, make sure step_metric is defined, if not define it and
+ # dispatch it locally first
+ metric = self._metric_defines[metric.name]
+ if metric.step_metric and metric.step_metric not in self._metric_defines:
+ m = MetricRecord(name=metric.step_metric)
+ self._metric_defines[metric.step_metric] = m
+ mr = Record()
+ mr.metric.CopyFrom(m)
+ mr.control.local = True # Don't store this, just send it
+ self._dispatch_record(mr)
+
+ self._dispatch_record(record)
+
+ def _handle_glob_metric(self, record: Record) -> None:
+ metric = record.metric
+ if metric._control.overwrite:
+ self._metric_globs[metric.glob_name].CopyFrom(metric)
+ else:
+ self._metric_globs[metric.glob_name].MergeFrom(metric)
+ self._dispatch_record(record)
+
+ def handle_metric(self, record: Record) -> None:
+ """Handle MetricRecord.
+
+ Walkthrough of the life of a MetricRecord:
+
+ Metric defined:
+ - run.define_metric() parses arguments create wandb_metric.Metric
+ - build MetricRecord publish to interface
+ - handler (this function) keeps list of metrics published:
+ - self._metric_defines: Fully defined metrics
+ - self._metric_globs: metrics that have a wildcard
+ - dispatch writer and sender thread
+ - writer: records are saved to persistent store
+ - sender: fully defined metrics get mapped into metadata for UI
+
+ History logged:
+ - handle_history
+ - check if metric matches _metric_defines
+ - if not, check if metric matches _metric_globs
+ - if _metric globs match, generate defined metric and call _handle_metric
+
+ Args:
+ record (Record): Metric record to process
+ """
+ if record.metric.name:
+ self._handle_defined_metric(record)
+ elif record.metric.glob_name:
+ self._handle_glob_metric(record)
+
+ def handle_request_sampled_history(self, record: Record) -> None:
+ result = proto_util._result_from_record(record)
+ for key, sampled in self._sampled_history.items():
+ item = SampledHistoryItem()
+ item.key = key
+ values: Iterable[Any] = sampled.get()
+ if all(isinstance(i, numbers.Integral) for i in values):
+ try:
+ item.values_int.extend(values)
+ except ValueError:
+ # it is safe to ignore these as this is for display information
+ pass
+ elif all(isinstance(i, numbers.Real) for i in values):
+ item.values_float.extend(values)
+ result.response.sampled_history_response.item.append(item)
+ self._respond_result(result)
+
+ def handle_request_server_info(self, record: Record) -> None:
+ self._dispatch_record(record, always_send=True)
+
+ def handle_request_keepalive(self, record: Record) -> None:
+ """Handle a keepalive request.
+
+ Keepalive is a noop, we just want to verify transport is alive.
+ """
+
+ def handle_request_run_status(self, record: Record) -> None:
+ self._dispatch_record(record, always_send=True)
+
+ def handle_request_shutdown(self, record: Record) -> None:
+ # TODO(jhr): should we drain things and stop new requests from coming in?
+ result = proto_util._result_from_record(record)
+ self._respond_result(result)
+ self._stopped.set()
+
+ def handle_request_job_info(self, record: Record) -> None:
+ self._dispatch_record(record, always_send=True)
+
+ def finish(self) -> None:
+ logger.info("shutting down handler")
+ if self._system_monitor is not None:
+ self._system_monitor.finish()
+ if self._tb_watcher:
+ self._tb_watcher.finish()
+ # self._context_keeper._debug_print_orphans()
+
+ def __next__(self) -> Record:
+ return self._record_q.get(block=True)
+
+ next = __next__
+
+ def _history_assign_runtime(
+ self,
+ history: HistoryRecord,
+ history_dict: Dict[str, Any],
+ ) -> None:
+ # _runtime calculation is meaningless if there is no _timestamp
+ if "_timestamp" not in history_dict:
+ return
+ # if it is offline sync, self._run_start_time is None
+ # in that case set it to the first tfevent timestamp
+ if self._run_start_time is None:
+ self._run_start_time = history_dict["_timestamp"]
+ history_dict["_runtime"] = history_dict["_timestamp"] - self._run_start_time
+ item = history.item.add()
+ item.key = "_runtime"
+ item.value_json = json.dumps(history_dict[item.key])
diff --git a/wandb/sdk/internal/internal.py b/wandb/sdk/internal/internal.py
new file mode 100644
index 0000000000000000000000000000000000000000..ac934753bf85d76274e43e2b9f6b849469db2824
--- /dev/null
+++ b/wandb/sdk/internal/internal.py
@@ -0,0 +1,418 @@
+#
+"""Internal process.
+
+This module implements the entrypoint for the internal process. The internal process
+is responsible for handling "record" requests, and responding with "results". Data is
+passed to the process over multiprocessing queues.
+
+Threads:
+ HandlerThread -- read from record queue and call handlers
+ SenderThread -- send to network
+ WriterThread -- write to disk
+
+"""
+
+
+import atexit
+import logging
+import os
+import queue
+import sys
+import threading
+import time
+import traceback
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, List, Optional
+
+import psutil
+
+import wandb
+
+from ..interface.interface_queue import InterfaceQueue
+from ..lib import tracelog
+from . import context, handler, internal_util, sender, writer
+
+if TYPE_CHECKING:
+ from queue import Queue
+ from threading import Event
+
+ from wandb.proto.wandb_internal_pb2 import Record, Result
+
+ from .internal_util import RecordLoopThread
+ from .settings_static import SettingsStatic
+
+
+logger = logging.getLogger(__name__)
+
+
+def wandb_internal(
+ settings: "SettingsStatic",
+ record_q: "Queue[Record]",
+ result_q: "Queue[Result]",
+ port: Optional[int] = None,
+ user_pid: Optional[int] = None,
+) -> None:
+ """Internal process function entrypoint.
+
+ Read from record queue and dispatch work to various threads.
+
+ Arguments:
+ settings: settings object
+ record_q: records to be handled
+ result_q: for sending results back
+
+ """
+ # mark this process as internal
+ wandb._set_internal_process()
+ _setup_tracelog()
+ started = time.time()
+
+ # any sentry events in the internal process will be tagged as such
+ wandb._sentry.configure_scope(process_context="internal", tags=dict(settings))
+
+ # register the exit handler only when wandb_internal is called, not on import
+ @atexit.register
+ def handle_exit(*args: "Any") -> None:
+ logger.info("Internal process exited")
+
+ # Let's make sure we don't modify settings so use a static object
+ _settings = settings
+ if _settings.log_internal:
+ configure_logging(_settings.log_internal, _settings._log_level)
+
+ user_pid = user_pid or os.getppid()
+ pid = os.getpid()
+
+ logger.info(
+ "W&B internal server running at pid: %s, started at: %s",
+ pid,
+ datetime.fromtimestamp(started),
+ )
+
+ tracelog.annotate_queue(record_q, "record_q")
+ tracelog.annotate_queue(result_q, "result_q")
+ publish_interface = InterfaceQueue(record_q=record_q)
+
+ stopped = threading.Event()
+ threads: List[RecordLoopThread] = []
+
+ context_keeper = context.ContextKeeper()
+
+ send_record_q: Queue[Record] = queue.Queue()
+ tracelog.annotate_queue(send_record_q, "send_q")
+
+ write_record_q: Queue[Record] = queue.Queue()
+ tracelog.annotate_queue(write_record_q, "write_q")
+
+ record_sender_thread = SenderThread(
+ settings=_settings,
+ record_q=send_record_q,
+ result_q=result_q,
+ stopped=stopped,
+ interface=publish_interface,
+ debounce_interval_ms=5000,
+ context_keeper=context_keeper,
+ )
+ threads.append(record_sender_thread)
+
+ record_writer_thread = WriterThread(
+ settings=_settings,
+ record_q=write_record_q,
+ result_q=result_q,
+ stopped=stopped,
+ interface=publish_interface,
+ sender_q=send_record_q,
+ context_keeper=context_keeper,
+ )
+ threads.append(record_writer_thread)
+
+ record_handler_thread = HandlerThread(
+ settings=_settings,
+ record_q=record_q,
+ result_q=result_q,
+ stopped=stopped,
+ writer_q=write_record_q,
+ interface=publish_interface,
+ context_keeper=context_keeper,
+ )
+ threads.append(record_handler_thread)
+
+ process_check = ProcessCheck(settings=_settings, user_pid=user_pid)
+
+ for thread in threads:
+ thread.start()
+
+ interrupt_count = 0
+ while not stopped.is_set():
+ try:
+ # wait for stop event
+ while not stopped.is_set():
+ time.sleep(1)
+ if process_check.is_dead():
+ logger.error("Internal process shutdown.")
+ stopped.set()
+ except KeyboardInterrupt:
+ interrupt_count += 1
+ logger.warning(f"Internal process interrupt: {interrupt_count}")
+ finally:
+ if interrupt_count >= 2:
+ logger.error("Internal process interrupted.")
+ stopped.set()
+
+ for thread in threads:
+ thread.join()
+
+ def close_internal_log() -> None:
+ root = logging.getLogger("wandb")
+ for _handler in root.handlers[:]:
+ _handler.close()
+ root.removeHandler(_handler)
+
+ for thread in threads:
+ exc_info = thread.get_exception()
+ if exc_info:
+ logger.error(f"Thread {thread.name}:", exc_info=exc_info)
+ print(f"Thread {thread.name}:", file=sys.stderr)
+ traceback.print_exception(*exc_info)
+ wandb._sentry.exception(exc_info)
+ wandb.termerror("Internal wandb error: file data was not synced")
+ if not settings._disable_service:
+ # TODO: We can make this more graceful by returning an error to streams.py
+ # and potentially just fail the one stream.
+ os._exit(-1)
+ sys.exit(-1)
+
+ close_internal_log()
+
+
+def _setup_tracelog() -> None:
+ # TODO: remove this temporary hack, need to find a better way to pass settings
+ # to the server. for now lets just look at the environment variable we need
+ tracelog_mode = os.environ.get("WANDB_TRACELOG")
+ if tracelog_mode:
+ tracelog.enable(tracelog_mode)
+
+
+def configure_logging(
+ log_fname: str, log_level: int, run_id: Optional[str] = None
+) -> None:
+ # TODO: we may want make prints and stdout make it into the logs
+ # sys.stdout = open(settings.log_internal, "a")
+ # sys.stderr = open(settings.log_internal, "a")
+ log_handler = logging.FileHandler(log_fname)
+ log_handler.setLevel(log_level)
+
+ class WBFilter(logging.Filter):
+ def filter(self, record: "Any") -> bool:
+ record.run_id = run_id
+ return True
+
+ if run_id:
+ formatter = logging.Formatter(
+ "%(asctime)s %(levelname)-7s %(threadName)-10s:%(process)d "
+ "[%(run_id)s:%(filename)s:%(funcName)s():%(lineno)s] %(message)s"
+ )
+ else:
+ formatter = logging.Formatter(
+ "%(asctime)s %(levelname)-7s %(threadName)-10s:%(process)d "
+ "[%(filename)s:%(funcName)s():%(lineno)s] %(message)s"
+ )
+
+ log_handler.setFormatter(formatter)
+ if run_id:
+ log_handler.addFilter(WBFilter())
+ # If this is called without "wandb", backend logs from this module
+ # are not streamed to `debug-internal.log` when we spawn with fork
+ # TODO: (cvp) we should really take another pass at logging in general
+ root = logging.getLogger("wandb")
+ root.propagate = False
+ root.setLevel(logging.DEBUG)
+ root.addHandler(log_handler)
+
+
+class HandlerThread(internal_util.RecordLoopThread):
+ """Read records from queue and dispatch to handler routines."""
+
+ _record_q: "Queue[Record]"
+ _result_q: "Queue[Result]"
+ _stopped: "Event"
+ _context_keeper: context.ContextKeeper
+
+ def __init__(
+ self,
+ settings: "SettingsStatic",
+ record_q: "Queue[Record]",
+ result_q: "Queue[Result]",
+ stopped: "Event",
+ writer_q: "Queue[Record]",
+ interface: "InterfaceQueue",
+ context_keeper: context.ContextKeeper,
+ debounce_interval_ms: "float" = 1000,
+ ) -> None:
+ super().__init__(
+ input_record_q=record_q,
+ result_q=result_q,
+ stopped=stopped,
+ debounce_interval_ms=debounce_interval_ms,
+ )
+ self.name = "HandlerThread"
+ self._settings = settings
+ self._record_q = record_q
+ self._result_q = result_q
+ self._stopped = stopped
+ self._writer_q = writer_q
+ self._interface = interface
+ self._context_keeper = context_keeper
+
+ def _setup(self) -> None:
+ self._hm = handler.HandleManager(
+ settings=self._settings,
+ record_q=self._record_q,
+ result_q=self._result_q,
+ stopped=self._stopped,
+ writer_q=self._writer_q,
+ interface=self._interface,
+ context_keeper=self._context_keeper,
+ )
+
+ def _process(self, record: "Record") -> None:
+ self._hm.handle(record)
+
+ def _finish(self) -> None:
+ self._hm.finish()
+
+ def _debounce(self) -> None:
+ self._hm.debounce()
+
+
+class SenderThread(internal_util.RecordLoopThread):
+ """Read records from queue and dispatch to sender routines."""
+
+ _record_q: "Queue[Record]"
+ _result_q: "Queue[Result]"
+ _context_keeper: context.ContextKeeper
+
+ def __init__(
+ self,
+ settings: "SettingsStatic",
+ record_q: "Queue[Record]",
+ result_q: "Queue[Result]",
+ stopped: "Event",
+ interface: "InterfaceQueue",
+ context_keeper: context.ContextKeeper,
+ debounce_interval_ms: "float" = 5000,
+ ) -> None:
+ super().__init__(
+ input_record_q=record_q,
+ result_q=result_q,
+ stopped=stopped,
+ debounce_interval_ms=debounce_interval_ms,
+ )
+ self.name = "SenderThread"
+ self._settings = settings
+ self._record_q = record_q
+ self._result_q = result_q
+ self._interface = interface
+ self._context_keeper = context_keeper
+
+ def _setup(self) -> None:
+ self._sm = sender.SendManager(
+ settings=self._settings,
+ record_q=self._record_q,
+ result_q=self._result_q,
+ interface=self._interface,
+ context_keeper=self._context_keeper,
+ )
+
+ def _process(self, record: "Record") -> None:
+ self._sm.send(record)
+
+ def _finish(self) -> None:
+ self._sm.finish()
+
+ def _debounce(self) -> None:
+ self._sm.debounce()
+
+
+class WriterThread(internal_util.RecordLoopThread):
+ """Read records from queue and dispatch to writer routines."""
+
+ _record_q: "Queue[Record]"
+ _result_q: "Queue[Result]"
+ _context_keeper: context.ContextKeeper
+
+ def __init__(
+ self,
+ settings: "SettingsStatic",
+ record_q: "Queue[Record]",
+ result_q: "Queue[Result]",
+ stopped: "Event",
+ interface: "InterfaceQueue",
+ sender_q: "Queue[Record]",
+ context_keeper: context.ContextKeeper,
+ debounce_interval_ms: "float" = 1000,
+ ) -> None:
+ super().__init__(
+ input_record_q=record_q,
+ result_q=result_q,
+ stopped=stopped,
+ debounce_interval_ms=debounce_interval_ms,
+ )
+ self.name = "WriterThread"
+ self._settings = settings
+ self._record_q = record_q
+ self._result_q = result_q
+ self._sender_q = sender_q
+ self._interface = interface
+ self._context_keeper = context_keeper
+
+ def _setup(self) -> None:
+ self._wm = writer.WriteManager(
+ settings=self._settings,
+ record_q=self._record_q,
+ result_q=self._result_q,
+ sender_q=self._sender_q,
+ interface=self._interface,
+ context_keeper=self._context_keeper,
+ )
+
+ def _process(self, record: "Record") -> None:
+ self._wm.write(record)
+
+ def _finish(self) -> None:
+ self._wm.finish()
+
+ def _debounce(self) -> None:
+ self._wm.debounce()
+
+
+class ProcessCheck:
+ """Class to help watch a process id to detect when it is dead."""
+
+ check_process_last: Optional[float]
+
+ def __init__(self, settings: "SettingsStatic", user_pid: Optional[int]) -> None:
+ self.settings = settings
+ self.pid = user_pid
+ self.check_process_last = None
+ self.check_process_interval = settings._internal_check_process
+
+ def is_dead(self) -> bool:
+ if not self.check_process_interval or not self.pid:
+ return False
+ time_now = time.time()
+ if (
+ self.check_process_last
+ and time_now < self.check_process_last + self.check_process_interval
+ ):
+ return False
+ self.check_process_last = time_now
+
+ # TODO(jhr): check for os.getppid on unix being 1?
+ exists = psutil.pid_exists(self.pid)
+ if not exists:
+ logger.warning(
+ f"Internal process exiting, parent pid {self.pid} disappeared"
+ )
+ return True
+ return False
diff --git a/wandb/sdk/internal/internal_api.py b/wandb/sdk/internal/internal_api.py
new file mode 100644
index 0000000000000000000000000000000000000000..2be956ee75399ae9b0a60dfcd094de91f8abd16b
--- /dev/null
+++ b/wandb/sdk/internal/internal_api.py
@@ -0,0 +1,4200 @@
+import ast
+import asyncio
+import base64
+import datetime
+import functools
+import http.client
+import json
+import logging
+import os
+import re
+import socket
+import sys
+import threading
+from copy import deepcopy
+from typing import (
+ IO,
+ TYPE_CHECKING,
+ Any,
+ Callable,
+ Dict,
+ Iterable,
+ List,
+ Mapping,
+ MutableMapping,
+ Optional,
+ Sequence,
+ TextIO,
+ Tuple,
+ Union,
+)
+
+import click
+import requests
+import yaml
+from wandb_gql import Client, gql
+from wandb_gql.client import RetryError
+
+import wandb
+from wandb import env, util
+from wandb.apis.normalize import normalize_exceptions, parse_backend_error_messages
+from wandb.errors import CommError, UnsupportedError, UsageError
+from wandb.integration.sagemaker import parse_sm_secrets
+from wandb.old.settings import Settings
+from wandb.sdk.internal.thread_local_settings import _thread_local_api_settings
+from wandb.sdk.lib.gql_request import GraphQLSession
+from wandb.sdk.lib.hashutil import B64MD5, md5_file_b64
+
+from ..lib import retry
+from ..lib.filenames import DIFF_FNAME, METADATA_FNAME
+from ..lib.gitlib import GitRepo
+from . import context
+from .progress import AsyncProgress, Progress
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+ if sys.version_info >= (3, 8):
+ from typing import Literal, TypedDict
+ else:
+ from typing_extensions import Literal, TypedDict
+
+ from .progress import ProgressFn
+
+ class CreateArtifactFileSpecInput(TypedDict, total=False):
+ """Corresponds to `type CreateArtifactFileSpecInput` in schema.graphql."""
+
+ artifactID: str # noqa: N815
+ name: str
+ md5: str
+ mimetype: Optional[str]
+ artifactManifestID: Optional[str] # noqa: N815
+ uploadPartsInput: Optional[List[Dict[str, object]]] # noqa: N815
+
+ class CreateArtifactFilesResponseFile(TypedDict):
+ id: str
+ name: str
+ displayName: str # noqa: N815
+ uploadUrl: Optional[str] # noqa: N815
+ uploadHeaders: Sequence[str] # noqa: N815
+ uploadMultipartUrls: "UploadPartsResponse" # noqa: N815
+ storagePath: str # noqa: N815
+ artifact: "CreateArtifactFilesResponseFileNode"
+
+ class CreateArtifactFilesResponseFileNode(TypedDict):
+ id: str
+
+ class UploadPartsResponse(TypedDict):
+ uploadUrlParts: List["UploadUrlParts"] # noqa: N815
+ uploadID: str # noqa: N815
+
+ class UploadUrlParts(TypedDict):
+ partNumber: int # noqa: N815
+ uploadUrl: str # noqa: N815
+
+ class CompleteMultipartUploadArtifactInput(TypedDict):
+ """Corresponds to `type CompleteMultipartUploadArtifactInput` in schema.graphql."""
+
+ completeMultipartAction: str # noqa: N815
+ completedParts: Dict[int, str] # noqa: N815
+ artifactID: str # noqa: N815
+ storagePath: str # noqa: N815
+ uploadID: str # noqa: N815
+ md5: str
+
+ class CompleteMultipartUploadArtifactResponse(TypedDict):
+ digest: str
+
+ class DefaultSettings(TypedDict):
+ section: str
+ git_remote: str
+ ignore_globs: Optional[List[str]]
+ base_url: Optional[str]
+ root_dir: Optional[str]
+ api_key: Optional[str]
+ entity: Optional[str]
+ project: Optional[str]
+ _extra_http_headers: Optional[Mapping[str, str]]
+ _proxies: Optional[Mapping[str, str]]
+
+ _Response = MutableMapping
+ SweepState = Literal["RUNNING", "PAUSED", "CANCELED", "FINISHED"]
+ Number = Union[int, float]
+
+# This funny if/else construction is the simplest thing I've found that
+# works at runtime, satisfies Mypy, and gives autocomplete in VSCode:
+if TYPE_CHECKING:
+ import httpx
+else:
+ httpx = util.get_module("httpx")
+
+# class _MappingSupportsCopy(Protocol):
+# def copy(self) -> "_MappingSupportsCopy": ...
+# def keys(self) -> Iterable: ...
+# def __getitem__(self, name: str) -> Any: ...
+
+httpclient_logger = logging.getLogger("http.client")
+if os.environ.get("WANDB_DEBUG"):
+ httpclient_logger.setLevel(logging.DEBUG)
+
+
+def check_httpclient_logger_handler() -> None:
+ # Only enable http.client logging if WANDB_DEBUG is set
+ if not os.environ.get("WANDB_DEBUG"):
+ return
+ if httpclient_logger.handlers:
+ return
+
+ # Enable HTTPConnection debug logging to the logging framework
+ level = logging.DEBUG
+
+ def httpclient_log(*args: Any) -> None:
+ httpclient_logger.log(level, " ".join(args))
+
+ # mask the print() built-in in the http.client module to use logging instead
+ http.client.print = httpclient_log # type: ignore[attr-defined]
+ # enable debugging
+ http.client.HTTPConnection.debuglevel = 1
+
+ root_logger = logging.getLogger("wandb")
+ if root_logger.handlers:
+ httpclient_logger.addHandler(root_logger.handlers[0])
+
+
+def check_httpx_exc_retriable(exc: Exception) -> bool:
+ retriable_codes = (308, 408, 409, 429, 500, 502, 503, 504)
+ return (
+ isinstance(exc, (httpx.TimeoutException, httpx.NetworkError))
+ or (
+ isinstance(exc, httpx.HTTPStatusError)
+ and exc.response.status_code in retriable_codes
+ )
+ or (
+ isinstance(exc, httpx.HTTPStatusError)
+ and exc.response.status_code == 400
+ and "x-amz-meta-md5" in exc.request.headers
+ and "RequestTimeout" in str(exc.response.content)
+ )
+ )
+
+
+class _ThreadLocalData(threading.local):
+ context: Optional[context.Context]
+
+ def __init__(self) -> None:
+ self.context = None
+
+
+class Api:
+ """W&B Internal Api wrapper.
+
+ Note:
+ Settings are automatically overridden by looking for
+ a `wandb/settings` file in the current working directory or its parent
+ directory. If none can be found, we look in the current user's home
+ directory.
+
+ Arguments:
+ default_settings(dict, optional): If you aren't using a settings
+ file, or you wish to override the section to use in the settings file
+ Override the settings here.
+ """
+
+ HTTP_TIMEOUT = env.get_http_timeout(20)
+ FILE_PUSHER_TIMEOUT = env.get_file_pusher_timeout()
+ _global_context: context.Context
+ _local_data: _ThreadLocalData
+
+ def __init__(
+ self,
+ default_settings: Optional[
+ Union[
+ "wandb.sdk.wandb_settings.Settings",
+ "wandb.sdk.internal.settings_static.SettingsStatic",
+ Settings,
+ dict,
+ ]
+ ] = None,
+ load_settings: bool = True,
+ retry_timedelta: datetime.timedelta = datetime.timedelta( # noqa: B008 # okay because it's immutable
+ days=7
+ ),
+ environ: MutableMapping = os.environ,
+ retry_callback: Optional[Callable[[int, str], Any]] = None,
+ ) -> None:
+ self._environ = environ
+ self._global_context = context.Context()
+ self._local_data = _ThreadLocalData()
+ self.default_settings: DefaultSettings = {
+ "section": "default",
+ "git_remote": "origin",
+ "ignore_globs": [],
+ "base_url": "https://api.wandb.ai",
+ "root_dir": None,
+ "api_key": None,
+ "entity": None,
+ "project": None,
+ "_extra_http_headers": None,
+ "_proxies": None,
+ }
+ self.retry_timedelta = retry_timedelta
+ # todo: Old Settings do not follow the SupportsKeysAndGetItem Protocol
+ default_settings = default_settings or {}
+ self.default_settings.update(default_settings) # type: ignore
+ self.retry_uploads = 10
+ self._settings = Settings(
+ load_settings=load_settings,
+ root_dir=self.default_settings.get("root_dir"),
+ )
+ self.git = GitRepo(remote=self.settings("git_remote"))
+ # Mutable settings set by the _file_stream_api
+ self.dynamic_settings = {
+ "system_sample_seconds": 2,
+ "system_samples": 15,
+ "heartbeat_seconds": 30,
+ }
+
+ # todo: remove these hacky hacks after settings refactor is complete
+ # keeping this code here to limit scope and so that it is easy to remove later
+ extra_http_headers = self.settings("_extra_http_headers") or json.loads(
+ self._environ.get("WANDB__EXTRA_HTTP_HEADERS", "{}")
+ )
+ proxies = self.settings("_proxies") or json.loads(
+ self._environ.get("WANDB__PROXIES", "{}")
+ )
+
+ auth = None
+ if _thread_local_api_settings.cookies is None:
+ auth = ("api", self.api_key or "")
+ extra_http_headers.update(_thread_local_api_settings.headers or {})
+ self.client = Client(
+ transport=GraphQLSession(
+ headers={
+ "User-Agent": self.user_agent,
+ "X-WANDB-USERNAME": env.get_username(env=self._environ),
+ "X-WANDB-USER-EMAIL": env.get_user_email(env=self._environ),
+ **extra_http_headers,
+ },
+ use_json=True,
+ # this timeout won't apply when the DNS lookup fails. in that case, it will be 60s
+ # https://bugs.python.org/issue22889
+ timeout=self.HTTP_TIMEOUT,
+ auth=auth,
+ url=f"{self.settings('base_url')}/graphql",
+ cookies=_thread_local_api_settings.cookies,
+ proxies=proxies,
+ )
+ )
+
+ # httpx is an optional dependency, so we lazily instantiate the client
+ # only when we need it
+ self._async_httpx_client: Optional[httpx.AsyncClient] = None
+
+ self.retry_callback = retry_callback
+ self._retry_gql = retry.Retry(
+ self.execute,
+ retry_timedelta=retry_timedelta,
+ check_retry_fn=util.no_retry_auth,
+ retryable_exceptions=(RetryError, requests.RequestException),
+ retry_callback=retry_callback,
+ )
+ self._current_run_id: Optional[str] = None
+ self._file_stream_api = None
+ self._upload_file_session = requests.Session()
+ if self.FILE_PUSHER_TIMEOUT:
+ self._upload_file_session.put = functools.partial( # type: ignore
+ self._upload_file_session.put,
+ timeout=self.FILE_PUSHER_TIMEOUT,
+ )
+ if proxies:
+ self._upload_file_session.proxies.update(proxies)
+ # This Retry class is initialized once for each Api instance, so this
+ # defaults to retrying 1 million times per process or 7 days
+ self.upload_file_retry = normalize_exceptions(
+ retry.retriable(retry_timedelta=retry_timedelta)(self.upload_file)
+ )
+ self.upload_multipart_file_chunk_retry = normalize_exceptions(
+ retry.retriable(retry_timedelta=retry_timedelta)(
+ self.upload_multipart_file_chunk
+ )
+ )
+ self._client_id_mapping: Dict[str, str] = {}
+ # Large file uploads to azure can optionally use their SDK
+ self._azure_blob_module = util.get_module("azure.storage.blob")
+
+ self.query_types: Optional[List[str]] = None
+ self.mutation_types: Optional[List[str]] = None
+ self.server_info_types: Optional[List[str]] = None
+ self.server_use_artifact_input_info: Optional[List[str]] = None
+ self.server_create_artifact_input_info: Optional[List[str]] = None
+ self.server_artifact_fields_info: Optional[List[str]] = None
+ self._max_cli_version: Optional[str] = None
+ self._server_settings_type: Optional[List[str]] = None
+ self.fail_run_queue_item_input_info: Optional[List[str]] = None
+ self.create_launch_agent_input_info: Optional[List[str]] = None
+ self.server_create_run_queue_supports_drc: Optional[bool] = None
+ self.server_create_run_queue_supports_priority: Optional[bool] = None
+ self.server_supports_template_variables: Optional[bool] = None
+ self.server_push_to_run_queue_supports_priority: Optional[bool] = None
+
+ def gql(self, *args: Any, **kwargs: Any) -> Any:
+ ret = self._retry_gql(
+ *args,
+ retry_cancel_event=self.context.cancel_event,
+ **kwargs,
+ )
+ return ret
+
+ def set_local_context(self, api_context: Optional[context.Context]) -> None:
+ self._local_data.context = api_context
+
+ def clear_local_context(self) -> None:
+ self._local_data.context = None
+
+ @property
+ def context(self) -> context.Context:
+ return self._local_data.context or self._global_context
+
+ def reauth(self) -> None:
+ """Ensure the current api key is set in the transport."""
+ self.client.transport.session.auth = ("api", self.api_key or "")
+
+ def relocate(self) -> None:
+ """Ensure the current api points to the right server."""
+ self.client.transport.url = "%s/graphql" % self.settings("base_url")
+
+ def execute(self, *args: Any, **kwargs: Any) -> "_Response":
+ """Wrapper around execute that logs in cases of failure."""
+ try:
+ return self.client.execute(*args, **kwargs) # type: ignore
+ except requests.exceptions.HTTPError as err:
+ response = err.response
+ assert response is not None
+ logger.error(f"{response.status_code} response executing GraphQL.")
+ logger.error(response.text)
+ for error in parse_backend_error_messages(response):
+ wandb.termerror(f"Error while calling W&B API: {error} ({response})")
+ raise
+
+ def disabled(self) -> Union[str, bool]:
+ return self._settings.get(Settings.DEFAULT_SECTION, "disabled", fallback=False) # type: ignore
+
+ def set_current_run_id(self, run_id: str) -> None:
+ self._current_run_id = run_id
+
+ @property
+ def current_run_id(self) -> Optional[str]:
+ return self._current_run_id
+
+ @property
+ def user_agent(self) -> str:
+ return f"W&B Internal Client {wandb.__version__}"
+
+ @property
+ def api_key(self) -> Optional[str]:
+ if _thread_local_api_settings.api_key:
+ return _thread_local_api_settings.api_key
+ auth = requests.utils.get_netrc_auth(self.api_url)
+ key = None
+ if auth:
+ key = auth[-1]
+
+ # Environment should take precedence
+ env_key: Optional[str] = self._environ.get(env.API_KEY)
+ sagemaker_key: Optional[str] = parse_sm_secrets().get(env.API_KEY)
+ default_key: Optional[str] = self.default_settings.get("api_key")
+ return env_key or key or sagemaker_key or default_key
+
+ @property
+ def api_url(self) -> str:
+ return self.settings("base_url") # type: ignore
+
+ @property
+ def app_url(self) -> str:
+ return wandb.util.app_url(self.api_url)
+
+ @property
+ def default_entity(self) -> str:
+ return self.viewer().get("entity") # type: ignore
+
+ def settings(self, key: Optional[str] = None, section: Optional[str] = None) -> Any:
+ """The settings overridden from the wandb/settings file.
+
+ Arguments:
+ key (str, optional): If provided only this setting is returned
+ section (str, optional): If provided this section of the setting file is
+ used, defaults to "default"
+
+ Returns:
+ A dict with the current settings
+
+ {
+ "entity": "models",
+ "base_url": "https://api.wandb.ai",
+ "project": None
+ }
+ """
+ result = self.default_settings.copy()
+ result.update(self._settings.items(section=section)) # type: ignore
+ result.update(
+ {
+ "entity": env.get_entity(
+ self._settings.get(
+ Settings.DEFAULT_SECTION,
+ "entity",
+ fallback=result.get("entity"),
+ ),
+ env=self._environ,
+ ),
+ "project": env.get_project(
+ self._settings.get(
+ Settings.DEFAULT_SECTION,
+ "project",
+ fallback=result.get("project"),
+ ),
+ env=self._environ,
+ ),
+ "base_url": env.get_base_url(
+ self._settings.get(
+ Settings.DEFAULT_SECTION,
+ "base_url",
+ fallback=result.get("base_url"),
+ ),
+ env=self._environ,
+ ),
+ "ignore_globs": env.get_ignore(
+ self._settings.get(
+ Settings.DEFAULT_SECTION,
+ "ignore_globs",
+ fallback=result.get("ignore_globs"),
+ ),
+ env=self._environ,
+ ),
+ }
+ )
+
+ return result if key is None else result[key] # type: ignore
+
+ def clear_setting(
+ self, key: str, globally: bool = False, persist: bool = False
+ ) -> None:
+ self._settings.clear(
+ Settings.DEFAULT_SECTION, key, globally=globally, persist=persist
+ )
+
+ def set_setting(
+ self, key: str, value: Any, globally: bool = False, persist: bool = False
+ ) -> None:
+ self._settings.set(
+ Settings.DEFAULT_SECTION, key, value, globally=globally, persist=persist
+ )
+ if key == "entity":
+ env.set_entity(value, env=self._environ)
+ elif key == "project":
+ env.set_project(value, env=self._environ)
+ elif key == "base_url":
+ self.relocate()
+
+ def parse_slug(
+ self, slug: str, project: Optional[str] = None, run: Optional[str] = None
+ ) -> Tuple[str, str]:
+ """Parse a slug into a project and run.
+
+ Arguments:
+ slug (str): The slug to parse
+ project (str, optional): The project to use, if not provided it will be
+ inferred from the slug
+ run (str, optional): The run to use, if not provided it will be inferred
+ from the slug
+
+ Returns:
+ A dict with the project and run
+ """
+ if slug and "/" in slug:
+ parts = slug.split("/")
+ project = parts[0]
+ run = parts[1]
+ else:
+ project = project or self.settings().get("project")
+ if project is None:
+ raise CommError("No default project configured.")
+ run = run or slug or self.current_run_id or env.get_run(env=self._environ)
+ assert run, "run must be specified"
+ return project, run
+
+ @normalize_exceptions
+ def server_info_introspection(self) -> Tuple[List[str], List[str], List[str]]:
+ query_string = """
+ query ProbeServerCapabilities {
+ QueryType: __type(name: "Query") {
+ ...fieldData
+ }
+ MutationType: __type(name: "Mutation") {
+ ...fieldData
+ }
+ ServerInfoType: __type(name: "ServerInfo") {
+ ...fieldData
+ }
+ }
+
+ fragment fieldData on __Type {
+ fields {
+ name
+ }
+ }
+ """
+ if (
+ self.query_types is None
+ or self.mutation_types is None
+ or self.server_info_types is None
+ ):
+ query = gql(query_string)
+ res = self.gql(query)
+
+ self.query_types = [
+ field.get("name", "")
+ for field in res.get("QueryType", {}).get("fields", [{}])
+ ]
+ self.mutation_types = [
+ field.get("name", "")
+ for field in res.get("MutationType", {}).get("fields", [{}])
+ ]
+ self.server_info_types = [
+ field.get("name", "")
+ for field in res.get("ServerInfoType", {}).get("fields", [{}])
+ ]
+ return self.query_types, self.server_info_types, self.mutation_types
+
+ @normalize_exceptions
+ def server_settings_introspection(self) -> None:
+ query_string = """
+ query ProbeServerSettings {
+ ServerSettingsType: __type(name: "ServerSettings") {
+ ...fieldData
+ }
+ }
+
+ fragment fieldData on __Type {
+ fields {
+ name
+ }
+ }
+ """
+ if self._server_settings_type is None:
+ query = gql(query_string)
+ res = self.gql(query)
+ self._server_settings_type = (
+ [
+ field.get("name", "")
+ for field in res.get("ServerSettingsType", {}).get("fields", [{}])
+ ]
+ if res
+ else []
+ )
+
+ def server_use_artifact_input_introspection(self) -> List:
+ query_string = """
+ query ProbeServerUseArtifactInput {
+ UseArtifactInputInfoType: __type(name: "UseArtifactInput") {
+ name
+ inputFields {
+ name
+ }
+ }
+ }
+ """
+
+ if self.server_use_artifact_input_info is None:
+ query = gql(query_string)
+ res = self.gql(query)
+ self.server_use_artifact_input_info = [
+ field.get("name", "")
+ for field in res.get("UseArtifactInputInfoType", {}).get(
+ "inputFields", [{}]
+ )
+ ]
+ return self.server_use_artifact_input_info
+
+ @normalize_exceptions
+ def launch_agent_introspection(self) -> Optional[str]:
+ query = gql(
+ """
+ query LaunchAgentIntrospection {
+ LaunchAgentType: __type(name: "LaunchAgent") {
+ name
+ }
+ }
+ """
+ )
+
+ res = self.gql(query)
+ return res.get("LaunchAgentType") or None
+
+ @normalize_exceptions
+ def create_run_queue_introspection(self) -> Tuple[bool, bool, bool]:
+ _, _, mutations = self.server_info_introspection()
+ query_string = """
+ query ProbeCreateRunQueueInput {
+ CreateRunQueueInputType: __type(name: "CreateRunQueueInput") {
+ name
+ inputFields {
+ name
+ }
+ }
+ }
+ """
+ if (
+ self.server_create_run_queue_supports_drc is None
+ or self.server_create_run_queue_supports_priority is None
+ ):
+ query = gql(query_string)
+ res = self.gql(query)
+ if res is None:
+ raise CommError("Could not get CreateRunQueue input from GQL.")
+ self.server_create_run_queue_supports_drc = "defaultResourceConfigID" in [
+ x["name"]
+ for x in (
+ res.get("CreateRunQueueInputType", {}).get("inputFields", [{}])
+ )
+ ]
+ self.server_create_run_queue_supports_priority = "prioritizationMode" in [
+ x["name"]
+ for x in (
+ res.get("CreateRunQueueInputType", {}).get("inputFields", [{}])
+ )
+ ]
+ return (
+ "createRunQueue" in mutations,
+ self.server_create_run_queue_supports_drc,
+ self.server_create_run_queue_supports_priority,
+ )
+
+ @normalize_exceptions
+ def push_to_run_queue_introspection(self) -> Tuple[bool, bool]:
+ query_string = """
+ query ProbePushToRunQueueInput {
+ PushToRunQueueInputType: __type(name: "PushToRunQueueInput") {
+ name
+ inputFields {
+ name
+ }
+ }
+ }
+ """
+
+ if (
+ self.server_supports_template_variables is None
+ or self.server_push_to_run_queue_supports_priority is None
+ ):
+ query = gql(query_string)
+ res = self.gql(query)
+ self.server_supports_template_variables = "templateVariableValues" in [
+ x["name"]
+ for x in (
+ res.get("PushToRunQueueInputType", {}).get("inputFields", [{}])
+ )
+ ]
+ self.server_push_to_run_queue_supports_priority = "priority" in [
+ x["name"]
+ for x in (
+ res.get("PushToRunQueueInputType", {}).get("inputFields", [{}])
+ )
+ ]
+
+ return (
+ self.server_supports_template_variables,
+ self.server_push_to_run_queue_supports_priority,
+ )
+
+ @normalize_exceptions
+ def create_default_resource_config_introspection(self) -> bool:
+ _, _, mutations = self.server_info_introspection()
+ return "createDefaultResourceConfig" in mutations
+
+ @normalize_exceptions
+ def fail_run_queue_item_introspection(self) -> bool:
+ _, _, mutations = self.server_info_introspection()
+ return "failRunQueueItem" in mutations
+
+ @normalize_exceptions
+ def fail_run_queue_item_fields_introspection(self) -> List:
+ if self.fail_run_queue_item_input_info:
+ return self.fail_run_queue_item_input_info
+ query_string = """
+ query ProbeServerFailRunQueueItemInput {
+ FailRunQueueItemInputInfoType: __type(name:"FailRunQueueItemInput") {
+ inputFields{
+ name
+ }
+ }
+ }
+ """
+
+ query = gql(query_string)
+ res = self.gql(query)
+
+ self.fail_run_queue_item_input_info = [
+ field.get("name", "")
+ for field in res.get("FailRunQueueItemInputInfoType", {}).get(
+ "inputFields", [{}]
+ )
+ ]
+ return self.fail_run_queue_item_input_info
+
+ @normalize_exceptions
+ def fail_run_queue_item(
+ self,
+ run_queue_item_id: str,
+ message: str,
+ stage: str,
+ file_paths: Optional[List[str]] = None,
+ ) -> bool:
+ if not self.fail_run_queue_item_introspection():
+ return False
+ variable_values: Dict[str, Union[str, Optional[List[str]]]] = {
+ "runQueueItemId": run_queue_item_id,
+ }
+ if "message" in self.fail_run_queue_item_fields_introspection():
+ variable_values.update({"message": message, "stage": stage})
+ if file_paths is not None:
+ variable_values["filePaths"] = file_paths
+ mutation_string = """
+ mutation failRunQueueItem($runQueueItemId: ID!, $message: String!, $stage: String!, $filePaths: [String!]) {
+ failRunQueueItem(
+ input: {
+ runQueueItemId: $runQueueItemId
+ message: $message
+ stage: $stage
+ filePaths: $filePaths
+ }
+ ) {
+ success
+ }
+ }
+ """
+ else:
+ mutation_string = """
+ mutation failRunQueueItem($runQueueItemId: ID!) {
+ failRunQueueItem(
+ input: {
+ runQueueItemId: $runQueueItemId
+ }
+ ) {
+ success
+ }
+ }
+ """
+
+ mutation = gql(mutation_string)
+ response = self.gql(mutation, variable_values=variable_values)
+ result: bool = response["failRunQueueItem"]["success"]
+ return result
+
+ @normalize_exceptions
+ def update_run_queue_item_warning_introspection(self) -> bool:
+ _, _, mutations = self.server_info_introspection()
+ return "updateRunQueueItemWarning" in mutations
+
+ @normalize_exceptions
+ def update_run_queue_item_warning(
+ self,
+ run_queue_item_id: str,
+ message: str,
+ stage: str,
+ file_paths: Optional[List[str]] = None,
+ ) -> bool:
+ if not self.update_run_queue_item_warning_introspection():
+ return False
+ mutation = gql(
+ """
+ mutation updateRunQueueItemWarning($runQueueItemId: ID!, $message: String!, $stage: String!, $filePaths: [String!]) {
+ updateRunQueueItemWarning(
+ input: {
+ runQueueItemId: $runQueueItemId
+ message: $message
+ stage: $stage
+ filePaths: $filePaths
+ }
+ ) {
+ success
+ }
+ }
+ """
+ )
+ response = self.gql(
+ mutation,
+ variable_values={
+ "runQueueItemId": run_queue_item_id,
+ "message": message,
+ "stage": stage,
+ "filePaths": file_paths,
+ },
+ )
+ result: bool = response["updateRunQueueItemWarning"]["success"]
+ return result
+
+ @normalize_exceptions
+ def viewer(self) -> Dict[str, Any]:
+ query = gql(
+ """
+ query Viewer{
+ viewer {
+ id
+ entity
+ username
+ flags
+ teams {
+ edges {
+ node {
+ name
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+ res = self.gql(query)
+ return res.get("viewer") or {}
+
+ @normalize_exceptions
+ def max_cli_version(self) -> Optional[str]:
+ if self._max_cli_version is not None:
+ return self._max_cli_version
+
+ query_types, server_info_types, _ = self.server_info_introspection()
+ cli_version_exists = (
+ "serverInfo" in query_types and "cliVersionInfo" in server_info_types
+ )
+ if not cli_version_exists:
+ return None
+
+ _, server_info = self.viewer_server_info()
+ self._max_cli_version = server_info.get("cliVersionInfo", {}).get(
+ "max_cli_version"
+ )
+ return self._max_cli_version
+
+ @normalize_exceptions
+ def viewer_server_info(self) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+ local_query = """
+ latestLocalVersionInfo {
+ outOfDate
+ latestVersionString
+ versionOnThisInstanceString
+ }
+ """
+ cli_query = """
+ serverInfo {
+ cliVersionInfo
+ _LOCAL_QUERY_
+ }
+ """
+ query_template = """
+ query Viewer{
+ viewer {
+ id
+ entity
+ username
+ email
+ flags
+ teams {
+ edges {
+ node {
+ name
+ }
+ }
+ }
+ }
+ _CLI_QUERY_
+ }
+ """
+ query_types, server_info_types, _ = self.server_info_introspection()
+
+ cli_version_exists = (
+ "serverInfo" in query_types and "cliVersionInfo" in server_info_types
+ )
+
+ local_version_exists = (
+ "serverInfo" in query_types
+ and "latestLocalVersionInfo" in server_info_types
+ )
+
+ cli_query_string = "" if not cli_version_exists else cli_query
+ local_query_string = "" if not local_version_exists else local_query
+
+ query_string = query_template.replace("_CLI_QUERY_", cli_query_string).replace(
+ "_LOCAL_QUERY_", local_query_string
+ )
+ query = gql(query_string)
+ res = self.gql(query)
+ return res.get("viewer") or {}, res.get("serverInfo") or {}
+
+ @normalize_exceptions
+ def list_projects(self, entity: Optional[str] = None) -> List[Dict[str, str]]:
+ """List projects in W&B scoped by entity.
+
+ Arguments:
+ entity (str, optional): The entity to scope this project to.
+
+ Returns:
+ [{"id","name","description"}]
+ """
+ query = gql(
+ """
+ query EntityProjects($entity: String) {
+ models(first: 10, entityName: $entity) {
+ edges {
+ node {
+ id
+ name
+ description
+ }
+ }
+ }
+ }
+ """
+ )
+ project_list: List[Dict[str, str]] = self._flatten_edges(
+ self.gql(
+ query, variable_values={"entity": entity or self.settings("entity")}
+ )["models"]
+ )
+ return project_list
+
+ @normalize_exceptions
+ def project(self, project: str, entity: Optional[str] = None) -> "_Response":
+ """Retrieve project.
+
+ Arguments:
+ project (str): The project to get details for
+ entity (str, optional): The entity to scope this project to.
+
+ Returns:
+ [{"id","name","repo","dockerImage","description"}]
+ """
+ query = gql(
+ """
+ query ProjectDetails($entity: String, $project: String) {
+ model(name: $project, entityName: $entity) {
+ id
+ name
+ repo
+ dockerImage
+ description
+ }
+ }
+ """
+ )
+ response: _Response = self.gql(
+ query, variable_values={"entity": entity, "project": project}
+ )["model"]
+ return response
+
+ @normalize_exceptions
+ def sweep(
+ self,
+ sweep: str,
+ specs: str,
+ project: Optional[str] = None,
+ entity: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ """Retrieve sweep.
+
+ Arguments:
+ sweep (str): The sweep to get details for
+ specs (str): history specs
+ project (str, optional): The project to scope this sweep to.
+ entity (str, optional): The entity to scope this sweep to.
+
+ Returns:
+ [{"id","name","repo","dockerImage","description"}]
+ """
+ query = gql(
+ """
+ query SweepWithRuns($entity: String, $project: String, $sweep: String!, $specs: [JSONString!]!) {
+ project(name: $project, entityName: $entity) {
+ sweep(sweepName: $sweep) {
+ id
+ name
+ method
+ state
+ description
+ config
+ createdAt
+ heartbeatAt
+ updatedAt
+ earlyStopJobRunning
+ bestLoss
+ controller
+ scheduler
+ runs {
+ edges {
+ node {
+ name
+ state
+ config
+ exitcode
+ heartbeatAt
+ shouldStop
+ failed
+ stopped
+ running
+ summaryMetrics
+ sampledHistory(specs: $specs)
+ }
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+ entity = entity or self.settings("entity")
+ project = project or self.settings("project")
+ response = self.gql(
+ query,
+ variable_values={
+ "entity": entity,
+ "project": project,
+ "sweep": sweep,
+ "specs": specs,
+ },
+ )
+ if response["project"] is None or response["project"]["sweep"] is None:
+ raise ValueError(f"Sweep {entity}/{project}/{sweep} not found")
+ data: Dict[str, Any] = response["project"]["sweep"]
+ if data:
+ data["runs"] = self._flatten_edges(data["runs"])
+ return data
+
+ @normalize_exceptions
+ def list_runs(
+ self, project: str, entity: Optional[str] = None
+ ) -> List[Dict[str, str]]:
+ """List runs in W&B scoped by project.
+
+ Arguments:
+ project (str): The project to scope the runs to
+ entity (str, optional): The entity to scope this project to. Defaults to public models
+
+ Returns:
+ [{"id","name","description"}]
+ """
+ query = gql(
+ """
+ query ProjectRuns($model: String!, $entity: String) {
+ model(name: $model, entityName: $entity) {
+ buckets(first: 10) {
+ edges {
+ node {
+ id
+ name
+ displayName
+ description
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+ return self._flatten_edges(
+ self.gql(
+ query,
+ variable_values={
+ "entity": entity or self.settings("entity"),
+ "model": project or self.settings("project"),
+ },
+ )["model"]["buckets"]
+ )
+
+ @normalize_exceptions
+ def run_config(
+ self, project: str, run: Optional[str] = None, entity: Optional[str] = None
+ ) -> Tuple[str, Dict[str, Any], Optional[str], Dict[str, Any]]:
+ """Get the relevant configs for a run.
+
+ Arguments:
+ project (str): The project to download, (can include bucket)
+ run (str, optional): The run to download
+ entity (str, optional): The entity to scope this project to.
+ """
+ check_httpclient_logger_handler()
+
+ query = gql(
+ """
+ query RunConfigs(
+ $name: String!,
+ $entity: String,
+ $run: String!,
+ $pattern: String!,
+ $includeConfig: Boolean!,
+ ) {
+ model(name: $name, entityName: $entity) {
+ bucket(name: $run) {
+ config @include(if: $includeConfig)
+ commit @include(if: $includeConfig)
+ files(pattern: $pattern) {
+ pageInfo {
+ hasNextPage
+ endCursor
+ }
+ edges {
+ node {
+ name
+ directUrl
+ }
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+
+ variable_values = {
+ "name": project,
+ "run": run,
+ "entity": entity,
+ "includeConfig": True,
+ }
+
+ commit: str = ""
+ config: Dict[str, Any] = {}
+ patch: Optional[str] = None
+ metadata: Dict[str, Any] = {}
+
+ # If we use the `names` parameter on the `files` node, then the server
+ # will helpfully give us and 'open' file handle to the files that don't
+ # exist. This is so that we can upload data to it. However, in this
+ # case, we just want to download that file and not upload to it, so
+ # let's instead query for the files that do exist using `pattern`
+ # (with no wildcards).
+ #
+ # Unfortunately we're unable to construct a single pattern that matches
+ # our 2 files, we would need something like regex for that.
+ for filename in [DIFF_FNAME, METADATA_FNAME]:
+ variable_values["pattern"] = filename
+ response = self.gql(query, variable_values=variable_values)
+ if response["model"] is None:
+ raise CommError(f"Run {entity}/{project}/{run} not found")
+ run_obj: Dict = response["model"]["bucket"]
+ # we only need to fetch this config once
+ if variable_values["includeConfig"]:
+ commit = run_obj["commit"]
+ config = json.loads(run_obj["config"] or "{}")
+ variable_values["includeConfig"] = False
+ if run_obj["files"] is not None:
+ for file_edge in run_obj["files"]["edges"]:
+ name = file_edge["node"]["name"]
+ url = file_edge["node"]["directUrl"]
+ res = requests.get(url)
+ res.raise_for_status()
+ if name == METADATA_FNAME:
+ metadata = res.json()
+ elif name == DIFF_FNAME:
+ patch = res.text
+
+ return commit, config, patch, metadata
+
+ @normalize_exceptions
+ def run_resume_status(
+ self, entity: str, project_name: str, name: str
+ ) -> Optional[Dict[str, Any]]:
+ """Check if a run exists and get resume information.
+
+ Arguments:
+ entity (str): The entity to scope this project to.
+ project_name (str): The project to download, (can include bucket)
+ name (str): The run to download
+ """
+ query = gql(
+ """
+ query RunResumeStatus($project: String, $entity: String, $name: String!) {
+ model(name: $project, entityName: $entity) {
+ id
+ name
+ entity {
+ id
+ name
+ }
+
+ bucket(name: $name, missingOk: true) {
+ id
+ name
+ summaryMetrics
+ displayName
+ logLineCount
+ historyLineCount
+ eventsLineCount
+ historyTail
+ eventsTail
+ config
+ tags
+ }
+ }
+ }
+ """
+ )
+
+ response = self.gql(
+ query,
+ variable_values={
+ "entity": entity,
+ "project": project_name,
+ "name": name,
+ },
+ )
+
+ if "model" not in response or "bucket" not in (response["model"] or {}):
+ return None
+
+ project = response["model"]
+ self.set_setting("project", project_name)
+ if "entity" in project:
+ self.set_setting("entity", project["entity"]["name"])
+
+ result: Dict[str, Any] = project["bucket"]
+
+ return result
+
+ @normalize_exceptions
+ def check_stop_requested(
+ self, project_name: str, entity_name: str, run_id: str
+ ) -> bool:
+ query = gql(
+ """
+ query RunStoppedStatus($projectName: String, $entityName: String, $runId: String!) {
+ project(name:$projectName, entityName:$entityName) {
+ run(name:$runId) {
+ stopped
+ }
+ }
+ }
+ """
+ )
+
+ response = self.gql(
+ query,
+ variable_values={
+ "projectName": project_name,
+ "entityName": entity_name,
+ "runId": run_id,
+ },
+ )
+
+ project = response.get("project", None)
+ if not project:
+ return False
+ run = project.get("run", None)
+ if not run:
+ return False
+
+ status: bool = run["stopped"]
+ return status
+
+ def format_project(self, project: str) -> str:
+ return re.sub(r"\W+", "-", project.lower()).strip("-_")
+
+ @normalize_exceptions
+ def upsert_project(
+ self,
+ project: str,
+ id: Optional[str] = None,
+ description: Optional[str] = None,
+ entity: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ """Create a new project.
+
+ Arguments:
+ project (str): The project to create
+ description (str, optional): A description of this project
+ entity (str, optional): The entity to scope this project to.
+ """
+ mutation = gql(
+ """
+ mutation UpsertModel($name: String!, $id: String, $entity: String!, $description: String, $repo: String) {
+ upsertModel(input: { id: $id, name: $name, entityName: $entity, description: $description, repo: $repo }) {
+ model {
+ name
+ description
+ }
+ }
+ }
+ """
+ )
+ response = self.gql(
+ mutation,
+ variable_values={
+ "name": self.format_project(project),
+ "entity": entity or self.settings("entity"),
+ "description": description,
+ "id": id,
+ },
+ )
+ # TODO(jhr): Commenting out 'repo' field for cling, add back
+ # 'description': description, 'repo': self.git.remote_url, 'id': id})
+ result: Dict[str, Any] = response["upsertModel"]["model"]
+ return result
+
+ @normalize_exceptions
+ def entity_is_team(self, entity: str) -> bool:
+ query = gql(
+ """
+ query EntityIsTeam($entity: String!) {
+ entity(name: $entity) {
+ id
+ isTeam
+ }
+ }
+ """
+ )
+ variable_values = {
+ "entity": entity,
+ }
+
+ res = self.gql(query, variable_values)
+ if res.get("entity") is None:
+ raise Exception(
+ f"Error fetching entity {entity} "
+ "check that you have access to this entity"
+ )
+
+ is_team: bool = res["entity"]["isTeam"]
+ return is_team
+
+ @normalize_exceptions
+ def get_project_run_queues(self, entity: str, project: str) -> List[Dict[str, str]]:
+ query = gql(
+ """
+ query ProjectRunQueues($entity: String!, $projectName: String!){
+ project(entityName: $entity, name: $projectName) {
+ runQueues {
+ id
+ name
+ createdBy
+ access
+ }
+ }
+ }
+ """
+ )
+ variable_values = {
+ "projectName": project,
+ "entity": entity,
+ }
+
+ res = self.gql(query, variable_values)
+ if res.get("project") is None:
+ # circular dependency: (LAUNCH_DEFAULT_PROJECT = model-registry)
+ if project == "model-registry":
+ msg = (
+ f"Error fetching run queues for {entity} "
+ "check that you have access to this entity and project"
+ )
+ else:
+ msg = (
+ f"Error fetching run queues for {entity}/{project} "
+ "check that you have access to this entity and project"
+ )
+
+ raise Exception(msg)
+
+ project_run_queues: List[Dict[str, str]] = res["project"]["runQueues"]
+ return project_run_queues
+
+ @normalize_exceptions
+ def create_default_resource_config(
+ self,
+ entity: str,
+ resource: str,
+ config: str,
+ template_variables: Optional[Dict[str, Union[float, int, str]]],
+ ) -> Optional[Dict[str, Any]]:
+ if not self.create_default_resource_config_introspection():
+ raise Exception()
+ supports_template_vars, _ = self.push_to_run_queue_introspection()
+
+ mutation_params = """
+ $entityName: String!,
+ $resource: String!,
+ $config: JSONString!
+ """
+ mutation_inputs = """
+ entityName: $entityName,
+ resource: $resource,
+ config: $config
+ """
+
+ if supports_template_vars:
+ mutation_params += ", $templateVariables: JSONString"
+ mutation_inputs += ", templateVariables: $templateVariables"
+ else:
+ if template_variables is not None:
+ raise UnsupportedError(
+ "server does not support template variables, please update server instance to >=0.46"
+ )
+
+ variable_values = {
+ "entityName": entity,
+ "resource": resource,
+ "config": config,
+ }
+ if supports_template_vars:
+ if template_variables is not None:
+ variable_values["templateVariables"] = json.dumps(template_variables)
+ else:
+ variable_values["templateVariables"] = "{}"
+
+ query = gql(
+ f"""
+ mutation createDefaultResourceConfig(
+ {mutation_params}
+ ) {{
+ createDefaultResourceConfig(
+ input: {{
+ {mutation_inputs}
+ }}
+ ) {{
+ defaultResourceConfigID
+ success
+ }}
+ }}
+ """
+ )
+
+ result: Optional[Dict[str, Any]] = self.gql(query, variable_values)[
+ "createDefaultResourceConfig"
+ ]
+ return result
+
+ @normalize_exceptions
+ def create_run_queue(
+ self,
+ entity: str,
+ project: str,
+ queue_name: str,
+ access: str,
+ prioritization_mode: Optional[str] = None,
+ config_id: Optional[str] = None,
+ ) -> Optional[Dict[str, Any]]:
+ (
+ create_run_queue,
+ supports_drc,
+ supports_prioritization,
+ ) = self.create_run_queue_introspection()
+ if not create_run_queue:
+ raise UnsupportedError(
+ "run queue creation is not supported by this version of "
+ "wandb server. Consider updating to the latest version."
+ )
+ if not supports_drc and config_id is not None:
+ raise UnsupportedError(
+ "default resource configurations are not supported by this version "
+ "of wandb server. Consider updating to the latest version."
+ )
+ if not supports_prioritization and prioritization_mode is not None:
+ raise UnsupportedError(
+ "launch prioritization is not supported by this version of "
+ "wandb server. Consider updating to the latest version."
+ )
+
+ if supports_prioritization:
+ query = gql(
+ """
+ mutation createRunQueue(
+ $entity: String!,
+ $project: String!,
+ $queueName: String!,
+ $access: RunQueueAccessType!,
+ $prioritizationMode: RunQueuePrioritizationMode,
+ $defaultResourceConfigID: ID,
+ ) {
+ createRunQueue(
+ input: {
+ entityName: $entity,
+ projectName: $project,
+ queueName: $queueName,
+ access: $access,
+ prioritizationMode: $prioritizationMode
+ defaultResourceConfigID: $defaultResourceConfigID
+ }
+ ) {
+ success
+ queueID
+ }
+ }
+ """
+ )
+ variable_values = {
+ "entity": entity,
+ "project": project,
+ "queueName": queue_name,
+ "access": access,
+ "prioritizationMode": prioritization_mode,
+ "defaultResourceConfigID": config_id,
+ }
+ else:
+ query = gql(
+ """
+ mutation createRunQueue(
+ $entity: String!,
+ $project: String!,
+ $queueName: String!,
+ $access: RunQueueAccessType!,
+ $defaultResourceConfigID: ID,
+ ) {
+ createRunQueue(
+ input: {
+ entityName: $entity,
+ projectName: $project,
+ queueName: $queueName,
+ access: $access,
+ defaultResourceConfigID: $defaultResourceConfigID
+ }
+ ) {
+ success
+ queueID
+ }
+ }
+ """
+ )
+ variable_values = {
+ "entity": entity,
+ "project": project,
+ "queueName": queue_name,
+ "access": access,
+ "defaultResourceConfigID": config_id,
+ }
+
+ result: Optional[Dict[str, Any]] = self.gql(query, variable_values)[
+ "createRunQueue"
+ ]
+ return result
+
+ @normalize_exceptions
+ def push_to_run_queue_by_name(
+ self,
+ entity: str,
+ project: str,
+ queue_name: str,
+ run_spec: str,
+ template_variables: Optional[Dict[str, Union[int, float, str]]],
+ priority: Optional[int] = None,
+ ) -> Optional[Dict[str, Any]]:
+ self.push_to_run_queue_introspection()
+ """Queryless mutation, should be used before legacy fallback method."""
+
+ mutation_params = """
+ $entityName: String!,
+ $projectName: String!,
+ $queueName: String!,
+ $runSpec: JSONString!
+ """
+
+ mutation_input = """
+ entityName: $entityName,
+ projectName: $projectName,
+ queueName: $queueName,
+ runSpec: $runSpec
+ """
+
+ variables: Dict[str, Any] = {
+ "entityName": entity,
+ "projectName": project,
+ "queueName": queue_name,
+ "runSpec": run_spec,
+ }
+ if self.server_push_to_run_queue_supports_priority:
+ if priority is not None:
+ variables["priority"] = priority
+ mutation_params += ", $priority: Int"
+ mutation_input += ", priority: $priority"
+ else:
+ if priority is not None:
+ raise UnsupportedError(
+ "server does not support priority, please update server instance to >=0.46"
+ )
+
+ if self.server_supports_template_variables:
+ if template_variables is not None:
+ variables.update(
+ {"templateVariableValues": json.dumps(template_variables)}
+ )
+ mutation_params += ", $templateVariableValues: JSONString"
+ mutation_input += ", templateVariableValues: $templateVariableValues"
+ else:
+ if template_variables is not None:
+ raise UnsupportedError(
+ "server does not support template variables, please update server instance to >=0.46"
+ )
+
+ mutation = gql(
+ f"""
+ mutation pushToRunQueueByName(
+ {mutation_params}
+ ) {{
+ pushToRunQueueByName(
+ input: {{
+ {mutation_input}
+ }}
+ ) {{
+ runQueueItemId
+ runSpec
+ }}
+ }}
+ """
+ )
+
+ try:
+ result: Optional[Dict[str, Any]] = self.gql(
+ mutation, variables, check_retry_fn=util.no_retry_4xx
+ ).get("pushToRunQueueByName")
+ if not result:
+ return None
+
+ if result.get("runSpec"):
+ run_spec = json.loads(str(result["runSpec"]))
+ result["runSpec"] = run_spec
+
+ return result
+ except Exception as e:
+ if (
+ 'Cannot query field "runSpec" on type "PushToRunQueueByNamePayload"'
+ not in str(e)
+ ):
+ return None
+
+ mutation_no_runspec = gql(
+ """
+ mutation pushToRunQueueByName(
+ $entityName: String!,
+ $projectName: String!,
+ $queueName: String!,
+ $runSpec: JSONString!,
+ ) {
+ pushToRunQueueByName(
+ input: {
+ entityName: $entityName,
+ projectName: $projectName,
+ queueName: $queueName,
+ runSpec: $runSpec
+ }
+ ) {
+ runQueueItemId
+ }
+ }
+ """
+ )
+
+ try:
+ result = self.gql(
+ mutation_no_runspec, variables, check_retry_fn=util.no_retry_4xx
+ ).get("pushToRunQueueByName")
+ except Exception:
+ result = None
+
+ return result
+
+ @normalize_exceptions
+ def push_to_run_queue(
+ self,
+ queue_name: str,
+ launch_spec: Dict[str, str],
+ template_variables: Optional[dict],
+ project_queue: str,
+ priority: Optional[int] = None,
+ ) -> Optional[Dict[str, Any]]:
+ self.push_to_run_queue_introspection()
+ entity = launch_spec.get("queue_entity") or launch_spec["entity"]
+ run_spec = json.dumps(launch_spec)
+
+ push_result = self.push_to_run_queue_by_name(
+ entity, project_queue, queue_name, run_spec, template_variables, priority
+ )
+
+ if push_result:
+ return push_result
+
+ if priority is not None:
+ # Cannot proceed with legacy method if priority is set
+ return None
+
+ """ Legacy Method """
+ queues_found = self.get_project_run_queues(entity, project_queue)
+ matching_queues = [
+ q
+ for q in queues_found
+ if q["name"] == queue_name
+ # ensure user has access to queue
+ and (
+ # TODO: User created queues in the UI have USER access
+ q["access"] in ["PROJECT", "USER"]
+ or q["createdBy"] == self.default_entity
+ )
+ ]
+ if not matching_queues:
+ # in the case of a missing default queue. create it
+ if queue_name == "default":
+ wandb.termlog(
+ f"No default queue existing for entity: {entity} in project: {project_queue}, creating one."
+ )
+ res = self.create_run_queue(
+ launch_spec["entity"],
+ project_queue,
+ queue_name,
+ access="PROJECT",
+ )
+
+ if res is None or res.get("queueID") is None:
+ wandb.termerror(
+ f"Unable to create default queue for entity: {entity} on project: {project_queue}. Run could not be added to a queue"
+ )
+ return None
+ queue_id = res["queueID"]
+
+ else:
+ if project_queue == "model-registry":
+ _msg = f"Unable to push to run queue {queue_name}. Queue not found."
+ else:
+ _msg = f"Unable to push to run queue {project_queue}/{queue_name}. Queue not found."
+ wandb.termwarn(_msg)
+ return None
+ elif len(matching_queues) > 1:
+ wandb.termerror(
+ f"Unable to push to run queue {queue_name}. More than one queue found with this name."
+ )
+ return None
+ else:
+ queue_id = matching_queues[0]["id"]
+ spec_json = json.dumps(launch_spec)
+ variables = {"queueID": queue_id, "runSpec": spec_json}
+
+ mutation_params = """
+ $queueID: ID!,
+ $runSpec: JSONString!
+ """
+ mutation_input = """
+ queueID: $queueID,
+ runSpec: $runSpec
+ """
+ if self.server_supports_template_variables:
+ if template_variables is not None:
+ mutation_params += ", $templateVariableValues: JSONString"
+ mutation_input += ", templateVariableValues: $templateVariableValues"
+ variables.update(
+ {"templateVariableValues": json.dumps(template_variables)}
+ )
+ else:
+ if template_variables is not None:
+ raise UnsupportedError(
+ "server does not support template variables, please update server instance to >=0.46"
+ )
+
+ mutation = gql(
+ f"""
+ mutation pushToRunQueue(
+ {mutation_params}
+ ) {{
+ pushToRunQueue(
+ input: {{{mutation_input}}}
+ ) {{
+ runQueueItemId
+ }}
+ }}
+ """
+ )
+
+ response = self.gql(mutation, variable_values=variables)
+ if not response.get("pushToRunQueue"):
+ raise CommError(f"Error pushing run queue item to queue {queue_name}.")
+
+ result: Optional[Dict[str, Any]] = response["pushToRunQueue"]
+ return result
+
+ @normalize_exceptions
+ def pop_from_run_queue(
+ self,
+ queue_name: str,
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+ agent_id: Optional[str] = None,
+ ) -> Optional[Dict[str, Any]]:
+ mutation = gql(
+ """
+ mutation popFromRunQueue($entity: String!, $project: String!, $queueName: String!, $launchAgentId: ID) {
+ popFromRunQueue(input: {
+ entityName: $entity,
+ projectName: $project,
+ queueName: $queueName,
+ launchAgentId: $launchAgentId
+ }) {
+ runQueueItemId
+ runSpec
+ }
+ }
+ """
+ )
+ response = self.gql(
+ mutation,
+ variable_values={
+ "entity": entity,
+ "project": project,
+ "queueName": queue_name,
+ "launchAgentId": agent_id,
+ },
+ )
+ result: Optional[Dict[str, Any]] = response["popFromRunQueue"]
+ return result
+
+ @normalize_exceptions
+ def ack_run_queue_item(self, item_id: str, run_id: Optional[str] = None) -> bool:
+ mutation = gql(
+ """
+ mutation ackRunQueueItem($itemId: ID!, $runId: String!) {
+ ackRunQueueItem(input: { runQueueItemId: $itemId, runName: $runId }) {
+ success
+ }
+ }
+ """
+ )
+ response = self.gql(
+ mutation, variable_values={"itemId": item_id, "runId": str(run_id)}
+ )
+ if not response["ackRunQueueItem"]["success"]:
+ raise CommError(
+ "Error acking run queue item. Item may have already been acknowledged by another process"
+ )
+ result: bool = response["ackRunQueueItem"]["success"]
+ return result
+
+ @normalize_exceptions
+ def create_launch_agent_fields_introspection(self) -> List:
+ if self.create_launch_agent_input_info:
+ return self.create_launch_agent_input_info
+ query_string = """
+ query ProbeServerCreateLaunchAgentInput {
+ CreateLaunchAgentInputInfoType: __type(name:"CreateLaunchAgentInput") {
+ inputFields{
+ name
+ }
+ }
+ }
+ """
+
+ query = gql(query_string)
+ res = self.gql(query)
+
+ self.create_launch_agent_input_info = [
+ field.get("name", "")
+ for field in res.get("CreateLaunchAgentInputInfoType", {}).get(
+ "inputFields", [{}]
+ )
+ ]
+ return self.create_launch_agent_input_info
+
+ @normalize_exceptions
+ def create_launch_agent(
+ self,
+ entity: str,
+ project: str,
+ queues: List[str],
+ agent_config: Dict[str, Any],
+ version: str,
+ gorilla_agent_support: bool,
+ ) -> dict:
+ project_queues = self.get_project_run_queues(entity, project)
+ if not project_queues:
+ # create default queue if it doesn't already exist
+ default = self.create_run_queue(
+ entity, project, "default", access="PROJECT"
+ )
+ if default is None or default.get("queueID") is None:
+ raise CommError(
+ "Unable to create default queue for {}/{}. No queues for agent to poll".format(
+ entity, project
+ )
+ )
+ project_queues = [{"id": default["queueID"], "name": "default"}]
+ polling_queue_ids = [
+ q["id"] for q in project_queues if q["name"] in queues
+ ] # filter to poll specified queues
+ if len(polling_queue_ids) != len(queues):
+ raise CommError(
+ f"Could not start launch agent: Not all of requested queues ({', '.join(queues)}) found. "
+ f"Available queues for this project: {','.join([q['name'] for q in project_queues])}"
+ )
+
+ if not gorilla_agent_support:
+ # if gorilla doesn't support launch agents, return a client-generated id
+ return {
+ "success": True,
+ "launchAgentId": None,
+ }
+
+ hostname = socket.gethostname()
+
+ variable_values = {
+ "entity": entity,
+ "project": project,
+ "queues": polling_queue_ids,
+ "hostname": hostname,
+ }
+
+ mutation_params = """
+ $entity: String!,
+ $project: String!,
+ $queues: [ID!]!,
+ $hostname: String!
+ """
+
+ mutation_input = """
+ entityName: $entity,
+ projectName: $project,
+ runQueues: $queues,
+ hostname: $hostname
+ """
+
+ if "agentConfig" in self.create_launch_agent_fields_introspection():
+ variable_values["agentConfig"] = json.dumps(agent_config)
+ mutation_params += ", $agentConfig: JSONString"
+ mutation_input += ", agentConfig: $agentConfig"
+ if "version" in self.create_launch_agent_fields_introspection():
+ variable_values["version"] = version
+ mutation_params += ", $version: String"
+ mutation_input += ", version: $version"
+
+ mutation = gql(
+ f"""
+ mutation createLaunchAgent(
+ {mutation_params}
+ ) {{
+ createLaunchAgent(
+ input: {{
+ {mutation_input}
+ }}
+ ) {{
+ launchAgentId
+ }}
+ }}
+ """
+ )
+ result: dict = self.gql(mutation, variable_values)["createLaunchAgent"]
+ return result
+
+ @normalize_exceptions
+ def update_launch_agent_status(
+ self,
+ agent_id: str,
+ status: str,
+ gorilla_agent_support: bool,
+ ) -> dict:
+ if not gorilla_agent_support:
+ # if gorilla doesn't support launch agents, this is a no-op
+ return {
+ "success": True,
+ }
+
+ mutation = gql(
+ """
+ mutation updateLaunchAgent($agentId: ID!, $agentStatus: String){
+ updateLaunchAgent(
+ input: {
+ launchAgentId: $agentId
+ agentStatus: $agentStatus
+ }
+ ) {
+ success
+ }
+ }
+ """
+ )
+ variable_values = {
+ "agentId": agent_id,
+ "agentStatus": status,
+ }
+ result: dict = self.gql(mutation, variable_values)["updateLaunchAgent"]
+ return result
+
+ @normalize_exceptions
+ def get_launch_agent(self, agent_id: str, gorilla_agent_support: bool) -> dict:
+ if not gorilla_agent_support:
+ return {
+ "id": None,
+ "name": "",
+ "stopPolling": False,
+ }
+ query = gql(
+ """
+ query LaunchAgent($agentId: ID!) {
+ launchAgent(id: $agentId) {
+ id
+ name
+ runQueues
+ hostname
+ agentStatus
+ stopPolling
+ heartbeatAt
+ }
+ }
+ """
+ )
+ variable_values = {
+ "agentId": agent_id,
+ }
+ result: dict = self.gql(query, variable_values)["launchAgent"]
+ return result
+
+ @normalize_exceptions
+ def upsert_run(
+ self,
+ id: Optional[str] = None,
+ name: Optional[str] = None,
+ project: Optional[str] = None,
+ host: Optional[str] = None,
+ group: Optional[str] = None,
+ tags: Optional[List[str]] = None,
+ config: Optional[dict] = None,
+ description: Optional[str] = None,
+ entity: Optional[str] = None,
+ state: Optional[str] = None,
+ display_name: Optional[str] = None,
+ notes: Optional[str] = None,
+ repo: Optional[str] = None,
+ job_type: Optional[str] = None,
+ program_path: Optional[str] = None,
+ commit: Optional[str] = None,
+ sweep_name: Optional[str] = None,
+ summary_metrics: Optional[str] = None,
+ num_retries: Optional[int] = None,
+ ) -> Tuple[dict, bool, Optional[List]]:
+ """Update a run.
+
+ Arguments:
+ id (str, optional): The existing run to update
+ name (str, optional): The name of the run to create
+ group (str, optional): Name of the group this run is a part of
+ project (str, optional): The name of the project
+ host (str, optional): The name of the host
+ tags (list, optional): A list of tags to apply to the run
+ config (dict, optional): The latest config params
+ description (str, optional): A description of this project
+ entity (str, optional): The entity to scope this project to.
+ display_name (str, optional): The display name of this project
+ notes (str, optional): Notes about this run
+ repo (str, optional): Url of the program's repository.
+ state (str, optional): State of the program.
+ job_type (str, optional): Type of job, e.g 'train'.
+ program_path (str, optional): Path to the program.
+ commit (str, optional): The Git SHA to associate the run with
+ sweep_name (str, optional): The name of the sweep this run is a part of
+ summary_metrics (str, optional): The JSON summary metrics
+ num_retries (int, optional): Number of retries
+ """
+ query_string = """
+ mutation UpsertBucket(
+ $id: String,
+ $name: String,
+ $project: String,
+ $entity: String,
+ $groupName: String,
+ $description: String,
+ $displayName: String,
+ $notes: String,
+ $commit: String,
+ $config: JSONString,
+ $host: String,
+ $debug: Boolean,
+ $program: String,
+ $repo: String,
+ $jobType: String,
+ $state: String,
+ $sweep: String,
+ $tags: [String!],
+ $summaryMetrics: JSONString,
+ ) {
+ upsertBucket(input: {
+ id: $id,
+ name: $name,
+ groupName: $groupName,
+ modelName: $project,
+ entityName: $entity,
+ description: $description,
+ displayName: $displayName,
+ notes: $notes,
+ config: $config,
+ commit: $commit,
+ host: $host,
+ debug: $debug,
+ jobProgram: $program,
+ jobRepo: $repo,
+ jobType: $jobType,
+ state: $state,
+ sweep: $sweep,
+ tags: $tags,
+ summaryMetrics: $summaryMetrics,
+ }) {
+ bucket {
+ id
+ name
+ displayName
+ description
+ config
+ sweepName
+ project {
+ id
+ name
+ entity {
+ id
+ name
+ }
+ }
+ }
+ inserted
+ _Server_Settings_
+ }
+ }
+ """
+ self.server_settings_introspection()
+
+ server_settings_string = (
+ """
+ serverSettings {
+ serverMessages{
+ utfText
+ plainText
+ htmlText
+ messageType
+ messageLevel
+ }
+ }
+ """
+ if self._server_settings_type
+ else ""
+ )
+
+ query_string = query_string.replace("_Server_Settings_", server_settings_string)
+ mutation = gql(query_string)
+ config_str = json.dumps(config) if config else None
+ if not description or description.isspace():
+ description = None
+
+ kwargs = {}
+ if num_retries is not None:
+ kwargs["num_retries"] = num_retries
+
+ variable_values = {
+ "id": id,
+ "entity": entity or self.settings("entity"),
+ "name": name,
+ "project": project or util.auto_project_name(program_path),
+ "groupName": group,
+ "tags": tags,
+ "description": description,
+ "config": config_str,
+ "commit": commit,
+ "displayName": display_name,
+ "notes": notes,
+ "host": None if self.settings().get("anonymous") == "true" else host,
+ "debug": env.is_debug(env=self._environ),
+ "repo": repo,
+ "program": program_path,
+ "jobType": job_type,
+ "state": state,
+ "sweep": sweep_name,
+ "summaryMetrics": summary_metrics,
+ }
+
+ # retry conflict errors for 2 minutes, default to no_auth_retry
+ check_retry_fn = util.make_check_retry_fn(
+ check_fn=util.check_retry_conflict_or_gone,
+ check_timedelta=datetime.timedelta(minutes=2),
+ fallback_retry_fn=util.no_retry_auth,
+ )
+
+ response = self.gql(
+ mutation,
+ variable_values=variable_values,
+ check_retry_fn=check_retry_fn,
+ **kwargs,
+ )
+
+ run_obj: Dict[str, Dict[str, Dict[str, str]]] = response["upsertBucket"][
+ "bucket"
+ ]
+ project_obj: Dict[str, Dict[str, str]] = run_obj.get("project", {})
+ if project_obj:
+ self.set_setting("project", project_obj["name"])
+ entity_obj = project_obj.get("entity", {})
+ if entity_obj:
+ self.set_setting("entity", entity_obj["name"])
+
+ server_messages = None
+ if self._server_settings_type:
+ server_messages = (
+ response["upsertBucket"]
+ .get("serverSettings", {})
+ .get("serverMessages", [])
+ )
+ return (
+ response["upsertBucket"]["bucket"],
+ response["upsertBucket"]["inserted"],
+ server_messages,
+ )
+
+ @normalize_exceptions
+ def get_run_info(
+ self,
+ entity: str,
+ project: str,
+ name: str,
+ ) -> dict:
+ query = gql(
+ """
+ query RunInfo($project: String!, $entity: String!, $name: String!) {
+ project(name: $project, entityName: $entity) {
+ run(name: $name) {
+ runInfo {
+ program
+ args
+ os
+ python
+ colab
+ executable
+ codeSaved
+ cpuCount
+ gpuCount
+ gpu
+ git {
+ remote
+ commit
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+ variable_values = {"project": project, "entity": entity, "name": name}
+ res = self.gql(query, variable_values)
+ if res.get("project") is None:
+ raise CommError(
+ "Error fetching run info for {}/{}/{}. Check that this project exists and you have access to this entity and project".format(
+ entity, project, name
+ )
+ )
+ elif res["project"].get("run") is None:
+ raise CommError(
+ "Error fetching run info for {}/{}/{}. Check that this run id exists".format(
+ entity, project, name
+ )
+ )
+ run_info: dict = res["project"]["run"]["runInfo"]
+ return run_info
+
+ @normalize_exceptions
+ def get_run_state(self, entity: str, project: str, name: str) -> str:
+ query = gql(
+ """
+ query RunState(
+ $project: String!,
+ $entity: String!,
+ $name: String!) {
+ project(name: $project, entityName: $entity) {
+ run(name: $name) {
+ state
+ }
+ }
+ }
+ """
+ )
+ variable_values = {
+ "project": project,
+ "entity": entity,
+ "name": name,
+ }
+ res = self.gql(query, variable_values)
+ if res.get("project") is None or res["project"].get("run") is None:
+ raise CommError(f"Error fetching run state for {entity}/{project}/{name}.")
+ run_state: str = res["project"]["run"]["state"]
+ return run_state
+
+ @normalize_exceptions
+ def create_run_files_introspection(self) -> bool:
+ _, _, mutations = self.server_info_introspection()
+ return "createRunFiles" in mutations
+
+ @normalize_exceptions
+ def upload_urls(
+ self,
+ project: str,
+ files: Union[List[str], Dict[str, IO]],
+ run: Optional[str] = None,
+ entity: Optional[str] = None,
+ description: Optional[str] = None,
+ ) -> Tuple[str, List[str], Dict[str, Dict[str, Any]]]:
+ """Generate temporary resumable upload urls.
+
+ Arguments:
+ project (str): The project to download
+ files (list or dict): The filenames to upload
+ run (str, optional): The run to upload to
+ entity (str, optional): The entity to scope this project to.
+ description (str, optional): description
+
+ Returns:
+ (run_id, upload_headers, file_info)
+ run_id: id of run we uploaded files to
+ upload_headers: A list of headers to use when uploading files.
+ file_info: A dict of filenames and urls.
+ {
+ "run_id": "run_id",
+ "upload_headers": [""],
+ "file_info": [
+ { "weights.h5": { "uploadUrl": "https://weights.url" } },
+ { "model.json": { "uploadUrl": "https://model.json" } }
+ ]
+ }
+ """
+ run_name = run or self.current_run_id
+ assert run_name, "run must be specified"
+ entity = entity or self.settings("entity")
+ assert entity, "entity must be specified"
+
+ has_create_run_files_mutation = self.create_run_files_introspection()
+ if not has_create_run_files_mutation:
+ return self.legacy_upload_urls(project, files, run, entity, description)
+
+ query = gql(
+ """
+ mutation CreateRunFiles($entity: String!, $project: String!, $run: String!, $files: [String!]!) {
+ createRunFiles(input: {entityName: $entity, projectName: $project, runName: $run, files: $files}) {
+ runID
+ uploadHeaders
+ files {
+ name
+ uploadUrl
+ }
+ }
+ }
+ """
+ )
+
+ query_result = self.gql(
+ query,
+ variable_values={
+ "project": project,
+ "run": run_name,
+ "entity": entity,
+ "files": [file for file in files],
+ },
+ )
+
+ result = query_result["createRunFiles"]
+ run_id = result["runID"]
+ if not run_id:
+ raise CommError(
+ f"Error uploading files to {entity}/{project}/{run_name}. Check that this project exists and you have access to this entity and project"
+ )
+ file_name_urls = {file["name"]: file for file in result["files"]}
+ return run_id, result["uploadHeaders"], file_name_urls
+
+ def legacy_upload_urls(
+ self,
+ project: str,
+ files: Union[List[str], Dict[str, IO]],
+ run: Optional[str] = None,
+ entity: Optional[str] = None,
+ description: Optional[str] = None,
+ ) -> Tuple[str, List[str], Dict[str, Dict[str, Any]]]:
+ """Generate temporary resumable upload urls.
+
+ A new mutation createRunFiles was introduced after 0.15.4.
+ This function is used to support older versions.
+ """
+ query = gql(
+ """
+ query RunUploadUrls($name: String!, $files: [String]!, $entity: String, $run: String!, $description: String) {
+ model(name: $name, entityName: $entity) {
+ bucket(name: $run, desc: $description) {
+ id
+ files(names: $files) {
+ uploadHeaders
+ edges {
+ node {
+ name
+ url(upload: true)
+ updatedAt
+ }
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+ run_id = run or self.current_run_id
+ assert run_id, "run must be specified"
+ entity = entity or self.settings("entity")
+ query_result = self.gql(
+ query,
+ variable_values={
+ "name": project,
+ "run": run_id,
+ "entity": entity,
+ "files": [file for file in files],
+ "description": description,
+ },
+ )
+
+ run_obj = query_result["model"]["bucket"]
+ if run_obj:
+ for file_node in run_obj["files"]["edges"]:
+ file = file_node["node"]
+ # we previously used "url" field but now use "uploadUrl"
+ # replace the "url" field with "uploadUrl for downstream compatibility
+ if "url" in file and "uploadUrl" not in file:
+ file["uploadUrl"] = file.pop("url")
+
+ result = {
+ file["name"]: file for file in self._flatten_edges(run_obj["files"])
+ }
+ return run_obj["id"], run_obj["files"]["uploadHeaders"], result
+ else:
+ raise CommError(f"Run does not exist {entity}/{project}/{run_id}.")
+
+ @normalize_exceptions
+ def download_urls(
+ self,
+ project: str,
+ run: Optional[str] = None,
+ entity: Optional[str] = None,
+ ) -> Dict[str, Dict[str, str]]:
+ """Generate download urls.
+
+ Arguments:
+ project (str): The project to download
+ run (str): The run to upload to
+ entity (str, optional): The entity to scope this project to. Defaults to wandb models
+
+ Returns:
+ A dict of extensions and urls
+
+ {
+ 'weights.h5': { "url": "https://weights.url", "updatedAt": '2013-04-26T22:22:23.832Z', 'md5': 'mZFLkyvTelC5g8XnyQrpOw==' },
+ 'model.json': { "url": "https://model.url", "updatedAt": '2013-04-26T22:22:23.832Z', 'md5': 'mZFLkyvTelC5g8XnyQrpOw==' }
+ }
+ """
+ query = gql(
+ """
+ query RunDownloadUrls($name: String!, $entity: String, $run: String!) {
+ model(name: $name, entityName: $entity) {
+ bucket(name: $run) {
+ files {
+ edges {
+ node {
+ name
+ url
+ md5
+ updatedAt
+ }
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+ run = run or self.current_run_id
+ assert run, "run must be specified"
+ entity = entity or self.settings("entity")
+ query_result = self.gql(
+ query,
+ variable_values={
+ "name": project,
+ "run": run,
+ "entity": entity,
+ },
+ )
+ if query_result["model"] is None:
+ raise CommError(f"Run does not exist {entity}/{project}/{run}.")
+ files = self._flatten_edges(query_result["model"]["bucket"]["files"])
+ return {file["name"]: file for file in files if file}
+
+ @normalize_exceptions
+ def download_url(
+ self,
+ project: str,
+ file_name: str,
+ run: Optional[str] = None,
+ entity: Optional[str] = None,
+ ) -> Optional[Dict[str, str]]:
+ """Generate download urls.
+
+ Arguments:
+ project (str): The project to download
+ file_name (str): The name of the file to download
+ run (str): The run to upload to
+ entity (str, optional): The entity to scope this project to. Defaults to wandb models
+
+ Returns:
+ A dict of extensions and urls
+
+ { "url": "https://weights.url", "updatedAt": '2013-04-26T22:22:23.832Z', 'md5': 'mZFLkyvTelC5g8XnyQrpOw==' }
+
+ """
+ query = gql(
+ """
+ query RunDownloadUrl($name: String!, $fileName: String!, $entity: String, $run: String!) {
+ model(name: $name, entityName: $entity) {
+ bucket(name: $run) {
+ files(names: [$fileName]) {
+ edges {
+ node {
+ name
+ url
+ md5
+ updatedAt
+ }
+ }
+ }
+ }
+ }
+ }
+ """
+ )
+ run = run or self.current_run_id
+ assert run, "run must be specified"
+ query_result = self.gql(
+ query,
+ variable_values={
+ "name": project,
+ "run": run,
+ "fileName": file_name,
+ "entity": entity or self.settings("entity"),
+ },
+ )
+ if query_result["model"]:
+ files = self._flatten_edges(query_result["model"]["bucket"]["files"])
+ return files[0] if len(files) > 0 and files[0].get("updatedAt") else None
+ else:
+ return None
+
+ @normalize_exceptions
+ def download_file(self, url: str) -> Tuple[int, requests.Response]:
+ """Initiate a streaming download.
+
+ Arguments:
+ url (str): The url to download
+
+ Returns:
+ A tuple of the content length and the streaming response
+ """
+ check_httpclient_logger_handler()
+ auth = None
+ if _thread_local_api_settings.cookies is None:
+ auth = ("user", self.api_key or "")
+ response = requests.get(
+ url,
+ auth=auth,
+ cookies=_thread_local_api_settings.cookies or {},
+ headers=_thread_local_api_settings.headers or {},
+ stream=True,
+ )
+ response.raise_for_status()
+ return int(response.headers.get("content-length", 0)), response
+
+ @normalize_exceptions
+ def download_write_file(
+ self,
+ metadata: Dict[str, str],
+ out_dir: Optional[str] = None,
+ ) -> Tuple[str, Optional[requests.Response]]:
+ """Download a file from a run and write it to wandb/.
+
+ Arguments:
+ metadata (obj): The metadata object for the file to download. Comes from Api.download_urls().
+ out_dir (str, optional): The directory to write the file to. Defaults to wandb/
+
+ Returns:
+ A tuple of the file's local path and the streaming response. The streaming response is None if the file
+ already existed and was up-to-date.
+ """
+ filename = metadata["name"]
+ path = os.path.join(out_dir or self.settings("wandb_dir"), filename)
+ if self.file_current(filename, B64MD5(metadata["md5"])):
+ return path, None
+
+ size, response = self.download_file(metadata["url"])
+
+ with util.fsync_open(path, "wb") as file:
+ for data in response.iter_content(chunk_size=1024):
+ file.write(data)
+
+ return path, response
+
+ def upload_file_azure(
+ self, url: str, file: Any, extra_headers: Dict[str, str]
+ ) -> None:
+ """Upload a file to azure."""
+ from azure.core.exceptions import AzureError # type: ignore
+
+ # Configure the client without retries so our existing logic can handle them
+ client = self._azure_blob_module.BlobClient.from_blob_url(
+ url, retry_policy=self._azure_blob_module.LinearRetry(retry_total=0)
+ )
+ try:
+ if extra_headers.get("Content-MD5") is not None:
+ md5: Optional[bytes] = base64.b64decode(extra_headers["Content-MD5"])
+ else:
+ md5 = None
+ content_settings = self._azure_blob_module.ContentSettings(
+ content_md5=md5,
+ content_type=extra_headers.get("Content-Type"),
+ )
+ client.upload_blob(
+ file,
+ max_concurrency=4,
+ length=len(file),
+ overwrite=True,
+ content_settings=content_settings,
+ )
+ except AzureError as e:
+ if hasattr(e, "response"):
+ response = requests.models.Response()
+ response.status_code = e.response.status_code
+ response.headers = e.response.headers
+ raise requests.exceptions.RequestException(e.message, response=response)
+ else:
+ raise requests.exceptions.ConnectionError(e.message)
+
+ def upload_multipart_file_chunk(
+ self,
+ url: str,
+ upload_chunk: bytes,
+ extra_headers: Optional[Dict[str, str]] = None,
+ ) -> Optional[requests.Response]:
+ """Upload a file chunk to S3 with failure resumption.
+
+ Arguments:
+ url: The url to download
+ upload_chunk: The path to the file you want to upload
+ extra_headers: A dictionary of extra headers to send with the request
+
+ Returns:
+ The `requests` library response object
+ """
+ check_httpclient_logger_handler()
+ try:
+ if env.is_debug(env=self._environ):
+ logger.debug("upload_file: %s", url)
+ response = self._upload_file_session.put(
+ url, data=upload_chunk, headers=extra_headers
+ )
+ if env.is_debug(env=self._environ):
+ logger.debug("upload_file: %s complete", url)
+ response.raise_for_status()
+ except requests.exceptions.RequestException as e:
+ logger.error(f"upload_file exception {url}: {e}")
+ request_headers = e.request.headers if e.request is not None else ""
+ logger.error(f"upload_file request headers: {request_headers}")
+ response_content = e.response.content if e.response is not None else ""
+ logger.error(f"upload_file response body: {response_content}")
+ status_code = e.response.status_code if e.response is not None else 0
+ # S3 reports retryable request timeouts out-of-band
+ is_aws_retryable = status_code == 400 and "RequestTimeout" in str(
+ response_content
+ )
+ # Retry errors from cloud storage or local network issues
+ if (
+ status_code in (308, 408, 409, 429, 500, 502, 503, 504)
+ or isinstance(
+ e,
+ (requests.exceptions.Timeout, requests.exceptions.ConnectionError),
+ )
+ or is_aws_retryable
+ ):
+ _e = retry.TransientError(exc=e)
+ raise _e.with_traceback(sys.exc_info()[2])
+ else:
+ wandb._sentry.reraise(e)
+ return response
+
+ def upload_file(
+ self,
+ url: str,
+ file: IO[bytes],
+ callback: Optional["ProgressFn"] = None,
+ extra_headers: Optional[Dict[str, str]] = None,
+ ) -> Optional[requests.Response]:
+ """Upload a file to W&B with failure resumption.
+
+ Arguments:
+ url: The url to download
+ file: The path to the file you want to upload
+ callback: A callback which is passed the number of
+ bytes uploaded since the last time it was called, used to report progress
+ extra_headers: A dictionary of extra headers to send with the request
+
+ Returns:
+ The `requests` library response object
+ """
+ check_httpclient_logger_handler()
+ extra_headers = extra_headers.copy() if extra_headers else {}
+ response: Optional[requests.Response] = None
+ progress = Progress(file, callback=callback)
+ try:
+ if "x-ms-blob-type" in extra_headers and self._azure_blob_module:
+ self.upload_file_azure(url, progress, extra_headers)
+ else:
+ if "x-ms-blob-type" in extra_headers:
+ wandb.termwarn(
+ "Azure uploads over 256MB require the azure SDK, install with pip install wandb[azure]",
+ repeat=False,
+ )
+ if env.is_debug(env=self._environ):
+ logger.debug("upload_file: %s", url)
+ response = self._upload_file_session.put(
+ url, data=progress, headers=extra_headers
+ )
+ if env.is_debug(env=self._environ):
+ logger.debug("upload_file: %s complete", url)
+ response.raise_for_status()
+ except requests.exceptions.RequestException as e:
+ logger.error(f"upload_file exception {url}: {e}")
+ request_headers = e.request.headers if e.request is not None else ""
+ logger.error(f"upload_file request headers: {request_headers}")
+ response_content = e.response.content if e.response is not None else ""
+ logger.error(f"upload_file response body: {response_content}")
+ status_code = e.response.status_code if e.response is not None else 0
+ # S3 reports retryable request timeouts out-of-band
+ is_aws_retryable = (
+ "x-amz-meta-md5" in extra_headers
+ and status_code == 400
+ and "RequestTimeout" in str(response_content)
+ )
+ # We need to rewind the file for the next retry (the file passed in is `seek`'ed to 0)
+ progress.rewind()
+ # Retry errors from cloud storage or local network issues
+ if (
+ status_code in (308, 408, 409, 429, 500, 502, 503, 504)
+ or isinstance(
+ e,
+ (requests.exceptions.Timeout, requests.exceptions.ConnectionError),
+ )
+ or is_aws_retryable
+ ):
+ _e = retry.TransientError(exc=e)
+ raise _e.with_traceback(sys.exc_info()[2])
+ else:
+ wandb._sentry.reraise(e)
+
+ return response
+
+ async def upload_file_async(
+ self,
+ url: str,
+ file: IO[bytes],
+ callback: Optional["ProgressFn"] = None,
+ extra_headers: Optional[Dict[str, str]] = None,
+ ) -> None:
+ """An async not-quite-equivalent version of `upload_file`.
+
+ Differences from `upload_file`:
+ - This method doesn't implement Azure uploads. (The Azure SDK supports
+ async, but it's nontrivial to use it here.) If the upload looks like
+ it's destined for Azure, this method will delegate to the sync impl.
+ - Consequently, this method doesn't return the response object.
+ (Because it might fall back to the sync impl, it would sometimes
+ return a `requests.Response` and sometimes an `httpx.Response`.)
+ - This method doesn't wrap retryable errors in `TransientError`.
+ It leaves that determination to the caller.
+ """
+ check_httpclient_logger_handler()
+ must_delegate = False
+
+ if httpx is None:
+ wandb.termwarn( # type: ignore[unreachable]
+ "async file-uploads require `pip install wandb[async]`; falling back to sync implementation",
+ repeat=False,
+ )
+ must_delegate = True
+
+ if extra_headers is not None and "x-ms-blob-type" in extra_headers:
+ wandb.termwarn(
+ "async file-uploads don't support Azure; falling back to sync implementation",
+ repeat=False,
+ )
+ must_delegate = True
+
+ if must_delegate:
+ await asyncio.get_event_loop().run_in_executor(
+ None,
+ lambda: self.upload_file_retry(
+ url=url,
+ file=file,
+ callback=callback,
+ extra_headers=extra_headers,
+ ),
+ )
+ return
+
+ if self._async_httpx_client is None:
+ self._async_httpx_client = httpx.AsyncClient()
+
+ progress = AsyncProgress(Progress(file, callback=callback))
+
+ try:
+ response = await self._async_httpx_client.put(
+ url=url,
+ content=progress,
+ headers={
+ "Content-Length": str(len(progress)),
+ **(extra_headers if extra_headers is not None else {}),
+ },
+ )
+ response.raise_for_status()
+ except Exception as e:
+ progress.rewind()
+ logger.error(f"upload_file_async exception {url}: {e}")
+ if isinstance(e, httpx.RequestError):
+ logger.error(f"upload_file_async request headers: {e.request.headers}")
+ if isinstance(e, httpx.HTTPStatusError):
+ logger.error(f"upload_file_async response body: {e.response.content!r}")
+ raise
+
+ async def upload_file_retry_async(
+ self,
+ url: str,
+ file: IO[bytes],
+ callback: Optional["ProgressFn"] = None,
+ extra_headers: Optional[Dict[str, str]] = None,
+ num_retries: int = 100,
+ ) -> None:
+ backoff = retry.FilteredBackoff(
+ filter=check_httpx_exc_retriable,
+ wrapped=retry.ExponentialBackoff(
+ initial_sleep=datetime.timedelta(seconds=1),
+ max_sleep=datetime.timedelta(seconds=60),
+ max_retries=num_retries,
+ timeout_at=datetime.datetime.now() + datetime.timedelta(days=7),
+ ),
+ )
+
+ await retry.retry_async(
+ backoff=backoff,
+ fn=self.upload_file_async,
+ url=url,
+ file=file,
+ callback=callback,
+ extra_headers=extra_headers,
+ )
+
+ @normalize_exceptions
+ def register_agent(
+ self,
+ host: str,
+ sweep_id: Optional[str] = None,
+ project_name: Optional[str] = None,
+ entity: Optional[str] = None,
+ ) -> dict:
+ """Register a new agent.
+
+ Arguments:
+ host (str): hostname
+ sweep_id (str): sweep id
+ project_name: (str): model that contains sweep
+ entity: (str): entity that contains sweep
+ """
+ mutation = gql(
+ """
+ mutation CreateAgent(
+ $host: String!
+ $projectName: String,
+ $entityName: String,
+ $sweep: String!
+ ) {
+ createAgent(input: {
+ host: $host,
+ projectName: $projectName,
+ entityName: $entityName,
+ sweep: $sweep,
+ }) {
+ agent {
+ id
+ }
+ }
+ }
+ """
+ )
+ if entity is None:
+ entity = self.settings("entity")
+ if project_name is None:
+ project_name = self.settings("project")
+
+ response = self.gql(
+ mutation,
+ variable_values={
+ "host": host,
+ "entityName": entity,
+ "projectName": project_name,
+ "sweep": sweep_id,
+ },
+ check_retry_fn=util.no_retry_4xx,
+ )
+ result: dict = response["createAgent"]["agent"]
+ return result
+
+ def agent_heartbeat(
+ self, agent_id: str, metrics: dict, run_states: dict
+ ) -> List[Dict[str, Any]]:
+ """Notify server about agent state, receive commands.
+
+ Arguments:
+ agent_id (str): agent_id
+ metrics (dict): system metrics
+ run_states (dict): run_id: state mapping
+ Returns:
+ List of commands to execute.
+ """
+ mutation = gql(
+ """
+ mutation Heartbeat(
+ $id: ID!,
+ $metrics: JSONString,
+ $runState: JSONString
+ ) {
+ agentHeartbeat(input: {
+ id: $id,
+ metrics: $metrics,
+ runState: $runState
+ }) {
+ agent {
+ id
+ }
+ commands
+ }
+ }
+ """
+ )
+
+ if agent_id is None:
+ raise ValueError("Cannot call heartbeat with an unregistered agent.")
+
+ try:
+ response = self.gql(
+ mutation,
+ variable_values={
+ "id": agent_id,
+ "metrics": json.dumps(metrics),
+ "runState": json.dumps(run_states),
+ },
+ timeout=60,
+ )
+ except Exception as e:
+ # GQL raises exceptions with stringified python dictionaries :/
+ message = ast.literal_eval(e.args[0])["message"]
+ logger.error("Error communicating with W&B: %s", message)
+ return []
+ else:
+ result: List[Dict[str, Any]] = json.loads(
+ response["agentHeartbeat"]["commands"]
+ )
+ return result
+
+ @staticmethod
+ def _validate_config_and_fill_distribution(config: dict) -> dict:
+ # verify that parameters are well specified.
+ # TODO(dag): deprecate this in favor of jsonschema validation once
+ # apiVersion 2 is released and local controller is integrated with
+ # wandb/client.
+
+ # avoid modifying the original config dict in
+ # case it is reused outside the calling func
+ config = deepcopy(config)
+
+ # explicitly cast to dict in case config was passed as a sweepconfig
+ # sweepconfig does not serialize cleanly to yaml and breaks graphql,
+ # but it is a subclass of dict, so this conversion is clean
+ config = dict(config)
+
+ if "parameters" not in config:
+ # still shows an anaconda warning, but doesn't error
+ return config
+
+ for parameter_name in config["parameters"]:
+ parameter = config["parameters"][parameter_name]
+ if "min" in parameter and "max" in parameter:
+ if "distribution" not in parameter:
+ if isinstance(parameter["min"], int) and isinstance(
+ parameter["max"], int
+ ):
+ parameter["distribution"] = "int_uniform"
+ elif isinstance(parameter["min"], float) and isinstance(
+ parameter["max"], float
+ ):
+ parameter["distribution"] = "uniform"
+ else:
+ raise ValueError(
+ "Parameter %s is ambiguous, please specify bounds as both floats (for a float_"
+ "uniform distribution) or ints (for an int_uniform distribution)."
+ % parameter_name
+ )
+ return config
+
+ @normalize_exceptions
+ def upsert_sweep(
+ self,
+ config: dict,
+ controller: Optional[str] = None,
+ launch_scheduler: Optional[str] = None,
+ scheduler: Optional[str] = None,
+ obj_id: Optional[str] = None,
+ project: Optional[str] = None,
+ entity: Optional[str] = None,
+ state: Optional[str] = None,
+ ) -> Tuple[str, List[str]]:
+ """Upsert a sweep object.
+
+ Arguments:
+ config (dict): sweep config (will be converted to yaml)
+ controller (str): controller to use
+ launch_scheduler (str): launch scheduler to use
+ scheduler (str): scheduler to use
+ obj_id (str): object id
+ project (str): project to use
+ entity (str): entity to use
+ state (str): state
+ """
+ project_query = """
+ project {
+ id
+ name
+ entity {
+ id
+ name
+ }
+ }
+ """
+ mutation_str = """
+ mutation UpsertSweep(
+ $id: ID,
+ $config: String,
+ $description: String,
+ $entityName: String,
+ $projectName: String,
+ $controller: JSONString,
+ $scheduler: JSONString,
+ $state: String
+ ) {
+ upsertSweep(input: {
+ id: $id,
+ config: $config,
+ description: $description,
+ entityName: $entityName,
+ projectName: $projectName,
+ controller: $controller,
+ scheduler: $scheduler,
+ state: $state
+ }) {
+ sweep {
+ name
+ _PROJECT_QUERY_
+ }
+ configValidationWarnings
+ }
+ }
+ """
+ # TODO(jhr): we need protocol versioning to know schema is not supported
+ # for now we will just try both new and old query
+
+ # launchScheduler was introduced in core v0.14.0
+ mutation_4 = gql(
+ mutation_str.replace(
+ "$controller: JSONString,",
+ "$controller: JSONString,$launchScheduler: JSONString,",
+ )
+ .replace(
+ "controller: $controller,",
+ "controller: $controller,launchScheduler: $launchScheduler,",
+ )
+ .replace("_PROJECT_QUERY_", project_query)
+ )
+
+ # mutation 3 maps to backend that can support CLI version of at least 0.10.31
+ mutation_3 = gql(mutation_str.replace("_PROJECT_QUERY_", project_query))
+ mutation_2 = gql(
+ mutation_str.replace("_PROJECT_QUERY_", project_query).replace(
+ "configValidationWarnings", ""
+ )
+ )
+ mutation_1 = gql(
+ mutation_str.replace("_PROJECT_QUERY_", "").replace(
+ "configValidationWarnings", ""
+ )
+ )
+
+ # TODO(dag): replace this with a query for protocol versioning
+ mutations = [mutation_4, mutation_3, mutation_2, mutation_1]
+
+ config = self._validate_config_and_fill_distribution(config)
+
+ # Silly, but attr-dicts like EasyDicts don't serialize correctly to yaml.
+ # This sanitizes them with a round trip pass through json to get a regular dict.
+ config_str = yaml.dump(json.loads(json.dumps(config)))
+
+ err: Optional[Exception] = None
+ for mutation in mutations:
+ try:
+ variables = {
+ "id": obj_id,
+ "config": config_str,
+ "description": config.get("description"),
+ "entityName": entity or self.settings("entity"),
+ "projectName": project or self.settings("project"),
+ "controller": controller,
+ "launchScheduler": launch_scheduler,
+ "scheduler": scheduler,
+ }
+ if state:
+ variables["state"] = state
+
+ response = self.gql(
+ mutation,
+ variable_values=variables,
+ check_retry_fn=util.no_retry_4xx,
+ )
+ except UsageError as e:
+ raise e
+ except Exception as e:
+ # graphql schema exception is generic
+ err = e
+ continue
+ err = None
+ break
+ if err:
+ raise err
+
+ sweep: Dict[str, Dict[str, Dict]] = response["upsertSweep"]["sweep"]
+ project_obj: Dict[str, Dict] = sweep.get("project", {})
+ if project_obj:
+ self.set_setting("project", project_obj["name"])
+ entity_obj: dict = project_obj.get("entity", {})
+ if entity_obj:
+ self.set_setting("entity", entity_obj["name"])
+
+ warnings = response["upsertSweep"].get("configValidationWarnings", [])
+ return response["upsertSweep"]["sweep"]["name"], warnings
+
+ @normalize_exceptions
+ def create_anonymous_api_key(self) -> str:
+ """Create a new API key belonging to a new anonymous user."""
+ mutation = gql(
+ """
+ mutation CreateAnonymousApiKey {
+ createAnonymousEntity(input: {}) {
+ apiKey {
+ name
+ }
+ }
+ }
+ """
+ )
+
+ response = self.gql(mutation, variable_values={})
+ key: str = str(response["createAnonymousEntity"]["apiKey"]["name"])
+ return key
+
+ @staticmethod
+ def file_current(fname: str, md5: B64MD5) -> bool:
+ """Checksum a file and compare the md5 with the known md5."""
+ return os.path.isfile(fname) and md5_file_b64(fname) == md5
+
+ @normalize_exceptions
+ def pull(
+ self, project: str, run: Optional[str] = None, entity: Optional[str] = None
+ ) -> "List[requests.Response]":
+ """Download files from W&B.
+
+ Arguments:
+ project (str): The project to download
+ run (str, optional): The run to upload to
+ entity (str, optional): The entity to scope this project to. Defaults to wandb models
+
+ Returns:
+ The `requests` library response object
+ """
+ project, run = self.parse_slug(project, run=run)
+ urls = self.download_urls(project, run, entity)
+ responses = []
+ for filename in urls:
+ _, response = self.download_write_file(urls[filename])
+ if response:
+ responses.append(response)
+
+ return responses
+
+ def get_project(self) -> str:
+ project: str = self.default_settings.get("project") or self.settings("project")
+ return project
+
+ @normalize_exceptions
+ def push(
+ self,
+ files: Union[List[str], Dict[str, IO]],
+ run: Optional[str] = None,
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+ description: Optional[str] = None,
+ force: bool = True,
+ progress: Union[TextIO, bool] = False,
+ ) -> "List[Optional[requests.Response]]":
+ """Uploads multiple files to W&B.
+
+ Arguments:
+ files (list or dict): The filenames to upload, when dict the values are open files
+ run (str, optional): The run to upload to
+ entity (str, optional): The entity to scope this project to. Defaults to wandb models
+ project (str, optional): The name of the project to upload to. Defaults to the one in settings.
+ description (str, optional): The description of the changes
+ force (bool, optional): Whether to prevent push if git has uncommitted changes
+ progress (callable, or stream): If callable, will be called with (chunk_bytes,
+ total_bytes) as argument else if True, renders a progress bar to stream.
+
+ Returns:
+ A list of `requests.Response` objects
+ """
+ if project is None:
+ project = self.get_project()
+ if project is None:
+ raise CommError("No project configured.")
+ if run is None:
+ run = self.current_run_id
+
+ # TODO(adrian): we use a retriable version of self.upload_file() so
+ # will never retry self.upload_urls() here. Instead, maybe we should
+ # make push itself retriable.
+ _, upload_headers, result = self.upload_urls(
+ project,
+ files,
+ run,
+ entity,
+ )
+ extra_headers = {}
+ for upload_header in upload_headers:
+ key, val = upload_header.split(":", 1)
+ extra_headers[key] = val
+ responses = []
+ for file_name, file_info in result.items():
+ file_url = file_info["uploadUrl"]
+
+ # If the upload URL is relative, fill it in with the base URL,
+ # since it's a proxied file store like the on-prem VM.
+ if file_url.startswith("/"):
+ file_url = f"{self.api_url}{file_url}"
+
+ try:
+ # To handle Windows paths
+ # TODO: this doesn't handle absolute paths...
+ normal_name = os.path.join(*file_name.split("/"))
+ open_file = (
+ files[file_name]
+ if isinstance(files, dict)
+ else open(normal_name, "rb")
+ )
+ except OSError:
+ print(f"{file_name} does not exist")
+ continue
+ if progress is False:
+ responses.append(
+ self.upload_file_retry(
+ file_info["uploadUrl"], open_file, extra_headers=extra_headers
+ )
+ )
+ else:
+ if callable(progress):
+ responses.append( # type: ignore
+ self.upload_file_retry(
+ file_url, open_file, progress, extra_headers=extra_headers
+ )
+ )
+ else:
+ length = os.fstat(open_file.fileno()).st_size
+ with click.progressbar(
+ file=progress, # type: ignore
+ length=length,
+ label=f"Uploading file: {file_name}",
+ fill_char=click.style("&", fg="green"),
+ ) as bar:
+ responses.append(
+ self.upload_file_retry(
+ file_url,
+ open_file,
+ lambda bites, _: bar.update(bites),
+ extra_headers=extra_headers,
+ )
+ )
+ open_file.close()
+ return responses
+
+ def link_artifact(
+ self,
+ client_id: str,
+ server_id: str,
+ portfolio_name: str,
+ entity: str,
+ project: str,
+ aliases: Sequence[str],
+ ) -> Dict[str, Any]:
+ template = """
+ mutation LinkArtifact(
+ $artifactPortfolioName: String!,
+ $entityName: String!,
+ $projectName: String!,
+ $aliases: [ArtifactAliasInput!],
+ ID_TYPE
+ ) {
+ linkArtifact(input: {
+ artifactPortfolioName: $artifactPortfolioName,
+ entityName: $entityName,
+ projectName: $projectName,
+ aliases: $aliases,
+ ID_VALUE
+ }) {
+ versionIndex
+ }
+ }
+ """
+
+ def replace(a: str, b: str) -> None:
+ nonlocal template
+ template = template.replace(a, b)
+
+ if server_id:
+ replace("ID_TYPE", "$artifactID: ID")
+ replace("ID_VALUE", "artifactID: $artifactID")
+ elif client_id:
+ replace("ID_TYPE", "$clientID: ID")
+ replace("ID_VALUE", "clientID: $clientID")
+
+ variable_values = {
+ "clientID": client_id,
+ "artifactID": server_id,
+ "artifactPortfolioName": portfolio_name,
+ "entityName": entity,
+ "projectName": project,
+ "aliases": [
+ {"alias": alias, "artifactCollectionName": portfolio_name}
+ for alias in aliases
+ ],
+ }
+
+ mutation = gql(template)
+ response = self.gql(mutation, variable_values=variable_values)
+ link_artifact: Dict[str, Any] = response["linkArtifact"]
+ return link_artifact
+
+ def use_artifact(
+ self,
+ artifact_id: str,
+ entity_name: Optional[str] = None,
+ project_name: Optional[str] = None,
+ run_name: Optional[str] = None,
+ use_as: Optional[str] = None,
+ ) -> Optional[Dict[str, Any]]:
+ query_template = """
+ mutation UseArtifact(
+ $entityName: String!,
+ $projectName: String!,
+ $runName: String!,
+ $artifactID: ID!,
+ _USED_AS_TYPE_
+ ) {
+ useArtifact(input: {
+ entityName: $entityName,
+ projectName: $projectName,
+ runName: $runName,
+ artifactID: $artifactID,
+ _USED_AS_VALUE_
+ }) {
+ artifact {
+ id
+ digest
+ description
+ state
+ createdAt
+ metadata
+ }
+ }
+ }
+ """
+
+ artifact_types = self.server_use_artifact_input_introspection()
+ if "usedAs" in artifact_types:
+ query_template = query_template.replace(
+ "_USED_AS_TYPE_", "$usedAs: String"
+ ).replace("_USED_AS_VALUE_", "usedAs: $usedAs")
+ else:
+ query_template = query_template.replace("_USED_AS_TYPE_", "").replace(
+ "_USED_AS_VALUE_", ""
+ )
+
+ query = gql(query_template)
+
+ entity_name = entity_name or self.settings("entity")
+ project_name = project_name or self.settings("project")
+ run_name = run_name or self.current_run_id
+
+ response = self.gql(
+ query,
+ variable_values={
+ "entityName": entity_name,
+ "projectName": project_name,
+ "runName": run_name,
+ "artifactID": artifact_id,
+ "usedAs": use_as,
+ },
+ )
+
+ if response["useArtifact"]["artifact"]:
+ artifact: Dict[str, Any] = response["useArtifact"]["artifact"]
+ return artifact
+ return None
+
+ def create_artifact_type(
+ self,
+ artifact_type_name: str,
+ entity_name: Optional[str] = None,
+ project_name: Optional[str] = None,
+ description: Optional[str] = None,
+ ) -> Optional[str]:
+ mutation = gql(
+ """
+ mutation CreateArtifactType(
+ $entityName: String!,
+ $projectName: String!,
+ $artifactTypeName: String!,
+ $description: String
+ ) {
+ createArtifactType(input: {
+ entityName: $entityName,
+ projectName: $projectName,
+ name: $artifactTypeName,
+ description: $description
+ }) {
+ artifactType {
+ id
+ }
+ }
+ }
+ """
+ )
+ entity_name = entity_name or self.settings("entity")
+ project_name = project_name or self.settings("project")
+ response = self.gql(
+ mutation,
+ variable_values={
+ "entityName": entity_name,
+ "projectName": project_name,
+ "artifactTypeName": artifact_type_name,
+ "description": description,
+ },
+ )
+ _id: Optional[str] = response["createArtifactType"]["artifactType"]["id"]
+ return _id
+
+ def server_artifact_introspection(self) -> List:
+ query_string = """
+ query ProbeServerArtifact {
+ ArtifactInfoType: __type(name:"Artifact") {
+ fields {
+ name
+ }
+ }
+ }
+ """
+
+ if self.server_artifact_fields_info is None:
+ query = gql(query_string)
+ res = self.gql(query)
+ input_fields = res.get("ArtifactInfoType", {}).get("fields", [{}])
+ self.server_artifact_fields_info = [
+ field["name"] for field in input_fields if "name" in field
+ ]
+
+ return self.server_artifact_fields_info
+
+ def server_create_artifact_introspection(self) -> List:
+ query_string = """
+ query ProbeServerCreateArtifactInput {
+ CreateArtifactInputInfoType: __type(name:"CreateArtifactInput") {
+ inputFields{
+ name
+ }
+ }
+ }
+ """
+
+ if self.server_create_artifact_input_info is None:
+ query = gql(query_string)
+ res = self.gql(query)
+ input_fields = res.get("CreateArtifactInputInfoType", {}).get(
+ "inputFields", [{}]
+ )
+ self.server_create_artifact_input_info = [
+ field["name"] for field in input_fields if "name" in field
+ ]
+
+ return self.server_create_artifact_input_info
+
+ def _get_create_artifact_mutation(
+ self,
+ fields: List,
+ history_step: Optional[int],
+ distributed_id: Optional[str],
+ ) -> str:
+ types = ""
+ values = ""
+
+ if "historyStep" in fields and history_step not in [0, None]:
+ types += "$historyStep: Int64!,"
+ values += "historyStep: $historyStep,"
+
+ if distributed_id:
+ types += "$distributedID: String,"
+ values += "distributedID: $distributedID,"
+
+ if "clientID" in fields:
+ types += "$clientID: ID,"
+ values += "clientID: $clientID,"
+
+ if "sequenceClientID" in fields:
+ types += "$sequenceClientID: ID,"
+ values += "sequenceClientID: $sequenceClientID,"
+
+ if "enableDigestDeduplication" in fields:
+ values += "enableDigestDeduplication: true,"
+
+ if "ttlDurationSeconds" in fields:
+ types += "$ttlDurationSeconds: Int64,"
+ values += "ttlDurationSeconds: $ttlDurationSeconds,"
+
+ query_template = """
+ mutation CreateArtifact(
+ $artifactTypeName: String!,
+ $artifactCollectionNames: [String!],
+ $entityName: String!,
+ $projectName: String!,
+ $runName: String,
+ $description: String,
+ $digest: String!,
+ $aliases: [ArtifactAliasInput!],
+ $metadata: JSONString,
+ _CREATE_ARTIFACT_ADDITIONAL_TYPE_
+ ) {
+ createArtifact(input: {
+ artifactTypeName: $artifactTypeName,
+ artifactCollectionNames: $artifactCollectionNames,
+ entityName: $entityName,
+ projectName: $projectName,
+ runName: $runName,
+ description: $description,
+ digest: $digest,
+ digestAlgorithm: MANIFEST_MD5,
+ aliases: $aliases,
+ metadata: $metadata,
+ _CREATE_ARTIFACT_ADDITIONAL_VALUE_
+ }) {
+ artifact {
+ id
+ state
+ artifactSequence {
+ id
+ latestArtifact {
+ id
+ versionIndex
+ }
+ }
+ }
+ }
+ }
+ """
+
+ return query_template.replace(
+ "_CREATE_ARTIFACT_ADDITIONAL_TYPE_", types
+ ).replace("_CREATE_ARTIFACT_ADDITIONAL_VALUE_", values)
+
+ def create_artifact(
+ self,
+ artifact_type_name: str,
+ artifact_collection_name: str,
+ digest: str,
+ client_id: Optional[str] = None,
+ sequence_client_id: Optional[str] = None,
+ entity_name: Optional[str] = None,
+ project_name: Optional[str] = None,
+ run_name: Optional[str] = None,
+ description: Optional[str] = None,
+ metadata: Optional[Dict] = None,
+ ttl_duration_seconds: Optional[int] = None,
+ aliases: Optional[List[Dict[str, str]]] = None,
+ distributed_id: Optional[str] = None,
+ is_user_created: Optional[bool] = False,
+ history_step: Optional[int] = None,
+ ) -> Tuple[Dict, Dict]:
+ fields = self.server_create_artifact_introspection()
+ artifact_fields = self.server_artifact_introspection()
+ if "ttlIsInherited" not in artifact_fields and ttl_duration_seconds:
+ wandb.termwarn(
+ "Server not compatible with setting Artifact TTLs, please upgrade the server to use Artifact TTL"
+ )
+ # ttlDurationSeconds is only usable if ttlIsInherited is also present
+ ttl_duration_seconds = None
+ query_template = self._get_create_artifact_mutation(
+ fields, history_step, distributed_id
+ )
+
+ entity_name = entity_name or self.settings("entity")
+ project_name = project_name or self.settings("project")
+ if not is_user_created:
+ run_name = run_name or self.current_run_id
+ if aliases is None:
+ aliases = []
+
+ mutation = gql(query_template)
+ response = self.gql(
+ mutation,
+ variable_values={
+ "entityName": entity_name,
+ "projectName": project_name,
+ "runName": run_name,
+ "artifactTypeName": artifact_type_name,
+ "artifactCollectionNames": [artifact_collection_name],
+ "clientID": client_id,
+ "sequenceClientID": sequence_client_id,
+ "digest": digest,
+ "description": description,
+ "aliases": [alias for alias in aliases],
+ "metadata": json.dumps(util.make_safe_for_json(metadata))
+ if metadata
+ else None,
+ "ttlDurationSeconds": ttl_duration_seconds,
+ "distributedID": distributed_id,
+ "historyStep": history_step,
+ },
+ )
+ av = response["createArtifact"]["artifact"]
+ latest = response["createArtifact"]["artifact"]["artifactSequence"].get(
+ "latestArtifact"
+ )
+ return av, latest
+
+ def commit_artifact(self, artifact_id: str) -> "_Response":
+ mutation = gql(
+ """
+ mutation CommitArtifact(
+ $artifactID: ID!,
+ ) {
+ commitArtifact(input: {
+ artifactID: $artifactID,
+ }) {
+ artifact {
+ id
+ digest
+ }
+ }
+ }
+ """
+ )
+
+ response: _Response = self.gql(
+ mutation,
+ variable_values={"artifactID": artifact_id},
+ timeout=60,
+ )
+ return response
+
+ def complete_multipart_upload_artifact(
+ self,
+ artifact_id: str,
+ storage_path: str,
+ completed_parts: List[Dict[str, Any]],
+ upload_id: str,
+ complete_multipart_action: str = "Complete",
+ ) -> Optional[str]:
+ mutation = gql(
+ """
+ mutation CompleteMultipartUploadArtifact(
+ $completeMultipartAction: CompleteMultipartAction!,
+ $completedParts: [UploadPartsInput!]!,
+ $artifactID: ID!
+ $storagePath: String!
+ $uploadID: String!
+ ) {
+ completeMultipartUploadArtifact(
+ input: {
+ completeMultipartAction: $completeMultipartAction,
+ completedParts: $completedParts,
+ artifactID: $artifactID,
+ storagePath: $storagePath
+ uploadID: $uploadID
+ }
+ ) {
+ digest
+ }
+ }
+ """
+ )
+ response = self.gql(
+ mutation,
+ variable_values={
+ "completeMultipartAction": complete_multipart_action,
+ "artifactID": artifact_id,
+ "storagePath": storage_path,
+ "completedParts": completed_parts,
+ "uploadID": upload_id,
+ },
+ )
+ digest: Optional[str] = response["completeMultipartUploadArtifact"]["digest"]
+ return digest
+
+ def create_artifact_manifest(
+ self,
+ name: str,
+ digest: str,
+ artifact_id: Optional[str],
+ base_artifact_id: Optional[str] = None,
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+ run: Optional[str] = None,
+ include_upload: bool = True,
+ type: str = "FULL",
+ ) -> Tuple[str, Dict[str, Any]]:
+ mutation = gql(
+ """
+ mutation CreateArtifactManifest(
+ $name: String!,
+ $digest: String!,
+ $artifactID: ID!,
+ $baseArtifactID: ID,
+ $entityName: String!,
+ $projectName: String!,
+ $runName: String!,
+ $includeUpload: Boolean!,
+ {}
+ ) {{
+ createArtifactManifest(input: {{
+ name: $name,
+ digest: $digest,
+ artifactID: $artifactID,
+ baseArtifactID: $baseArtifactID,
+ entityName: $entityName,
+ projectName: $projectName,
+ runName: $runName,
+ {}
+ }}) {{
+ artifactManifest {{
+ id
+ file {{
+ id
+ name
+ displayName
+ uploadUrl @include(if: $includeUpload)
+ uploadHeaders @include(if: $includeUpload)
+ }}
+ }}
+ }}
+ }}
+ """.format(
+ "$type: ArtifactManifestType = FULL" if type != "FULL" else "",
+ "type: $type" if type != "FULL" else "",
+ )
+ )
+
+ entity_name = entity or self.settings("entity")
+ project_name = project or self.settings("project")
+ run_name = run or self.current_run_id
+
+ response = self.gql(
+ mutation,
+ variable_values={
+ "name": name,
+ "digest": digest,
+ "artifactID": artifact_id,
+ "baseArtifactID": base_artifact_id,
+ "entityName": entity_name,
+ "projectName": project_name,
+ "runName": run_name,
+ "includeUpload": include_upload,
+ "type": type,
+ },
+ )
+ return (
+ response["createArtifactManifest"]["artifactManifest"]["id"],
+ response["createArtifactManifest"]["artifactManifest"]["file"],
+ )
+
+ def update_artifact_manifest(
+ self,
+ artifact_manifest_id: str,
+ base_artifact_id: Optional[str] = None,
+ digest: Optional[str] = None,
+ include_upload: Optional[bool] = True,
+ ) -> Tuple[str, Dict[str, Any]]:
+ mutation = gql(
+ """
+ mutation UpdateArtifactManifest(
+ $artifactManifestID: ID!,
+ $digest: String,
+ $baseArtifactID: ID,
+ $includeUpload: Boolean!,
+ ) {
+ updateArtifactManifest(input: {
+ artifactManifestID: $artifactManifestID,
+ digest: $digest,
+ baseArtifactID: $baseArtifactID,
+ }) {
+ artifactManifest {
+ id
+ file {
+ id
+ name
+ displayName
+ uploadUrl @include(if: $includeUpload)
+ uploadHeaders @include(if: $includeUpload)
+ }
+ }
+ }
+ }
+ """
+ )
+
+ response = self.gql(
+ mutation,
+ variable_values={
+ "artifactManifestID": artifact_manifest_id,
+ "digest": digest,
+ "baseArtifactID": base_artifact_id,
+ "includeUpload": include_upload,
+ },
+ )
+
+ return (
+ response["updateArtifactManifest"]["artifactManifest"]["id"],
+ response["updateArtifactManifest"]["artifactManifest"]["file"],
+ )
+
+ def _resolve_client_id(
+ self,
+ client_id: str,
+ ) -> Optional[str]:
+ if client_id in self._client_id_mapping:
+ return self._client_id_mapping[client_id]
+
+ query = gql(
+ """
+ query ClientIDMapping($clientID: ID!) {
+ clientIDMapping(clientID: $clientID) {
+ serverID
+ }
+ }
+ """
+ )
+ response = self.gql(
+ query,
+ variable_values={
+ "clientID": client_id,
+ },
+ )
+ server_id = None
+ if response is not None:
+ client_id_mapping = response.get("clientIDMapping")
+ if client_id_mapping is not None:
+ server_id = client_id_mapping.get("serverID")
+ if server_id is not None:
+ self._client_id_mapping[client_id] = server_id
+ return server_id
+
+ def server_create_artifact_file_spec_input_introspection(self) -> List:
+ query_string = """
+ query ProbeServerCreateArtifactFileSpecInput {
+ CreateArtifactFileSpecInputInfoType: __type(name:"CreateArtifactFileSpecInput") {
+ inputFields{
+ name
+ }
+ }
+ }
+ """
+
+ query = gql(query_string)
+ res = self.gql(query)
+ create_artifact_file_spec_input_info = [
+ field.get("name", "")
+ for field in res.get("CreateArtifactFileSpecInputInfoType", {}).get(
+ "inputFields", [{}]
+ )
+ ]
+ return create_artifact_file_spec_input_info
+
+ @normalize_exceptions
+ def create_artifact_files(
+ self, artifact_files: Iterable["CreateArtifactFileSpecInput"]
+ ) -> Mapping[str, "CreateArtifactFilesResponseFile"]:
+ query_template = """
+ mutation CreateArtifactFiles(
+ $storageLayout: ArtifactStorageLayout!
+ $artifactFiles: [CreateArtifactFileSpecInput!]!
+ ) {
+ createArtifactFiles(input: {
+ artifactFiles: $artifactFiles,
+ storageLayout: $storageLayout,
+ }) {
+ files {
+ edges {
+ node {
+ id
+ name
+ displayName
+ uploadUrl
+ uploadHeaders
+ _MULTIPART_UPLOAD_FIELDS_
+ artifact {
+ id
+ }
+ }
+ }
+ }
+ }
+ }
+ """
+ multipart_upload_url_query = """
+ storagePath
+ uploadMultipartUrls {
+ uploadID
+ uploadUrlParts {
+ partNumber
+ uploadUrl
+ }
+ }
+ """
+
+ # TODO: we should use constants here from interface/artifacts.py
+ # but probably don't want the dependency. We're going to remove
+ # this setting in a future release, so I'm just hard-coding the strings.
+ storage_layout = "V2"
+ if env.get_use_v1_artifacts():
+ storage_layout = "V1"
+
+ create_artifact_file_spec_input_fields = (
+ self.server_create_artifact_file_spec_input_introspection()
+ )
+ if "uploadPartsInput" in create_artifact_file_spec_input_fields:
+ query_template = query_template.replace(
+ "_MULTIPART_UPLOAD_FIELDS_", multipart_upload_url_query
+ )
+ else:
+ query_template = query_template.replace("_MULTIPART_UPLOAD_FIELDS_", "")
+
+ mutation = gql(query_template)
+ response = self.gql(
+ mutation,
+ variable_values={
+ "storageLayout": storage_layout,
+ "artifactFiles": [af for af in artifact_files],
+ },
+ )
+
+ result = {}
+ for edge in response["createArtifactFiles"]["files"]["edges"]:
+ node = edge["node"]
+ result[node["displayName"]] = node
+ return result
+
+ @normalize_exceptions
+ def notify_scriptable_run_alert(
+ self,
+ title: str,
+ text: str,
+ level: Optional[str] = None,
+ wait_duration: Optional["Number"] = None,
+ ) -> bool:
+ mutation = gql(
+ """
+ mutation NotifyScriptableRunAlert(
+ $entityName: String!,
+ $projectName: String!,
+ $runName: String!,
+ $title: String!,
+ $text: String!,
+ $severity: AlertSeverity = INFO,
+ $waitDuration: Duration
+ ) {
+ notifyScriptableRunAlert(input: {
+ entityName: $entityName,
+ projectName: $projectName,
+ runName: $runName,
+ title: $title,
+ text: $text,
+ severity: $severity,
+ waitDuration: $waitDuration
+ }) {
+ success
+ }
+ }
+ """
+ )
+
+ response = self.gql(
+ mutation,
+ variable_values={
+ "entityName": self.settings("entity"),
+ "projectName": self.settings("project"),
+ "runName": self.current_run_id,
+ "title": title,
+ "text": text,
+ "severity": level,
+ "waitDuration": wait_duration,
+ },
+ )
+ success: bool = response["notifyScriptableRunAlert"]["success"]
+ return success
+
+ def get_sweep_state(
+ self, sweep: str, entity: Optional[str] = None, project: Optional[str] = None
+ ) -> "SweepState":
+ state: SweepState = self.sweep(
+ sweep=sweep, entity=entity, project=project, specs="{}"
+ )["state"]
+ return state
+
+ def set_sweep_state(
+ self,
+ sweep: str,
+ state: "SweepState",
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+ ) -> None:
+ assert state in ("RUNNING", "PAUSED", "CANCELED", "FINISHED")
+ s = self.sweep(sweep=sweep, entity=entity, project=project, specs="{}")
+ curr_state = s["state"].upper()
+ if state == "PAUSED" and curr_state not in ("PAUSED", "RUNNING"):
+ raise Exception("Cannot pause %s sweep." % curr_state.lower())
+ elif state != "RUNNING" and curr_state not in ("RUNNING", "PAUSED", "PENDING"):
+ raise Exception("Sweep already %s." % curr_state.lower())
+ sweep_id = s["id"]
+ mutation = gql(
+ """
+ mutation UpsertSweep(
+ $id: ID,
+ $state: String,
+ $entityName: String,
+ $projectName: String
+ ) {
+ upsertSweep(input: {
+ id: $id,
+ state: $state,
+ entityName: $entityName,
+ projectName: $projectName
+ }){
+ sweep {
+ name
+ }
+ }
+ }
+ """
+ )
+ self.gql(
+ mutation,
+ variable_values={
+ "id": sweep_id,
+ "state": state,
+ "entityName": entity or self.settings("entity"),
+ "projectName": project or self.settings("project"),
+ },
+ )
+
+ def stop_sweep(
+ self,
+ sweep: str,
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+ ) -> None:
+ """Finish the sweep to stop running new runs and let currently running runs finish."""
+ self.set_sweep_state(
+ sweep=sweep, state="FINISHED", entity=entity, project=project
+ )
+
+ def cancel_sweep(
+ self,
+ sweep: str,
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+ ) -> None:
+ """Cancel the sweep to kill all running runs and stop running new runs."""
+ self.set_sweep_state(
+ sweep=sweep, state="CANCELED", entity=entity, project=project
+ )
+
+ def pause_sweep(
+ self,
+ sweep: str,
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+ ) -> None:
+ """Pause the sweep to temporarily stop running new runs."""
+ self.set_sweep_state(
+ sweep=sweep, state="PAUSED", entity=entity, project=project
+ )
+
+ def resume_sweep(
+ self,
+ sweep: str,
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+ ) -> None:
+ """Resume the sweep to continue running new runs."""
+ self.set_sweep_state(
+ sweep=sweep, state="RUNNING", entity=entity, project=project
+ )
+
+ def _status_request(self, url: str, length: int) -> requests.Response:
+ """Ask google how much we've uploaded."""
+ check_httpclient_logger_handler()
+ return requests.put(
+ url=url,
+ headers={"Content-Length": "0", "Content-Range": "bytes */%i" % length},
+ )
+
+ def _flatten_edges(self, response: "_Response") -> List[Dict]:
+ """Return an array from the nested graphql relay structure."""
+ return [node["node"] for node in response["edges"]]
+
+ @normalize_exceptions
+ def stop_run(
+ self,
+ run_id: str,
+ ) -> bool:
+ mutation = gql(
+ """
+ mutation stopRun($id: ID!) {
+ stopRun(input: {
+ id: $id
+ }) {
+ clientMutationId
+ success
+ }
+ }
+ """
+ )
+
+ response = self.gql(
+ mutation,
+ variable_values={
+ "id": run_id,
+ },
+ )
+
+ success: bool = response["stopRun"].get("success")
+
+ return success
diff --git a/wandb/sdk/internal/internal_util.py b/wandb/sdk/internal/internal_util.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e8443805bc7d375469a82507cbebb9845ab8367
--- /dev/null
+++ b/wandb/sdk/internal/internal_util.py
@@ -0,0 +1,101 @@
+"""Internal utility routines.
+
+Collection of classes to support the internal process.
+
+"""
+
+
+import logging
+import queue
+import sys
+import threading
+import time
+from typing import TYPE_CHECKING, Optional, Tuple, Type, Union
+
+from ..lib import tracelog
+
+if TYPE_CHECKING:
+ from queue import Queue
+ from threading import Event
+ from types import TracebackType
+
+ from wandb.proto.wandb_internal_pb2 import Record, Result
+
+ ExceptionType = Union[
+ Tuple[Type[BaseException], BaseException, TracebackType],
+ Tuple[None, None, None],
+ ]
+
+
+logger = logging.getLogger(__name__)
+
+
+class ExceptionThread(threading.Thread):
+ """Class to catch exceptions when running a thread."""
+
+ __stopped: "Event"
+ __exception: Optional["ExceptionType"]
+
+ def __init__(self, stopped: "Event") -> None:
+ threading.Thread.__init__(self)
+ self.__stopped = stopped
+ self.__exception = None
+
+ def _run(self) -> None:
+ raise NotImplementedError
+
+ def run(self) -> None:
+ try:
+ self._run()
+ except Exception:
+ self.__exception = sys.exc_info()
+ finally:
+ if self.__exception and self.__stopped:
+ self.__stopped.set()
+
+ def get_exception(self) -> Optional["ExceptionType"]:
+ return self.__exception
+
+
+class RecordLoopThread(ExceptionThread):
+ """Class to manage reading from queues safely."""
+
+ def __init__(
+ self,
+ input_record_q: "Queue[Record]",
+ result_q: "Queue[Result]",
+ stopped: "Event",
+ debounce_interval_ms: "float" = 1000,
+ ) -> None:
+ ExceptionThread.__init__(self, stopped=stopped)
+ self._input_record_q = input_record_q
+ self._result_q = result_q
+ self._stopped = stopped
+ self._debounce_interval_ms = debounce_interval_ms
+
+ def _setup(self) -> None:
+ raise NotImplementedError
+
+ def _process(self, record: "Record") -> None:
+ raise NotImplementedError
+
+ def _finish(self) -> None:
+ raise NotImplementedError
+
+ def _debounce(self) -> None:
+ raise NotImplementedError
+
+ def _run(self) -> None:
+ self._setup()
+ start = time.time()
+ while not self._stopped.is_set():
+ if time.time() - start >= self._debounce_interval_ms / 1000.0:
+ self._debounce()
+ start = time.time()
+ try:
+ record = self._input_record_q.get(timeout=1)
+ except queue.Empty:
+ continue
+ tracelog.log_message_dequeue(record, self._input_record_q)
+ self._process(record)
+ self._finish()
diff --git a/wandb/sdk/internal/job_builder.py b/wandb/sdk/internal/job_builder.py
new file mode 100644
index 0000000000000000000000000000000000000000..728c0c3828a7693d39264d1c78cf59f780171eea
--- /dev/null
+++ b/wandb/sdk/internal/job_builder.py
@@ -0,0 +1,523 @@
+"""job builder."""
+import json
+import logging
+import os
+import re
+import sys
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+
+import wandb
+from wandb.sdk.artifacts.artifact import Artifact
+from wandb.sdk.data_types._dtypes import TypeRegistry
+from wandb.sdk.lib.filenames import DIFF_FNAME, METADATA_FNAME, REQUIREMENTS_FNAME
+from wandb.util import make_artifact_name_safe
+
+from .settings_static import SettingsStatic
+
+if sys.version_info >= (3, 8):
+ from typing import Literal, TypedDict
+else:
+ from typing_extensions import Literal, TypedDict
+
+_logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+ from wandb.proto.wandb_internal_pb2 import ArtifactRecord, UseArtifactRecord
+
+FROZEN_REQUIREMENTS_FNAME = "requirements.frozen.txt"
+JOB_FNAME = "wandb-job.json"
+JOB_ARTIFACT_TYPE = "job"
+
+
+class GitInfo(TypedDict):
+ remote: str
+ commit: str
+
+
+class GitSourceDict(TypedDict):
+ git: GitInfo
+ entrypoint: List[str]
+ notebook: bool
+
+
+class ArtifactSourceDict(TypedDict):
+ artifact: str
+ entrypoint: List[str]
+ notebook: bool
+
+
+class ImageSourceDict(TypedDict):
+ image: str
+
+
+class JobSourceDict(TypedDict, total=False):
+ _version: str
+ source_type: str
+ source: Union[GitSourceDict, ArtifactSourceDict, ImageSourceDict]
+ input_types: Dict[str, Any]
+ output_types: Dict[str, Any]
+ runtime: Optional[str]
+ _partial: Optional[str] # flag to indicate incomplete job
+
+
+class PartialJobSourceDict(TypedDict):
+ job_name: str
+ job_source_info: JobSourceDict
+
+
+class ArtifactInfoForJob(TypedDict):
+ id: str
+ name: str
+
+
+class JobArtifact(Artifact):
+ def __init__(self, name: str, *args: Any, **kwargs: Any):
+ super().__init__(name, "placeholder", *args, **kwargs)
+ self._type = JOB_ARTIFACT_TYPE # Get around type restriction.
+
+
+class JobBuilder:
+ _settings: SettingsStatic
+ _metadatafile_path: Optional[str]
+ _requirements_path: Optional[str]
+ _config: Optional[Dict[str, Any]]
+ _summary: Optional[Dict[str, Any]]
+ _logged_code_artifact: Optional[ArtifactInfoForJob]
+ _disable: bool
+ _partial_source: Optional[PartialJobSourceDict]
+ _aliases: List[str]
+ _job_seq_id: Optional[str]
+ _job_version_alias: Optional[str]
+ _is_notebook_run: bool
+
+ def __init__(self, settings: SettingsStatic):
+ self._settings = settings
+ self._metadatafile_path = None
+ self._requirements_path = None
+ self._config = None
+ self._summary = None
+ self._logged_code_artifact = None
+ self._job_seq_id = None
+ self._job_version_alias = None
+ self._disable = settings.disable_job_creation
+ self._partial_source = None
+ self._aliases = []
+ self._source_type: Optional[
+ Literal["repo", "artifact", "image"]
+ ] = settings.job_source # type: ignore[assignment]
+ self._is_notebook_run = self._get_is_notebook_run()
+
+ def set_config(self, config: Dict[str, Any]) -> None:
+ self._config = config
+
+ def set_summary(self, summary: Dict[str, Any]) -> None:
+ self._summary = summary
+
+ @property
+ def disable(self) -> bool:
+ return self._disable
+
+ @disable.setter
+ def disable(self, val: bool) -> None:
+ self._disable = val
+
+ def _handle_server_artifact(self, res: Dict, artifact: "ArtifactRecord") -> None:
+ if artifact.type == "job" and res is not None:
+ try:
+ if res["artifactSequence"]["latestArtifact"] is None:
+ self._job_version_alias = "v0"
+ elif res["artifactSequence"]["latestArtifact"]["id"] == res["id"]:
+ self._job_version_alias = (
+ f"v{res['artifactSequence']['latestArtifact']['versionIndex']}"
+ )
+ else:
+ self._job_version_alias = f"v{res['artifactSequence']['latestArtifact']['versionIndex'] + 1}"
+ self._job_seq_id = res["artifactSequence"]["id"]
+ except KeyError as e:
+ _logger.info(f"Malformed response from ArtifactSaver.save {e}")
+ if artifact.type == "code" and "id" in res:
+ self._logged_code_artifact = ArtifactInfoForJob(
+ {
+ "id": res["id"],
+ "name": artifact.name,
+ }
+ )
+
+ def _build_repo_job_source(
+ self,
+ program_relpath: str,
+ metadata: Dict[str, Any],
+ ) -> Tuple[Optional[GitSourceDict], Optional[str]]:
+ git_info: Dict[str, str] = metadata.get("git", {})
+ remote = git_info.get("remote")
+ commit = git_info.get("commit")
+ root = metadata.get("root")
+ assert remote is not None
+ assert commit is not None
+ if self._is_notebook_run:
+ if not os.path.exists(
+ os.path.join(os.getcwd(), os.path.basename(program_relpath))
+ ):
+ return None, None
+
+ if root is None or self._settings._jupyter_root is None:
+ _logger.info("target path does not exist, exiting")
+ return None, None
+ assert self._settings._jupyter_root is not None
+ # git notebooks set the root to the git root,
+ # jupyter_root contains the path where the jupyter notebook was started
+ # program_relpath contains the path from jupyter_root to the file
+ # full program path here is actually the relpath from the program to the git root
+ full_program_path = os.path.join(
+ os.path.relpath(str(self._settings._jupyter_root), root),
+ program_relpath,
+ )
+ full_program_path = os.path.normpath(full_program_path)
+ # if the notebook server is started above the git repo need to clear all the ..s
+ if full_program_path.startswith(".."):
+ split_path = full_program_path.split("/")
+ count_dots = 0
+ for p in split_path:
+ if p == "..":
+ count_dots += 1
+ full_program_path = "/".join(split_path[2 * count_dots :])
+ else:
+ full_program_path = program_relpath
+
+ entrypoint = self._get_entrypoint(full_program_path, metadata)
+ # TODO: update executable to a method that supports pex
+ source: GitSourceDict = {
+ "git": {"remote": remote, "commit": commit},
+ "entrypoint": entrypoint,
+ "notebook": self._is_notebook_run,
+ }
+ name = self._make_job_name(f"{remote}_{program_relpath}")
+
+ return source, name
+
+ def _build_artifact_job_source(
+ self,
+ program_relpath: str,
+ metadata: Dict[str, Any],
+ ) -> Tuple[Optional[ArtifactSourceDict], Optional[str]]:
+ assert isinstance(self._logged_code_artifact, dict)
+ # TODO: should we just always exit early if the path doesn't exist?
+ if self._is_notebook_run and not self._is_colab_run():
+ full_program_relpath = os.path.relpath(program_relpath, os.getcwd())
+ # if the resolved path doesn't exist, then we shouldn't make a job because it will fail
+ if not os.path.exists(full_program_relpath):
+ # when users call log code in a notebook the code artifact starts
+ # at the directory the notebook is in instead of the jupyter core
+ if not os.path.exists(os.path.basename(program_relpath)):
+ _logger.info("target path does not exist, exiting")
+ wandb.termwarn(
+ "No program path found when generating artifact job source for a non-colab notebook run. See https://docs.wandb.ai/guides/launch/create-job"
+ )
+ return None, None
+ full_program_relpath = os.path.basename(program_relpath)
+ else:
+ full_program_relpath = program_relpath
+
+ entrypoint = self._get_entrypoint(full_program_relpath, metadata)
+ # TODO: update executable to a method that supports pex
+ source: ArtifactSourceDict = {
+ "entrypoint": entrypoint,
+ "notebook": self._is_notebook_run,
+ "artifact": f"wandb-artifact://_id/{self._logged_code_artifact['id']}",
+ }
+ name = self._make_job_name(self._logged_code_artifact["name"])
+
+ return source, name
+
+ def _build_image_job_source(
+ self, metadata: Dict[str, Any]
+ ) -> Tuple[ImageSourceDict, str]:
+ image_name = metadata.get("docker")
+ assert isinstance(image_name, str)
+
+ raw_image_name = image_name
+ if ":" in image_name:
+ tag = image_name.split(":")[-1]
+
+ # if tag looks properly formatted, assume its a tag
+ # regex: alphanumeric and "_" "-" "."
+ if re.fullmatch(r"([a-zA-Z0-9_\-\.]+)", tag):
+ raw_image_name = raw_image_name.replace(f":{tag}", "")
+ self._aliases += [tag]
+
+ source: ImageSourceDict = {
+ "image": image_name,
+ }
+ name = self._make_job_name(raw_image_name)
+
+ return source, name
+
+ def _make_job_name(self, input_str: str) -> str:
+ """Use job name from settings if provided, else use programatic name."""
+ if self._settings.job_name:
+ return self._settings.job_name
+
+ return make_artifact_name_safe(f"job-{input_str}")
+
+ def _get_entrypoint(
+ self,
+ program_relpath: str,
+ metadata: Dict[str, Any],
+ ) -> List[str]:
+ # if building a partial job from CLI, overwrite entrypoint and notebook
+ # should already be in metadata from create_job
+ if metadata.get("_partial"):
+ if metadata.get("entrypoint"):
+ entrypoint: List[str] = metadata["entrypoint"]
+ return entrypoint
+
+ # if entrypoint is not in metadata, then construct from python
+ assert metadata.get("python")
+
+ python = metadata["python"]
+ if python.count(".") > 1:
+ python = ".".join(python.split(".")[:2])
+
+ entrypoint = [f"python{python}", program_relpath]
+ return entrypoint
+
+ # job is being built from a run
+ entrypoint = [os.path.basename(sys.executable), program_relpath]
+
+ return entrypoint
+
+ def _get_is_notebook_run(self) -> bool:
+ return hasattr(self._settings, "_jupyter") and bool(self._settings._jupyter)
+
+ def _is_colab_run(self) -> bool:
+ return hasattr(self._settings, "_colab") and bool(self._settings._colab)
+
+ def build(self) -> Optional[Artifact]:
+ _logger.info("Attempting to build job artifact")
+ if not os.path.exists(
+ os.path.join(self._settings.files_dir, REQUIREMENTS_FNAME)
+ ):
+ wandb.termwarn(
+ "No requirements.txt found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"
+ )
+ return None
+ metadata = self._handle_metadata_file()
+ if metadata is None:
+ wandb.termwarn(
+ f"Ensure read and write access to run files dir: {self._settings.files_dir}, control this via the WANDB_DIR env var. See https://docs.wandb.ai/guides/track/environment-variables"
+ )
+ return None
+
+ runtime: Optional[str] = metadata.get("python")
+ # can't build a job without a python version
+ if runtime is None:
+ wandb.termwarn(
+ "No python version found in metadata, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"
+ )
+ return None
+
+ input_types = TypeRegistry.type_of(self._config).to_json()
+ output_types = TypeRegistry.type_of(self._summary).to_json()
+
+ name: Optional[str] = None
+ source_info: Optional[JobSourceDict] = None
+
+ if self._partial_source is not None:
+ # construct source from downloaded partial job metadata
+ name = self._partial_source["job_name"]
+ source_info = self._partial_source["job_source_info"]
+ # add input/output types now that we are actually running a run
+ source_info.update(
+ {"input_types": input_types, "output_types": output_types}
+ )
+ # set source_type to determine whether to add diff file to artifact
+ source_type = source_info.get("source_type")
+ else:
+ # configure job from environment
+ source_type = self._get_source_type(metadata)
+ if not source_type:
+ # if source_type is None, then we don't have enough information to build a job
+ # if the user intended to create a job, warn.
+ if (
+ self._settings.job_name
+ or self._settings.job_source
+ or self._source_type
+ ):
+ wandb.termwarn("No source type found, not creating job artifact")
+ return None
+
+ program_relpath = self._get_program_relpath(source_type, metadata)
+ if source_type != "image" and not program_relpath:
+ wandb.termwarn(
+ "No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"
+ )
+ return None
+
+ source: Union[
+ Optional[GitSourceDict],
+ Optional[ArtifactSourceDict],
+ Optional[ImageSourceDict],
+ ] = None
+
+ # make source dict
+ if source_type == "repo":
+ assert program_relpath
+ source, name = self._build_repo_job_source(program_relpath, metadata)
+ elif source_type == "artifact":
+ assert program_relpath
+ source, name = self._build_artifact_job_source(
+ program_relpath, metadata
+ )
+ elif source_type == "image" and self._has_image_job_ingredients(metadata):
+ source, name = self._build_image_job_source(metadata)
+ else:
+ source = None
+
+ if source is None:
+ if source_type:
+ wandb.termwarn(
+ f"Source type is set to '{source_type}' but some required information is missing "
+ "from the environment. A job will not be created from this run. See "
+ "https://docs.wandb.ai/guides/launch/create-job"
+ )
+ return None
+
+ source_info = {
+ "_version": "v0",
+ "source_type": source_type,
+ "source": source,
+ "input_types": input_types,
+ "output_types": output_types,
+ "runtime": runtime,
+ }
+
+ assert source_info is not None
+ assert name is not None
+ if metadata.get("_partial"):
+ assert not self._partial_source, "partial job has partial output"
+ source_info.update({"_partial": metadata["_partial"]})
+
+ artifact = JobArtifact(name)
+
+ _logger.info("adding wandb-job metadata file")
+ with artifact.new_file("wandb-job.json") as f:
+ f.write(json.dumps(source_info, indent=4))
+
+ artifact.add_file(
+ os.path.join(self._settings.files_dir, REQUIREMENTS_FNAME),
+ name=FROZEN_REQUIREMENTS_FNAME,
+ )
+
+ if source_type == "repo":
+ # add diff
+ if os.path.exists(os.path.join(self._settings.files_dir, DIFF_FNAME)):
+ artifact.add_file(
+ os.path.join(self._settings.files_dir, DIFF_FNAME),
+ name=DIFF_FNAME,
+ )
+
+ return artifact
+
+ def _get_source_type(self, metadata: Dict[str, Any]) -> Optional[str]:
+ if self._source_type:
+ return self._source_type
+
+ if self._has_git_job_ingredients(metadata):
+ _logger.info("is repo sourced job")
+ return "repo"
+
+ if self._has_artifact_job_ingredients():
+ _logger.info("is artifact sourced job")
+ return "artifact"
+
+ if self._has_image_job_ingredients(metadata):
+ _logger.info("is image sourced job")
+ return "image"
+
+ _logger.info("no source found")
+ return None
+
+ def _get_program_relpath(
+ self, source_type: str, metadata: Dict[str, Any]
+ ) -> Optional[str]:
+ if self._is_notebook_run:
+ _logger.info("run is notebook based run")
+ program = metadata.get("program")
+
+ if not program:
+ wandb.termwarn(
+ "Notebook 'program' path not found in metadata. See https://docs.wandb.ai/guides/launch/create-job"
+ )
+
+ return program
+
+ if source_type == "artifact" or self._settings.job_source == "artifact":
+ # if the job is set to be an artifact, use relpath guaranteed
+ # to be correct. 'codePath' uses the root path when in git repo
+ # fallback to codePath if strictly local relpath not present
+ return metadata.get("codePathLocal") or metadata.get("codePath")
+
+ return metadata.get("codePath")
+
+ def _handle_metadata_file(
+ self,
+ ) -> Optional[Dict]:
+ if os.path.exists(os.path.join(self._settings.files_dir, METADATA_FNAME)):
+ with open(os.path.join(self._settings.files_dir, METADATA_FNAME)) as f:
+ metadata: Dict = json.load(f)
+ return metadata
+
+ return None
+
+ def _has_git_job_ingredients(self, metadata: Dict[str, Any]) -> bool:
+ git_info: Dict[str, str] = metadata.get("git", {})
+ if self._is_notebook_run and metadata.get("root") is None:
+ return False
+ return git_info.get("remote") is not None and git_info.get("commit") is not None
+
+ def _has_artifact_job_ingredients(self) -> bool:
+ return self._logged_code_artifact is not None
+
+ def _has_image_job_ingredients(self, metadata: Dict[str, Any]) -> bool:
+ return metadata.get("docker") is not None
+
+
+def convert_use_artifact_to_job_source(
+ use_artifact: "UseArtifactRecord",
+) -> PartialJobSourceDict:
+ source_info = use_artifact.partial.source_info
+ source_info_dict: JobSourceDict = {
+ "_version": "v0",
+ "source_type": source_info.source_type,
+ "runtime": source_info.runtime,
+ }
+ if source_info.source_type == "repo":
+ entrypoint = [str(x) for x in source_info.source.git.entrypoint]
+ git_source: GitSourceDict = {
+ "git": {
+ "remote": source_info.source.git.git_info.remote,
+ "commit": source_info.source.git.git_info.commit,
+ },
+ "entrypoint": entrypoint,
+ "notebook": source_info.source.git.notebook,
+ }
+ source_info_dict.update({"source": git_source})
+ elif source_info.source_type == "artifact":
+ entrypoint = [str(x) for x in source_info.source.artifact.entrypoint]
+ artifact_source: ArtifactSourceDict = {
+ "artifact": source_info.source.artifact.artifact,
+ "entrypoint": entrypoint,
+ "notebook": source_info.source.artifact.notebook,
+ }
+ source_info_dict.update({"source": artifact_source})
+ elif source_info.source_type == "image":
+ image_source: ImageSourceDict = {
+ "image": source_info.source.image.image,
+ }
+ source_info_dict.update({"source": image_source})
+
+ partal_job_source_dict: PartialJobSourceDict = {
+ "job_name": use_artifact.partial.job_name.split(":")[0],
+ "job_source_info": source_info_dict,
+ }
+ return partal_job_source_dict
diff --git a/wandb/sdk/internal/profiler.py b/wandb/sdk/internal/profiler.py
new file mode 100644
index 0000000000000000000000000000000000000000..e74b3223e11cce42a8378fbcb19c5272c1110fb9
--- /dev/null
+++ b/wandb/sdk/internal/profiler.py
@@ -0,0 +1,77 @@
+"""Integration with pytorch profiler."""
+import os
+
+import wandb
+from wandb.errors import Error, UsageError
+from wandb.sdk.lib import telemetry
+
+PYTORCH_MODULE = "torch"
+PYTORCH_PROFILER_MODULE = "torch.profiler"
+
+
+def torch_trace_handler():
+ """Create a trace handler for traces generated by the profiler.
+
+ Provide as an argument to `torch.profiler.profile`:
+ ```python
+ torch.profiler.profile(..., on_trace_ready=wandb.profiler.torch_trace_handler())
+ ```
+
+ Calling this function ensures that profiler charts & tables can be viewed in your run dashboard
+ on wandb.ai.
+
+ Please note that `wandb.init()` must be called before this function is invoked.
+ The PyTorch (torch) version must also be at least 1.9, in order to ensure stability
+ of their Profiler API.
+
+ Args:
+ None
+
+ Returns:
+ None
+
+ Raises:
+ UsageError if wandb.init() hasn't been called before profiling.
+ Error if torch version is less than 1.9.0.
+
+ Examples:
+ ```python
+ run = wandb.init()
+ run.config.id = "profile_code"
+
+ with torch.profiler.profile(
+ schedule=torch.profiler.schedule(wait=1, warmup=1, active=3, repeat=1),
+ on_trace_ready=wandb.profiler.torch_trace_handler(),
+ record_shapes=True,
+ with_stack=True,
+ ) as prof:
+ for i, batch in enumerate(dataloader):
+ if step >= 5:
+ break
+ train(batch)
+ prof.step()
+ ```
+ """
+ from pkg_resources import parse_version
+
+ torch = wandb.util.get_module(PYTORCH_MODULE, required=True)
+ torch_profiler = wandb.util.get_module(PYTORCH_PROFILER_MODULE, required=True)
+
+ if parse_version(torch.__version__) < parse_version("1.9.0"):
+ raise Error(
+ f"torch version must be at least 1.9 in order to use the PyTorch Profiler API.\
+ \nVersion of torch currently installed: {torch.__version__}"
+ )
+
+ try:
+ logdir = os.path.join(wandb.run.dir, "pytorch_traces") # type: ignore
+ os.mkdir(logdir)
+ except AttributeError:
+ raise UsageError(
+ "Please call `wandb.init()` before `wandb.profiler.torch_trace_handler()`"
+ ) from None
+
+ with telemetry.context() as tel:
+ tel.feature.torch_profiler_trace = True
+
+ return torch_profiler.tensorboard_trace_handler(logdir)
diff --git a/wandb/sdk/internal/progress.py b/wandb/sdk/internal/progress.py
new file mode 100644
index 0000000000000000000000000000000000000000..14b42fbd04da7752aaf88df004a31cbcd2baee4f
--- /dev/null
+++ b/wandb/sdk/internal/progress.py
@@ -0,0 +1,111 @@
+"""progress."""
+
+import os
+import sys
+from typing import IO, TYPE_CHECKING, Optional
+
+from wandb.errors import CommError
+
+if TYPE_CHECKING:
+ if sys.version_info >= (3, 8):
+ from typing import Protocol
+ else:
+ from typing_extensions import Protocol
+
+ class ProgressFn(Protocol):
+ def __call__(self, new_bytes: int, total_bytes: int) -> None:
+ pass
+
+
+class Progress:
+ """A helper class for displaying progress."""
+
+ ITER_BYTES = 1024 * 1024
+
+ def __init__(
+ self, file: IO[bytes], callback: Optional["ProgressFn"] = None
+ ) -> None:
+ self.file = file
+ if callback is None:
+
+ def callback_(new_bytes: int, total_bytes: int) -> None:
+ pass
+
+ callback = callback_
+
+ self.callback: ProgressFn = callback
+ self.bytes_read = 0
+ self.len = os.fstat(file.fileno()).st_size
+
+ def read(self, size=-1):
+ """Read bytes and call the callback."""
+ bites = self.file.read(size)
+ self.bytes_read += len(bites)
+ if not bites and self.bytes_read < self.len:
+ # Files shrinking during uploads causes request timeouts. Maybe
+ # we could avoid those by updating the self.len in real-time, but
+ # files getting truncated while uploading seems like something
+ # that shouldn't really be happening anyway.
+ raise CommError(
+ "File {} size shrank from {} to {} while it was being uploaded.".format(
+ self.file.name, self.len, self.bytes_read
+ )
+ )
+ # Growing files are also likely to be bad, but our code didn't break
+ # on those in the past, so it's riskier to make that an error now.
+ self.callback(len(bites), self.bytes_read)
+ return bites
+
+ def rewind(self) -> None:
+ self.callback(-self.bytes_read, 0)
+ self.bytes_read = 0
+ self.file.seek(0)
+
+ def __getattr__(self, name):
+ """Fallback to the file object for attrs not defined here."""
+ if hasattr(self.file, name):
+ return getattr(self.file, name)
+ else:
+ raise AttributeError
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ bites = self.read(self.ITER_BYTES)
+ if len(bites) == 0:
+ raise StopIteration
+ return bites
+
+ def __len__(self):
+ return self.len
+
+ next = __next__
+
+
+class AsyncProgress:
+ """Wrapper around Progress, to make it async iterable.
+
+ httpx, for streaming uploads, requires the data source to be an async iterable.
+ If we pass in a sync iterable (like a bare `Progress` instance), httpx will
+ get confused, think we're trying to make a synchronous request, and raise.
+ So we need this wrapper class to be an async iterable but *not* a sync iterable.
+ """
+
+ def __init__(self, progress: Progress) -> None:
+ self._progress = progress
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ try:
+ return next(self._progress)
+ except StopIteration:
+ raise StopAsyncIteration
+
+ def __len__(self):
+ return len(self._progress)
+
+ def rewind(self) -> None:
+ self._progress.rewind()
diff --git a/wandb/sdk/internal/run.py b/wandb/sdk/internal/run.py
new file mode 100644
index 0000000000000000000000000000000000000000..3304d938ff2221aa9639bf2b7931b075e56380c6
--- /dev/null
+++ b/wandb/sdk/internal/run.py
@@ -0,0 +1,24 @@
+#
+"""InternalRun - Internal-only run object.
+
+Semi-stubbed run for internal process use.
+
+"""
+from wandb._globals import _datatypes_set_callback
+
+from .. import wandb_run
+
+
+class InternalRun(wandb_run.Run):
+ def __init__(self, run_obj, settings, datatypes_cb):
+ super().__init__(settings=settings)
+ self._run_obj = run_obj
+
+ # TODO: This overwrites what's done in the constructor of wandb_run.Run.
+ # We really want a common interface for wandb_run.Run and InternalRun.
+ _datatypes_set_callback(datatypes_cb)
+
+ def _set_backend(self, backend):
+ # This type of run object can't have a backend
+ # or do any writes.
+ pass
diff --git a/wandb/sdk/internal/sample.py b/wandb/sdk/internal/sample.py
new file mode 100644
index 0000000000000000000000000000000000000000..0432f3ad7713e8bd11f727ae4a8e6406b055b59f
--- /dev/null
+++ b/wandb/sdk/internal/sample.py
@@ -0,0 +1,70 @@
+"""sample."""
+
+import math
+
+
+class UniformSampleAccumulator:
+ def __init__(self, min_samples=None):
+ self._samples = min_samples or 64
+ # force power of 2 samples
+ self._samples = 2 ** int(math.ceil(math.log(self._samples, 2)))
+ # target oversample by factor of 2
+ self._samples2 = self._samples * 2
+ # max size of each buffer
+ self._max = self._samples2 // 2
+ self._shift = 0
+ self._mask = (1 << self._shift) - 1
+ self._buckets = int(math.log(self._samples2, 2))
+ self._buckets_bits = int(math.log(self._buckets, 2))
+ self._buckets_mask = (1 << self._buckets_bits + 1) - 1
+ self._buckets_index = 0
+ self._bucket = []
+ self._index = [0] * self._buckets
+ self._count = 0
+ self._log2 = [0]
+
+ # pre-allocate buckets
+ for _ in range(self._buckets):
+ self._bucket.append([0] * self._max)
+ # compute integer log2
+ self._log2 += [int(math.log(i, 2)) for i in range(1, 2**self._buckets + 1)]
+
+ def _show(self):
+ print("=" * 20)
+ for b in range(self._buckets):
+ b = (b + self._buckets_index) % self._buckets
+ vals = [self._bucket[b][i] for i in range(self._index[b])]
+ print(f"{b}: {vals}")
+
+ def add(self, val):
+ self._count += 1
+ cnt = self._count
+ if cnt & self._mask:
+ return
+ b = cnt >> self._shift
+ b = self._log2[b] # b = int(math.log(b, 2))
+ if b >= self._buckets:
+ self._index[self._buckets_index] = 0
+ self._buckets_index = (self._buckets_index + 1) % self._buckets
+ self._shift += 1
+ self._mask = (self._mask << 1) | 1
+ b += self._buckets - 1
+ b = (b + self._buckets_index) % self._buckets
+ self._bucket[b][self._index[b]] = val
+ self._index[b] += 1
+
+ def get(self):
+ full = []
+ sampled = []
+ # self._show()
+ for b in range(self._buckets):
+ max_num = 2**b
+ b = (b + self._buckets_index) % self._buckets
+ modb = self._index[b] // max_num
+ for i in range(self._index[b]):
+ if not modb or i % modb == 0:
+ sampled.append(self._bucket[b][i])
+ full.append(self._bucket[b][i])
+ if len(sampled) < self._samples:
+ return tuple(full)
+ return tuple(sampled)
diff --git a/wandb/sdk/internal/sender.py b/wandb/sdk/internal/sender.py
new file mode 100644
index 0000000000000000000000000000000000000000..217f32bc02d78af356710582b030aebecea57b59
--- /dev/null
+++ b/wandb/sdk/internal/sender.py
@@ -0,0 +1,1670 @@
+"""sender."""
+
+import concurrent.futures
+import json
+import logging
+import os
+import queue
+import sys
+import threading
+import time
+import traceback
+from collections import defaultdict
+from datetime import datetime
+from queue import Queue
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Dict,
+ Generator,
+ List,
+ NewType,
+ Optional,
+ Tuple,
+ Type,
+ Union,
+ cast,
+)
+
+import requests
+
+import wandb
+from wandb import util
+from wandb.errors import CommError, UsageError
+from wandb.errors.util import ProtobufErrorHandler
+from wandb.filesync.dir_watcher import DirWatcher
+from wandb.proto import wandb_internal_pb2
+from wandb.sdk.artifacts.artifact_saver import ArtifactSaver
+from wandb.sdk.interface import interface
+from wandb.sdk.interface.interface_queue import InterfaceQueue
+from wandb.sdk.internal import (
+ context,
+ datastore,
+ file_stream,
+ internal_api,
+ job_builder,
+ update,
+)
+from wandb.sdk.internal.file_pusher import FilePusher
+from wandb.sdk.internal.job_builder import JobBuilder
+from wandb.sdk.internal.settings_static import SettingsStatic
+from wandb.sdk.lib import (
+ config_util,
+ filenames,
+ filesystem,
+ printer,
+ proto_util,
+ redirect,
+ telemetry,
+ tracelog,
+)
+from wandb.sdk.lib.mailbox import ContextCancelledError
+from wandb.sdk.lib.proto_util import message_to_dict
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+if TYPE_CHECKING:
+ from wandb.proto.wandb_internal_pb2 import (
+ ArtifactManifest,
+ ArtifactRecord,
+ HttpResponse,
+ LocalInfo,
+ Record,
+ Result,
+ RunExitResult,
+ RunRecord,
+ SummaryRecord,
+ )
+
+ StreamLiterals = Literal["stdout", "stderr"]
+
+
+logger = logging.getLogger(__name__)
+
+
+DictWithValues = NewType("DictWithValues", Dict[str, Any])
+DictNoValues = NewType("DictNoValues", Dict[str, Any])
+
+_OUTPUT_MIN_CALLBACK_INTERVAL = 2 # seconds
+
+
+def _framework_priority() -> Generator[Tuple[str, str], None, None]:
+ yield from [
+ ("lightgbm", "lightgbm"),
+ ("catboost", "catboost"),
+ ("xgboost", "xgboost"),
+ ("transformers_huggingface", "huggingface"), # backwards compatibility
+ ("transformers", "huggingface"),
+ ("pytorch_ignite", "ignite"), # backwards compatibility
+ ("ignite", "ignite"),
+ ("pytorch_lightning", "lightning"),
+ ("fastai", "fastai"),
+ ("torch", "torch"),
+ ("keras", "keras"),
+ ("tensorflow", "tensorflow"),
+ ("sklearn", "sklearn"),
+ ]
+
+
+def _manifest_json_from_proto(manifest: "ArtifactManifest") -> Dict:
+ if manifest.version == 1:
+ contents = {
+ content.path: {
+ "digest": content.digest,
+ "birthArtifactID": content.birth_artifact_id
+ if content.birth_artifact_id
+ else None,
+ "ref": content.ref if content.ref else None,
+ "size": content.size if content.size is not None else None,
+ "local_path": content.local_path if content.local_path else None,
+ "extra": {
+ extra.key: json.loads(extra.value_json) for extra in content.extra
+ },
+ }
+ for content in manifest.contents
+ }
+ else:
+ raise ValueError(f"unknown artifact manifest version: {manifest.version}")
+
+ return {
+ "version": manifest.version,
+ "storagePolicy": manifest.storage_policy,
+ "storagePolicyConfig": {
+ config.key: json.loads(config.value_json)
+ for config in manifest.storage_policy_config
+ },
+ "contents": contents,
+ }
+
+
+class ResumeState:
+ resumed: bool
+ step: int
+ history: int
+ events: int
+ output: int
+ runtime: float
+ wandb_runtime: Optional[int]
+ summary: Optional[Dict[str, Any]]
+ config: Optional[Dict[str, Any]]
+ tags: Optional[List[str]]
+
+ def __init__(self) -> None:
+ self.resumed = False
+ self.step = 0
+ self.history = 0
+ self.events = 0
+ self.output = 0
+ self.runtime = 0
+ # wandb_runtime is the canonical runtime (stored in summary._wandb.runtime)
+ self.wandb_runtime = None
+ self.summary = None
+ self.config = None
+ self.tags = None
+
+ def __str__(self) -> str:
+ obj = ",".join(map(lambda it: f"{it[0]}={it[1]}", vars(self).items()))
+ return f"ResumeState({obj})"
+
+
+class _OutputRawStream:
+ _stopped: threading.Event
+ _queue: queue.Queue
+ _emulator: redirect.TerminalEmulator
+ _writer_thr: threading.Thread
+ _reader_thr: threading.Thread
+
+ def __init__(self, stream: str, sm: "SendManager"):
+ self._stopped = threading.Event()
+ self._queue = queue.Queue()
+ self._emulator = redirect.TerminalEmulator()
+ self._writer_thr = threading.Thread(
+ target=sm._output_raw_writer_thread,
+ kwargs=dict(stream=stream),
+ daemon=True,
+ name=f"OutRawWr-{stream}",
+ )
+ self._reader_thr = threading.Thread(
+ target=sm._output_raw_reader_thread,
+ kwargs=dict(stream=stream),
+ daemon=True,
+ name=f"OutRawRd-{stream}",
+ )
+
+ def start(self) -> None:
+ self._writer_thr.start()
+ self._reader_thr.start()
+
+
+class SendManager:
+ UPDATE_CONFIG_TIME: int = 30
+ UPDATE_STATUS_TIME: int = 5
+
+ _settings: SettingsStatic
+ _record_q: "Queue[Record]"
+ _result_q: "Queue[Result]"
+ _interface: InterfaceQueue
+ _api_settings: Dict[str, str]
+ _partial_output: Dict[str, str]
+ _context_keeper: context.ContextKeeper
+
+ _telemetry_obj: telemetry.TelemetryRecord
+ _fs: Optional["file_stream.FileStreamApi"]
+ _run: Optional["RunRecord"]
+ _entity: Optional[str]
+ _project: Optional[str]
+ _dir_watcher: Optional["DirWatcher"]
+ _pusher: Optional["FilePusher"]
+ _record_exit: Optional["Record"]
+ _exit_result: Optional["RunExitResult"]
+ _resume_state: ResumeState
+ _cached_server_info: Dict[str, Any]
+ _cached_viewer: Dict[str, Any]
+ _server_messages: List[Dict[str, Any]]
+ _ds: Optional[datastore.DataStore]
+ _output_raw_streams: Dict["StreamLiterals", _OutputRawStream]
+ _output_raw_file: Optional[filesystem.CRDedupedFile]
+ _send_record_num: int
+ _send_end_offset: int
+ _debounce_config_time: float
+ _debounce_status_time: float
+
+ def __init__(
+ self,
+ settings: SettingsStatic,
+ record_q: "Queue[Record]",
+ result_q: "Queue[Result]",
+ interface: InterfaceQueue,
+ context_keeper: context.ContextKeeper,
+ ) -> None:
+ self._settings = settings
+ self._record_q = record_q
+ self._result_q = result_q
+ self._interface = interface
+ self._context_keeper = context_keeper
+
+ self._ds = None
+ self._send_record_num = 0
+ self._send_end_offset = 0
+
+ self._fs = None
+ self._pusher = None
+ self._dir_watcher = None
+
+ # State updated by login
+ self._entity = None
+ self._flags = None
+
+ # State updated by wandb.init
+ self._run = None
+ self._project = None
+
+ # keep track of config from key/val updates
+ self._consolidated_config: DictNoValues = cast(DictNoValues, dict())
+ self._start_time: float = 0
+ self._telemetry_obj = telemetry.TelemetryRecord()
+ self._config_metric_pbdict_list: List[Dict[int, Any]] = []
+ self._metadata_summary: Dict[str, Any] = defaultdict()
+ self._cached_summary: Dict[str, Any] = dict()
+ self._config_metric_index_dict: Dict[str, int] = {}
+ self._config_metric_dict: Dict[str, wandb_internal_pb2.MetricRecord] = {}
+ self._consolidated_summary: Dict[str, Any] = dict()
+
+ self._cached_server_info = dict()
+ self._cached_viewer = dict()
+ self._server_messages = []
+
+ # State updated by resuming
+ self._resume_state = ResumeState()
+
+ # State added when run_exit is initiated and complete
+ self._record_exit = None
+ self._exit_result = None
+
+ self._api = internal_api.Api(
+ default_settings=settings, retry_callback=self.retry_callback
+ )
+ self._api_settings = dict()
+
+ # queue filled by retry_callback
+ self._retry_q: Queue[HttpResponse] = queue.Queue()
+
+ # do we need to debounce?
+ self._config_needs_debounce: bool = False
+
+ # TODO(jhr): do something better, why do we need to send full lines?
+ self._partial_output = dict()
+
+ self._exit_code = 0
+
+ # internal vars for handing raw console output
+ self._output_raw_streams = dict()
+ self._output_raw_file = None
+
+ # job builder
+ self._job_builder = JobBuilder(settings)
+
+ time_now = time.monotonic()
+ self._debounce_config_time = time_now
+ self._debounce_status_time = time_now
+
+ @classmethod
+ def setup(
+ cls,
+ root_dir: str,
+ resume: Union[None, bool, str],
+ ) -> "SendManager":
+ """Set up a standalone SendManager.
+
+ Currently, we're using this primarily for `sync.py`.
+ """
+ files_dir = os.path.join(root_dir, "files")
+ settings = wandb.Settings(
+ files_dir=files_dir,
+ root_dir=root_dir,
+ # _start_time=0,
+ resume=resume,
+ # ignore_globs=(),
+ _sync=True,
+ disable_job_creation=False,
+ _async_upload_concurrency_limit=None,
+ _file_stream_timeout_seconds=0,
+ )
+ record_q: Queue[Record] = queue.Queue()
+ result_q: Queue[Result] = queue.Queue()
+ publish_interface = InterfaceQueue(record_q=record_q)
+ context_keeper = context.ContextKeeper()
+ return SendManager(
+ settings=SettingsStatic(settings.to_proto()),
+ record_q=record_q,
+ result_q=result_q,
+ interface=publish_interface,
+ context_keeper=context_keeper,
+ )
+
+ def __len__(self) -> int:
+ return self._record_q.qsize()
+
+ def __enter__(self) -> "SendManager":
+ return self
+
+ def __exit__(
+ self,
+ exc_type: Optional[Type[BaseException]],
+ exc_value: Optional[BaseException],
+ exc_traceback: Optional[traceback.TracebackException],
+ ) -> Literal[False]:
+ while self:
+ data = next(self)
+ self.send(data)
+ self.finish()
+ return False
+
+ def retry_callback(self, status: int, response_text: str) -> None:
+ response = wandb_internal_pb2.HttpResponse()
+ response.http_status_code = status
+ response.http_response_text = response_text
+ self._retry_q.put(response)
+
+ def send(self, record: "Record") -> None:
+ self._update_record_num(record.num)
+ self._update_end_offset(record.control.end_offset)
+
+ record_type = record.WhichOneof("record_type")
+ assert record_type
+ handler_str = "send_" + record_type
+ send_handler = getattr(self, handler_str, None)
+ # Don't log output to reduce log noise
+ if record_type not in {"output", "request", "output_raw"}:
+ logger.debug(f"send: {record_type}")
+ assert send_handler, f"unknown send handler: {handler_str}"
+
+ context_id = context.context_id_from_record(record)
+ api_context = self._context_keeper.get(context_id)
+ try:
+ self._api.set_local_context(api_context)
+ send_handler(record)
+ except ContextCancelledError:
+ logger.debug(f"Record cancelled: {record_type}")
+ self._context_keeper.release(context_id)
+ finally:
+ self._api.clear_local_context()
+
+ def send_preempting(self, _: "Record") -> None:
+ if self._fs:
+ self._fs.enqueue_preempting()
+
+ def send_request_sender_mark(self, _: "Record") -> None:
+ self._maybe_report_status(always=True)
+
+ def send_request(self, record: "Record") -> None:
+ request_type = record.request.WhichOneof("request_type")
+ assert request_type
+ handler_str = "send_request_" + request_type
+ send_handler = getattr(self, handler_str, None)
+ if request_type != "network_status":
+ logger.debug(f"send_request: {request_type}")
+ assert send_handler, f"unknown handle: {handler_str}"
+ send_handler(record)
+
+ def _respond_result(self, result: "Result") -> None:
+ tracelog.log_message_queue(result, self._result_q)
+ context_id = context.context_id_from_result(result)
+ self._context_keeper.release(context_id)
+ self._result_q.put(result)
+
+ def _flatten(self, dictionary: Dict) -> None:
+ if isinstance(dictionary, dict):
+ for k, v in list(dictionary.items()):
+ if isinstance(v, dict):
+ self._flatten(v)
+ dictionary.pop(k)
+ for k2, v2 in v.items():
+ dictionary[k + "." + k2] = v2
+
+ def _update_record_num(self, record_num: int) -> None:
+ if not record_num:
+ return
+ # Currently how we handle offline mode and syncing is not
+ # compatible with this assertion due to how the exit record
+ # is (mis)handled:
+ # - using "always_send" in offline mode to trigger defer
+ # state machine
+ # - skipping the exit record in `wandb sync` mode so that
+ # it is always executed as the last record
+ if not self._settings._offline and not self._settings._sync:
+ assert record_num == self._send_record_num + 1
+ self._send_record_num = record_num
+
+ def _update_end_offset(self, end_offset: int) -> None:
+ if not end_offset:
+ return
+ self._send_end_offset = end_offset
+
+ def send_request_sender_read(self, record: "Record") -> None:
+ if self._ds is None:
+ self._ds = datastore.DataStore()
+ self._ds.open_for_scan(self._settings.sync_file)
+
+ # TODO(cancel_paused): implement cancel_set logic
+ # The idea is that there is an active request to cancel a
+ # message that is being read from the transaction log below
+
+ start_offset = record.request.sender_read.start_offset
+ final_offset = record.request.sender_read.final_offset
+ self._ds.seek(start_offset)
+
+ current_end_offset = 0
+ while current_end_offset < final_offset:
+ data = self._ds.scan_data()
+ assert data
+ current_end_offset = self._ds.get_offset()
+
+ send_record = wandb_internal_pb2.Record()
+ send_record.ParseFromString(data)
+ self._update_end_offset(current_end_offset)
+ self.send(send_record)
+
+ # make sure we perform deferred operations
+ self.debounce()
+
+ # make sure that we always update writer for every sended read request
+ self._maybe_report_status(always=True)
+
+ def send_request_check_version(self, record: "Record") -> None:
+ assert record.control.req_resp or record.control.mailbox_slot
+ result = proto_util._result_from_record(record)
+ current_version = (
+ record.request.check_version.current_version or wandb.__version__
+ )
+ messages = update.check_available(current_version)
+ if messages:
+ upgrade_message = messages.get("upgrade_message")
+ if upgrade_message:
+ result.response.check_version_response.upgrade_message = upgrade_message
+ yank_message = messages.get("yank_message")
+ if yank_message:
+ result.response.check_version_response.yank_message = yank_message
+ delete_message = messages.get("delete_message")
+ if delete_message:
+ result.response.check_version_response.delete_message = delete_message
+ self._respond_result(result)
+
+ def send_request_stop_status(self, record: "Record") -> None:
+ result = proto_util._result_from_record(record)
+ status_resp = result.response.stop_status_response
+ status_resp.run_should_stop = False
+ if self._entity and self._project and self._run and self._run.run_id:
+ try:
+ status_resp.run_should_stop = self._api.check_stop_requested(
+ self._project, self._entity, self._run.run_id
+ )
+ except Exception as e:
+ logger.warning("Failed to check stop requested status: %s", e)
+ self._respond_result(result)
+
+ def _maybe_update_config(self, always: bool = False) -> None:
+ time_now = time.monotonic()
+ if (
+ not always
+ and time_now < self._debounce_config_time + self.UPDATE_CONFIG_TIME
+ ):
+ return
+ if self._config_needs_debounce:
+ self._debounce_config()
+ self._debounce_config_time = time_now
+
+ def _maybe_report_status(self, always: bool = False) -> None:
+ time_now = time.monotonic()
+ if (
+ not always
+ and time_now < self._debounce_status_time + self.UPDATE_STATUS_TIME
+ ):
+ return
+ self._debounce_status_time = time_now
+
+ status_report = wandb_internal_pb2.StatusReportRequest(
+ record_num=self._send_record_num,
+ sent_offset=self._send_end_offset,
+ )
+ status_time = time.time()
+ status_report.sync_time.FromMicroseconds(int(status_time * 1e6))
+ record = self._interface._make_request(status_report=status_report)
+ self._interface._publish(record)
+
+ def debounce(self, final: bool = False) -> None:
+ self._maybe_report_status(always=final)
+ self._maybe_update_config(always=final)
+
+ def _debounce_config(self) -> None:
+ config_value_dict = self._config_format(self._consolidated_config)
+ # TODO(jhr): check result of upsert_run?
+ if self._run:
+ self._api.upsert_run(
+ name=self._run.run_id,
+ config=config_value_dict,
+ **self._api_settings, # type: ignore
+ )
+ self._config_save(config_value_dict)
+ self._config_needs_debounce = False
+
+ def send_request_network_status(self, record: "Record") -> None:
+ result = proto_util._result_from_record(record)
+ status_resp = result.response.network_status_response
+ while True:
+ try:
+ status_resp.network_responses.append(self._retry_q.get_nowait())
+ except queue.Empty:
+ break
+ except Exception as e:
+ logger.warning(f"Error emptying retry queue: {e}")
+ self._respond_result(result)
+
+ def send_request_login(self, record: "Record") -> None:
+ # TODO: do something with api_key or anonymous?
+ # TODO: return an error if we aren't logged in?
+ self._api.reauth()
+ viewer = self.get_viewer_info()
+ server_info = self.get_server_info()
+ # self._login_flags = json.loads(viewer.get("flags", "{}"))
+ # self._login_entity = viewer.get("entity")
+ if server_info:
+ logger.info(f"Login server info: {server_info}")
+ self._entity = viewer.get("entity")
+ if record.control.req_resp:
+ result = proto_util._result_from_record(record)
+ if self._entity:
+ result.response.login_response.active_entity = self._entity
+ self._respond_result(result)
+
+ def send_exit(self, record: "Record") -> None:
+ # track where the exit came from
+ self._record_exit = record
+
+ run_exit = record.exit
+ self._exit_code = run_exit.exit_code
+ logger.info("handling exit code: %s", run_exit.exit_code)
+ runtime = run_exit.runtime
+ logger.info("handling runtime: %s", run_exit.runtime)
+ self._metadata_summary["runtime"] = runtime
+ self._update_summary()
+
+ # We need to give the request queue a chance to empty between states
+ # so use handle_request_defer as a state machine.
+ logger.info("send defer")
+ self._interface.publish_defer()
+
+ def send_final(self, record: "Record") -> None:
+ pass
+
+ def _flush_run(self) -> None:
+ pass
+
+ def send_request_status_report(self, record: "Record") -> None:
+ # todo? this is just a noop to please wandb sync
+ pass
+
+ def send_request_defer(self, record: "Record") -> None: # noqa: C901
+ defer = record.request.defer
+ state = defer.state
+ logger.info(f"handle sender defer: {state}")
+
+ def transition_state() -> None:
+ state = defer.state + 1
+ logger.info(f"send defer: {state}")
+ self._interface.publish_defer(state)
+
+ done = False
+ if state == defer.BEGIN:
+ transition_state()
+ elif state == defer.FLUSH_RUN:
+ self._flush_run()
+ transition_state()
+ elif state == defer.FLUSH_STATS:
+ # NOTE: this is handled in handler.py:handle_request_defer()
+ transition_state()
+ elif state == defer.FLUSH_PARTIAL_HISTORY:
+ # NOTE: this is handled in handler.py:handle_request_defer()
+ transition_state()
+ elif state == defer.FLUSH_TB:
+ # NOTE: this is handled in handler.py:handle_request_defer()
+ transition_state()
+ elif state == defer.FLUSH_SUM:
+ # NOTE: this is handled in handler.py:handle_request_defer()
+ transition_state()
+ elif state == defer.FLUSH_DEBOUNCER:
+ self.debounce(final=True)
+ transition_state()
+ elif state == defer.FLUSH_OUTPUT:
+ self._output_raw_finish()
+ transition_state()
+ elif state == defer.FLUSH_JOB:
+ self._flush_job()
+ transition_state()
+ elif state == defer.FLUSH_DIR:
+ if self._dir_watcher:
+ self._dir_watcher.finish()
+ self._dir_watcher = None
+ transition_state()
+ elif state == defer.FLUSH_FP:
+ if self._pusher:
+ # FilePusher generates some events for FileStreamApi, so we
+ # need to wait for pusher to finish before going to the next
+ # state to ensure that filestream gets all the events that we
+ # want before telling it to finish up
+ self._pusher.finish(transition_state)
+ else:
+ transition_state()
+ elif state == defer.JOIN_FP:
+ if self._pusher:
+ self._pusher.join()
+ transition_state()
+ elif state == defer.FLUSH_FS:
+ if self._fs:
+ # TODO(jhr): now is a good time to output pending output lines
+ self._fs.finish(self._exit_code)
+ self._fs = None
+ transition_state()
+ elif state == defer.FLUSH_FINAL:
+ self._interface.publish_final()
+ self._interface.publish_footer()
+ transition_state()
+ elif state == defer.END:
+ done = True
+ else:
+ raise AssertionError("unknown state")
+
+ if not done:
+ return
+
+ exit_result = wandb_internal_pb2.RunExitResult()
+
+ # mark exit done in case we are polling on exit
+ self._exit_result = exit_result
+
+ # Report response to mailbox
+ if self._record_exit and self._record_exit.control.mailbox_slot:
+ result = proto_util._result_from_record(self._record_exit)
+ result.exit_result.CopyFrom(exit_result)
+ self._respond_result(result)
+
+ def send_request_poll_exit(self, record: "Record") -> None:
+ if not record.control.req_resp and not record.control.mailbox_slot:
+ return
+
+ result = proto_util._result_from_record(record)
+
+ if self._pusher:
+ _alive, status = self._pusher.get_status()
+ file_counts = self._pusher.file_counts_by_category()
+ resp = result.response.poll_exit_response
+ resp.pusher_stats.uploaded_bytes = status.uploaded_bytes
+ resp.pusher_stats.total_bytes = status.total_bytes
+ resp.pusher_stats.deduped_bytes = status.deduped_bytes
+ resp.file_counts.wandb_count = file_counts.wandb
+ resp.file_counts.media_count = file_counts.media
+ resp.file_counts.artifact_count = file_counts.artifact
+ resp.file_counts.other_count = file_counts.other
+
+ if self._exit_result:
+ result.response.poll_exit_response.done = True
+ result.response.poll_exit_response.exit_result.CopyFrom(self._exit_result)
+
+ self._respond_result(result)
+
+ def send_request_server_info(self, record: "Record") -> None:
+ assert record.control.req_resp or record.control.mailbox_slot
+ result = proto_util._result_from_record(record)
+
+ result.response.server_info_response.local_info.CopyFrom(self.get_local_info())
+ for message in self._server_messages:
+ # guard against the case the message level returns malformed from server
+ message_level = str(message.get("messageLevel"))
+ message_level_sanitized = int(
+ printer.INFO if not message_level.isdigit() else message_level
+ )
+ result.response.server_info_response.server_messages.item.append(
+ wandb_internal_pb2.ServerMessage(
+ utf_text=message.get("utfText", ""),
+ plain_text=message.get("plainText", ""),
+ html_text=message.get("htmlText", ""),
+ type=message.get("messageType", ""),
+ level=message_level_sanitized,
+ )
+ )
+ self._respond_result(result)
+
+ def send_request_job_info(self, record: "Record") -> None:
+ """Respond to a request for a job link."""
+ result = proto_util._result_from_record(record)
+ result.response.job_info_response.sequenceId = (
+ self._job_builder._job_seq_id or ""
+ )
+ result.response.job_info_response.version = (
+ self._job_builder._job_version_alias or ""
+ )
+ self._respond_result(result)
+
+ def _maybe_setup_resume(
+ self, run: "RunRecord"
+ ) -> Optional["wandb_internal_pb2.ErrorInfo"]:
+ """Queries the backend for a run; fail if the settings are incompatible."""
+ if not self._settings.resume:
+ return None
+
+ # TODO: This causes a race, we need to make the upsert atomically
+ # only create or update depending on the resume config
+ # we use the runs entity if set, otherwise fallback to users entity
+ # todo: ensure entity is not None as self._entity is Optional[str]
+ entity = run.entity or self._entity
+ logger.info(
+ "checking resume status for %s/%s/%s", entity, run.project, run.run_id
+ )
+ resume_status = self._api.run_resume_status(
+ entity=entity, # type: ignore
+ project_name=run.project,
+ name=run.run_id,
+ )
+
+ if not resume_status:
+ if self._settings.resume == "must":
+ error = wandb_internal_pb2.ErrorInfo()
+ error.code = wandb_internal_pb2.ErrorInfo.ErrorCode.USAGE
+ error.message = (
+ "You provided an invalid value for the `resume` argument."
+ f" The value 'must' is not a valid option for resuming a run ({run.run_id}) that does not exist."
+ " Please check your inputs and try again with a valid run ID."
+ " If you are trying to start a new run, please omit the `resume` argument or use `resume='allow'`."
+ )
+ return error
+ return None
+
+ #
+ # handle cases where we have resume_status
+ #
+ if self._settings.resume == "never":
+ error = wandb_internal_pb2.ErrorInfo()
+ error.code = wandb_internal_pb2.ErrorInfo.ErrorCode.USAGE
+ error.message = (
+ "You provided an invalid value for the `resume` argument."
+ f" The value 'never' is not a valid option for resuming a run ({run.run_id}) that already exists."
+ " Please check your inputs and try again with a valid value for the `resume` argument."
+ )
+ return error
+
+ history = {}
+ events = {}
+ config = {}
+ summary = {}
+ try:
+ events_rt = 0
+ history_rt = 0
+ history = json.loads(resume_status["historyTail"])
+ if history:
+ history = json.loads(history[-1])
+ history_rt = history.get("_runtime", 0)
+ events = json.loads(resume_status["eventsTail"])
+ if events:
+ events = json.loads(events[-1])
+ events_rt = events.get("_runtime", 0)
+ config = json.loads(resume_status["config"] or "{}")
+ summary = json.loads(resume_status["summaryMetrics"] or "{}")
+ new_runtime = summary.get("_wandb", {}).get("runtime", None)
+ if new_runtime is not None:
+ self._resume_state.wandb_runtime = new_runtime
+ tags = resume_status.get("tags") or []
+
+ except (IndexError, ValueError) as e:
+ logger.error("unable to load resume tails", exc_info=e)
+ if self._settings.resume == "must":
+ error = wandb_internal_pb2.ErrorInfo()
+ error.code = wandb_internal_pb2.ErrorInfo.ErrorCode.USAGE
+ error.message = "resume='must' but could not resume (%s) " % run.run_id
+ return error
+
+ # TODO: Do we need to restore config / summary?
+ # System metrics runtime is usually greater than history
+ self._resume_state.runtime = max(events_rt, history_rt)
+ last_step = history.get("_step", 0)
+ history_line_count = resume_status["historyLineCount"]
+ self._resume_state.step = last_step + 1 if history_line_count > 0 else last_step
+ self._resume_state.history = history_line_count
+ self._resume_state.events = resume_status["eventsLineCount"]
+ self._resume_state.output = resume_status["logLineCount"]
+ self._resume_state.config = config
+ self._resume_state.summary = summary
+ self._resume_state.tags = tags
+ self._resume_state.resumed = True
+ logger.info("configured resuming with: %s" % self._resume_state)
+ return None
+
+ def _telemetry_get_framework(self) -> str:
+ """Get telemetry data for internal config structure."""
+ # detect framework by checking what is loaded
+ imports: telemetry.TelemetryImports
+ if self._telemetry_obj.HasField("imports_finish"):
+ imports = self._telemetry_obj.imports_finish
+ elif self._telemetry_obj.HasField("imports_init"):
+ imports = self._telemetry_obj.imports_init
+ else:
+ return ""
+ framework = next(
+ (n for f, n in _framework_priority() if getattr(imports, f, False)), ""
+ )
+ return framework
+
+ def _config_telemetry_update(self, config_dict: Dict[str, Any]) -> None:
+ """Add legacy telemetry to config object."""
+ wandb_key = "_wandb"
+ config_dict.setdefault(wandb_key, dict())
+ s: str
+ b: bool
+ s = self._telemetry_obj.python_version
+ if s:
+ config_dict[wandb_key]["python_version"] = s
+ s = self._telemetry_obj.cli_version
+ if s:
+ config_dict[wandb_key]["cli_version"] = s
+ s = self._telemetry_get_framework()
+ if s:
+ config_dict[wandb_key]["framework"] = s
+ s = self._telemetry_obj.huggingface_version
+ if s:
+ config_dict[wandb_key]["huggingface_version"] = s
+ b = self._telemetry_obj.env.jupyter
+ config_dict[wandb_key]["is_jupyter_run"] = b
+ b = self._telemetry_obj.env.kaggle
+ config_dict[wandb_key]["is_kaggle_kernel"] = b
+
+ config_dict[wandb_key]["start_time"] = self._start_time
+
+ t: Dict[int, Any] = proto_util.proto_encode_to_dict(self._telemetry_obj)
+ config_dict[wandb_key]["t"] = t
+
+ def _config_metric_update(self, config_dict: Dict[str, Any]) -> None:
+ """Add default xaxis to config."""
+ if not self._config_metric_pbdict_list:
+ return
+ wandb_key = "_wandb"
+ config_dict.setdefault(wandb_key, dict())
+ config_dict[wandb_key]["m"] = self._config_metric_pbdict_list
+
+ def _config_format(self, config_data: Optional[DictNoValues]) -> DictWithValues:
+ """Format dict into value dict with telemetry info."""
+ config_dict: Dict[str, Any] = config_data.copy() if config_data else dict()
+ self._config_telemetry_update(config_dict)
+ self._config_metric_update(config_dict)
+ config_value_dict: DictWithValues = config_util.dict_add_value_dict(config_dict)
+ return config_value_dict
+
+ def _config_save(self, config_value_dict: DictWithValues) -> None:
+ config_path = os.path.join(self._settings.files_dir, "config.yaml")
+ config_util.save_config_file_from_dict(config_path, config_value_dict)
+
+ def _sync_spell(self) -> None:
+ """Sync this run with spell."""
+ if not self._run:
+ return
+ try:
+ env = os.environ
+ self._interface.publish_config(
+ key=("_wandb", "spell_url"), val=env.get("SPELL_RUN_URL")
+ )
+ url = "{}/{}/{}/runs/{}".format(
+ self._api.app_url, self._run.entity, self._run.project, self._run.run_id
+ )
+ requests.put(
+ env.get("SPELL_API_URL", "https://api.spell.run") + "/wandb_url",
+ json={"access_token": env.get("WANDB_ACCESS_TOKEN"), "url": url},
+ timeout=2,
+ )
+ except requests.RequestException:
+ pass
+ # TODO: do something if sync spell is not successful?
+
+ def send_run(self, record: "Record", file_dir: Optional[str] = None) -> None:
+ run = record.run
+ error = None
+ is_wandb_init = self._run is None
+
+ # save start time of a run
+ self._start_time = run.start_time.ToMicroseconds() / 1e6
+
+ # update telemetry
+ if run.telemetry:
+ self._telemetry_obj.MergeFrom(run.telemetry)
+ if self._settings._sync:
+ self._telemetry_obj.feature.sync = True
+
+ # build config dict
+ config_value_dict: Optional[DictWithValues] = None
+ if run.config:
+ config_util.update_from_proto(self._consolidated_config, run.config)
+ config_value_dict = self._config_format(self._consolidated_config)
+ self._config_save(config_value_dict)
+
+ if is_wandb_init:
+ # Ensure we have a project to query for status
+ if run.project == "":
+ run.project = util.auto_project_name(self._settings.program)
+ # Only check resume status on `wandb.init`
+ error = self._maybe_setup_resume(run)
+
+ if error is not None:
+ if record.control.req_resp or record.control.mailbox_slot:
+ result = proto_util._result_from_record(record)
+ result.run_result.run.CopyFrom(run)
+ result.run_result.error.CopyFrom(error)
+ self._respond_result(result)
+ else:
+ logger.error("Got error in async mode: %s", error.message)
+ return
+
+ # Save the resumed config
+ if self._resume_state.config is not None:
+ # TODO: should we merge this with resumed config?
+ config_override = self._consolidated_config
+ config_dict = self._resume_state.config
+ config_dict = config_util.dict_strip_value_dict(config_dict)
+ config_dict.update(config_override)
+ self._consolidated_config.update(config_dict)
+ config_value_dict = self._config_format(self._consolidated_config)
+ self._config_save(config_value_dict)
+
+ # handle empty config
+ # TODO(jhr): consolidate the 4 ways config is built:
+ # (passed config, empty config, resume config, send_config)
+ if not config_value_dict:
+ config_value_dict = self._config_format(None)
+ self._config_save(config_value_dict)
+
+ try:
+ self._init_run(run, config_value_dict)
+ except (CommError, UsageError) as e:
+ logger.error(e, exc_info=True)
+ if record.control.req_resp or record.control.mailbox_slot:
+ result = proto_util._result_from_record(record)
+ result.run_result.run.CopyFrom(run)
+ error = ProtobufErrorHandler.from_exception(e)
+ result.run_result.error.CopyFrom(error)
+ self._respond_result(result)
+ return
+
+ assert self._run # self._run is configured in _init_run()
+
+ if record.control.req_resp or record.control.mailbox_slot:
+ result = proto_util._result_from_record(record)
+ # TODO: we could do self._interface.publish_defer(resp) to notify
+ # the handler not to actually perform server updates for this uuid
+ # because the user process will send a summary update when we resume
+ result.run_result.run.CopyFrom(self._run)
+ self._respond_result(result)
+
+ # Only spin up our threads on the first run message
+ if is_wandb_init:
+ self._start_run_threads(file_dir)
+ else:
+ logger.info("updated run: %s", self._run.run_id)
+
+ def _init_run(
+ self,
+ run: "RunRecord",
+ config_dict: Optional[DictWithValues],
+ ) -> None:
+ # We subtract the previous runs runtime when resuming
+ start_time = (
+ run.start_time.ToMicroseconds() / 1e6
+ ) - self._resume_state.runtime
+ # TODO: we don't check inserted currently, ultimately we should make
+ # the upsert know the resume state and fail transactionally
+
+ if self._resume_state and self._resume_state.tags and not run.tags:
+ run.tags.extend(self._resume_state.tags)
+
+ server_run, inserted, server_messages = self._api.upsert_run(
+ name=run.run_id,
+ entity=run.entity or None,
+ project=run.project or None,
+ group=run.run_group or None,
+ job_type=run.job_type or None,
+ display_name=run.display_name or None,
+ notes=run.notes or None,
+ tags=run.tags[:] or None,
+ config=config_dict or None,
+ sweep_name=run.sweep_id or None,
+ host=run.host or None,
+ program_path=self._settings.program or None,
+ repo=run.git.remote_url or None,
+ commit=run.git.commit or None,
+ )
+ # TODO: we don't want to create jobs in sweeps, since the
+ # executable doesn't appear to be consistent
+ if run.sweep_id:
+ self._job_builder.disable = True
+
+ self._server_messages = server_messages or []
+ self._run = run
+ if self._resume_state.resumed:
+ self._run.resumed = True
+ if self._resume_state.wandb_runtime is not None:
+ self._run.runtime = self._resume_state.wandb_runtime
+ else:
+ # If the user is not resuming, and we didn't insert on upsert_run then
+ # it is likely that we are overwriting the run which we might want to
+ # prevent in the future. This could be a false signal since an upsert_run
+ # message which gets retried in the network could also show up as not
+ # inserted.
+ if not inserted:
+ # no need to flush this, it will get updated eventually
+ self._telemetry_obj.feature.maybe_run_overwrite = True
+ self._run.starting_step = self._resume_state.step
+ self._run.start_time.FromMicroseconds(int(start_time * 1e6))
+ self._run.config.CopyFrom(self._interface._make_config(config_dict))
+ if self._resume_state.summary is not None:
+ self._run.summary.CopyFrom(
+ self._interface._make_summary_from_dict(self._resume_state.summary)
+ )
+ storage_id = server_run.get("id")
+ if storage_id:
+ self._run.storage_id = storage_id
+ id = server_run.get("name")
+ if id:
+ self._api.set_current_run_id(id)
+ display_name = server_run.get("displayName")
+ if display_name:
+ self._run.display_name = display_name
+ project = server_run.get("project")
+ # TODO: remove self._api.set_settings, and make self._project a property?
+ if project:
+ project_name = project.get("name")
+ if project_name:
+ self._run.project = project_name
+ self._project = project_name
+ self._api_settings["project"] = project_name
+ self._api.set_setting("project", project_name)
+ entity = project.get("entity")
+ if entity:
+ entity_name = entity.get("name")
+ if entity_name:
+ self._run.entity = entity_name
+ self._entity = entity_name
+ self._api_settings["entity"] = entity_name
+ self._api.set_setting("entity", entity_name)
+ sweep_id = server_run.get("sweepName")
+ if sweep_id:
+ self._run.sweep_id = sweep_id
+ if os.getenv("SPELL_RUN_URL"):
+ self._sync_spell()
+
+ def _start_run_threads(self, file_dir: Optional[str] = None) -> None:
+ assert self._run # self._run is configured by caller
+ self._fs = file_stream.FileStreamApi(
+ self._api,
+ self._run.run_id,
+ self._run.start_time.ToMicroseconds() / 1e6,
+ timeout=self._settings._file_stream_timeout_seconds,
+ settings=self._api_settings,
+ )
+ # Ensure the streaming polices have the proper offsets
+ self._fs.set_file_policy("wandb-summary.json", file_stream.SummaryFilePolicy())
+ self._fs.set_file_policy(
+ "wandb-history.jsonl",
+ file_stream.JsonlFilePolicy(start_chunk_id=self._resume_state.history),
+ )
+ self._fs.set_file_policy(
+ "wandb-events.jsonl",
+ file_stream.JsonlFilePolicy(start_chunk_id=self._resume_state.events),
+ )
+ self._fs.set_file_policy(
+ "output.log",
+ file_stream.CRDedupeFilePolicy(start_chunk_id=self._resume_state.output),
+ )
+
+ # hack to merge run_settings and self._settings object together
+ # so that fields like entity or project are available to be attached to Sentry events.
+ run_settings = message_to_dict(self._run)
+ _settings = dict(self._settings)
+ _settings.update(run_settings)
+ wandb._sentry.configure_scope(tags=_settings, process_context="internal")
+
+ self._fs.start()
+ self._pusher = FilePusher(self._api, self._fs, settings=self._settings)
+ self._dir_watcher = DirWatcher(self._settings, self._pusher, file_dir)
+ logger.info(
+ "run started: %s with start time %s",
+ self._run.run_id,
+ self._run.start_time.ToMicroseconds() / 1e6,
+ )
+
+ def _save_history(self, history_dict: Dict[str, Any]) -> None:
+ if self._fs:
+ self._fs.push(filenames.HISTORY_FNAME, json.dumps(history_dict))
+
+ def send_history(self, record: "Record") -> None:
+ history = record.history
+ history_dict = proto_util.dict_from_proto_list(history.item)
+ self._save_history(history_dict)
+
+ def _update_summary_record(self, summary: "SummaryRecord") -> None:
+ summary_dict = proto_util.dict_from_proto_list(summary.update)
+ self._cached_summary = summary_dict
+ self._update_summary()
+
+ def send_summary(self, record: "Record") -> None:
+ self._update_summary_record(record.summary)
+
+ def send_request_summary_record(self, record: "Record") -> None:
+ self._update_summary_record(record.request.summary_record.summary)
+
+ def _update_summary(self) -> None:
+ summary_dict = self._cached_summary.copy()
+ summary_dict.pop("_wandb", None)
+ if self._metadata_summary:
+ summary_dict["_wandb"] = self._metadata_summary
+ # merge with consolidated summary
+ self._consolidated_summary.update(summary_dict)
+ json_summary = json.dumps(self._consolidated_summary)
+ if self._fs:
+ self._fs.push(filenames.SUMMARY_FNAME, json_summary)
+ # TODO(jhr): we should only write this at the end of the script
+ summary_path = os.path.join(self._settings.files_dir, filenames.SUMMARY_FNAME)
+ with open(summary_path, "w") as f:
+ f.write(json_summary)
+ self._save_file(interface.GlobStr(filenames.SUMMARY_FNAME))
+
+ def send_stats(self, record: "Record") -> None:
+ stats = record.stats
+ if stats.stats_type != wandb_internal_pb2.StatsRecord.StatsType.SYSTEM:
+ return
+ if not self._fs:
+ return
+ if not self._run:
+ return
+ now_us = stats.timestamp.ToMicroseconds()
+ start_us = self._run.start_time.ToMicroseconds()
+ d = dict()
+ for item in stats.item:
+ d[item.key] = json.loads(item.value_json)
+ row: Dict[str, Any] = dict(system=d)
+ self._flatten(row)
+ row["_wandb"] = True
+ row["_timestamp"] = now_us / 1e6
+ row["_runtime"] = (now_us - start_us) / 1e6
+ self._fs.push(filenames.EVENTS_FNAME, json.dumps(row))
+ # TODO(jhr): check fs.push results?
+
+ def _output_raw_finish(self) -> None:
+ for stream, output_raw in self._output_raw_streams.items():
+ output_raw._stopped.set()
+
+ # shut down threads
+ output_raw._writer_thr.join(timeout=5)
+ if output_raw._writer_thr.is_alive():
+ logger.info("processing output...")
+ output_raw._writer_thr.join()
+ output_raw._reader_thr.join()
+
+ # flush output buffers and files
+ self._output_raw_flush(stream)
+ self._output_raw_streams = {}
+ if self._output_raw_file:
+ self._output_raw_file.close()
+ self._output_raw_file = None
+
+ def _output_raw_writer_thread(self, stream: "StreamLiterals") -> None:
+ while True:
+ output_raw = self._output_raw_streams[stream]
+ if output_raw._queue.empty():
+ if output_raw._stopped.is_set():
+ return
+ time.sleep(0.5)
+ continue
+ data = []
+ while not output_raw._queue.empty():
+ data.append(output_raw._queue.get())
+ if output_raw._stopped.is_set() and sum(map(len, data)) > 100000:
+ logger.warning("Terminal output too large. Logging without processing.")
+ self._output_raw_flush(stream)
+ for line in data:
+ self._output_raw_flush(stream, line)
+ # TODO: lets mark that this happened in telemetry
+ return
+ try:
+ output_raw._emulator.write("".join(data))
+ except Exception as e:
+ logger.warning(f"problem writing to output_raw emulator: {e}")
+
+ def _output_raw_reader_thread(self, stream: "StreamLiterals") -> None:
+ output_raw = self._output_raw_streams[stream]
+ while not (output_raw._stopped.is_set() and output_raw._queue.empty()):
+ self._output_raw_flush(stream)
+ time.sleep(_OUTPUT_MIN_CALLBACK_INTERVAL)
+
+ def _output_raw_flush(
+ self, stream: "StreamLiterals", data: Optional[str] = None
+ ) -> None:
+ if data is None:
+ output_raw = self._output_raw_streams[stream]
+ try:
+ data = output_raw._emulator.read()
+ except Exception as e:
+ logger.warning(f"problem reading from output_raw emulator: {e}")
+ if data:
+ self._send_output_line(stream, data)
+ if self._output_raw_file:
+ self._output_raw_file.write(data.encode("utf-8"))
+
+ def send_request_python_packages(self, record: "Record") -> None:
+ import os
+
+ from wandb.sdk.lib.filenames import REQUIREMENTS_FNAME
+
+ installed_packages_list = sorted(
+ f"{r.name}=={r.version}" for r in record.request.python_packages.package
+ )
+ with open(os.path.join(self._settings.files_dir, REQUIREMENTS_FNAME), "w") as f:
+ f.write("\n".join(installed_packages_list))
+
+ def send_output(self, record: "Record") -> None:
+ if not self._fs:
+ return
+ out = record.output
+ stream: StreamLiterals = "stdout"
+ if out.output_type == wandb_internal_pb2.OutputRecord.OutputType.STDERR:
+ stream = "stderr"
+ line = out.line
+ self._send_output_line(stream, line)
+
+ def send_output_raw(self, record: "Record") -> None:
+ if not self._fs:
+ return
+ out = record.output_raw
+ stream: StreamLiterals = "stdout"
+ if out.output_type == wandb_internal_pb2.OutputRawRecord.OutputType.STDERR:
+ stream = "stderr"
+ line = out.line
+
+ output_raw = self._output_raw_streams.get(stream)
+ if not output_raw:
+ output_raw = _OutputRawStream(stream=stream, sm=self)
+ self._output_raw_streams[stream] = output_raw
+
+ # open the console output file shared between both streams
+ if not self._output_raw_file:
+ output_log_path = os.path.join(
+ self._settings.files_dir, filenames.OUTPUT_FNAME
+ )
+ output_raw_file = None
+ try:
+ output_raw_file = filesystem.CRDedupedFile(
+ open(output_log_path, "wb")
+ )
+ except OSError as e:
+ logger.warning(f"could not open output_raw_file: {e}")
+ if output_raw_file:
+ self._output_raw_file = output_raw_file
+ output_raw.start()
+
+ output_raw._queue.put(line)
+
+ def _send_output_line(self, stream: "StreamLiterals", line: str) -> None:
+ """Combined writer for raw and non raw output lines.
+
+ This is combined because they are both post emulator.
+ """
+ prepend = ""
+ if stream == "stderr":
+ prepend = "ERROR "
+ if not line.endswith("\n"):
+ self._partial_output.setdefault(stream, "")
+ if line.startswith("\r"):
+ # TODO: maybe we shouldnt just drop this, what if there was some \ns in the partial
+ # that should probably be the check instead of not line.endswith(\n")
+ # logger.info(f"Dropping data {self._partial_output[stream]}")
+ self._partial_output[stream] = ""
+ self._partial_output[stream] += line
+ # TODO(jhr): how do we make sure this gets flushed?
+ # we might need this for other stuff like telemetry
+ else:
+ # TODO(jhr): use time from timestamp proto
+ # TODO(jhr): do we need to make sure we write full lines?
+ # seems to be some issues with line breaks
+ cur_time = time.time()
+ timestamp = datetime.utcfromtimestamp(cur_time).isoformat() + " "
+ prev_str = self._partial_output.get(stream, "")
+ line = f"{prepend}{timestamp}{prev_str}{line}"
+ if self._fs:
+ self._fs.push(filenames.OUTPUT_FNAME, line)
+ self._partial_output[stream] = ""
+
+ def _update_config(self) -> None:
+ self._config_needs_debounce = True
+
+ def send_config(self, record: "Record") -> None:
+ cfg = record.config
+ config_util.update_from_proto(self._consolidated_config, cfg)
+ self._update_config()
+
+ def send_metric(self, record: "Record") -> None:
+ metric = record.metric
+ if metric.glob_name:
+ logger.warning("Seen metric with glob (shouldn't happen)")
+ return
+
+ # merge or overwrite
+ old_metric = self._config_metric_dict.get(
+ metric.name, wandb_internal_pb2.MetricRecord()
+ )
+ if metric._control.overwrite:
+ old_metric.CopyFrom(metric)
+ else:
+ old_metric.MergeFrom(metric)
+ self._config_metric_dict[metric.name] = old_metric
+ metric = old_metric
+
+ # convert step_metric to index
+ if metric.step_metric:
+ find_step_idx = self._config_metric_index_dict.get(metric.step_metric)
+ if find_step_idx is not None:
+ # make a copy of this metric as we will be modifying it
+ rec = wandb_internal_pb2.Record()
+ rec.metric.CopyFrom(metric)
+ metric = rec.metric
+
+ metric.ClearField("step_metric")
+ metric.step_metric_index = find_step_idx + 1
+
+ md: Dict[int, Any] = proto_util.proto_encode_to_dict(metric)
+ find_idx = self._config_metric_index_dict.get(metric.name)
+ if find_idx is not None:
+ self._config_metric_pbdict_list[find_idx] = md
+ else:
+ next_idx = len(self._config_metric_pbdict_list)
+ self._config_metric_pbdict_list.append(md)
+ self._config_metric_index_dict[metric.name] = next_idx
+ self._update_config()
+
+ def _update_telemetry_record(self, telemetry: telemetry.TelemetryRecord) -> None:
+ self._telemetry_obj.MergeFrom(telemetry)
+ self._update_config()
+
+ def send_telemetry(self, record: "Record") -> None:
+ self._update_telemetry_record(record.telemetry)
+
+ def send_request_telemetry_record(self, record: "Record") -> None:
+ self._update_telemetry_record(record.request.telemetry_record.telemetry)
+
+ def _save_file(
+ self, fname: interface.GlobStr, policy: "interface.PolicyName" = "end"
+ ) -> None:
+ logger.info("saving file %s with policy %s", fname, policy)
+ if self._dir_watcher:
+ self._dir_watcher.update_policy(fname, policy)
+
+ def send_files(self, record: "Record") -> None:
+ files = record.files
+ for k in files.files:
+ # TODO(jhr): fix paths with directories
+ self._save_file(
+ interface.GlobStr(k.path), interface.file_enum_to_policy(k.policy)
+ )
+
+ def send_header(self, record: "Record") -> None:
+ pass
+
+ def send_footer(self, record: "Record") -> None:
+ pass
+
+ def send_tbrecord(self, record: "Record") -> None:
+ # tbrecord watching threads are handled by handler.py
+ pass
+
+ def send_link_artifact(self, record: "Record") -> None:
+ link = record.link_artifact
+ client_id = link.client_id
+ server_id = link.server_id
+ portfolio_name = link.portfolio_name
+ entity = link.portfolio_entity
+ project = link.portfolio_project
+ aliases = link.portfolio_aliases
+ logger.debug(
+ f"link_artifact params - client_id={client_id}, server_id={server_id}, pfolio={portfolio_name}, entity={entity}, project={project}"
+ )
+ if (client_id or server_id) and portfolio_name and entity and project:
+ try:
+ self._api.link_artifact(
+ client_id, server_id, portfolio_name, entity, project, aliases
+ )
+ except Exception as e:
+ logger.warning("Failed to link artifact to portfolio: %s", e)
+
+ def send_use_artifact(self, record: "Record") -> None:
+ """Pretend to send a used artifact.
+
+ This function doesn't actually send anything, it is just used internally.
+ """
+ use = record.use_artifact
+
+ if use.type == "job" and not use.partial.job_name:
+ self._job_builder.disable = True
+ elif use.partial.job_name:
+ # job is partial, let job builder rebuild job, set job source dict
+ self._job_builder._partial_source = (
+ job_builder.convert_use_artifact_to_job_source(record.use_artifact)
+ )
+
+ def send_request_log_artifact(self, record: "Record") -> None:
+ assert record.control.mailbox_slot
+ result = proto_util._result_from_record(record)
+ artifact = record.request.log_artifact.artifact
+ history_step = record.request.log_artifact.history_step
+
+ future = None
+ try:
+ res, future = self._send_artifact(artifact, history_step)
+ assert res, "Unable to send artifact"
+ result.response.log_artifact_response.artifact_id = res.get("id", None)
+ logger.info(f"logged artifact {artifact.name} - {res}")
+ except Exception as e:
+ result.response.log_artifact_response.error_message = (
+ f'error logging artifact "{artifact.type}/{artifact.name}": {e}'
+ )
+
+ def _respond_result(fut: concurrent.futures.Future):
+ if fut.exception() is not None:
+ result.response.log_artifact_response.error_message = f'error logging artifact "{artifact.type}/{artifact.name}": {fut.exception()}'
+ self._respond_result(result)
+
+ if future is not None:
+ # respond to the request only after the artifact is fully committed
+ future.add_done_callback(_respond_result)
+ else:
+ self._respond_result(result)
+
+ def send_artifact(self, record: "Record") -> None:
+ artifact = record.artifact
+ try:
+ res, future = self._send_artifact(artifact)
+ # wait for future to complete in send artifact
+ if future is not None:
+ future.result()
+ logger.info(f"sent artifact {artifact.name} - {res}")
+ except Exception as e:
+ logger.error(
+ 'send_artifact: failed for artifact "{}/{}": {}'.format(
+ artifact.type, artifact.name, e
+ )
+ )
+
+ def _send_artifact(
+ self, artifact: "ArtifactRecord", history_step: Optional[int] = None
+ ) -> Tuple[Dict, Optional[concurrent.futures.Future]]:
+ from pkg_resources import parse_version
+
+ assert self._pusher
+ saver = ArtifactSaver(
+ api=self._api,
+ digest=artifact.digest,
+ manifest_json=_manifest_json_from_proto(artifact.manifest),
+ file_pusher=self._pusher,
+ is_user_created=artifact.user_created,
+ )
+
+ if artifact.distributed_id:
+ max_cli_version = self._max_cli_version()
+ if max_cli_version is None or parse_version(
+ max_cli_version
+ ) < parse_version("0.10.16"):
+ logger.warning(
+ "This W&B Server doesn't support distributed artifacts, "
+ "have your administrator install wandb/local >= 0.9.37"
+ )
+ return {}, None
+
+ metadata = json.loads(artifact.metadata) if artifact.metadata else None
+ res, future = saver.save(
+ type=artifact.type,
+ name=artifact.name,
+ client_id=artifact.client_id,
+ sequence_client_id=artifact.sequence_client_id,
+ metadata=metadata,
+ ttl_duration_seconds=artifact.ttl_duration_seconds or None,
+ description=artifact.description or None,
+ aliases=artifact.aliases,
+ use_after_commit=artifact.use_after_commit,
+ distributed_id=artifact.distributed_id,
+ finalize=artifact.finalize,
+ incremental=artifact.incremental_beta1,
+ history_step=history_step,
+ base_id=artifact.base_id or None,
+ )
+
+ self._job_builder._handle_server_artifact(res, artifact)
+ return res, future
+
+ def send_alert(self, record: "Record") -> None:
+ from pkg_resources import parse_version
+
+ alert = record.alert
+ max_cli_version = self._max_cli_version()
+ if max_cli_version is None or parse_version(max_cli_version) < parse_version(
+ "0.10.9"
+ ):
+ logger.warning(
+ "This W&B server doesn't support alerts, "
+ "have your administrator install wandb/local >= 0.9.31"
+ )
+ else:
+ try:
+ self._api.notify_scriptable_run_alert(
+ title=alert.title,
+ text=alert.text,
+ level=alert.level,
+ wait_duration=alert.wait_duration,
+ )
+ except Exception as e:
+ logger.error(f"send_alert: failed for alert {alert.title!r}: {e}")
+
+ def finish(self) -> None:
+ logger.info("shutting down sender")
+ # if self._tb_watcher:
+ # self._tb_watcher.finish()
+ self._output_raw_finish()
+ if self._dir_watcher:
+ self._dir_watcher.finish()
+ self._dir_watcher = None
+ if self._pusher:
+ self._pusher.finish()
+ self._pusher.join()
+ self._pusher = None
+ if self._fs:
+ self._fs.finish(self._exit_code)
+ self._fs = None
+ wandb._sentry.end_session()
+
+ def _max_cli_version(self) -> Optional[str]:
+ server_info = self.get_server_info()
+ max_cli_version = server_info.get("cliVersionInfo", {}).get(
+ "max_cli_version", None
+ )
+ if not isinstance(max_cli_version, str):
+ return None
+ return max_cli_version
+
+ def get_viewer_server_info(self) -> None:
+ if self._cached_server_info and self._cached_viewer:
+ return
+ self._cached_viewer, self._cached_server_info = self._api.viewer_server_info()
+
+ def get_viewer_info(self) -> Dict[str, Any]:
+ if not self._cached_viewer:
+ self.get_viewer_server_info()
+ return self._cached_viewer
+
+ def get_server_info(self) -> Dict[str, Any]:
+ if not self._cached_server_info:
+ self.get_viewer_server_info()
+ return self._cached_server_info
+
+ def get_local_info(self) -> "LocalInfo":
+ """Queries the server to get the local version information.
+
+ First, we perform an introspection, if it returns empty we deduce that the
+ docker image is out-of-date. Otherwise, we use the returned values to deduce the
+ state of the local server.
+ """
+ local_info = wandb_internal_pb2.LocalInfo()
+ if self._settings._offline:
+ local_info.out_of_date = False
+ return local_info
+
+ latest_local_version = "latest"
+
+ # Assuming the query is successful if the result is empty it indicates that
+ # the backend is out of date since it doesn't have the desired field
+ server_info = self.get_server_info()
+ latest_local_version_info = server_info.get("latestLocalVersionInfo", {})
+ if latest_local_version_info is None:
+ local_info.out_of_date = False
+ else:
+ local_info.out_of_date = latest_local_version_info.get("outOfDate", True)
+ local_info.version = latest_local_version_info.get(
+ "latestVersionString", latest_local_version
+ )
+ return local_info
+
+ def _flush_job(self) -> None:
+ if self._job_builder.disable or self._settings._offline:
+ return
+ self._job_builder.set_config(
+ {k: v for k, v in self._consolidated_config.items() if k != "_wandb"}
+ )
+ summary_dict = self._cached_summary.copy()
+ summary_dict.pop("_wandb", None)
+ self._job_builder.set_summary(summary_dict)
+ artifact = self._job_builder.build()
+ if artifact is not None and self._run is not None:
+ proto_artifact = self._interface._make_artifact(artifact)
+ proto_artifact.run_id = self._run.run_id
+ proto_artifact.project = self._run.project
+ proto_artifact.entity = self._run.entity
+ # TODO: this should be removed when the latest tag is handled
+ # by the backend (WB-12116)
+ proto_artifact.aliases.append("latest")
+ # add docker image tag
+ for alias in self._job_builder._aliases:
+ proto_artifact.aliases.append(alias)
+
+ proto_artifact.user_created = True
+ proto_artifact.use_after_commit = True
+ proto_artifact.finalize = True
+
+ self._interface._publish_artifact(proto_artifact)
+
+ def __next__(self) -> "Record":
+ return self._record_q.get(block=True)
+
+ next = __next__
diff --git a/wandb/sdk/internal/settings_static.py b/wandb/sdk/internal/settings_static.py
new file mode 100644
index 0000000000000000000000000000000000000000..5d71018c69868e98ee94b05632d1f534d4545fc9
--- /dev/null
+++ b/wandb/sdk/internal/settings_static.py
@@ -0,0 +1,74 @@
+from dataclasses import fields
+from typing import Any, Iterable, Sequence, Tuple
+
+from wandb.proto import wandb_settings_pb2
+from wandb.sdk.wandb_settings import SettingsData
+
+
+class SettingsStatic(SettingsData):
+ """A readonly object that wraps a protobuf Settings message.
+
+ Implements the mapping protocol, so you can access settings as
+ attributes or items.
+ """
+
+ def __init__(self, proto: wandb_settings_pb2.Settings) -> None:
+ self._from_proto(proto)
+ object.__setattr__(self, "_proto", proto)
+
+ def _from_proto(self, proto: wandb_settings_pb2.Settings) -> None:
+ for field in fields(SettingsData):
+ key = field.name
+ value: Any = None
+ if key == "_stats_open_metrics_filters":
+ # todo: it's an underscored field, refactor into
+ # something more elegant?
+ # I'm really about this. It's ugly, but it works.
+ # Do not try to repeat this at home.
+ value_type = getattr(proto, key).WhichOneof("value")
+ if value_type == "sequence":
+ value = list(getattr(proto, key).sequence.value)
+ elif value_type == "mapping":
+ unpacked_mapping = {}
+ for outer_key, outer_value in getattr(
+ proto, key
+ ).mapping.value.items():
+ unpacked_inner = {}
+ for inner_key, inner_value in outer_value.value.items():
+ unpacked_inner[inner_key] = inner_value
+ unpacked_mapping[outer_key] = unpacked_inner
+ value = unpacked_mapping
+ else:
+ if proto.HasField(key): # type: ignore [arg-type]
+ value = getattr(proto, key).value
+ if field.type == Sequence[str]:
+ value = list(value)
+ elif field.type == Tuple[str]:
+ value = tuple(value)
+ else:
+ value = None
+ object.__setattr__(self, key, value)
+
+ def __setattr__(self, name: str, value: object) -> None:
+ raise AttributeError("Error: SettingsStatic is a readonly object")
+
+ def __setitem__(self, key: str, val: object) -> None:
+ raise AttributeError("Error: SettingsStatic is a readonly object")
+
+ def keys(self) -> "Iterable[str]":
+ return self.__dict__.keys()
+
+ def __getitem__(self, key: str) -> Any:
+ return self.__dict__[key]
+
+ def __getattr__(self, name: str) -> Any:
+ try:
+ return self.__dict__[name]
+ except KeyError:
+ raise AttributeError(f"SettingsStatic has no attribute {name}")
+
+ def __str__(self) -> str:
+ return str(self.__dict__)
+
+ def __contains__(self, key: str) -> bool:
+ return key in self.__dict__
diff --git a/wandb/sdk/internal/system/__init__.py b/wandb/sdk/internal/system/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sdk/internal/system/assets/__init__.py b/wandb/sdk/internal/system/assets/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..3c0ce23bbaa338ac2da29ec342de1c54185ed62a
--- /dev/null
+++ b/wandb/sdk/internal/system/assets/__init__.py
@@ -0,0 +1,27 @@
+__all__ = (
+ "asset_registry",
+ "CPU",
+ "Disk",
+ "GPU",
+ "GPUAMD",
+ "GPUApple",
+ "IPU",
+ "Memory",
+ "Network",
+ "OpenMetrics",
+ "TPU",
+ "Trainium",
+)
+
+from .asset_registry import asset_registry
+from .cpu import CPU
+from .disk import Disk
+from .gpu import GPU
+from .gpu_amd import GPUAMD
+from .gpu_apple import GPUApple
+from .ipu import IPU
+from .memory import Memory
+from .network import Network
+from .open_metrics import OpenMetrics
+from .tpu import TPU
+from .trainium import Trainium
diff --git a/wandb/sdk/internal/system/assets/aggregators.py b/wandb/sdk/internal/system/assets/aggregators.py
new file mode 100644
index 0000000000000000000000000000000000000000..a292e34c6b3374fbf57e774cbfc3bfec1a329fbd
--- /dev/null
+++ b/wandb/sdk/internal/system/assets/aggregators.py
@@ -0,0 +1,37 @@
+import sys
+from typing import Union
+
+if sys.version_info >= (3, 9):
+ from collections.abc import Sequence
+else:
+ from typing import Sequence
+
+Number = Union[int, float]
+
+
+def aggregate_mean(samples: Sequence[Number], precision: int = 2) -> float:
+ return round(sum(samples) / len(samples), precision)
+
+
+def aggregate_last(samples: Sequence[Number], precision: int = 2) -> Union[float, int]:
+ if isinstance(samples[-1], int):
+ return samples[-1]
+ return round(samples[-1], precision)
+
+
+def aggregate_max(samples: Sequence[Number], precision: int = 2) -> Union[float, int]:
+ if isinstance(samples[-1], int):
+ return max(samples)
+ return round(max(samples), precision)
+
+
+def aggregate_min(samples: Sequence[Number], precision: int = 2) -> Union[float, int]:
+ if isinstance(samples[-1], int):
+ return min(samples)
+ return round(min(samples), precision)
+
+
+def aggregate_sum(samples: Sequence[Number], precision: int = 2) -> Union[float, int]:
+ if isinstance(samples[-1], int):
+ return sum(samples)
+ return round(sum(samples), precision)
diff --git a/wandb/sdk/internal/system/assets/asset_registry.py b/wandb/sdk/internal/system/assets/asset_registry.py
new file mode 100644
index 0000000000000000000000000000000000000000..4114adadd72465efeb0ac37fc393fa5feb09c471
--- /dev/null
+++ b/wandb/sdk/internal/system/assets/asset_registry.py
@@ -0,0 +1,20 @@
+from typing import Iterator, List, Type
+
+from .interfaces import Asset
+
+
+class AssetRegistry:
+ def __init__(self) -> None:
+ self._registry: List[Type[Asset]] = []
+
+ def register(self, asset: Type[Asset]) -> Type[Asset]:
+ self._registry.append(asset)
+ return asset
+
+ def __iter__(self) -> Iterator[Type[Asset]]:
+ for asset in self._registry:
+ if asset.is_available():
+ yield asset
+
+
+asset_registry = AssetRegistry()
diff --git a/wandb/sdk/internal/system/assets/cpu.py b/wandb/sdk/internal/system/assets/cpu.py
new file mode 100644
index 0000000000000000000000000000000000000000..02d3dbbca4d789b138bb136614f4c37a5beb0aec
--- /dev/null
+++ b/wandb/sdk/internal/system/assets/cpu.py
@@ -0,0 +1,163 @@
+import threading
+from collections import deque
+from typing import TYPE_CHECKING, List, Optional
+
+try:
+ import psutil
+except ImportError:
+ psutil = None
+from .aggregators import aggregate_last, aggregate_mean
+from .asset_registry import asset_registry
+from .interfaces import Interface, Metric, MetricsMonitor
+
+if TYPE_CHECKING:
+ from typing import Deque
+
+ from wandb.sdk.internal.settings_static import SettingsStatic
+
+
+# CPU Metrics
+
+
+class ProcessCpuPercent:
+ """CPU usage of the process in percent normalized by the number of CPUs."""
+
+ # name = "process_cpu_percent"
+ name = "cpu"
+
+ def __init__(self, pid: int) -> None:
+ self.pid = pid
+ self.samples: Deque[float] = deque([])
+ self.process: Optional[psutil.Process] = None
+
+ def sample(self) -> None:
+ # todo: this is what we'd eventually want to do
+ # self.samples.append(
+ # (
+ # datetime.datetime.utcnow(),
+ # psutil.Process(self.pid).cpu_percent(),
+ # )
+ # )
+ if self.process is None:
+ self.process = psutil.Process(self.pid)
+
+ self.samples.append(self.process.cpu_percent() / psutil.cpu_count())
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ # todo: create a statistics class with helper methods to compute
+ # mean, median, min, max, etc.
+ if not self.samples:
+ return {}
+ aggregate = aggregate_mean(self.samples)
+ return {self.name: aggregate}
+
+
+class CpuPercent:
+ """CPU usage of the system in percent per core."""
+
+ name = "cpu.{i}.cpu_percent"
+
+ def __init__(self, interval: Optional[float] = None) -> None:
+ self.samples: Deque[List[float]] = deque([])
+ self.interval = interval
+
+ def sample(self) -> None:
+ self.samples.append(psutil.cpu_percent(interval=self.interval, percpu=True))
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ num_cpu = len(self.samples[0])
+ cpu_metrics = {}
+ for i in range(num_cpu):
+ aggregate_i = aggregate_mean([sample[i] for sample in self.samples])
+ cpu_metrics[self.name.format(i=i)] = aggregate_i
+
+ return cpu_metrics
+
+
+class ProcessCpuThreads:
+ """Number of threads used by the process."""
+
+ name = "proc.cpu.threads"
+
+ def __init__(self, pid: int) -> None:
+ self.samples: Deque[int] = deque([])
+ self.pid = pid
+ self.process: Optional[psutil.Process] = None
+
+ def sample(self) -> None:
+ if self.process is None:
+ self.process = psutil.Process(self.pid)
+
+ self.samples.append(self.process.num_threads())
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ return {self.name: aggregate_last(self.samples)}
+
+
+@asset_registry.register
+class CPU:
+ def __init__(
+ self,
+ interface: "Interface",
+ settings: "SettingsStatic",
+ shutdown_event: threading.Event,
+ ) -> None:
+ self.name: str = self.__class__.__name__.lower()
+ self.metrics: List[Metric] = [
+ ProcessCpuPercent(settings._stats_pid),
+ CpuPercent(),
+ ProcessCpuThreads(settings._stats_pid),
+ ]
+ self.metrics_monitor: MetricsMonitor = MetricsMonitor(
+ self.name,
+ self.metrics,
+ interface,
+ settings,
+ shutdown_event,
+ )
+
+ @classmethod
+ def is_available(cls) -> bool:
+ return psutil is not None
+
+ def probe(self) -> dict:
+ asset_info = {
+ "cpu_count": psutil.cpu_count(logical=False),
+ "cpu_count_logical": psutil.cpu_count(logical=True),
+ }
+ try:
+ asset_info["cpu_freq"] = {
+ "current": psutil.cpu_freq().current,
+ "min": psutil.cpu_freq().min,
+ "max": psutil.cpu_freq().max,
+ }
+ asset_info["cpu_freq_per_core"] = [
+ {
+ "current": freq.current,
+ "min": freq.min,
+ "max": freq.max,
+ }
+ for freq in psutil.cpu_freq(percpu=True)
+ ]
+ except Exception:
+ pass
+ return asset_info
+
+ def start(self) -> None:
+ self.metrics_monitor.start()
+
+ def finish(self) -> None:
+ self.metrics_monitor.finish()
diff --git a/wandb/sdk/internal/system/assets/disk.py b/wandb/sdk/internal/system/assets/disk.py
new file mode 100644
index 0000000000000000000000000000000000000000..80f1cb12b5e5077298bae2124b7457c1fe20c66f
--- /dev/null
+++ b/wandb/sdk/internal/system/assets/disk.py
@@ -0,0 +1,210 @@
+import threading
+from collections import deque
+from typing import TYPE_CHECKING, List, Optional
+
+try:
+ import psutil
+except ImportError:
+ psutil = None
+
+from wandb.errors.term import termwarn
+
+from .aggregators import aggregate_mean
+from .asset_registry import asset_registry
+from .interfaces import Interface, Metric, MetricsMonitor
+
+if TYPE_CHECKING:
+ from typing import Deque
+
+ from wandb.sdk.internal.settings_static import SettingsStatic
+
+
+class DiskUsagePercent:
+ """Total system disk usage in percent."""
+
+ name = "disk.{path}.usagePercent"
+ samples: "Deque[List[float]]"
+
+ def __init__(self, paths: List[str]) -> None:
+ self.samples = deque([])
+ # check if we have access to the disk paths:
+ self.paths: List[str] = []
+ for path in paths:
+ try:
+ psutil.disk_usage(path)
+ self.paths.append(path)
+ except Exception as e: # noqa
+ termwarn(f"Could not access disk path {path}: {e}", repeat=False)
+
+ def sample(self) -> None:
+ # self.samples.append(psutil.disk_usage("/").percent)
+ disk_usage: List[float] = []
+ for path in self.paths:
+ disk_usage.append(psutil.disk_usage(path).percent)
+ if disk_usage:
+ self.samples.append(disk_usage)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ disk_metrics = {}
+ for i, _path in enumerate(self.paths):
+ aggregate_i = aggregate_mean([sample[i] for sample in self.samples])
+ # ugly hack to please the frontend:
+ _path = _path.replace("/", "\\")
+ disk_metrics[self.name.format(path=_path)] = aggregate_i
+
+ return disk_metrics
+
+
+class DiskUsage:
+ """Total system disk usage in GB."""
+
+ name = "disk.{path}.usageGB"
+ samples: "Deque[List[float]]"
+
+ def __init__(self, paths: List[str]) -> None:
+ self.samples = deque([])
+ # check if we have access to the disk paths:
+ self.paths: List[str] = []
+ for path in paths:
+ try:
+ psutil.disk_usage(path)
+ self.paths.append(path)
+ except Exception as e: # noqa
+ termwarn(f"Could not access disk path {path}: {e}", repeat=False)
+
+ def sample(self) -> None:
+ disk_usage: List[float] = []
+ for path in self.paths:
+ disk_usage.append(psutil.disk_usage(path).used / 1024 / 1024 / 1024)
+ if disk_usage:
+ self.samples.append(disk_usage)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ disk_metrics = {}
+ for i, _path in enumerate(self.paths):
+ aggregate_i = aggregate_mean([sample[i] for sample in self.samples])
+ # ugly hack to please the frontend:
+ _path = _path.replace("/", "\\")
+ disk_metrics[self.name.format(path=_path)] = aggregate_i
+
+ return disk_metrics
+
+
+class DiskIn:
+ """Total system disk read in MB."""
+
+ name = "disk.in"
+ samples: "Deque[float]"
+
+ def __init__(self) -> None:
+ self.samples = deque([])
+ self.read_init: Optional[int] = None
+
+ def sample(self) -> None:
+ if self.read_init is None:
+ # initialize the read_init value on first sample
+ self.read_init = psutil.disk_io_counters().read_bytes
+ self.samples.append(
+ (psutil.disk_io_counters().read_bytes - self.read_init) / 1024 / 1024
+ )
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ aggregate = aggregate_mean(self.samples)
+ return {self.name: aggregate}
+
+
+class DiskOut:
+ """Total system disk write in MB."""
+
+ name = "disk.out"
+ samples: "Deque[float]"
+
+ def __init__(self) -> None:
+ self.samples = deque([])
+ self.write_init: Optional[int] = None
+
+ def sample(self) -> None:
+ if self.write_init is None:
+ # init on first sample
+ self.write_init = psutil.disk_io_counters().write_bytes
+ self.samples.append(
+ (psutil.disk_io_counters().write_bytes - self.write_init) / 1024 / 1024
+ )
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ aggregate = aggregate_mean(self.samples)
+ return {self.name: aggregate}
+
+
+@asset_registry.register
+class Disk:
+ def __init__(
+ self,
+ interface: "Interface",
+ settings: "SettingsStatic",
+ shutdown_event: threading.Event,
+ ) -> None:
+ self.name = self.__class__.__name__.lower()
+ self.settings = settings
+ self.metrics: List[Metric] = [
+ DiskUsagePercent(list(settings._stats_disk_paths or ["/"])),
+ DiskUsage(list(settings._stats_disk_paths or ["/"])),
+ DiskIn(),
+ DiskOut(),
+ ]
+ self.metrics_monitor = MetricsMonitor(
+ self.name,
+ self.metrics,
+ interface,
+ settings,
+ shutdown_event,
+ )
+
+ @classmethod
+ def is_available(cls) -> bool:
+ """Return a new instance of the CPU metrics."""
+ return psutil is not None
+
+ def probe(self) -> dict:
+ disk_paths = list(self.settings._stats_disk_paths or ["/"])
+ disk_metrics = {}
+ for disk_path in disk_paths:
+ try:
+ # total disk space in GB:
+ total = psutil.disk_usage(disk_path).total / 1024 / 1024 / 1024
+ # total disk space used in GB:
+ used = psutil.disk_usage(disk_path).used / 1024 / 1024 / 1024
+ disk_metrics[disk_path] = {
+ "total": total,
+ "used": used,
+ }
+ except Exception as e: # noqa
+ termwarn(f"Could not access disk path {disk_path}: {e}", repeat=False)
+
+ return {self.name: disk_metrics}
+
+ def start(self) -> None:
+ self.metrics_monitor.start()
+
+ def finish(self) -> None:
+ self.metrics_monitor.finish()
diff --git a/wandb/sdk/internal/system/assets/gpu.py b/wandb/sdk/internal/system/assets/gpu.py
new file mode 100644
index 0000000000000000000000000000000000000000..3c7a98782fff423974f53d6f515ebea7c4d40769
--- /dev/null
+++ b/wandb/sdk/internal/system/assets/gpu.py
@@ -0,0 +1,414 @@
+import logging
+import threading
+from collections import deque
+from typing import TYPE_CHECKING, List
+
+try:
+ import psutil
+except ImportError:
+ psutil = None
+
+from wandb.vendor.pynvml import pynvml
+
+from .aggregators import aggregate_mean
+from .asset_registry import asset_registry
+from .interfaces import Interface, Metric, MetricsMonitor
+
+if TYPE_CHECKING:
+ from typing import Deque
+
+ from wandb.sdk.internal.settings_static import SettingsStatic
+
+ GPUHandle = object
+
+
+logger = logging.getLogger(__name__)
+
+
+def gpu_in_use_by_this_process(gpu_handle: "GPUHandle", pid: int) -> bool:
+ if psutil is None:
+ return False
+
+ try:
+ base_process = psutil.Process(pid=pid)
+ except psutil.NoSuchProcess:
+ # do not report any gpu metrics if the base process cant be found
+ return False
+
+ our_processes = base_process.children(recursive=True)
+ our_processes.append(base_process)
+
+ our_pids = {process.pid for process in our_processes}
+
+ compute_pids = {
+ process.pid
+ for process in pynvml.nvmlDeviceGetComputeRunningProcesses(gpu_handle) # type: ignore
+ }
+ graphics_pids = {
+ process.pid
+ for process in pynvml.nvmlDeviceGetGraphicsRunningProcesses(gpu_handle) # type: ignore
+ }
+
+ pids_using_device = compute_pids | graphics_pids
+
+ return len(pids_using_device & our_pids) > 0
+
+
+class GPUMemoryUtilization:
+ """GPU memory utilization in percent for each GPU."""
+
+ # name = "memory_utilization"
+ name = "gpu.{}.memory"
+ # samples: Deque[Tuple[datetime.datetime, float]]
+ samples: "Deque[List[float]]"
+
+ def __init__(self, pid: int) -> None:
+ self.pid = pid
+ self.samples = deque([])
+
+ def sample(self) -> None:
+ memory_utilization_rate = []
+ device_count = pynvml.nvmlDeviceGetCount() # type: ignore
+ for i in range(device_count):
+ handle = pynvml.nvmlDeviceGetHandleByIndex(i) # type: ignore
+ memory_utilization_rate.append(
+ pynvml.nvmlDeviceGetUtilizationRates(handle).memory # type: ignore
+ )
+ self.samples.append(memory_utilization_rate)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ stats = {}
+ device_count = pynvml.nvmlDeviceGetCount() # type: ignore
+ for i in range(device_count):
+ samples = [sample[i] for sample in self.samples]
+ aggregate = aggregate_mean(samples)
+ stats[self.name.format(i)] = aggregate
+
+ handle = pynvml.nvmlDeviceGetHandleByIndex(i) # type: ignore
+ if gpu_in_use_by_this_process(handle, self.pid):
+ stats[self.name.format(f"process.{i}")] = aggregate
+
+ return stats
+
+
+class GPUMemoryAllocated:
+ """GPU memory allocated in percent for each GPU."""
+
+ # name = "memory_allocated"
+ name = "gpu.{}.memoryAllocated"
+ # samples: Deque[Tuple[datetime.datetime, float]]
+ samples: "Deque[List[float]]"
+
+ def __init__(self, pid: int) -> None:
+ self.pid = pid
+ self.samples = deque([])
+
+ def sample(self) -> None:
+ memory_allocated = []
+ device_count = pynvml.nvmlDeviceGetCount() # type: ignore
+ for i in range(device_count):
+ handle = pynvml.nvmlDeviceGetHandleByIndex(i) # type: ignore
+ memory_info = pynvml.nvmlDeviceGetMemoryInfo(handle) # type: ignore
+ memory_allocated.append(memory_info.used / memory_info.total * 100)
+ self.samples.append(memory_allocated)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ stats = {}
+ device_count = pynvml.nvmlDeviceGetCount() # type: ignore
+ for i in range(device_count):
+ samples = [sample[i] for sample in self.samples]
+ aggregate = aggregate_mean(samples)
+ stats[self.name.format(i)] = aggregate
+
+ handle = pynvml.nvmlDeviceGetHandleByIndex(i) # type: ignore
+ if gpu_in_use_by_this_process(handle, self.pid):
+ stats[self.name.format(f"process.{i}")] = aggregate
+
+ return stats
+
+
+class GPUMemoryAllocatedBytes:
+ """GPU memory allocated in bytes for each GPU."""
+
+ # name = "memory_allocated"
+ name = "gpu.{}.memoryAllocatedBytes"
+ # samples: Deque[Tuple[datetime.datetime, float]]
+ samples: "Deque[List[float]]"
+
+ def __init__(self, pid: int) -> None:
+ self.pid = pid
+ self.samples = deque([])
+
+ def sample(self) -> None:
+ memory_allocated = []
+ device_count = pynvml.nvmlDeviceGetCount() # type: ignore
+ for i in range(device_count):
+ handle = pynvml.nvmlDeviceGetHandleByIndex(i) # type: ignore
+ memory_info = pynvml.nvmlDeviceGetMemoryInfo(handle) # type: ignore
+ memory_allocated.append(memory_info.used)
+ self.samples.append(memory_allocated)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ stats = {}
+ device_count = pynvml.nvmlDeviceGetCount() # type: ignore
+ for i in range(device_count):
+ samples = [sample[i] for sample in self.samples]
+ aggregate = aggregate_mean(samples)
+ stats[self.name.format(i)] = aggregate
+
+ handle = pynvml.nvmlDeviceGetHandleByIndex(i) # type: ignore
+ if gpu_in_use_by_this_process(handle, self.pid):
+ stats[self.name.format(f"process.{i}")] = aggregate
+
+ return stats
+
+
+class GPUUtilization:
+ """GPU utilization in percent for each GPU."""
+
+ # name = "gpu_utilization"
+ name = "gpu.{}.gpu"
+ # samples: Deque[Tuple[datetime.datetime, float]]
+ samples: "Deque[List[float]]"
+
+ def __init__(self, pid: int) -> None:
+ self.pid = pid
+ self.samples = deque([])
+
+ def sample(self) -> None:
+ gpu_utilization_rate = []
+ device_count = pynvml.nvmlDeviceGetCount() # type: ignore
+ for i in range(device_count):
+ handle = pynvml.nvmlDeviceGetHandleByIndex(i) # type: ignore
+ gpu_utilization_rate.append(
+ pynvml.nvmlDeviceGetUtilizationRates(handle).gpu # type: ignore
+ )
+ self.samples.append(gpu_utilization_rate)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ stats = {}
+ device_count = pynvml.nvmlDeviceGetCount() # type: ignore
+ for i in range(device_count):
+ samples = [sample[i] for sample in self.samples]
+ aggregate = aggregate_mean(samples)
+ stats[self.name.format(i)] = aggregate
+
+ handle = pynvml.nvmlDeviceGetHandleByIndex(i) # type: ignore
+ if gpu_in_use_by_this_process(handle, self.pid):
+ stats[self.name.format(f"process.{i}")] = aggregate
+
+ return stats
+
+
+class GPUTemperature:
+ """GPU temperature in Celsius for each GPU."""
+
+ # name = "gpu_temperature"
+ name = "gpu.{}.temp"
+ # samples: Deque[Tuple[datetime.datetime, float]]
+ samples: "Deque[List[float]]"
+
+ def __init__(self, pid: int) -> None:
+ self.pid = pid
+ self.samples = deque([])
+
+ def sample(self) -> None:
+ temperature = []
+ device_count = pynvml.nvmlDeviceGetCount() # type: ignore
+ for i in range(device_count):
+ handle = pynvml.nvmlDeviceGetHandleByIndex(i) # type: ignore
+ temperature.append(
+ pynvml.nvmlDeviceGetTemperature( # type: ignore
+ handle,
+ pynvml.NVML_TEMPERATURE_GPU,
+ )
+ )
+ self.samples.append(temperature)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ stats = {}
+ device_count = pynvml.nvmlDeviceGetCount() # type: ignore
+ for i in range(device_count):
+ samples = [sample[i] for sample in self.samples]
+ aggregate = aggregate_mean(samples)
+ stats[self.name.format(i)] = aggregate
+
+ handle = pynvml.nvmlDeviceGetHandleByIndex(i) # type: ignore
+ if gpu_in_use_by_this_process(handle, self.pid):
+ stats[self.name.format(f"process.{i}")] = aggregate
+
+ return stats
+
+
+class GPUPowerUsageWatts:
+ """GPU power usage in Watts for each GPU."""
+
+ name = "gpu.{}.powerWatts"
+ # samples: Deque[Tuple[datetime.datetime, float]]
+ samples: "Deque[List[float]]"
+
+ def __init__(self, pid: int) -> None:
+ self.pid = pid
+ self.samples = deque([])
+
+ def sample(self) -> None:
+ power_usage = []
+ device_count = pynvml.nvmlDeviceGetCount() # type: ignore
+ for i in range(device_count):
+ handle = pynvml.nvmlDeviceGetHandleByIndex(i) # type: ignore
+ power_watts = pynvml.nvmlDeviceGetPowerUsage(handle) / 1000 # type: ignore
+ power_usage.append(power_watts)
+ self.samples.append(power_usage)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ stats = {}
+ device_count = pynvml.nvmlDeviceGetCount() # type: ignore
+ for i in range(device_count):
+ samples = [sample[i] for sample in self.samples]
+ aggregate = aggregate_mean(samples)
+ stats[self.name.format(i)] = aggregate
+
+ handle = pynvml.nvmlDeviceGetHandleByIndex(i) # type: ignore
+ if gpu_in_use_by_this_process(handle, self.pid):
+ stats[self.name.format(f"process.{i}")] = aggregate
+
+ return stats
+
+
+class GPUPowerUsagePercent:
+ """GPU power usage in percent for each GPU."""
+
+ name = "gpu.{}.powerPercent"
+ # samples: Deque[Tuple[datetime.datetime, float]]
+ samples: "Deque[List[float]]"
+
+ def __init__(self, pid: int) -> None:
+ self.pid = pid
+ self.samples = deque([])
+
+ def sample(self) -> None:
+ power_usage = []
+ device_count = pynvml.nvmlDeviceGetCount() # type: ignore
+ for i in range(device_count):
+ handle = pynvml.nvmlDeviceGetHandleByIndex(i) # type: ignore
+ power_watts = pynvml.nvmlDeviceGetPowerUsage(handle) # type: ignore
+ power_capacity_watts = pynvml.nvmlDeviceGetEnforcedPowerLimit(handle) # type: ignore
+ power_usage.append((power_watts / power_capacity_watts) * 100)
+ self.samples.append(power_usage)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ stats = {}
+ device_count = pynvml.nvmlDeviceGetCount() # type: ignore
+ for i in range(device_count):
+ samples = [sample[i] for sample in self.samples]
+ aggregate = aggregate_mean(samples)
+ stats[self.name.format(i)] = aggregate
+
+ handle = pynvml.nvmlDeviceGetHandleByIndex(i) # type: ignore
+ if gpu_in_use_by_this_process(handle, self.pid):
+ stats[self.name.format(f"process.{i}")] = aggregate
+
+ return stats
+
+
+@asset_registry.register
+class GPU:
+ def __init__(
+ self,
+ interface: "Interface",
+ settings: "SettingsStatic",
+ shutdown_event: threading.Event,
+ ) -> None:
+ self.name = self.__class__.__name__.lower()
+ self.metrics: List[Metric] = [
+ GPUMemoryAllocated(settings._stats_pid),
+ GPUMemoryAllocatedBytes(settings._stats_pid),
+ GPUMemoryUtilization(settings._stats_pid),
+ GPUUtilization(settings._stats_pid),
+ GPUTemperature(settings._stats_pid),
+ GPUPowerUsageWatts(settings._stats_pid),
+ GPUPowerUsagePercent(settings._stats_pid),
+ ]
+ self.metrics_monitor = MetricsMonitor(
+ self.name,
+ self.metrics,
+ interface,
+ settings,
+ shutdown_event,
+ )
+
+ @classmethod
+ def is_available(cls) -> bool:
+ try:
+ pynvml.nvmlInit() # type: ignore
+ return True
+ except pynvml.NVMLError_LibraryNotFound: # type: ignore
+ return False
+ except Exception as e:
+ logger.error(f"Error initializing NVML: {e}")
+ return False
+
+ def start(self) -> None:
+ self.metrics_monitor.start()
+
+ def finish(self) -> None:
+ self.metrics_monitor.finish()
+
+ def probe(self) -> dict:
+ info = {}
+ try:
+ pynvml.nvmlInit() # type: ignore
+ # todo: this is an adapter for the legacy stats system:
+ info["gpu"] = pynvml.nvmlDeviceGetName(pynvml.nvmlDeviceGetHandleByIndex(0)) # type: ignore
+ info["gpu_count"] = pynvml.nvmlDeviceGetCount() # type: ignore
+
+ device_count = pynvml.nvmlDeviceGetCount() # type: ignore
+ devices = []
+ for i in range(device_count):
+ handle = pynvml.nvmlDeviceGetHandleByIndex(i) # type: ignore
+ gpu_info = pynvml.nvmlDeviceGetMemoryInfo(handle) # type: ignore
+ devices.append(
+ {
+ "name": pynvml.nvmlDeviceGetName(handle),
+ "memory_total": gpu_info.total,
+ }
+ )
+ info["gpu_devices"] = devices
+
+ except pynvml.NVMLError:
+ pass
+
+ return info
diff --git a/wandb/sdk/internal/system/assets/gpu_amd.py b/wandb/sdk/internal/system/assets/gpu_amd.py
new file mode 100644
index 0000000000000000000000000000000000000000..d186ca36b6cbde196392bc313c3a15e0a01e3fab
--- /dev/null
+++ b/wandb/sdk/internal/system/assets/gpu_amd.py
@@ -0,0 +1,214 @@
+import json
+import logging
+import shutil
+import subprocess
+import sys
+import threading
+from collections import deque
+from typing import TYPE_CHECKING, Any, Dict, List, Union
+
+if sys.version_info >= (3, 8):
+ from typing import Final, Literal
+else:
+ from typing_extensions import Final, Literal
+
+from wandb.sdk.lib import telemetry
+
+from .aggregators import aggregate_mean
+from .asset_registry import asset_registry
+from .interfaces import Interface, Metric, MetricsMonitor
+
+if TYPE_CHECKING:
+ from typing import Deque
+
+ from wandb.sdk.internal.settings_static import SettingsStatic
+
+
+logger = logging.getLogger(__name__)
+ROCM_SMI_CMD: Final[str] = shutil.which("rocm-smi") or "/usr/bin/rocm-smi"
+
+
+def get_rocm_smi_stats() -> Dict[str, Any]:
+ command = [str(ROCM_SMI_CMD), "-a", "--json"]
+ output = (
+ subprocess.check_output(command, universal_newlines=True).strip().split("\n")
+ )[0]
+ return json.loads(output) # type: ignore
+
+
+_StatsKeys = Literal[
+ "gpu",
+ "memoryAllocated",
+ "temp",
+ "powerWatts",
+ "powerPercent",
+]
+_Stats = Dict[_StatsKeys, float]
+
+
+_InfoDict = Dict[str, Union[int, List[Dict[str, Any]]]]
+
+
+class GPUAMDStats:
+ """Stats for AMD GPU devices."""
+
+ name = "gpu.{gpu_id}.{key}"
+ samples: "Deque[List[_Stats]]"
+
+ def __init__(self) -> None:
+ self.samples = deque()
+
+ @staticmethod
+ def parse_stats(stats: Dict[str, str]) -> _Stats:
+ """Parse stats from rocm-smi output."""
+ parsed_stats: _Stats = {}
+
+ try:
+ parsed_stats["gpu"] = float(stats.get("GPU use (%)")) # type: ignore
+ except (TypeError, ValueError):
+ logger.warning("Could not parse GPU usage as float")
+ try:
+ parsed_stats["memoryAllocated"] = float(stats.get("GPU memory use (%)")) # type: ignore
+ except (TypeError, ValueError):
+ logger.warning("Could not parse GPU memory allocation as float")
+ try:
+ parsed_stats["temp"] = float(stats.get("Temperature (Sensor memory) (C)")) # type: ignore
+ except (TypeError, ValueError):
+ logger.warning("Could not parse GPU temperature as float")
+ try:
+ parsed_stats["powerWatts"] = float(
+ stats.get("Average Graphics Package Power (W)") # type: ignore
+ )
+ except (TypeError, ValueError):
+ logger.warning("Could not parse GPU power as float")
+ try:
+ parsed_stats["powerPercent"] = (
+ float(stats.get("Average Graphics Package Power (W)")) # type: ignore
+ / float(stats.get("Max Graphics Package Power (W)")) # type: ignore
+ * 100
+ )
+ except (TypeError, ValueError):
+ logger.warning("Could not parse GPU average/max power as float")
+
+ return parsed_stats
+
+ def sample(self) -> None:
+ try:
+ raw_stats = get_rocm_smi_stats()
+ cards = []
+
+ card_keys = [
+ key for key in sorted(raw_stats.keys()) if key.startswith("card")
+ ]
+
+ for card_key in card_keys:
+ card_stats = raw_stats[card_key]
+ stats = self.parse_stats(card_stats)
+ if stats:
+ cards.append(stats)
+
+ if cards:
+ self.samples.append(cards)
+
+ except (OSError, ValueError, TypeError, subprocess.CalledProcessError) as e:
+ logger.exception(f"GPU stats error: {e}")
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ stats = {}
+ device_count = len(self.samples[0])
+
+ for i in range(device_count):
+ samples = [sample[i] for sample in self.samples]
+
+ for key in samples[0].keys():
+ samples_key = [s[key] for s in samples]
+ aggregate = aggregate_mean(samples_key)
+ stats[self.name.format(gpu_id=i, key=key)] = aggregate
+
+ return stats
+
+
+@asset_registry.register
+class GPUAMD:
+ """GPUAMD is a class for monitoring AMD GPU devices.
+
+ Uses AMD's rocm_smi tool to get GPU stats.
+ For the list of supported environments and devices, see
+ https://github.com/RadeonOpenCompute/ROCm/blob/develop/docs/deploy/
+ """
+
+ def __init__(
+ self,
+ interface: "Interface",
+ settings: "SettingsStatic",
+ shutdown_event: threading.Event,
+ ) -> None:
+ self.name = self.__class__.__name__.lower()
+ self.metrics: List[Metric] = [
+ GPUAMDStats(),
+ ]
+ self.metrics_monitor = MetricsMonitor(
+ self.name,
+ self.metrics,
+ interface,
+ settings,
+ shutdown_event,
+ )
+ telemetry_record = telemetry.TelemetryRecord()
+ telemetry_record.env.amd_gpu = True
+ interface._publish_telemetry(telemetry_record)
+
+ @classmethod
+ def is_available(cls) -> bool:
+ rocm_smi_available = shutil.which(ROCM_SMI_CMD) is not None
+ if rocm_smi_available:
+ try:
+ _ = get_rocm_smi_stats()
+ return True
+ except Exception:
+ pass
+ return False
+
+ def start(self) -> None:
+ self.metrics_monitor.start()
+
+ def finish(self) -> None:
+ self.metrics_monitor.finish()
+
+ def probe(self) -> dict:
+ info: _InfoDict = {}
+ try:
+ stats = get_rocm_smi_stats()
+
+ info["gpu_count"] = len(
+ [key for key in stats.keys() if key.startswith("card")]
+ )
+ key_mapping = {
+ "id": "GPU ID",
+ "unique_id": "Unique ID",
+ "vbios_version": "VBIOS version",
+ "performance_level": "Performance Level",
+ "gpu_overdrive": "GPU OverDrive value (%)",
+ "gpu_memory_overdrive": "GPU Memory OverDrive value (%)",
+ "max_power": "Max Graphics Package Power (W)",
+ "series": "Card series",
+ "model": "Card model",
+ "vendor": "Card vendor",
+ "sku": "Card SKU",
+ "sclk_range": "Valid sclk range",
+ "mclk_range": "Valid mclk range",
+ }
+
+ info["gpu_devices"] = [
+ {k: stats[key][v] for k, v in key_mapping.items() if stats[key].get(v)}
+ for key in stats.keys()
+ if key.startswith("card")
+ ]
+ except Exception as e:
+ logger.exception(f"GPUAMD probe error: {e}")
+ return info
diff --git a/wandb/sdk/internal/system/assets/gpu_apple.py b/wandb/sdk/internal/system/assets/gpu_apple.py
new file mode 100644
index 0000000000000000000000000000000000000000..75c3c6db1b3bf77550a147090ead1b9a07c2f8ac
--- /dev/null
+++ b/wandb/sdk/internal/system/assets/gpu_apple.py
@@ -0,0 +1,132 @@
+import json
+import logging
+import pathlib
+import platform
+import subprocess
+import sys
+import threading
+from collections import deque
+from typing import TYPE_CHECKING, List
+
+if sys.version_info >= (3, 8):
+ from typing import TypedDict
+else:
+ from typing_extensions import TypedDict
+
+from wandb.sdk.lib import telemetry
+
+from .aggregators import aggregate_mean
+from .asset_registry import asset_registry
+from .interfaces import Interface, Metric, MetricsMonitor
+
+if TYPE_CHECKING:
+ from typing import Deque
+
+ from wandb.sdk.internal.settings_static import SettingsStatic
+
+
+logger = logging.getLogger(__name__)
+
+
+class _Stats(TypedDict):
+ gpu: float
+ memoryAllocated: float # noqa: N815
+ temp: float
+ powerWatts: float # noqa: N815
+ powerPercent: float # noqa: N815
+ # cpuWaitMs: float
+
+
+class GPUAppleStats:
+ """Apple GPU stats available on Arm Macs."""
+
+ name = "gpu.0.{}"
+ samples: "Deque[_Stats]"
+
+ # TODO: hard coded max watts as 16.5, found this number in the SMC list.
+ # Eventually we can have the apple_gpu_stats binary query for this.
+ MAX_POWER_WATTS = 16.5
+
+ def __init__(self) -> None:
+ self.samples = deque()
+ self.binary_path = (
+ pathlib.Path(sys.modules["wandb"].__path__[0]) / "bin" / "apple_gpu_stats"
+ ).resolve()
+
+ def sample(self) -> None:
+ try:
+ command = [str(self.binary_path), "--json"]
+ output = (
+ subprocess.check_output(command, universal_newlines=True)
+ .strip()
+ .split("\n")
+ )[0]
+ raw_stats = json.loads(output)
+
+ stats: _Stats = {
+ "gpu": raw_stats["utilization"],
+ "memoryAllocated": raw_stats["mem_used"],
+ "temp": raw_stats["temperature"],
+ "powerWatts": raw_stats["power"],
+ "powerPercent": (raw_stats["power"] / self.MAX_POWER_WATTS) * 100,
+ # TODO: this stat could be useful eventually, it was consistently
+ # 0 in my experimentation and requires a frontend change
+ # so leaving it out for now.
+ # "cpuWaitMs": raw_stats["cpu_wait_ms"],
+ }
+
+ self.samples.append(stats)
+
+ except (OSError, ValueError, TypeError, subprocess.CalledProcessError) as e:
+ logger.exception(f"GPU stats error: {e}")
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ stats = {}
+ for key in self.samples[0].keys():
+ samples = [s[key] for s in self.samples] # type: ignore
+ aggregate = aggregate_mean(samples)
+ stats[self.name.format(key)] = aggregate
+ return stats
+
+
+@asset_registry.register
+class GPUApple:
+ def __init__(
+ self,
+ interface: "Interface",
+ settings: "SettingsStatic",
+ shutdown_event: threading.Event,
+ ) -> None:
+ self.name = self.__class__.__name__.lower()
+ self.metrics: List[Metric] = [
+ GPUAppleStats(),
+ ]
+ self.metrics_monitor = MetricsMonitor(
+ self.name,
+ self.metrics,
+ interface,
+ settings,
+ shutdown_event,
+ )
+ telemetry_record = telemetry.TelemetryRecord()
+ telemetry_record.env.m1_gpu = True
+ interface._publish_telemetry(telemetry_record)
+
+ @classmethod
+ def is_available(cls) -> bool:
+ return platform.system() == "Darwin" and platform.processor() == "arm"
+
+ def start(self) -> None:
+ self.metrics_monitor.start()
+
+ def finish(self) -> None:
+ self.metrics_monitor.finish()
+
+ def probe(self) -> dict:
+ # todo: make this actually meaningful
+ return {self.name: {"type": "arm", "vendor": "Apple"}}
diff --git a/wandb/sdk/internal/system/assets/interfaces.py b/wandb/sdk/internal/system/assets/interfaces.py
new file mode 100644
index 0000000000000000000000000000000000000000..445ef3dce3af63aebcd58f30406c440e71f90f44
--- /dev/null
+++ b/wandb/sdk/internal/system/assets/interfaces.py
@@ -0,0 +1,209 @@
+import datetime
+import logging
+import sys
+import threading
+from typing import TYPE_CHECKING, Any, List, Optional, TypeVar
+
+if sys.version_info >= (3, 8):
+ from typing import Protocol, runtime_checkable
+else:
+ from typing_extensions import Protocol, runtime_checkable
+
+if TYPE_CHECKING:
+ from typing import Deque
+
+ from wandb.proto.wandb_telemetry_pb2 import TelemetryRecord
+ from wandb.sdk.interface.interface import FilesDict
+ from wandb.sdk.internal.settings_static import SettingsStatic
+
+import psutil
+
+TimeStamp = TypeVar("TimeStamp", bound=datetime.datetime)
+
+
+logger = logging.getLogger(__name__)
+
+
+class Metric(Protocol):
+ """Base protocol for individual metrics."""
+
+ name: str
+ # samples: Sequence[Tuple[TimeStamp, Sample]]
+ samples: "Deque[Any]"
+
+ def sample(self) -> None:
+ """Sample the metric."""
+ ... # pragma: no cover
+
+ def clear(self) -> None:
+ """Clear the samples."""
+ ... # pragma: no cover
+
+ def aggregate(self) -> dict:
+ """Aggregate the samples."""
+ ... # pragma: no cover
+
+
+@runtime_checkable
+class SetupTeardown(Protocol):
+ """Protocol for classes that require setup and teardown."""
+
+ def setup(self) -> None:
+ """Extra setup required for the metric beyond __init__."""
+ ... # pragma: no cover
+
+ def teardown(self) -> None:
+ """Extra teardown required for the metric."""
+ ... # pragma: no cover
+
+
+@runtime_checkable
+class Asset(Protocol):
+ """Base protocol encapsulate everything relating to an "Asset".
+
+ An asset can be CPU, GPU, TPU, Network, I/O etc.
+ """
+
+ name: str
+ metrics: List[Metric]
+ metrics_monitor: "MetricsMonitor"
+
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ ... # pragma: no cover
+
+ @classmethod
+ def is_available(cls) -> bool:
+ """Check if the resource is available."""
+ ... # pragma: no cover
+
+ def start(self) -> None:
+ """Start monitoring the resource."""
+ ... # pragma: no cover
+
+ def finish(self) -> None:
+ """Finish monitoring the resource."""
+ ... # pragma: no cover
+
+ def probe(self) -> dict:
+ """Get static information about the resource."""
+ ... # pragma: no cover
+
+
+class Interface(Protocol):
+ def publish_stats(self, stats: dict) -> None:
+ ... # pragma: no cover
+
+ def _publish_telemetry(self, telemetry: "TelemetryRecord") -> None:
+ ... # pragma: no cover
+
+ def publish_files(self, files_dict: "FilesDict") -> None:
+ ... # pragma: no cover
+
+
+class MetricsMonitor:
+ """Takes care of collecting, sampling, serializing, and publishing a set of metrics."""
+
+ def __init__(
+ self,
+ asset_name: str,
+ metrics: List[Metric],
+ interface: Interface,
+ settings: "SettingsStatic",
+ shutdown_event: threading.Event,
+ ) -> None:
+ self.metrics = metrics
+ self.asset_name = asset_name
+ self._interface = interface
+ self._process: Optional[threading.Thread] = None
+ self._shutdown_event: threading.Event = shutdown_event
+
+ self.sampling_interval: float = float(
+ max(
+ 0.1,
+ settings._stats_sample_rate_seconds,
+ )
+ ) # seconds
+ # The number of samples to aggregate (e.g. average or compute max/min etc.)
+ # before publishing; defaults to 15; valid range: [1:30]
+ self.samples_to_aggregate: int = min(
+ 30, max(1, settings._stats_samples_to_average)
+ )
+
+ def monitor(self) -> None:
+ """Poll the Asset metrics."""
+ while not self._shutdown_event.is_set():
+ for _ in range(self.samples_to_aggregate):
+ for metric in self.metrics:
+ try:
+ metric.sample()
+ except psutil.NoSuchProcess:
+ logger.info(f"Process {metric.name} has exited.")
+ self._shutdown_event.set()
+ break
+ except Exception as e:
+ logger.error(f"Failed to sample metric: {e}")
+ self._shutdown_event.wait(self.sampling_interval)
+ if self._shutdown_event.is_set():
+ break
+ self.publish()
+
+ def aggregate(self) -> dict:
+ """Return a dict of metrics."""
+ aggregated_metrics = {}
+ for metric in self.metrics:
+ try:
+ serialized_metric = metric.aggregate()
+ aggregated_metrics.update(serialized_metric)
+ # aggregated_metrics = wandb.util.merge_dicts(
+ # aggregated_metrics, metric.serialize()
+ # )
+ except Exception as e:
+ logger.error(f"Failed to serialize metric: {e}")
+ return aggregated_metrics
+
+ def publish(self) -> None:
+ """Publish the Asset metrics."""
+ try:
+ aggregated_metrics = self.aggregate()
+ if aggregated_metrics:
+ self._interface.publish_stats(aggregated_metrics)
+ for metric in self.metrics:
+ metric.clear()
+ except Exception as e:
+ logger.error(f"Failed to publish metrics: {e}")
+
+ def start(self) -> None:
+ if (self._process is not None) or self._shutdown_event.is_set():
+ return None
+
+ thread_name = f"{self.asset_name[:15]}" # thread names are limited to 15 chars
+ try:
+ for metric in self.metrics:
+ if isinstance(metric, SetupTeardown):
+ metric.setup()
+ self._process = threading.Thread(
+ target=self.monitor,
+ daemon=True,
+ name=thread_name,
+ )
+ self._process.start()
+ logger.info(f"Started {thread_name} monitoring")
+ except Exception as e:
+ logger.warning(f"Failed to start {thread_name} monitoring: {e}")
+ self._process = None
+
+ def finish(self) -> None:
+ if self._process is None:
+ return None
+
+ thread_name = f"{self.asset_name[:15]}"
+ try:
+ self._process.join()
+ logger.info(f"Joined {thread_name} monitor")
+ for metric in self.metrics:
+ if isinstance(metric, SetupTeardown):
+ metric.teardown()
+ except Exception as e:
+ logger.warning(f"Failed to finish {thread_name} monitoring: {e}")
+ finally:
+ self._process = None
diff --git a/wandb/sdk/internal/system/assets/ipu.py b/wandb/sdk/internal/system/assets/ipu.py
new file mode 100644
index 0000000000000000000000000000000000000000..ecda659b0cb1c071889fc0cd4909e3b8dc5ace1d
--- /dev/null
+++ b/wandb/sdk/internal/system/assets/ipu.py
@@ -0,0 +1,177 @@
+import threading
+from collections import deque
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Union
+
+try:
+ import gcipuinfo # type: ignore
+except ImportError:
+ gcipuinfo = None
+
+import wandb
+
+from .aggregators import aggregate_mean
+from .asset_registry import asset_registry
+from .interfaces import Interface, Metric, MetricsMonitor
+
+if TYPE_CHECKING:
+ from typing import Deque
+
+ from wandb.sdk.internal.settings_static import SettingsStatic
+
+
+class IPUStats:
+ """Stats for Graphcore IPU devices."""
+
+ name = "ipu.{}.{}"
+ samples: "Deque[dict]"
+
+ # The metrics that change over time.
+ # Only these are returned on each invocation
+ # to avoid sending a load of unnecessary data.
+ variable_metric_keys = {
+ "average board temp",
+ "average die temp",
+ "clock",
+ "ipu power",
+ "ipu utilisation",
+ "ipu utilisation (session)",
+ }
+
+ def __init__(self, pid: int, gc_ipu_info: Optional[Any] = None) -> None:
+ self.samples: Deque[dict] = deque()
+
+ if gc_ipu_info is None:
+ if not gcipuinfo:
+ raise ImportError(
+ "Monitoring IPU stats requires gcipuinfo to be installed"
+ )
+
+ self._gc_ipu_info = gcipuinfo.gcipuinfo()
+ else:
+ self._gc_ipu_info = gc_ipu_info
+ self._gc_ipu_info.setUpdateMode(True)
+
+ self._pid = pid
+ self._devices_called: Set[str] = set()
+
+ @staticmethod
+ def parse_metric(key: str, value: str) -> Optional[Tuple[str, Union[int, float]]]:
+ metric_suffixes = {
+ "temp": "C",
+ "clock": "MHz",
+ "power": "W",
+ "utilisation": "%",
+ "utilisation (session)": "%",
+ "speed": "GT/s",
+ }
+
+ for metric, suffix in metric_suffixes.items():
+ if key.endswith(metric) and value.endswith(suffix):
+ value = value[: -len(suffix)]
+ key = f"{key} ({suffix})"
+
+ try:
+ float_value = float(value)
+ num_value = int(float_value) if float_value.is_integer() else float_value
+ except ValueError:
+ return None
+
+ return key, num_value
+
+ def sample(self) -> None:
+ try:
+ stats = {}
+ devices = self._gc_ipu_info.getDevices()
+
+ for device in devices:
+ device_metrics: Dict[str, str] = dict(device)
+
+ pid = device_metrics.get("user process id")
+ if pid is None or int(pid) != self._pid:
+ continue
+
+ device_id = device_metrics.get("id")
+ initial_call = device_id not in self._devices_called
+ if device_id is not None:
+ self._devices_called.add(device_id)
+
+ for key, value in device_metrics.items():
+ log_metric = initial_call or key in self.variable_metric_keys
+ if not log_metric:
+ continue
+ parsed = self.parse_metric(key, value)
+ if parsed is None:
+ continue
+ parsed_key, parsed_value = parsed
+ stats[self.name.format(device_id, parsed_key)] = parsed_value
+
+ self.samples.append(stats)
+
+ except Exception as e:
+ wandb.termwarn(f"IPU stats error {e}", repeat=False)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ stats = {}
+ for key in self.samples[0].keys():
+ samples = [s[key] for s in self.samples if key in s]
+ aggregate = aggregate_mean(samples)
+ stats[key] = aggregate
+ return stats
+
+
+@asset_registry.register
+class IPU:
+ def __init__(
+ self,
+ interface: "Interface",
+ settings: "SettingsStatic",
+ shutdown_event: threading.Event,
+ ) -> None:
+ self.name = self.__class__.__name__.lower()
+ self.metrics: List[Metric] = [
+ IPUStats(settings._stats_pid),
+ ]
+ self.metrics_monitor = MetricsMonitor(
+ self.name,
+ self.metrics,
+ interface,
+ settings,
+ shutdown_event,
+ )
+
+ @classmethod
+ def is_available(cls) -> bool:
+ return gcipuinfo is not None
+
+ def start(self) -> None:
+ self.metrics_monitor.start()
+
+ def finish(self) -> None:
+ self.metrics_monitor.finish()
+
+ def probe(self) -> dict:
+ device_data = self.metrics[0]._gc_ipu_info.getDevices() # type: ignore
+ device_count = len(device_data)
+ devices = []
+ for i, device in enumerate(device_data):
+ device_metrics: Dict[str, str] = dict(device)
+ devices.append(
+ {
+ "id": device_metrics.get("id") or i,
+ "board ipu index": device_metrics.get("board ipu index"),
+ "board type": device_metrics.get("board type") or "unknown",
+ }
+ )
+
+ return {
+ self.name: {
+ "device_count": device_count,
+ "devices": devices,
+ "vendor": "Graphcore",
+ }
+ }
diff --git a/wandb/sdk/internal/system/assets/memory.py b/wandb/sdk/internal/system/assets/memory.py
new file mode 100644
index 0000000000000000000000000000000000000000..d8648eb893f4ba10f3774ecfdc175d09d032194b
--- /dev/null
+++ b/wandb/sdk/internal/system/assets/memory.py
@@ -0,0 +1,166 @@
+import threading
+from collections import deque
+from typing import TYPE_CHECKING, List, Optional
+
+try:
+ import psutil
+except ImportError:
+ psutil = None
+
+from .aggregators import aggregate_mean
+from .asset_registry import asset_registry
+from .interfaces import Interface, Metric, MetricsMonitor
+
+if TYPE_CHECKING:
+ from typing import Deque
+
+ from wandb.sdk.internal.settings_static import SettingsStatic
+
+
+class ProcessMemoryRSS:
+ """Memory resident set size (RSS) in MB.
+
+ RSS is the portion of memory occupied by a process that is held in main memory (RAM).
+ """
+
+ # name = "memory_rss"
+ name = "proc.memory.rssMB"
+ samples: "Deque[float]"
+
+ def __init__(self, pid: int) -> None:
+ self.pid = pid
+ self.process: Optional[psutil.Process] = None
+ self.samples = deque([])
+
+ def sample(self) -> None:
+ if self.process is None:
+ self.process = psutil.Process(self.pid)
+
+ self.samples.append(self.process.memory_info().rss / 1024 / 1024)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ aggregate = aggregate_mean(self.samples)
+ return {self.name: aggregate}
+
+
+class ProcessMemoryPercent:
+ """Process memory usage in percent."""
+
+ # name = "process_memory_percent"
+ name = "proc.memory.percent"
+ samples: "Deque[float]"
+
+ def __init__(self, pid: int) -> None:
+ self.pid = pid
+ self.process: Optional[psutil.Process] = None
+ self.samples = deque([])
+
+ def sample(self) -> None:
+ if self.process is None:
+ self.process = psutil.Process(self.pid)
+
+ self.samples.append(self.process.memory_percent())
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ aggregate = aggregate_mean(self.samples)
+ return {self.name: aggregate}
+
+
+class MemoryPercent:
+ """Total system memory usage in percent."""
+
+ # name = "memory_percent"
+ name = "memory"
+ samples: "Deque[float]"
+
+ def __init__(self) -> None:
+ self.samples = deque([])
+
+ def sample(self) -> None:
+ self.samples.append(psutil.virtual_memory().percent)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ aggregate = aggregate_mean(self.samples)
+ return {self.name: aggregate}
+
+
+class MemoryAvailable:
+ """Total system memory available in MB."""
+
+ # name = "memory_available"
+ name = "proc.memory.availableMB"
+ samples: "Deque[float]"
+
+ def __init__(self) -> None:
+ self.samples = deque([])
+
+ def sample(self) -> None:
+ self.samples.append(psutil.virtual_memory().available / 1024 / 1024)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ aggregate = aggregate_mean(self.samples)
+ return {self.name: aggregate}
+
+
+@asset_registry.register
+class Memory:
+ def __init__(
+ self,
+ interface: "Interface",
+ settings: "SettingsStatic",
+ shutdown_event: threading.Event,
+ ) -> None:
+ self.name = self.__class__.__name__.lower()
+ self.metrics: List[Metric] = [
+ MemoryAvailable(),
+ MemoryPercent(),
+ ProcessMemoryRSS(settings._stats_pid),
+ ProcessMemoryPercent(settings._stats_pid),
+ ]
+ self.metrics_monitor = MetricsMonitor(
+ self.name,
+ self.metrics,
+ interface,
+ settings,
+ shutdown_event,
+ )
+
+ def start(self) -> None:
+ self.metrics_monitor.start()
+
+ def finish(self) -> None:
+ self.metrics_monitor.finish()
+
+ @classmethod
+ def is_available(cls) -> bool:
+ """Return a new instance of the CPU metrics."""
+ return psutil is not None
+
+ def probe(self) -> dict:
+ """Return a dict of the hardware information."""
+ # total available memory in gigabytes
+ return {
+ "memory": {
+ "total": psutil.virtual_memory().total / 1024 / 1024 / 1024,
+ }
+ }
diff --git a/wandb/sdk/internal/system/assets/network.py b/wandb/sdk/internal/system/assets/network.py
new file mode 100644
index 0000000000000000000000000000000000000000..ccb90831f5410a41444c24bd21cd1606d569b488
--- /dev/null
+++ b/wandb/sdk/internal/system/assets/network.py
@@ -0,0 +1,125 @@
+import threading
+from collections import deque
+from typing import TYPE_CHECKING, List
+
+try:
+ import psutil
+except ImportError:
+ psutil = None
+
+from .aggregators import aggregate_mean
+from .asset_registry import asset_registry
+from .interfaces import Interface, Metric, MetricsMonitor
+
+if TYPE_CHECKING:
+ from typing import Deque
+
+ from wandb.sdk.internal.settings_static import SettingsStatic
+
+
+class NetworkSent:
+ """Network bytes sent."""
+
+ name = "network.sent"
+ samples: "Deque[float]"
+
+ def __init__(self) -> None:
+ self.samples = deque([])
+ self.sent_init = psutil.net_io_counters().bytes_sent
+
+ def sample(self) -> None:
+ self.samples.append(psutil.net_io_counters().bytes_sent - self.sent_init)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ aggregate = aggregate_mean(self.samples)
+ # todo: this is an adapter for the legacy metrics system
+ # return {"network": {self.name: aggregate}}
+ return {self.name: aggregate}
+
+
+class NetworkRecv:
+ """Network bytes received."""
+
+ name = "network.recv"
+ samples: "Deque[float]"
+
+ def __init__(self) -> None:
+ self.samples = deque([])
+ self.recv_init = psutil.net_io_counters().bytes_recv
+
+ def sample(self) -> None:
+ self.samples.append(psutil.net_io_counters().bytes_recv - self.recv_init)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ aggregate = aggregate_mean(self.samples)
+ # todo: this is an adapter for the legacy metrics system
+ # return {"network": {self.name: aggregate}}
+
+ return {self.name: aggregate}
+
+
+@asset_registry.register
+class Network:
+ def __init__(
+ self,
+ interface: "Interface",
+ settings: "SettingsStatic",
+ shutdown_event: threading.Event,
+ ) -> None:
+ self.name = self.__class__.__name__.lower()
+ self.metrics: List[Metric] = [
+ NetworkSent(),
+ NetworkRecv(),
+ ]
+ self.metrics_monitor = MetricsMonitor(
+ self.name,
+ self.metrics,
+ interface,
+ settings,
+ shutdown_event,
+ )
+
+ def start(self) -> None:
+ self.metrics_monitor.start()
+
+ def finish(self) -> None:
+ self.metrics_monitor.finish()
+
+ @classmethod
+ def is_available(cls) -> bool:
+ """Return a new instance of the CPU metrics."""
+ return psutil is not None
+
+ def probe(self) -> dict:
+ """Return a dict of the hardware information."""
+ # net_if_addrs = psutil.net_if_addrs()
+
+ # return {
+ # self.name: {
+ # "interfaces": {
+ # k: {
+ # "addresses": [
+ # {
+ # "address": v.address,
+ # "netmask": v.netmask,
+ # "broadcast": v.broadcast,
+ # "ptp": v.ptp,
+ # }
+ # for v in v
+ # ]
+ # }
+ # for k, v in net_if_addrs.items()
+ # }
+ # }
+ # }
+ return {}
diff --git a/wandb/sdk/internal/system/assets/open_metrics.py b/wandb/sdk/internal/system/assets/open_metrics.py
new file mode 100644
index 0000000000000000000000000000000000000000..3b54c7c39cdac803130153059ea120fc1eac5411
--- /dev/null
+++ b/wandb/sdk/internal/system/assets/open_metrics.py
@@ -0,0 +1,299 @@
+import logging
+import re
+import sys
+import threading
+from collections import defaultdict, deque
+from functools import lru_cache
+from types import ModuleType
+from typing import TYPE_CHECKING, Dict, List, Mapping, Sequence, Tuple, Union
+
+if sys.version_info >= (3, 8):
+ from typing import Final
+else:
+ from typing_extensions import Final
+
+import requests
+import requests.adapters
+import urllib3
+
+import wandb
+from wandb.sdk.lib import hashutil, telemetry
+
+from .aggregators import aggregate_last, aggregate_mean
+from .interfaces import Interface, Metric, MetricsMonitor
+
+if TYPE_CHECKING:
+ from typing import Deque, Optional
+
+ from wandb.sdk.internal.settings_static import SettingsStatic
+
+
+_PREFIX: Final[str] = "openmetrics"
+
+_REQUEST_RETRY_STRATEGY = urllib3.util.retry.Retry(
+ backoff_factor=1,
+ total=3,
+ status_forcelist=(408, 409, 429, 500, 502, 503, 504),
+)
+_REQUEST_POOL_CONNECTIONS = 4
+_REQUEST_POOL_MAXSIZE = 4
+_REQUEST_TIMEOUT = 3
+
+
+logger = logging.getLogger(__name__)
+
+
+prometheus_client_parser: "Optional[ModuleType]" = None
+try:
+ import prometheus_client.parser # type: ignore
+
+ prometheus_client_parser = prometheus_client.parser
+except ImportError:
+ pass
+
+
+def _setup_requests_session() -> requests.Session:
+ session = requests.Session()
+ adapter = requests.adapters.HTTPAdapter(
+ max_retries=_REQUEST_RETRY_STRATEGY,
+ pool_connections=_REQUEST_POOL_CONNECTIONS,
+ pool_maxsize=_REQUEST_POOL_MAXSIZE,
+ )
+ session.mount("http://", adapter)
+ session.mount("https://", adapter)
+ return session
+
+
+def _nested_dict_to_tuple(
+ nested_dict: Mapping[str, Mapping[str, str]]
+) -> Tuple[Tuple[str, Tuple[str, str]], ...]:
+ return tuple((k, *v.items()) for k, v in nested_dict.items()) # type: ignore
+
+
+def _tuple_to_nested_dict(
+ nested_tuple: Tuple[Tuple[str, Tuple[str, str]], ...]
+) -> Dict[str, Dict[str, str]]:
+ return {k: dict(v) for k, *v in nested_tuple}
+
+
+@lru_cache(maxsize=128)
+def _should_capture_metric(
+ endpoint_name: str,
+ metric_name: str,
+ metric_labels: Tuple[str, ...],
+ filters: Tuple[Tuple[str, Tuple[str, str]], ...],
+) -> bool:
+ # we use tuples to make the function arguments hashable => usable with lru_cache
+ should_capture = False
+
+ if not filters:
+ return should_capture
+
+ # self.filters keys are regexes, check the name against them
+ # and for the first match, check the labels against the label filters.
+ # assume that if at least one label filter doesn't match, the metric
+ # should not be captured.
+ # it's up to the user to make sure that the filters are not conflicting etc.
+ metric_labels_dict = {t[0]: t[1] for t in metric_labels}
+ filters_dict = _tuple_to_nested_dict(filters)
+ for metric_name_regex, label_filters in filters_dict.items():
+ if not re.match(metric_name_regex, f"{endpoint_name}.{metric_name}"):
+ continue
+
+ should_capture = True
+
+ for label, label_filter in label_filters.items():
+ if not re.match(label_filter, metric_labels_dict.get(label, "")):
+ should_capture = False
+ break
+ break
+
+ return should_capture
+
+
+class OpenMetricsMetric:
+ """Container for all the COUNTER and GAUGE metrics extracted from an OpenMetrics endpoint."""
+
+ def __init__(
+ self,
+ name: str,
+ url: str,
+ filters: Union[Mapping[str, Mapping[str, str]], Sequence[str], None],
+ ) -> None:
+ self.name = name # user-defined name for the endpoint
+ self.url = url # full URL
+
+ # - filters can be a dict {"<metric regex>": {"<label>": "<filter regex>"}}
+ # or a sequence of metric regexes. we convert the latter to a dict
+ # to make it easier to work with.
+ # - the metric regexes are matched against the full metric name,
+ # i.e. "<endpoint name>.<metric name>".
+ # - by default, all metrics are captured.
+ self.filters = (
+ filters
+ if isinstance(filters, Mapping)
+ else {k: {} for k in filters or [".*"]}
+ )
+ self.filters_tuple = _nested_dict_to_tuple(self.filters) if self.filters else ()
+
+ self._session: Optional[requests.Session] = None
+ self.samples: Deque[dict] = deque([])
+ # {"<metric name>": {"<labels hash>": <index>}}
+ self.label_map: Dict[str, Dict[str, int]] = defaultdict(dict)
+ # {"<labels hash>": <labels>}
+ self.label_hashes: Dict[str, dict] = {}
+
+ def setup(self) -> None:
+ if self._session is not None:
+ return
+
+ self._session = _setup_requests_session()
+
+ def teardown(self) -> None:
+ if self._session is None:
+ return
+
+ self._session.close()
+ self._session = None
+
+ def parse_open_metrics_endpoint(self) -> Dict[str, Union[str, int, float]]:
+ assert prometheus_client_parser is not None
+ assert self._session is not None
+
+ response = self._session.get(self.url, timeout=_REQUEST_TIMEOUT)
+ response.raise_for_status()
+
+ text = response.text
+ measurement = {}
+ for family in prometheus_client_parser.text_string_to_metric_families(text):
+ if family.type not in ("counter", "gauge"):
+ # todo: add support for other metric types?
+ # todo: log warning about that?
+ continue
+ for sample in family.samples:
+ name, labels, value = sample.name, sample.labels, sample.value
+
+ if not _should_capture_metric(
+ self.name,
+ name,
+ tuple(labels.items()),
+ self.filters_tuple,
+ ):
+ continue
+
+ # md5 hash of the labels
+ label_hash = hashutil._md5(str(labels).encode("utf-8")).hexdigest()
+ if label_hash not in self.label_map[name]:
+ # store the index of the label hash in the label map
+ self.label_map[name][label_hash] = len(self.label_map[name])
+ # store the labels themselves
+ self.label_hashes[label_hash] = labels
+ index = self.label_map[name][label_hash]
+ measurement[f"{name}.{index}"] = value
+
+ return measurement
+
+ def sample(self) -> None:
+ s = self.parse_open_metrics_endpoint()
+ self.samples.append(s)
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+
+ prefix = f"{_PREFIX}.{self.name}."
+
+ stats = {}
+ for key in self.samples[0].keys():
+ samples = [s[key] for s in self.samples if key in s]
+ if samples and all(isinstance(s, (int, float)) for s in samples):
+ stats[f"{prefix}{key}"] = aggregate_mean(samples)
+ else:
+ stats[f"{prefix}{key}"] = aggregate_last(samples)
+ return stats
+
+
+class OpenMetrics:
+ # Poll an OpenMetrics endpoint, parse the response and return a dict of metrics
+ # Implements the same Protocol interface as Asset
+
+ def __init__(
+ self,
+ interface: "Interface",
+ settings: "SettingsStatic",
+ shutdown_event: threading.Event,
+ name: str,
+ url: str,
+ ) -> None:
+ self.name = name
+ self.url = url
+ self.interface = interface
+ self.settings = settings
+ self.shutdown_event = shutdown_event
+
+ self.metrics: List[Metric] = [
+ OpenMetricsMetric(name, url, settings._stats_open_metrics_filters)
+ ]
+
+ self.metrics_monitor: MetricsMonitor = MetricsMonitor(
+ asset_name=self.name,
+ metrics=self.metrics,
+ interface=interface,
+ settings=settings,
+ shutdown_event=shutdown_event,
+ )
+
+ telemetry_record = telemetry.TelemetryRecord()
+ telemetry_record.feature.open_metrics = True
+ interface._publish_telemetry(telemetry_record)
+
+ @classmethod
+ def is_available(cls, url: str) -> bool:
+ _is_available: bool = False
+
+ ret = prometheus_client_parser is not None
+ if not ret:
+ wandb.termwarn(
+ "Monitoring OpenMetrics endpoints requires the `prometheus_client` package. "
+ "To install it, run `pip install prometheus_client`.",
+ repeat=False,
+ )
+ return _is_available
+ # check if the endpoint is available and is a valid OpenMetrics endpoint
+ _session: Optional[requests.Session] = None
+ try:
+ assert prometheus_client_parser is not None
+ _session = _setup_requests_session()
+ response = _session.get(url, timeout=_REQUEST_TIMEOUT)
+ response.raise_for_status()
+
+ # check if the response is a valid OpenMetrics response
+ # text_string_to_metric_families returns a generator
+ if list(
+ prometheus_client_parser.text_string_to_metric_families(response.text)
+ ):
+ _is_available = True
+ except Exception as e:
+ logger.debug(
+ f"OpenMetrics endpoint {url} is not available: {e}", exc_info=True
+ )
+
+ if _session is not None:
+ try:
+ _session.close()
+ except Exception:
+ pass
+ return _is_available
+
+ def start(self) -> None:
+ self.metrics_monitor.start()
+
+ def finish(self) -> None:
+ self.metrics_monitor.finish()
+
+ def probe(self) -> dict:
+ # todo: also return self.label_hashes
+ return {self.name: self.url}
diff --git a/wandb/sdk/internal/system/assets/tpu.py b/wandb/sdk/internal/system/assets/tpu.py
new file mode 100644
index 0000000000000000000000000000000000000000..5a52b3a135769405ee4bac09b357d4ed6d30fb63
--- /dev/null
+++ b/wandb/sdk/internal/system/assets/tpu.py
@@ -0,0 +1,154 @@
+import logging
+import os
+import threading
+from collections import deque
+from typing import TYPE_CHECKING, List, Optional
+
+from .aggregators import aggregate_mean
+from .asset_registry import asset_registry
+from .interfaces import Interface, Metric, MetricsMonitor
+
+if TYPE_CHECKING:
+ from typing import Deque
+
+ from wandb.sdk.internal.settings_static import SettingsStatic
+
+logger = logging.getLogger(__name__)
+
+
+class TPUUtilization:
+ """Google Cloud TPU utilization in percent."""
+
+ name = "tpu"
+ samples: "Deque[float]"
+
+ def __init__(
+ self,
+ service_addr: str,
+ duration_ms: int = 100,
+ ) -> None:
+ self.samples = deque([])
+
+ self.duration_ms = duration_ms
+ self.service_addr = service_addr
+
+ try:
+ from tensorflow.python.profiler import profiler_client # type: ignore
+
+ self._profiler_client = profiler_client
+ except ImportError:
+ logger.warning(
+ "Unable to import `tensorflow.python.profiler.profiler_client`. "
+ "TPU metrics will not be reported."
+ )
+ self._profiler_client = None
+
+ def sample(self) -> None:
+ result = self._profiler_client.monitor(
+ self.service_addr, duration_ms=self.duration_ms, level=2
+ )
+
+ self.samples.append(
+ float(result.split("Utilization ")[1].split(": ")[1].split("%")[0])
+ )
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+ aggregate = aggregate_mean(self.samples)
+ return {self.name: aggregate}
+
+
+@asset_registry.register
+class TPU:
+ def __init__(
+ self,
+ interface: "Interface",
+ settings: "SettingsStatic",
+ shutdown_event: threading.Event,
+ ) -> None:
+ self.name = self.__class__.__name__.lower()
+ self.service_addr = self.get_service_addr()
+ self.metrics: List[Metric] = [TPUUtilization(self.service_addr)]
+
+ self.metrics_monitor = MetricsMonitor(
+ self.name,
+ self.metrics,
+ interface,
+ settings,
+ shutdown_event,
+ )
+
+ @staticmethod
+ def get_service_addr(
+ service_addr: Optional[str] = None,
+ tpu_name: Optional[str] = None,
+ compute_zone: Optional[str] = None,
+ core_project: Optional[str] = None,
+ ) -> str:
+ if service_addr is not None:
+ if tpu_name is not None:
+ logger.warn(
+ "Both service_addr and tpu_name arguments provided. "
+ "Ignoring tpu_name and using service_addr."
+ )
+ else:
+ tpu_name = tpu_name or os.environ.get("TPU_NAME")
+ if tpu_name is None:
+ raise Exception("Required environment variable TPU_NAME.")
+ compute_zone = compute_zone or os.environ.get("CLOUDSDK_COMPUTE_ZONE")
+ core_project = core_project or os.environ.get("CLOUDSDK_CORE_PROJECT")
+ try:
+ from tensorflow.python.distribute.cluster_resolver import ( # type: ignore
+ tpu_cluster_resolver,
+ )
+
+ service_addr = tpu_cluster_resolver.TPUClusterResolver(
+ [tpu_name], zone=compute_zone, project=core_project
+ ).get_master()
+ except (ValueError, TypeError):
+ raise ValueError(
+ "Failed to find TPU. Try specifying TPU zone "
+ "(via CLOUDSDK_COMPUTE_ZONE environment variable)"
+ " and GCP project (via CLOUDSDK_CORE_PROJECT "
+ "environment variable)."
+ )
+ service_addr = service_addr.replace("grpc://", "").replace(":8470", ":8466")
+ return service_addr
+
+ def start(self) -> None:
+ if self.metrics:
+ self.metrics_monitor.start()
+
+ def finish(self) -> None:
+ self.metrics_monitor.finish()
+
+ @classmethod
+ def is_available(cls) -> bool:
+ if os.environ.get("TPU_NAME", False) is False:
+ return False
+
+ try:
+ from tensorflow.python.distribute.cluster_resolver import ( # noqa: F401
+ tpu_cluster_resolver,
+ )
+ from tensorflow.python.profiler import profiler_client # noqa: F401
+
+ cls.get_service_addr()
+ except (
+ ImportError,
+ TypeError,
+ AttributeError,
+ ValueError,
+ ): # Saw type error when iterating paths on colab...
+ # TODO: Saw error in sentry where module 'tensorflow.python.pywrap_tensorflow'
+ # has no attribute 'TFE_DEVICE_PLACEMENT_EXPLICIT'
+ return False
+
+ return True
+
+ def probe(self) -> dict:
+ return {self.name: {"service_address": self.service_addr}}
diff --git a/wandb/sdk/internal/system/assets/trainium.py b/wandb/sdk/internal/system/assets/trainium.py
new file mode 100644
index 0000000000000000000000000000000000000000..bd868c826cab484613537170c6b2bba7d77d3cba
--- /dev/null
+++ b/wandb/sdk/internal/system/assets/trainium.py
@@ -0,0 +1,400 @@
+import collections
+import dataclasses
+import json
+import logging
+import os
+import pathlib
+import shutil
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+from collections import deque
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+
+if sys.version_info >= (3, 8):
+ from typing import Final
+else:
+ from typing_extensions import Final
+
+from wandb.sdk.lib import telemetry
+
+from .aggregators import aggregate_mean
+from .asset_registry import asset_registry
+from .interfaces import Interface, Metric, MetricsMonitor
+
+if TYPE_CHECKING:
+ from typing import Deque
+
+ from wandb.sdk.internal.settings_static import SettingsStatic
+
+
+logger = logging.getLogger(__name__)
+
+
+NEURON_MONITOR_DEFAULT_CONFIG: Final[dict] = {
+ "period": "1s",
+ "neuron_runtimes": [
+ {
+ "tag_filter": ".*",
+ "metrics": [
+ {"type": "neuroncore_counters"},
+ {"type": "memory_used"},
+ {"type": "neuron_runtime_vcpu_usage"},
+ # {"type": "execution_stats"},
+ ],
+ }
+ ],
+ "system_metrics": [
+ {"type": "vcpu_usage"},
+ {"type": "memory_info"},
+ {"type": "neuron_hw_counters"},
+ ],
+}
+
+# todo: once a python sdk is released with the Neuron utils, rewrite this
+NEURON_LS_COMMAND: Final[Tuple[str, str]] = (
+ shutil.which("neuron-ls") or "/opt/aws/neuron/bin/neuron-ls",
+ "-j",
+)
+NEURON_MONITOR_PATH: Final[str] = (
+ shutil.which("neuron-monitor") or "/opt/aws/neuron/bin/neuron-monitor"
+)
+
+
+@dataclasses.dataclass
+class _NeuronCoreMemoryUsage:
+ constants: int
+ model_code: int
+ model_shared_scratchpad: int
+ runtime_memory: int
+ tensors: int
+
+
+@dataclasses.dataclass
+class _HostMemoryUsage:
+ application_memory: int
+ constants: int
+ dma_buffers: int
+ tensors: int
+
+
+@dataclasses.dataclass
+class _Stats:
+ neuroncore_utilization: Dict[int, float] # per neuron core utilization
+ host_total_memory_usage: int # total memory usage in bytes
+ neuron_device_total_memory_usage: int # total memory usage
+ host_memory_usage: _HostMemoryUsage # host memory usage breakdown
+ neuroncore_memory_usage: Dict[
+ int, _NeuronCoreMemoryUsage
+ ] # per core memory usage breakdown
+
+
+class NeuronCoreStats:
+ """AWS Trainium stats."""
+
+ name: str = "trn.{key}"
+ samples: "Deque[_Stats]"
+
+ def write_neuron_monitor_config(self) -> None:
+ """Write neuron monitor config file."""
+ # mkdir if not exists
+ pathlib.Path(self.neuron_monitor_config_path).parent.mkdir(
+ parents=True, exist_ok=True
+ )
+ # write default config
+ with open(self.neuron_monitor_config_path, "w") as f:
+ json.dump(NEURON_MONITOR_DEFAULT_CONFIG, f, indent=4)
+
+ def neuron_monitor(self) -> None:
+ """Run neuron-monitor in a separate process to collect raw data."""
+ self.write_neuron_monitor_config()
+
+ try:
+ command = [
+ NEURON_MONITOR_PATH,
+ "-c",
+ self.neuron_monitor_config_path,
+ ]
+ with subprocess.Popen(
+ command,
+ stdout=subprocess.PIPE,
+ stderr=None,
+ ) as process:
+ while not self.shutdown_event.is_set():
+ if process.stdout is None:
+ self.shutdown_event.wait(0.1)
+ continue
+
+ raw_data = process.stdout.readline()
+ if raw_data:
+ self.raw_samples.append(raw_data)
+ process.kill()
+ process.wait()
+ except Exception as e:
+ logger.error("neuron-monitor failed: %s" % e)
+
+ def __init__(
+ self,
+ pid: int,
+ neuron_monitor_config_path: Optional[str],
+ ) -> None:
+ self.pid = pid
+ # neuron-monitor requires a config file (json)
+ # we provide an option to supply a custom config file path
+ # in case the default temp file path is not writable
+ self.neuron_monitor_config_path = (
+ neuron_monitor_config_path or tempfile.NamedTemporaryFile(delete=False).name
+ )
+ self.raw_samples: Deque[bytes] = deque(maxlen=10)
+ self.samples: Deque[_Stats] = deque()
+ self.shutdown_event = threading.Event()
+
+ self.neuron_monitor_thread: Optional[threading.Thread] = None
+
+ def setup(self) -> None:
+ """Start the neuron-monitor thread for collecting raw data."""
+ if self.neuron_monitor_thread is not None:
+ return
+
+ logger.debug("Starting neuron-monitor thread")
+ self.shutdown_event.clear()
+ self.neuron_monitor_thread = threading.Thread(
+ name="NeuronCoreMntr",
+ target=self.neuron_monitor,
+ daemon=True,
+ )
+ self.neuron_monitor_thread.start()
+
+ def teardown(self) -> None:
+ """Stop the neuron-monitor thread."""
+ logger.debug("Stopping neuron-monitor thread")
+ try:
+ self.shutdown_event.set()
+ assert self.neuron_monitor_thread is not None
+ self.neuron_monitor_thread.join()
+ except Exception as e:
+ logger.error("neuron-monitor thread failed to stop: %s" % e)
+ finally:
+ self.neuron_monitor_thread = None
+
+ def _is_matching_entry(self, entry: dict) -> bool:
+ """Check if the entry should be saved.
+
+ Checks if the pid in the entry matches the pid of the process.
+ If not (as in the case of multi-process training with torchrun),
+ checks if the LOCAL_RANK environment variable is set.
+
+ todo: add matching by neuron_runtime_tag
+ """
+ return (int(entry["pid"]) == int(self.pid)) or "LOCAL_RANK" in os.environ
+
+ def sample(self) -> None:
+ try:
+ raw_stats = json.loads(self.raw_samples[-1])
+ neuron_runtime_data = [
+ entry["report"]
+ for entry in raw_stats["neuron_runtime_data"]
+ if self._is_matching_entry(entry)
+ ][
+ 0
+ ] # there should be only one entry with the pid
+
+ neuroncores_in_use = neuron_runtime_data["neuroncore_counters"][
+ "neuroncores_in_use"
+ ]
+ # per-core utilization stats:
+ neuroncore_utilization = {
+ int(k): v["neuroncore_utilization"]
+ for k, v in neuroncores_in_use.items()
+ }
+ # memory usage
+ neuron_runtime_used_bytes = neuron_runtime_data["memory_used"][
+ "neuron_runtime_used_bytes"
+ ]
+ # memory usage totals
+ host_total_memory_usage = neuron_runtime_used_bytes["host"]
+ neuron_device_total_memory_usage = neuron_runtime_used_bytes[
+ "neuron_device"
+ ]
+ # memory usage breakdown
+ usage_breakdown = neuron_runtime_used_bytes["usage_breakdown"]
+ host_memory_usage = _HostMemoryUsage(**usage_breakdown["host"])
+ neuroncore_memory_usage = {
+ int(k): _NeuronCoreMemoryUsage(**v)
+ for k, v in usage_breakdown["neuroncore_memory_usage"].items()
+ }
+
+ # executed with torchrun? only keep the local_rank stats
+ local_rank = int(os.environ.get("LOCAL_RANK", -1337))
+ if local_rank >= 0:
+ neuroncore_utilization = {
+ local_rank: neuroncore_utilization[local_rank]
+ }
+ neuroncore_memory_usage = {
+ local_rank: neuroncore_memory_usage[local_rank]
+ }
+
+ stats: _Stats = _Stats(
+ neuroncore_utilization=neuroncore_utilization,
+ host_total_memory_usage=host_total_memory_usage,
+ neuron_device_total_memory_usage=neuron_device_total_memory_usage,
+ host_memory_usage=host_memory_usage,
+ neuroncore_memory_usage=neuroncore_memory_usage,
+ )
+ self.samples.append(stats)
+
+ except Exception as e: # noqa
+ pass
+
+ def clear(self) -> None:
+ self.samples.clear()
+
+ @staticmethod
+ def flatten_stats(sample: _Stats) -> dict:
+ """Flatten _Stats object into a flat dict of numbers."""
+ flattened = {}
+
+ def helper(key: str, value: Any) -> None:
+ if isinstance(value, (int, float)):
+ ret = {f"{key}": value}
+ flattened.update(ret)
+ return
+ elif isinstance(value, dict):
+ for kk, vv in value.items():
+ if isinstance(kk, int):
+ # top-level keys are neuron core ids,
+ # so we swap the order to comply with the
+ # frontend expectations
+ helper(f"{kk}.{key}", vv)
+ else:
+ helper(f"{key}.{kk}", vv)
+ return
+ elif isinstance(value, list):
+ for i, val in enumerate(value):
+ helper(f"{i}.{key}", val)
+
+ for kkk, vvv in dataclasses.asdict(sample).items():
+ helper(kkk, vvv)
+
+ return flattened
+
+ def aggregate(self) -> dict:
+ if not self.samples:
+ return {}
+
+ stats = {}
+
+ # Stats could be: numbers or dataclass objects or lists of such.
+ # In the latter case that means per-core stats.
+ # The dataclass objects are flat containers of numbers.
+
+ # flatten samples and merge the corresponding values into lists
+ merged_samples: Dict[str, List[Union[int, float]]] = collections.defaultdict(
+ list
+ )
+ for flattened_sample in (self.flatten_stats(sample) for sample in self.samples):
+ for k, v in flattened_sample.items():
+ merged_samples[k].append(v)
+
+ # aggregate the lists
+ for k, v in merged_samples.items():
+ stats[self.name.format(key=k)] = aggregate_mean(v)
+
+ return stats
+
+
+@asset_registry.register
+class Trainium:
+ def __init__(
+ self,
+ interface: "Interface",
+ settings: "SettingsStatic",
+ shutdown_event: threading.Event,
+ ) -> None:
+ self.name = self.__class__.__name__.lower()
+ self.metrics: List[Metric] = [
+ NeuronCoreStats(
+ settings._stats_pid,
+ settings._stats_neuron_monitor_config_path,
+ ),
+ ]
+ self.metrics_monitor = MetricsMonitor(
+ self.name,
+ self.metrics,
+ interface,
+ settings,
+ shutdown_event,
+ )
+ telemetry_record = telemetry.TelemetryRecord()
+ telemetry_record.env.trainium = True
+ interface._publish_telemetry(telemetry_record)
+
+ @classmethod
+ def is_available(cls) -> bool:
+ # todo: check if neuron-ls is available and if yes, what it reports. see:
+ # https://awsdocs-neuron.readthedocs-hosted.com/en/latest/tools/neuron-sys-tools/neuron-ls.html
+ if not pathlib.Path(NEURON_LS_COMMAND[0]).exists():
+ return False
+ # need to be extra careful as neuron tools could be pre-installed
+ # on some systems that do not have the hardware
+ try:
+ # redirect stderr to null to avoid printing errors to the console
+ # todo: alternative: check /dev/neuron0 ? sysfs support coming soon in neuron tools
+ output = subprocess.check_output(
+ NEURON_LS_COMMAND,
+ universal_newlines=True,
+ stderr=subprocess.DEVNULL,
+ ).strip()
+ if len(json.loads(output)) > 0:
+ return True
+ except (OSError, ValueError, TypeError, subprocess.CalledProcessError):
+ pass
+
+ return False
+
+ def start(self) -> None:
+ self.metrics_monitor.start()
+
+ def finish(self) -> None:
+ self.metrics_monitor.finish()
+
+ def probe(self) -> dict:
+ try:
+ self.metrics[0].check_neuron_monitor_config() # type: ignore
+ neuron_hardware_info: dict = {}
+ command = [
+ NEURON_MONITOR_PATH,
+ "-c",
+ self.metrics[0].neuron_monitor_config_path, # type: ignore
+ ]
+ with subprocess.Popen(
+ command,
+ stdout=subprocess.PIPE,
+ stderr=None,
+ ) as process:
+ while True:
+ if process.stdout is None:
+ time.sleep(0.1)
+ continue
+
+ raw_data = process.stdout.readline()
+ if raw_data:
+ parsed_data = json.loads(raw_data)
+ neuron_hardware_info = parsed_data.get(
+ "neuron_hardware_info", {}
+ )
+ neuron_hardware_info.pop("error", None)
+ break
+
+ try:
+ process.kill()
+ process.wait()
+ except: # noqa
+ pass
+
+ return {self.name: neuron_hardware_info}
+ except Exception as e:
+ logger.error("neuron-monitor failed: %s" % e)
+ return {}
diff --git a/wandb/sdk/internal/system/env_probe_helpers.py b/wandb/sdk/internal/system/env_probe_helpers.py
new file mode 100644
index 0000000000000000000000000000000000000000..90bc12470dcc9e1f21d1bf1fb001debcfe8bf3ae
--- /dev/null
+++ b/wandb/sdk/internal/system/env_probe_helpers.py
@@ -0,0 +1,13 @@
+import logging
+
+from sentry_sdk.integrations.aws_lambda import get_lambda_bootstrap # type: ignore
+
+logger = logging.getLogger(__name__)
+
+
+def is_aws_lambda() -> bool:
+ """Check if we are running in a lambda environment."""
+ lambda_bootstrap = get_lambda_bootstrap()
+ if not lambda_bootstrap or not hasattr(lambda_bootstrap, "handle_event_request"):
+ return False
+ return True
diff --git a/wandb/sdk/internal/system/system_info.py b/wandb/sdk/internal/system/system_info.py
new file mode 100644
index 0000000000000000000000000000000000000000..e52d5e108dc96179617727b37d6ccec5b8457afe
--- /dev/null
+++ b/wandb/sdk/internal/system/system_info.py
@@ -0,0 +1,245 @@
+# Information about the system and the environment
+import datetime
+import glob
+import json
+import logging
+import os
+import subprocess
+import sys
+from shutil import copyfile
+from typing import Any, Dict, List, Optional
+from urllib.parse import unquote
+
+from wandb.sdk.internal.settings_static import SettingsStatic
+from wandb.sdk.lib import filesystem
+from wandb.sdk.lib.filenames import CONDA_ENVIRONMENTS_FNAME, DIFF_FNAME, METADATA_FNAME
+from wandb.sdk.lib.gitlib import GitRepo
+from wandb.sdk.wandb_settings import _get_program_relpath
+
+from .assets.interfaces import Interface
+
+logger = logging.getLogger(__name__)
+
+
+class SystemInfo:
+ # todo: this is mostly a copy of the legacy Meta class, but it should be refactored
+ def __init__(self, settings: SettingsStatic, interface: Interface) -> None:
+ logger.debug("System info init")
+ self.settings = settings
+
+ self.metadata_file_name = os.path.join(self.settings.files_dir, METADATA_FNAME)
+ self.backend_interface = interface
+ self.git = GitRepo(
+ root=self.settings.git_root,
+ remote=self.settings.git_remote,
+ remote_url=self.settings.git_remote_url,
+ commit=self.settings.git_commit,
+ )
+ # Location under "code" directory in files where program was saved.
+ self.saved_program: Optional[os.PathLike] = None
+ # Locations under files directory where diff patches were saved.
+ self.saved_patches: List[str] = []
+ logger.debug("System info init done")
+
+ def _save_code(self) -> None:
+ logger.debug("Saving code")
+ if not self.settings.program_relpath:
+ logger.warning("unable to save code -- program entry not found")
+ return None
+
+ root: str = self.git.root or os.getcwd()
+ program_relative: str = self.settings.program_relpath
+ filesystem.mkdir_exists_ok(
+ os.path.join(
+ self.settings.files_dir, "code", os.path.dirname(program_relative)
+ )
+ )
+ program_absolute = os.path.join(root, program_relative)
+ if not os.path.exists(program_absolute):
+ logger.warning("unable to save code -- can't find %s" % program_absolute)
+ return None
+ saved_program = os.path.join(self.settings.files_dir, "code", program_relative)
+ self.saved_program = program_relative # type: ignore
+
+ if not os.path.exists(saved_program):
+ copyfile(program_absolute, saved_program)
+ logger.debug("Saving code done")
+
+ def _save_patches(self) -> None:
+ """Save the current state of this repository to one or more patches.
+
+ Makes one patch against HEAD and another one against the most recent
+ commit that occurs in an upstream branch. This way we can be robust
+ to history editing as long as the user never does "push -f" to break
+ history on an upstream branch.
+
+ Writes the first patch to <files_dir>/<DIFF_FNAME> and the second to
+ <files_dir>/upstream_diff_<commit_id>.patch.
+
+ """
+ if not self.git.enabled:
+ return None
+
+ logger.debug("Saving git patches")
+ try:
+ root = self.git.root
+ diff_args = ["git", "diff"]
+ if self.git.has_submodule_diff:
+ diff_args.append("--submodule=diff")
+
+ if self.git.dirty:
+ patch_path = os.path.join(self.settings.files_dir, DIFF_FNAME)
+ with open(patch_path, "wb") as patch:
+ # we diff against HEAD to ensure we get changes in the index
+ subprocess.check_call(
+ diff_args + ["HEAD"], stdout=patch, cwd=root, timeout=5
+ )
+ self.saved_patches.append(
+ os.path.relpath(patch_path, start=self.settings.files_dir)
+ )
+
+ upstream_commit = self.git.get_upstream_fork_point()
+ if upstream_commit and upstream_commit != self.git.repo.head.commit: # type: ignore
+ sha = upstream_commit.hexsha
+ upstream_patch_path = os.path.join(
+ self.settings.files_dir, f"upstream_diff_{sha}.patch"
+ )
+ with open(upstream_patch_path, "wb") as upstream_patch:
+ subprocess.check_call(
+ diff_args + [sha], stdout=upstream_patch, cwd=root, timeout=5
+ )
+ self.saved_patches.append(
+ os.path.relpath(
+ upstream_patch_path, start=self.settings.files_dir
+ )
+ )
+ # TODO: A customer saw `ValueError: Reference at 'refs/remotes/origin/foo'
+ # does not exist` so we now catch ValueError. Catching this error feels
+ # too generic.
+ except (
+ ValueError,
+ subprocess.CalledProcessError,
+ subprocess.TimeoutExpired,
+ ) as e:
+ logger.error("Error generating diff: %s" % e)
+ logger.debug("Saving git patches done")
+
+ def _probe_git(self, data: Dict[str, Any]) -> Dict[str, Any]:
+ if self.settings.disable_git:
+ return data
+
+ # in case of manually passing the git repo info, `enabled` would be False,
+ # but we still want to save the git repo info
+ if not self.git.enabled and self.git.auto:
+ return data
+
+ logger.debug("Probing git")
+
+ data["git"] = {
+ "remote": self.git.remote_url,
+ "commit": self.git.last_commit,
+ }
+ data["email"] = self.git.email
+ data["root"] = self.git.root or data.get("root") or os.getcwd()
+ logger.debug("Probing git done")
+
+ return data
+
+ def probe(self) -> Dict[str, Any]:
+ """Probe the system for information about the current environment."""
+ # todo: refactor this quality code 🤮🤮🤮🤮🤮
+ logger.debug("Probing system")
+ data: Dict[str, Any] = dict()
+
+ data["os"] = self.settings._os
+ data["python"] = self.settings._python
+ data["heartbeatAt"] = datetime.datetime.utcnow().isoformat()
+ data["startedAt"] = datetime.datetime.utcfromtimestamp(
+ self.settings._start_time
+ ).isoformat()
+
+ data["docker"] = self.settings.docker
+
+ data["cuda"] = self.settings._cuda
+ data["args"] = tuple(self.settings._args or ())
+ data["state"] = "running"
+
+ if self.settings.program is not None:
+ data["program"] = self.settings.program
+ # Used during artifact-job creation, always points to the relpath
+ # of code execution, even when in a git repo
+ data["codePathLocal"] = _get_program_relpath(self.settings.program)
+ if not self.settings.disable_code:
+ if self.settings.program_relpath:
+ data["codePath"] = self.settings.program_relpath
+ elif self.settings._jupyter:
+ if self.settings.notebook_name:
+ data["program"] = self.settings.notebook_name
+ elif self.settings._jupyter_path:
+ if self.settings._jupyter_path.startswith("fileId="):
+ unescaped = unquote(self.settings._jupyter_path)
+ data["colab"] = (
+ "https://colab.research.google.com/notebook#" + unescaped
+ )
+ data["program"] = self.settings._jupyter_name
+ else:
+ data["program"] = self.settings._jupyter_path
+ data["root"] = self.settings._jupyter_root
+ # get the git repo info
+ data = self._probe_git(data)
+
+ if self.settings.anonymous != "true":
+ data["host"] = self.settings.host
+ data["username"] = self.settings.username
+ data["executable"] = sys.executable
+ else:
+ data.pop("email", None)
+ data.pop("root", None)
+
+ logger.debug("Probing system done")
+
+ return data
+
+ def _save_conda(self) -> None:
+ current_shell_is_conda = os.path.exists(os.path.join(sys.prefix, "conda-meta"))
+ if not current_shell_is_conda:
+ return None
+
+ logger.debug(
+ "Saving list of conda packages installed into the current environment"
+ )
+ try:
+ with open(
+ os.path.join(self.settings.files_dir, CONDA_ENVIRONMENTS_FNAME), "w"
+ ) as f:
+ subprocess.call(
+ ["conda", "env", "export"], stdout=f, stderr=subprocess.DEVNULL
+ )
+ except Exception as e:
+ logger.exception(f"Error saving conda packages: {e}")
+ logger.debug("Saving conda packages done")
+
+ def publish(self, system_info: dict) -> None:
+ # save pip, conda, code patches to disk
+ if self.settings._save_requirements:
+ self._save_conda()
+ if self.settings.save_code:
+ self._save_code()
+ self._save_patches()
+
+ # save system_info to disk
+ with open(self.metadata_file_name, "w") as f:
+ s = json.dumps(system_info, indent=4)
+ f.write(s)
+ f.write("\n")
+ base_name = os.path.basename(self.metadata_file_name)
+ files = dict(files=[(base_name, "now")])
+
+ if self.saved_program:
+ saved_program = os.path.join("code", self.saved_program)
+ files["files"].append((glob.escape(saved_program), "now"))
+ for patch in self.saved_patches:
+ files["files"].append((glob.escape(patch), "now"))
+
+ # publish files to the backend
+ self.backend_interface.publish_files(files) # type: ignore
diff --git a/wandb/sdk/internal/system/system_monitor.py b/wandb/sdk/internal/system/system_monitor.py
new file mode 100644
index 0000000000000000000000000000000000000000..0427f88836d3fcd40eb91efd42aa1bf764e98f63
--- /dev/null
+++ b/wandb/sdk/internal/system/system_monitor.py
@@ -0,0 +1,229 @@
+import datetime
+import logging
+import queue
+import threading
+from collections import defaultdict, deque
+from typing import TYPE_CHECKING, Deque, Dict, List, Optional, Tuple
+
+from .assets.asset_registry import asset_registry
+from .assets.interfaces import Asset, Interface
+from .assets.open_metrics import OpenMetrics
+from .system_info import SystemInfo
+
+if TYPE_CHECKING:
+ from wandb.proto.wandb_telemetry_pb2 import TelemetryRecord
+ from wandb.sdk.interface.interface import FilesDict
+ from wandb.sdk.internal.settings_static import SettingsStatic
+
+
+logger = logging.getLogger(__name__)
+
+
+class AssetInterface:
+ def __init__(self) -> None:
+ self.metrics_queue: queue.Queue[dict] = queue.Queue()
+ self.telemetry_queue: queue.Queue[TelemetryRecord] = queue.Queue()
+
+ def publish_stats(self, stats: dict) -> None:
+ self.metrics_queue.put(stats)
+
+ def _publish_telemetry(self, telemetry: "TelemetryRecord") -> None:
+ self.telemetry_queue.put(telemetry)
+
+ def publish_files(self, files_dict: "FilesDict") -> None:
+ pass
+
+
+class SystemMonitor:
+ # SystemMonitor is responsible for managing system metrics data.
+
+ # if joining assets, wait for publishing_interval times this many seconds
+ PUBLISHING_INTERVAL_DELAY_FACTOR = 2
+
+ def __init__(
+ self,
+ settings: "SettingsStatic",
+ interface: "Interface",
+ ) -> None:
+ self._shutdown_event: threading.Event = threading.Event()
+ self._process: Optional[threading.Thread] = None
+
+ self.settings = settings
+
+ # settings._stats_join_assets controls whether we should join stats from different assets
+ # before publishing them to the backend. If set to False, we will publish stats from each
+ # asset separately, using the backend interface. If set to True, we will aggregate stats from
+ # all assets before publishing them, using an internal queue interface, and then publish
+ # them using the interface to the backend.
+ # This is done to improve compatibility with older versions of the backend as it used to
+ # collect the names of the metrics to be displayed in the UI from the first stats message.
+
+ # compute the global publishing interval if _stats_join_assets is requested
+ sampling_interval: float = float(
+ max(
+ 0.1,
+ self.settings._stats_sample_rate_seconds,
+ )
+ ) # seconds
+ # The number of samples to aggregate (e.g. average or compute max/min etc.)
+ # before publishing; defaults to 15; valid range: [1:30]
+ samples_to_aggregate: int = min(
+ 30, max(1, self.settings._stats_samples_to_average)
+ )
+ self.publishing_interval: float = sampling_interval * samples_to_aggregate
+ self.join_assets: bool = self.settings._stats_join_assets
+
+ self.backend_interface = interface
+ self.asset_interface: Optional[AssetInterface] = (
+ AssetInterface() if self.join_assets else None
+ )
+
+ # hardware assets
+ self.assets: List[Asset] = self._get_assets()
+
+ # OpenMetrics/Prometheus-compatible endpoints
+ self.assets.extend(self._get_open_metrics_assets())
+
+ # static system info, both hardware and software
+ self.system_info: SystemInfo = SystemInfo(
+ settings=self.settings, interface=interface
+ )
+
+ self.buffer: Dict[str, Deque[Tuple[float, float]]] = defaultdict(
+ lambda: deque([], maxlen=self.settings._stats_buffer_size)
+ )
+
+ def _get_assets(self) -> List["Asset"]:
+ return [
+ asset_class(
+ interface=self.asset_interface or self.backend_interface,
+ settings=self.settings,
+ shutdown_event=self._shutdown_event,
+ )
+ for asset_class in asset_registry
+ ]
+
+ def _get_open_metrics_assets(self) -> List["Asset"]:
+ open_metrics_endpoints = self.settings._stats_open_metrics_endpoints
+ if not open_metrics_endpoints:
+ return []
+
+ assets: List[Asset] = []
+ for name, endpoint in open_metrics_endpoints.items():
+ if not OpenMetrics.is_available(url=endpoint):
+ continue
+ logger.debug(f"Monitoring OpenMetrics endpoint: {endpoint}")
+ open_metrics = OpenMetrics(
+ interface=self.asset_interface or self.backend_interface,
+ settings=self.settings,
+ shutdown_event=self._shutdown_event,
+ name=name,
+ url=endpoint,
+ )
+ assets.append(open_metrics) # type: ignore
+
+ return assets
+
+ def aggregate_and_publish_asset_metrics(self) -> None:
+ if self.asset_interface is None:
+ return None
+ # only extract as many items as are available in the queue at the moment
+ size = self.asset_interface.metrics_queue.qsize()
+
+ aggregated_metrics = {}
+ for _ in range(size):
+ item = self.asset_interface.metrics_queue.get()
+ aggregated_metrics.update(item)
+
+ if aggregated_metrics:
+ # update buffer:
+ # todo: get it from publish_stats instead?
+ # either is not too accurate, just use wandb-core!
+ t = datetime.datetime.now().timestamp()
+ for k, v in aggregated_metrics.items():
+ self.buffer[k].append((t, v))
+ # publish aggregated metrics
+ self.backend_interface.publish_stats(aggregated_metrics)
+
+ def publish_telemetry(self) -> None:
+ if self.asset_interface is None:
+ return None
+ # get everything from the self.asset_interface.telemetry_queue,
+ # merge into a single dictionary and publish on the backend_interface
+ while not self.asset_interface.telemetry_queue.empty():
+ telemetry_record = self.asset_interface.telemetry_queue.get()
+ self.backend_interface._publish_telemetry(telemetry_record)
+
+ def _start(self) -> None:
+ logger.info("Starting system asset monitoring threads")
+ for asset in self.assets:
+ asset.start()
+
+ # compatibility mode: join stats from different assets before publishing
+ if not (self.join_assets and self.asset_interface is not None):
+ return None
+
+ # give the assets a chance to accumulate and publish their first stats
+ # this will provide a constant offset for the following accumulation events below
+ self._shutdown_event.wait(
+ self.publishing_interval * self.PUBLISHING_INTERVAL_DELAY_FACTOR
+ )
+
+ logger.debug("Starting system metrics aggregation loop")
+
+ while not self._shutdown_event.is_set():
+ self.publish_telemetry()
+ self.aggregate_and_publish_asset_metrics()
+ self._shutdown_event.wait(self.publishing_interval)
+
+ logger.debug("Finished system metrics aggregation loop")
+
+ # try to publish the last batch of metrics + telemetry
+ try:
+ logger.debug("Publishing last batch of metrics")
+ # publish telemetry
+ self.publish_telemetry()
+ self.aggregate_and_publish_asset_metrics()
+ except Exception as e:
+ logger.error(f"Error publishing last batch of metrics: {e}")
+
+ def start(self) -> None:
+ self._shutdown_event.clear()
+ if self._process is not None:
+ return None
+ logger.info("Starting system monitor")
+ self._process = threading.Thread(
+ target=self._start, daemon=True, name="SystemMonitor"
+ )
+ self._process.start()
+
+ def finish(self) -> None:
+ if self._process is None:
+ return None
+ logger.info("Stopping system monitor")
+ self._shutdown_event.set()
+ for asset in self.assets:
+ asset.finish()
+ try:
+ self._process.join()
+ except Exception as e:
+ logger.error(f"Error joining system monitor process: {e}")
+ self._process = None
+
+ def probe(self, publish: bool = True) -> None:
+ logger.info("Collecting system info")
+ # collect static info about the hardware from registered assets
+ hardware_info: dict = {
+ k: v for d in [asset.probe() for asset in self.assets] for k, v in d.items()
+ }
+ # collect static info about the software environment
+ software_info: dict = self.system_info.probe()
+ # merge the two dictionaries
+ system_info = {**software_info, **hardware_info}
+ logger.debug(system_info)
+ logger.info("Finished collecting system info")
+
+ if publish:
+ logger.info("Publishing system info")
+ self.system_info.publish(system_info)
+ logger.info("Finished publishing system info")
diff --git a/wandb/sdk/internal/tb_watcher.py b/wandb/sdk/internal/tb_watcher.py
new file mode 100644
index 0000000000000000000000000000000000000000..af46641380533fac6f762a9b4483b0413927c18c
--- /dev/null
+++ b/wandb/sdk/internal/tb_watcher.py
@@ -0,0 +1,518 @@
+"""tensorboard watcher."""
+
+import glob
+import logging
+import os
+import queue
+import socket
+import sys
+import threading
+import time
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+import wandb
+from wandb import util
+from wandb.sdk.interface.interface import GlobStr
+from wandb.sdk.lib import filesystem
+from wandb.viz import CustomChart
+
+from . import run as internal_run
+
+if TYPE_CHECKING:
+ from queue import PriorityQueue
+
+ from tensorboard.backend.event_processing.event_file_loader import EventFileLoader
+ from tensorboard.compat.proto.event_pb2 import ProtoEvent
+
+ from wandb.proto.wandb_internal_pb2 import RunRecord
+ from wandb.sdk.interface.interface import FilesDict
+
+ from ..interface.interface_queue import InterfaceQueue
+ from .settings_static import SettingsStatic
+
+ HistoryDict = Dict[str, Any]
+
+# Give some time for tensorboard data to be flushed
+SHUTDOWN_DELAY = 5
+ERROR_DELAY = 5
+REMOTE_FILE_TOKEN = "://"
+logger = logging.getLogger(__name__)
+
+
+def _link_and_save_file(
+ path: str, base_path: str, interface: "InterfaceQueue", settings: "SettingsStatic"
+) -> None:
+ # TODO(jhr): should this logic be merged with Run.save()
+ files_dir = settings.files_dir
+ file_name = os.path.relpath(path, base_path)
+ abs_path = os.path.abspath(path)
+ wandb_path = os.path.join(files_dir, file_name)
+ filesystem.mkdir_exists_ok(os.path.dirname(wandb_path))
+ # We overwrite existing symlinks because namespaces can change in Tensorboard
+ if os.path.islink(wandb_path) and abs_path != os.readlink(wandb_path):
+ os.remove(wandb_path)
+ os.symlink(abs_path, wandb_path)
+ elif not os.path.exists(wandb_path):
+ os.symlink(abs_path, wandb_path)
+ # TODO(jhr): need to figure out policy, live/throttled?
+ interface.publish_files(dict(files=[(GlobStr(glob.escape(file_name)), "live")]))
+
+
+def is_tfevents_file_created_by(
+ path: str, hostname: Optional[str], start_time: Optional[float]
+) -> bool:
+ """Check if a path is a tfevents file.
+
+ Optionally checks that it was created by [hostname] after [start_time].
+
+ tensorboard tfevents filename format:
+ https://github.com/tensorflow/tensorboard/blob/f3f26b46981da5bd46a5bb93fcf02d9eb7608bc1/tensorboard/summary/writer/event_file_writer.py#L81
+ tensorflow tfevents filename format:
+ https://github.com/tensorflow/tensorflow/blob/8f597046dc30c14b5413813d02c0e0aed399c177/tensorflow/core/util/events_writer.cc#L68
+ """
+ if not path:
+ raise ValueError("Path must be a nonempty string")
+ basename = os.path.basename(path)
+ if basename.endswith(".profile-empty") or basename.endswith(".sagemaker-uploaded"):
+ return False
+ fname_components = basename.split(".")
+ try:
+ tfevents_idx = fname_components.index("tfevents")
+ except ValueError:
+ return False
+ # check the hostname, which may have dots
+ if hostname is not None:
+ for i, part in enumerate(hostname.split(".")):
+ try:
+ fname_component_part = fname_components[tfevents_idx + 2 + i]
+ except IndexError:
+ return False
+ if part != fname_component_part:
+ return False
+ if start_time is not None:
+ try:
+ created_time = int(fname_components[tfevents_idx + 1])
+ except (ValueError, IndexError):
+ return False
+ # Ensure that the file is newer then our start time, and that it was
+ # created from the same hostname.
+ # TODO: we should also check the PID (also contained in the tfevents
+ # filename). Can we assume that our parent pid is the user process
+ # that wrote these files?
+ if created_time < int(start_time):
+ return False
+ return True
+
+
+class TBWatcher:
+ _logdirs: "Dict[str, TBDirWatcher]"
+ _watcher_queue: "PriorityQueue"
+
+ def __init__(
+ self,
+ settings: "SettingsStatic",
+ run_proto: "RunRecord",
+ interface: "InterfaceQueue",
+ force: bool = False,
+ ) -> None:
+ self._logdirs = {}
+ self._consumer: Optional[TBEventConsumer] = None
+ self._settings = settings
+ self._interface = interface
+ self._run_proto = run_proto
+ self._force = force
+ # TODO(jhr): do we need locking in this queue?
+ self._watcher_queue = queue.PriorityQueue()
+ wandb.tensorboard.reset_state()
+
+ def _calculate_namespace(self, logdir: str, rootdir: str) -> Optional[str]:
+ namespace: Optional[str]
+ dirs = list(self._logdirs) + [logdir]
+
+ if os.path.isfile(logdir):
+ filename = os.path.basename(logdir)
+ else:
+ filename = ""
+
+ if rootdir == "":
+ rootdir = util.to_forward_slash_path(
+ os.path.dirname(os.path.commonprefix(dirs))
+ )
+ # Tensorboard loads all tfevents files in a directory and prepends
+ # their values with the path. Passing namespace to log allows us
+ # to nest the values in wandb
+ # Note that we strip '/' instead of os.sep, because elsewhere we've
+ # converted paths to forward slash.
+ namespace = logdir.replace(filename, "").replace(rootdir, "").strip("/")
+
+ # TODO: revisit this heuristic, it exists because we don't know the
+ # root log directory until more than one tfevents file is written to
+ if len(dirs) == 1 and namespace not in ["train", "validation"]:
+ namespace = None
+ else:
+ namespace = logdir.replace(filename, "").replace(rootdir, "").strip("/")
+
+ return namespace
+
+ def add(self, logdir: str, save: bool, root_dir: str) -> None:
+ logdir = util.to_forward_slash_path(logdir)
+ root_dir = util.to_forward_slash_path(root_dir)
+ if logdir in self._logdirs:
+ return
+ namespace = self._calculate_namespace(logdir, root_dir)
+ # TODO(jhr): implement the deferred tbdirwatcher to find namespace
+
+ if not self._consumer:
+ self._consumer = TBEventConsumer(
+ self, self._watcher_queue, self._run_proto, self._settings
+ )
+ self._consumer.start()
+
+ tbdir_watcher = TBDirWatcher(
+ self, logdir, save, namespace, self._watcher_queue, self._force
+ )
+ self._logdirs[logdir] = tbdir_watcher
+ tbdir_watcher.start()
+
+ def finish(self) -> None:
+ for tbdirwatcher in self._logdirs.values():
+ tbdirwatcher.shutdown()
+ for tbdirwatcher in self._logdirs.values():
+ tbdirwatcher.finish()
+ if self._consumer:
+ self._consumer.finish()
+
+
+class TBDirWatcher:
+ def __init__(
+ self,
+ tbwatcher: "TBWatcher",
+ logdir: str,
+ save: bool,
+ namespace: Optional[str],
+ queue: "PriorityQueue",
+ force: bool = False,
+ ) -> None:
+ self.directory_watcher = util.get_module(
+ "tensorboard.backend.event_processing.directory_watcher",
+ required="Please install tensorboard package",
+ )
+ # self.event_file_loader = util.get_module(
+ # "tensorboard.backend.event_processing.event_file_loader",
+ # required="Please install tensorboard package",
+ # )
+ self.tf_compat = util.get_module(
+ "tensorboard.compat", required="Please install tensorboard package"
+ )
+ self._tbwatcher = tbwatcher
+ self._generator = self.directory_watcher.DirectoryWatcher(
+ logdir, self._loader(save, namespace), self._is_our_tfevents_file
+ )
+ self._thread = threading.Thread(target=self._thread_except_body)
+ self._first_event_timestamp = None
+ self._shutdown = threading.Event()
+ self._queue = queue
+ self._file_version = None
+ self._namespace = namespace
+ self._logdir = logdir
+ self._hostname = socket.gethostname()
+ self._force = force
+ self._process_events_lock = threading.Lock()
+
+ def start(self) -> None:
+ self._thread.start()
+
+ def _is_our_tfevents_file(self, path: str) -> bool:
+ """Check if a path has been modified since launch and contains tfevents."""
+ if not path:
+ raise ValueError("Path must be a nonempty string")
+ path = self.tf_compat.tf.compat.as_str_any(path)
+ if self._force:
+ return is_tfevents_file_created_by(path, None, None)
+ else:
+ return is_tfevents_file_created_by(
+ path, self._hostname, self._tbwatcher._settings._start_time
+ )
+
+ def _loader(
+ self, save: bool = True, namespace: Optional[str] = None
+ ) -> "EventFileLoader":
+ """Incredibly hacky class generator to optionally save / prefix tfevent files."""
+ _loader_interface = self._tbwatcher._interface
+ _loader_settings = self._tbwatcher._settings
+ try:
+ from tensorboard.backend.event_processing import event_file_loader
+ except ImportError:
+ raise Exception("Please install tensorboard package")
+
+ class EventFileLoader(event_file_loader.EventFileLoader):
+ def __init__(self, file_path: str) -> None:
+ super().__init__(file_path)
+ if save:
+ if REMOTE_FILE_TOKEN in file_path:
+ logger.warning(
+ "Not persisting remote tfevent file: %s", file_path
+ )
+ else:
+ # TODO: save plugins?
+ logdir = os.path.dirname(file_path)
+ parts = list(os.path.split(logdir))
+ if namespace and parts[-1] == namespace:
+ parts.pop()
+ logdir = os.path.join(*parts)
+ _link_and_save_file(
+ path=file_path,
+ base_path=logdir,
+ interface=_loader_interface,
+ settings=_loader_settings,
+ )
+
+ return EventFileLoader
+
+ def _process_events(self, shutdown_call: bool = False) -> None:
+ try:
+ with self._process_events_lock:
+ for event in self._generator.Load():
+ self.process_event(event)
+ except (
+ self.directory_watcher.DirectoryDeletedError,
+ StopIteration,
+ RuntimeError,
+ OSError,
+ ) as e:
+ # When listing s3 the directory may not yet exist, or could be empty
+ logger.debug("Encountered tensorboard directory watcher error: %s", e)
+ if not self._shutdown.is_set() and not shutdown_call:
+ time.sleep(ERROR_DELAY)
+
+ def _thread_except_body(self) -> None:
+ try:
+ self._thread_body()
+ except Exception as e:
+ logger.exception("generic exception in TBDirWatcher thread")
+ raise e
+
+ def _thread_body(self) -> None:
+ """Check for new events every second."""
+ shutdown_time: Optional[float] = None
+ while True:
+ self._process_events()
+ if self._shutdown.is_set():
+ now = time.time()
+ if not shutdown_time:
+ shutdown_time = now + SHUTDOWN_DELAY
+ elif now > shutdown_time:
+ break
+ time.sleep(1)
+
+ def process_event(self, event: "ProtoEvent") -> None:
+ # print("\nEVENT:::", self._logdir, self._namespace, event, "\n")
+ if self._first_event_timestamp is None:
+ self._first_event_timestamp = event.wall_time
+
+ if event.HasField("file_version"):
+ self._file_version = event.file_version
+
+ if event.HasField("summary"):
+ self._queue.put(Event(event, self._namespace))
+
+ def shutdown(self) -> None:
+ self._process_events(shutdown_call=True)
+ self._shutdown.set()
+
+ def finish(self) -> None:
+ self.shutdown()
+ self._thread.join()
+
+
+class Event:
+ """An event wrapper to enable priority queueing."""
+
+ def __init__(self, event: "ProtoEvent", namespace: Optional[str]):
+ self.event = event
+ self.namespace = namespace
+ self.created_at = time.time()
+
+ def __lt__(self, other: "Event") -> bool:
+ if self.event.wall_time < other.event.wall_time:
+ return True
+ return False
+
+
+class TBEventConsumer:
+ """Consume tfevents from a priority queue.
+
+ There should always only be one of these per run_manager. We wait for 10 seconds of
+ queued events to reduce the chance of multiple tfevent files triggering out of order
+ steps.
+ """
+
+ def __init__(
+ self,
+ tbwatcher: TBWatcher,
+ queue: "PriorityQueue",
+ run_proto: "RunRecord",
+ settings: "SettingsStatic",
+ delay: int = 10,
+ ) -> None:
+ self._tbwatcher = tbwatcher
+ self._queue = queue
+ self._thread = threading.Thread(target=self._thread_except_body)
+ self._shutdown = threading.Event()
+ self.tb_history = TBHistory()
+ self._delay = delay
+
+ # This is a bit of a hack to get file saving to work as it does in the user
+ # process. Since we don't have a real run object, we have to define the
+ # datatypes callback ourselves.
+ def datatypes_cb(fname: GlobStr) -> None:
+ files: FilesDict = dict(files=[(fname, "now")])
+ self._tbwatcher._interface.publish_files(files)
+
+ # this is only used for logging artifacts
+ self._internal_run = internal_run.InternalRun(run_proto, settings, datatypes_cb)
+ self._internal_run._set_internal_run_interface(self._tbwatcher._interface)
+
+ def start(self) -> None:
+ self._start_time = time.time()
+ self._thread.start()
+
+ def finish(self) -> None:
+ self._delay = 0
+ self._shutdown.set()
+ self._thread.join()
+ while not self._queue.empty():
+ event = self._queue.get(True, 1)
+ if event:
+ self._handle_event(event, history=self.tb_history)
+ items = self.tb_history._get_and_reset()
+ for item in items:
+ self._save_row(
+ item,
+ )
+
+ def _thread_except_body(self) -> None:
+ try:
+ self._thread_body()
+ except Exception as e:
+ logger.exception("generic exception in TBEventConsumer thread")
+ raise e
+
+ def _thread_body(self) -> None:
+ while True:
+ try:
+ event = self._queue.get(True, 1)
+ # Wait self._delay seconds from consumer start before logging events
+ if (
+ time.time() < self._start_time + self._delay
+ and not self._shutdown.is_set()
+ ):
+ self._queue.put(event)
+ time.sleep(0.1)
+ continue
+ except queue.Empty:
+ event = None
+ if self._shutdown.is_set():
+ break
+ if event:
+ self._handle_event(event, history=self.tb_history)
+ items = self.tb_history._get_and_reset()
+ for item in items:
+ self._save_row(
+ item,
+ )
+ # flush uncommitted data
+ self.tb_history._flush()
+ items = self.tb_history._get_and_reset()
+ for item in items:
+ self._save_row(item)
+
+ def _handle_event(
+ self, event: "ProtoEvent", history: Optional["TBHistory"] = None
+ ) -> None:
+ wandb.tensorboard._log(
+ event.event,
+ step=event.event.step,
+ namespace=event.namespace,
+ history=history,
+ )
+
+ def _save_row(self, row: "HistoryDict") -> None:
+ chart_keys = set()
+ for k in row:
+ if isinstance(row[k], CustomChart):
+ chart_keys.add(k)
+ key = row[k].get_config_key(k)
+ value = row[k].get_config_value(
+ "Vega2", row[k].user_query(f"{k}_table")
+ )
+ row[k] = row[k]._data
+ self._tbwatcher._interface.publish_config(val=value, key=key)
+
+ for k in chart_keys:
+ row[f"{k}_table"] = row.pop(k)
+
+ self._tbwatcher._interface.publish_history(
+ row, run=self._internal_run, publish_step=False
+ )
+
+
+class TBHistory:
+ _data: "HistoryDict"
+ _added: "List[HistoryDict]"
+
+ def __init__(self) -> None:
+ self._step = 0
+ self._step_size = 0
+ self._data = dict()
+ self._added = []
+
+ def _flush(self) -> None:
+ if not self._data:
+ return
+ # A single tensorboard step may have too much data
+ # we just drop the largest keys in the step if it does.
+ # TODO: we could flush the data across multiple steps
+ if self._step_size > util.MAX_LINE_BYTES:
+ metrics = [(k, sys.getsizeof(v)) for k, v in self._data.items()]
+ metrics.sort(key=lambda t: t[1], reverse=True)
+ bad = 0
+ dropped_keys = []
+ for k, v in metrics:
+ # TODO: (cvp) Added a buffer of 100KiB, this feels rather brittle.
+ if self._step_size - bad < util.MAX_LINE_BYTES - 100000:
+ break
+ else:
+ bad += v
+ dropped_keys.append(k)
+ del self._data[k]
+ wandb.termwarn(
+ "Step {} exceeds max data limit, dropping {} of the largest keys:".format(
+ self._step, len(dropped_keys)
+ )
+ )
+ print("\t" + ("\n\t".join(dropped_keys)))
+ self._data["_step"] = self._step
+ self._added.append(self._data)
+ self._step += 1
+ self._step_size = 0
+
+ def add(self, d: "HistoryDict") -> None:
+ self._flush()
+ self._data = dict()
+ self._data.update(self._track_history_dict(d))
+
+ def _track_history_dict(self, d: "HistoryDict") -> "HistoryDict":
+ e = {}
+ for k in d.keys():
+ e[k] = d[k]
+ self._step_size += sys.getsizeof(e[k])
+ return e
+
+ def _row_update(self, d: "HistoryDict") -> None:
+ self._data.update(self._track_history_dict(d))
+
+ def _get_and_reset(self) -> "List[HistoryDict]":
+ added = self._added[:]
+ self._added = []
+ return added
diff --git a/wandb/sdk/internal/thread_local_settings.py b/wandb/sdk/internal/thread_local_settings.py
new file mode 100644
index 0000000000000000000000000000000000000000..2ee0e74cc44151617cb90e7eb45996e0d1a3859d
--- /dev/null
+++ b/wandb/sdk/internal/thread_local_settings.py
@@ -0,0 +1,18 @@
+import threading
+from typing import Dict, Optional
+
+
+# Context variable for setting API settings (api keys, etc.) for internal and public apis thread-locally
+# TODO: move this into actual settings
+class _ThreadLocalApiSettings(threading.local):
+ api_key: Optional[str]
+ cookies: Optional[Dict]
+ headers: Optional[Dict]
+
+ def __init__(self) -> None:
+ self.api_key = None
+ self.cookies = None
+ self.headers = None
+
+
+_thread_local_api_settings: _ThreadLocalApiSettings = _ThreadLocalApiSettings()
diff --git a/wandb/sdk/internal/update.py b/wandb/sdk/internal/update.py
new file mode 100644
index 0000000000000000000000000000000000000000..6950e53b08a19bcbb3e1d889a4e5081c699aa4bc
--- /dev/null
+++ b/wandb/sdk/internal/update.py
@@ -0,0 +1,113 @@
+from typing import Dict, Optional, Tuple
+
+import requests
+
+import wandb
+
+
+def _find_available(
+ current_version: str,
+) -> Optional[Tuple[str, bool, bool, bool, Optional[str]]]:
+ from pkg_resources import parse_version
+
+ pypi_url = f"https://pypi.org/pypi/{wandb._wandb_module}/json"
+
+ yanked_dict = {}
+ try:
+ # raise Exception("test")
+ async_requests_get = wandb.util.async_call(requests.get, timeout=5)
+ data, thread = async_requests_get(pypi_url, timeout=3)
+ if not data or isinstance(data, Exception):
+ return None
+ data = data.json()
+ latest_version = data["info"]["version"]
+ release_list = data["releases"].keys()
+ for version, fields in data["releases"].items():
+ for item in fields:
+ yanked = item.get("yanked")
+ yanked_reason = item.get("yanked_reason")
+ if yanked:
+ yanked_dict[version] = yanked_reason
+ except Exception:
+ # Any issues whatsoever, just skip the latest version check.
+ return None
+
+ # Return if no update is available
+ pip_prerelease = False
+ deleted = False
+ yanked = False
+ yanked_reason = None
+ parsed_current_version = parse_version(current_version)
+
+ # Check if current version has been yanked or deleted
+ # NOTE: we will not return yanked or deleted if there is nothing to upgrade to
+ if current_version in release_list:
+ yanked = current_version in yanked_dict
+ yanked_reason = yanked_dict.get(current_version)
+ else:
+ deleted = True
+
+ # Check pre-releases
+ if parse_version(latest_version) <= parsed_current_version:
+ # pre-releases are not included in latest_version
+ # so if we are currently running a pre-release we check more
+ if not parsed_current_version.is_prerelease:
+ return None
+ # Candidates are pre-releases with the same base_version
+ release_list = map(parse_version, release_list)
+ release_list = filter(lambda v: v.is_prerelease, release_list)
+ release_list = filter(
+ lambda v: v.base_version == parsed_current_version.base_version,
+ release_list,
+ )
+ release_list = sorted(release_list)
+ if not release_list:
+ return None
+
+ parsed_latest_version = release_list[-1]
+ if parsed_latest_version <= parsed_current_version:
+ return None
+ latest_version = str(parsed_latest_version)
+ pip_prerelease = True
+
+ return latest_version, pip_prerelease, deleted, yanked, yanked_reason
+
+
+def check_available(current_version: str) -> Optional[Dict[str, Optional[str]]]:
+ package_info = _find_available(current_version)
+ if not package_info:
+ return None
+
+ wandb_module_name = wandb._wandb_module
+
+ latest_version, pip_prerelease, deleted, yanked, yanked_reason = package_info
+ upgrade_message = (
+ "{} version {} is available! To upgrade, please run:\n"
+ " $ pip install {} --upgrade{}".format(
+ wandb_module_name,
+ latest_version,
+ wandb_module_name,
+ " --pre" if pip_prerelease else "",
+ )
+ )
+ delete_message = None
+ if deleted:
+ delete_message = "{} version {} has been retired! Please upgrade.".format(
+ wandb_module_name,
+ current_version,
+ )
+ yank_message = None
+ if yanked:
+ reason_message = "(%s) " % yanked_reason if yanked_reason else ""
+ yank_message = "{} version {} has been recalled! {}Please upgrade.".format(
+ wandb_module_name,
+ current_version,
+ reason_message,
+ )
+
+ # A new version is available!
+ return {
+ "upgrade_message": upgrade_message,
+ "yank_message": yank_message,
+ "delete_message": delete_message,
+ }
diff --git a/wandb/sdk/internal/writer.py b/wandb/sdk/internal/writer.py
new file mode 100644
index 0000000000000000000000000000000000000000..c2a26257e8da3f06268d9817ddacac9fe488acea
--- /dev/null
+++ b/wandb/sdk/internal/writer.py
@@ -0,0 +1,206 @@
+"""Writer thread."""
+
+import logging
+from typing import TYPE_CHECKING, Callable, Optional
+
+from wandb.proto import wandb_internal_pb2 as pb
+from wandb.proto import wandb_telemetry_pb2 as tpb
+
+from ..interface.interface_queue import InterfaceQueue
+from ..lib import proto_util, telemetry, tracelog
+from . import context, datastore, flow_control
+from .settings_static import SettingsStatic
+
+if TYPE_CHECKING:
+ from queue import Queue
+
+
+logger = logging.getLogger(__name__)
+
+
+class WriteManager:
+ _settings: SettingsStatic
+ _record_q: "Queue[pb.Record]"
+ _result_q: "Queue[pb.Result]"
+ _sender_q: "Queue[pb.Record]"
+ _interface: InterfaceQueue
+ _context_keeper: context.ContextKeeper
+
+ _ds: Optional[datastore.DataStore]
+ _flow_control: Optional[flow_control.FlowControl]
+ _status_report: Optional["pb.StatusReportRequest"]
+ _record_num: int
+ _telemetry_obj: tpb.TelemetryRecord
+ _telemetry_overflow: bool
+ _use_flow_control: bool
+
+ # TODO(cancel_paused): implement me
+ # _sender_cancel_set: Set[str]
+
+ def __init__(
+ self,
+ settings: SettingsStatic,
+ record_q: "Queue[pb.Record]",
+ result_q: "Queue[pb.Result]",
+ sender_q: "Queue[pb.Record]",
+ interface: InterfaceQueue,
+ context_keeper: context.ContextKeeper,
+ ):
+ self._settings = settings
+ self._record_q = record_q
+ self._result_q = result_q
+ self._sender_q = sender_q
+ self._interface = interface
+ self._context_keeper = context_keeper
+
+ # TODO(cancel_paused): implement me
+ # self._sender_cancel_set = set()
+
+ self._ds = None
+ self._flow_control = None
+ self._status_report = None
+ self._record_num = 0
+ self._telemetry_obj = tpb.TelemetryRecord()
+ self._telemetry_overflow = False
+ self._use_flow_control = not (
+ self._settings._flow_control_disabled or self._settings._offline
+ )
+
+ def open(self) -> None:
+ self._ds = datastore.DataStore()
+ self._ds.open_for_write(self._settings.sync_file)
+ self._flow_control = flow_control.FlowControl(
+ settings=self._settings,
+ write_record=self._write_record,
+ forward_record=self._forward_record,
+ pause_marker=self._pause_marker,
+ recover_records=self._recover_records,
+ )
+
+ def _forward_record(self, record: "pb.Record") -> None:
+ self._context_keeper.add_from_record(record)
+ tracelog.log_message_queue(record, self._sender_q)
+ self._sender_q.put(record)
+
+ def _send_mark(self) -> None:
+ sender_mark = pb.SenderMarkRequest()
+ record = self._interface._make_request(sender_mark=sender_mark)
+ self._forward_record(record)
+
+ def _maybe_send_telemetry(self) -> None:
+ if self._telemetry_overflow:
+ return
+ self._telemetry_overflow = True
+ with telemetry.context(obj=self._telemetry_obj) as tel:
+ tel.feature.flow_control_overflow = True
+ telemetry_record = pb.TelemetryRecordRequest(telemetry=self._telemetry_obj)
+ record = self._interface._make_request(telemetry_record=telemetry_record)
+ self._forward_record(record)
+
+ def _pause_marker(self) -> None:
+ self._maybe_send_telemetry()
+ self._send_mark()
+
+ def _write_record(self, record: "pb.Record") -> int:
+ assert self._ds
+
+ self._record_num += 1
+ proto_util._assign_record_num(record, self._record_num)
+ ret = self._ds.write(record)
+ assert ret is not None
+
+ _start_offset, end_offset, _flush_offset = ret
+ proto_util._assign_end_offset(record, end_offset)
+ return end_offset
+
+ def _ensure_flushed(self, offset: int) -> None:
+ if self._ds:
+ self._ds.ensure_flushed(offset)
+
+ def _recover_records(self, start: int, end: int) -> None:
+ sender_read = pb.SenderReadRequest(start_offset=start, final_offset=end)
+ # TODO(cancel_paused): implement me
+ # for cancel_id in self._sender_cancel_set:
+ # sender_read.cancel_list.append(cancel_id)
+ record = self._interface._make_request(sender_read=sender_read)
+ self._ensure_flushed(end)
+ self._forward_record(record)
+
+ def _write(self, record: "pb.Record") -> None:
+ if not self._ds:
+ self.open()
+ assert self._flow_control
+
+ if not record.control.local:
+ self._write_record(record)
+
+ if self._use_flow_control:
+ self._flow_control.flow(record)
+ elif not self._settings._offline or record.control.always_send:
+ # when flow_control is disabled we pass through all records to
+ # the sender as long as we are online. The exception is there
+ # are special records that we always pass to the sender
+ # (namely the exit record so we can trigger the defer shutdown
+ # state machine)
+ self._forward_record(record)
+
+ def write(self, record: "pb.Record") -> None:
+ record_type = record.WhichOneof("record_type")
+ assert record_type
+ writer_str = "write_" + record_type
+ write_handler: Callable[[pb.Record], None] = getattr(
+ self, writer_str, self._write
+ )
+ write_handler(record)
+
+ def write_request(self, record: "pb.Record") -> None:
+ request_type = record.request.WhichOneof("request_type")
+ assert request_type
+ write_request_str = "write_request_" + request_type
+ write_request_handler: Optional[Callable[[pb.Record], None]] = getattr(
+ self, write_request_str, None
+ )
+ if write_request_handler:
+ return write_request_handler(record)
+ self._write(record)
+
+ def write_request_run_status(self, record: "pb.Record") -> None:
+ result = proto_util._result_from_record(record)
+ if self._status_report:
+ result.response.run_status_response.sync_time.CopyFrom(
+ self._status_report.sync_time
+ )
+ send_record_num = self._status_report.record_num
+ result.response.run_status_response.sync_items_total = self._record_num
+ result.response.run_status_response.sync_items_pending = (
+ self._record_num - send_record_num
+ )
+ self._respond_result(result)
+
+ def write_request_status_report(self, record: "pb.Record") -> None:
+ self._status_report = record.request.status_report
+ self._write(record)
+
+ def write_request_cancel(self, record: "pb.Record") -> None:
+ cancel_id = record.request.cancel.cancel_slot
+ self._context_keeper.cancel(cancel_id)
+
+ # TODO(cancel_paused): implement me
+ # cancelled = self._context_keeper.cancel(cancel_id)
+ # if not cancelled:
+ # self._sender_cancel_set.add(cancel_id)
+
+ def _respond_result(self, result: "pb.Result") -> None:
+ tracelog.log_message_queue(result, self._result_q)
+ self._result_q.put(result)
+
+ def finish(self) -> None:
+ if self._flow_control:
+ self._flow_control.flush()
+ if self._ds:
+ self._ds.close()
+ # TODO(debug_context) see context.py
+ # self._context_keeper._debug_print_orphans(print_to_stdout=self._settings._debug)
+
+ def debounce(self) -> None:
+ pass
diff --git a/wandb/sdk/launch/__init__.py b/wandb/sdk/launch/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..baa5936bbd9117f7bcf14369fbea510018474317
--- /dev/null
+++ b/wandb/sdk/launch/__init__.py
@@ -0,0 +1,6 @@
+from ._launch import launch
+from ._launch_add import launch_add
+from .agent.agent import LaunchAgent
+from .utils import load_wandb_config
+
+__all__ = ["LaunchAgent", "launch", "launch_add", "load_wandb_config"]
diff --git a/wandb/sdk/launch/_launch.py b/wandb/sdk/launch/_launch.py
new file mode 100644
index 0000000000000000000000000000000000000000..9d9f8419400b550b2d1ef278a81580abba7f5ee5
--- /dev/null
+++ b/wandb/sdk/launch/_launch.py
@@ -0,0 +1,343 @@
+import asyncio
+import logging
+import os
+import sys
+from typing import Any, Dict, List, Optional, Tuple
+
+import yaml
+
+import wandb
+from wandb.apis.internal import Api
+
+from . import loader
+from ._project_spec import create_project_from_spec, fetch_and_validate_project
+from .agent import LaunchAgent
+from .builder.build import construct_agent_configs
+from .environment.local_environment import LocalEnvironment
+from .errors import ExecutionError, LaunchError
+from .runner.abstract import AbstractRun
+from .utils import (
+ LAUNCH_CONFIG_FILE,
+ LAUNCH_DEFAULT_PROJECT,
+ PROJECT_SYNCHRONOUS,
+ construct_launch_spec,
+ validate_launch_spec_source,
+)
+
+_logger = logging.getLogger(__name__)
+
+
+def set_launch_logfile(logfile: str) -> None:
+ """Set the logfile for the launch agent."""
+ # Get logger of parent module
+ _launch_logger = logging.getLogger("wandb.sdk.launch")
+ if logfile == "-":
+ logfile_stream = sys.stdout
+ else:
+ try:
+ logfile_stream = open(logfile, "w")
+ # check if file is writable
+ except Exception as e:
+ wandb.termerror(
+ f"Could not open {logfile} for writing logs. Please check "
+ f"the path and permissions.\nError: {e}"
+ )
+ return
+
+ wandb.termlog(
+ f"Internal agent logs printing to {'stdout' if logfile == '-' else logfile}. "
+ )
+ handler = logging.StreamHandler(logfile_stream)
+ handler.formatter = logging.Formatter(
+ "%(asctime)s %(levelname)-7s %(threadName)-10s:%(process)d "
+ "[%(filename)s:%(funcName)s():%(lineno)s] %(message)s"
+ )
+ _launch_logger.addHandler(handler)
+ _launch_logger.log(logging.INFO, "Internal agent logs printing to %s", logfile)
+
+
+def resolve_agent_config( # noqa: C901
+ entity: Optional[str],
+ project: Optional[str],
+ max_jobs: Optional[int],
+ queues: Optional[Tuple[str]],
+ config: Optional[str],
+) -> Tuple[Dict[str, Any], Api]:
+ """Resolve the agent config.
+
+ Arguments:
+ api (Api): The api.
+ entity (str): The entity.
+ project (str): The project.
+ max_jobs (int): The max number of jobs.
+ queues (Tuple[str]): The queues.
+ config (str): The config.
+
+ Returns:
+ Tuple[Dict[str, Any], Api]: The resolved config and api.
+ """
+ defaults = {
+ "project": LAUNCH_DEFAULT_PROJECT,
+ "max_jobs": 1,
+ "max_schedulers": 1,
+ "queues": [],
+ "registry": {},
+ "builder": {},
+ }
+ user_set_project = False
+ resolved_config: Dict[str, Any] = defaults
+ config_path = config or os.path.expanduser(LAUNCH_CONFIG_FILE)
+ if os.path.isfile(config_path):
+ launch_config = {}
+ with open(config_path) as f:
+ try:
+ launch_config = yaml.safe_load(f)
+ # This is considered unreachable by mypy, but it's not.
+ if launch_config is None:
+ launch_config = {} # type: ignore
+ except yaml.YAMLError as e:
+ raise LaunchError(f"Invalid launch agent config: {e}")
+ if launch_config.get("project") is not None:
+ user_set_project = True
+ resolved_config.update(launch_config.items())
+ elif config is not None:
+ raise LaunchError(
+ f"Could not find use specified launch config file: {config_path}"
+ )
+ if os.environ.get("WANDB_PROJECT") is not None:
+ resolved_config.update({"project": os.environ.get("WANDB_PROJECT")})
+ user_set_project = True
+ if os.environ.get("WANDB_ENTITY") is not None:
+ resolved_config.update({"entity": os.environ.get("WANDB_ENTITY")})
+ if os.environ.get("WANDB_LAUNCH_MAX_JOBS") is not None:
+ resolved_config.update(
+ {"max_jobs": int(os.environ.get("WANDB_LAUNCH_MAX_JOBS", 1))}
+ )
+
+ if project is not None:
+ resolved_config.update({"project": project})
+ user_set_project = True
+ if entity is not None:
+ resolved_config.update({"entity": entity})
+ if max_jobs is not None:
+ resolved_config.update({"max_jobs": int(max_jobs)})
+ if queues:
+ resolved_config.update({"queues": list(queues)})
+ # queue -> queues
+ if resolved_config.get("queue"):
+ if isinstance(resolved_config.get("queue"), str):
+ resolved_config["queues"].append(resolved_config["queue"])
+ else:
+ raise LaunchError(
+ f"Invalid launch agent config for key 'queue' with type: {type(resolved_config.get('queue'))}"
+ + " (expected str). Specify multiple queues with the 'queues' key"
+ )
+
+ keys = ["project", "entity"]
+ settings = {
+ k: resolved_config.get(k) for k in keys if resolved_config.get(k) is not None
+ }
+
+ api = Api(default_settings=settings)
+
+ if resolved_config.get("entity") is None:
+ resolved_config.update({"entity": api.default_entity})
+ if user_set_project:
+ wandb.termwarn(
+ "Specifying a project for the launch agent is deprecated. Please use queues found in the Launch application at https://wandb.ai/launch."
+ )
+
+ return resolved_config, api
+
+
+def create_and_run_agent(
+ api: Api,
+ config: Dict[str, Any],
+) -> None:
+ try:
+ from wandb.sdk.launch.agent import config as agent_config
+ except ModuleNotFoundError:
+ raise LaunchError(
+ "wandb launch-agent requires pydantic to be installed. "
+ "Please install with `pip install wandb[launch]`"
+ )
+ try:
+ agent_config.AgentConfig(**config)
+ except agent_config.ValidationError as e:
+ errors = e.errors()
+ for error in errors:
+ loc = ".".join([str(x) for x in error.get("loc", [])])
+ msg = f"Agent config error in field {loc}"
+ value = error.get("input")
+ if not isinstance(value, dict):
+ msg += f" (value: {value})"
+ msg += f": {error['msg']}"
+ wandb.termerror(msg)
+ raise LaunchError("Invalid launch agent config")
+ agent = LaunchAgent(api, config)
+ try:
+ asyncio.run(agent.loop())
+ except asyncio.CancelledError:
+ pass
+
+
+async def _launch(
+ api: Api,
+ uri: Optional[str] = None,
+ job: Optional[str] = None,
+ name: Optional[str] = None,
+ project: Optional[str] = None,
+ entity: Optional[str] = None,
+ docker_image: Optional[str] = None,
+ entry_point: Optional[List[str]] = None,
+ version: Optional[str] = None,
+ resource: Optional[str] = None,
+ resource_args: Optional[Dict[str, Any]] = None,
+ launch_config: Optional[Dict[str, Any]] = None,
+ synchronous: Optional[bool] = None,
+ run_id: Optional[str] = None,
+ repository: Optional[str] = None,
+) -> AbstractRun:
+ """Helper that delegates to the project-running method corresponding to the passed-in backend."""
+ if launch_config is None:
+ launch_config = {}
+ if resource is None:
+ resource = "local-container"
+ launch_spec = construct_launch_spec(
+ uri,
+ job,
+ api,
+ name,
+ project,
+ entity,
+ docker_image,
+ resource,
+ entry_point,
+ version,
+ resource_args,
+ launch_config,
+ run_id,
+ repository,
+ author=None,
+ )
+ validate_launch_spec_source(launch_spec)
+ launch_project = create_project_from_spec(launch_spec, api)
+ launch_project = fetch_and_validate_project(launch_project, api)
+ entrypoint = launch_project.get_single_entry_point()
+ image_uri = launch_project.docker_image # Either set by user or None.
+
+ # construct runner config.
+ runner_config: Dict[str, Any] = {}
+ runner_config[PROJECT_SYNCHRONOUS] = synchronous
+
+ config = launch_config or {}
+ environment_config, build_config, registry_config = construct_agent_configs(config)
+ environment = loader.environment_from_config(environment_config)
+ if environment is not None and not isinstance(environment, LocalEnvironment):
+ await environment.verify()
+ registry = loader.registry_from_config(registry_config, environment)
+ builder = loader.builder_from_config(build_config, environment, registry)
+ if not launch_project.docker_image:
+ assert entrypoint
+ image_uri = await builder.build_image(launch_project, entrypoint, None)
+ backend = loader.runner_from_config(
+ resource, api, runner_config, environment, registry
+ )
+ if backend:
+ assert image_uri
+ submitted_run = await backend.run(launch_project, image_uri)
+ # this check will always pass, run is only optional in the agent case where
+ # a run queue id is present on the backend config
+ assert submitted_run
+ return submitted_run
+ else:
+ raise ExecutionError(
+ f"Unavailable backend {resource}, available backends: {', '.join(loader.WANDB_RUNNERS)}"
+ )
+
+
+def launch(
+ api: Api,
+ job: Optional[str] = None,
+ entry_point: Optional[List[str]] = None,
+ version: Optional[str] = None,
+ name: Optional[str] = None,
+ resource: Optional[str] = None,
+ resource_args: Optional[Dict[str, Any]] = None,
+ project: Optional[str] = None,
+ entity: Optional[str] = None,
+ docker_image: Optional[str] = None,
+ config: Optional[Dict[str, Any]] = None,
+ synchronous: Optional[bool] = True,
+ run_id: Optional[str] = None,
+ repository: Optional[str] = None,
+) -> AbstractRun:
+ """Launch a W&B launch experiment.
+
+ Arguments:
+ job: string reference to a wandb.Job eg: wandb/test/my-job:latest
+ api: An instance of a wandb Api from wandb.apis.internal.
+ entry_point: Entry point to run within the project. Defaults to using the entry point used
+ in the original run for wandb URIs, or main.py for git repository URIs.
+ version: For Git-based projects, either a commit hash or a branch name.
+ name: Name run under which to launch the run.
+ resource: Execution backend for the run.
+ resource_args: Resource related arguments for launching runs onto a remote backend.
+ Will be stored on the constructed launch config under ``resource_args``.
+ project: Target project to send launched run to
+ entity: Target entity to send launched run to
+ config: A dictionary containing the configuration for the run. May also contain
+ resource specific arguments under the key "resource_args".
+ synchronous: Whether to block while waiting for a run to complete. Defaults to True.
+ Note that if ``synchronous`` is False and ``backend`` is "local-container", this
+ method will return, but the current process will block when exiting until
+ the local run completes. If the current process is interrupted, any
+ asynchronous runs launched via this method will be terminated. If
+ ``synchronous`` is True and the run fails, the current process will
+ error out as well.
+ run_id: ID for the run (To ultimately replace the :name: field)
+ repository: string name of repository path for remote registry
+
+ Example:
+ ```python
+ from wandb.sdk.launch import launch
+
+ job = "wandb/jobs/Hello World:latest"
+ params = {"epochs": 5}
+ # Run W&B project and create a reproducible docker environment
+ # on a local host
+ api = wandb.apis.internal.Api()
+ launch(api, job, parameters=params)
+ ```
+
+
+ Returns:
+ an instance of`wandb.launch.SubmittedRun` exposing information (e.g. run ID)
+ about the launched run.
+
+ Raises:
+ `wandb.exceptions.ExecutionError` If a run launched in blocking mode
+ is unsuccessful.
+ """
+ submitted_run_obj = asyncio.run(
+ _launch(
+ # TODO: fully deprecate URI path
+ uri=None,
+ job=job,
+ name=name,
+ project=project,
+ entity=entity,
+ docker_image=docker_image,
+ entry_point=entry_point,
+ version=version,
+ resource=resource,
+ resource_args=resource_args,
+ launch_config=config,
+ synchronous=synchronous,
+ api=api,
+ run_id=run_id,
+ repository=repository,
+ )
+ )
+
+ return submitted_run_obj
diff --git a/wandb/sdk/launch/_launch_add.py b/wandb/sdk/launch/_launch_add.py
new file mode 100644
index 0000000000000000000000000000000000000000..e178cf35d8a92d725989f2e4c7a6ddac9d93db1b
--- /dev/null
+++ b/wandb/sdk/launch/_launch_add.py
@@ -0,0 +1,258 @@
+import asyncio
+import pprint
+from typing import Any, Dict, List, Optional, Union
+
+import wandb
+import wandb.apis.public as public
+from wandb.apis.internal import Api
+from wandb.errors import CommError
+from wandb.sdk.launch._project_spec import create_project_from_spec
+from wandb.sdk.launch.builder.build import build_image_from_project
+from wandb.sdk.launch.errors import LaunchError
+from wandb.sdk.launch.utils import (
+ LAUNCH_DEFAULT_PROJECT,
+ LOG_PREFIX,
+ construct_launch_spec,
+ validate_launch_spec_source,
+)
+
+
+def push_to_queue(
+ api: Api,
+ queue_name: str,
+ launch_spec: Dict[str, Any],
+ template_variables: Optional[dict],
+ project_queue: str,
+ priority: Optional[int] = None,
+) -> Any:
+ return api.push_to_run_queue(
+ queue_name, launch_spec, template_variables, project_queue, priority
+ )
+
+
+def launch_add(
+ uri: Optional[str] = None,
+ job: Optional[str] = None,
+ config: Optional[Dict[str, Any]] = None,
+ template_variables: Optional[Dict[str, Union[float, int, str]]] = None,
+ project: Optional[str] = None,
+ entity: Optional[str] = None,
+ queue_name: Optional[str] = None,
+ resource: Optional[str] = None,
+ entry_point: Optional[List[str]] = None,
+ name: Optional[str] = None,
+ version: Optional[str] = None,
+ docker_image: Optional[str] = None,
+ project_queue: Optional[str] = None,
+ resource_args: Optional[Dict[str, Any]] = None,
+ run_id: Optional[str] = None,
+ build: Optional[bool] = False,
+ repository: Optional[str] = None,
+ sweep_id: Optional[str] = None,
+ author: Optional[str] = None,
+ priority: Optional[int] = None,
+) -> "public.QueuedRun":
+ """Enqueue a W&B launch experiment. With either a source uri, job or docker_image.
+
+ Arguments:
+ uri: URI of experiment to run. A wandb run uri or a Git repository URI.
+ job: string reference to a wandb.Job eg: wandb/test/my-job:latest
+ config: A dictionary containing the configuration for the run. May also contain
+ resource specific arguments under the key "resource_args"
+ template_variables: A dictionary containing values of template variables for a run queue.
+ Expected format of {"<var-name>": <var-value>}
+ project: Target project to send launched run to
+ entity: Target entity to send launched run to
+ queue: the name of the queue to enqueue the run to
+ priority: the priority level of the job, where 1 is the highest priority
+ resource: Execution backend for the run: W&B provides built-in support for "local-container" backend
+ entry_point: Entry point to run within the project. Defaults to using the entry point used
+ in the original run for wandb URIs, or main.py for git repository URIs.
+ name: Name run under which to launch the run.
+ version: For Git-based projects, either a commit hash or a branch name.
+ docker_image: The name of the docker image to use for the run.
+ resource_args: Resource related arguments for launching runs onto a remote backend.
+ Will be stored on the constructed launch config under ``resource_args``.
+ run_id: optional string indicating the id of the launched run
+ build: optional flag defaulting to false, requires queue to be set
+ if build, an image is created, creates a job artifact, pushes a reference
+ to that job artifact to queue
+ repository: optional string to control the name of the remote repository, used when
+ pushing images to a registry
+ project_queue: optional string to control the name of the project for the queue. Primarily used
+ for back compatibility with project scoped queues
+
+
+ Example:
+ ```python
+ from wandb.sdk.launch import launch_add
+
+ project_uri = "https://github.com/wandb/examples"
+ params = {"alpha": 0.5, "l1_ratio": 0.01}
+ # Run W&B project and create a reproducible docker environment
+ # on a local host
+ api = wandb.apis.internal.Api()
+ launch_add(uri=project_uri, parameters=params)
+ ```
+
+
+ Returns:
+ an instance of`wandb.api.public.QueuedRun` which gives information about the
+ queued run, or if `wait_until_started` or `wait_until_finished` are called, gives access
+ to the underlying Run information.
+
+ Raises:
+ `wandb.exceptions.LaunchError` if unsuccessful
+ """
+ api = Api()
+
+ return asyncio.run(
+ _launch_add(
+ api,
+ uri,
+ job,
+ config,
+ template_variables,
+ project,
+ entity,
+ queue_name,
+ resource,
+ entry_point,
+ name,
+ version,
+ docker_image,
+ project_queue,
+ resource_args,
+ run_id=run_id,
+ build=build,
+ repository=repository,
+ sweep_id=sweep_id,
+ author=author,
+ priority=priority,
+ )
+ )
+
+
+async def _launch_add(
+ api: Api,
+ uri: Optional[str],
+ job: Optional[str],
+ config: Optional[Dict[str, Any]],
+ template_variables: Optional[dict],
+ project: Optional[str],
+ entity: Optional[str],
+ queue_name: Optional[str],
+ resource: Optional[str],
+ entry_point: Optional[List[str]],
+ name: Optional[str],
+ version: Optional[str],
+ docker_image: Optional[str],
+ project_queue: Optional[str],
+ resource_args: Optional[Dict[str, Any]] = None,
+ run_id: Optional[str] = None,
+ build: Optional[bool] = False,
+ repository: Optional[str] = None,
+ sweep_id: Optional[str] = None,
+ author: Optional[str] = None,
+ priority: Optional[int] = None,
+) -> "public.QueuedRun":
+ launch_spec = construct_launch_spec(
+ uri,
+ job,
+ api,
+ name,
+ project,
+ entity,
+ docker_image,
+ resource,
+ entry_point,
+ version,
+ resource_args,
+ config,
+ run_id,
+ repository,
+ author,
+ sweep_id,
+ )
+
+ if build:
+ if resource == "local-process":
+ raise LaunchError(
+ "Cannot build a docker image for the resource: local-process"
+ )
+
+ if launch_spec.get("job") is not None:
+ wandb.termwarn("Build doesn't support setting a job. Overwriting job.")
+ launch_spec["job"] = None
+
+ launch_project = create_project_from_spec(launch_spec, api)
+ docker_image_uri = await build_image_from_project(
+ launch_project, api, config or {}
+ )
+ run = wandb.run or wandb.init(
+ project=launch_spec["project"],
+ entity=launch_spec["entity"],
+ job_type="launch_job",
+ )
+
+ job_artifact = run._log_job_artifact_with_image( # type: ignore
+ docker_image_uri, launch_project.override_args
+ )
+ job_name = job_artifact.wait().name
+
+ job = f"{launch_spec['entity']}/{launch_spec['project']}/{job_name}"
+ launch_spec["job"] = job
+ launch_spec["uri"] = None # Remove given URI --> now in job
+
+ if queue_name is None:
+ queue_name = "default"
+ if project_queue is None:
+ project_queue = LAUNCH_DEFAULT_PROJECT
+ spec_template_vars = launch_spec.get("template_variables")
+ if isinstance(spec_template_vars, dict):
+ launch_spec.pop("template_variables")
+ if template_variables is None:
+ template_variables = spec_template_vars
+ else:
+ template_variables = {
+ **spec_template_vars,
+ **template_variables,
+ }
+
+ validate_launch_spec_source(launch_spec)
+ res = push_to_queue(
+ api, queue_name, launch_spec, template_variables, project_queue, priority
+ )
+
+ if res is None or "runQueueItemId" not in res:
+ raise LaunchError("Error adding run to queue")
+
+ updated_spec = res.get("runSpec")
+ if updated_spec:
+ if updated_spec.get("resource_args"):
+ launch_spec["resource_args"] = updated_spec.get("resource_args")
+ if updated_spec.get("resource"):
+ launch_spec["resource"] = updated_spec.get("resource")
+
+ if project_queue == LAUNCH_DEFAULT_PROJECT:
+ wandb.termlog(f"{LOG_PREFIX}Added run to queue {queue_name}.")
+ else:
+ wandb.termlog(f"{LOG_PREFIX}Added run to queue {project_queue}/{queue_name}.")
+ wandb.termlog(f"{LOG_PREFIX}Launch spec:\n{pprint.pformat(launch_spec)}\n")
+
+ public_api = public.Api()
+ if job is not None:
+ try:
+ public_api.artifact(job, type="job")
+ except (ValueError, CommError) as e:
+ raise LaunchError(f"Unable to fetch job with name {job}: {e}")
+
+ queued_run = public_api.queued_run(
+ launch_spec["entity"],
+ launch_spec["project"],
+ queue_name,
+ res["runQueueItemId"],
+ project_queue,
+ priority,
+ )
+ return queued_run # type: ignore
diff --git a/wandb/sdk/launch/_project_spec.py b/wandb/sdk/launch/_project_spec.py
new file mode 100644
index 0000000000000000000000000000000000000000..c0764e5305f3c9ed994d4cb2b17974202dc17892
--- /dev/null
+++ b/wandb/sdk/launch/_project_spec.py
@@ -0,0 +1,579 @@
+"""Convert launch arguments into a runnable wandb launch script.
+
+Arguments can come from a launch spec or call to wandb launch.
+"""
+import enum
+import json
+import logging
+import os
+import tempfile
+from copy import deepcopy
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, cast
+
+import wandb
+import wandb.docker as docker
+from wandb.apis.internal import Api
+from wandb.errors import CommError
+from wandb.sdk.launch import utils
+from wandb.sdk.lib.runid import generate_id
+
+from .errors import LaunchError
+from .utils import LOG_PREFIX, recursive_macro_sub
+
+if TYPE_CHECKING:
+ from wandb.sdk.artifacts.artifact import Artifact
+
+_logger = logging.getLogger(__name__)
+
+DEFAULT_LAUNCH_METADATA_PATH = "launch_metadata.json"
+
+# need to make user root for sagemaker, so users have access to /opt/ml directories
+# that let users create artifacts and access input data
+RESOURCE_UID_MAP = {"local": 1000, "sagemaker": 0}
+IMAGE_TAG_MAX_LENGTH = 32
+
+
+class LaunchSource(enum.IntEnum):
+ WANDB: int = 1
+ GIT: int = 2
+ LOCAL: int = 3
+ DOCKER: int = 4
+ JOB: int = 5
+
+
+class EntrypointDefaults(List[str]):
+ PYTHON = ["python", "main.py"]
+
+
+class LaunchProject:
+ """A launch project specification."""
+
+ def __init__(
+ self,
+ uri: Optional[str],
+ job: Optional[str],
+ api: Api,
+ launch_spec: Dict[str, Any],
+ target_entity: str,
+ target_project: str,
+ name: Optional[str],
+ docker_config: Dict[str, Any],
+ git_info: Dict[str, str],
+ overrides: Dict[str, Any],
+ resource: str,
+ resource_args: Dict[str, Any],
+ run_id: Optional[str],
+ sweep_id: Optional[str] = None,
+ ):
+ if uri is not None and utils.is_bare_wandb_uri(uri):
+ uri = api.settings("base_url") + uri
+ _logger.info(f"{LOG_PREFIX}Updating uri with base uri: {uri}")
+ self.uri = uri
+ self.job = job
+ if job is not None:
+ wandb.termlog(f"{LOG_PREFIX}Launching job: {job}")
+ self._job_artifact: Optional[Artifact] = None
+ self.api = api
+ self.launch_spec = launch_spec
+ self.target_entity = target_entity
+ self.target_project = target_project.lower()
+ self.name = name # TODO: replace with run_id
+ # the builder key can be passed in through the resource args
+ # but these resource_args are then passed to the appropriate
+ # runner, so we need to pop the builder key out
+ resource_args_copy = deepcopy(resource_args)
+ resource_args_build = resource_args_copy.get(resource, {}).pop("builder", {})
+ self.resource = resource
+ self.resource_args = resource_args_copy
+ self.sweep_id = sweep_id
+ self.author = launch_spec.get("author")
+ self.python_version: Optional[str] = launch_spec.get("python_version")
+ self.accelerator_base_image: Optional[str] = resource_args_build.get(
+ "accelerator", {}
+ ).get("base_image") or resource_args_build.get("cuda", {}).get("base_image")
+ self._base_image: Optional[str] = launch_spec.get("base_image")
+ self.docker_image: Optional[str] = docker_config.get(
+ "docker_image"
+ ) or launch_spec.get("image_uri")
+ uid = RESOURCE_UID_MAP.get(resource, 1000)
+ if self._base_image:
+ uid = docker.get_image_uid(self._base_image)
+ _logger.info(f"{LOG_PREFIX}Retrieved base image uid {uid}")
+ self.docker_user_id: int = docker_config.get("user_id", uid)
+ self.git_version: Optional[str] = git_info.get("version")
+ self.git_repo: Optional[str] = git_info.get("repo")
+ self.overrides = overrides
+ self.override_args: List[str] = overrides.get("args", [])
+ self.override_config: Dict[str, Any] = overrides.get("run_config", {})
+ self.override_artifacts: Dict[str, Any] = overrides.get("artifacts", {})
+ self.override_entrypoint: Optional[EntryPoint] = None
+ self.override_dockerfile: Optional[str] = overrides.get("dockerfile")
+ self.deps_type: Optional[str] = None
+ self._runtime: Optional[str] = None
+ self.run_id = run_id or generate_id()
+ self._queue_name: Optional[str] = None
+ self._queue_entity: Optional[str] = None
+ self._run_queue_item_id: Optional[str] = None
+ self._entry_point: Optional[
+ EntryPoint
+ ] = None # todo: keep multiple entrypoint support?
+
+ override_entrypoint = overrides.get("entry_point")
+ if override_entrypoint:
+ _logger.info("Adding override entry point")
+ self.override_entrypoint = EntryPoint(
+ name=self._get_entrypoint_file(override_entrypoint),
+ command=override_entrypoint,
+ )
+
+ if overrides.get("sweep_id") is not None:
+ _logger.info("Adding override sweep id")
+ self.sweep_id = overrides["sweep_id"]
+ if self.docker_image is not None:
+ self.source = LaunchSource.DOCKER
+ self.project_dir = None
+ elif self.job is not None:
+ self.source = LaunchSource.JOB
+ self.project_dir = tempfile.mkdtemp()
+ elif self.uri is not None and utils._is_wandb_uri(self.uri):
+ _logger.info(f"URI {self.uri} indicates a wandb uri")
+ self.source = LaunchSource.WANDB
+ self.project_dir = tempfile.mkdtemp()
+ elif self.uri is not None and utils._is_git_uri(self.uri):
+ _logger.info(f"URI {self.uri} indicates a git uri")
+ self.source = LaunchSource.GIT
+ self.project_dir = tempfile.mkdtemp()
+ elif self.uri is not None and "placeholder-" in self.uri:
+ wandb.termlog(
+ f"{LOG_PREFIX}Launch received placeholder URI, replacing with local path."
+ )
+ self.uri = os.getcwd()
+ self.source = LaunchSource.LOCAL
+ self.project_dir = self.uri
+ else:
+ _logger.info(f"URI {self.uri} indicates a local uri")
+ # assume local
+ if self.uri is not None and not os.path.exists(self.uri):
+ raise LaunchError(
+ "Assumed URI supplied is a local path but path is not valid"
+ )
+ self.source = LaunchSource.LOCAL
+ self.project_dir = self.uri
+
+ self.aux_dir = tempfile.mkdtemp()
+
+ @property
+ def base_image(self) -> str:
+ """Returns {PROJECT}_base:{PYTHON_VERSION}."""
+ # TODO: this should likely be source_project when we have it...
+
+ # don't make up a separate base image name if user provides a docker image
+ if self.docker_image is not None:
+ return self.docker_image
+
+ python_version = (self.python_version or "3").replace("+", "dev")
+ generated_name = "{}_base:{}".format(
+ self.target_project.replace(" ", "-"), python_version
+ )
+ return self._base_image or generated_name
+
+ @property
+ def image_name(self) -> str:
+ if self.docker_image is not None:
+ return self.docker_image
+ elif self.uri is not None:
+ cleaned_uri = self.uri.replace("https://", "/")
+ first_sep = cleaned_uri.find("/")
+ shortened_uri = cleaned_uri[first_sep:]
+ return wandb.util.make_docker_image_name_safe(shortened_uri)
+ else:
+ # this will always pass since one of these 3 is required
+ assert self.job is not None
+ return wandb.util.make_docker_image_name_safe(self.job.split(":")[0])
+
+ @property
+ def queue_name(self) -> Optional[str]:
+ return self._queue_name
+
+ @queue_name.setter
+ def queue_name(self, value: str) -> None:
+ self._queue_name = value
+
+ @property
+ def queue_entity(self) -> Optional[str]:
+ return self._queue_entity
+
+ @queue_entity.setter
+ def queue_entity(self, value: str) -> None:
+ self._queue_entity = value
+
+ @property
+ def run_queue_item_id(self) -> Optional[str]:
+ return self._run_queue_item_id
+
+ @run_queue_item_id.setter
+ def run_queue_item_id(self, value: str) -> None:
+ self._run_queue_item_id = value
+
+ def _get_entrypoint_file(self, entrypoint: List[str]) -> Optional[str]:
+ if not entrypoint:
+ return None
+ if entrypoint[0].endswith(".py") or entrypoint[0].endswith(".sh"):
+ return entrypoint[0]
+ if len(entrypoint) < 2:
+ return None
+ return entrypoint[1]
+
+ def fill_macros(self, image: str) -> Dict[str, Any]:
+ """Substitute values for macros in resource arguments.
+
+ Certain macros can be used in resource args. These macros allow the
+ user to set resource args dynamically in the context of the
+ run being launched. The macros are given in the ${macro} format. The
+ following macros are currently supported:
+
+ ${project_name} - the name of the project the run is being launched to.
+ ${entity_name} - the owner of the project the run being launched to.
+ ${run_id} - the id of the run being launched.
+ ${run_name} - the name of the run that is launching.
+ ${image_uri} - the URI of the container image for this run.
+
+ Additionally, you may use ${<ENV-VAR-NAME>} to refer to the value of any
+ environment variables that you plan to set in the environment of any
+ agents that will receive these resource args.
+
+ Calling this method will overwrite the contents of self.resource_args
+ with the substituted values.
+
+ Args:
+ image (str): The image name to fill in for ${wandb-image}.
+
+ Returns:
+ None
+ """
+ update_dict = {
+ "project_name": self.target_project,
+ "entity_name": self.target_entity,
+ "run_id": self.run_id,
+ "run_name": self.name,
+ "image_uri": image,
+ "author": self.author,
+ }
+ update_dict.update(os.environ)
+ result = recursive_macro_sub(self.resource_args, update_dict)
+ # recursive_macro_sub given a dict returns a dict with the same keys
+ # but with other input types behaves differently. The cast is for mypy.
+ return cast(Dict[str, Any], result)
+
+ def build_required(self) -> bool:
+ """Checks the source to see if a build is required."""
+ # since the image tag for images built from jobs
+ # is based on the job version index, which is immutable
+ # we don't need to build the image for a job if that tag
+ # already exists
+ if self.source != LaunchSource.JOB:
+ return True
+ return False
+
+ @property
+ def docker_image(self) -> Optional[str]:
+ return self._docker_image
+
+ @docker_image.setter
+ def docker_image(self, value: str) -> None:
+ self._docker_image = value
+ self._ensure_not_docker_image_and_local_process()
+
+ def get_single_entry_point(self) -> Optional["EntryPoint"]:
+ """Returns the first entrypoint for the project, or None if no entry point was provided because a docker image was provided."""
+ # assuming project only has 1 entry point, pull that out
+ # tmp fn until we figure out if we want to support multiple entry points or not
+ if not self._entry_point:
+ if not self.docker_image:
+ raise LaunchError(
+ "Project must have at least one entry point unless docker image is specified."
+ )
+ return None
+ return self._entry_point
+
+ def set_entry_point(self, command: List[str]) -> "EntryPoint":
+ """Add an entry point to the project."""
+ assert (
+ self._entry_point is None
+ ), "Cannot set entry point twice. Use LaunchProject.override_entrypoint"
+ new_entrypoint = EntryPoint(name=command[-1], command=command)
+ self._entry_point = new_entrypoint
+ return new_entrypoint
+
+ def _ensure_not_docker_image_and_local_process(self) -> None:
+ if self.docker_image is not None and self.resource == "local-process":
+ raise LaunchError(
+ "Cannot specify docker image with local-process resource runner"
+ )
+
+ def _fetch_job(self) -> None:
+ public_api = wandb.apis.public.Api()
+ job_dir = tempfile.mkdtemp()
+ try:
+ job = public_api.job(self.job, path=job_dir)
+ except CommError as e:
+ msg = e.message
+ raise LaunchError(
+ f"Error accessing job {self.job}: {msg} on {public_api.settings.get('base_url')}"
+ )
+ job.configure_launch_project(self)
+ self._job_artifact = job._job_artifact
+
+ def get_image_source_string(self) -> str:
+ """Returns a unique string identifying the source of an image."""
+ if self.source == LaunchSource.LOCAL:
+ # TODO: more correct to get a hash of local uri contents
+ assert isinstance(self.uri, str)
+ return self.uri
+ elif self.source == LaunchSource.JOB:
+ assert self._job_artifact is not None
+ return f"{self._job_artifact.name}:v{self._job_artifact.version}"
+ elif self.source == LaunchSource.GIT:
+ assert isinstance(self.uri, str)
+ ret = self.uri
+ if self.git_version:
+ ret += self.git_version
+ return ret
+ elif self.source == LaunchSource.WANDB:
+ assert isinstance(self.uri, str)
+ return self.uri
+ elif self.source == LaunchSource.DOCKER:
+ assert isinstance(self.docker_image, str)
+ _logger.debug("")
+ return self.docker_image
+ else:
+ raise LaunchError("Unknown source type when determing image source string")
+
+ def _fetch_project_local(self, internal_api: Api) -> None:
+ """Fetch a project (either wandb run or git repo) into a local directory, returning the path to the local project directory."""
+ # these asserts are all guaranteed to pass, but are required by mypy
+ assert self.source != LaunchSource.LOCAL and self.source != LaunchSource.JOB
+ assert isinstance(self.uri, str)
+ assert self.project_dir is not None
+ _logger.info("Fetching project locally...")
+ if utils._is_wandb_uri(self.uri):
+ source_entity, source_project, source_run_name = utils.parse_wandb_uri(
+ self.uri
+ )
+ run_info = utils.fetch_wandb_project_run_info(
+ source_entity, source_project, source_run_name, internal_api
+ )
+ program_name = run_info.get("codePath") or run_info["program"]
+
+ self.python_version = run_info.get("python", "3")
+ downloaded_code_artifact = utils.check_and_download_code_artifacts(
+ source_entity,
+ source_project,
+ source_run_name,
+ internal_api,
+ self.project_dir,
+ )
+ if not downloaded_code_artifact:
+ if not run_info["git"]:
+ raise LaunchError(
+ "Reproducing a run requires either an associated git repo or a code artifact logged with `run.log_code()`"
+ )
+ branch_name = utils._fetch_git_repo(
+ self.project_dir,
+ run_info["git"]["remote"],
+ run_info["git"]["commit"],
+ )
+ if self.git_version is None:
+ self.git_version = branch_name
+ patch = utils.fetch_project_diff(
+ source_entity, source_project, source_run_name, internal_api
+ )
+ if patch:
+ utils.apply_patch(patch, self.project_dir)
+
+ # For cases where the entry point wasn't checked into git
+ if not os.path.exists(os.path.join(self.project_dir, program_name)):
+ downloaded_entrypoint = utils.download_entry_point(
+ source_entity,
+ source_project,
+ source_run_name,
+ internal_api,
+ program_name,
+ self.project_dir,
+ )
+
+ if not downloaded_entrypoint:
+ raise LaunchError(
+ f"Entrypoint file: {program_name} does not exist, "
+ "and could not be downloaded. Please specify the entrypoint for this run."
+ )
+
+ if (
+ "_session_history.ipynb" in os.listdir(self.project_dir)
+ or ".ipynb" in program_name
+ ):
+ program_name = utils.convert_jupyter_notebook_to_script(
+ program_name, self.project_dir
+ )
+
+ # Download any frozen requirements
+ utils.download_wandb_python_deps(
+ source_entity,
+ source_project,
+ source_run_name,
+ internal_api,
+ self.project_dir,
+ )
+
+ if not self._entry_point:
+ _, ext = os.path.splitext(program_name)
+ if ext == ".py":
+ entry_point = ["python", program_name]
+ elif ext == ".sh":
+ command = os.environ.get("SHELL", "bash")
+ entry_point = [command, program_name]
+ else:
+ raise LaunchError(f"Unsupported entrypoint: {program_name}")
+ self.set_entry_point(entry_point)
+ if not self.override_args:
+ self.override_args = run_info["args"]
+ else:
+ assert utils._GIT_URI_REGEX.match(self.uri), (
+ "Non-wandb URI %s should be a Git URI" % self.uri
+ )
+ if not self._entry_point:
+ wandb.termlog(
+ f"{LOG_PREFIX}Entry point for repo not specified, defaulting to python main.py"
+ )
+ self.set_entry_point(EntrypointDefaults.PYTHON)
+ branch_name = utils._fetch_git_repo(
+ self.project_dir, self.uri, self.git_version
+ )
+ if self.git_version is None:
+ self.git_version = branch_name
+
+
+class EntryPoint:
+ """An entry point into a wandb launch specification."""
+
+ def __init__(self, name: Optional[str], command: List[str]):
+ self.name = name
+ self.command = command
+
+ def compute_command(self, user_parameters: Optional[List[str]]) -> List[str]:
+ """Converts user parameter dictionary to a string."""
+ ret = self.command
+ if user_parameters:
+ return ret + user_parameters
+ return ret
+
+
+def get_entry_point_command(
+ entry_point: Optional["EntryPoint"], parameters: List[str]
+) -> List[str]:
+ """Returns the shell command to execute in order to run the specified entry point.
+
+ Arguments:
+ entry_point: Entry point to run
+ parameters: Parameters (dictionary) for the entry point command
+
+ Returns:
+ List of strings representing the shell command to be executed
+ """
+ if entry_point is None:
+ return []
+ return entry_point.compute_command(parameters)
+
+
+def create_project_from_spec(launch_spec: Dict[str, Any], api: Api) -> LaunchProject:
+ """Constructs a LaunchProject instance using a launch spec.
+
+ Arguments:
+ launch_spec: Dictionary representation of launch spec
+ api: Instance of wandb.apis.internal Api
+
+ Returns:
+ An initialized `LaunchProject` object
+ """
+ name: Optional[str] = None
+ if launch_spec.get("name"):
+ name = launch_spec["name"]
+ return LaunchProject(
+ launch_spec.get("uri"),
+ launch_spec.get("job"),
+ api,
+ launch_spec,
+ launch_spec["entity"],
+ launch_spec["project"],
+ name,
+ launch_spec.get("docker", {}),
+ launch_spec.get("git", {}),
+ launch_spec.get("overrides", {}),
+ launch_spec.get("resource", None),
+ launch_spec.get("resource_args", {}),
+ launch_spec.get("run_id", None),
+ launch_spec.get("sweep_id", {}),
+ )
+
+
+def fetch_and_validate_project(
+ launch_project: LaunchProject, api: Api
+) -> LaunchProject:
+ """Fetches a project into a local directory, adds the config values to the directory, and validates the first entrypoint for the project.
+
+ Arguments:
+ launch_project: LaunchProject to fetch and validate.
+ api: Instance of wandb.apis.internal Api
+
+ Returns:
+ A validated `LaunchProject` object.
+
+ """
+ if launch_project.source == LaunchSource.DOCKER:
+ return launch_project
+ if launch_project.source == LaunchSource.LOCAL:
+ if not launch_project._entry_point:
+ wandb.termlog(
+ f"{LOG_PREFIX}Entry point for repo not specified, defaulting to `python main.py`"
+ )
+ launch_project.set_entry_point(EntrypointDefaults.PYTHON)
+ elif launch_project.source == LaunchSource.JOB:
+ launch_project._fetch_job()
+ else:
+ launch_project._fetch_project_local(internal_api=api)
+
+ assert launch_project.project_dir is not None
+ # this prioritizes pip, and we don't support any cases where both are present conda projects when uploaded to
+ # wandb become pip projects via requirements.frozen.txt, wandb doesn't preserve conda envs
+ if os.path.exists(
+ os.path.join(launch_project.project_dir, "requirements.txt")
+ ) or os.path.exists(
+ os.path.join(launch_project.project_dir, "requirements.frozen.txt")
+ ):
+ launch_project.deps_type = "pip"
+ elif os.path.exists(os.path.join(launch_project.project_dir, "environment.yml")):
+ launch_project.deps_type = "conda"
+
+ return launch_project
+
+
+def create_metadata_file(
+ launch_project: LaunchProject,
+ image_uri: str,
+ sanitized_entrypoint_str: str,
+ sanitized_dockerfile_contents: str,
+) -> None:
+ assert launch_project.project_dir is not None
+ with open(
+ os.path.join(launch_project.project_dir, DEFAULT_LAUNCH_METADATA_PATH),
+ "w",
+ ) as f:
+ json.dump(
+ {
+ **launch_project.launch_spec,
+ "image_uri": image_uri,
+ "command": sanitized_entrypoint_str,
+ "dockerfile_contents": sanitized_dockerfile_contents,
+ },
+ f,
+ )
diff --git a/wandb/sdk/launch/agent/README.md b/wandb/sdk/launch/agent/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..30f23da61afc7400ba2c3518f27943f87dface32
--- /dev/null
+++ b/wandb/sdk/launch/agent/README.md
@@ -0,0 +1,4 @@
+An agent is broadly defined as a *"program that collects information or performs a task in the background at a particular schedule. The term agent is often thought of as a software abstraction that is capable of acting with a certain degree of autonomy to perform a particular task on behalf of its host"* ([Source](https://www.techopedia.com/definition/1292/agent#:~:text=Techopedia%20Explains%20Agent-,What%20Does%20Agent%20Mean%3F,on%20behalf%20of%20its%20host.))
+
+Within Launch, an agent continually polls a queue of jobs, executing runs on the specified resource and then seamlessly report back to wandb
+
diff --git a/wandb/sdk/launch/agent/__init__.py b/wandb/sdk/launch/agent/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..838a7886c290fced892b06b8fee5137cbcb278af
--- /dev/null
+++ b/wandb/sdk/launch/agent/__init__.py
@@ -0,0 +1,5 @@
+from .agent import LaunchAgent
+
+LaunchAgent = LaunchAgent
+
+__all__ = ["LaunchAgent"]
diff --git a/wandb/sdk/launch/agent/agent.py b/wandb/sdk/launch/agent/agent.py
new file mode 100644
index 0000000000000000000000000000000000000000..b8f3b4ba9eb3c42d76a3807f6b77f69d8341cf68
--- /dev/null
+++ b/wandb/sdk/launch/agent/agent.py
@@ -0,0 +1,838 @@
+"""Implementation of launch agent."""
+import asyncio
+import logging
+import os
+import pprint
+import threading
+import time
+import traceback
+from dataclasses import dataclass
+from multiprocessing import Event
+from typing import Any, Dict, List, Optional, Union
+
+import wandb
+from wandb.apis.internal import Api
+from wandb.errors import CommError
+from wandb.sdk.launch._launch_add import launch_add
+from wandb.sdk.launch.runner.local_container import LocalSubmittedRun
+from wandb.sdk.launch.runner.local_process import LocalProcessRunner
+from wandb.sdk.launch.sweeps.scheduler import Scheduler
+from wandb.sdk.lib import runid
+
+from .. import loader
+from .._project_spec import (
+ LaunchProject,
+ create_project_from_spec,
+ fetch_and_validate_project,
+)
+from ..builder.build import construct_agent_configs
+from ..errors import LaunchDockerError, LaunchError
+from ..utils import (
+ LAUNCH_DEFAULT_PROJECT,
+ LOG_PREFIX,
+ PROJECT_SYNCHRONOUS,
+ event_loop_thread_exec,
+)
+from .job_status_tracker import JobAndRunStatusTracker
+from .run_queue_item_file_saver import RunQueueItemFileSaver
+
+AGENT_POLLING_INTERVAL = 10
+RECEIVED_JOB_POLLING_INTERVAL = 0.0 # more frequent when we know we have jobs
+
+AGENT_POLLING = "POLLING"
+AGENT_RUNNING = "RUNNING"
+AGENT_KILLED = "KILLED"
+
+HIDDEN_AGENT_RUN_TYPE = "sweep-controller"
+
+MAX_RESUME_COUNT = 5
+
+RUN_INFO_GRACE_PERIOD = 60
+
+MAX_WAIT_RUN_STOPPED = 60
+
+_env_timeout = os.environ.get("WANDB_LAUNCH_START_TIMEOUT")
+if _env_timeout:
+ try:
+ RUN_START_TIMEOUT = float(_env_timeout)
+ except ValueError:
+ raise LaunchError(
+ f"Invalid value for WANDB_LAUNCH_START_TIMEOUT: {_env_timeout}"
+ )
+else:
+ RUN_START_TIMEOUT = 60 * 30 # default 30 minutes
+
+_logger = logging.getLogger(__name__)
+
+
+@dataclass
+class JobSpecAndQueue:
+ job: Dict[str, Any]
+ queue: str
+
+
+def _convert_access(access: str) -> str:
+ """Convert access string to a value accepted by wandb."""
+ access = access.upper()
+ assert (
+ access == "PROJECT" or access == "USER"
+ ), "Queue access must be either project or user"
+ return access
+
+
+def _max_from_config(
+ config: Dict[str, Any], key: str, default: int = 1
+) -> Union[int, float]:
+ """Get an integer from the config, or float.inf if -1.
+
+ Utility for parsing integers from the agent config with a default, infinity
+ handling, and integer parsing. Raises more informative error if parse error.
+ """
+ try:
+ val = config.get(key)
+ if val is None:
+ val = default
+ max_from_config = int(val)
+ except ValueError as e:
+ raise LaunchError(
+ f"Error when parsing LaunchAgent config key: ['{key}': "
+ f"{config.get(key)}]. Error: {str(e)}"
+ )
+ if max_from_config == -1:
+ return float("inf")
+
+ if max_from_config < 0:
+ raise LaunchError(
+ f"Error when parsing LaunchAgent config key: ['{key}': "
+ f"{config.get(key)}]. Error: negative value."
+ )
+ return max_from_config
+
+
+def _is_scheduler_job(run_spec: Dict[str, Any]) -> bool:
+ """Determine whether a job/runSpec is a sweep scheduler."""
+ if not run_spec:
+ _logger.debug("Recieved runSpec in _is_scheduler_job that was empty")
+
+ if run_spec.get("uri") != Scheduler.PLACEHOLDER_URI:
+ return False
+
+ if run_spec.get("resource") == "local-process":
+ # Any job pushed to a run queue that has a scheduler uri is
+ # allowed to use local-process
+ if run_spec.get("job"):
+ return True
+
+ # If a scheduler is local-process and run through CLI, also
+ # confirm command is in format: [wandb scheduler <sweep>]
+ cmd = run_spec.get("overrides", {}).get("entry_point", [])
+ if len(cmd) < 3:
+ return False
+
+ if cmd[:2] != ["wandb", "scheduler"]:
+ return False
+
+ return True
+
+
+class LaunchAgent:
+ """Launch agent class which polls run given run queues and launches runs for wandb launch."""
+
+ _instance = None
+
+ def __new__(cls, *args: Any, **kwargs: Any) -> "LaunchAgent":
+ """Create a new instance of the LaunchAgent.
+
+ This method ensures that only one instance of the LaunchAgent is created.
+ This is done so that information about the agent can be accessed from
+ elsewhere in the library.
+ """
+ if cls._instance is None:
+ cls._instance = super().__new__(cls)
+ return cls._instance
+
+ @classmethod
+ def name(cls) -> str:
+ """Return the name of the agent."""
+ if cls._instance is None:
+ raise LaunchError("LaunchAgent has not been initialized")
+ name = cls._instance._name
+ if isinstance(name, str):
+ return name
+ raise LaunchError(f"Found invalid name for agent {name}")
+
+ @classmethod
+ def initialized(cls) -> bool:
+ """Return whether the agent is initialized."""
+ return cls._instance is not None
+
+ def __init__(self, api: Api, config: Dict[str, Any]):
+ """Initialize a launch agent.
+
+ Arguments:
+ api: Api object to use for making requests to the backend.
+ config: Config dictionary for the agent.
+ """
+ self._entity = config["entity"]
+ self._project = config.get("project", LAUNCH_DEFAULT_PROJECT)
+ self._api = api
+ self._base_url = self._api.settings().get("base_url")
+ self._ticks = 0
+ self._jobs: Dict[int, JobAndRunStatusTracker] = {}
+ self._jobs_lock = threading.Lock()
+ self._jobs_event = Event()
+ self._jobs_event.set()
+ self._cwd = os.getcwd()
+ self._namespace = runid.generate_id()
+ self._access = _convert_access("project")
+ self._max_jobs = _max_from_config(config, "max_jobs")
+ self._max_schedulers = _max_from_config(config, "max_schedulers")
+ self._secure_mode = config.get("secure_mode", False)
+ self.default_config: Dict[str, Any] = config
+
+ # Get agent version from env var if present, otherwise wandb version
+ self.version: str = "wandb@" + wandb.__version__
+ env_agent_version = os.environ.get("WANDB_AGENT_VERSION")
+ if env_agent_version and env_agent_version != "wandb-launch-agent":
+ self.version = env_agent_version
+
+ # serverside creation
+ self.gorilla_supports_agents = (
+ self._api.launch_agent_introspection() is not None
+ )
+ self._gorilla_supports_fail_run_queue_items = (
+ self._api.fail_run_queue_item_introspection()
+ )
+
+ self._queues: List[str] = config.get("queues", ["default"])
+
+ # remove project field from agent config before sending to back end
+ # because otherwise it shows up in the config in the UI and confuses users
+ sent_config = config.copy()
+ if "project" in sent_config:
+ del sent_config["project"]
+
+ create_response = self._api.create_launch_agent(
+ self._entity,
+ self._project,
+ self._queues,
+ sent_config,
+ self.version,
+ self.gorilla_supports_agents,
+ )
+ self._id = create_response["launchAgentId"]
+ if self._api.entity_is_team(self._entity):
+ wandb.termwarn(
+ f"{LOG_PREFIX}Agent is running on team entity ({self._entity}). Members of this team will be able to run code on this device."
+ )
+
+ agent_response = self._api.get_launch_agent(
+ self._id, self.gorilla_supports_agents
+ )
+ self._name = agent_response["name"]
+ self._init_agent_run()
+
+ async def fail_run_queue_item(
+ self,
+ run_queue_item_id: str,
+ message: str,
+ phase: str,
+ files: Optional[List[str]] = None,
+ ) -> None:
+ if self._gorilla_supports_fail_run_queue_items:
+ fail_rqi = event_loop_thread_exec(self._api.fail_run_queue_item)
+ await fail_rqi(run_queue_item_id, message, phase, files)
+
+ def _init_agent_run(self) -> None:
+ # TODO: has it been long enough that all backends support agents?
+ if self.gorilla_supports_agents:
+ settings = wandb.Settings(silent=True, disable_git=True)
+ self._wandb_run = wandb.init(
+ project=self._project,
+ entity=self._entity,
+ settings=settings,
+ id=self._name,
+ job_type=HIDDEN_AGENT_RUN_TYPE,
+ )
+ else:
+ self._wandb_run = None
+
+ @property
+ def thread_ids(self) -> List[int]:
+ """Returns a list of keys running thread ids for the agent."""
+ with self._jobs_lock:
+ return list(self._jobs.keys())
+
+ @property
+ def num_running_schedulers(self) -> int:
+ """Return just the number of schedulers."""
+ with self._jobs_lock:
+ return len([x for x in self._jobs if self._jobs[x].is_scheduler])
+
+ @property
+ def num_running_jobs(self) -> int:
+ """Return the number of jobs not including schedulers."""
+ with self._jobs_lock:
+ return len([x for x in self._jobs if not self._jobs[x].is_scheduler])
+
+ async def pop_from_queue(self, queue: str) -> Any:
+ """Pops an item off the runqueue to run as a job.
+
+ Arguments:
+ queue: Queue to pop from.
+
+ Returns:
+ Item popped off the queue.
+
+ Raises:
+ Exception: if there is an error popping from the queue.
+ """
+ try:
+ pop = event_loop_thread_exec(self._api.pop_from_run_queue)
+ ups = await pop(
+ queue,
+ entity=self._entity,
+ project=self._project,
+ agent_id=self._id,
+ )
+ return ups
+ except Exception as e:
+ print("Exception:", e)
+ return None
+
+ def print_status(self) -> None:
+ """Prints the current status of the agent."""
+ output_str = "agent "
+ if self._name:
+ output_str += f"{self._name} "
+ if self.num_running_jobs < self._max_jobs:
+ output_str += "polling on "
+ if self._project != LAUNCH_DEFAULT_PROJECT:
+ output_str += f"project {self._project}, "
+ output_str += f"queues {','.join(self._queues)}, "
+ output_str += (
+ f"running {self.num_running_jobs} out of a maximum of {self._max_jobs} jobs"
+ )
+
+ wandb.termlog(f"{LOG_PREFIX}{output_str}")
+ if self.num_running_jobs > 0:
+ output_str += f": {','.join(str(job_id) for job_id in self.thread_ids)}"
+
+ _logger.info(output_str)
+
+ async def update_status(self, status: str) -> None:
+ """Update the status of the agent.
+
+ Arguments:
+ status: Status to update the agent to.
+ """
+ _update_status = event_loop_thread_exec(self._api.update_launch_agent_status)
+ update_ret = await _update_status(
+ self._id, status, self.gorilla_supports_agents
+ )
+ if not update_ret["success"]:
+ wandb.termerror(f"{LOG_PREFIX}Failed to update agent status to {status}")
+
+ def _check_run_exists_and_inited(
+ self, entity: str, project: str, run_id: str, rqi_id: str
+ ) -> bool:
+ """Checks the stateof the run to ensure it has been inited. Note this will not behave well with resuming."""
+ # Checks the _wandb key in the run config for the run queue item id. If it exists, the
+ # submitted run definitely called init. Falls back to checking state of run.
+ # TODO: handle resuming runs
+
+ # Sweep runs exist but are in pending state, normal launch runs won't exist
+ # so will raise a CommError.
+ try:
+ run_state = self._api.get_run_state(entity, project, run_id)
+ if run_state.lower() != "pending":
+ return True
+ except CommError:
+ _logger.info(
+ f"Run {entity}/{project}/{run_id} with rqi id: {rqi_id} did not have associated run"
+ )
+ return False
+
+ async def finish_thread_id(
+ self,
+ thread_id: int,
+ exception: Optional[Union[Exception, LaunchDockerError]] = None,
+ ) -> None:
+ """Removes the job from our list for now."""
+ with self._jobs_lock:
+ job_and_run_status = self._jobs[thread_id]
+ if (
+ job_and_run_status.entity is not None
+ and job_and_run_status.entity != self._entity
+ ):
+ _logger.info(
+ "Skipping check for completed run status because run is on a different entity than agent"
+ )
+ elif exception is not None:
+ tb_str = traceback.format_exception(
+ type(exception), value=exception, tb=exception.__traceback__
+ )
+ fnames = job_and_run_status.saver.save_contents(
+ "".join(tb_str), "error.log", "error"
+ )
+ await self.fail_run_queue_item(
+ job_and_run_status.run_queue_item_id,
+ str(exception),
+ job_and_run_status.err_stage,
+ fnames,
+ )
+ elif job_and_run_status.project is None or job_and_run_status.run_id is None:
+ _logger.error(
+ f"called finish_thread_id on thread whose tracker has no project or run id. RunQueueItemID: {job_and_run_status.run_queue_item_id}"
+ )
+ wandb.termerror(
+ "Missing project or run id on thread called finish thread id"
+ )
+ await self.fail_run_queue_item(
+ job_and_run_status.run_queue_item_id,
+ "submitted job was finished without assigned project or run id",
+ "agent",
+ )
+ elif job_and_run_status.run is not None:
+ called_init = False
+ # We do some weird stuff here getting run info to check for a
+ # created in run in W&B.
+ #
+ # We retry for 60 seconds with an exponential backoff in case
+ # upsert run is taking a while.
+ logs = None
+ start_time = time.time()
+ interval = 1
+ while True:
+ called_init = self._check_run_exists_and_inited(
+ self._entity,
+ job_and_run_status.project,
+ job_and_run_status.run_id,
+ job_and_run_status.run_queue_item_id,
+ )
+ if called_init or time.time() - start_time > RUN_INFO_GRACE_PERIOD:
+ break
+ if not called_init:
+ # Fetch the logs now if we don't get run info on the
+ # first try, in case the logs are cleaned from the runner
+ # environment (e.g. k8s) during the run info grace period.
+ if interval == 1:
+ logs = await job_and_run_status.run.get_logs()
+ await asyncio.sleep(interval)
+ interval *= 2
+ if not called_init:
+ fnames = None
+ if job_and_run_status.completed_status == "finished":
+ _msg = "The submitted job exited successfully but failed to call wandb.init"
+ else:
+ _msg = "The submitted run was not successfully started"
+ if logs:
+ fnames = job_and_run_status.saver.save_contents(
+ logs, "error.log", "error"
+ )
+ await self.fail_run_queue_item(
+ job_and_run_status.run_queue_item_id, _msg, "run", fnames
+ )
+ else:
+ _logger.info(f"Finish thread id {thread_id} had no exception and no run")
+ wandb._sentry.exception(
+ "launch agent called finish thread id on thread without run or exception"
+ )
+
+ # TODO: keep logs or something for the finished jobs
+ with self._jobs_lock:
+ del self._jobs[thread_id]
+
+ # update status back to polling if no jobs are running
+ if len(self.thread_ids) == 0:
+ await self.update_status(AGENT_POLLING)
+
+ async def run_job(
+ self, job: Dict[str, Any], queue: str, file_saver: RunQueueItemFileSaver
+ ) -> None:
+ """Set up project and run the job.
+
+ Arguments:
+ job: Job to run.
+ """
+ _msg = f"{LOG_PREFIX}Launch agent received job:\n{pprint.pformat(job)}\n"
+ wandb.termlog(_msg)
+ _logger.info(_msg)
+ # update agent status
+ await self.update_status(AGENT_RUNNING)
+
+ # parse job
+ _logger.info("Parsing launch spec")
+ launch_spec = job["runSpec"]
+
+ # Abort if this job attempts to override secure mode
+ self._assert_secure(launch_spec)
+ job_tracker = JobAndRunStatusTracker(job["runQueueItemId"], queue, file_saver)
+
+ asyncio.create_task(
+ self.task_run_job(
+ launch_spec,
+ job,
+ self.default_config,
+ self._api,
+ job_tracker,
+ )
+ )
+
+ def _assert_secure(self, launch_spec: Dict[str, Any]) -> None:
+ """If secure mode is set, make sure no vulnerable keys are overridden."""
+ if not self._secure_mode:
+ return
+ k8s_config = launch_spec.get("resource_args", {}).get("kubernetes", {})
+
+ pod_secure_keys = ["hostPID", "hostIPC", "hostNetwork", "initContainers"]
+ pod_spec = k8s_config.get("spec", {}).get("template", {}).get("spec", {})
+ for key in pod_secure_keys:
+ if key in pod_spec:
+ raise ValueError(
+ f'This agent is configured to lock "{key}" in pod spec '
+ "but the job specification attempts to override it."
+ )
+
+ container_specs = pod_spec.get("containers", [])
+ for container_spec in container_specs:
+ if "command" in container_spec:
+ raise ValueError(
+ 'This agent is configured to lock "command" in container spec '
+ "but the job specification attempts to override it."
+ )
+
+ if launch_spec.get("overrides", {}).get("entry_point"):
+ raise ValueError(
+ 'This agent is configured to lock the "entrypoint" override '
+ "but the job specification attempts to override it."
+ )
+
+ async def loop(self) -> None:
+ """Loop infinitely to poll for jobs and run them.
+
+ Raises:
+ KeyboardInterrupt: if the agent is requested to stop.
+ """
+ self.print_status()
+ try:
+ while True:
+ job = None
+ self._ticks += 1
+ agent_response = self._api.get_launch_agent(
+ self._id, self.gorilla_supports_agents
+ )
+ if agent_response["stopPolling"]:
+ # shutdown process and all jobs if requested from ui
+ raise KeyboardInterrupt
+ if self.num_running_jobs < self._max_jobs:
+ # only check for new jobs if we're not at max
+ job_and_queue = await self.get_job_and_queue()
+ # these will either both be None, or neither will be None
+ if job_and_queue is not None:
+ job = job_and_queue.job
+ queue = job_and_queue.queue
+ try:
+ file_saver = RunQueueItemFileSaver(
+ self._wandb_run, job["runQueueItemId"]
+ )
+ if _is_scheduler_job(job.get("runSpec", {})):
+ # If job is a scheduler, and we are already at the cap, ignore,
+ # don't ack, and it will be pushed back onto the queue in 1 min
+ if self.num_running_schedulers >= self._max_schedulers:
+ wandb.termwarn(
+ f"{LOG_PREFIX}Agent already running the maximum number "
+ f"of sweep schedulers: {self._max_schedulers}. To set "
+ "this value use `max_schedulers` key in the agent config"
+ )
+ continue
+ await self.run_job(job, queue, file_saver)
+ except Exception as e:
+ wandb.termerror(
+ f"{LOG_PREFIX}Error running job: {traceback.format_exc()}"
+ )
+ wandb._sentry.exception(e)
+
+ # always the first phase, because we only enter phase 2 within the thread
+ files = file_saver.save_contents(
+ contents=traceback.format_exc(),
+ fname="error.log",
+ file_sub_type="error",
+ )
+ await self.fail_run_queue_item(
+ run_queue_item_id=job["runQueueItemId"],
+ message=str(e),
+ phase="agent",
+ files=files,
+ )
+
+ if self._ticks % 2 == 0:
+ if len(self.thread_ids) == 0:
+ await self.update_status(AGENT_POLLING)
+ else:
+ await self.update_status(AGENT_RUNNING)
+ self.print_status()
+
+ if self.num_running_jobs == self._max_jobs or job is None:
+ # all threads busy or did not receive job
+ await asyncio.sleep(AGENT_POLLING_INTERVAL)
+ else:
+ await asyncio.sleep(RECEIVED_JOB_POLLING_INTERVAL)
+
+ except KeyboardInterrupt:
+ await self.update_status(AGENT_KILLED)
+ wandb.termlog(f"{LOG_PREFIX}Shutting down, active jobs:")
+ self.print_status()
+ finally:
+ self._jobs_event.clear()
+
+ # Threaded functions
+ async def task_run_job(
+ self,
+ launch_spec: Dict[str, Any],
+ job: Dict[str, Any],
+ default_config: Dict[str, Any],
+ api: Api,
+ job_tracker: JobAndRunStatusTracker,
+ ) -> None:
+ rqi_id = job["runQueueItemId"]
+ assert rqi_id
+ exception: Optional[Union[LaunchDockerError, Exception]] = None
+ try:
+ with self._jobs_lock:
+ self._jobs[rqi_id] = job_tracker
+ await self._task_run_job(
+ launch_spec, job, default_config, api, rqi_id, job_tracker
+ )
+ except LaunchDockerError as e:
+ wandb.termerror(
+ f"{LOG_PREFIX}agent {self._name} encountered an issue while starting Docker, see above output for details."
+ )
+ exception = e
+ wandb._sentry.exception(e)
+ except LaunchError as e:
+ wandb.termerror(f"{LOG_PREFIX}Error running job: {e}")
+ exception = e
+ wandb._sentry.exception(e)
+ except Exception as e:
+ wandb.termerror(f"{LOG_PREFIX}Error running job: {traceback.format_exc()}")
+ exception = e
+ wandb._sentry.exception(e)
+ finally:
+ await self.finish_thread_id(rqi_id, exception)
+
+ async def _task_run_job(
+ self,
+ launch_spec: Dict[str, Any],
+ job: Dict[str, Any],
+ default_config: Dict[str, Any],
+ api: Api,
+ thread_id: int,
+ job_tracker: JobAndRunStatusTracker,
+ ) -> None:
+ project = create_project_from_spec(launch_spec, api)
+ self._set_queue_and_rqi_in_project(project, job, job_tracker.queue)
+ ack = event_loop_thread_exec(api.ack_run_queue_item)
+ await ack(job["runQueueItemId"], project.run_id)
+ # don't launch sweep runs if the sweep isn't healthy
+ await self.check_sweep_state(launch_spec, api)
+
+ job_tracker.update_run_info(project)
+ _logger.info("Fetching and validating project...")
+ project = fetch_and_validate_project(project, api)
+ _logger.info("Fetching resource...")
+ resource = launch_spec.get("resource") or "local-container"
+ backend_config: Dict[str, Any] = {
+ PROJECT_SYNCHRONOUS: False, # agent always runs async
+ }
+ _logger.info("Loading backend")
+ override_build_config = launch_spec.get("builder")
+
+ _, build_config, registry_config = construct_agent_configs(
+ default_config, override_build_config
+ )
+ image_uri = project.docker_image
+ entrypoint = project.get_single_entry_point()
+ environment = loader.environment_from_config(
+ default_config.get("environment", {})
+ )
+ registry = loader.registry_from_config(registry_config, environment)
+ builder = loader.builder_from_config(build_config, environment, registry)
+ backend = loader.runner_from_config(
+ resource, api, backend_config, environment, registry
+ )
+ if not (project.docker_image or isinstance(backend, LocalProcessRunner)):
+ assert entrypoint is not None
+ image_uri = await builder.build_image(project, entrypoint, job_tracker)
+
+ _logger.info("Backend loaded...")
+ if isinstance(backend, LocalProcessRunner):
+ run = await backend.run(project, image_uri)
+ else:
+ assert image_uri
+ run = await backend.run(project, image_uri)
+ if _is_scheduler_job(launch_spec):
+ with self._jobs_lock:
+ self._jobs[thread_id].is_scheduler = True
+ wandb.termlog(
+ f"{LOG_PREFIX}Preparing to run sweep scheduler "
+ f"({self.num_running_schedulers}/{self._max_schedulers})"
+ )
+
+ if not run:
+ with self._jobs_lock:
+ job_tracker.failed_to_start = True
+ return
+ with self._jobs_lock:
+ job_tracker.run = run
+ start_time = time.time()
+ stopped_time: Optional[float] = None
+ while self._jobs_event.is_set():
+ # If run has failed to start before timeout, kill it
+ state = (await run.get_status()).state
+ if state == "starting" and RUN_START_TIMEOUT > 0:
+ if time.time() - start_time > RUN_START_TIMEOUT:
+ await run.cancel()
+ raise LaunchError(
+ f"Run failed to start within {RUN_START_TIMEOUT} seconds. "
+ "If you want to increase this timeout, set WANDB_LAUNCH_START_TIMEOUT "
+ "to a larger value."
+ )
+ if await self._check_run_finished(job_tracker, launch_spec):
+ return
+ if await job_tracker.check_wandb_run_stopped(self._api):
+ if stopped_time is None:
+ stopped_time = time.time()
+ else:
+ if time.time() - stopped_time > MAX_WAIT_RUN_STOPPED:
+ await run.cancel()
+ await asyncio.sleep(AGENT_POLLING_INTERVAL)
+
+ # temp: for local, kill all jobs. we don't yet have good handling for different
+ # types of runners in general
+ if isinstance(run, LocalSubmittedRun) and run._command_proc is not None:
+ run._command_proc.kill()
+
+ async def check_sweep_state(self, launch_spec: Dict[str, Any], api: Api) -> None:
+ """Check the state of a sweep before launching a run for the sweep."""
+ if launch_spec.get("sweep_id"):
+ try:
+ get_sweep_state = event_loop_thread_exec(api.get_sweep_state)
+ state = await get_sweep_state(
+ sweep=launch_spec["sweep_id"],
+ entity=launch_spec["entity"],
+ project=launch_spec["project"],
+ )
+ except Exception as e:
+ _logger.debug(f"Fetch sweep state error: {e}")
+ state = None
+
+ if state != "RUNNING" and state != "PAUSED":
+ raise LaunchError(
+ f"Launch agent picked up sweep job, but sweep ({launch_spec['sweep_id']}) was in a terminal state ({state})"
+ )
+
+ async def _check_run_finished(
+ self, job_tracker: JobAndRunStatusTracker, launch_spec: Dict[str, Any]
+ ) -> bool:
+ if job_tracker.completed_status:
+ return True
+
+ # the run can be done before the run has started
+ # but can also be none if the run failed to start
+ # so if there is no run, either the run hasn't started yet
+ # or it has failed
+ if job_tracker.run is None:
+ if job_tracker.failed_to_start:
+ return True
+ return False
+
+ known_error = False
+ try:
+ run = job_tracker.run
+ status = (await run.get_status()).state
+
+ if status == "preempted" and job_tracker.entity == self._entity:
+ config = launch_spec.copy()
+ config["run_id"] = job_tracker.run_id
+ config["_resume_count"] = config.get("_resume_count", 0) + 1
+ with self._jobs_lock:
+ job_tracker.completed_status = status
+ if config["_resume_count"] > MAX_RESUME_COUNT:
+ wandb.termlog(
+ f"{LOG_PREFIX}Run {job_tracker.run_id} has already resumed {MAX_RESUME_COUNT} times."
+ )
+ return True
+ wandb.termlog(
+ f"{LOG_PREFIX}Run {job_tracker.run_id} was preempted, requeueing..."
+ )
+
+ if "sweep_id" in config:
+ # allow resumed runs from sweeps that have already completed by removing
+ # the sweep id before pushing to queue
+ del config["sweep_id"]
+
+ launch_add(
+ config=config,
+ project_queue=self._project,
+ queue_name=job_tracker.queue,
+ )
+ return True
+ # TODO change these statuses to an enum
+ if status in ["stopped", "failed", "finished", "preempted"]:
+ if job_tracker.is_scheduler:
+ wandb.termlog(f"{LOG_PREFIX}Scheduler finished with ID: {run.id}")
+ if status == "failed":
+ # on fail, update sweep state. scheduler run_id should == sweep_id
+ try:
+ self._api.set_sweep_state(
+ sweep=job_tracker.run_id,
+ entity=job_tracker.entity,
+ project=job_tracker.project,
+ state="CANCELED",
+ )
+ except Exception as e:
+ raise LaunchError(f"Failed to update sweep state: {e}")
+ else:
+ wandb.termlog(f"{LOG_PREFIX}Job finished with ID: {run.id}")
+ with self._jobs_lock:
+ job_tracker.completed_status = status
+ return True
+
+ return False
+ except LaunchError as e:
+ wandb.termerror(
+ f"{LOG_PREFIX}Terminating job {run.id} because it failed to start: {str(e)}"
+ )
+ known_error = True
+ with self._jobs_lock:
+ job_tracker.failed_to_start = True
+ # TODO: make get_status robust to errors for each runner, and handle them
+ except Exception as e:
+ wandb.termerror(f"{LOG_PREFIX}Error getting status for job {run.id}")
+ wandb.termerror(traceback.format_exc())
+ _logger.info("---")
+ _logger.info("Caught exception while getting status.")
+ _logger.info(f"Job ID: {run.id}")
+ _logger.info(traceback.format_exc())
+ _logger.info("---")
+ wandb._sentry.exception(e)
+ return known_error
+
+ async def get_job_and_queue(self) -> Optional[JobSpecAndQueue]:
+ for queue in self._queues:
+ job = await self.pop_from_queue(queue)
+ if job is not None:
+ self._queues.remove(queue)
+ self._queues.append(queue)
+ return JobSpecAndQueue(job, queue)
+ return None
+
+ def _set_queue_and_rqi_in_project(
+ self, project: LaunchProject, job: Dict[str, Any], queue: str
+ ) -> None:
+ project.queue_name = queue
+
+ # queue entity currently always matches the agent
+ project.queue_entity = self._entity
+ project.run_queue_item_id = job["runQueueItemId"]
diff --git a/wandb/sdk/launch/agent/config.py b/wandb/sdk/launch/agent/config.py
new file mode 100644
index 0000000000000000000000000000000000000000..8cd68a18b06629eda8ad07efee343a5dad60866d
--- /dev/null
+++ b/wandb/sdk/launch/agent/config.py
@@ -0,0 +1,257 @@
+"""Definition of the config object used by the Launch agent."""
+
+from enum import Enum
+from typing import List, Optional
+
+# ValidationError is imported for exception type checking purposes only.
+from pydantic import ( # type: ignore
+ BaseModel,
+ Field,
+ ValidationError, # noqa: F401
+ root_validator,
+ validator,
+)
+
+import wandb
+from wandb.sdk.launch.utils import (
+ AZURE_BLOB_REGEX,
+ AZURE_CONTAINER_REGISTRY_URI_REGEX,
+ ELASTIC_CONTAINER_REGISTRY_URI_REGEX,
+ GCP_ARTIFACT_REGISTRY_URI_REGEX,
+ GCS_URI_RE,
+ S3_URI_RE,
+)
+
+__all__ = [
+ "ValidationError",
+ "AgentConfig",
+]
+
+
+class EnvironmentType(str, Enum):
+ """Enum of valid environment types."""
+
+ aws = "aws"
+ gcp = "gcp"
+ azure = "azure"
+
+
+class RegistryType(str, Enum):
+ """Enum of valid registry types."""
+
+ ecr = "ecr"
+ acr = "acr"
+ gcr = "gcr"
+
+
+class BuilderType(str, Enum):
+ """Enum of valid builder types."""
+
+ docker = "docker"
+ kaniko = "kaniko"
+ noop = "noop"
+
+
+class TargetPlatform(str, Enum):
+ """Enum of valid target platforms."""
+
+ linux_amd64 = "linux/amd64"
+ linux_arm64 = "linux/arm64"
+
+
+class RegistryConfig(BaseModel):
+ """Configuration for registry block.
+
+ Note that we don't forbid extra fields here because:
+ - We want to allow all fields supported by each registry
+ - We will perform validation on the registry object itself later
+ - Registry block is being deprecated in favor of destination field in builder
+ """
+
+ type: Optional[RegistryType] = Field(
+ None,
+ description="The type of registry to use.",
+ )
+ uri: Optional[str] = Field(
+ None,
+ description="The URI of the registry.",
+ )
+
+ @validator("uri") # type: ignore
+ @classmethod
+ def validate_uri(cls, uri: str) -> str:
+ for regex in [
+ GCP_ARTIFACT_REGISTRY_URI_REGEX,
+ AZURE_CONTAINER_REGISTRY_URI_REGEX,
+ ELASTIC_CONTAINER_REGISTRY_URI_REGEX,
+ ]:
+ if regex.match(uri):
+ return uri
+ raise ValueError(
+ "Invalid uri. URI must be a repository URI for an "
+ "ECR, ACR, or GCP Artifact Registry."
+ )
+
+
+class EnvironmentConfig(BaseModel):
+ """Configuration for the environment block."""
+
+ type: Optional[EnvironmentType] = Field(
+ None,
+ description="The type of environment to use.",
+ )
+ region: Optional[str] = Field(..., description="The region to use.")
+
+ class Config:
+ extra = "allow"
+
+ @root_validator(pre=True) # type: ignore
+ @classmethod
+ def check_extra_fields(cls, values: dict) -> dict:
+ """Check for extra fields and print a warning."""
+ for key in values:
+ if key not in ["type", "region"]:
+ wandb.termwarn(
+ f"Unrecognized field {key} in environment block. Please check your config file."
+ )
+ return values
+
+
+class BuilderConfig(BaseModel):
+ type: Optional[BuilderType] = Field(
+ None,
+ description="The type of builder to use.",
+ )
+ destination: Optional[str] = Field(
+ None,
+ description="The destination to use for the built image. If not provided, "
+ "the image will be pushed to the registry.",
+ )
+
+ @validator("destination") # type: ignore
+ @classmethod
+ def validate_destination(cls, destination: str) -> str:
+ """Validate that the destination is a valid container registry URI."""
+ for regex in [
+ GCP_ARTIFACT_REGISTRY_URI_REGEX,
+ AZURE_CONTAINER_REGISTRY_URI_REGEX,
+ ELASTIC_CONTAINER_REGISTRY_URI_REGEX,
+ ]:
+ if regex.match(destination):
+ return destination
+ raise ValueError(
+ "Invalid destination. Destination must be a repository URI for an "
+ "ECR, ACR, or GCP Artifact Registry."
+ )
+
+ platform: Optional[TargetPlatform] = Field(
+ None,
+ description="The platform to use for the built image. If not provided, "
+ "the platform will be detected automatically.",
+ )
+
+ build_context_store: Optional[str] = Field(
+ None,
+ description="The build context store to use. Required for kaniko builds.",
+ alias="build-context-store",
+ )
+ build_job_name: Optional[str] = Field(
+ "wandb-launch-container-build",
+ description="Name prefix of the build job.",
+ alias="build-job-name",
+ )
+ secret_name: Optional[str] = Field(
+ None,
+ description="The name of the secret to use for the build job.",
+ alias="secret-name",
+ )
+ secret_key: Optional[str] = Field(
+ None,
+ description="The key of the secret to use for the build job.",
+ alias="secret-key",
+ )
+ kaniko_image: Optional[str] = Field(
+ "gcr.io/kaniko-project/executor:latest",
+ description="The image to use for the kaniko executor.",
+ alias="kaniko-image",
+ )
+
+ @validator("build_context_store") # type: ignore
+ @classmethod
+ def validate_build_context_store(
+ cls, build_context_store: Optional[str]
+ ) -> Optional[str]:
+ """Validate that the build context store is a valid container registry URI."""
+ if build_context_store is None:
+ return None
+ for regex in [
+ S3_URI_RE,
+ GCS_URI_RE,
+ AZURE_BLOB_REGEX,
+ ]:
+ if regex.match(build_context_store):
+ return build_context_store
+ raise ValueError(
+ "Invalid build context store. Build context store must be a URI for an "
+ "S3 bucket, GCS bucket, or Azure blob."
+ )
+
+ @root_validator(pre=True) # type: ignore
+ @classmethod
+ def validate_kaniko(cls, values: dict) -> dict:
+ """Validate that kaniko is configured correctly."""
+ if values.get("type") == BuilderType.kaniko:
+ if values.get("build-context-store") is None:
+ raise ValueError(
+ "builder.build-context-store is required if builder.type is set to kaniko."
+ )
+ return values
+
+ @root_validator(pre=True) # type: ignore
+ @classmethod
+ def validate_docker(cls, values: dict) -> dict:
+ """Right now there are no required fields for docker builds."""
+ return values
+
+
+class AgentConfig(BaseModel):
+ """Configuration for the Launch agent."""
+
+ queues: List[str] = Field(
+ default=[],
+ description="The queues to use for this agent.",
+ )
+ project: Optional[str] = Field(
+ description="The W&B project to use for this agent.",
+ )
+ entity: Optional[str] = Field(
+ description="The W&B entity to use for this agent.",
+ )
+ max_jobs: Optional[int] = Field(
+ 1,
+ description="The maximum number of jobs to run concurrently.",
+ )
+ max_schedulers: Optional[int] = Field(
+ 1,
+ description="The maximum number of sweep schedulers to run concurrently.",
+ )
+ secure_mode: Optional[bool] = Field(
+ False,
+ description="Whether to use secure mode for this agent. If True, the "
+ "agent will reject runs that attempt to override the entrypoint or image.",
+ )
+ registry: Optional[RegistryConfig] = Field(
+ None,
+ description="The registry to use.",
+ )
+ environment: Optional[EnvironmentConfig] = Field(
+ None,
+ description="The environment to use.",
+ )
+ builder: Optional[BuilderConfig] = Field(
+ None,
+ description="The builder to use.",
+ )
+
+ class Config:
+ extra = "forbid"
diff --git a/wandb/sdk/launch/agent/job_status_tracker.py b/wandb/sdk/launch/agent/job_status_tracker.py
new file mode 100644
index 0000000000000000000000000000000000000000..2b25b5bc637b6a09fadd260d8d7eb702e9f54d32
--- /dev/null
+++ b/wandb/sdk/launch/agent/job_status_tracker.py
@@ -0,0 +1,53 @@
+import logging
+from dataclasses import dataclass
+from typing import Optional
+
+from wandb.apis.internal import Api
+from wandb.errors import CommError
+from wandb.sdk.launch._project_spec import LaunchProject
+
+from ..runner.abstract import AbstractRun
+from ..utils import event_loop_thread_exec
+from .run_queue_item_file_saver import RunQueueItemFileSaver
+
+_logger = logging.getLogger(__name__)
+
+
+@dataclass
+class JobAndRunStatusTracker:
+ run_queue_item_id: str
+ queue: str
+ saver: RunQueueItemFileSaver
+ run_id: Optional[str] = None
+ project: Optional[str] = None
+ entity: Optional[str] = None
+ run: Optional[AbstractRun] = None
+ failed_to_start: bool = False
+ completed_status: Optional[str] = None
+ is_scheduler: bool = False
+ err_stage: str = "agent"
+
+ @property
+ def job_completed(self) -> bool:
+ return self.failed_to_start or self.completed_status is not None
+
+ def update_run_info(self, launch_project: LaunchProject) -> None:
+ self.run_id = launch_project.run_id
+ self.project = launch_project.target_project
+ self.entity = launch_project.target_entity
+
+ def set_err_stage(self, stage: str) -> None:
+ self.err_stage = stage
+
+ async def check_wandb_run_stopped(self, api: Api) -> bool:
+ assert (
+ self.run_id is not None
+ and self.project is not None
+ and self.entity is not None
+ ), "Job tracker does not contain run info. Update with run info before checking if run stopped"
+ check_stop = event_loop_thread_exec(api.api.check_stop_requested)
+ try:
+ return bool(await check_stop(self.project, self.entity, self.run_id))
+ except CommError as e:
+ _logger.error(f"CommError when checking if wandb run stopped: {e}")
+ return False
diff --git a/wandb/sdk/launch/agent/run_queue_item_file_saver.py b/wandb/sdk/launch/agent/run_queue_item_file_saver.py
new file mode 100644
index 0000000000000000000000000000000000000000..f8bebd0f51c97b13f359bba1c3cb75f65509e270
--- /dev/null
+++ b/wandb/sdk/launch/agent/run_queue_item_file_saver.py
@@ -0,0 +1,47 @@
+"""Implementation of the run queue item file saver class."""
+
+import os
+import sys
+from typing import List, Optional, Union
+
+import wandb
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+FileSubtypes = Literal["warning", "error"]
+
+
+class RunQueueItemFileSaver:
+ def __init__(
+ self,
+ agent_run: Optional[
+ Union["wandb.sdk.wandb_run.Run", "wandb.sdk.lib.RunDisabled"]
+ ],
+ run_queue_item_id: str,
+ ):
+ self.run_queue_item_id = run_queue_item_id
+ self.run = agent_run
+
+ def save_contents(
+ self, contents: str, fname: str, file_sub_type: FileSubtypes
+ ) -> Optional[List[str]]:
+ if not isinstance(self.run, wandb.sdk.wandb_run.Run):
+ wandb.termwarn("Not saving file contents because agent has no run")
+ return None
+ root_dir = self.run._settings.files_dir
+ saved_run_path = os.path.join(self.run_queue_item_id, file_sub_type, fname)
+ local_path = os.path.join(root_dir, saved_run_path)
+ os.makedirs(os.path.dirname(local_path), exist_ok=True)
+ with open(local_path, "w") as f:
+ f.write(contents)
+ res = self.run.save(local_path, base_path=root_dir, policy="now")
+ if isinstance(res, list):
+ return [saved_run_path]
+ else:
+ wandb.termwarn(
+ f"Failed to save files for run queue item: {self.run_queue_item_id}"
+ )
+ return None
diff --git a/wandb/sdk/launch/builder/README.md b/wandb/sdk/launch/builder/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..3e3f350bfbb9bdc092e87ac2b2d22c8db7073799
--- /dev/null
+++ b/wandb/sdk/launch/builder/README.md
@@ -0,0 +1,3 @@
+The builder creates a docker image for execution of the run. (If necessary)
+Prebuilt docker images can be used to skip this step.
+
diff --git a/wandb/sdk/launch/builder/__init__.py b/wandb/sdk/launch/builder/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sdk/launch/builder/abstract.py b/wandb/sdk/launch/builder/abstract.py
new file mode 100644
index 0000000000000000000000000000000000000000..64bb4cb96679de13a7603533181e09798afae296
--- /dev/null
+++ b/wandb/sdk/launch/builder/abstract.py
@@ -0,0 +1,88 @@
+"""Abstract plugin class defining the interface needed to build container images for W&B Launch."""
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+from wandb.sdk.launch.environment.abstract import AbstractEnvironment
+from wandb.sdk.launch.registry.abstract import AbstractRegistry
+
+from .._project_spec import EntryPoint, LaunchProject
+
+if TYPE_CHECKING:
+ from wandb.sdk.launch.agent.job_status_tracker import JobAndRunStatusTracker
+
+
+class AbstractBuilder(ABC):
+ """Abstract plugin class defining the interface needed to build container images for W&B Launch."""
+
+ builder_type: str
+ environment: AbstractEnvironment
+ registry: AbstractRegistry
+ builder_config: Dict[str, Any]
+
+ @abstractmethod
+ def __init__(
+ self,
+ environment: AbstractEnvironment,
+ registry: AbstractRegistry,
+ verify: bool = True,
+ ) -> None:
+ """Initialize a builder.
+
+ Arguments:
+ builder_config: The builder config.
+ registry: The registry to use.
+ verify: Whether to verify the functionality of the builder.
+
+ Raises:
+ LaunchError: If the builder cannot be intialized or verified.
+ """
+ raise NotImplementedError
+
+ @classmethod
+ @abstractmethod
+ def from_config(
+ cls,
+ config: dict,
+ environment: AbstractEnvironment,
+ registry: AbstractRegistry,
+ ) -> "AbstractBuilder":
+ """Create a builder from a config dictionary.
+
+ Arguments:
+ config: The config dictionary.
+ environment: The environment to use.
+ registry: The registry to use.
+ verify: Whether to verify the functionality of the builder.
+ login: Whether to login to the registry immediately.
+
+ Returns:
+ The builder.
+ """
+ raise NotImplementedError
+
+ @abstractmethod
+ async def build_image(
+ self,
+ launch_project: LaunchProject,
+ entrypoint: EntryPoint,
+ job_tracker: Optional["JobAndRunStatusTracker"] = None,
+ ) -> str:
+ """Build the image for the given project.
+
+ Arguments:
+ launch_project: The project to build.
+ build_ctx_path: The path to the build context.
+
+ Returns:
+ The image name.
+ """
+ raise NotImplementedError
+
+ @abstractmethod
+ async def verify(self) -> None:
+ """Verify that the builder can be used to build images.
+
+ Raises:
+ LaunchError: If the builder cannot be used to build images.
+ """
+ raise NotImplementedError
diff --git a/wandb/sdk/launch/builder/build.py b/wandb/sdk/launch/builder/build.py
new file mode 100644
index 0000000000000000000000000000000000000000..7846273ac21e31665735a36d290d369192fc12f0
--- /dev/null
+++ b/wandb/sdk/launch/builder/build.py
@@ -0,0 +1,658 @@
+import hashlib
+import json
+import logging
+import os
+import shlex
+import shutil
+import sys
+import tempfile
+from typing import Any, Dict, List, Optional, Tuple
+
+import pkg_resources
+import yaml
+from dockerpycreds.utils import find_executable # type: ignore
+from six.moves import shlex_quote
+
+import wandb
+import wandb.docker as docker
+import wandb.env
+from wandb.apis.internal import Api
+from wandb.sdk.launch.loader import (
+ builder_from_config,
+ environment_from_config,
+ registry_from_config,
+)
+
+from .._project_spec import (
+ EntryPoint,
+ EntrypointDefaults,
+ LaunchProject,
+ fetch_and_validate_project,
+)
+from ..errors import ExecutionError, LaunchError
+from ..registry.abstract import AbstractRegistry
+from ..utils import (
+ AZURE_CONTAINER_REGISTRY_URI_REGEX,
+ ELASTIC_CONTAINER_REGISTRY_URI_REGEX,
+ GCP_ARTIFACT_REGISTRY_URI_REGEX,
+ LAUNCH_CONFIG_FILE,
+ LOG_PREFIX,
+ event_loop_thread_exec,
+ resolve_build_and_registry_config,
+)
+
+_logger = logging.getLogger(__name__)
+
+
+_WANDB_DOCKERFILE_NAME = "Dockerfile.wandb"
+
+
+def registry_from_uri(uri: str) -> AbstractRegistry:
+ """Create a registry helper object from a uri.
+
+ This function parses the URI and determines which supported registry it
+ belongs to. It then creates a registry helper object for that registry.
+ The supported remote registry types are:
+ - Azure Container Registry
+ - Google Container Registry
+ - AWS Elastic Container Registry
+
+ The format of the URI is as follows:
+ - Azure Container Registry: <registry-name>.azurecr.io/<repo-name>/<image-name>
+ - Google Container Registry: <location>-docker.pkg.dev/<project-id>/<repo-name>/<image-name>
+ - AWS Elastic Container Registry: <account-id>.dkr.ecr.<region>.amazonaws.com/<repo-name>/<image-name>
+
+ Our classification of the registry is based on the domain name. For example,
+ if the uri contains `.azurecr.io`, we classify it as an Azure
+ Container Registry. If the uri contains `.dkr.ecr`, we classify
+ it as an AWS Elastic Container Registry. If the uri contains
+ `-docker.pkg.dev`, we classify it as a Google Artifact Registry.
+
+ This function will attempt to load the approriate cloud helpers for the
+
+ `https://` prefix is optional for all of the above.
+
+ Arguments:
+ uri: The uri to create a registry from.
+
+ Returns:
+ The registry.
+
+ Raises:
+ LaunchError: If the registry helper cannot be loaded for the given URI.
+ """
+ if uri.startswith("https://"):
+ uri = uri[len("https://") :]
+
+ if AZURE_CONTAINER_REGISTRY_URI_REGEX.match(uri) is not None:
+ from wandb.sdk.launch.registry.azure_container_registry import (
+ AzureContainerRegistry,
+ )
+
+ return AzureContainerRegistry(uri=uri)
+
+ elif GCP_ARTIFACT_REGISTRY_URI_REGEX.match(uri) is not None:
+ from wandb.sdk.launch.registry.google_artifact_registry import (
+ GoogleArtifactRegistry,
+ )
+
+ return GoogleArtifactRegistry(uri=uri)
+
+ elif ELASTIC_CONTAINER_REGISTRY_URI_REGEX.match(uri) is not None:
+ from wandb.sdk.launch.registry.elastic_container_registry import (
+ ElasticContainerRegistry,
+ )
+
+ return ElasticContainerRegistry(uri=uri)
+
+ else:
+ raise LaunchError(f"Unsupported registry URI: {uri}. Unable to load helper.")
+
+
+async def validate_docker_installation() -> None:
+ """Verify if Docker is installed on host machine."""
+ find_exec = event_loop_thread_exec(find_executable)
+ if not await find_exec("docker"):
+ raise ExecutionError(
+ "Could not find Docker executable. "
+ "Ensure Docker is installed as per the instructions "
+ "at https://docs.docker.com/install/overview/."
+ )
+
+
+def get_docker_user(launch_project: LaunchProject, runner_type: str) -> Tuple[str, int]:
+ import getpass
+
+ username = getpass.getuser()
+
+ if runner_type == "sagemaker" and not launch_project.docker_image:
+ # unless user has provided their own image, sagemaker must run as root but keep the name for workdir etc
+ return username, 0
+
+ userid = launch_project.docker_user_id or os.geteuid()
+ return username, userid
+
+
+DOCKERFILE_TEMPLATE = """
+# ----- stage 1: build -----
+FROM {py_build_image} as build
+
+# requirements section depends on pip vs conda, and presence of buildx
+ENV PIP_PROGRESS_BAR off
+{requirements_section}
+
+# ----- stage 2: base -----
+{base_setup}
+
+COPY --from=build /env /env
+ENV PATH="/env/bin:$PATH"
+
+ENV SHELL /bin/bash
+
+# some resources (eg sagemaker) must run on root
+{user_setup}
+
+WORKDIR {workdir}
+RUN chown -R {uid} {workdir}
+
+# make artifacts cache dir unrelated to build
+RUN mkdir -p {workdir}/.cache && chown -R {uid} {workdir}/.cache
+
+# copy code/etc
+COPY --chown={uid} src/ {workdir}
+
+ENV PYTHONUNBUFFERED=1
+
+{entrypoint_section}
+"""
+
+# this goes into base_setup in TEMPLATE
+PYTHON_SETUP_TEMPLATE = """
+FROM {py_base_image} as base
+"""
+
+# this goes into base_setup in TEMPLATE
+ACCELERATOR_SETUP_TEMPLATE = """
+FROM {accelerator_base_image} as base
+
+# make non-interactive so build doesn't block on questions
+ENV DEBIAN_FRONTEND=noninteractive
+
+# TODO: once NVIDIA their linux repository keys for all docker images
+RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/$(cat /etc/os-release | grep ^ID= | cut -d "=" -f2 )$(cat /etc/os-release | grep ^VERSION_ID= | cut -d "=" -f2 | sed -e 's/[\".]//g' )/$(uname -i)/3bf863cc.pub
+RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/$(cat /etc/os-release | grep ^ID= | cut -d "=" -f2 )$(cat /etc/os-release | grep ^VERSION_ID= | cut -d "=" -f2 | sed -e 's/[\".]//g' )/$(uname -i)/7fa2af80.pub
+RUN apt-get update -qq && apt-get install -y software-properties-common && add-apt-repository -y ppa:deadsnakes/ppa
+
+# install python
+RUN apt-get update -qq && apt-get install --no-install-recommends -y \
+ {python_packages} \
+ && apt-get -qq purge && apt-get -qq clean \
+ && rm -rf /var/lib/apt/lists/*
+
+# make sure `python` points at the right version
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python{py_version} 1 \
+ && update-alternatives --install /usr/local/bin/python python /usr/bin/python{py_version} 1
+"""
+
+# this goes into requirements_section in TEMPLATE
+PIP_TEMPLATE = """
+RUN python -m venv /env
+# make sure we install into the env
+ENV PATH="/env/bin:$PATH"
+
+COPY {requirements_files} ./
+{buildx_optional_prefix} {pip_install}
+"""
+
+# this goes into requirements_section in TEMPLATE
+CONDA_TEMPLATE = """
+COPY src/environment.yml .
+{buildx_optional_prefix} conda env create -f environment.yml -n env
+
+# pack the environment so that we can transfer to the base image
+RUN conda install -c conda-forge conda-pack
+RUN conda pack -n env -o /tmp/env.tar && \
+ mkdir /env && cd /env && tar xf /tmp/env.tar && \
+ rm /tmp/env.tar
+RUN /env/bin/conda-unpack
+"""
+
+USER_CREATE_TEMPLATE = """
+RUN useradd \
+ --create-home \
+ --no-log-init \
+ --shell /bin/bash \
+ --gid 0 \
+ --uid {uid} \
+ {user} || echo ""
+"""
+
+ENTRYPOINT_TEMPLATE = """
+ENTRYPOINT {entrypoint}
+"""
+
+
+def get_current_python_version() -> Tuple[str, str]:
+ full_version = sys.version.split()[0].split(".")
+ major = full_version[0]
+ version = ".".join(full_version[:2]) if len(full_version) >= 2 else major + ".0"
+ return version, major
+
+
+def get_base_setup(
+ launch_project: LaunchProject, py_version: str, py_major: str
+) -> str:
+ """Fill in the Dockerfile templates for stage 2 of build.
+
+ CPU version is built on python, Accelerator version is built on user provided.
+ """
+ python_base_image = f"python:{py_version}-buster"
+ if launch_project.accelerator_base_image:
+ _logger.info(
+ f"Using accelerator base image: {launch_project.accelerator_base_image}"
+ )
+ # accelerator base images doesn't come with python tooling
+ if py_major == "2":
+ python_packages = [
+ f"python{py_version}",
+ f"libpython{py_version}",
+ "python-pip",
+ "python-setuptools",
+ ]
+ else:
+ python_packages = [
+ f"python{py_version}",
+ f"libpython{py_version}",
+ "python3-pip",
+ "python3-setuptools",
+ ]
+ base_setup = ACCELERATOR_SETUP_TEMPLATE.format(
+ accelerator_base_image=launch_project.accelerator_base_image,
+ python_packages=" \\\n".join(python_packages),
+ py_version=py_version,
+ )
+ else:
+ python_packages = [
+ "python3-dev" if py_major == "3" else "python-dev",
+ "gcc",
+ ] # gcc required for python < 3.7 for some reason
+ base_setup = PYTHON_SETUP_TEMPLATE.format(py_base_image=python_base_image)
+ return base_setup
+
+
+def get_env_vars_dict(
+ launch_project: LaunchProject, api: Api, max_env_length: int
+) -> Dict[str, str]:
+ """Generate environment variables for the project.
+
+ Arguments:
+ launch_project: LaunchProject to generate environment variables for.
+
+ Returns:
+ Dictionary of environment variables.
+ """
+ env_vars = {}
+ env_vars["WANDB_BASE_URL"] = api.settings("base_url")
+ override_api_key = launch_project.launch_spec.get("_wandb_api_key")
+ env_vars["WANDB_API_KEY"] = override_api_key or api.api_key
+ if launch_project.target_project:
+ env_vars["WANDB_PROJECT"] = launch_project.target_project
+ env_vars["WANDB_ENTITY"] = launch_project.target_entity
+ env_vars["WANDB_LAUNCH"] = "True"
+ env_vars["WANDB_RUN_ID"] = launch_project.run_id
+ if launch_project.docker_image:
+ env_vars["WANDB_DOCKER"] = launch_project.docker_image
+ if launch_project.name is not None:
+ env_vars["WANDB_NAME"] = launch_project.name
+ if "author" in launch_project.launch_spec and not override_api_key:
+ env_vars["WANDB_USERNAME"] = launch_project.launch_spec["author"]
+ if launch_project.sweep_id:
+ env_vars["WANDB_SWEEP_ID"] = launch_project.sweep_id
+ if launch_project.launch_spec.get("_resume_count", 0) > 0:
+ env_vars["WANDB_RESUME"] = "allow"
+ if launch_project.queue_name:
+ env_vars[wandb.env.LAUNCH_QUEUE_NAME] = launch_project.queue_name
+ if launch_project.queue_entity:
+ env_vars[wandb.env.LAUNCH_QUEUE_ENTITY] = launch_project.queue_entity
+ if launch_project.run_queue_item_id:
+ env_vars[wandb.env.LAUNCH_TRACE_ID] = launch_project.run_queue_item_id
+
+ _inject_wandb_config_env_vars(
+ launch_project.override_config, env_vars, max_env_length
+ )
+ # env_vars["WANDB_CONFIG"] = json.dumps(launch_project.override_config)
+ artifacts = {}
+ # if we're spinning up a launch process from a job
+ # we should tell the run to use that artifact
+ if launch_project.job:
+ artifacts = {wandb.util.LAUNCH_JOB_ARTIFACT_SLOT_NAME: launch_project.job}
+ env_vars["WANDB_ARTIFACTS"] = json.dumps(
+ {**artifacts, **launch_project.override_artifacts}
+ )
+ return env_vars
+
+
+def get_requirements_section(launch_project: LaunchProject, builder_type: str) -> str:
+ if builder_type == "docker":
+ buildx_installed = docker.is_buildx_installed()
+ if not buildx_installed:
+ wandb.termwarn(
+ "Docker BuildX is not installed, for faster builds upgrade docker: https://github.com/docker/buildx#installing"
+ )
+ prefix = "RUN WANDB_DISABLE_CACHE=true"
+ elif builder_type == "kaniko":
+ prefix = "RUN WANDB_DISABLE_CACHE=true"
+ buildx_installed = False
+ if launch_project.deps_type == "pip":
+ requirements_files = []
+ if launch_project.project_dir is not None and os.path.exists(
+ os.path.join(launch_project.project_dir, "requirements.txt")
+ ):
+ requirements_files += ["src/requirements.txt"]
+ pip_install_line = "pip install -r requirements.txt"
+ elif launch_project.project_dir is not None and os.path.exists(
+ os.path.join(launch_project.project_dir, "requirements.frozen.txt")
+ ):
+ # if we have frozen requirements stored, copy those over and have them take precedence
+ requirements_files += ["src/requirements.frozen.txt", "_wandb_bootstrap.py"]
+ pip_install_line = (
+ _parse_existing_requirements(launch_project)
+ + "python _wandb_bootstrap.py"
+ )
+ if buildx_installed:
+ prefix = "RUN --mount=type=cache,mode=0777,target=/root/.cache/pip"
+
+ requirements_line = PIP_TEMPLATE.format(
+ buildx_optional_prefix=prefix,
+ requirements_files=" ".join(requirements_files),
+ pip_install=pip_install_line,
+ )
+ elif launch_project.deps_type == "conda":
+ if buildx_installed:
+ prefix = "RUN --mount=type=cache,mode=0777,target=/opt/conda/pkgs"
+ requirements_line = CONDA_TEMPLATE.format(buildx_optional_prefix=prefix)
+ else:
+ # this means no deps file was found
+ requirements_line = "RUN mkdir -p env/" # Docker fails otherwise
+ wandb.termwarn("No requirements file found. No packages will be installed.")
+
+ return requirements_line
+
+
+def get_user_setup(username: str, userid: int, runner_type: str) -> str:
+ if runner_type == "sagemaker":
+ # sagemaker must run as root
+ return "USER root"
+ user_create = USER_CREATE_TEMPLATE.format(uid=userid, user=username)
+ user_create += f"\nUSER {username}"
+ return user_create
+
+
+def get_entrypoint_setup(
+ entry_point: EntryPoint,
+) -> str:
+ return ENTRYPOINT_TEMPLATE.format(entrypoint=json.dumps(entry_point.command))
+
+
+def generate_dockerfile(
+ launch_project: LaunchProject,
+ entry_point: EntryPoint,
+ runner_type: str,
+ builder_type: str,
+ dockerfile: Optional[str] = None,
+) -> str:
+ if launch_project.project_dir is not None and dockerfile:
+ path = os.path.join(launch_project.project_dir, dockerfile)
+ if not os.path.exists(path):
+ raise LaunchError(f"Dockerfile does not exist at {path}")
+ launch_project.project_dir = os.path.dirname(path)
+ wandb.termlog(f"Using dockerfile: {dockerfile}")
+ return open(path).read()
+
+ # get python versions truncated to major.minor to ensure image availability
+ if launch_project.python_version:
+ spl = launch_project.python_version.split(".")[:2]
+ py_version, py_major = (".".join(spl), spl[0])
+ else:
+ py_version, py_major = get_current_python_version()
+
+ # ----- stage 1: build -----
+ if launch_project.deps_type == "pip" or launch_project.deps_type is None:
+ python_build_image = (
+ f"python:{py_version}" # use full python image for package installation
+ )
+ elif launch_project.deps_type == "conda":
+ # neither of these images are receiving regular updates, latest should be pretty stable
+ python_build_image = (
+ "continuumio/miniconda3:latest"
+ if py_major == "3"
+ else "continuumio/miniconda:latest"
+ )
+ requirements_section = get_requirements_section(launch_project, builder_type)
+ # ----- stage 2: base -----
+ python_base_setup = get_base_setup(launch_project, py_version, py_major)
+
+ # set up user info
+ username, userid = get_docker_user(launch_project, runner_type)
+ user_setup = get_user_setup(username, userid, runner_type)
+ workdir = f"/home/{username}"
+
+ entrypoint_section = get_entrypoint_setup(entry_point)
+
+ dockerfile_contents = DOCKERFILE_TEMPLATE.format(
+ py_build_image=python_build_image,
+ requirements_section=requirements_section,
+ base_setup=python_base_setup,
+ uid=userid,
+ user_setup=user_setup,
+ workdir=workdir,
+ entrypoint_section=entrypoint_section,
+ )
+ return dockerfile_contents
+
+
+def construct_gcp_registry_uri(
+ gcp_repo: str, gcp_project: str, gcp_registry: str
+) -> str:
+ return "/".join([gcp_registry, gcp_project, gcp_repo])
+
+
+def _parse_existing_requirements(launch_project: LaunchProject) -> str:
+ requirements_line = ""
+ assert launch_project.project_dir is not None
+ base_requirements = os.path.join(launch_project.project_dir, "requirements.txt")
+ if os.path.exists(base_requirements):
+ include_only = set()
+ with open(base_requirements) as f:
+ iter = pkg_resources.parse_requirements(f)
+ while True:
+ try:
+ pkg = next(iter)
+ if hasattr(pkg, "name"):
+ name = pkg.name.lower()
+ else:
+ name = str(pkg)
+ include_only.add(shlex_quote(name))
+ except StopIteration:
+ break
+ # Different versions of pkg_resources throw different errors
+ # just catch them all and ignore packages we can't parse
+ except Exception as e:
+ _logger.warn(f"Unable to parse requirements.txt: {e}")
+ continue
+ requirements_line += "WANDB_ONLY_INCLUDE={} ".format(",".join(include_only))
+ return requirements_line
+
+
+def _create_docker_build_ctx(
+ launch_project: LaunchProject,
+ dockerfile_contents: str,
+) -> str:
+ """Create a build context temp dir for a Dockerfile and project code."""
+ assert launch_project.project_dir is not None
+ directory = tempfile.mkdtemp()
+ entrypoint = launch_project.get_single_entry_point()
+ if entrypoint is not None:
+ assert entrypoint.name is not None
+ entrypoint_dir = os.path.dirname(entrypoint.name)
+ if entrypoint_dir:
+ path = os.path.join(
+ launch_project.project_dir, entrypoint_dir, _WANDB_DOCKERFILE_NAME
+ )
+ else:
+ path = os.path.join(launch_project.project_dir, _WANDB_DOCKERFILE_NAME)
+ if os.path.exists(
+ path
+ ): # We found a Dockerfile.wandb adjacent to the entrypoint.
+ shutil.copytree(
+ os.path.dirname(path),
+ directory,
+ symlinks=True,
+ dirs_exist_ok=True,
+ ignore=shutil.ignore_patterns("fsmonitor--daemon.ipc"),
+ )
+ return directory
+
+ dst_path = os.path.join(directory, "src")
+ assert launch_project.project_dir is not None
+ shutil.copytree(
+ src=launch_project.project_dir,
+ dst=dst_path,
+ symlinks=True,
+ ignore=shutil.ignore_patterns("fsmonitor--daemon.ipc"),
+ )
+ shutil.copy(
+ os.path.join(os.path.dirname(__file__), "templates", "_wandb_bootstrap.py"),
+ os.path.join(directory),
+ )
+ if launch_project.python_version:
+ runtime_path = os.path.join(dst_path, "runtime.txt")
+ with open(runtime_path, "w") as fp:
+ fp.write(f"python-{launch_project.python_version}")
+ # TODO: we likely don't need to pass the whole git repo into the container
+ # with open(os.path.join(directory, ".dockerignore"), "w") as f:
+ # f.write("**/.git")
+ with open(os.path.join(directory, _WANDB_DOCKERFILE_NAME), "w") as handle:
+ handle.write(dockerfile_contents)
+ return directory
+
+
+def join(split_command: List[str]) -> str:
+ """Return a shell-escaped string from *split_command*.
+
+ Also remove quotes from double quoted strings. Ex:
+ "'local container queue'" --> "local container queue"
+ """
+ return " ".join(shlex.quote(arg.replace("'", "")) for arg in split_command)
+
+
+def construct_agent_configs(
+ launch_config: Optional[Dict] = None,
+ build_config: Optional[Dict] = None,
+) -> Tuple[Optional[Dict[str, Any]], Dict[str, Any], Dict[str, Any]]:
+ registry_config = None
+ environment_config = None
+ if launch_config is not None:
+ build_config = launch_config.get("builder")
+ registry_config = launch_config.get("registry")
+
+ default_launch_config = None
+ if os.path.exists(os.path.expanduser(LAUNCH_CONFIG_FILE)):
+ with open(os.path.expanduser(LAUNCH_CONFIG_FILE)) as f:
+ default_launch_config = (
+ yaml.safe_load(f) or {}
+ ) # In case the config is empty, we want it to be {} instead of None.
+ environment_config = default_launch_config.get("environment")
+
+ build_config, registry_config = resolve_build_and_registry_config(
+ default_launch_config, build_config, registry_config
+ )
+
+ return environment_config, build_config, registry_config
+
+
+async def build_image_from_project(
+ launch_project: LaunchProject,
+ api: Api,
+ launch_config: Dict[str, Any],
+) -> str:
+ """Construct a docker image from a project and returns the URI of the image.
+
+ Arguments:
+ launch_project: The project to build an image from.
+ api: The API object to use for fetching the project.
+ launch_config: The launch config to use for building the image.
+
+ Returns:
+ The URI of the built image.
+ """
+ assert launch_project.uri, "To build an image on queue a URI must be set."
+ launch_config = launch_config or {}
+ env_config = launch_config.get("environment", {})
+ if not isinstance(env_config, dict):
+ wrong_type = type(env_config).__name__
+ raise LaunchError(
+ f"Invalid environment config: {env_config} of type {wrong_type} "
+ "loaded from launch config. Expected dict."
+ )
+ environment = environment_from_config(env_config)
+
+ registry_config = launch_config.get("registry", {})
+ if not isinstance(registry_config, dict):
+ wrong_type = type(registry_config).__name__
+ raise LaunchError(
+ f"Invalid registry config: {registry_config} of type {wrong_type}"
+ " loaded from launch config. Expected dict."
+ )
+ registry = registry_from_config(registry_config, environment)
+
+ builder_config = launch_config.get("builder", {})
+ if not isinstance(builder_config, dict):
+ wrong_type = type(builder_config).__name__
+ raise LaunchError(
+ f"Invalid builder config: {builder_config} of type {wrong_type} "
+ "loaded from launch config. Expected dict."
+ )
+ builder = builder_from_config(builder_config, environment, registry)
+
+ if not builder:
+ raise LaunchError("Unable to build image. No builder found.")
+
+ launch_project = fetch_and_validate_project(launch_project, api)
+
+ entry_point: EntryPoint = launch_project.get_single_entry_point() or EntryPoint(
+ name=EntrypointDefaults.PYTHON[-1],
+ command=EntrypointDefaults.PYTHON,
+ )
+ wandb.termlog(f"{LOG_PREFIX}Building docker image from uri source")
+ image_uri = await builder.build_image(launch_project, entry_point)
+ if not image_uri:
+ raise LaunchError("Error building image uri")
+ else:
+ return image_uri
+
+
+def image_tag_from_dockerfile_and_source(
+ launch_project: LaunchProject, dockerfile_contents: str
+) -> str:
+ """Hashes the source and dockerfile contents into a unique tag."""
+ image_source_string = launch_project.get_image_source_string()
+ unique_id_string = image_source_string + dockerfile_contents
+ image_tag = hashlib.sha256(unique_id_string.encode("utf-8")).hexdigest()[:8]
+ return image_tag
+
+
+def _inject_wandb_config_env_vars(
+ config: Dict[str, Any], env_dict: Dict[str, Any], maximum_env_length: int
+) -> None:
+ str_config = json.dumps(config)
+ if len(str_config) <= maximum_env_length:
+ env_dict["WANDB_CONFIG"] = str_config
+ return
+
+ chunks = [
+ str_config[i : i + maximum_env_length]
+ for i in range(0, len(str_config), maximum_env_length)
+ ]
+ config_chunks_dict = {f"WANDB_CONFIG_{i}": chunk for i, chunk in enumerate(chunks)}
+ env_dict.update(config_chunks_dict)
diff --git a/wandb/sdk/launch/builder/docker_builder.py b/wandb/sdk/launch/builder/docker_builder.py
new file mode 100644
index 0000000000000000000000000000000000000000..7ade38155fea5822bb4c15315eb4309fa3bcf360
--- /dev/null
+++ b/wandb/sdk/launch/builder/docker_builder.py
@@ -0,0 +1,203 @@
+"""Implementation of the docker builder."""
+import logging
+import os
+from typing import Any, Dict, Optional
+
+import wandb
+import wandb.docker as docker
+from wandb.sdk.launch.agent.job_status_tracker import JobAndRunStatusTracker
+from wandb.sdk.launch.builder.abstract import AbstractBuilder
+from wandb.sdk.launch.builder.build import registry_from_uri
+from wandb.sdk.launch.environment.abstract import AbstractEnvironment
+from wandb.sdk.launch.registry.abstract import AbstractRegistry
+
+from .._project_spec import (
+ EntryPoint,
+ LaunchProject,
+ create_metadata_file,
+ get_entry_point_command,
+)
+from ..errors import LaunchDockerError, LaunchError
+from ..registry.local_registry import LocalRegistry
+from ..utils import (
+ LOG_PREFIX,
+ event_loop_thread_exec,
+ sanitize_wandb_api_key,
+ warn_failed_packages_from_build_logs,
+)
+from .build import (
+ _WANDB_DOCKERFILE_NAME,
+ _create_docker_build_ctx,
+ generate_dockerfile,
+ image_tag_from_dockerfile_and_source,
+ validate_docker_installation,
+)
+
+_logger = logging.getLogger(__name__)
+
+
+class DockerBuilder(AbstractBuilder):
+ """Builds a docker image for a project.
+
+ Attributes:
+ builder_config (Dict[str, Any]): The builder config.
+
+ """
+
+ builder_type = "docker"
+ base_image = "python:3.8"
+ target_platform = "linux/amd64"
+
+ def __init__(
+ self,
+ environment: AbstractEnvironment,
+ registry: AbstractRegistry,
+ config: Dict[str, Any],
+ ):
+ """Initialize a DockerBuilder.
+
+ Arguments:
+ environment (AbstractEnvironment): The environment to use.
+ registry (AbstractRegistry): The registry to use.
+
+ Raises:
+ LaunchError: If docker is not installed
+ """
+ self.environment = environment # Docker builder doesn't actually use this.
+ self.registry = registry
+ self.config = config
+
+ @classmethod
+ def from_config(
+ cls,
+ config: Dict[str, Any],
+ environment: AbstractEnvironment,
+ registry: AbstractRegistry,
+ ) -> "DockerBuilder":
+ """Create a DockerBuilder from a config.
+
+ Arguments:
+ config (Dict[str, Any]): The config.
+ registry (AbstractRegistry): The registry to use.
+ verify (bool, optional): Whether to verify the functionality of the builder.
+ login (bool, optional): Whether to login to the registry.
+
+ Returns:
+ DockerBuilder: The DockerBuilder.
+ """
+ # If the user provided a destination URI in the builder config
+ # we use that as the registry.
+ image_uri = config.get("destination")
+ if image_uri:
+ if registry is not None:
+ wandb.termwarn(
+ f"{LOG_PREFIX}Overriding registry from registry config"
+ f" with {image_uri} from builder config."
+ )
+ registry = registry_from_uri(image_uri)
+
+ return cls(environment, registry, config)
+
+ async def verify(self) -> None:
+ """Verify the builder."""
+ await validate_docker_installation()
+
+ async def login(self) -> None:
+ """Login to the registry."""
+ if isinstance(self.registry, LocalRegistry):
+ _logger.info(f"{LOG_PREFIX}No registry configured, skipping login.")
+ else:
+ username, password = await self.registry.get_username_password()
+ login = event_loop_thread_exec(docker.login)
+ await login(username, password, self.registry.uri)
+
+ async def build_image(
+ self,
+ launch_project: LaunchProject,
+ entrypoint: EntryPoint,
+ job_tracker: Optional[JobAndRunStatusTracker] = None,
+ ) -> str:
+ """Build the image for the given project.
+
+ Arguments:
+ launch_project (LaunchProject): The project to build.
+ entrypoint (EntryPoint): The entrypoint to use.
+ """
+ await self.verify()
+ await self.login()
+
+ dockerfile_str = generate_dockerfile(
+ launch_project=launch_project,
+ entry_point=entrypoint,
+ runner_type=launch_project.resource,
+ builder_type="docker",
+ dockerfile=launch_project.override_dockerfile,
+ )
+
+ image_tag = image_tag_from_dockerfile_and_source(launch_project, dockerfile_str)
+
+ repository = None if not self.registry else await self.registry.get_repo_uri()
+ # if repo is set, use the repo name as the image name
+ if repository:
+ image_uri = f"{repository}:{image_tag}"
+ # otherwise, base the image name off of the source
+ # which the launch_project checks in image_name
+ else:
+ image_uri = f"{launch_project.image_name}:{image_tag}"
+
+ if (
+ not launch_project.build_required()
+ and await self.registry.check_image_exists(image_uri)
+ ):
+ return image_uri
+
+ _logger.info(
+ f"image {image_uri} does not already exist in repository, building."
+ )
+
+ entry_cmd = get_entry_point_command(entrypoint, launch_project.override_args)
+
+ create_metadata_file(
+ launch_project,
+ image_uri,
+ sanitize_wandb_api_key(" ".join(entry_cmd)),
+ dockerfile_str,
+ )
+ build_ctx_path = _create_docker_build_ctx(launch_project, dockerfile_str)
+ dockerfile = os.path.join(build_ctx_path, _WANDB_DOCKERFILE_NAME)
+ try:
+ output = await event_loop_thread_exec(docker.build)(
+ tags=[image_uri],
+ file=dockerfile,
+ context_path=build_ctx_path,
+ platform=self.config.get("platform"),
+ )
+
+ warn_failed_packages_from_build_logs(
+ output, image_uri, launch_project.api, job_tracker
+ )
+
+ except docker.DockerError as e:
+ if job_tracker:
+ job_tracker.set_err_stage("build")
+ raise LaunchDockerError(f"Error communicating with docker client: {e}")
+
+ try:
+ os.remove(build_ctx_path)
+ except Exception:
+ _msg = f"{LOG_PREFIX}Temporary docker context file {build_ctx_path} was not deleted."
+ _logger.info(_msg)
+
+ if repository:
+ reg, tag = image_uri.split(":")
+ wandb.termlog(f"{LOG_PREFIX}Pushing image {image_uri}")
+ push_resp = await event_loop_thread_exec(docker.push)(reg, tag)
+ if push_resp is None:
+ raise LaunchError("Failed to push image to repository")
+ elif (
+ launch_project.resource == "sagemaker"
+ and f"The push refers to repository [{repository}]" not in push_resp
+ ):
+ raise LaunchError(f"Unable to push image to ECR, response: {push_resp}")
+
+ return image_uri
diff --git a/wandb/sdk/launch/builder/kaniko_builder.py b/wandb/sdk/launch/builder/kaniko_builder.py
new file mode 100644
index 0000000000000000000000000000000000000000..2c12f62f0be9c42a9d072cb35140879018445bcb
--- /dev/null
+++ b/wandb/sdk/launch/builder/kaniko_builder.py
@@ -0,0 +1,537 @@
+import asyncio
+import base64
+import json
+import logging
+import os
+import tarfile
+import tempfile
+import time
+import traceback
+from typing import Optional
+
+import wandb
+from wandb.sdk.launch.agent.job_status_tracker import JobAndRunStatusTracker
+from wandb.sdk.launch.builder.abstract import AbstractBuilder
+from wandb.sdk.launch.builder.build import registry_from_uri
+from wandb.sdk.launch.environment.abstract import AbstractEnvironment
+from wandb.sdk.launch.environment.azure_environment import AzureEnvironment
+from wandb.sdk.launch.registry.abstract import AbstractRegistry
+from wandb.sdk.launch.registry.azure_container_registry import AzureContainerRegistry
+from wandb.sdk.launch.registry.elastic_container_registry import (
+ ElasticContainerRegistry,
+)
+from wandb.sdk.launch.registry.google_artifact_registry import GoogleArtifactRegistry
+from wandb.util import get_module
+
+from .._project_spec import (
+ EntryPoint,
+ LaunchProject,
+ create_metadata_file,
+ get_entry_point_command,
+)
+from ..errors import LaunchError
+from ..utils import (
+ LOG_PREFIX,
+ get_kube_context_and_api_client,
+ sanitize_wandb_api_key,
+ warn_failed_packages_from_build_logs,
+)
+from .build import (
+ _WANDB_DOCKERFILE_NAME,
+ _create_docker_build_ctx,
+ generate_dockerfile,
+ image_tag_from_dockerfile_and_source,
+)
+
+get_module(
+ "kubernetes_asyncio",
+ required="Kaniko builder requires the kubernetes_asyncio package. Please install it with `pip install wandb[launch]`.",
+)
+
+import kubernetes_asyncio as kubernetes # type: ignore # noqa: E402
+from kubernetes_asyncio import client # noqa: E402
+
+_logger = logging.getLogger(__name__)
+
+_DEFAULT_BUILD_TIMEOUT_SECS = 1800 # 30 minute build timeout
+
+SERVICE_ACCOUNT_NAME = os.environ.get("WANDB_LAUNCH_SERVICE_ACCOUNT_NAME", "default")
+
+if os.path.exists("/var/run/secrets/kubernetes.io/serviceaccount/namespace"):
+ with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as f:
+ NAMESPACE = f.read().strip()
+else:
+ NAMESPACE = "wandb"
+
+
+async def _wait_for_completion(
+ batch_client: client.BatchV1Api, job_name: str, deadline_secs: Optional[int] = None
+) -> bool:
+ start_time = time.time()
+ while True:
+ job = await batch_client.read_namespaced_job_status(job_name, NAMESPACE)
+ if job.status.succeeded is not None and job.status.succeeded >= 1:
+ return True
+ elif job.status.failed is not None and job.status.failed >= 1:
+ wandb.termerror(f"{LOG_PREFIX}Build job {job.status.failed} failed {job}")
+ return False
+ wandb.termlog(f"{LOG_PREFIX}Waiting for build job to complete...")
+ if deadline_secs is not None and time.time() - start_time > deadline_secs:
+ return False
+
+ await asyncio.sleep(5)
+
+
+class KanikoBuilder(AbstractBuilder):
+ """Builds a docker image for a project using Kaniko."""
+
+ type = "kaniko"
+
+ build_job_name: str
+ build_context_store: str
+ secret_name: Optional[str]
+ secret_key: Optional[str]
+ image: str
+
+ def __init__(
+ self,
+ environment: AbstractEnvironment,
+ registry: AbstractRegistry,
+ build_job_name: str = "wandb-launch-container-build",
+ build_context_store: str = "",
+ secret_name: str = "",
+ secret_key: str = "",
+ image: str = "gcr.io/kaniko-project/executor:v1.11.0",
+ ):
+ """Initialize a KanikoBuilder.
+
+ Arguments:
+ environment (AbstractEnvironment): The environment to use.
+ registry (AbstractRegistry): The registry to use.
+ build_job_name (str, optional): The name of the build job.
+ build_context_store (str, optional): The name of the build context store.
+ secret_name (str, optional): The name of the secret to use for the registry.
+ secret_key (str, optional): The key of the secret to use for the registry.
+ verify (bool, optional): Whether to verify the functionality of the builder.
+ Defaults to True.
+ """
+ if build_context_store is None:
+ raise LaunchError(
+ "You are required to specify an external build "
+ "context store for Kaniko builds. Please specify a storage url "
+ "in the 'build-context-store' field of your builder config."
+ )
+ self.environment = environment
+ self.registry = registry
+ self.build_job_name = build_job_name
+ self.build_context_store = build_context_store.rstrip("/")
+ self.secret_name = secret_name
+ self.secret_key = secret_key
+ self.image = image
+
+ @classmethod
+ def from_config(
+ cls,
+ config: dict,
+ environment: AbstractEnvironment,
+ registry: AbstractRegistry,
+ verify: bool = True,
+ login: bool = True,
+ ) -> "AbstractBuilder":
+ """Create a KanikoBuilder from a config dict.
+
+ Arguments:
+ config: A dict containing the builder config. Must contain a "type" key
+ with value "kaniko".
+ environment: The environment to use for the build.
+ registry: The registry to use for the build.
+ verify: Whether to verify the builder config.
+
+ Returns:
+ A KanikoBuilder instance.
+ """
+ if config.get("type") != "kaniko":
+ raise LaunchError(
+ "Builder config must include 'type':'kaniko' to create a KanikoBuilder."
+ )
+ build_context_store = config.get("build-context-store")
+ if build_context_store is None:
+ raise LaunchError(
+ "You are required to specify an external build "
+ "context store for Kaniko builds. Please specify a "
+ "storage url in the 'build_context_store' field of your builder config."
+ )
+ build_job_name = config.get("build-job-name", "wandb-launch-container-build")
+ secret_name = config.get("secret-name", "")
+ secret_key = config.get("secret-key", "")
+ kaniko_image = config.get(
+ "kaniko-image", "gcr.io/kaniko-project/executor:v1.11.0"
+ )
+ image_uri = config.get("destination")
+ if image_uri is not None:
+ registry = registry_from_uri(image_uri)
+ return cls(
+ environment,
+ registry,
+ build_context_store=build_context_store,
+ build_job_name=build_job_name,
+ secret_name=secret_name,
+ secret_key=secret_key,
+ image=kaniko_image,
+ )
+
+ async def verify(self) -> None:
+ """Verify that the builder config is valid.
+
+ Raises:
+ LaunchError: If the builder config is invalid.
+ """
+ if self.environment is None:
+ raise LaunchError("No environment specified for Kaniko build.")
+ await self.environment.verify_storage_uri(self.build_context_store)
+
+ def login(self) -> None:
+ """Login to the registry."""
+ pass
+
+ async def _create_docker_ecr_config_map(
+ self, job_name: str, corev1_client: client.CoreV1Api, repository: str
+ ) -> None:
+ if self.registry is None:
+ raise LaunchError("No registry specified for Kaniko build.")
+ username, password = await self.registry.get_username_password()
+ encoded = base64.b64encode(f"{username}:{password}".encode()).decode("utf-8")
+ ecr_config_map = client.V1ConfigMap(
+ api_version="v1",
+ kind="ConfigMap",
+ metadata=client.V1ObjectMeta(
+ name=f"docker-config-{job_name}",
+ namespace=NAMESPACE,
+ ),
+ data={
+ "config.json": json.dumps(
+ {
+ "auths": {
+ f"{await self.registry.get_repo_uri()}": {"auth": encoded}
+ }
+ }
+ )
+ },
+ immutable=True,
+ )
+ await corev1_client.create_namespaced_config_map(NAMESPACE, ecr_config_map)
+
+ async def _delete_docker_ecr_config_map(
+ self, job_name: str, client: client.CoreV1Api
+ ) -> None:
+ if self.secret_name:
+ await client.delete_namespaced_config_map(
+ f"docker-config-{job_name}", NAMESPACE
+ )
+
+ async def _upload_build_context(self, run_id: str, context_path: str) -> str:
+ # creat a tar archive of the build context and upload it to s3
+ context_file = tempfile.NamedTemporaryFile(delete=False)
+ with tarfile.TarFile.open(fileobj=context_file, mode="w:gz") as context_tgz:
+ context_tgz.add(context_path, arcname=".")
+ context_file.close()
+ destination = f"{self.build_context_store}/{run_id}.tgz"
+ if self.environment is None:
+ raise LaunchError("No environment specified for Kaniko build.")
+ await self.environment.upload_file(context_file.name, destination)
+ return destination
+
+ async def build_image(
+ self,
+ launch_project: LaunchProject,
+ entrypoint: EntryPoint,
+ job_tracker: Optional[JobAndRunStatusTracker] = None,
+ ) -> str:
+ await self.verify()
+ # TODO: this should probably throw an error if the registry is a local registry
+ if not self.registry:
+ raise LaunchError("No registry specified for Kaniko build.")
+ # kaniko builder doesn't seem to work with a custom user id, need more investigation
+ dockerfile_str = generate_dockerfile(
+ launch_project=launch_project,
+ entry_point=entrypoint,
+ runner_type=launch_project.resource,
+ builder_type="kaniko",
+ dockerfile=launch_project.override_dockerfile,
+ )
+ image_tag = image_tag_from_dockerfile_and_source(launch_project, dockerfile_str)
+ repo_uri = await self.registry.get_repo_uri()
+ image_uri = repo_uri + ":" + image_tag
+
+ if (
+ not launch_project.build_required()
+ and await self.registry.check_image_exists(image_uri)
+ ):
+ return image_uri
+
+ _logger.info(f"Building image {image_uri}...")
+
+ entry_cmd = " ".join(
+ get_entry_point_command(entrypoint, launch_project.override_args)
+ )
+
+ create_metadata_file(
+ launch_project,
+ image_uri,
+ sanitize_wandb_api_key(entry_cmd),
+ sanitize_wandb_api_key(dockerfile_str),
+ )
+ context_path = _create_docker_build_ctx(launch_project, dockerfile_str)
+ run_id = launch_project.run_id
+
+ _, api_client = await get_kube_context_and_api_client(
+ kubernetes, launch_project.resource_args
+ )
+ # TODO: use same client as kuberentes_runner.py
+ batch_v1 = client.BatchV1Api(api_client)
+ core_v1 = client.CoreV1Api(api_client)
+
+ build_job_name = f"{self.build_job_name}-{run_id}"
+
+ build_context = await self._upload_build_context(run_id, context_path)
+ build_job = await self._create_kaniko_job(
+ build_job_name, repo_uri, image_uri, build_context, core_v1
+ )
+ wandb.termlog(f"{LOG_PREFIX}Created kaniko job {build_job_name}")
+
+ try:
+ if isinstance(self.registry, AzureContainerRegistry):
+ dockerfile_config_map = client.V1ConfigMap(
+ metadata=client.V1ObjectMeta(
+ name=f"docker-config-{build_job_name}"
+ ),
+ data={
+ "config.json": json.dumps(
+ {
+ "credHelpers": {
+ f"{self.registry.registry_name}.azurecr.io": "acr-env"
+ }
+ }
+ )
+ },
+ )
+ await core_v1.create_namespaced_config_map(
+ "wandb", dockerfile_config_map
+ )
+ if self.secret_name:
+ await self._create_docker_ecr_config_map(
+ build_job_name, core_v1, repo_uri
+ )
+ await batch_v1.create_namespaced_job(NAMESPACE, build_job)
+
+ # wait for double the job deadline since it might take time to schedule
+ if not await _wait_for_completion(
+ batch_v1, build_job_name, 3 * _DEFAULT_BUILD_TIMEOUT_SECS
+ ):
+ if job_tracker:
+ job_tracker.set_err_stage("build")
+ raise Exception(f"Failed to build image in kaniko for job {run_id}")
+ try:
+ pods_from_job = await core_v1.list_namespaced_pod(
+ namespace=NAMESPACE, label_selector=f"job-name={build_job_name}"
+ )
+ if len(pods_from_job.items) != 1:
+ raise Exception(
+ f"Expected 1 pod for job {build_job_name}, found {len(pods_from_job.items)}"
+ )
+ pod_name = pods_from_job.items[0].metadata.name
+ logs = await core_v1.read_namespaced_pod_log(pod_name, NAMESPACE)
+ warn_failed_packages_from_build_logs(
+ logs, image_uri, launch_project.api, job_tracker
+ )
+ except Exception as e:
+ wandb.termwarn(
+ f"{LOG_PREFIX}Failed to get logs for kaniko job {build_job_name}: {e}"
+ )
+ except Exception as e:
+ wandb.termerror(
+ f"{LOG_PREFIX}Exception when creating Kubernetes resources: {e}\n"
+ )
+ raise e
+ finally:
+ wandb.termlog(f"{LOG_PREFIX}Cleaning up resources")
+ try:
+ if isinstance(self.registry, AzureContainerRegistry):
+ await core_v1.delete_namespaced_config_map(
+ f"docker-config-{build_job_name}", "wandb"
+ )
+ if self.secret_name:
+ await self._delete_docker_ecr_config_map(build_job_name, core_v1)
+ await batch_v1.delete_namespaced_job(build_job_name, NAMESPACE)
+ except Exception as e:
+ traceback.print_exc()
+ raise LaunchError(
+ f"Exception during Kubernetes resource clean up {e}"
+ ) from e
+ return image_uri
+
+ async def _create_kaniko_job(
+ self,
+ job_name: str,
+ repository: str,
+ image_tag: str,
+ build_context_path: str,
+ core_client: client.CoreV1Api,
+ ) -> "client.V1Job":
+ env = []
+ volume_mounts = []
+ volumes = []
+ if bool(self.secret_name) != bool(self.secret_key):
+ raise LaunchError(
+ "Both secret_name and secret_key or neither must be specified "
+ "for kaniko build. You provided only one of them."
+ )
+ if isinstance(self.registry, ElasticContainerRegistry):
+ env += [
+ client.V1EnvVar(
+ name="AWS_REGION",
+ value=self.registry.region,
+ )
+ ]
+ # TODO: Refactor all of this environment/registry
+ # specific stuff into methods of those classes.
+ if isinstance(self.environment, AzureEnvironment):
+ # Use the core api to check if the secret exists
+ try:
+ await core_client.read_namespaced_secret(
+ "azure-storage-access-key",
+ "wandb",
+ )
+ except Exception as e:
+ raise LaunchError(
+ "Secret azure-storage-access-key does not exist in "
+ "namespace wandb. Please create it with the key password "
+ "set to your azure storage access key."
+ ) from e
+ env += [
+ client.V1EnvVar(
+ name="AZURE_STORAGE_ACCESS_KEY",
+ value_from=client.V1EnvVarSource(
+ secret_key_ref=client.V1SecretKeySelector(
+ name="azure-storage-access-key",
+ key="password",
+ )
+ ),
+ )
+ ]
+
+ if self.secret_name and self.secret_key:
+ volumes += [
+ client.V1Volume(
+ name="docker-config",
+ config_map=client.V1ConfigMapVolumeSource(
+ name=f"docker-config-{job_name}",
+ ),
+ ),
+ ]
+ volume_mounts += [
+ client.V1VolumeMount(
+ name="docker-config", mount_path="/kaniko/.docker/"
+ ),
+ ]
+ # TODO: I don't like conditioning on the registry type here. As a
+ # future change I want the registry and environment classes to provide
+ # a list of environment variables and volume mounts that need to be
+ # added to the job. The environment class provides credentials for
+ # build context access, and the registry class provides credentials
+ # for pushing the image. This way we can have separate secrets for
+ # each and support build contexts and registries that require
+ # different credentials.
+ if isinstance(self.registry, ElasticContainerRegistry):
+ mount_path = "/root/.aws"
+ key = "credentials"
+ elif isinstance(self.registry, GoogleArtifactRegistry):
+ mount_path = "/kaniko/.config/gcloud"
+ key = "config.json"
+ env += [
+ client.V1EnvVar(
+ name="GOOGLE_APPLICATION_CREDENTIALS",
+ value="/kaniko/.config/gcloud/config.json",
+ )
+ ]
+ else:
+ raise LaunchError(
+ f"Registry type {type(self.registry)} not supported by kaniko"
+ )
+ volume_mounts += [
+ client.V1VolumeMount(
+ name=self.secret_name,
+ mount_path=mount_path,
+ read_only=True,
+ )
+ ]
+ volumes += [
+ client.V1Volume(
+ name=self.secret_name,
+ secret=client.V1SecretVolumeSource(
+ secret_name=self.secret_name,
+ items=[client.V1KeyToPath(key=self.secret_key, path=key)],
+ ),
+ )
+ ]
+ if isinstance(self.registry, AzureContainerRegistry):
+ # ADd the docker config map
+ volume_mounts += [
+ client.V1VolumeMount(
+ name="docker-config", mount_path="/kaniko/.docker/"
+ ),
+ ]
+ volumes += [
+ client.V1Volume(
+ name="docker-config",
+ config_map=client.V1ConfigMapVolumeSource(
+ name=f"docker-config-{job_name}",
+ ),
+ ),
+ ]
+ # Kaniko doesn't want https:// at the begining of the image tag.
+ destination = image_tag
+ if destination.startswith("https://"):
+ destination = destination.replace("https://", "")
+ args = [
+ f"--context={build_context_path}",
+ f"--dockerfile={_WANDB_DOCKERFILE_NAME}",
+ f"--destination={destination}",
+ "--cache=true",
+ f"--cache-repo={repository.replace('https://', '')}",
+ "--snapshotMode=redo",
+ "--compressed-caching=false",
+ ]
+ container = client.V1Container(
+ name="wandb-container-build",
+ image=self.image,
+ args=args,
+ volume_mounts=volume_mounts,
+ env=env if env else None,
+ )
+ # Create and configure a spec section
+ labels = {"wandb": "launch"}
+ # This annotation is required to enable azure workload identity.
+ if isinstance(self.registry, AzureContainerRegistry):
+ labels["azure.workload.identity/use"] = "true"
+ template = client.V1PodTemplateSpec(
+ metadata=client.V1ObjectMeta(labels=labels),
+ spec=client.V1PodSpec(
+ restart_policy="Never",
+ active_deadline_seconds=_DEFAULT_BUILD_TIMEOUT_SECS,
+ containers=[container],
+ volumes=volumes,
+ service_account_name=SERVICE_ACCOUNT_NAME,
+ ),
+ )
+ # Create the specification of job
+ spec = client.V1JobSpec(template=template, backoff_limit=0)
+ job = client.V1Job(
+ api_version="batch/v1",
+ kind="Job",
+ metadata=client.V1ObjectMeta(
+ name=job_name, namespace=NAMESPACE, labels={"wandb": "launch"}
+ ),
+ spec=spec,
+ )
+ return job
diff --git a/wandb/sdk/launch/builder/noop.py b/wandb/sdk/launch/builder/noop.py
new file mode 100644
index 0000000000000000000000000000000000000000..30e9218d504d084bc4c4d104ea69f7e22e410856
--- /dev/null
+++ b/wandb/sdk/launch/builder/noop.py
@@ -0,0 +1,57 @@
+"""NoOp builder implementation."""
+from typing import Any, Dict, Optional
+
+from wandb.sdk.launch.builder.abstract import AbstractBuilder
+from wandb.sdk.launch.environment.abstract import AbstractEnvironment
+from wandb.sdk.launch.errors import LaunchError
+from wandb.sdk.launch.registry.abstract import AbstractRegistry
+
+from .._project_spec import EntryPoint, LaunchProject
+from ..agent.job_status_tracker import JobAndRunStatusTracker
+
+
+class NoOpBuilder(AbstractBuilder):
+ """NoOp builder."""
+
+ type = "noop"
+
+ def __init__(
+ self,
+ builder_config: Dict[str, Any],
+ environment: AbstractEnvironment,
+ registry: AbstractRegistry,
+ ) -> None:
+ """Initialize a NoOpBuilder."""
+ self.environment = environment
+ self.registry = registry
+
+ @classmethod
+ def from_config(
+ cls,
+ config: dict,
+ environment: AbstractEnvironment,
+ registry: AbstractRegistry,
+ verify: bool = True,
+ ) -> "AbstractBuilder":
+ """Create a noop builder from a config."""
+ return cls(config, environment, registry)
+
+ async def verify(self) -> None:
+ """Verify the builder."""
+ raise LaunchError("Attempted to verify noop builder.")
+
+ async def build_image(
+ self,
+ launch_project: LaunchProject,
+ entrypoint: EntryPoint,
+ job_tracker: Optional[JobAndRunStatusTracker] = None,
+ ) -> str:
+ """Build the image.
+
+ For this we raise a launch error since it can't build.
+ """
+ raise LaunchError(
+ "Attempted build with noop builder. Specify a builder in your launch config at ~/.config/wandb/launch-config.yaml.\n"
+ "Note: Jobs sourced from git repos and code artifacts require a builder, while jobs sourced from Docker images do not.\n"
+ "See https://docs.wandb.ai/guides/launch/create-job."
+ )
diff --git a/wandb/sdk/launch/builder/templates/_wandb_bootstrap.py b/wandb/sdk/launch/builder/templates/_wandb_bootstrap.py
new file mode 100644
index 0000000000000000000000000000000000000000..e0bc05b58404f95d7abb77c706f3a5ddd0462fde
--- /dev/null
+++ b/wandb/sdk/launch/builder/templates/_wandb_bootstrap.py
@@ -0,0 +1,187 @@
+import json
+import os
+import re
+import subprocess
+import sys
+from typing import List, Optional, Set
+
+FAILED_PACKAGES_PREFIX = "ERROR: Failed to install: "
+FAILED_PACKAGES_POSTFIX = ". During automated build process."
+ONLY_INCLUDE = {x for x in os.getenv("WANDB_ONLY_INCLUDE", "").split(",") if x != ""}
+OPTS = []
+# If the builder doesn't support buildx no need to use the cache
+if os.getenv("WANDB_DISABLE_CACHE"):
+ OPTS.append("--no-cache-dir")
+# When installing all packages from requirements.frozen.txt no need to resolve deps
+if len(ONLY_INCLUDE) == 0:
+ OPTS.append("--no-deps")
+# When installing the intersection of requirements.frozen.txt and requirements.txt
+# force the frozen versions
+else:
+ OPTS.append("--force")
+
+TORCH_DEP_REGEX = r"torch(vision|audio)?==\d+\.\d+\.\d+(\+(?:cu[\d]{2,3})|(?:\+cpu))?"
+
+
+def install_deps(
+ deps: List[str],
+ failed: Optional[Set[str]] = None,
+ extra_index: Optional[str] = None,
+ opts: Optional[List[str]] = None,
+) -> Optional[Set[str]]:
+ """Install pip dependencies.
+
+ Arguments:
+ deps {List[str]} -- List of dependencies to install
+ failed (set, None): The libraries that failed to install
+
+ Returns:
+ deps (str[], None): The dependencies that failed to install
+ """
+ try:
+ # Include only uri if @ is present
+ clean_deps = [d.split("@")[-1].strip() if "@" in d else d for d in deps]
+ index_args = ["--extra-index-url", extra_index] if extra_index else []
+ print("installing {}...".format(", ".join(clean_deps)))
+ opts = opts or []
+ args = ["pip", "install"] + opts + clean_deps + index_args
+ sys.stdout.flush()
+ subprocess.check_output(args, stderr=subprocess.STDOUT)
+ return failed
+ except subprocess.CalledProcessError as e:
+ if failed is None:
+ failed = set()
+ num_failed = len(failed)
+ current_pkg = None
+ for line in e.output.decode("utf8").splitlines():
+ # Since the name of the package might not be on the same line as
+ # the error msg, keep track of the currently installing package
+ current_pkg = get_current_package(line, clean_deps, current_pkg)
+
+ if "error: subprocess-exited-with-error" in line:
+ if current_pkg is not None:
+ failed.add(current_pkg)
+ elif line.startswith("ERROR:"):
+ clean_dep = find_package_in_error_string(clean_deps, line)
+ if clean_dep is not None:
+ if clean_dep in deps:
+ failed.add(clean_dep)
+ else:
+ for d in deps:
+ if clean_dep in d:
+ failed.add(d.replace(" ", ""))
+ break
+ if len(set(clean_deps) - failed) == 0:
+ return failed
+ elif len(failed) > num_failed:
+ return install_deps(
+ list(set(clean_deps) - failed),
+ failed,
+ extra_index=extra_index,
+ opts=opts,
+ )
+ else:
+ return failed
+
+
+def main() -> None:
+ """Install deps in requirements.frozen.txt."""
+ extra_index = None
+ torch_reqs = []
+ if os.path.exists("requirements.frozen.txt"):
+ with open("requirements.frozen.txt") as f:
+ print("Installing frozen dependencies...")
+ reqs = []
+ for req in f:
+ if (
+ len(ONLY_INCLUDE) == 0
+ or req in ONLY_INCLUDE
+ or req.split("=")[0].lower() in ONLY_INCLUDE
+ ):
+ # can't pip install wandb==0.*.*.dev1 through pip. Lets just install wandb for now
+ if req.startswith("wandb==") and "dev1" in req:
+ req = "wandb"
+ match = re.match(
+ TORCH_DEP_REGEX,
+ req,
+ )
+ if match:
+ variant = match.group(2)
+ if variant:
+ extra_index = (
+ f"https://download.pytorch.org/whl/{variant[1:]}"
+ )
+ torch_reqs.append(req.strip().replace(" ", ""))
+ else:
+ reqs.append(req.strip().replace(" ", ""))
+ else:
+ print(f"Ignoring requirement: {req} from frozen requirements")
+ failed = install_deps(reqs, opts=OPTS) or set()
+ with open("_wandb_bootstrap_errors.json", "w") as f:
+ f.write(json.dumps({"pip": list(failed)}))
+ if len(failed) > 0:
+ sys.stderr.write(
+ FAILED_PACKAGES_PREFIX + ",".join(failed) + FAILED_PACKAGES_POSTFIX
+ )
+ sys.stderr.flush()
+ install_deps(torch_reqs, extra_index=extra_index)
+ else:
+ print("No frozen requirements found")
+
+
+def add_version_to_package_name(deps: List[str], package: str) -> Optional[str]:
+ """Add the associated version to a package name.
+
+ For example: `my-package` -> `my-package==1.0.0`
+ """
+ for dep in deps:
+ if dep.split("==")[0] == package:
+ return dep
+ return None
+
+
+def get_current_package(
+ line: str, deps: List[str], current_pkg: Optional[str]
+) -> Optional[str]:
+ """Tries to pull a package name from the line.
+
+ Used to keep track of what the currently-installing package is,
+ in case an error message isn't on the same line as the package
+ """
+ # "Collecting my-package==1.0.0"
+ if line.startswith("Collecting"):
+ return line.split(" ")[1]
+ # "Building wheel for my-package (pyproject.toml): finished with status 'error'"
+ elif line.strip().startswith("Building wheel") and line.strip().endswith(
+ "finished with status 'error'"
+ ):
+ return add_version_to_package_name(deps, line.strip().split(" ")[3])
+ # "Running setup.py install for my-package: finished with status 'error'"
+ elif line.strip().startswith("Running setup.py install") and line.strip().endswith(
+ "finished with status 'error'"
+ ):
+ return add_version_to_package_name(deps, line.strip().split(" ")[4][:-1])
+ return current_pkg
+
+
+# hacky way to get the name of the requirement that failed
+# attempt last word which is the name of the package often
+# fall back to checking all words in the line for the package name
+def find_package_in_error_string(deps: List[str], line: str) -> Optional[str]:
+ # if the last word in the error string is in the list of deps, return it
+ last_word = line.split(" ")[-1]
+ if last_word in deps:
+ return last_word
+ # if the last word is not in the list of deps, check all words
+ # TODO: this could report the wrong package if the error string
+ # contains a reference to another package in the deps
+ # before the package that failed to install
+ for word in line.split(" "):
+ if word.strip(",") in deps:
+ return word
+ # if we can't find the package, return None
+ return None
+
+
+if __name__ == "__main__":
+ main()
diff --git a/wandb/sdk/launch/create_job.py b/wandb/sdk/launch/create_job.py
new file mode 100644
index 0000000000000000000000000000000000000000..93b1b854f8aabaf3ae0973072205190b6eb52bc8
--- /dev/null
+++ b/wandb/sdk/launch/create_job.py
@@ -0,0 +1,541 @@
+import json
+import logging
+import os
+import sys
+import tempfile
+from typing import Any, Dict, List, Optional, Tuple
+
+import wandb
+from wandb.apis.internal import Api
+from wandb.sdk.artifacts.artifact import Artifact
+from wandb.sdk.internal.job_builder import JobBuilder
+from wandb.sdk.launch.builder.build import get_current_python_version
+from wandb.sdk.launch.git_reference import GitReference
+from wandb.sdk.launch.utils import _is_git_uri
+from wandb.sdk.lib import filesystem
+from wandb.util import make_artifact_name_safe
+
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+_logger = logging.getLogger("wandb")
+
+
+def create_job(
+ path: str,
+ job_type: str,
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+ name: Optional[str] = None,
+ description: Optional[str] = None,
+ aliases: Optional[List[str]] = None,
+ runtime: Optional[str] = None,
+ entrypoint: Optional[str] = None,
+ git_hash: Optional[str] = None,
+) -> Optional[Artifact]:
+ """Create a job from a path, not as the output of a run.
+
+ Arguments:
+ path (str): Path to the job directory.
+ job_type (str): Type of the job. One of "git", "code", or "image".
+ entity (Optional[str]): Entity to create the job under.
+ project (Optional[str]): Project to create the job under.
+ name (Optional[str]): Name of the job.
+ description (Optional[str]): Description of the job.
+ aliases (Optional[List[str]]): Aliases for the job.
+ runtime (Optional[str]): Python runtime of the job, like 3.9.
+ entrypoint (Optional[str]): Entrypoint of the job.
+ git_hash (Optional[str]): Git hash of a specific commit, when using git type jobs.
+
+
+ Returns:
+ Optional[Artifact]: The artifact created by the job, the action (for printing), and job aliases.
+ None if job creation failed.
+
+ Example:
+ ```python
+ artifact_job = wandb.create_job(
+ job_type="code",
+ path=".",
+ entity="wandb",
+ project="jobs",
+ name="my-train-job",
+ description="My training job",
+ aliases=["train"],
+ runtime="3.9",
+ entrypoint="train.py",
+ )
+ # then run the newly created job
+ artifact_job.call()
+ ```
+ """
+ api = Api()
+
+ artifact_job, _action, _aliases = _create_job(
+ api,
+ job_type,
+ path,
+ entity,
+ project,
+ name,
+ description,
+ aliases,
+ runtime,
+ entrypoint,
+ git_hash,
+ )
+
+ return artifact_job
+
+
+def _create_job(
+ api: Api,
+ job_type: str,
+ path: str,
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+ name: Optional[str] = None,
+ description: Optional[str] = None,
+ aliases: Optional[List[str]] = None,
+ runtime: Optional[str] = None,
+ entrypoint: Optional[str] = None,
+ git_hash: Optional[str] = None,
+) -> Tuple[Optional[Artifact], str, List[str]]:
+ wandb.termlog(f"Creating launch job of type: {job_type}...")
+
+ if name and name != make_artifact_name_safe(name):
+ wandb.termerror(
+ f"Artifact names may only contain alphanumeric characters, dashes, underscores, and dots. Did you mean: {make_artifact_name_safe(name)}"
+ )
+ return None, "", []
+
+ aliases = aliases or []
+ tempdir = tempfile.TemporaryDirectory()
+ try:
+ metadata, requirements = _make_metadata_for_partial_job(
+ job_type=job_type,
+ tempdir=tempdir,
+ git_hash=git_hash,
+ runtime=runtime,
+ path=path,
+ entrypoint=entrypoint,
+ )
+ if not metadata:
+ return None, "", []
+ except Exception as e:
+ wandb.termerror(f"Error creating job: {e}")
+ return None, "", []
+
+ _dump_metadata_and_requirements(
+ metadata=metadata,
+ tmp_path=tempdir.name,
+ requirements=requirements,
+ )
+
+ try:
+ # init hidden wandb run with job building disabled (handled manually)
+ run = wandb.init(
+ dir=tempdir.name,
+ settings={"silent": True, "disable_job_creation": True},
+ entity=entity,
+ project=project,
+ job_type="cli_create_job",
+ )
+ except Exception:
+ # Error printed by wandb.init
+ return None, "", []
+
+ job_builder = _configure_job_builder_for_partial(tempdir.name, job_source=job_type)
+ if job_type == "code":
+ job_name = _make_code_artifact(
+ api=api,
+ job_builder=job_builder,
+ path=path,
+ entrypoint=entrypoint,
+ run=run, # type: ignore
+ entity=entity,
+ project=project,
+ name=name,
+ )
+ if not job_name:
+ return None, "", []
+ name = job_name
+
+ # build job artifact, loads wandb-metadata and creates wandb-job.json here
+ artifact = job_builder.build()
+ if not artifact:
+ wandb.termerror("JobBuilder failed to build a job")
+ _logger.debug("Failed to build job, check job source and metadata")
+ return None, "", []
+
+ if not name:
+ name = artifact.name
+
+ aliases += job_builder._aliases
+ if "latest" not in aliases:
+ aliases += ["latest"]
+
+ res, _ = api.create_artifact(
+ artifact_type_name="job",
+ artifact_collection_name=name,
+ digest=artifact.digest,
+ client_id=artifact._client_id,
+ sequence_client_id=artifact._sequence_client_id,
+ entity_name=entity,
+ project_name=project,
+ run_name=run.id, # type: ignore # run will be deleted after creation
+ description=description,
+ metadata=metadata,
+ is_user_created=True,
+ aliases=[{"artifactCollectionName": name, "alias": a} for a in aliases],
+ )
+ action = "No changes detected for"
+ if not res.get("artifactSequence", {}).get("latestArtifact"):
+ # When there is no latestArtifact, we are creating new
+ action = "Created"
+ elif res.get("state") == "PENDING":
+ # updating an existing artifafct, state is pending awaiting call to
+ # log_artifact to upload and finalize artifact. If not pending, digest
+ # is the same as latestArtifact, so no changes detected
+ action = "Updated"
+
+ run.log_artifact(artifact, aliases=aliases) # type: ignore
+ artifact.wait()
+ run.finish() # type: ignore
+
+ # fetch, then delete hidden run
+ _run = wandb.Api().run(f"{entity}/{project}/{run.id}") # type: ignore
+ _run.delete()
+
+ return artifact, action, aliases
+
+
+def _make_metadata_for_partial_job(
+ job_type: str,
+ tempdir: tempfile.TemporaryDirectory,
+ git_hash: Optional[str],
+ runtime: Optional[str],
+ path: str,
+ entrypoint: Optional[str],
+) -> Tuple[Optional[Dict[str, Any]], Optional[List[str]]]:
+ """Create metadata for partial jobs, return metadata and requirements."""
+ metadata = {"_partial": "v0"}
+ if job_type == "git":
+ repo_metadata = _create_repo_metadata(
+ path=path,
+ tempdir=tempdir.name,
+ entrypoint=entrypoint,
+ git_hash=git_hash,
+ runtime=runtime,
+ )
+ if not repo_metadata:
+ tempdir.cleanup() # otherwise git can pollute
+ return None, None
+ metadata.update(repo_metadata)
+ return metadata, None
+
+ if job_type == "code":
+ path, entrypoint = _handle_artifact_entrypoint(path, entrypoint)
+ if not entrypoint:
+ wandb.termerror(
+ "Artifact jobs must have an entrypoint, either included in the path or specified with -E"
+ )
+ return None, None
+
+ artifact_metadata, requirements = _create_artifact_metadata(
+ path=path, entrypoint=entrypoint, runtime=runtime
+ )
+ if not artifact_metadata:
+ return None, None
+ metadata.update(artifact_metadata)
+ return metadata, requirements
+
+ if job_type == "image":
+ if runtime:
+ wandb.termwarn(
+ "Setting runtime is not supported for image jobs, ignoring runtime"
+ )
+ # TODO(gst): support entrypoint for image based jobs
+ if entrypoint:
+ wandb.termwarn(
+ "Setting an entrypoint is not currently supported for image jobs, ignoring entrypoint argument"
+ )
+ metadata.update({"python": runtime or "", "docker": path})
+ return metadata, None
+
+ wandb.termerror(f"Invalid job type: {job_type}")
+ return None, None
+
+
+def _create_repo_metadata(
+ path: str,
+ tempdir: str,
+ entrypoint: Optional[str] = None,
+ git_hash: Optional[str] = None,
+ runtime: Optional[str] = None,
+) -> Optional[Dict[str, Any]]:
+ # Make sure the entrypoint doesn't contain any backward path traversal
+ if entrypoint and ".." in entrypoint:
+ wandb.termerror("Entrypoint cannot contain backward path traversal")
+ return None
+ if not _is_git_uri(path):
+ wandb.termerror("Path must be a git URI")
+ return None
+
+ ref = GitReference(path, git_hash)
+ if not ref:
+ wandb.termerror("Could not parse git URI")
+ return None
+
+ ref.fetch(tempdir)
+
+ commit = ref.commit_hash
+ if not commit:
+ if not ref.commit_hash:
+ wandb.termerror("Could not find git commit hash")
+ return None
+ commit = ref.commit_hash
+
+ local_dir = os.path.join(tempdir, ref.path or "")
+ python_version = runtime
+ if not python_version:
+ if os.path.exists(os.path.join(local_dir, "runtime.txt")):
+ with open(os.path.join(local_dir, "runtime.txt")) as f:
+ python_version = f.read().strip()
+ elif os.path.exists(os.path.join(local_dir, ".python-version")):
+ with open(os.path.join(local_dir, ".python-version")) as f:
+ python_version = f.read().strip().splitlines()[0]
+ else:
+ major, minor = get_current_python_version()
+ python_version = f"{major}.{minor}"
+
+ python_version = _clean_python_version(python_version)
+
+ # check if entrypoint is valid
+ assert entrypoint is not None
+ if not os.path.exists(os.path.join(local_dir, entrypoint)):
+ wandb.termerror(f"Entrypoint {entrypoint} not found in git repo")
+ return None
+
+ # check if requirements.txt exists
+ # start at the location of the python file and recurse up to the git root
+ entrypoint_dir = os.path.dirname(entrypoint)
+ if entrypoint_dir:
+ req_dir = os.path.join(local_dir, entrypoint_dir)
+ else:
+ req_dir = local_dir
+
+ # If there is a Dockerfile.wandb in the starting rec dir, don't require a requirements.txt
+ if os.path.exists(os.path.join(req_dir, "Dockerfile.wandb")):
+ wandb.termlog(
+ f"Using Dockerfile.wandb in {req_dir.replace(tempdir, '') or 'repository root'}"
+ )
+ else:
+ while (
+ not os.path.exists(os.path.join(req_dir, "requirements.txt"))
+ and req_dir != tempdir
+ ):
+ req_dir = os.path.dirname(req_dir)
+
+ if not os.path.exists(os.path.join(req_dir, "requirements.txt")):
+ path_with_subdir = os.path.dirname(
+ os.path.join(path or "", entrypoint or "")
+ )
+ wandb.termerror(
+ f"Could not find requirements.txt file in git repo at {path_with_subdir}"
+ )
+ return None
+
+ wandb.termlog(
+ f"Using requirements.txt in {req_dir.replace(tempdir, '') or 'repository root'}"
+ )
+
+ metadata = {
+ "git": {
+ "commit": commit,
+ "remote": ref.url,
+ },
+ "codePathLocal": entrypoint, # not in git context, optionally also set local
+ "codePath": entrypoint,
+ "entrypoint": [f"python{python_version}", entrypoint],
+ "python": python_version, # used to build container
+ "notebook": False, # partial jobs from notebooks not supported
+ }
+
+ return metadata
+
+
+def _create_artifact_metadata(
+ path: str, entrypoint: str, runtime: Optional[str] = None
+) -> Tuple[Dict[str, Any], List[str]]:
+ if not os.path.exists(path):
+ wandb.termerror("Path must be a valid file or directory")
+ return {}, []
+
+ if not os.path.exists(os.path.join(path, "requirements.txt")):
+ wandb.termerror(f"Could not find requirements.txt file in: {path}")
+ return {}, []
+
+ # read local requirements.txt and dump to temp dir for builder
+ requirements = []
+ with open(os.path.join(path, "requirements.txt")) as f:
+ requirements = f.read().splitlines()
+
+ if runtime:
+ python_version = _clean_python_version(runtime)
+ else:
+ python_version = ".".join(get_current_python_version())
+
+ metadata = {"python": python_version, "codePath": entrypoint}
+ return metadata, requirements
+
+
+def _handle_artifact_entrypoint(
+ path: str, entrypoint: Optional[str] = None
+) -> Tuple[str, Optional[str]]:
+ if os.path.isfile(path):
+ if entrypoint and path.endswith(entrypoint):
+ path = path.replace(entrypoint, "")
+ wandb.termwarn(
+ f"Both entrypoint provided and path contains file. Using provided entrypoint: {entrypoint}, path is now: {path}"
+ )
+ elif entrypoint:
+ wandb.termwarn(
+ f"Ignoring passed in entrypoint as it does not match file path found in 'path'. Path entrypoint: {path.split('/')[-1]}"
+ )
+ entrypoint = path.split("/")[-1]
+ path = "/".join(path.split("/")[:-1])
+ elif not entrypoint:
+ wandb.termerror("Entrypoint not valid")
+ return "", None
+ path = path or "." # when path is just an entrypoint, use cdw
+
+ if not os.path.exists(os.path.join(path, entrypoint)):
+ wandb.termerror(
+ f"Could not find execution point: {os.path.join(path, entrypoint)}"
+ )
+ return "", None
+
+ return path, entrypoint
+
+
+def _configure_job_builder_for_partial(tmpdir: str, job_source: str) -> JobBuilder:
+ """Configure job builder with temp dir and job source."""
+ # adjust git source to repo
+ if job_source == "git":
+ job_source = "repo"
+
+ # adjust code source to artifact
+ if job_source == "code":
+ job_source = "artifact"
+
+ settings = wandb.Settings()
+ settings.update({"files_dir": tmpdir, "job_source": job_source})
+ job_builder = JobBuilder(
+ settings=settings, # type: ignore
+ )
+ # never allow notebook runs
+ job_builder._is_notebook_run = False
+ # set run inputs and outputs to empty dicts
+ job_builder.set_config({})
+ job_builder.set_summary({})
+ return job_builder
+
+
+def _make_code_artifact(
+ api: Api,
+ job_builder: JobBuilder,
+ run: "wandb.sdk.wandb_run.Run",
+ path: str,
+ entrypoint: Optional[str],
+ entity: Optional[str],
+ project: Optional[str],
+ name: Optional[str],
+) -> Optional[str]:
+ """Helper for creating and logging code artifacts.
+
+ Returns the name of the eventual job.
+ """
+ artifact_name = _make_code_artifact_name(os.path.join(path, entrypoint or ""), name)
+ code_artifact = wandb.Artifact(
+ name=artifact_name,
+ type="code",
+ description="Code artifact for job",
+ )
+
+ # Update path and entrypoint vars to match metadata
+ # TODO(gst): consolidate into one place
+ path, entrypoint = _handle_artifact_entrypoint(path, entrypoint)
+
+ try:
+ code_artifact.add_dir(path)
+ except Exception as e:
+ if os.path.islink(path):
+ wandb.termerror(
+ "Symlinks are not supported for code artifact jobs, please copy the code into a directory and try again"
+ )
+ wandb.termerror(f"Error adding to code artifact: {e}")
+ return None
+
+ res, _ = api.create_artifact(
+ artifact_type_name="code",
+ artifact_collection_name=artifact_name,
+ digest=code_artifact.digest,
+ client_id=code_artifact._client_id,
+ sequence_client_id=code_artifact._sequence_client_id,
+ entity_name=entity,
+ project_name=project,
+ run_name=run.id, # run will be deleted after creation
+ description="Code artifact for job",
+ metadata={"codePath": path, "entrypoint": entrypoint},
+ is_user_created=True,
+ aliases=[
+ {"artifactCollectionName": artifact_name, "alias": a} for a in ["latest"]
+ ],
+ )
+ run.log_artifact(code_artifact)
+ code_artifact.wait()
+ job_builder._handle_server_artifact(res, code_artifact) # type: ignore
+
+ # code artifacts have "code" prefix, remove it and alias
+ if not name:
+ name = code_artifact.name.replace("code", "job").split(":")[0]
+
+ return name
+
+
+def _make_code_artifact_name(path: str, name: Optional[str]) -> str:
+ """Make a code artifact name from a path and user provided name."""
+ if name:
+ return f"code-{name}"
+
+ clean_path = path.replace("./", "")
+ if clean_path[0] == "/":
+ clean_path = clean_path[1:]
+ if clean_path[-1] == "/":
+ clean_path = clean_path[:-1]
+
+ path_name = f"code-{make_artifact_name_safe(clean_path)}"
+ return path_name
+
+
+def _dump_metadata_and_requirements(
+ tmp_path: str, metadata: Dict[str, Any], requirements: Optional[List[str]]
+) -> None:
+ """Dump manufactured metadata and requirements.txt.
+
+ File used by the job_builder to create a job from provided metadata.
+ """
+ filesystem.mkdir_exists_ok(tmp_path)
+ with open(os.path.join(tmp_path, "wandb-metadata.json"), "w") as f:
+ json.dump(metadata, f)
+
+ requirements = requirements or []
+ with open(os.path.join(tmp_path, "requirements.txt"), "w") as f:
+ f.write("\n".join(requirements))
+
+
+def _clean_python_version(python_version: str) -> str:
+ # remove micro if present
+ if python_version.count(".") > 1:
+ python_version = ".".join(python_version.split(".")[:2])
+ _logger.debug(f"micro python version stripped. Now: {python_version}")
+ return python_version
diff --git a/wandb/sdk/launch/deploys/Dockerfile b/wandb/sdk/launch/deploys/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..350ce059695a3f24b02eaed4ee487737b090008f
--- /dev/null
+++ b/wandb/sdk/launch/deploys/Dockerfile
@@ -0,0 +1,18 @@
+FROM python:3.9-slim-bullseye
+LABEL maintainer='Weights & Biases <support@wandb.com>'
+
+# install git
+RUN apt-get update && apt-get upgrade -y \
+ && apt-get install -y git \
+ && apt-get -qy autoremove \
+ && apt-get clean && rm -r /var/lib/apt/lists/*
+
+# required pip packages
+RUN pip install --no-cache-dir wandb[launch]
+# user set up
+RUN useradd -m -s /bin/bash --create-home --no-log-init -u 1000 -g 0 launch_agent
+USER launch_agent
+WORKDIR /home/launch_agent
+RUN chown -R launch_agent /home/launch_agent
+
+ENTRYPOINT ["wandb", "launch-agent"]
diff --git a/wandb/sdk/launch/deploys/aws-ec2/launch-config.yaml b/wandb/sdk/launch/deploys/aws-ec2/launch-config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c3245808f6f3ca500d6c368efb46daff564ac7dd
--- /dev/null
+++ b/wandb/sdk/launch/deploys/aws-ec2/launch-config.yaml
@@ -0,0 +1,12 @@
+max_jobs: -1 # TODO: set the maximum number of concurrent jobs
+base_url: https://api.wandb.ai # TODO: set api url
+queues: # TODO: set to the names of your sagemaker queues
+ - sagemaker
+environment:
+ type: aws
+ region: us-east-2 # TODO: fill in aws region
+registry:
+ type: ecr
+ repository: my-container-repository # TODO: fill in ecr repository name
+builder:
+ type: docker
diff --git a/wandb/sdk/launch/deploys/aws-eks/launch-agent.yaml b/wandb/sdk/launch/deploys/aws-eks/launch-agent.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0043762cd1e389839242aa892133c642d84974f7
--- /dev/null
+++ b/wandb/sdk/launch/deploys/aws-eks/launch-agent.yaml
@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: launch-agent
+ namespace: wandb
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: launch-agent
+ template:
+ metadata:
+ labels:
+ app: launch-agent
+ spec:
+ serviceAccountName: wandb-launch-serviceaccount
+ containers:
+ - name: launch-agent
+ image: wandb/launch-agent-dev:df552430
+ resources:
+ limits:
+ memory: "2Gi"
+ cpu: "1000m"
+ securityContext:
+ allowPrivilegeEscalation: false
+ runAsNonRoot: true
+ runAsUser: 1000
+ capabilities:
+ drop: ["ALL"]
+ seccompProfile:
+ type: RuntimeDefault
+ env:
+ - name: WANDB_API_KEY
+ valueFrom:
+ secretKeyRef:
+ name: wandb-api-key
+ key: password
+ - name: WANDB_BASE_URL
+ valueFrom:
+ configMapKeyRef:
+ name: wandb-launch-configmap
+ key: wandb-base-url
+ volumeMounts:
+ - name: wandb-launch-config
+ mountPath: /home/launch_agent/.config/wandb
+ readOnly: true
+ volumes:
+ - name: wandb-launch-config
+ configMap:
+ name: wandb-launch-configmap
diff --git a/wandb/sdk/launch/deploys/aws-eks/launch-config.yaml b/wandb/sdk/launch/deploys/aws-eks/launch-config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d9cba173a51df952cd2b79c7a01aaf8bf894ede7
--- /dev/null
+++ b/wandb/sdk/launch/deploys/aws-eks/launch-config.yaml
@@ -0,0 +1,103 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: wandb
+ labels:
+ pod-security.kubernetes.io/enforce: baseline
+ pod-security.kubernetes.io/enforce-version: latest
+ pod-security.kubernetes.io/warn: baseline
+ pod-security.kubernetes.io/warn-version: latest
+---
+# role for handling builds and jobs within the wandb namespace
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ namespace: wandb
+ name: wandb-launch-agent
+rules:
+ - apiGroups: [""]
+ resources: ["pods", "configmaps", "secrets", "pods/log"]
+ verbs: ["create", "get", "watch", "list", "update", "delete", "patch"]
+ - apiGroups: ["batch"]
+ resources: ["jobs", "jobs/status"]
+ verbs: ["create", "get", "watch", "list", "update", "delete", "patch"]
+---
+# cluster role to creating ML jobs in desired namespaces
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: job-creator
+rules:
+ - apiGroups: [""]
+ resources: ["pods", "pods/log", "secrets"]
+ verbs: ["create", "get", "watch", "list", "update", "delete", "patch"]
+ - apiGroups: ["batch"]
+ resources: ["jobs", "jobs/status"]
+ verbs: ["create", "get", "watch", "list", "update", "delete", "patch"]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: wandb-launch-serviceaccount
+ namespace: wandb
+---
+# role binding for namespaced role
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: wandb-launch-role-binding
+ namespace: wandb
+subjects:
+ - kind: ServiceAccount
+ name: wandb-launch-serviceaccount
+ namespace: wandb
+roleRef:
+ kind: Role
+ name: wandb-launch-agent
+ apiGroup: rbac.authorization.k8s.io
+---
+# role binding to create ML jobs in another namespace (could use cluster role binding if we want to launch cluster wide)
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: wandb-launch-cluster-role-binding
+ namespace: default #TODO: SET YOUR TRAINING NAMESPACE
+subjects:
+ - kind: ServiceAccount
+ name: wandb-launch-serviceaccount
+ namespace: wandb
+roleRef:
+ kind: ClusterRole
+ name: job-creator
+ apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+data:
+ wandb-base-url: https://api.wandb.ai # TODO: set your base_url here
+ launch-config.yaml: |
+ max_jobs: -1 # TODO: set max concurrent jobs here
+ queues:
+ - default # TODO: set queue name here
+ environment:
+ type: aws
+ region: us-east-1 # TODO: set aws region here
+ registry:
+ type: ecr
+ repository: # TODO: set ecr repository name here
+ builder:
+ type: kaniko
+ build-context-store: s3://my-bucket/... # TODO: set your build context store here
+
+kind: ConfigMap
+metadata:
+ name: wandb-launch-configmap
+ namespace: wandb
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: wandb-api-key
+ namespace: wandb
+type: kubernetes.io/basic-auth
+stringData:
+ password: "" ### API KEY HERE
diff --git a/wandb/sdk/launch/deploys/gcp-gke/launch-agent.yaml b/wandb/sdk/launch/deploys/gcp-gke/launch-agent.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..35b500f06bcfad7fc1092435ce6693ecf140bd78
--- /dev/null
+++ b/wandb/sdk/launch/deploys/gcp-gke/launch-agent.yaml
@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: launch-agent
+ namespace: wandb
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: launch-agent
+ template:
+ metadata:
+ labels:
+ app: launch-agent
+ spec:
+ serviceAccountName: wandb-launch-serviceaccount
+ containers:
+ - name: launch-agent
+ image: wandb/launch-agent-dev:df552430
+ resources:
+ limits:
+ memory: "2Gi"
+ cpu: "1000m"
+ securityContext:
+ allowPrivilegeEscalation: false
+ runAsNonRoot: true
+ runAsUser: 1000
+ capabilities:
+ drop: ["ALL"]
+ seccompProfile:
+ type: RuntimeDefault
+ env:
+ - name: WANDB_API_KEY
+ valueFrom:
+ secretKeyRef:
+ name: wandb-api-key
+ key: password
+ - name: WANDB_BASE_URL
+ valueFrom:
+ configMapKeyRef:
+ name: wandb-launch-configmap
+ key: wandb-base-url
+ - name: GOOGLE_APPLICATION_CREDENTIALS
+ value: /home/launch_agent/.config/gcp/gcp-creds.json # TODO: change the last part of this path to match the key for the service account json in your secret
+ volumeMounts:
+ - name: wandb-launch-config
+ mountPath: /home/launch_agent/.config/wandb
+ readOnly: true
+ - name: gcp-creds
+ mountPath: /home/launch_agent/.config/gcp
+ readOnly: true
+ volumes:
+ - name: wandb-launch-config
+ configMap:
+ name: wandb-launch-configmap
+ - name: gcp-creds
+ secret:
+ secretName: my-secret-name # TODO: set name of your secret here - the secret should contain the contents of a gcp service account json file
diff --git a/wandb/sdk/launch/deploys/gcp-gke/launch-config.yaml b/wandb/sdk/launch/deploys/gcp-gke/launch-config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7dff18d99bd19a010fe5552235ce7e6153b15d82
--- /dev/null
+++ b/wandb/sdk/launch/deploys/gcp-gke/launch-config.yaml
@@ -0,0 +1,106 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: wandb
+ labels:
+ pod-security.kubernetes.io/enforce: baseline
+ pod-security.kubernetes.io/enforce-version: latest
+ pod-security.kubernetes.io/warn: baseline
+ pod-security.kubernetes.io/warn-version: latest
+---
+# role for handling builds and jobs within the wandb namespace
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ namespace: wandb
+ name: wandb-launch-agent
+rules:
+ - apiGroups: [""]
+ resources: ["pods", "configmaps", "secrets", "pods/log"]
+ verbs: ["create", "get", "watch", "list", "update", "delete", "patch"]
+ - apiGroups: ["batch"]
+ resources: ["jobs", "jobs/status"]
+ verbs: ["create", "get", "watch", "list", "update", "delete", "patch"]
+---
+# cluster role to creating ML jobs in desired namespaces
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: job-creator
+rules:
+ - apiGroups: [""]
+ resources: ["pods", "pods/log", "secrets"]
+ verbs: ["create", "get", "watch", "list", "update", "delete", "patch"]
+ - apiGroups: ["batch"]
+ resources: ["jobs", "jobs/status"]
+ verbs: ["create", "get", "watch", "list", "update", "delete", "patch"]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: wandb-launch-serviceaccount
+ namespace: wandb
+---
+# role binding for namespaced role
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: wandb-launch-role-binding
+ namespace: wandb
+subjects:
+ - kind: ServiceAccount
+ name: wandb-launch-serviceaccount
+ namespace: wandb
+roleRef:
+ kind: Role
+ name: wandb-launch-agent
+ apiGroup: rbac.authorization.k8s.io
+---
+# role binding to create ML jobs in another namespace (could use cluster role binding if we want to launch cluster wide)
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: wandb-launch-cluster-role-binding
+ namespace: default #TODO: SET YOUR TRAINING NAMESPACE
+subjects:
+ - kind: ServiceAccount
+ name: wandb-launch-serviceaccount
+ namespace: wandb
+roleRef:
+ kind: ClusterRole
+ name: job-creator
+ apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+data:
+ wandb-base-url: https://api.wandb.ai # TODO: set your base_url here
+ launch-config.yaml: |
+ max_jobs: -1 # TODO: set max concurrent jobs here
+ queues:
+ - default # TODO: set queue name here
+ environment:
+ type: gcp
+ region: us-central1 # TODO: set gcp region here
+ registry:
+ type: gcr
+ repository: # TODO: set name of artifact repository name here
+ image_name: launch-images # TODO: set name of image here
+ builder:
+ type: kaniko
+ build-context-store: gs://my-bucket/... # TODO: set your build context store here
+ secret-name: gcp-creds # TODO: set your secret name here
+ secret-key: service_account.json # TODO: set your secret key here, i.e. the key in the secret that contains your service account json
+
+kind: ConfigMap
+metadata:
+ name: wandb-launch-configmap
+ namespace: wandb
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: wandb-api-key
+ namespace: wandb
+type: kubernetes.io/basic-auth
+stringData:
+ password: "" ### API KEY HERE
diff --git a/wandb/sdk/launch/deploys/gcp-vertex/launch-config.yaml b/wandb/sdk/launch/deploys/gcp-vertex/launch-config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e96a74bcb66494e57c70fe37c26ca530592441fe
--- /dev/null
+++ b/wandb/sdk/launch/deploys/gcp-vertex/launch-config.yaml
@@ -0,0 +1,14 @@
+# This is a launch config template for running an agent on a GCE instance or your local machine.
+max_jobs: -1 # TODO: set the maximum number of concurrent jobs
+base_url: https://api.wandb.ai # TODO: set api url
+queues:
+ - vertex_queue # TODO: set to the names of your vertex queue
+environment:
+ type: gcp
+ region: us-central1 # TODO: fill in gcp region
+registry:
+ type: gcr
+ repository: # TODO: fill in artifact repository name
+ image_name: launch-images # TODO: fill in image name
+builder:
+ type: docker
diff --git a/wandb/sdk/launch/deploys/kubernetes/launch-agent.yaml b/wandb/sdk/launch/deploys/kubernetes/launch-agent.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..316c5f305b564dd710119c6f5fe0008b1ca9c820
--- /dev/null
+++ b/wandb/sdk/launch/deploys/kubernetes/launch-agent.yaml
@@ -0,0 +1,56 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: launch-agent
+ namespace: wandb
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: launch-agent
+ template:
+ metadata:
+ labels:
+ app: launch-agent
+ spec:
+ serviceAccountName: wandb-launch-serviceaccount
+ containers:
+ - name: launch-agent
+ image: wandb/launch-agent-dev:df552430
+ resources:
+ limits:
+ memory: "2Gi"
+ cpu: "1000m"
+ securityContext:
+ allowPrivilegeEscalation: false
+ runAsNonRoot: true
+ runAsUser: 1000
+ capabilities:
+ drop: ["ALL"]
+ seccompProfile:
+ type: RuntimeDefault
+ env:
+ - name: WANDB_API_KEY
+ valueFrom:
+ secretKeyRef:
+ name: wandb-api-key
+ key: password
+ - name: WANDB_BASE_URL
+ valueFrom:
+ configMapKeyRef:
+ name: wandb-launch-configmap
+ key: wandb-base-url
+ volumeMounts:
+ - name: wandb-launch-config
+ mountPath: /home/launch_agent/.config/wandb
+ - name: aws-secret # if you intend on using kaniko with ecr create a secret containing your aws credentials, with this name
+ mountPath: /home/launch_agent/.aws/
+ readOnly: true
+ volumes:
+ - name: wandb-launch-config
+ configMap:
+ name: wandb-launch-configmap
+ - name: aws-secret # if you intend on using kaniko with ecr create a secret containing your aws credentials, with this name
+ secret:
+ secretName: aws-secret # if you intend on using kaniko with ecr create a secret containing your aws credentials, with this name
+ optional: true
diff --git a/wandb/sdk/launch/deploys/kubernetes/launch-config.yaml b/wandb/sdk/launch/deploys/kubernetes/launch-config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4fe8d5471f8dd0e95ffa46b90c7b569d9913af1d
--- /dev/null
+++ b/wandb/sdk/launch/deploys/kubernetes/launch-config.yaml
@@ -0,0 +1,102 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: wandb
+ labels:
+ pod-security.kubernetes.io/enforce: baseline
+ pod-security.kubernetes.io/enforce-version: latest
+ pod-security.kubernetes.io/warn: baseline
+ pod-security.kubernetes.io/warn-version: latest
+---
+# role for handling builds and jobs within the wandb namespace
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ namespace: wandb
+ name: wandb-launch-agent
+rules:
+ - apiGroups: [""]
+ resources: ["pods", "configmaps", "secrets", "pods/log"]
+ verbs: ["create", "get", "watch", "list", "update", "delete", "patch"]
+ - apiGroups: ["batch"]
+ resources: ["jobs", "jobs/status"]
+ verbs: ["create", "get", "watch", "list", "update", "delete", "patch"]
+---
+# cluster role to creating ML jobs in desired namespaces
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: job-creator
+rules:
+ - apiGroups: [""]
+ resources: ["pods", "pods/log", "secrets"]
+ verbs: ["create", "get", "watch", "list", "update", "delete", "patch"]
+ - apiGroups: ["batch"]
+ resources: ["jobs", "jobs/status"]
+ verbs: ["create", "get", "watch", "list", "update", "delete", "patch"]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: wandb-launch-serviceaccount
+ namespace: wandb
+---
+# role binding for namespaced role
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: wandb-launch-role-binding
+ namespace: wandb
+subjects:
+ - kind: ServiceAccount
+ name: wandb-launch-serviceaccount
+ namespace: wandb
+roleRef:
+ kind: Role
+ name: wandb-launch-agent
+ apiGroup: rbac.authorization.k8s.io
+---
+# role binding to create ML jobs in another namespace (could use cluster role binding if we want to launch cluster wide)
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: wandb-launch-cluster-role-binding
+ namespace: default #TODO: SET YOUR TRAINING NAMESPACE
+subjects:
+ - kind: ServiceAccount
+ name: wandb-launch-serviceaccount
+ namespace: wandb
+roleRef:
+ kind: ClusterRole
+ name: job-creator
+ apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+data:
+ wandb-base-url: https://api.wandb.ai # TODO: set your base_url here
+ launch-config.yaml: |
+ max_jobs: -1 # TODO: set max concurrent jobs here
+ queues:
+ - default # TODO: set queue name here
+ environment:
+ type: aws
+ region: us-east-1 # TODO: set aws region here
+ registry:
+ type: ecr
+ repository: # TODO: set ecr repository name here
+ builder:
+ type: kaniko
+ build-context-store: s3://my-bucket/... # TODO: set your build context store here
+kind: ConfigMap
+metadata:
+ name: wandb-launch-configmap
+ namespace: wandb
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: wandb-api-key
+ namespace: wandb
+type: kubernetes.io/basic-auth
+stringData:
+ password: "" ### API KEY HERE
diff --git a/wandb/sdk/launch/environment/abstract.py b/wandb/sdk/launch/environment/abstract.py
new file mode 100644
index 0000000000000000000000000000000000000000..0a8c4936acc5351cc00570f13c7e8109ad196767
--- /dev/null
+++ b/wandb/sdk/launch/environment/abstract.py
@@ -0,0 +1,28 @@
+"""Abstract base class for environments."""
+from abc import ABC, abstractmethod
+
+
+class AbstractEnvironment(ABC):
+ """Abstract base class for environments."""
+
+ region: str
+
+ @abstractmethod
+ async def verify(self) -> None:
+ """Verify that the environment is configured correctly."""
+ raise NotImplementedError
+
+ @abstractmethod
+ async def upload_file(self, source: str, destination: str) -> None:
+ """Upload a file from the local filesystem to storage in the environment."""
+ raise NotImplementedError
+
+ @abstractmethod
+ async def upload_dir(self, source: str, destination: str) -> None:
+ """Upload the contents of a directory from the local filesystem to the environment."""
+ raise NotImplementedError
+
+ @abstractmethod
+ async def verify_storage_uri(self, uri: str) -> None:
+ """Verify that the storage URI is configured correctly."""
+ raise NotImplementedError
diff --git a/wandb/sdk/launch/environment/aws_environment.py b/wandb/sdk/launch/environment/aws_environment.py
new file mode 100644
index 0000000000000000000000000000000000000000..20eab2ca052f43aef4c56f68cba8a4d568b80793
--- /dev/null
+++ b/wandb/sdk/launch/environment/aws_environment.py
@@ -0,0 +1,297 @@
+"""Implements the AWS environment."""
+
+import logging
+import os
+from typing import Dict, Optional
+
+from wandb.sdk.launch.errors import LaunchError
+from wandb.util import get_module
+
+from ..utils import S3_URI_RE, event_loop_thread_exec
+from .abstract import AbstractEnvironment
+
+boto3 = get_module(
+ "boto3",
+ required="AWS environment requires boto3 to be installed. Please install "
+ "it with `pip install wandb[launch]`.",
+)
+botocore = get_module(
+ "botocore",
+ required="AWS environment requires botocore to be installed. Please install "
+ "it with `pip install wandb[launch]`.",
+)
+
+_logger = logging.getLogger(__name__)
+
+
+class AwsEnvironment(AbstractEnvironment):
+ """AWS environment."""
+
+ def __init__(
+ self,
+ region: str,
+ access_key: str,
+ secret_key: str,
+ session_token: str,
+ ) -> None:
+ """Initialize the AWS environment.
+
+ Arguments:
+ region (str): The AWS region.
+
+ Raises:
+ LaunchError: If the AWS environment is not configured correctly.
+ """
+ super().__init__()
+ _logger.info(f"Initializing AWS environment in region {region}.")
+ self._region = region
+ self._access_key = access_key
+ self._secret_key = secret_key
+ self._session_token = session_token
+ self._account = None
+
+ @classmethod
+ def from_default(cls, region: Optional[str] = None) -> "AwsEnvironment":
+ """Create an AWS environment from the default AWS environment.
+
+ Arguments:
+ region (str, optional): The AWS region.
+ verify (bool, optional): Whether to verify the AWS environment. Defaults to True.
+
+ Returns:
+ AwsEnvironment: The AWS environment.
+ """
+ _logger.info("Creating AWS environment from default credentials.")
+ try:
+ session = boto3.Session()
+ if hasattr(session, "region"):
+ region = region or session.region
+ region = region or os.environ.get("AWS_REGION")
+ credentials = session.get_credentials()
+ if not credentials:
+ raise LaunchError(
+ "Could not create AWS environment from default environment. Please verify that your AWS credentials are configured correctly."
+ )
+ access_key = credentials.access_key
+ secret_key = credentials.secret_key
+ session_token = credentials.token
+ except botocore.client.ClientError as e:
+ raise LaunchError(
+ f"Could not create AWS environment from default environment. Please verify that your AWS credentials are configured correctly. {e}"
+ )
+ if not region:
+ raise LaunchError(
+ "Could not create AWS environment from default environment. Region not specified."
+ )
+ return cls(
+ region=region,
+ access_key=access_key,
+ secret_key=secret_key,
+ session_token=session_token,
+ )
+
+ @classmethod
+ def from_config(
+ cls,
+ config: Dict[str, str],
+ ) -> "AwsEnvironment":
+ """Create an AWS environment from the default AWS environment.
+
+ Arguments:
+ config (dict): Configuration dictionary.
+ verify (bool, optional): Whether to verify the AWS environment. Defaults to True.
+
+ Returns:
+ AwsEnvironment: The AWS environment.
+ """
+ region = str(config.get("region", ""))
+ if not region:
+ raise LaunchError(
+ "Could not create AWS environment from config. Region not specified."
+ )
+ return cls.from_default(
+ region=region,
+ )
+
+ @property
+ def region(self) -> str:
+ """The AWS region."""
+ return self._region
+
+ @region.setter
+ def region(self, region: str) -> None:
+ self._region = region
+
+ async def verify(self) -> None:
+ """Verify that the AWS environment is configured correctly.
+
+ Raises:
+ LaunchError: If the AWS environment is not configured correctly.
+ """
+ _logger.debug("Verifying AWS environment.")
+ try:
+ session = await self.get_session()
+ client = await event_loop_thread_exec(session.client)("sts")
+ get_caller_identity = event_loop_thread_exec(client.get_caller_identity)
+ self._account = (await get_caller_identity()).get("Account")
+ # TODO: log identity details from the response
+ except botocore.exceptions.ClientError as e:
+ raise LaunchError(
+ f"Could not verify AWS environment. Please verify that your AWS credentials are configured correctly. {e}"
+ ) from e
+
+ async def get_session(self) -> "boto3.Session": # type: ignore
+ """Get an AWS session.
+
+ Returns:
+ boto3.Session: The AWS session.
+
+ Raises:
+ LaunchError: If the AWS session could not be created.
+ """
+ _logger.debug(f"Creating AWS session in region {self._region}")
+ try:
+ session = event_loop_thread_exec(boto3.Session)
+ return await session(
+ region_name=self._region,
+ aws_access_key_id=self._access_key,
+ aws_secret_access_key=self._secret_key,
+ aws_session_token=self._session_token,
+ )
+ except botocore.exceptions.ClientError as e:
+ raise LaunchError(f"Could not create AWS session. {e}")
+
+ async def upload_file(self, source: str, destination: str) -> None:
+ """Upload a file to s3 from local storage.
+
+ The destination is a valid s3 URI, e.g. s3://bucket/key and will
+ be used as a prefix for the uploaded file. Only the filename of the source
+ is kept in the upload key. So if the source is "foo/bar" and the
+ destination is "s3://bucket/key", the file "foo/bar" will be uploaded
+ to "s3://bucket/key/bar".
+
+ Arguments:
+ source (str): The path to the file or directory.
+ destination (str): The uri of the storage destination. This should
+ be a valid s3 URI, e.g. s3://bucket/key.
+
+ Raises:
+ LaunchError: If the copy fails, the source path does not exist, or the
+ destination is not a valid s3 URI, or the upload fails.
+ """
+ _logger.debug(f"Uploading {source} to {destination}")
+ _err_prefix = f"Error attempting to copy {source} to {destination}."
+ if not os.path.isfile(source):
+ raise LaunchError(f"{_err_prefix}: Source {source} does not exist.")
+ match = S3_URI_RE.match(destination)
+ if not match:
+ raise LaunchError(
+ f"{_err_prefix}: Destination {destination} is not a valid s3 URI."
+ )
+ bucket = match.group(1)
+ key = match.group(2).lstrip("/")
+ if not key:
+ key = ""
+ session = await self.get_session()
+ try:
+ client = await event_loop_thread_exec(session.client)("s3")
+ client.upload_file(source, bucket, key)
+ except botocore.exceptions.ClientError as e:
+ raise LaunchError(
+ f"{_err_prefix}: botocore error attempting to copy {source} to {destination}. {e}"
+ )
+
+ async def upload_dir(self, source: str, destination: str) -> None:
+ """Upload a directory to s3 from local storage.
+
+ The upload will place the contents of the source directory in the destination
+ with the same directory structure. So if the source is "foo/bar" and the
+ destination is "s3://bucket/key", the contents of "foo/bar" will be uploaded
+ to "s3://bucket/key/bar".
+
+ Arguments:
+ source (str): The path to the file or directory.
+ destination (str): The URI of the storage.
+ recursive (bool, optional): If True, copy the directory recursively. Defaults to False.
+
+ Raises:
+ LaunchError: If the copy fails, the source path does not exist, or the
+ destination is not a valid s3 URI.
+ """
+ _logger.debug(f"Uploading {source} to {destination}")
+ _err_prefix = f"Error attempting to copy {source} to {destination}."
+ if not os.path.isdir(source):
+ raise LaunchError(f"{_err_prefix}: Source {source} does not exist.")
+ match = S3_URI_RE.match(destination)
+ if not match:
+ raise LaunchError(
+ f"{_err_prefix}: Destination {destination} is not a valid s3 URI."
+ )
+ bucket = match.group(1)
+ key = match.group(2).lstrip("/")
+ if not key:
+ key = ""
+ session = await self.get_session()
+ try:
+ client = await event_loop_thread_exec(session.client)("s3")
+ for path, _, files in os.walk(source):
+ for file in files:
+ abs_path = os.path.join(path, file)
+ key_path = (
+ abs_path.replace(source, "").replace("\\", "/").lstrip("/")
+ )
+ client.upload_file(
+ abs_path,
+ bucket,
+ key_path,
+ )
+ except botocore.exceptions.ClientError as e:
+ raise LaunchError(
+ f"{_err_prefix}: botocore error attempting to copy {source} to {destination}. {e}"
+ ) from e
+ except Exception as e:
+ raise LaunchError(
+ f"{_err_prefix}: Unexpected error attempting to copy {source} to {destination}. {e}"
+ ) from e
+
+ async def verify_storage_uri(self, uri: str) -> None:
+ """Verify that s3 storage is configured correctly.
+
+ This will check that the bucket exists and that the credentials are
+ configured correctly.
+
+ Arguments:
+ uri (str): The URI of the storage.
+
+ Raises:
+ LaunchError: If the storage is not configured correctly or the URI is
+ not a valid s3 URI.
+
+ Returns:
+ None
+ """
+ _logger.debug(f"Verifying storage {uri}")
+ match = S3_URI_RE.match(uri)
+ if not match:
+ raise LaunchError(
+ f"Failed to validate storage uri: {uri} is not a valid s3 URI."
+ )
+ bucket = match.group(1)
+ try:
+ session = await self.get_session()
+ client = await event_loop_thread_exec(session.client)("s3")
+ client.head_bucket(Bucket=bucket)
+ except botocore.exceptions.ClientError as e:
+ if e.response["Error"]["Code"] == "404":
+ raise LaunchError(
+ f"Could not verify AWS storage uri {uri}. Bucket {bucket} does not exist."
+ )
+ if e.response["Error"]["Code"] == "403":
+ raise LaunchError(
+ f"Could not verify AWS storage uri {uri}. "
+ "Bucket {bucket} is not accessible. Please check that this "
+ "client is authenticated with permission to access the bucket."
+ )
+ raise LaunchError(
+ f"Failed to verify AWS storage uri {uri}. Response: {e.response} Please verify that your AWS credentials are configured correctly."
+ )
diff --git a/wandb/sdk/launch/environment/azure_environment.py b/wandb/sdk/launch/environment/azure_environment.py
new file mode 100644
index 0000000000000000000000000000000000000000..2dbfebbe14d499302b4c22dd2321cdfb4ff8df63
--- /dev/null
+++ b/wandb/sdk/launch/environment/azure_environment.py
@@ -0,0 +1,105 @@
+"""Implementation of AzureEnvironment class."""
+
+from typing import Tuple
+
+from azure.core.exceptions import HttpResponseError # type: ignore
+from azure.identity import DefaultAzureCredential # type: ignore
+from azure.storage.blob import BlobClient, BlobServiceClient # type: ignore
+
+from ..errors import LaunchError
+from ..utils import AZURE_BLOB_REGEX
+from .abstract import AbstractEnvironment
+
+
+class AzureEnvironment(AbstractEnvironment):
+ """AzureEnvironment is a helper for accessing Azure resources."""
+
+ def __init__(
+ self,
+ ) -> None:
+ """Initialize an AzureEnvironment."""
+
+ @classmethod
+ def from_config(cls, config: dict, verify: bool = True) -> "AzureEnvironment":
+ """Create an AzureEnvironment from a config dict."""
+ return cls()
+
+ @classmethod
+ def get_credentials(cls) -> DefaultAzureCredential:
+ """Get Azure credentials."""
+ try:
+ return DefaultAzureCredential()
+ except Exception as e:
+ raise LaunchError(
+ f"Could not get Azure credentials. Please make sure you have "
+ f"configured your Azure CLI correctly.\n{e}"
+ ) from e
+
+ async def upload_file(self, source: str, destination: str) -> None:
+ """Upload a file to Azure blob storage.
+
+ Arguments:
+ source (str): The path to the file to upload.
+ destination (str): The destination path in Azure blob storage. Ex:
+ https://<storage_account>.blob.core.windows.net/<storage_container>/<path>
+ Raise:
+ LaunchError: If the file could not be uploaded.
+ """
+ storage_account, storage_container, path = self.parse_uri(destination)
+ _err_prefix = f"Could not upload file {source} to Azure blob {destination}"
+ creds = self.get_credentials()
+ try:
+ client = BlobClient(
+ f"https://{storage_account}.blob.core.windows.net",
+ storage_container,
+ path,
+ credential=creds,
+ )
+ with open(source, "rb") as f:
+ client.upload_blob(f, overwrite=True)
+ except HttpResponseError as e:
+ raise LaunchError(f"{_err_prefix}: {e.message}") from e
+ except Exception as e:
+ raise LaunchError(f"{_err_prefix}: {e.__class__.__name__}: {e}") from e
+
+ async def upload_dir(self, source: str, destination: str) -> None:
+ """Upload a directory to Azure blob storage."""
+ raise NotImplementedError()
+
+ async def verify_storage_uri(self, uri: str) -> None:
+ """Verify that the given blob storage prefix exists.
+
+ Args:
+ uri (str): The URI to verify.
+ """
+ creds = self.get_credentials()
+ storage_account, storage_container, _ = self.parse_uri(uri)
+ try:
+ client = BlobServiceClient(
+ f"https://{storage_account}.blob.core.windows.net",
+ credential=creds,
+ )
+ client.get_container_client(storage_container)
+ except Exception as e:
+ raise LaunchError(
+ f"Could not verify storage URI {uri} in container {storage_container}."
+ ) from e
+
+ async def verify(self) -> None:
+ """Verify that the AzureEnvironment is valid."""
+ self.get_credentials()
+
+ @staticmethod
+ def parse_uri(uri: str) -> Tuple[str, str, str]:
+ """Parse an Azure blob storage URI into a storage account and container.
+
+ Args:
+ uri (str): The URI to parse.
+
+ Returns:
+ Tuple[str, str, prefix]: The storage account, container, and path.
+ """
+ match = AZURE_BLOB_REGEX.match(uri)
+ if match is None:
+ raise LaunchError(f"Could not parse Azure blob URI {uri}.")
+ return match.group(1), match.group(2), match.group(3)
diff --git a/wandb/sdk/launch/environment/gcp_environment.py b/wandb/sdk/launch/environment/gcp_environment.py
new file mode 100644
index 0000000000000000000000000000000000000000..dcf785284b8cb4108b3534532fab06d9080f57cc
--- /dev/null
+++ b/wandb/sdk/launch/environment/gcp_environment.py
@@ -0,0 +1,334 @@
+"""Implementation of the GCP environment for wandb launch."""
+import logging
+import os
+import subprocess
+from typing import Optional
+
+from wandb.sdk.launch.errors import LaunchError
+from wandb.util import get_module
+
+from ..utils import GCS_URI_RE, event_loop_thread_exec
+from .abstract import AbstractEnvironment
+
+google = get_module(
+ "google",
+ required="Google Cloud Platform support requires the google package. Please"
+ " install it with `pip install wandb[launch]`.",
+)
+google.cloud.compute_v1 = get_module(
+ "google.cloud.compute_v1",
+ required="Google Cloud Platform support requires the google-cloud-compute package. "
+ "Please install it with `pip install wandb[launch]`.",
+)
+google.auth.credentials = get_module(
+ "google.auth.credentials",
+ required="Google Cloud Platform support requires google-auth. "
+ "Please install it with `pip install wandb[launch]`.",
+)
+google.auth.transport.requests = get_module(
+ "google.auth.transport.requests",
+ required="Google Cloud Platform support requires google-auth. "
+ "Please install it with `pip install wandb[launch]`.",
+)
+google.api_core.exceptions = get_module(
+ "google.api_core.exceptions",
+ required="Google Cloud Platform support requires google-api-core. "
+ "Please install it with `pip install wandb[launch]`.",
+)
+google.cloud.storage = get_module(
+ "google.cloud.storage",
+ required="Google Cloud Platform support requires google-cloud-storage. "
+ "Please install it with `pip install wandb[launch].",
+)
+
+
+_logger = logging.getLogger(__name__)
+
+GCP_REGION_ENV_VAR = "GOOGLE_CLOUD_REGION"
+
+
+class GcpEnvironment(AbstractEnvironment):
+ """GCP Environment.
+
+ Attributes:
+ region: The GCP region.
+ """
+
+ region: str
+
+ def __init__(
+ self,
+ region: str,
+ ) -> None:
+ """Initialize the GCP environment.
+
+ Arguments:
+ region: The GCP region.
+ verify: Whether to verify the credentials, region, and project.
+
+ Raises:
+ LaunchError: If verify is True and the environment is not properly
+ configured.
+ """
+ super().__init__()
+ _logger.info(f"Initializing GcpEnvironment in region {region}")
+ self.region: str = region
+ self._project = ""
+
+ @classmethod
+ def from_config(cls, config: dict) -> "GcpEnvironment":
+ """Create a GcpEnvironment from a config dictionary.
+
+ Arguments:
+ config: The config dictionary.
+
+ Returns:
+ GcpEnvironment: The GcpEnvironment.
+ """
+ if config.get("type") != "gcp":
+ raise LaunchError(
+ f"Could not create GcpEnvironment from config. Expected type 'gcp' "
+ f"but got '{config.get('type')}'."
+ )
+ region = config.get("region", None)
+ if not region:
+ raise LaunchError(
+ "Could not create GcpEnvironment from config. Missing 'region' "
+ "field."
+ )
+ return cls(region=region)
+
+ @classmethod
+ def from_default(
+ cls,
+ ) -> "GcpEnvironment":
+ """Create a GcpEnvironment from the default configuration.
+
+ Returns:
+ GcpEnvironment: The GcpEnvironment.
+ """
+ region = get_default_region()
+ if region is None:
+ raise LaunchError(
+ "Could not create GcpEnvironment from user's gcloud configuration. "
+ "Please set the default region with `gcloud config set compute/region` "
+ "or set the environment variable {GCP_REGION_ENV_VAR}. "
+ "Alternatively, you may specify the region explicitly in your "
+ "wandb launch configuration at `$HOME/.config/wandb/launch-config.yaml`. "
+ "See https://docs.wandb.ai/guides/launch/run-agent#environments for more information."
+ )
+ return cls(region=region)
+
+ @property
+ def project(self) -> str:
+ """Get the name of the gcp project associated with the credentials.
+
+ Returns:
+ str: The name of the gcp project.
+
+ Raises:
+ LaunchError: If the launch environment cannot be verified.
+ """
+ return self._project
+
+ async def get_credentials(self) -> google.auth.credentials.Credentials: # type: ignore
+ """Get the GCP credentials.
+
+ Uses google.auth.default() to get the credentials. If the credentials
+ are invalid, this method will refresh them. If the credentials are
+ still invalid after refreshing, this method will raise an error.
+
+ Returns:
+ google.auth.credentials.Credentials: The GCP credentials.
+
+ Raises:
+ LaunchError: If the GCP credentials are invalid.
+ """
+ _logger.debug("Getting GCP credentials")
+ # TODO: Figure out a minimal set of scopes.
+ try:
+ google_auth_default = event_loop_thread_exec(google.auth.default)
+ creds, project = await google_auth_default()
+ if not self._project:
+ self._project = project
+ _logger.debug("Refreshing GCP credentials")
+ await event_loop_thread_exec(creds.refresh)(
+ google.auth.transport.requests.Request()
+ )
+ except google.auth.exceptions.DefaultCredentialsError as e:
+ raise LaunchError(
+ "No Google Cloud Platform credentials found. Please run "
+ "`gcloud auth application-default login` or set the environment "
+ "variable GOOGLE_APPLICATION_CREDENTIALS to the path of a valid "
+ "service account key file."
+ ) from e
+ except google.auth.exceptions.RefreshError as e:
+ raise LaunchError(
+ "Could not refresh Google Cloud Platform credentials. Please run "
+ "`gcloud auth application-default login` or set the environment "
+ "variable GOOGLE_APPLICATION_CREDENTIALS to the path of a valid "
+ "service account key file."
+ ) from e
+ if not creds.valid:
+ raise LaunchError(
+ "Invalid Google Cloud Platform credentials. Please run "
+ "`gcloud auth application-default login` or set the environment "
+ "variable GOOGLE_APPLICATION_CREDENTIALS to the path of a valid "
+ "service account key file."
+ )
+ return creds
+
+ async def verify(self) -> None:
+ """Verify the credentials, region, and project.
+
+ Credentials and region are verified by calling get_credentials(). The
+ region and is verified by calling the compute API.
+
+ Raises:
+ LaunchError: If the credentials, region, or project are invalid.
+
+ Returns:
+ None
+ """
+ _logger.debug("Verifying GCP environment")
+ await self.get_credentials()
+
+ async def verify_storage_uri(self, uri: str) -> None:
+ """Verify that a storage URI is valid.
+
+ Arguments:
+ uri: The storage URI.
+
+ Raises:
+ LaunchError: If the storage URI is invalid.
+ """
+ match = GCS_URI_RE.match(uri)
+ if not match:
+ raise LaunchError(f"Invalid GCS URI: {uri}")
+ bucket = match.group(1)
+ cloud_storage_client = event_loop_thread_exec(google.cloud.storage.Client)
+ try:
+ credentials = await self.get_credentials()
+ storage_client = await cloud_storage_client(credentials=credentials)
+ bucket = await event_loop_thread_exec(storage_client.get_bucket)(bucket)
+ except google.api_core.exceptions.GoogleAPICallError as e:
+ raise LaunchError(
+ f"Failed verifying storage uri {uri}: bucket {bucket} does not exist."
+ ) from e
+ except google.api_core.exceptions.Forbidden as e:
+ raise LaunchError(
+ f"Failed verifying storage uri {uri}: bucket {bucket} is not accessible. Please check your permissions and try again."
+ ) from e
+
+ async def upload_file(self, source: str, destination: str) -> None:
+ """Upload a file to GCS.
+
+ Arguments:
+ source: The path to the local file.
+ destination: The path to the GCS file.
+
+ Raises:
+ LaunchError: If the file cannot be uploaded.
+ """
+ _logger.debug(f"Uploading file {source} to {destination}")
+ _err_prefix = f"Could not upload file {source} to GCS destination {destination}"
+ if not os.path.isfile(source):
+ raise LaunchError(f"{_err_prefix}: File {source} does not exist.")
+ match = GCS_URI_RE.match(destination)
+ if not match:
+ raise LaunchError(f"{_err_prefix}: Invalid GCS URI: {destination}")
+ bucket = match.group(1)
+ key = match.group(2).lstrip("/")
+ google_storage_client = event_loop_thread_exec(google.cloud.storage.Client)
+ credentials = await self.get_credentials()
+ try:
+ storage_client = await google_storage_client(credentials=credentials)
+ bucket = await event_loop_thread_exec(storage_client.bucket)(bucket)
+ blob = await event_loop_thread_exec(bucket.blob)(key)
+ await event_loop_thread_exec(blob.upload_from_filename)(source)
+ except google.api_core.exceptions.GoogleAPICallError as e:
+ resp = e.response
+ assert resp is not None
+ try:
+ message = resp.json()["error"]["message"]
+ except Exception:
+ message = str(resp)
+ raise LaunchError(f"{_err_prefix}: {message}") from e
+
+ async def upload_dir(self, source: str, destination: str) -> None:
+ """Upload a directory to GCS.
+
+ Arguments:
+ source: The path to the local directory.
+ destination: The path to the GCS directory.
+
+ Raises:
+ LaunchError: If the directory cannot be uploaded.
+ """
+ _logger.debug(f"Uploading directory {source} to {destination}")
+ _err_prefix = (
+ f"Could not upload directory {source} to GCS destination {destination}"
+ )
+ if not os.path.isdir(source):
+ raise LaunchError(f"{_err_prefix}: Directory {source} does not exist.")
+ match = GCS_URI_RE.match(destination)
+ if not match:
+ raise LaunchError(f"{_err_prefix}: Invalid GCS URI: {destination}")
+ bucket = match.group(1)
+ key = match.group(2).lstrip("/")
+ google_storage_client = event_loop_thread_exec(google.cloud.storage.Client)
+ credentials = await self.get_credentials()
+ try:
+ storage_client = await google_storage_client(credentials=credentials)
+ bucket = await event_loop_thread_exec(storage_client.bucket)(bucket)
+ for root, _, files in os.walk(source):
+ for file in files:
+ local_path = os.path.join(root, file)
+ gcs_path = os.path.join(
+ key, os.path.relpath(local_path, source)
+ ).replace("\\", "/")
+ blob = await event_loop_thread_exec(bucket.blob)(gcs_path)
+ await event_loop_thread_exec(blob.upload_from_filename)(local_path)
+ except google.api_core.exceptions.GoogleAPICallError as e:
+ resp = e.response
+ assert resp is not None
+ try:
+ message = resp.json()["error"]["message"]
+ except Exception:
+ message = str(resp)
+ raise LaunchError(f"{_err_prefix}: {message}") from e
+ except Exception as e:
+ raise LaunchError(f"{_err_prefix}: GCS upload failed: {e}") from e
+
+
+def get_gcloud_config_value(config_name: str) -> Optional[str]:
+ """Get a value from gcloud config.
+
+ Arguments:
+ config_name: The name of the config value.
+
+ Returns:
+ str: The config value, or None if the value is not set.
+ """
+ try:
+ output = subprocess.check_output(
+ ["gcloud", "config", "get-value", config_name], stderr=subprocess.STDOUT
+ )
+ value = str(output.decode("utf-8").strip())
+ if value and "unset" not in value:
+ return value
+ return None
+ except subprocess.CalledProcessError:
+ return None
+
+
+def get_default_region() -> Optional[str]:
+ """Get the default region from gcloud config or environment variables.
+
+ Returns:
+ str: The default region, or None if it cannot be determined.
+ """
+ region = get_gcloud_config_value("compute/region")
+ if not region:
+ region = os.environ.get(GCP_REGION_ENV_VAR)
+ return region
diff --git a/wandb/sdk/launch/environment/local_environment.py b/wandb/sdk/launch/environment/local_environment.py
new file mode 100644
index 0000000000000000000000000000000000000000..49efe8ff18d8dae587003322fe559b4190368d1d
--- /dev/null
+++ b/wandb/sdk/launch/environment/local_environment.py
@@ -0,0 +1,65 @@
+"""Dummy local environment implementation. This is the default environment."""
+from typing import Any, Dict, Union
+
+from wandb.sdk.launch.errors import LaunchError
+
+from .abstract import AbstractEnvironment
+
+
+class LocalEnvironment(AbstractEnvironment):
+ """Local environment class."""
+
+ def __init__(self) -> None:
+ """Initialize a local environment by doing nothing."""
+ pass
+
+ @classmethod
+ def from_config(
+ cls, config: Dict[str, Union[Dict[str, Any], str]]
+ ) -> "LocalEnvironment":
+ """Create a local environment from a config.
+
+ Arguments:
+ config (dict): The config. This is ignored.
+
+ Returns:
+ LocalEnvironment: The local environment.
+ """
+ return cls()
+
+ async def verify(self) -> None:
+ """Verify that the local environment is configured correctly."""
+ raise LaunchError("Attempted to verify LocalEnvironment.")
+
+ async def verify_storage_uri(self, uri: str) -> None:
+ """Verify that the storage URI is configured correctly.
+
+ Arguments:
+ uri (str): The storage URI. This is ignored.
+ """
+ raise LaunchError("Attempted to verify storage uri for LocalEnvironment.")
+
+ async def upload_file(self, source: str, destination: str) -> None:
+ """Upload a file from the local filesystem to storage in the environment.
+
+ Arguments:
+ source (str): The source file. This is ignored.
+ destination (str): The destination file. This is ignored.
+ """
+ raise LaunchError("Attempted to upload file for LocalEnvironment.")
+
+ async def upload_dir(self, source: str, destination: str) -> None:
+ """Upload the contents of a directory from the local filesystem to the environment.
+
+ Arguments:
+ source (str): The source directory. This is ignored.
+ destination (str): The destination directory. This is ignored.
+ """
+ raise LaunchError("Attempted to upload directory for LocalEnvironment.")
+
+ async def get_project(self) -> str:
+ """Get the project of the local environment.
+
+ Returns: An empty string.
+ """
+ raise LaunchError("Attempted to get project for LocalEnvironment.")
diff --git a/wandb/sdk/launch/errors.py b/wandb/sdk/launch/errors.py
new file mode 100644
index 0000000000000000000000000000000000000000..36e98600a102472f4cd998d8fd8170182279ee1f
--- /dev/null
+++ b/wandb/sdk/launch/errors.py
@@ -0,0 +1,19 @@
+from wandb.errors import Error
+
+
+class LaunchError(Error):
+ """Raised when a known error occurs in wandb launch."""
+
+ pass
+
+
+class LaunchDockerError(Error):
+ """Raised when Docker daemon is not running."""
+
+ pass
+
+
+class ExecutionError(Error):
+ """Generic execution exception."""
+
+ pass
diff --git a/wandb/sdk/launch/git_reference.py b/wandb/sdk/launch/git_reference.py
new file mode 100644
index 0000000000000000000000000000000000000000..0b44596a357b3b8fb63a9cbc948f71e532cecfc0
--- /dev/null
+++ b/wandb/sdk/launch/git_reference.py
@@ -0,0 +1,109 @@
+"""Support for parsing GitHub URLs (which might be user provided) into constituent parts."""
+
+import re
+from dataclasses import dataclass
+from enum import IntEnum
+from typing import Optional, Tuple, Union
+
+from wandb.sdk.launch.errors import LaunchError
+
+PREFIX_HTTPS = "https://"
+PREFIX_SSH = "git@"
+SUFFIX_GIT = ".git"
+
+
+GIT_COMMIT_REGEX = re.compile(r"[0-9a-f]{40}")
+
+
+class ReferenceType(IntEnum):
+ BRANCH = 1
+ COMMIT = 2
+
+
+def _parse_netloc(netloc: str) -> Tuple[Optional[str], Optional[str], str]:
+ """Parse netloc into username, password, and host.
+
+ github.com => None, None, "@github.com"
+ username@github.com => "username", None, "github.com"
+ username:password@github.com => "username", "password", "github.com"
+ """
+ parts = netloc.split("@", 1)
+ if len(parts) == 1:
+ return None, None, parts[0]
+ auth, host = parts
+ parts = auth.split(":", 1)
+ if len(parts) == 1:
+ return parts[0], None, host
+ return parts[0], parts[1], host
+
+
+@dataclass
+class GitReference:
+ def __init__(self, remote: str, ref: Optional[str] = None) -> None:
+ """Initialize a reference from a remote and ref.
+
+ Arguments:
+ remote: A remote URL or URI.
+ ref: A branch, tag, or commit hash.
+ """
+ self.uri = remote
+ self.ref = ref
+
+ @property
+ def url(self) -> Optional[str]:
+ return self.uri
+
+ def fetch(self, dst_dir: str) -> None:
+ """Fetch the repo into dst_dir and refine githubref based on what we learn."""
+ # We defer importing git until the last moment, because the import requires that the git
+ # executable is available on the PATH, so we only want to fail if we actually need it.
+ import git # type: ignore
+
+ repo = git.Repo.init(dst_dir)
+ self.path = repo.working_dir
+ origin = repo.create_remote("origin", self.uri or "")
+
+ try:
+ # We fetch the origin so that we have branch and tag references
+ origin.fetch(depth=1)
+ except git.exc.GitCommandError as e:
+ raise LaunchError(
+ f"Unable to fetch from git remote repository {self.url}:\n{e}"
+ )
+
+ ref: Union[git.RemoteReference, str]
+ if self.ref:
+ if self.ref in origin.refs:
+ ref = origin.refs[self.ref]
+ else:
+ ref = self.ref
+ head = repo.create_head(self.ref, ref)
+ head.checkout()
+ self.commit_hash = head.commit.hexsha
+
+ else:
+ # TODO: Is there a better way to do this?
+ default_branch = None
+ for ref in repo.references:
+ if hasattr(ref, "tag"): # Skip tag references
+ continue
+ refname = ref.name
+ if refname.startswith("origin/"): # Trim off "origin/"
+ refname = refname[7:]
+ if refname == "main":
+ default_branch = "main"
+ break
+ if refname == "master":
+ default_branch = "master"
+ # Keep looking in case we also have a main, which we let take precedence
+ # (While the references appear to be sorted, not clear if that's guaranteed.)
+ if not default_branch:
+ raise LaunchError(
+ f"Unable to determine branch or commit to checkout from {self.url}"
+ )
+ self.default_branch = default_branch
+ self.ref = default_branch
+ head = repo.create_head(default_branch, origin.refs[default_branch])
+ head.checkout()
+ self.commit_hash = head.commit.hexsha
+ repo.submodule_update(init=True, recursive=True)
diff --git a/wandb/sdk/launch/loader.py b/wandb/sdk/launch/loader.py
new file mode 100644
index 0000000000000000000000000000000000000000..6deb6e72f5918ab8f03dc35d246d9b2099d178e8
--- /dev/null
+++ b/wandb/sdk/launch/loader.py
@@ -0,0 +1,238 @@
+"""Utilities for the agent."""
+from typing import Any, Dict, Optional
+
+from wandb.apis.internal import Api
+from wandb.docker import is_docker_installed
+from wandb.sdk.launch.errors import LaunchError
+
+from .builder.abstract import AbstractBuilder
+from .environment.abstract import AbstractEnvironment
+from .registry.abstract import AbstractRegistry
+from .runner.abstract import AbstractRunner
+
+WANDB_RUNNERS = {
+ "local-container",
+ "local-process",
+ "kubernetes",
+ "vertex",
+ "sagemaker",
+}
+
+
+def environment_from_config(config: Optional[Dict[str, Any]]) -> AbstractEnvironment:
+ """Create an environment from a config.
+
+ This helper function is used to create an environment from a config. The
+ config should have a "type" key that specifies the type of environment to
+ create. The remaining keys are passed to the environment's from_config
+ method. If the config is None or empty, a LocalEnvironment is returned.
+
+ Arguments:
+ config (Dict[str, Any]): The config.
+
+ Returns:
+ Environment: The environment constructed.
+ """
+ if not config:
+ from .environment.local_environment import LocalEnvironment
+
+ return LocalEnvironment() # This is the default, dummy environment.
+ env_type = config.get("type")
+ if not env_type:
+ raise LaunchError(
+ "Could not create environment from config. Environment type not specified!"
+ )
+ if env_type == "local":
+ from .environment.local_environment import LocalEnvironment
+
+ return LocalEnvironment.from_config(config)
+ if env_type == "aws":
+ from .environment.aws_environment import AwsEnvironment
+
+ return AwsEnvironment.from_config(config)
+ if env_type == "gcp":
+ from .environment.gcp_environment import GcpEnvironment
+
+ return GcpEnvironment.from_config(config)
+ if env_type == "azure":
+ from .environment.azure_environment import AzureEnvironment
+
+ return AzureEnvironment.from_config(config)
+ raise LaunchError(
+ f"Could not create environment from config. Invalid type: {env_type}"
+ )
+
+
+def registry_from_config(
+ config: Optional[Dict[str, Any]], environment: AbstractEnvironment
+) -> AbstractRegistry:
+ """Create a registry from a config.
+
+ This helper function is used to create a registry from a config. The
+ config should have a "type" key that specifies the type of registry to
+ create. The remaining keys are passed to the registry's from_config
+ method. If the config is None or empty, a LocalRegistry is returned.
+
+ Arguments:
+ config (Dict[str, Any]): The registry config.
+ environment (Environment): The environment of the registry.
+
+ Returns:
+ The registry if config is not None, otherwise None.
+
+ Raises:
+ LaunchError: If the registry is not configured correctly.
+ """
+ if not config:
+ from .registry.local_registry import LocalRegistry
+
+ return LocalRegistry() # This is the default, dummy registry.
+ registry_type = config.get("type")
+ if registry_type is None or registry_type == "local":
+ from .registry.local_registry import LocalRegistry
+
+ return LocalRegistry() # This is the default, dummy registry.
+ if registry_type == "ecr":
+ from .registry.elastic_container_registry import ElasticContainerRegistry
+
+ return ElasticContainerRegistry.from_config(config)
+ if registry_type == "gcr":
+ from .registry.google_artifact_registry import GoogleArtifactRegistry
+
+ return GoogleArtifactRegistry.from_config(config)
+ if registry_type == "acr":
+ from .registry.azure_container_registry import AzureContainerRegistry
+
+ return AzureContainerRegistry.from_config(config)
+ raise LaunchError(
+ f"Could not create registry from config. Invalid registry type: {registry_type}"
+ )
+
+
+def builder_from_config(
+ config: Optional[Dict[str, Any]],
+ environment: AbstractEnvironment,
+ registry: AbstractRegistry,
+) -> AbstractBuilder:
+ """Create a builder from a config.
+
+ This helper function is used to create a builder from a config. The
+ config should have a "type" key that specifies the type of builder to import
+ and create. The remaining keys are passed to the builder's from_config
+ method. If the config is None or empty, a default builder is returned.
+
+ The default builder will be a DockerBuilder if we find a working docker cli
+ on the system, otherwise it will be a NoOpBuilder.
+
+ Arguments:
+ config (Dict[str, Any]): The builder config.
+ registry (Registry): The registry of the builder.
+
+ Returns:
+ The builder.
+
+ Raises:
+ LaunchError: If the builder is not configured correctly.
+ """
+ if not config:
+ if is_docker_installed():
+ from .builder.docker_builder import DockerBuilder
+
+ return DockerBuilder.from_config(
+ {}, environment, registry
+ ) # This is the default builder.
+
+ from .builder.noop import NoOpBuilder
+
+ return NoOpBuilder.from_config({}, environment, registry)
+
+ builder_type = config.get("type")
+ if builder_type is None:
+ raise LaunchError(
+ "Could not create builder from config. Builder type not specified"
+ )
+ if builder_type == "docker":
+ from .builder.docker_builder import DockerBuilder
+
+ return DockerBuilder.from_config(config, environment, registry)
+ if builder_type == "kaniko":
+ from .builder.kaniko_builder import KanikoBuilder
+
+ return KanikoBuilder.from_config(config, environment, registry)
+ if builder_type == "noop":
+ from .builder.noop import NoOpBuilder
+
+ return NoOpBuilder.from_config(config, environment, registry)
+ raise LaunchError(
+ f"Could not create builder from config. Invalid builder type: {builder_type}"
+ )
+
+
+def runner_from_config(
+ runner_name: str,
+ api: Api,
+ runner_config: Dict[str, Any],
+ environment: AbstractEnvironment,
+ registry: AbstractRegistry,
+) -> AbstractRunner:
+ """Create a runner from a config.
+
+ This helper function is used to create a runner from a config. The
+ config should have a "type" key that specifies the type of runner to import
+ and create. The remaining keys are passed to the runner's from_config
+ method. If the config is None or empty, a LocalContainerRunner is returned.
+
+ Arguments:
+ runner_name (str): The name of the backend.
+ api (Api): The API.
+ runner_config (Dict[str, Any]): The backend config.
+
+ Returns:
+ The runner.
+
+ Raises:
+ LaunchError: If the runner is not configured correctly.
+ """
+ if not runner_name or runner_name in ["local-container", "local"]:
+ from .runner.local_container import LocalContainerRunner
+
+ return LocalContainerRunner(api, runner_config, environment, registry)
+ if runner_name == "local-process":
+ from .runner.local_process import LocalProcessRunner
+
+ return LocalProcessRunner(api, runner_config)
+ if runner_name == "sagemaker":
+ from .environment.aws_environment import AwsEnvironment
+
+ if not isinstance(environment, AwsEnvironment):
+ try:
+ environment = AwsEnvironment.from_default()
+ except LaunchError as e:
+ raise LaunchError(
+ "Could not create Sagemaker runner. "
+ "Environment must be an instance of AwsEnvironment."
+ ) from e
+ from .runner.sagemaker_runner import SageMakerRunner
+
+ return SageMakerRunner(api, runner_config, environment, registry)
+ if runner_name in ["vertex", "gcp-vertex"]:
+ from .environment.gcp_environment import GcpEnvironment
+
+ if not isinstance(environment, GcpEnvironment):
+ try:
+ environment = GcpEnvironment.from_default()
+ except LaunchError as e:
+ raise LaunchError(
+ "Could not create Vertex runner. "
+ "Environment must be an instance of GcpEnvironment."
+ ) from e
+ from .runner.vertex_runner import VertexRunner
+
+ return VertexRunner(api, runner_config, environment, registry)
+ if runner_name == "kubernetes":
+ from .runner.kubernetes_runner import KubernetesRunner
+
+ return KubernetesRunner(api, runner_config, environment, registry)
+ raise LaunchError(
+ f"Could not create runner from config. Invalid runner name: {runner_name}"
+ )
diff --git a/wandb/sdk/launch/registry/abstract.py b/wandb/sdk/launch/registry/abstract.py
new file mode 100644
index 0000000000000000000000000000000000000000..55a70806abb67da1287f12821b8685a6a6d644e0
--- /dev/null
+++ b/wandb/sdk/launch/registry/abstract.py
@@ -0,0 +1,47 @@
+"""Abstract base class for registries."""
+from abc import ABC, abstractmethod
+from typing import Tuple
+
+
+class AbstractRegistry(ABC):
+ """Abstract base class for registries."""
+
+ uri: str
+
+ async def get_username_password(self) -> Tuple[str, str]:
+ """Get the username and password for the registry.
+
+ Returns:
+ (str, str): The username and password.
+ """
+ raise NotImplementedError
+
+ @abstractmethod
+ async def get_repo_uri(self) -> str:
+ """Get the URI for a repository.
+
+ Returns:
+ str: The URI.
+ """
+ raise NotImplementedError
+
+ @abstractmethod
+ async def check_image_exists(self, image_uri: str) -> bool:
+ """Check if an image exists in the registry.
+
+ Arguments:
+ image_uri (str): The URI of the image.
+
+ Returns:
+ bool: True if the image exists.
+ """
+ raise NotImplementedError
+
+ @classmethod
+ @abstractmethod
+ def from_config(
+ cls,
+ config: dict,
+ ) -> "AbstractRegistry":
+ """Create a registry from a config."""
+ raise NotImplementedError
diff --git a/wandb/sdk/launch/registry/azure_container_registry.py b/wandb/sdk/launch/registry/azure_container_registry.py
new file mode 100644
index 0000000000000000000000000000000000000000..de79fa6fd0aec1b6d1bf24f7fe26e47659adef4a
--- /dev/null
+++ b/wandb/sdk/launch/registry/azure_container_registry.py
@@ -0,0 +1,123 @@
+"""Implementation of AzureContainerRegistry class."""
+import re
+from typing import TYPE_CHECKING, Optional, Tuple
+
+from wandb.sdk.launch.environment.azure_environment import AzureEnvironment
+from wandb.sdk.launch.errors import LaunchError
+from wandb.sdk.launch.utils import AZURE_CONTAINER_REGISTRY_URI_REGEX
+from wandb.util import get_module
+
+from .abstract import AbstractRegistry
+
+if TYPE_CHECKING:
+ from azure.containerregistry import ContainerRegistryClient # type: ignore
+ from azure.core.exceptions import ResourceNotFoundError # type: ignore
+
+
+ContainerRegistryClient = get_module( # noqa: F811
+ "azure.containerregistry",
+ required="The azure-containerregistry package is required to use launch with Azure. Please install it with `pip install azure-containerregistry`.",
+).ContainerRegistryClient
+
+ResourceNotFoundError = get_module( # noqa: F811
+ "azure.core.exceptions",
+ required="The azure-core package is required to use launch with Azure. Please install it with `pip install azure-core`.",
+).ResourceNotFoundError
+
+
+class AzureContainerRegistry(AbstractRegistry):
+ """Helper for accessing Azure Container Registry resources."""
+
+ def __init__(
+ self,
+ uri: Optional[str] = None,
+ registry_name: Optional[str] = None,
+ repo_name: Optional[str] = None,
+ ):
+ """Initialize an AzureContainerRegistry."""
+ if uri is not None:
+ self.uri = uri
+ if any(x is not None for x in (registry_name, repo_name)):
+ raise LaunchError(
+ "Please specify either a registry name and repo name or a registry URI."
+ )
+ if self.uri.startswith("https://"):
+ self.uri = self.uri[len("https://") :]
+ match = AZURE_CONTAINER_REGISTRY_URI_REGEX.match(self.uri)
+ if match is None:
+ raise LaunchError(
+ f"Unable to parse Azure Container Registry URI: {self.uri}"
+ )
+ self.registry_name = match.group(1)
+ self.repo_name = match.group(2)
+ else:
+ if any(x is None for x in (registry_name, repo_name)):
+ raise LaunchError(
+ "Please specify both a registry name and repo name or a registry URI."
+ )
+ self.registry_name = registry_name
+ self.repo_name = repo_name
+ self.uri = f"{self.registry_name}.azurecr.io/{self.repo_name}"
+
+ @classmethod
+ def from_config(
+ cls,
+ config: dict,
+ ) -> "AzureContainerRegistry":
+ """Create an AzureContainerRegistry from a config dict.
+
+ Args:
+ config (dict): The config dict.
+ environment (AbstractEnvironment): The environment to use.
+ verify (bool, optional): Whether to verify the registry. Defaults to True.
+
+ Returns:
+ AzureContainerRegistry: The registry.
+
+ Raises:
+ LaunchError: If the config is invalid.
+ """
+ uri = config.get("uri")
+ if uri is None:
+ raise LaunchError(
+ "Please specify a registry name to use under the registry.uri."
+ )
+ return cls(
+ uri=uri,
+ )
+
+ async def get_username_password(self) -> Tuple[str, str]:
+ """Get username and password for container registry."""
+ raise NotImplementedError
+
+ async def check_image_exists(self, image_uri: str) -> bool:
+ """Check if image exists in container registry.
+
+ Args:
+ image_uri (str): Image URI to check.
+
+ Returns:
+ bool: True if image exists, False otherwise.
+ """
+ match = re.match(AZURE_CONTAINER_REGISTRY_URI_REGEX, image_uri)
+ if match is None:
+ raise LaunchError(
+ f"Unable to parse Azure Container Registry URI: {image_uri}"
+ )
+ registry = match.group(1)
+ repository = match.group(2)
+ tag = match.group(3)
+ credential = AzureEnvironment.get_credentials()
+ client = ContainerRegistryClient(f"https://{registry}.azurecr.io", credential)
+ try:
+ client.get_manifest_properties(repository, tag)
+ return True
+ except ResourceNotFoundError:
+ return False
+ except Exception as e:
+ raise LaunchError(
+ f"Unable to check if image exists in Azure Container Registry: {e}"
+ ) from e
+
+ async def get_repo_uri(self) -> str:
+ return f"{self.registry_name}.azurecr.io/{self.repo_name}"
diff --git a/wandb/sdk/launch/registry/elastic_container_registry.py b/wandb/sdk/launch/registry/elastic_container_registry.py
new file mode 100644
index 0000000000000000000000000000000000000000..eecbcc2041035682b60c67abfe83ab0b9548aa01
--- /dev/null
+++ b/wandb/sdk/launch/registry/elastic_container_registry.py
@@ -0,0 +1,191 @@
+"""Implementation of Elastic Container Registry class for wandb launch."""
+import base64
+import logging
+from typing import Dict, Optional, Tuple
+
+from wandb.sdk.launch.errors import LaunchError
+from wandb.sdk.launch.registry.abstract import AbstractRegistry
+from wandb.sdk.launch.utils import (
+ ELASTIC_CONTAINER_REGISTRY_URI_REGEX,
+ event_loop_thread_exec,
+)
+from wandb.util import get_module
+
+_logger = logging.getLogger(__name__)
+
+botocore = get_module( # noqa: F811
+ "botocore",
+ required="The boto3 package is required to use launch with AWS. Please install it with `pip install wandb[launch]`.",
+)
+boto3 = get_module( # noqa: F811
+ "boto3",
+ required="The boto3 package is required to use launch with AWS. Please install it with `pip install wandb[launch]`.",
+)
+
+
+class ElasticContainerRegistry(AbstractRegistry):
+ """Elastic Container Registry class."""
+
+ def __init__(
+ self,
+ uri: Optional[str] = None,
+ account_id: Optional[str] = None,
+ region: Optional[str] = None,
+ repo_name: Optional[str] = None,
+ ) -> None:
+ """Initialize the Elastic Container Registry.
+
+ Arguments:
+ uri: The uri of the repository.
+ account_id: The AWS account id.
+ region: The AWS region of the container registry.
+ repository: The name of the repository.
+
+ Raises:
+ LaunchError: If there is an error initializing the Elastic Container Registry helper.
+ """
+ if uri:
+ self.uri = uri
+ if any([account_id, region, repo_name]):
+ raise LaunchError(
+ "Could not create ElasticContainerRegistry from config. Either 'uri' or "
+ "'account_id', 'region', and 'repo_name' are required."
+ )
+ match = ELASTIC_CONTAINER_REGISTRY_URI_REGEX.match(
+ self.uri,
+ )
+ if not match:
+ raise LaunchError(
+ f"Could not create ElasticContainerRegistry from config. The uri "
+ f"{self.uri} is invalid."
+ )
+ self.account_id = match.group("account")
+ self.region = match.group("region")
+ self.repo_name = match.group("repository")
+ else:
+ if not all([account_id, region, repo_name]):
+ raise LaunchError(
+ "Could not create ElasticContainerRegistry from config. Either 'uri' or "
+ "'account_id', 'region', and 'repo_name' are required."
+ )
+ self.account_id = account_id
+ self.region = region
+ self.repo_name = repo_name
+ self.uri = f"{self.account_id}.dkr.ecr.{self.region}.amazonaws.com/{self.repo_name}"
+ if self.account_id is None:
+ raise LaunchError(
+ "Could not create ElasticContainerRegistry from config. Either 'uri' or "
+ "'account_id' is required."
+ )
+ if self.region is None:
+ raise LaunchError(
+ "Could not create ElasticContainerRegistry from config. Either 'uri' or "
+ "'region' is required."
+ )
+ if self.repo_name is None:
+ raise LaunchError(
+ "Could not create ElasticContainerRegistry from config. Either 'uri' or "
+ "'repository' is required."
+ )
+
+ @classmethod
+ def from_config(
+ cls,
+ config: Dict[str, str],
+ ) -> "ElasticContainerRegistry":
+ """Create an Elastic Container Registry from a config.
+
+ Arguments:
+ config (dict): The config.
+
+ Returns:
+ ElasticContainerRegistry: The Elastic Container Registry.
+ """
+ # TODO: Replace this with pydantic.
+ acceptable_keys = {
+ "uri",
+ "type",
+ "account_id",
+ "region",
+ "repo_name",
+ }
+ unsupported_keys = set(config.keys()) - acceptable_keys
+ if unsupported_keys:
+ raise LaunchError(
+ f"The Elastic Container Registry config contains unsupported keys: "
+ f"{unsupported_keys}. Please remove these keys. The acceptable "
+ f"keys are: {acceptable_keys}."
+ )
+ return cls(
+ uri=config.get("uri"),
+ account_id=config.get("account_id"),
+ region=config.get("region"),
+ repo_name=config.get("repository"),
+ )
+
+ async def get_username_password(self) -> Tuple[str, str]:
+ """Get the username and password for the registry.
+
+ Returns:
+ (str, str): The username and password.
+
+ Raises:
+ RegistryError: If there is an error getting the username and password.
+ """
+ _logger.debug("Getting username and password for Elastic Container Registry.")
+ try:
+ session = boto3.Session(region_name=self.region)
+ client = await event_loop_thread_exec(session.client)("ecr")
+ response = await event_loop_thread_exec(client.get_authorization_token)()
+ username, password = base64.standard_b64decode(
+ response["authorizationData"][0]["authorizationToken"]
+ ).split(b":")
+ return username.decode("utf-8"), password.decode("utf-8")
+
+ except botocore.exceptions.ClientError as e:
+ code = e.response["Error"]["Code"]
+ msg = e.response["Error"]["Message"]
+ # TODO: Log the code and the message here?
+ raise LaunchError(f"Error getting username and password: {code} {msg}")
+
+ async def get_repo_uri(self) -> str:
+ """Get the uri of the repository.
+
+ Returns:
+ str: The uri of the repository.
+ """
+ return f"{self.account_id}.dkr.ecr.{self.region}.amazonaws.com/{self.repo_name}"
+
+ async def check_image_exists(self, image_uri: str) -> bool:
+ """Check if the image tag exists.
+
+ Arguments:
+ image_uri (str): The full image_uri.
+
+ Returns:
+ bool: True if the image tag exists.
+ """
+ if ":" not in image_uri:
+ tag = image_uri
+ else:
+ uri, tag = image_uri.split(":")
+ repo_uri = await self.get_repo_uri()
+ if uri != repo_uri:
+ raise LaunchError(
+ f"Image uri {image_uri} does not match Elastic Container Registry uri {repo_uri}."
+ )
+ _logger.debug(f"Checking if image tag {tag} exists in repository {self.uri}")
+ try:
+ session = boto3.Session(region_name=self.region)
+ client = await event_loop_thread_exec(session.client)("ecr")
+ response = await event_loop_thread_exec(client.describe_images)(
+ repositoryName=self.repo_name, imageIds=[{"imageTag": tag}]
+ )
+ return len(response["imageDetails"]) > 0
+
+ except botocore.exceptions.ClientError as e:
+ code = e.response["Error"]["Code"]
+ if code == "ImageNotFoundException":
+ return False
+ msg = e.response["Error"]["Message"]
+ raise LaunchError(f"Error checking if image tag exists: {code} {msg}")
diff --git a/wandb/sdk/launch/registry/google_artifact_registry.py b/wandb/sdk/launch/registry/google_artifact_registry.py
new file mode 100644
index 0000000000000000000000000000000000000000..90eba602b710d66e468c407dec65a837c12cec0b
--- /dev/null
+++ b/wandb/sdk/launch/registry/google_artifact_registry.py
@@ -0,0 +1,218 @@
+"""Implementation of Google Artifact Registry for wandb launch."""
+import logging
+from typing import Optional, Tuple
+
+import google.auth # type: ignore
+import google.cloud.artifactregistry # type: ignore
+
+from wandb.sdk.launch.errors import LaunchError
+from wandb.sdk.launch.utils import (
+ GCP_ARTIFACT_REGISTRY_URI_REGEX,
+ event_loop_thread_exec,
+)
+from wandb.util import get_module
+
+from .abstract import AbstractRegistry
+
+_logger = logging.getLogger(__name__)
+
+google = get_module( # noqa: F811
+ "google",
+ required="The google package is required to use launch with Google. Please install it with `pip install wandb[launch]`.",
+)
+google.auth = get_module( # noqa: F811
+ "google.auth",
+ required="The google-auth package is required to use launch with Google. Please install it with `pip install wandb[launch]`.",
+)
+
+google.cloud.artifactregistry = get_module( # noqa: F811
+ "google.cloud.artifactregistry",
+ required="The google-cloud-artifactregistry package is required to use launch with Google. Please install it with `pip install wandb[launch]`.",
+)
+
+
+class GoogleArtifactRegistry(AbstractRegistry):
+ """Google Artifact Registry helper for interacting with the registry.
+
+ This helper should be constructed from either a uri or a repository,
+ project, and optional image-name. If constructed from a uri, the uri
+ must be of the form REGION-docker.pkg.dev/PROJECT/REPOSITORY/[IMAGE_NAME],
+ with an optional https:// preceding.
+ """
+
+ def __init__(
+ self,
+ uri: Optional[str] = None,
+ repository: Optional[str] = None,
+ image_name: Optional[str] = None,
+ project: Optional[str] = None,
+ region: Optional[str] = None,
+ ) -> None:
+ """Initialize the Google Artifact Registry.
+
+ Either uri or repository and image_name must be provided. Project and
+ region are optional, and will be inferred from the uri if provided, or
+ from the default credentials if not.
+
+ Arguments:
+ uri (optional): The uri of the repository.
+ repository (optional): The repository name.
+ image_name (optional): The image name.
+ project (optional): The GCP project name.
+ region (optional): The GCP region name.
+
+ Raises:
+ LaunchError: If verify is True and the container registry or its
+ environment have not been properly configured. Or if the environment
+ is not an instance of GcpEnvironment.
+ """
+ _logger.info(
+ f"Initializing Google Artifact Registry with repository {repository} "
+ f"and image name {image_name}"
+ )
+
+ if uri is not None:
+ self.uri = uri
+ # Raise an error if any other kwargs were provided in addition to uri.
+ if any([repository, image_name, project, region]):
+ raise LaunchError(
+ "The Google Artifact Registry must be specified with either "
+ "the uri key or the repository, image-name, project and region "
+ "keys, but not both."
+ )
+ match = GCP_ARTIFACT_REGISTRY_URI_REGEX.match(self.uri)
+ if not match:
+ raise LaunchError(
+ f"The Google Artifact Registry uri {self.uri} is invalid. "
+ "Please provide a uri of the form "
+ "REGION-docker.pkg.dev/PROJECT/REPOSITORY/IMAGE_NAME."
+ )
+ self.project = match.group("project")
+ self.region = match.group("region")
+ self.repository = match.group("repository")
+ self.image_name = match.group("image_name")
+ else:
+ if any(x is None for x in (repository, region, image_name)):
+ raise LaunchError(
+ "The Google Artifact Registry must be specified with either "
+ "the uri key or the repository, image-name, project and region "
+ "keys."
+ )
+ self.project = project
+ self.region = region
+ self.repository = repository
+ self.image_name = image_name
+ self.uri = f"{self.region}-docker.pkg.dev/{self.project}/{self.repository}/{self.image_name}"
+
+ _missing_kwarg_msg = (
+ "The Google Artifact Registry is missing the {} kwarg. "
+ "Please specify it by name or as part of the uri argument."
+ )
+ if not self.region:
+ raise LaunchError(_missing_kwarg_msg.format("region"))
+ if not self.repository:
+ raise LaunchError(_missing_kwarg_msg.format("repository"))
+ if not self.image_name:
+ raise LaunchError(_missing_kwarg_msg.format("image-name"))
+ # Try to load default project from the default credentials.
+ self.credentials, project = google.auth.default()
+ self.project = self.project or project
+ self.credentials.refresh(google.auth.transport.requests.Request())
+
+ @classmethod
+ def from_config(
+ cls,
+ config: dict,
+ ) -> "GoogleArtifactRegistry":
+ """Create a Google Artifact Registry from a config.
+
+ Arguments:
+ config: A dictionary containing the following keys:
+ repository: The repository name.
+ image-name: The image name.
+ environment: A GcpEnvironment configured for access to this registry.
+
+ Returns:
+ A GoogleArtifactRegistry.
+ """
+ # TODO: Replace this with pydantic.
+ acceptable_keys = {
+ "uri",
+ "type",
+ "repository",
+ "image-name",
+ "region",
+ "project",
+ }
+ unacceptable_keys = set(config.keys()) - acceptable_keys
+ if unacceptable_keys:
+ raise LaunchError(
+ f"The Google Artifact Registry config contains unacceptable keys: "
+ f"{unacceptable_keys}. Please remove these keys. The acceptable "
+ f"keys are: {acceptable_keys}."
+ )
+ return cls(
+ uri=config.get("uri"),
+ repository=config.get("repository"),
+ image_name=config.get("image-name"),
+ project=config.get("project"),
+ region=config.get("region"),
+ )
+
+ async def get_username_password(self) -> Tuple[str, str]:
+ """Get the username and password for the registry.
+
+ Returns:
+ A tuple of the username and password.
+ """
+ if not self.credentials.token:
+ self.credentials.refresh(google.auth.transport.requests.Request())
+ return "oauth2accesstoken", self.credentials.token
+
+ async def get_repo_uri(self) -> str:
+ """Get the URI for the given repository.
+
+ Arguments:
+ repo_name: The repository name.
+
+ Returns:
+ The repository URI.
+ """
+ return (
+ f"{self.region}-docker.pkg.dev/"
+ f"{self.project}/{self.repository}/{self.image_name}"
+ )
+
+ async def check_image_exists(self, image_uri: str) -> bool:
+ """Check if the image exists.
+
+ Arguments:
+ image_uri: The image URI.
+
+ Returns:
+ True if the image exists, False otherwise.
+ """
+ _logger.info(f"Checking if image {image_uri} exists")
+ repo_uri, tag = image_uri.split(":")
+ self_repo_uri = await self.get_repo_uri()
+ if repo_uri != self_repo_uri:
+ raise LaunchError(
+ f"The image {image_uri} does not match to the image uri "
+ f"repository {self.uri}."
+ )
+ parent = f"projects/{self.project}/locations/{self.region}/repositories/{self.repository}"
+ artifact_registry_client = event_loop_thread_exec(
+ google.cloud.artifactregistry.ArtifactRegistryClient
+ )
+ client = await artifact_registry_client(credentials=self.credentials)
+ list_images = event_loop_thread_exec(client.list_docker_images)
+ try:
+ for image in await list_images(request={"parent": parent}):
+ if tag in image.tags:
+ return True
+ except google.api_core.exceptions.NotFound as e:
+ raise LaunchError(
+ f"The Google Artifact Registry repository {self.repository} "
+ f"does not exist. Please create it or modify your registry configuration."
+ ) from e
+ return False
diff --git a/wandb/sdk/launch/registry/local_registry.py b/wandb/sdk/launch/registry/local_registry.py
new file mode 100644
index 0000000000000000000000000000000000000000..fd4bf9166868299b4455f856c372a385c482562d
--- /dev/null
+++ b/wandb/sdk/launch/registry/local_registry.py
@@ -0,0 +1,63 @@
+"""Local registry implementation."""
+import logging
+from typing import Tuple
+
+from wandb.sdk.launch.errors import LaunchError
+from wandb.sdk.launch.utils import docker_image_exists
+
+from .abstract import AbstractRegistry
+
+_logger = logging.getLogger(__name__)
+
+
+class LocalRegistry(AbstractRegistry):
+ """A local registry.
+
+ This is a dummy registry that is used when no registry is configured.
+ """
+
+ def __init__(self) -> None:
+ """Initialize a local registry."""
+ pass
+
+ @classmethod
+ def from_config(
+ cls,
+ config: dict,
+ ) -> "LocalRegistry":
+ """Create a local registry from a config.
+
+ Arguments:
+ config (dict): The config. This is ignored.
+ environment (AbstractEnvironment): The environment. This is ignored.
+
+ Returns:
+ LocalRegistry: The local registry.
+ """
+ return cls()
+
+ async def verify(self) -> None:
+ """Verify the local registry by doing nothing."""
+ pass
+
+ async def get_username_password(self) -> Tuple[str, str]:
+ """Get the username and password of the local registry."""
+ raise LaunchError("Attempted to get username and password for LocalRegistry.")
+
+ async def get_repo_uri(self) -> str:
+ """Get the uri of the local registry.
+
+ Returns: An empty string.
+ """
+ return ""
+
+ async def check_image_exists(self, image_uri: str) -> bool:
+ """Check if an image exists in the local registry.
+
+ Arguments:
+ image_uri (str): The uri of the image.
+
+ Returns:
+ bool: True.
+ """
+ return docker_image_exists(image_uri)
diff --git a/wandb/sdk/launch/runner/README.md b/wandb/sdk/launch/runner/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..3d32e3ad6f6f5dc67faa40ae917887d7dd161057
--- /dev/null
+++ b/wandb/sdk/launch/runner/README.md
@@ -0,0 +1,2 @@
+A runner is responsible for executing and launching the container. Each runner launches a container in a different way depending on the target environment.
+
diff --git a/wandb/sdk/launch/runner/__init__.py b/wandb/sdk/launch/runner/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sdk/launch/runner/abstract.py b/wandb/sdk/launch/runner/abstract.py
new file mode 100644
index 0000000000000000000000000000000000000000..e8dfb182a3e6af22ee17474f495325a2ee814286
--- /dev/null
+++ b/wandb/sdk/launch/runner/abstract.py
@@ -0,0 +1,194 @@
+"""Implementation of the abstract runner class.
+
+This class defines the interface that the W&B launch runner uses to manage the lifecycle
+of runs launched in different environments (e.g. runs launched locally or in a cluster).
+"""
+import logging
+import os
+import subprocess
+import sys
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Union
+
+from dockerpycreds.utils import find_executable # type: ignore
+
+import wandb
+from wandb.apis.internal import Api
+from wandb.sdk.lib import runid
+
+from .._project_spec import LaunchProject
+
+_logger = logging.getLogger(__name__)
+
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+State = Literal[
+ "unknown",
+ "starting",
+ "running",
+ "failed",
+ "finished",
+ "stopping",
+ "stopped",
+ "preempted",
+]
+
+
+class Status:
+ def __init__(self, state: "State" = "unknown", data=None): # type: ignore
+ self.state = state
+ self.data = data or {}
+
+ def __repr__(self) -> "State":
+ return self.state
+
+ def __str__(self) -> str:
+ return self.state
+
+ def __eq__(self, __value: object) -> bool:
+ if isinstance(__value, Status):
+ return self.state == __value.state
+ else:
+ return self.state == __value
+
+ def __hash__(self) -> int:
+ return hash(self.state)
+
+
+class AbstractRun(ABC):
+ """Wrapper around a W&B launch run.
+
+ A launched run is a subprocess running an entry point
+ command, that exposes methods for waiting on and cancelling the run.
+ This class defines the interface that the W&B launch runner uses to manage the lifecycle
+ of runs launched in different environments (e.g. runs launched locally or in a cluster).
+ ``AbstractRun`` is not thread-safe. That is, concurrent calls to wait() / cancel()
+ from multiple threads may inadvertently kill resources (e.g. local processes) unrelated to the
+ run.
+ """
+
+ def __init__(self) -> None:
+ self._status = Status()
+
+ @property
+ def status(self) -> Status:
+ return self._status
+
+ @abstractmethod
+ async def get_logs(self) -> Optional[str]:
+ """Return the logs associated with the run."""
+ pass
+
+ def _run_cmd(
+ self, cmd: List[str], output_only: Optional[bool] = False
+ ) -> Optional[Union["subprocess.Popen[bytes]", bytes]]:
+ """Run the command and returns a popen object or the stdout of the command.
+
+ Arguments:
+ cmd: The command to run
+ output_only: If true just return the stdout bytes
+ """
+ try:
+ env = os.environ
+ popen = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE)
+ if output_only:
+ popen.wait()
+ if popen.stdout is not None:
+ return popen.stdout.read()
+ return popen
+ except subprocess.CalledProcessError as e:
+ wandb.termerror(f"Command failed: {e}")
+ return None
+
+ @abstractmethod
+ async def wait(self) -> bool:
+ """Wait for the run to finish, returning True if the run succeeded and false otherwise.
+
+ Note that in some cases, we may wait until the remote job completes rather than until the W&B run completes.
+ """
+ pass
+
+ @abstractmethod
+ async def get_status(self) -> Status:
+ """Get status of the run."""
+ pass
+
+ @abstractmethod
+ async def cancel(self) -> None:
+ """Cancel the run (interrupts the command subprocess, cancels the run, etc).
+
+ Cancels the run and waits for it to terminate. The W&B run status may not be
+ set correctly upon run cancellation.
+ """
+ pass
+
+ @property
+ @abstractmethod
+ def id(self) -> Optional[str]:
+ pass
+
+
+class AbstractRunner(ABC):
+ """Abstract plugin class defining the interface needed to execute W&B Launches.
+
+ You can define subclasses of ``AbstractRunner`` and expose them as third-party
+ plugins to enable running W&B projects against custom execution backends
+ (e.g. to run projects against your team's in-house cluster or job scheduler).
+ """
+
+ _type: str
+
+ def __init__(
+ self,
+ api: Api,
+ backend_config: Dict[str, Any],
+ ) -> None:
+ self._api = api
+ self.backend_config = backend_config
+ self._cwd = os.getcwd()
+ self._namespace = runid.generate_id()
+
+ def find_executable(
+ self, cmd: str
+ ) -> Any: # should return a string, but mypy doesn't trust find_executable
+ """Cross platform utility for checking if a program is available."""
+ return find_executable(cmd)
+
+ @property
+ def api_key(self) -> Any:
+ return self._api.api_key
+
+ def verify(self) -> bool:
+ """This is called on first boot to verify the needed commands, and permissions are available.
+
+ For now just call `wandb.termerror` and `sys.exit(1)`
+ """
+ if self._api.api_key is None:
+ wandb.termerror(
+ "Couldn't find W&B api key, run wandb login or set WANDB_API_KEY"
+ )
+ sys.exit(1)
+ return True
+
+ @abstractmethod
+ async def run(
+ self,
+ launch_project: LaunchProject,
+ image_uri: str,
+ ) -> Optional[AbstractRun]:
+ """Submit an LaunchProject to be run.
+
+ Returns a SubmittedRun object to track the execution
+ Arguments:
+ launch_project: Object of _project_spec.LaunchProject class representing a wandb launch project
+
+ Returns:
+ A :py:class:`wandb.sdk.launch.runners.SubmittedRun`. This function is expected to run
+ the project asynchronously, i.e. it should trigger project execution and then
+ immediately return a `SubmittedRun` to track execution status.
+ """
+ pass
diff --git a/wandb/sdk/launch/runner/kubernetes_monitor.py b/wandb/sdk/launch/runner/kubernetes_monitor.py
new file mode 100644
index 0000000000000000000000000000000000000000..6b75947810a900f42418de7f7c822856107a1e3d
--- /dev/null
+++ b/wandb/sdk/launch/runner/kubernetes_monitor.py
@@ -0,0 +1,438 @@
+"""Monitors kubernetes resources managed by the launch agent."""
+import asyncio
+import logging
+import sys
+import traceback
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import kubernetes_asyncio # type: ignore # noqa: F401
+import urllib3
+from kubernetes_asyncio import watch
+from kubernetes_asyncio.client import ( # type: ignore # noqa: F401
+ ApiException,
+ BatchV1Api,
+ CoreV1Api,
+ CustomObjectsApi,
+ V1PodStatus,
+)
+
+import wandb
+from wandb.sdk.launch.agent import LaunchAgent
+from wandb.sdk.launch.errors import LaunchError
+from wandb.sdk.launch.runner.abstract import State, Status
+from wandb.sdk.launch.utils import get_kube_context_and_api_client
+
+WANDB_K8S_LABEL_NAMESPACE = "wandb.ai"
+WANDB_K8S_RUN_ID = f"{WANDB_K8S_LABEL_NAMESPACE}/run-id"
+WANDB_K8S_LABEL_AGENT = f"{WANDB_K8S_LABEL_NAMESPACE}/agent"
+WANDB_K8S_LABEL_MONITOR = f"{WANDB_K8S_LABEL_NAMESPACE}/monitor"
+
+
+class Resources:
+ JOBS = "jobs"
+ PODS = "pods"
+
+
+class CustomResource:
+ """Class for custom resources."""
+
+ def __init__(self, group: str, version: str, plural: str) -> None:
+ """Initialize the CustomResource."""
+ self.group = group
+ self.version = version
+ self.plural = plural
+
+ def __str__(self) -> str:
+ """Return a string representation of the CustomResource."""
+ return f"{self.group}/{self.version}/{self.plural}"
+
+ def __hash__(self) -> int:
+ """Return a hash of the CustomResource."""
+ return hash(str(self))
+
+
+# Maps phases and conditions of custom objects to agent's internal run states.
+CRD_STATE_DICT: Dict[str, State] = {
+ "created": "starting",
+ "pending": "starting",
+ "running": "running",
+ "completing": "running",
+ "succeeded": "finished",
+ "completed": "finished",
+ "failed": "failed",
+ "aborted": "failed",
+ "timeout": "failed",
+ "terminated": "failed",
+ "terminating": "stopping",
+}
+
+_logger = logging.getLogger(__name__)
+
+
+def create_named_task(name: str, coro: Any, *args: Any, **kwargs: Any) -> asyncio.Task:
+ """Create a named task."""
+ task = asyncio.create_task(coro(*args, **kwargs))
+ if sys.version_info >= (3, 8):
+ task.set_name(name)
+ task.add_done_callback(_log_err_task_callback)
+ return task
+
+
+def _log_err_task_callback(task: asyncio.Task) -> None:
+ """Callback to log exceptions from tasks."""
+ exec = task.exception()
+ if exec is not None:
+ if isinstance(exec, asyncio.CancelledError):
+ wandb.termlog(f"Task {task.get_name()} was cancelled")
+ return
+ name = str(task) if sys.version_info < (3, 8) else task.get_name()
+ wandb.termerror(f"Exception in task {name}")
+ tb = exec.__traceback__
+ tb_str = "".join(traceback.format_tb(tb))
+ wandb.termerror(tb_str)
+
+
+def _is_preempted(status: "V1PodStatus") -> bool:
+ """Check if this pod has been preempted."""
+ if hasattr(status, "conditions") and status.conditions is not None:
+ for condition in status.conditions:
+ if condition.type == "DisruptionTarget" and condition.reason in [
+ "EvictionByEvictionAPI",
+ "PreemptionByScheduler",
+ "TerminationByKubelet",
+ ]:
+ return True
+ return False
+
+
+def _is_container_creating(status: "V1PodStatus") -> bool:
+ """Check if this pod has started creating containers."""
+ for container_status in status.container_statuses or []:
+ if (
+ container_status.state
+ and container_status.state.waiting
+ and container_status.state.waiting.reason == "ContainerCreating"
+ ):
+ return True
+ return False
+
+
+def _state_from_conditions(conditions: List[Dict[str, Any]]) -> Optional[State]:
+ """Get the status from the pod conditions."""
+ true_conditions = [
+ c.get("type", "").lower() for c in conditions if c.get("status") == "True"
+ ]
+ detected_states = {
+ CRD_STATE_DICT[c] for c in true_conditions if c in CRD_STATE_DICT
+ }
+ # The list below is ordered so that returning the first state detected
+ # will accurately reflect the state of the job.
+ states_in_order: List[State] = [
+ "finished",
+ "failed",
+ "stopping",
+ "running",
+ "starting",
+ ]
+ for state in states_in_order:
+ if state in detected_states:
+ return state
+ return None
+
+
+def _state_from_replicated_status(status_dict: Dict[str, int]) -> Optional[State]:
+ """Infer overall job status from replicated job status for jobsets.
+
+ More info on jobset:
+ https://github.com/kubernetes-sigs/jobset/blob/main/docs/concepts/README.md
+
+ This is useful for detecting when jobsets are starting.
+ """
+ pods_ready = status_dict.get("ready", 0)
+ pods_active = status_dict.get("active", 0)
+ if pods_ready >= 1:
+ return "running"
+ elif pods_active >= 1:
+ return "starting"
+ return None
+
+
+class LaunchKubernetesMonitor:
+ """Monitors kubernetes resources managed by the launch agent.
+
+ Note: this class is forced to be a singleton in order to prevent multiple
+ threads from being created that monitor the same kubernetes resources.
+ """
+
+ _instance = None # This is used to ensure only one instance is created.
+
+ def __new__(cls, *args: Any, **kwargs: Any) -> "LaunchKubernetesMonitor":
+ """Create a new instance of the LaunchKubernetesMonitor.
+
+ This method ensures that only one instance of the LaunchKubernetesMonitor
+ is created. This is done to prevent multiple threads from being created
+ that monitor the same kubernetes resources.
+ """
+ if cls._instance is None:
+ cls._instance = super().__new__(cls)
+ return cls._instance
+
+ def __init__(
+ self,
+ core_api: CoreV1Api,
+ batch_api: BatchV1Api,
+ custom_api: CustomObjectsApi,
+ label_selector: str,
+ ):
+ """Initialize the LaunchKubernetesMonitor."""
+ self._core_api: CoreV1Api = core_api
+ self._batch_api: BatchV1Api = batch_api
+ self._custom_api: CustomObjectsApi = custom_api
+
+ self._label_selector: str = label_selector
+
+ # Dict mapping a tuple of (namespace, resource_type) to an
+ # asyncio.Task that is monitoring that resource type in that namespace.
+ self._monitor_tasks: Dict[
+ Tuple[str, Union[str, CustomResource]], asyncio.Task
+ ] = dict()
+
+ # Map from job name to job state.
+ self._job_states: Dict[str, Status] = dict()
+
+ @classmethod
+ async def ensure_initialized(
+ cls,
+ ) -> None:
+ """Initialize the LaunchKubernetesMonitor."""
+ if cls._instance is None:
+ _, api_client = await get_kube_context_and_api_client(
+ kubernetes_asyncio, {}
+ )
+ core_api = CoreV1Api(api_client)
+ batch_api = BatchV1Api(api_client)
+ custom_api = CustomObjectsApi(api_client)
+ label_selector = f"{WANDB_K8S_LABEL_MONITOR}=true"
+ if LaunchAgent.initialized():
+ label_selector += f",{WANDB_K8S_LABEL_AGENT}={LaunchAgent.name()}"
+ cls(
+ core_api=core_api,
+ batch_api=batch_api,
+ custom_api=custom_api,
+ label_selector=label_selector,
+ )
+
+ @classmethod
+ def monitor_namespace(
+ cls, namespace: str, custom_resource: Optional[CustomResource] = None
+ ) -> None:
+ """Start monitoring a namespaces for resources."""
+ if cls._instance is None:
+ raise LaunchError(
+ "LaunchKubernetesMonitor not initialized, cannot monitor namespace."
+ )
+ cls._instance.__monitor_namespace(namespace, custom_resource=custom_resource)
+
+ @classmethod
+ def get_status(cls, job_name: str) -> Status:
+ """Get the status of a job."""
+ if cls._instance is None:
+ raise LaunchError(
+ "LaunchKubernetesMonitor not initialized, cannot get status."
+ )
+ return cls._instance.__get_status(job_name)
+
+ @classmethod
+ def status_count(cls) -> Dict[State, int]:
+ """Get a dictionary mapping statuses to the # monitored jobs with each status."""
+ if cls._instance is None:
+ raise ValueError(
+ "LaunchKubernetesMonitor not initialized, cannot get status counts."
+ )
+ return cls._instance.__status_count()
+
+ def __monitor_namespace(
+ self, namespace: str, custom_resource: Optional[CustomResource] = None
+ ) -> None:
+ """Start monitoring a namespaces for resources."""
+ if (namespace, Resources.PODS) not in self._monitor_tasks:
+ self._monitor_tasks[(namespace, Resources.PODS)] = create_named_task(
+ f"monitor_pods_{namespace}",
+ self._monitor_pods,
+ namespace,
+ )
+ # If a custom resource is specified then we will start monitoring
+ # that resource type in the namespace instead of jobs.
+ if custom_resource is not None:
+ if (namespace, custom_resource) not in self._monitor_tasks:
+ self._monitor_tasks[(namespace, custom_resource)] = create_named_task(
+ f"monitor_{custom_resource}_{namespace}",
+ self._monitor_crd,
+ namespace,
+ custom_resource=custom_resource,
+ )
+ else:
+ if (namespace, Resources.JOBS) not in self._monitor_tasks:
+ self._monitor_tasks[(namespace, Resources.JOBS)] = create_named_task(
+ f"monitor_jobs_{namespace}",
+ self._monitor_jobs,
+ namespace,
+ )
+
+ def __get_status(self, job_name: str) -> Status:
+ """Get the status of a job."""
+ if job_name not in self._job_states:
+ return Status("unknown")
+ state = self._job_states[job_name]
+ return state
+
+ def __status_count(self) -> Dict[State, int]:
+ """Get a dictionary mapping statuses to the # monitored jobs with each status."""
+ counts = dict()
+ for _, status in self._job_states.items():
+ state = status.state
+ if state not in counts:
+ counts[state] = 1
+ else:
+ counts[state] += 1
+ return counts
+
+ def _set_status(self, job_name: str, status: Status) -> None:
+ """Set the status of the run."""
+ if self._job_states.get(job_name) != status:
+ self._job_states[job_name] = status
+
+ async def _monitor_pods(self, namespace: str) -> None:
+ """Monitor a namespace for changes."""
+ watcher = SafeWatch(watch.Watch())
+ async for event in watcher.stream(
+ self._core_api.list_namespaced_pod,
+ namespace=namespace,
+ label_selector=self._label_selector,
+ ):
+ obj = event.get("object")
+ job_name = obj.metadata.labels.get("job-name")
+ if job_name is None or not hasattr(obj, "status"):
+ continue
+ if self.__get_status(job_name) in ["finished", "failed"]:
+ continue
+ if obj.status.phase == "Running" or _is_container_creating(obj.status):
+ self._set_status(job_name, Status("running"))
+ elif _is_preempted(obj.status):
+ self._set_status(job_name, Status("preempted"))
+
+ async def _monitor_jobs(self, namespace: str) -> None:
+ """Monitor a namespace for changes."""
+ watcher = SafeWatch(watch.Watch())
+ async for event in watcher.stream(
+ self._batch_api.list_namespaced_job,
+ namespace=namespace,
+ label_selector=self._label_selector,
+ ):
+ obj = event.get("object")
+ job_name = obj.metadata.name
+
+ if obj.status.succeeded == 1:
+ self._set_status(job_name, Status("finished"))
+ elif obj.status.failed is not None and obj.status.failed >= 1:
+ self._set_status(job_name, Status("failed"))
+
+ # If the job is deleted and we haven't seen a terminal state
+ # then we will consider the job failed.
+ if event.get("type") == "DELETED":
+ if self._job_states.get(job_name) != Status("finished"):
+ self._set_status(job_name, Status("failed"))
+
+ async def _monitor_crd(
+ self, namespace: str, custom_resource: CustomResource
+ ) -> None:
+ """Monitor a namespace for changes."""
+ watcher = SafeWatch(watch.Watch())
+ async for event in watcher.stream(
+ self._custom_api.list_namespaced_custom_object,
+ namespace=namespace,
+ plural=custom_resource.plural,
+ group=custom_resource.group,
+ version=custom_resource.version,
+ label_selector=self._label_selector, # TODO: Label selector doesn't work for CRDs.
+ ):
+ object = event.get("object")
+ name = object.get("metadata", dict()).get("name")
+ status = object.get("status")
+ state = None
+ if status is None:
+ continue
+ replicated_jobs_status = status.get("ReplicatedJobsStatus")
+ if isinstance(replicated_jobs_status, dict):
+ state = _state_from_replicated_status(replicated_jobs_status)
+ state_dict = status.get("state")
+ if isinstance(state_dict, dict):
+ phase = state_dict.get("phase")
+ if phase:
+ state = CRD_STATE_DICT.get(phase.lower())
+ else:
+ conditions = status.get("conditions")
+ if isinstance(conditions, list):
+ state = _state_from_conditions(conditions)
+ else:
+ # This should never happen.
+ _logger.warning(
+ f"Unexpected conditions type {type(conditions)} "
+ f"for CRD watcher in {namespace}"
+ )
+ if state is None:
+ continue
+ status = Status(state)
+ self._set_status(name, status)
+
+
+class SafeWatch:
+ """Wrapper for the kubernetes watch class that can recover in more situations."""
+
+ def __init__(self, watcher: watch.Watch) -> None:
+ """Initialize the SafeWatch."""
+ self._watcher = watcher
+ self._last_seen_resource_version: Optional[str] = None
+ self._stopped = False
+
+ async def stream(self, func: Any, *args: Any, **kwargs: Any) -> Any:
+ """Stream the watcher.
+
+ This method will automatically resume the stream if it breaks. It will
+ also save the resource version so that the stream can be resumed from
+ the last seen resource version.
+ """
+ while True:
+ try:
+ async for event in self._watcher.stream(
+ func, *args, **kwargs, timeout_seconds=30
+ ):
+ if self._stopped:
+ break
+ # Save the resource version so that we can resume the stream
+ # if it breaks.
+ object = event.get("object")
+ if isinstance(object, dict):
+ self._last_seen_resource_version = object.get(
+ "metadata", dict()
+ ).get("resourceVersion")
+ else:
+ self._last_seen_resource_version = (
+ object.metadata.resource_version
+ )
+ kwargs["resource_version"] = self._last_seen_resource_version
+ yield event
+ # If stream ends after stop just break
+ if self._stopped:
+ break
+ except urllib3.exceptions.ProtocolError as e:
+ wandb.termwarn(f"Broken event stream: {e}, attempting to recover")
+ except ApiException as e:
+ if e.status == 410:
+ # If resource version is too old we need to start over.
+ del kwargs["resource_version"]
+ self._last_seen_resource_version = None
+ except Exception as E:
+ wandb.termerror(
+ f"Unknown exception in event stream: {E}, attempting to recover"
+ )
diff --git a/wandb/sdk/launch/runner/kubernetes_runner.py b/wandb/sdk/launch/runner/kubernetes_runner.py
new file mode 100644
index 0000000000000000000000000000000000000000..69d3c39e58a43a57cffd0e3955b7b37eea7d23f4
--- /dev/null
+++ b/wandb/sdk/launch/runner/kubernetes_runner.py
@@ -0,0 +1,874 @@
+"""Implementation of KubernetesRunner class for wandb launch."""
+import asyncio
+import base64
+import json
+import logging
+import os
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
+
+import yaml
+
+import wandb
+from wandb.apis.internal import Api
+from wandb.sdk.launch.agent.agent import LaunchAgent
+from wandb.sdk.launch.environment.abstract import AbstractEnvironment
+from wandb.sdk.launch.registry.abstract import AbstractRegistry
+from wandb.sdk.launch.registry.azure_container_registry import AzureContainerRegistry
+from wandb.sdk.launch.registry.local_registry import LocalRegistry
+from wandb.sdk.launch.runner.abstract import Status
+from wandb.sdk.launch.runner.kubernetes_monitor import (
+ WANDB_K8S_LABEL_AGENT,
+ WANDB_K8S_LABEL_MONITOR,
+ WANDB_K8S_RUN_ID,
+ CustomResource,
+ LaunchKubernetesMonitor,
+)
+from wandb.util import get_module
+
+from .._project_spec import EntryPoint, LaunchProject
+from ..builder.build import get_env_vars_dict
+from ..errors import LaunchError
+from ..utils import (
+ LOG_PREFIX,
+ MAX_ENV_LENGTHS,
+ PROJECT_SYNCHRONOUS,
+ get_kube_context_and_api_client,
+ make_name_dns_safe,
+)
+from .abstract import AbstractRun, AbstractRunner
+
+get_module(
+ "kubernetes_asyncio",
+ required="Kubernetes runner requires the kubernetes package. Please install it with `pip install wandb[launch]`.",
+)
+
+import kubernetes_asyncio # type: ignore # noqa: E402
+from kubernetes_asyncio import client # noqa: E402
+from kubernetes_asyncio.client.api.batch_v1_api import ( # type: ignore # noqa: E402
+ BatchV1Api,
+)
+from kubernetes_asyncio.client.api.core_v1_api import ( # type: ignore # noqa: E402
+ CoreV1Api,
+)
+from kubernetes_asyncio.client.api.custom_objects_api import ( # type: ignore # noqa: E402
+ CustomObjectsApi,
+)
+from kubernetes_asyncio.client.models.v1_secret import ( # type: ignore # noqa: E402
+ V1Secret,
+)
+from kubernetes_asyncio.client.rest import ApiException # type: ignore # noqa: E402
+
+TIMEOUT = 5
+
+_logger = logging.getLogger(__name__)
+
+
+class KubernetesSubmittedRun(AbstractRun):
+ """Wrapper for a launched run on Kubernetes."""
+
+ def __init__(
+ self,
+ batch_api: "BatchV1Api",
+ core_api: "CoreV1Api",
+ name: str,
+ namespace: Optional[str] = "default",
+ secret: Optional["V1Secret"] = None,
+ ) -> None:
+ """Initialize a KubernetesSubmittedRun.
+
+ Other implementations of the AbstractRun interface poll on the run
+ when `get_status` is called, but KubernetesSubmittedRun uses
+ Kubernetes watch streams to update the run status. One thread handles
+ events from the job object and another thread handles events from the
+ rank 0 pod. These threads updated the `_status` attributed of the
+ KubernetesSubmittedRun object. When `get_status` is called, the
+ `_status` attribute is returned.
+
+ Arguments:
+ batch_api: Kubernetes BatchV1Api object.
+ core_api: Kubernetes CoreV1Api object.
+ name: Name of the job.
+ namespace: Kubernetes namespace.
+ secret: Kubernetes secret.
+
+ Returns:
+ None.
+ """
+ self.batch_api = batch_api
+ self.core_api = core_api
+ self.name = name
+ self.namespace = namespace
+ self._fail_count = 0
+ self.secret = secret
+
+ @property
+ def id(self) -> str:
+ """Return the run id."""
+ return self.name
+
+ async def get_logs(self) -> Optional[str]:
+ try:
+ pods = await self.core_api.list_namespaced_pod(
+ label_selector=f"job-name={self.name}", namespace=self.namespace
+ )
+ pod_names = [pi.metadata.name for pi in pods.items]
+ if not pod_names:
+ wandb.termwarn(f"Found no pods for kubernetes job: {self.name}")
+ return None
+ logs = await self.core_api.read_namespaced_pod_log(
+ name=pod_names[0], namespace=self.namespace
+ )
+ if logs:
+ return str(logs)
+ else:
+ wandb.termwarn(f"No logs for kubernetes pod(s): {pod_names}")
+ return None
+ except Exception as e:
+ wandb.termerror(f"{LOG_PREFIX}Failed to get pod logs: {e}")
+ return None
+
+ async def wait(self) -> bool:
+ """Wait for the run to finish.
+
+ Returns:
+ True if the run finished successfully, False otherwise.
+ """
+ while True:
+ status = await self.get_status()
+ wandb.termlog(f"{LOG_PREFIX}Job {self.name} status: {status.state}")
+ if status.state in ["finished", "failed", "preempted"]:
+ break
+ await asyncio.sleep(5)
+
+ await self._delete_secret()
+ return (
+ status.state == "finished"
+ ) # todo: not sure if this (copied from aws runner) is the right approach? should we return false on failure
+
+ async def get_status(self) -> Status:
+ status = LaunchKubernetesMonitor.get_status(self.name)
+ if status in ["stopped", "failed", "finished", "preempted"]:
+ await self._delete_secret()
+ return status
+
+ async def cancel(self) -> None:
+ """Cancel the run."""
+ try:
+ await self.batch_api.delete_namespaced_job(
+ namespace=self.namespace,
+ name=self.name,
+ )
+ await self._delete_secret()
+ except ApiException as e:
+ raise LaunchError(
+ f"Failed to delete Kubernetes Job {self.name} in namespace {self.namespace}: {str(e)}"
+ ) from e
+
+ async def _delete_secret(self) -> None:
+ # Cleanup secret if not running in a helm-managed context
+ if not os.environ.get("WANDB_RELEASE_NAME") and self.secret:
+ await self.core_api.delete_namespaced_secret(
+ name=self.secret.metadata.name,
+ namespace=self.secret.metadata.namespace,
+ )
+ self.secret = None
+
+
+class CrdSubmittedRun(AbstractRun):
+ """Run submitted to a CRD backend, e.g. Volcano."""
+
+ def __init__(
+ self,
+ group: str,
+ version: str,
+ plural: str,
+ name: str,
+ namespace: str,
+ core_api: CoreV1Api,
+ custom_api: CustomObjectsApi,
+ ) -> None:
+ """Create a run object for tracking the progress of a CRD.
+
+ Arguments:
+ group: The API group of the CRD.
+ version: The API version of the CRD.
+ plural: The plural name of the CRD.
+ name: The name of the CRD instance.
+ namespace: The namespace of the CRD instance.
+ core_api: The Kubernetes core API client.
+ custom_api: The Kubernetes custom object API client.
+
+ Raises:
+ LaunchError: If the CRD instance does not exist.
+ """
+ self.group = group
+ self.version = version
+ self.plural = plural
+ self.name = name
+ self.namespace = namespace
+ self.core_api = core_api
+ self.custom_api = custom_api
+ self._fail_count = 0
+
+ @property
+ def id(self) -> str:
+ """Get the name of the custom object."""
+ return self.name
+
+ async def get_logs(self) -> Optional[str]:
+ """Get logs for custom object."""
+ # TODO: test more carefully once we release multi-node support
+ logs: Dict[str, Optional[str]] = {}
+ try:
+ pods = await self.core_api.list_namespaced_pod(
+ label_selector=f"wandb/run-id={self.name}", namespace=self.namespace
+ )
+ pod_names = [pi.metadata.name for pi in pods.items]
+ for pod_name in pod_names:
+ logs[pod_name] = await self.core_api.read_namespaced_pod_log(
+ name=pod_name, namespace=self.namespace
+ )
+ except ApiException as e:
+ wandb.termwarn(f"Failed to get logs for {self.name}: {str(e)}")
+ return None
+ if not logs:
+ return None
+ logs_as_array = [f"Pod {pod_name}:\n{log}" for pod_name, log in logs.items()]
+ return "\n".join(logs_as_array)
+
+ async def get_status(self) -> Status:
+ """Get status of custom object."""
+ return LaunchKubernetesMonitor.get_status(self.name)
+
+ async def cancel(self) -> None:
+ """Cancel the custom object."""
+ try:
+ await self.custom_api.delete_namespaced_custom_object(
+ group=self.group,
+ version=self.version,
+ namespace=self.namespace,
+ plural=self.plural,
+ name=self.name,
+ )
+ except ApiException as e:
+ raise LaunchError(
+ f"Failed to delete CRD {self.name} in namespace {self.namespace}: {str(e)}"
+ ) from e
+
+ async def wait(self) -> bool:
+ """Wait for this custom object to finish running."""
+ while True:
+ status = await self.get_status()
+ wandb.termlog(f"{LOG_PREFIX}Job {self.name} status: {status}")
+ if status.state in ["finished", "failed", "preempted"]:
+ return status.state == "finished"
+ await asyncio.sleep(5)
+
+
+class KubernetesRunner(AbstractRunner):
+ """Launches runs onto kubernetes."""
+
+ def __init__(
+ self,
+ api: Api,
+ backend_config: Dict[str, Any],
+ environment: AbstractEnvironment,
+ registry: AbstractRegistry,
+ ) -> None:
+ """Create a Kubernetes runner.
+
+ Arguments:
+ api: The API client object.
+ backend_config: The backend configuration.
+ environment: The environment to launch runs into.
+
+ Raises:
+ LaunchError: If the Kubernetes configuration is invalid.
+ """
+ super().__init__(api, backend_config)
+ self.environment = environment
+ self.registry = registry
+
+ def get_namespace(
+ self, resource_args: Dict[str, Any], context: Dict[str, Any]
+ ) -> str:
+ """Get the namespace to launch into.
+
+ Arguments:
+ resource_args: The resource args to launch.
+ context: The k8s config context.
+
+ Returns:
+ The namespace to launch into.
+ """
+ default_namespace = (
+ context["context"].get("namespace", "default") if context else "default"
+ )
+ return ( # type: ignore[no-any-return]
+ resource_args.get("metadata", {}).get("namespace")
+ or resource_args.get(
+ "namespace"
+ ) # continue support for malformed namespace
+ or self.backend_config.get("runner", {}).get("namespace")
+ or default_namespace
+ )
+
+ async def _inject_defaults(
+ self,
+ resource_args: Dict[str, Any],
+ launch_project: LaunchProject,
+ image_uri: str,
+ namespace: str,
+ core_api: "CoreV1Api",
+ ) -> Tuple[Dict[str, Any], Optional["V1Secret"]]:
+ """Apply our default values, return job dict and api key secret.
+
+ Arguments:
+ resource_args (Dict[str, Any]): The resource args to launch.
+ launch_project (LaunchProject): The launch project.
+ builder (Optional[AbstractBuilder]): The builder.
+ namespace (str): The namespace.
+ core_api (CoreV1Api): The core api.
+
+ Returns:
+ Tuple[Dict[str, Any], Optional["V1Secret"]]: The resource args and api key secret.
+ """
+ job: Dict[str, Any] = {
+ "apiVersion": "batch/v1",
+ "kind": "Job",
+ }
+ job.update(resource_args)
+
+ job_metadata: Dict[str, Any] = job.get("metadata", {})
+ job_spec: Dict[str, Any] = {"backoffLimit": 0, "ttlSecondsAfterFinished": 60}
+ job_spec.update(job.get("spec", {}))
+ pod_template: Dict[str, Any] = job_spec.get("template", {})
+ pod_spec: Dict[str, Any] = {"restartPolicy": "Never"}
+ pod_spec.update(pod_template.get("spec", {}))
+ containers: List[Dict[str, Any]] = pod_spec.get("containers", [{}])
+
+ # Add labels to job metadata
+ job_metadata.setdefault("labels", {})
+ job_metadata["labels"][WANDB_K8S_RUN_ID] = launch_project.run_id
+ job_metadata["labels"][WANDB_K8S_LABEL_MONITOR] = "true"
+ if LaunchAgent.initialized():
+ job_metadata["labels"][WANDB_K8S_LABEL_AGENT] = LaunchAgent.name()
+ # name precedence: name in spec > generated name
+ if not job_metadata.get("name"):
+ job_metadata["generateName"] = make_name_dns_safe(
+ f"launch-{launch_project.target_entity}-{launch_project.target_project}-"
+ )
+
+ for i, cont in enumerate(containers):
+ if "name" not in cont:
+ cont["name"] = cont.get("name", "launch" + str(i))
+ if "securityContext" not in cont:
+ cont["securityContext"] = {
+ "allowPrivilegeEscalation": False,
+ "capabilities": {"drop": ["ALL"]},
+ "seccompProfile": {"type": "RuntimeDefault"},
+ }
+
+ entry_point = (
+ launch_project.override_entrypoint
+ or launch_project.get_single_entry_point()
+ )
+ if launch_project.docker_image:
+ # dont specify run id if user provided image, could have multiple runs
+ containers[0]["image"] = image_uri
+ # TODO: handle secret pulling image from registry
+ elif not any(["image" in cont for cont in containers]):
+ assert entry_point is not None
+ # in the non instance case we need to make an imagePullSecret
+ # so the new job can pull the image
+ containers[0]["image"] = image_uri
+ secret = await maybe_create_imagepull_secret(
+ core_api, self.registry, launch_project.run_id, namespace
+ )
+ if secret is not None:
+ pod_spec["imagePullSecrets"] = [
+ {"name": f"regcred-{launch_project.run_id}"}
+ ]
+
+ inject_entrypoint_and_args(
+ containers,
+ entry_point,
+ launch_project.override_args,
+ launch_project.override_entrypoint is not None,
+ )
+
+ env_vars = get_env_vars_dict(
+ launch_project, self._api, MAX_ENV_LENGTHS[self.__class__.__name__]
+ )
+ api_key_secret = None
+ for cont in containers:
+ # Add our env vars to user supplied env vars
+ env = cont.get("env") or []
+ for key, value in env_vars.items():
+ if (
+ key == "WANDB_API_KEY"
+ and value
+ and (
+ LaunchAgent.initialized()
+ or self.backend_config[PROJECT_SYNCHRONOUS]
+ )
+ ):
+ # Override API key with secret. TODO: Do the same for other runners
+ release_name = os.environ.get("WANDB_RELEASE_NAME")
+ secret_name = "wandb-api-key"
+ if release_name:
+ secret_name += f"-{release_name}"
+ else:
+ secret_name += f"-{launch_project.run_id}"
+
+ api_key_secret = await ensure_api_key_secret(
+ core_api, secret_name, namespace, value
+ )
+ env.append(
+ {
+ "name": key,
+ "valueFrom": {
+ "secretKeyRef": {
+ "name": secret_name,
+ "key": "password",
+ }
+ },
+ }
+ )
+ else:
+ env.append({"name": key, "value": value})
+ cont["env"] = env
+
+ pod_spec["containers"] = containers
+ pod_template["spec"] = pod_spec
+ job_spec["template"] = pod_template
+ job["spec"] = job_spec
+ job["metadata"] = job_metadata
+
+ add_label_to_pods(
+ job,
+ WANDB_K8S_LABEL_MONITOR,
+ "true",
+ )
+
+ # Add wandb.ai/agent: current agent label on all pods
+ if LaunchAgent.initialized():
+ add_label_to_pods(
+ job,
+ WANDB_K8S_LABEL_AGENT,
+ LaunchAgent.name(),
+ )
+
+ return job, api_key_secret
+
+ async def run(
+ self, launch_project: LaunchProject, image_uri: str
+ ) -> Optional[AbstractRun]: # noqa: C901
+ """Execute a launch project on Kubernetes.
+
+ Arguments:
+ launch_project: The launch project to execute.
+ builder: The builder to use to build the image.
+
+ Returns:
+ The run object if the run was successful, otherwise None.
+ """
+ await LaunchKubernetesMonitor.ensure_initialized()
+ resource_args = launch_project.fill_macros(image_uri).get("kubernetes", {})
+ if not resource_args:
+ wandb.termlog(
+ f"{LOG_PREFIX}Note: no resource args specified. Add a "
+ "Kubernetes yaml spec or other options in a json file "
+ "with --resource-args <json>."
+ )
+ _logger.info(f"Running Kubernetes job with resource args: {resource_args}")
+
+ context, api_client = await get_kube_context_and_api_client(
+ kubernetes_asyncio, resource_args
+ )
+
+ # If the user specified an alternate api, we need will execute this
+ # run by creating a custom object.
+ api_version = resource_args.get("apiVersion", "batch/v1")
+
+ if api_version not in ["batch/v1", "batch/v1beta1"]:
+ env_vars = get_env_vars_dict(
+ launch_project, self._api, MAX_ENV_LENGTHS[self.__class__.__name__]
+ )
+ # Crawl the resource args and add our env vars to the containers.
+ add_wandb_env(resource_args, env_vars)
+
+ # Add our labels to the resource args. This is necessary for the
+ # agent to find the custom object later on.
+ resource_args["metadata"] = resource_args.get("metadata", {})
+ resource_args["metadata"]["labels"] = resource_args["metadata"].get(
+ "labels", {}
+ )
+ resource_args["metadata"]["labels"][WANDB_K8S_LABEL_MONITOR] = "true"
+
+ # Crawl the resource arsg and add our labels to the pods. This is
+ # necessary for the agent to find the pods later on.
+ add_label_to_pods(
+ resource_args,
+ WANDB_K8S_LABEL_MONITOR,
+ "true",
+ )
+
+ # Add wandb.ai/agent: current agent label on all pods
+ if LaunchAgent.initialized():
+ add_label_to_pods(
+ resource_args,
+ WANDB_K8S_LABEL_MONITOR,
+ LaunchAgent.name(),
+ )
+ resource_args["metadata"]["labels"][
+ WANDB_K8S_LABEL_AGENT
+ ] = LaunchAgent.name()
+
+ overrides = {}
+ if launch_project.override_args:
+ overrides["args"] = launch_project.override_args
+ if launch_project.override_entrypoint:
+ overrides["command"] = launch_project.override_entrypoint.command
+ add_entrypoint_args_overrides(
+ resource_args,
+ overrides,
+ )
+ api = client.CustomObjectsApi(api_client)
+ # Infer the attributes of a custom object from the apiVersion and/or
+ # a kind: attribute in the resource args.
+ namespace = self.get_namespace(resource_args, context)
+ group, version, *_ = api_version.split("/")
+ group = resource_args.get("group", group)
+ version = resource_args.get("version", version)
+ kind = resource_args.get("kind", version)
+ plural = f"{kind.lower()}s"
+ custom_resource = CustomResource(
+ group=group,
+ version=version,
+ plural=plural,
+ )
+ LaunchKubernetesMonitor.monitor_namespace(
+ namespace, custom_resource=custom_resource
+ )
+
+ try:
+ response = await api.create_namespaced_custom_object(
+ group=group,
+ version=version,
+ namespace=namespace,
+ plural=plural,
+ body=resource_args,
+ )
+ except ApiException as e:
+ body = json.loads(e.body)
+ body_yaml = yaml.dump(body)
+ raise LaunchError(
+ f"Error creating CRD of kind {kind}: {e.status} {e.reason}\n{body_yaml}"
+ ) from e
+ name = response.get("metadata", {}).get("name")
+ _logger.info(f"Created {kind} {response['metadata']['name']}")
+ submitted_run = CrdSubmittedRun(
+ name=name,
+ group=group,
+ version=version,
+ namespace=namespace,
+ plural=plural,
+ core_api=client.CoreV1Api(api_client),
+ custom_api=api,
+ )
+ if self.backend_config[PROJECT_SYNCHRONOUS]:
+ await submitted_run.wait()
+ return submitted_run
+
+ batch_api = kubernetes_asyncio.client.BatchV1Api(api_client)
+ core_api = kubernetes_asyncio.client.CoreV1Api(api_client)
+ namespace = self.get_namespace(resource_args, context)
+ job, secret = await self._inject_defaults(
+ resource_args, launch_project, image_uri, namespace, core_api
+ )
+ msg = "Creating Kubernetes job"
+ if "name" in resource_args:
+ msg += f": {resource_args['name']}"
+ _logger.info(msg)
+ try:
+ response = await kubernetes_asyncio.utils.create_from_dict(
+ api_client, job, namespace=namespace
+ )
+ except kubernetes_asyncio.utils.FailToCreateError as e:
+ for exc in e.api_exceptions:
+ resp = json.loads(exc.body)
+ msg = resp.get("message")
+ code = resp.get("code")
+ raise LaunchError(
+ f"Failed to create Kubernetes job for run {launch_project.run_id} ({code} {exc.reason}): {msg}"
+ )
+ except Exception as e:
+ raise LaunchError(
+ f"Unexpected exception when creating Kubernetes job: {str(e)}\n"
+ )
+ job_response = response[0]
+ job_name = job_response.metadata.name
+ LaunchKubernetesMonitor.monitor_namespace(namespace)
+ submitted_job = KubernetesSubmittedRun(
+ batch_api, core_api, job_name, namespace, secret
+ )
+ if self.backend_config[PROJECT_SYNCHRONOUS]:
+ await submitted_job.wait()
+
+ return submitted_job
+
+
+def inject_entrypoint_and_args(
+ containers: List[dict],
+ entry_point: Optional[EntryPoint],
+ override_args: List[str],
+ should_override_entrypoint: bool,
+) -> None:
+ """Inject the entrypoint and args into the containers.
+
+ Arguments:
+ containers: The containers to inject the entrypoint and args into.
+ entry_point: The entrypoint to inject.
+ override_args: The args to inject.
+ should_override_entrypoint: Whether to override the entrypoint.
+
+ Returns:
+ None
+ """
+ for i in range(len(containers)):
+ if override_args:
+ containers[i]["args"] = override_args
+ if entry_point and (
+ not containers[i].get("command") or should_override_entrypoint
+ ):
+ containers[i]["command"] = entry_point.command
+
+
+async def ensure_api_key_secret(
+ core_api: "CoreV1Api",
+ secret_name: str,
+ namespace: str,
+ api_key: str,
+) -> "V1Secret":
+ """Create a secret containing a user's wandb API key.
+
+ Arguments:
+ core_api: The Kubernetes CoreV1Api object.
+ secret_name: The name to use for the secret.
+ namespace: The namespace to create the secret in.
+ api_key: The user's wandb API key
+
+ Returns:
+ The created secret
+ """
+ secret_data = {"password": base64.b64encode(api_key.encode()).decode()}
+ labels = {"wandb.ai/created-by": "launch-agent"}
+ secret = client.V1Secret(
+ data=secret_data,
+ metadata=client.V1ObjectMeta(
+ name=secret_name, namespace=namespace, labels=labels
+ ),
+ kind="Secret",
+ type="kubernetes.io/basic-auth",
+ )
+
+ try:
+ try:
+ return await core_api.create_namespaced_secret(namespace, secret)
+ except ApiException as e:
+ # 409 = conflict = secret already exists
+ if e.status == 409:
+ existing_secret = await core_api.read_namespaced_secret(
+ name=secret_name, namespace=namespace
+ )
+ if existing_secret.data != secret_data:
+ # If it's a previous secret made by launch agent, clean it up
+ if (
+ existing_secret.metadata.labels.get("wandb.ai/created-by")
+ == "launch-agent"
+ ):
+ await core_api.delete_namespaced_secret(
+ name=secret_name, namespace=namespace
+ )
+ return await core_api.create_namespaced_secret(
+ namespace, secret
+ )
+ else:
+ raise LaunchError(
+ f"Kubernetes secret already exists in namespace {namespace} with incorrect data: {secret_name}"
+ )
+ return existing_secret
+ raise
+ except Exception as e:
+ raise LaunchError(
+ f"Exception when ensuring Kubernetes API key secret: {str(e)}\n"
+ )
+
+
+async def maybe_create_imagepull_secret(
+ core_api: "CoreV1Api",
+ registry: AbstractRegistry,
+ run_id: str,
+ namespace: str,
+) -> Optional["V1Secret"]:
+ """Create a secret for pulling images from a private registry.
+
+ Arguments:
+ core_api: The Kubernetes CoreV1Api object.
+ registry: The registry to pull from.
+ run_id: The run id.
+ namespace: The namespace to create the secret in.
+
+ Returns:
+ A secret if one was created, otherwise None.
+ """
+ secret = None
+ if isinstance(registry, LocalRegistry) or isinstance(
+ registry, AzureContainerRegistry
+ ):
+ # Secret not required
+ return None
+ uname, token = await registry.get_username_password()
+ creds_info = {
+ "auths": {
+ registry.uri: {
+ "auth": base64.b64encode(f"{uname}:{token}".encode()).decode(),
+ # need an email but the use is deprecated
+ "email": "deprecated@wandblaunch.com",
+ }
+ }
+ }
+ secret_data = {
+ ".dockerconfigjson": base64.b64encode(json.dumps(creds_info).encode()).decode()
+ }
+ secret = client.V1Secret(
+ data=secret_data,
+ metadata=client.V1ObjectMeta(name=f"regcred-{run_id}", namespace=namespace),
+ kind="Secret",
+ type="kubernetes.io/dockerconfigjson",
+ )
+ try:
+ try:
+ return await core_api.create_namespaced_secret(namespace, secret)
+ except ApiException as e:
+ # 409 = conflict = secret already exists
+ if e.status == 409:
+ return await core_api.read_namespaced_secret(
+ name=f"regcred-{run_id}", namespace=namespace
+ )
+ raise
+ except Exception as e:
+ raise LaunchError(f"Exception when creating Kubernetes secret: {str(e)}\n")
+
+
+def yield_containers(root: Any) -> Iterator[dict]:
+ """Yield all container specs in a manifest.
+
+ Recursively traverses the manifest and yields all container specs. Container
+ specs are identified by the presence of a "containers" key in the value.
+ """
+ if isinstance(root, dict):
+ for k, v in root.items():
+ if k == "containers":
+ if isinstance(v, list):
+ yield from v
+ elif isinstance(v, (dict, list)):
+ yield from yield_containers(v)
+ elif isinstance(root, list):
+ for item in root:
+ yield from yield_containers(item)
+
+
+def add_wandb_env(root: Union[dict, list], env_vars: Dict[str, str]) -> None:
+ """Injects wandb environment variables into specs.
+
+ Recursively walks the spec and injects the environment variables into
+ every container spec. Containers are identified by the "containers" key.
+
+ This function treats the WANDB_RUN_ID and WANDB_GROUP_ID environment variables
+ specially. If they are present in the spec, they will be overwritten. If a setting
+ for WANDB_RUN_ID is provided in env_vars, then that environment variable will only be
+ set in the first container modified by this function.
+
+ Arguments:
+ root: The spec to modify.
+ env_vars: The environment variables to inject.
+
+ Returns: None.
+ """
+ for cont in yield_containers(root):
+ env = cont.setdefault("env", [])
+ env.extend([{"name": key, "value": value} for key, value in env_vars.items()])
+ cont["env"] = env
+ # After we have set WANDB_RUN_ID once, we don't want to set it again
+ if "WANDB_RUN_ID" in env_vars:
+ env_vars.pop("WANDB_RUN_ID")
+
+
+def yield_pods(manifest: Any) -> Iterator[dict]:
+ """Yield all pod specs in a manifest.
+
+ Recursively traverses the manifest and yields all pod specs. Pod specs are
+ identified by the presence of a "spec" key with a "containers" key in the
+ value.
+ """
+ if isinstance(manifest, list):
+ for item in manifest:
+ yield from yield_pods(item)
+ elif isinstance(manifest, dict):
+ if "spec" in manifest and "containers" in manifest["spec"]:
+ yield manifest
+ for value in manifest.values():
+ if isinstance(value, (dict, list)):
+ yield from yield_pods(value)
+
+
+def add_label_to_pods(
+ manifest: Union[dict, list], label_key: str, label_value: str
+) -> None:
+ """Add a label to all pod specs in a manifest.
+
+ Recursively traverses the manifest and adds the label to all pod specs.
+ Pod specs are identified by the presence of a "spec" key with a "containers"
+ key in the value.
+
+ Arguments:
+ manifest: The manifest to modify.
+ label_key: The label key to add.
+ label_value: The label value to add.
+
+ Returns: None.
+ """
+ for pod in yield_pods(manifest):
+ metadata = pod.setdefault("metadata", {})
+ labels = metadata.setdefault("labels", {})
+ labels[label_key] = label_value
+
+
+def add_entrypoint_args_overrides(manifest: Union[dict, list], overrides: dict) -> None:
+ """Add entrypoint and args overrides to all containers in a manifest.
+
+ Recursively traverses the manifest and adds the entrypoint and args overrides
+ to all containers. Containers are identified by the presence of a "spec" key
+ with a "containers" key in the value.
+
+ Arguments:
+ manifest: The manifest to modify.
+ overrides: Dictionary with args and entrypoint keys.
+
+ Returns: None.
+ """
+ if isinstance(manifest, list):
+ for item in manifest:
+ add_entrypoint_args_overrides(item, overrides)
+ elif isinstance(manifest, dict):
+ if "spec" in manifest and "containers" in manifest["spec"]:
+ containers = manifest["spec"]["containers"]
+ for container in containers:
+ if "command" in overrides:
+ container["command"] = overrides["command"]
+ if "args" in overrides:
+ container["args"] = overrides["args"]
+ for value in manifest.values():
+ add_entrypoint_args_overrides(value, overrides)
diff --git a/wandb/sdk/launch/runner/local_container.py b/wandb/sdk/launch/runner/local_container.py
new file mode 100644
index 0000000000000000000000000000000000000000..66d189733419cdd96d7e8527237069533d8a2979
--- /dev/null
+++ b/wandb/sdk/launch/runner/local_container.py
@@ -0,0 +1,294 @@
+import asyncio
+import logging
+import os
+import shlex
+import subprocess
+import sys
+import threading
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+import wandb
+from wandb.sdk.launch.environment.abstract import AbstractEnvironment
+from wandb.sdk.launch.registry.abstract import AbstractRegistry
+
+from .._project_spec import LaunchProject
+from ..builder.build import get_env_vars_dict
+from ..errors import LaunchError
+from ..utils import (
+ LOG_PREFIX,
+ MAX_ENV_LENGTHS,
+ PROJECT_SYNCHRONOUS,
+ _is_wandb_dev_uri,
+ _is_wandb_local_uri,
+ docker_image_exists,
+ event_loop_thread_exec,
+ pull_docker_image,
+ sanitize_wandb_api_key,
+)
+from .abstract import AbstractRun, AbstractRunner, Status
+
+if TYPE_CHECKING:
+ from wandb.apis.internal import Api
+
+_logger = logging.getLogger(__name__)
+
+
+class LocalSubmittedRun(AbstractRun):
+ """Instance of ``AbstractRun`` corresponding to a subprocess launched to run an entry point command locally."""
+
+ def __init__(self) -> None:
+ super().__init__()
+ self._command_proc: Optional[subprocess.Popen] = None
+ self._stdout: Optional[str] = None
+ self._terminate_flag: bool = False
+ self._thread: Optional[threading.Thread] = None
+
+ def set_command_proc(self, command_proc: subprocess.Popen) -> None:
+ self._command_proc = command_proc
+
+ def set_thread(self, thread: threading.Thread) -> None:
+ self._thread = thread
+
+ @property
+ def id(self) -> Optional[str]:
+ if self._command_proc is None:
+ return None
+ return str(self._command_proc.pid)
+
+ async def wait(self) -> bool:
+ assert self._thread is not None
+ # if command proc is not set
+ # wait for thread to set it
+ if self._command_proc is None:
+ while self._thread.is_alive():
+ await asyncio.sleep(5)
+ # command proc can be updated by another thread
+ if self._command_proc is not None:
+ break # type: ignore # mypy thinks this is unreachable
+ else:
+ return False
+ wait = event_loop_thread_exec(self._command_proc.wait)
+ return int(await wait()) == 0
+
+ async def get_logs(self) -> Optional[str]:
+ return self._stdout
+
+ async def cancel(self) -> None:
+ # thread is set immediately after starting, should always exist
+ assert self._thread is not None
+
+ # cancel called before the thread subprocess has started
+ # indicates to thread to not start command proc if not already started
+ self._terminate_flag = True
+
+ async def get_status(self) -> Status:
+ assert self._thread is not None, "Failed to get status, self._thread = None"
+ if self._command_proc is None:
+ if self._thread.is_alive():
+ return Status("running")
+ return Status("stopped")
+ exit_code = self._command_proc.poll()
+ if exit_code is None:
+ return Status("running")
+ if exit_code == 0:
+ return Status("finished")
+ return Status("failed")
+
+
+class LocalContainerRunner(AbstractRunner):
+ """Runner class, uses a project to create a LocallySubmittedRun."""
+
+ def __init__(
+ self,
+ api: "Api",
+ backend_config: Dict[str, Any],
+ environment: AbstractEnvironment,
+ registry: AbstractRegistry,
+ ) -> None:
+ super().__init__(api, backend_config)
+ self.environment = environment
+ self.registry = registry
+
+ def _populate_docker_args(
+ self, launch_project: LaunchProject, image_uri: str
+ ) -> Dict[str, Any]:
+ docker_args: Dict[str, Any] = launch_project.fill_macros(image_uri).get(
+ "local-container", {}
+ )
+ if _is_wandb_local_uri(self._api.settings("base_url")):
+ if sys.platform == "win32":
+ docker_args["net"] = "host"
+ else:
+ docker_args["network"] = "host"
+ if sys.platform == "linux" or sys.platform == "linux2":
+ docker_args["add-host"] = "host.docker.internal:host-gateway"
+
+ return docker_args
+
+ async def run(
+ self,
+ launch_project: LaunchProject,
+ image_uri: str,
+ ) -> Optional[AbstractRun]:
+ docker_args = self._populate_docker_args(launch_project, image_uri)
+ synchronous: bool = self.backend_config[PROJECT_SYNCHRONOUS]
+
+ env_vars = get_env_vars_dict(
+ launch_project, self._api, MAX_ENV_LENGTHS[self.__class__.__name__]
+ )
+
+ # When running against local port, need to swap to local docker host
+ if (
+ _is_wandb_local_uri(self._api.settings("base_url"))
+ and sys.platform == "darwin"
+ ):
+ _, _, port = self._api.settings("base_url").split(":")
+ env_vars["WANDB_BASE_URL"] = f"http://host.docker.internal:{port}"
+ elif _is_wandb_dev_uri(self._api.settings("base_url")):
+ env_vars["WANDB_BASE_URL"] = "http://host.docker.internal:9001"
+
+ if launch_project.docker_image:
+ if image_uri.endswith(":latest") or not docker_image_exists(image_uri):
+ try:
+ pull_docker_image(image_uri)
+ except Exception as e:
+ wandb.termwarn(f"Error attempting to pull docker image {image_uri}")
+ if not docker_image_exists(image_uri):
+ raise LaunchError(
+ f"Failed to pull docker image {image_uri} with error: {e}"
+ )
+
+ assert launch_project.docker_image == image_uri
+
+ entry_cmd = (
+ launch_project.override_entrypoint.command
+ if launch_project.override_entrypoint is not None
+ else None
+ )
+
+ command_str = " ".join(
+ get_docker_command(
+ image_uri,
+ env_vars,
+ docker_args=docker_args,
+ entry_cmd=entry_cmd,
+ additional_args=launch_project.override_args,
+ )
+ ).strip()
+ sanitized_cmd_str = sanitize_wandb_api_key(command_str)
+ _msg = f"{LOG_PREFIX}Launching run in docker with command: {sanitized_cmd_str}"
+ wandb.termlog(_msg)
+ run = _run_entry_point(command_str, launch_project.project_dir)
+ if synchronous:
+ await run.wait()
+ return run
+
+
+def _run_entry_point(command: str, work_dir: Optional[str]) -> AbstractRun:
+ """Run an entry point command in a subprocess.
+
+ Arguments:
+ command: Entry point command to run
+ work_dir: Working directory in which to run the command
+
+ Returns:
+ An instance of `LocalSubmittedRun`
+ """
+ if work_dir is None:
+ work_dir = os.getcwd()
+ env = os.environ.copy()
+ run = LocalSubmittedRun()
+ thread = threading.Thread(
+ target=_thread_process_runner,
+ args=(run, ["bash", "-c", command], work_dir, env),
+ )
+ run.set_thread(thread)
+ thread.start()
+ return run
+
+
+def _thread_process_runner(
+ run: LocalSubmittedRun, args: List[str], work_dir: str, env: Dict[str, str]
+) -> None:
+ # cancel was called before we started the subprocess
+ if run._terminate_flag:
+ return
+ # TODO: Make this async
+ process = subprocess.Popen(
+ args,
+ close_fds=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ universal_newlines=True,
+ bufsize=1,
+ cwd=work_dir,
+ env=env,
+ )
+ run.set_command_proc(process)
+ run._stdout = ""
+ while True:
+ # the agent thread could set the terminate flag
+ if run._terminate_flag:
+ process.terminate() # type: ignore
+ chunk = os.read(process.stdout.fileno(), 4096) # type: ignore
+ if not chunk:
+ break
+ index = chunk.find(b"\r")
+ decoded_chunk = chunk.decode()
+ if index != -1:
+ run._stdout += decoded_chunk
+ print(chunk.decode(), end="")
+ else:
+ run._stdout += decoded_chunk + "\r"
+ print(chunk.decode(), end="\r")
+
+
+def get_docker_command(
+ image: str,
+ env_vars: Dict[str, str],
+ entry_cmd: Optional[List[str]] = None,
+ docker_args: Optional[Dict[str, Any]] = None,
+ additional_args: Optional[List[str]] = None,
+) -> List[str]:
+ """Construct the docker command using the image and docker args.
+
+ Arguments:
+ image: a Docker image to be run
+ env_vars: a dictionary of environment variables for the command
+ entry_cmd: the entry point command to run
+ docker_args: a dictionary of additional docker args for the command
+ """
+ docker_path = "docker"
+ cmd: List[Any] = [docker_path, "run", "--rm"]
+
+ # hacky handling of env vars, needs to be improved
+ for env_key, env_value in env_vars.items():
+ cmd += ["-e", f"{shlex.quote(env_key)}={shlex.quote(env_value)}"]
+
+ if docker_args:
+ for name, value in docker_args.items():
+ if len(name) == 1:
+ prefix = "-" + shlex.quote(name)
+ else:
+ prefix = "--" + shlex.quote(name)
+ if isinstance(value, list):
+ for v in value:
+ cmd += [prefix, shlex.quote(str(v))]
+ elif isinstance(value, bool) and value:
+ cmd += [prefix]
+ else:
+ cmd += [prefix, shlex.quote(str(value))]
+
+ if entry_cmd:
+ cmd += ["--entrypoint", entry_cmd[0]]
+ cmd += [shlex.quote(image)]
+ if entry_cmd and len(entry_cmd) > 1:
+ cmd += entry_cmd[1:]
+ if additional_args:
+ cmd += additional_args
+ return cmd
+
+
+def join(split_command: List[str]) -> str:
+ """Return a shell-escaped string from *split_command*."""
+ return " ".join(shlex.quote(arg) for arg in split_command)
diff --git a/wandb/sdk/launch/runner/local_process.py b/wandb/sdk/launch/runner/local_process.py
new file mode 100644
index 0000000000000000000000000000000000000000..babb8bf316a899725fef0cd00c1b2b0a88f667c0
--- /dev/null
+++ b/wandb/sdk/launch/runner/local_process.py
@@ -0,0 +1,99 @@
+import logging
+import shlex
+from typing import Any, List, Optional
+
+import wandb
+
+from .._project_spec import LaunchProject, get_entry_point_command
+from ..builder.build import get_env_vars_dict
+from ..errors import LaunchError
+from ..utils import (
+ LOG_PREFIX,
+ MAX_ENV_LENGTHS,
+ PROJECT_SYNCHRONOUS,
+ _is_wandb_uri,
+ download_wandb_python_deps,
+ parse_wandb_uri,
+ sanitize_wandb_api_key,
+ validate_wandb_python_deps,
+)
+from .abstract import AbstractRun, AbstractRunner
+from .local_container import _run_entry_point
+
+_logger = logging.getLogger(__name__)
+
+
+class LocalProcessRunner(AbstractRunner):
+ """Runner class, uses a project to create a LocallySubmittedRun.
+
+ LocalProcessRunner is very similar to a LocalContainerRunner, except it does not
+ run the command inside a docker container. Instead, it runs the
+ command specified as a process directly on the bare metal machine.
+
+ """
+
+ async def run( # type: ignore
+ self,
+ launch_project: LaunchProject,
+ *args,
+ **kwargs,
+ ) -> Optional[AbstractRun]:
+ if args is not None:
+ _msg = f"{LOG_PREFIX}LocalProcessRunner.run received unused args {args}"
+ _logger.warning(_msg)
+ if kwargs is not None:
+ _msg = f"{LOG_PREFIX}LocalProcessRunner.run received unused kwargs {kwargs}"
+ _logger.warning(_msg)
+
+ synchronous: bool = self.backend_config[PROJECT_SYNCHRONOUS]
+ entry_point = (
+ launch_project.override_entrypoint
+ or launch_project.get_single_entry_point()
+ )
+
+ cmd: List[Any] = []
+
+ if launch_project.project_dir is None:
+ raise LaunchError("Launch LocalProcessRunner received empty project dir")
+
+ # Check to make sure local python dependencies match run's requirement.txt
+ if launch_project.uri and _is_wandb_uri(launch_project.uri):
+ source_entity, source_project, run_name = parse_wandb_uri(
+ launch_project.uri
+ )
+ run_requirements_file = download_wandb_python_deps(
+ source_entity,
+ source_project,
+ run_name,
+ self._api,
+ launch_project.project_dir,
+ )
+ validate_wandb_python_deps(
+ run_requirements_file,
+ launch_project.project_dir,
+ )
+ elif launch_project.job:
+ assert launch_project._job_artifact is not None
+ try:
+ validate_wandb_python_deps(
+ "requirements.frozen.txt",
+ launch_project.project_dir,
+ )
+ except Exception:
+ wandb.termwarn("Unable to validate python dependencies")
+ env_vars = get_env_vars_dict(
+ launch_project, self._api, MAX_ENV_LENGTHS[self.__class__.__name__]
+ )
+ for env_key, env_value in env_vars.items():
+ cmd += [f"{shlex.quote(env_key)}={shlex.quote(env_value)}"]
+
+ entry_cmd = get_entry_point_command(entry_point, launch_project.override_args)
+ cmd += entry_cmd
+
+ command_str = " ".join(cmd).strip()
+ _msg = f"{LOG_PREFIX}Launching run as a local-process with command {sanitize_wandb_api_key(command_str)}"
+ wandb.termlog(_msg)
+ run = _run_entry_point(command_str, launch_project.project_dir)
+ if synchronous:
+ await run.wait()
+ return run
diff --git a/wandb/sdk/launch/runner/sagemaker_runner.py b/wandb/sdk/launch/runner/sagemaker_runner.py
new file mode 100644
index 0000000000000000000000000000000000000000..0966d72afc9d19bc0c950fb2dae114170cc81c31
--- /dev/null
+++ b/wandb/sdk/launch/runner/sagemaker_runner.py
@@ -0,0 +1,421 @@
+"""Implementation of the SageMakerRunner class."""
+import asyncio
+import logging
+from typing import Any, Dict, List, Optional, cast
+
+if False:
+ import boto3 # type: ignore
+
+import wandb
+from wandb.apis.internal import Api
+from wandb.sdk.launch.environment.aws_environment import AwsEnvironment
+from wandb.sdk.launch.errors import LaunchError
+
+from .._project_spec import EntryPoint, LaunchProject, get_entry_point_command
+from ..builder.build import get_env_vars_dict
+from ..registry.abstract import AbstractRegistry
+from ..utils import (
+ LOG_PREFIX,
+ MAX_ENV_LENGTHS,
+ PROJECT_SYNCHRONOUS,
+ event_loop_thread_exec,
+ to_camel_case,
+)
+from .abstract import AbstractRun, AbstractRunner, Status
+
+_logger = logging.getLogger(__name__)
+
+
+class SagemakerSubmittedRun(AbstractRun):
+ """Instance of ``AbstractRun`` corresponding to a subprocess launched to run an entry point command on aws sagemaker."""
+
+ def __init__(
+ self,
+ training_job_name: str,
+ client: "boto3.Client",
+ log_client: Optional["boto3.Client"] = None,
+ ) -> None:
+ super().__init__()
+ self.client = client
+ self.log_client = log_client
+ self.training_job_name = training_job_name
+ self._status = Status("running")
+
+ @property
+ def id(self) -> str:
+ return f"sagemaker-{self.training_job_name}"
+
+ async def get_logs(self) -> Optional[str]:
+ if self.log_client is None:
+ return None
+ try:
+ describe_log_streams = event_loop_thread_exec(
+ self.log_client.describe_log_streams
+ )
+ describe_res = await describe_log_streams(
+ logGroupName="/aws/sagemaker/TrainingJobs",
+ logStreamNamePrefix=self.training_job_name,
+ )
+ if len(describe_res["logStreams"]) == 0:
+ wandb.termwarn(
+ f"Failed to get logs for training job: {self.training_job_name}"
+ )
+ return None
+ log_name = describe_res["logStreams"][0]["logStreamName"]
+ get_log_events = event_loop_thread_exec(self.log_client.get_log_events)
+ res = await get_log_events(
+ logGroupName="/aws/sagemaker/TrainingJobs",
+ logStreamName=log_name,
+ )
+ return "\n".join(
+ [f'{event["timestamp"]}:{event["message"]}' for event in res["events"]]
+ )
+ except self.log_client.exceptions.ResourceNotFoundException:
+ wandb.termwarn(
+ f"Failed to get logs for training job: {self.training_job_name}"
+ )
+ return None
+ except Exception as e:
+ wandb.termwarn(
+ f"Failed to handle logs for training job: {self.training_job_name} with error {str(e)}"
+ )
+ return None
+
+ async def wait(self) -> bool:
+ while True:
+ status_state = (await self.get_status()).state
+ wandb.termlog(
+ f"{LOG_PREFIX}Training job {self.training_job_name} status: {status_state}"
+ )
+ if status_state in ["stopped", "failed", "finished"]:
+ break
+ await asyncio.sleep(5)
+ return status_state == "finished"
+
+ async def cancel(self) -> None:
+ # Interrupt child process if it hasn't already exited
+ status = await self.get_status()
+ if status.state == "running":
+ self.client.stop_training_job(TrainingJobName=self.training_job_name)
+ await self.wait()
+
+ async def get_status(self) -> Status:
+ describe_training_job = event_loop_thread_exec(
+ self.client.describe_training_job
+ )
+ job_status = (
+ await describe_training_job(TrainingJobName=self.training_job_name)
+ )["TrainingJobStatus"]
+ if job_status == "Completed" or job_status == "Stopped":
+ self._status = Status("finished")
+ elif job_status == "Failed":
+ self._status = Status("failed")
+ elif job_status == "Stopping":
+ self._status = Status("stopping")
+ elif job_status == "InProgress":
+ self._status = Status("running")
+ return self._status
+
+
+class SageMakerRunner(AbstractRunner):
+ """Runner class, uses a project to create a SagemakerSubmittedRun."""
+
+ def __init__(
+ self,
+ api: Api,
+ backend_config: Dict[str, Any],
+ environment: AwsEnvironment,
+ registry: AbstractRegistry,
+ ) -> None:
+ """Initialize the SagemakerRunner.
+
+ Arguments:
+ api (Api): The API instance.
+ backend_config (Dict[str, Any]): The backend configuration.
+ environment (AwsEnvironment): The AWS environment.
+
+ Raises:
+ LaunchError: If the runner cannot be initialized.
+ """
+ super().__init__(api, backend_config)
+ self.environment = environment
+ self.registry = registry
+
+ async def run(
+ self,
+ launch_project: LaunchProject,
+ image_uri: str,
+ ) -> Optional[AbstractRun]:
+ """Run a project on Amazon Sagemaker.
+
+ Arguments:
+ launch_project (LaunchProject): The project to run.
+
+ Returns:
+ Optional[AbstractRun]: The run instance.
+
+ Raises:
+ LaunchError: If the launch is unsuccessful.
+ """
+ _logger.info("using AWSSagemakerRunner")
+
+ given_sagemaker_args = launch_project.resource_args.get("sagemaker")
+ if given_sagemaker_args is None:
+ raise LaunchError(
+ "No sagemaker args specified. Specify sagemaker args in resource_args"
+ )
+
+ default_output_path = self.backend_config.get("runner", {}).get(
+ "s3_output_path"
+ )
+ if default_output_path is not None and not default_output_path.startswith(
+ "s3://"
+ ):
+ default_output_path = f"s3://{default_output_path}"
+
+ session = await self.environment.get_session()
+ client = await event_loop_thread_exec(session.client)("sts")
+ caller_id = client.get_caller_identity()
+ account_id = caller_id["Account"]
+ _logger.info(f"Using account ID {account_id}")
+ role_arn = get_role_arn(given_sagemaker_args, self.backend_config, account_id)
+
+ # Create a sagemaker client to launch the job.
+ sagemaker_client = session.client("sagemaker")
+ log_client = None
+ try:
+ log_client = session.client("logs")
+ except Exception as e:
+ wandb.termwarn(
+ f"Failed to connect to cloudwatch logs with error {str(e)}, logs will not be available"
+ )
+
+ # if the user provided the image they want to use, use that, but warn it won't have swappable artifacts
+ if (
+ given_sagemaker_args.get("AlgorithmSpecification", {}).get("TrainingImage")
+ is not None
+ ):
+ sagemaker_args = build_sagemaker_args(
+ launch_project,
+ self._api,
+ role_arn,
+ launch_project.override_entrypoint,
+ launch_project.override_args,
+ MAX_ENV_LENGTHS[self.__class__.__name__],
+ given_sagemaker_args.get("AlgorithmSpecification", {}).get(
+ "TrainingImage"
+ ),
+ default_output_path,
+ )
+ _logger.info(
+ f"Launching sagemaker job on user supplied image with args: {sagemaker_args}"
+ )
+ run = await launch_sagemaker_job(
+ launch_project, sagemaker_args, sagemaker_client, log_client
+ )
+ if self.backend_config[PROJECT_SYNCHRONOUS]:
+ await run.wait()
+ return run
+
+ launch_project.fill_macros(image_uri)
+ _logger.info("Connecting to sagemaker client")
+ entry_point = (
+ launch_project.override_entrypoint
+ or launch_project.get_single_entry_point()
+ )
+ command_args = get_entry_point_command(
+ entry_point, launch_project.override_args
+ )
+ if command_args:
+ command_str = " ".join(command_args)
+ wandb.termlog(
+ f"{LOG_PREFIX}Launching run on sagemaker with entrypoint: {command_str}"
+ )
+ else:
+ wandb.termlog(
+ f"{LOG_PREFIX}Launching run on sagemaker with user-provided entrypoint in image"
+ )
+ sagemaker_args = build_sagemaker_args(
+ launch_project,
+ self._api,
+ role_arn,
+ launch_project.override_entrypoint,
+ launch_project.override_args,
+ MAX_ENV_LENGTHS[self.__class__.__name__],
+ image_uri,
+ default_output_path,
+ )
+ _logger.info(f"Launching sagemaker job with args: {sagemaker_args}")
+ run = await launch_sagemaker_job(
+ launch_project, sagemaker_args, sagemaker_client, log_client
+ )
+ if self.backend_config[PROJECT_SYNCHRONOUS]:
+ await run.wait()
+ return run
+
+
+def merge_image_uri_with_algorithm_specification(
+ algorithm_specification: Optional[Dict[str, Any]],
+ image_uri: Optional[str],
+ entrypoint_command: List[str],
+ args: Optional[List[str]],
+) -> Dict[str, Any]:
+ """Create an AWS AlgorithmSpecification.
+
+ AWS Sagemaker algorithms require a training image and an input mode. If the user
+ does not specify the specification themselves, define the spec minimally using these
+ two fields. Otherwise, if they specify the AlgorithmSpecification set the training
+ image if it is not set.
+ """
+ if algorithm_specification is None:
+ algorithm_specification = {
+ "TrainingImage": image_uri,
+ "TrainingInputMode": "File",
+ }
+ else:
+ if image_uri:
+ algorithm_specification["TrainingImage"] = image_uri
+ if entrypoint_command:
+ algorithm_specification["ContainerEntrypoint"] = entrypoint_command
+ if args:
+ algorithm_specification["ContainerArguments"] = args
+
+ if algorithm_specification["TrainingImage"] is None:
+ raise LaunchError("Failed determine tag for training image")
+ return algorithm_specification
+
+
+def build_sagemaker_args(
+ launch_project: LaunchProject,
+ api: Api,
+ role_arn: str,
+ entry_point: Optional[EntryPoint],
+ args: Optional[List[str]],
+ max_env_length: int,
+ image_uri: Optional[str] = None,
+ default_output_path: Optional[str] = None,
+) -> Dict[str, Any]:
+ sagemaker_args: Dict[str, Any] = {}
+ given_sagemaker_args: Optional[Dict[str, Any]] = launch_project.resource_args.get(
+ "sagemaker"
+ )
+
+ if given_sagemaker_args is None:
+ raise LaunchError(
+ "No sagemaker args specified. Specify sagemaker args in resource_args"
+ )
+ if (
+ given_sagemaker_args.get("OutputDataConfig") is None
+ and default_output_path is not None
+ ):
+ sagemaker_args["OutputDataConfig"] = {"S3OutputPath": default_output_path}
+ else:
+ sagemaker_args["OutputDataConfig"] = given_sagemaker_args.get(
+ "OutputDataConfig"
+ )
+
+ if sagemaker_args.get("OutputDataConfig") is None:
+ raise LaunchError(
+ "Sagemaker launcher requires an OutputDataConfig Sagemaker resource argument"
+ )
+ training_job_name = cast(
+ str, (given_sagemaker_args.get("TrainingJobName") or launch_project.run_id)
+ )
+ sagemaker_args["TrainingJobName"] = training_job_name
+ entry_cmd = entry_point.command if entry_point else []
+
+ sagemaker_args[
+ "AlgorithmSpecification"
+ ] = merge_image_uri_with_algorithm_specification(
+ given_sagemaker_args.get(
+ "AlgorithmSpecification",
+ given_sagemaker_args.get("algorithm_specification"),
+ ),
+ image_uri,
+ entry_cmd,
+ args,
+ )
+
+ sagemaker_args["RoleArn"] = role_arn
+
+ camel_case_args = {
+ to_camel_case(key): item for key, item in given_sagemaker_args.items()
+ }
+ sagemaker_args = {
+ **camel_case_args,
+ **sagemaker_args,
+ }
+
+ if sagemaker_args.get("ResourceConfig") is None:
+ raise LaunchError(
+ "Sagemaker launcher requires a ResourceConfig Sagemaker resource argument"
+ )
+
+ if sagemaker_args.get("StoppingCondition") is None:
+ raise LaunchError(
+ "Sagemaker launcher requires a StoppingCondition Sagemaker resource argument"
+ )
+
+ given_env = given_sagemaker_args.get(
+ "Environment", sagemaker_args.get("environment", {})
+ )
+ calced_env = get_env_vars_dict(launch_project, api, max_env_length)
+ total_env = {**calced_env, **given_env}
+ sagemaker_args["Environment"] = total_env
+
+ # Add wandb tag
+ tags = sagemaker_args.get("Tags", [])
+ tags.append({"Key": "WandbRunId", "Value": launch_project.run_id})
+ sagemaker_args["Tags"] = tags
+
+ # remove args that were passed in for launch but not passed to sagemaker
+ sagemaker_args.pop("EcrRepoName", None)
+ sagemaker_args.pop("region", None)
+ sagemaker_args.pop("profile", None)
+
+ # clear the args that are None so they are not passed
+ filtered_args = {k: v for k, v in sagemaker_args.items() if v is not None}
+
+ return filtered_args
+
+
+async def launch_sagemaker_job(
+ launch_project: LaunchProject,
+ sagemaker_args: Dict[str, Any],
+ sagemaker_client: "boto3.Client",
+ log_client: Optional["boto3.Client"] = None,
+) -> SagemakerSubmittedRun:
+ training_job_name = sagemaker_args.get("TrainingJobName") or launch_project.run_id
+ create_training_job = event_loop_thread_exec(sagemaker_client.create_training_job)
+ resp = await create_training_job(**sagemaker_args)
+
+ if resp.get("TrainingJobArn") is None:
+ raise LaunchError("Failed to create training job when submitting to SageMaker")
+
+ run = SagemakerSubmittedRun(training_job_name, sagemaker_client, log_client)
+ wandb.termlog(
+ f"{LOG_PREFIX}Run job submitted with arn: {resp.get('TrainingJobArn')}"
+ )
+ url = "https://{region}.console.aws.amazon.com/sagemaker/home?region={region}#/jobs/{job_name}".format(
+ region=sagemaker_client.meta.region_name, job_name=training_job_name
+ )
+ wandb.termlog(f"{LOG_PREFIX}See training job status at: {url}")
+ return run
+
+
+def get_role_arn(
+ sagemaker_args: Dict[str, Any], backend_config: Dict[str, Any], account_id: str
+) -> str:
+ """Get the role arn from the sagemaker args or the backend config."""
+ role_arn = sagemaker_args.get("RoleArn") or sagemaker_args.get("role_arn")
+ if role_arn is None:
+ role_arn = backend_config.get("runner", {}).get("role_arn")
+ if role_arn is None or not isinstance(role_arn, str):
+ raise LaunchError(
+ "AWS sagemaker require a string RoleArn set this by adding a `RoleArn` key to the sagemaker"
+ "field of resource_args"
+ )
+ if role_arn.startswith("arn:aws:iam::"):
+ return role_arn # type: ignore
+
+ return f"arn:aws:iam::{account_id}:role/{role_arn}"
diff --git a/wandb/sdk/launch/runner/vertex_runner.py b/wandb/sdk/launch/runner/vertex_runner.py
new file mode 100644
index 0000000000000000000000000000000000000000..607a11344eca302b17ac3c92460e655ce2917819
--- /dev/null
+++ b/wandb/sdk/launch/runner/vertex_runner.py
@@ -0,0 +1,229 @@
+import asyncio
+import logging
+from typing import Any, Dict, Optional
+
+if False:
+ from google.cloud import aiplatform # type: ignore # noqa: F401
+
+from wandb.apis.internal import Api
+from wandb.util import get_module
+
+from .._project_spec import LaunchProject, get_entry_point_command
+from ..builder.build import get_env_vars_dict
+from ..environment.gcp_environment import GcpEnvironment
+from ..errors import LaunchError
+from ..registry.abstract import AbstractRegistry
+from ..utils import MAX_ENV_LENGTHS, PROJECT_SYNCHRONOUS, event_loop_thread_exec
+from .abstract import AbstractRun, AbstractRunner, Status
+
+GCP_CONSOLE_URI = "https://console.cloud.google.com"
+
+_logger = logging.getLogger(__name__)
+
+
+WANDB_RUN_ID_KEY = "wandb-run-id"
+
+
+class VertexSubmittedRun(AbstractRun):
+ def __init__(self, job: Any) -> None:
+ self._job = job
+
+ @property
+ def id(self) -> str:
+ # numeric ID of the custom training job
+ return self._job.name # type: ignore
+
+ async def get_logs(self) -> Optional[str]:
+ # TODO: implement
+ return None
+
+ @property
+ def name(self) -> str:
+ return self._job.display_name # type: ignore
+
+ @property
+ def gcp_region(self) -> str:
+ return self._job.location # type: ignore
+
+ @property
+ def gcp_project(self) -> str:
+ return self._job.project # type: ignore
+
+ def get_page_link(self) -> str:
+ return "{console_uri}/vertex-ai/locations/{region}/training/{job_id}?project={project}".format(
+ console_uri=GCP_CONSOLE_URI,
+ region=self.gcp_region,
+ job_id=self.id,
+ project=self.gcp_project,
+ )
+
+ async def wait(self) -> bool:
+ # TODO: run this in a separate thread.
+ await self._job.wait()
+ return (await self.get_status()).state == "finished"
+
+ async def get_status(self) -> Status:
+ job_state = str(self._job.state) # extract from type PipelineState
+ if job_state == "JobState.JOB_STATE_SUCCEEDED":
+ return Status("finished")
+ if job_state == "JobState.JOB_STATE_FAILED":
+ return Status("failed")
+ if job_state == "JobState.JOB_STATE_RUNNING":
+ return Status("running")
+ if job_state == "JobState.JOB_STATE_PENDING":
+ return Status("starting")
+ return Status("unknown")
+
+ async def cancel(self) -> None:
+ self._job.cancel()
+
+
+class VertexRunner(AbstractRunner):
+ """Runner class, uses a project to create a VertexSubmittedRun."""
+
+ def __init__(
+ self,
+ api: Api,
+ backend_config: Dict[str, Any],
+ environment: GcpEnvironment,
+ registry: AbstractRegistry,
+ ) -> None:
+ """Initialize a VertexRunner instance."""
+ super().__init__(api, backend_config)
+ self.environment = environment
+ self.registry = registry
+
+ async def run(
+ self, launch_project: LaunchProject, image_uri: str
+ ) -> Optional[AbstractRun]:
+ """Run a Vertex job."""
+ full_resource_args = launch_project.fill_macros(image_uri)
+ resource_args = full_resource_args.get("vertex")
+ # We support setting under gcp-vertex for historical reasons.
+ if not resource_args:
+ resource_args = full_resource_args.get("gcp-vertex")
+ if not resource_args:
+ raise LaunchError(
+ "No Vertex resource args specified. Specify args via --resource-args with a JSON file or string under top-level key gcp_vertex"
+ )
+
+ spec_args = resource_args.get("spec", {})
+ run_args = resource_args.get("run", {})
+
+ synchronous: bool = self.backend_config[PROJECT_SYNCHRONOUS]
+
+ entry_point = (
+ launch_project.override_entrypoint
+ or launch_project.get_single_entry_point()
+ )
+
+ # TODO: Set entrypoint in each container
+ entry_cmd = get_entry_point_command(entry_point, launch_project.override_args)
+ env_vars = get_env_vars_dict(
+ launch_project=launch_project,
+ api=self._api,
+ max_env_length=MAX_ENV_LENGTHS[self.__class__.__name__],
+ )
+
+ worker_specs = spec_args.get("worker_pool_specs", [])
+ if not worker_specs:
+ raise LaunchError(
+ "Vertex requires at least one worker pool spec. Please specify "
+ "a worker pool spec in resource arguments under the key "
+ "`vertex.spec.worker_pool_specs`."
+ )
+
+ # TODO: Add entrypoint + args to each worker pool spec
+ for spec in worker_specs:
+ if not spec.get("container_spec"):
+ raise LaunchError(
+ "Vertex requires a container spec for each worker pool spec. "
+ "Please specify a container spec in resource arguments under "
+ "the key `vertex.spec.worker_pool_specs[].container_spec`."
+ )
+ spec["container_spec"]["command"] = entry_cmd
+
+ # Add our env vars to user supplied env vars
+ env = spec["container_spec"].get("env", [])
+ env.extend(
+ [{"name": key, "value": value} for key, value in env_vars.items()]
+ )
+ spec["container_spec"]["env"] = env
+
+ if not spec_args.get("staging_bucket"):
+ raise LaunchError(
+ "Vertex requires a staging bucket. Please specify a staging bucket "
+ "in resource arguments under the key `vertex.spec.staging_bucket`."
+ )
+
+ _logger.info("Launching Vertex job...")
+ submitted_run = await launch_vertex_job(
+ launch_project,
+ spec_args,
+ run_args,
+ self.environment,
+ synchronous,
+ )
+ return submitted_run
+
+
+async def launch_vertex_job(
+ launch_project: LaunchProject,
+ spec_args: Dict[str, Any],
+ run_args: Dict[str, Any],
+ environment: GcpEnvironment,
+ synchronous: bool = False,
+) -> VertexSubmittedRun:
+ try:
+ await environment.verify()
+ aiplatform = get_module( # noqa: F811
+ "google.cloud.aiplatform",
+ "VertexRunner requires google.cloud.aiplatform to be installed",
+ )
+ init = event_loop_thread_exec(aiplatform.init)
+ await init(
+ project=environment.project,
+ location=environment.region,
+ staging_bucket=spec_args.get("staging_bucket"),
+ credentials=await environment.get_credentials(),
+ )
+ labels = spec_args.get("labels", {})
+ labels[WANDB_RUN_ID_KEY] = launch_project.run_id
+ job = aiplatform.CustomJob(
+ display_name=launch_project.name,
+ worker_pool_specs=spec_args.get("worker_pool_specs"),
+ base_output_dir=spec_args.get("base_output_dir"),
+ encryption_spec_key_name=spec_args.get("encryption_spec_key_name"),
+ labels=labels,
+ )
+ execution_kwargs = dict(
+ timeout=run_args.get("timeout"),
+ service_account=run_args.get("service_account"),
+ network=run_args.get("network"),
+ enable_web_access=run_args.get("enable_web_access", False),
+ experiment=run_args.get("experiment"),
+ experiment_run=run_args.get("experiment_run"),
+ tensorboard=run_args.get("tensorboard"),
+ restart_job_on_worker_restart=run_args.get(
+ "restart_job_on_worker_restart", False
+ ),
+ )
+ # Unclear if there are exceptions that can be thrown where we should
+ # retry instead of erroring. For now, just catch all exceptions and they
+ # go to the UI for the user to interpret.
+ except Exception as e:
+ raise LaunchError(f"Failed to create Vertex job: {e}")
+
+ if synchronous:
+ run = event_loop_thread_exec(job.run)
+ await run(**execution_kwargs, sync=True)
+ else:
+ submit = event_loop_thread_exec(job.submit)
+ await submit(**execution_kwargs)
+ submitted_run = VertexSubmittedRun(job)
+ interval = 1
+ while not getattr(job._gca_resource, "name", None):
+ # give time for the gcp job object to be created and named, this should only loop a couple times max
+ await asyncio.sleep(interval)
+ interval = min(30, interval * 2)
+ return submitted_run
diff --git a/wandb/sdk/launch/sweeps/Dockerfile.scheduler.sweep b/wandb/sdk/launch/sweeps/Dockerfile.scheduler.sweep
new file mode 100644
index 0000000000000000000000000000000000000000..c8a1d6f665ae11cd256f54c740a3d1471a3a8afb
--- /dev/null
+++ b/wandb/sdk/launch/sweeps/Dockerfile.scheduler.sweep
@@ -0,0 +1,19 @@
+FROM python:3.9-slim-bullseye
+LABEL maintainer='Weights & Biases <support@wandb.com>'
+LABEL version="0.1"
+
+# install git
+RUN apt-get update && apt-get upgrade -y \
+ && apt-get install -y git \
+ && apt-get -qy autoremove \
+ && apt-get clean && rm -r /var/lib/apt/lists/*
+
+# required pip packages
+RUN pip install --no-cache-dir wandb[launch]
+# user set up
+RUN useradd -m -s /bin/bash --create-home --no-log-init -u 1000 -g 0 wandb_scheduler
+USER wandb_scheduler
+WORKDIR /home/wandb_scheduler
+RUN chown -R wandb_scheduler /home/wandb_scheduler
+
+ENTRYPOINT ["wandb", "scheduler"]
diff --git a/wandb/sdk/launch/sweeps/__init__.py b/wandb/sdk/launch/sweeps/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..3cdc29ddb4ffc4358ee1eb464564a81bd4e3519c
--- /dev/null
+++ b/wandb/sdk/launch/sweeps/__init__.py
@@ -0,0 +1,39 @@
+import logging
+from typing import Any, Callable, Dict
+
+log = logging.getLogger(__name__)
+
+
+class SchedulerError(Exception):
+ """Raised when a known error occurs with wandb sweep scheduler."""
+
+ pass
+
+
+def _import_sweep_scheduler() -> Any:
+ from .scheduler_sweep import SweepScheduler
+
+ return SweepScheduler
+
+
+_WANDB_SCHEDULERS: Dict[str, Callable] = {
+ "wandb": _import_sweep_scheduler,
+}
+
+
+def load_scheduler(scheduler_type: str) -> Any:
+ scheduler_type = scheduler_type.lower()
+ if scheduler_type not in _WANDB_SCHEDULERS:
+ raise SchedulerError(
+ f"The `scheduler_name` argument must be one of "
+ f"{list(_WANDB_SCHEDULERS.keys())}, got: {scheduler_type}"
+ )
+
+ log.warn(f"Loading dependencies for Scheduler of type: {scheduler_type}")
+ import_func = _WANDB_SCHEDULERS[scheduler_type]
+ return import_func()
+
+
+__all__ = [
+ "load_scheduler",
+]
diff --git a/wandb/sdk/launch/sweeps/scheduler.py b/wandb/sdk/launch/sweeps/scheduler.py
new file mode 100644
index 0000000000000000000000000000000000000000..391086d5a9d7a3ab438ef21d9a381f79badc21b0
--- /dev/null
+++ b/wandb/sdk/launch/sweeps/scheduler.py
@@ -0,0 +1,727 @@
+"""Abstract Scheduler class."""
+import asyncio
+import base64
+import logging
+import os
+import socket
+import threading
+import time
+import traceback
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Union
+
+import click
+import yaml
+
+import wandb
+from wandb.errors import CommError
+from wandb.sdk.launch._launch_add import launch_add
+from wandb.sdk.launch.errors import LaunchError
+from wandb.sdk.launch.sweeps import SchedulerError
+from wandb.sdk.launch.sweeps.utils import (
+ create_sweep_command_args,
+ make_launch_sweep_entrypoint,
+)
+from wandb.sdk.launch.utils import event_loop_thread_exec
+from wandb.sdk.lib.runid import generate_id
+
+if TYPE_CHECKING:
+ import wandb.apis.public as public
+ from wandb.apis.internal import Api
+ from wandb.apis.public import QueuedRun, Run
+ from wandb.sdk.wandb_run import Run as SdkRun
+
+
+_logger = logging.getLogger(__name__)
+LOG_PREFIX = f"{click.style('sched:', fg='cyan')} "
+
+DEFAULT_POLLING_SLEEP = 5.0
+
+
+class SchedulerState(Enum):
+ PENDING = 0
+ STARTING = 1
+ RUNNING = 2
+ FLUSH_RUNS = 3
+ COMPLETED = 4
+ FAILED = 5
+ STOPPED = 6
+ CANCELLED = 7
+
+
+class RunState(Enum):
+ RUNNING = "running", "alive"
+ PENDING = "pending", "alive"
+ PREEMPTING = "preempting", "alive"
+ CRASHED = "crashed", "dead"
+ FAILED = "failed", "dead"
+ KILLED = "killed", "dead"
+ FINISHED = "finished", "dead"
+ PREEMPTED = "preempted", "dead"
+ # unknown when api.get_run_state fails or returns unexpected state
+ # assumed alive, unless we get unknown 2x then move to failed (dead)
+ UNKNOWN = "unknown", "alive"
+
+ def __new__(cls: Any, *args: List, **kwds: Any) -> "RunState":
+ obj: RunState = object.__new__(cls)
+ obj._value_ = args[0]
+ return obj
+
+ def __init__(self, _: str, life: str = "unknown") -> None:
+ self._life = life
+
+ @property
+ def is_alive(self) -> bool:
+ return self._life == "alive"
+
+
+@dataclass
+class _Worker:
+ agent_config: Dict[str, Any]
+ agent_id: str
+
+
+@dataclass
+class SweepRun:
+ id: str
+ worker_id: int
+ state: RunState = RunState.RUNNING
+ queued_run: Optional["public.QueuedRun"] = None
+ args: Optional[Dict[str, Any]] = None
+ logs: Optional[List[str]] = None
+
+
+class Scheduler(ABC):
+ """A controller/agent that populates a Launch RunQueue from a hyperparameter sweep."""
+
+ PLACEHOLDER_URI = "placeholder-uri-scheduler"
+ SWEEP_JOB_TYPE = "sweep-controller"
+ ENTRYPOINT = ["wandb", "scheduler", "WANDB_SWEEP_ID"]
+
+ def __init__(
+ self,
+ api: "Api",
+ *args: Optional[Any],
+ polling_sleep: Optional[float] = None,
+ sweep_id: Optional[str] = None,
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+ project_queue: Optional[str] = None,
+ num_workers: Optional[Union[int, str]] = None,
+ **kwargs: Optional[Any],
+ ):
+ from wandb.apis.public import Api as PublicApi
+
+ self._api = api
+ self._public_api = PublicApi()
+ self._entity = (
+ entity
+ or os.environ.get("WANDB_ENTITY")
+ or api.settings("entity")
+ or api.default_entity
+ )
+ self._project = (
+ project or os.environ.get("WANDB_PROJECT") or api.settings("project")
+ )
+ self._sweep_id: str = sweep_id or "empty-sweep-id"
+ self._state: SchedulerState = SchedulerState.PENDING
+
+ # Make sure the provided sweep_id corresponds to a valid sweep
+ try:
+ resp = self._api.sweep(
+ sweep_id, "{}", entity=self._entity, project=self._project
+ )
+ if resp.get("state") == SchedulerState.CANCELLED.name:
+ self._state = SchedulerState.CANCELLED
+ self._sweep_config = yaml.safe_load(resp["config"])
+ self._num_runs_launched: int = self._get_num_runs_launched(resp["runs"])
+ if self._num_runs_launched > 0:
+ wandb.termlog(
+ f"{LOG_PREFIX}Found {self._num_runs_launched} previous valid runs for sweep {self._sweep_id}"
+ )
+ except Exception as e:
+ raise SchedulerError(
+ f"{LOG_PREFIX}Exception when finding sweep ({sweep_id}) {e}"
+ )
+
+ # Scheduler may receive additional kwargs which will be piped into the launch command
+ self._kwargs: Dict[str, Any] = kwargs
+
+ # Dictionary of the runs being managed by the scheduler
+ self._runs: Dict[str, SweepRun] = {}
+ # Threading lock to ensure thread-safe access to the runs dictionary
+ self._threading_lock: threading.Lock = threading.Lock()
+ self._polling_sleep = polling_sleep or DEFAULT_POLLING_SLEEP
+ self._project_queue = project_queue
+ # Optionally run multiple workers in (pseudo-)parallel. Workers do not
+ # actually run training workloads, they simply send heartbeat messages
+ # (emulating a real agent) and add new runs to the launch queue. The
+ # launch agent is the one that actually runs the training workloads.
+ self._workers: Dict[int, _Worker] = {}
+
+ # Init wandb scheduler run
+ self._wandb_run = self._init_wandb_run()
+
+ # Grab params from scheduler wandb run config
+ num_workers = num_workers or self._wandb_run.config.get("scheduler", {}).get(
+ "num_workers"
+ )
+ self._num_workers = int(num_workers) if str(num_workers).isdigit() else 8
+ self._settings_config: Dict[str, Any] = self._wandb_run.config.get(
+ "settings", {}
+ )
+
+ @abstractmethod
+ def _get_next_sweep_run(self, worker_id: int) -> Optional[SweepRun]:
+ """Called when worker available."""
+ pass
+
+ @abstractmethod
+ def _poll(self) -> None:
+ """Called every polling loop."""
+ pass
+
+ @abstractmethod
+ def _exit(self) -> None:
+ pass
+
+ @abstractmethod
+ def _load_state(self) -> None:
+ pass
+
+ @abstractmethod
+ def _save_state(self) -> None:
+ pass
+
+ @property
+ def state(self) -> SchedulerState:
+ _logger.debug(f"{LOG_PREFIX}Scheduler state is {self._state.name}")
+ return self._state
+
+ @state.setter
+ def state(self, value: SchedulerState) -> None:
+ _logger.debug(f"{LOG_PREFIX}Scheduler was {self.state.name} is {value.name}")
+ self._state = value
+
+ @property
+ def is_alive(self) -> bool:
+ if self.state in [
+ SchedulerState.COMPLETED,
+ SchedulerState.FAILED,
+ SchedulerState.STOPPED,
+ SchedulerState.CANCELLED,
+ ]:
+ return False
+ return True
+
+ @property
+ def at_runcap(self) -> bool:
+ """False if under user-specified cap on # of runs."""
+ run_cap = self._sweep_config.get("run_cap")
+ if not run_cap:
+ return False
+ at_runcap: bool = self._num_runs_launched >= run_cap
+ return at_runcap
+
+ @property
+ def num_active_runs(self) -> int:
+ return len(self._runs)
+
+ @property
+ def busy_workers(self) -> Dict[int, _Worker]:
+ """Returns dict of id:worker already assigned to a launch run.
+
+ runs should always have a worker_id, but are created before
+ workers are assigned to the run
+ """
+ busy_workers = {}
+ for _, r in self._yield_runs():
+ busy_workers[r.worker_id] = self._workers[r.worker_id]
+ return busy_workers
+
+ @property
+ def available_workers(self) -> Dict[int, _Worker]:
+ """Returns dict of id:worker ready to launch another run."""
+ if len(self._workers) == 0:
+ return {}
+ return {
+ _id: w for _id, w in self._workers.items() if _id not in self.busy_workers
+ }
+
+ def _init_wandb_run(self) -> "SdkRun":
+ """Controls resume or init logic for a scheduler wandb run."""
+ run: SdkRun = wandb.init( # type: ignore
+ name=f"Scheduler.{self._sweep_id}",
+ resume="allow",
+ config=self._kwargs, # when run as a job, this sets config
+ )
+ return run
+
+ def stop_sweep(self) -> None:
+ """Stop the sweep."""
+ self._state = SchedulerState.STOPPED
+
+ def fail_sweep(self, err: Optional[str]) -> None:
+ """Fail the sweep w/ optional exception."""
+ self._state = SchedulerState.FAILED
+ if err:
+ raise SchedulerError(err)
+
+ def start(self) -> None:
+ """Start a scheduler, confirms prerequisites, begins execution loop."""
+ wandb.termlog(f"{LOG_PREFIX}Scheduler starting.")
+ if not self.is_alive:
+ wandb.termerror(
+ f"{LOG_PREFIX}Sweep already in end state ({self.state.name.lower()}). Exiting..."
+ )
+ self.exit()
+ return
+
+ self._state = SchedulerState.STARTING
+ if not self._try_load_executable():
+ wandb.termerror(
+ f"{LOG_PREFIX}No 'job' or 'image_uri' loaded from sweep config."
+ )
+ self.exit()
+ return
+
+ # For resuming sweeps
+ self._load_state()
+ asyncio.run(self._register_agents())
+ self.run()
+
+ def run(self) -> None:
+ """Main run function."""
+ wandb.termlog(f"{LOG_PREFIX}Scheduler running")
+ self.state = SchedulerState.RUNNING
+ try:
+ while True:
+ self._update_scheduler_run_state()
+ if not self.is_alive:
+ break
+
+ wandb.termlog(f"{LOG_PREFIX}Polling for new runs to launch")
+
+ self._update_run_states()
+ self._poll()
+ if self.state == SchedulerState.FLUSH_RUNS:
+ if self.num_active_runs == 0:
+ wandb.termlog(f"{LOG_PREFIX}Done polling on runs, exiting")
+ break
+ time.sleep(self._polling_sleep)
+ continue
+
+ for worker_id in self.available_workers:
+ if self.at_runcap:
+ wandb.termlog(
+ f"{LOG_PREFIX}Sweep at run_cap ({self._num_runs_launched})"
+ )
+ self.state = SchedulerState.FLUSH_RUNS
+ break
+
+ try:
+ run: Optional[SweepRun] = self._get_next_sweep_run(worker_id)
+ if not run:
+ break
+ except SchedulerError as e:
+ raise SchedulerError(e)
+ except Exception as e:
+ wandb.termerror(
+ f"{LOG_PREFIX}Failed to get next sweep run: {e}"
+ )
+ self.state = SchedulerState.FAILED
+ break
+
+ if self._add_to_launch_queue(run):
+ self._num_runs_launched += 1
+
+ time.sleep(self._polling_sleep)
+ except KeyboardInterrupt:
+ wandb.termwarn(f"{LOG_PREFIX}Scheduler received KeyboardInterrupt. Exiting")
+ self.state = SchedulerState.STOPPED
+ self.exit()
+ return
+ except Exception as e:
+ wandb.termlog(f"{LOG_PREFIX}Scheduler failed with exception {e}")
+ self.state = SchedulerState.FAILED
+ self.exit()
+ raise e
+ else:
+ # scheduler succeeds if at runcap
+ if self.state == SchedulerState.FLUSH_RUNS and self.at_runcap:
+ self.state = SchedulerState.COMPLETED
+ self.exit()
+
+ def exit(self) -> None:
+ self._exit()
+ # _save_state isn't controlled, possibly fails
+ try:
+ self._save_state()
+ except Exception:
+ wandb.termerror(
+ f"{LOG_PREFIX}Failed to save state: {traceback.format_exc()}"
+ )
+
+ status = ""
+ if self.state == SchedulerState.FLUSH_RUNS:
+ self._set_sweep_state("PAUSED")
+ status = "paused"
+ elif self.state == SchedulerState.COMPLETED:
+ self._set_sweep_state("FINISHED")
+ status = "completed"
+ elif self.state in [SchedulerState.CANCELLED, SchedulerState.STOPPED]:
+ self._set_sweep_state("CANCELED") # one L
+ status = "cancelled"
+ self._stop_runs()
+ else:
+ self.state = SchedulerState.FAILED
+ self._set_sweep_state("CRASHED")
+ status = "crashed"
+ self._stop_runs()
+
+ wandb.termlog(f"{LOG_PREFIX}Scheduler {status}")
+ self._wandb_run.finish()
+
+ def _get_num_runs_launched(self, runs: List[Dict[str, Any]]) -> int:
+ """Returns the number of valid runs in the sweep."""
+ count = 0
+ for run in runs:
+ # if bad run, shouldn't be counted against run cap
+ if run.get("state", "") in ["killed", "crashed"] and not run.get(
+ "summaryMetrics"
+ ):
+ _logger.debug(
+ f"excluding run: {run['name']} with state: {run['state']} from run cap \n{run}"
+ )
+ continue
+ count += 1
+
+ return count
+
+ def _try_load_executable(self) -> bool:
+ """Check existance of valid executable for a run.
+
+ logs and returns False when job is unreachable
+ """
+ if self._kwargs.get("job"):
+ try:
+ _job_artifact = self._public_api.job(self._kwargs["job"])
+ wandb.termlog(
+ f"{LOG_PREFIX}Successfully loaded job ({_job_artifact.name}) in scheduler"
+ )
+ except Exception:
+ wandb.termerror(f"{LOG_PREFIX}{traceback.format_exc()}")
+ return False
+ return True
+ elif self._kwargs.get("image_uri"):
+ # TODO(gst): check docker existance? Use registry in launch config?
+ return True
+ else:
+ return False
+
+ async def _register_agents(self) -> None:
+ tasks = []
+ register_agent = event_loop_thread_exec(self._api.register_agent)
+ for worker_id in range(self._num_workers):
+ _logger.debug(f"{LOG_PREFIX}Starting AgentHeartbeat worker ({worker_id})")
+ try:
+ worker = register_agent(
+ f"{socket.gethostname()}-{worker_id}", # host
+ sweep_id=self._sweep_id,
+ project_name=self._project,
+ entity=self._entity,
+ )
+ tasks.append(worker)
+ except Exception as e:
+ _logger.debug(f"failed to register agent: {e}")
+ self.fail_sweep(f"failed to register agent: {e}")
+
+ finished_tasks = await asyncio.gather(*tasks)
+ for idx, agent_config in enumerate(finished_tasks):
+ self._workers[idx] = _Worker(
+ agent_config=agent_config,
+ agent_id=agent_config["id"],
+ )
+
+ def _yield_runs(self) -> Iterator[Tuple[str, SweepRun]]:
+ """Thread-safe way to iterate over the runs."""
+ with self._threading_lock:
+ yield from self._runs.items()
+
+ def _cleanup_runs(self, runs_to_remove: List[str]) -> None:
+ """Helper for removing runs from memory.
+
+ Can be overloaded to prevent deletion of runs, which is useful
+ for debugging or when polling on completed runs.
+ """
+ with self._threading_lock:
+ for run_id in runs_to_remove:
+ wandb.termlog(f"{LOG_PREFIX}Cleaning up finished run ({run_id})")
+ del self._runs[run_id]
+
+ def _stop_runs(self) -> None:
+ to_delete = []
+ for run_id, _ in self._yield_runs():
+ to_delete += [run_id]
+
+ for run_id in to_delete:
+ wandb.termlog(f"{LOG_PREFIX}Stopping run ({run_id})")
+ if not self._stop_run(run_id):
+ wandb.termwarn(f"{LOG_PREFIX}Failed to stop run ({run_id})")
+
+ def _stop_run(self, run_id: str) -> bool:
+ """Stops a run and removes it from the scheduler."""
+ if run_id not in self._runs:
+ _logger.debug(f"run: {run_id} not in _runs: {self._runs}")
+ return False
+
+ run = self._runs[run_id]
+ del self._runs[run_id]
+
+ if not run.queued_run:
+ _logger.debug(
+ f"tried to _stop_run but run not queued yet (run_id:{run.id})"
+ )
+ return False
+
+ if not run.state.is_alive:
+ # run already dead, just delete reference
+ return True
+
+ # run still alive, send stop signal
+ encoded_run_id = base64.standard_b64encode(
+ f"Run:v1:{run_id}:{self._project}:{self._entity}".encode()
+ ).decode("utf-8")
+
+ try:
+ success: bool = self._api.stop_run(run_id=encoded_run_id)
+ if success:
+ wandb.termlog(f"{LOG_PREFIX}Stopped run {run_id}.")
+ return True
+ except Exception as e:
+ _logger.debug(f"error stopping run ({run_id}): {e}")
+
+ return False
+
+ def _update_scheduler_run_state(self) -> None:
+ """Update the scheduler state from state of scheduler run and sweep state."""
+ state: RunState = self._get_run_state(self._wandb_run.id)
+
+ # map scheduler run-state to scheduler-state
+ if state == RunState.KILLED:
+ self.state = SchedulerState.STOPPED
+ elif state in [RunState.FAILED, RunState.CRASHED]:
+ self.state = SchedulerState.FAILED
+ elif state == RunState.FINISHED:
+ self.state = SchedulerState.COMPLETED
+
+ # check sweep state for completed states, overwrite scheduler state
+ try:
+ sweep_state = self._api.get_sweep_state(
+ self._sweep_id, self._entity, self._project
+ )
+ except Exception as e:
+ _logger.debug(f"sweep state error: {e}")
+ return
+
+ if sweep_state == "FINISHED":
+ self.state = SchedulerState.COMPLETED
+ elif sweep_state in ["CANCELLED", "STOPPED"]:
+ self.state = SchedulerState.CANCELLED
+ elif sweep_state == "PAUSED":
+ self.state = SchedulerState.FLUSH_RUNS
+
+ def _update_run_states(self) -> None:
+ """Iterate through runs.
+
+ Get state from backend and deletes runs if not in running state. Threadsafe.
+ """
+ runs_to_remove: List[str] = []
+ for run_id, run in self._yield_runs():
+ run.state = self._get_run_state(run_id, run.state)
+
+ try:
+ rqi_state = run.queued_run.state if run.queued_run else None
+ except (CommError, LaunchError) as e:
+ _logger.debug(f"Failed to get queued_run.state: {e}")
+ rqi_state = None
+
+ if not run.state.is_alive or rqi_state == "failed":
+ _logger.debug(f"({run_id}) states: ({run.state}, {rqi_state})")
+ runs_to_remove.append(run_id)
+ self._cleanup_runs(runs_to_remove)
+
+ def _get_metrics_from_run(self, run_id: str) -> List[Any]:
+ """Use the public api to get metrics from a run.
+
+ Uses the metric name found in the sweep config, any
+ misspellings will result in an empty list.
+ """
+ try:
+ queued_run: Optional[QueuedRun] = self._runs[run_id].queued_run
+ if not queued_run:
+ return []
+
+ api_run: Run = self._public_api.run(
+ f"{queued_run.entity}/{queued_run.project}/{run_id}"
+ )
+ metric_name = self._sweep_config["metric"]["name"]
+ history = api_run.scan_history(keys=["_step", metric_name])
+ metrics = [x[metric_name] for x in history]
+
+ return metrics
+ except Exception as e:
+ _logger.debug(f"[_get_metrics_from_run] {e}")
+ return []
+
+ def _get_run_info(self, run_id: str) -> Dict[str, Any]:
+ """Use the public api to get info about a run."""
+ try:
+ info: Dict[str, Any] = self._api.get_run_info(
+ self._entity, self._project, run_id
+ )
+ if info:
+ return info
+ except Exception as e:
+ _logger.debug(f"[_get_run_info] {e}")
+ return {}
+
+ def _get_run_state(
+ self, run_id: str, prev_run_state: RunState = RunState.UNKNOWN
+ ) -> RunState:
+ """Use the public api to get state of a run."""
+ run_state = None
+ try:
+ state = self._api.get_run_state(self._entity, self._project, run_id)
+ run_state = RunState(state)
+ except CommError as e:
+ _logger.debug(f"error getting state for run ({run_id}): {e}")
+ if prev_run_state == RunState.UNKNOWN:
+ # triggers when we get an unknown state for the second time
+ wandb.termwarn(
+ f"Failed to get runstate for run ({run_id}). Error: {traceback.format_exc()}"
+ )
+ run_state = RunState.FAILED
+ else: # first time we get unknwon state
+ run_state = RunState.UNKNOWN
+ except (AttributeError, ValueError):
+ wandb.termwarn(
+ f"Bad state ({run_state}) for run ({run_id}). Error: {traceback.format_exc()}"
+ )
+ run_state = RunState.UNKNOWN
+ return run_state
+
+ def _create_run(self) -> Dict[str, Any]:
+ """Use the public api to create a blank run."""
+ try:
+ run: List[Dict[str, Any]] = self._api.upsert_run(
+ project=self._project,
+ entity=self._entity,
+ sweep_name=self._sweep_id,
+ )
+ if run:
+ return run[0]
+ except Exception as e:
+ _logger.debug(f"[_create_run] {e}")
+ raise SchedulerError(
+ "Error creating run from scheduler, check API connection and CLI version."
+ )
+ return {}
+
+ def _set_sweep_state(self, state: str) -> None:
+ wandb.termlog(f"{LOG_PREFIX}Updating sweep state to: {state.lower()}")
+ try:
+ self._api.set_sweep_state(sweep=self._sweep_id, state=state)
+ except Exception as e:
+ _logger.debug(f"[set_sweep_state] {e}")
+
+ def _encode(self, _id: str) -> str:
+ return (
+ base64.b64decode(bytes(_id.encode("utf-8"))).decode("utf-8").split(":")[2]
+ )
+
+ def _make_entry_and_launch_config(
+ self, run: SweepRun
+ ) -> Tuple[Optional[List[str]], Dict[str, Dict[str, Any]]]:
+ args = create_sweep_command_args({"args": run.args})
+ entry_point, macro_args = make_launch_sweep_entrypoint(
+ args, self._sweep_config.get("command")
+ )
+ # handle program macro
+ if entry_point and "${program}" in entry_point:
+ if not self._sweep_config.get("program"):
+ raise SchedulerError(
+ f"{LOG_PREFIX}Program macro in command has no corresponding 'program' in sweep config."
+ )
+ pidx = entry_point.index("${program}")
+ entry_point[pidx] = self._sweep_config["program"]
+
+ launch_config = self._wandb_run.config.get("launch", {})
+ if "overrides" not in launch_config:
+ launch_config["overrides"] = {"run_config": {}}
+ launch_config["overrides"]["run_config"].update(args["args_dict"])
+
+ if macro_args: # pipe in hyperparam args as params to launch
+ launch_config["overrides"]["args"] = macro_args
+
+ if entry_point:
+ unresolved = [x for x in entry_point if str(x).startswith("${")]
+ if unresolved:
+ wandb.termwarn(
+ f"{LOG_PREFIX}Sweep command contains unresolved macros: "
+ f"{unresolved}, see launch docs for supported macros."
+ )
+ return entry_point, launch_config
+
+ def _add_to_launch_queue(self, run: SweepRun) -> bool:
+ """Convert a sweeprun into a launch job then push to runqueue."""
+ # job and image first from CLI args, then from sweep config
+ _job = self._kwargs.get("job") or self._sweep_config.get("job")
+ _sweep_config_uri = self._sweep_config.get("image_uri")
+ _image_uri = self._kwargs.get("image_uri") or _sweep_config_uri
+ if _job is None and _image_uri is None:
+ raise SchedulerError(f"{LOG_PREFIX}No 'job' nor 'image_uri' ({run.id})")
+ elif _job is not None and _image_uri is not None:
+ raise SchedulerError(f"{LOG_PREFIX}Sweep has both 'job' and 'image_uri'")
+
+ entry_point, launch_config = self._make_entry_and_launch_config(run)
+ if entry_point:
+ wandb.termwarn(
+ f"{LOG_PREFIX}Sweep command {entry_point} will override"
+ f' {"job" if _job else "image_uri"} entrypoint'
+ )
+
+ # override resource and args of job
+ _job_launch_config = self._wandb_run.config.get("launch") or {}
+
+ # default priority is "medium"
+ _priority = int(launch_config.get("priority", 2)) # type: ignore
+
+ run_id = run.id or generate_id()
+ queued_run = launch_add(
+ run_id=run_id,
+ entry_point=entry_point,
+ config=launch_config,
+ docker_image=_image_uri, # TODO(gst): make agnostic (github? run uri?)
+ job=_job,
+ project=self._project,
+ entity=self._entity,
+ queue_name=self._kwargs.get("queue"),
+ project_queue=self._project_queue,
+ resource=_job_launch_config.get("resource"),
+ resource_args=_job_launch_config.get("resource_args"),
+ author=self._kwargs.get("author"),
+ sweep_id=self._sweep_id,
+ priority=_priority,
+ )
+ run.queued_run = queued_run
+ # TODO(gst): unify run and queued_run state
+ run.state = RunState.RUNNING # assume it will get picked up
+ self._runs[run_id] = run
+
+ wandb.termlog(
+ f"{LOG_PREFIX}Added run ({run_id}) to queue ({self._kwargs.get('queue')})"
+ )
+ return True
diff --git a/wandb/sdk/launch/sweeps/scheduler_sweep.py b/wandb/sdk/launch/sweeps/scheduler_sweep.py
new file mode 100644
index 0000000000000000000000000000000000000000..8cd30fb94e80b02271d3b70998b58f7cee287764
--- /dev/null
+++ b/wandb/sdk/launch/sweeps/scheduler_sweep.py
@@ -0,0 +1,90 @@
+"""Scheduler for classic wandb Sweeps."""
+import logging
+from pprint import pformat as pf
+from typing import Any, Dict, List, Optional
+
+import wandb
+from wandb.sdk.launch.sweeps.scheduler import LOG_PREFIX, RunState, Scheduler, SweepRun
+
+_logger = logging.getLogger(__name__)
+
+
+class SweepScheduler(Scheduler):
+ """A controller/agent that populates a Launch RunQueue from a sweeps RunQueue."""
+
+ def __init__(
+ self,
+ *args: Any,
+ **kwargs: Any,
+ ):
+ super().__init__(*args, **kwargs)
+
+ def _get_next_sweep_run(self, worker_id: int) -> Optional[SweepRun]:
+ """Called by the main scheduler execution loop.
+
+ Expected to return a properly formatted SweepRun if the scheduler
+ is alive, or None and set the appropriate scheduler state:
+
+ FAILED: self.fail_sweep()
+ STOPPED: self.stop_sweep()
+ """
+ commands: List[Dict[str, Any]] = self._get_sweep_commands(worker_id)
+ for command in commands:
+ # The command "type" can be one of "run", "resume", "stop", "exit"
+ _type = command.get("type")
+ if _type in ["exit", "stop"]:
+ self.stop_sweep()
+ return None
+
+ if _type not in ["run", "resume"]:
+ self.fail_sweep(f"AgentHeartbeat unknown command: {_type}")
+
+ _run_id: Optional[str] = command.get("run_id")
+ if not _run_id:
+ self.fail_sweep(f"No run id in agent heartbeat: {command}")
+ return None
+
+ if _run_id in self._runs:
+ wandb.termlog(f"{LOG_PREFIX}Skipping duplicate run: {_run_id}")
+ continue
+
+ return SweepRun(
+ id=_run_id,
+ state=RunState.PENDING,
+ args=command.get("args", {}),
+ logs=command.get("logs", []),
+ worker_id=worker_id,
+ )
+ return None
+
+ def _get_sweep_commands(self, worker_id: int) -> List[Dict[str, Any]]:
+ """Helper to recieve sweep command from backend."""
+ # AgentHeartbeat wants a Dict of runs which are running or queued
+ _run_states: Dict[str, bool] = {}
+ for run_id, run in self._yield_runs():
+ # Filter out runs that are from a different worker thread
+ if run.worker_id == worker_id and run.state.is_alive:
+ _run_states[run_id] = True
+
+ _logger.debug(f"Sending states: \n{pf(_run_states)}\n")
+ commands: List[Dict[str, Any]] = self._api.agent_heartbeat(
+ agent_id=self._workers[worker_id].agent_id,
+ metrics={},
+ run_states=_run_states,
+ )
+ _logger.debug(f"AgentHeartbeat commands: \n{pf(commands)}\n")
+
+ return commands
+
+ def _exit(self) -> None:
+ pass
+
+ def _poll(self) -> None:
+ _logger.debug(f"_poll. _runs: {self._runs}")
+ pass
+
+ def _load_state(self) -> None:
+ pass
+
+ def _save_state(self) -> None:
+ pass
diff --git a/wandb/sdk/launch/sweeps/utils.py b/wandb/sdk/launch/sweeps/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..9761a32e1e415fc6811f6c999f57e59b92075743
--- /dev/null
+++ b/wandb/sdk/launch/sweeps/utils.py
@@ -0,0 +1,316 @@
+import json
+import os
+import re
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+
+import yaml
+
+import wandb
+from wandb import util
+from wandb.sdk.launch.errors import LaunchError
+
+if TYPE_CHECKING:
+ from wandb.apis.public import Api as PublicApi
+
+DEFAULT_SWEEP_COMMAND: List[str] = [
+ "${env}",
+ "${interpreter}",
+ "${program}",
+ "${args}",
+]
+SWEEP_COMMAND_ENV_VAR_REGEX = re.compile(r"\$\{envvar\:([A-Z0-9_]*)\}")
+
+
+def parse_sweep_id(parts_dict: dict) -> Optional[str]:
+ """In place parse sweep path from parts dict.
+
+ Arguments:
+ parts_dict (dict): dict(entity=,project=,name=). Modifies dict inplace.
+
+ Returns:
+ None or str if there is an error
+ """
+ entity = None
+ project = None
+ sweep_id = parts_dict.get("name")
+ if not isinstance(sweep_id, str):
+ return "Expected string sweep_id"
+
+ sweep_split = sweep_id.split("/")
+ if len(sweep_split) == 1:
+ pass
+ elif len(sweep_split) == 2:
+ split_project, sweep_id = sweep_split
+ project = split_project or project
+ elif len(sweep_split) == 3:
+ split_entity, split_project, sweep_id = sweep_split
+ project = split_project or project
+ entity = split_entity or entity
+ else:
+ return (
+ "Expected sweep_id in form of sweep, project/sweep, or entity/project/sweep"
+ )
+ parts_dict.update(dict(name=sweep_id, project=project, entity=entity))
+ return None
+
+
+def sweep_config_err_text_from_jsonschema_violations(violations: List[str]) -> str:
+ """Consolidate schema violation strings from wandb/sweeps into a single string.
+
+ Parameters
+ ----------
+ violations: list of str
+ The warnings to render.
+
+ Returns:
+ -------
+ violation: str
+ The consolidated violation text.
+
+ """
+ violation_base = (
+ "Malformed sweep config detected! This may cause your sweep to behave in unexpected ways.\n"
+ "To avoid this, please fix the sweep config schema violations below:"
+ )
+
+ for i, warning in enumerate(violations):
+ violations[i] = f" Violation {i + 1}. {warning}"
+ violation = "\n".join([violation_base] + violations)
+
+ return violation
+
+
+def handle_sweep_config_violations(warnings: List[str]) -> None:
+ """Echo sweep config schema violation warnings from Gorilla to the terminal.
+
+ Parameters
+ ----------
+ warnings: list of str
+ The warnings to render.
+ """
+ warning = sweep_config_err_text_from_jsonschema_violations(warnings)
+ if len(warnings) > 0:
+ wandb.termwarn(warning)
+
+
+def load_sweep_config(sweep_config_path: str) -> Optional[Dict[str, Any]]:
+ """Load a sweep yaml from path."""
+ try:
+ yaml_file = open(sweep_config_path)
+ except OSError:
+ wandb.termerror(f"Couldn't open sweep file: {sweep_config_path}")
+ return None
+ try:
+ config: Optional[Dict[str, Any]] = yaml.safe_load(yaml_file)
+ except yaml.YAMLError as err:
+ wandb.termerror(f"Error in configuration file: {err}")
+ return None
+ if not config:
+ wandb.termerror("Configuration file is empty")
+ return None
+ return config
+
+
+def load_launch_sweep_config(config: Optional[str]) -> Any:
+ if not config:
+ return {}
+
+ parsed_config = util.load_json_yaml_dict(config)
+ if parsed_config is None:
+ raise LaunchError(f"Could not load config from {config}. Check formatting")
+ return parsed_config
+
+
+def construct_scheduler_args(
+ sweep_config: Dict[str, Any],
+ queue: str,
+ project: str,
+ author: Optional[str] = None,
+ return_job: bool = False,
+) -> Union[List[str], Dict[str, str], None]:
+ """Construct sweep scheduler args.
+
+ logs error and returns None if misconfigured,
+ otherwise returns args as a dict if is_job else a list of strings.
+ """
+ job = sweep_config.get("job")
+ image_uri = sweep_config.get("image_uri")
+ if not job and not image_uri: # don't allow empty string
+ wandb.termerror(
+ "No 'job' nor 'image_uri' top-level key found in sweep config, exactly one is required for a launch-sweep"
+ )
+ return None
+ elif job and image_uri:
+ wandb.termerror(
+ "Sweep config has both 'job' and 'image_uri' but a launch-sweep can use only one"
+ )
+ return None
+
+ # if scheduler is a job, return args as dict
+ if return_job:
+ args_dict: Dict[str, str] = {
+ "sweep_id": "WANDB_SWEEP_ID",
+ "queue": queue,
+ "project": project,
+ }
+ if job:
+ args_dict["job"] = job
+ elif image_uri:
+ args_dict["image_uri"] = image_uri
+
+ if author:
+ args_dict["author"] = author
+
+ return args_dict
+
+ # scheduler uses cli commands, pass args as param list
+ args = [
+ "--queue",
+ f"{queue!r}",
+ "--project",
+ f"{project!r}",
+ ]
+ if author:
+ args += [
+ "--author",
+ f"{author!r}",
+ ]
+ if job:
+ args += [
+ "--job",
+ f"{job!r}",
+ ]
+ elif image_uri:
+ args += ["--image_uri", image_uri]
+
+ return args
+
+
+def create_sweep_command(command: Optional[List] = None) -> List:
+ """Return sweep command, filling in environment variable macros."""
+ # Start from default sweep command
+ command = command or DEFAULT_SWEEP_COMMAND
+ for i, chunk in enumerate(command):
+ # Replace environment variable macros
+ # Search a str(chunk), but allow matches to be of any (ex: int) type
+ if SWEEP_COMMAND_ENV_VAR_REGEX.search(str(chunk)):
+ # Replace from backwards forwards
+ matches = list(SWEEP_COMMAND_ENV_VAR_REGEX.finditer(chunk))
+ for m in matches[::-1]:
+ # Default to just leaving as is if environment variable does not exist
+ _var: str = os.environ.get(m.group(1), m.group(1))
+ command[i] = f"{command[i][:m.start()]}{_var}{command[i][m.end():]}"
+ return command
+
+
+def create_sweep_command_args(command: Dict) -> Dict[str, Any]:
+ """Create various formats of command arguments for the agent.
+
+ Raises:
+ ValueError: improperly formatted command dict
+
+ """
+ if "args" not in command:
+ raise ValueError('No "args" found in command: %s' % command)
+ # four different formats of command args
+ # (1) standard command line flags (e.g. --foo=bar)
+ flags: List[str] = []
+ # (2) flags without hyphens (e.g. foo=bar)
+ flags_no_hyphens: List[str] = []
+ # (3) flags with false booleans ommited (e.g. --foo)
+ flags_no_booleans: List[str] = []
+ # (4) flags as a dictionary (used for constructing a json)
+ flags_dict: Dict[str, Any] = {}
+ # (5) flags without equals (e.g. --foo bar)
+ args_no_equals: List[str] = []
+ for param, config in command["args"].items():
+ # allow 'None' as a valid value, but error if no value is found
+ try:
+ _value: Any = config["value"]
+ except KeyError:
+ raise ValueError('No "value" found for command["args"]["%s"]' % param)
+
+ _flag: str = f"{param}={_value}"
+ flags.append("--" + _flag)
+ flags_no_hyphens.append(_flag)
+ args_no_equals += [f"--{param}", str(_value)]
+ if isinstance(_value, bool):
+ # omit flags if they are boolean and false
+ if _value:
+ flags_no_booleans.append("--" + param)
+ else:
+ flags_no_booleans.append("--" + _flag)
+ flags_dict[param] = _value
+ return {
+ "args": flags,
+ "args_no_equals": args_no_equals,
+ "args_no_hyphens": flags_no_hyphens,
+ "args_no_boolean_flags": flags_no_booleans,
+ "args_json": [json.dumps(flags_dict)],
+ "args_dict": flags_dict,
+ }
+
+
+def make_launch_sweep_entrypoint(
+ args: Dict[str, Any], command: Optional[List[str]]
+) -> Tuple[Optional[List[str]], Any]:
+ """Use args dict from create_sweep_command_args to construct entrypoint.
+
+ If replace is True, remove macros from entrypoint, fill them in with args
+ and then return the args in seperate return value.
+ """
+ if not command:
+ return None, None
+
+ entry_point = create_sweep_command(command)
+ macro_args = {}
+ for macro in args:
+ mstr = "${" + macro + "}"
+ if mstr in entry_point:
+ idx = entry_point.index(mstr)
+ # only supports 1 macro per entrypoint
+ macro_args = args[macro]
+ entry_point = entry_point[:idx] + entry_point[idx + 1 :]
+
+ if len(entry_point) == 0:
+ return None, macro_args
+
+ return entry_point, macro_args
+
+
+def check_job_exists(public_api: "PublicApi", job: Optional[str]) -> bool:
+ """Check if the job exists using the public api.
+
+ Returns: True if no job is passed, or if the job exists.
+ Returns: False if the job is misformatted or doesn't exist.
+ """
+ if not job:
+ return True
+
+ try:
+ public_api.job(job)
+ except Exception as e:
+ wandb.termerror(f"Failed to load job. {e}")
+ return False
+ return True
+
+
+def get_previous_args(
+ run_spec: Dict[str, Any]
+) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+ """Parse through previous scheduler run_spec.
+
+ returns scheduler_args and settings.
+ """
+ scheduler_args = (
+ run_spec.get("overrides", {}).get("run_config", {}).get("scheduler", {})
+ )
+ # also pipe through top level resource setup
+ if run_spec.get("resource"):
+ scheduler_args["resource"] = run_spec["resource"]
+ if run_spec.get("resource_args"):
+ scheduler_args["resource_args"] = run_spec["resource_args"]
+
+ settings = run_spec.get("overrides", {}).get("run_config", {}).get("settings", {})
+
+ return scheduler_args, settings
diff --git a/wandb/sdk/launch/utils.py b/wandb/sdk/launch/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..69f4e57bb3fbb23e01024ed2a3ac9508eefeb885
--- /dev/null
+++ b/wandb/sdk/launch/utils.py
@@ -0,0 +1,834 @@
+# heavily inspired by https://github.com/mlflow/mlflow/blob/master/mlflow/projects/utils.py
+import asyncio
+import json
+import logging
+import os
+import platform
+import re
+import subprocess
+import sys
+from collections import defaultdict
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
+
+import click
+
+import wandb
+import wandb.docker as docker
+from wandb import util
+from wandb.apis.internal import Api
+from wandb.errors import CommError
+from wandb.sdk.launch.errors import LaunchError
+from wandb.sdk.launch.git_reference import GitReference
+from wandb.sdk.launch.wandb_reference import WandbReference
+from wandb.sdk.wandb_config import Config
+
+from .builder.templates._wandb_bootstrap import (
+ FAILED_PACKAGES_POSTFIX,
+ FAILED_PACKAGES_PREFIX,
+)
+
+FAILED_PACKAGES_REGEX = re.compile(
+ f"{re.escape(FAILED_PACKAGES_PREFIX)}(.*){re.escape(FAILED_PACKAGES_POSTFIX)}"
+)
+
+if TYPE_CHECKING: # pragma: no cover
+ from wandb.sdk.artifacts.artifact import Artifact
+ from wandb.sdk.launch.agent.job_status_tracker import JobAndRunStatusTracker
+
+
+# TODO: this should be restricted to just Git repos and not S3 and stuff like that
+_GIT_URI_REGEX = re.compile(r"^[^/|^~|^\.].*(git|bitbucket)")
+_VALID_IP_REGEX = r"^https?://[0-9]+(?:\.[0-9]+){3}(:[0-9]+)?"
+_VALID_PIP_PACKAGE_REGEX = r"^[a-zA-Z0-9_.-]+$"
+_VALID_WANDB_REGEX = r"^https?://(api.)?wandb"
+_WANDB_URI_REGEX = re.compile(r"|".join([_VALID_WANDB_REGEX, _VALID_IP_REGEX]))
+_WANDB_QA_URI_REGEX = re.compile(
+ r"^https?://ap\w.qa.wandb"
+) # for testing, not sure if we wanna keep this
+_WANDB_DEV_URI_REGEX = re.compile(
+ r"^https?://ap\w.wandb.test"
+) # for testing, not sure if we wanna keep this
+_WANDB_LOCAL_DEV_URI_REGEX = re.compile(
+ r"^https?://localhost"
+) # for testing, not sure if we wanna keep this
+
+API_KEY_REGEX = r"WANDB_API_KEY=\w+(-\w+)?"
+
+MACRO_REGEX = re.compile(r"\$\{(\w+)\}")
+
+AZURE_CONTAINER_REGISTRY_URI_REGEX = re.compile(
+ r"(?:https://)?([\w]+)\.azurecr\.io/([\w\-]+):?(.*)"
+)
+
+ELASTIC_CONTAINER_REGISTRY_URI_REGEX = re.compile(
+ r"^(?P<account>.*)\.dkr\.ecr\.(?P<region>.*)\.amazonaws\.com/(?P<repository>.*)/?$"
+)
+
+GCP_ARTIFACT_REGISTRY_URI_REGEX = re.compile(
+ r"^(?P<region>[\w-]+)-docker\.pkg\.dev/(?P<project>[\w-]+)/(?P<repository>[\w-]+)/(?P<image_name>[\w-]+)$",
+ re.IGNORECASE,
+)
+
+S3_URI_RE = re.compile(r"s3://([^/]+)(/(.*))?")
+GCS_URI_RE = re.compile(r"gs://([^/]+)(?:/(.*))?")
+AZURE_BLOB_REGEX = re.compile(
+ r"^https://([^\.]+)\.blob\.core\.windows\.net/([^/]+)/?(.*)$"
+)
+
+
+PROJECT_SYNCHRONOUS = "SYNCHRONOUS"
+
+LAUNCH_CONFIG_FILE = "~/.config/wandb/launch-config.yaml"
+LAUNCH_DEFAULT_PROJECT = "model-registry"
+
+_logger = logging.getLogger(__name__)
+LOG_PREFIX = f"{click.style('launch:', fg='magenta')} "
+
+MAX_ENV_LENGTHS: Dict[str, int] = defaultdict(lambda: 32670)
+MAX_ENV_LENGTHS["SageMakerRunner"] = 512
+
+
+def load_wandb_config() -> Config:
+ """Load wandb config from WANDB_CONFIG environment variable(s).
+
+ The WANDB_CONFIG environment variable is a json string that can contain
+ multiple config keys. The WANDB_CONFIG_[0-9]+ environment variables are
+ used for environments where there is a limit on the length of environment
+ variables. In that case, we shard the contents of WANDB_CONFIG into
+ multiple environment variables numbered from 0.
+
+ Returns:
+ A dictionary of wandb config values.
+ """
+ config_str = os.environ.get("WANDB_CONFIG")
+ if config_str is None:
+ config_str = ""
+ idx = 0
+ while True:
+ chunk = os.environ.get(f"WANDB_CONFIG_{idx}")
+ if chunk is None:
+ break
+ config_str += chunk
+ idx += 1
+ if idx < 1:
+ raise LaunchError(
+ "No WANDB_CONFIG or WANDB_CONFIG_[0-9]+ environment variables found"
+ )
+ wandb_config = Config()
+ try:
+ env_config = json.loads(config_str)
+ except json.JSONDecodeError as e:
+ raise LaunchError(f"Failed to parse WANDB_CONFIG: {e}") from e
+
+ wandb_config.update(env_config)
+ return wandb_config
+
+
+def event_loop_thread_exec(func: Any) -> Any:
+ """Wrapper for running any function in an awaitable thread on an event loop.
+
+ Example usage:
+ ```
+ def my_func(arg1, arg2):
+ return arg1 + arg2
+
+ future = event_loop_thread_exec(my_func)(2, 2)
+ assert await future == 4
+ ```
+
+ The returned function must be called within an active event loop.
+ """
+
+ async def wrapper(*args: Any, **kwargs: Any) -> Any:
+ loop = asyncio.get_event_loop()
+ result = cast(
+ Any, await loop.run_in_executor(None, lambda: func(*args, **kwargs))
+ )
+ return result
+
+ return wrapper
+
+
+def _is_wandb_uri(uri: str) -> bool:
+ return (
+ _WANDB_URI_REGEX.match(uri)
+ or _WANDB_DEV_URI_REGEX.match(uri)
+ or _WANDB_LOCAL_DEV_URI_REGEX.match(uri)
+ or _WANDB_QA_URI_REGEX.match(uri)
+ ) is not None
+
+
+def _is_wandb_dev_uri(uri: str) -> bool:
+ return bool(_WANDB_DEV_URI_REGEX.match(uri))
+
+
+def _is_wandb_local_uri(uri: str) -> bool:
+ return bool(_WANDB_LOCAL_DEV_URI_REGEX.match(uri))
+
+
+def _is_git_uri(uri: str) -> bool:
+ return bool(_GIT_URI_REGEX.match(uri))
+
+
+def sanitize_wandb_api_key(s: str) -> str:
+ return str(re.sub(API_KEY_REGEX, "WANDB_API_KEY", s))
+
+
+def get_project_from_job(job: str) -> Optional[str]:
+ job_parts = job.split("/")
+ if len(job_parts) == 3:
+ return job_parts[1]
+ return None
+
+
+def set_project_entity_defaults(
+ uri: Optional[str],
+ job: Optional[str],
+ api: Api,
+ project: Optional[str],
+ entity: Optional[str],
+ launch_config: Optional[Dict[str, Any]],
+) -> Tuple[Optional[str], str]:
+ # set the target project and entity if not provided
+ source_uri = None
+ if uri is not None:
+ if _is_wandb_uri(uri):
+ _, source_uri, _ = parse_wandb_uri(uri)
+ elif _is_git_uri(uri):
+ source_uri = os.path.splitext(os.path.basename(uri))[0]
+ elif job is not None:
+ source_uri = get_project_from_job(job)
+ if project is None:
+ config_project = None
+ if launch_config:
+ config_project = launch_config.get("project")
+ project = config_project or source_uri or ""
+ if entity is None:
+ entity = get_default_entity(api, launch_config)
+ prefix = ""
+ if platform.system() != "Windows" and sys.stdout.encoding == "UTF-8":
+ prefix = "🚀 "
+ wandb.termlog(
+ f"{LOG_PREFIX}{prefix}Launching run into {entity}{'/' + project if project else ''}"
+ )
+ return project, entity
+
+
+def get_default_entity(api: Api, launch_config: Optional[Dict[str, Any]]):
+ config_entity = None
+ if launch_config:
+ config_entity = launch_config.get("entity")
+ return config_entity or api.default_entity
+
+
+def construct_launch_spec(
+ uri: Optional[str],
+ job: Optional[str],
+ api: Api,
+ name: Optional[str],
+ project: Optional[str],
+ entity: Optional[str],
+ docker_image: Optional[str],
+ resource: Optional[str],
+ entry_point: Optional[List[str]],
+ version: Optional[str],
+ resource_args: Optional[Dict[str, Any]],
+ launch_config: Optional[Dict[str, Any]],
+ run_id: Optional[str],
+ repository: Optional[str],
+ author: Optional[str],
+ sweep_id: Optional[str] = None,
+) -> Dict[str, Any]:
+ """Construct the launch specification from CLI arguments."""
+ # override base config (if supplied) with supplied args
+ launch_spec = launch_config if launch_config is not None else {}
+ if uri is not None:
+ launch_spec["uri"] = uri
+ if job is not None:
+ launch_spec["job"] = job
+ project, entity = set_project_entity_defaults(
+ uri,
+ job,
+ api,
+ project,
+ entity,
+ launch_config,
+ )
+ launch_spec["entity"] = entity
+ if author:
+ launch_spec["author"] = author
+
+ launch_spec["project"] = project
+ if name:
+ launch_spec["name"] = name
+ if "docker" not in launch_spec:
+ launch_spec["docker"] = {}
+ if docker_image:
+ launch_spec["docker"]["docker_image"] = docker_image
+ if sweep_id: # all runs in a sweep have this set
+ launch_spec["sweep_id"] = sweep_id
+
+ if "resource" not in launch_spec:
+ launch_spec["resource"] = resource if resource else None
+
+ if "git" not in launch_spec:
+ launch_spec["git"] = {}
+ if version:
+ launch_spec["git"]["version"] = version
+
+ if "overrides" not in launch_spec:
+ launch_spec["overrides"] = {}
+
+ if not isinstance(launch_spec["overrides"].get("args", []), list):
+ raise LaunchError("override args must be a list of strings")
+
+ if resource_args:
+ launch_spec["resource_args"] = resource_args
+
+ if entry_point:
+ launch_spec["overrides"]["entry_point"] = entry_point
+
+ if run_id is not None:
+ launch_spec["run_id"] = run_id
+
+ if repository:
+ launch_config = launch_config or {}
+ if launch_config.get("registry"):
+ launch_config["registry"]["url"] = repository
+ else:
+ launch_config["registry"] = {"url": repository}
+
+ return launch_spec
+
+
+def validate_launch_spec_source(launch_spec: Dict[str, Any]) -> None:
+ uri = launch_spec.get("uri")
+ job = launch_spec.get("job")
+ docker_image = launch_spec.get("docker", {}).get("docker_image")
+
+ if not bool(uri) and not bool(job) and not bool(docker_image):
+ raise LaunchError("Must specify a uri, job or docker image")
+ elif bool(uri) and bool(docker_image):
+ raise LaunchError("Found both uri and docker-image, only one can be set")
+ elif sum(map(bool, [uri, job, docker_image])) > 1:
+ raise LaunchError("Must specify exactly one of uri, job or image")
+
+
+def parse_wandb_uri(uri: str) -> Tuple[str, str, str]:
+ """Parse wandb uri to retrieve entity, project and run name."""
+ ref = WandbReference.parse(uri)
+ if not ref or not ref.entity or not ref.project or not ref.run_id:
+ raise LaunchError(f"Trouble parsing wandb uri {uri}")
+ return (ref.entity, ref.project, ref.run_id)
+
+
+def is_bare_wandb_uri(uri: str) -> bool:
+ """Check that a wandb uri is valid.
+
+ URI must be in the format
+ `/<entity>/<project>/runs/<run_name>[other stuff]`
+ or
+ `/<entity>/<project>/artifacts/job/<job_name>[other stuff]`.
+ """
+ _logger.info(f"Checking if uri {uri} is bare...")
+ return uri.startswith("/") and WandbReference.is_uri_job_or_run(uri)
+
+
+def fetch_wandb_project_run_info(
+ entity: str, project: str, run_name: str, api: Api
+) -> Any:
+ _logger.info("Fetching run info...")
+ try:
+ result = api.get_run_info(entity, project, run_name)
+ except CommError:
+ result = None
+ if result is None:
+ raise LaunchError(
+ f"Run info is invalid or doesn't exist for {api.settings('base_url')}/{entity}/{project}/runs/{run_name}"
+ )
+ if result.get("codePath") is None:
+ # TODO: we don't currently expose codePath in the runInfo endpoint, this downloads
+ # it from wandb-metadata.json if we can.
+ metadata = api.download_url(
+ project, "wandb-metadata.json", run=run_name, entity=entity
+ )
+ if metadata is not None:
+ _, response = api.download_file(metadata["url"])
+ data = response.json()
+ result["codePath"] = data.get("codePath")
+ result["cudaVersion"] = data.get("cuda", None)
+
+ return result
+
+
+def download_entry_point(
+ entity: str, project: str, run_name: str, api: Api, entry_point: str, dir: str
+) -> bool:
+ metadata = api.download_url(
+ project, f"code/{entry_point}", run=run_name, entity=entity
+ )
+ if metadata is not None:
+ _, response = api.download_file(metadata["url"])
+ with util.fsync_open(os.path.join(dir, entry_point), "wb") as file:
+ for data in response.iter_content(chunk_size=1024):
+ file.write(data)
+ return True
+ return False
+
+
+def download_wandb_python_deps(
+ entity: str, project: str, run_name: str, api: Api, dir: str
+) -> Optional[str]:
+ reqs = api.download_url(project, "requirements.txt", run=run_name, entity=entity)
+ if reqs is not None:
+ _logger.info("Downloading python dependencies")
+ _, response = api.download_file(reqs["url"])
+
+ with util.fsync_open(
+ os.path.join(dir, "requirements.frozen.txt"), "wb"
+ ) as file:
+ for data in response.iter_content(chunk_size=1024):
+ file.write(data)
+ return "requirements.frozen.txt"
+ return None
+
+
+def get_local_python_deps(
+ dir: str, filename: str = "requirements.local.txt"
+) -> Optional[str]:
+ try:
+ env = os.environ
+ with open(os.path.join(dir, filename), "w") as f:
+ subprocess.call(["pip", "freeze"], env=env, stdout=f)
+ return filename
+ except subprocess.CalledProcessError as e:
+ wandb.termerror(f"Command failed: {e}")
+ return None
+
+
+def diff_pip_requirements(req_1: List[str], req_2: List[str]) -> Dict[str, str]:
+ """Return a list of pip requirements that are not in req_1 but are in req_2."""
+
+ def _parse_req(req: List[str]) -> Dict[str, str]:
+ # TODO: This can be made more exhaustive, but for 99% of cases this is fine
+ # see https://pip.pypa.io/en/stable/reference/requirements-file-format/#example
+ d: Dict[str, str] = dict()
+ for line in req:
+ _name: str = None # type: ignore
+ _version: str = None # type: ignore
+ if line.startswith("#"): # Ignore comments
+ continue
+ elif "git+" in line or "hg+" in line:
+ _name = line.split("#egg=")[1]
+ _version = line.split("@")[-1].split("#")[0]
+ elif "==" in line:
+ _s = line.split("==")
+ _name = _s[0].lower()
+ _version = _s[1].split("#")[0].strip()
+ elif ">=" in line:
+ _s = line.split(">=")
+ _name = _s[0].lower()
+ _version = _s[1].split("#")[0].strip()
+ elif ">" in line:
+ _s = line.split(">")
+ _name = _s[0].lower()
+ _version = _s[1].split("#")[0].strip()
+ elif re.match(_VALID_PIP_PACKAGE_REGEX, line) is not None:
+ _name = line
+ else:
+ raise ValueError(f"Unable to parse pip requirements file line: {line}")
+ if _name is not None:
+ assert re.match(
+ _VALID_PIP_PACKAGE_REGEX, _name
+ ), f"Invalid pip package name {_name}"
+ d[_name] = _version
+ return d
+
+ # Use symmetric difference between dict representation to print errors
+ try:
+ req_1_dict: Dict[str, str] = _parse_req(req_1)
+ req_2_dict: Dict[str, str] = _parse_req(req_2)
+ except (AssertionError, ValueError, IndexError, KeyError) as e:
+ raise LaunchError(f"Failed to parse pip requirements: {e}")
+ diff: List[Tuple[str, str]] = []
+ for item in set(req_1_dict.items()) ^ set(req_2_dict.items()):
+ diff.append(item)
+ # Parse through the diff to make it pretty
+ pretty_diff: Dict[str, str] = {}
+ for name, version in diff:
+ if pretty_diff.get(name) is None:
+ pretty_diff[name] = version
+ else:
+ pretty_diff[name] = f"v{version} and v{pretty_diff[name]}"
+ return pretty_diff
+
+
+def validate_wandb_python_deps(
+ requirements_file: Optional[str],
+ dir: str,
+) -> None:
+ """Warn if local python dependencies differ from wandb requirements.txt."""
+ if requirements_file is not None:
+ requirements_path = os.path.join(dir, requirements_file)
+ with open(requirements_path) as f:
+ wandb_python_deps: List[str] = f.read().splitlines()
+
+ local_python_file = get_local_python_deps(dir)
+ if local_python_file is not None:
+ local_python_deps_path = os.path.join(dir, local_python_file)
+ with open(local_python_deps_path) as f:
+ local_python_deps: List[str] = f.read().splitlines()
+
+ diff_pip_requirements(wandb_python_deps, local_python_deps)
+ return
+ _logger.warning("Unable to validate local python dependencies")
+
+
+def fetch_project_diff(
+ entity: str, project: str, run_name: str, api: Api
+) -> Optional[str]:
+ """Fetches project diff from wandb servers."""
+ _logger.info("Searching for diff.patch")
+ patch = None
+ try:
+ (_, _, patch, _) = api.run_config(project, run_name, entity)
+ except CommError:
+ pass
+ return patch
+
+
+def apply_patch(patch_string: str, dst_dir: str) -> None:
+ """Applies a patch file to a directory."""
+ _logger.info("Applying diff.patch")
+ with open(os.path.join(dst_dir, "diff.patch"), "w") as fp:
+ fp.write(patch_string)
+ try:
+ subprocess.check_call(
+ [
+ "patch",
+ "-s",
+ f"--directory={dst_dir}",
+ "-p1",
+ "-i",
+ "diff.patch",
+ ]
+ )
+ except subprocess.CalledProcessError:
+ raise wandb.Error("Failed to apply diff.patch associated with run.")
+
+
+def _make_refspec_from_version(version: Optional[str]) -> List[str]:
+ """Create a refspec that checks for the existence of origin/main and the version."""
+ if version:
+ return [f"+{version}"]
+
+ return [
+ "+refs/heads/main*:refs/remotes/origin/main*",
+ "+refs/heads/master*:refs/remotes/origin/master*",
+ ]
+
+
+def _fetch_git_repo(dst_dir: str, uri: str, version: Optional[str]) -> Optional[str]:
+ """Clones the git repo at ``uri`` into ``dst_dir``.
+
+ checks out commit ``version``. Assumes authentication parameters are
+ specified by the environment, e.g. by a Git credential helper.
+ """
+ # We defer importing git until the last moment, because the import requires that the git
+ # executable is available on the PATH, so we only want to fail if we actually need it.
+
+ _logger.info("Fetching git repo")
+ ref = GitReference(uri, version)
+ if ref is None:
+ raise LaunchError(f"Unable to parse git uri: {uri}")
+ ref.fetch(dst_dir)
+ if version is None:
+ version = ref.ref
+ return version
+
+
+def merge_parameters(
+ higher_priority_params: Dict[str, Any], lower_priority_params: Dict[str, Any]
+) -> Dict[str, Any]:
+ """Merge the contents of two dicts, keeping values from higher_priority_params if there are conflicts."""
+ return {**lower_priority_params, **higher_priority_params}
+
+
+def convert_jupyter_notebook_to_script(fname: str, project_dir: str) -> str:
+ nbconvert = wandb.util.get_module(
+ "nbconvert", "nbformat and nbconvert are required to use launch with notebooks"
+ )
+ nbformat = wandb.util.get_module(
+ "nbformat", "nbformat and nbconvert are required to use launch with notebooks"
+ )
+
+ _logger.info("Converting notebook to script")
+ new_name = fname.replace(".ipynb", ".py")
+ with open(os.path.join(project_dir, fname)) as fh:
+ nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT)
+ for cell in nb.cells:
+ if cell.cell_type == "code":
+ source_lines = cell.source.split("\n")
+ modified_lines = []
+ for line in source_lines:
+ if not line.startswith("!"):
+ modified_lines.append(line)
+ cell.source = "\n".join(modified_lines)
+
+ exporter = nbconvert.PythonExporter()
+ source, meta = exporter.from_notebook_node(nb)
+
+ with open(os.path.join(project_dir, new_name), "w+") as fh:
+ fh.writelines(source)
+ return new_name
+
+
+def check_and_download_code_artifacts(
+ entity: str, project: str, run_name: str, internal_api: Api, project_dir: str
+) -> Optional["Artifact"]:
+ _logger.info("Checking for code artifacts")
+ public_api = wandb.PublicApi(
+ overrides={"base_url": internal_api.settings("base_url")}
+ )
+
+ run = public_api.run(f"{entity}/{project}/{run_name}")
+ run_artifacts = run.logged_artifacts()
+
+ for artifact in run_artifacts:
+ if hasattr(artifact, "type") and artifact.type == "code":
+ artifact.download(project_dir)
+ return artifact # type: ignore
+
+ return None
+
+
+def to_camel_case(maybe_snake_str: str) -> str:
+ if "_" not in maybe_snake_str:
+ return maybe_snake_str
+ components = maybe_snake_str.split("_")
+ return "".join(x.title() if x else "_" for x in components)
+
+
+def run_shell(args: List[str]) -> Tuple[str, str]:
+ out = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ return out.stdout.decode("utf-8").strip(), out.stderr.decode("utf-8").strip()
+
+
+def validate_build_and_registry_configs(
+ build_config: Dict[str, Any], registry_config: Dict[str, Any]
+) -> None:
+ build_config_credentials = build_config.get("credentials", {})
+ registry_config_credentials = registry_config.get("credentials", {})
+ if (
+ build_config_credentials
+ and registry_config_credentials
+ and build_config_credentials != registry_config_credentials
+ ):
+ raise LaunchError("registry and build config credential mismatch")
+
+
+async def get_kube_context_and_api_client(
+ kubernetes: Any,
+ resource_args: Dict[str, Any],
+) -> Tuple[Any, Any]:
+ config_file = resource_args.get("configFile", None)
+ context = None
+ if config_file is not None or os.path.exists(os.path.expanduser("~/.kube/config")):
+ # context only exist in the non-incluster case
+ (
+ all_contexts,
+ active_context,
+ ) = kubernetes.config.list_kube_config_contexts(config_file)
+ context = None
+ if resource_args.get("context"):
+ context_name = resource_args["context"]
+ for c in all_contexts:
+ if c["name"] == context_name:
+ context = c
+ break
+ raise LaunchError(f"Specified context {context_name} was not found.")
+ else:
+ context = active_context
+ # TODO: We should not really be performing this check if the user is not
+ # using EKS but I don't see an obvious way to make an eks specific code path
+ # right here.
+ util.get_module(
+ "awscli",
+ "awscli is required to load a kubernetes context "
+ "from eks. Please run `pip install wandb[launch]` to install it.",
+ )
+ await kubernetes.config.load_kube_config(config_file, context["name"])
+ api_client = await kubernetes.config.new_client_from_config(
+ config_file, context=context["name"]
+ )
+ return context, api_client
+ else:
+ kubernetes.config.load_incluster_config()
+ api_client = kubernetes.client.api_client.ApiClient()
+ return context, api_client
+
+
+def resolve_build_and_registry_config(
+ default_launch_config: Optional[Dict[str, Any]],
+ build_config: Optional[Dict[str, Any]],
+ registry_config: Optional[Dict[str, Any]],
+) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+ resolved_build_config: Dict[str, Any] = {}
+ if build_config is None and default_launch_config is not None:
+ resolved_build_config = default_launch_config.get("builder", {})
+ elif build_config is not None:
+ resolved_build_config = build_config
+ resolved_registry_config: Dict[str, Any] = {}
+ if registry_config is None and default_launch_config is not None:
+ resolved_registry_config = default_launch_config.get("registry", {})
+ elif registry_config is not None:
+ resolved_registry_config = registry_config
+ validate_build_and_registry_configs(resolved_build_config, resolved_registry_config)
+ return resolved_build_config, resolved_registry_config
+
+
+def check_logged_in(api: Api) -> bool:
+ """Check if a user is logged in.
+
+ Raises an error if the viewer doesn't load (likely a broken API key). Expected time
+ cost is 0.1-0.2 seconds.
+ """
+ res = api.api.viewer()
+ if not res:
+ raise LaunchError(
+ "Could not connect with current API-key. "
+ "Please relogin using `wandb login --relogin`"
+ " and try again (see `wandb login --help` for more options)"
+ )
+
+ return True
+
+
+def make_name_dns_safe(name: str) -> str:
+ resp = name.replace("_", "-").lower()
+ resp = re.sub(r"[^a-z\.\-]", "", resp)
+ # Actual length limit is 253, but we want to leave room for the generated suffix
+ resp = resp[:200]
+ return resp
+
+
+def warn_failed_packages_from_build_logs(
+ log: str, image_uri: str, api: Api, job_tracker: Optional["JobAndRunStatusTracker"]
+) -> None:
+ match = FAILED_PACKAGES_REGEX.search(log)
+ if match:
+ _msg = f"Failed to install the following packages: {match.group(1)} for image: {image_uri}. Will attempt to launch image without them."
+ wandb.termwarn(_msg)
+ if job_tracker is not None:
+ res = job_tracker.saver.save_contents(
+ _msg, "failed-packages.log", "warning"
+ )
+ api.update_run_queue_item_warning(
+ job_tracker.run_queue_item_id,
+ "Some packages were not successfully installed during the build",
+ "build",
+ res,
+ )
+
+
+def docker_image_exists(docker_image: str, should_raise: bool = False) -> bool:
+ """Check if a specific image is already available.
+
+ Optionally raises an exception if the image is not found.
+ """
+ _logger.info("Checking if base image exists...")
+ try:
+ docker.run(["docker", "image", "inspect", docker_image])
+ return True
+ except (docker.DockerError, ValueError) as e:
+ if should_raise:
+ raise e
+ _logger.info("Base image not found. Generating new base image")
+ return False
+
+
+def pull_docker_image(docker_image: str) -> None:
+ """Pull the requested docker image."""
+ try:
+ docker.run(["docker", "pull", docker_image])
+ except docker.DockerError as e:
+ raise LaunchError(f"Docker server returned error: {e}")
+
+
+def macro_sub(original: str, sub_dict: Dict[str, Optional[str]]) -> str:
+ """Substitute macros in a string.
+
+ Macros occur in the string in the ${macro} format. The macro names are
+ substituted with their values from the given dictionary. If a macro
+ is not found in the dictionary, it is left unchanged.
+
+ Args:
+ original: The string to substitute macros in.
+ sub_dict: A dictionary mapping macro names to their values.
+
+ Returns:
+ The string with the macros substituted.
+ """
+ return MACRO_REGEX.sub(
+ lambda match: str(sub_dict.get(match.group(1), match.group(0))), original
+ )
+
+
+def recursive_macro_sub(source: Any, sub_dict: Dict[str, Optional[str]]) -> Any:
+ """Recursively substitute macros in a parsed JSON or YAML blob.
+
+ Macros occur in strings at leaves of the blob in the ${macro} format.
+ The macro names are substituted with their values from the given dictionary.
+ If a macro is not found in the dictionary, it is left unchanged.
+
+ Arguments:
+ source: The JSON or YAML blob to substitute macros in.
+ sub_dict: A dictionary mapping macro names to their values.
+
+ Returns:
+ The blob with the macros substituted.
+ """
+ if isinstance(source, str):
+ return macro_sub(source, sub_dict)
+ elif isinstance(source, list):
+ return [recursive_macro_sub(item, sub_dict) for item in source]
+ elif isinstance(source, dict):
+ return {
+ key: recursive_macro_sub(value, sub_dict) for key, value in source.items()
+ }
+ else:
+ return source
+
+
+def fetch_and_validate_template_variables(
+ runqueue: Any, fields: dict
+) -> Dict[str, Any]:
+ template_variables = {}
+
+ variable_schemas = {}
+ for tv in runqueue.template_variables:
+ variable_schemas[tv["name"]] = json.loads(tv["schema"])
+
+ for field in fields:
+ field_parts = field.split("=")
+ if len(field_parts) != 2:
+ raise LaunchError(
+ f'--set-var value must be in the format "--set-var key1=value1", instead got: {field}'
+ )
+ key, val = field_parts
+ if key not in variable_schemas:
+ raise LaunchError(
+ f"Queue {runqueue.name} does not support overriding {key}."
+ )
+ schema = variable_schemas.get(key, {})
+ field_type = schema.get("type")
+ try:
+ if field_type == "integer":
+ val = int(val)
+ elif field_type == "number":
+ val = float(val)
+
+ except ValueError:
+ raise LaunchError(f"Value for {key} must be of type {field_type}.")
+ template_variables[key] = val
+ return template_variables
diff --git a/wandb/sdk/launch/wandb_reference.py b/wandb/sdk/launch/wandb_reference.py
new file mode 100644
index 0000000000000000000000000000000000000000..5de34c04bf3aa77068da19cb6b4af8f1e163709d
--- /dev/null
+++ b/wandb/sdk/launch/wandb_reference.py
@@ -0,0 +1,138 @@
+"""Support for parsing W&B URLs (which might be user provided) into constituent parts."""
+
+from dataclasses import dataclass
+from enum import IntEnum
+from typing import Optional
+from urllib.parse import urlparse
+
+PREFIX_HTTP = "http://"
+PREFIX_HTTPS = "https://"
+
+
+class ReferenceType(IntEnum):
+ RUN = 1
+ JOB = 2
+
+
+# Ideally we would not overload the URL paths as we do.
+# TODO: Not sure these are exhaustive, and even if so more special paths might get added.
+# Would be good to have restrictions that we could check.
+RESERVED_NON_ENTITIES = (
+ "create-team",
+ "fully-connected",
+ "registry",
+ "settings",
+ "subscriptions",
+)
+RESERVED_NON_PROJECTS = (
+ "likes",
+ "projects",
+)
+RESERVED_JOB_PATHS = ("_view",)
+
+
+@dataclass
+class WandbReference:
+ # TODO: This will include port, should we separate that out?
+ host: Optional[str] = None
+
+ entity: Optional[str] = None
+ project: Optional[str] = None
+
+ # Set when we don't know how to parse yet
+ path: Optional[str] = None
+
+ # Reference type will determine what other fields are set
+ ref_type: Optional[ReferenceType] = None
+
+ run_id: Optional[str] = None
+
+ job_name: Optional[str] = None
+ job_alias: str = "latest" # In addition to an alias can be a version specifier
+
+ def is_bare(self) -> bool:
+ return self.host is None
+
+ def is_job(self) -> bool:
+ return self.ref_type == ReferenceType.JOB
+
+ def is_run(self) -> bool:
+ return self.ref_type == ReferenceType.RUN
+
+ def is_job_or_run(self) -> bool:
+ return self.is_job() or self.is_run()
+
+ def job_reference(self) -> str:
+ assert self.is_job()
+ return f"{self.job_name}:{self.job_alias}"
+
+ def job_reference_scoped(self) -> str:
+ assert self.entity
+ assert self.project
+ unscoped = self.job_reference()
+ return f"{self.entity}/{self.project}/{unscoped}"
+
+ def url_host(self) -> str:
+ return f"{PREFIX_HTTPS}{self.host}" if self.host else ""
+
+ def url_entity(self) -> str:
+ assert self.entity
+ return f"{self.url_host()}/{self.entity}"
+
+ def url_project(self) -> str:
+ assert self.project
+ return f"{self.url_entity()}/{self.project}"
+
+ @staticmethod
+ def parse(uri: str) -> Optional["WandbReference"]:
+ """Attempt to parse a string as a W&B URL."""
+ # TODO: Error if HTTP and host is not localhost?
+ if (
+ not uri.startswith("/")
+ and not uri.startswith(PREFIX_HTTP)
+ and not uri.startswith(PREFIX_HTTPS)
+ ):
+ return None
+
+ ref = WandbReference()
+
+ # This takes care of things like query and fragment
+ parsed = urlparse(uri)
+ if parsed.netloc:
+ ref.host = parsed.netloc
+
+ if not parsed.path.startswith("/"):
+ return ref
+
+ ref.path = parsed.path[1:]
+ parts = ref.path.split("/")
+ if len(parts) > 0:
+ if parts[0] not in RESERVED_NON_ENTITIES:
+ ref.path = None
+ ref.entity = parts[0]
+ if len(parts) > 1:
+ if parts[1] not in RESERVED_NON_PROJECTS:
+ ref.project = parts[1]
+ if len(parts) > 3 and parts[2] == "runs":
+ ref.ref_type = ReferenceType.RUN
+ ref.run_id = parts[3]
+ elif (
+ len(parts) > 4
+ and parts[2] == "artifacts"
+ and parts[3] == "job"
+ ):
+ ref.ref_type = ReferenceType.JOB
+ ref.job_name = parts[4]
+ if len(parts) > 5 and parts[5] not in RESERVED_JOB_PATHS:
+ ref.job_alias = parts[5]
+ # TODO: Right now we are not tracking selection as part of URL state in the Jobs tab.
+ # If that changes we'll want to update this.
+
+ return ref
+
+ @staticmethod
+ def is_uri_job_or_run(uri: str) -> bool:
+ ref = WandbReference.parse(uri)
+ if ref and ref.is_job_or_run():
+ return True
+ return False
diff --git a/wandb/sdk/lib/__init__.py b/wandb/sdk/lib/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..86f8867913dcfee60638e8f493ddc5bbb9dd3eee
--- /dev/null
+++ b/wandb/sdk/lib/__init__.py
@@ -0,0 +1,8 @@
+from . import lazyloader
+from .disabled import RunDisabled, SummaryDisabled
+
+__all__ = (
+ "lazyloader",
+ "RunDisabled",
+ "SummaryDisabled",
+)
diff --git a/wandb/sdk/lib/_settings_toposort_generate.py b/wandb/sdk/lib/_settings_toposort_generate.py
new file mode 100644
index 0000000000000000000000000000000000000000..9afe707d673fd282a1d1af44d74dae1b272b7d5f
--- /dev/null
+++ b/wandb/sdk/lib/_settings_toposort_generate.py
@@ -0,0 +1,159 @@
+import inspect
+import sys
+from typing import Dict, List, Set, Tuple
+
+from wandb.errors import UsageError
+from wandb.sdk.wandb_settings import Settings
+
+if sys.version_info >= (3, 8):
+ from typing import get_type_hints
+else:
+ from typing_extensions import get_type_hints
+
+
+template = """
+__all__ = ("SETTINGS_TOPOLOGICALLY_SORTED", "_Setting")
+
+import sys
+from typing import Tuple
+
+if sys.version_info >= (3, 8):
+ from typing import Final, Literal
+else:
+ from typing_extensions import Final, Literal
+
+
+_Setting = Literal[
+ $settings_literal_list
+]
+
+SETTINGS_TOPOLOGICALLY_SORTED: Final[Tuple[_Setting, ...]] = (
+ $settings_topologically_sorted
+)
+"""
+
+
+class Graph:
+ # A simple class representing an unweighted directed graph
+ # that uses an adjacency list representation.
+ # We use to ensure that we don't have cyclic dependencies in the settings
+ # and that modifications to the settings are applied in the correct order.
+ def __init__(self) -> None:
+ self.adj_list: Dict[str, Set[str]] = {}
+
+ def add_node(self, node: str) -> None:
+ if node not in self.adj_list:
+ self.adj_list[node] = set()
+
+ def add_edge(self, node1: str, node2: str) -> None:
+ self.adj_list[node1].add(node2)
+
+ def get_neighbors(self, node: str) -> Set[str]:
+ return self.adj_list[node]
+
+ # return a list of nodes sorted in topological order
+ def topological_sort_dfs(self) -> List[str]:
+ sorted_copy = {k: sorted(v) for k, v in self.adj_list.items()}
+
+ sorted_nodes: List[str] = []
+ visited_nodes: Set[str] = set()
+ current_nodes: Set[str] = set()
+
+ def visit(n: str) -> None:
+ if n in visited_nodes:
+ return None
+ if n in current_nodes:
+ raise UsageError("Cyclic dependency detected in wandb.Settings")
+
+ current_nodes.add(n)
+ for neighbor in sorted_copy[n]:
+ visit(neighbor)
+
+ current_nodes.remove(n)
+ visited_nodes.add(n)
+ sorted_nodes.append(n)
+
+ return None
+
+ for node in self.adj_list:
+ if node not in visited_nodes:
+ visit(node)
+
+ return sorted_nodes
+
+
+def _get_modification_order(
+ settings: Settings,
+) -> Tuple[Tuple[str, ...], Tuple[str, ...]]:
+ """Return the order in which settings should be modified, based on dependencies."""
+ dependency_graph = Graph()
+
+ props = tuple(get_type_hints(Settings).keys())
+
+ # discover prop dependencies from validator methods and runtime hooks
+
+ prefix = "_validate_"
+ symbols = set(dir(settings))
+ validator_methods = tuple(sorted(m for m in symbols if m.startswith(prefix)))
+
+ # extract dependencies from validator methods
+ for m in validator_methods:
+ setting = m.split(prefix)[1]
+ dependency_graph.add_node(setting)
+ # if the method is not static, inspect its code to find the attributes it depends on
+ if (
+ not isinstance(Settings.__dict__[m], staticmethod)
+ and not isinstance(Settings.__dict__[m], classmethod)
+ and Settings.__dict__[m].__code__.co_argcount > 0
+ ):
+ unbound_closure_vars = inspect.getclosurevars(Settings.__dict__[m]).unbound
+ dependencies = (v for v in unbound_closure_vars if v in props)
+ for d in dependencies:
+ dependency_graph.add_node(d)
+ dependency_graph.add_edge(setting, d)
+
+ # extract dependencies from props' runtime hooks
+ default_props = settings._default_props()
+ for prop, spec in default_props.items():
+ if "hook" not in spec:
+ continue
+
+ dependency_graph.add_node(prop)
+
+ hook = spec["hook"]
+ if callable(hook):
+ hook = [hook]
+
+ for h in hook:
+ unbound_closure_vars = inspect.getclosurevars(h).unbound
+ dependencies = (v for v in unbound_closure_vars if v in props)
+ for d in dependencies:
+ dependency_graph.add_node(d)
+ dependency_graph.add_edge(prop, d)
+
+ modification_order = dependency_graph.topological_sort_dfs()
+ return props, tuple(modification_order)
+
+
+def generate(settings: Settings) -> None:
+ _settings_literal_list, _settings_topologically_sorted = _get_modification_order(
+ settings
+ )
+ settings_literal_list = ", ".join(f'"{s}"' for s in _settings_literal_list)
+ settings_topologically_sorted = ", ".join(
+ f'"{s}"' for s in _settings_topologically_sorted
+ )
+
+ print(
+ template.replace(
+ "$settings_literal_list",
+ settings_literal_list,
+ ).replace(
+ "$settings_topologically_sorted",
+ settings_topologically_sorted,
+ )
+ )
+
+
+if __name__ == "__main__":
+ generate(Settings())
diff --git a/wandb/sdk/lib/_settings_toposort_generated.py b/wandb/sdk/lib/_settings_toposort_generated.py
new file mode 100644
index 0000000000000000000000000000000000000000..d6bb331b16f7a057c12fb0bcefe85a5a383dfdef
--- /dev/null
+++ b/wandb/sdk/lib/_settings_toposort_generated.py
@@ -0,0 +1,240 @@
+# DO NOT EDIT -- GENERATED BY: `generate-tool.py --generate`
+__all__ = ("SETTINGS_TOPOLOGICALLY_SORTED", "_Setting")
+
+import sys
+from typing import Tuple
+
+if sys.version_info >= (3, 8):
+ from typing import Final, Literal
+else:
+ from typing_extensions import Final, Literal
+
+
+_Setting = Literal[
+ "_args",
+ "_aws_lambda",
+ "_async_upload_concurrency_limit",
+ "_cli_only_mode",
+ "_colab",
+ "_cuda",
+ "_disable_meta",
+ "_disable_service",
+ "_disable_setproctitle",
+ "_disable_stats",
+ "_disable_viewer",
+ "_disable_machine_info",
+ "_except_exit",
+ "_executable",
+ "_extra_http_headers",
+ "_file_stream_retry_max",
+ "_file_stream_retry_wait_min_seconds",
+ "_file_stream_retry_wait_max_seconds",
+ "_file_stream_timeout_seconds",
+ "_file_transfer_retry_max",
+ "_file_transfer_retry_wait_min_seconds",
+ "_file_transfer_retry_wait_max_seconds",
+ "_file_transfer_timeout_seconds",
+ "_flow_control_custom",
+ "_flow_control_disabled",
+ "_graphql_retry_max",
+ "_graphql_retry_wait_min_seconds",
+ "_graphql_retry_wait_max_seconds",
+ "_graphql_timeout_seconds",
+ "_internal_check_process",
+ "_internal_queue_timeout",
+ "_ipython",
+ "_jupyter",
+ "_jupyter_name",
+ "_jupyter_path",
+ "_jupyter_root",
+ "_kaggle",
+ "_live_policy_rate_limit",
+ "_live_policy_wait_time",
+ "_log_level",
+ "_network_buffer",
+ "_noop",
+ "_notebook",
+ "_offline",
+ "_sync",
+ "_os",
+ "_platform",
+ "_proxies",
+ "_python",
+ "_runqueue_item_id",
+ "_require_core",
+ "_save_requirements",
+ "_service_transport",
+ "_service_wait",
+ "_shared",
+ "_start_datetime",
+ "_start_time",
+ "_stats_pid",
+ "_stats_sample_rate_seconds",
+ "_stats_samples_to_average",
+ "_stats_join_assets",
+ "_stats_neuron_monitor_config_path",
+ "_stats_open_metrics_endpoints",
+ "_stats_open_metrics_filters",
+ "_stats_disk_paths",
+ "_stats_buffer_size",
+ "_tmp_code_dir",
+ "_tracelog",
+ "_unsaved_keys",
+ "_windows",
+ "allow_val_change",
+ "anonymous",
+ "api_key",
+ "azure_account_url_to_access_key",
+ "base_url",
+ "code_dir",
+ "colab_url",
+ "config_paths",
+ "console",
+ "deployment",
+ "disable_code",
+ "disable_git",
+ "disable_hints",
+ "disable_job_creation",
+ "disabled",
+ "docker",
+ "email",
+ "entity",
+ "files_dir",
+ "force",
+ "git_commit",
+ "git_remote",
+ "git_remote_url",
+ "git_root",
+ "heartbeat_seconds",
+ "host",
+ "ignore_globs",
+ "init_timeout",
+ "is_local",
+ "job_name",
+ "job_source",
+ "label_disable",
+ "launch",
+ "launch_config_path",
+ "log_dir",
+ "log_internal",
+ "log_symlink_internal",
+ "log_symlink_user",
+ "log_user",
+ "login_timeout",
+ "mode",
+ "notebook_name",
+ "problem",
+ "program",
+ "program_abspath",
+ "program_relpath",
+ "project",
+ "project_url",
+ "quiet",
+ "reinit",
+ "relogin",
+ "resume",
+ "resume_fname",
+ "resumed",
+ "root_dir",
+ "run_group",
+ "run_id",
+ "run_job_type",
+ "run_mode",
+ "run_name",
+ "run_notes",
+ "run_tags",
+ "run_url",
+ "sagemaker_disable",
+ "save_code",
+ "settings_system",
+ "settings_workspace",
+ "show_colors",
+ "show_emoji",
+ "show_errors",
+ "show_info",
+ "show_warnings",
+ "silent",
+ "start_method",
+ "strict",
+ "summary_errors",
+ "summary_timeout",
+ "summary_warnings",
+ "sweep_id",
+ "sweep_param_path",
+ "sweep_url",
+ "symlink",
+ "sync_dir",
+ "sync_file",
+ "sync_symlink_latest",
+ "system_sample",
+ "system_sample_seconds",
+ "table_raise_on_max_row_limit_exceeded",
+ "timespec",
+ "tmp_dir",
+ "username",
+ "wandb_dir",
+]
+
+SETTINGS_TOPOLOGICALLY_SORTED: Final[Tuple[_Setting, ...]] = (
+ "_async_upload_concurrency_limit",
+ "_service_wait",
+ "_stats_sample_rate_seconds",
+ "_stats_samples_to_average",
+ "anonymous",
+ "api_key",
+ "base_url",
+ "console",
+ "job_source",
+ "mode",
+ "problem",
+ "project",
+ "run_id",
+ "start_method",
+ "_aws_lambda",
+ "_colab",
+ "_disable_machine_info",
+ "_disable_meta",
+ "_disable_stats",
+ "_network_buffer",
+ "_flow_control_disabled",
+ "_flow_control_custom",
+ "_ipython",
+ "_jupyter",
+ "_kaggle",
+ "_noop",
+ "_notebook",
+ "disabled",
+ "_offline",
+ "_shared",
+ "_stats_neuron_monitor_config_path",
+ "run_mode",
+ "_start_datetime",
+ "timespec",
+ "root_dir",
+ "wandb_dir",
+ "tmp_dir",
+ "_tmp_code_dir",
+ "_windows",
+ "colab_url",
+ "is_local",
+ "deployment",
+ "disable_code",
+ "disable_git",
+ "disable_job_creation",
+ "files_dir",
+ "log_dir",
+ "log_internal",
+ "log_symlink_internal",
+ "log_symlink_user",
+ "log_user",
+ "program",
+ "project_url",
+ "resume_fname",
+ "run_url",
+ "settings_system",
+ "settings_workspace",
+ "sweep_url",
+ "sync_dir",
+ "sync_file",
+ "sync_symlink_latest",
+)
diff --git a/wandb/sdk/lib/_wburls_generate.py b/wandb/sdk/lib/_wburls_generate.py
new file mode 100644
index 0000000000000000000000000000000000000000..2d246ef0ad85eef152e16cc7bffa59116a7e698c
--- /dev/null
+++ b/wandb/sdk/lib/_wburls_generate.py
@@ -0,0 +1,25 @@
+from wburls import wburls # type: ignore
+
+template = """
+import sys
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+
+URLS = Literal[
+ $literal_list
+]
+"""
+
+
+def generate() -> None:
+ urls = wburls._get_urls()
+ literal_list = ", ".join([f"{key!r}" for key in urls])
+ print(template.replace("$literal_list", literal_list))
+
+
+if __name__ == "__main__":
+ generate()
diff --git a/wandb/sdk/lib/_wburls_generated.py b/wandb/sdk/lib/_wburls_generated.py
new file mode 100644
index 0000000000000000000000000000000000000000..ebfd87ec9750d5aca892a5f999aceb092f5aa4bd
--- /dev/null
+++ b/wandb/sdk/lib/_wburls_generated.py
@@ -0,0 +1,21 @@
+# DO NOT EDIT -- GENERATED BY: `generate-tool.py --generate`
+import sys
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+
+URLS = Literal[
+ "cli_launch",
+ "doc_run",
+ "doc_require",
+ "doc_start_err",
+ "doc_artifacts_guide",
+ "upgrade_server",
+ "multiprocess",
+ "wandb_init",
+ "wandb_server",
+ "wandb_define_metric",
+]
diff --git a/wandb/sdk/lib/apikey.py b/wandb/sdk/lib/apikey.py
new file mode 100644
index 0000000000000000000000000000000000000000..ad058dfc380116692b309246e7346685738a47b3
--- /dev/null
+++ b/wandb/sdk/lib/apikey.py
@@ -0,0 +1,263 @@
+"""apikey util."""
+
+import os
+import stat
+import sys
+import textwrap
+from functools import partial
+from typing import TYPE_CHECKING, Callable, Dict, Optional, Union
+from urllib.parse import urlparse
+
+# import Literal
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+import click
+import requests.utils
+
+import wandb
+from wandb.apis import InternalApi
+from wandb.errors import term
+from wandb.util import _is_databricks, isatty, prompt_choices
+
+from .wburls import wburls
+
+LOGIN_CHOICE_ANON = "Private W&B dashboard, no account required"
+LOGIN_CHOICE_NEW = "Create a W&B account"
+LOGIN_CHOICE_EXISTS = "Use an existing W&B account"
+LOGIN_CHOICE_DRYRUN = "Don't visualize my results"
+LOGIN_CHOICE_NOTTY = "Unconfigured"
+LOGIN_CHOICES = [
+ LOGIN_CHOICE_ANON,
+ LOGIN_CHOICE_NEW,
+ LOGIN_CHOICE_EXISTS,
+ LOGIN_CHOICE_DRYRUN,
+]
+
+Mode = Literal["allow", "must", "never", "false", "true"]
+
+if TYPE_CHECKING:
+ from wandb.sdk.wandb_settings import Settings
+
+
+getpass = partial(click.prompt, hide_input=True, err=True)
+
+
+def _fixup_anon_mode(default: Optional[Mode]) -> Optional[Mode]:
+ # Convert weird anonymode values from legacy settings files
+ # into one of our expected values.
+ anon_mode = default or "never"
+ mapping: Dict[Mode, Mode] = {"true": "allow", "false": "never"}
+ return mapping.get(anon_mode, anon_mode)
+
+
+def get_netrc_file_path() -> str:
+ netrc_file = os.environ.get("NETRC")
+ if netrc_file:
+ return os.path.expanduser(netrc_file)
+ return os.path.join(os.path.expanduser("~"), ".netrc")
+
+
+def prompt_api_key( # noqa: C901
+ settings: "Settings",
+ api: Optional[InternalApi] = None,
+ input_callback: Optional[Callable] = None,
+ browser_callback: Optional[Callable] = None,
+ no_offline: bool = False,
+ no_create: bool = False,
+ local: bool = False,
+) -> Union[str, bool, None]:
+ """Prompt for api key.
+
+ Returns:
+ str - if key is configured
+ None - if dryrun is selected
+ False - if unconfigured (notty)
+ """
+ input_callback = input_callback or getpass
+ log_string = term.LOG_STRING
+ api = api or InternalApi(settings)
+ anon_mode = _fixup_anon_mode(settings.anonymous) # type: ignore
+ jupyter = settings._jupyter or False
+ app_url = api.app_url
+
+ choices = [choice for choice in LOGIN_CHOICES]
+ if anon_mode == "never":
+ # Omit LOGIN_CHOICE_ANON as a choice if the env var is set to never
+ choices.remove(LOGIN_CHOICE_ANON)
+ if (jupyter and not settings.login_timeout) or no_offline:
+ choices.remove(LOGIN_CHOICE_DRYRUN)
+ if (jupyter and not settings.login_timeout) or no_create:
+ choices.remove(LOGIN_CHOICE_NEW)
+
+ if jupyter and "google.colab" in sys.modules:
+ log_string = term.LOG_STRING_NOCOLOR
+ key = wandb.jupyter.attempt_colab_login(app_url)
+ if key is not None:
+ write_key(settings, key, api=api)
+ return key # type: ignore
+
+ if anon_mode == "must":
+ result = LOGIN_CHOICE_ANON
+ # If we're not in an interactive environment, default to dry-run.
+ elif (
+ not jupyter and (not isatty(sys.stdout) or not isatty(sys.stdin))
+ ) or _is_databricks():
+ result = LOGIN_CHOICE_NOTTY
+ elif local:
+ result = LOGIN_CHOICE_EXISTS
+ elif len(choices) == 1:
+ result = choices[0]
+ else:
+ result = prompt_choices(
+ choices, input_timeout=settings.login_timeout, jupyter=jupyter
+ )
+
+ api_ask = (
+ f"{log_string}: Paste an API key from your profile and hit enter, "
+ "or press ctrl+c to quit"
+ )
+ if result == LOGIN_CHOICE_ANON:
+ key = api.create_anonymous_api_key()
+
+ write_key(settings, key, api=api, anonymous=True)
+ return key # type: ignore
+ elif result == LOGIN_CHOICE_NEW:
+ key = browser_callback(signup=True) if browser_callback else None
+
+ if not key:
+ wandb.termlog(f"Create an account here: {app_url}/authorize?signup=true")
+ key = input_callback(api_ask).strip()
+
+ write_key(settings, key, api=api)
+ return key # type: ignore
+ elif result == LOGIN_CHOICE_EXISTS:
+ key = browser_callback() if browser_callback else None
+
+ if not key:
+ if not (settings.is_local or local):
+ host = app_url
+ for prefix in ("http://", "https://"):
+ if app_url.startswith(prefix):
+ host = app_url[len(prefix) :]
+ wandb.termlog(
+ f"Logging into {host}. (Learn how to deploy a W&B server locally: {wburls.get('wandb_server')})"
+ )
+ wandb.termlog(
+ f"You can find your API key in your browser here: {app_url}/authorize"
+ )
+ key = input_callback(api_ask).strip()
+ write_key(settings, key, api=api)
+ return key # type: ignore
+ elif result == LOGIN_CHOICE_NOTTY:
+ # TODO: Needs refactor as this needs to be handled by caller
+ return False
+ elif result == LOGIN_CHOICE_DRYRUN:
+ return None
+ else:
+ # Jupyter environments don't have a tty, but we can still try logging in using
+ # the browser callback if one is supplied.
+ key, anonymous = (
+ browser_callback() if jupyter and browser_callback else (None, False)
+ )
+
+ write_key(settings, key, api=api)
+ return key # type: ignore
+
+
+def write_netrc(host: str, entity: str, key: str) -> Optional[bool]:
+ """Add our host and key to .netrc."""
+ _, key_suffix = key.split("-", 1) if "-" in key else ("", key)
+ if len(key_suffix) != 40:
+ wandb.termerror(
+ "API-key must be exactly 40 characters long: {} ({} chars)".format(
+ key_suffix, len(key_suffix)
+ )
+ )
+ return None
+ try:
+ normalized_host = urlparse(host).netloc.split(":")[0]
+ if normalized_host != "localhost" and "." not in normalized_host:
+ wandb.termerror(
+ f"Host must be a url in the form https://some.address.com, received {host}"
+ )
+ return None
+ netrc_path = get_netrc_file_path()
+ wandb.termlog(
+ f"Appending key for {normalized_host} to your netrc file: {netrc_path}"
+ )
+ machine_line = f"machine {normalized_host}"
+ orig_lines = None
+ try:
+ with open(netrc_path) as f:
+ orig_lines = f.read().strip().split("\n")
+ except OSError:
+ pass
+ with open(netrc_path, "w") as f:
+ if orig_lines:
+ # delete this machine from the file if it's already there.
+ skip = 0
+ for line in orig_lines:
+ # we fix invalid netrc files with an empty host that we wrote before
+ # verifying host...
+ if line == "machine " or machine_line in line:
+ skip = 2
+ elif skip:
+ skip -= 1
+ else:
+ f.write("%s\n" % line)
+ f.write(
+ textwrap.dedent(
+ """\
+ machine {host}
+ login {entity}
+ password {key}
+ """
+ ).format(host=normalized_host, entity=entity, key=key)
+ )
+ os.chmod(netrc_path, stat.S_IRUSR | stat.S_IWUSR)
+ return True
+ except OSError:
+ wandb.termerror(f"Unable to read {netrc_path}")
+ return None
+
+
+def write_key(
+ settings: "Settings",
+ key: Optional[str],
+ api: Optional["InternalApi"] = None,
+ anonymous: bool = False,
+) -> None:
+ if not key:
+ raise ValueError("No API key specified.")
+
+ # TODO(jhr): api shouldn't be optional or it shouldn't be passed, clean up callers
+ api = api or InternalApi()
+
+ # Normal API keys are 40-character hex strings. On-prem API keys have a
+ # variable-length prefix, a dash, then the 40-char string.
+ _, suffix = key.split("-", 1) if "-" in key else ("", key)
+
+ if len(suffix) != 40:
+ raise ValueError("API key must be 40 characters long, yours was %s" % len(key))
+
+ if anonymous:
+ api.set_setting("anonymous", "true", globally=True, persist=True)
+ else:
+ api.clear_setting("anonymous", globally=True, persist=True)
+
+ write_netrc(settings.base_url, "user", key)
+
+
+def api_key(settings: Optional["Settings"] = None) -> Optional[str]:
+ if settings is None:
+ settings = wandb.setup().settings # type: ignore
+ assert settings is not None
+ if settings.api_key:
+ return settings.api_key
+ auth = requests.utils.get_netrc_auth(settings.base_url)
+ if auth:
+ return auth[-1]
+ return None
diff --git a/wandb/sdk/lib/capped_dict.py b/wandb/sdk/lib/capped_dict.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e162109926cf7c115bf835323de4d292f8725e8
--- /dev/null
+++ b/wandb/sdk/lib/capped_dict.py
@@ -0,0 +1,26 @@
+import collections
+from typing import Any, Optional
+
+
+class CappedDict(collections.OrderedDict):
+ default_max_size = 50
+
+ def __init__(self, max_size: Optional[int] = None) -> None:
+ self.max_size = max_size or self.default_max_size
+ super().__init__()
+
+ def __setitem__(self, key: str, val: Any) -> None:
+ if key not in self:
+ max_size = self.max_size - 1
+ self._prune_dict(max_size)
+ super().__setitem__(key, val)
+
+ def update(self, **kwargs: Any) -> None: # type: ignore[override]
+ super().update(**kwargs)
+ self._prune_dict(self.max_size)
+
+ def _prune_dict(self, max_size: int) -> None:
+ if len(self) >= max_size:
+ diff = len(self) - max_size
+ for k in list(self.keys())[:diff]:
+ del self[k]
diff --git a/wandb/sdk/lib/config_util.py b/wandb/sdk/lib/config_util.py
new file mode 100644
index 0000000000000000000000000000000000000000..0995acecc1ddf3361672ae44042face9252ceb88
--- /dev/null
+++ b/wandb/sdk/lib/config_util.py
@@ -0,0 +1,132 @@
+import json
+import logging
+import os
+from typing import Any, Dict, Optional
+
+import yaml
+
+import wandb
+from wandb.errors import Error
+from wandb.util import load_yaml
+
+from . import filesystem
+
+logger = logging.getLogger("wandb")
+
+
+class ConfigError(Error):
+ pass
+
+
+def dict_from_proto_list(obj_list):
+ d = dict()
+ for item in obj_list:
+ d[item.key] = dict(desc=None, value=json.loads(item.value_json))
+ return d
+
+
+def update_from_proto(config_dict, config_proto):
+ for item in config_proto.update:
+ key_list = item.nested_key or (item.key,)
+ assert key_list, "key or nested key must be set"
+ target = config_dict
+ # recurse down the dictionary structure:
+ for prop in key_list[:-1]:
+ if not target.get(prop):
+ target[prop] = {}
+ target = target[prop]
+ # use the last element of the key to write the leaf:
+ target[key_list[-1]] = json.loads(item.value_json)
+ for item in config_proto.remove:
+ key_list = item.nested_key or (item.key,)
+ assert key_list, "key or nested key must be set"
+ target = config_dict
+ # recurse down the dictionary structure:
+ for prop in key_list[:-1]:
+ target = target[prop]
+ # use the last element of the key to write the leaf:
+ del target[key_list[-1]]
+ # TODO(jhr): should we delete empty parents?
+
+
+def dict_add_value_dict(config_dict):
+ d = dict()
+ for k, v in config_dict.items():
+ d[k] = dict(desc=None, value=v)
+ return d
+
+
+def dict_strip_value_dict(config_dict):
+ d = dict()
+ for k, v in config_dict.items():
+ d[k] = v["value"]
+ return d
+
+
+def dict_no_value_from_proto_list(obj_list):
+ d = dict()
+ for item in obj_list:
+ possible_dict = json.loads(item.value_json)
+ if not isinstance(possible_dict, dict) or "value" not in possible_dict:
+ continue
+ d[item.key] = possible_dict["value"]
+
+ return d
+
+
+# TODO(jhr): these functions should go away once we merge jobspec PR
+def save_config_file_from_dict(config_filename, config_dict):
+ s = b"wandb_version: 1"
+ if config_dict: # adding an empty dictionary here causes a parse error
+ s += b"\n\n" + yaml.dump(
+ config_dict,
+ Dumper=yaml.SafeDumper,
+ default_flow_style=False,
+ allow_unicode=True,
+ encoding="utf-8",
+ sort_keys=False,
+ )
+ data = s.decode("utf-8")
+ filesystem.mkdir_exists_ok(os.path.dirname(config_filename))
+ with open(config_filename, "w") as conf_file:
+ conf_file.write(data)
+
+
+def dict_from_config_file(
+ filename: str, must_exist: bool = False
+) -> Optional[Dict[str, Any]]:
+ if not os.path.exists(filename):
+ if must_exist:
+ raise ConfigError("config file %s doesn't exist" % filename)
+ logger.debug("no default config file found in %s" % filename)
+ return None
+ try:
+ conf_file = open(filename)
+ except OSError:
+ raise ConfigError("Couldn't read config file: %s" % filename)
+ try:
+ loaded = load_yaml(conf_file)
+ except yaml.parser.ParserError:
+ raise ConfigError("Invalid YAML in config yaml")
+ if loaded is None:
+ wandb.termwarn(
+ "Found an empty default config file (config-defaults.yaml). Proceeding with no defaults."
+ )
+ return None
+ config_version = loaded.pop("wandb_version", None)
+ if config_version is not None and config_version != 1:
+ raise ConfigError("Unknown config version")
+ data = dict()
+ for k, v in loaded.items():
+ data[k] = v["value"]
+ return data
+
+
+def merge_dicts(dest: dict, src: dict) -> dict:
+ """Recursively merge two dictionaries. Similar to Lodash's _.merge()."""
+ for key, value in src.items():
+ if isinstance(value, dict) and key in dest and isinstance(dest[key], dict):
+ merge_dicts(dest[key], value)
+ else:
+ dest[key] = value
+ return dest
diff --git a/wandb/sdk/lib/console.py b/wandb/sdk/lib/console.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e84a5b05e3fa64aef23e4d5a235817e6d85d5b0
--- /dev/null
+++ b/wandb/sdk/lib/console.py
@@ -0,0 +1,39 @@
+"""console."""
+
+import os
+
+
+def win32_redirect(stdout_slave_fd, stderr_slave_fd):
+ # import win32api
+
+ # save for later
+ # fd_stdout = os.dup(1)
+ # fd_stderr = os.dup(2)
+
+ # std_out = win32api.GetStdHandle(win32api.STD_OUTPUT_HANDLE)
+ # std_err = win32api.GetStdHandle(win32api.STD_ERROR_HANDLE)
+
+ # os.dup2(stdout_slave_fd, 1)
+ # os.dup2(stderr_slave_fd, 2)
+
+ # TODO(jhr): do something about current stdout, stderr file handles
+ pass
+
+
+def win32_create_pipe():
+ # import pywintypes
+ # import win32pipe
+
+ # sa=pywintypes.SECURITY_ATTRIBUTES()
+ # sa.bInheritHandle=1
+
+ # read_fd, write_fd = win32pipe.FdCreatePipe(sa, 0, os.O_TEXT)
+ # read_fd, write_fd = win32pipe.FdCreatePipe(sa, 0, os.O_BINARY)
+ read_fd, write_fd = os.pipe()
+ # http://timgolden.me.uk/pywin32-docs/win32pipe__FdCreatePipe_meth.html
+ # https://stackoverflow.com/questions/17942874/stdout-redirection-with-ctypes
+
+ # f = open("testing.txt", "rb")
+ # read_fd = f.fileno()
+
+ return read_fd, write_fd
diff --git a/wandb/sdk/lib/deprecate.py b/wandb/sdk/lib/deprecate.py
new file mode 100644
index 0000000000000000000000000000000000000000..14afb06206855784d0b00e0aa78ca5945d9915bb
--- /dev/null
+++ b/wandb/sdk/lib/deprecate.py
@@ -0,0 +1,42 @@
+__all__ = ["deprecate", "Deprecated"]
+
+from typing import TYPE_CHECKING, Optional, Tuple
+
+import wandb
+from wandb.proto.wandb_deprecated import DEPRECATED_FEATURES, Deprecated
+from wandb.proto.wandb_telemetry_pb2 import Deprecated as TelemetryDeprecated
+
+# avoid cycle, use string type reference
+if TYPE_CHECKING:
+ from .. import wandb_run
+
+
+deprecated_field_names: Tuple[str, ...] = tuple(
+ str(v) for k, v in Deprecated.__dict__.items() if not k.startswith("_")
+)
+
+
+def deprecate(
+ field_name: DEPRECATED_FEATURES,
+ warning_message: str,
+ run: Optional["wandb_run.Run"] = None,
+) -> None:
+ """Warn the user that a feature has been deprecated.
+
+ Also stores the information about the event in telemetry.
+
+ Args:
+ field_name: The name of the feature that has been deprecated.
+ Defined in wandb/proto/wandb_telemetry.proto::Deprecated
+ warning_message: The message to display to the user.
+ run: The run to whose telemetry the event will be added.
+ """
+ known_fields = TelemetryDeprecated.DESCRIPTOR.fields_by_name.keys()
+ if field_name not in known_fields:
+ raise ValueError(
+ f"Unknown field name: {field_name}. Known fields: {known_fields}"
+ )
+ _run = run or wandb.run
+ with wandb.wandb_lib.telemetry.context(run=_run) as tel: # type: ignore[attr-defined]
+ setattr(tel.deprecated, field_name, True)
+ wandb.termwarn(warning_message, repeat=False)
diff --git a/wandb/sdk/lib/disabled.py b/wandb/sdk/lib/disabled.py
new file mode 100644
index 0000000000000000000000000000000000000000..3b32975efeb8ca3d1b597356a19cc1ccb4f762d4
--- /dev/null
+++ b/wandb/sdk/lib/disabled.py
@@ -0,0 +1,190 @@
+#
+
+
+class RunDisabled(str):
+ def __init__(self, *args, **kwargs):
+ object.__setattr__(self, "___dict", {})
+
+ def __add__(self, other):
+ return self
+
+ def __sub__(self, other):
+ return self
+
+ def __mul__(self, other):
+ return self
+
+ def __truediv__(self, other):
+ return self
+
+ def __floordiv__(self, other):
+ return self
+
+ def __mod__(self, other):
+ return self
+
+ def __pow__(self, other, modulo=None):
+ return self
+
+ def __lshift__(self, other):
+ return self
+
+ def __rshift__(self, other):
+ return self
+
+ def __and__(self, other):
+ return self
+
+ def __xor__(self, other):
+ return self
+
+ def __or__(self, other):
+ return self
+
+ def __iadd__(self, other):
+ return self
+
+ def __isub__(self, other):
+ return self
+
+ def __imul__(self, other):
+ return self
+
+ def __idiv__(self, other):
+ return self
+
+ def __ifloordiv__(self, other):
+ return self
+
+ def __imod__(self, other):
+ return self
+
+ def __ipow__(self, other, modulo=None):
+ return self
+
+ def __ilshift__(self, other):
+ return self
+
+ def __irshift__(self, other):
+ return self
+
+ def __iand__(self, other):
+ return self
+
+ def __ixor__(self, other):
+ return self
+
+ def __ior__(self, other):
+ return self
+
+ def __neg__(self):
+ return self
+
+ def __pos__(self):
+ return self
+
+ def __abs__(self):
+ return self
+
+ def __invert__(self):
+ return self
+
+ def __complex__(self):
+ return 1 + 0j
+
+ def __int__(self):
+ return 1
+
+ def __long__(self):
+ return 1
+
+ def __float__(self):
+ return 1.0
+
+ def __oct__(self):
+ return oct(1)
+
+ def __hex__(self):
+ return hex(1)
+
+ def __lt__(self, other):
+ return True
+
+ def __le__(self, other):
+ return True
+
+ def __eq__(self, other):
+ return True
+
+ def __ne__(self, other):
+ return True
+
+ def __gt__(self, other):
+ return True
+
+ def __ge__(self, other):
+ return True
+
+ def __getattr__(self, attr):
+ return self[attr]
+
+ def __getitem__(self, key):
+ d = object.__getattribute__(self, "___dict")
+ try:
+ if key in d:
+ return d[key]
+ except TypeError:
+ key = str(key)
+ if key in d:
+ return d[key]
+ dummy = RunDisabled()
+ d[key] = dummy
+ return dummy
+
+ def __setitem__(self, key, value):
+ object.__getattribute__(self, "___dict")[key] = value
+
+ def __setattr__(self, key, value):
+ self[key] = value
+
+ def __call__(self, *args, **kwargs):
+ return RunDisabled()
+
+ def __len__(self):
+ return 1
+
+ def __str__(self):
+ return ""
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ return exc_type is None
+
+ def __repr__(self):
+ return ""
+
+ def __nonzero__(self):
+ return True
+
+ def __bool__(self):
+ return True
+
+ def __getstate__(self):
+ return 1
+
+
+class SummaryDisabled(dict):
+ __setattr__ = dict.__setitem__
+ __delattr__ = dict.__delitem__
+
+ def __getattr__(self, key):
+ return self[key]
+
+ def __getitem__(self, key):
+ val = dict.__getitem__(self, key)
+ if isinstance(val, dict) and not isinstance(val, SummaryDisabled):
+ val = SummaryDisabled(val)
+ self[key] = val
+ return val
diff --git a/wandb/sdk/lib/exit_hooks.py b/wandb/sdk/lib/exit_hooks.py
new file mode 100644
index 0000000000000000000000000000000000000000..aa747495299a646041f7deaeb72c1a9bcc050660
--- /dev/null
+++ b/wandb/sdk/lib/exit_hooks.py
@@ -0,0 +1,54 @@
+import sys
+import traceback
+from types import TracebackType
+from typing import TYPE_CHECKING, Optional, Type
+
+import wandb
+from wandb.errors import Error
+
+if TYPE_CHECKING:
+ from typing import NoReturn
+
+
+class ExitHooks:
+ exception: Optional[BaseException] = None
+
+ def __init__(self) -> None:
+ self.exit_code = 0
+ self.exception = None
+
+ def hook(self) -> None:
+ self._orig_exit = sys.exit
+ sys.exit = self.exit
+ self._orig_excepthook = (
+ sys.excepthook
+ if sys.excepthook
+ != sys.__excepthook__ # respect hooks by other libraries like pdb
+ else None
+ )
+ sys.excepthook = self.exc_handler # type: ignore
+
+ def exit(self, code: object = 0) -> "NoReturn":
+ orig_code = code
+ code = code if code is not None else 0
+ code = code if isinstance(code, int) else 1
+ self.exit_code = code
+ self._orig_exit(orig_code) # type: ignore
+
+ def was_ctrl_c(self) -> bool:
+ return isinstance(self.exception, KeyboardInterrupt)
+
+ def exc_handler(
+ self, exc_type: Type[BaseException], exc: BaseException, tb: TracebackType
+ ) -> None:
+ self.exit_code = 1
+ self.exception = exc
+ if issubclass(exc_type, Error):
+ wandb.termerror(str(exc), repeat=False)
+
+ if self.was_ctrl_c():
+ self.exit_code = 255
+
+ traceback.print_exception(exc_type, exc, tb)
+ if self._orig_excepthook:
+ self._orig_excepthook(exc_type, exc, tb)
diff --git a/wandb/sdk/lib/file_stream_utils.py b/wandb/sdk/lib/file_stream_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..4f295e63f6da2cf6317f154b3e6c34c6be0b9869
--- /dev/null
+++ b/wandb/sdk/lib/file_stream_utils.py
@@ -0,0 +1,118 @@
+#
+from typing import Any, Dict, Iterable
+
+
+def split_files(
+ files: Dict[str, Any], max_bytes: int = 10 * 1024 * 1024
+) -> Iterable[Dict[str, Dict]]:
+ """Split a file's dict (see `files` arg) into smaller dicts.
+
+ Each smaller dict will have at most `MAX_BYTES` size.
+
+ This method is used in `FileStreamAPI._send()` to limit the size of post requests
+ sent to wandb server.
+
+ Arguments:
+ files (dict): `dict` of form {file_name: {'content': ".....", 'offset': 0}}
+ The key `file_name` can also be mapped to a List [{"offset": int, "content": str}]
+ `max_bytes`: max size for chunk in bytes
+ """
+ current_volume: Dict[str, Dict] = {}
+ current_size = 0
+
+ def _str_size(x):
+ return len(x) if isinstance(x, bytes) else len(x.encode("utf-8"))
+
+ def _file_size(file):
+ size = file.get("_size")
+ if size is None:
+ size = sum(map(_str_size, file["content"]))
+ file["_size"] = size
+ return size
+
+ def _split_file(file, num_lines):
+ offset = file["offset"]
+ content = file["content"]
+ name = file["name"]
+ f1 = {"offset": offset, "content": content[:num_lines], "name": name}
+ f2 = {
+ "offset": offset + num_lines,
+ "content": content[num_lines:],
+ "name": name,
+ }
+ return f1, f2
+
+ def _num_lines_from_num_bytes(file, num_bytes):
+ size = 0
+ num_lines = 0
+ content = file["content"]
+ while num_lines < len(content):
+ size += _str_size(content[num_lines])
+ if size > num_bytes:
+ break
+ num_lines += 1
+ return num_lines
+
+ files_stack = []
+ for k, v in files.items():
+ if isinstance(v, list):
+ for item in v:
+ files_stack.append(
+ {"name": k, "offset": item["offset"], "content": item["content"]}
+ )
+ else:
+ files_stack.append(
+ {"name": k, "offset": v["offset"], "content": v["content"]}
+ )
+
+ while files_stack:
+ f = files_stack.pop()
+ if f["name"] in current_volume:
+ files_stack.append(f)
+ yield current_volume
+ current_volume = {}
+ current_size = 0
+ continue
+ # For each file, we have to do 1 of 4 things:
+ # - Add the file as such to the current volume if possible.
+ # - Split the file and add the first part to the current volume and push the second part back onto the stack.
+ # - If that's not possible, check if current volume is empty:
+ # - If empty, add first line of file to current volume and push rest onto stack (This volume will exceed MAX_MB).
+ # - If not, push file back to stack and yield current volume.
+ fsize = _file_size(f)
+ rem = max_bytes - current_size
+ if fsize <= rem:
+ current_volume[f["name"]] = {
+ "offset": f["offset"],
+ "content": f["content"],
+ }
+ current_size += fsize
+ else:
+ num_lines = _num_lines_from_num_bytes(f, rem)
+ if not num_lines and not current_volume:
+ num_lines = 1
+ if num_lines:
+ f1, f2 = _split_file(f, num_lines)
+ current_volume[f1["name"]] = {
+ "offset": f1["offset"],
+ "content": f1["content"],
+ }
+ files_stack.append(f2)
+ yield current_volume
+ current_volume = {}
+ current_size = 0
+ continue
+ else:
+ files_stack.append(f)
+ yield current_volume
+ current_volume = {}
+ current_size = 0
+ continue
+ if current_size >= max_bytes:
+ yield current_volume
+ current_volume = {}
+ current_size = 0
+ continue
+
+ if current_volume:
+ yield current_volume
diff --git a/wandb/sdk/lib/filenames.py b/wandb/sdk/lib/filenames.py
new file mode 100644
index 0000000000000000000000000000000000000000..c272b7a2c3337ddb6d91ce03ed800e84affcc5ff
--- /dev/null
+++ b/wandb/sdk/lib/filenames.py
@@ -0,0 +1,64 @@
+import os
+from typing import Callable, Generator, Union
+
+WANDB_DIRS = ("wandb", ".wandb")
+
+CONFIG_FNAME = "config.yaml"
+OUTPUT_FNAME = "output.log"
+DIFF_FNAME = "diff.patch"
+SUMMARY_FNAME = "wandb-summary.json"
+METADATA_FNAME = "wandb-metadata.json"
+REQUIREMENTS_FNAME = "requirements.txt"
+HISTORY_FNAME = "wandb-history.jsonl"
+EVENTS_FNAME = "wandb-events.jsonl"
+JOBSPEC_FNAME = "wandb-jobspec.json"
+CONDA_ENVIRONMENTS_FNAME = "conda-environment.yaml"
+
+
+def is_wandb_file(name: str) -> bool:
+ return (
+ name.startswith("wandb")
+ or name == METADATA_FNAME
+ or name == CONFIG_FNAME
+ or name == REQUIREMENTS_FNAME
+ or name == OUTPUT_FNAME
+ or name == DIFF_FNAME
+ or name == CONDA_ENVIRONMENTS_FNAME
+ )
+
+
+def filtered_dir(
+ root: str,
+ include_fn: Union[Callable[[str, str], bool], Callable[[str], bool]],
+ exclude_fn: Union[Callable[[str, str], bool], Callable[[str], bool]],
+) -> Generator[str, None, None]:
+ """Simple generator to walk a directory."""
+ import inspect
+
+ # compatibility with old API, which didn't pass root
+ def _include_fn(path: str, root: str) -> bool:
+ return (
+ include_fn(path, root) # type: ignore
+ if len(inspect.signature(include_fn).parameters) == 2
+ else include_fn(path) # type: ignore
+ )
+
+ def _exclude_fn(path: str, root: str) -> bool:
+ return (
+ exclude_fn(path, root) # type: ignore
+ if len(inspect.signature(exclude_fn).parameters) == 2
+ else exclude_fn(path) # type: ignore
+ )
+
+ for dirpath, _, files in os.walk(root):
+ for fname in files:
+ file_path = os.path.join(dirpath, fname)
+ if _include_fn(file_path, root) and not _exclude_fn(file_path, root):
+ yield file_path
+
+
+def exclude_wandb_fn(path: str, root: str) -> bool:
+ return any(
+ os.path.relpath(path, root).startswith(wandb_dir + os.sep)
+ for wandb_dir in WANDB_DIRS
+ )
diff --git a/wandb/sdk/lib/filesystem.py b/wandb/sdk/lib/filesystem.py
new file mode 100644
index 0000000000000000000000000000000000000000..ef641435879ef887335a7dc3554f0e427bc01aeb
--- /dev/null
+++ b/wandb/sdk/lib/filesystem.py
@@ -0,0 +1,362 @@
+import contextlib
+import ctypes
+import errno
+import logging
+import os
+import platform
+import re
+import shutil
+import tempfile
+import threading
+from pathlib import Path
+from typing import IO, Any, BinaryIO, Generator, Optional
+
+from wandb.sdk.lib.paths import StrPath
+
+logger = logging.getLogger(__name__)
+
+# https://en.wikipedia.org/wiki/Filename#Comparison_of_filename_limitations
+PROBLEMATIC_PATH_CHARS = "".join(chr(i) for i in range(0, 32)) + ':"*<>?|'
+
+
+def mkdir_exists_ok(dir_name: StrPath) -> None:
+ """Create `dir_name` and any parent directories if they don't exist.
+
+ Raises:
+ FileExistsError: if `dir_name` exists and is not a directory.
+ PermissionError: if `dir_name` is not writable.
+ """
+ try:
+ os.makedirs(dir_name, exist_ok=True)
+ except FileExistsError as e:
+ raise FileExistsError(f"{dir_name!s} exists and is not a directory") from e
+ except PermissionError as e:
+ raise PermissionError(f"{dir_name!s} is not writable") from e
+
+
+def path_fallbacks(path: StrPath) -> Generator[str, None, None]:
+ """Yield variations of `path` that may exist on the filesystem.
+
+ Return a sequence of paths that should be checked in order for existence or
+ create-ability. Essentially, keep replacing "suspect" characters until we run out.
+ """
+ path = str(path)
+ root, tail = os.path.splitdrive(path)
+ yield os.path.join(root, tail)
+ for char in PROBLEMATIC_PATH_CHARS:
+ if char in tail:
+ tail = tail.replace(char, "-")
+ yield os.path.join(root, tail)
+
+
+def mkdir_allow_fallback(dir_name: StrPath) -> StrPath:
+ """Create `dir_name`, removing invalid path characters if necessary.
+
+ Returns:
+ The path to the created directory, which may not be the original path.
+ """
+ for new_name in path_fallbacks(dir_name):
+ try:
+ os.makedirs(new_name, exist_ok=True)
+ if Path(new_name) != Path(dir_name):
+ logger.warning(f"Creating '{new_name}' instead of '{dir_name}'")
+ return Path(new_name) if isinstance(dir_name, Path) else new_name
+ except (ValueError, NotADirectoryError):
+ pass
+ except OSError as e:
+ if e.errno != 22:
+ raise
+
+ raise OSError(f"Unable to create directory '{dir_name}'")
+
+
+def files_in(path: StrPath) -> Generator[os.DirEntry, None, None]:
+ """Yield a directory entry for each file under a given path (recursive)."""
+ if not os.path.isdir(path):
+ return
+ for entry in os.scandir(path):
+ if entry.is_dir():
+ yield from files_in(entry.path)
+ else:
+ yield entry
+
+
+class WriteSerializingFile:
+ """Wrapper for a file object that serializes writes."""
+
+ def __init__(self, f: BinaryIO) -> None:
+ self.lock = threading.Lock()
+ self.f = f
+
+ def write(self, *args, **kargs) -> None: # type: ignore
+ self.lock.acquire()
+ try:
+ self.f.write(*args, **kargs)
+ self.f.flush()
+ finally:
+ self.lock.release()
+
+ def close(self) -> None:
+ self.lock.acquire() # wait for pending writes
+ try:
+ self.f.close()
+ finally:
+ self.lock.release()
+
+
+class CRDedupedFile(WriteSerializingFile):
+ def __init__(self, f: BinaryIO) -> None:
+ super().__init__(f=f)
+ self._buff = b""
+
+ def write(self, data) -> None: # type: ignore
+ lines = re.split(b"\r\n|\n", data)
+ ret = [] # type: ignore
+ for line in lines:
+ if line[:1] == b"\r":
+ if ret:
+ ret.pop()
+ elif self._buff:
+ self._buff = b""
+ line = line.split(b"\r")[-1]
+ if line:
+ ret.append(line)
+ if self._buff:
+ ret.insert(0, self._buff)
+ if ret:
+ self._buff = ret.pop()
+ super().write(b"\n".join(ret) + b"\n")
+
+ def close(self) -> None:
+ if self._buff:
+ super().write(self._buff)
+ super().close()
+
+
+def copy_or_overwrite_changed(source_path: StrPath, target_path: StrPath) -> StrPath:
+ """Copy source_path to target_path, unless it already exists with the same mtime.
+
+ We liberally add write permissions to deal with the case of multiple users needing
+ to share the same cache or run directory.
+
+ Args:
+ source_path: The path to the file to copy.
+ target_path: The path to copy the file to.
+
+ Returns:
+ The path to the copied file (which may be different from target_path).
+ """
+ return_type = type(target_path)
+
+ target_path = system_preferred_path(target_path, warn=True)
+
+ need_copy = (
+ not os.path.isfile(target_path)
+ or os.stat(source_path).st_mtime != os.stat(target_path).st_mtime
+ )
+
+ permissions_plus_write = os.stat(source_path).st_mode
+ if need_copy:
+ dir_name, file_name = os.path.split(target_path)
+ target_path = os.path.join(mkdir_allow_fallback(dir_name), file_name)
+ try:
+ # Use copy2 to preserve file metadata (including modified time).
+ shutil.copy2(source_path, target_path)
+ except PermissionError:
+ # If the file is read-only try to make it writable.
+ try:
+ os.chmod(target_path, permissions_plus_write)
+ shutil.copy2(source_path, target_path)
+ except PermissionError as e:
+ raise PermissionError("Unable to overwrite '{target_path!s}'") from e
+ # Prevent future permissions issues by universal write permissions now.
+ os.chmod(target_path, permissions_plus_write)
+
+ return return_type(target_path) # type: ignore # 'os.PathLike' is abstract.
+
+
+@contextlib.contextmanager
+def safe_open(
+ path: StrPath, mode: str = "r", *args: Any, **kwargs: Any
+) -> Generator[IO, None, None]:
+ """Open a file, ensuring any changes only apply atomically after close.
+
+ This context manager ensures that even unsuccessful writes will not leave a "dirty"
+ file or overwrite good data, and that all temp data is cleaned up.
+
+ The semantics and behavior are intended to be nearly identical to the built-in
+ open() function. Differences:
+ - It creates any parent directories that don't exist, rather than raising.
+ - In 'x' mode, it checks at the beginning AND end of the write and fails if the
+ file exists either time.
+ """
+ path = Path(path).resolve()
+ path.parent.mkdir(parents=True, exist_ok=True)
+
+ if "x" in mode and path.exists():
+ raise FileExistsError(f"{path!s} already exists")
+
+ if "r" in mode and "+" not in mode:
+ # This is read-only, so we can just open the original file.
+ # TODO (hugh): create a reflink and read from that.
+ with path.open(mode, *args, **kwargs) as f:
+ yield f
+ return
+
+ with tempfile.TemporaryDirectory(dir=path.parent) as tmp_dir:
+ tmp_path = Path(tmp_dir) / path.name
+
+ if ("r" in mode or "a" in mode) and path.exists():
+ # We need to copy the original file in order to support reads and appends.
+ # TODO (hugh): use reflinks to avoid the copy on platforms that support it.
+ shutil.copy2(path, tmp_path)
+
+ with tmp_path.open(mode, *args, **kwargs) as f:
+ yield f
+ f.flush()
+ os.fsync(f.fileno())
+
+ if "x" in mode:
+ # Ensure that if another process has beaten us to writing the file we raise
+ # rather than overwrite. os.link() atomically creates a hard link to the
+ # target file and will raise FileExistsError if the target already exists.
+ os.link(tmp_path, path)
+ os.unlink(tmp_path)
+ else:
+ tmp_path.replace(path)
+
+
+def safe_copy(source_path: StrPath, target_path: StrPath) -> StrPath:
+ """Copy a file, ensuring any changes only apply atomically once finished."""
+ # TODO (hugh): check that there is enough free space.
+ output_path = Path(target_path).resolve()
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+ with tempfile.TemporaryDirectory(dir=output_path.parent) as tmp_dir:
+ tmp_path = (Path(tmp_dir) / Path(source_path).name).with_suffix(".tmp")
+ shutil.copy2(source_path, tmp_path)
+ tmp_path.replace(output_path)
+ return target_path
+
+
+def _reflink_linux(existing_path: Path, new_path: Path) -> None:
+ """Create a reflink to `existing_path` at `new_path` on Linux."""
+ import fcntl
+
+ FICLONE = 0x40049409 # magic number from <linux/fs.h> # noqa: N806
+ with open(existing_path, "rb") as t_f, open(new_path, "wb+") as l_f:
+ fcntl.ioctl(l_f.fileno(), FICLONE, t_f.fileno())
+
+
+def _reflink_macos(existing_path: Path, new_path: Path) -> None:
+ try:
+ clib = ctypes.CDLL("libc.dylib", use_errno=True)
+ except (FileNotFoundError, OSError) as e:
+ if ctypes.get_errno() != errno.ENOENT and not isinstance(e, FileNotFoundError):
+ raise
+ # Before macOS 11 (<Nov 2020) clib was in libSystem.dylib, so we can try there.
+ clib = ctypes.CDLL("/usr/lib/libSystem.dylib", use_errno=True)
+
+ try:
+ clonefile = clib.clonefile
+ except AttributeError:
+ raise OSError(errno.ENOTSUP, "'clonefile' is not available on this system")
+
+ clonefile.argtypes = (ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int)
+ clonefile.restype = ctypes.c_int
+
+ if clonefile(os.fsencode(existing_path), os.fsencode(new_path), ctypes.c_int(0)):
+ # Anything other than 0 is an error.
+ err = ctypes.get_errno()
+ raise OSError(err, os.strerror(err), existing_path)
+
+
+def reflink(existing_path: StrPath, new_path: StrPath, overwrite: bool = False) -> None:
+ """Create a reflink to `existing_path` at `new_path`.
+
+ A reflink (reflective link) is a copy-on-write reference to a file. Once linked, the
+ file and link are both "real" files (not symbolic or hard links) and each can be
+ modified independently without affecting the other; however, they share the same
+ underlying data blocks on disk so until one is modified they are "zero-cost" copies.
+
+ Reflinks have all the functionality of copies, so we should use them wherever they
+ are supported if we would otherwise copy a file. (This is not particularly radical--
+ GNU `cp` defaults to `reflink=auto`, using it whenever available) However, support
+ for them is limited to a small number of filesystems. They should work on:
+ - Linux with a Btrfs or XFS filesystem (NOT ext4)
+ - macOS 10.13 or later with an APFS filesystem (called clone files)
+
+ Reflinks are also supported on Solaris and Windows with ReFSv2, but we haven't
+ implemented support for them.
+
+ Like hard links, a reflink can only be created on the same filesystem as the target.
+ """
+ if platform.system() == "Linux":
+ link_fn = _reflink_linux
+ elif platform.system() == "Darwin":
+ link_fn = _reflink_macos
+ else:
+ raise OSError(
+ errno.ENOTSUP, f"reflinks are not supported on {platform.system()}"
+ )
+
+ new_path = Path(new_path).resolve()
+ existing_path = Path(existing_path).resolve()
+ if new_path.exists():
+ if not overwrite:
+ raise FileExistsError(f"{new_path} already exists")
+ logger.warning(f"Overwriting existing file {new_path}.")
+ new_path.unlink()
+
+ # Create any missing parent directories.
+ new_path.parent.mkdir(parents=True, exist_ok=True)
+
+ try:
+ link_fn(existing_path, new_path)
+ except OSError as e:
+ base_msg = f"failed to create reflink from {existing_path} to {new_path}."
+ if e.errno in (errno.EPERM, errno.EACCES):
+ raise PermissionError(f"Insufficient permissions; {base_msg}") from e
+ if e.errno == errno.ENOENT:
+ raise FileNotFoundError(f"File not found; {base_msg}") from e
+ if e.errno == errno.EXDEV:
+ raise ValueError(f"Cannot link across filesystems; {base_msg}") from e
+ if e.errno == errno.EISDIR:
+ raise IsADirectoryError(f"Cannot reflink a directory; {base_msg}") from e
+ if e.errno in (errno.EOPNOTSUPP, errno.ENOTSUP):
+ raise OSError(
+ errno.ENOTSUP,
+ f"Filesystem does not support reflinks; {base_msg}",
+ ) from e
+ if e.errno == errno.EINVAL:
+ raise ValueError(f"Cannot link file ranges; {base_msg}") from e
+ raise
+
+
+def check_exists(path: StrPath) -> Optional[StrPath]:
+ """Look for variations of `path` and return the first found.
+
+ This exists to support former behavior around system-dependent paths; we used to use
+ ':' in Artifact paths unless we were on Windows, but this has issues when e.g. a
+ Linux machine is accessing an NTFS filesystem; we might need to look for the
+ alternate path. This checks all the possible directories we would consider creating.
+ """
+ for dest in path_fallbacks(path):
+ if os.path.exists(dest):
+ return Path(dest) if isinstance(path, Path) else dest
+ return None
+
+
+def system_preferred_path(path: StrPath, warn: bool = False) -> StrPath:
+ """Replace ':' with '-' in paths on Windows.
+
+ Args:
+ path: The path to convert.
+ warn: Whether to warn if ':' is replaced.
+ """
+ if platform.system() != "Windows":
+ return path
+ head, tail = os.path.splitdrive(path)
+ if warn and ":" in tail:
+ logger.warning(f"Replacing ':' in {tail} with '-'")
+ new_path = head + tail.replace(":", "-")
+ return Path(new_path) if isinstance(path, Path) else new_path
diff --git a/wandb/sdk/lib/fsm.py b/wandb/sdk/lib/fsm.py
new file mode 100755
index 0000000000000000000000000000000000000000..9957a681eb440815232ed5b04d2d29d74666305e
--- /dev/null
+++ b/wandb/sdk/lib/fsm.py
@@ -0,0 +1,178 @@
+#!/usr/bin/env python
+"""Finite state machine.
+
+Simple FSM implementation.
+
+Usage:
+ ```python
+ class A:
+ def on_output(self, inputs) -> None:
+ pass
+
+
+ class B:
+ def on_output(self, inputs) -> None:
+ pass
+
+
+ def to_b(inputs) -> bool:
+ return True
+
+
+ def to_a(inputs) -> bool:
+ return True
+
+
+ f = Fsm(states=[A(), B()], table={A: [(to_b, B)], B: [(to_a, A)]})
+ f.run({"input1": 1, "input2": 2})
+ ```
+"""
+
+import sys
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Callable, Dict, Generic, Optional, Sequence, Type, TypeVar, Union
+
+if sys.version_info >= (3, 8):
+ from typing import Protocol, runtime_checkable
+else:
+ from typing_extensions import Protocol, runtime_checkable
+
+if sys.version_info >= (3, 10):
+ from typing import TypeAlias
+else:
+ from typing_extensions import TypeAlias
+
+T_FsmInputs = TypeVar("T_FsmInputs", contravariant=True)
+T_FsmContext = TypeVar("T_FsmContext")
+T_FsmContext_cov = TypeVar("T_FsmContext_cov", covariant=True)
+T_FsmContext_contra = TypeVar("T_FsmContext_contra", contravariant=True)
+
+
+@runtime_checkable
+class FsmStateCheck(Protocol[T_FsmInputs]):
+ @abstractmethod
+ def on_check(self, inputs: T_FsmInputs) -> None:
+ ... # pragma: no cover
+
+
+@runtime_checkable
+class FsmStateOutput(Protocol[T_FsmInputs]):
+ @abstractmethod
+ def on_state(self, inputs: T_FsmInputs) -> None:
+ ... # pragma: no cover
+
+
+@runtime_checkable
+class FsmStateEnter(Protocol[T_FsmInputs]):
+ @abstractmethod
+ def on_enter(self, inputs: T_FsmInputs) -> None:
+ ... # pragma: no cover
+
+
+@runtime_checkable
+class FsmStateEnterWithContext(Protocol[T_FsmInputs, T_FsmContext_contra]):
+ @abstractmethod
+ def on_enter(self, inputs: T_FsmInputs, context: T_FsmContext_contra) -> None:
+ ... # pragma: no cover
+
+
+@runtime_checkable
+class FsmStateStay(Protocol[T_FsmInputs]):
+ @abstractmethod
+ def on_stay(self, inputs: T_FsmInputs) -> None:
+ ... # pragma: no cover
+
+
+@runtime_checkable
+class FsmStateExit(Protocol[T_FsmInputs, T_FsmContext_cov]):
+ @abstractmethod
+ def on_exit(self, inputs: T_FsmInputs) -> T_FsmContext_cov:
+ ... # pragma: no cover
+
+
+# It would be nice if python provided optional protocol members, but it doesnt as described here:
+# https://peps.python.org/pep-0544/#support-optional-protocol-members
+# Until then, we can only enforce that a state at least supports one protocol interface. This
+# unfortunately will not check the signature of other potential protocols.
+FsmState: TypeAlias = Union[
+ FsmStateCheck[T_FsmInputs],
+ FsmStateOutput[T_FsmInputs],
+ FsmStateEnter[T_FsmInputs],
+ FsmStateEnterWithContext[T_FsmInputs, T_FsmContext],
+ FsmStateStay[T_FsmInputs],
+ FsmStateExit[T_FsmInputs, T_FsmContext],
+]
+
+
+@dataclass
+class FsmEntry(Generic[T_FsmInputs, T_FsmContext]):
+ condition: Callable[[T_FsmInputs], bool]
+ target_state: Type[FsmState[T_FsmInputs, T_FsmContext]]
+ action: Optional[Callable[[T_FsmInputs], None]] = None
+
+
+FsmTableWithContext: TypeAlias = Dict[
+ Type[FsmState[T_FsmInputs, T_FsmContext]],
+ Sequence[FsmEntry[T_FsmInputs, T_FsmContext]],
+]
+
+
+FsmTable: TypeAlias = FsmTableWithContext[T_FsmInputs, None]
+
+
+class FsmWithContext(Generic[T_FsmInputs, T_FsmContext]):
+ _state_dict: Dict[Type[FsmState], FsmState]
+ _table: FsmTableWithContext[T_FsmInputs, T_FsmContext]
+ _state: FsmState[T_FsmInputs, T_FsmContext]
+ _states: Sequence[FsmState]
+
+ def __init__(
+ self,
+ states: Sequence[FsmState],
+ table: FsmTableWithContext[T_FsmInputs, T_FsmContext],
+ ) -> None:
+ self._states = states
+ self._table = table
+ self._state_dict = {type(s): s for s in states}
+ self._state = self._state_dict[type(states[0])]
+
+ def _transition(
+ self,
+ inputs: T_FsmInputs,
+ new_state: Type[FsmState[T_FsmInputs, T_FsmContext]],
+ action: Optional[Callable[[T_FsmInputs], None]],
+ ) -> None:
+ if action:
+ action(inputs)
+
+ context = None
+ if isinstance(self._state, FsmStateExit):
+ context = self._state.on_exit(inputs)
+
+ prev_state = type(self._state)
+ if prev_state == new_state:
+ if isinstance(self._state, FsmStateStay):
+ self._state.on_stay(inputs)
+ else:
+ self._state = self._state_dict[new_state]
+ if context and isinstance(self._state, FsmStateEnterWithContext):
+ self._state.on_enter(inputs, context=context)
+ elif isinstance(self._state, FsmStateEnter):
+ self._state.on_enter(inputs)
+
+ def _check_transitions(self, inputs: T_FsmInputs) -> None:
+ for entry in self._table[type(self._state)]:
+ if entry.condition(inputs):
+ self._transition(inputs, entry.target_state, entry.action)
+ return
+
+ def input(self, inputs: T_FsmInputs) -> None:
+ if isinstance(self._state, FsmStateCheck):
+ self._state.on_check(inputs)
+ self._check_transitions(inputs)
+ if isinstance(self._state, FsmStateOutput):
+ self._state.on_state(inputs)
+
+
+Fsm: TypeAlias = FsmWithContext[T_FsmInputs, None]
diff --git a/wandb/sdk/lib/gitlib.py b/wandb/sdk/lib/gitlib.py
new file mode 100644
index 0000000000000000000000000000000000000000..5cb079842c67dc45db12c4e83234160b0282a062
--- /dev/null
+++ b/wandb/sdk/lib/gitlib.py
@@ -0,0 +1,239 @@
+import configparser
+import logging
+import os
+from typing import TYPE_CHECKING, Any, Optional
+from urllib.parse import urlparse, urlunparse
+
+import wandb
+
+try:
+ from git import ( # type: ignore
+ GitCommandError,
+ InvalidGitRepositoryError,
+ NoSuchPathError,
+ Repo,
+ )
+except ImportError:
+ Repo = None
+
+if TYPE_CHECKING:
+ from git import Repo
+
+
+logger = logging.getLogger(__name__)
+
+
+class GitRepo:
+ def __init__(
+ self,
+ root: Optional[str] = None,
+ remote: str = "origin",
+ lazy: bool = True,
+ remote_url: Optional[str] = None,
+ commit: Optional[str] = None,
+ ) -> None:
+ self.remote_name = remote if remote_url is None else None
+ self._root = root
+ self._remote_url = remote_url
+ self._commit = commit
+ self._repo = None
+ self._repo_initialized = False
+ if not lazy:
+ self._repo = self._init_repo()
+
+ def _init_repo(self) -> Optional[Repo]:
+ self._repo_initialized = True
+ if Repo is None:
+ return None
+ if self.remote_name is None:
+ return None
+ try:
+ return Repo(self._root or os.getcwd(), search_parent_directories=True)
+ except FileNotFoundError:
+ wandb.termwarn("current working directory has been invalidated")
+ logger.warn("current working directory has been invalidated")
+ except InvalidGitRepositoryError:
+ logger.debug("git repository is invalid")
+ except NoSuchPathError:
+ wandb.termwarn(f"git root {self._root} does not exist")
+ logger.warn(f"git root {self._root} does not exist")
+ return None
+
+ @property
+ def repo(self) -> Optional[Repo]:
+ if not self._repo_initialized:
+ self._repo = self._init_repo()
+ return self._repo
+
+ @property
+ def auto(self) -> bool:
+ return self._remote_url is None
+
+ def is_untracked(self, file_name: str) -> Optional[bool]:
+ if not self.repo:
+ return True
+ try:
+ return file_name in self.repo.untracked_files
+ except GitCommandError:
+ return None
+
+ @property
+ def enabled(self) -> bool:
+ return bool(self.repo)
+
+ @property
+ def root(self) -> Any:
+ if not self.repo:
+ return None
+ try:
+ return self.repo.git.rev_parse("--show-toplevel")
+ except GitCommandError as e:
+ # todo: collect telemetry on this
+ logger.error(f"git root error: {e}")
+ return None
+
+ @property
+ def dirty(self) -> Any:
+ if not self.repo:
+ return False
+ try:
+ return self.repo.is_dirty()
+ except GitCommandError:
+ return False
+
+ @property
+ def email(self) -> Optional[str]:
+ if not self.repo:
+ return None
+ try:
+ return self.repo.config_reader().get_value("user", "email") # type: ignore
+ except configparser.Error:
+ return None
+
+ @property
+ def last_commit(self) -> Any:
+ if self._commit:
+ return self._commit
+ if not self.repo:
+ return None
+ if not self.repo.head or not self.repo.head.is_valid():
+ return None
+ # TODO: Saw a user getting a Unicode decode error when parsing refs,
+ # more details on implementing a real fix in [WB-4064]
+ try:
+ if len(self.repo.refs) > 0:
+ return self.repo.head.commit.hexsha
+ else:
+ return self.repo.git.show_ref("--head").split(" ")[0]
+ except Exception:
+ logger.exception("Unable to find most recent commit in git")
+ return None
+
+ @property
+ def branch(self) -> Any:
+ if not self.repo:
+ return None
+ return self.repo.head.ref.name
+
+ @property
+ def remote(self) -> Any:
+ if not self.repo:
+ return None
+ try:
+ return self.repo.remotes[self.remote_name]
+ except IndexError:
+ return None
+
+ # the --submodule=diff option doesn't exist in pre-2.11 versions of git (november 2016)
+ # https://stackoverflow.com/questions/10757091/git-list-of-all-changed-files-including-those-in-submodules
+ @property
+ def has_submodule_diff(self) -> bool:
+ if not self.repo:
+ return False
+ return bool(self.repo.git.version_info >= (2, 11, 0))
+
+ @property
+ def remote_url(self) -> Any:
+ if self._remote_url:
+ return self._remote_url
+ if not self.remote:
+ return None
+ parsed = urlparse(self.remote.url)
+ hostname = parsed.hostname
+ if parsed.port is not None:
+ hostname = f"{hostname}:{parsed.port}"
+ if parsed.password is not None:
+ return urlunparse(parsed._replace(netloc=f"{parsed.username}:@{hostname}"))
+ return urlunparse(parsed._replace(netloc=hostname))
+
+ @property
+ def root_dir(self) -> Any:
+ if not self.repo:
+ return None
+ try:
+ return self.repo.git.rev_parse("--show-toplevel")
+ except GitCommandError:
+ return None
+
+ def get_upstream_fork_point(self) -> Any:
+ """Get the most recent ancestor of HEAD that occurs on an upstream branch.
+
+ First looks at the current branch's tracking branch, if applicable. If
+ that doesn't work, looks at every other branch to find the most recent
+ ancestor of HEAD that occurs on a tracking branch.
+
+ Returns:
+ git.Commit object or None
+ """
+ possible_relatives = []
+ try:
+ if not self.repo:
+ return None
+ try:
+ active_branch = self.repo.active_branch
+ except (TypeError, ValueError):
+ logger.debug("git is in a detached head state")
+ return None # detached head
+ else:
+ tracking_branch = active_branch.tracking_branch()
+ if tracking_branch:
+ possible_relatives.append(tracking_branch.commit)
+
+ if not possible_relatives:
+ for branch in self.repo.branches:
+ tracking_branch = branch.tracking_branch()
+ if tracking_branch is not None:
+ possible_relatives.append(tracking_branch.commit)
+
+ head = self.repo.head
+ most_recent_ancestor = None
+ for possible_relative in possible_relatives:
+ # at most one:
+ for ancestor in self.repo.merge_base(head, possible_relative):
+ if most_recent_ancestor is None:
+ most_recent_ancestor = ancestor
+ elif self.repo.is_ancestor(most_recent_ancestor, ancestor): # type: ignore
+ most_recent_ancestor = ancestor
+ return most_recent_ancestor
+ except GitCommandError as e:
+ logger.debug("git remote upstream fork point could not be found")
+ logger.debug(str(e))
+ return None
+
+ def tag(self, name: str, message: Optional[str]) -> Any:
+ if not self.repo:
+ return None
+ try:
+ return self.repo.create_tag(f"wandb/{name}", message=message, force=True)
+ except GitCommandError:
+ print("Failed to tag repository.")
+ return None
+
+ def push(self, name: str) -> Any:
+ if not self.remote:
+ return None
+ try:
+ return self.remote.push(f"wandb/{name}", force=True)
+ except GitCommandError:
+ logger.debug("failed to push git")
+ return None
diff --git a/wandb/sdk/lib/gql_request.py b/wandb/sdk/lib/gql_request.py
new file mode 100644
index 0000000000000000000000000000000000000000..381097447c805593f7710887b765e475f3f4a904
--- /dev/null
+++ b/wandb/sdk/lib/gql_request.py
@@ -0,0 +1,65 @@
+"""A simple GraphQL client for sending queries and mutations.
+
+Note: This was originally wandb/vendor/gql-0.2.0/wandb_gql/transport/requests.py
+The only substantial change is to re-use a requests.Session object.
+"""
+
+from typing import Any, Callable, Dict, Optional, Tuple, Union
+
+import requests
+from wandb_gql.transport.http import HTTPTransport
+from wandb_graphql.execution import ExecutionResult
+from wandb_graphql.language import ast
+from wandb_graphql.language.printer import print_ast
+
+
+class GraphQLSession(HTTPTransport):
+ def __init__(
+ self,
+ url: str,
+ auth: Optional[Union[Tuple[str, str], Callable]] = None,
+ use_json: bool = False,
+ timeout: Optional[Union[int, float]] = None,
+ proxies: Optional[Dict[str, str]] = None,
+ **kwargs: Any,
+ ) -> None:
+ """Setup a session for sending GraphQL queries and mutations.
+
+ Args:
+ url (str): The GraphQL URL
+ auth (tuple or callable): Auth tuple or callable for Basic/Digest/Custom HTTP Auth
+ use_json (bool): Send request body as JSON instead of form-urlencoded
+ timeout (int, float): Specifies a default timeout for requests (Default: None)
+ """
+ super().__init__(url, **kwargs)
+ self.session = requests.Session()
+ if proxies:
+ self.session.proxies.update(proxies)
+ self.session.auth = auth
+ self.default_timeout = timeout
+ self.use_json = use_json
+
+ def execute(
+ self,
+ document: ast.Node,
+ variable_values: Optional[Dict] = None,
+ timeout: Optional[Union[int, float]] = None,
+ ) -> ExecutionResult:
+ query_str = print_ast(document)
+ payload = {"query": query_str, "variables": variable_values or {}}
+
+ data_key = "json" if self.use_json else "data"
+ post_args = {
+ "headers": self.headers,
+ "cookies": self.cookies,
+ "timeout": timeout or self.default_timeout,
+ data_key: payload,
+ }
+ request = self.session.post(self.url, **post_args)
+ request.raise_for_status()
+
+ result = request.json()
+ data, errors = result.get("data"), result.get("errors")
+ if data is None and errors is None:
+ raise RuntimeError(f"Received non-compatible response: {result}")
+ return ExecutionResult(data=data, errors=errors)
diff --git a/wandb/sdk/lib/handler_util.py b/wandb/sdk/lib/handler_util.py
new file mode 100644
index 0000000000000000000000000000000000000000..b4efd8d571baebab80f9a7702efdd962f0c9028d
--- /dev/null
+++ b/wandb/sdk/lib/handler_util.py
@@ -0,0 +1,21 @@
+import wandb.data_types as data_types
+
+
+def get_types():
+ classes = map(data_types.__dict__.get, data_types.__all__)
+ types = []
+ for cls in classes:
+ if hasattr(cls, "_log_type") and cls._log_type is not None:
+ types.append(cls._log_type)
+ # add table-file type because this is a special case
+ # that does not have a matching _log_type for artifacts
+ # and files
+ types.append("table-file")
+ return types
+
+
+WANDB_TYPES = get_types()
+
+
+def metric_is_wandb_dict(metric):
+ return "_type" in list(metric.keys()) and metric["_type"] in WANDB_TYPES
diff --git a/wandb/sdk/lib/hashutil.py b/wandb/sdk/lib/hashutil.py
new file mode 100644
index 0000000000000000000000000000000000000000..579c86c9116207e8bfd7e02307ed2da4460822d3
--- /dev/null
+++ b/wandb/sdk/lib/hashutil.py
@@ -0,0 +1,62 @@
+import base64
+import hashlib
+import mmap
+import os
+import sys
+from pathlib import Path
+from typing import NewType, Union
+
+from wandb.sdk.lib.paths import StrPath
+
+ETag = NewType("ETag", str)
+HexMD5 = NewType("HexMD5", str)
+B64MD5 = NewType("B64MD5", str)
+
+
+def _md5(data: bytes = b"") -> "hashlib._Hash":
+ """Allow FIPS-compliant md5 hash when supported."""
+ if sys.version_info >= (3, 9):
+ return hashlib.md5(data, usedforsecurity=False)
+ else:
+ return hashlib.md5(data)
+
+
+def md5_string(string: str) -> B64MD5:
+ return _b64_from_hasher(_md5(string.encode("utf-8")))
+
+
+def _b64_from_hasher(hasher: "hashlib._Hash") -> B64MD5:
+ return B64MD5(base64.b64encode(hasher.digest()).decode("ascii"))
+
+
+def b64_to_hex_id(string: B64MD5) -> HexMD5:
+ return HexMD5(base64.standard_b64decode(string).hex())
+
+
+def hex_to_b64_id(encoded_string: Union[str, bytes]) -> B64MD5:
+ if isinstance(encoded_string, bytes):
+ encoded_string = encoded_string.decode("utf-8")
+ as_str = bytes.fromhex(encoded_string)
+ return B64MD5(base64.standard_b64encode(as_str).decode("utf-8"))
+
+
+def md5_file_b64(*paths: StrPath) -> B64MD5:
+ return _b64_from_hasher(_md5_file_hasher(*paths))
+
+
+def md5_file_hex(*paths: StrPath) -> HexMD5:
+ return HexMD5(_md5_file_hasher(*paths).hexdigest())
+
+
+def _md5_file_hasher(*paths: StrPath) -> "hashlib._Hash":
+ md5_hash = _md5()
+
+ for path in sorted(Path(p) for p in paths):
+ with path.open("rb") as f:
+ if os.stat(f.fileno()).st_size <= 1024 * 1024:
+ md5_hash.update(f.read())
+ else:
+ with mmap.mmap(f.fileno(), length=0, access=mmap.ACCESS_READ) as mview:
+ md5_hash.update(mview)
+
+ return md5_hash
diff --git a/wandb/sdk/lib/import_hooks.py b/wandb/sdk/lib/import_hooks.py
new file mode 100644
index 0000000000000000000000000000000000000000..b7da5d7d82ffa464190f8a1901dd67b0fcfd684c
--- /dev/null
+++ b/wandb/sdk/lib/import_hooks.py
@@ -0,0 +1,275 @@
+"""Implements a post-import hook mechanism.
+
+Styled as per PEP-369. Note that it doesn't cope with modules being reloaded.
+
+Note: This file is based on
+https://github.com/GrahamDumpleton/wrapt/blob/1.12.1/src/wrapt/importer.py
+and manual backports of later patches up to 1.15.0 in the wrapt repository
+(with slight modifications).
+"""
+
+import sys
+import threading
+from importlib.util import find_spec
+from typing import Any, Callable, Dict, Optional, Union
+
+# The dictionary registering any post import hooks to be triggered once
+# the target module has been imported. Once a module has been imported
+# and the hooks fired, the list of hooks recorded against the target
+# module will be truncated but the list left in the dictionary. This
+# acts as a flag to indicate that the module had already been imported.
+
+_post_import_hooks: Dict = {}
+_post_import_hooks_init: bool = False
+_post_import_hooks_lock = threading.RLock()
+
+# Register a new post import hook for the target module name. This
+# differs from the PEP-369 implementation in that it also allows the
+# hook function to be specified as a string consisting of the name of
+# the callback in the form 'module:function'. This will result in a
+# proxy callback being registered which will defer loading of the
+# specified module containing the callback function until required.
+
+
+def _create_import_hook_from_string(name: str) -> Callable:
+ def import_hook(module: Any) -> Callable:
+ module_name, function = name.split(":")
+ attrs = function.split(".")
+ __import__(module_name)
+ callback = sys.modules[module_name]
+ for attr in attrs:
+ callback = getattr(callback, attr)
+ return callback(module) # type: ignore
+
+ return import_hook
+
+
+def register_post_import_hook(
+ hook: Union[str, Callable], hook_id: str, name: str
+) -> None:
+ # Create a deferred import hook if hook is a string name rather than
+ # a callable function.
+
+ if isinstance(hook, (str,)):
+ hook = _create_import_hook_from_string(hook)
+
+ # Automatically install the import hook finder if it has not already
+ # been installed.
+
+ with _post_import_hooks_lock:
+ global _post_import_hooks_init
+
+ if not _post_import_hooks_init:
+ _post_import_hooks_init = True
+ sys.meta_path.insert(0, ImportHookFinder()) # type: ignore
+
+ # Check if the module is already imported. If not, register the hook
+ # to be called after import.
+
+ module = sys.modules.get(name, None)
+
+ if module is None:
+ _post_import_hooks.setdefault(name, {}).update({hook_id: hook})
+
+ # If the module is already imported, we fire the hook right away. Note that
+ # the hook is called outside of the lock to avoid deadlocks if code run as a
+ # consequence of calling the module import hook in turn triggers a separate
+ # thread which tries to register an import hook.
+
+ if module is not None:
+ hook(module)
+
+
+def unregister_post_import_hook(name: str, hook_id: Optional[str]) -> None:
+ # Remove the import hook if it has been registered.
+ with _post_import_hooks_lock:
+ hooks = _post_import_hooks.get(name)
+
+ if hooks is not None:
+ if hook_id is not None:
+ hooks.pop(hook_id, None)
+
+ if not hooks:
+ del _post_import_hooks[name]
+ else:
+ del _post_import_hooks[name]
+
+
+def unregister_all_post_import_hooks() -> None:
+ with _post_import_hooks_lock:
+ _post_import_hooks.clear()
+
+
+# Indicate that a module has been loaded. Any post import hooks which
+# were registered against the target module will be invoked. If an
+# exception is raised in any of the post import hooks, that will cause
+# the import of the target module to fail.
+
+
+def notify_module_loaded(module: Any) -> None:
+ name = getattr(module, "__name__", None)
+
+ with _post_import_hooks_lock:
+ hooks = _post_import_hooks.pop(name, {})
+
+ # Note that the hook is called outside of the lock to avoid deadlocks if
+ # code run as a consequence of calling the module import hook in turn
+ # triggers a separate thread which tries to register an import hook.
+ for hook in hooks.values():
+ if hook:
+ hook(module)
+
+
+# A custom module import finder. This intercepts attempts to import
+# modules and watches out for attempts to import target modules of
+# interest. When a module of interest is imported, then any post import
+# hooks which are registered will be invoked.
+
+
+class _ImportHookChainedLoader:
+ def __init__(self, loader: Any) -> None:
+ self.loader = loader
+
+ if hasattr(loader, "load_module"):
+ self.load_module = self._load_module
+ if hasattr(loader, "create_module"):
+ self.create_module = self._create_module
+ if hasattr(loader, "exec_module"):
+ self.exec_module = self._exec_module
+
+ def _set_loader(self, module: Any) -> None:
+ # Set module's loader to self.loader unless it's already set to
+ # something else. Import machinery will set it to spec.loader if it is
+ # None, so handle None as well. The module may not support attribute
+ # assignment, in which case we simply skip it. Note that we also deal
+ # with __loader__ not existing at all. This is to future proof things
+ # due to proposal to remove the attribue as described in the GitHub
+ # issue at https://github.com/python/cpython/issues/77458. Also prior
+ # to Python 3.3, the __loader__ attribute was only set if a custom
+ # module loader was used. It isn't clear whether the attribute still
+ # existed in that case or was set to None.
+
+ class UNDEFINED:
+ pass
+
+ if getattr(module, "__loader__", UNDEFINED) in (None, self):
+ try:
+ module.__loader__ = self.loader
+ except AttributeError:
+ pass
+
+ if (
+ getattr(module, "__spec__", None) is not None
+ and getattr(module.__spec__, "loader", None) is self
+ ):
+ module.__spec__.loader = self.loader
+
+ def _load_module(self, fullname: str) -> Any:
+ module = self.loader.load_module(fullname)
+ self._set_loader(module)
+ notify_module_loaded(module)
+
+ return module
+
+ # Python 3.4 introduced create_module() and exec_module() instead of
+ # load_module() alone. Splitting the two steps.
+
+ def _create_module(self, spec: Any) -> Any:
+ return self.loader.create_module(spec)
+
+ def _exec_module(self, module: Any) -> None:
+ self._set_loader(module)
+ self.loader.exec_module(module)
+ notify_module_loaded(module)
+
+
+class ImportHookFinder:
+ def __init__(self) -> None:
+ self.in_progress: Dict = {}
+
+ def find_module( # type: ignore
+ self,
+ fullname: str,
+ path: Optional[str] = None,
+ ) -> Optional["_ImportHookChainedLoader"]:
+ # If the module being imported is not one we have registered
+ # post import hooks for, we can return immediately. We will
+ # take no further part in the importing of this module.
+
+ with _post_import_hooks_lock:
+ if fullname not in _post_import_hooks:
+ return None
+
+ # When we are interested in a specific module, we will call back
+ # into the import system a second time to defer to the import
+ # finder that is supposed to handle the importing of the module.
+ # We set an in progress flag for the target module so that on
+ # the second time through we don't trigger another call back
+ # into the import system and cause a infinite loop.
+
+ if fullname in self.in_progress:
+ return None
+
+ self.in_progress[fullname] = True
+
+ # Now call back into the import system again.
+
+ try:
+ # For Python 3 we need to use find_spec().loader
+ # from the importlib.util module. It doesn't actually
+ # import the target module and only finds the
+ # loader. If a loader is found, we need to return
+ # our own loader which will then in turn call the
+ # real loader to import the module and invoke the
+ # post import hooks.
+ loader = getattr(find_spec(fullname), "loader", None)
+
+ if loader and not isinstance(loader, _ImportHookChainedLoader):
+ return _ImportHookChainedLoader(loader)
+
+ finally:
+ del self.in_progress[fullname]
+
+ def find_spec(
+ self, fullname: str, path: Optional[str] = None, target: Any = None
+ ) -> Any:
+ # Since Python 3.4, you are meant to implement find_spec() method
+ # instead of find_module() and since Python 3.10 you get deprecation
+ # warnings if you don't define find_spec().
+
+ # If the module being imported is not one we have registered
+ # post import hooks for, we can return immediately. We will
+ # take no further part in the importing of this module.
+
+ with _post_import_hooks_lock:
+ if fullname not in _post_import_hooks:
+ return None
+
+ # When we are interested in a specific module, we will call back
+ # into the import system a second time to defer to the import
+ # finder that is supposed to handle the importing of the module.
+ # We set an in progress flag for the target module so that on
+ # the second time through we don't trigger another call back
+ # into the import system and cause a infinite loop.
+
+ if fullname in self.in_progress:
+ return None
+
+ self.in_progress[fullname] = True
+
+ # Now call back into the import system again.
+
+ try:
+ # This should only be Python 3 so find_spec() should always
+ # exist so don't need to check.
+ spec = find_spec(fullname)
+ loader = getattr(spec, "loader", None)
+
+ if loader and not isinstance(loader, _ImportHookChainedLoader):
+ assert spec is not None
+ spec.loader = _ImportHookChainedLoader(loader) # type: ignore
+
+ return spec
+
+ finally:
+ del self.in_progress[fullname]
diff --git a/wandb/sdk/lib/ipython.py b/wandb/sdk/lib/ipython.py
new file mode 100644
index 0000000000000000000000000000000000000000..19d5cd9067b4b6294fed5119ddde2d56b97c7d3f
--- /dev/null
+++ b/wandb/sdk/lib/ipython.py
@@ -0,0 +1,146 @@
+import logging
+import sys
+import warnings
+from typing import Optional
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+import wandb
+
+PythonType = Literal["python", "ipython", "jupyter"]
+
+logger = logging.getLogger(__name__)
+
+
+TABLE_STYLES = """<style>
+ table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}
+ .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }
+ .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }
+ </style>
+"""
+
+
+def toggle_button(what="run"):
+ return f"<button onClick=\"this.nextSibling.style.display='block';this.style.display='none';\">Display W&B {what}</button>"
+
+
+def _get_python_type() -> PythonType:
+ if "IPython" not in sys.modules:
+ return "python"
+
+ try:
+ from IPython import get_ipython # type: ignore
+
+ # Calling get_ipython can cause an ImportError
+ if get_ipython() is None:
+ return "python"
+ except ImportError:
+ return "python"
+
+ # jupyter-based environments (e.g. jupyter itself, colab, kaggle, etc) have a connection file
+ ip_kernel_app_connection_file = (
+ (get_ipython().config.get("IPKernelApp", {}) or {})
+ .get("connection_file", "")
+ .lower()
+ ) or (
+ (get_ipython().config.get("ColabKernelApp", {}) or {})
+ .get("connection_file", "")
+ .lower()
+ )
+
+ if (
+ ("terminal" in get_ipython().__module__)
+ or ("jupyter" not in ip_kernel_app_connection_file)
+ or ("spyder" in sys.modules)
+ ):
+ return "ipython"
+ else:
+ return "jupyter"
+
+
+def in_jupyter() -> bool:
+ return _get_python_type() == "jupyter"
+
+
+def in_notebook() -> bool:
+ return _get_python_type() != "python"
+
+
+def display_html(html: str): # type: ignore
+ """Display HTML in notebooks, is a noop outside a jupyter context."""
+ if wandb.run and wandb.run._settings.silent:
+ return
+ try:
+ from IPython.core.display import HTML, display # type: ignore
+ except ImportError:
+ wandb.termwarn("Unable to render HTML, can't import display from ipython.core")
+ return False
+ return display(HTML(html))
+
+
+def display_widget(widget):
+ """Display ipywidgets in notebooks, is a noop outside of a jupyter context."""
+ if wandb.run and wandb.run._settings.silent:
+ return
+ try:
+ from IPython.core.display import display
+ except ImportError:
+ wandb.termwarn(
+ "Unable to render Widget, can't import display from ipython.core"
+ )
+ return False
+ return display(widget)
+
+
+class ProgressWidget:
+ """A simple wrapper to render a nice progress bar with a label."""
+
+ def __init__(self, widgets, min, max):
+ self.widgets = widgets
+ self._progress = widgets.FloatProgress(min=min, max=max)
+ self._label = widgets.Label()
+ self._widget = self.widgets.VBox([self._label, self._progress])
+ self._displayed = False
+ self._disabled = False
+
+ def update(self, value: float, label: str) -> None:
+ if self._disabled:
+ return
+ try:
+ self._progress.value = value
+ self._label.value = label
+ if not self._displayed:
+ self._displayed = True
+ display_widget(self._widget)
+ except Exception as e:
+ self._disabled = True
+ logger.exception(e)
+ wandb.termwarn(
+ "Unable to render progress bar, see the user log for details"
+ )
+
+ def close(self) -> None:
+ if self._disabled or not self._displayed:
+ return
+ self._widget.close()
+
+
+def jupyter_progress_bar(min: float = 0, max: float = 1.0) -> Optional[ProgressWidget]:
+ """Return an ipywidget progress bar or None if we can't import it."""
+ widgets = wandb.util.get_module("ipywidgets")
+ try:
+ if widgets is None:
+ # TODO: this currently works in iPython but it's deprecated since 4.0
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ from IPython.html import widgets # type: ignore
+
+ assert hasattr(widgets, "VBox")
+ assert hasattr(widgets, "Label")
+ assert hasattr(widgets, "FloatProgress")
+ return ProgressWidget(widgets, min=min, max=max)
+ except (ImportError, AssertionError):
+ return None
diff --git a/wandb/sdk/lib/json_util.py b/wandb/sdk/lib/json_util.py
new file mode 100644
index 0000000000000000000000000000000000000000..6407319221118b822d7b047f1599f7cfe3b954d0
--- /dev/null
+++ b/wandb/sdk/lib/json_util.py
@@ -0,0 +1,80 @@
+import json
+import logging
+import os
+from typing import Any, Union
+
+logger = logging.getLogger(__name__)
+
+
+try:
+ import orjson # type: ignore
+
+ # todo: orjson complies with the json standard and does not support
+ # NaN, Infinity, and -Infinity. Should be fixed in the future.
+
+ # additional safeguard for now
+ if os.environ.get("_WANDB_ORJSON"):
+
+ def dumps(obj: Any, **kwargs: Any) -> str:
+ """Wrapper for <json|orjson>.dumps."""
+ cls = kwargs.pop("cls", None)
+ try:
+ _kwargs = kwargs.copy()
+ if cls:
+ _kwargs["default"] = cls.default
+ encoded = orjson.dumps(
+ obj, option=orjson.OPT_NON_STR_KEYS, **_kwargs
+ ).decode()
+ except Exception as e:
+ logger.exception(f"Error using orjson.dumps: {e}")
+ if cls:
+ kwargs["cls"] = cls
+ encoded = json.dumps(obj, **kwargs)
+
+ return encoded # type: ignore[no-any-return]
+
+ def dump(obj: Any, fp: Any, **kwargs: Any) -> None:
+ """Wrapper for <json|orjson>.dump."""
+ cls = kwargs.pop("cls", None)
+ try:
+ _kwargs = kwargs.copy()
+ if cls:
+ _kwargs["default"] = cls.default
+ encoded = orjson.dumps(obj, option=orjson.OPT_NON_STR_KEYS, **_kwargs)
+ fp.write(encoded)
+ except Exception as e:
+ logger.exception(f"Error using orjson.dump: {e}")
+ if cls:
+ kwargs["cls"] = cls
+ json.dump(obj, fp, **kwargs)
+
+ def loads(obj: Union[str, bytes]) -> Any:
+ """Wrapper for orjson.loads."""
+ try:
+ decoded = orjson.loads(obj)
+ except Exception as e:
+ logger.exception(f"Error using orjson.loads: {e}")
+ decoded = json.loads(obj)
+
+ return decoded
+
+ def load(fp: Any) -> Any:
+ """Wrapper for orjson.load."""
+ try:
+ decoded = orjson.loads(fp.read())
+ except Exception as e:
+ logger.exception(f"Error using orjson.load: {e}")
+ decoded = json.load(fp)
+
+ return decoded
+
+ else:
+ from json import ( # type: ignore[assignment] # noqa: F401
+ dump,
+ dumps,
+ load,
+ loads,
+ )
+
+except ImportError:
+ from json import dump, dumps, load, loads # type: ignore[assignment] # noqa: F401
diff --git a/wandb/sdk/lib/lazyloader.py b/wandb/sdk/lib/lazyloader.py
new file mode 100644
index 0000000000000000000000000000000000000000..fe7482174ac2e6242aa0a2c6500a27215de49d73
--- /dev/null
+++ b/wandb/sdk/lib/lazyloader.py
@@ -0,0 +1,64 @@
+"""module lazyloader."""
+
+
+import importlib
+import sys
+import types
+
+
+class LazyLoader(types.ModuleType):
+ """Lazily import a module, mainly to avoid pulling in large dependencies.
+
+ We use this for tensorflow and other optional libraries primarily at the
+ top module level.
+ """
+
+ # The lint error here is incorrect.
+ def __init__(
+ self,
+ local_name, # pylint: disable=super-on-old-class
+ parent_module_globals,
+ name,
+ warning=None,
+ ):
+ self._local_name = local_name
+ self._parent_module_globals = parent_module_globals
+ self._warning = warning
+
+ super().__init__(str(name))
+
+ def _load(self):
+ """Load the module and insert it into the parent's globals."""
+ # Import the target module and insert it into the parent's namespace
+ module = importlib.import_module(self.__name__)
+ self._parent_module_globals[self._local_name] = module
+ # print("import", self.__name__)
+ # print("Set global", self._local_name)
+ # print("mod", module)
+ sys.modules[self._local_name] = module
+
+ # Emit a warning if one was specified
+ if self._warning:
+ print(self._warning)
+ # Make sure to only warn once.
+ self._warning = None
+
+ # Update this object's dict so that if someone keeps a reference to the
+ # LazyLoader, lookups are efficient (__getattr__ is only called on lookups
+ # that fail).
+ self.__dict__.update(module.__dict__)
+
+ return module
+
+ # def __getattribute__(self, item):
+ # print("getattribute", item)
+
+ def __getattr__(self, item):
+ # print("getattr", item)
+ module = self._load()
+ return getattr(module, item)
+
+ def __dir__(self):
+ # print("dir")
+ module = self._load()
+ return dir(module)
diff --git a/wandb/sdk/lib/mailbox.py b/wandb/sdk/lib/mailbox.py
new file mode 100644
index 0000000000000000000000000000000000000000..e0dbb49c4b7567825b3b43fcb71fef6f0f926039
--- /dev/null
+++ b/wandb/sdk/lib/mailbox.py
@@ -0,0 +1,460 @@
+"""mailbox."""
+
+import secrets
+import string
+import threading
+import time
+from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple
+
+from wandb.errors import Error
+from wandb.proto import wandb_internal_pb2 as pb
+
+if TYPE_CHECKING:
+ from wandb.sdk.interface.interface_shared import InterfaceShared
+
+
+def _generate_address(length: int = 12) -> str:
+ address = "".join(
+ secrets.choice(string.ascii_lowercase + string.digits) for i in range(length)
+ )
+ return address
+
+
+class MailboxError(Error):
+ """Generic Mailbox Exception."""
+
+ pass
+
+
+class ContextCancelledError(MailboxError):
+ """Context cancelled Exception."""
+
+ pass
+
+
+class _MailboxWaitAll:
+ _event: threading.Event
+ _lock: threading.Lock
+ _handles: List["MailboxHandle"]
+ _failed_handles: int
+
+ def __init__(self) -> None:
+ self._event = threading.Event()
+ self._lock = threading.Lock()
+ self._handles = []
+ self._failed_handles = 0
+
+ def notify(self) -> None:
+ with self._lock:
+ self._event.set()
+
+ def _add_handle(self, handle: "MailboxHandle") -> None:
+ handle._slot._set_wait_all(self)
+ self._handles.append(handle)
+
+ # set wait_all event if an event has already been set before added to wait_all
+ if handle._slot._event.is_set():
+ self._event.set()
+
+ @property
+ def active_handles(self) -> List["MailboxHandle"]:
+ return [h for h in self._handles if not h._is_failed]
+
+ @property
+ def active_handles_count(self) -> int:
+ return len(self.active_handles)
+
+ @property
+ def failed_handles_count(self) -> int:
+ return self._failed_handles
+
+ def _mark_handle_failed(self, handle: "MailboxHandle") -> None:
+ handle._mark_failed()
+ self._failed_handles += 1
+
+ def clear_handles(self) -> None:
+ for handle in self._handles:
+ handle._slot._clear_wait_all()
+ self._handles = []
+
+ def _wait(self, timeout: float) -> bool:
+ return self._event.wait(timeout=timeout)
+
+ def _get_and_clear(self, timeout: float) -> List["MailboxHandle"]:
+ found: List[MailboxHandle] = []
+ if self._wait(timeout=timeout):
+ with self._lock:
+ remove_handles = []
+
+ # Look through handles for triggered events
+ for handle in self._handles:
+ if handle._slot._event.is_set():
+ found.append(handle)
+ remove_handles.append(handle)
+
+ for handle in remove_handles:
+ self._handles.remove(handle)
+
+ self._event.clear()
+ return found
+
+
+class _MailboxSlot:
+ _result: Optional[pb.Result]
+ _event: threading.Event
+ _lock: threading.Lock
+ _wait_all: Optional[_MailboxWaitAll]
+ _address: str
+ _abandoned: bool
+
+ def __init__(self, address: str) -> None:
+ self._result = None
+ self._event = threading.Event()
+ self._lock = threading.Lock()
+ self._address = address
+ self._wait_all = None
+ self._abandoned = False
+
+ def _set_wait_all(self, wait_all: _MailboxWaitAll) -> None:
+ assert not self._wait_all, "Only one caller can wait_all for a slot at a time"
+ self._wait_all = wait_all
+
+ def _clear_wait_all(self) -> None:
+ self._wait_all = None
+
+ def _wait(self, timeout: float) -> bool:
+ return self._event.wait(timeout=timeout)
+
+ def _get_and_clear(self, timeout: float) -> Tuple[Optional[pb.Result], bool]:
+ found = None
+ if self._wait(timeout=timeout):
+ with self._lock:
+ found = self._result
+ self._event.clear()
+ abandoned = self._abandoned
+ return found, abandoned
+
+ def _deliver(self, result: pb.Result) -> None:
+ with self._lock:
+ self._result = result
+ self._event.set()
+
+ if self._wait_all:
+ self._wait_all.notify()
+
+ def _notify_abandon(self) -> None:
+ self._abandoned = True
+ with self._lock:
+ self._event.set()
+
+ if self._wait_all:
+ self._wait_all.notify()
+
+
+class MailboxProbe:
+ _result: Optional[pb.Result]
+ _handle: Optional["MailboxHandle"]
+
+ def __init__(self) -> None:
+ self._handle = None
+ self._result = None
+
+ def set_probe_result(self, result: pb.Result) -> None:
+ self._result = result
+
+ def get_probe_result(self) -> Optional[pb.Result]:
+ return self._result
+
+ def get_mailbox_handle(self) -> Optional["MailboxHandle"]:
+ return self._handle
+
+ def set_mailbox_handle(self, handle: "MailboxHandle") -> None:
+ self._handle = handle
+
+
+class MailboxProgress:
+ _percent_done: float
+ _handle: "MailboxHandle"
+ _probe_handles: List[MailboxProbe]
+ _stopped: bool
+
+ def __init__(self, _handle: "MailboxHandle") -> None:
+ self._handle = _handle
+ self._percent_done = 0.0
+ self._probe_handles = []
+ self._stopped = False
+
+ @property
+ def percent_done(self) -> float:
+ return self._percent_done
+
+ def set_percent_done(self, percent_done: float) -> None:
+ self._percent_done = percent_done
+
+ def add_probe_handle(self, probe_handle: MailboxProbe) -> None:
+ self._probe_handles.append(probe_handle)
+
+ def get_probe_handles(self) -> List[MailboxProbe]:
+ return self._probe_handles
+
+ def wait_stop(self) -> None:
+ self._stopped = True
+
+ @property
+ def _is_stopped(self) -> bool:
+ return self._stopped
+
+
+class MailboxProgressAll:
+ _progress_handles: List[MailboxProgress]
+
+ def __init__(self) -> None:
+ self._progress_handles = []
+
+ def add_progress_handle(self, progress_handle: MailboxProgress) -> None:
+ self._progress_handles.append(progress_handle)
+
+ def get_progress_handles(self) -> List[MailboxProgress]:
+ # only return progress handles for not failed handles
+ return [ph for ph in self._progress_handles if not ph._handle._is_failed]
+
+
+class MailboxHandle:
+ _mailbox: "Mailbox"
+ _slot: _MailboxSlot
+ _on_probe: Optional[Callable[[MailboxProbe], None]]
+ _on_progress: Optional[Callable[[MailboxProgress], None]]
+ _interface: Optional["InterfaceShared"]
+ _keepalive: bool
+ _failed: bool
+
+ def __init__(self, mailbox: "Mailbox", slot: _MailboxSlot) -> None:
+ self._mailbox = mailbox
+ self._slot = slot
+ self._on_probe = None
+ self._on_progress = None
+ self._interface = None
+ self._keepalive = False
+ self._failed = False
+
+ def add_probe(self, on_probe: Callable[[MailboxProbe], None]) -> None:
+ self._on_probe = on_probe
+
+ def add_progress(self, on_progress: Callable[[MailboxProgress], None]) -> None:
+ self._on_progress = on_progress
+
+ def _time(self) -> float:
+ return time.monotonic()
+
+ def wait( # noqa: C901
+ self,
+ *,
+ timeout: float,
+ on_probe: Optional[Callable[[MailboxProbe], None]] = None,
+ on_progress: Optional[Callable[[MailboxProgress], None]] = None,
+ release: bool = True,
+ cancel: bool = False,
+ ) -> Optional[pb.Result]:
+ probe_handle: Optional[MailboxProbe] = None
+ progress_handle: Optional[MailboxProgress] = None
+ found: Optional[pb.Result] = None
+ start_time = self._time()
+ percent_done = 0.0
+ progress_sent = False
+ wait_timeout = 1.0
+ if timeout >= 0:
+ wait_timeout = min(timeout, wait_timeout)
+
+ on_progress = on_progress or self._on_progress
+ if on_progress:
+ progress_handle = MailboxProgress(_handle=self)
+
+ on_probe = on_probe or self._on_probe
+ if on_probe:
+ probe_handle = MailboxProbe()
+ if progress_handle:
+ progress_handle.add_probe_handle(probe_handle)
+
+ while True:
+ if self._keepalive and self._interface:
+ if self._interface._transport_keepalive_failed():
+ raise MailboxError("transport failed")
+
+ found, abandoned = self._slot._get_and_clear(timeout=wait_timeout)
+ if found:
+ # Always update progress to 100% when done
+ if on_progress and progress_handle and progress_sent:
+ progress_handle.set_percent_done(100)
+ on_progress(progress_handle)
+ break
+ if abandoned:
+ break
+ now = self._time()
+ if timeout >= 0:
+ if now >= start_time + timeout:
+ # todo: communicate that we timed out
+ break
+ if on_probe and probe_handle:
+ on_probe(probe_handle)
+ if on_progress and progress_handle:
+ if timeout > 0:
+ percent_done = min((now - start_time) / timeout, 1.0)
+ progress_handle.set_percent_done(percent_done)
+ on_progress(progress_handle)
+ if progress_handle._is_stopped:
+ break
+ progress_sent = True
+ if not found and cancel:
+ self._cancel()
+ if release:
+ self._release()
+ return found
+
+ def _cancel(self) -> None:
+ mailbox_slot = self.address
+ if self._interface:
+ self._interface.publish_cancel(mailbox_slot)
+
+ def _release(self) -> None:
+ self._mailbox._release_slot(self.address)
+
+ def abandon(self) -> None:
+ self._slot._notify_abandon()
+ self._release()
+
+ @property
+ def _is_failed(self) -> bool:
+ return self._failed
+
+ def _mark_failed(self) -> None:
+ self._failed = True
+
+ @property
+ def address(self) -> str:
+ return self._slot._address
+
+
+class Mailbox:
+ _slots: Dict[str, _MailboxSlot]
+ _keepalive: bool
+
+ def __init__(self) -> None:
+ self._slots = {}
+ self._keepalive = False
+
+ def enable_keepalive(self) -> None:
+ self._keepalive = True
+
+ def wait(
+ self,
+ handle: MailboxHandle,
+ *,
+ timeout: float,
+ on_progress: Optional[Callable[[MailboxProgress], None]] = None,
+ cancel: bool = False,
+ ) -> Optional[pb.Result]:
+ return handle.wait(timeout=timeout, on_progress=on_progress, cancel=cancel)
+
+ def _time(self) -> float:
+ return time.monotonic()
+
+ def wait_all(
+ self,
+ handles: List[MailboxHandle],
+ *,
+ timeout: float,
+ on_progress_all: Optional[Callable[[MailboxProgressAll], None]] = None,
+ ) -> bool:
+ progress_all_handle: Optional[MailboxProgressAll] = None
+
+ if on_progress_all:
+ progress_all_handle = MailboxProgressAll()
+
+ wait_all = _MailboxWaitAll()
+ for handle in handles:
+ wait_all._add_handle(handle)
+ if progress_all_handle and handle._on_progress:
+ progress_handle = MailboxProgress(_handle=handle)
+ if handle._on_probe:
+ probe_handle = MailboxProbe()
+ progress_handle.add_probe_handle(probe_handle)
+ progress_all_handle.add_progress_handle(progress_handle)
+
+ start_time = self._time()
+
+ while wait_all.active_handles_count > 0:
+ # Make sure underlying interfaces are still up
+ if self._keepalive:
+ for handle in wait_all.active_handles:
+ if not handle._interface:
+ continue
+ if handle._interface._transport_keepalive_failed():
+ wait_all._mark_handle_failed(handle)
+
+ # if there are no valid handles left, either break or raise exception
+ if not wait_all.active_handles_count:
+ if wait_all.failed_handles_count:
+ wait_all.clear_handles()
+ raise MailboxError("transport failed")
+ break
+
+ # wait for next event
+ wait_all._get_and_clear(timeout=1)
+
+ # TODO: we can do more careful timekeeping and not run probes and progress
+ # indications until a full second elapses in the case where we found a wait_all
+ # event. Extra probes should be ok for now.
+
+ if progress_all_handle and on_progress_all:
+ # Run all probe handles
+ for progress_handle in progress_all_handle.get_progress_handles():
+ for probe_handle in progress_handle.get_probe_handles():
+ if (
+ progress_handle._handle
+ and progress_handle._handle._on_probe
+ ):
+ progress_handle._handle._on_probe(probe_handle)
+
+ on_progress_all(progress_all_handle)
+
+ now = self._time()
+ if timeout >= 0 and now >= start_time + timeout:
+ break
+
+ return wait_all.active_handles_count == 0
+
+ def deliver(self, result: pb.Result) -> None:
+ mailbox = result.control.mailbox_slot
+ slot = self._slots.get(mailbox)
+ if not slot:
+ return
+ slot._deliver(result)
+
+ def _allocate_slot(self) -> _MailboxSlot:
+ address = _generate_address()
+ slot = _MailboxSlot(address=address)
+ self._slots[address] = slot
+ return slot
+
+ def _release_slot(self, address: str) -> None:
+ self._slots.pop(address, None)
+
+ def get_handle(self) -> MailboxHandle:
+ slot = self._allocate_slot()
+ handle = MailboxHandle(mailbox=self, slot=slot)
+ return handle
+
+ def _deliver_record(
+ self, record: pb.Record, interface: "InterfaceShared"
+ ) -> MailboxHandle:
+ handle = self.get_handle()
+ handle._interface = interface
+ handle._keepalive = self._keepalive
+ record.control.mailbox_slot = handle.address
+ try:
+ interface._publish(record)
+ except Exception:
+ interface._transport_mark_failed()
+ raise
+ interface._transport_mark_success()
+ return handle
diff --git a/wandb/sdk/lib/module.py b/wandb/sdk/lib/module.py
new file mode 100644
index 0000000000000000000000000000000000000000..3958b82775444bd35fc8bbf60892bd915ee1510b
--- /dev/null
+++ b/wandb/sdk/lib/module.py
@@ -0,0 +1,69 @@
+#
+import wandb
+
+from . import preinit
+
+
+def set_global(
+ run=None,
+ config=None,
+ log=None,
+ summary=None,
+ save=None,
+ use_artifact=None,
+ log_artifact=None,
+ define_metric=None,
+ alert=None,
+ plot_table=None,
+ mark_preempting=None,
+ log_model=None,
+ use_model=None,
+ link_model=None,
+):
+ if run:
+ wandb.run = run
+ if config is not None:
+ wandb.config = config
+ if log:
+ wandb.log = log
+ if summary is not None:
+ wandb.summary = summary
+ if save:
+ wandb.save = save
+ if use_artifact:
+ wandb.use_artifact = use_artifact
+ if log_artifact:
+ wandb.log_artifact = log_artifact
+ if define_metric:
+ wandb.define_metric = define_metric
+ if plot_table:
+ wandb.plot_table = plot_table
+ if alert:
+ wandb.alert = alert
+ if mark_preempting:
+ wandb.mark_preempting = mark_preempting
+ if log_model:
+ wandb.log_model = log_model
+ if use_model:
+ wandb.use_model = use_model
+ if link_model:
+ wandb.link_model = link_model
+
+
+def unset_globals():
+ wandb.run = None
+ wandb.config = preinit.PreInitObject("wandb.config")
+ wandb.summary = preinit.PreInitObject("wandb.summary")
+ wandb.log = preinit.PreInitCallable("wandb.log", wandb.wandb_sdk.wandb_run.Run.log)
+ wandb.save = preinit.PreInitCallable(
+ "wandb.save", wandb.wandb_sdk.wandb_run.Run.save
+ )
+ wandb.use_artifact = preinit.PreInitCallable(
+ "wandb.use_artifact", wandb.wandb_sdk.wandb_run.Run.use_artifact
+ )
+ wandb.log_artifact = preinit.PreInitCallable(
+ "wandb.log_artifact", wandb.wandb_sdk.wandb_run.Run.log_artifact
+ )
+ wandb.define_metric = preinit.PreInitCallable(
+ "wandb.define_metric", wandb.wandb_sdk.wandb_run.Run.define_metric
+ )
diff --git a/wandb/sdk/lib/paths.py b/wandb/sdk/lib/paths.py
new file mode 100644
index 0000000000000000000000000000000000000000..1fae7110ec5ffe83d46c18c4263b0745a44264c9
--- /dev/null
+++ b/wandb/sdk/lib/paths.py
@@ -0,0 +1,106 @@
+import os
+import platform
+from functools import wraps
+from pathlib import PurePath, PurePosixPath
+from typing import Any, NewType, Union
+
+# Path _inputs_ should generally accept any kind of path. This is named the same and
+# modeled after the hint defined in the Python standard library's `typeshed`:
+# https://github.com/python/typeshed/blob/0b1cd5989669544866213807afa833a88f649ee7/stdlib/_typeshed/__init__.pyi#L56-L65
+StrPath = Union[str, "os.PathLike[str]"]
+
+# A native path to a file on a local filesystem.
+FilePathStr = NewType("FilePathStr", str)
+
+URIStr = NewType("URIStr", str)
+
+
+class LogicalPath(str):
+ """A string that represents a path relative to an artifact or run.
+
+ The format of the string is always as a POSIX path, e.g. "foo/bar.txt".
+
+ A neat trick is that you can use this class as if it were a PurePosixPath. E.g.:
+ ```
+ >>> path = LogicalPath("foo/bar.txt")
+ >>> path.parts
+ ('foo', 'bar.txt')
+ >>> path.parent / "baz.txt"
+ 'foo/baz.txt'
+ >>> type(path.relative_to("foo"))
+ LogicalPath
+ ```
+ """
+
+ # It should probably always be a relative path, but that would be a behavior change.
+ #
+ # These strings used to be the output of `to_forward_slash_path`, which only works
+ # with strings and whose behavior is pretty simple:
+ # ```
+ # if platform.system() == "Windows":
+ # path = path.replace("\\", "/")
+ # ```
+ #
+ # This results in some weird things, such as backslashes being allowed from
+ # non-Windows platforms (which would probably break if such an artifact was used
+ # from Windows) and anchors or absolute paths being allowed. E.g., the Windows path
+ # "C:\foo\bar.txt" becomes "C:/foo/bar.txt", which then would mount as
+ # "./artifacts/artifact_name:v0/C:/foo/bar.txt" on MacOS and as
+ # "./artifacts/artifact_name-v0/C-/foo/bar.txt" on Windows.
+ #
+ # This implementation preserves behavior for strings but attempts to sanitize other
+ # formerly unsupported inputs more aggressively. It uses the `.as_posix()` form of
+ # pathlib objects rather than the `str()` form to reduce how often identical inputs
+ # will result in different outputs on different platforms; however, it doesn't alter
+ # absolute paths or check for prohibited characters etc.
+
+ def __new__(cls, path: StrPath) -> "LogicalPath":
+ if isinstance(path, LogicalPath):
+ return super().__new__(cls, path)
+ if hasattr(path, "as_posix"):
+ path = PurePosixPath(path.as_posix())
+ return super().__new__(cls, str(path))
+ if hasattr(path, "__fspath__"):
+ path = path.__fspath__() # Can be str or bytes.
+ if isinstance(path, bytes):
+ path = os.fsdecode(path)
+ # For historical reasons we have to convert backslashes to forward slashes, but
+ # only on Windows, and need to do it before any pathlib operations.
+ if platform.system() == "Windows":
+ path = path.replace("\\", "/")
+ # This weird contortion and the one above are because in some unusual cases
+ # PurePosixPath(path.as_posix()).as_posix() != path.as_posix().
+ path = PurePath(path).as_posix()
+ return super().__new__(cls, str(PurePosixPath(path)))
+
+ def to_path(self) -> PurePosixPath:
+ """Convert this path to a PurePosixPath."""
+ return PurePosixPath(self)
+
+ def __getattr__(self, attr: str) -> Any:
+ """Act like a subclass of PurePosixPath for all methods not defined on str."""
+ try:
+ result = getattr(self.to_path(), attr)
+ except AttributeError as e:
+ raise AttributeError(f"LogicalPath has no attribute {attr!r}") from e
+
+ if isinstance(result, PurePosixPath):
+ return LogicalPath(result)
+
+ # If the result is a callable (a method), wrap it so that it has the same
+ # behavior: if the call result returns a PurePosixPath, return a LogicalPath.
+ if callable(result):
+
+ @wraps(result)
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
+ inner_result = result(*args, **kwargs)
+ if isinstance(inner_result, PurePosixPath):
+ return LogicalPath(inner_result)
+ return inner_result
+
+ return wrapper
+ return result
+
+ def __truediv__(self, other: StrPath) -> "LogicalPath":
+ """Act like a PurePosixPath for the / operator, but return a LogicalPath."""
+ return LogicalPath(self.to_path() / LogicalPath(other))
diff --git a/wandb/sdk/lib/preinit.py b/wandb/sdk/lib/preinit.py
new file mode 100644
index 0000000000000000000000000000000000000000..624528198b1cab580f95d29eb91b0531378041e8
--- /dev/null
+++ b/wandb/sdk/lib/preinit.py
@@ -0,0 +1,42 @@
+from typing import Any, Callable, Optional
+
+import wandb
+
+
+class PreInitObject:
+ def __init__(self, name: str, destination: Optional[Any] = None) -> None:
+ self._name = name
+
+ if destination is not None:
+ self.__doc__ = destination.__doc__
+
+ def __getitem__(self, key: str) -> None:
+ raise wandb.Error(f"You must call wandb.init() before {self._name}[{key!r}]")
+
+ def __setitem__(self, key: str, value: Any) -> Any:
+ raise wandb.Error(f"You must call wandb.init() before {self._name}[{key!r}]")
+
+ def __setattr__(self, key: str, value: Any) -> Any:
+ if not key.startswith("_"):
+ raise wandb.Error(f"You must call wandb.init() before {self._name}.{key}")
+ else:
+ return object.__setattr__(self, key, value)
+
+ def __getattr__(self, key: str) -> Any:
+ if not key.startswith("_"):
+ raise wandb.Error(f"You must call wandb.init() before {self._name}.{key}")
+ else:
+ raise AttributeError
+
+
+def PreInitCallable( # noqa: N802
+ name: str, destination: Optional[Any] = None
+) -> Callable:
+ def preinit_wrapper(*args: Any, **kwargs: Any) -> Any:
+ raise wandb.Error(f"You must call wandb.init() before {name}()")
+
+ preinit_wrapper.__name__ = str(name)
+ if destination:
+ preinit_wrapper.__wrapped__ = destination # type: ignore
+ preinit_wrapper.__doc__ = destination.__doc__
+ return preinit_wrapper
diff --git a/wandb/sdk/lib/printer.py b/wandb/sdk/lib/printer.py
new file mode 100644
index 0000000000000000000000000000000000000000..0aeddc126d2a1ff232036aab275e38aa267cdc25
--- /dev/null
+++ b/wandb/sdk/lib/printer.py
@@ -0,0 +1,313 @@
+# Note: this is a helper printer class, this file might go away once we switch to rich console printing
+
+import itertools
+import platform
+import sys
+from abc import abstractmethod
+from typing import Callable, List, Optional, Tuple, Union
+
+import click
+
+import wandb
+
+from . import ipython, sparkline
+
+# Follow the same logic as the python logging module
+CRITICAL = 50
+FATAL = CRITICAL
+ERROR = 40
+WARNING = 30
+WARN = WARNING
+INFO = 20
+DEBUG = 10
+NOTSET = 0
+
+_level_to_name = {
+ CRITICAL: "CRITICAL",
+ ERROR: "ERROR",
+ WARNING: "WARNING",
+ INFO: "INFO",
+ DEBUG: "DEBUG",
+ NOTSET: "NOTSET",
+}
+
+_name_to_level = {
+ "CRITICAL": CRITICAL,
+ "FATAL": FATAL,
+ "ERROR": ERROR,
+ "WARN": WARNING,
+ "WARNING": WARNING,
+ "INFO": INFO,
+ "DEBUG": DEBUG,
+ "NOTSET": NOTSET,
+}
+
+
+class _Printer:
+ def sparklines(self, series: List[Union[int, float]]) -> Optional[str]:
+ # Only print sparklines if the terminal is utf-8
+ if wandb.util.is_unicode_safe(sys.stdout):
+ return sparkline.sparkify(series)
+ return None
+
+ def abort(
+ self,
+ ) -> str:
+ return "Control-C" if platform.system() != "Windows" else "Ctrl-C"
+
+ def display(
+ self,
+ text: Union[str, List[str], Tuple[str]],
+ *,
+ level: Optional[Union[str, int]] = None,
+ off: Optional[bool] = None,
+ default_text: Optional[Union[str, List[str], Tuple[str]]] = None,
+ ) -> None:
+ if off:
+ return
+ self._display(text, level=level, default_text=default_text)
+
+ @abstractmethod
+ def _display(
+ self,
+ text: Union[str, List[str], Tuple[str]],
+ *,
+ level: Optional[Union[str, int]] = None,
+ default_text: Optional[Union[str, List[str], Tuple[str]]] = None,
+ ) -> None:
+ raise NotImplementedError
+
+ @staticmethod
+ def _sanitize_level(name_or_level: Optional[Union[str, int]]) -> int:
+ if isinstance(name_or_level, str):
+ try:
+ return _name_to_level[name_or_level.upper()]
+ except KeyError:
+ raise ValueError(
+ f"Unknown level name: {name_or_level}, supported levels: {_name_to_level.keys()}"
+ )
+
+ if isinstance(name_or_level, int):
+ return name_or_level
+
+ if name_or_level is None:
+ return INFO
+
+ raise ValueError(f"Unknown status level {name_or_level}")
+
+ @abstractmethod
+ def code(self, text: str) -> str:
+ raise NotImplementedError
+
+ @abstractmethod
+ def name(self, text: str) -> str:
+ raise NotImplementedError
+
+ @abstractmethod
+ def link(self, link: str, text: Optional[str] = None) -> str:
+ raise NotImplementedError
+
+ @abstractmethod
+ def emoji(self, name: str) -> str:
+ raise NotImplementedError
+
+ @abstractmethod
+ def status(self, text: str, failure: Optional[bool] = None) -> str:
+ raise NotImplementedError
+
+ @abstractmethod
+ def files(self, text: str) -> str:
+ raise NotImplementedError
+
+ @abstractmethod
+ def grid(self, rows: List[List[str]], title: Optional[str] = None) -> str:
+ raise NotImplementedError
+
+ @abstractmethod
+ def panel(self, columns: List[str]) -> str:
+ raise NotImplementedError
+
+
+class PrinterTerm(_Printer):
+ def __init__(self) -> None:
+ super().__init__()
+ self._html = False
+ self._progress = itertools.cycle(["-", "\\", "|", "/"])
+
+ def _display(
+ self,
+ text: Union[str, List[str], Tuple[str]],
+ *,
+ level: Optional[Union[str, int]] = None,
+ default_text: Optional[Union[str, List[str], Tuple[str]]] = None,
+ ) -> None:
+ text = "\n".join(text) if isinstance(text, (list, tuple)) else text
+ if default_text is not None:
+ default_text = (
+ "\n".join(default_text)
+ if isinstance(default_text, (list, tuple))
+ else default_text
+ )
+ text = text or default_text
+ self._display_fn_mapping(level)(text)
+
+ @staticmethod
+ def _display_fn_mapping(level: Optional[Union[str, int]]) -> Callable[[str], None]:
+ level = _Printer._sanitize_level(level)
+
+ if level >= CRITICAL:
+ return wandb.termerror
+ elif ERROR <= level < CRITICAL:
+ return wandb.termerror
+ elif WARNING <= level < ERROR:
+ return wandb.termwarn
+ elif INFO <= level < WARNING:
+ return wandb.termlog
+ elif DEBUG <= level < INFO:
+ return wandb.termlog
+ else:
+ return wandb.termlog
+
+ def progress_update(self, text: str, percent_done: Optional[float] = None) -> None:
+ wandb.termlog(f"{next(self._progress)} {text}", newline=False)
+
+ def progress_close(self, text: Optional[str] = None) -> None:
+ text = text or " " * 79
+ wandb.termlog(text)
+
+ def code(self, text: str) -> str:
+ ret: str = click.style(text, bold=True)
+ return ret
+
+ def name(self, text: str) -> str:
+ ret: str = click.style(text, fg="yellow")
+ return ret
+
+ def link(self, link: str, text: Optional[str] = None) -> str:
+ ret: str = click.style(link, fg="blue", underline=True)
+ # ret = f"\x1b[m{text or link}\x1b[0m"
+ # ret = f"\x1b]8;;{link}\x1b\\{ret}\x1b]8;;\x1b\\"
+ return ret
+
+ def emoji(self, name: str) -> str:
+ emojis = dict()
+ if platform.system() != "Windows" and wandb.util.is_unicode_safe(sys.stdout):
+ emojis = dict(
+ star="âï¸",
+ broom="🧹",
+ rocket="🚀",
+ gorilla="ðŸ¦",
+ turtle="ðŸ¢",
+ lightning="ï¸âš¡",
+ )
+
+ return emojis.get(name, "")
+
+ def status(self, text: str, failure: Optional[bool] = None) -> str:
+ color = "red" if failure else "green"
+ ret: str = click.style(text, fg=color)
+ return ret
+
+ def files(self, text: str) -> str:
+ ret: str = click.style(text, fg="magenta", bold=True)
+ return ret
+
+ def grid(self, rows: List[List[str]], title: Optional[str] = None) -> str:
+ max_len = max(len(row[0]) for row in rows)
+ format_row = " ".join(["{:>{max_len}}", "{}" * (len(rows[0]) - 1)])
+ grid = "\n".join([format_row.format(*row, max_len=max_len) for row in rows])
+ if title:
+ return f"{title}\n{grid}\n"
+ return f"{grid}\n"
+
+ def panel(self, columns: List[str]) -> str:
+ return "\n" + "\n".join(columns)
+
+
+class PrinterJupyter(_Printer):
+ def __init__(self) -> None:
+ super().__init__()
+ self._html = True
+ self._progress = ipython.jupyter_progress_bar()
+
+ def _display(
+ self,
+ text: Union[str, List[str], Tuple[str]],
+ *,
+ level: Optional[Union[str, int]] = None,
+ default_text: Optional[Union[str, List[str], Tuple[str]]] = None,
+ ) -> None:
+ text = "<br/>".join(text) if isinstance(text, (list, tuple)) else text
+ if default_text is not None:
+ default_text = (
+ "<br/>".join(default_text)
+ if isinstance(default_text, (list, tuple))
+ else default_text
+ )
+ text = text or default_text
+ self._display_fn_mapping(level)(text)
+
+ @staticmethod
+ def _display_fn_mapping(level: Optional[Union[str, int]]) -> Callable[[str], None]:
+ level = _Printer._sanitize_level(level)
+
+ if level >= CRITICAL:
+ return ipython.display_html
+ elif ERROR <= level < CRITICAL:
+ return ipython.display_html
+ elif WARNING <= level < ERROR:
+ return ipython.display_html
+ elif INFO <= level < WARNING:
+ return ipython.display_html
+ elif DEBUG <= level < INFO:
+ return ipython.display_html
+ else:
+ return ipython.display_html
+
+ def code(self, text: str) -> str:
+ return f"<code>{text}<code>"
+
+ def name(self, text: str) -> str:
+ return f'<strong style="color:#cdcd00">{text}</strong>'
+
+ def link(self, link: str, text: Optional[str] = None) -> str:
+ return f'<a href={link!r} target="_blank">{text or link}</a>'
+
+ def emoji(self, name: str) -> str:
+ return ""
+
+ def status(self, text: str, failure: Optional[bool] = None) -> str:
+ color = "red" if failure else "green"
+ return f'<strong style="color:{color}">{text}</strong>'
+
+ def files(self, text: str) -> str:
+ return f"<code>{text}</code>"
+
+ def progress_update(self, text: str, percent_done: float) -> None:
+ if self._progress:
+ self._progress.update(percent_done, text)
+
+ def progress_close(self, _: Optional[str] = None) -> None:
+ if self._progress:
+ self._progress.close()
+
+ def grid(self, rows: List[List[str]], title: Optional[str] = None) -> str:
+ format_row = "".join(["<tr>", "<td>{}</td>" * len(rows[0]), "</tr>"])
+ grid = "".join([format_row.format(*row) for row in rows])
+ grid = f'<table class="wandb">{grid}</table>'
+ if title:
+ return f"<h3>{title}</h3><br/>{grid}<br/>"
+ return f"{grid}<br/>"
+
+ def panel(self, columns: List[str]) -> str:
+ row = "".join([f'<div class="wandb-col">{col}</div>' for col in columns])
+ return f'{ipython.TABLE_STYLES}<div class="wandb-row">{row}</div>'
+
+
+Printer = Union[PrinterTerm, PrinterJupyter]
+
+
+def get_printer(_jupyter: bool) -> Printer:
+ if _jupyter:
+ return PrinterJupyter()
+ return PrinterTerm()
diff --git a/wandb/sdk/lib/proto_util.py b/wandb/sdk/lib/proto_util.py
new file mode 100644
index 0000000000000000000000000000000000000000..7231ce25db0bc6fe2317d16ab6779a2a8500fe85
--- /dev/null
+++ b/wandb/sdk/lib/proto_util.py
@@ -0,0 +1,69 @@
+#
+import json
+from typing import TYPE_CHECKING, Any, Dict, Union
+
+from wandb.proto import wandb_internal_pb2 as pb
+
+if TYPE_CHECKING: # pragma: no cover
+ from google.protobuf.internal.containers import RepeatedCompositeFieldContainer
+ from google.protobuf.message import Message
+
+ from wandb.proto import wandb_telemetry_pb2 as tpb
+
+
+def dict_from_proto_list(obj_list: "RepeatedCompositeFieldContainer") -> Dict[str, Any]:
+ return {item.key: json.loads(item.value_json) for item in obj_list}
+
+
+def _result_from_record(record: "pb.Record") -> "pb.Result":
+ result = pb.Result(uuid=record.uuid, control=record.control)
+ return result
+
+
+def _assign_record_num(record: "pb.Record", record_num: int) -> None:
+ record.num = record_num
+
+
+def _assign_end_offset(record: "pb.Record", end_offset: int) -> None:
+ record.control.end_offset = end_offset
+
+
+def proto_encode_to_dict(
+ pb_obj: Union["tpb.TelemetryRecord", "pb.MetricRecord"]
+) -> Dict[int, Any]:
+ data: Dict[int, Any] = dict()
+ fields = pb_obj.ListFields()
+ for desc, value in fields:
+ if desc.name.startswith("_"):
+ continue
+ if desc.type == desc.TYPE_STRING:
+ data[desc.number] = value
+ elif desc.type == desc.TYPE_INT32:
+ data[desc.number] = value
+ elif desc.type == desc.TYPE_ENUM:
+ data[desc.number] = value
+ elif desc.type == desc.TYPE_MESSAGE:
+ nested = value.ListFields()
+ bool_msg = all(d.type == d.TYPE_BOOL for d, _ in nested)
+ if bool_msg:
+ items = [d.number for d, v in nested if v]
+ if items:
+ data[desc.number] = items
+ else:
+ # TODO: for now this code only handles sub-messages with strings
+ md = {}
+ for d, v in nested:
+ if not v or d.type != d.TYPE_STRING:
+ continue
+ md[d.number] = v
+ data[desc.number] = md
+ return data
+
+
+def message_to_dict(
+ message: "Message",
+) -> Dict[str, Any]:
+ """Convert a protobuf message into a dictionary."""
+ from google.protobuf.json_format import MessageToDict
+
+ return MessageToDict(message, preserving_proto_field_name=True)
diff --git a/wandb/sdk/lib/redirect.py b/wandb/sdk/lib/redirect.py
new file mode 100644
index 0000000000000000000000000000000000000000..0adea0fd52bbeeed64b167c9067d58d29a287793
--- /dev/null
+++ b/wandb/sdk/lib/redirect.py
@@ -0,0 +1,840 @@
+try:
+ import fcntl
+ import pty
+ import termios
+ import tty
+except ImportError: # windows
+ pty = tty = termios = fcntl = None # type: ignore
+
+import itertools
+import logging
+import os
+import queue
+import re
+import signal
+import struct
+import sys
+import threading
+import time
+from collections import defaultdict
+
+import wandb
+
+
+class _Numpy: # fallback in case numpy is not available
+ def where(self, x):
+ return ([i for i in range(len(x)) if x[i]],)
+
+ def diff(self, x):
+ return [x[i + 1] - x[i] for i in range(len(x) - 1)]
+
+ def arange(self, x):
+ class Arr(list):
+ def __getitem__(self, s):
+ if isinstance(s, slice):
+ self._start = s.start
+ return self
+ return super().__getitem__(s)
+
+ def __getslice__(self, i, j):
+ self._start = i
+ return self
+
+ def __iadd__(self, i): # type: ignore
+ for j in range(self._start, len(self)):
+ self[j] += i
+
+ return Arr(range(x))
+
+
+try:
+ import numpy as np # type: ignore
+except ImportError:
+ np = _Numpy() # type: ignore
+
+
+logger = logging.getLogger("wandb")
+
+_redirects = {"stdout": None, "stderr": None}
+
+
+ANSI_CSI_RE = re.compile("\001?\033\\[((?:\\d|;)*)([a-zA-Z])\002?")
+ANSI_OSC_RE = re.compile("\001?\033\\]([^\a]*)(\a)\002?")
+
+_LAST_WRITE_TOKEN = b"L@stWr!t3T0k3n"
+
+SEP_RE = re.compile(
+ "\r|\n|"
+ # Unprintable ascii characters:
+ + "|".join([chr(i) for i in range(2**8) if repr(chr(i)).startswith("'\\x")])
+)
+
+ANSI_FG = list(map(str, itertools.chain(range(30, 40), range(90, 98))))
+ANSI_BG = list(map(str, itertools.chain(range(40, 50), range(100, 108))))
+
+ANSI_FG_DEFAULT = "39"
+ANSI_BG_DEFAULT = "49"
+
+ANSI_RESET = "0"
+
+ANSI_STYLES = {
+ "1": "bold",
+ "2": "/bold",
+ "3": "italics",
+ "4": "underscore",
+ "5": "blink",
+ "7": "reverse",
+ "9": "strikethrough",
+ "22": "/bold",
+ "23": "/italics",
+ "24": "/underscore",
+ "25": "/blink",
+ "27": "/reverse",
+ "29": "/strikethrough",
+}
+
+ANSI_STYLES_REV = {v: k for k, v in ANSI_STYLES.items()}
+
+
+CSI = "\033["
+
+
+def _get_char(code):
+ return "\033[" + str(code) + "m"
+
+
+class Char:
+ """Class encapsulating a single character, its foreground, background and style attributes."""
+
+ __slots__ = (
+ "data",
+ "fg",
+ "bg",
+ "bold",
+ "italics",
+ "underscore",
+ "blink",
+ "strikethrough",
+ "reverse",
+ )
+
+ def __init__(
+ self,
+ data=" ",
+ fg=ANSI_FG_DEFAULT,
+ bg=ANSI_BG_DEFAULT,
+ bold=False,
+ italics=False,
+ underscore=False,
+ blink=False,
+ strikethrough=False,
+ reverse=False,
+ ):
+ self.data = data
+ self.fg = fg
+ self.bg = bg
+ self.bold = bold
+ self.italics = italics
+ self.underscore = underscore
+ self.blink = blink
+ self.strikethrough = strikethrough
+ self.reverse = reverse
+
+ def reset(self):
+ # Reset everything other than data to defaults
+ default = self.__class__()
+ for k in self.__slots__[1:]:
+ self[k] = default[k]
+
+ def __getitem__(self, k):
+ return getattr(self, k)
+
+ def __setitem__(self, k, v):
+ setattr(self, k, v)
+
+ def copy(self, **kwargs):
+ attrs = {}
+ for k in self.__slots__:
+ if k in kwargs:
+ attrs[k] = kwargs[k]
+ else:
+ attrs[k] = self[k]
+ return self.__class__(**attrs)
+
+ def __eq__(self, other):
+ for k in self.__slots__:
+ if self[k] != other[k]:
+ return False
+ return True
+
+
+_defchar = Char()
+
+
+class Cursor:
+ """A 2D cursor.
+
+ Attributes:
+ x: x-coordinate.
+ y: y-coordinate.
+ char: the character to inherit colors and styles from.
+ """
+
+ __slots__ = ("x", "y", "char")
+
+ def __init__(self, x=0, y=0, char=None):
+ if char is None:
+ char = Char()
+ self.x = x
+ self.y = y
+ self.char = char
+
+
+class TerminalEmulator:
+ """An FSM emulating a terminal.
+
+ Characters are stored in a 2D matrix (buffer) indexed by the cursor.
+ """
+
+ _MAX_LINES = 100
+
+ def __init__(self):
+ self.buffer = defaultdict(lambda: defaultdict(lambda: _defchar))
+ self.cursor = Cursor()
+ self._num_lines = None # Cache
+
+ # For diffing:
+ self._prev_num_lines = None
+ self._prev_last_line = None
+
+ def cursor_up(self, n=1):
+ n = min(n, self.cursor.y)
+ self.cursor.y -= n
+
+ def cursor_down(self, n=1):
+ self.cursor.y += n
+
+ def cursor_left(self, n=1):
+ n = min(n, self.cursor.x)
+ self.cursor.x -= n
+
+ def cursor_right(self, n=1):
+ self.cursor.x += n
+
+ def carriage_return(self):
+ self.cursor.x = 0
+
+ def cursor_postion(self, line, column):
+ self.cursor.x = min(column, 1) - 1
+ self.cursor.y = min(line, 1) - 1
+
+ def cursor_column(self, column):
+ self.cursor.x = min(column, 1) - 1
+
+ def cursor_line(self, line):
+ self.cursor.y = min(line, 1) - 1
+
+ def linefeed(self):
+ self.cursor_down()
+ self.carriage_return()
+
+ def _get_line_len(self, n):
+ if n not in self.buffer:
+ return 0
+ line = self.buffer[n]
+ if not line:
+ return 0
+ n = max(line.keys())
+ for i in range(n, -1, -1):
+ if line[i] != _defchar:
+ return i + 1
+ return 0
+
+ @property
+ def num_lines(self):
+ if self._num_lines is not None:
+ return self._num_lines
+ ret = 0
+ if self.buffer:
+ n = max(self.buffer.keys())
+ for i in range(n, -1, -1):
+ if self._get_line_len(i):
+ ret = i + 1
+ break
+ self._num_lines = ret
+ return ret
+
+ def display(self):
+ return [
+ [self.buffer[i][j].data for j in range(self._get_line_len(i))]
+ for i in range(self.num_lines)
+ ]
+
+ def erase_screen(self, mode=0):
+ if mode == 0:
+ for i in range(self.cursor.y + 1, self.num_lines):
+ if i in self.buffer:
+ del self.buffer[i]
+ self.erase_line(mode)
+ if mode == 1:
+ for i in range(self.cursor.y):
+ if i in self.buffer:
+ del self.buffer[i]
+ self.erase_line(mode)
+ elif mode == 2 or mode == 3:
+ self.buffer.clear()
+
+ def erase_line(self, mode=0):
+ curr_line = self.buffer[self.cursor.y]
+ if mode == 0:
+ for i in range(self.cursor.x, self._get_line_len(self.cursor.y)):
+ if i in curr_line:
+ del curr_line[i]
+ elif mode == 1:
+ for i in range(self.cursor.x + 1):
+ if i in curr_line:
+ del curr_line[i]
+ else:
+ curr_line.clear()
+
+ def insert_lines(self, n=1):
+ for i in range(self.num_lines - 1, self.cursor.y, -1):
+ self.buffer[i + n] = self.buffer[i]
+ for i in range(self.cursor.y + 1, self.cursor.y + 1 + n):
+ if i in self.buffer:
+ del self.buffer[i]
+
+ def _write_plain_text(self, plain_text):
+ self.buffer[self.cursor.y].update(
+ [
+ (self.cursor.x + i, self.cursor.char.copy(data=c))
+ for i, c in enumerate(plain_text)
+ ]
+ )
+ self.cursor.x += len(plain_text)
+
+ def _write_text(self, text):
+ prev_end = 0
+ for match in SEP_RE.finditer(text):
+ start, end = match.span()
+ self._write_plain_text(text[prev_end:start])
+ prev_end = end
+ c = match.group()
+ if c == "\n":
+ self.linefeed()
+ elif c == "\r":
+ self.carriage_return()
+ elif c == "\b":
+ self.cursor_left()
+ else:
+ continue
+ self._write_plain_text(text[prev_end:])
+
+ def _remove_osc(self, text):
+ return re.sub(ANSI_OSC_RE, "", text)
+
+ def write(self, data):
+ self._num_lines = None # invalidate cache
+ data = self._remove_osc(data)
+ prev_end = 0
+ for match in ANSI_CSI_RE.finditer(data):
+ start, end = match.span()
+ text = data[prev_end:start]
+ csi = data[start:end]
+ prev_end = end
+ self._write_text(text)
+ self._handle_csi(csi, *match.groups())
+ self._write_text(data[prev_end:])
+
+ def _handle_csi(self, csi, params, command):
+ try:
+ if command == "m":
+ p = params.split(";")[0]
+ if not p:
+ p = "0"
+ if p in ANSI_FG:
+ self.cursor.char.fg = p
+ elif p in ANSI_BG:
+ self.cursor.char.bg = p
+ elif p == ANSI_RESET:
+ self.cursor.char.reset()
+ elif p in ANSI_STYLES:
+ style = ANSI_STYLES[p]
+ off = style.startswith("/")
+ if off:
+ style = style[1:]
+ self.cursor.char[style] = not off
+ else:
+ abcd = {
+ "A": "cursor_up",
+ "B": "cursor_down",
+ "C": "cursor_right",
+ "D": "cursor_left",
+ }
+ cursor_fn = abcd.get(command)
+ if cursor_fn:
+ getattr(self, cursor_fn)(int(params) if params else 1)
+ elif command == "J":
+ p = params.split(";")[0]
+ p = int(p) if p else 0
+ self.erase_screen(p)
+ elif command == "K":
+ p = params.split(";")[0]
+ p = int(p) if p else 0
+ self.erase_line(p)
+ elif command == "L":
+ p = int(params) if params else 1
+ self.insert_lines(p)
+ elif command in "Hf":
+ p = params.split(";")
+ if len(p) == 2:
+ p = (int(p[0]), int(p[1]))
+ elif len(p) == 1:
+ p = (int(p[0]), 1)
+ else:
+ p = (1, 1)
+ self.cursor_postion(*p)
+ except Exception:
+ pass
+
+ def _get_line(self, n):
+ line = self.buffer[n]
+ line_len = self._get_line_len(n)
+ # We have to loop through each character in the line and check if foreground, background and
+ # other attributes (italics, bold, underline, etc) of the ith character are different from those of the
+ # (i-1)th character. If different, the appropriate ascii character for switching the color/attribute
+ # should be appended to the output string before appending the actual character. This loop and subsequent
+ # checks can be expensive, especially because 99% of terminal output use default colors and formatting. Even
+ # in outputs that do contain colors and styles, its unlikely that they will change on a per character basis.
+
+ # So instead we create a character list without any ascii codes (`out`), and a list of all the foregrounds
+ # in the line (`fgs`) on which we call np.diff() and np.where() to find the indices where the foreground change,
+ # and insert the ascii characters in the output list (`out`) on those indices. All of this is the done ony if
+ # there are more than 1 foreground color in the line in the first place (`if len(set(fgs)) > 1 else None`).
+ # Same logic is repeated for background colors and other attributes.
+
+ out = [line[i].data for i in range(line_len)]
+
+ # for dynamic insert using original indices
+ idxs = np.arange(line_len)
+ insert = lambda i, c: (out.insert(idxs[i], c), idxs[i:].__iadd__(1)) # noqa
+
+ fgs = [int(_defchar.fg)] + [int(line[i].fg) for i in range(line_len)]
+ [
+ insert(i, _get_char(line[int(i)].fg)) for i in np.where(np.diff(fgs))[0]
+ ] if len(set(fgs)) > 1 else None
+ bgs = [int(_defchar.bg)] + [int(line[i].bg) for i in range(line_len)]
+ [
+ insert(i, _get_char(line[int(i)].bg)) for i in np.where(np.diff(bgs))[0]
+ ] if len(set(bgs)) > 1 else None
+ attrs = {
+ k: [False] + [line[i][k] for i in range(line_len)]
+ for k in Char.__slots__[3:]
+ }
+ [
+ [
+ insert(i, _get_char(ANSI_STYLES_REV[k if line[int(i)][k] else "/" + k]))
+ for i in np.where(np.diff(v))[0]
+ ]
+ for k, v in attrs.items()
+ if any(v)
+ ]
+ return "".join(out)
+
+ def read(self):
+ num_lines = self.num_lines
+ if self._prev_num_lines is None:
+ ret = os.linesep.join(map(self._get_line, range(num_lines)))
+ if ret:
+ ret += os.linesep
+ else:
+ return ret
+ else:
+ curr_line = self._get_line(self._prev_num_lines - 1)
+ if curr_line == self._prev_last_line:
+ if num_lines == self._prev_num_lines:
+ return ""
+ ret = (
+ os.linesep.join(
+ map(self._get_line, range(self._prev_num_lines, num_lines))
+ )
+ + os.linesep
+ )
+ else:
+ ret = (
+ "\r"
+ + os.linesep.join(
+ map(self._get_line, range(self._prev_num_lines - 1, num_lines))
+ )
+ + os.linesep
+ )
+ if num_lines > self._MAX_LINES:
+ shift = num_lines - self._MAX_LINES
+ for i in range(shift, num_lines):
+ self.buffer[i - shift] = self.buffer[i]
+ for i in range(self._MAX_LINES, max(self.buffer.keys())):
+ if i in self.buffer:
+ del self.buffer[i]
+ self.cursor.y -= min(self.cursor.y, shift)
+ self._num_lines = num_lines = self._MAX_LINES
+ self._prev_num_lines = num_lines
+ self._prev_last_line = self._get_line(num_lines - 1)
+ return ret
+
+
+_MIN_CALLBACK_INTERVAL = 2 # seconds
+
+
+class RedirectBase:
+ def __init__(self, src, cbs=()):
+ """# Arguments.
+
+ `src`: Source stream to be redirected. "stdout" or "stderr".
+ `cbs`: tuple/list of callbacks. Each callback should take exactly 1 argument (bytes).
+
+ """
+ assert hasattr(sys, src)
+ self.src = src
+ self.cbs = cbs
+
+ @property
+ def src_stream(self):
+ return getattr(sys, "__%s__" % self.src)
+
+ @property
+ def src_fd(self):
+ return self.src_stream.fileno()
+
+ @property
+ def src_wrapped_stream(self):
+ return getattr(sys, self.src)
+
+ def save(self):
+ pass
+
+ def install(self):
+ curr_redirect = _redirects.get(self.src)
+ if curr_redirect and curr_redirect != self:
+ curr_redirect.uninstall()
+ _redirects[self.src] = self
+
+ def uninstall(self):
+ if _redirects[self.src] != self:
+ return
+ _redirects[self.src] = None
+
+
+class StreamWrapper(RedirectBase):
+ """Patches the write method of current sys.stdout/sys.stderr."""
+
+ def __init__(self, src, cbs=()):
+ super().__init__(src=src, cbs=cbs)
+ self._installed = False
+ self._emulator = TerminalEmulator()
+
+ def _emulator_write(self):
+ while True:
+ if self._queue.empty():
+ if self._stopped.is_set():
+ return
+ time.sleep(0.5)
+ continue
+ data = []
+ while not self._queue.empty():
+ data.append(self._queue.get())
+ if self._stopped.is_set() and sum(map(len, data)) > 100000:
+ wandb.termlog("Terminal output too large. Logging without processing.")
+ self.flush()
+ [self.flush(line.encode("utf-8")) for line in data]
+ return
+ try:
+ self._emulator.write("".join(data))
+ except Exception:
+ pass
+
+ def _callback(self):
+ while not (self._stopped.is_set() and self._queue.empty()):
+ self.flush()
+ time.sleep(_MIN_CALLBACK_INTERVAL)
+
+ def install(self):
+ super().install()
+ if self._installed:
+ return
+ stream = self.src_wrapped_stream
+ old_write = stream.write
+ self._prev_callback_timestamp = time.time()
+ self._old_write = old_write
+
+ def write(data):
+ self._old_write(data)
+ self._queue.put(data)
+
+ stream.write = write
+
+ self._queue = queue.Queue()
+ self._stopped = threading.Event()
+ self._emulator_write_thread = threading.Thread(target=self._emulator_write)
+ self._emulator_write_thread.daemon = True
+ self._emulator_write_thread.start()
+
+ if not wandb.run or wandb.run._settings.mode == "online":
+ self._callback_thread = threading.Thread(target=self._callback)
+ self._callback_thread.daemon = True
+ self._callback_thread.start()
+
+ self._installed = True
+
+ def flush(self, data=None):
+ if data is None:
+ try:
+ data = self._emulator.read().encode("utf-8")
+ except Exception:
+ pass
+ if data:
+ for cb in self.cbs:
+ try:
+ cb(data)
+ except Exception:
+ pass # TODO(frz)
+
+ def uninstall(self):
+ if not self._installed:
+ return
+ self.src_wrapped_stream.write = self._old_write
+
+ self._stopped.set()
+ self._emulator_write_thread.join(timeout=5)
+ if self._emulator_write_thread.is_alive():
+ wandb.termlog(f"Processing terminal output ({self.src})...")
+ self._emulator_write_thread.join()
+ wandb.termlog("Done.")
+ self.flush()
+
+ self._installed = False
+ super().uninstall()
+
+
+class StreamRawWrapper(RedirectBase):
+ """Patches the write method of current sys.stdout/sys.stderr.
+
+ Captures data in a raw form rather than using the emulator
+ """
+
+ def __init__(self, src, cbs=()):
+ super().__init__(src=src, cbs=cbs)
+ self._installed = False
+
+ def save(self):
+ stream = self.src_wrapped_stream
+ self._old_write = stream.write
+
+ def install(self):
+ super().install()
+ if self._installed:
+ return
+ stream = self.src_wrapped_stream
+ self._prev_callback_timestamp = time.time()
+
+ def write(data):
+ self._old_write(data)
+ for cb in self.cbs:
+ try:
+ cb(data)
+ except Exception:
+ # TODO: Figure out why this was needed and log or error out appropriately
+ # it might have been strange terminals? maybe shutdown cases?
+ pass
+
+ stream.write = write
+ self._installed = True
+
+ def uninstall(self):
+ if not self._installed:
+ return
+ self.src_wrapped_stream.write = self._old_write
+ self._installed = False
+ super().uninstall()
+
+
+class _WindowSizeChangeHandler:
+ def __init__(self):
+ self._fds = set()
+
+ def _register(self):
+ old_handler = signal.signal(signal.SIGWINCH, lambda *_: None)
+
+ def handler(signum, frame):
+ if callable(old_handler):
+ old_handler(signum, frame)
+ self.handle_window_size_change()
+
+ signal.signal(signal.SIGWINCH, handler)
+ self._old_handler = old_handler
+
+ def _unregister(self):
+ signal.signal(signal.SIGWINCH, self._old_handler)
+
+ def add_fd(self, fd):
+ if not self._fds:
+ self._register()
+ self._fds.add(fd)
+ self.handle_window_size_change()
+
+ def remove_fd(self, fd):
+ if fd in self._fds:
+ self._fds.remove(fd)
+ if not self._fds:
+ self._unregister()
+
+ def handle_window_size_change(self):
+ try:
+ win_size = fcntl.ioctl(0, termios.TIOCGWINSZ, "\0" * 8)
+ rows, cols, xpix, ypix = struct.unpack("HHHH", win_size)
+ # Note: IOError not subclass of OSError in python 2.x
+ except OSError: # eg. in MPI we can't do this.
+ return
+ if cols == 0:
+ return
+ win_size = struct.pack("HHHH", rows, cols, xpix, ypix)
+ for fd in self._fds:
+ fcntl.ioctl(fd, termios.TIOCSWINSZ, win_size)
+
+
+_WSCH = _WindowSizeChangeHandler()
+
+
+class Redirect(RedirectBase):
+ """Redirect low level file descriptors."""
+
+ def __init__(self, src, cbs=()):
+ super().__init__(src=src, cbs=cbs)
+ self._installed = False
+ self._emulator = TerminalEmulator()
+
+ def _pipe(self):
+ if pty:
+ r, w = pty.openpty()
+ else:
+ r, w = os.pipe()
+ return r, w
+
+ def install(self):
+ super().install()
+ if self._installed:
+ return
+ self._pipe_read_fd, self._pipe_write_fd = self._pipe()
+ if os.isatty(self._pipe_read_fd):
+ _WSCH.add_fd(self._pipe_read_fd)
+ self._orig_src_fd = os.dup(self.src_fd)
+ self._orig_src = os.fdopen(self._orig_src_fd, "wb", 0)
+ os.dup2(self._pipe_write_fd, self.src_fd)
+ self._installed = True
+ self._queue = queue.Queue()
+ self._stopped = threading.Event()
+ self._pipe_relay_thread = threading.Thread(target=self._pipe_relay)
+ self._pipe_relay_thread.daemon = True
+ self._pipe_relay_thread.start()
+ self._emulator_write_thread = threading.Thread(target=self._emulator_write)
+ self._emulator_write_thread.daemon = True
+ self._emulator_write_thread.start()
+ if not wandb.run or wandb.run._settings.mode == "online":
+ self._callback_thread = threading.Thread(target=self._callback)
+ self._callback_thread.daemon = True
+ self._callback_thread.start()
+
+ def uninstall(self):
+ if not self._installed:
+ return
+ self._installed = False
+ # If the user printed a very long string (millions of chars) right before wandb.finish(),
+ # it will take a while for it to reach pipe relay. 1 second is enough time for ~5 million chars.
+ time.sleep(1)
+ self._stopped.set()
+ os.dup2(self._orig_src_fd, self.src_fd)
+ os.write(self._pipe_write_fd, _LAST_WRITE_TOKEN)
+ self._pipe_relay_thread.join()
+ os.close(self._pipe_read_fd)
+ os.close(self._pipe_write_fd)
+
+ t = threading.Thread(
+ target=self.src_wrapped_stream.flush
+ ) # Calling flush() from the current thread does not flush the buffer instantly.
+ t.start()
+ t.join(timeout=10)
+
+ self._emulator_write_thread.join(timeout=5)
+ if self._emulator_write_thread.is_alive():
+ wandb.termlog(f"Processing terminal output ({self.src})...")
+ self._emulator_write_thread.join()
+ wandb.termlog("Done.")
+ self.flush()
+
+ _WSCH.remove_fd(self._pipe_read_fd)
+ super().uninstall()
+
+ def flush(self, data=None):
+ if data is None:
+ try:
+ data = self._emulator.read().encode("utf-8")
+ except Exception:
+ pass
+ if data:
+ for cb in self.cbs:
+ try:
+ cb(data)
+ except Exception:
+ pass # TODO(frz)
+
+ def _callback(self):
+ while not self._stopped.is_set():
+ self.flush()
+ time.sleep(_MIN_CALLBACK_INTERVAL)
+
+ def _pipe_relay(self):
+ while True:
+ try:
+ brk = False
+ data = os.read(self._pipe_read_fd, 4096)
+ if self._stopped.is_set():
+ if _LAST_WRITE_TOKEN not in data:
+ # _LAST_WRITE_TOKEN could have gotten split up at the 4096 border
+ n = len(_LAST_WRITE_TOKEN)
+ while n and data[-n:] != _LAST_WRITE_TOKEN[:n]:
+ n -= 1
+ if n:
+ data += os.read(
+ self._pipe_read_fd, len(_LAST_WRITE_TOKEN) - n
+ )
+ if _LAST_WRITE_TOKEN in data:
+ data = data.replace(_LAST_WRITE_TOKEN, b"")
+ brk = True
+ i = self._orig_src.write(data)
+ if i is not None: # python 3 w/ unbuffered i/o: we need to keep writing
+ while i < len(data):
+ i += self._orig_src.write(data[i:])
+ self._queue.put(data)
+ if brk:
+ return
+ except OSError:
+ return
+
+ def _emulator_write(self):
+ while True:
+ if self._queue.empty():
+ if self._stopped.is_set():
+ return
+ time.sleep(0.5)
+ continue
+ data = []
+ while not self._queue.empty():
+ data.append(self._queue.get())
+ if self._stopped.is_set() and sum(map(len, data)) > 100000:
+ wandb.termlog("Terminal output too large. Logging without processing.")
+ self.flush()
+ [self.flush(line) for line in data]
+ return
+ try:
+ self._emulator.write(b"".join(data).decode("utf-8"))
+ except Exception:
+ pass
diff --git a/wandb/sdk/lib/reporting.py b/wandb/sdk/lib/reporting.py
new file mode 100644
index 0000000000000000000000000000000000000000..a9e6cad9d2a328986d6eca79d2600f98e4613ef5
--- /dev/null
+++ b/wandb/sdk/lib/reporting.py
@@ -0,0 +1,99 @@
+"""reporting."""
+
+import logging
+
+logger = logging.getLogger("wandb")
+
+
+class _Reporter:
+ def __init__(self, settings):
+ self._settings = settings
+ self._errors = []
+ self._warnings = []
+ self._num_errors = 0
+ self._num_warnings = 0
+ self._context = dict()
+
+ def error(self, __s, *args):
+ pass
+
+ def warning(self, __s, *args):
+ show = self._settings.show_warnings
+ summary = self._settings.summary_warnings
+ if show is not None or summary is not None:
+ s = __s % args
+ self._num_warnings += 1
+ if show is not None:
+ if self._num_warnings <= show or show == 0:
+ print("[WARNING]", s)
+ if self._num_warnings == show:
+ print("not showing any more warnings")
+ if summary is not None:
+ if self._num_warnings <= summary or summary == 0:
+ self._warnings.append(s)
+
+ def info(self, __s, *args):
+ if self._settings.show_info:
+ print(("[INFO]" + __s) % args)
+
+ def internal(self, __s, *args):
+ pass
+
+ def problem(self, bool, __s=None, *args):
+ pass
+
+ def set_context(self, __d=None, **kwargs):
+ if __d:
+ self._context.update(__d)
+ self._context.update(**kwargs)
+
+ def clear_context(self, keys=None):
+ if keys is None:
+ self._context = dict()
+ return
+ for k in keys:
+ self._context.pop(k, None)
+
+ @property
+ def warning_count(self):
+ return self._num_warnings
+
+ @property
+ def error_count(self):
+ return self._num_errors
+
+ @property
+ def warning_lines(self):
+ return self._warnings
+
+ @property
+ def error_lines(self):
+ return self._errors
+
+
+class Reporter:
+ _instance = None
+
+ def __init__(self, settings=None):
+ if Reporter._instance is not None:
+ return
+ if settings is None:
+ logging.error("internal issue: reporter not setup")
+
+ Reporter._instance = _Reporter(settings)
+
+ def __getattr__(self, name):
+ return getattr(self._instance, name)
+
+
+def setup_reporter(settings):
+ # fixme: why?
+ # if not settings.is_frozen():
+ # logging.error("internal issue: settings not frozen")
+ r = Reporter(settings=settings)
+ return r
+
+
+def get_reporter():
+ r = Reporter()
+ return r
diff --git a/wandb/sdk/lib/retry.py b/wandb/sdk/lib/retry.py
new file mode 100644
index 0000000000000000000000000000000000000000..a3e1470dd7efc8822dceb081fe8e88f152dce5d0
--- /dev/null
+++ b/wandb/sdk/lib/retry.py
@@ -0,0 +1,288 @@
+import abc
+import asyncio
+import datetime
+import functools
+import logging
+import os
+import random
+import threading
+import time
+from typing import Any, Awaitable, Callable, Generic, Optional, Tuple, Type, TypeVar
+
+from requests import HTTPError
+
+import wandb
+from wandb.util import CheckRetryFnType
+
+from .mailbox import ContextCancelledError
+
+logger = logging.getLogger(__name__)
+
+
+# To let tests mock out the retry logic's now()/sleep() funcs, this file
+# should only use these variables, not call the stdlib funcs directly.
+NOW_FN = datetime.datetime.now
+SLEEP_FN = time.sleep
+SLEEP_ASYNC_FN = asyncio.sleep
+
+
+class TransientError(Exception):
+ """Exception type designated for errors that may only be temporary.
+
+ Can have its own message and/or wrap another exception.
+ """
+
+ def __init__(
+ self, msg: Optional[str] = None, exc: Optional[BaseException] = None
+ ) -> None:
+ super().__init__(msg)
+ self.message = msg
+ self.exception = exc
+
+
+_R = TypeVar("_R")
+
+
+class Retry(Generic[_R]):
+ """Create a retryable version of a function.
+
+ Calling this will call the passed function, retrying if any exceptions in
+ retryable_exceptions are caught, with exponential backoff.
+ """
+
+ MAX_SLEEP_SECONDS = 5 * 60
+
+ def __init__(
+ self,
+ call_fn: Callable[..., _R],
+ retry_timedelta: Optional[datetime.timedelta] = None,
+ retry_cancel_event: Optional[threading.Event] = None,
+ num_retries: Optional[int] = None,
+ check_retry_fn: CheckRetryFnType = lambda e: True,
+ retryable_exceptions: Optional[Tuple[Type[Exception], ...]] = None,
+ error_prefix: str = "Network error",
+ retry_callback: Optional[Callable[[int, str], Any]] = None,
+ ) -> None:
+ self._call_fn = call_fn
+ self._check_retry_fn = check_retry_fn
+ self._error_prefix = error_prefix
+ self._last_print = datetime.datetime.now() - datetime.timedelta(minutes=1)
+ self._retry_timedelta = retry_timedelta
+ self._retry_cancel_event = retry_cancel_event
+ self._num_retries = num_retries
+ if retryable_exceptions is not None:
+ self._retryable_exceptions = retryable_exceptions
+ else:
+ self._retryable_exceptions = (TransientError,)
+ self._index = 0
+ self.retry_callback = retry_callback
+
+ def _sleep_check_cancelled(
+ self, wait_seconds: float, cancel_event: Optional[threading.Event]
+ ) -> bool:
+ if not cancel_event:
+ SLEEP_FN(wait_seconds)
+ return False
+ cancelled = cancel_event.wait(wait_seconds)
+ return cancelled
+
+ @property
+ def num_iters(self) -> int:
+ """The number of iterations the previous __call__ retried."""
+ return self._num_iter
+
+ def __call__(self, *args: Any, **kwargs: Any) -> _R: # noqa: C901
+ """Call the wrapped function, with retries.
+
+ Arguments:
+ retry_timedelta (kwarg): amount of time to retry before giving up.
+ sleep_base (kwarg): amount of time to sleep upon first failure, all other sleeps
+ are derived from this one.
+ """
+ retry_timedelta = kwargs.pop("retry_timedelta", self._retry_timedelta)
+ if retry_timedelta is None:
+ retry_timedelta = datetime.timedelta(days=365)
+
+ retry_cancel_event = kwargs.pop("retry_cancel_event", self._retry_cancel_event)
+
+ num_retries = kwargs.pop("num_retries", self._num_retries)
+ if num_retries is None:
+ num_retries = 1000000
+
+ if os.environ.get("WANDB_TEST"):
+ num_retries = 0
+
+ sleep_base: float = kwargs.pop("retry_sleep_base", 1)
+
+ # an extra function to allow performing more logic on the filtered exception
+ check_retry_fn: CheckRetryFnType = kwargs.pop(
+ "check_retry_fn", self._check_retry_fn
+ )
+
+ sleep = sleep_base
+ now = NOW_FN()
+ start_time = now
+ start_time_triggered = None
+
+ self._num_iter = 0
+
+ while True:
+ try:
+ result = self._call_fn(*args, **kwargs)
+ # Only print resolved attempts once every minute
+ if self._num_iter > 2 and now - self._last_print > datetime.timedelta(
+ minutes=1
+ ):
+ self._last_print = NOW_FN()
+ if self.retry_callback:
+ self.retry_callback(
+ 200,
+ "{} resolved after {}, resuming normal operation.".format(
+ self._error_prefix, NOW_FN() - start_time
+ ),
+ )
+ return result
+ except self._retryable_exceptions as e:
+ # if the secondary check fails, re-raise
+ retry_timedelta_triggered = check_retry_fn(e)
+ if not retry_timedelta_triggered:
+ raise
+
+ # always enforce num_retries no matter which type of exception was seen
+ if self._num_iter >= num_retries:
+ raise
+
+ now = NOW_FN()
+
+ # handle a triggered secondary check which could have a shortened timeout
+ if isinstance(retry_timedelta_triggered, datetime.timedelta):
+ # save the time of the first secondary trigger
+ if not start_time_triggered:
+ start_time_triggered = now
+
+ # make sure that we haven't run out of time from secondary trigger
+ if now - start_time_triggered >= retry_timedelta_triggered:
+ raise
+
+ # always enforce the default timeout from start of retries
+ if now - start_time >= retry_timedelta:
+ raise
+
+ if self._num_iter == 2:
+ logger.info("Retry attempt failed:", exc_info=e)
+ if (
+ isinstance(e, HTTPError)
+ and e.response is not None
+ and self.retry_callback is not None
+ ):
+ self.retry_callback(e.response.status_code, e.response.text)
+ else:
+ # todo: would like to catch other errors, eg wandb.errors.Error, ConnectionError etc
+ # but some of these can be raised before the retry handler thread (RunStatusChecker) is
+ # spawned in wandb_init
+ wandb.termlog(
+ "{} ({}), entering retry loop.".format(
+ self._error_prefix, e.__class__.__name__
+ )
+ )
+ # if wandb.env.is_debug():
+ # traceback.print_exc()
+ cancelled = self._sleep_check_cancelled(
+ sleep + random.random() * 0.25 * sleep, cancel_event=retry_cancel_event
+ )
+ if cancelled:
+ raise ContextCancelledError("retry timeout")
+ sleep *= 2
+ if sleep > self.MAX_SLEEP_SECONDS:
+ sleep = self.MAX_SLEEP_SECONDS
+ now = NOW_FN()
+
+ self._num_iter += 1
+
+
+_F = TypeVar("_F", bound=Callable)
+
+
+def retriable(*args: Any, **kargs: Any) -> Callable[[_F], _F]:
+ def decorator(fn: _F) -> _F:
+ retrier: Retry[Any] = Retry(fn, *args, **kargs)
+
+ @functools.wraps(fn)
+ def wrapped_fn(*args: Any, **kargs: Any) -> Any:
+ return retrier(*args, **kargs)
+
+ return wrapped_fn # type: ignore
+
+ return decorator
+
+
+class Backoff(abc.ABC):
+ """A backoff strategy: decides whether to sleep or give up when an exception is raised."""
+
+ @abc.abstractmethod
+ def next_sleep_or_reraise(self, exc: Exception) -> datetime.timedelta:
+ raise NotImplementedError # pragma: no cover
+
+
+class ExponentialBackoff(Backoff):
+ """Jittered exponential backoff: sleep times increase ~exponentially up to some limit."""
+
+ def __init__(
+ self,
+ initial_sleep: datetime.timedelta,
+ max_sleep: datetime.timedelta,
+ max_retries: Optional[int] = None,
+ timeout_at: Optional[datetime.datetime] = None,
+ ) -> None:
+ self._next_sleep = min(max_sleep, initial_sleep)
+ self._max_sleep = max_sleep
+ self._remaining_retries = max_retries
+ self._timeout_at = timeout_at
+
+ def next_sleep_or_reraise(self, exc: Exception) -> datetime.timedelta:
+ if self._remaining_retries is not None:
+ if self._remaining_retries <= 0:
+ raise exc
+ self._remaining_retries -= 1
+
+ if self._timeout_at is not None and NOW_FN() > self._timeout_at:
+ raise exc
+
+ result, self._next_sleep = self._next_sleep, min(
+ self._max_sleep, self._next_sleep * (1 + random.random())
+ )
+
+ return result
+
+
+class FilteredBackoff(Backoff):
+ """Re-raise any exceptions that fail a predicate; delegate others to another Backoff."""
+
+ def __init__(self, filter: Callable[[Exception], bool], wrapped: Backoff) -> None:
+ self._filter = filter
+ self._wrapped = wrapped
+
+ def next_sleep_or_reraise(self, exc: Exception) -> datetime.timedelta:
+ if not self._filter(exc):
+ raise exc
+ return self._wrapped.next_sleep_or_reraise(exc)
+
+
+async def retry_async(
+ backoff: Backoff,
+ fn: Callable[..., Awaitable[_R]],
+ *args: Any,
+ on_exc: Optional[Callable[[Exception], None]] = None,
+ **kwargs: Any,
+) -> _R:
+ """Call `fn` repeatedly until either it succeeds, or `backoff` decides we should give up.
+
+ Each time `fn` fails, `on_exc` is called with the exception.
+ """
+ while True:
+ try:
+ return await fn(*args, **kwargs)
+ except Exception as e:
+ if on_exc is not None:
+ on_exc(e)
+ await SLEEP_ASYNC_FN(backoff.next_sleep_or_reraise(e).total_seconds())
diff --git a/wandb/sdk/lib/runid.py b/wandb/sdk/lib/runid.py
new file mode 100644
index 0000000000000000000000000000000000000000..355d24df01a2fc9b5a6174a343e84ed083696277
--- /dev/null
+++ b/wandb/sdk/lib/runid.py
@@ -0,0 +1,12 @@
+"""runid util."""
+
+import secrets
+import string
+
+
+def generate_id(length: int = 8) -> str:
+ """Generate a random base-36 string of `length` digits."""
+ # There are ~2.8T base-36 8-digit strings. If we generate 210k ids,
+ # we'll have a ~1% chance of collision.
+ alphabet = string.ascii_lowercase + string.digits
+ return "".join(secrets.choice(alphabet) for _ in range(length))
diff --git a/wandb/sdk/lib/server.py b/wandb/sdk/lib/server.py
new file mode 100644
index 0000000000000000000000000000000000000000..fa7d95df9b4ab817c74838bb6083df98f435e284
--- /dev/null
+++ b/wandb/sdk/lib/server.py
@@ -0,0 +1,52 @@
+"""module server."""
+
+import json
+from typing import TYPE_CHECKING, Any, Dict, Optional, Union
+
+from wandb import util
+from wandb.apis import InternalApi
+
+if TYPE_CHECKING:
+ from wandb.sdk.wandb_settings import Settings
+
+
+class ServerError(Exception):
+ pass
+
+
+class Server:
+ def __init__(
+ self,
+ api: Optional[InternalApi] = None,
+ settings: Optional["Settings"] = None,
+ ) -> None:
+ self._api = api or InternalApi(default_settings=settings)
+ self._error_network: Optional[bool] = None
+ self._viewer: Dict[str, Any] = {}
+ self._flags: Dict[str, Any] = {}
+ self._settings = settings
+
+ def query_with_timeout(self, timeout: Union[int, float, None] = None) -> None:
+ if self._settings and self._settings._disable_viewer:
+ return
+ timeout = timeout or 5
+ async_viewer = util.async_call(self._api.viewer_server_info, timeout=timeout)
+ try:
+ viewer_tuple, viewer_thread = async_viewer()
+ except Exception:
+ # TODO: currently a bare exception as lots can happen, we should classify
+ self._error_network = True
+ return
+ if viewer_thread.is_alive():
+ # this is likely a DNS hang
+ self._error_network = True
+ return
+ self._error_network = False
+ # TODO(jhr): should we kill the thread?
+ self._viewer, self._serverinfo = viewer_tuple
+ self._flags = json.loads(self._viewer.get("flags", "{}"))
+
+ def is_valid(self) -> bool:
+ if self._error_network is None:
+ raise Exception("invalid usage: must query server")
+ return self._error_network
diff --git a/wandb/sdk/lib/sock_client.py b/wandb/sdk/lib/sock_client.py
new file mode 100644
index 0000000000000000000000000000000000000000..188f87646bafc710b2d90e93ceb9c037243fc987
--- /dev/null
+++ b/wandb/sdk/lib/sock_client.py
@@ -0,0 +1,291 @@
+import socket
+import struct
+import threading
+import time
+import uuid
+from typing import TYPE_CHECKING, Any, List, Optional
+
+from wandb.proto import wandb_server_pb2 as spb
+
+from . import tracelog
+
+if TYPE_CHECKING:
+ from wandb.proto import wandb_internal_pb2 as pb
+
+
+class SockClientClosedError(Exception):
+ """Socket has been closed."""
+
+ pass
+
+
+class SockBuffer:
+ _buf_list: List[bytes]
+ _buf_lengths: List[int]
+ _buf_total: int
+
+ def __init__(self) -> None:
+ self._buf_list = []
+ self._buf_lengths = []
+ self._buf_total = 0
+
+ @property
+ def length(self) -> int:
+ return self._buf_total
+
+ def _get(self, start: int, end: int, peek: bool = False) -> bytes:
+ index: Optional[int] = None
+ buffers = []
+ need = end
+
+ # compute buffers needed
+ for i, (buf_len, buf_data) in enumerate(zip(self._buf_lengths, self._buf_list)):
+ buffers.append(buf_data[:need] if need < buf_len else buf_data)
+ if need <= buf_len:
+ index = i
+ break
+ need -= buf_len
+
+ # buffer not large enough, caller should have made sure there was enough data
+ if index is None:
+ raise IndexError("SockBuffer index out of range")
+
+ # advance buffer internals if we are not peeking into the data
+ if not peek:
+ self._buf_total -= end
+ if need < buf_len:
+ # update partially used buffer list
+ self._buf_list = self._buf_list[index:]
+ self._buf_lengths = self._buf_lengths[index:]
+ self._buf_list[0] = self._buf_list[0][need:]
+ self._buf_lengths[0] -= need
+ else:
+ # update fully used buffer list
+ self._buf_list = self._buf_list[index + 1 :]
+ self._buf_lengths = self._buf_lengths[index + 1 :]
+
+ return b"".join(buffers)[start:end]
+
+ def get(self, start: int, end: int) -> bytes:
+ return self._get(start, end)
+
+ def peek(self, start: int, end: int) -> bytes:
+ return self._get(start, end, peek=True)
+
+ def put(self, data: bytes, data_len: int) -> None:
+ self._buf_list.append(data)
+ self._buf_lengths.append(data_len)
+ self._buf_total += data_len
+
+
+class SockClient:
+ _sock: socket.socket
+ _sockid: str
+ _retry_delay: float
+ _lock: "threading.Lock"
+ _bufsize: int
+ _buffer: SockBuffer
+
+ # current header is magic byte "W" followed by 4 byte length of the message
+ HEADLEN = 1 + 4
+
+ def __init__(self) -> None:
+ # TODO: use safe uuid's (python3.7+) or emulate this
+ self._sockid = uuid.uuid4().hex
+ self._retry_delay = 0.1
+ self._lock = threading.Lock()
+ self._bufsize = 4096
+ self._buffer = SockBuffer()
+
+ def connect(self, port: int) -> None:
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.connect(("localhost", port))
+ self._sock = s
+ self._detect_bufsize()
+
+ def _detect_bufsize(self) -> None:
+ sndbuf_size = self._sock.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF)
+ rcvbuf_size = self._sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF)
+ self._bufsize = min(sndbuf_size, rcvbuf_size, 65536)
+
+ def close(self) -> None:
+ self._sock.close()
+
+ def shutdown(self, val: int) -> None:
+ self._sock.shutdown(val)
+
+ def set_socket(self, sock: socket.socket) -> None:
+ self._sock = sock
+ self._detect_bufsize()
+
+ def _sendall_with_error_handle(self, data: bytes) -> None:
+ # This is a helper function for sending data in a retry fashion.
+ # Similar to the sendall() function in the socket module, but with
+ # an error handling in case of timeout.
+ total_sent = 0
+ total_data = len(data)
+ while total_sent < total_data:
+ start_time = time.monotonic()
+ try:
+ sent = self._sock.send(data)
+ # sent equal to 0 indicates a closed socket
+ if sent == 0:
+ raise SockClientClosedError("socket connection broken")
+ total_sent += sent
+ # truncate our data to save memory
+ data = data[sent:]
+ # we handle the timeout case for the cases when timeout is set
+ # on a system level by another application
+ except socket.timeout:
+ # adding sleep to avoid tight loop
+ delta_time = time.monotonic() - start_time
+ if delta_time < self._retry_delay:
+ time.sleep(self._retry_delay - delta_time)
+
+ def _send_message(self, msg: Any) -> None:
+ tracelog.log_message_send(msg, self._sockid)
+ raw_size = msg.ByteSize()
+ data = msg.SerializeToString()
+ assert len(data) == raw_size, "invalid serialization"
+ header = struct.pack("<BI", ord("W"), raw_size)
+ with self._lock:
+ self._sendall_with_error_handle(header + data)
+
+ def send_server_request(self, msg: Any) -> None:
+ self._send_message(msg)
+
+ def send_server_response(self, msg: Any) -> None:
+ try:
+ self._send_message(msg)
+ except BrokenPipeError:
+ # TODO(jhr): user thread might no longer be around to receive responses to
+ # things like network status poll loop, there might be a better way to quiesce
+ pass
+
+ def send_and_recv(
+ self,
+ *,
+ inform_init: Optional[spb.ServerInformInitRequest] = None,
+ inform_start: Optional[spb.ServerInformStartRequest] = None,
+ inform_attach: Optional[spb.ServerInformAttachRequest] = None,
+ inform_finish: Optional[spb.ServerInformFinishRequest] = None,
+ inform_teardown: Optional[spb.ServerInformTeardownRequest] = None,
+ ) -> spb.ServerResponse:
+ self.send(
+ inform_init=inform_init,
+ inform_start=inform_start,
+ inform_attach=inform_attach,
+ inform_finish=inform_finish,
+ inform_teardown=inform_teardown,
+ )
+ # TODO: this solution is fragile, but for checking attach
+ # it should be relatively stable.
+ # This pass would be solved as part of the fix in https://wandb.atlassian.net/browse/WB-8709
+ response = self.read_server_response(timeout=1)
+ if response is None:
+ raise Exception("No response")
+ return response
+
+ def send(
+ self,
+ *,
+ inform_init: Optional[spb.ServerInformInitRequest] = None,
+ inform_start: Optional[spb.ServerInformStartRequest] = None,
+ inform_attach: Optional[spb.ServerInformAttachRequest] = None,
+ inform_finish: Optional[spb.ServerInformFinishRequest] = None,
+ inform_teardown: Optional[spb.ServerInformTeardownRequest] = None,
+ ) -> None:
+ server_req = spb.ServerRequest()
+ if inform_init:
+ server_req.inform_init.CopyFrom(inform_init)
+ elif inform_start:
+ server_req.inform_start.CopyFrom(inform_start)
+ elif inform_attach:
+ server_req.inform_attach.CopyFrom(inform_attach)
+ elif inform_finish:
+ server_req.inform_finish.CopyFrom(inform_finish)
+ elif inform_teardown:
+ server_req.inform_teardown.CopyFrom(inform_teardown)
+ else:
+ raise Exception("unmatched")
+ self.send_server_request(server_req)
+
+ def send_record_communicate(self, record: "pb.Record") -> None:
+ server_req = spb.ServerRequest()
+ server_req.record_communicate.CopyFrom(record)
+ self.send_server_request(server_req)
+
+ def send_record_publish(self, record: "pb.Record") -> None:
+ server_req = spb.ServerRequest()
+ server_req.record_publish.CopyFrom(record)
+ self.send_server_request(server_req)
+
+ def _extract_packet_bytes(self) -> Optional[bytes]:
+ # Do we have enough data to read the header?
+ start_offset = self.HEADLEN
+ if self._buffer.length >= start_offset:
+ header = self._buffer.peek(0, start_offset)
+ fields = struct.unpack("<BI", header)
+ magic, dlength = fields
+ assert magic == ord("W")
+ # Do we have enough data to read the full record?
+ end_offset = self.HEADLEN + dlength
+ if self._buffer.length >= end_offset:
+ rec_data = self._buffer.get(start_offset, end_offset)
+ return rec_data
+ return None
+
+ def _read_packet_bytes(self, timeout: Optional[int] = None) -> Optional[bytes]:
+ """Read full message from socket.
+
+ Args:
+ timeout: number of seconds to wait on socket data.
+
+ Raises:
+ SockClientClosedError: socket has been closed.
+ """
+ while True:
+ rec = self._extract_packet_bytes()
+ if rec:
+ return rec
+
+ if timeout:
+ self._sock.settimeout(timeout)
+ try:
+ data = self._sock.recv(self._bufsize)
+ except socket.timeout:
+ break
+ except ConnectionResetError:
+ raise SockClientClosedError
+ except OSError:
+ raise SockClientClosedError
+ finally:
+ if timeout:
+ self._sock.settimeout(None)
+ data_len = len(data)
+ if data_len == 0:
+ # socket.recv() will return 0 bytes if socket was shutdown
+ # caller will handle this condition like other connection problems
+ raise SockClientClosedError
+ self._buffer.put(data, data_len)
+ return None
+
+ def read_server_request(self) -> Optional[spb.ServerRequest]:
+ data = self._read_packet_bytes()
+ if not data:
+ return None
+ rec = spb.ServerRequest()
+ rec.ParseFromString(data)
+ tracelog.log_message_recv(rec, self._sockid)
+ return rec
+
+ def read_server_response(
+ self, timeout: Optional[int] = None
+ ) -> Optional[spb.ServerResponse]:
+ data = self._read_packet_bytes(timeout=timeout)
+ if not data:
+ return None
+ rec = spb.ServerResponse()
+ rec.ParseFromString(data)
+ tracelog.log_message_recv(rec, self._sockid)
+ return rec
diff --git a/wandb/sdk/lib/sparkline.py b/wandb/sdk/lib/sparkline.py
new file mode 100644
index 0000000000000000000000000000000000000000..8f4e131b619482ee84860c6a25457dc7d3307d50
--- /dev/null
+++ b/wandb/sdk/lib/sparkline.py
@@ -0,0 +1,45 @@
+#
+# From pysparklines (BSD License): https://pypi.python.org/pypi/pysparklines
+
+import math
+from typing import List, Union
+
+spark_chars = "â–▂▃▄▅▆▇█"
+
+
+# math.isfinite doesn't exist in python2, so provider our own
+def isfinite(f):
+ return not (math.isinf(f) or math.isnan(f))
+
+
+def sparkify(series: List[Union[float, int]]) -> str:
+ """Convert <series> to a sparkline string.
+
+ Example:
+ >>> sparkify([ 0.5, 1.2, 3.5, 7.3, 8.0, 12.5, 13.2, 15.0, 14.2, 11.8, 6.1,
+ ... 1.9 ])
+ u'â–â–▂▄▅▇▇██▆▄▂'
+
+ >>> sparkify([1, 1, -2, 3, -5, 8, -13])
+ u'▆▆▅▆▄█â–'
+
+ Raises ValueError if input data cannot be converted to float.
+ Raises TypeError if series is not an iterable.
+ """
+ series = [float(i) for i in series]
+ finite_series = [x for x in series if isfinite(x)]
+ if not finite_series:
+ return ""
+ minimum = min(finite_series)
+ maximum = max(finite_series)
+ data_range = maximum - minimum
+ if data_range == 0.0:
+ # Graph a baseline if every input value is equal.
+ return "".join([spark_chars[0] if isfinite(x) else " " for x in series])
+ coefficient = (len(spark_chars) - 1.0) / data_range
+ return "".join(
+ [
+ spark_chars[int(round((x - minimum) * coefficient))] if isfinite(x) else " "
+ for x in series
+ ]
+ )
diff --git a/wandb/sdk/lib/telemetry.py b/wandb/sdk/lib/telemetry.py
new file mode 100644
index 0000000000000000000000000000000000000000..12c844fe4f1e2854d929606b13970aecefa1964e
--- /dev/null
+++ b/wandb/sdk/lib/telemetry.py
@@ -0,0 +1,100 @@
+import re
+import sys
+from types import TracebackType
+from typing import TYPE_CHECKING, ContextManager, Dict, List, Optional, Set, Type
+
+import wandb
+from wandb.proto.wandb_telemetry_pb2 import Imports as TelemetryImports
+from wandb.proto.wandb_telemetry_pb2 import TelemetryRecord
+
+# avoid cycle, use string type reference
+
+if TYPE_CHECKING:
+ from .. import wandb_run
+
+
+_LABEL_TOKEN: str = "@wandbcode{"
+
+
+class _TelemetryObject:
+ _run: Optional["wandb_run.Run"]
+ _obj: TelemetryRecord
+
+ def __init__(
+ self,
+ run: Optional["wandb_run.Run"] = None,
+ obj: Optional[TelemetryRecord] = None,
+ ) -> None:
+ self._run = run or wandb.run
+ self._obj = obj or TelemetryRecord()
+
+ def __enter__(self) -> TelemetryRecord:
+ return self._obj
+
+ def __exit__(
+ self,
+ exctype: Optional[Type[BaseException]],
+ excinst: Optional[BaseException],
+ exctb: Optional[TracebackType],
+ ) -> None:
+ if not self._run:
+ return
+ self._run._telemetry_callback(self._obj)
+
+
+def context(
+ run: Optional["wandb_run.Run"] = None, obj: Optional[TelemetryRecord] = None
+) -> ContextManager[TelemetryRecord]:
+ return _TelemetryObject(run=run, obj=obj)
+
+
+MATCH_RE = re.compile(r"(?P<code>[a-zA-Z0-9_-]+)[,}](?P<rest>.*)")
+
+
+def _parse_label_lines(lines: List[str]) -> Dict[str, str]:
+ seen = False
+ ret = {}
+ for line in lines:
+ idx = line.find(_LABEL_TOKEN)
+ if idx < 0:
+ # Stop parsing on first non token line after match
+ if seen:
+ break
+ continue
+ seen = True
+ label_str = line[idx + len(_LABEL_TOKEN) :]
+
+ # match identifier (first token without key=value syntax (optional)
+ # Note: Parse is fairly permissive as it doesnt enforce strict syntax
+ r = MATCH_RE.match(label_str)
+ if r:
+ ret["code"] = r.group("code").replace("-", "_")
+ label_str = r.group("rest")
+
+ # match rest of tokens on one line
+ tokens = re.findall(
+ r'([a-zA-Z0-9_]+)\s*=\s*("[a-zA-Z0-9_-]*"|[a-zA-Z0-9_-]*)[,}]', label_str
+ )
+ for k, v in tokens:
+ ret[k] = v.strip('"').replace("-", "_")
+ return ret
+
+
+def list_telemetry_imports(only_imported: bool = False) -> Set[str]:
+ import_telemetry_set = {
+ desc.name
+ for desc in TelemetryImports.DESCRIPTOR.fields
+ if desc.type == desc.TYPE_BOOL
+ }
+ if only_imported:
+ imported_modules_set = set(sys.modules)
+ return imported_modules_set.intersection(import_telemetry_set)
+ return import_telemetry_set
+
+
+__all__ = [
+ "TelemetryImports",
+ "TelemetryRecord",
+ "context",
+ "list_telemetry_imports",
+]
diff --git a/wandb/sdk/lib/timed_input.py b/wandb/sdk/lib/timed_input.py
new file mode 100644
index 0000000000000000000000000000000000000000..6d963ae0521263453ac7f68ed2906934dd7c0f95
--- /dev/null
+++ b/wandb/sdk/lib/timed_input.py
@@ -0,0 +1,133 @@
+"""timed_input: add a timeout to standard input.
+
+Approach was inspired by: https://github.com/johejo/inputimeout
+"""
+
+import sys
+import threading
+
+import wandb
+
+SP = " "
+CR = "\r"
+LF = "\n"
+CRLF = CR + LF
+
+
+def _echo(prompt: str) -> None:
+ sys.stdout.write(prompt)
+ sys.stdout.flush()
+
+
+def _posix_timed_input(prompt: str, timeout: float) -> str:
+ _echo(prompt)
+ sel = selectors.DefaultSelector()
+ sel.register(sys.stdin, selectors.EVENT_READ, data=sys.stdin.readline)
+ events = sel.select(timeout=timeout)
+
+ for key, _ in events:
+ input_callback = key.data
+ input_data: str = input_callback()
+ if not input_data: # end-of-file - treat as timeout
+ raise TimeoutError
+ return input_data.rstrip(LF)
+
+ _echo(LF)
+ termios.tcflush(sys.stdin, termios.TCIFLUSH)
+ raise TimeoutError
+
+
+def _windows_timed_input(prompt: str, timeout: float) -> str:
+ interval = 0.1
+
+ _echo(prompt)
+ begin = time.monotonic()
+ end = begin + timeout
+ line = ""
+
+ while time.monotonic() < end:
+ if msvcrt.kbhit(): # type: ignore[attr-defined]
+ c = msvcrt.getwche() # type: ignore[attr-defined]
+ if c in (CR, LF):
+ _echo(CRLF)
+ return line
+ if c == "\003":
+ raise KeyboardInterrupt
+ if c == "\b":
+ line = line[:-1]
+ cover = SP * len(prompt + line + SP)
+ _echo("".join([CR, cover, CR, prompt, line]))
+ else:
+ line += c
+ time.sleep(interval)
+
+ _echo(CRLF)
+ raise TimeoutError
+
+
+def _jupyter_timed_input(prompt: str, timeout: float) -> str:
+ clear = True
+ try:
+ from IPython.core.display import clear_output # type: ignore
+ except ImportError:
+ clear = False
+ wandb.termwarn(
+ "Unable to clear output, can't import clear_output from ipython.core"
+ )
+
+ _echo(prompt)
+
+ user_inp = None
+ event = threading.Event()
+
+ def get_input() -> None:
+ nonlocal user_inp
+ raw = input()
+ if event.is_set():
+ return
+ user_inp = raw
+
+ t = threading.Thread(target=get_input)
+ t.start()
+ t.join(timeout)
+ event.set()
+ if user_inp:
+ return user_inp
+ if clear:
+ clear_output()
+ raise TimeoutError
+
+
+def timed_input(
+ prompt: str, timeout: float, show_timeout: bool = True, jupyter: bool = False
+) -> str:
+ """Behaves like builtin `input()` but adds timeout.
+
+ Args:
+ prompt (str): Prompt to output to stdout.
+ timeout (float): Timeout to wait for input.
+ show_timeout (bool): Show timeout in prompt
+ jupyter (bool): If True, use jupyter specific code.
+
+ Raises:
+ TimeoutError: exception raised if timeout occurred.
+ """
+ if show_timeout:
+ prompt = f"{prompt}({timeout:.0f} second timeout) "
+ if jupyter:
+ return _jupyter_timed_input(prompt=prompt, timeout=timeout)
+
+ return _timed_input(prompt=prompt, timeout=timeout)
+
+
+try:
+ import msvcrt
+except ImportError:
+ import selectors
+ import termios
+
+ _timed_input = _posix_timed_input
+else:
+ import time
+
+ _timed_input = _windows_timed_input
diff --git a/wandb/sdk/lib/timer.py b/wandb/sdk/lib/timer.py
new file mode 100644
index 0000000000000000000000000000000000000000..0ff2be95dc9dcfd5a307558ad936d1b22242f882
--- /dev/null
+++ b/wandb/sdk/lib/timer.py
@@ -0,0 +1,19 @@
+import time
+from typing import Any
+
+
+class Timer:
+ def __init__(self) -> None:
+ self.start_time: float = time.time()
+ self.start: float = time.perf_counter()
+ self.stop: float = self.start
+
+ def __enter__(self) -> "Timer":
+ return self
+
+ def __exit__(self, *args: Any) -> None:
+ self.stop = time.perf_counter()
+
+ @property
+ def elapsed(self) -> float:
+ return self.stop - self.start
diff --git a/wandb/sdk/lib/tracelog.py b/wandb/sdk/lib/tracelog.py
new file mode 100644
index 0000000000000000000000000000000000000000..bb67944aca4704481d836ab93125a0bce75ca795
--- /dev/null
+++ b/wandb/sdk/lib/tracelog.py
@@ -0,0 +1,255 @@
+"""tracelog.
+
+Functions:
+ log_message_queue - message put() to queue
+ log_message_dequeue - message get() from queue
+ log_message_send - message sent to socket
+ log_message_recv - message received from socket
+ log_message_process - message processed by thread
+ log_message_link - message linked to another mesage
+ log_message_assert - message encountered problem
+
+"""
+
+import datetime
+import logging
+import secrets
+import sys
+import threading
+from typing import TYPE_CHECKING, Optional, cast
+
+if TYPE_CHECKING:
+ import multiprocessing
+ import queue
+ import socket
+ from typing import Union
+
+ from wandb.proto import wandb_internal_pb2 as pb
+ from wandb.proto import wandb_server_pb2 as spb
+
+ MessageQueueType = Union[pb.Record, pb.Result]
+ MessageType = Union[pb.Record, pb.Result, spb.ServerRequest, spb.ServerResponse]
+ QueueType = Union[multiprocessing.Queue, queue.Queue]
+ TransportType = Union[socket.socket, str]
+
+
+# Supported modes:
+# logger - tracelog output goes to python logging (default)
+# stdout - tracelog output goes to stdout
+# stderr - tracelog output goes to stderr
+tracelog_mode: Optional[str] = "logger"
+
+logger = logging.getLogger(__name__)
+
+
+ANNOTATE_QUEUE_NAME = "_DEBUGLOG_QUEUE_NAME"
+
+# capture stdout and stderr before anyone messes with them
+stdout_write = sys.__stdout__.write
+stderr_write = sys.__stderr__.write
+
+
+def _log(
+ msg_type: str,
+ log_type: str,
+ is_response: bool = False,
+ record: Optional["pb.Record"] = None,
+ result: Optional["pb.Result"] = None,
+ resource: Optional[str] = None,
+) -> None:
+ prefix = "TRACELOG(1)"
+ tname = threading.current_thread().name
+ now = datetime.datetime.now()
+ ts = now.strftime("%H%M%S.%f")
+ arrow = "<-" if is_response else "->"
+ resource = resource or "unknown"
+ uuid = ""
+ data = record or result
+ record_id = ""
+ if data:
+ uuid = data.uuid or uuid
+ record_id = data._info._tracelog_id
+ uuid = uuid or "-"
+ record_id = record_id or "-"
+ relay = ""
+ if data and data.control and data.control.relay_id:
+ relay = data.control.relay_id
+ relay = relay or "-"
+ line = f"{prefix} {arrow} {ts} {record_id:16} {log_type:7} {resource:8} {tname:16} {msg_type:32} {uuid:32} {relay:32}"
+ if tracelog_mode == "stdout":
+ stdout_write(f"{line}\n")
+ elif tracelog_mode == "stderr":
+ stderr_write(f"{line}\n")
+ elif tracelog_mode == "logger":
+ logger.info(line)
+
+
+def _record_msg_type(record: "pb.Record") -> str:
+ msg_type = str(record.WhichOneof("record_type"))
+ if msg_type == "request":
+ request = record.request
+ msg_type = str(request.WhichOneof("request_type"))
+ return msg_type
+
+
+def _result_msg_type(result: "pb.Result") -> str:
+ msg_type = str(result.WhichOneof("result_type"))
+ if msg_type == "response":
+ response = result.response
+ msg_type = str(response.WhichOneof("response_type"))
+ return msg_type
+
+
+def _log_message(
+ msg: "MessageType", log_type: str, resource: Optional[str] = None
+) -> None:
+ record: Optional[pb.Record] = None
+ result: Optional[pb.Result] = None
+ is_response = False
+ msg_type: str
+ # Note: using strings to avoid an import
+ message_type_str = type(msg).__name__
+ if message_type_str == "Record":
+ record = cast("pb.Record", msg)
+ msg_type = _record_msg_type(record)
+ elif message_type_str == "Result":
+ is_response = True
+ result = cast("pb.Result", msg)
+ msg_type = _result_msg_type(result)
+ elif message_type_str == "ServerRequest":
+ server_request = cast("spb.ServerRequest", msg)
+ msg_type = str(server_request.WhichOneof("server_request_type"))
+ if msg_type == "record_publish":
+ record = server_request.record_publish
+ sub_msg_type = _record_msg_type(record)
+ msg_type = f"pub-{sub_msg_type}"
+ elif msg_type == "record_communicate":
+ record = server_request.record_communicate
+ sub_msg_type = _record_msg_type(record)
+ msg_type = f"comm-{sub_msg_type}"
+ # print("SRV", server_request)
+ elif message_type_str == "ServerResponse":
+ is_response = True
+ server_response = cast("spb.ServerResponse", msg)
+ msg_type = str(server_response.WhichOneof("server_response_type"))
+ if msg_type == "result_communicate":
+ result = server_response.result_communicate
+ sub_msg_type = _result_msg_type(result)
+ msg_type = f"comm-{sub_msg_type}"
+ else:
+ raise AssertionError(f"Unknown message type {message_type_str}")
+ _log(
+ msg_type,
+ is_response=is_response,
+ record=record,
+ result=result,
+ log_type=log_type,
+ resource=resource,
+ )
+
+
+def _log_message_queue(msg: "MessageQueueType", q: "QueueType") -> None:
+ _annotate_message(msg)
+ resource = getattr(q, ANNOTATE_QUEUE_NAME, None)
+ _log_message(msg, "queue", resource=resource)
+
+
+def _log_message_dequeue(msg: "MessageQueueType", q: "QueueType") -> None:
+ resource = getattr(q, ANNOTATE_QUEUE_NAME, None)
+ _log_message(msg, "dequeue", resource=resource)
+
+
+def _log_message_send(msg: "MessageType", t: "TransportType") -> None:
+ _log_message(msg, "send")
+
+
+def _log_message_recv(msg: "MessageType", t: "TransportType") -> None:
+ _log_message(msg, "recv")
+
+
+def _log_message_process(msg: "MessageType") -> None:
+ _log_message(msg, "process")
+
+
+def _log_message_link(src: "MessageType", dest: "MessageType") -> None:
+ _log_message(src, "source")
+ _log_message(dest, "dest")
+
+
+def _log_message_assert(msg: "MessageType") -> None:
+ _log_message(msg, "assert")
+
+
+def _annotate_queue(q: "QueueType", name: str) -> None:
+ setattr(q, ANNOTATE_QUEUE_NAME, name)
+
+
+def _annotate_message(msg: "MessageQueueType") -> None:
+ record_id = secrets.token_hex(8)
+ msg._info._tracelog_id = record_id
+
+
+#
+# Default functions when logging is disabled
+#
+
+
+def log_message_queue(msg: "MessageQueueType", q: "QueueType") -> None:
+ return None
+
+
+def log_message_dequeue(msg: "MessageQueueType", q: "QueueType") -> None:
+ return None
+
+
+def log_message_send(msg: "MessageType", t: "TransportType") -> None:
+ return None
+
+
+def log_message_recv(msg: "MessageType", t: "TransportType") -> None:
+ return None
+
+
+def log_message_process(msg: "MessageType") -> None:
+ return None
+
+
+def log_message_link(src: "MessageType", dest: "MessageType") -> None:
+ return None
+
+
+def log_message_assert(msg: "MessageType") -> None:
+ return None
+
+
+def annotate_queue(q: "QueueType", name: str) -> None:
+ return None
+
+
+def annotate_message(msg: "MessageQueueType") -> None:
+ return None
+
+
+def enable(log_mode: Optional[str] = None) -> None:
+ global tracelog_mode
+ if log_mode:
+ tracelog_mode = log_mode
+
+ global log_message_queue
+ global log_message_dequeue
+ global log_message_send
+ global log_message_recv
+ global log_message_process
+ global log_message_link
+ global log_message_assert
+ global annotate_queue
+ global annotate_message
+ log_message_queue = _log_message_queue
+ log_message_dequeue = _log_message_dequeue
+ log_message_send = _log_message_send
+ log_message_recv = _log_message_recv
+ log_message_process = _log_message_process
+ log_message_link = _log_message_link
+ log_message_assert = _log_message_assert
+ annotate_queue = _annotate_queue
+ annotate_message = _annotate_message
diff --git a/wandb/sdk/lib/wburls.py b/wandb/sdk/lib/wburls.py
new file mode 100644
index 0000000000000000000000000000000000000000..a4f4a5943b5dba34dbd9ff14405f359017d29d5e
--- /dev/null
+++ b/wandb/sdk/lib/wburls.py
@@ -0,0 +1,45 @@
+"""Container for urls used in the wandb package.
+
+Use this anytime a URL is displayed to the user.
+
+Usage:
+ ```python
+ from wandb.sdk.lib.wburls import wburls
+
+ print(f"This is a url {wburls.get('cli_launch')}")
+ ```
+"""
+
+from typing import TYPE_CHECKING, Dict, Optional
+
+if TYPE_CHECKING:
+ from ._wburls_generated import URLS
+
+
+class WBURLs:
+ _urls_dict: Optional[Dict["URLS", str]]
+
+ def __init__(self) -> None:
+ self._urls_dict = None
+
+ def _get_urls(self) -> Dict["URLS", str]:
+ return dict(
+ cli_launch="https://wandb.me/launch",
+ doc_run="https://wandb.me/run",
+ doc_require="https://wandb.me/library-require",
+ doc_start_err="https://docs.wandb.ai/guides/track/tracking-faq#initstarterror-error-communicating-with-wandb-process-",
+ doc_artifacts_guide="https://docs.wandb.ai/guides/artifacts",
+ upgrade_server="https://wandb.me/server-upgrade",
+ multiprocess="http://wandb.me/init-multiprocess",
+ wandb_init="https://wandb.me/wandb-init",
+ wandb_server="https://wandb.me/wandb-server",
+ wandb_define_metric="https://wandb.me/define-metric",
+ )
+
+ def get(self, s: "URLS") -> str:
+ if self._urls_dict is None:
+ self._urls_dict = self._get_urls()
+ return self._urls_dict[s]
+
+
+wburls = WBURLs()
diff --git a/wandb/sdk/service/__init__.py b/wandb/sdk/service/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sdk/service/_startup_debug.py b/wandb/sdk/service/_startup_debug.py
new file mode 100644
index 0000000000000000000000000000000000000000..766a6f1cf3dac53fbfa3cf0a444e4e4e32f62496
--- /dev/null
+++ b/wandb/sdk/service/_startup_debug.py
@@ -0,0 +1,22 @@
+"""_startup_debug.
+
+Temporary helper to debug issues with wandb service startup
+"""
+
+import os
+import time
+
+
+def is_enabled() -> bool:
+ # This is very temporary to help diagnose problems seen by some
+ # customers which we are having trouble reproducing. It should be
+ # replaced by something more permanent in the future when we have
+ # proper logging for wandb-service
+ if os.environ.get("_WANDB_STARTUP_DEBUG"):
+ return True
+ return False
+
+
+def print_message(message: str) -> None:
+ time_now = time.time()
+ print("WANDB_STARTUP_DEBUG", time_now, message)
diff --git a/wandb/sdk/service/port_file.py b/wandb/sdk/service/port_file.py
new file mode 100644
index 0000000000000000000000000000000000000000..1980a84dd29af08d4c402a932231b7b04615337e
--- /dev/null
+++ b/wandb/sdk/service/port_file.py
@@ -0,0 +1,53 @@
+"""port_file: write/read file containing port info."""
+
+import os
+import tempfile
+from typing import Optional
+
+
+class PortFile:
+ _sock_port: Optional[int]
+ _valid: bool
+
+ SOCK_TOKEN = "sock="
+ EOF_TOKEN = "EOF"
+
+ def __init__(self, sock_port: Optional[int] = None) -> None:
+ self._sock_port = sock_port
+ self._valid = False
+
+ def write(self, fname: str) -> None:
+ dname, bname = os.path.split(fname)
+ f = tempfile.NamedTemporaryFile(prefix=bname, dir=dname, mode="w", delete=False)
+ try:
+ tmp_filename = f.name
+ with f:
+ data = []
+ if self._sock_port:
+ data.append(f"{self.SOCK_TOKEN}{self._sock_port}")
+ data.append(self.EOF_TOKEN)
+ port_str = "\n".join(data)
+ written = f.write(port_str)
+ assert written == len(port_str)
+ os.rename(tmp_filename, fname)
+ except Exception:
+ os.unlink(tmp_filename)
+ raise
+
+ def read(self, fname: str) -> None:
+ with open(fname) as f:
+ lines = f.readlines()
+ if lines[-1] != self.EOF_TOKEN:
+ return
+ for ln in lines:
+ if ln.startswith(self.SOCK_TOKEN):
+ self._sock_port = int(ln[len(self.SOCK_TOKEN) :])
+ self._valid = True
+
+ @property
+ def sock_port(self) -> Optional[int]:
+ return self._sock_port
+
+ @property
+ def is_valid(self) -> bool:
+ return self._valid
diff --git a/wandb/sdk/service/server.py b/wandb/sdk/service/server.py
new file mode 100755
index 0000000000000000000000000000000000000000..6d6beae965174049c3d1ada921c3af4ac5047ec7
--- /dev/null
+++ b/wandb/sdk/service/server.py
@@ -0,0 +1,119 @@
+"""wandb server.
+
+Start up socket transport servers.
+"""
+
+import logging
+import os
+import sys
+from typing import Optional
+
+import wandb
+
+from ..lib import tracelog
+from . import _startup_debug, port_file
+from .server_sock import SocketServer
+from .streams import StreamMux
+
+
+class WandbServer:
+ _pid: Optional[int]
+ _sock_port: Optional[int]
+ _debug: bool
+ _serve_sock: bool
+ _sock_server: Optional[SocketServer]
+ _startup_debug_enabled: bool
+
+ def __init__(
+ self,
+ sock_port: Optional[int] = None,
+ port_fname: Optional[str] = None,
+ address: Optional[str] = None,
+ pid: Optional[int] = None,
+ debug: bool = True,
+ serve_sock: bool = False,
+ ) -> None:
+ self._sock_port = sock_port
+ self._port_fname = port_fname
+ self._address = address
+ self._pid = pid
+ self._debug = debug
+ self._serve_sock = serve_sock
+ self._sock_server = None
+ self._startup_debug_enabled = _startup_debug.is_enabled()
+
+ if debug:
+ logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+
+ def _inform_used_ports(self, sock_port: Optional[int]) -> None:
+ if not self._port_fname:
+ return
+ pf = port_file.PortFile(sock_port=sock_port)
+ pf.write(self._port_fname)
+
+ def _start_sock(self, mux: StreamMux) -> int:
+ address: str = self._address or "127.0.0.1"
+ port: int = self._sock_port or 0
+ self._sock_server = SocketServer(mux=mux, address=address, port=port)
+ try:
+ self._sock_server.start()
+ port = self._sock_server.port
+ if self._pid:
+ mux.set_pid(self._pid)
+ except KeyboardInterrupt:
+ mux.cleanup()
+ raise
+ except Exception:
+ mux.cleanup()
+ raise
+ return port
+
+ def _stop_servers(self) -> None:
+ if self._sock_server:
+ self._sock_server.stop()
+
+ def _setup_tracelog(self) -> None:
+ # TODO: remove this temporary hack, need to find a better way to pass settings
+ # to the server. for now lets just look at the environment variable we need
+ tracelog_mode = os.environ.get("WANDB_TRACELOG")
+ if tracelog_mode:
+ tracelog.enable(tracelog_mode)
+
+ def _startup_debug_print(self, message: str) -> None:
+ if not self._startup_debug_enabled:
+ return
+ _startup_debug.print_message(message)
+
+ def _setup_proctitle(self, sock_port: Optional[int]) -> None:
+ # TODO: similar to _setup_tracelog, the internal_process should have
+ # a better way to have access to settings.
+ disable_setproctitle = os.environ.get("WANDB__DISABLE_SETPROCTITLE")
+ if disable_setproctitle:
+ return
+
+ setproctitle = wandb.util.get_optional_module("setproctitle")
+ if setproctitle:
+ service_ver = 2
+ pid = str(self._pid or 0)
+ transport = "s" if sock_port else "g"
+ port = sock_port or 0
+ # this format is similar to wandb_manager token, but it's purely informative now
+ # (consider unifying this in the future)
+ service_id = f"{service_ver}-{pid}-{transport}-{port}"
+ proc_title = f"wandb-service({service_id})"
+ self._startup_debug_print("before_setproctitle")
+ setproctitle.setproctitle(proc_title)
+ self._startup_debug_print("after_setproctitle")
+
+ def serve(self) -> None:
+ self._setup_tracelog()
+ mux = StreamMux()
+ self._startup_debug_print("before_network")
+ sock_port = self._start_sock(mux=mux) if self._serve_sock else None
+ self._startup_debug_print("after_network")
+ self._inform_used_ports(sock_port=sock_port)
+ self._startup_debug_print("after_inform")
+ self._setup_proctitle(sock_port=sock_port)
+ self._startup_debug_print("before_loop")
+ mux.loop()
+ self._stop_servers()
diff --git a/wandb/sdk/service/server_sock.py b/wandb/sdk/service/server_sock.py
new file mode 100644
index 0000000000000000000000000000000000000000..3249d31ca3294c7a3218e56801963e921e89b4f5
--- /dev/null
+++ b/wandb/sdk/service/server_sock.py
@@ -0,0 +1,276 @@
+import queue
+import socket
+import threading
+import time
+from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
+
+from wandb.proto import wandb_server_pb2 as spb
+from wandb.sdk.internal.settings_static import SettingsStatic
+
+from ..lib import tracelog
+from ..lib.sock_client import SockClient, SockClientClosedError
+from .streams import StreamMux
+
+if TYPE_CHECKING:
+ from threading import Event
+
+ from ..interface.interface_relay import InterfaceRelay
+
+
+class ClientDict:
+ _client_dict: Dict[str, SockClient]
+ _lock: threading.Lock
+
+ def __init__(self) -> None:
+ self._client_dict = {}
+ self._lock = threading.Lock()
+
+ def get_client(self, client_id: str) -> Optional[SockClient]:
+ with self._lock:
+ client = self._client_dict.get(client_id)
+ return client
+
+ def add_client(self, client: SockClient) -> None:
+ with self._lock:
+ self._client_dict[client._sockid] = client
+
+ def del_client(self, client: SockClient) -> None:
+ with self._lock:
+ del self._client_dict[client._sockid]
+
+
+class SockServerInterfaceReaderThread(threading.Thread):
+ _socket_client: SockClient
+ _stopped: "Event"
+
+ def __init__(
+ self, clients: ClientDict, iface: "InterfaceRelay", stopped: "Event"
+ ) -> None:
+ self._iface = iface
+ self._clients = clients
+ threading.Thread.__init__(self)
+ self.name = "SockSrvIntRdThr"
+ self._stopped = stopped
+
+ def run(self) -> None:
+ assert self._iface.relay_q
+ while not self._stopped.is_set():
+ try:
+ result = self._iface.relay_q.get(timeout=1)
+ except queue.Empty:
+ continue
+ except OSError:
+ # handle is closed
+ break
+ except ValueError:
+ # queue is closed
+ break
+ tracelog.log_message_dequeue(result, self._iface.relay_q)
+ sockid = result.control.relay_id
+ assert sockid
+ sock_client = self._clients.get_client(sockid)
+ assert sock_client
+ sresp = spb.ServerResponse()
+ sresp.result_communicate.CopyFrom(result)
+ sock_client.send_server_response(sresp)
+
+
+class SockServerReadThread(threading.Thread):
+ _sock_client: SockClient
+ _mux: StreamMux
+ _stopped: "Event"
+ _clients: ClientDict
+
+ def __init__(
+ self, conn: socket.socket, mux: StreamMux, clients: ClientDict
+ ) -> None:
+ self._mux = mux
+ threading.Thread.__init__(self)
+ self.name = "SockSrvRdThr"
+ sock_client = SockClient()
+ sock_client.set_socket(conn)
+ self._sock_client = sock_client
+ self._stopped = mux._get_stopped_event()
+ self._clients = clients
+
+ def run(self) -> None:
+ while not self._stopped.is_set():
+ try:
+ sreq = self._sock_client.read_server_request()
+ except SockClientClosedError:
+ # socket has been closed
+ # TODO: shut down other threads serving this socket?
+ break
+ assert sreq, "read_server_request should never timeout"
+ sreq_type = sreq.WhichOneof("server_request_type")
+ shandler_str = "server_" + sreq_type # type: ignore
+ shandler: Callable[[spb.ServerRequest], None] = getattr( # type: ignore
+ self, shandler_str, None
+ )
+ assert shandler, f"unknown handle: {shandler_str}" # type: ignore
+ shandler(sreq)
+
+ def stop(self) -> None:
+ try:
+ # See shutdown notes in class SocketServer for a discussion about this mechanism
+ self._sock_client.shutdown(socket.SHUT_RDWR)
+ except OSError:
+ pass
+ self._sock_client.close()
+
+ def server_inform_init(self, sreq: "spb.ServerRequest") -> None:
+ request = sreq.inform_init
+ stream_id = request._info.stream_id
+ settings = SettingsStatic(request.settings)
+ self._mux.add_stream(stream_id, settings=settings)
+
+ iface = self._mux.get_stream(stream_id).interface
+ self._clients.add_client(self._sock_client)
+ iface_reader_thread = SockServerInterfaceReaderThread(
+ clients=self._clients,
+ iface=iface,
+ stopped=self._stopped,
+ )
+ iface_reader_thread.start()
+
+ def server_inform_start(self, sreq: "spb.ServerRequest") -> None:
+ request = sreq.inform_start
+ stream_id = request._info.stream_id
+ settings = SettingsStatic(request.settings)
+ self._mux.update_stream(stream_id, settings=settings)
+ self._mux.start_stream(stream_id)
+
+ def server_inform_attach(self, sreq: "spb.ServerRequest") -> None:
+ request = sreq.inform_attach
+ stream_id = request._info.stream_id
+
+ self._clients.add_client(self._sock_client)
+ inform_attach_response = spb.ServerInformAttachResponse()
+ inform_attach_response.settings.CopyFrom(
+ self._mux._streams[stream_id]._settings._proto,
+ )
+ response = spb.ServerResponse(inform_attach_response=inform_attach_response)
+ self._sock_client.send_server_response(response)
+ iface = self._mux.get_stream(stream_id).interface
+
+ assert iface
+
+ def server_record_communicate(self, sreq: "spb.ServerRequest") -> None:
+ record = sreq.record_communicate
+ # encode relay information so the right socket picks up the data
+ record.control.relay_id = self._sock_client._sockid
+ stream_id = record._info.stream_id
+ iface = self._mux.get_stream(stream_id).interface
+ assert iface.record_q
+ iface.record_q.put(record)
+
+ def server_record_publish(self, sreq: "spb.ServerRequest") -> None:
+ record = sreq.record_publish
+ # encode relay information so the right socket picks up the data
+ record.control.relay_id = self._sock_client._sockid
+ stream_id = record._info.stream_id
+ iface = self._mux.get_stream(stream_id).interface
+ assert iface.record_q
+ iface.record_q.put(record)
+
+ def server_inform_finish(self, sreq: "spb.ServerRequest") -> None:
+ request = sreq.inform_finish
+ stream_id = request._info.stream_id
+ self._mux.drop_stream(stream_id)
+
+ def server_inform_teardown(self, sreq: "spb.ServerRequest") -> None:
+ request = sreq.inform_teardown
+ exit_code = request.exit_code
+ self._mux.teardown(exit_code)
+
+
+class SockAcceptThread(threading.Thread):
+ _sock: socket.socket
+ _mux: StreamMux
+ _stopped: "Event"
+ _clients: ClientDict
+
+ def __init__(self, sock: socket.socket, mux: StreamMux) -> None:
+ self._sock = sock
+ self._mux = mux
+ self._stopped = mux._get_stopped_event()
+ threading.Thread.__init__(self)
+ self.name = "SockAcceptThr"
+ self._clients = ClientDict()
+
+ def run(self) -> None:
+ self._sock.listen(5)
+ read_threads = []
+
+ while not self._stopped.is_set():
+ try:
+ conn, addr = self._sock.accept()
+ except ConnectionAbortedError:
+ break
+ except OSError:
+ # on shutdown
+ break
+ sr = SockServerReadThread(conn=conn, mux=self._mux, clients=self._clients)
+ sr.start()
+ read_threads.append(sr)
+
+ for rt in read_threads:
+ rt.stop()
+
+
+class DebugThread(threading.Thread):
+ def __init__(self, mux: "StreamMux") -> None:
+ threading.Thread.__init__(self)
+ self.daemon = True
+ self.name = "DebugThr"
+
+ def run(self) -> None:
+ while True:
+ time.sleep(30)
+ for thread in threading.enumerate():
+ print(f"DEBUG: {thread.name}")
+
+
+class SocketServer:
+ _mux: StreamMux
+ _address: str
+ _port: int
+ _sock: socket.socket
+
+ def __init__(self, mux: Any, address: str, port: int) -> None:
+ self._mux = mux
+ self._address = address
+ self._port = port
+ # This is the server socket that we accept new connections from
+ self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+ def _bind(self) -> None:
+ self._sock.bind((self._address, self._port))
+ self._port = self._sock.getsockname()[1]
+
+ @property
+ def port(self) -> int:
+ return self._port
+
+ def start(self) -> None:
+ self._bind()
+ self._thread = SockAcceptThread(sock=self._sock, mux=self._mux)
+ self._thread.start()
+ # Note: Uncomment to figure out what thread is not exiting properly
+ # self._dbg_thread = DebugThread(mux=self._mux)
+ # self._dbg_thread.start()
+
+ def stop(self) -> None:
+ if self._sock:
+ # we need to stop the SockAcceptThread
+ try:
+ # TODO(jhr): consider a more graceful shutdown in the future
+ # socket.shutdown() is a more heavy handed approach to interrupting socket.accept()
+ # in the future we might want to consider a more graceful shutdown which would involve setting
+ # a threading Event and then initiating one last connection just to close down the thread
+ # The advantage of the heavy handed approach is that it doesnt depend on the threads functioning
+ # properly, that is, if something has gone wrong, we probably want to use this hammer to shut things down
+ self._sock.shutdown(socket.SHUT_RDWR)
+ except OSError:
+ pass
+ self._sock.close()
diff --git a/wandb/sdk/service/service.py b/wandb/sdk/service/service.py
new file mode 100644
index 0000000000000000000000000000000000000000..072b3a7d9b80c1ca2e5a129446d99f3948feab8e
--- /dev/null
+++ b/wandb/sdk/service/service.py
@@ -0,0 +1,251 @@
+"""Reliably launch and connect to backend server process (wandb service).
+
+Backend server process can be connected to using tcp sockets transport.
+"""
+import datetime
+import os
+import pathlib
+import platform
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+from wandb import _sentry, termlog
+from wandb.env import error_reporting_enabled
+from wandb.errors import Error
+from wandb.util import get_core_path, get_module
+
+from . import _startup_debug, port_file
+from .service_base import ServiceInterface
+from .service_sock import ServiceSockInterface
+
+if TYPE_CHECKING:
+ from wandb.sdk.wandb_settings import Settings
+
+
+class ServiceStartProcessError(Error):
+ """Raised when a known error occurs when launching wandb service."""
+
+ pass
+
+
+class ServiceStartTimeoutError(Error):
+ """Raised when service start times out."""
+
+ pass
+
+
+class ServiceStartPortError(Error):
+ """Raised when service start fails to find a port."""
+
+ pass
+
+
+class _Service:
+ _settings: "Settings"
+ _sock_port: Optional[int]
+ _service_interface: ServiceInterface
+ _internal_proc: Optional[subprocess.Popen]
+ _startup_debug_enabled: bool
+
+ def __init__(
+ self,
+ settings: "Settings",
+ ) -> None:
+ self._settings = settings
+ self._stub = None
+ self._sock_port = None
+ self._internal_proc = None
+ self._startup_debug_enabled = _startup_debug.is_enabled()
+
+ _sentry.configure_scope(tags=dict(settings), process_context="service")
+
+ # current code only supports socket server implementation, in the
+ # future we might be able to support both
+ self._service_interface = ServiceSockInterface()
+
+ def _startup_debug_print(self, message: str) -> None:
+ if not self._startup_debug_enabled:
+ return
+ _startup_debug.print_message(message)
+
+ def _wait_for_ports(
+ self, fname: str, proc: Optional[subprocess.Popen] = None
+ ) -> None:
+ """Wait for the service to write the port file and then read it.
+
+ Args:
+ fname: The path to the port file.
+ proc: The process to wait for.
+
+ Raises:
+ ServiceStartTimeoutError: If the service takes too long to start.
+ ServiceStartPortError: If the service writes an invalid port file or unable to read it.
+ ServiceStartProcessError: If the service process exits unexpectedly.
+
+ """
+ time_max = time.monotonic() + self._settings._service_wait
+ while time.monotonic() < time_max:
+ if proc and proc.poll():
+ # process finished
+ # define these variables for sentry context grab:
+ # command = proc.args
+ # sys_executable = sys.executable
+ # which_python = shutil.which("python3")
+ # proc_out = proc.stdout.read()
+ # proc_err = proc.stderr.read()
+ context = dict(
+ command=proc.args,
+ sys_executable=sys.executable,
+ which_python=shutil.which("python3"),
+ proc_out=proc.stdout.read() if proc.stdout else "",
+ proc_err=proc.stderr.read() if proc.stderr else "",
+ )
+ raise ServiceStartProcessError(
+ f"The wandb service process exited with {proc.returncode}. "
+ "Ensure that `sys.executable` is a valid python interpreter. "
+ "You can override it with the `_executable` setting "
+ "or with the `WANDB__EXECUTABLE` environment variable.",
+ context=context,
+ )
+ if not os.path.isfile(fname):
+ time.sleep(0.2)
+ continue
+ try:
+ pf = port_file.PortFile()
+ pf.read(fname)
+ if not pf.is_valid:
+ time.sleep(0.2)
+ continue
+ self._sock_port = pf.sock_port
+ except Exception as e:
+ # todo: point at the docs. this could be due to a number of reasons,
+ # for example, being unable to write to the port file etc.
+ raise ServiceStartPortError(
+ f"Failed to allocate port for wandb service: {e}."
+ )
+ return
+ raise ServiceStartTimeoutError(
+ "Timed out waiting for wandb service to start after "
+ f"{self._settings._service_wait} seconds. "
+ "Try increasing the timeout with the `_service_wait` setting."
+ )
+
+ def _launch_server(self) -> None:
+ """Launch server and set ports."""
+ # References for starting processes
+ # - https://github.com/wandb/wandb/blob/archive/old-cli/wandb/__init__.py
+ # - https://stackoverflow.com/questions/1196074/how-to-start-a-background-process-in-python
+ self._startup_debug_print("launch")
+
+ kwargs: Dict[str, Any] = dict(close_fds=True)
+ # flags to handle keyboard interrupt signal that is causing a hang
+ if platform.system() == "Windows":
+ kwargs.update(creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) # type: ignore [attr-defined]
+ else:
+ kwargs.update(start_new_session=True)
+
+ pid = str(os.getpid())
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ fname = os.path.join(tmpdir, f"port-{pid}.txt")
+
+ executable = self._settings._executable
+ exec_cmd_list = [executable, "-m"]
+ # Add coverage collection if needed
+ if os.environ.get("YEA_RUN_COVERAGE") and os.environ.get("COVERAGE_RCFILE"):
+ exec_cmd_list += ["coverage", "run", "-m"]
+
+ service_args = []
+ # NOTE: "wandb-core" is the name of the package that will be distributed
+ # as the stable version of the wandb core library.
+ #
+ # Environment variable _WANDB_CORE_PATH is a temporary development feature
+ # to assist in running the core service from a live development directory.
+ core_path = get_core_path()
+ if core_path:
+ service_args.extend([core_path])
+ if not error_reporting_enabled():
+ service_args.append("--no-observability")
+ exec_cmd_list = []
+ else:
+ service_args.extend(["wandb", "service"])
+
+ service_args += [
+ "--port-filename",
+ fname,
+ "--pid",
+ pid,
+ "--debug",
+ ]
+ service_args.append("--serve-sock")
+
+ if os.environ.get("WANDB_SERVICE_PROFILE") == "memray":
+ _ = get_module(
+ "memray",
+ required=(
+ "wandb service memory profiling requires memray, "
+ "install with `pip install memray`"
+ ),
+ )
+
+ time_tag = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+ output_file = f"wandb_service.memray.{time_tag}.bin"
+ cli_executable = (
+ pathlib.Path(__file__).parent.parent.parent.parent
+ / "tools"
+ / "cli.py"
+ )
+ exec_cmd_list = [
+ executable,
+ "-m",
+ "memray",
+ "run",
+ "-o",
+ output_file,
+ ]
+ service_args[0] = str(cli_executable)
+ termlog(
+ f"wandb service memory profiling enabled, output file: {output_file}"
+ )
+ termlog(
+ f"Convert to flamegraph with: `python -m memray flamegraph {output_file}`"
+ )
+
+ try:
+ internal_proc = subprocess.Popen(
+ exec_cmd_list + service_args,
+ env=os.environ,
+ **kwargs,
+ )
+ except Exception as e:
+ _sentry.reraise(e)
+
+ self._startup_debug_print("wait_ports")
+ try:
+ self._wait_for_ports(fname, proc=internal_proc)
+ except Exception as e:
+ _sentry.reraise(e)
+ self._startup_debug_print("wait_ports_done")
+ self._internal_proc = internal_proc
+ self._startup_debug_print("launch_done")
+
+ def start(self) -> None:
+ self._launch_server()
+
+ @property
+ def sock_port(self) -> Optional[int]:
+ return self._sock_port
+
+ @property
+ def service_interface(self) -> ServiceInterface:
+ return self._service_interface
+
+ def join(self) -> int:
+ ret = 0
+ if self._internal_proc:
+ ret = self._internal_proc.wait()
+ return ret
diff --git a/wandb/sdk/service/service_base.py b/wandb/sdk/service/service_base.py
new file mode 100644
index 0000000000000000000000000000000000000000..855f798a936c2204fc65dc2d1e9ce657385308af
--- /dev/null
+++ b/wandb/sdk/service/service_base.py
@@ -0,0 +1,50 @@
+"""Base service abstract class.
+
+Derived classes for socket service interfaces classes should implement
+abstract methods.
+"""
+
+from abc import abstractmethod
+from typing import TYPE_CHECKING, Optional
+
+from wandb.proto import wandb_server_pb2 as spb
+
+if TYPE_CHECKING:
+ from wandb.proto import wandb_settings_pb2
+
+
+class ServiceInterface:
+ def __init__(self) -> None:
+ pass
+
+ @abstractmethod
+ def get_transport(self) -> str:
+ raise NotImplementedError
+
+ @abstractmethod
+ def _svc_inform_init(
+ self, settings: "wandb_settings_pb2.Settings", run_id: str
+ ) -> None:
+ raise NotImplementedError
+
+ @abstractmethod
+ def _svc_inform_start(
+ self, settings: "wandb_settings_pb2.Settings", run_id: str
+ ) -> None:
+ raise NotImplementedError
+
+ @abstractmethod
+ def _svc_inform_attach(self, attach_id: str) -> spb.ServerInformAttachResponse:
+ raise NotImplementedError
+
+ @abstractmethod
+ def _svc_inform_finish(self, run_id: Optional[str] = None) -> None:
+ raise NotImplementedError
+
+ @abstractmethod
+ def _svc_inform_teardown(self, exit_code: int) -> None:
+ raise NotImplementedError
+
+ @abstractmethod
+ def _svc_connect(self, port: int) -> None:
+ raise NotImplementedError
diff --git a/wandb/sdk/service/service_sock.py b/wandb/sdk/service/service_sock.py
new file mode 100644
index 0000000000000000000000000000000000000000..36c21dd89be1f409005e1453a3f298e1cee9ac01
--- /dev/null
+++ b/wandb/sdk/service/service_sock.py
@@ -0,0 +1,70 @@
+"""socket service.
+
+Implement ServiceInterface for socket transport.
+"""
+
+from typing import TYPE_CHECKING, Optional
+
+from wandb.proto import wandb_server_pb2 as spb
+
+from ..lib.sock_client import SockClient
+from .service_base import ServiceInterface
+
+if TYPE_CHECKING:
+ from wandb.proto import wandb_settings_pb2
+
+
+class ServiceSockInterface(ServiceInterface):
+ _sock_client: SockClient
+
+ def __init__(self) -> None:
+ self._sock_client = SockClient()
+
+ def get_transport(self) -> str:
+ return "tcp"
+
+ def _get_sock_client(self) -> SockClient:
+ return self._sock_client
+
+ def _svc_connect(self, port: int) -> None:
+ self._sock_client.connect(port=port)
+
+ def _svc_inform_init(
+ self, settings: "wandb_settings_pb2.Settings", run_id: str
+ ) -> None:
+ inform_init = spb.ServerInformInitRequest()
+ inform_init.settings.CopyFrom(settings)
+ inform_init._info.stream_id = run_id
+ assert self._sock_client
+ self._sock_client.send(inform_init=inform_init)
+
+ def _svc_inform_start(
+ self, settings: "wandb_settings_pb2.Settings", run_id: str
+ ) -> None:
+ inform_start = spb.ServerInformStartRequest()
+ inform_start.settings.CopyFrom(settings)
+ inform_start._info.stream_id = run_id
+ assert self._sock_client
+ self._sock_client.send(inform_start=inform_start)
+
+ def _svc_inform_finish(self, run_id: Optional[str] = None) -> None:
+ assert run_id
+ inform_finish = spb.ServerInformFinishRequest()
+ inform_finish._info.stream_id = run_id
+
+ assert self._sock_client
+ self._sock_client.send(inform_finish=inform_finish)
+
+ def _svc_inform_attach(self, attach_id: str) -> spb.ServerInformAttachResponse:
+ inform_attach = spb.ServerInformAttachRequest()
+ inform_attach._info.stream_id = attach_id
+
+ assert self._sock_client
+ response = self._sock_client.send_and_recv(inform_attach=inform_attach)
+ return response.inform_attach_response
+
+ def _svc_inform_teardown(self, exit_code: int) -> None:
+ inform_teardown = spb.ServerInformTeardownRequest(exit_code=exit_code)
+
+ assert self._sock_client
+ self._sock_client.send(inform_teardown=inform_teardown)
diff --git a/wandb/sdk/service/streams.py b/wandb/sdk/service/streams.py
new file mode 100755
index 0000000000000000000000000000000000000000..3189b57c58941d1a0a7da5aa99d18be987bb0943
--- /dev/null
+++ b/wandb/sdk/service/streams.py
@@ -0,0 +1,431 @@
+"""streams: class that manages internal threads for each run.
+
+StreamThread: Thread that runs internal.wandb_internal()
+StreamRecord: All the external state for the internal thread (queues, etc)
+StreamAction: Lightweight record for stream ops for thread safety
+StreamMux: Container for dictionary of stream threads per runid
+"""
+import functools
+import multiprocessing
+import queue
+import threading
+import time
+from threading import Event
+from typing import Any, Callable, Dict, List, Optional
+
+import psutil
+
+import wandb
+import wandb.util
+from wandb.proto import wandb_internal_pb2 as pb
+from wandb.sdk.internal.settings_static import SettingsStatic
+from wandb.sdk.lib.mailbox import (
+ Mailbox,
+ MailboxProbe,
+ MailboxProgress,
+ MailboxProgressAll,
+)
+from wandb.sdk.lib.printer import get_printer
+from wandb.sdk.wandb_run import Run
+
+from ..interface.interface_relay import InterfaceRelay
+
+# from wandb.sdk.wandb_settings import Settings
+
+
+class StreamThread(threading.Thread):
+ """Class to running internal process as a thread."""
+
+ def __init__(self, target: Callable, kwargs: Dict[str, Any]) -> None:
+ threading.Thread.__init__(self)
+ self.name = "StreamThr"
+ self._target = target
+ self._kwargs = kwargs
+ self.daemon = True
+
+ def run(self) -> None:
+ # TODO: catch exceptions and report errors to scheduler
+ self._target(**self._kwargs)
+
+
+class StreamRecord:
+ _record_q: "queue.Queue[pb.Record]"
+ _result_q: "queue.Queue[pb.Result]"
+ _relay_q: "queue.Queue[pb.Result]"
+ _iface: InterfaceRelay
+ _thread: StreamThread
+ _settings: SettingsStatic
+ _started: bool
+
+ def __init__(self, settings: SettingsStatic, mailbox: Mailbox) -> None:
+ self._started = False
+ self._mailbox = mailbox
+ self._record_q = queue.Queue()
+ self._result_q = queue.Queue()
+ self._relay_q = queue.Queue()
+ process = multiprocessing.current_process()
+ self._iface = InterfaceRelay(
+ record_q=self._record_q,
+ result_q=self._result_q,
+ relay_q=self._relay_q,
+ process=process,
+ process_check=False,
+ mailbox=self._mailbox,
+ )
+ self._settings = settings
+
+ def start_thread(self, thread: StreamThread) -> None:
+ self._thread = thread
+ thread.start()
+ self._wait_thread_active()
+
+ def _wait_thread_active(self) -> None:
+ result = self._iface.communicate_status()
+ # TODO: using the default communicate timeout, is that enough? retries?
+ assert result
+
+ def join(self) -> None:
+ self._iface.join()
+ if self._thread:
+ self._thread.join()
+
+ def drop(self) -> None:
+ self._iface._drop = True
+
+ @property
+ def interface(self) -> InterfaceRelay:
+ return self._iface
+
+ def mark_started(self) -> None:
+ self._started = True
+
+ def update(self, settings: SettingsStatic) -> None:
+ # Note: Currently just overriding the _settings attribute
+ # once we use Settings Class we might want to properly update it
+ self._settings = settings
+
+
+class StreamAction:
+ _action: str
+ _stream_id: str
+ _processed: Event
+ _data: Any
+
+ def __init__(self, action: str, stream_id: str, data: Optional[Any] = None):
+ self._action = action
+ self._stream_id = stream_id
+ self._data = data
+ self._processed = Event()
+
+ def __repr__(self) -> str:
+ return f"StreamAction({self._action},{self._stream_id})"
+
+ def wait_handled(self) -> None:
+ self._processed.wait()
+
+ def set_handled(self) -> None:
+ self._processed.set()
+
+ @property
+ def stream_id(self) -> str:
+ return self._stream_id
+
+
+class StreamMux:
+ _streams_lock: threading.Lock
+ _streams: Dict[str, StreamRecord]
+ _port: Optional[int]
+ _pid: Optional[int]
+ _action_q: "queue.Queue[StreamAction]"
+ _stopped: Event
+ _pid_checked_ts: Optional[float]
+ _mailbox: Mailbox
+
+ def __init__(self) -> None:
+ self._streams_lock = threading.Lock()
+ self._streams = dict()
+ self._port = None
+ self._pid = None
+ self._stopped = Event()
+ self._action_q = queue.Queue()
+ self._pid_checked_ts = None
+ self._mailbox = Mailbox()
+ self._mailbox.enable_keepalive()
+
+ def _get_stopped_event(self) -> "Event":
+ # TODO: clean this up, there should be a better way to abstract this
+ return self._stopped
+
+ def set_port(self, port: int) -> None:
+ self._port = port
+
+ def set_pid(self, pid: int) -> None:
+ self._pid = pid
+
+ def add_stream(self, stream_id: str, settings: SettingsStatic) -> None:
+ action = StreamAction(action="add", stream_id=stream_id, data=settings)
+ self._action_q.put(action)
+ action.wait_handled()
+
+ def start_stream(self, stream_id: str) -> None:
+ action = StreamAction(action="start", stream_id=stream_id)
+ self._action_q.put(action)
+ action.wait_handled()
+
+ def update_stream(self, stream_id: str, settings: SettingsStatic) -> None:
+ action = StreamAction(action="update", stream_id=stream_id, data=settings)
+ self._action_q.put(action)
+ action.wait_handled()
+
+ def del_stream(self, stream_id: str) -> None:
+ action = StreamAction(action="del", stream_id=stream_id)
+ self._action_q.put(action)
+ action.wait_handled()
+
+ def drop_stream(self, stream_id: str) -> None:
+ action = StreamAction(action="drop", stream_id=stream_id)
+ self._action_q.put(action)
+ action.wait_handled()
+
+ def teardown(self, exit_code: int) -> None:
+ action = StreamAction(action="teardown", stream_id="na", data=exit_code)
+ self._action_q.put(action)
+ action.wait_handled()
+
+ def stream_names(self) -> List[str]:
+ with self._streams_lock:
+ names = list(self._streams.keys())
+ return names
+
+ def has_stream(self, stream_id: str) -> bool:
+ with self._streams_lock:
+ return stream_id in self._streams
+
+ def get_stream(self, stream_id: str) -> StreamRecord:
+ with self._streams_lock:
+ stream = self._streams[stream_id]
+ return stream
+
+ def _process_add(self, action: StreamAction) -> None:
+ stream = StreamRecord(action._data, mailbox=self._mailbox)
+ # run_id = action.stream_id # will want to fix if a streamid != runid
+ settings = action._data
+ thread = StreamThread(
+ target=wandb.wandb_sdk.internal.internal.wandb_internal,
+ kwargs=dict(
+ settings=settings,
+ record_q=stream._record_q,
+ result_q=stream._result_q,
+ port=self._port,
+ user_pid=self._pid,
+ ),
+ )
+ stream.start_thread(thread)
+ with self._streams_lock:
+ self._streams[action._stream_id] = stream
+
+ def _process_start(self, action: StreamAction) -> None:
+ with self._streams_lock:
+ self._streams[action._stream_id].mark_started()
+
+ def _process_update(self, action: StreamAction) -> None:
+ with self._streams_lock:
+ self._streams[action._stream_id].update(action._data)
+
+ def _process_del(self, action: StreamAction) -> None:
+ with self._streams_lock:
+ stream = self._streams.pop(action._stream_id)
+ stream.join()
+ # TODO: we assume stream has already been shutdown. should we verify?
+
+ def _process_drop(self, action: StreamAction) -> None:
+ with self._streams_lock:
+ if action._stream_id in self._streams:
+ stream = self._streams.pop(action._stream_id)
+ stream.drop()
+ stream.join()
+
+ def _on_probe_exit(self, probe_handle: MailboxProbe, stream: StreamRecord) -> None:
+ handle = probe_handle.get_mailbox_handle()
+ if handle:
+ result = handle.wait(timeout=0, release=False)
+ if not result:
+ return
+ probe_handle.set_probe_result(result)
+ handle = stream.interface.deliver_poll_exit()
+ probe_handle.set_mailbox_handle(handle)
+
+ def _on_progress_exit(self, progress_handle: MailboxProgress) -> None:
+ pass
+
+ def _on_progress_exit_all(self, progress_all_handle: MailboxProgressAll) -> None:
+ probe_handles = []
+ progress_handles = progress_all_handle.get_progress_handles()
+ for progress_handle in progress_handles:
+ probe_handles.extend(progress_handle.get_probe_handles())
+
+ assert probe_handles
+
+ if self._check_orphaned():
+ self._stopped.set()
+
+ poll_exit_responses: List[Optional[pb.PollExitResponse]] = []
+ for probe_handle in probe_handles:
+ result = probe_handle.get_probe_result()
+ if result:
+ poll_exit_responses.append(result.response.poll_exit_response)
+
+ Run._footer_file_pusher_status_info(poll_exit_responses, printer=self._printer)
+
+ def _finish_all(self, streams: Dict[str, StreamRecord], exit_code: int) -> None:
+ if not streams:
+ return
+
+ printer = get_printer(
+ all(stream._settings._jupyter for stream in streams.values())
+ )
+ self._printer = printer
+
+ # fixme: for now we have a single printer for all streams,
+ # and jupyter is disabled if at least single stream's setting set `_jupyter` to false
+ exit_handles = []
+
+ # only finish started streams, non started streams failed early
+ started_streams: Dict[str, StreamRecord] = {}
+ not_started_streams: Dict[str, StreamRecord] = {}
+ for stream_id, stream in streams.items():
+ d = started_streams if stream._started else not_started_streams
+ d[stream_id] = stream
+
+ for stream in started_streams.values():
+ handle = stream.interface.deliver_exit(exit_code)
+ handle.add_progress(self._on_progress_exit)
+ handle.add_probe(functools.partial(self._on_probe_exit, stream=stream))
+ exit_handles.append(handle)
+
+ # this message is confusing, we should remove it
+ # Run._footer_exit_status_info(
+ # exit_code, settings=stream._settings, printer=printer # type: ignore
+ # )
+
+ # todo: should we wait for the max timeout (?) of all exit handles or just wait forever?
+ # timeout = max(stream._settings._exit_timeout for stream in streams.values())
+ got_result = self._mailbox.wait_all(
+ handles=exit_handles, timeout=-1, on_progress_all=self._on_progress_exit_all
+ )
+ assert got_result
+
+ # These could be done in parallel in the future
+ for _sid, stream in started_streams.items():
+ # dispatch all our final requests
+ poll_exit_handle = stream.interface.deliver_poll_exit()
+ server_info_handle = stream.interface.deliver_request_server_info()
+ final_summary_handle = stream.interface.deliver_get_summary()
+ sampled_history_handle = stream.interface.deliver_request_sampled_history()
+ internal_messages_handle = stream.interface.deliver_internal_messages()
+
+ result = internal_messages_handle.wait(timeout=-1)
+ assert result
+ internal_messages_response = result.response.internal_messages_response
+ job_info_handle = stream.interface.deliver_request_job_info()
+
+ # wait for them, it's ok to do this serially but this can be improved
+ result = poll_exit_handle.wait(timeout=-1)
+ assert result
+ poll_exit_response = result.response.poll_exit_response
+
+ result = server_info_handle.wait(timeout=-1)
+ assert result
+ server_info_response = result.response.server_info_response
+
+ result = sampled_history_handle.wait(timeout=-1)
+ assert result
+ sampled_history = result.response.sampled_history_response
+
+ result = final_summary_handle.wait(timeout=-1)
+ assert result
+ final_summary = result.response.get_summary_response
+
+ result = job_info_handle.wait(timeout=-1)
+ assert result
+ job_info = result.response.job_info_response
+
+ Run._footer(
+ sampled_history=sampled_history,
+ final_summary=final_summary,
+ poll_exit_response=poll_exit_response,
+ server_info_response=server_info_response,
+ internal_messages_response=internal_messages_response,
+ job_info=job_info,
+ settings=stream._settings, # type: ignore
+ printer=printer,
+ )
+ stream.join()
+
+ # not started streams need to be cleaned up
+ for stream in not_started_streams.values():
+ stream.join()
+
+ def _process_teardown(self, action: StreamAction) -> None:
+ exit_code: int = action._data
+ with self._streams_lock:
+ # TODO: mark streams to prevent new modifications?
+ streams_copy = self._streams.copy()
+ self._finish_all(streams_copy, exit_code)
+ with self._streams_lock:
+ self._streams = dict()
+ self._stopped.set()
+
+ def _process_action(self, action: StreamAction) -> None:
+ if action._action == "add":
+ self._process_add(action)
+ return
+ if action._action == "update":
+ self._process_update(action)
+ return
+ if action._action == "start":
+ self._process_start(action)
+ return
+ if action._action == "del":
+ self._process_del(action)
+ return
+ if action._action == "drop":
+ self._process_drop(action)
+ return
+ if action._action == "teardown":
+ self._process_teardown(action)
+ return
+ raise AssertionError(f"Unsupported action: {action._action}")
+
+ def _check_orphaned(self) -> bool:
+ if not self._pid:
+ return False
+ time_now = time.time()
+ # if we have checked already and it was less than 2 seconds ago
+ if self._pid_checked_ts and time_now < self._pid_checked_ts + 2:
+ return False
+ self._pid_checked_ts = time_now
+ return not psutil.pid_exists(self._pid)
+
+ def _loop(self) -> None:
+ while not self._stopped.is_set():
+ if self._check_orphaned():
+ # parent process is gone, let other threads know we need to shut down
+ self._stopped.set()
+ try:
+ action = self._action_q.get(timeout=1)
+ except queue.Empty:
+ continue
+ self._process_action(action)
+ action.set_handled()
+ self._action_q.task_done()
+ self._action_q.join()
+
+ def loop(self) -> None:
+ try:
+ self._loop()
+ except Exception as e:
+ raise e
+
+ def cleanup(self) -> None:
+ pass
diff --git a/wandb/sdk/verify/__init__.py b/wandb/sdk/verify/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/sdk/verify/verify.py b/wandb/sdk/verify/verify.py
new file mode 100644
index 0000000000000000000000000000000000000000..bbc3a4ebd846bfa86ec41e44b143d7567071eba7
--- /dev/null
+++ b/wandb/sdk/verify/verify.py
@@ -0,0 +1,499 @@
+"""Utilities for wandb verify."""
+import getpass
+import os
+import time
+from functools import partial
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+import click
+import requests
+from pkg_resources import parse_version
+from wandb_gql import gql
+
+import wandb
+from wandb.sdk.artifacts.artifact import Artifact
+from wandb.sdk.lib import runid
+
+from ...apis.internal import Api
+
+PROJECT_NAME = "verify"
+GET_RUN_MAX_TIME = 10
+MIN_RETRYS = 3
+CHECKMARK = "\u2705"
+RED_X = "\u274C"
+ID_PREFIX = runid.generate_id()
+
+
+def nice_id(name):
+ return ID_PREFIX + "-" + name
+
+
+def print_results(
+ failed_test_or_tests: Optional[Union[str, List[str]]], warning: bool
+) -> None:
+ if warning:
+ color = "yellow"
+ else:
+ color = "red"
+ if isinstance(failed_test_or_tests, str):
+ print(RED_X)
+ print(click.style(failed_test_or_tests, fg=color, bold=True))
+ elif isinstance(failed_test_or_tests, list) and len(failed_test_or_tests) > 0:
+ print(RED_X)
+ print(
+ "\n".join(
+ [click.style(f, fg=color, bold=True) for f in failed_test_or_tests]
+ )
+ )
+ else:
+ print(CHECKMARK)
+
+
+def check_host(host: str) -> bool:
+ if host in ("api.wandb.ai", "http://api.wandb.ai", "https://api.wandb.ai"):
+ print_results("Cannot run wandb verify against api.wandb.ai", False)
+ return False
+ return True
+
+
+def check_logged_in(api: Api, host: str) -> bool:
+ print("Checking if logged in".ljust(72, "."), end="")
+ login_doc_url = "https://docs.wandb.ai/ref/cli/wandb-login"
+ fail_string = None
+ if api.api_key is None:
+ fail_string = (
+ "Not logged in. Please log in using `wandb login`. See the docs: {}".format(
+ click.style(login_doc_url, underline=True, fg="blue")
+ )
+ )
+ # check that api key is correct
+ # TODO: Better check for api key is correct
+ else:
+ res = api.api.viewer()
+ if not res:
+ fail_string = (
+ "Could not get viewer with default API key. "
+ f"Please relogin using `WANDB_BASE_URL={host} wandb login --relogin` and try again"
+ )
+
+ print_results(fail_string, False)
+ return fail_string is None
+
+
+def check_secure_requests(url: str, test_url_string: str, failure_output: str) -> None:
+ # check if request is over https
+ print(test_url_string.ljust(72, "."), end="")
+ fail_string = None
+ if not url.startswith("https"):
+ fail_string = failure_output
+ print_results(fail_string, True)
+
+
+def check_cors_configuration(url: str, origin: str) -> None:
+ print("Checking CORs configuration of the bucket".ljust(72, "."), end="")
+ fail_string = None
+ res_get = requests.options(
+ url, headers={"Origin": origin, "Access-Control-Request-Method": "GET"}
+ )
+
+ if res_get.headers.get("Access-Control-Allow-Origin") is None:
+ fail_string = (
+ "Your object store does not have a valid CORs configuration, "
+ f"you must allow GET and PUT to Origin: {origin}"
+ )
+
+ print_results(fail_string, True)
+
+
+def check_run(api: Api) -> bool:
+ print(
+ "Checking logged metrics, saving and downloading a file".ljust(72, "."), end=""
+ )
+ failed_test_strings = []
+
+ # set up config
+ n_epochs = 4
+ string_test = "A test config"
+ dict_test = {"config_val": 2, "config_string": "config string"}
+ list_test = [0, "one", "2"]
+ config = {
+ "epochs": n_epochs,
+ "stringTest": string_test,
+ "dictTest": dict_test,
+ "listTest": list_test,
+ }
+ # create a file to save
+ filepath = "./test with_special-characters.txt"
+ f = open(filepath, "w")
+ f.write("test")
+ f.close()
+
+ with wandb.init(
+ id=nice_id("check_run"), reinit=True, config=config, project=PROJECT_NAME
+ ) as run:
+ run_id = run.id
+ entity = run.entity
+ logged = True
+ try:
+ for i in range(1, 11):
+ run.log({"loss": 1.0 / i}, step=i)
+ log_dict = {"val1": 1.0, "val2": 2}
+ run.log({"dict": log_dict}, step=i + 1)
+ except Exception:
+ logged = False
+ failed_test_strings.append(
+ "Failed to log values to run. Contact W&B for support."
+ )
+
+ try:
+ run.log({"HT%3ML ": wandb.Html('<a href="https://mysite">Link</a>')})
+ except Exception:
+ failed_test_strings.append(
+ "Failed to log to media. Contact W&B for support."
+ )
+
+ wandb.save(filepath)
+ public_api = wandb.Api()
+ prev_run = public_api.run(f"{entity}/{PROJECT_NAME}/{run_id}")
+ # raise Exception(prev_run.__dict__)
+ if prev_run is None:
+ failed_test_strings.append(
+ "Failed to access run through API. Contact W&B for support."
+ )
+ print_results(failed_test_strings, False)
+ return False
+ for key, value in prev_run.config.items():
+ if config[key] != value:
+ failed_test_strings.append(
+ "Read config values don't match run config. Contact W&B for support."
+ )
+ break
+ if logged and (
+ prev_run.history_keys["keys"]["loss"]["previousValue"] != 0.1
+ or prev_run.history_keys["lastStep"] != 11
+ or prev_run.history_keys["keys"]["dict.val1"]["previousValue"] != 1.0
+ or prev_run.history_keys["keys"]["dict.val2"]["previousValue"] != 2
+ ):
+ failed_test_strings.append(
+ "History metrics don't match logged values. Check database encoding."
+ )
+
+ if logged and prev_run.summary["loss"] != 1.0 / 10:
+ failed_test_strings.append(
+ "Read summary values don't match expected value. Check database encoding, or contact W&B for support."
+ )
+ # TODO: (kdg) refactor this so it doesn't rely on an exception handler
+ try:
+ read_file = retry_fn(partial(prev_run.file, filepath))
+ # There's a race where the file hasn't been processed in the queue,
+ # we just retry until we get a download
+ read_file = retry_fn(partial(read_file.download, replace=True))
+ except Exception:
+ failed_test_strings.append(
+ "Unable to download file. Check SQS configuration, topic configuration and bucket permissions."
+ )
+
+ print_results(failed_test_strings, False)
+ return False
+ contents = read_file.read()
+ if contents != "test":
+ failed_test_strings.append(
+ "Contents of downloaded file do not match uploaded contents. Contact W&B for support."
+ )
+ print_results(failed_test_strings, False)
+ return len(failed_test_strings) == 0
+
+
+def verify_manifest(
+ downloaded_manifest: Dict[str, Any],
+ computed_manifest: Dict[str, Any],
+ fails_list: List[str],
+) -> None:
+ try:
+ for key in computed_manifest.keys():
+ assert (
+ computed_manifest[key]["digest"] == downloaded_manifest[key]["digest"]
+ )
+ assert computed_manifest[key]["size"] == downloaded_manifest[key]["size"]
+ except AssertionError:
+ fails_list.append(
+ "Artifact manifest does not appear as expected. Contact W&B for support."
+ )
+
+
+def verify_digest(
+ downloaded: "Artifact", computed: "Artifact", fails_list: List[str]
+) -> None:
+ if downloaded.digest != computed.digest:
+ fails_list.append(
+ "Artifact digest does not appear as expected. Contact W&B for support."
+ )
+
+
+def artifact_with_path_or_paths(
+ name: str, verify_dir: Optional[str] = None, singular: bool = False
+) -> "Artifact":
+ art = wandb.Artifact(type="artsy", name=name)
+ # internal file
+ with open("verify_int_test.txt", "w") as f:
+ f.write("test 1")
+ f.close()
+ art.add_file(f.name)
+ if singular:
+ return art
+ if verify_dir is None:
+ verify_dir = "./"
+ with art.new_file("verify_a.txt") as f:
+ f.write("test 2")
+ if not os.path.exists(verify_dir):
+ os.makedirs(verify_dir)
+ with open(f"{verify_dir}/verify_1.txt", "w") as f:
+ f.write("1")
+ art.add_dir(verify_dir)
+ file3 = Path(verify_dir) / "verify_3.txt"
+ file3.write_text("3")
+
+ # reference to local file
+ art.add_reference(file3.resolve().as_uri())
+
+ return art
+
+
+def log_use_download_artifact(
+ artifact: "Artifact",
+ alias: str,
+ name: str,
+ download_dir: str,
+ failed_test_strings: List[str],
+ add_extra_file: bool,
+) -> Tuple[bool, Optional["Artifact"], List[str]]:
+ with wandb.init(
+ id=nice_id("log_artifact"),
+ reinit=True,
+ project=PROJECT_NAME,
+ config={"test": "artifact log"},
+ ) as log_art_run:
+ if add_extra_file:
+ with open("verify_2.txt", "w") as f:
+ f.write("2")
+ f.close()
+ artifact.add_file(f.name)
+
+ try:
+ log_art_run.log_artifact(artifact, aliases=alias)
+ except Exception as e:
+ failed_test_strings.append(f"Unable to log artifact. {e}")
+ return False, None, failed_test_strings
+
+ with wandb.init(
+ id=nice_id("use_artifact"),
+ project=PROJECT_NAME,
+ config={"test": "artifact use"},
+ ) as use_art_run:
+ try:
+ used_art = use_art_run.use_artifact(f"{name}:{alias}")
+ except Exception as e:
+ failed_test_strings.append(f"Unable to use artifact. {e}")
+ return False, None, failed_test_strings
+ try:
+ used_art.download(root=download_dir)
+ except Exception:
+ failed_test_strings.append(
+ "Unable to download artifact. Check bucket permissions."
+ )
+ return False, None, failed_test_strings
+
+ return True, used_art, failed_test_strings
+
+
+def check_artifacts() -> bool:
+ print("Checking artifact save and download workflows".ljust(72, "."), end="")
+ failed_test_strings: List[str] = []
+
+ # test checksum
+ sing_art_dir = "./verify_sing_art"
+ alias = "sing_art1"
+ name = nice_id("sing-artys")
+ singular_art = artifact_with_path_or_paths(name, singular=True)
+ cont_test, download_artifact, failed_test_strings = log_use_download_artifact(
+ singular_art, alias, name, sing_art_dir, failed_test_strings, False
+ )
+ if not cont_test or download_artifact is None:
+ print_results(failed_test_strings, False)
+ return False
+ try:
+ download_artifact.verify(root=sing_art_dir)
+ except ValueError:
+ failed_test_strings.append(
+ "Artifact does not contain expected checksum. Contact W&B for support."
+ )
+
+ # test manifest and digest
+ multi_art_dir = "./verify_art"
+ alias = "art1"
+ name = nice_id("my-artys")
+ art1 = artifact_with_path_or_paths(name, "./verify_art_dir", singular=False)
+ cont_test, download_artifact, failed_test_strings = log_use_download_artifact(
+ art1, alias, name, multi_art_dir, failed_test_strings, True
+ )
+ if not cont_test or download_artifact is None:
+ print_results(failed_test_strings, False)
+ return False
+ if set(os.listdir(multi_art_dir)) != {
+ "verify_a.txt",
+ "verify_2.txt",
+ "verify_1.txt",
+ "verify_3.txt",
+ "verify_int_test.txt",
+ }:
+ failed_test_strings.append(
+ "Artifact directory is missing files. Contact W&B for support."
+ )
+
+ computed = wandb.Artifact("computed", type="dataset")
+ computed.add_dir(multi_art_dir)
+ verify_digest(download_artifact, computed, failed_test_strings)
+
+ computed_manifest = computed.manifest.to_manifest_json()["contents"]
+ downloaded_manifest = download_artifact.manifest.to_manifest_json()["contents"]
+ verify_manifest(downloaded_manifest, computed_manifest, failed_test_strings)
+
+ print_results(failed_test_strings, False)
+ return len(failed_test_strings) == 0
+
+
+def check_graphql_put(api: Api, host: str) -> Tuple[bool, Optional[str]]:
+ # check graphql endpoint using an upload
+ print("Checking signed URL upload".ljust(72, "."), end="")
+ failed_test_strings = []
+ gql_fp = "gql_test_file.txt"
+ f = open(gql_fp, "w")
+ f.write("test2")
+ f.close()
+ with wandb.init(
+ id=nice_id("graphql_put"),
+ reinit=True,
+ project=PROJECT_NAME,
+ config={"test": "put to graphql"},
+ ) as run:
+ wandb.save(gql_fp)
+ public_api = wandb.Api()
+ prev_run = public_api.run(f"{run.entity}/{PROJECT_NAME}/{run.id}")
+ if prev_run is None:
+ failed_test_strings.append(
+ "Unable to access previous run through public API. Contact W&B for support."
+ )
+ print_results(failed_test_strings, False)
+ return False, None
+ # TODO: (kdg) refactor this so it doesn't rely on an exception handler
+ try:
+ read_file = retry_fn(partial(prev_run.file, gql_fp))
+ url = read_file.url
+ read_file = retry_fn(partial(read_file.download, replace=True))
+ except Exception:
+ failed_test_strings.append(
+ "Unable to read file successfully saved through a put request. Check SQS configurations, bucket permissions and topic configs."
+ )
+ print_results(failed_test_strings, False)
+ return False, None
+ contents = read_file.read()
+ try:
+ assert contents == "test2"
+ except AssertionError:
+ failed_test_strings.append(
+ "Read file contents do not match saved file contents. Contact W&B for support."
+ )
+
+ print_results(failed_test_strings, False)
+ return len(failed_test_strings) == 0, url
+
+
+def check_large_post() -> bool:
+ print(
+ "Checking ability to send large payloads through proxy".ljust(72, "."), end=""
+ )
+ descy = "a" * int(10**7)
+
+ username = getpass.getuser()
+ failed_test_strings = []
+ query = gql(
+ """
+ query Project($entity: String!, $name: String!, $runName: String!, $desc: String!){
+ project(entityName: $entity, name: $name) {
+ run(name: $runName, desc: $desc) {
+ name
+ summaryMetrics
+ }
+ }
+ }
+ """
+ )
+ public_api = wandb.Api()
+ client = public_api._base_client
+
+ try:
+ client._get_result(
+ query,
+ variable_values={
+ "entity": username,
+ "name": PROJECT_NAME,
+ "runName": "",
+ "desc": descy,
+ },
+ timeout=60,
+ )
+ except Exception as e:
+ if (
+ isinstance(e, requests.HTTPError)
+ and e.response is not None
+ and e.response.status_code == 413
+ ):
+ failed_test_strings.append(
+ 'Failed to send a large payload. Check nginx.ingress.kubernetes.io/proxy-body-size is "0".'
+ )
+ else:
+ failed_test_strings.append(
+ f"Failed to send a large payload with error: {e}."
+ )
+ print_results(failed_test_strings, False)
+ return len(failed_test_strings) == 0
+
+
+def check_wandb_version(api: Api) -> None:
+ print("Checking wandb package version is up to date".ljust(72, "."), end="")
+ _, server_info = api.viewer_server_info()
+ fail_string = None
+ warning = False
+ max_cli_version = server_info.get("cliVersionInfo", {}).get("max_cli_version", None)
+ min_cli_version = server_info.get("cliVersionInfo", {}).get(
+ "min_cli_version", "0.0.1"
+ )
+ if parse_version(wandb.__version__) < parse_version(min_cli_version):
+ fail_string = "wandb version out of date, please run pip install --upgrade wandb=={}".format(
+ max_cli_version
+ )
+ elif parse_version(wandb.__version__) > parse_version(max_cli_version):
+ fail_string = (
+ "wandb version is not supported by your local installation. This could "
+ "cause some issues. If you're having problems try: please run `pip "
+ f"install --upgrade wandb=={max_cli_version}`"
+ )
+ warning = True
+
+ print_results(fail_string, warning)
+
+
+def retry_fn(fn: Callable) -> Any:
+ ini_time = time.time()
+ res = None
+ i = 0
+ while i < MIN_RETRYS or time.time() - ini_time < GET_RUN_MAX_TIME:
+ i += 1
+ try:
+ res = fn()
+ break
+ except Exception:
+ time.sleep(1)
+ continue
+ return res
diff --git a/wandb/sdk/wandb_alerts.py b/wandb/sdk/wandb_alerts.py
new file mode 100644
index 0000000000000000000000000000000000000000..072547ab219e907efede3399d660817926074490
--- /dev/null
+++ b/wandb/sdk/wandb_alerts.py
@@ -0,0 +1,12 @@
+#
+from enum import Enum
+
+"""
+Call run.alert() to generate an email or Slack notification programmatically.
+"""
+
+
+class AlertLevel(Enum):
+ INFO = "INFO"
+ WARN = "WARN"
+ ERROR = "ERROR"
diff --git a/wandb/sdk/wandb_config.py b/wandb/sdk/wandb_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..e44332899034779e8cdd9903d4138edb6c470b9d
--- /dev/null
+++ b/wandb/sdk/wandb_config.py
@@ -0,0 +1,319 @@
+"""config."""
+
+import logging
+from typing import Optional
+
+import wandb
+from wandb.util import (
+ _is_artifact_representation,
+ check_dict_contains_nested_artifact,
+ json_friendly_val,
+)
+
+from . import wandb_helper
+from .lib import config_util
+
+logger = logging.getLogger("wandb")
+
+
+# TODO(jhr): consider a callback for persisting changes?
+# if this is done right we might make sure this is pickle-able
+# we might be able to do this on other objects like Run?
+class Config:
+ """Config object.
+
+ Config objects are intended to hold all of the hyperparameters associated with
+ a wandb run and are saved with the run object when `wandb.init` is called.
+
+ We recommend setting `wandb.config` once at the top of your training experiment or
+ setting the config as a parameter to init, ie. `wandb.init(config=my_config_dict)`
+
+ You can create a file called `config-defaults.yaml`, and it will automatically be
+ loaded into `wandb.config`. See https://docs.wandb.com/guides/track/config#file-based-configs.
+
+ You can also load a config YAML file with your custom name and pass the filename
+ into `wandb.init(config="special_config.yaml")`.
+ See https://docs.wandb.com/guides/track/config#file-based-configs.
+
+ Examples:
+ Basic usage
+ ```
+ wandb.config.epochs = 4
+ wandb.init()
+ for x in range(wandb.config.epochs):
+ # train
+ ```
+
+ Using wandb.init to set config
+ ```
+ wandb.init(config={"epochs": 4, "batch_size": 32})
+ for x in range(wandb.config.epochs):
+ # train
+ ```
+
+ Nested configs
+ ```
+ wandb.config['train']['epochs'] = 4
+ wandb.init()
+ for x in range(wandb.config['train']['epochs']):
+ # train
+ ```
+
+ Using absl flags
+ ```
+ flags.DEFINE_string(‘model’, None, ‘model to run’) # name, default, help
+ wandb.config.update(flags.FLAGS) # adds all absl flags to config
+ ```
+
+ Argparse flags
+ ```python
+ wandb.init()
+ wandb.config.epochs = 4
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "-b",
+ "--batch-size",
+ type=int,
+ default=8,
+ metavar="N",
+ help="input batch size for training (default: 8)",
+ )
+ args = parser.parse_args()
+ wandb.config.update(args)
+ ```
+
+ Using TensorFlow flags (deprecated in tensorflow v2)
+ ```python
+ flags = tf.app.flags
+ flags.DEFINE_string("data_dir", "/tmp/data")
+ flags.DEFINE_integer("batch_size", 128, "Batch size.")
+ wandb.config.update(flags.FLAGS) # adds all of the tensorflow flags to config
+ ```
+ """
+
+ def __init__(self):
+ object.__setattr__(self, "_items", dict())
+ object.__setattr__(self, "_locked", dict())
+ object.__setattr__(self, "_users", dict())
+ object.__setattr__(self, "_users_inv", dict())
+ object.__setattr__(self, "_users_cnt", 0)
+ object.__setattr__(self, "_callback", None)
+ object.__setattr__(self, "_settings", None)
+ object.__setattr__(self, "_artifact_callback", None)
+
+ self._load_defaults()
+
+ def _set_callback(self, cb):
+ object.__setattr__(self, "_callback", cb)
+
+ def _set_artifact_callback(self, cb):
+ object.__setattr__(self, "_artifact_callback", cb)
+
+ def _set_settings(self, settings):
+ object.__setattr__(self, "_settings", settings)
+
+ def __repr__(self):
+ return str(dict(self))
+
+ def keys(self):
+ return [k for k in self._items.keys() if not k.startswith("_")]
+
+ def _as_dict(self):
+ return self._items
+
+ def as_dict(self):
+ # TODO: add telemetry, deprecate, then remove
+ return dict(self)
+
+ def __getitem__(self, key):
+ return self._items[key]
+
+ def _check_locked(self, key, ignore_locked=False) -> bool:
+ locked = self._locked.get(key)
+ if locked is not None:
+ locked_user = self._users_inv[locked]
+ if not ignore_locked:
+ wandb.termwarn(
+ f"Config item '{key}' was locked by '{locked_user}' (ignored update)."
+ )
+ return True
+ return False
+
+ def __setitem__(self, key, val):
+ if self._check_locked(key):
+ return
+ with wandb.sdk.lib.telemetry.context() as tel:
+ tel.feature.set_config_item = True
+ self._raise_value_error_on_nested_artifact(val, nested=True)
+ key, val = self._sanitize(key, val)
+ self._items[key] = val
+ logger.info("config set %s = %s - %s", key, val, self._callback)
+ if self._callback:
+ self._callback(key=key, val=val)
+
+ def items(self):
+ return [(k, v) for k, v in self._items.items() if not k.startswith("_")]
+
+ __setattr__ = __setitem__
+
+ def __getattr__(self, key):
+ try:
+ return self.__getitem__(key)
+ except KeyError as ke:
+ raise AttributeError(
+ f"{self.__class__!r} object has no attribute {key!r}"
+ ) from ke
+
+ def __contains__(self, key):
+ return key in self._items
+
+ def _update(self, d, allow_val_change=None, ignore_locked=None):
+ parsed_dict = wandb_helper.parse_config(d)
+ locked_keys = set()
+ for key in list(parsed_dict):
+ if self._check_locked(key, ignore_locked=ignore_locked):
+ locked_keys.add(key)
+ sanitized = self._sanitize_dict(
+ parsed_dict, allow_val_change, ignore_keys=locked_keys
+ )
+ self._items.update(sanitized)
+ return sanitized
+
+ def update(self, d, allow_val_change=None):
+ sanitized = self._update(d, allow_val_change)
+ if self._callback:
+ self._callback(data=sanitized)
+
+ def get(self, *args):
+ return self._items.get(*args)
+
+ def persist(self):
+ """Call the callback if it's set."""
+ if self._callback:
+ self._callback(data=self._as_dict())
+
+ def setdefaults(self, d):
+ d = wandb_helper.parse_config(d)
+ # strip out keys already configured
+ d = {k: v for k, v in d.items() if k not in self._items}
+ d = self._sanitize_dict(d)
+ self._items.update(d)
+ if self._callback:
+ self._callback(data=d)
+
+ def _get_user_id(self, user) -> int:
+ if user not in self._users:
+ self._users[user] = self._users_cnt
+ self._users_inv[self._users_cnt] = user
+ object.__setattr__(self, "_users_cnt", self._users_cnt + 1)
+
+ return self._users[user]
+
+ def update_locked(self, d, user=None, _allow_val_change=None):
+ """Shallow-update config with `d` and lock config updates on d's keys."""
+ num = self._get_user_id(user)
+
+ for k, v in d.items():
+ k, v = self._sanitize(k, v, allow_val_change=_allow_val_change)
+ self._locked[k] = num
+ self._items[k] = v
+
+ if self._callback:
+ self._callback(data=d)
+
+ def merge_locked(self, d, user=None, _allow_val_change=None):
+ """Recursively merge-update config with `d` and lock config updates on d's keys."""
+ num = self._get_user_id(user)
+ callback_d = {}
+
+ for k, v in d.items():
+ k, v = self._sanitize(k, v, allow_val_change=_allow_val_change)
+ self._locked[k] = num
+
+ if (
+ k in self._items
+ and isinstance(self._items[k], dict)
+ and isinstance(v, dict)
+ ):
+ self._items[k] = config_util.merge_dicts(self._items[k], v)
+ else:
+ self._items[k] = v
+
+ callback_d[k] = self._items[k]
+
+ if self._callback:
+ self._callback(data=callback_d)
+
+ def _load_defaults(self):
+ conf_dict = config_util.dict_from_config_file("config-defaults.yaml")
+ if conf_dict is not None:
+ self.update(conf_dict)
+
+ def _sanitize_dict(
+ self,
+ config_dict,
+ allow_val_change=None,
+ ignore_keys: Optional[set] = None,
+ ):
+ sanitized = {}
+ self._raise_value_error_on_nested_artifact(config_dict)
+ for k, v in config_dict.items():
+ if ignore_keys and k in ignore_keys:
+ continue
+ k, v = self._sanitize(k, v, allow_val_change)
+ sanitized[k] = v
+ return sanitized
+
+ def _sanitize(self, key, val, allow_val_change=None):
+ # TODO: enable WBValues in the config in the future
+ # refuse all WBValues which is all Media and Histograms
+ if isinstance(val, wandb.sdk.data_types.base_types.wb_value.WBValue):
+ raise ValueError("WBValue objects cannot be added to the run config")
+ # Let jupyter change config freely by default
+ if self._settings and self._settings._jupyter and allow_val_change is None:
+ allow_val_change = True
+ # We always normalize keys by stripping '-'
+ key = key.strip("-")
+ if _is_artifact_representation(val):
+ val = self._artifact_callback(key, val)
+ # if the user inserts an artifact into the config
+ if not isinstance(val, wandb.Artifact):
+ val = json_friendly_val(val)
+ if not allow_val_change:
+ if key in self._items and val != self._items[key]:
+ raise config_util.ConfigError(
+ f'Attempted to change value of key "{key}" '
+ f"from {self._items[key]} to {val}\n"
+ "If you really want to do this, pass"
+ " allow_val_change=True to config.update()"
+ )
+ return key, val
+
+ def _raise_value_error_on_nested_artifact(self, v, nested=False):
+ # we can't swap nested artifacts because their root key can be locked by other values
+ # best if we don't allow nested artifacts until we can lock nested keys in the config
+ if isinstance(v, dict) and check_dict_contains_nested_artifact(v, nested):
+ raise ValueError(
+ "Instances of wandb.Artifact can only be top level keys in wandb.config"
+ )
+
+
+class ConfigStatic:
+ def __init__(self, config):
+ object.__setattr__(self, "__dict__", dict(config))
+
+ def __setattr__(self, name, value):
+ raise AttributeError("Error: wandb.run.config_static is a readonly object")
+
+ def __setitem__(self, key, val):
+ raise AttributeError("Error: wandb.run.config_static is a readonly object")
+
+ def keys(self):
+ return self.__dict__.keys()
+
+ def __getitem__(self, key):
+ return self.__dict__[key]
+
+ def __str__(self):
+ return str(self.__dict__)
diff --git a/wandb/sdk/wandb_helper.py b/wandb/sdk/wandb_helper.py
new file mode 100644
index 0000000000000000000000000000000000000000..5c5e5250f71b2b96db0e1300653ba1f94d62d682
--- /dev/null
+++ b/wandb/sdk/wandb_helper.py
@@ -0,0 +1,54 @@
+import inspect
+import types
+
+from wandb.errors import UsageError
+
+from .lib import config_util
+
+
+def parse_config(params, exclude=None, include=None):
+ if exclude and include:
+ raise UsageError("Expected at most only one of exclude or include")
+ if isinstance(params, str):
+ params = config_util.dict_from_config_file(params, must_exist=True)
+ params = _to_dict(params)
+ if include:
+ params = {key: value for key, value in params.items() if key in include}
+ if exclude:
+ params = {key: value for key, value in params.items() if key not in exclude}
+ return params
+
+
+def _to_dict(params):
+ if isinstance(params, dict):
+ return params
+
+ # Handle some cases where params is not a dictionary
+ # by trying to convert it into a dictionary
+ meta = inspect.getmodule(params)
+ if meta:
+ is_tf_flags_module = (
+ isinstance(params, types.ModuleType)
+ and meta.__name__ == "tensorflow.python.platform.flags"
+ )
+ if is_tf_flags_module or meta.__name__ == "absl.flags":
+ params = params.FLAGS
+ meta = inspect.getmodule(params)
+
+ # newer tensorflow flags (post 1.4) uses absl.flags
+ if meta and meta.__name__ == "absl.flags._flagvalues":
+ params = {name: params[name].value for name in dir(params)}
+ elif not hasattr(params, "__dict__"):
+ raise TypeError("config must be a dict or have a __dict__ attribute.")
+ elif "__flags" in vars(params):
+ # for older tensorflow flags (pre 1.4)
+ if not "__parsed" not in vars(params):
+ params._parse_flags()
+ params = vars(params)["__flags"]
+ else:
+ # params is a Namespace object (argparse)
+ # or something else
+ params = vars(params)
+
+ # assume argparse Namespace
+ return params
diff --git a/wandb/sdk/wandb_init.py b/wandb/sdk/wandb_init.py
new file mode 100644
index 0000000000000000000000000000000000000000..2a55a443e96fa3933263c305b573e8584783465f
--- /dev/null
+++ b/wandb/sdk/wandb_init.py
@@ -0,0 +1,1215 @@
+"""Defines wandb.init() and associated classes and methods.
+
+`wandb.init()` indicates the beginning of a new run. In an ML training pipeline,
+you could add `wandb.init()` to the beginning of your training script as well as
+your evaluation script, and each step would be tracked as a run in W&B.
+
+For more on using `wandb.init()`, including code snippets, check out our
+[guide and FAQs](https://docs.wandb.ai/guides/track/launch).
+"""
+import copy
+import json
+import logging
+import os
+import platform
+import sys
+import tempfile
+import traceback
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union
+
+import wandb
+import wandb.env
+from wandb import trigger
+from wandb.errors import CommError, Error, UsageError
+from wandb.errors.util import ProtobufErrorHandler
+from wandb.integration import sagemaker
+from wandb.integration.magic import magic_install
+from wandb.sdk.lib import runid
+from wandb.sdk.lib.paths import StrPath
+from wandb.util import _is_artifact_representation
+
+from . import wandb_login, wandb_setup
+from .backend.backend import Backend
+from .lib import (
+ RunDisabled,
+ SummaryDisabled,
+ filesystem,
+ ipython,
+ module,
+ reporting,
+ telemetry,
+)
+from .lib.deprecate import Deprecated, deprecate
+from .lib.mailbox import Mailbox, MailboxProgress
+from .lib.printer import Printer, get_printer
+from .lib.wburls import wburls
+from .wandb_helper import parse_config
+from .wandb_run import Run, TeardownHook, TeardownStage
+from .wandb_settings import Settings, Source
+
+if TYPE_CHECKING:
+ from wandb.proto import wandb_internal_pb2 as pb
+
+logger: Optional[logging.Logger] = None # logger configured during wandb.init()
+
+
+def _set_logger(log_object: logging.Logger) -> None:
+ """Configure module logger."""
+ global logger
+ logger = log_object
+
+
+def _huggingface_version() -> Optional[str]:
+ if "transformers" in sys.modules:
+ trans = wandb.util.get_module("transformers")
+ if hasattr(trans, "__version__"):
+ return str(trans.__version__)
+ return None
+
+
+def _maybe_mp_process(backend: Backend) -> bool:
+ parent_process = getattr(
+ backend._multiprocessing, "parent_process", None
+ ) # New in version 3.8.
+ if parent_process:
+ return parent_process() is not None
+ process = backend._multiprocessing.current_process()
+ if process.name == "MainProcess":
+ return False
+ if process.name.startswith("Process-"):
+ return True
+ return False
+
+
+def _handle_launch_config(settings: "Settings") -> Dict[str, Any]:
+ launch_run_config: Dict[str, Any] = {}
+ if not settings.launch:
+ return launch_run_config
+ if os.environ.get("WANDB_CONFIG") is not None:
+ try:
+ launch_run_config = json.loads(os.environ.get("WANDB_CONFIG", "{}"))
+ except (ValueError, SyntaxError):
+ wandb.termwarn("Malformed WANDB_CONFIG, using original config")
+ elif settings.launch_config_path and os.path.exists(settings.launch_config_path):
+ with open(settings.launch_config_path) as fp:
+ launch_config = json.loads(fp.read())
+ launch_run_config = launch_config.get("overrides", {}).get("run_config")
+ else:
+ i = 0
+ chunks = []
+ while True:
+ key = f"WANDB_CONFIG_{i}"
+ if key in os.environ:
+ chunks.append(os.environ[key])
+ i += 1
+ else:
+ break
+ if len(chunks) > 0:
+ config_string = "".join(chunks)
+ try:
+ launch_run_config = json.loads(config_string)
+ except (ValueError, SyntaxError):
+ wandb.termwarn("Malformed WANDB_CONFIG, using original config")
+
+ return launch_run_config
+
+
+class _WandbInit:
+ _init_telemetry_obj: telemetry.TelemetryRecord
+
+ def __init__(self) -> None:
+ self.kwargs = None
+ self.settings: Optional[Settings] = None
+ self.sweep_config: Dict[str, Any] = {}
+ self.launch_config: Dict[str, Any] = {}
+ self.config: Dict[str, Any] = {}
+ self.run: Optional[Run] = None
+ self.backend: Optional[Backend] = None
+
+ self._teardown_hooks: List[TeardownHook] = []
+ self._wl: Optional[wandb_setup._WandbSetup] = None
+ self._reporter: Optional[wandb.sdk.lib.reporting.Reporter] = None
+ self.notebook: Optional[wandb.jupyter.Notebook] = None # type: ignore
+ self.printer: Optional[Printer] = None
+
+ self._init_telemetry_obj = telemetry.TelemetryRecord()
+
+ self.deprecated_features_used: Dict[str, str] = dict()
+
+ def _setup_printer(self, settings: Settings) -> None:
+ if self.printer:
+ return
+ self.printer = get_printer(settings._jupyter)
+
+ def setup(self, kwargs: Any) -> None: # noqa: C901
+ """Complete setup for `wandb.init()`.
+
+ This includes parsing all arguments, applying them with settings and enabling logging.
+ """
+ self.kwargs = kwargs
+
+ # if the user ran, for example, `wandb.login(`) before `wandb.init()`,
+ # the singleton will already be set up and so if e.g. env vars are set
+ # in between, they will be ignored, which we need to inform the user about.
+ singleton = wandb_setup._WandbSetup._instance
+ if singleton is not None:
+ self._setup_printer(settings=singleton._settings)
+ assert self.printer
+ exclude_env_vars = {"WANDB_SERVICE", "WANDB_KUBEFLOW_URL"}
+ # check if environment variables have changed
+ singleton_env = {
+ k: v
+ for k, v in singleton._environ.items()
+ if k.startswith("WANDB_") and k not in exclude_env_vars
+ }
+ os_env = {
+ k: v
+ for k, v in os.environ.items()
+ if k.startswith("WANDB_") and k not in exclude_env_vars
+ }
+ if set(singleton_env.keys()) != set(os_env.keys()) or set(
+ singleton_env.values()
+ ) != set(os_env.values()):
+ line = (
+ "Changes to your `wandb` environment variables will be ignored "
+ "because your `wandb` session has already started. "
+ "For more information on how to modify your settings with "
+ "`wandb.init()` arguments, please refer to "
+ f"{self.printer.link(wburls.get('wandb_init'), 'the W&B docs')}."
+ )
+ self.printer.display(line, level="warn")
+
+ # we add this logic to be backward compatible with the old behavior of disable
+ # where it would disable the service if the mode was set to disabled
+ mode = kwargs.get("mode")
+ settings_mode = (kwargs.get("settings") or {}).get("mode")
+ _disable_service = mode == "disabled" or settings_mode == "disabled"
+ setup_settings = {"_disable_service": _disable_service}
+
+ self._wl = wandb_setup.setup(settings=setup_settings)
+ # Make sure we have a logger setup (might be an early logger)
+ assert self._wl is not None
+ _set_logger(self._wl._get_logger())
+
+ # Start with settings from wandb library singleton
+ settings: Settings = self._wl.settings.copy()
+
+ # when using launch, we don't want to reuse the same run id from the singleton
+ # since users might launch multiple runs in the same process
+ # TODO(kdg): allow users to control this via launch settings
+ if settings.launch and singleton is not None:
+ settings.update({"run_id": None}, source=Source.INIT)
+
+ settings_param = kwargs.pop("settings", None)
+ if settings_param is not None and isinstance(settings_param, (Settings, dict)):
+ settings.update(settings_param, source=Source.INIT)
+
+ self._setup_printer(settings)
+ self._reporter = reporting.setup_reporter(settings=settings)
+
+ sagemaker_config: Dict = (
+ dict() if settings.sagemaker_disable else sagemaker.parse_sm_config()
+ )
+ if sagemaker_config:
+ sagemaker_api_key = sagemaker_config.get("wandb_api_key", None)
+ sagemaker_run, sagemaker_env = sagemaker.parse_sm_resources()
+ if sagemaker_env:
+ if sagemaker_api_key:
+ sagemaker_env["WANDB_API_KEY"] = sagemaker_api_key
+ settings._apply_env_vars(sagemaker_env)
+ wandb.setup(settings=settings)
+ settings.update(sagemaker_run, source=Source.SETUP)
+ with telemetry.context(obj=self._init_telemetry_obj) as tel:
+ tel.feature.sagemaker = True
+
+ with telemetry.context(obj=self._init_telemetry_obj) as tel:
+ if kwargs.get("config"):
+ tel.feature.set_init_config = True
+ if kwargs.get("name"):
+ tel.feature.set_init_name = True
+ if kwargs.get("id"):
+ tel.feature.set_init_id = True
+ if kwargs.get("tags"):
+ tel.feature.set_init_tags = True
+
+ # Remove parameters that are not part of settings
+ init_config = kwargs.pop("config", None) or dict()
+
+ # todo: remove this once officially deprecated
+ deprecated_kwargs = {
+ "config_include_keys": (
+ "Use `config=wandb.helper.parse_config(config_object, include=('key',))` instead."
+ ),
+ "config_exclude_keys": (
+ "Use `config=wandb.helper.parse_config(config_object, exclude=('key',))` instead."
+ ),
+ }
+ for deprecated_kwarg, msg in deprecated_kwargs.items():
+ if kwargs.get(deprecated_kwarg):
+ self.deprecated_features_used[deprecated_kwarg] = msg
+
+ init_config = parse_config(
+ init_config,
+ include=kwargs.pop("config_include_keys", None),
+ exclude=kwargs.pop("config_exclude_keys", None),
+ )
+
+ # merge config with sweep or sagemaker (or config file)
+ self.sweep_config = dict()
+ sweep_config = self._wl._sweep_config or dict()
+ self.config = dict()
+ self.init_artifact_config: Dict[str, Any] = dict()
+ for config_data in (
+ sagemaker_config,
+ self._wl._config,
+ init_config,
+ ):
+ if not config_data:
+ continue
+ # split out artifacts, since when inserted into
+ # config they will trigger use_artifact
+ # but the run is not yet upserted
+ self._split_artifacts_from_config(config_data, self.config)
+
+ if sweep_config:
+ self._split_artifacts_from_config(sweep_config, self.sweep_config)
+
+ monitor_gym = kwargs.pop("monitor_gym", None)
+ if monitor_gym and len(wandb.patched["gym"]) == 0:
+ wandb.gym.monitor()
+
+ if wandb.patched["tensorboard"]:
+ with telemetry.context(obj=self._init_telemetry_obj) as tel:
+ tel.feature.tensorboard_patch = True
+
+ tensorboard = kwargs.pop("tensorboard", None)
+ sync_tensorboard = kwargs.pop("sync_tensorboard", None)
+ if tensorboard or sync_tensorboard and len(wandb.patched["tensorboard"]) == 0:
+ wandb.tensorboard.patch()
+ with telemetry.context(obj=self._init_telemetry_obj) as tel:
+ tel.feature.tensorboard_sync = True
+
+ magic = kwargs.get("magic")
+ if magic not in (None, False):
+ magic_install(kwargs)
+
+ # handle login related parameters as these are applied to global state
+ init_settings = {
+ key: kwargs[key]
+ for key in ["anonymous", "force", "mode", "resume"]
+ if kwargs.get(key) is not None
+ }
+ if init_settings:
+ settings.update(init_settings, source=Source.INIT)
+
+ if not settings._offline and not settings._noop:
+ wandb_login._login(
+ anonymous=kwargs.pop("anonymous", None),
+ force=kwargs.pop("force", None),
+ _disable_warning=True,
+ _silent=settings.quiet or settings.silent,
+ _entity=kwargs.get("entity") or settings.entity,
+ )
+
+ # apply updated global state after login was handled
+ wl = wandb.setup()
+ assert wl is not None
+ settings._apply_settings(wl.settings)
+
+ # get status of code saving before applying user settings
+ save_code_pre_user_settings = settings.save_code
+
+ settings._apply_init(kwargs)
+ if not settings._offline and not settings._noop:
+ user_settings = self._wl._load_user_settings()
+ settings._apply_user(user_settings)
+
+ # ensure that user settings don't set saving to true
+ # if user explicitly set these to false in UI
+ if save_code_pre_user_settings is False:
+ settings.update({"save_code": False}, source=Source.INIT)
+
+ # TODO(jhr): should this be moved? probably.
+ settings._set_run_start_time(source=Source.INIT)
+
+ if not settings._noop:
+ self._log_setup(settings)
+
+ if settings._jupyter:
+ self._jupyter_setup(settings)
+ launch_config = _handle_launch_config(settings)
+ if launch_config:
+ self._split_artifacts_from_config(launch_config, self.launch_config)
+
+ self.settings = settings
+
+ # self.settings.freeze()
+
+ def teardown(self) -> None:
+ # TODO: currently this is only called on failed wandb.init attempts
+ # normally this happens on the run object
+ assert logger
+ logger.info("tearing down wandb.init")
+ for hook in self._teardown_hooks:
+ hook.call()
+
+ def _split_artifacts_from_config(
+ self, config_source: dict, config_target: dict
+ ) -> None:
+ for k, v in config_source.items():
+ if _is_artifact_representation(v):
+ self.init_artifact_config[k] = v
+ else:
+ config_target.setdefault(k, v)
+
+ def _enable_logging(self, log_fname: str, run_id: Optional[str] = None) -> None:
+ """Enable logging to the global debug log.
+
+ This adds a run_id to the log, in case of multiple processes on the same machine.
+ Currently, there is no way to disable logging after it's enabled.
+ """
+ handler = logging.FileHandler(log_fname)
+ handler.setLevel(logging.INFO)
+
+ class WBFilter(logging.Filter):
+ def filter(self, record: logging.LogRecord) -> bool:
+ record.run_id = run_id
+ return True
+
+ if run_id:
+ formatter = logging.Formatter(
+ "%(asctime)s %(levelname)-7s %(threadName)-10s:%(process)d "
+ "[%(run_id)s:%(filename)s:%(funcName)s():%(lineno)s] %(message)s"
+ )
+ else:
+ formatter = logging.Formatter(
+ "%(asctime)s %(levelname)-7s %(threadName)-10s:%(process)d "
+ "[%(filename)s:%(funcName)s():%(lineno)s] %(message)s"
+ )
+
+ handler.setFormatter(formatter)
+ if run_id:
+ handler.addFilter(WBFilter())
+ assert logger is not None
+ logger.propagate = False
+ logger.addHandler(handler)
+ # TODO: make me configurable
+ logger.setLevel(logging.DEBUG)
+ self._teardown_hooks.append(
+ TeardownHook(
+ lambda: (handler.close(), logger.removeHandler(handler)), # type: ignore
+ TeardownStage.LATE,
+ )
+ )
+
+ def _safe_symlink(
+ self, base: str, target: str, name: str, delete: bool = False
+ ) -> None:
+ # TODO(jhr): do this with relpaths, but i cant figure it out on no sleep
+ if not hasattr(os, "symlink"):
+ return
+
+ pid = os.getpid()
+ tmp_name = os.path.join(base, "%s.%d" % (name, pid))
+
+ if delete:
+ try:
+ os.remove(os.path.join(base, name))
+ except OSError:
+ pass
+ target = os.path.relpath(target, base)
+ try:
+ os.symlink(target, tmp_name)
+ os.rename(tmp_name, os.path.join(base, name))
+ except OSError:
+ pass
+
+ def _pause_backend(self, *args: Any, **kwargs: Any) -> None: # noqa
+ if self.backend is None:
+ return None
+
+ # Attempt to save the code on every execution
+ if self.notebook.save_ipynb(): # type: ignore
+ assert self.run is not None
+ res = self.run.log_code(root=None)
+ logger.info("saved code: %s", res) # type: ignore
+ if self.backend.interface is not None:
+ logger.info("pausing backend") # type: ignore
+ self.backend.interface.publish_pause()
+
+ def _resume_backend(self, *args: Any, **kwargs: Any) -> None: # noqa
+ if self.backend is not None and self.backend.interface is not None:
+ logger.info("resuming backend") # type: ignore
+ self.backend.interface.publish_resume()
+
+ def _jupyter_teardown(self) -> None:
+ """Teardown hooks and display saving, called with wandb.finish."""
+ assert self.notebook
+ ipython = self.notebook.shell
+ self.notebook.save_history()
+ if self.notebook.save_ipynb():
+ assert self.run is not None
+ res = self.run.log_code(root=None)
+ logger.info("saved code and history: %s", res) # type: ignore
+ logger.info("cleaning up jupyter logic") # type: ignore
+ # because of how we bind our methods we manually find them to unregister
+ for hook in ipython.events.callbacks["pre_run_cell"]:
+ if "_resume_backend" in hook.__name__:
+ ipython.events.unregister("pre_run_cell", hook)
+ for hook in ipython.events.callbacks["post_run_cell"]:
+ if "_pause_backend" in hook.__name__:
+ ipython.events.unregister("post_run_cell", hook)
+ ipython.display_pub.publish = ipython.display_pub._orig_publish
+ del ipython.display_pub._orig_publish
+
+ def _jupyter_setup(self, settings: Settings) -> None:
+ """Add hooks, and session history saving."""
+ self.notebook = wandb.jupyter.Notebook(settings)
+ ipython = self.notebook.shell
+
+ # Monkey patch ipython publish to capture displayed outputs
+ if not hasattr(ipython.display_pub, "_orig_publish"):
+ logger.info("configuring jupyter hooks %s", self) # type: ignore
+ ipython.display_pub._orig_publish = ipython.display_pub.publish
+ # Registering resume and pause hooks
+
+ ipython.events.register("pre_run_cell", self._resume_backend)
+ ipython.events.register("post_run_cell", self._pause_backend)
+ self._teardown_hooks.append(
+ TeardownHook(self._jupyter_teardown, TeardownStage.EARLY)
+ )
+
+ def publish(data, metadata=None, **kwargs) -> None: # type: ignore
+ ipython.display_pub._orig_publish(data, metadata=metadata, **kwargs)
+ assert self.notebook is not None
+ self.notebook.save_display(
+ ipython.execution_count, {"data": data, "metadata": metadata}
+ )
+
+ ipython.display_pub.publish = publish
+
+ def _log_setup(self, settings: Settings) -> None:
+ """Set up logging from settings."""
+ filesystem.mkdir_exists_ok(os.path.dirname(settings.log_user))
+ filesystem.mkdir_exists_ok(os.path.dirname(settings.log_internal))
+ filesystem.mkdir_exists_ok(os.path.dirname(settings.sync_file))
+ filesystem.mkdir_exists_ok(settings.files_dir)
+ filesystem.mkdir_exists_ok(settings._tmp_code_dir)
+
+ if settings.symlink:
+ self._safe_symlink(
+ os.path.dirname(settings.sync_symlink_latest),
+ os.path.dirname(settings.sync_file),
+ os.path.basename(settings.sync_symlink_latest),
+ delete=True,
+ )
+ self._safe_symlink(
+ os.path.dirname(settings.log_symlink_user),
+ settings.log_user,
+ os.path.basename(settings.log_symlink_user),
+ delete=True,
+ )
+ self._safe_symlink(
+ os.path.dirname(settings.log_symlink_internal),
+ settings.log_internal,
+ os.path.basename(settings.log_symlink_internal),
+ delete=True,
+ )
+
+ _set_logger(logging.getLogger("wandb"))
+ self._enable_logging(settings.log_user)
+
+ assert self._wl
+ assert logger
+
+ self._wl._early_logger_flush(logger)
+ logger.info(f"Logging user logs to {settings.log_user}")
+ logger.info(f"Logging internal logs to {settings.log_internal}")
+
+ def _make_run_disabled(self) -> RunDisabled:
+ drun = RunDisabled()
+ drun.config = wandb.wandb_sdk.wandb_config.Config()
+ drun.config.update(self.sweep_config)
+ drun.config.update(self.config)
+ drun.summary = SummaryDisabled()
+ drun.log = lambda data, *_, **__: drun.summary.update(data)
+ drun.finish = lambda *_, **__: module.unset_globals()
+ drun.step = 0
+ drun.resumed = False
+ drun.disabled = True
+ drun.id = runid.generate_id()
+ drun.name = "dummy-" + drun.id
+ drun.dir = tempfile.gettempdir()
+ module.set_global(
+ run=drun,
+ config=drun.config,
+ log=drun.log,
+ summary=drun.summary,
+ save=drun.save,
+ use_artifact=drun.use_artifact,
+ log_artifact=drun.log_artifact,
+ define_metric=drun.define_metric,
+ plot_table=drun.plot_table,
+ alert=drun.alert,
+ )
+ return drun
+
+ def _on_progress_init(self, handle: MailboxProgress) -> None:
+ assert self.printer
+ line = "Waiting for wandb.init()...\r"
+ percent_done = handle.percent_done
+ self.printer.progress_update(line, percent_done=percent_done)
+
+ def init(self) -> Union[Run, RunDisabled, None]: # noqa: C901
+ if logger is None:
+ raise RuntimeError("Logger not initialized")
+ logger.info("calling init triggers")
+ trigger.call("on_init", **self.kwargs) # type: ignore
+
+ assert self.settings is not None
+ assert self._wl is not None
+ assert self._reporter is not None
+
+ logger.info(
+ f"wandb.init called with sweep_config: {self.sweep_config}\nconfig: {self.config}"
+ )
+
+ if self.settings._noop:
+ return self._make_run_disabled()
+ if self.settings.reinit or (
+ self.settings._jupyter and self.settings.reinit is not False
+ ):
+ if len(self._wl._global_run_stack) > 0:
+ if len(self._wl._global_run_stack) > 1:
+ wandb.termwarn(
+ "If you want to track multiple runs concurrently in wandb, "
+ "you should use multi-processing not threads"
+ )
+
+ latest_run = self._wl._global_run_stack[-1]
+
+ logger.info(
+ f"re-initializing run, found existing run on stack: {latest_run._run_id}"
+ )
+
+ jupyter = self.settings._jupyter
+ if jupyter and not self.settings.silent:
+ ipython.display_html(
+ f"Finishing last run (ID:{latest_run._run_id}) before initializing another..."
+ )
+
+ latest_run.finish()
+
+ if jupyter and not self.settings.silent:
+ ipython.display_html(
+ f"Successfully finished last run (ID:{latest_run._run_id}). Initializing new run:<br/>"
+ )
+ elif isinstance(wandb.run, Run):
+ manager = self._wl._get_manager()
+ # We shouldn't return a stale global run if we are in a new pid
+ if not manager or os.getpid() == wandb.run._init_pid:
+ logger.info("wandb.init() called when a run is still active")
+ with telemetry.context() as tel:
+ tel.feature.init_return_run = True
+ return wandb.run
+
+ logger.info("starting backend")
+
+ manager = self._wl._get_manager()
+ if manager:
+ logger.info("setting up manager")
+ manager._inform_init(
+ settings=self.settings.to_proto(), run_id=self.settings.run_id
+ )
+
+ mailbox = Mailbox()
+ backend = Backend(settings=self.settings, manager=manager, mailbox=mailbox)
+ backend.ensure_launched()
+ logger.info("backend started and connected")
+ # Make sure we are logged in
+ # wandb_login._login(_backend=backend, _settings=self.settings)
+
+ # resuming needs access to the server, check server_status()?
+ run = Run(
+ config=self.config,
+ settings=self.settings,
+ sweep_config=self.sweep_config,
+ launch_config=self.launch_config,
+ )
+
+ # Populate initial telemetry
+ with telemetry.context(run=run, obj=self._init_telemetry_obj) as tel:
+ tel.cli_version = wandb.__version__
+ tel.python_version = platform.python_version()
+ tel.platform = f"{platform.system()}-{platform.machine()}".lower()
+ hf_version = _huggingface_version()
+ if hf_version:
+ tel.huggingface_version = hf_version
+ if self.settings._jupyter:
+ tel.env.jupyter = True
+ if self.settings._ipython:
+ tel.env.ipython = True
+ if self.settings._colab:
+ tel.env.colab = True
+ if self.settings._kaggle:
+ tel.env.kaggle = True
+ if self.settings._windows:
+ tel.env.windows = True
+
+ if self.settings.launch:
+ tel.feature.launch = True
+
+ if self.settings._async_upload_concurrency_limit:
+ tel.feature.async_uploads = True
+
+ for module_name in telemetry.list_telemetry_imports(only_imported=True):
+ setattr(tel.imports_init, module_name, True)
+
+ # probe the active start method
+ active_start_method: Optional[str] = None
+ if self.settings.start_method == "thread":
+ active_start_method = self.settings.start_method
+ else:
+ active_start_method = getattr(
+ backend._multiprocessing, "get_start_method", lambda: None
+ )()
+
+ if active_start_method == "spawn":
+ tel.env.start_spawn = True
+ elif active_start_method == "fork":
+ tel.env.start_fork = True
+ elif active_start_method == "forkserver":
+ tel.env.start_forkserver = True
+ elif active_start_method == "thread":
+ tel.env.start_thread = True
+
+ if os.environ.get("PEX"):
+ tel.env.pex = True
+
+ if self.settings._aws_lambda:
+ tel.env.aws_lambda = True
+
+ if os.environ.get(wandb.env._DISABLE_SERVICE):
+ tel.feature.service_disabled = True
+
+ if manager:
+ tel.feature.service = True
+ if self.settings._flow_control_disabled:
+ tel.feature.flow_control_disabled = True
+ if self.settings._flow_control_custom:
+ tel.feature.flow_control_custom = True
+ if self.settings._require_core:
+ tel.feature.core = True
+
+ tel.env.maybe_mp = _maybe_mp_process(backend)
+
+ if not self.settings.label_disable:
+ if self.notebook:
+ run._label_probe_notebook(self.notebook)
+ else:
+ run._label_probe_main()
+
+ for deprecated_feature, msg in self.deprecated_features_used.items():
+ warning_message = f"`{deprecated_feature}` is deprecated. {msg}"
+ deprecate(
+ field_name=getattr(Deprecated, "init__" + deprecated_feature),
+ warning_message=warning_message,
+ run=run,
+ )
+
+ logger.info("updated telemetry")
+
+ run._set_library(self._wl)
+ run._set_backend(backend)
+ run._set_reporter(self._reporter)
+ run._set_teardown_hooks(self._teardown_hooks)
+
+ backend._hack_set_run(run)
+ assert backend.interface
+ mailbox.enable_keepalive()
+ backend.interface.publish_header()
+
+ # Using GitRepo() blocks & can be slow, depending on user's current git setup.
+ # We don't want to block run initialization/start request, so populate run's git
+ # info beforehand.
+ if not self.settings.disable_git:
+ run._populate_git_info()
+
+ run_result: Optional[pb.RunUpdateResult] = None
+
+ if self.settings._offline:
+ with telemetry.context(run=run) as tel:
+ tel.feature.offline = True
+
+ if self.settings.resume:
+ wandb.termwarn(
+ "`resume` will be ignored since W&B syncing is set to `offline`. "
+ f"Starting a new run with run id {run.id}."
+ )
+ error: Optional[wandb.errors.Error] = None
+
+ timeout = self.settings.init_timeout
+
+ logger.info(f"communicating run to backend with {timeout} second timeout")
+
+ run_init_handle = backend.interface.deliver_run(run)
+ result = run_init_handle.wait(
+ timeout=timeout,
+ on_progress=self._on_progress_init,
+ cancel=True,
+ )
+ if result:
+ run_result = result.run_result
+
+ if run_result is None:
+ error_message = (
+ f"Run initialization has timed out after {timeout} sec. "
+ f"\nPlease refer to the documentation for additional information: {wburls.get('doc_start_err')}"
+ )
+ # We're not certain whether the error we encountered is due to an issue
+ # with the server (a "CommError") or if it's a problem within the SDK (an "Error").
+ # This means that the error could be a result of the server being unresponsive,
+ # or it could be because we were unable to communicate with the wandb service.
+ error = CommError(error_message)
+ run_init_handle._cancel()
+ elif run_result.HasField("error"):
+ error = ProtobufErrorHandler.to_exception(run_result.error)
+
+ if error is not None:
+ logger.error(f"encountered error: {error}")
+ if not manager:
+ # Shutdown the backend and get rid of the logger
+ # we don't need to do console cleanup at this point
+ backend.cleanup()
+ self.teardown()
+ raise error
+
+ assert run_result is not None # for mypy
+
+ if not run_result.HasField("run"):
+ raise Error(
+ "It appears that something have gone wrong during the program execution as an unexpected missing field was encountered. "
+ "(run_result is missing the 'run' field)"
+ )
+
+ if run_result.run.resumed:
+ logger.info("run resumed")
+ with telemetry.context(run=run) as tel:
+ tel.feature.resumed = run_result.run.resumed
+
+ run._set_run_obj(run_result.run)
+
+ run._on_init()
+
+ logger.info("starting run threads in backend")
+ # initiate run (stats and metadata probing)
+
+ if manager:
+ manager._inform_start(
+ settings=self.settings.to_proto(), run_id=self.settings.run_id
+ )
+
+ assert backend.interface
+ assert run._run_obj
+
+ run_start_handle = backend.interface.deliver_run_start(run._run_obj)
+ # TODO: add progress to let user know we are doing something
+ run_start_result = run_start_handle.wait(timeout=30)
+ if run_start_result is None:
+ run_start_handle.abandon()
+
+ assert self._wl is not None
+ self._wl._global_run_stack.append(run)
+ self.run = run
+
+ run._handle_launch_artifact_overrides()
+ if (
+ self.settings.launch
+ and self.settings.launch_config_path
+ and os.path.exists(self.settings.launch_config_path)
+ ):
+ run._save(self.settings.launch_config_path)
+ # put artifacts in run config here
+ # since doing so earlier will cause an error
+ # as the run is not upserted
+ for k, v in self.init_artifact_config.items():
+ run.config.update({k: v}, allow_val_change=True)
+ job_artifact = run._launch_artifact_mapping.get(
+ wandb.util.LAUNCH_JOB_ARTIFACT_SLOT_NAME
+ )
+ if job_artifact:
+ run.use_artifact(job_artifact)
+
+ self.backend = backend
+ assert self._reporter
+ self._reporter.set_context(run=run)
+ run._on_start()
+ logger.info("run started, returning control to user process")
+ return run
+
+
+def getcaller() -> None:
+ if not logger:
+ return None
+ src, line, func, stack = logger.findCaller(stack_info=True)
+ print("Problem at:", src, line, func)
+
+
+def _attach(
+ attach_id: Optional[str] = None,
+ run_id: Optional[str] = None,
+ *,
+ run: Optional["Run"] = None,
+) -> Union[Run, RunDisabled, None]:
+ """Attach to a run currently executing in another process/thread.
+
+ Arguments:
+ attach_id: (str, optional) The id of the run or an attach identifier
+ that maps to a run.
+ run_id: (str, optional) The id of the run to attach to.
+ run: (Run, optional) The run instance to attach
+ """
+ attach_id = attach_id or run_id
+ if not ((attach_id is None) ^ (run is None)):
+ raise UsageError("Either (`attach_id` or `run_id`) or `run` must be specified")
+
+ attach_id = attach_id or (run._attach_id if run else None)
+
+ if attach_id is None:
+ raise UsageError(
+ "Either `attach_id` or `run_id` must be specified or `run` must have `_attach_id`"
+ )
+ wandb._assert_is_user_process()
+
+ _wl = wandb_setup._setup()
+ assert _wl
+
+ _set_logger(_wl._get_logger())
+ if logger is None:
+ raise UsageError("logger is not initialized")
+
+ manager = _wl._get_manager()
+ response = manager._inform_attach(attach_id=attach_id) if manager else None
+ if response is None:
+ raise UsageError(f"Unable to attach to run {attach_id}")
+
+ settings: Settings = copy.copy(_wl._settings)
+
+ settings.update(
+ {
+ "run_id": attach_id,
+ "_start_time": response._start_time.value,
+ "_start_datetime": response._start_datetime.value,
+ "_offline": response._offline.value,
+ },
+ source=Source.INIT,
+ )
+
+ # TODO: consolidate this codepath with wandb.init()
+ mailbox = Mailbox()
+ backend = Backend(settings=settings, manager=manager, mailbox=mailbox)
+ backend.ensure_launched()
+ logger.info("attach backend started and connected")
+
+ if run is None:
+ run = Run(settings=settings)
+ else:
+ run._init(settings=settings)
+ run._set_library(_wl)
+ run._set_backend(backend)
+ backend._hack_set_run(run)
+ assert backend.interface
+
+ mailbox.enable_keepalive()
+
+ attach_handle = backend.interface.deliver_attach(attach_id)
+ # TODO: add progress to let user know we are doing something
+ attach_result = attach_handle.wait(timeout=30)
+ if not attach_result:
+ attach_handle.abandon()
+ raise UsageError("Timeout attaching to run")
+ attach_response = attach_result.response.attach_response
+ if attach_response.error and attach_response.error.message:
+ raise UsageError(f"Failed to attach to run: {attach_response.error.message}")
+
+ run._set_run_obj(attach_response.run)
+ run._on_attach()
+ return run
+
+
+def init(
+ job_type: Optional[str] = None,
+ dir: Optional[StrPath] = None,
+ config: Union[Dict, str, None] = None,
+ project: Optional[str] = None,
+ entity: Optional[str] = None,
+ reinit: Optional[bool] = None,
+ tags: Optional[Sequence] = None,
+ group: Optional[str] = None,
+ name: Optional[str] = None,
+ notes: Optional[str] = None,
+ magic: Optional[Union[dict, str, bool]] = None,
+ config_exclude_keys: Optional[List[str]] = None,
+ config_include_keys: Optional[List[str]] = None,
+ anonymous: Optional[str] = None,
+ mode: Optional[str] = None,
+ allow_val_change: Optional[bool] = None,
+ resume: Optional[Union[bool, str]] = None,
+ force: Optional[bool] = None,
+ tensorboard: Optional[bool] = None, # alias for sync_tensorboard
+ sync_tensorboard: Optional[bool] = None,
+ monitor_gym: Optional[bool] = None,
+ save_code: Optional[bool] = None,
+ id: Optional[str] = None,
+ settings: Union[Settings, Dict[str, Any], None] = None,
+) -> Union[Run, RunDisabled, None]:
+ r"""Start a new run to track and log to W&B.
+
+ In an ML training pipeline, you could add `wandb.init()`
+ to the beginning of your training script as well as your evaluation
+ script, and each piece would be tracked as a run in W&B.
+
+ `wandb.init()` spawns a new background process to log data to a run, and it
+ also syncs data to wandb.ai by default, so you can see live visualizations.
+
+ Call `wandb.init()` to start a run before logging data with `wandb.log()`:
+ <!--yeadoc-test:init-method-log-->
+ ```python
+ import wandb
+
+ wandb.init()
+ # ... calculate metrics, generate media
+ wandb.log({"accuracy": 0.9})
+ ```
+
+ `wandb.init()` returns a run object, and you can also access the run object
+ via `wandb.run`:
+ <!--yeadoc-test:init-and-assert-global-->
+ ```python
+ import wandb
+
+ run = wandb.init()
+
+ assert run is wandb.run
+ ```
+
+ At the end of your script, we will automatically call `wandb.finish` to
+ finalize and cleanup the run. However, if you call `wandb.init` from a
+ child process, you must explicitly call `wandb.finish` at the end of the
+ child process.
+
+ For more on using `wandb.init()`, including detailed examples, check out our
+ [guide and FAQs](https://docs.wandb.ai/guides/track/launch).
+
+ Arguments:
+ project: (str, optional) The name of the project where you're sending
+ the new run. If the project is not specified, the run is put in an
+ "Uncategorized" project.
+ entity: (str, optional) An entity is a username or team name where
+ you're sending runs. This entity must exist before you can send runs
+ there, so make sure to create your account or team in the UI before
+ starting to log runs.
+ If you don't specify an entity, the run will be sent to your default
+ entity, which is usually your username. Change your default entity
+ in [your settings](https://wandb.ai/settings) under "default location
+ to create new projects".
+ config: (dict, argparse, absl.flags, str, optional)
+ This sets `wandb.config`, a dictionary-like object for saving inputs
+ to your job, like hyperparameters for a model or settings for a data
+ preprocessing job. The config will show up in a table in the UI that
+ you can use to group, filter, and sort runs. Keys should not contain
+ `.` in their names, and values should be under 10 MB.
+ If dict, argparse or absl.flags: will load the key value pairs into
+ the `wandb.config` object.
+ If str: will look for a yaml file by that name, and load config from
+ that file into the `wandb.config` object.
+ save_code: (bool, optional) Turn this on to save the main script or
+ notebook to W&B. This is valuable for improving experiment
+ reproducibility and to diff code across experiments in the UI. By
+ default this is off, but you can flip the default behavior to on
+ in [your settings page](https://wandb.ai/settings).
+ group: (str, optional) Specify a group to organize individual runs into
+ a larger experiment. For example, you might be doing cross
+ validation, or you might have multiple jobs that train and evaluate
+ a model against different test sets. Group gives you a way to
+ organize runs together into a larger whole, and you can toggle this
+ on and off in the UI. For more details, see our
+ [guide to grouping runs](https://docs.wandb.com/guides/runs/grouping).
+ job_type: (str, optional) Specify the type of run, which is useful when
+ you're grouping runs together into larger experiments using group.
+ For example, you might have multiple jobs in a group, with job types
+ like train and eval. Setting this makes it easy to filter and group
+ similar runs together in the UI so you can compare apples to apples.
+ tags: (list, optional) A list of strings, which will populate the list
+ of tags on this run in the UI. Tags are useful for organizing runs
+ together, or applying temporary labels like "baseline" or
+ "production". It's easy to add and remove tags in the UI, or filter
+ down to just runs with a specific tag.
+ If you are resuming a run, its tags will be overwritten by the tags
+ you pass to `wandb.init()`. If you want to add tags to a resumed run
+ without overwriting its existing tags, use `run.tags += ["new_tag"]`
+ after `wandb.init()`.
+ name: (str, optional) A short display name for this run, which is how
+ you'll identify this run in the UI. By default, we generate a random
+ two-word name that lets you easily cross-reference runs from the
+ table to charts. Keeping these run names short makes the chart
+ legends and tables easier to read. If you're looking for a place to
+ save your hyperparameters, we recommend saving those in config.
+ notes: (str, optional) A longer description of the run, like a `-m` commit
+ message in git. This helps you remember what you were doing when you
+ ran this run.
+ dir: (str or pathlib.Path, optional) An absolute path to a directory where
+ metadata will be stored. When you call `download()` on an artifact,
+ this is the directory where downloaded files will be saved. By default,
+ this is the `./wandb` directory.
+ resume: (bool, str, optional) Sets the resuming behavior. Options:
+ `"allow"`, `"must"`, `"never"`, `"auto"` or `None`. Defaults to `None`.
+ Cases:
+ - `None` (default): If the new run has the same ID as a previous run,
+ this run overwrites that data.
+ - `"auto"` (or `True`): if the previous run on this machine crashed,
+ automatically resume it. Otherwise, start a new run.
+ - `"allow"`: if id is set with `init(id="UNIQUE_ID")` or
+ `WANDB_RUN_ID="UNIQUE_ID"` and it is identical to a previous run,
+ wandb will automatically resume the run with that id. Otherwise,
+ wandb will start a new run.
+ - `"never"`: if id is set with `init(id="UNIQUE_ID")` or
+ `WANDB_RUN_ID="UNIQUE_ID"` and it is identical to a previous run,
+ wandb will crash.
+ - `"must"`: if id is set with `init(id="UNIQUE_ID")` or
+ `WANDB_RUN_ID="UNIQUE_ID"` and it is identical to a previous run,
+ wandb will automatically resume the run with the id. Otherwise,
+ wandb will crash.
+ See [our guide to resuming runs](https://docs.wandb.com/guides/runs/resuming)
+ for more.
+ reinit: (bool, optional) Allow multiple `wandb.init()` calls in the same
+ process. (default: `False`)
+ magic: (bool, dict, or str, optional) The bool controls whether we try to
+ auto-instrument your script, capturing basic details of your run
+ without you having to add more wandb code. (default: `False`)
+ You can also pass a dict, json string, or yaml filename.
+ config_exclude_keys: (list, optional) string keys to exclude from
+ `wandb.config`.
+ config_include_keys: (list, optional) string keys to include in
+ `wandb.config`.
+ anonymous: (str, optional) Controls anonymous data logging. Options:
+ - `"never"` (default): requires you to link your W&B account before
+ tracking the run, so you don't accidentally create an anonymous
+ run.
+ - `"allow"`: lets a logged-in user track runs with their account, but
+ lets someone who is running the script without a W&B account see
+ the charts in the UI.
+ - `"must"`: sends the run to an anonymous account instead of to a
+ signed-up user account.
+ mode: (str, optional) Can be `"online"`, `"offline"` or `"disabled"`. Defaults to
+ online.
+ allow_val_change: (bool, optional) Whether to allow config values to
+ change after setting the keys once. By default, we throw an exception
+ if a config value is overwritten. If you want to track something
+ like a varying learning rate at multiple times during training, use
+ `wandb.log()` instead. (default: `False` in scripts, `True` in Jupyter)
+ force: (bool, optional) If `True`, this crashes the script if a user isn't
+ logged in to W&B. If `False`, this will let the script run in offline
+ mode if a user isn't logged in to W&B. (default: `False`)
+ sync_tensorboard: (bool, optional) Synchronize wandb logs from tensorboard or
+ tensorboardX and save the relevant events file. (default: `False`)
+ monitor_gym: (bool, optional) Automatically log videos of environment when
+ using OpenAI Gym. (default: `False`)
+ See [our guide to this integration](https://docs.wandb.com/guides/integrations/openai-gym).
+ id: (str, optional) A unique ID for this run, used for resuming. It must
+ be unique in the project, and if you delete a run you can't reuse
+ the ID. Use the `name` field for a short descriptive name, or `config`
+ for saving hyperparameters to compare across runs. The ID cannot
+ contain the following special characters: `/\#?%:`.
+ See [our guide to resuming runs](https://docs.wandb.com/guides/runs/resuming).
+
+ Examples:
+ ### Set where the run is logged
+
+ You can change where the run is logged, just like changing
+ the organization, repository, and branch in git:
+ ```python
+ import wandb
+
+ user = "geoff"
+ project = "capsules"
+ display_name = "experiment-2021-10-31"
+
+ wandb.init(entity=user, project=project, name=display_name)
+ ```
+
+ ### Add metadata about the run to the config
+
+ Pass a dictionary-style object as the `config` keyword argument to add
+ metadata, like hyperparameters, to your run.
+ <!--yeadoc-test:init-set-config-->
+ ```python
+ import wandb
+
+ config = {"lr": 3e-4, "batch_size": 32}
+ config.update({"architecture": "resnet", "depth": 34})
+ wandb.init(config=config)
+ ```
+
+ Raises:
+ Error: if some unknown or internal error happened during the run initialization.
+ AuthenticationError: if the user failed to provide valid credentials.
+ CommError: if there was a problem communicating with the WandB server.
+ UsageError: if the user provided invalid arguments.
+ KeyboardInterrupt: if user interrupts the run.
+
+ Returns:
+ A `Run` object.
+ """
+ wandb._assert_is_user_process()
+
+ kwargs = dict(locals())
+ error_seen = None
+ except_exit = None
+ run: Optional[Union[Run, RunDisabled]] = None
+ try:
+ wi = _WandbInit()
+ wi.setup(kwargs)
+ assert wi.settings
+ except_exit = wi.settings._except_exit
+ try:
+ run = wi.init()
+ except_exit = wi.settings._except_exit
+ except (KeyboardInterrupt, Exception) as e:
+ if not isinstance(e, KeyboardInterrupt):
+ wandb._sentry.exception(e)
+ if not (
+ wandb.wandb_agent._is_running() and isinstance(e, KeyboardInterrupt)
+ ):
+ getcaller()
+ assert logger
+ if wi.settings.problem == "fatal":
+ raise
+ if wi.settings.problem == "warn":
+ pass
+ # TODO(jhr): figure out how to make this RunDummy
+ run = None
+ except Error as e:
+ if logger is not None:
+ logger.exception(str(e))
+ raise e
+ except KeyboardInterrupt as e:
+ assert logger
+ logger.warning("interrupted", exc_info=e)
+ raise e
+ except Exception as e:
+ error_seen = e
+ traceback.print_exc()
+ assert logger
+ logger.error("error", exc_info=e)
+ # Need to build delay into this sentry capture because our exit hooks
+ # mess with sentry's ability to send out errors before the program ends.
+ wandb._sentry.exception(e)
+ # reraise(*sys.exc_info())
+ finally:
+ if error_seen:
+ if except_exit:
+ wandb.termerror("Abnormal program exit")
+ os._exit(1)
+ raise Error("An unexpected error occurred") from error_seen
+ return run
diff --git a/wandb/sdk/wandb_login.py b/wandb/sdk/wandb_login.py
new file mode 100644
index 0000000000000000000000000000000000000000..01b32bde82307dd8d6a00f9dec77621d3ba3eb82
--- /dev/null
+++ b/wandb/sdk/wandb_login.py
@@ -0,0 +1,322 @@
+"""Log in to Weights & Biases.
+
+This authenticates your machine to log data to your account.
+"""
+
+import enum
+import os
+import sys
+from typing import Dict, Optional, Tuple
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
+
+import click
+
+import wandb
+from wandb.errors import AuthenticationError, UsageError
+from wandb.old.settings import Settings as OldSettings
+
+from ..apis import InternalApi
+from .internal.internal_api import Api
+from .lib import apikey
+from .wandb_settings import Settings, Source
+
+
+def _handle_host_wandb_setting(host: Optional[str], cloud: bool = False) -> None:
+ """Write the host parameter to the global settings file.
+
+ This takes the parameter from wandb.login or wandb login for use by the
+ application's APIs.
+ """
+ _api = InternalApi()
+ if host == "https://api.wandb.ai" or (host is None and cloud):
+ _api.clear_setting("base_url", globally=True, persist=True)
+ # To avoid writing an empty local settings file, we only clear if it exists
+ if os.path.exists(OldSettings._local_path()):
+ _api.clear_setting("base_url", persist=True)
+ elif host:
+ host = host.rstrip("/")
+ # force relogin if host is specified
+ _api.set_setting("base_url", host, globally=True, persist=True)
+
+
+def login(
+ anonymous: Optional[Literal["must", "allow", "never"]] = None,
+ key: Optional[str] = None,
+ relogin: Optional[bool] = None,
+ host: Optional[str] = None,
+ force: Optional[bool] = None,
+ timeout: Optional[int] = None,
+ verify: bool = False,
+) -> bool:
+ """Set up W&B login credentials.
+
+ By default, this will only store the credentials locally without
+ verifying them with the W&B server. To verify credentials, pass
+ verify=True.
+
+ Arguments:
+ anonymous: (string, optional) Can be "must", "allow", or "never".
+ If set to "must" we'll always log in anonymously, if set to
+ "allow" we'll only create an anonymous user if the user
+ isn't already logged in.
+ key: (string, optional) authentication key.
+ relogin: (bool, optional) If true, will re-prompt for API key.
+ host: (string, optional) The host to connect to.
+ force: (bool, optional) If true, will force a relogin.
+ timeout: (int, optional) Number of seconds to wait for user input.
+ verify: (bool) Verify the credentials with the W&B server.
+
+ Returns:
+ bool: if key is configured
+
+ Raises:
+ AuthenticationError - if api_key fails verification with the server
+ UsageError - if api_key cannot be configured and no tty
+ """
+ _handle_host_wandb_setting(host)
+ if wandb.setup()._settings._noop:
+ return True
+ kwargs = dict(locals())
+ _verify = kwargs.pop("verify", False)
+ configured = _login(**kwargs)
+
+ if _verify:
+ from . import wandb_setup
+
+ singleton = wandb_setup._WandbSetup._instance
+ assert singleton is not None
+ viewer = singleton._server._viewer
+ if not viewer:
+ raise AuthenticationError(
+ "API key verification failed. Make sure your API key is valid."
+ )
+ return True if configured else False
+
+
+class ApiKeyStatus(enum.Enum):
+ VALID = 1
+ NOTTY = 2
+ OFFLINE = 3
+ DISABLED = 4
+
+
+class _WandbLogin:
+ def __init__(self):
+ self.kwargs: Optional[Dict] = None
+ self._settings: Optional[Settings] = None
+ self._backend = None
+ self._silent = None
+ self._entity = None
+ self._wl = None
+ self._key = None
+ self._relogin = None
+
+ def setup(self, kwargs):
+ self.kwargs = kwargs
+
+ # built up login settings
+ login_settings: Settings = wandb.Settings()
+ settings_param = kwargs.pop("_settings", None)
+ # note that this case does not come up anywhere except for the tests
+ if settings_param is not None:
+ if isinstance(settings_param, Settings):
+ login_settings._apply_settings(settings_param)
+ elif isinstance(settings_param, dict):
+ login_settings.update(settings_param, source=Source.LOGIN)
+ _logger = wandb.setup()._get_logger()
+ # Do not save relogin into settings as we just want to relogin once
+ self._relogin = kwargs.pop("relogin", None)
+ login_settings._apply_login(kwargs, _logger=_logger)
+
+ # make sure they are applied globally
+ self._wl = wandb.setup(settings=login_settings)
+ self._settings = self._wl.settings
+
+ def is_apikey_configured(self):
+ return apikey.api_key(settings=self._settings) is not None
+
+ def set_backend(self, backend):
+ self._backend = backend
+
+ def set_silent(self, silent: bool):
+ self._silent = silent
+
+ def set_entity(self, entity: str):
+ self._entity = entity
+
+ def login(self):
+ apikey_configured = self.is_apikey_configured()
+ if self._settings.relogin or self._relogin:
+ apikey_configured = False
+ if not apikey_configured:
+ return False
+
+ if not self._silent:
+ self.login_display()
+
+ return apikey_configured
+
+ def login_display(self):
+ username = self._wl._get_username()
+
+ if username:
+ # check to see if we got an entity from the setup call or from the user
+ entity = self._entity or self._wl._get_entity()
+
+ entity_str = ""
+ # check if entity exist, valid (is part of a certain team) and different from the username
+ if entity and entity in self._wl._get_teams() and entity != username:
+ entity_str = f" ({click.style(entity, fg='yellow')})"
+
+ login_state_str = f"Currently logged in as: {click.style(username, fg='yellow')}{entity_str}"
+ else:
+ login_state_str = "W&B API key is configured"
+
+ login_info_str = (
+ f"Use {click.style('`wandb login --relogin`', bold=True)} to force relogin"
+ )
+ wandb.termlog(
+ f"{login_state_str}. {login_info_str}",
+ repeat=False,
+ )
+
+ def configure_api_key(self, key):
+ if self._settings._notebook and not self._settings.silent:
+ wandb.termwarn(
+ "If you're specifying your api key in code, ensure this "
+ "code is not shared publicly.\nConsider setting the "
+ "WANDB_API_KEY environment variable, or running "
+ "`wandb login` from the command line."
+ )
+ apikey.write_key(self._settings, key)
+ self.update_session(key)
+ self._key = key
+
+ def update_session(
+ self, key: Optional[str], status: ApiKeyStatus = ApiKeyStatus.VALID
+ ) -> None:
+ _logger = wandb.setup()._get_logger()
+ login_settings = dict()
+ if status == ApiKeyStatus.OFFLINE:
+ login_settings = dict(mode="offline")
+ elif status == ApiKeyStatus.DISABLED:
+ login_settings = dict(mode="disabled")
+ elif key:
+ login_settings = dict(api_key=key)
+ self._wl._settings._apply_login(login_settings, _logger=_logger)
+ # Whenever the key changes, make sure to pull in user settings
+ # from server.
+ if not self._wl.settings._offline:
+ self._wl._update_user_settings()
+
+ def _prompt_api_key(self) -> Tuple[Optional[str], ApiKeyStatus]:
+ api = Api(self._settings)
+ while True:
+ try:
+ key = apikey.prompt_api_key(
+ self._settings,
+ api=api,
+ no_offline=self._settings.force if self._settings else None,
+ no_create=self._settings.force if self._settings else None,
+ )
+ except ValueError as e:
+ # invalid key provided, try again
+ wandb.termerror(e.args[0])
+ continue
+ except TimeoutError:
+ wandb.termlog("W&B disabled due to login timeout.")
+ return None, ApiKeyStatus.DISABLED
+ if key is False:
+ return None, ApiKeyStatus.NOTTY
+ if not key:
+ return None, ApiKeyStatus.OFFLINE
+ return key, ApiKeyStatus.VALID
+
+ def prompt_api_key(self):
+ key, status = self._prompt_api_key()
+ if status == ApiKeyStatus.NOTTY:
+ directive = (
+ "wandb login [your_api_key]"
+ if self._settings._cli_only_mode
+ else "wandb.login(key=[your_api_key])"
+ )
+ raise UsageError("api_key not configured (no-tty). call " + directive)
+
+ self.update_session(key, status=status)
+ self._key = key
+
+ def propogate_login(self):
+ # TODO(jhr): figure out if this is really necessary
+ if self._backend:
+ # TODO: calling this twice is gross, this deserves a refactor
+ # Make sure our backend picks up the new creds
+ # _ = self._backend.interface.communicate_login(key, anonymous)
+ pass
+
+
+def _login(
+ anonymous: Optional[Literal["must", "allow", "never"]] = None,
+ key: Optional[str] = None,
+ relogin: Optional[bool] = None,
+ host: Optional[str] = None,
+ force: Optional[bool] = None,
+ timeout: Optional[int] = None,
+ _backend=None,
+ _silent: Optional[bool] = None,
+ _disable_warning: Optional[bool] = None,
+ _entity: Optional[str] = None,
+):
+ kwargs = dict(locals())
+ _disable_warning = kwargs.pop("_disable_warning", None)
+
+ if wandb.run is not None:
+ if not _disable_warning:
+ wandb.termwarn("Calling wandb.login() after wandb.init() has no effect.")
+ return True
+
+ wlogin = _WandbLogin()
+
+ _backend = kwargs.pop("_backend", None)
+ if _backend:
+ wlogin.set_backend(_backend)
+
+ _silent = kwargs.pop("_silent", None)
+ if _silent:
+ wlogin.set_silent(_silent)
+
+ _entity = kwargs.pop("_entity", None)
+ if _entity:
+ wlogin.set_entity(_entity)
+
+ # configure login object
+ wlogin.setup(kwargs)
+
+ if wlogin._settings._offline:
+ return False
+ elif wandb.util._is_kaggle() and not wandb.util._has_internet():
+ wandb.termerror(
+ "To use W&B in kaggle you must enable internet in the settings panel on the right."
+ )
+ return False
+
+ # perform a login
+ logged_in = wlogin.login()
+
+ key = kwargs.get("key")
+ if key:
+ wlogin.configure_api_key(key)
+
+ if logged_in:
+ return logged_in
+
+ if not key:
+ wlogin.prompt_api_key()
+
+ # make sure login credentials get to the backend
+ wlogin.propogate_login()
+
+ return wlogin._key or False
diff --git a/wandb/sdk/wandb_manager.py b/wandb/sdk/wandb_manager.py
new file mode 100644
index 0000000000000000000000000000000000000000..d9df513e4d1f59d517885625d350e6aab0d1c1b2
--- /dev/null
+++ b/wandb/sdk/wandb_manager.py
@@ -0,0 +1,222 @@
+"""Manage wandb processes.
+
+Create a manager channel.
+"""
+
+import atexit
+import os
+from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
+
+import psutil
+
+import wandb
+from wandb import env, trigger
+from wandb.errors import Error
+from wandb.sdk.lib.exit_hooks import ExitHooks
+from wandb.sdk.lib.import_hooks import unregister_all_post_import_hooks
+
+if TYPE_CHECKING:
+ from wandb.proto import wandb_settings_pb2
+ from wandb.sdk.service import service
+ from wandb.sdk.service.service_base import ServiceInterface
+ from wandb.sdk.wandb_settings import Settings
+
+
+class ManagerConnectionError(Error):
+ """Raised when service process is not running."""
+
+ pass
+
+
+class ManagerConnectionRefusedError(ManagerConnectionError):
+ """Raised when service process is not running."""
+
+ pass
+
+
+class _ManagerToken:
+ _version = "2"
+ _supported_transports = {"tcp"}
+ _token_str: str
+ _pid: int
+ _transport: str
+ _host: str
+ _port: int
+
+ def __init__(self, token: str) -> None:
+ self._token_str = token
+ self._parse()
+
+ @classmethod
+ def from_environment(cls) -> Optional["_ManagerToken"]:
+ token = os.environ.get(env.SERVICE)
+ if not token:
+ return None
+ return cls(token=token)
+
+ @classmethod
+ def from_params(cls, transport: str, host: str, port: int) -> "_ManagerToken":
+ version = cls._version
+ pid = os.getpid()
+ token = "-".join([version, str(pid), transport, host, str(port)])
+ return cls(token=token)
+
+ def set_environment(self) -> None:
+ os.environ[env.SERVICE] = self._token_str
+
+ def _parse(self) -> None:
+ assert self._token_str
+ parts = self._token_str.split("-")
+ assert len(parts) == 5, f"token must have 5 parts: {parts}"
+ # TODO: make more robust?
+ version, pid_str, transport, host, port_str = parts
+ assert version == self._version
+ assert transport in self._supported_transports
+ self._pid = int(pid_str)
+ self._transport = transport
+ self._host = host
+ self._port = int(port_str)
+
+ def reset_environment(self) -> None:
+ os.environ.pop(env.SERVICE, None)
+
+ @property
+ def token(self) -> str:
+ return self._token_str
+
+ @property
+ def pid(self) -> int:
+ return self._pid
+
+ @property
+ def transport(self) -> str:
+ return self._transport
+
+ @property
+ def host(self) -> str:
+ return self._host
+
+ @property
+ def port(self) -> int:
+ return self._port
+
+
+class _Manager:
+ _token: _ManagerToken
+ _atexit_lambda: Optional[Callable[[], None]]
+ _hooks: Optional[ExitHooks]
+ _settings: "Settings"
+ _service: "service._Service"
+
+ def _service_connect(self) -> None:
+ port = self._token.port
+ svc_iface = self._get_service_interface()
+
+ try:
+ svc_iface._svc_connect(port=port)
+ except ConnectionRefusedError as e:
+ if not psutil.pid_exists(self._token.pid):
+ message = (
+ "Connection to wandb service failed "
+ "since the process is not available. "
+ )
+ else:
+ message = f"Connection to wandb service failed: {e}. "
+ raise ManagerConnectionRefusedError(message)
+ except Exception as e:
+ raise ManagerConnectionError(f"Connection to wandb service failed: {e}")
+
+ def __init__(self, settings: "Settings") -> None:
+ from wandb.sdk.service import service
+
+ self._settings = settings
+ self._atexit_lambda = None
+ self._hooks = None
+
+ self._service = service._Service(settings=self._settings)
+ token = _ManagerToken.from_environment()
+ if not token:
+ self._service.start()
+ host = "localhost"
+ transport = "tcp"
+ port = self._service.sock_port
+ assert port
+ token = _ManagerToken.from_params(transport=transport, host=host, port=port)
+ token.set_environment()
+ self._atexit_setup()
+
+ self._token = token
+
+ try:
+ self._service_connect()
+ except ManagerConnectionError as e:
+ wandb._sentry.reraise(e)
+
+ def _atexit_setup(self) -> None:
+ self._atexit_lambda = lambda: self._atexit_teardown()
+
+ self._hooks = ExitHooks()
+ self._hooks.hook()
+ atexit.register(self._atexit_lambda)
+
+ def _atexit_teardown(self) -> None:
+ trigger.call("on_finished")
+ exit_code = self._hooks.exit_code if self._hooks else 0
+ self._teardown(exit_code)
+
+ def _teardown(self, exit_code: int) -> None:
+ unregister_all_post_import_hooks()
+
+ if self._atexit_lambda:
+ atexit.unregister(self._atexit_lambda)
+ self._atexit_lambda = None
+
+ try:
+ self._inform_teardown(exit_code)
+ result = self._service.join()
+ if result and not self._settings._notebook:
+ os._exit(result)
+ except Exception as e:
+ wandb.termlog(
+ f"While tearing down the service manager. The following error has occurred: {e}",
+ repeat=False,
+ )
+ finally:
+ self._token.reset_environment()
+
+ def _get_service(self) -> "service._Service":
+ return self._service
+
+ def _get_service_interface(self) -> "ServiceInterface":
+ assert self._service
+ svc_iface = self._service.service_interface
+ assert svc_iface
+ return svc_iface
+
+ def _inform_init(
+ self, settings: "wandb_settings_pb2.Settings", run_id: str
+ ) -> None:
+ svc_iface = self._get_service_interface()
+ svc_iface._svc_inform_init(settings=settings, run_id=run_id)
+
+ def _inform_start(
+ self, settings: "wandb_settings_pb2.Settings", run_id: str
+ ) -> None:
+ svc_iface = self._get_service_interface()
+ svc_iface._svc_inform_start(settings=settings, run_id=run_id)
+
+ def _inform_attach(self, attach_id: str) -> Optional[Dict[str, Any]]:
+ svc_iface = self._get_service_interface()
+ try:
+ response = svc_iface._svc_inform_attach(attach_id=attach_id)
+ except Exception:
+ return None
+ return response.settings
+
+ def _inform_finish(self, run_id: Optional[str] = None) -> None:
+ svc_iface = self._get_service_interface()
+ svc_iface._svc_inform_finish(run_id=run_id)
+
+ def _inform_teardown(self, exit_code: int) -> None:
+ svc_iface = self._get_service_interface()
+ svc_iface._svc_inform_teardown(exit_code)
diff --git a/wandb/sdk/wandb_metric.py b/wandb/sdk/wandb_metric.py
new file mode 100644
index 0000000000000000000000000000000000000000..f01e4b7a30bbbc070a2a00b174d7c2f9f4fa27e0
--- /dev/null
+++ b/wandb/sdk/wandb_metric.py
@@ -0,0 +1,110 @@
+"""metric."""
+
+import logging
+from typing import Callable, Optional, Sequence, Tuple
+
+from wandb.proto import wandb_internal_pb2 as pb
+
+logger = logging.getLogger("wandb")
+
+
+class Metric:
+ """Metric object."""
+
+ _callback: Optional[Callable[[pb.MetricRecord], None]]
+ _name: str
+ _step_metric: Optional[str]
+ _step_sync: Optional[bool]
+ _hidden: Optional[bool]
+ _summary: Optional[Sequence[str]]
+ _goal: Optional[str]
+ _overwrite: Optional[bool]
+
+ def __init__(
+ self,
+ name: str,
+ step_metric: Optional[str] = None,
+ step_sync: Optional[bool] = None,
+ hidden: Optional[bool] = None,
+ summary: Optional[Sequence[str]] = None,
+ goal: Optional[str] = None,
+ overwrite: Optional[bool] = None,
+ ) -> None:
+ self._callback = None
+ self._name = name
+ self._step_metric = step_metric
+ # default to step_sync=True if step metric is set
+ step_sync = step_sync if step_sync is not None else step_metric is not None
+ self._step_sync = step_sync
+ self._hidden = hidden
+ self._summary = summary
+ self._goal = goal
+ self._overwrite = overwrite
+
+ def _set_callback(self, cb: Callable[[pb.MetricRecord], None]) -> None:
+ self._callback = cb
+
+ @property
+ def name(self) -> str:
+ return self._name
+
+ @property
+ def step_metric(self) -> Optional[str]:
+ return self._step_metric
+
+ @property
+ def step_sync(self) -> Optional[bool]:
+ return self._step_sync
+
+ @property
+ def summary(self) -> Optional[Tuple[str, ...]]:
+ if self._summary is None:
+ return None
+ return tuple(self._summary)
+
+ @property
+ def hidden(self) -> Optional[bool]:
+ return self._hidden
+
+ @property
+ def goal(self) -> Optional[str]:
+ goal_dict = dict(min="minimize", max="maximize")
+ return goal_dict[self._goal] if self._goal else None
+
+ def _commit(self) -> None:
+ m = pb.MetricRecord()
+ m.options.defined = True
+ if self._name.endswith("*"):
+ m.glob_name = self._name
+ else:
+ m.name = self._name
+ if self._step_metric:
+ m.step_metric = self._step_metric
+ if self._step_sync:
+ m.options.step_sync = self._step_sync
+ if self._hidden:
+ m.options.hidden = self._hidden
+ if self._summary:
+ summary_set = set(self._summary)
+ if "min" in summary_set:
+ m.summary.min = True
+ if "max" in summary_set:
+ m.summary.max = True
+ if "mean" in summary_set:
+ m.summary.mean = True
+ if "last" in summary_set:
+ m.summary.last = True
+ if "copy" in summary_set:
+ m.summary.copy = True
+ if "none" in summary_set:
+ m.summary.none = True
+ if "best" in summary_set:
+ m.summary.best = True
+ if self._goal == "min":
+ m.goal = m.GOAL_MINIMIZE
+ if self._goal == "max":
+ m.goal = m.GOAL_MAXIMIZE
+ if self._overwrite:
+ m._control.overwrite = self._overwrite
+ if self._callback:
+ self._callback(m)
diff --git a/wandb/sdk/wandb_require.py b/wandb/sdk/wandb_require.py
new file mode 100644
index 0000000000000000000000000000000000000000..7b1f03cf214462948fc8a1e5586adc88a4a7bd0d
--- /dev/null
+++ b/wandb/sdk/wandb_require.py
@@ -0,0 +1,87 @@
+"""Feature Flags Module.
+
+This module implements a feature flag system for the wandb library to require experimental features
+and notify the user when features have been deprecated.
+
+Example:
+ import wandb
+ wandb.require("wandb-service@beta")
+ wandb.require("incremental-artifacts@beta")
+"""
+
+from typing import Optional, Sequence, Union
+
+import wandb
+from wandb.errors import UnsupportedError
+from wandb.sdk import wandb_run
+from wandb.sdk.lib.wburls import wburls
+
+
+class _Requires:
+ """Internal feature class."""
+
+ _features: Sequence[str]
+
+ def __init__(self, features: Union[str, Sequence[str]]) -> None:
+ self._features = (
+ tuple([features]) if isinstance(features, str) else tuple(features)
+ )
+
+ def require_require(self) -> None:
+ pass
+
+ def _require_service(self) -> None:
+ wandb.teardown = wandb._teardown # type: ignore
+ wandb.attach = wandb._attach # type: ignore
+ wandb_run.Run.detach = wandb_run.Run._detach # type: ignore
+
+ def require_service(self) -> None:
+ self._require_service()
+
+ def apply(self) -> None:
+ """Call require_* method for supported features."""
+ last_message: str = ""
+ for feature_item in self._features:
+ full_feature = feature_item.split("@", 2)[0]
+ feature = full_feature.split(":", 2)[0]
+ func_str = "require_{}".format(feature.replace("-", "_"))
+ func = getattr(self, func_str, None)
+ if not func:
+ last_message = f"require() unsupported requirement: {feature}"
+ wandb.termwarn(last_message)
+ continue
+ func()
+
+ if last_message:
+ wandb.termerror(
+ f"Supported wandb.require() features can be found at: {wburls.get('doc_require')}"
+ )
+ raise UnsupportedError(last_message)
+
+
+def require(
+ requirement: Optional[Union[str, Sequence[str]]] = None,
+ experiment: Optional[Union[str, Sequence[str]]] = None,
+) -> None:
+ """Indicate which experimental features are used by the script.
+
+ Args:
+ requirement: (str or list) Features to require
+ experiment: (str or list) Features to require
+
+ Raises:
+ wandb.errors.UnsupportedError: if not supported
+ """
+ features = requirement or experiment
+ if not features:
+ return
+
+ f = _Requires(features=features)
+ f.apply()
+
+
+def _import_module_hook() -> None:
+ """On wandb import, setup anything needed based on parent process require calls."""
+ # TODO: optimize by caching which pids this has been done for or use real import hooks
+ # TODO: make this more generic, but for now this works
+ require("service")
diff --git a/wandb/sdk/wandb_require_helpers.py b/wandb/sdk/wandb_require_helpers.py
new file mode 100644
index 0000000000000000000000000000000000000000..3f14583ac53efd1b2b49b76ea1a0a27e65e88627
--- /dev/null
+++ b/wandb/sdk/wandb_require_helpers.py
@@ -0,0 +1,44 @@
+import os
+from functools import wraps
+from typing import Any, Callable, Dict, TypeVar, cast
+
+FuncT = TypeVar("FuncT", bound=Callable[..., Any])
+
+requirement_env_var_mapping: Dict[str, str] = {
+ "report-editing:v0": "WANDB_REQUIRE_REPORT_EDITING_V0"
+}
+
+
+def requires(requirement: str) -> FuncT: # type: ignore
+ """Decorate functions to gate features with wandb.require."""
+ env_var = requirement_env_var_mapping[requirement]
+
+ def deco(func: FuncT) -> FuncT:
+ @wraps(func)
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
+ if not os.getenv(env_var):
+ raise Exception(
+ f"You need to enable this feature with `wandb.require({requirement!r})`"
+ )
+ return func(*args, **kwargs)
+
+ return cast(FuncT, wrapper)
+
+ return cast(FuncT, deco)
+
+
+class RequiresMixin:
+ requirement = ""
+
+ def __init__(self) -> None:
+ self._check_if_requirements_met()
+
+ def __post_init__(self) -> None:
+ self._check_if_requirements_met()
+
+ def _check_if_requirements_met(self) -> None:
+ env_var = requirement_env_var_mapping[self.requirement]
+ if not os.getenv(env_var):
+ raise Exception(
+ f'You must explicitly enable this feature with `wandb.require("{self.requirement})"'
+ )
diff --git a/wandb/sdk/wandb_run.py b/wandb/sdk/wandb_run.py
new file mode 100644
index 0000000000000000000000000000000000000000..72a3a63f250ff759002d1c1421e75a5db74b5114
--- /dev/null
+++ b/wandb/sdk/wandb_run.py
@@ -0,0 +1,4119 @@
+import _thread as thread
+import atexit
+import functools
+import glob
+import json
+import logging
+import numbers
+import os
+import re
+import sys
+import threading
+import time
+import traceback
+import warnings
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+from enum import IntEnum
+from types import TracebackType
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Callable,
+ Dict,
+ List,
+ NamedTuple,
+ Optional,
+ Sequence,
+ TextIO,
+ Tuple,
+ Type,
+ Union,
+)
+
+import requests
+
+import wandb
+import wandb.env
+from wandb import errors, trigger
+from wandb._globals import _datatypes_set_callback
+from wandb.apis import internal, public
+from wandb.apis.internal import Api
+from wandb.apis.public import Api as PublicApi
+from wandb.proto.wandb_internal_pb2 import (
+ JobInfoResponse,
+ MetricRecord,
+ PollExitResponse,
+ Result,
+ RunRecord,
+ ServerInfoResponse,
+)
+from wandb.sdk.artifacts.artifact import Artifact
+from wandb.sdk.internal import job_builder
+from wandb.sdk.lib.import_hooks import (
+ register_post_import_hook,
+ unregister_post_import_hook,
+)
+from wandb.sdk.lib.paths import FilePathStr, LogicalPath, StrPath
+from wandb.util import (
+ _is_artifact_object,
+ _is_artifact_string,
+ _is_artifact_version_weave_dict,
+ _is_py_or_dockerfile,
+ _resolve_aliases,
+ add_import_hook,
+ parse_artifact_string,
+)
+from wandb.viz import CustomChart, Visualize, custom_chart
+
+from . import wandb_config, wandb_metric, wandb_summary
+from .data_types._dtypes import TypeRegistry
+from .interface.interface import GlobStr, InterfaceBase
+from .interface.summary_record import SummaryRecord
+from .lib import (
+ config_util,
+ deprecate,
+ filenames,
+ filesystem,
+ ipython,
+ module,
+ proto_util,
+ redirect,
+ telemetry,
+)
+from .lib.exit_hooks import ExitHooks
+from .lib.gitlib import GitRepo
+from .lib.mailbox import MailboxError, MailboxHandle, MailboxProbe, MailboxProgress
+from .lib.printer import get_printer
+from .lib.proto_util import message_to_dict
+from .lib.reporting import Reporter
+from .lib.wburls import wburls
+from .wandb_settings import Settings
+from .wandb_setup import _WandbSetup
+
+if TYPE_CHECKING:
+ if sys.version_info >= (3, 8):
+ from typing import TypedDict
+ else:
+ from typing_extensions import TypedDict
+
+ import wandb.apis.public
+ import wandb.sdk.backend.backend
+ import wandb.sdk.interface.interface_queue
+ from wandb.proto.wandb_internal_pb2 import (
+ CheckVersionResponse,
+ GetSummaryResponse,
+ InternalMessagesResponse,
+ SampledHistoryResponse,
+ )
+
+ from .interface.interface import FilesDict, PolicyName
+ from .lib.printer import PrinterJupyter, PrinterTerm
+ from .wandb_alerts import AlertLevel
+
+ class GitSourceDict(TypedDict):
+ remote: str
+ commit: str
+ entrypoint: List[str]
+ args: Sequence[str]
+
+ class ArtifactSourceDict(TypedDict):
+ artifact: str
+ entrypoint: List[str]
+ args: Sequence[str]
+
+ class ImageSourceDict(TypedDict):
+ image: str
+ args: Sequence[str]
+
+ class JobSourceDict(TypedDict, total=False):
+ _version: str
+ source_type: str
+ source: Union[GitSourceDict, ArtifactSourceDict, ImageSourceDict]
+ input_types: Dict[str, Any]
+ output_types: Dict[str, Any]
+ runtime: Optional[str]
+
+
+logger = logging.getLogger("wandb")
+EXIT_TIMEOUT = 60
+RE_LABEL = re.compile(r"[a-zA-Z0-9_-]+$")
+
+
+class TeardownStage(IntEnum):
+ EARLY = 1
+ LATE = 2
+
+
+class TeardownHook(NamedTuple):
+ call: Callable[[], None]
+ stage: TeardownStage
+
+
+class RunStatusChecker:
+ """Periodically polls the background process for relevant updates.
+
+ - check if the user has requested a stop.
+ - check the network status.
+ - check the run sync status.
+ """
+
+ _stop_status_lock: threading.Lock
+ _stop_status_handle: Optional[MailboxHandle]
+ _network_status_lock: threading.Lock
+ _network_status_handle: Optional[MailboxHandle]
+ _internal_messages_lock: threading.Lock
+ _internal_messages_handle: Optional[MailboxHandle]
+
+ def __init__(
+ self,
+ interface: InterfaceBase,
+ stop_polling_interval: int = 15,
+ retry_polling_interval: int = 5,
+ ) -> None:
+ self._interface = interface
+ self._stop_polling_interval = stop_polling_interval
+ self._retry_polling_interval = retry_polling_interval
+
+ self._join_event = threading.Event()
+
+ self._stop_status_lock = threading.Lock()
+ self._stop_status_handle = None
+ self._stop_thread = threading.Thread(
+ target=self.check_stop_status,
+ name="ChkStopThr",
+ daemon=True,
+ )
+
+ self._network_status_lock = threading.Lock()
+ self._network_status_handle = None
+ self._network_status_thread = threading.Thread(
+ target=self.check_network_status,
+ name="NetStatThr",
+ daemon=True,
+ )
+
+ self._internal_messages_lock = threading.Lock()
+ self._internal_messages_handle = None
+ self._internal_messages_thread = threading.Thread(
+ target=self.check_internal_messages,
+ name="IntMsgThr",
+ daemon=True,
+ )
+
+ def start(self) -> None:
+ self._stop_thread.start()
+ self._network_status_thread.start()
+ self._internal_messages_thread.start()
+
+ def _loop_check_status(
+ self,
+ *,
+ lock: threading.Lock,
+ set_handle: Any,
+ timeout: int,
+ request: Any,
+ process: Any,
+ ) -> None:
+ local_handle: Optional[MailboxHandle] = None
+ join_requested = False
+ while not join_requested:
+ time_probe = time.monotonic()
+ if not local_handle:
+ local_handle = request()
+ assert local_handle
+
+ with lock:
+ if self._join_event.is_set():
+ return
+ set_handle(local_handle)
+ try:
+ result = local_handle.wait(timeout=timeout)
+ except MailboxError:
+ # background threads are oportunistically getting results
+ # from the internal process but the internal process could
+ # be shutdown at any time. In this case assume that the
+ # thread should exit silently. This is possible
+ # because we do not have an atexit handler for the user
+ # process which quiesces active threads.
+ break
+ with lock:
+ set_handle(None)
+
+ if result:
+ process(result)
+ # if request finished, clear the handle to send on the next interval
+ local_handle = None
+
+ time_elapsed = time.monotonic() - time_probe
+ wait_time = max(self._stop_polling_interval - time_elapsed, 0)
+ join_requested = self._join_event.wait(timeout=wait_time)
+
+ def check_network_status(self) -> None:
+ def _process_network_status(result: Result) -> None:
+ network_status = result.response.network_status_response
+ for hr in network_status.network_responses:
+ if (
+ hr.http_status_code == 200 or hr.http_status_code == 0
+ ): # we use 0 for non-http errors (eg wandb errors)
+ wandb.termlog(f"{hr.http_response_text}")
+ else:
+ wandb.termlog(
+ "{} encountered ({}), retrying request".format(
+ hr.http_status_code, hr.http_response_text.rstrip()
+ )
+ )
+
+ self._loop_check_status(
+ lock=self._network_status_lock,
+ set_handle=lambda x: setattr(self, "_network_status_handle", x),
+ timeout=self._retry_polling_interval,
+ request=self._interface.deliver_network_status,
+ process=_process_network_status,
+ )
+
+ def check_stop_status(self) -> None:
+ def _process_stop_status(result: Result) -> None:
+ stop_status = result.response.stop_status_response
+ if stop_status.run_should_stop:
+ # TODO(frz): This check is required
+ # until WB-3606 is resolved on server side.
+ if not wandb.agents.pyagent.is_running():
+ thread.interrupt_main()
+ return
+
+ self._loop_check_status(
+ lock=self._stop_status_lock,
+ set_handle=lambda x: setattr(self, "_stop_status_handle", x),
+ timeout=self._stop_polling_interval,
+ request=self._interface.deliver_stop_status,
+ process=_process_stop_status,
+ )
+
+ def check_internal_messages(self) -> None:
+ def _process_internal_messages(result: Result) -> None:
+ internal_messages = result.response.internal_messages_response
+ for msg in internal_messages.messages.warning:
+ wandb.termwarn(msg)
+
+ self._loop_check_status(
+ lock=self._internal_messages_lock,
+ set_handle=lambda x: setattr(self, "_internal_messages_handle", x),
+ timeout=1,
+ request=self._interface.deliver_internal_messages,
+ process=_process_internal_messages,
+ )
+
+ def stop(self) -> None:
+ self._join_event.set()
+ with self._stop_status_lock:
+ if self._stop_status_handle:
+ self._stop_status_handle.abandon()
+ with self._network_status_lock:
+ if self._network_status_handle:
+ self._network_status_handle.abandon()
+ with self._internal_messages_lock:
+ if self._internal_messages_handle:
+ self._internal_messages_handle.abandon()
+
+ def join(self) -> None:
+ self.stop()
+ self._stop_thread.join()
+ self._network_status_thread.join()
+ self._internal_messages_thread.join()
+
+
+class _run_decorator: # noqa: N801
+ _is_attaching: str = ""
+
+ class Dummy:
+ ...
+
+ @classmethod
+ def _attach(cls, func: Callable) -> Callable:
+ @functools.wraps(func)
+ def wrapper(self: Type["Run"], *args: Any, **kwargs: Any) -> Any:
+ # * `_attach_id` is only assigned in service hence for all non-service cases
+ # it will be a passthrough.
+ # * `_attach_pid` is only assigned in _init (using _attach_pid guarantees single attach):
+ # - for non-fork case the object is shared through pickling so will be None.
+ # - for fork case the new process share mem space hence the value would be of parent process.
+ if (
+ getattr(self, "_attach_id", None)
+ and getattr(self, "_attach_pid", None) != os.getpid()
+ ):
+ if cls._is_attaching:
+ message = (
+ f"Trying to attach `{func.__name__}` "
+ f"while in the middle of attaching `{cls._is_attaching}`"
+ )
+ raise RuntimeError(message)
+ cls._is_attaching = func.__name__
+ try:
+ wandb._attach(run=self) # type: ignore
+ except Exception as e:
+ # In case the attach fails we will raise the exception that caused the issue.
+ # This exception should be caught and fail the execution of the program.
+ cls._is_attaching = ""
+ raise e
+ cls._is_attaching = ""
+ return func(self, *args, **kwargs)
+
+ return wrapper
+
+ @classmethod
+ def _noop_on_finish(cls, message: str = "", only_warn: bool = False) -> Callable:
+ def decorator_fn(func: Callable) -> Callable:
+ @functools.wraps(func)
+ def wrapper_fn(self: Type["Run"], *args: Any, **kwargs: Any) -> Any:
+ if not getattr(self, "_is_finished", False):
+ return func(self, *args, **kwargs)
+
+ default_message = (
+ f"Run ({self.id}) is finished. The call to `{func.__name__}` will be ignored. "
+ f"Please make sure that you are using an active run."
+ )
+ resolved_message = message or default_message
+ if only_warn:
+ warnings.warn(resolved_message, UserWarning, stacklevel=2)
+ else:
+ raise errors.UsageError(resolved_message)
+
+ return wrapper_fn
+
+ return decorator_fn
+
+ @classmethod
+ def _noop(cls, func: Callable) -> Callable:
+ @functools.wraps(func)
+ def wrapper(self: Type["Run"], *args: Any, **kwargs: Any) -> Any:
+ # `_attach_id` is only assigned in service hence for all service cases
+ # it will be a passthrough. We don't pickle non-service so again a way
+ # to see that we are in non-service case
+ if getattr(self, "_attach_id", None) is None:
+ # `_init_pid` is only assigned in __init__ (this will be constant check for mp):
+ # - for non-fork case the object is shared through pickling,
+ # and we don't pickle non-service so will be None
+ # - for fork case the new process share mem space hence the value would be of parent process.
+ _init_pid = getattr(self, "_init_pid", None)
+ if _init_pid != os.getpid():
+ message = "`{}` ignored (called from pid={}, `init` called from pid={}). See: {}".format(
+ func.__name__,
+ os.getpid(),
+ _init_pid,
+ wburls.get("multiprocess"),
+ )
+ # - if this process was pickled in non-service case,
+ # we ignore the attributes (since pickle is not supported)
+ # - for fork case will use the settings of the parent process
+ # - only point of inconsistent behavior from forked and non-forked cases
+ settings = getattr(self, "_settings", None)
+ if settings and settings["strict"]:
+ wandb.termerror(message, repeat=False)
+ raise errors.UnsupportedError(
+ f"`{func.__name__}` does not support multiprocessing"
+ )
+ wandb.termwarn(message, repeat=False)
+ return cls.Dummy()
+
+ return func(self, *args, **kwargs)
+
+ return wrapper
+
+
+@dataclass
+class RunStatus:
+ sync_items_total: int = field(default=0)
+ sync_items_pending: int = field(default=0)
+ sync_time: Optional[datetime] = field(default=None)
+
+
+class Run:
+ """A unit of computation logged by wandb. Typically, this is an ML experiment.
+
+ Create a run with `wandb.init()`:
+ <!--yeadoc-test:run-object-basic-->
+ ```python
+ import wandb
+
+ run = wandb.init()
+ ```
+
+ There is only ever at most one active `wandb.Run` in any process,
+ and it is accessible as `wandb.run`:
+ <!--yeadoc-test:global-run-object-->
+ ```python
+ import wandb
+
+ assert wandb.run is None
+
+ wandb.init()
+
+ assert wandb.run is not None
+ ```
+ anything you log with `wandb.log` will be sent to that run.
+
+ If you want to start more runs in the same script or notebook, you'll need to
+ finish the run that is in-flight. Runs can be finished with `wandb.finish` or
+ by using them in a `with` block:
+ <!--yeadoc-test:run-context-manager-->
+ ```python
+ import wandb
+
+ wandb.init()
+ wandb.finish()
+
+ assert wandb.run is None
+
+ with wandb.init() as run:
+ pass # log data here
+
+ assert wandb.run is None
+ ```
+
+ See the documentation for `wandb.init` for more on creating runs, or check out
+ [our guide to `wandb.init`](https://docs.wandb.ai/guides/track/launch).
+
+ In distributed training, you can either create a single run in the rank 0 process
+ and then log information only from that process, or you can create a run in each process,
+ logging from each separately, and group the results together with the `group` argument
+ to `wandb.init`. For more details on distributed training with W&B, check out
+ [our guide](https://docs.wandb.ai/guides/track/log/distributed-training).
+
+ Currently, there is a parallel `Run` object in the `wandb.Api`. Eventually these
+ two objects will be merged.
+
+ Attributes:
+ summary: (Summary) Single values set for each `wandb.log()` key. By
+ default, summary is set to the last value logged. You can manually
+ set summary to the best value, like max accuracy, instead of the
+ final value.
+ """
+
+ _telemetry_obj: telemetry.TelemetryRecord
+ _telemetry_obj_active: bool
+ _telemetry_obj_dirty: bool
+ _telemetry_obj_flushed: bytes
+
+ _teardown_hooks: List[TeardownHook]
+ _tags: Optional[Tuple[Any, ...]]
+
+ _entity: Optional[str]
+ _project: Optional[str]
+ _group: Optional[str]
+ _job_type: Optional[str]
+ _name: Optional[str]
+ _notes: Optional[str]
+ _sweep_id: Optional[str]
+
+ _run_obj: Optional[RunRecord]
+ # Use string literal annotation because of type reference loop
+ _backend: Optional["wandb.sdk.backend.backend.Backend"]
+ _internal_run_interface: Optional[
+ "wandb.sdk.interface.interface_queue.InterfaceQueue"
+ ]
+ _wl: Optional[_WandbSetup]
+
+ _out_redir: Optional[redirect.RedirectBase]
+ _err_redir: Optional[redirect.RedirectBase]
+ _redirect_cb: Optional[Callable[[str, str], None]]
+ _redirect_raw_cb: Optional[Callable[[str, str], None]]
+ _output_writer: Optional["filesystem.CRDedupedFile"]
+ _quiet: Optional[bool]
+
+ _atexit_cleanup_called: bool
+ _hooks: Optional[ExitHooks]
+ _exit_code: Optional[int]
+
+ _run_status_checker: Optional[RunStatusChecker]
+
+ _check_version: Optional["CheckVersionResponse"]
+ _sampled_history: Optional["SampledHistoryResponse"]
+ _final_summary: Optional["GetSummaryResponse"]
+ _job_info: Optional["JobInfoResponse"]
+ _poll_exit_handle: Optional[MailboxHandle]
+ _poll_exit_response: Optional[PollExitResponse]
+ _server_info_response: Optional[ServerInfoResponse]
+ _internal_messages_response: Optional["InternalMessagesResponse"]
+
+ _stdout_slave_fd: Optional[int]
+ _stderr_slave_fd: Optional[int]
+ _artifact_slots: List[str]
+
+ _init_pid: int
+ _attach_pid: int
+ _iface_pid: Optional[int]
+ _iface_port: Optional[int]
+
+ _attach_id: Optional[str]
+ _is_attached: bool
+ _is_finished: bool
+ _settings: Settings
+
+ _launch_artifacts: Optional[Dict[str, Any]]
+ _printer: Union["PrinterTerm", "PrinterJupyter"]
+
+ def __init__(
+ self,
+ settings: Settings,
+ config: Optional[Dict[str, Any]] = None,
+ sweep_config: Optional[Dict[str, Any]] = None,
+ launch_config: Optional[Dict[str, Any]] = None,
+ ) -> None:
+ # pid is set, so we know if this run object was initialized by this process
+ self._init_pid = os.getpid()
+ self._init(
+ settings=settings,
+ config=config,
+ sweep_config=sweep_config,
+ launch_config=launch_config,
+ )
+
+ def _init(
+ self,
+ settings: Settings,
+ config: Optional[Dict[str, Any]] = None,
+ sweep_config: Optional[Dict[str, Any]] = None,
+ launch_config: Optional[Dict[str, Any]] = None,
+ ) -> None:
+ self._settings = settings
+ self._config = wandb_config.Config()
+ self._config._set_callback(self._config_callback)
+ self._config._set_artifact_callback(self._config_artifact_callback)
+ self._config._set_settings(self._settings)
+ self._backend = None
+ self._internal_run_interface = None
+ # todo: perhaps this should be a property that is a noop on a finished run
+ self.summary = wandb_summary.Summary(
+ self._summary_get_current_summary_callback,
+ )
+ self.summary._set_update_callback(self._summary_update_callback)
+ self._step = 0
+ self._torch_history: Optional[wandb.wandb_torch.TorchHistory] = None
+
+ # todo: eventually would be nice to make this configurable using self._settings._start_time
+ # need to test (jhr): if you set start time to 2 days ago and run a test for 15 minutes,
+ # does the total time get calculated right (not as 2 days and 15 minutes)?
+ self._start_time = time.time()
+
+ _datatypes_set_callback(self._datatypes_callback)
+
+ self._printer = get_printer(self._settings._jupyter)
+ self._wl = None
+ self._reporter: Optional[Reporter] = None
+
+ self._entity = None
+ self._project = None
+ self._group = None
+ self._job_type = None
+ self._run_id = self._settings.run_id
+ self._starting_step = 0
+ self._name = None
+ self._notes = None
+ self._tags = None
+ self._remote_url = None
+ self._commit = None
+ self._sweep_id = None
+
+ self._hooks = None
+ self._teardown_hooks = []
+ self._out_redir = None
+ self._err_redir = None
+ self._stdout_slave_fd = None
+ self._stderr_slave_fd = None
+ self._exit_code = None
+ self._exit_result = None
+ self._quiet = self._settings.quiet
+
+ self._output_writer = None
+ self._used_artifact_slots: Dict[str, str] = {}
+
+ # Returned from backend request_run(), set from wandb_init?
+ self._run_obj = None
+
+ # Created when the run "starts".
+ self._run_status_checker = None
+
+ self._check_version = None
+ self._sampled_history = None
+ self._final_summary = None
+ self._poll_exit_response = None
+ self._server_info_response = None
+ self._internal_messages_response = None
+ self._poll_exit_handle = None
+ self._job_info = None
+
+ # Initialize telemetry object
+ self._telemetry_obj = telemetry.TelemetryRecord()
+ self._telemetry_obj_active = False
+ self._telemetry_obj_flushed = b""
+ self._telemetry_obj_dirty = False
+
+ self._atexit_cleanup_called = False
+
+ # Pull info from settings
+ self._init_from_settings(self._settings)
+
+ # Initial scope setup for sentry.
+ # This might get updated when the actual run comes back.
+ wandb._sentry.configure_scope(
+ tags=dict(self._settings),
+ process_context="user",
+ )
+
+ # Populate config
+ config = config or dict()
+ wandb_key = "_wandb"
+ config.setdefault(wandb_key, dict())
+ self._launch_artifact_mapping: Dict[str, Any] = {}
+ self._unique_launch_artifact_sequence_names: Dict[str, Any] = {}
+ if self._settings.save_code and self._settings.program_relpath:
+ config[wandb_key]["code_path"] = LogicalPath(
+ os.path.join("code", self._settings.program_relpath)
+ )
+
+ self._config._update(config, ignore_locked=True)
+
+ if sweep_config:
+ self._config.merge_locked(
+ sweep_config, user="sweep", _allow_val_change=True
+ )
+
+ if launch_config:
+ self._config.merge_locked(
+ launch_config, user="launch", _allow_val_change=True
+ )
+
+ # if run is from a launch queue, add queue id to _wandb config
+ launch_queue_name = wandb.env.get_launch_queue_name()
+ if launch_queue_name:
+ self._config[wandb_key]["launch_queue_name"] = launch_queue_name
+
+ launch_queue_entity = wandb.env.get_launch_queue_entity()
+ if launch_queue_entity:
+ self._config[wandb_key]["launch_queue_entity"] = launch_queue_entity
+
+ launch_trace_id = wandb.env.get_launch_trace_id()
+ if launch_trace_id:
+ self._config[wandb_key]["launch_trace_id"] = launch_trace_id
+
+ # interface pid and port configured when backend is configured (See _hack_set_run)
+ # TODO: using pid isn't the best for windows as pid reuse can happen more often than unix
+ self._iface_pid = None
+ self._iface_port = None
+ self._attach_id = None
+ self._is_attached = False
+ self._is_finished = False
+
+ self._attach_pid = os.getpid()
+
+ # for now, use runid as attach id, this could/should be versioned in the future
+ if not self._settings._disable_service:
+ self._attach_id = self._settings.run_id
+
+ def _set_iface_pid(self, iface_pid: int) -> None:
+ self._iface_pid = iface_pid
+
+ def _set_iface_port(self, iface_port: int) -> None:
+ self._iface_port = iface_port
+
+ def _handle_launch_artifact_overrides(self) -> None:
+ if self._settings.launch and (os.environ.get("WANDB_ARTIFACTS") is not None):
+ try:
+ artifacts: Dict[str, Any] = json.loads(
+ os.environ.get("WANDB_ARTIFACTS", "{}")
+ )
+ except (ValueError, SyntaxError):
+ wandb.termwarn("Malformed WANDB_ARTIFACTS, using original artifacts")
+ else:
+ self._initialize_launch_artifact_maps(artifacts)
+
+ elif (
+ self._settings.launch
+ and self._settings.launch_config_path
+ and os.path.exists(self._settings.launch_config_path)
+ ):
+ self._save(self._settings.launch_config_path)
+ with open(self._settings.launch_config_path) as fp:
+ launch_config = json.loads(fp.read())
+ if launch_config.get("overrides", {}).get("artifacts") is not None:
+ artifacts = launch_config.get("overrides").get("artifacts")
+ self._initialize_launch_artifact_maps(artifacts)
+
+ def _initialize_launch_artifact_maps(self, artifacts: Dict[str, Any]) -> None:
+ for key, item in artifacts.items():
+ self._launch_artifact_mapping[key] = item
+ artifact_sequence_tuple_or_slot = key.split(":")
+
+ if len(artifact_sequence_tuple_or_slot) == 2:
+ sequence_name = artifact_sequence_tuple_or_slot[0].split("/")[-1]
+ if self._unique_launch_artifact_sequence_names.get(sequence_name):
+ self._unique_launch_artifact_sequence_names.pop(sequence_name)
+ else:
+ self._unique_launch_artifact_sequence_names[sequence_name] = item
+
+ def _telemetry_callback(self, telem_obj: telemetry.TelemetryRecord) -> None:
+ self._telemetry_obj.MergeFrom(telem_obj)
+ self._telemetry_obj_dirty = True
+ self._telemetry_flush()
+
+ def _telemetry_flush(self) -> None:
+ if not self._telemetry_obj_active:
+ return
+ if not self._telemetry_obj_dirty:
+ return
+ if self._backend and self._backend.interface:
+ serialized = self._telemetry_obj.SerializeToString()
+ if serialized == self._telemetry_obj_flushed:
+ return
+ self._backend.interface._publish_telemetry(self._telemetry_obj)
+ self._telemetry_obj_flushed = serialized
+ self._telemetry_obj_dirty = False
+
+ def _freeze(self) -> None:
+ self._frozen = True
+
+ def __setattr__(self, attr: str, value: object) -> None:
+ if getattr(self, "_frozen", None) and not hasattr(self, attr):
+ raise Exception(f"Attribute {attr} is not supported on Run object.")
+ super().__setattr__(attr, value)
+
+ def _update_settings(self, settings: Settings) -> None:
+ self._settings = settings
+ self._init_from_settings(settings)
+
+ def _init_from_settings(self, settings: Settings) -> None:
+ if settings.entity is not None:
+ self._entity = settings.entity
+ if settings.project is not None:
+ self._project = settings.project
+ if settings.run_group is not None:
+ self._group = settings.run_group
+ if settings.run_job_type is not None:
+ self._job_type = settings.run_job_type
+ if settings.run_name is not None:
+ self._name = settings.run_name
+ if settings.run_notes is not None:
+ self._notes = settings.run_notes
+ if settings.run_tags is not None:
+ self._tags = settings.run_tags
+ if settings.sweep_id is not None:
+ self._sweep_id = settings.sweep_id
+
+ def _make_proto_run(self, run: RunRecord) -> None:
+ """Populate protocol buffer RunData for interface/interface."""
+ if self._entity is not None:
+ run.entity = self._entity
+ if self._project is not None:
+ run.project = self._project
+ if self._group is not None:
+ run.run_group = self._group
+ if self._job_type is not None:
+ run.job_type = self._job_type
+ if self._run_id is not None:
+ run.run_id = self._run_id
+ if self._name is not None:
+ run.display_name = self._name
+ if self._notes is not None:
+ run.notes = self._notes
+ if self._tags is not None:
+ for tag in self._tags:
+ run.tags.append(tag)
+ if self._start_time is not None:
+ run.start_time.FromMicroseconds(int(self._start_time * 1e6))
+ if self._remote_url is not None:
+ run.git.remote_url = self._remote_url
+ if self._commit is not None:
+ run.git.commit = self._commit
+ if self._sweep_id is not None:
+ run.sweep_id = self._sweep_id
+ # Note: run.config is set in interface/interface:_make_run()
+
+ def _populate_git_info(self) -> None:
+ # Use user provided git info if available otherwise resolve it from the environment
+ try:
+ repo = GitRepo(
+ root=self._settings.git_root,
+ remote=self._settings.git_remote,
+ remote_url=self._settings.git_remote_url,
+ commit=self._settings.git_commit,
+ lazy=False,
+ )
+ self._remote_url, self._commit = repo.remote_url, repo.last_commit
+ except Exception:
+ wandb.termwarn("Cannot find valid git repo associated with this directory.")
+
+ def __deepcopy__(self, memo: Dict[int, Any]) -> "Run":
+ return self
+
+ def __getstate__(self) -> Any:
+ """Return run state as a custom pickle."""
+ # We only pickle in service mode
+ if not self._settings or self._settings._disable_service:
+ return
+
+ _attach_id = self._attach_id
+ if not _attach_id:
+ return
+
+ return dict(
+ _attach_id=_attach_id,
+ _init_pid=self._init_pid,
+ _is_finished=self._is_finished,
+ )
+
+ def __setstate__(self, state: Any) -> None:
+ """Set run state from a custom pickle."""
+ if not state:
+ return
+
+ _attach_id = state.get("_attach_id")
+ if not _attach_id:
+ return
+
+ if state["_init_pid"] == os.getpid():
+ raise RuntimeError("attach in the same process is not supported currently")
+
+ self.__dict__.update(state)
+
+ @property
+ def _torch(self) -> "wandb.wandb_torch.TorchHistory":
+ if self._torch_history is None:
+ self._torch_history = wandb.wandb_torch.TorchHistory()
+ return self._torch_history
+
+ @property
+ @_run_decorator._attach
+ def settings(self) -> Settings:
+ """A frozen copy of run's Settings object."""
+ cp = self._settings.copy()
+ cp.freeze()
+ return cp
+
+ @property
+ @_run_decorator._attach
+ def dir(self) -> str:
+ """The directory where files associated with the run are saved."""
+ return self._settings.files_dir
+
+ @property
+ @_run_decorator._attach
+ def config(self) -> wandb_config.Config:
+ """Config object associated with this run."""
+ return self._config
+
+ @property
+ @_run_decorator._attach
+ def config_static(self) -> wandb_config.ConfigStatic:
+ return wandb_config.ConfigStatic(self._config)
+
+ @property
+ @_run_decorator._attach
+ def name(self) -> Optional[str]:
+ """Display name of the run.
+
+ Display names are not guaranteed to be unique and may be descriptive.
+ By default, they are randomly generated.
+ """
+ if self._name:
+ return self._name
+ if not self._run_obj:
+ return None
+ return self._run_obj.display_name
+
+ @name.setter
+ @_run_decorator._noop_on_finish()
+ def name(self, name: str) -> None:
+ with telemetry.context(run=self) as tel:
+ tel.feature.set_run_name = True
+ self._name = name
+ if self._backend and self._backend.interface:
+ self._backend.interface.publish_run(self)
+
+ @property
+ @_run_decorator._attach
+ def notes(self) -> Optional[str]:
+ """Notes associated with the run, if there are any.
+
+ Notes can be a multiline string and can also use markdown and latex equations
+ inside `$$`, like `$x + 3$`.
+ """
+ if self._notes:
+ return self._notes
+ if not self._run_obj:
+ return None
+ return self._run_obj.notes
+
+ @notes.setter
+ @_run_decorator._noop_on_finish()
+ def notes(self, notes: str) -> None:
+ self._notes = notes
+ if self._backend and self._backend.interface:
+ self._backend.interface.publish_run(self)
+
+ @property
+ @_run_decorator._attach
+ def tags(self) -> Optional[Tuple]:
+ """Tags associated with the run, if there are any."""
+ if self._tags:
+ return self._tags
+ if self._run_obj:
+ return tuple(self._run_obj.tags)
+ return None
+
+ @tags.setter
+ @_run_decorator._noop_on_finish()
+ def tags(self, tags: Sequence) -> None:
+ with telemetry.context(run=self) as tel:
+ tel.feature.set_run_tags = True
+ self._tags = tuple(tags)
+ if self._backend and self._backend.interface:
+ self._backend.interface.publish_run(self)
+
+ @property
+ @_run_decorator._attach
+ def id(self) -> str:
+ """Identifier for this run."""
+ if TYPE_CHECKING:
+ assert self._run_id is not None
+ return self._run_id
+
+ @property
+ @_run_decorator._attach
+ def sweep_id(self) -> Optional[str]:
+ """ID of the sweep associated with the run, if there is one."""
+ if not self._run_obj:
+ return None
+ return self._run_obj.sweep_id or None
+
+ def _get_path(self) -> str:
+ parts = [
+ e for e in [self._entity, self._project, self._run_id] if e is not None
+ ]
+ return "/".join(parts)
+
+ @property
+ @_run_decorator._attach
+ def path(self) -> str:
+ """Path to the run.
+
+ Run paths include entity, project, and run ID, in the format
+ `entity/project/run_id`.
+ """
+ return self._get_path()
+
+ def _get_start_time(self) -> float:
+ return (
+ self._start_time
+ if not self._run_obj
+ else (self._run_obj.start_time.ToMicroseconds() / 1e6)
+ )
+
+ @property
+ @_run_decorator._attach
+ def start_time(self) -> float:
+ """Unix timestamp (in seconds) of when the run started."""
+ return self._get_start_time()
+
+ def _get_starting_step(self) -> int:
+ return self._starting_step if not self._run_obj else self._run_obj.starting_step
+
+ @property
+ @_run_decorator._attach
+ def starting_step(self) -> int:
+ """The first step of the run."""
+ return self._get_starting_step()
+
+ @property
+ @_run_decorator._attach
+ def resumed(self) -> bool:
+ """True if the run was resumed, False otherwise."""
+ return self._run_obj.resumed if self._run_obj else False
+
+ @property
+ @_run_decorator._attach
+ def step(self) -> int:
+ """Current value of the step.
+
+ This counter is incremented by `wandb.log`.
+ """
+ return self._step
+
+ def project_name(self) -> str:
+ return self._run_obj.project if self._run_obj else ""
+
+ @property
+ @_run_decorator._attach
+ def mode(self) -> str:
+ """For compatibility with `0.9.x` and earlier, deprecate eventually."""
+ deprecate.deprecate(
+ field_name=deprecate.Deprecated.run__mode,
+ warning_message=(
+ "The mode property of wandb.run is deprecated "
+ "and will be removed in a future release."
+ ),
+ )
+ return "dryrun" if self._settings._offline else "run"
+
+ @property
+ @_run_decorator._attach
+ def offline(self) -> bool:
+ return self._settings._offline
+
+ @property
+ @_run_decorator._attach
+ def disabled(self) -> bool:
+ return self._settings._noop
+
+ def _get_group(self) -> str:
+ return self._run_obj.run_group if self._run_obj else ""
+
+ @property
+ @_run_decorator._attach
+ def group(self) -> str:
+ """Name of the group associated with the run.
+
+ Setting a group helps the W&B UI organize runs in a sensible way.
+
+ If you are doing a distributed training you should give all of the
+ runs in the training the same group.
+ If you are doing cross-validation you should give all the cross-validation
+ folds the same group.
+ """
+ return self._get_group()
+
+ @property
+ @_run_decorator._attach
+ def job_type(self) -> str:
+ return self._run_obj.job_type if self._run_obj else ""
+
+ @property
+ @_run_decorator._attach
+ def project(self) -> str:
+ """Name of the W&B project associated with the run."""
+ return self.project_name()
+
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def log_code(
+ self,
+ root: Optional[str] = ".",
+ name: Optional[str] = None,
+ include_fn: Union[
+ Callable[[str, str], bool], Callable[[str], bool]
+ ] = _is_py_or_dockerfile,
+ exclude_fn: Union[
+ Callable[[str, str], bool], Callable[[str], bool]
+ ] = filenames.exclude_wandb_fn,
+ ) -> Optional[Artifact]:
+ """Save the current state of your code to a W&B Artifact.
+
+ By default, it walks the current directory and logs all files that end with `.py`.
+
+ Arguments:
+ root: The relative (to `os.getcwd()`) or absolute path to recursively find code from.
+ name: (str, optional) The name of our code artifact. By default, we'll name
+ the artifact `source-$PROJECT_ID-$ENTRYPOINT_RELPATH`. There may be scenarios where you want
+ many runs to share the same artifact. Specifying name allows you to achieve that.
+ include_fn: A callable that accepts a file path and (optionally) root path and
+ returns True when it should be included and False otherwise. This
+ defaults to: `lambda path, root: path.endswith(".py")`
+ exclude_fn: A callable that accepts a file path and (optionally) root path and
+ returns `True` when it should be excluded and `False` otherwise. This
+ defaults to a function that excludes all files within `<root>/.wandb/`
+ and `<root>/wandb/` directories.
+
+ Examples:
+ Basic usage
+ ```python
+ run.log_code()
+ ```
+
+ Advanced usage
+ ```python
+ run.log_code(
+ "../",
+ include_fn=lambda path: path.endswith(".py") or path.endswith(".ipynb"),
+ exclude_fn=lambda path, root: os.path.relpath(path, root).startswith("cache/"),
+ )
+ ```
+
+ Returns:
+ An `Artifact` object if code was logged
+ """
+ if name is None:
+ if self.settings._jupyter:
+ notebook_name = None
+ if self.settings.notebook_name:
+ notebook_name = self.settings.notebook_name
+ elif self.settings._jupyter_path:
+ if self.settings._jupyter_path.startswith("fileId="):
+ notebook_name = self.settings._jupyter_name
+ else:
+ notebook_name = self.settings._jupyter_path
+ name_string = f"{self._project}-{notebook_name}"
+ else:
+ name_string = f"{self._project}-{self._settings.program_relpath}"
+ name = wandb.util.make_artifact_name_safe(f"source-{name_string}")
+ art = wandb.Artifact(name, "code")
+ files_added = False
+ if root is not None:
+ root = os.path.abspath(root)
+ for file_path in filenames.filtered_dir(root, include_fn, exclude_fn):
+ files_added = True
+ save_name = os.path.relpath(file_path, root)
+ art.add_file(file_path, name=save_name)
+ # Add any manually staged files such as ipynb notebooks
+ for dirpath, _, files in os.walk(self._settings._tmp_code_dir):
+ for fname in files:
+ file_path = os.path.join(dirpath, fname)
+ save_name = os.path.relpath(file_path, self._settings._tmp_code_dir)
+ files_added = True
+ art.add_file(file_path, name=save_name)
+ if not files_added:
+ wandb.termwarn(
+ "No relevant files were detected in the specified directory. No code will be logged to your run."
+ )
+ return None
+
+ return self._log_artifact(art)
+
+ def get_url(self) -> Optional[str]:
+ """Return the url for the W&B run, if there is one.
+
+ Offline runs will not have a url.
+ """
+ if self._settings._offline:
+ wandb.termwarn("URL not available in offline run")
+ return None
+ return self._settings.run_url
+
+ def get_project_url(self) -> Optional[str]:
+ """Return the url for the W&B project associated with the run, if there is one.
+
+ Offline runs will not have a project url.
+ """
+ if self._settings._offline:
+ wandb.termwarn("URL not available in offline run")
+ return None
+ return self._settings.project_url
+
+ def get_sweep_url(self) -> Optional[str]:
+ """Return the url for the sweep associated with the run, if there is one."""
+ if self._settings._offline:
+ wandb.termwarn("URL not available in offline run")
+ return None
+ return self._settings.sweep_url
+
+ @property
+ @_run_decorator._attach
+ def url(self) -> Optional[str]:
+ """The W&B url associated with the run."""
+ return self.get_url()
+
+ @property
+ @_run_decorator._attach
+ def entity(self) -> str:
+ """The name of the W&B entity associated with the run.
+
+ Entity can be a username or the name of a team or organization.
+ """
+ return self._entity or ""
+
+ def _label_internal(
+ self,
+ code: Optional[str] = None,
+ repo: Optional[str] = None,
+ code_version: Optional[str] = None,
+ ) -> None:
+ with telemetry.context(run=self) as tel:
+ if code and RE_LABEL.match(code):
+ tel.label.code_string = code
+ if repo and RE_LABEL.match(repo):
+ tel.label.repo_string = repo
+ if code_version and RE_LABEL.match(code_version):
+ tel.label.code_version = code_version
+
+ def _label(
+ self,
+ code: Optional[str] = None,
+ repo: Optional[str] = None,
+ code_version: Optional[str] = None,
+ **kwargs: str,
+ ) -> None:
+ if self._settings.label_disable:
+ return
+ for k, v in (("code", code), ("repo", repo), ("code_version", code_version)):
+ if v and not RE_LABEL.match(v):
+ wandb.termwarn(
+ f"Label added for '{k}' with invalid identifier '{v}' (ignored).",
+ repeat=False,
+ )
+ for v in kwargs:
+ wandb.termwarn(
+ f"Label added for unsupported key {v!r} (ignored).",
+ repeat=False,
+ )
+
+ self._label_internal(code=code, repo=repo, code_version=code_version)
+
+ # update telemetry in the backend immediately for _label() callers
+ self._telemetry_flush()
+
+ def _label_probe_lines(self, lines: List[str]) -> None:
+ if not lines:
+ return
+ parsed = telemetry._parse_label_lines(lines)
+ if not parsed:
+ return
+ label_dict = {}
+ code = parsed.get("code") or parsed.get("c")
+ if code:
+ label_dict["code"] = code
+ repo = parsed.get("repo") or parsed.get("r")
+ if repo:
+ label_dict["repo"] = repo
+ code_ver = parsed.get("version") or parsed.get("v")
+ if code_ver:
+ label_dict["code_version"] = code_ver
+ self._label_internal(**label_dict)
+
+ def _label_probe_main(self) -> None:
+ m = sys.modules.get("__main__")
+ if not m:
+ return
+ doc = getattr(m, "__doc__", None)
+ if not doc:
+ return
+
+ doclines = doc.splitlines()
+ self._label_probe_lines(doclines)
+
+ # TODO: annotate jupyter Notebook class
+ def _label_probe_notebook(self, notebook: Any) -> None:
+ logger.info("probe notebook")
+ lines = None
+ try:
+ data = notebook.probe_ipynb()
+ cell0 = data.get("cells", [])[0]
+ lines = cell0.get("source")
+ # kaggle returns a string instead of a list
+ if isinstance(lines, str):
+ lines = lines.split()
+ except Exception as e:
+ logger.info(f"Unable to probe notebook: {e}")
+ return
+ if lines:
+ self._label_probe_lines(lines)
+
+ @_run_decorator._attach
+ def display(self, height: int = 420, hidden: bool = False) -> bool:
+ """Display this run in jupyter."""
+ if self._settings._jupyter:
+ ipython.display_html(self.to_html(height, hidden))
+ return True
+ else:
+ wandb.termwarn(".display() only works in jupyter environments")
+ return False
+
+ @_run_decorator._attach
+ def to_html(self, height: int = 420, hidden: bool = False) -> str:
+ """Generate HTML containing an iframe displaying the current run."""
+ url = self._settings.run_url + "?jupyter=true"
+ style = f"border:none;width:100%;height:{height}px;"
+ prefix = ""
+ if hidden:
+ style += "display:none;"
+ prefix = ipython.toggle_button()
+ return prefix + f"<iframe src={url!r} style={style!r}></iframe>"
+
+ def _repr_mimebundle_(
+ self, include: Optional[Any] = None, exclude: Optional[Any] = None
+ ) -> Dict[str, str]:
+ return {"text/html": self.to_html(hidden=True)}
+
+ @_run_decorator._noop_on_finish()
+ def _config_callback(
+ self,
+ key: Optional[Union[Tuple[str, ...], str]] = None,
+ val: Optional[Any] = None,
+ data: Optional[Dict[str, object]] = None,
+ ) -> None:
+ logger.info(f"config_cb {key} {val} {data}")
+ if self._backend and self._backend.interface:
+ self._backend.interface.publish_config(key=key, val=val, data=data)
+
+ def _config_artifact_callback(
+ self, key: str, val: Union[str, Artifact, dict]
+ ) -> Artifact:
+ # artifacts can look like dicts as they are passed into the run config
+ # since the run config stores them on the backend as a dict with fields shown
+ # in wandb.util.artifact_to_json
+ if _is_artifact_version_weave_dict(val):
+ assert isinstance(val, dict)
+ public_api = self._public_api()
+ artifact = Artifact._from_id(val["id"], public_api.client)
+ return self.use_artifact(artifact, use_as=key)
+ elif _is_artifact_string(val):
+ # this will never fail, but is required to make mypy happy
+ assert isinstance(val, str)
+ artifact_string, base_url, is_id = parse_artifact_string(val)
+ overrides = {}
+ if base_url is not None:
+ overrides = {"base_url": base_url}
+ public_api = public.Api(overrides)
+ else:
+ public_api = self._public_api()
+ if is_id:
+ artifact = Artifact._from_id(artifact_string, public_api._client)
+ else:
+ artifact = public_api.artifact(name=artifact_string)
+ # in the future we'll need to support using artifacts from
+ # different instances of wandb.
+
+ return self.use_artifact(artifact, use_as=key)
+ elif _is_artifact_object(val):
+ return self.use_artifact(val, use_as=key)
+ else:
+ raise ValueError(
+ f"Cannot call _config_artifact_callback on type {type(val)}"
+ )
+
+ def _set_config_wandb(self, key: str, val: Any) -> None:
+ self._config_callback(key=("_wandb", key), val=val)
+
+ @_run_decorator._noop_on_finish()
+ def _summary_update_callback(self, summary_record: SummaryRecord) -> None:
+ if self._backend and self._backend.interface:
+ self._backend.interface.publish_summary(summary_record)
+
+ def _on_progress_get_summary(self, handle: MailboxProgress) -> None:
+ pass
+ # TODO(jhr): enable printing for get_summary in later mailbox dev phase
+ # line = "Waiting for run.summary data..."
+ # self._printer.display(line)
+
+ def _summary_get_current_summary_callback(self) -> Dict[str, Any]:
+ if not self._backend or not self._backend.interface:
+ return {}
+ handle = self._backend.interface.deliver_get_summary()
+ result = handle.wait(
+ timeout=self._settings.summary_timeout,
+ on_progress=self._on_progress_get_summary,
+ )
+ if not result:
+ return {}
+ get_summary_response = result.response.get_summary_response
+ return proto_util.dict_from_proto_list(get_summary_response.item)
+
+ def _metric_callback(self, metric_record: MetricRecord) -> None:
+ if self._backend and self._backend.interface:
+ self._backend.interface._publish_metric(metric_record)
+
+ def _datatypes_callback(self, fname: str) -> None:
+ if not self._backend or not self._backend.interface:
+ return
+ files: FilesDict = dict(files=[(GlobStr(glob.escape(fname)), "now")])
+ self._backend.interface.publish_files(files)
+
+ def _visualization_hack(self, row: Dict[str, Any]) -> Dict[str, Any]:
+ # TODO(jhr): move visualize hack somewhere else
+ chart_keys = set()
+ split_table_set = set()
+ for k in row:
+ if isinstance(row[k], Visualize):
+ key = row[k].get_config_key(k)
+ value = row[k].get_config_value(k)
+ row[k] = row[k]._data
+ self._config_callback(val=value, key=key)
+ elif isinstance(row[k], CustomChart):
+ chart_keys.add(k)
+ key = row[k].get_config_key(k)
+ if row[k]._split_table:
+ value = row[k].get_config_value(
+ "Vega2", row[k].user_query(f"Custom Chart Tables/{k}_table")
+ )
+ split_table_set.add(k)
+ else:
+ value = row[k].get_config_value(
+ "Vega2", row[k].user_query(f"{k}_table")
+ )
+ row[k] = row[k]._data
+ self._config_callback(val=value, key=key)
+
+ for k in chart_keys:
+ # remove the chart key from the row
+ # TODO: is this really the right move? what if the user logs
+ # a non-custom chart to this key?
+ if k in split_table_set:
+ row[f"Custom Chart Tables/{k}_table"] = row.pop(k)
+ else:
+ row[f"{k}_table"] = row.pop(k)
+ return row
+
+ def _partial_history_callback(
+ self,
+ row: Dict[str, Any],
+ step: Optional[int] = None,
+ commit: Optional[bool] = None,
+ ) -> None:
+ row = row.copy()
+ if row:
+ row = self._visualization_hack(row)
+
+ if self._backend and self._backend.interface:
+ not_using_tensorboard = len(wandb.patched["tensorboard"]) == 0
+
+ self._backend.interface.publish_partial_history(
+ row,
+ user_step=self._step,
+ step=step,
+ flush=commit,
+ publish_step=not_using_tensorboard,
+ )
+
+ def _console_callback(self, name: str, data: str) -> None:
+ # logger.info("console callback: %s, %s", name, data)
+ if self._backend and self._backend.interface:
+ self._backend.interface.publish_output(name, data)
+
+ @_run_decorator._noop_on_finish(only_warn=True)
+ def _console_raw_callback(self, name: str, data: str) -> None:
+ # logger.info("console callback: %s, %s", name, data)
+
+ # NOTE: console output is only allowed on the process which installed the callback
+ # this will prevent potential corruption in the socket to the service. Other methods
+ # are protected by the _attach run decorator, but this callback was installed on the
+ # write function of stdout and stderr streams.
+ console_pid = getattr(self, "_attach_pid", 0)
+ if console_pid != os.getpid():
+ return
+
+ if self._backend and self._backend.interface:
+ self._backend.interface.publish_output_raw(name, data)
+
+ def _tensorboard_callback(
+ self, logdir: str, save: bool = True, root_logdir: str = ""
+ ) -> None:
+ logger.info("tensorboard callback: %s, %s", logdir, save)
+ if self._backend and self._backend.interface:
+ self._backend.interface.publish_tbdata(logdir, save, root_logdir)
+
+ def _set_library(self, library: _WandbSetup) -> None:
+ self._wl = library
+
+ def _set_backend(self, backend: "wandb.sdk.backend.backend.Backend") -> None:
+ self._backend = backend
+
+ def _set_internal_run_interface(
+ self,
+ interface: "wandb.sdk.interface.interface_queue.InterfaceQueue",
+ ) -> None:
+ self._internal_run_interface = interface
+
+ def _set_reporter(self, reporter: Reporter) -> None:
+ self._reporter = reporter
+
+ def _set_teardown_hooks(self, hooks: List[TeardownHook]) -> None:
+ self._teardown_hooks = hooks
+
+ def _set_run_obj(self, run_obj: RunRecord) -> None:
+ self._run_obj = run_obj
+ if self.settings._offline:
+ return
+
+ self._entity = run_obj.entity
+ self._project = run_obj.project
+
+ # Grab the config from resuming
+ if run_obj.config:
+ c_dict = config_util.dict_no_value_from_proto_list(run_obj.config.update)
+ # TODO: Windows throws a wild error when this is set...
+ if "_wandb" in c_dict:
+ del c_dict["_wandb"]
+ # We update the config object here without triggering the callback
+ self._config._update(c_dict, allow_val_change=True, ignore_locked=True)
+ # Update the summary, this will trigger an un-needed graphql request :(
+ if run_obj.summary:
+ summary_dict = {}
+ for orig in run_obj.summary.update:
+ summary_dict[orig.key] = json.loads(orig.value_json)
+ if summary_dict:
+ self.summary.update(summary_dict)
+ self._step = self._get_starting_step()
+
+ # update settings from run_obj
+ self._settings._apply_run_start(message_to_dict(self._run_obj))
+ self._update_settings(self._settings)
+
+ wandb._sentry.configure_scope(
+ process_context="user",
+ tags=dict(self._settings),
+ )
+
+ def _add_singleton(
+ self, data_type: str, key: str, value: Dict[Union[int, str], str]
+ ) -> None:
+ """Store a singleton item to wandb config.
+
+ A singleton in this context is a piece of data that is continually
+ logged with the same value in each history step, but represented
+ as a single item in the config.
+
+ We do this to avoid filling up history with a lot of repeated unnecessary data
+
+ Add singleton can be called many times in one run, and it will only be
+ updated when the value changes. The last value logged will be the one
+ persisted to the server.
+ """
+ value_extra = {"type": data_type, "key": key, "value": value}
+
+ if data_type not in self._config["_wandb"]:
+ self._config["_wandb"][data_type] = {}
+
+ if data_type in self._config["_wandb"][data_type]:
+ old_value = self._config["_wandb"][data_type][key]
+ else:
+ old_value = None
+
+ if value_extra != old_value:
+ self._config["_wandb"][data_type][key] = value_extra
+ self._config.persist()
+
+ def _log(
+ self,
+ data: Dict[str, Any],
+ step: Optional[int] = None,
+ commit: Optional[bool] = None,
+ ) -> None:
+ if not isinstance(data, Mapping):
+ raise ValueError("wandb.log must be passed a dictionary")
+
+ if any(not isinstance(key, str) for key in data.keys()):
+ raise ValueError("Key values passed to `wandb.log` must be strings.")
+
+ self._partial_history_callback(data, step, commit)
+
+ if step is not None:
+ if os.getpid() != self._init_pid or self._is_attached:
+ wandb.termwarn(
+ "Note that setting step in multiprocessing can result in data loss. "
+ "Please log your step values as a metric such as 'global_step'",
+ repeat=False,
+ )
+ # if step is passed in when tensorboard_sync is used we honor the step passed
+ # to make decisions about how to close out the history record, but will strip
+ # this history later on in publish_history()
+ if len(wandb.patched["tensorboard"]) > 0:
+ wandb.termwarn(
+ "Step cannot be set when using syncing with tensorboard. "
+ "Please log your step values as a metric such as 'global_step'",
+ repeat=False,
+ )
+ if step > self._step:
+ self._step = step
+
+ if (step is None and commit is None) or commit:
+ self._step += 1
+
+ @_run_decorator._noop
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def log(
+ self,
+ data: Dict[str, Any],
+ step: Optional[int] = None,
+ commit: Optional[bool] = None,
+ sync: Optional[bool] = None,
+ ) -> None:
+ """Log a dictionary of data to the current run's history.
+
+ Use `wandb.log` to log data from runs, such as scalars, images, video,
+ histograms, plots, and tables.
+
+ See our [guides to logging](https://docs.wandb.ai/guides/track/log) for
+ live examples, code snippets, best practices, and more.
+
+ The most basic usage is `wandb.log({"train-loss": 0.5, "accuracy": 0.9})`.
+ This will save the loss and accuracy to the run's history and update
+ the summary values for these metrics.
+
+ Visualize logged data in the workspace at [wandb.ai](https://wandb.ai),
+ or locally on a [self-hosted instance](https://docs.wandb.ai/guides/hosting)
+ of the W&B app, or export data to visualize and explore locally, e.g. in
+ Jupyter notebooks, with [our API](https://docs.wandb.ai/guides/track/public-api-guide).
+
+ In the UI, summary values show up in the run table to compare single values across runs.
+ Summary values can also be set directly with `wandb.run.summary["key"] = value`.
+
+ Logged values don't have to be scalars. Logging any wandb object is supported.
+ For example `wandb.log({"example": wandb.Image("myimage.jpg")})` will log an
+ example image which will be displayed nicely in the W&B UI.
+ See the [reference documentation](https://docs.wandb.com/ref/python/data-types)
+ for all of the different supported types or check out our
+ [guides to logging](https://docs.wandb.ai/guides/track/log) for examples,
+ from 3D molecular structures and segmentation masks to PR curves and histograms.
+ `wandb.Table`s can be used to logged structured data. See our
+ [guide to logging tables](https://docs.wandb.ai/guides/data-vis/log-tables)
+ for details.
+
+ Logging nested metrics is encouraged and is supported in the W&B UI.
+ If you log with a nested dictionary like `wandb.log({"train":
+ {"acc": 0.9}, "val": {"acc": 0.8}})`, the metrics will be organized into
+ `train` and `val` sections in the W&B UI.
+
+ wandb keeps track of a global step, which by default increments with each
+ call to `wandb.log`, so logging related metrics together is encouraged.
+ If it's inconvenient to log related metrics together
+ calling `wandb.log({"train-loss": 0.5}, commit=False)` and then
+ `wandb.log({"accuracy": 0.9})` is equivalent to calling
+ `wandb.log({"train-loss": 0.5, "accuracy": 0.9})`.
+
+ `wandb.log` is not intended to be called more than a few times per second.
+ If you want to log more frequently than that it's better to aggregate
+ the data on the client side or you may get degraded performance.
+
+ Arguments:
+ data: (dict, optional) A dict of serializable python objects i.e `str`,
+ `ints`, `floats`, `Tensors`, `dicts`, or any of the `wandb.data_types`.
+ commit: (boolean, optional) Save the metrics dict to the wandb server
+ and increment the step. If false `wandb.log` just updates the current
+ metrics dict with the data argument and metrics won't be saved until
+ `wandb.log` is called with `commit=True`.
+ step: (integer, optional) The global step in processing. This persists
+ any non-committed earlier steps but defaults to not committing the
+ specified step.
+ sync: (boolean, True) This argument is deprecated and currently doesn't
+ change the behaviour of `wandb.log`.
+
+ Examples:
+ For more and more detailed examples, see
+ [our guides to logging](https://docs.wandb.com/guides/track/log).
+
+ ### Basic usage
+ <!--yeadoc-test:init-and-log-basic-->
+ ```python
+ import wandb
+
+ run = wandb.init()
+ run.log({"accuracy": 0.9, "epoch": 5})
+ ```
+
+ ### Incremental logging
+ <!--yeadoc-test:init-and-log-incremental-->
+ ```python
+ import wandb
+
+ run = wandb.init()
+ run.log({"loss": 0.2}, commit=False)
+ # Somewhere else when I'm ready to report this step:
+ run.log({"accuracy": 0.8})
+ ```
+
+ ### Histogram
+ <!--yeadoc-test:init-and-log-histogram-->
+ ```python
+ import numpy as np
+ import wandb
+
+ # sample gradients at random from normal distribution
+ gradients = np.random.randn(100, 100)
+ run = wandb.init()
+ run.log({"gradients": wandb.Histogram(gradients)})
+ ```
+
+ ### Image from numpy
+ <!--yeadoc-test:init-and-log-image-numpy-->
+ ```python
+ import numpy as np
+ import wandb
+
+ run = wandb.init()
+ examples = []
+ for i in range(3):
+ pixels = np.random.randint(low=0, high=256, size=(100, 100, 3))
+ image = wandb.Image(pixels, caption=f"random field {i}")
+ examples.append(image)
+ run.log({"examples": examples})
+ ```
+
+ ### Image from PIL
+ <!--yeadoc-test:init-and-log-image-pillow-->
+ ```python
+ import numpy as np
+ from PIL import Image as PILImage
+ import wandb
+
+ run = wandb.init()
+ examples = []
+ for i in range(3):
+ pixels = np.random.randint(low=0, high=256, size=(100, 100, 3), dtype=np.uint8)
+ pil_image = PILImage.fromarray(pixels, mode="RGB")
+ image = wandb.Image(pil_image, caption=f"random field {i}")
+ examples.append(image)
+ run.log({"examples": examples})
+ ```
+
+ ### Video from numpy
+ <!--yeadoc-test:init-and-log-video-numpy-->
+ ```python
+ import numpy as np
+ import wandb
+
+ run = wandb.init()
+ # axes are (time, channel, height, width)
+ frames = np.random.randint(low=0, high=256, size=(10, 3, 100, 100), dtype=np.uint8)
+ run.log({"video": wandb.Video(frames, fps=4)})
+ ```
+
+ ### Matplotlib Plot
+ <!--yeadoc-test:init-and-log-matplotlib-->
+ ```python
+ from matplotlib import pyplot as plt
+ import numpy as np
+ import wandb
+
+ run = wandb.init()
+ fig, ax = plt.subplots()
+ x = np.linspace(0, 10)
+ y = x * x
+ ax.plot(x, y) # plot y = x^2
+ run.log({"chart": fig})
+ ```
+
+ ### PR Curve
+ ```python
+ import wandb
+
+ run = wandb.init()
+ run.log({"pr": wandb.plots.precision_recall(y_test, y_probas, labels)})
+ ```
+
+ ### 3D Object
+ ```python
+ import wandb
+
+ run = wandb.init()
+ run.log(
+ {
+ "generated_samples": [
+ wandb.Object3D(open("sample.obj")),
+ wandb.Object3D(open("sample.gltf")),
+ wandb.Object3D(open("sample.glb")),
+ ]
+ }
+ )
+ ```
+
+ Raises:
+ wandb.Error: if called before `wandb.init`
+ ValueError: if invalid data is passed
+
+ """
+ if sync is not None:
+ deprecate.deprecate(
+ field_name=deprecate.Deprecated.run__log_sync,
+ warning_message=(
+ "`sync` argument is deprecated and does not affect the behaviour of `wandb.log`"
+ ),
+ )
+ if self._settings._shared and step is not None:
+ wandb.termwarn(
+ "In shared mode, the use of `wandb.log` with the step argument is not supported "
+ f"and will be ignored. Please refer to {wburls.get('wandb_define_metric')} "
+ "on how to customize your x-axis.",
+ repeat=False,
+ )
+ self._log(data=data, step=step, commit=commit)
+
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def save(
+ self,
+ glob_str: Optional[str] = None,
+ base_path: Optional[str] = None,
+ policy: "PolicyName" = "live",
+ ) -> Union[bool, List[str]]:
+ """Ensure all files matching `glob_str` are synced to wandb with the policy specified.
+
+ Arguments:
+ glob_str: (string) a relative or absolute path to a unix glob or regular
+ path. If this isn't specified the method is a noop.
+ base_path: (string) the base path to run the glob relative to
+ policy: (string) one of `live`, `now`, or `end`
+ - live: upload the file as it changes, overwriting the previous version
+ - now: upload the file once now
+ - end: only upload file when the run ends
+ """
+ if glob_str is None:
+ # noop for historical reasons, run.save() may be called in legacy code
+ deprecate.deprecate(
+ field_name=deprecate.Deprecated.run__save_no_args,
+ warning_message=(
+ "Calling wandb.run.save without any arguments is deprecated."
+ "Changes to attributes are automatically persisted."
+ ),
+ )
+ return True
+
+ return self._save(glob_str, base_path, policy)
+
+ def _save(
+ self,
+ glob_str: Optional[str] = None,
+ base_path: Optional[str] = None,
+ policy: "PolicyName" = "live",
+ ) -> Union[bool, List[str]]:
+ if policy not in ("live", "end", "now"):
+ raise ValueError(
+ 'Only "live" "end" and "now" policies are currently supported.'
+ )
+ if isinstance(glob_str, bytes):
+ glob_str = glob_str.decode("utf-8")
+ if not isinstance(glob_str, str):
+ raise ValueError("Must call wandb.save(glob_str) with glob_str a str")
+
+ if base_path is None:
+ if os.path.isabs(glob_str):
+ base_path = os.path.dirname(glob_str)
+ wandb.termwarn(
+ "Saving files without folders. If you want to preserve "
+ "sub directories pass base_path to wandb.save, i.e. "
+ 'wandb.save("/mnt/folder/file.h5", base_path="/mnt")',
+ repeat=False,
+ )
+ else:
+ base_path = "."
+ wandb_glob_str = GlobStr(os.path.relpath(glob_str, base_path))
+ if ".." + os.sep in wandb_glob_str:
+ raise ValueError("globs can't walk above base_path")
+
+ with telemetry.context(run=self) as tel:
+ tel.feature.save = True
+
+ if glob_str.startswith("gs://") or glob_str.startswith("s3://"):
+ wandb.termlog(
+ "%s is a cloud storage url, can't save file to wandb." % glob_str
+ )
+ return []
+ files = glob.glob(os.path.join(self._settings.files_dir, wandb_glob_str))
+ warn = False
+ if len(files) == 0 and "*" in wandb_glob_str:
+ warn = True
+ for path in glob.glob(glob_str):
+ file_name = os.path.relpath(path, base_path)
+ abs_path = os.path.abspath(path)
+ wandb_path = os.path.join(self._settings.files_dir, file_name)
+ filesystem.mkdir_exists_ok(os.path.dirname(wandb_path))
+ # We overwrite symlinks because namespaces can change in Tensorboard
+ if os.path.islink(wandb_path) and abs_path != os.readlink(wandb_path):
+ os.remove(wandb_path)
+ os.symlink(abs_path, wandb_path)
+ elif not os.path.exists(wandb_path):
+ os.symlink(abs_path, wandb_path)
+ files.append(wandb_path)
+ if warn:
+ file_str = "%i file" % len(files)
+ if len(files) > 1:
+ file_str += "s"
+ wandb.termwarn(
+ (
+ "Symlinked %s into the W&B run directory, "
+ "call wandb.save again to sync new files."
+ )
+ % file_str
+ )
+ files_dict: FilesDict = dict(files=[(wandb_glob_str, policy)])
+ if self._backend and self._backend.interface:
+ self._backend.interface.publish_files(files_dict)
+ return files
+
+ @_run_decorator._attach
+ def restore(
+ self,
+ name: str,
+ run_path: Optional[str] = None,
+ replace: bool = False,
+ root: Optional[str] = None,
+ ) -> Union[None, TextIO]:
+ return restore(
+ name,
+ run_path or self._get_path(),
+ replace,
+ root or self._settings.files_dir,
+ )
+
+ @_run_decorator._noop
+ @_run_decorator._attach
+ def finish(
+ self, exit_code: Optional[int] = None, quiet: Optional[bool] = None
+ ) -> None:
+ """Mark a run as finished, and finish uploading all data.
+
+ This is used when creating multiple runs in the same process. We automatically
+ call this method when your script exits or if you use the run context manager.
+
+ Arguments:
+ exit_code: Set to something other than 0 to mark a run as failed
+ quiet: Set to true to minimize log output
+ """
+ return self._finish(exit_code, quiet)
+
+ def _finish(
+ self, exit_code: Optional[int] = None, quiet: Optional[bool] = None
+ ) -> None:
+ if quiet is not None:
+ self._quiet = quiet
+ with telemetry.context(run=self) as tel:
+ tel.feature.finish = True
+ logger.info(f"finishing run {self._get_path()}")
+ # detach jupyter hooks / others that needs to happen before backend shutdown
+ for hook in self._teardown_hooks:
+ if hook.stage == TeardownStage.EARLY:
+ hook.call()
+
+ self._atexit_cleanup(exit_code=exit_code)
+ if self._wl and len(self._wl._global_run_stack) > 0:
+ self._wl._global_run_stack.pop()
+ # detach logger / others meant to be run after we've shutdown the backend
+ for hook in self._teardown_hooks:
+ if hook.stage == TeardownStage.LATE:
+ hook.call()
+ self._teardown_hooks = []
+ module.unset_globals()
+
+ # inform manager this run is finished
+ manager = self._wl and self._wl._get_manager()
+ if manager:
+ manager._inform_finish(run_id=self._run_id)
+
+ self._is_finished = True
+ # end sentry session
+ wandb._sentry.end_session()
+
+ @_run_decorator._noop
+ @_run_decorator._attach
+ def join(self, exit_code: Optional[int] = None) -> None:
+ """Deprecated alias for `finish()` - use finish instead."""
+ deprecate.deprecate(
+ field_name=deprecate.Deprecated.run__join,
+ warning_message=(
+ "wandb.run.join() is deprecated, please use wandb.run.finish()."
+ ),
+ )
+ self._finish(exit_code=exit_code)
+
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def status(
+ self,
+ ) -> RunStatus:
+ """Get sync info from the internal backend, about the current run's sync status."""
+ if not self._backend or not self._backend.interface:
+ return RunStatus()
+
+ handle_run_status = self._backend.interface.deliver_request_run_status()
+ result = handle_run_status.wait(timeout=-1)
+ assert result
+ sync_data = result.response.run_status_response
+
+ sync_time = None
+ if sync_data.sync_time.seconds:
+ sync_time = datetime.fromtimestamp(
+ sync_data.sync_time.seconds + sync_data.sync_time.nanos / 1e9
+ )
+ return RunStatus(
+ sync_items_total=sync_data.sync_items_total,
+ sync_items_pending=sync_data.sync_items_pending,
+ sync_time=sync_time,
+ )
+
+ @staticmethod
+ def plot_table(
+ vega_spec_name: str,
+ data_table: "wandb.Table",
+ fields: Dict[str, Any],
+ string_fields: Optional[Dict[str, Any]] = None,
+ split_table: Optional[bool] = False,
+ ) -> CustomChart:
+ """Create a custom plot on a table.
+
+ Arguments:
+ vega_spec_name: the name of the spec for the plot
+ data_table: a wandb.Table object containing the data to
+ be used on the visualization
+ fields: a dict mapping from table keys to fields that the custom
+ visualization needs
+ string_fields: a dict that provides values for any string constants
+ the custom visualization needs
+ """
+ return custom_chart(
+ vega_spec_name, data_table, fields, string_fields or {}, split_table
+ )
+
+ def _add_panel(
+ self, visualize_key: str, panel_type: str, panel_config: dict
+ ) -> None:
+ config = {
+ "panel_type": panel_type,
+ "panel_config": panel_config,
+ }
+ self._config_callback(val=config, key=("_wandb", "visualize", visualize_key))
+
+ def _set_globals(self) -> None:
+ module.set_global(
+ run=self,
+ config=self.config,
+ log=self.log,
+ summary=self.summary,
+ save=self.save,
+ use_artifact=self.use_artifact,
+ log_artifact=self.log_artifact,
+ define_metric=self.define_metric,
+ plot_table=self.plot_table,
+ alert=self.alert,
+ mark_preempting=self.mark_preempting,
+ log_model=self.log_model,
+ use_model=self.use_model,
+ link_model=self.link_model,
+ )
+
+ def _redirect(
+ self,
+ stdout_slave_fd: Optional[int],
+ stderr_slave_fd: Optional[int],
+ console: Optional[str] = None,
+ ) -> None:
+ if console is None:
+ console = self._settings.console
+ # only use raw for service to minimize potential changes
+ if console == "wrap":
+ if not self._settings._disable_service:
+ console = "wrap_raw"
+ else:
+ console = "wrap_emu"
+ logger.info("redirect: %s", console)
+
+ out_redir: redirect.RedirectBase
+ err_redir: redirect.RedirectBase
+
+ # raw output handles the output_log writing in the internal process
+ if console in {"redirect", "wrap_emu"}:
+ output_log_path = os.path.join(
+ self._settings.files_dir, filenames.OUTPUT_FNAME
+ )
+ # output writer might have been set up, see wrap_fallback case
+ if not self._output_writer:
+ self._output_writer = filesystem.CRDedupedFile(
+ open(output_log_path, "wb")
+ )
+
+ if console == "redirect":
+ logger.info("Redirecting console.")
+ out_redir = redirect.Redirect(
+ src="stdout",
+ cbs=[
+ lambda data: self._console_callback("stdout", data),
+ self._output_writer.write, # type: ignore
+ ],
+ )
+ err_redir = redirect.Redirect(
+ src="stderr",
+ cbs=[
+ lambda data: self._console_callback("stderr", data),
+ self._output_writer.write, # type: ignore
+ ],
+ )
+ if os.name == "nt":
+
+ def wrap_fallback() -> None:
+ if self._out_redir:
+ self._out_redir.uninstall()
+ if self._err_redir:
+ self._err_redir.uninstall()
+ msg = (
+ "Tensorflow detected. Stream redirection is not supported "
+ "on Windows when tensorflow is imported. Falling back to "
+ "wrapping stdout/err."
+ )
+ wandb.termlog(msg)
+ self._redirect(None, None, console="wrap")
+
+ add_import_hook("tensorflow", wrap_fallback)
+ elif console == "wrap_emu":
+ logger.info("Wrapping output streams.")
+ out_redir = redirect.StreamWrapper(
+ src="stdout",
+ cbs=[
+ lambda data: self._console_callback("stdout", data),
+ self._output_writer.write, # type: ignore
+ ],
+ )
+ err_redir = redirect.StreamWrapper(
+ src="stderr",
+ cbs=[
+ lambda data: self._console_callback("stderr", data),
+ self._output_writer.write, # type: ignore
+ ],
+ )
+ elif console == "wrap_raw":
+ logger.info("Wrapping output streams.")
+ out_redir = redirect.StreamRawWrapper(
+ src="stdout",
+ cbs=[
+ lambda data: self._console_raw_callback("stdout", data),
+ ],
+ )
+ err_redir = redirect.StreamRawWrapper(
+ src="stderr",
+ cbs=[
+ lambda data: self._console_raw_callback("stderr", data),
+ ],
+ )
+ elif console == "off":
+ return
+ else:
+ raise ValueError("unhandled console")
+ try:
+ # save stdout and stderr before installing new write functions
+ out_redir.save()
+ err_redir.save()
+ out_redir.install()
+ err_redir.install()
+ self._out_redir = out_redir
+ self._err_redir = err_redir
+ logger.info("Redirects installed.")
+ except Exception as e:
+ print(e)
+ logger.error("Failed to redirect.", exc_info=e)
+ return
+
+ def _restore(self) -> None:
+ logger.info("restore")
+ # TODO(jhr): drain and shutdown all threads
+ if self._out_redir:
+ self._out_redir.uninstall()
+ if self._err_redir:
+ self._err_redir.uninstall()
+ logger.info("restore done")
+
+ def _atexit_cleanup(self, exit_code: Optional[int] = None) -> None:
+ if self._backend is None:
+ logger.warning("process exited without backend configured")
+ return
+ if self._atexit_cleanup_called:
+ return
+ self._atexit_cleanup_called = True
+
+ exit_code = exit_code or self._hooks.exit_code if self._hooks else 0
+ logger.info(f"got exitcode: {exit_code}")
+ if exit_code == 0:
+ # Cleanup our resume file on a clean exit
+ if os.path.exists(self._settings.resume_fname):
+ os.remove(self._settings.resume_fname)
+
+ self._exit_code = exit_code
+ report_failure = False
+ try:
+ self._on_finish()
+ except KeyboardInterrupt as ki:
+ if wandb.wandb_agent._is_running():
+ raise ki
+ wandb.termerror("Control-C detected -- Run data was not synced")
+ if not self._settings._notebook:
+ os._exit(-1)
+ except Exception as e:
+ if not self._settings._notebook:
+ report_failure = True
+ self._console_stop()
+ self._backend.cleanup()
+ logger.error("Problem finishing run", exc_info=e)
+ wandb.termerror("Problem finishing run")
+ traceback.print_exc()
+ else:
+ self._on_final()
+ finally:
+ if report_failure:
+ os._exit(-1)
+
+ def _console_start(self) -> None:
+ logger.info("atexit reg")
+ self._hooks = ExitHooks()
+
+ manager = self._wl and self._wl._get_manager()
+ if not manager:
+ self._hooks.hook()
+ # NB: manager will perform atexit hook like behavior for outstanding runs
+ atexit.register(lambda: self._atexit_cleanup())
+
+ self._redirect(self._stdout_slave_fd, self._stderr_slave_fd)
+
+ def _console_stop(self) -> None:
+ self._restore()
+ if self._output_writer:
+ self._output_writer.close()
+ self._output_writer = None
+
+ def _on_init(self) -> None:
+ if self._settings._offline:
+ return
+ if self._backend and self._backend.interface:
+ logger.info("communicating current version")
+ version_handle = self._backend.interface.deliver_check_version(
+ current_version=wandb.__version__
+ )
+ version_result = version_handle.wait(timeout=30)
+ if not version_result:
+ version_handle.abandon()
+ return
+ self._check_version = version_result.response.check_version_response
+ logger.info(f"got version response {self._check_version}")
+
+ def _on_start(self) -> None:
+ # would like to move _set_global to _on_ready to unify _on_start and _on_attach
+ # (we want to do the set globals after attach)
+ # TODO(console) However _console_start calls Redirect that uses `wandb.run` hence breaks
+ # TODO(jupyter) However _header calls _header_run_info that uses wandb.jupyter that uses
+ # `wandb.run` and hence breaks
+ self._set_globals()
+ self._header(
+ self._check_version, settings=self._settings, printer=self._printer
+ )
+
+ if self._settings.save_code and self._settings.code_dir is not None:
+ self.log_code(self._settings.code_dir)
+
+ if self._settings._save_requirements:
+ if self._backend and self._backend.interface:
+ import pkg_resources
+
+ logger.debug(
+ "Saving list of pip packages installed into the current environment"
+ )
+ self._backend.interface.publish_python_packages(
+ pkg_resources.working_set
+ )
+
+ if self._backend and self._backend.interface and not self._settings._offline:
+ self._run_status_checker = RunStatusChecker(
+ interface=self._backend.interface,
+ )
+ self._run_status_checker.start()
+
+ self._console_start()
+ self._on_ready()
+
+ def _on_attach(self) -> None:
+ """Event triggered when run is attached to another run."""
+ with telemetry.context(run=self) as tel:
+ tel.feature.attach = True
+
+ self._set_globals()
+ self._is_attached = True
+ self._on_ready()
+
+ def _register_telemetry_import_hooks(
+ self,
+ ) -> None:
+ def _telemetry_import_hook(
+ run: "Run",
+ module: Any,
+ ) -> None:
+ with telemetry.context(run=run) as tel:
+ try:
+ name = getattr(module, "__name__", None)
+ if name is not None:
+ setattr(tel.imports_finish, name, True)
+ except AttributeError:
+ return
+
+ import_telemetry_set = telemetry.list_telemetry_imports()
+ import_hook_fn = functools.partial(_telemetry_import_hook, self)
+ for module_name in import_telemetry_set:
+ register_post_import_hook(
+ import_hook_fn,
+ self._run_id,
+ module_name,
+ )
+
+ def _on_ready(self) -> None:
+ """Event triggered when run is ready for the user."""
+ self._register_telemetry_import_hooks()
+
+ # start reporting any telemetry changes
+ self._telemetry_obj_active = True
+ self._telemetry_flush()
+
+ # object is about to be returned to the user, don't let them modify it
+ self._freeze()
+
+ if not self._settings.resume:
+ if os.path.exists(self._settings.resume_fname):
+ os.remove(self._settings.resume_fname)
+
+ def _make_job_source_reqs(self) -> Tuple[List[str], Dict[str, Any], Dict[str, Any]]:
+ import pkg_resources
+
+ installed_packages_list = sorted(
+ f"{d.key}=={d.version}" for d in iter(pkg_resources.working_set)
+ )
+ input_types = TypeRegistry.type_of(self.config.as_dict()).to_json()
+ output_types = TypeRegistry.type_of(self.summary._as_dict()).to_json()
+
+ return installed_packages_list, input_types, output_types
+
+ def _construct_job_artifact(
+ self,
+ name: str,
+ source_dict: "JobSourceDict",
+ installed_packages_list: List[str],
+ patch_path: Optional[os.PathLike] = None,
+ ) -> "Artifact":
+ job_artifact = job_builder.JobArtifact(name)
+ if patch_path and os.path.exists(patch_path):
+ job_artifact.add_file(FilePathStr(str(patch_path)), "diff.patch")
+ with job_artifact.new_file("requirements.frozen.txt") as f:
+ f.write("\n".join(installed_packages_list))
+ with job_artifact.new_file("wandb-job.json") as f:
+ f.write(json.dumps(source_dict))
+
+ return job_artifact
+
+ def _create_image_job(
+ self,
+ input_types: Dict[str, Any],
+ output_types: Dict[str, Any],
+ installed_packages_list: List[str],
+ docker_image_name: Optional[str] = None,
+ args: Optional[List[str]] = None,
+ ) -> Optional["Artifact"]:
+ docker_image_name = docker_image_name or os.getenv("WANDB_DOCKER")
+
+ if not docker_image_name:
+ return None
+
+ name = wandb.util.make_artifact_name_safe(f"job-{docker_image_name}")
+ s_args: Sequence[str] = args if args is not None else self._settings._args
+ source_info: JobSourceDict = {
+ "_version": "v0",
+ "source_type": "image",
+ "source": {"image": docker_image_name, "args": s_args},
+ "input_types": input_types,
+ "output_types": output_types,
+ "runtime": self._settings._python,
+ }
+ job_artifact = self._construct_job_artifact(
+ name, source_info, installed_packages_list
+ )
+
+ return job_artifact
+
+ def _log_job_artifact_with_image(
+ self, docker_image_name: str, args: Optional[List[str]] = None
+ ) -> Artifact:
+ packages, in_types, out_types = self._make_job_source_reqs()
+ job_artifact = self._create_image_job(
+ in_types,
+ out_types,
+ packages,
+ args=args,
+ docker_image_name=docker_image_name,
+ )
+
+ artifact = self.log_artifact(job_artifact)
+
+ if not artifact:
+ raise wandb.Error(f"Job Artifact log unsuccessful: {artifact}")
+ else:
+ return artifact
+
+ # Add a recurring callback (probe) to poll the backend process
+ # for its status using the "poll_exit" message.
+ def _on_probe_exit(self, probe_handle: MailboxProbe) -> None:
+ handle = probe_handle.get_mailbox_handle()
+ if handle:
+ result = handle.wait(timeout=0, release=False)
+ if not result:
+ return
+ probe_handle.set_probe_result(result)
+ assert self._backend and self._backend.interface
+ handle = self._backend.interface.deliver_poll_exit()
+ probe_handle.set_mailbox_handle(handle)
+
+ # Handles the progress message from the backend process and prints
+ # the current status to the terminal footer
+ def _on_progress_exit(self, progress_handle: MailboxProgress) -> None:
+ probe_handles = progress_handle.get_probe_handles()
+ assert probe_handles and len(probe_handles) == 1
+
+ result = probe_handles[0].get_probe_result()
+ if not result:
+ return
+ self._footer_file_pusher_status_info(
+ result.response.poll_exit_response, printer=self._printer
+ )
+
+ def _on_finish(self) -> None:
+ trigger.call("on_finished")
+
+ if self._run_status_checker is not None:
+ self._run_status_checker.stop()
+
+ self._console_stop() # TODO: there's a race here with jupyter console logging
+
+ assert self._backend and self._backend.interface
+ exit_handle = self._backend.interface.deliver_exit(self._exit_code)
+ exit_handle.add_probe(on_probe=self._on_probe_exit)
+
+ # this message is confusing, we should remove it
+ # self._footer_exit_status_info(
+ # self._exit_code, settings=self._settings, printer=self._printer
+ # )
+
+ _ = exit_handle.wait(timeout=-1, on_progress=self._on_progress_exit)
+
+ poll_exit_handle = self._backend.interface.deliver_poll_exit()
+ # wait for them, it's ok to do this serially but this can be improved
+ result = poll_exit_handle.wait(timeout=-1)
+ assert result
+ self._footer_file_pusher_status_info(
+ result.response.poll_exit_response, printer=self._printer
+ )
+ self._poll_exit_response = result.response.poll_exit_response
+ internal_messages_handle = self._backend.interface.deliver_internal_messages()
+ result = internal_messages_handle.wait(timeout=-1)
+ assert result
+ self._internal_messages_response = result.response.internal_messages_response
+
+ # dispatch all our final requests
+
+ server_info_handle = self._backend.interface.deliver_request_server_info()
+ final_summary_handle = self._backend.interface.deliver_get_summary()
+ sampled_history_handle = (
+ self._backend.interface.deliver_request_sampled_history()
+ )
+ job_info_handle = self._backend.interface.deliver_request_job_info()
+
+ result = server_info_handle.wait(timeout=-1)
+ assert result
+ self._server_info_response = result.response.server_info_response
+
+ result = sampled_history_handle.wait(timeout=-1)
+ assert result
+ self._sampled_history = result.response.sampled_history_response
+
+ result = final_summary_handle.wait(timeout=-1)
+ assert result
+ self._final_summary = result.response.get_summary_response
+
+ result = job_info_handle.wait(timeout=-1)
+ assert result
+ self._job_info = result.response.job_info_response
+
+ if self._backend:
+ self._backend.cleanup()
+
+ if self._run_status_checker:
+ self._run_status_checker.join()
+
+ self._unregister_telemetry_import_hooks(self._run_id)
+
+ @staticmethod
+ def _unregister_telemetry_import_hooks(run_id: str) -> None:
+ import_telemetry_set = telemetry.list_telemetry_imports()
+ for module_name in import_telemetry_set:
+ unregister_post_import_hook(module_name, run_id)
+
+ def _on_final(self) -> None:
+ self._footer(
+ sampled_history=self._sampled_history,
+ final_summary=self._final_summary,
+ poll_exit_response=self._poll_exit_response,
+ server_info_response=self._server_info_response,
+ check_version_response=self._check_version,
+ internal_messages_response=self._internal_messages_response,
+ job_info=self._job_info,
+ reporter=self._reporter,
+ quiet=self._quiet,
+ settings=self._settings,
+ printer=self._printer,
+ )
+
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def define_metric(
+ self,
+ name: str,
+ step_metric: Union[str, wandb_metric.Metric, None] = None,
+ step_sync: Optional[bool] = None,
+ hidden: Optional[bool] = None,
+ summary: Optional[str] = None,
+ goal: Optional[str] = None,
+ overwrite: Optional[bool] = None,
+ **kwargs: Any,
+ ) -> wandb_metric.Metric:
+ """Define metric properties which will later be logged with `wandb.log()`.
+
+ Arguments:
+ name: Name of the metric.
+ step_metric: Independent variable associated with the metric.
+ step_sync: Automatically add `step_metric` to history if needed.
+ Defaults to True if step_metric is specified.
+ hidden: Hide this metric from automatic plots.
+ summary: Specify aggregate metrics added to summary.
+ Supported aggregations: "min,max,mean,best,last,none"
+ Default aggregation is `copy`
+ Aggregation `best` defaults to `goal`==`minimize`
+ goal: Specify direction for optimizing the metric.
+ Supported directions: "minimize,maximize"
+
+ Returns:
+ A metric object is returned that can be further specified.
+
+ """
+ return self._define_metric(
+ name, step_metric, step_sync, hidden, summary, goal, overwrite, **kwargs
+ )
+
+ def _define_metric(
+ self,
+ name: str,
+ step_metric: Union[str, wandb_metric.Metric, None] = None,
+ step_sync: Optional[bool] = None,
+ hidden: Optional[bool] = None,
+ summary: Optional[str] = None,
+ goal: Optional[str] = None,
+ overwrite: Optional[bool] = None,
+ **kwargs: Any,
+ ) -> wandb_metric.Metric:
+ if not name:
+ raise wandb.Error("define_metric() requires non-empty name argument")
+ for k in kwargs:
+ wandb.termwarn(f"Unhandled define_metric() arg: {k}")
+ if isinstance(step_metric, wandb_metric.Metric):
+ step_metric = step_metric.name
+ for arg_name, arg_val, exp_type in (
+ ("name", name, str),
+ ("step_metric", step_metric, str),
+ ("step_sync", step_sync, bool),
+ ("hidden", hidden, bool),
+ ("summary", summary, str),
+ ("goal", goal, str),
+ ("overwrite", overwrite, bool),
+ ):
+ # NOTE: type checking is broken for isinstance and str
+ if arg_val is not None and not isinstance(arg_val, exp_type):
+ arg_type = type(arg_val).__name__
+ raise wandb.Error(
+ f"Unhandled define_metric() arg: {arg_name} type: {arg_type}"
+ )
+ stripped = name[:-1] if name.endswith("*") else name
+ if "*" in stripped:
+ raise wandb.Error(
+ f"Unhandled define_metric() arg: name (glob suffixes only): {name}"
+ )
+ summary_ops: Optional[Sequence[str]] = None
+ if summary:
+ summary_items = [s.lower() for s in summary.split(",")]
+ summary_ops = []
+ valid = {"min", "max", "mean", "best", "last", "copy", "none"}
+ for i in summary_items:
+ if i not in valid:
+ raise wandb.Error(f"Unhandled define_metric() arg: summary op: {i}")
+ summary_ops.append(i)
+ goal_cleaned: Optional[str] = None
+ if goal is not None:
+ goal_cleaned = goal[:3].lower()
+ valid_goal = {"min", "max"}
+ if goal_cleaned not in valid_goal:
+ raise wandb.Error(f"Unhandled define_metric() arg: goal: {goal}")
+ m = wandb_metric.Metric(
+ name=name,
+ step_metric=step_metric,
+ step_sync=step_sync,
+ summary=summary_ops,
+ hidden=hidden,
+ goal=goal_cleaned,
+ overwrite=overwrite,
+ )
+ m._set_callback(self._metric_callback)
+ m._commit()
+ with telemetry.context(run=self) as tel:
+ tel.feature.metric = True
+ return m
+
+ # TODO(jhr): annotate this
+ @_run_decorator._attach
+ def watch( # type: ignore
+ self,
+ models,
+ criterion=None,
+ log="gradients",
+ log_freq=100,
+ idx=None,
+ log_graph=False,
+ ) -> None:
+ wandb.watch(models, criterion, log, log_freq, idx, log_graph)
+
+ # TODO(jhr): annotate this
+ @_run_decorator._attach
+ def unwatch(self, models=None) -> None: # type: ignore
+ wandb.unwatch(models=models)
+
+ # TODO(kdg): remove all artifact swapping logic
+ def _swap_artifact_name(self, artifact_name: str, use_as: Optional[str]) -> str:
+ artifact_key_string = use_as or artifact_name
+ replacement_artifact_info = self._launch_artifact_mapping.get(
+ artifact_key_string
+ )
+ if replacement_artifact_info is not None:
+ new_name = replacement_artifact_info.get("name")
+ entity = replacement_artifact_info.get("entity")
+ project = replacement_artifact_info.get("project")
+ if new_name is None or entity is None or project is None:
+ raise ValueError(
+ "Misconfigured artifact in launch config. Must include name, project and entity keys."
+ )
+ return f"{entity}/{project}/{new_name}"
+ elif replacement_artifact_info is None and use_as is None:
+ sequence_name = artifact_name.split(":")[0].split("/")[-1]
+ unique_artifact_replacement_info = (
+ self._unique_launch_artifact_sequence_names.get(sequence_name)
+ )
+ if unique_artifact_replacement_info is not None:
+ new_name = unique_artifact_replacement_info.get("name")
+ entity = unique_artifact_replacement_info.get("entity")
+ project = unique_artifact_replacement_info.get("project")
+ if new_name is None or entity is None or project is None:
+ raise ValueError(
+ "Misconfigured artifact in launch config. Must include name, project and entity keys."
+ )
+ return f"{entity}/{project}/{new_name}"
+
+ else:
+ return artifact_name
+
+ return artifact_name
+
+ def _detach(self) -> None:
+ pass
+
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def link_artifact(
+ self,
+ artifact: Artifact,
+ target_path: str,
+ aliases: Optional[List[str]] = None,
+ ) -> None:
+ """Link the given artifact to a portfolio (a promoted collection of artifacts).
+
+ The linked artifact will be visible in the UI for the specified portfolio.
+
+ Arguments:
+ artifact: the (public or local) artifact which will be linked
+ target_path: `str` - takes the following forms: {portfolio}, {project}/{portfolio},
+ or {entity}/{project}/{portfolio}
+ aliases: `List[str]` - optional alias(es) that will only be applied on this linked artifact
+ inside the portfolio.
+ The alias "latest" will always be applied to the latest version of an artifact that is linked.
+
+ Returns:
+ None
+
+ """
+ portfolio, project, entity = wandb.util._parse_entity_project_item(target_path)
+ if aliases is None:
+ aliases = []
+
+ if self._backend and self._backend.interface:
+ if artifact.is_draft() and not artifact._is_draft_save_started():
+ artifact = self._log_artifact(artifact)
+ # artifact logging is async, wait until the artifact is committed
+ # before trying to link it
+ artifact.wait()
+ if not self._settings._offline:
+ self._backend.interface.publish_link_artifact(
+ self,
+ artifact,
+ portfolio,
+ aliases,
+ entity,
+ project,
+ )
+ if artifact._ttl_duration_seconds is not None:
+ wandb.termwarn(
+ "Artifact TTL will be disabled for source artifacts that are linked to portfolios."
+ )
+ else:
+ # TODO: implement offline mode + sync
+ raise NotImplementedError
+
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def use_artifact(
+ self,
+ artifact_or_name: Union[str, Artifact],
+ type: Optional[str] = None,
+ aliases: Optional[List[str]] = None,
+ use_as: Optional[str] = None,
+ ) -> Artifact:
+ """Declare an artifact as an input to a run.
+
+ Call `download` or `file` on the returned object to get the contents locally.
+
+ Arguments:
+ artifact_or_name: (str or Artifact) An artifact name.
+ May be prefixed with entity/project/. Valid names
+ can be in the following forms:
+ - name:version
+ - name:alias
+ - digest
+ You can also pass an Artifact object created by calling `wandb.Artifact`
+ type: (str, optional) The type of artifact to use.
+ aliases: (list, optional) Aliases to apply to this artifact
+ use_as: (string, optional) Optional string indicating what purpose the artifact was used with.
+ Will be shown in UI.
+
+ Returns:
+ An `Artifact` object.
+ """
+ if self._settings._offline:
+ raise TypeError("Cannot use artifact when in offline mode.")
+ r = self._run_obj
+ assert r is not None
+ api = internal.Api(default_settings={"entity": r.entity, "project": r.project})
+ api.set_current_run_id(self._run_id)
+
+ if isinstance(artifact_or_name, str):
+ if self._launch_artifact_mapping:
+ name = self._swap_artifact_name(artifact_or_name, use_as)
+ else:
+ name = artifact_or_name
+ public_api = self._public_api()
+ artifact = public_api.artifact(type=type, name=name)
+ if type is not None and type != artifact.type:
+ raise ValueError(
+ "Supplied type {} does not match type {} of artifact {}".format(
+ type, artifact.type, artifact.name
+ )
+ )
+ artifact._use_as = use_as or artifact_or_name
+ if use_as:
+ if (
+ use_as in self._used_artifact_slots.keys()
+ and self._used_artifact_slots[use_as] != artifact.id
+ ):
+ raise ValueError(
+ "Cannot call use_artifact with the same use_as argument more than once"
+ )
+ elif ":" in use_as or "/" in use_as:
+ raise ValueError(
+ "use_as cannot contain special characters ':' or '/'"
+ )
+ self._used_artifact_slots[use_as] = artifact.id
+ api.use_artifact(
+ artifact.id,
+ entity_name=r.entity,
+ use_as=use_as or artifact_or_name,
+ )
+ else:
+ artifact = artifact_or_name
+ if aliases is None:
+ aliases = []
+ elif isinstance(aliases, str):
+ aliases = [aliases]
+ if isinstance(artifact_or_name, Artifact) and artifact.is_draft():
+ if use_as is not None:
+ wandb.termwarn(
+ "Indicating use_as is not supported when using a draft artifact"
+ )
+ self._log_artifact(
+ artifact,
+ aliases=aliases,
+ is_user_created=True,
+ use_after_commit=True,
+ )
+ artifact.wait()
+ artifact._use_as = use_as or artifact.name
+ elif isinstance(artifact, Artifact) and not artifact.is_draft():
+ if (
+ self._launch_artifact_mapping
+ and artifact.name in self._launch_artifact_mapping.keys()
+ ):
+ wandb.termwarn(
+ "Swapping artifacts is not supported when using a non-draft artifact. "
+ f"Using {artifact.name}."
+ )
+ artifact._use_as = use_as or artifact.name
+ api.use_artifact(
+ artifact.id, use_as=use_as or artifact._use_as or artifact.name
+ )
+ else:
+ raise ValueError(
+ 'You must pass an artifact name (e.g. "pedestrian-dataset:v1"), '
+ "an instance of `wandb.Artifact`, or `wandb.Api().artifact()` to `use_artifact`"
+ )
+ if self._backend and self._backend.interface:
+ self._backend.interface.publish_use_artifact(artifact)
+ return artifact
+
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def log_artifact(
+ self,
+ artifact_or_path: Union[Artifact, StrPath],
+ name: Optional[str] = None,
+ type: Optional[str] = None,
+ aliases: Optional[List[str]] = None,
+ ) -> Artifact:
+ """Declare an artifact as an output of a run.
+
+ Arguments:
+ artifact_or_path: (str or Artifact) A path to the contents of this artifact,
+ can be in the following forms:
+ - `/local/directory`
+ - `/local/directory/file.txt`
+ - `s3://bucket/path`
+ You can also pass an Artifact object created by calling
+ `wandb.Artifact`.
+ name: (str, optional) An artifact name. May be prefixed with entity/project.
+ Valid names can be in the following forms:
+ - name:version
+ - name:alias
+ - digest
+ This will default to the basename of the path prepended with the current
+ run id if not specified.
+ type: (str) The type of artifact to log, examples include `dataset`, `model`
+ aliases: (list, optional) Aliases to apply to this artifact,
+ defaults to `["latest"]`
+
+ Returns:
+ An `Artifact` object.
+ """
+ return self._log_artifact(
+ artifact_or_path, name=name, type=type, aliases=aliases
+ )
+
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def upsert_artifact(
+ self,
+ artifact_or_path: Union[Artifact, str],
+ name: Optional[str] = None,
+ type: Optional[str] = None,
+ aliases: Optional[List[str]] = None,
+ distributed_id: Optional[str] = None,
+ ) -> Artifact:
+ """Declare (or append to) a non-finalized artifact as output of a run.
+
+ Note that you must call run.finish_artifact() to finalize the artifact.
+ This is useful when distributed jobs need to all contribute to the same artifact.
+
+ Arguments:
+ artifact_or_path: (str or Artifact) A path to the contents of this artifact,
+ can be in the following forms:
+ - `/local/directory`
+ - `/local/directory/file.txt`
+ - `s3://bucket/path`
+ You can also pass an Artifact object created by calling
+ `wandb.Artifact`.
+ name: (str, optional) An artifact name. May be prefixed with entity/project.
+ Valid names can be in the following forms:
+ - name:version
+ - name:alias
+ - digest
+ This will default to the basename of the path prepended with the current
+ run id if not specified.
+ type: (str) The type of artifact to log, examples include `dataset`, `model`
+ aliases: (list, optional) Aliases to apply to this artifact,
+ defaults to `["latest"]`
+ distributed_id: (string, optional) Unique string that all distributed jobs share. If None,
+ defaults to the run's group name.
+
+ Returns:
+ An `Artifact` object.
+ """
+ if self._get_group() == "" and distributed_id is None:
+ raise TypeError(
+ "Cannot upsert artifact unless run is in a group or distributed_id is provided"
+ )
+ if distributed_id is None:
+ distributed_id = self._get_group()
+ return self._log_artifact(
+ artifact_or_path,
+ name=name,
+ type=type,
+ aliases=aliases,
+ distributed_id=distributed_id,
+ finalize=False,
+ )
+
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def finish_artifact(
+ self,
+ artifact_or_path: Union[Artifact, str],
+ name: Optional[str] = None,
+ type: Optional[str] = None,
+ aliases: Optional[List[str]] = None,
+ distributed_id: Optional[str] = None,
+ ) -> Artifact:
+ """Finishes a non-finalized artifact as output of a run.
+
+ Subsequent "upserts" with the same distributed ID will result in a new version.
+
+ Arguments:
+ artifact_or_path: (str or Artifact) A path to the contents of this artifact,
+ can be in the following forms:
+ - `/local/directory`
+ - `/local/directory/file.txt`
+ - `s3://bucket/path`
+ You can also pass an Artifact object created by calling
+ `wandb.Artifact`.
+ name: (str, optional) An artifact name. May be prefixed with entity/project.
+ Valid names can be in the following forms:
+ - name:version
+ - name:alias
+ - digest
+ This will default to the basename of the path prepended with the current
+ run id if not specified.
+ type: (str) The type of artifact to log, examples include `dataset`, `model`
+ aliases: (list, optional) Aliases to apply to this artifact,
+ defaults to `["latest"]`
+ distributed_id: (string, optional) Unique string that all distributed jobs share. If None,
+ defaults to the run's group name.
+
+ Returns:
+ An `Artifact` object.
+ """
+ if self._get_group() == "" and distributed_id is None:
+ raise TypeError(
+ "Cannot finish artifact unless run is in a group or distributed_id is provided"
+ )
+ if distributed_id is None:
+ distributed_id = self._get_group()
+
+ return self._log_artifact(
+ artifact_or_path,
+ name,
+ type,
+ aliases,
+ distributed_id=distributed_id,
+ finalize=True,
+ )
+
+ def _log_artifact(
+ self,
+ artifact_or_path: Union[Artifact, StrPath],
+ name: Optional[str] = None,
+ type: Optional[str] = None,
+ aliases: Optional[List[str]] = None,
+ distributed_id: Optional[str] = None,
+ finalize: bool = True,
+ is_user_created: bool = False,
+ use_after_commit: bool = False,
+ ) -> Artifact:
+ api = internal.Api()
+ if api.settings().get("anonymous") == "true":
+ wandb.termwarn(
+ "Artifacts logged anonymously cannot be claimed and expire after 7 days."
+ )
+ if not finalize and distributed_id is None:
+ raise TypeError("Must provide distributed_id if artifact is not finalize")
+ if aliases is not None:
+ if any(invalid in alias for alias in aliases for invalid in ["/", ":"]):
+ raise ValueError(
+ "Aliases must not contain any of the following characters: /, :"
+ )
+ artifact, aliases = self._prepare_artifact(
+ artifact_or_path, name, type, aliases
+ )
+ artifact.distributed_id = distributed_id
+ self._assert_can_log_artifact(artifact)
+ if self._backend and self._backend.interface:
+ if not self._settings._offline:
+ handle = self._backend.interface.deliver_artifact(
+ self,
+ artifact,
+ aliases,
+ self.step,
+ finalize=finalize,
+ is_user_created=is_user_created,
+ use_after_commit=use_after_commit,
+ )
+ handle.add_probe(self._on_probe_exit)
+ handle.add_progress(self._on_progress_exit)
+ artifact._set_save_handle(handle, self._public_api().client)
+ else:
+ self._backend.interface.publish_artifact(
+ self,
+ artifact,
+ aliases,
+ finalize=finalize,
+ is_user_created=is_user_created,
+ use_after_commit=use_after_commit,
+ )
+ elif self._internal_run_interface:
+ self._internal_run_interface.publish_artifact(
+ self,
+ artifact,
+ aliases,
+ finalize=finalize,
+ is_user_created=is_user_created,
+ use_after_commit=use_after_commit,
+ )
+ return artifact
+
+ def _public_api(self, overrides: Optional[Dict[str, str]] = None) -> PublicApi:
+ overrides = {"run": self._run_id}
+ if not (self._settings._offline or self._run_obj is None):
+ overrides["entity"] = self._run_obj.entity
+ overrides["project"] = self._run_obj.project
+ return public.Api(overrides)
+
+ # TODO(jhr): annotate this
+ def _assert_can_log_artifact(self, artifact) -> None: # type: ignore
+ if self._settings._offline:
+ return
+ try:
+ public_api = self._public_api()
+ entity = public_api.settings["entity"]
+ project = public_api.settings["project"]
+ expected_type = Artifact._expected_type(
+ entity, project, artifact.name, public_api.client
+ )
+ except requests.exceptions.RequestException:
+ # Just return early if there is a network error. This is
+ # ok, as this function is intended to help catch an invalid
+ # type early, but not a hard requirement for valid operation.
+ return
+ if expected_type is not None and artifact.type != expected_type:
+ raise ValueError(
+ f"Artifact {artifact.name} already exists with type '{expected_type}'; "
+ f"cannot create another with type '{artifact.type}'"
+ )
+ if entity and artifact._source_entity and entity != artifact._source_entity:
+ raise ValueError(
+ f"Artifact {artifact.name} is owned by entity '{entity}'; it can't be "
+ f"moved to '{artifact._source_entity}'"
+ )
+ if project and artifact._source_project and project != artifact._source_project:
+ raise ValueError(
+ f"Artifact {artifact.name} exists in project '{project}'; it can't be "
+ f"moved to '{artifact._source_project}'"
+ )
+
+ def _prepare_artifact(
+ self,
+ artifact_or_path: Union[Artifact, StrPath],
+ name: Optional[str] = None,
+ type: Optional[str] = None,
+ aliases: Optional[List[str]] = None,
+ ) -> Tuple[Artifact, List[str]]:
+ if isinstance(artifact_or_path, (str, os.PathLike)):
+ name = name or f"run-{self._run_id}-{os.path.basename(artifact_or_path)}"
+ artifact = wandb.Artifact(name, type or "unspecified")
+ if os.path.isfile(artifact_or_path):
+ artifact.add_file(str(artifact_or_path))
+ elif os.path.isdir(artifact_or_path):
+ artifact.add_dir(str(artifact_or_path))
+ elif "://" in str(artifact_or_path):
+ artifact.add_reference(str(artifact_or_path))
+ else:
+ raise ValueError(
+ "path must be a file, directory or external"
+ "reference like s3://bucket/path"
+ )
+ else:
+ artifact = artifact_or_path
+ if not isinstance(artifact, wandb.Artifact):
+ raise ValueError(
+ "You must pass an instance of wandb.Artifact or a "
+ "valid file path to log_artifact"
+ )
+ artifact.finalize()
+ return artifact, _resolve_aliases(aliases)
+
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def log_model(
+ self,
+ path: StrPath,
+ name: Optional[str] = None,
+ aliases: Optional[List[str]] = None,
+ ) -> None:
+ """Logs a model artifact containing the contents inside the 'path' to a run and marks it as an output to this run.
+
+ Arguments:
+ path: (str) A path to the contents of this model,
+ can be in the following forms:
+ - `/local/directory`
+ - `/local/directory/file.txt`
+ - `s3://bucket/path`
+ name: (str, optional) A name to assign to the model artifact that the file contents will be added to.
+ The string must contain only the following alphanumeric characters: dashes, underscores, and dots.
+ This will default to the basename of the path prepended with the current
+ run id if not specified.
+ aliases: (list, optional) Aliases to apply to the created model artifact,
+ defaults to `["latest"]`
+
+ Examples:
+ ```python
+ run.log_model(
+ path="/local/directory",
+ name="my_model_artifact",
+ aliases=["production"],
+ )
+ ```
+
+ Invalid usage
+ ```python
+ run.log_model(
+ path="/local/directory",
+ name="my_entity/my_project/my_model_artifact",
+ aliases=["production"],
+ )
+ ```
+
+ Raises:
+ ValueError: if name has invalid special characters
+
+ Returns:
+ None
+ """
+ self._log_artifact(
+ artifact_or_path=path, name=name, type="model", aliases=aliases
+ )
+
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def use_model(self, name: str) -> FilePathStr:
+ """Download the files logged in a model artifact 'name'.
+
+ Arguments:
+ name: (str) A model artifact name. 'name' must match the name of an existing logged
+ model artifact.
+ May be prefixed with entity/project/. Valid names
+ can be in the following forms:
+ - model_artifact_name:version
+ - model_artifact_name:alias
+ - model_artifact_name:digest.
+
+ Examples:
+ ```python
+ run.use_model(
+ name="my_model_artifact:latest",
+ )
+
+ run.use_model(
+ name="my_project/my_model_artifact:v0",
+ )
+
+ run.use_model(
+ name="my_entity/my_project/my_model_artifact:<digest>",
+ )
+ ```
+
+ Invalid usage
+ ```python
+ run.use_model(
+ name="my_entity/my_project/my_model_artifact",
+ )
+ ```
+
+ Raises:
+ AssertionError: if model artifact 'name' is of a type that does not contain the substring 'model'.
+
+ Returns:
+ path: (str) path to downloaded model artifact file(s).
+ """
+ artifact = self.use_artifact(artifact_or_name=name)
+ assert "model" in str(
+ artifact.type.lower()
+ ), "You can only use this method for 'model' artifacts. For an artifact to be a 'model' artifact, its type property must contain the substring 'model'."
+ path = artifact.download()
+
+ # If returned directory contains only one file, return path to that file
+ dir_list = os.listdir(path)
+ if len(dir_list) == 1:
+ return FilePathStr(os.path.join(path, dir_list[0]))
+ return path
+
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def link_model(
+ self,
+ path: StrPath,
+ registered_model_name: str,
+ name: Optional[str] = None,
+ aliases: Optional[List[str]] = None,
+ ) -> None:
+ """Log a model artifact version and link it to a registered model in the model registry.
+
+ The linked model version will be visible in the UI for the specified registered model.
+
+ Steps:
+ - Check if 'name' model artifact has been logged. If so, use the artifact version that matches the files
+ located at 'path' or log a new version. Otherwise log files under 'path' as a new model artifact, 'name'
+ of type 'model'.
+ - Check if registered model with name 'registered_model_name' exists in the 'model-registry' project.
+ If not, create a new registered model with name 'registered_model_name'.
+ - Link version of model artifact 'name' to registered model, 'registered_model_name'.
+ - Attach aliases from 'aliases' list to the newly linked model artifact version.
+
+ Arguments:
+ path: (str) A path to the contents of this model,
+ can be in the following forms:
+ - `/local/directory`
+ - `/local/directory/file.txt`
+ - `s3://bucket/path`
+ registered_model_name: (str) - the name of the registered model that the model is to be linked to.
+ A registered model is a collection of model versions linked to the model registry, typically representing a
+ team's specific ML Task. The entity that this registered model belongs to will be derived from the run
+ name: (str, optional) - the name of the model artifact that files in 'path' will be logged to. This will
+ default to the basename of the path prepended with the current run id if not specified.
+ aliases: (List[str], optional) - alias(es) that will only be applied on this linked artifact
+ inside the registered model.
+ The alias "latest" will always be applied to the latest version of an artifact that is linked.
+
+ Examples:
+ ```python
+ run.link_model(
+ path="/local/directory",
+ registered_model_name="my_reg_model",
+ name="my_model_artifact",
+ aliases=["production"],
+ )
+ ```
+
+ Invalid usage
+ ```python
+ run.link_model(
+ path="/local/directory",
+ registered_model_name="my_entity/my_project/my_reg_model",
+ name="my_model_artifact",
+ aliases=["production"],
+ )
+
+ run.link_model(
+ path="/local/directory",
+ registered_model_name="my_reg_model",
+ name="my_entity/my_project/my_model_artifact",
+ aliases=["production"],
+ )
+ ```
+
+ Raises:
+ AssertionError: if registered_model_name is a path or
+ if model artifact 'name' is of a type that does not contain the substring 'model'
+ ValueError: if name has invalid special characters
+
+ Returns:
+ None
+ """
+ name_parts = registered_model_name.split("/")
+ assert (
+ len(name_parts) == 1
+ ), "Please provide only the name of the registered model. Do not append the entity or project name."
+ project = "model-registry"
+ target_path = self.entity + "/" + project + "/" + registered_model_name
+
+ public_api = self._public_api()
+ try:
+ artifact = public_api.artifact(name=f"{name}:latest")
+ assert "model" in str(
+ artifact.type.lower()
+ ), "You can only use this method for 'model' artifacts. For an artifact to be a 'model' artifact, its type property must contain the substring 'model'."
+ artifact = self._log_artifact(
+ artifact_or_path=path, name=name, type=artifact.type
+ )
+ except (ValueError, wandb.CommError):
+ artifact = self._log_artifact(
+ artifact_or_path=path, name=name, type="model"
+ )
+ self.link_artifact(artifact=artifact, target_path=target_path, aliases=aliases)
+
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def alert(
+ self,
+ title: str,
+ text: str,
+ level: Optional[Union[str, "AlertLevel"]] = None,
+ wait_duration: Union[int, float, timedelta, None] = None,
+ ) -> None:
+ """Launch an alert with the given title and text.
+
+ Arguments:
+ title: (str) The title of the alert, must be less than 64 characters long.
+ text: (str) The text body of the alert.
+ level: (str or wandb.AlertLevel, optional) The alert level to use, either: `INFO`, `WARN`, or `ERROR`.
+ wait_duration: (int, float, or timedelta, optional) The time to wait (in seconds) before sending another
+ alert with this title.
+ """
+ level = level or wandb.AlertLevel.INFO
+ level_str: str = level.value if isinstance(level, wandb.AlertLevel) else level
+ if level_str not in {lev.value for lev in wandb.AlertLevel}:
+ raise ValueError("level must be one of 'INFO', 'WARN', or 'ERROR'")
+
+ wait_duration = wait_duration or timedelta(minutes=1)
+ if isinstance(wait_duration, int) or isinstance(wait_duration, float):
+ wait_duration = timedelta(seconds=wait_duration)
+ elif not callable(getattr(wait_duration, "total_seconds", None)):
+ raise ValueError(
+ "wait_duration must be an int, float, or datetime.timedelta"
+ )
+ wait_duration = int(wait_duration.total_seconds() * 1000)
+
+ if self._backend and self._backend.interface:
+ self._backend.interface.publish_alert(title, text, level_str, wait_duration)
+
+ def __enter__(self) -> "Run":
+ return self
+
+ def __exit__(
+ self,
+ exc_type: Type[BaseException],
+ exc_val: BaseException,
+ exc_tb: TracebackType,
+ ) -> bool:
+ exception_raised = exc_type is not None
+ if exception_raised:
+ traceback.print_exception(exc_type, exc_val, exc_tb)
+ exit_code = 1 if exception_raised else 0
+ self._finish(exit_code=exit_code)
+ return not exception_raised
+
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def mark_preempting(self) -> None:
+ """Mark this run as preempting.
+
+ Also tells the internal process to immediately report this to server.
+ """
+ if self._backend and self._backend.interface:
+ self._backend.interface.publish_preempting()
+
+ @property
+ @_run_decorator._noop_on_finish()
+ @_run_decorator._attach
+ def _system_metrics(self) -> Dict[str, List[Tuple[datetime, float]]]:
+ """Returns a dictionary of system metrics.
+
+ Returns:
+ A dictionary of system metrics.
+ """
+
+ def pb_to_dict(
+ system_metrics_pb: wandb.proto.wandb_internal_pb2.GetSystemMetricsResponse,
+ ) -> Dict[str, List[Tuple[datetime, float]]]:
+ res = {}
+
+ for metric, records in system_metrics_pb.system_metrics.items():
+ measurements = []
+ for record in records.record:
+ # Convert timestamp to datetime
+ dt = datetime.fromtimestamp(
+ record.timestamp.seconds, tz=timezone.utc
+ )
+ dt = dt.replace(microsecond=record.timestamp.nanos // 1000)
+
+ measurements.append((dt, record.value))
+
+ res[metric] = measurements
+
+ return res
+
+ if not self._backend or not self._backend.interface:
+ return {}
+
+ handle = self._backend.interface.deliver_get_system_metrics()
+ result = handle.wait(timeout=1)
+
+ if result:
+ try:
+ response = result.response.get_system_metrics_response
+ if response:
+ return pb_to_dict(response)
+ except Exception as e:
+ logger.error("Error getting system metrics: %s", e)
+ return {}
+
+ # ------------------------------------------------------------------------------
+ # HEADER
+ # ------------------------------------------------------------------------------
+ # Note: All the header methods are static methods since we want to share the printing logic
+ # with the service execution path that doesn't have access to the run instance
+ @staticmethod
+ def _header(
+ check_version: Optional["CheckVersionResponse"] = None,
+ *,
+ settings: "Settings",
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ Run._header_version_check_info(
+ check_version, settings=settings, printer=printer
+ )
+ Run._header_wandb_version_info(settings=settings, printer=printer)
+ Run._header_sync_info(settings=settings, printer=printer)
+ Run._header_run_info(settings=settings, printer=printer)
+
+ @staticmethod
+ def _header_version_check_info(
+ check_version: Optional["CheckVersionResponse"] = None,
+ *,
+ settings: "Settings",
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ if not check_version or settings._offline:
+ return
+
+ if check_version.delete_message:
+ printer.display(check_version.delete_message, level="error")
+ elif check_version.yank_message:
+ printer.display(check_version.yank_message, level="warn")
+
+ printer.display(
+ check_version.upgrade_message, off=not check_version.upgrade_message
+ )
+
+ @staticmethod
+ def _header_wandb_version_info(
+ *,
+ settings: "Settings",
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ if settings.quiet or settings.silent:
+ return
+
+ # TODO: add this to a higher verbosity level
+ printer.display(
+ f"Tracking run with wandb version {wandb.__version__}", off=False
+ )
+
+ @staticmethod
+ def _header_sync_info(
+ *,
+ settings: "Settings",
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ if settings._offline:
+ printer.display(
+ [
+ f"W&B syncing is set to {printer.code('`offline`')} in this directory. ",
+ f"Run {printer.code('`wandb online`')} or set {printer.code('WANDB_MODE=online')} "
+ "to enable cloud syncing.",
+ ]
+ )
+ else:
+ info = [f"Run data is saved locally in {printer.files(settings.sync_dir)}"]
+ if not printer._html:
+ info.append(
+ f"Run {printer.code('`wandb offline`')} to turn off syncing."
+ )
+ printer.display(info, off=settings.quiet or settings.silent)
+
+ @staticmethod
+ def _header_run_info(
+ *,
+ settings: "Settings",
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ if settings._offline or settings.silent:
+ return
+
+ run_url = settings.run_url
+ project_url = settings.project_url
+ sweep_url = settings.sweep_url
+
+ run_state_str = "Resuming run" if settings.resumed else "Syncing run"
+ run_name = settings.run_name
+ if not run_name:
+ return
+
+ if printer._html:
+ if not wandb.jupyter.maybe_display():
+ run_line = f"<strong>{printer.link(run_url, run_name)}</strong>"
+ project_line, sweep_line = "", ""
+
+ # TODO(settings): make settings the source of truth
+ if not wandb.jupyter.quiet():
+ doc_html = printer.link(wburls.get("doc_run"), "docs")
+
+ project_html = printer.link(project_url, "Weights & Biases")
+ project_line = f"to {project_html} ({doc_html})"
+
+ if sweep_url:
+ sweep_line = f"Sweep page: {printer.link(sweep_url, sweep_url)}"
+
+ printer.display(
+ [f"{run_state_str} {run_line} {project_line}", sweep_line],
+ )
+
+ else:
+ printer.display(
+ f"{run_state_str} {printer.name(run_name)}", off=not run_name
+ )
+
+ if not settings.quiet:
+ # TODO: add verbosity levels and add this to higher levels
+ printer.display(
+ f'{printer.emoji("star")} View project at {printer.link(project_url)}'
+ )
+ if sweep_url:
+ printer.display(
+ f'{printer.emoji("broom")} View sweep at {printer.link(sweep_url)}'
+ )
+ printer.display(
+ f'{printer.emoji("rocket")} View run at {printer.link(run_url)}',
+ )
+
+ # TODO(settings) use `wandb_settings` (if self.settings.anonymous == "true":)
+ if Api().api.settings().get("anonymous") == "true":
+ printer.display(
+ "Do NOT share these links with anyone. They can be used to claim your runs.",
+ level="warn",
+ off=not run_name,
+ )
+
+ # ------------------------------------------------------------------------------
+ # FOOTER
+ # ------------------------------------------------------------------------------
+ # Note: All the footer methods are static methods since we want to share the printing logic
+ # with the service execution path that doesn't have acess to the run instance
+ @staticmethod
+ def _footer(
+ sampled_history: Optional["SampledHistoryResponse"] = None,
+ final_summary: Optional["GetSummaryResponse"] = None,
+ poll_exit_response: Optional[PollExitResponse] = None,
+ server_info_response: Optional[ServerInfoResponse] = None,
+ check_version_response: Optional["CheckVersionResponse"] = None,
+ internal_messages_response: Optional["InternalMessagesResponse"] = None,
+ job_info: Optional["JobInfoResponse"] = None,
+ reporter: Optional[Reporter] = None,
+ quiet: Optional[bool] = None,
+ *,
+ settings: "Settings",
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ Run._footer_history_summary_info(
+ history=sampled_history,
+ summary=final_summary,
+ quiet=quiet,
+ settings=settings,
+ printer=printer,
+ )
+
+ Run._footer_sync_info(
+ poll_exit_response=poll_exit_response,
+ job_info=job_info,
+ quiet=quiet,
+ settings=settings,
+ printer=printer,
+ )
+ Run._footer_log_dir_info(quiet=quiet, settings=settings, printer=printer)
+ Run._footer_version_check_info(
+ check_version=check_version_response,
+ quiet=quiet,
+ settings=settings,
+ printer=printer,
+ )
+ Run._footer_local_warn(
+ server_info_response=server_info_response,
+ quiet=quiet,
+ settings=settings,
+ printer=printer,
+ )
+ Run._footer_internal_messages(
+ internal_messages_response=internal_messages_response,
+ quiet=quiet,
+ settings=settings,
+ printer=printer,
+ )
+ Run._footer_reporter_warn_err(
+ reporter=reporter, quiet=quiet, settings=settings, printer=printer
+ )
+ Run._footer_server_messages(
+ server_info_response=server_info_response,
+ quiet=quiet,
+ settings=settings,
+ printer=printer,
+ )
+
+ @staticmethod
+ def _footer_exit_status_info(
+ exit_code: Optional[int],
+ *,
+ settings: "Settings",
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ if settings.silent:
+ return
+
+ status = "(success)." if not exit_code else f"(failed {exit_code})."
+ info = [
+ f"Waiting for W&B process to finish... {printer.status(status, bool(exit_code))}"
+ ]
+
+ if not settings._offline and exit_code:
+ info.append(f"Press {printer.abort()} to abort syncing.")
+
+ printer.display(f'{" ".join(info)}')
+
+ # fixme: Temporary hack until we move to rich which allows multiple spinners
+ @staticmethod
+ def _footer_file_pusher_status_info(
+ poll_exit_responses: Optional[
+ Union[PollExitResponse, List[Optional[PollExitResponse]]]
+ ] = None,
+ *,
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ if not poll_exit_responses:
+ return
+ if isinstance(poll_exit_responses, PollExitResponse):
+ Run._footer_single_run_file_pusher_status_info(
+ poll_exit_responses, printer=printer
+ )
+ elif isinstance(poll_exit_responses, list):
+ poll_exit_responses_list = poll_exit_responses
+ assert all(
+ response is None or isinstance(response, PollExitResponse)
+ for response in poll_exit_responses_list
+ )
+ if len(poll_exit_responses_list) == 0:
+ return
+ elif len(poll_exit_responses_list) == 1:
+ Run._footer_single_run_file_pusher_status_info(
+ poll_exit_responses_list[0], printer=printer
+ )
+ else:
+ Run._footer_multiple_runs_file_pusher_status_info(
+ poll_exit_responses_list, printer=printer
+ )
+ else:
+ logger.error(
+ f"Got the type `{type(poll_exit_responses)}` for `poll_exit_responses`. "
+ "Expected either None, PollExitResponse or a List[Union[PollExitResponse, None]]"
+ )
+
+ @staticmethod
+ def _footer_single_run_file_pusher_status_info(
+ poll_exit_response: Optional[PollExitResponse] = None,
+ *,
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ # todo: is this same as settings._offline?
+ if not poll_exit_response:
+ return
+
+ progress = poll_exit_response.pusher_stats
+ done = poll_exit_response.done
+
+ megabyte = wandb.util.POW_2_BYTES[2][1]
+ line = f"{progress.uploaded_bytes / megabyte :.3f} MB of {progress.total_bytes / megabyte:.3f} MB uploaded"
+ if progress.deduped_bytes > 0:
+ line += f" ({progress.deduped_bytes / megabyte:.3f} MB deduped)\r"
+ else:
+ line += "\r"
+
+ percent_done = (
+ 1.0
+ if progress.total_bytes == 0
+ else progress.uploaded_bytes / progress.total_bytes
+ )
+
+ printer.progress_update(line, percent_done)
+ if done:
+ printer.progress_close()
+
+ dedupe_fraction = (
+ progress.deduped_bytes / float(progress.total_bytes)
+ if progress.total_bytes > 0
+ else 0
+ )
+ if dedupe_fraction > 0.01:
+ printer.display(
+ f"W&B sync reduced upload amount by {dedupe_fraction * 100:.1f}% "
+ )
+
+ @staticmethod
+ def _footer_multiple_runs_file_pusher_status_info(
+ poll_exit_responses: List[Optional[PollExitResponse]],
+ *,
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ # todo: is this same as settings._offline?
+ if not all(poll_exit_responses):
+ return
+
+ megabyte = wandb.util.POW_2_BYTES[2][1]
+ total_files: int = sum(
+ sum(
+ [
+ response.file_counts.wandb_count,
+ response.file_counts.media_count,
+ response.file_counts.artifact_count,
+ response.file_counts.other_count,
+ ]
+ )
+ for response in poll_exit_responses
+ if response is not None and response.file_counts is not None
+ )
+ uploaded = sum(
+ response.pusher_stats.uploaded_bytes
+ for response in poll_exit_responses
+ if response is not None and response.pusher_stats is not None
+ )
+ total = sum(
+ response.pusher_stats.total_bytes
+ for response in poll_exit_responses
+ if response is not None and response.pusher_stats is not None
+ )
+
+ line = (
+ f"Processing {len(poll_exit_responses)} runs with {total_files} files "
+ f"({uploaded/megabyte :.2f} MB/{total/megabyte :.2f} MB)\r"
+ )
+ # line = "{}{:<{max_len}}\r".format(line, " ", max_len=(80 - len(line)))
+ printer.progress_update(line) # type:ignore[call-arg]
+
+ done = all(
+ [
+ poll_exit_response.done
+ for poll_exit_response in poll_exit_responses
+ if poll_exit_response
+ ]
+ )
+ if done:
+ printer.progress_close()
+
+ @staticmethod
+ def _footer_sync_info(
+ poll_exit_response: Optional[PollExitResponse] = None,
+ job_info: Optional["JobInfoResponse"] = None,
+ quiet: Optional[bool] = None,
+ *,
+ settings: "Settings",
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ if settings.silent:
+ return
+
+ if settings._offline:
+ printer.display(
+ [
+ "You can sync this run to the cloud by running:",
+ printer.code(f"wandb sync {settings.sync_dir}"),
+ ],
+ off=(quiet or settings.quiet),
+ )
+ else:
+ info = []
+ if settings.run_name and settings.run_url:
+ info = [
+ f"{printer.emoji('rocket')} View run {printer.name(settings.run_name)} at: {printer.link(settings.run_url)}"
+ ]
+ if job_info and job_info.version and job_info.sequenceId:
+ link = f"{settings.project_url}/jobs/{job_info.sequenceId}/version_details/{job_info.version}"
+ info.append(
+ f"{printer.emoji('lightning')} View job at {printer.link(link)}",
+ )
+ if poll_exit_response and poll_exit_response.file_counts:
+ logger.info("logging synced files")
+ file_counts = poll_exit_response.file_counts
+ info.append(
+ f"Synced {file_counts.wandb_count} W&B file(s), {file_counts.media_count} media file(s), "
+ f"{file_counts.artifact_count} artifact file(s) and {file_counts.other_count} other file(s)",
+ )
+ printer.display(info)
+
+ @staticmethod
+ def _footer_log_dir_info(
+ quiet: Optional[bool] = None,
+ *,
+ settings: "Settings",
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ if (quiet or settings.quiet) or settings.silent:
+ return
+
+ log_dir = settings.log_user or settings.log_internal
+ if log_dir:
+ log_dir = os.path.dirname(log_dir.replace(os.getcwd(), "."))
+ printer.display(
+ f"Find logs at: {printer.files(log_dir)}",
+ )
+
+ @staticmethod
+ def _footer_history_summary_info(
+ history: Optional["SampledHistoryResponse"] = None,
+ summary: Optional["GetSummaryResponse"] = None,
+ quiet: Optional[bool] = None,
+ *,
+ settings: "Settings",
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ if (quiet or settings.quiet) or settings.silent:
+ return
+
+ panel = []
+
+ # Render history if available
+ if history:
+ logger.info("rendering history")
+
+ sampled_history = {
+ item.key: wandb.util.downsample(
+ item.values_float or item.values_int, 40
+ )
+ for item in history.item
+ if not item.key.startswith("_")
+ }
+
+ history_rows = []
+ for key, values in sorted(sampled_history.items()):
+ if any(not isinstance(value, numbers.Number) for value in values):
+ continue
+ sparkline = printer.sparklines(values)
+ if sparkline:
+ history_rows.append([key, sparkline])
+ if history_rows:
+ history_grid = printer.grid(
+ history_rows,
+ "Run history:",
+ )
+ panel.append(history_grid)
+
+ # Render summary if available
+ if summary:
+ final_summary = {
+ item.key: json.loads(item.value_json)
+ for item in summary.item
+ if not item.key.startswith("_")
+ }
+
+ logger.info("rendering summary")
+ summary_rows = []
+ for key, value in sorted(final_summary.items()):
+ # arrays etc. might be too large. for now, we just don't print them
+ if isinstance(value, str):
+ value = value[:20] + "..." * (len(value) >= 20)
+ summary_rows.append([key, value])
+ elif isinstance(value, numbers.Number):
+ value = round(value, 5) if isinstance(value, float) else value
+ summary_rows.append([key, str(value)])
+ else:
+ continue
+
+ if summary_rows:
+ summary_grid = printer.grid(
+ summary_rows,
+ "Run summary:",
+ )
+ panel.append(summary_grid)
+
+ if panel:
+ printer.display(printer.panel(panel))
+
+ @staticmethod
+ def _footer_local_warn(
+ server_info_response: Optional[ServerInfoResponse] = None,
+ quiet: Optional[bool] = None,
+ *,
+ settings: "Settings",
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ if (quiet or settings.quiet) or settings.silent:
+ return
+
+ if settings._offline:
+ return
+
+ if not server_info_response or not server_info_response.local_info:
+ return
+
+ if settings.is_local:
+ local_info = server_info_response.local_info
+ latest_version, out_of_date = local_info.version, local_info.out_of_date
+ if out_of_date:
+ printer.display(
+ f"Upgrade to the {latest_version} version of W&B Server to get the latest features. "
+ f"Learn more: {printer.link(wburls.get('upgrade_server'))}",
+ level="warn",
+ )
+
+ @staticmethod
+ def _footer_internal_messages(
+ internal_messages_response: Optional["InternalMessagesResponse"] = None,
+ quiet: Optional[bool] = None,
+ *,
+ settings: "Settings",
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ if (quiet or settings.quiet) or settings.silent:
+ return
+
+ if not internal_messages_response:
+ return
+
+ for message in internal_messages_response.messages.warning:
+ printer.display(message, level="warn")
+
+ @staticmethod
+ def _footer_server_messages(
+ server_info_response: Optional[ServerInfoResponse] = None,
+ quiet: Optional[bool] = None,
+ *,
+ settings: "Settings",
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ if (quiet or settings.quiet) or settings.silent:
+ return
+
+ if settings.disable_hints:
+ return
+
+ if server_info_response and server_info_response.server_messages:
+ for message in server_info_response.server_messages.item:
+ printer.display(
+ message.html_text if printer._html else message.utf_text,
+ default_text=message.plain_text,
+ level=message.level,
+ off=message.type.lower() != "footer",
+ )
+
+ @staticmethod
+ def _footer_version_check_info(
+ check_version: Optional["CheckVersionResponse"] = None,
+ quiet: Optional[bool] = None,
+ *,
+ settings: "Settings",
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ if not check_version:
+ return
+
+ if settings._offline:
+ return
+
+ if (quiet or settings.quiet) or settings.silent:
+ return
+
+ if check_version.delete_message:
+ printer.display(check_version.delete_message, level="error")
+ elif check_version.yank_message:
+ printer.display(check_version.yank_message, level="warn")
+
+ # only display upgrade message if packages are bad
+ package_problem = check_version.delete_message or check_version.yank_message
+ if package_problem and check_version.upgrade_message:
+ printer.display(check_version.upgrade_message)
+
+ @staticmethod
+ def _footer_reporter_warn_err(
+ reporter: Optional[Reporter] = None,
+ quiet: Optional[bool] = None,
+ *,
+ settings: "Settings",
+ printer: Union["PrinterTerm", "PrinterJupyter"],
+ ) -> None:
+ if (quiet or settings.quiet) or settings.silent:
+ return
+
+ if not reporter:
+ return
+
+ warning_lines = reporter.warning_lines
+ if warning_lines:
+ warnings = ["Warnings:"] + [f"{line}" for line in warning_lines]
+ if len(warning_lines) < reporter.warning_count:
+ warnings.append("More warnings...")
+ printer.display(warnings)
+
+ error_lines = reporter.error_lines
+ if error_lines:
+ errors = ["Errors:"] + [f"{line}" for line in error_lines]
+ if len(error_lines) < reporter.error_count:
+ errors.append("More errors...")
+ printer.display(errors)
+
+
+# We define this outside of the run context to support restoring before init
+def restore(
+ name: str,
+ run_path: Optional[str] = None,
+ replace: bool = False,
+ root: Optional[str] = None,
+) -> Union[None, TextIO]:
+ """Download the specified file from cloud storage.
+
+ File is placed into the current directory or run directory.
+ By default, will only download the file if it doesn't already exist.
+
+ Arguments:
+ name: the name of the file
+ run_path: optional path to a run to pull files from, i.e. `username/project_name/run_id`
+ if wandb.init has not been called, this is required.
+ replace: whether to download the file even if it already exists locally
+ root: the directory to download the file to. Defaults to the current
+ directory or the run directory if wandb.init was called.
+
+ Returns:
+ None if it can't find the file, otherwise a file object open for reading
+
+ Raises:
+ wandb.CommError: if we can't connect to the wandb backend
+ ValueError: if the file is not found or can't find run_path
+ """
+ is_disabled = wandb.run is not None and wandb.run.disabled
+ run = None if is_disabled else wandb.run
+ if run_path is None:
+ if run is not None:
+ run_path = run.path
+ else:
+ raise ValueError(
+ "run_path required when calling wandb.restore before wandb.init"
+ )
+ if root is None:
+ if run is not None:
+ root = run.dir
+ api = public.Api()
+ api_run = api.run(run_path)
+ if root is None:
+ root = os.getcwd()
+ path = os.path.join(root, name)
+ if os.path.exists(path) and replace is False:
+ return open(path)
+ if is_disabled:
+ return None
+ files = api_run.files([name])
+ if len(files) == 0:
+ return None
+ # if the file does not exist, the file has an md5 of 0
+ if files[0].md5 == "0":
+ raise ValueError(f"File {name} not found in {run_path or root}.")
+ return files[0].download(root=root, replace=True)
+
+
+# propagate our doc string to the runs restore method
+try:
+ Run.restore.__doc__ = restore.__doc__
+except AttributeError:
+ pass
+
+
+def finish(exit_code: Optional[int] = None, quiet: Optional[bool] = None) -> None:
+ """Mark a run as finished, and finish uploading all data.
+
+ This is used when creating multiple runs in the same process.
+ We automatically call this method when your script exits.
+
+ Arguments:
+ exit_code: Set to something other than 0 to mark a run as failed
+ quiet: Set to true to minimize log output
+ """
+ if wandb.run:
+ wandb.run.finish(exit_code=exit_code, quiet=quiet)
diff --git a/wandb/sdk/wandb_settings.py b/wandb/sdk/wandb_settings.py
new file mode 100644
index 0000000000000000000000000000000000000000..7c152389d76b4d3d2b1c8ec12e13c7c1af63f941
--- /dev/null
+++ b/wandb/sdk/wandb_settings.py
@@ -0,0 +1,1930 @@
+import collections.abc
+import configparser
+import enum
+import getpass
+import json
+import logging
+import multiprocessing
+import os
+import platform
+import re
+import shutil
+import socket
+import sys
+import tempfile
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from distutils.util import strtobool
+from functools import reduce
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ FrozenSet,
+ ItemsView,
+ Iterable,
+ Mapping,
+ Optional,
+ Sequence,
+ Set,
+ Tuple,
+ Union,
+ no_type_check,
+)
+from urllib.parse import quote, unquote, urlencode, urlparse, urlsplit
+
+from google.protobuf.wrappers_pb2 import BoolValue, DoubleValue, Int32Value, StringValue
+
+import wandb
+import wandb.env
+from wandb import util
+from wandb.apis.internal import Api
+from wandb.errors import UsageError
+from wandb.proto import wandb_settings_pb2
+from wandb.sdk.internal.system.env_probe_helpers import is_aws_lambda
+from wandb.sdk.lib import filesystem
+from wandb.sdk.lib._settings_toposort_generated import SETTINGS_TOPOLOGICALLY_SORTED
+from wandb.sdk.wandb_setup import _EarlyLogger
+
+from .lib import apikey
+from .lib.gitlib import GitRepo
+from .lib.ipython import _get_python_type
+from .lib.runid import generate_id
+
+if sys.version_info >= (3, 8):
+ from typing import get_args, get_origin, get_type_hints
+else:
+ from typing_extensions import get_args, get_origin, get_type_hints
+
+
+class SettingsPreprocessingError(UsageError):
+ """Raised when the value supplied to a wandb.Settings() setting does not pass preprocessing."""
+
+
+class SettingsValidationError(UsageError):
+ """Raised when the value supplied to a wandb.Settings() setting does not pass validation."""
+
+
+class SettingsUnexpectedArgsError(UsageError):
+ """Raised when unexpected arguments are passed to wandb.Settings()."""
+
+
+def _get_wandb_dir(root_dir: str) -> str:
+ """Get the full path to the wandb directory.
+
+ The setting exposed to users as `dir=` or `WANDB_DIR` is the `root_dir`.
+ We add the `__stage_dir__` to it to get the full `wandb_dir`
+ """
+ # We use the hidden version if it already exists, otherwise non-hidden.
+ if os.path.exists(os.path.join(root_dir, ".wandb")):
+ __stage_dir__ = ".wandb" + os.sep
+ else:
+ __stage_dir__ = "wandb" + os.sep
+
+ path = os.path.join(root_dir, __stage_dir__)
+ if not os.access(root_dir or ".", os.W_OK):
+ wandb.termwarn(
+ f"Path {path} wasn't writable, using system temp directory.",
+ repeat=False,
+ )
+ path = os.path.join(tempfile.gettempdir(), __stage_dir__ or ("wandb" + os.sep))
+
+ return os.path.expanduser(path)
+
+
+def _str_as_bool(val: Union[str, bool]) -> bool:
+ """Parse a string as a bool."""
+ if isinstance(val, bool):
+ return val
+ try:
+ ret_val = bool(strtobool(str(val)))
+ return ret_val
+ except (AttributeError, ValueError):
+ pass
+
+ raise UsageError(f"Could not parse value {val} as a bool.")
+
+
+def _str_as_json(val: Union[str, Dict[str, Any]]) -> Any:
+ """Parse a string as a json object."""
+ if not isinstance(val, str):
+ return val
+ try:
+ return json.loads(val)
+ except (AttributeError, ValueError):
+ pass
+
+ raise UsageError(f"Could not parse value {val} as JSON.")
+
+
+def _str_as_tuple(val: Union[str, Sequence[str]]) -> Tuple[str, ...]:
+ """Parse a (potentially comma-separated) string as a tuple."""
+ if isinstance(val, str):
+ return tuple(val.split(","))
+ return tuple(val)
+
+
+def _datetime_as_str(val: Union[datetime, str]) -> str:
+ """Parse a datetime object as a string."""
+ if isinstance(val, datetime):
+ return datetime.strftime(val, "%Y%m%d_%H%M%S")
+ return val
+
+
+def _redact_dict(
+ d: Dict[str, Any],
+ unsafe_keys: Union[Set[str], FrozenSet[str]] = frozenset({"api_key"}),
+ redact_str: str = "***REDACTED***",
+) -> Dict[str, Any]:
+ """Redact a dict of unsafe values specified by their key."""
+ if not d or unsafe_keys.isdisjoint(d):
+ return d
+ safe_dict = d.copy()
+ safe_dict.update({k: redact_str for k in unsafe_keys.intersection(d)})
+ return safe_dict
+
+
+def _get_program() -> Optional[str]:
+ program = os.getenv(wandb.env.PROGRAM)
+ if program is not None:
+ return program
+ try:
+ import __main__
+
+ if __main__.__spec__ is None:
+ return __main__.__file__
+ # likely run as `python -m ...`
+ return f"-m {__main__.__spec__.name}"
+ except (ImportError, AttributeError):
+ return None
+
+
+def _get_program_relpath(
+ program: str, root: Optional[str] = None, _logger: Optional[_EarlyLogger] = None
+) -> Optional[str]:
+ if not program:
+ if _logger is not None:
+ _logger.warning("Empty program passed to get_program_relpath")
+ return None
+
+ root = root or os.getcwd()
+ if not root:
+ return None
+
+ full_path_to_program = os.path.join(
+ root, os.path.relpath(os.getcwd(), root), program
+ )
+ if os.path.exists(full_path_to_program):
+ relative_path = os.path.relpath(full_path_to_program, start=root)
+ if "../" in relative_path:
+ if _logger is not None:
+ _logger.warning(f"Could not save program above cwd: {program}")
+ return None
+ return relative_path
+
+ if _logger is not None:
+ _logger.warning(f"Could not find program at {program}")
+ return None
+
+
+def is_instance_recursive(obj: Any, type_hint: Any) -> bool: # noqa: C901
+ if type_hint is Any:
+ return True
+
+ origin = get_origin(type_hint)
+ args = get_args(type_hint)
+
+ if origin is None:
+ return isinstance(obj, type_hint)
+
+ if origin is Union:
+ return any(is_instance_recursive(obj, arg) for arg in args)
+
+ if issubclass(origin, collections.abc.Mapping):
+ if not isinstance(obj, collections.abc.Mapping):
+ return False
+ key_type, value_type = args
+
+ for key, value in obj.items():
+ if not is_instance_recursive(key, key_type) or not is_instance_recursive(
+ value, value_type
+ ):
+ return False
+
+ return True
+
+ if issubclass(origin, collections.abc.Sequence):
+ if not isinstance(obj, collections.abc.Sequence) or isinstance(
+ obj, (str, bytes, bytearray)
+ ):
+ return False
+
+ if len(args) == 1 and args[0] != ...:
+ (item_type,) = args
+ for item in obj:
+ if not is_instance_recursive(item, item_type):
+ return False
+ elif len(args) == 2 and args[-1] == ...:
+ item_type = args[0]
+ for item in obj:
+ if not is_instance_recursive(item, item_type):
+ return False
+ elif len(args) == len(obj):
+ for item, item_type in zip(obj, args):
+ if not is_instance_recursive(item, item_type):
+ return False
+ else:
+ return False
+
+ return True
+
+ if issubclass(origin, collections.abc.Set):
+ if not isinstance(obj, collections.abc.Set):
+ return False
+
+ (item_type,) = args
+ for item in obj:
+ if not is_instance_recursive(item, item_type):
+ return False
+
+ return True
+
+ return False
+
+
+@enum.unique
+class Source(enum.IntEnum):
+ OVERRIDE: int = 0
+ BASE: int = 1 # todo: audit this
+ ORG: int = 2
+ ENTITY: int = 3
+ PROJECT: int = 4
+ USER: int = 5
+ SYSTEM: int = 6
+ WORKSPACE: int = 7
+ ENV: int = 8
+ SETUP: int = 9
+ LOGIN: int = 10
+ INIT: int = 11
+ SETTINGS: int = 12
+ ARGS: int = 13
+ RUN: int = 14
+
+
+ConsoleValue = {
+ "auto",
+ "off",
+ "wrap",
+ "redirect",
+ # internal console states
+ "wrap_raw",
+ "wrap_emu",
+}
+
+
+@dataclass()
+class SettingsData:
+ """Settings for the W&B SDK."""
+
+ _args: Sequence[str]
+ _aws_lambda: bool
+ _async_upload_concurrency_limit: int
+ _cli_only_mode: bool # Avoid running any code specific for runs
+ _colab: bool
+ # _config_dict: Config
+ _cuda: str
+ _disable_meta: bool # Do not collect system metadata
+ _disable_service: (
+ bool
+ ) # Disable wandb-service, spin up internal process the old way
+ _disable_setproctitle: bool # Do not use setproctitle on internal process
+ _disable_stats: bool # Do not collect system metrics
+ _disable_viewer: bool # Prevent early viewer query
+ _disable_machine_info: bool # Disable automatic machine info collection
+ _except_exit: bool
+ _executable: str
+ _extra_http_headers: Mapping[str, str]
+ # file stream retry client configuration
+ _file_stream_retry_max: int # max number of retries
+ _file_stream_retry_wait_min_seconds: float # min wait time between retries
+ _file_stream_retry_wait_max_seconds: float # max wait time between retries
+ _file_stream_timeout_seconds: float # timeout for individual HTTP requests
+ # file transfer retry client configuration
+ _file_transfer_retry_max: int
+ _file_transfer_retry_wait_min_seconds: float
+ _file_transfer_retry_wait_max_seconds: float
+ _file_transfer_timeout_seconds: float
+ _flow_control_custom: bool
+ _flow_control_disabled: bool
+ # graphql retry client configuration
+ _graphql_retry_max: int
+ _graphql_retry_wait_min_seconds: float
+ _graphql_retry_wait_max_seconds: float
+ _graphql_timeout_seconds: float
+ _internal_check_process: float
+ _internal_queue_timeout: float
+ _ipython: bool
+ _jupyter: bool
+ _jupyter_name: str
+ _jupyter_path: str
+ _jupyter_root: str
+ _kaggle: bool
+ _live_policy_rate_limit: int
+ _live_policy_wait_time: int
+ _log_level: int
+ _network_buffer: int
+ _noop: bool
+ _notebook: bool
+ _offline: bool
+ _sync: bool
+ _os: str
+ _platform: str
+ _proxies: Mapping[str, str] # dedicated global proxy servers [scheme -> url]
+ _python: str
+ _runqueue_item_id: str
+ _require_core: bool
+ _save_requirements: bool
+ _service_transport: str
+ _service_wait: float
+ _shared: bool
+ _start_datetime: str
+ _start_time: float
+ _stats_pid: int # (internal) base pid for system stats
+ _stats_sample_rate_seconds: float
+ _stats_samples_to_average: int
+ _stats_join_assets: (
+ bool
+ ) # join metrics from different assets before sending to backend
+ _stats_neuron_monitor_config_path: (
+ str
+ ) # path to place config file for neuron-monitor (AWS Trainium)
+ _stats_open_metrics_endpoints: Mapping[str, str] # open metrics endpoint names/urls
+ # open metrics filters in one of the two formats:
+ # - {"metric regex pattern, including endpoint name as prefix": {"label": "label value regex pattern"}}
+ # - ("metric regex pattern 1", "metric regex pattern 2", ...)
+ _stats_open_metrics_filters: Union[Sequence[str], Mapping[str, Mapping[str, str]]]
+ _stats_disk_paths: Sequence[str] # paths to monitor disk usage
+ _stats_buffer_size: (
+ int
+ ) # number of consolidated samples to buffer before flushing, available in run obj
+ _tmp_code_dir: str
+ _tracelog: str
+ _unsaved_keys: Sequence[str]
+ _windows: bool
+ allow_val_change: bool
+ anonymous: str
+ api_key: str
+ azure_account_url_to_access_key: Dict[str, str]
+ base_url: str # The base url for the wandb api
+ code_dir: str
+ colab_url: str
+ config_paths: Sequence[str]
+ console: str
+ deployment: str
+ disable_code: bool
+ disable_git: bool
+ disable_hints: bool
+ disable_job_creation: bool
+ disabled: bool # Alias for mode=dryrun, not supported yet
+ docker: str
+ email: str
+ entity: str
+ files_dir: str
+ force: bool
+ git_commit: str
+ git_remote: str
+ git_remote_url: str
+ git_root: str
+ heartbeat_seconds: int
+ host: str
+ ignore_globs: Tuple[str]
+ init_timeout: float
+ is_local: bool
+ job_name: str
+ job_source: str
+ label_disable: bool
+ launch: bool
+ launch_config_path: str
+ log_dir: str
+ log_internal: str
+ log_symlink_internal: str
+ log_symlink_user: str
+ log_user: str
+ login_timeout: float
+ # magic: Union[str, bool, dict] # never used in code, deprecated
+ mode: str
+ notebook_name: str
+ problem: str
+ program: str
+ program_abspath: str
+ program_relpath: str
+ project: str
+ project_url: str
+ quiet: bool
+ reinit: bool
+ relogin: bool
+ # todo: add a preprocessing step to convert this to string
+ resume: Union[str, bool]
+ resume_fname: str
+ resumed: bool # indication from the server about the state of the run (different from resume - user provided flag)
+ root_dir: str
+ run_group: str
+ run_id: str
+ run_job_type: str
+ run_mode: str
+ run_name: str
+ run_notes: str
+ run_tags: Tuple[str]
+ run_url: str
+ sagemaker_disable: bool
+ save_code: bool
+ settings_system: str
+ settings_workspace: str
+ show_colors: bool
+ show_emoji: bool
+ show_errors: bool
+ show_info: bool
+ show_warnings: bool
+ silent: bool
+ start_method: str
+ strict: bool
+ summary_errors: int
+ summary_timeout: int
+ summary_warnings: int
+ sweep_id: str
+ sweep_param_path: str
+ sweep_url: str
+ symlink: bool
+ sync_dir: str
+ sync_file: str
+ sync_symlink_latest: str
+ system_sample: int
+ system_sample_seconds: int
+ table_raise_on_max_row_limit_exceeded: bool
+ timespec: str
+ tmp_dir: str
+ username: str
+ wandb_dir: str
+
+
+class Property:
+ """A class to represent attributes (individual settings) of the Settings object.
+
+ - Encapsulates the logic of how to preprocess and validate values of settings
+ throughout the lifetime of a class instance.
+ - Allows for runtime modification of settings with hooks, e.g. in the case when
+ a setting depends on another setting.
+ - The update() method is used to update the value of a setting.
+ - The `is_policy` attribute determines the source priority when updating the property value.
+ E.g. if `is_policy` is True, the smallest `Source` value takes precedence.
+ """
+
+ def __init__( # pylint: disable=unused-argument
+ self,
+ name: str,
+ value: Optional[Any] = None,
+ preprocessor: Union[Callable, Sequence[Callable], None] = None,
+ # validators allow programming by contract
+ validator: Union[Callable, Sequence[Callable], None] = None,
+ # runtime converter (hook): properties can be e.g. tied to other properties
+ hook: Union[Callable, Sequence[Callable], None] = None,
+ # always apply hook even if value is None. can be used to replace @property's
+ auto_hook: bool = False,
+ is_policy: bool = False,
+ frozen: bool = False,
+ source: int = Source.BASE,
+ **kwargs: Any,
+ ):
+ self.name = name
+ self._preprocessor = preprocessor
+ self._validator = validator
+ self._hook = hook
+ self._auto_hook = auto_hook
+ self._is_policy = is_policy
+ self._source = source
+
+ # preprocess and validate value
+ self._value = self._validate(self._preprocess(value))
+
+ self.__frozen = frozen
+
+ @property
+ def value(self) -> Any:
+ """Apply the runtime modifier(s) (if any) and return the value."""
+ _value = self._value
+ if (_value is not None or self._auto_hook) and self._hook is not None:
+ _hook = [self._hook] if callable(self._hook) else self._hook
+ for h in _hook:
+ _value = h(_value)
+ return _value
+
+ @property
+ def is_policy(self) -> bool:
+ return self._is_policy
+
+ @property
+ def source(self) -> int:
+ return self._source
+
+ def _preprocess(self, value: Any) -> Any:
+ if value is not None and self._preprocessor is not None:
+ _preprocessor = (
+ [self._preprocessor]
+ if callable(self._preprocessor)
+ else self._preprocessor
+ )
+ for p in _preprocessor:
+ try:
+ value = p(value)
+ except Exception:
+ raise SettingsPreprocessingError(
+ f"Unable to preprocess value for property {self.name}: {value}."
+ )
+ return value
+
+ def _validate(self, value: Any) -> Any:
+ if value is not None and self._validator is not None:
+ _validator = (
+ [self._validator] if callable(self._validator) else self._validator
+ )
+ for v in _validator:
+ if not v(value):
+ # failed validation will likely cause a downstream error
+ # when trying to convert to protobuf, so we raise a hard error
+ raise SettingsValidationError(
+ f"Invalid value for property {self.name}: {value}."
+ )
+ return value
+
+ def update(self, value: Any, source: int = Source.OVERRIDE) -> None:
+ """Update the value of the property."""
+ if self.__frozen:
+ raise TypeError("Property object is frozen")
+ # - always update value if source == Source.OVERRIDE
+ # - if not previously overridden:
+ # - update value if source is lower than or equal to current source and property is policy
+ # - update value if source is higher than or equal to current source and property is not policy
+ if (
+ (source == Source.OVERRIDE)
+ or (
+ self._is_policy
+ and self._source != Source.OVERRIDE
+ and source <= self._source
+ )
+ or (
+ not self._is_policy
+ and self._source != Source.OVERRIDE
+ and source >= self._source
+ )
+ ):
+ # self.__dict__["_value"] = self._validate(self._preprocess(value))
+ self._value = self._validate(self._preprocess(value))
+ self._source = source
+
+ def __setattr__(self, key: str, value: Any) -> None:
+ if "_Property__frozen" in self.__dict__ and self.__frozen:
+ raise TypeError(f"Property object {self.name} is frozen")
+ if key == "value":
+ raise AttributeError("Use update() to update property value")
+ self.__dict__[key] = value
+
+ def __str__(self) -> str:
+ return f"{self.value!r}" if isinstance(self.value, str) else f"{self.value}"
+
+ def __repr__(self) -> str:
+ return (
+ f"<Property {self.name}: value={self.value} "
+ f"_value={self._value} source={self._source} is_policy={self._is_policy}>"
+ )
+ # return f"<Property {self.name}: value={self.value}>"
+ # return self.__dict__.__repr__()
+
+
+class Settings(SettingsData):
+ """A class to represent modifiable settings."""
+
+ def _default_props(self) -> Dict[str, Dict[str, Any]]:
+ """Initialize instance attributes (individual settings) as Property objects.
+
+ Helper method that is used in `__init__` together with the class attributes.
+ Note that key names must be the same as the class attribute names.
+ """
+ props: Dict[str, Dict[str, Any]] = dict(
+ _async_upload_concurrency_limit={
+ "preprocessor": int,
+ "validator": self._validate__async_upload_concurrency_limit,
+ },
+ _aws_lambda={
+ "hook": lambda _: is_aws_lambda(),
+ "auto_hook": True,
+ },
+ _colab={
+ "hook": lambda _: "google.colab" in sys.modules,
+ "auto_hook": True,
+ },
+ _disable_machine_info={
+ "value": False,
+ "preprocessor": _str_as_bool,
+ },
+ _disable_meta={
+ "value": False,
+ "preprocessor": _str_as_bool,
+ "hook": lambda x: self._disable_machine_info or x,
+ },
+ _disable_service={
+ "value": False,
+ "preprocessor": _str_as_bool,
+ "is_policy": True,
+ },
+ _disable_setproctitle={"value": False, "preprocessor": _str_as_bool},
+ _disable_stats={
+ "value": False,
+ "preprocessor": _str_as_bool,
+ "hook": lambda x: self._disable_machine_info or x,
+ },
+ _disable_viewer={"preprocessor": _str_as_bool},
+ _extra_http_headers={"preprocessor": _str_as_json},
+ # Retry filestream requests for 2 hours before dropping chunk (how do we recover?)
+ # retry_count = seconds_in_2_hours / max_retry_time + num_retries_until_max_60_sec
+ # = 7200 / 60 + ceil(log2(60/2))
+ # = 120 + 5
+ _file_stream_retry_max={"value": 125, "preprocessor": int},
+ _file_stream_retry_wait_min_seconds={"value": 2, "preprocessor": float},
+ _file_stream_retry_wait_max_seconds={"value": 60, "preprocessor": float},
+ # A 3 minute timeout for all filestream post requests
+ _file_stream_timeout_seconds={"value": 180, "preprocessor": float},
+ _file_transfer_retry_max={"value": 20, "preprocessor": int},
+ _file_transfer_retry_wait_min_seconds={"value": 2, "preprocessor": float},
+ _file_transfer_retry_wait_max_seconds={"value": 60, "preprocessor": float},
+ _file_transfer_timeout_seconds={"value": 0, "preprocessor": float},
+ _flow_control_disabled={
+ "hook": lambda _: self._network_buffer == 0,
+ "auto_hook": True,
+ },
+ _flow_control_custom={
+ "hook": lambda _: bool(self._network_buffer),
+ "auto_hook": True,
+ },
+ _graphql_retry_max={"value": 20, "preprocessor": int},
+ _graphql_retry_wait_min_seconds={"value": 2, "preprocessor": float},
+ _graphql_retry_wait_max_seconds={"value": 60, "preprocessor": float},
+ _graphql_timeout_seconds={"value": 30.0, "preprocessor": float},
+ _internal_check_process={"value": 8, "preprocessor": float},
+ _internal_queue_timeout={"value": 2, "preprocessor": float},
+ _ipython={
+ "hook": lambda _: _get_python_type() == "ipython",
+ "auto_hook": True,
+ },
+ _jupyter={
+ "hook": lambda _: _get_python_type() == "jupyter",
+ "auto_hook": True,
+ },
+ _kaggle={"hook": lambda _: util._is_likely_kaggle(), "auto_hook": True},
+ _log_level={"value": logging.DEBUG},
+ _network_buffer={"preprocessor": int},
+ _noop={"hook": lambda _: self.mode == "disabled", "auto_hook": True},
+ _notebook={
+ "hook": lambda _: self._ipython
+ or self._jupyter
+ or self._colab
+ or self._kaggle,
+ "auto_hook": True,
+ },
+ _offline={
+ "hook": (
+ lambda _: True
+ if self.disabled or (self.mode in ("dryrun", "offline"))
+ else False
+ ),
+ "auto_hook": True,
+ },
+ _platform={"value": util.get_platform_name()},
+ _proxies={
+ "preprocessor": _str_as_json,
+ },
+ _require_core={"value": False, "preprocessor": _str_as_bool},
+ _save_requirements={"value": True, "preprocessor": _str_as_bool},
+ _service_wait={
+ "value": 30,
+ "preprocessor": float,
+ "validator": self._validate__service_wait,
+ },
+ _shared={
+ "hook": lambda _: self.mode == "shared",
+ "auto_hook": True,
+ },
+ _start_datetime={"preprocessor": _datetime_as_str},
+ _stats_sample_rate_seconds={
+ "value": 2.0,
+ "preprocessor": float,
+ "validator": self._validate__stats_sample_rate_seconds,
+ },
+ _stats_samples_to_average={
+ "value": 15,
+ "preprocessor": int,
+ "validator": self._validate__stats_samples_to_average,
+ },
+ _stats_join_assets={"value": True, "preprocessor": _str_as_bool},
+ _stats_neuron_monitor_config_path={
+ "hook": lambda x: self._path_convert(x),
+ },
+ _stats_open_metrics_endpoints={
+ "preprocessor": _str_as_json,
+ },
+ _stats_open_metrics_filters={
+ # capture all metrics on all endpoints by default
+ "value": (".*",),
+ "preprocessor": _str_as_json,
+ },
+ _stats_disk_paths={
+ "value": ("/",),
+ "preprocessor": _str_as_json,
+ },
+ _stats_buffer_size={
+ "value": 0,
+ "preprocessor": int,
+ },
+ _sync={"value": False},
+ _tmp_code_dir={
+ "value": "code",
+ "hook": lambda x: self._path_convert(self.tmp_dir, x),
+ },
+ _windows={
+ "hook": lambda _: platform.system() == "Windows",
+ "auto_hook": True,
+ },
+ anonymous={"validator": self._validate_anonymous},
+ api_key={"validator": self._validate_api_key},
+ base_url={
+ "value": "https://api.wandb.ai",
+ "preprocessor": lambda x: str(x).strip().rstrip("/"),
+ "validator": self._validate_base_url,
+ },
+ colab_url={
+ "hook": lambda _: self._get_colab_url(),
+ "auto_hook": True,
+ },
+ config_paths={"preprocessor": _str_as_tuple},
+ console={
+ "value": "auto",
+ "validator": self._validate_console,
+ "hook": lambda x: self._convert_console(x),
+ "auto_hook": True,
+ },
+ deployment={
+ "hook": lambda _: "local" if self.is_local else "cloud",
+ "auto_hook": True,
+ },
+ disable_code={
+ "value": False,
+ "preprocessor": _str_as_bool,
+ "hook": lambda x: self._disable_machine_info or x,
+ },
+ disable_hints={"preprocessor": _str_as_bool},
+ disable_git={
+ "value": False,
+ "preprocessor": _str_as_bool,
+ "hook": lambda x: self._disable_machine_info or x,
+ },
+ disable_job_creation={
+ "value": False,
+ "preprocessor": _str_as_bool,
+ "hook": lambda x: self._disable_machine_info or x,
+ },
+ disabled={"value": False, "preprocessor": _str_as_bool},
+ files_dir={
+ "value": "files",
+ "hook": lambda x: self._path_convert(
+ self.wandb_dir, f"{self.run_mode}-{self.timespec}-{self.run_id}", x
+ ),
+ },
+ force={"preprocessor": _str_as_bool},
+ git_remote={"value": "origin"},
+ heartbeat_seconds={"value": 30},
+ ignore_globs={
+ "value": tuple(),
+ "preprocessor": lambda x: tuple(x) if not isinstance(x, tuple) else x,
+ },
+ init_timeout={"value": 90, "preprocessor": lambda x: float(x)},
+ is_local={
+ "hook": (
+ lambda _: self.base_url != "https://api.wandb.ai"
+ if self.base_url is not None
+ else False
+ ),
+ "auto_hook": True,
+ },
+ job_name={"preprocessor": str},
+ job_source={"validator": self._validate_job_source},
+ label_disable={"preprocessor": _str_as_bool},
+ launch={"preprocessor": _str_as_bool},
+ log_dir={
+ "value": "logs",
+ "hook": lambda x: self._path_convert(
+ self.wandb_dir, f"{self.run_mode}-{self.timespec}-{self.run_id}", x
+ ),
+ },
+ log_internal={
+ "value": "debug-internal.log",
+ "hook": lambda x: self._path_convert(self.log_dir, x),
+ },
+ log_symlink_internal={
+ "value": "debug-internal.log",
+ "hook": lambda x: self._path_convert(self.wandb_dir, x),
+ },
+ log_symlink_user={
+ "value": "debug.log",
+ "hook": lambda x: self._path_convert(self.wandb_dir, x),
+ },
+ log_user={
+ "value": "debug.log",
+ "hook": lambda x: self._path_convert(self.log_dir, x),
+ },
+ login_timeout={"preprocessor": lambda x: float(x)},
+ mode={"value": "online", "validator": self._validate_mode},
+ problem={"value": "fatal", "validator": self._validate_problem},
+ program={
+ "hook": lambda x: self._get_program(x),
+ },
+ project={"validator": self._validate_project},
+ project_url={"hook": lambda _: self._project_url(), "auto_hook": True},
+ quiet={"preprocessor": _str_as_bool},
+ reinit={"preprocessor": _str_as_bool},
+ relogin={"preprocessor": _str_as_bool},
+ # todo: hack to make to_proto() always happy
+ resume={"preprocessor": lambda x: None if x is False else x},
+ resume_fname={
+ "value": "wandb-resume.json",
+ "hook": lambda x: self._path_convert(self.wandb_dir, x),
+ },
+ resumed={"value": "False", "preprocessor": _str_as_bool},
+ root_dir={
+ "preprocessor": lambda x: str(x),
+ "value": os.path.abspath(os.getcwd()),
+ },
+ run_id={
+ "validator": self._validate_run_id,
+ },
+ run_mode={
+ "hook": lambda _: "offline-run" if self._offline else "run",
+ "auto_hook": True,
+ },
+ run_tags={
+ "preprocessor": lambda x: tuple(x) if not isinstance(x, tuple) else x,
+ },
+ run_url={"hook": lambda _: self._run_url(), "auto_hook": True},
+ sagemaker_disable={"preprocessor": _str_as_bool},
+ save_code={"preprocessor": _str_as_bool},
+ settings_system={
+ "value": os.path.join("~", ".config", "wandb", "settings"),
+ "hook": lambda x: self._path_convert(x),
+ },
+ settings_workspace={
+ "value": "settings",
+ "hook": lambda x: self._path_convert(self.wandb_dir, x),
+ },
+ show_colors={"preprocessor": _str_as_bool},
+ show_emoji={"preprocessor": _str_as_bool},
+ show_errors={"value": "True", "preprocessor": _str_as_bool},
+ show_info={"value": "True", "preprocessor": _str_as_bool},
+ show_warnings={"value": "True", "preprocessor": _str_as_bool},
+ silent={"value": "False", "preprocessor": _str_as_bool},
+ start_method={"validator": self._validate_start_method},
+ strict={"preprocessor": _str_as_bool},
+ summary_timeout={"value": 60, "preprocessor": lambda x: int(x)},
+ summary_warnings={
+ "value": 5,
+ "preprocessor": lambda x: int(x),
+ "is_policy": True,
+ },
+ sweep_url={"hook": lambda _: self._sweep_url(), "auto_hook": True},
+ symlink={"preprocessor": _str_as_bool},
+ sync_dir={
+ "hook": [
+ lambda _: self._path_convert(
+ self.wandb_dir, f"{self.run_mode}-{self.timespec}-{self.run_id}"
+ )
+ ],
+ "auto_hook": True,
+ },
+ sync_file={
+ "hook": lambda _: self._path_convert(
+ self.sync_dir, f"run-{self.run_id}.wandb"
+ ),
+ "auto_hook": True,
+ },
+ sync_symlink_latest={
+ "value": "latest-run",
+ "hook": lambda x: self._path_convert(self.wandb_dir, x),
+ },
+ system_sample={"value": 15},
+ system_sample_seconds={"value": 2},
+ table_raise_on_max_row_limit_exceeded={
+ "value": False,
+ "preprocessor": _str_as_bool,
+ },
+ timespec={
+ "hook": lambda _: self._start_datetime,
+ "auto_hook": True,
+ },
+ tmp_dir={
+ "value": "tmp",
+ "hook": lambda x: (
+ self._path_convert(
+ self.wandb_dir,
+ f"{self.run_mode}-{self.timespec}-{self.run_id}",
+ x,
+ )
+ or tempfile.gettempdir()
+ ),
+ },
+ wandb_dir={
+ "hook": lambda _: _get_wandb_dir(self.root_dir or ""),
+ "auto_hook": True,
+ },
+ )
+ return props
+
+ # helper methods for validating values
+ @staticmethod
+ def _validator_factory(hint: Any) -> Callable[[Any], bool]: # noqa: C901
+ """Return a factory for setting type validators."""
+
+ def helper(value: Any) -> bool:
+ try:
+ is_valid = is_instance_recursive(value, hint)
+ except Exception:
+ # instance check failed, but let's not crash and only print a warning
+ is_valid = False
+
+ return is_valid
+
+ return helper
+
+ @staticmethod
+ def _validate_mode(value: str) -> bool:
+ choices: Set[str] = {"dryrun", "run", "offline", "online", "disabled", "shared"}
+ if value not in choices:
+ raise UsageError(f"Settings field `mode`: {value!r} not in {choices}")
+ return True
+
+ @staticmethod
+ def _validate_project(value: Optional[str]) -> bool:
+ invalid_chars_list = list("/\\#?%:")
+ if value is not None:
+ if len(value) > 128:
+ raise UsageError(
+ f"Invalid project name {value!r}: exceeded 128 characters"
+ )
+ invalid_chars = {char for char in invalid_chars_list if char in value}
+ if invalid_chars:
+ raise UsageError(
+ f"Invalid project name {value!r}: "
+ f"cannot contain characters {','.join(invalid_chars_list)!r}, "
+ f"found {','.join(invalid_chars)!r}"
+ )
+ return True
+
+ @staticmethod
+ def _validate_start_method(value: str) -> bool:
+ available_methods = ["thread"]
+ if hasattr(multiprocessing, "get_all_start_methods"):
+ available_methods += multiprocessing.get_all_start_methods()
+ if value not in available_methods:
+ raise UsageError(
+ f"Settings field `start_method`: {value!r} not in {available_methods}"
+ )
+ return True
+
+ @staticmethod
+ def _validate_console(value: str) -> bool:
+ choices = ConsoleValue
+ if value not in choices:
+ # do not advertise internal console states
+ choices -= {"wrap_emu", "wrap_raw"}
+ raise UsageError(f"Settings field `console`: {value!r} not in {choices}")
+ return True
+
+ @staticmethod
+ def _validate_problem(value: str) -> bool:
+ choices: Set[str] = {"fatal", "warn", "silent"}
+ if value not in choices:
+ raise UsageError(f"Settings field `problem`: {value!r} not in {choices}")
+ return True
+
+ @staticmethod
+ def _validate_anonymous(value: str) -> bool:
+ choices: Set[str] = {"allow", "must", "never", "false", "true"}
+ if value not in choices:
+ raise UsageError(f"Settings field `anonymous`: {value!r} not in {choices}")
+ return True
+
+ @staticmethod
+ def _validate_run_id(value: str) -> bool:
+ # if len(value) > len(value.strip()):
+ # raise UsageError("Run ID cannot start or end with whitespace")
+ return bool(value.strip())
+
+ @staticmethod
+ def _validate_api_key(value: str) -> bool:
+ if len(value) > len(value.strip()):
+ raise UsageError("API key cannot start or end with whitespace")
+
+ # todo: move this check to the post-init validation step
+ # if value.startswith("local") and not self.is_local:
+ # raise UsageError(
+ # "Attempting to use a local API key to connect to https://api.wandb.ai"
+ # )
+ # todo: move here the logic from sdk/lib/apikey.py
+
+ return True
+
+ @staticmethod
+ def _validate_base_url(value: Optional[str]) -> bool:
+ """Validate the base url of the wandb server.
+
+ param value: URL to validate
+
+ Based on the Django URLValidator, but with a few additional checks.
+
+ Copyright (c) Django Software Foundation and individual contributors.
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without modification,
+ are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ 3. Neither the name of Django nor the names of its contributors may be used
+ to endorse or promote products derived from this software without
+ specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ """
+ if value is None:
+ return True
+
+ ul = "\u00a1-\uffff" # Unicode letters range (must not be a raw string).
+
+ # IP patterns
+ ipv4_re = (
+ r"(?:0|25[0-5]|2[0-4][0-9]|1[0-9]?[0-9]?|[1-9][0-9]?)"
+ r"(?:\.(?:0|25[0-5]|2[0-4][0-9]|1[0-9]?[0-9]?|[1-9][0-9]?)){3}"
+ )
+ ipv6_re = r"\[[0-9a-f:.]+\]" # (simple regex, validated later)
+
+ # Host patterns
+ hostname_re = (
+ r"[a-z" + ul + r"0-9](?:[a-z" + ul + r"0-9-]{0,61}[a-z" + ul + r"0-9])?"
+ )
+ # Max length for domain name labels is 63 characters per RFC 1034 sec. 3.1
+ domain_re = r"(?:\.(?!-)[a-z" + ul + r"0-9-]{1,63}(?<!-))*"
+ tld_re = (
+ r"\." # dot
+ r"(?!-)" # can't start with a dash
+ r"(?:[a-z" + ul + "-]{2,63}" # domain label
+ r"|xn--[a-z0-9]{1,59})" # or punycode label
+ r"(?<!-)" # can't end with a dash
+ r"\.?" # may have a trailing dot
+ )
+ # host_re = "(" + hostname_re + domain_re + tld_re + "|localhost)"
+ # todo?: allow hostname to be just a hostname (no tld)?
+ host_re = "(" + hostname_re + domain_re + f"({tld_re})?" + "|localhost)"
+
+ regex = re.compile(
+ r"^(?:[a-z0-9.+-]*)://" # scheme is validated separately
+ r"(?:[^\s:@/]+(?::[^\s:@/]*)?@)?" # user:pass authentication
+ r"(?:" + ipv4_re + "|" + ipv6_re + "|" + host_re + ")"
+ r"(?::[0-9]{1,5})?" # port
+ r"(?:[/?#][^\s]*)?" # resource path
+ r"\Z",
+ re.IGNORECASE,
+ )
+ schemes = {"http", "https"}
+ unsafe_chars = frozenset("\t\r\n")
+
+ scheme = value.split("://")[0].lower()
+ split_url = urlsplit(value)
+ parsed_url = urlparse(value)
+
+ if re.match(r".*wandb\.ai[^\.]*$", value) and "api." not in value:
+ # user might guess app.wandb.ai or wandb.ai is the default cloud server
+ raise UsageError(
+ f"{value} is not a valid server address, did you mean https://api.wandb.ai?"
+ )
+ elif re.match(r".*wandb\.ai[^\.]*$", value) and scheme != "https":
+ raise UsageError("http is not secure, please use https://api.wandb.ai")
+ elif parsed_url.netloc == "":
+ raise UsageError(f"Invalid URL: {value}")
+ elif unsafe_chars.intersection(value):
+ raise UsageError("URL cannot contain unsafe characters")
+ elif scheme not in schemes:
+ raise UsageError("URL must start with `http(s)://`")
+ elif not regex.search(value):
+ raise UsageError(f"{value} is not a valid server address")
+ elif split_url.hostname is None or len(split_url.hostname) > 253:
+ raise UsageError("hostname is invalid")
+
+ return True
+
+ @staticmethod
+ def _validate__service_wait(value: float) -> bool:
+ if value <= 0:
+ raise UsageError("_service_wait must be a positive number")
+ return True
+
+ @staticmethod
+ def _validate__stats_sample_rate_seconds(value: float) -> bool:
+ if value < 0.1:
+ raise UsageError("_stats_sample_rate_seconds must be >= 0.1")
+ return True
+
+ @staticmethod
+ def _validate__stats_samples_to_average(value: int) -> bool:
+ if value < 1 or value > 30:
+ raise UsageError("_stats_samples_to_average must be between 1 and 30")
+ return True
+
+ @staticmethod
+ def _validate__async_upload_concurrency_limit(value: int) -> bool:
+ if value <= 0:
+ raise UsageError("_async_upload_concurrency_limit must be positive")
+
+ try:
+ import resource # not always available on Windows
+
+ file_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
+ except Exception:
+ # Couldn't get the open-file-limit for some reason,
+ # probably very platform-specific. Not a problem,
+ # we just won't use it to cap the concurrency.
+ pass
+ else:
+ if value > file_limit:
+ wandb.termwarn(
+ (
+ "_async_upload_concurrency_limit setting of"
+ f" {value} exceeds this process's limit"
+ f" on open files ({file_limit}); may cause file-upload failures."
+ " Try decreasing _async_upload_concurrency_limit,"
+ " or increasing your file limit with `ulimit -n`."
+ ),
+ repeat=False,
+ )
+
+ return True
+
+ @staticmethod
+ def _validate_job_source(value: str) -> bool:
+ valid_sources = ["repo", "artifact", "image"]
+ if value not in valid_sources:
+ raise UsageError(
+ f"Settings field `job_source`: {value!r} not in {valid_sources}"
+ )
+ return True
+
+ # other helper methods
+ @staticmethod
+ def _path_convert(*args: str) -> str:
+ """Join path and apply os.path.expanduser to it."""
+ return os.path.expanduser(os.path.join(*args))
+
+ def _convert_console(self, console: str) -> str:
+ if console == "auto":
+ if (
+ self._jupyter
+ or (self.start_method == "thread")
+ or not self._disable_service
+ or self._windows
+ ):
+ console = "wrap"
+ else:
+ console = "redirect"
+ return console
+
+ def _get_colab_url(self) -> Optional[str]:
+ if not self._colab:
+ return None
+ if self._jupyter_path and self._jupyter_path.startswith("fileId="):
+ unescaped = unquote(self._jupyter_path)
+ return "https://colab.research.google.com/notebook#" + unescaped
+ return None
+
+ def _get_program(self, program: Optional[str]) -> Optional[str]:
+ if program is not None and program != "<python with no main file>":
+ return program
+
+ if not self._jupyter:
+ return program
+
+ if self.notebook_name:
+ return self.notebook_name
+
+ if not self._jupyter_path:
+ return program
+
+ if self._jupyter_path.startswith("fileId="):
+ return self._jupyter_name
+ else:
+ return self._jupyter_path
+
+ def _get_url_query_string(self) -> str:
+ # TODO(settings) use `wandb_setting` (if self.anonymous != "true":)
+ if Api().settings().get("anonymous") != "true":
+ return ""
+
+ api_key = apikey.api_key(settings=self)
+
+ return f"?{urlencode({'apiKey': api_key})}"
+
+ def _project_url_base(self) -> str:
+ if not all([self.entity, self.project]):
+ return ""
+
+ app_url = wandb.util.app_url(self.base_url)
+ return f"{app_url}/{quote(self.entity)}/{quote(self.project)}"
+
+ def _project_url(self) -> str:
+ project_url = self._project_url_base()
+ if not project_url:
+ return ""
+
+ query = self._get_url_query_string()
+
+ return f"{project_url}{query}"
+
+ def _run_url(self) -> str:
+ """Return the run url."""
+ project_url = self._project_url_base()
+ if not all([project_url, self.run_id]):
+ return ""
+
+ query = self._get_url_query_string()
+ return f"{project_url}/runs/{quote(self.run_id)}{query}"
+
+ def _set_run_start_time(self, source: int = Source.BASE) -> None:
+ """Set the time stamps for the settings.
+
+ Called once the run is initialized.
+ """
+ time_stamp: float = time.time()
+ datetime_now: datetime = datetime.fromtimestamp(time_stamp)
+ datetime_now_str = _datetime_as_str(datetime_now)
+ object.__setattr__(self, "_Settings_start_datetime", datetime_now_str)
+ object.__setattr__(self, "_Settings_start_time", time_stamp)
+ self.update(
+ _start_datetime=datetime_now_str,
+ _start_time=time_stamp,
+ source=source,
+ )
+
+ def _sweep_url(self) -> str:
+ """Return the sweep url."""
+ project_url = self._project_url_base()
+ if not all([project_url, self.sweep_id]):
+ return ""
+
+ query = self._get_url_query_string()
+ return f"{project_url}/sweeps/{quote(self.sweep_id)}{query}"
+
+ def __init__(self, **kwargs: Any) -> None:
+ self.__frozen: bool = False
+ self.__initialized: bool = False
+
+ self.__modification_order = SETTINGS_TOPOLOGICALLY_SORTED
+
+ # Set default settings values
+ # We start off with the class attributes and `default_props` dicts
+ # and then create Property objects.
+ # Once initialized, attributes are to only be updated using the `update` method
+ default_props = self._default_props()
+
+ # Init instance attributes as Property objects.
+ # Type hints of class attributes are used to generate a type validator function
+ # for runtime checks for each attribute.
+ # These are defaults, using Source.BASE for non-policy attributes and Source.RUN for policies.
+ for prop, type_hint in get_type_hints(SettingsData).items():
+ validators = [self._validator_factory(type_hint)]
+
+ if prop in default_props:
+ validator = default_props[prop].pop("validator", [])
+ # Property validator could be either Callable or Sequence[Callable]
+ if callable(validator):
+ validators.append(validator)
+ elif isinstance(validator, Sequence):
+ validators.extend(list(validator))
+ object.__setattr__(
+ self,
+ prop,
+ Property(
+ name=prop,
+ **default_props[prop],
+ validator=validators,
+ # todo: double-check this logic:
+ source=Source.RUN
+ if default_props[prop].get("is_policy", False)
+ else Source.BASE,
+ ),
+ )
+ else:
+ object.__setattr__(
+ self,
+ prop,
+ Property(
+ name=prop,
+ validator=validators,
+ source=Source.BASE,
+ ),
+ )
+
+ # update overridden defaults from kwargs
+ unexpected_arguments = [k for k in kwargs.keys() if k not in self.__dict__]
+ # allow only explicitly defined arguments
+ if unexpected_arguments:
+ raise SettingsUnexpectedArgsError(
+ f"Got unexpected arguments: {unexpected_arguments}. "
+ )
+
+ # automatically inspect setting validators and runtime hooks and topologically sort them
+ # so that we can safely update them. throw error if there are cycles.
+ for prop in self.__modification_order:
+ if prop in kwargs:
+ source = Source.RUN if self.__dict__[prop].is_policy else Source.BASE
+ self.update({prop: kwargs[prop]}, source=source)
+ kwargs.pop(prop)
+
+ for k, v in kwargs.items():
+ # todo: double-check this logic:
+ source = Source.RUN if self.__dict__[k].is_policy else Source.BASE
+ self.update({k: v}, source=source)
+
+ # setup private attributes
+ object.__setattr__(self, "_Settings_start_datetime", None)
+ object.__setattr__(self, "_Settings_start_time", None)
+
+ # done with init, use self.update() to update attributes from now on
+ self.__initialized = True
+
+ # todo? freeze settings to prevent accidental changes
+ # self.freeze()
+
+ def __str__(self) -> str:
+ # get attributes that are instances of the Property class:
+ representation = {
+ k: v.value for k, v in self.__dict__.items() if isinstance(v, Property)
+ }
+ return f"<Settings {_redact_dict(representation)}>"
+
+ def __repr__(self) -> str:
+ # private attributes
+ private = {k: v for k, v in self.__dict__.items() if k.startswith("_Settings")}
+ # get attributes that are instances of the Property class:
+ attributes = {
+ k: f"<Property value={v.value} source={v.source}>"
+ for k, v in self.__dict__.items()
+ if isinstance(v, Property)
+ }
+ representation = {**private, **attributes}
+ return f"<Settings {representation}>"
+
+ def __copy__(self) -> "Settings":
+ """Ensure that a copy of the settings object is a truly deep copy.
+
+ Note that the copied object will not be frozen todo? why is this needed?
+ """
+ # get attributes that are instances of the Property class:
+ attributes = {k: v for k, v in self.__dict__.items() if isinstance(v, Property)}
+ new = Settings()
+ # update properties that have deps or are dependent on in the topologically-sorted order
+ for prop in self.__modification_order:
+ new.update({prop: attributes[prop]._value}, source=attributes[prop].source)
+ attributes.pop(prop)
+
+ # update the remaining attributes
+ for k, v in attributes.items():
+ # make sure to use the raw property value (v._value),
+ # not the potential result of runtime hooks applied to it (v.value)
+ new.update({k: v._value}, source=v.source)
+ new.unfreeze()
+
+ return new
+
+ def __deepcopy__(self, memo: dict) -> "Settings":
+ return self.__copy__()
+
+ # attribute access methods
+ @no_type_check # this is a hack to make mypy happy
+ def __getattribute__(self, name: str) -> Any:
+ """Expose `attribute.value` if `attribute` is a Property."""
+ item = object.__getattribute__(self, name)
+ if isinstance(item, Property):
+ return item.value
+ return item
+
+ def __setattr__(self, key: str, value: Any) -> None:
+ if "_Settings__initialized" in self.__dict__ and self.__initialized:
+ raise TypeError(f"Please use update() to update attribute `{key}` value")
+ object.__setattr__(self, key, value)
+
+ def __iter__(self) -> Iterable:
+ return iter(self.to_dict())
+
+ def copy(self) -> "Settings":
+ return self.__copy__()
+
+ # implement the Mapping interface
+ def keys(self) -> Iterable[str]:
+ return self.to_dict().keys()
+
+ @no_type_check # this is a hack to make mypy happy
+ def __getitem__(self, name: str) -> Any:
+ """Expose attribute.value if attribute is a Property."""
+ item = object.__getattribute__(self, name)
+ if isinstance(item, Property):
+ return item.value
+ return item
+
+ def update(
+ self,
+ settings: Optional[Union[Dict[str, Any], "Settings"]] = None,
+ source: int = Source.OVERRIDE,
+ **kwargs: Any,
+ ) -> None:
+ """Update individual settings."""
+ if "_Settings__frozen" in self.__dict__ and self.__frozen:
+ raise TypeError("Settings object is frozen")
+
+ if isinstance(settings, Settings):
+ # If a Settings object is passed, detect the settings that differ
+ # from defaults, collect them into a dict, and apply them using `source`.
+ # This comes up in `wandb.init(settings=wandb.Settings(...))` and
+ # seems like the behavior that the user would expect when calling init that way.
+ defaults = Settings()
+ settings_dict = dict()
+ for k, v in settings.__dict__.items():
+ if isinstance(v, Property):
+ if v._value != defaults.__dict__[k]._value:
+ settings_dict[k] = v._value
+ # replace with the generated dict
+ settings = settings_dict
+
+ # add kwargs to settings
+ settings = settings or dict()
+ # explicit kwargs take precedence over settings
+ settings = {**settings, **kwargs}
+ unknown_properties = []
+ for key in settings.keys():
+ # only allow updating known Properties
+ if key not in self.__dict__ or not isinstance(self.__dict__[key], Property):
+ unknown_properties.append(key)
+ if unknown_properties:
+ raise KeyError(f"Unknown settings: {unknown_properties}")
+ # only if all keys are valid, update them
+
+ # store settings to be updated in a dict to preserve stats on preprocessing and validation errors
+ settings.copy()
+
+ # update properties that have deps or are dependent on in the topologically-sorted order
+ for key in self.__modification_order:
+ if key in settings:
+ self.__dict__[key].update(settings.pop(key), source=source)
+
+ # update the remaining properties
+ for key, value in settings.items():
+ self.__dict__[key].update(value, source)
+
+ def items(self) -> ItemsView[str, Any]:
+ return self.to_dict().items()
+
+ def get(self, key: str, default: Optional[Any] = None) -> Any:
+ return self.to_dict().get(key, default)
+
+ def freeze(self) -> None:
+ object.__setattr__(self, "_Settings__frozen", True)
+
+ def unfreeze(self) -> None:
+ object.__setattr__(self, "_Settings__frozen", False)
+
+ def is_frozen(self) -> bool:
+ return self.__frozen
+
+ def to_dict(self) -> Dict[str, Any]:
+ """Return a dict representation of the settings."""
+ # get attributes that are instances of the Property class:
+ attributes = {
+ k: v.value for k, v in self.__dict__.items() if isinstance(v, Property)
+ }
+ return attributes
+
+ def to_proto(self) -> wandb_settings_pb2.Settings:
+ """Generate a protobuf representation of the settings."""
+ from dataclasses import fields
+
+ settings = wandb_settings_pb2.Settings()
+ for field in fields(SettingsData):
+ k = field.name
+ v = getattr(self, k)
+ # special case for _stats_open_metrics_filters
+ if k == "_stats_open_metrics_filters":
+ if isinstance(v, (list, set, tuple)):
+ setting = getattr(settings, k)
+ setting.sequence.value.extend(v)
+ elif isinstance(v, dict):
+ setting = getattr(settings, k)
+ for key, value in v.items():
+ for kk, vv in value.items():
+ setting.mapping.value[key].value[kk] = vv
+ else:
+ raise TypeError(f"Unsupported type {type(v)} for setting {k}")
+ continue
+
+ if isinstance(v, bool):
+ getattr(settings, k).CopyFrom(BoolValue(value=v))
+ elif isinstance(v, int):
+ getattr(settings, k).CopyFrom(Int32Value(value=v))
+ elif isinstance(v, float):
+ getattr(settings, k).CopyFrom(DoubleValue(value=v))
+ elif isinstance(v, str):
+ getattr(settings, k).CopyFrom(StringValue(value=v))
+ elif isinstance(v, (list, set, tuple)):
+ # we only support sequences of strings for now
+ sequence = getattr(settings, k)
+ sequence.value.extend(v)
+ elif isinstance(v, dict):
+ mapping = getattr(settings, k)
+ for key, value in v.items():
+ # we only support dicts with string values for now
+ mapping.value[key] = value
+ elif v is None:
+ # None is the default value for all settings, so we don't need to set it,
+ # i.e. None means that the value was not set.
+ pass
+ else:
+ raise TypeError(f"Unsupported type {type(v)} for setting {k}")
+ # TODO: store property sources in the protobuf so that we can reconstruct the
+ # settings object from the protobuf
+ return settings
+
+ # apply settings from different sources
+ # TODO(dd): think about doing some|all of that at init
+ def _apply_settings(
+ self,
+ settings: "Settings",
+ _logger: Optional[_EarlyLogger] = None,
+ ) -> None:
+ """Apply settings from a Settings object."""
+ if _logger is not None:
+ _logger.info(f"Applying settings from {settings}")
+ attributes = {
+ k: v for k, v in settings.__dict__.items() if isinstance(v, Property)
+ }
+ # update properties that have deps or are dependent on in the topologically-sorted order
+ for prop in self.__modification_order:
+ self.update({prop: attributes[prop]._value}, source=attributes[prop].source)
+ attributes.pop(prop)
+ # update the remaining properties
+ for k, v in attributes.items():
+ # note that only the same/higher priority settings are propagated
+ self.update({k: v._value}, source=v.source)
+
+ @staticmethod
+ def _load_config_file(file_name: str, section: str = "default") -> dict:
+ parser = configparser.ConfigParser()
+ parser.add_section(section)
+ parser.read(file_name)
+ config: Dict[str, Any] = dict()
+ for k in parser[section]:
+ config[k] = parser[section][k]
+ # TODO (cvp): we didn't do this in the old cli, but it seems necessary
+ if k == "ignore_globs":
+ config[k] = config[k].split(",")
+ return config
+
+ def _apply_base(self, pid: int, _logger: Optional[_EarlyLogger] = None) -> None:
+ if _logger is not None:
+ _logger.info(f"Current SDK version is {wandb.__version__}")
+ _logger.info(f"Configure stats pid to {pid}")
+ self.update({"_stats_pid": pid}, source=Source.SETUP)
+
+ def _apply_config_files(self, _logger: Optional[_EarlyLogger] = None) -> None:
+ # TODO(jhr): permit setting of config in system and workspace
+ if self.settings_system is not None:
+ if _logger is not None:
+ _logger.info(f"Loading settings from {self.settings_system}")
+ self.update(
+ self._load_config_file(self.settings_system),
+ source=Source.SYSTEM,
+ )
+ if self.settings_workspace is not None:
+ if _logger is not None:
+ _logger.info(f"Loading settings from {self.settings_workspace}")
+ self.update(
+ self._load_config_file(self.settings_workspace),
+ source=Source.WORKSPACE,
+ )
+
+ def _apply_env_vars(
+ self,
+ environ: Mapping[str, Any],
+ _logger: Optional[_EarlyLogger] = None,
+ ) -> None:
+ env_prefix: str = "WANDB_"
+ special_env_var_names = {
+ "WANDB_TRACELOG": "_tracelog",
+ "WANDB_DISABLE_SERVICE": "_disable_service",
+ "WANDB_SERVICE_TRANSPORT": "_service_transport",
+ "WANDB_REQUIRE_CORE": "_require_core",
+ "WANDB_DIR": "root_dir",
+ "WANDB_NAME": "run_name",
+ "WANDB_NOTES": "run_notes",
+ "WANDB_TAGS": "run_tags",
+ "WANDB_JOB_TYPE": "run_job_type",
+ "WANDB_HTTP_TIMEOUT": "_graphql_timeout_seconds",
+ "WANDB_FILE_PUSHER_TIMEOUT": "_file_transfer_timeout_seconds",
+ "WANDB_USER_EMAIL": "email",
+ }
+ env = dict()
+ for setting, value in environ.items():
+ if not setting.startswith(env_prefix):
+ continue
+
+ if setting in special_env_var_names:
+ key = special_env_var_names[setting]
+ else:
+ # otherwise, strip the prefix and convert to lowercase
+ key = setting[len(env_prefix) :].lower()
+
+ if key in self.__dict__:
+ if key in ("ignore_globs", "run_tags"):
+ value = value.split(",")
+ env[key] = value
+ elif _logger is not None:
+ _logger.warning(f"Unknown environment variable: {setting}")
+
+ if _logger is not None:
+ _logger.info(
+ f"Loading settings from environment variables: {_redact_dict(env)}"
+ )
+ self.update(env, source=Source.ENV)
+
+ def _infer_settings_from_environment(
+ self, _logger: Optional[_EarlyLogger] = None
+ ) -> None:
+ """Modify settings based on environment (for runs and cli)."""
+ settings: Dict[str, Union[bool, str, Sequence, None]] = dict()
+ # disable symlinks if on windows (requires admin or developer setup)
+ settings["symlink"] = True
+ if self._windows:
+ settings["symlink"] = False
+
+ # TODO(jhr): this needs to be moved last in setting up settings ?
+ # (dd): loading order does not matter as long as source is set correctly
+
+ # For code saving, only allow env var override if value from server is true, or
+ # if no preference was specified.
+ if (self.save_code is True or self.save_code is None) and (
+ os.getenv(wandb.env.SAVE_CODE) is not None
+ or os.getenv(wandb.env.DISABLE_CODE) is not None
+ ):
+ settings["save_code"] = wandb.env.should_save_code()
+
+ settings["disable_git"] = wandb.env.disable_git()
+
+ # Attempt to get notebook information if not already set by the user
+ if self._jupyter and (self.notebook_name is None or self.notebook_name == ""):
+ meta = wandb.jupyter.notebook_metadata(self.silent)
+ settings["_jupyter_path"] = meta.get("path")
+ settings["_jupyter_name"] = meta.get("name")
+ settings["_jupyter_root"] = meta.get("root")
+ elif (
+ self._jupyter
+ and self.notebook_name is not None
+ and os.path.exists(self.notebook_name)
+ ):
+ settings["_jupyter_path"] = self.notebook_name
+ settings["_jupyter_name"] = self.notebook_name
+ settings["_jupyter_root"] = os.getcwd()
+ elif self._jupyter:
+ wandb.termwarn(
+ "WANDB_NOTEBOOK_NAME should be a path to a notebook file, "
+ f"couldn't find {self.notebook_name}.",
+ )
+
+ # host and username are populated by apply_env_vars if corresponding env
+ # vars exist -- but if they don't, we'll fill them in here
+ if self.host is None:
+ settings["host"] = socket.gethostname() # type: ignore
+
+ if self.username is None:
+ try: # type: ignore
+ settings["username"] = getpass.getuser()
+ except KeyError:
+ # getuser() could raise KeyError in restricted environments like
+ # chroot jails or docker containers. Return user id in these cases.
+ settings["username"] = str(os.getuid())
+
+ _executable = (
+ self._executable
+ or os.environ.get(wandb.env._EXECUTABLE)
+ or sys.executable
+ or shutil.which("python3")
+ or "python3"
+ )
+ settings["_executable"] = _executable
+
+ settings["docker"] = wandb.env.get_docker(wandb.util.image_id_from_k8s())
+
+ # TODO: we should use the cuda library to collect this
+ if os.path.exists("/usr/local/cuda/version.txt"):
+ with open("/usr/local/cuda/version.txt") as f:
+ settings["_cuda"] = f.read().split(" ")[-1].strip()
+ if not self._jupyter:
+ settings["_args"] = sys.argv[1:]
+ settings["_os"] = platform.platform(aliased=True)
+ settings["_python"] = platform.python_version()
+ # hack to make sure we don't hang on windows
+ if self._windows and self._except_exit is None:
+ settings["_except_exit"] = True # type: ignore
+
+ if _logger is not None:
+ _logger.info(
+ f"Inferring settings from compute environment: {_redact_dict(settings)}"
+ )
+
+ self.update(settings, source=Source.ENV)
+
+ def _infer_run_settings_from_environment(
+ self,
+ _logger: Optional[_EarlyLogger] = None,
+ ) -> None:
+ """Modify settings based on environment (for runs only)."""
+ # If there's not already a program file, infer it now.
+ settings: Dict[str, Union[bool, str, None]] = dict()
+ program = self.program or _get_program()
+ if program is not None:
+ repo = GitRepo()
+ root = repo.root or os.getcwd()
+
+ program_relpath = self.program_relpath or _get_program_relpath(
+ program, repo.root, _logger=_logger
+ )
+ settings["program_relpath"] = program_relpath
+ program_abspath = os.path.abspath(
+ os.path.join(root, os.path.relpath(os.getcwd(), root), program)
+ )
+ if os.path.exists(program_abspath):
+ settings["program_abspath"] = program_abspath
+ else:
+ program = "<python with no main file>"
+
+ settings["program"] = program
+
+ if _logger is not None:
+ _logger.info(
+ f"Inferring run settings from compute environment: {_redact_dict(settings)}"
+ )
+
+ self.update(settings, source=Source.ENV)
+
+ def _apply_setup(
+ self, setup_settings: Dict[str, Any], _logger: Optional[_EarlyLogger] = None
+ ) -> None:
+ if _logger:
+ _logger.info(f"Applying setup settings: {_redact_dict(setup_settings)}")
+ self.update(setup_settings, source=Source.SETUP)
+
+ def _apply_user(
+ self, user_settings: Dict[str, Any], _logger: Optional[_EarlyLogger] = None
+ ) -> None:
+ if _logger:
+ _logger.info(f"Applying user settings: {_redact_dict(user_settings)}")
+ self.update(user_settings, source=Source.USER)
+
+ def _apply_init(self, init_settings: Dict[str, Union[str, int, None]]) -> None:
+ # pop magic from init settings
+ init_settings.pop("magic", None)
+
+ # prevent setting project, entity if in sweep
+ # TODO(jhr): these should be locked elements in the future
+ if self.sweep_id:
+ for key in ("project", "entity", "id"):
+ val = init_settings.pop(key, None)
+ if val:
+ wandb.termwarn(
+ f"Ignored wandb.init() arg {key} when running a sweep."
+ )
+ if self.launch:
+ if self.project is not None and init_settings.pop("project", None):
+ wandb.termwarn(
+ "Project is ignored when running from wandb launch context. "
+ "Ignored wandb.init() arg project when running running from launch.",
+ )
+ for key in ("entity", "id"):
+ # Init settings cannot override launch settings.
+ if init_settings.pop(key, None):
+ wandb.termwarn(
+ "Project, entity and id are ignored when running from wandb launch context. "
+ f"Ignored wandb.init() arg {key} when running running from launch.",
+ )
+
+ # strip out items where value is None
+ param_map = dict(
+ name="run_name",
+ id="run_id",
+ tags="run_tags",
+ group="run_group",
+ job_type="run_job_type",
+ notes="run_notes",
+ dir="root_dir",
+ sweep_id="sweep_id",
+ )
+ init_settings = {
+ param_map.get(k, k): v for k, v in init_settings.items() if v is not None
+ }
+ # fun logic to convert the resume init arg
+ if init_settings.get("resume"):
+ if isinstance(init_settings["resume"], str):
+ if init_settings["resume"] not in ("allow", "must", "never", "auto"):
+ if init_settings.get("run_id") is None:
+ # TODO: deprecate or don't support
+ init_settings["run_id"] = init_settings["resume"]
+ init_settings["resume"] = "allow"
+ elif init_settings["resume"] is True:
+ # todo: add deprecation warning, switch to literal strings for resume
+ init_settings["resume"] = "auto"
+
+ # update settings
+ self.update(init_settings, source=Source.INIT)
+
+ # handle auto resume logic
+ if self.resume == "auto":
+ if os.path.exists(self.resume_fname):
+ with open(self.resume_fname) as f:
+ resume_run_id = json.load(f)["run_id"]
+ if self.run_id is None:
+ self.update({"run_id": resume_run_id}, source=Source.INIT) # type: ignore
+ elif self.run_id != resume_run_id:
+ wandb.termwarn(
+ "Tried to auto resume run with "
+ f"id {resume_run_id} but id {self.run_id} is set.",
+ )
+ self.update({"run_id": self.run_id or generate_id()}, source=Source.INIT)
+ # persist our run id in case of failure
+ # check None for mypy
+ if self.resume == "auto" and self.resume_fname is not None:
+ filesystem.mkdir_exists_ok(self.wandb_dir)
+ with open(self.resume_fname, "w") as f:
+ f.write(json.dumps({"run_id": self.run_id}))
+
+ def _apply_login(
+ self, login_settings: Dict[str, Any], _logger: Optional[_EarlyLogger] = None
+ ) -> None:
+ param_map = dict(key="api_key", host="base_url", timeout="login_timeout")
+ login_settings = {
+ param_map.get(k, k): v for k, v in login_settings.items() if v is not None
+ }
+ if login_settings:
+ if _logger:
+ _logger.info(f"Applying login settings: {_redact_dict(login_settings)}")
+ self.update(login_settings, source=Source.LOGIN)
+
+ def _apply_run_start(self, run_start_settings: Dict[str, Any]) -> None:
+ # This dictionary maps from the "run message dict" to relevant fields in settings
+ # Note: that config is missing
+ param_map = {
+ "run_id": "run_id",
+ "entity": "entity",
+ "project": "project",
+ "run_group": "run_group",
+ "job_type": "run_job_type",
+ "display_name": "run_name",
+ "notes": "run_notes",
+ "tags": "run_tags",
+ "sweep_id": "sweep_id",
+ "host": "host",
+ "resumed": "resumed",
+ "git.remote_url": "git_remote_url",
+ "git.commit": "git_commit",
+ }
+ run_settings = {
+ name: reduce(lambda d, k: d.get(k, {}), attr.split("."), run_start_settings)
+ for attr, name in param_map.items()
+ }
+ run_settings = {key: value for key, value in run_settings.items() if value}
+ if run_settings:
+ self.update(run_settings, source=Source.RUN)
diff --git a/wandb/sdk/wandb_setup.py b/wandb/sdk/wandb_setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..1c87cf5f03091a6007c4e3742113d1167d487843
--- /dev/null
+++ b/wandb/sdk/wandb_setup.py
@@ -0,0 +1,335 @@
+#
+"""Setup wandb session.
+
+This module configures a wandb session which can extend to mutiple wandb runs.
+
+Functions:
+ setup(): Configure wandb session.
+
+Early logging keeps track of logger output until the call to wandb.init() when the
+run_id can be resolved.
+
+"""
+
+import logging
+import os
+import sys
+import threading
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+
+import wandb
+
+from . import wandb_manager, wandb_settings
+from .lib import config_util, server, tracelog
+
+Settings = Union["wandb.sdk.wandb_settings.Settings", Dict[str, Any]]
+
+Logger = Union[logging.Logger, "_EarlyLogger"]
+
+if TYPE_CHECKING:
+ from . import wandb_run
+
+# logger will be configured to be either a standard logger instance or _EarlyLogger
+logger: Optional[Logger] = None
+
+
+def _set_logger(log_object: Logger) -> None:
+ """Configure module logger."""
+ global logger
+ logger = log_object
+
+
+class _EarlyLogger:
+ """Early logger which captures logs in memory until logging can be configured."""
+
+ def __init__(self) -> None:
+ self._log: List[tuple] = []
+ self._exception: List[tuple] = []
+ # support old warn() as alias of warning()
+ self.warn = self.warning
+
+ def debug(self, msg: str, *args: Any, **kwargs: Any) -> None:
+ self._log.append((logging.DEBUG, msg, args, kwargs))
+
+ def info(self, msg: str, *args: Any, **kwargs: Any) -> None:
+ self._log.append((logging.INFO, msg, args, kwargs))
+
+ def warning(self, msg: str, *args: Any, **kwargs: Any) -> None:
+ self._log.append((logging.WARNING, msg, args, kwargs))
+
+ def error(self, msg: str, *args: Any, **kwargs: Any) -> None:
+ self._log.append((logging.ERROR, msg, args, kwargs))
+
+ def critical(self, msg: str, *args: Any, **kwargs: Any) -> None:
+ self._log.append((logging.CRITICAL, msg, args, kwargs))
+
+ def exception(self, msg: str, *args: Any, **kwargs: Any) -> None:
+ self._exception.append((msg, args, kwargs))
+
+ def log(self, level: str, msg: str, *args: Any, **kwargs: Any) -> None:
+ self._log.append((level, msg, args, kwargs))
+
+ def _flush(self) -> None:
+ assert self is not logger
+ assert logger is not None
+ for level, msg, args, kwargs in self._log:
+ logger.log(level, msg, *args, **kwargs)
+ for msg, args, kwargs in self._exception:
+ logger.exception(msg, *args, **kwargs)
+
+
+class _WandbSetup__WandbSetup: # noqa: N801
+ """Inner class of _WandbSetup."""
+
+ _manager: Optional[wandb_manager._Manager]
+ _pid: int
+
+ def __init__(
+ self,
+ pid: int,
+ settings: Optional[Settings] = None,
+ environ: Optional[Dict[str, Any]] = None,
+ ) -> None:
+ self._environ = environ or dict(os.environ)
+ self._sweep_config: Optional[Dict[str, Any]] = None
+ self._config: Optional[Dict[str, Any]] = None
+ self._server: Optional[server.Server] = None
+ self._manager: Optional[wandb_manager._Manager] = None
+ self._pid = pid
+
+ # keep track of multiple runs, so we can unwind with join()s
+ self._global_run_stack: List[wandb_run.Run] = []
+
+ # TODO(jhr): defer strict checks until settings are fully initialized
+ # and logging is ready
+ self._early_logger = _EarlyLogger()
+ _set_logger(self._early_logger)
+
+ self._settings = self._settings_setup(settings, self._early_logger)
+ # self._settings.freeze()
+
+ wandb.termsetup(self._settings, logger)
+
+ self._check()
+ self._setup()
+
+ tracelog_mode = self._settings._tracelog
+ if tracelog_mode:
+ tracelog.enable(tracelog_mode)
+
+ def _settings_setup(
+ self,
+ settings: Optional[Settings] = None,
+ early_logger: Optional[_EarlyLogger] = None,
+ ) -> "wandb_settings.Settings":
+ s = wandb_settings.Settings()
+ s._apply_base(pid=self._pid, _logger=early_logger)
+ s._apply_config_files(_logger=early_logger)
+ s._apply_env_vars(self._environ, _logger=early_logger)
+
+ if isinstance(settings, wandb_settings.Settings):
+ s._apply_settings(settings, _logger=early_logger)
+ elif isinstance(settings, dict):
+ # if passed settings arg is a mapping, update the settings with it
+ s._apply_setup(settings, _logger=early_logger)
+
+ s._infer_settings_from_environment()
+ if not s._cli_only_mode:
+ s._infer_run_settings_from_environment(_logger=early_logger)
+
+ return s
+
+ def _update(
+ self,
+ settings: Optional[Settings] = None,
+ ) -> None:
+ if settings is None:
+ return
+ # self._settings.unfreeze()
+ if isinstance(settings, wandb_settings.Settings):
+ # todo: check the logic here. this _only_ comes up in tests?
+ self._settings._apply_settings(settings)
+ elif isinstance(settings, dict):
+ # if it is a mapping, update the settings with it
+ self._settings.update(settings, source=wandb_settings.Source.SETUP)
+ # self._settings.freeze()
+
+ def _update_user_settings(self, settings: Optional[Settings] = None) -> None:
+ settings = settings or self._settings
+ # Get rid of cached results to force a refresh.
+ self._server = None
+ user_settings = self._load_user_settings(settings=settings)
+ if user_settings is not None:
+ # self._settings.unfreeze()
+ self._settings._apply_user(user_settings)
+ # self._settings.freeze()
+
+ def _early_logger_flush(self, new_logger: Logger) -> None:
+ if not self._early_logger:
+ return
+ _set_logger(new_logger)
+ # self._settings._clear_early_logger()
+ self._early_logger._flush()
+
+ def _get_logger(self) -> Optional[Logger]:
+ return logger
+
+ @property
+ def settings(self) -> "wandb_settings.Settings":
+ return self._settings
+
+ def _get_entity(self) -> Optional[str]:
+ if self._settings and self._settings._offline:
+ return None
+ if self._server is None:
+ self._load_viewer()
+ assert self._server is not None
+ entity = self._server._viewer.get("entity")
+ return entity
+
+ def _get_username(self) -> Optional[str]:
+ if self._settings and self._settings._offline:
+ return None
+ if self._server is None:
+ self._load_viewer()
+ assert self._server is not None
+ username = self._server._viewer.get("username")
+ return username
+
+ def _get_teams(self) -> List[str]:
+ if self._settings and self._settings._offline:
+ return []
+ if self._server is None:
+ self._load_viewer()
+ assert self._server is not None
+ teams = self._server._viewer.get("teams")
+ if teams:
+ teams = [team["node"]["name"] for team in teams["edges"]]
+ return teams or []
+
+ def _load_viewer(self, settings: Optional[Settings] = None) -> None:
+ if self._settings and self._settings._offline:
+ return
+ if isinstance(settings, dict):
+ settings = wandb_settings.Settings(**settings)
+ s = server.Server(settings=settings)
+ s.query_with_timeout()
+ self._server = s
+
+ def _load_user_settings(
+ self, settings: Optional[Settings] = None
+ ) -> Optional[Dict[str, Any]]:
+ if self._server is None:
+ self._load_viewer(settings=settings)
+
+ # offline?
+ if self._server is None:
+ return None
+
+ flags = self._server._flags
+ user_settings = dict()
+ if "code_saving_enabled" in flags:
+ user_settings["save_code"] = flags["code_saving_enabled"]
+
+ email = self._server._viewer.get("email", None)
+ if email:
+ user_settings["email"] = email
+
+ return user_settings
+
+ def _check(self) -> None:
+ if hasattr(threading, "main_thread"):
+ if threading.current_thread() is not threading.main_thread():
+ pass
+ elif threading.current_thread().name != "MainThread":
+ print("bad thread2", threading.current_thread().name)
+ if getattr(sys, "frozen", False):
+ print("frozen, could be trouble")
+
+ def _setup(self) -> None:
+ self._setup_manager()
+
+ sweep_path = self._settings.sweep_param_path
+ if sweep_path:
+ self._sweep_config = config_util.dict_from_config_file(
+ sweep_path, must_exist=True
+ )
+
+ # if config_paths was set, read in config dict
+ if self._settings.config_paths:
+ # TODO(jhr): handle load errors, handle list of files
+ for config_path in self._settings.config_paths:
+ config_dict = config_util.dict_from_config_file(config_path)
+ if config_dict is None:
+ continue
+ if self._config is not None:
+ self._config.update(config_dict)
+ else:
+ self._config = config_dict
+
+ def _teardown(self, exit_code: Optional[int] = None) -> None:
+ exit_code = exit_code or 0
+ self._teardown_manager(exit_code=exit_code)
+
+ def _setup_manager(self) -> None:
+ if self._settings._disable_service:
+ return
+ self._manager = wandb_manager._Manager(settings=self._settings)
+
+ def _teardown_manager(self, exit_code: int) -> None:
+ if not self._manager:
+ return
+ self._manager._teardown(exit_code)
+ self._manager = None
+
+ def _get_manager(self) -> Optional[wandb_manager._Manager]:
+ return self._manager
+
+
+class _WandbSetup:
+ """Wandb singleton class.
+
+ Note: This is a process local singleton.
+ (Forked processes will get a new copy of the object)
+ """
+
+ _instance: Optional["_WandbSetup__WandbSetup"] = None
+
+ def __init__(self, settings: Optional[Settings] = None) -> None:
+ pid = os.getpid()
+ if _WandbSetup._instance and _WandbSetup._instance._pid == pid:
+ _WandbSetup._instance._update(settings=settings)
+ return
+ _WandbSetup._instance = _WandbSetup__WandbSetup(settings=settings, pid=pid)
+
+ def __getattr__(self, name: str) -> Any:
+ return getattr(self._instance, name)
+
+
+def _setup(
+ settings: Optional[Settings] = None,
+ _reset: bool = False,
+) -> Optional["_WandbSetup"]:
+ """Set up library context."""
+ if _reset:
+ setup_instance = _WandbSetup._instance
+ if setup_instance:
+ setup_instance._teardown()
+ _WandbSetup._instance = None
+ return None
+ wl = _WandbSetup(settings=settings)
+ return wl
+
+
+def setup(
+ settings: Optional[Settings] = None,
+) -> Optional["_WandbSetup"]:
+ ret = _setup(settings=settings)
+ return ret
+
+
+def teardown(exit_code: Optional[int] = None) -> None:
+ setup_instance = _WandbSetup._instance
+ if setup_instance:
+ setup_instance._teardown(exit_code=exit_code)
+ _WandbSetup._instance = None
diff --git a/wandb/sdk/wandb_summary.py b/wandb/sdk/wandb_summary.py
new file mode 100644
index 0000000000000000000000000000000000000000..f15c7cc041959115b0cfc0024f5ec2b690209558
--- /dev/null
+++ b/wandb/sdk/wandb_summary.py
@@ -0,0 +1,150 @@
+import abc
+import typing as t
+
+from .interface.summary_record import SummaryItem, SummaryRecord
+
+
+def _get_dict(d):
+ if isinstance(d, dict):
+ return d
+ # assume argparse Namespace
+ return vars(d)
+
+
+class SummaryDict(metaclass=abc.ABCMeta):
+ """dict-like wrapper for the nested dictionaries in a SummarySubDict.
+
+ Triggers self._root._callback on property changes.
+ """
+
+ @abc.abstractmethod
+ def _as_dict(self):
+ raise NotImplementedError
+
+ @abc.abstractmethod
+ def _update(self, record: SummaryRecord):
+ raise NotImplementedError
+
+ def keys(self):
+ return [k for k in self._as_dict().keys() if k != "_wandb"]
+
+ def get(self, key, default=None):
+ return self._as_dict().get(key, default)
+
+ def __getitem__(self, key):
+ item = self._as_dict()[key]
+
+ if isinstance(item, dict):
+ # this nested dict needs to be wrapped:
+ wrapped_item = SummarySubDict()
+ object.__setattr__(wrapped_item, "_items", item)
+ object.__setattr__(wrapped_item, "_parent", self)
+ object.__setattr__(wrapped_item, "_parent_key", key)
+
+ return wrapped_item
+
+ # this item isn't a nested dict
+ return item
+
+ __getattr__ = __getitem__
+
+ def __setitem__(self, key, val):
+ self.update({key: val})
+
+ __setattr__ = __setitem__
+
+ def __delattr__(self, key):
+ record = SummaryRecord()
+ item = SummaryItem()
+ item.key = (key,)
+ record.remove = (item,)
+ self._update(record)
+
+ __delitem__ = __delattr__
+
+ def update(self, d: t.Dict):
+ # import ipdb; ipdb.set_trace()
+ record = SummaryRecord()
+ for key, value in d.items():
+ item = SummaryItem()
+ item.key = (key,)
+ item.value = value
+ record.update.append(item)
+
+ self._update(record)
+
+
+class Summary(SummaryDict):
+ """Track single values for each metric for each run.
+
+ By default, a metric's summary is the last value of its History.
+
+ For example, `wandb.log({'accuracy': 0.9})` will add a new step to History and
+ update Summary to the latest value. In some cases, it's more useful to have
+ the maximum or minimum of a metric instead of the final value. You can set
+ history manually `(wandb.summary['accuracy'] = best_acc)`.
+
+ In the UI, summary metrics appear in the table to compare across runs.
+ Summary metrics are also used in visualizations like the scatter plot and
+ parallel coordinates chart.
+
+ After training has completed, you may want to save evaluation metrics to a
+ run. Summary can handle numpy arrays and PyTorch/TensorFlow tensors. When
+ you save one of these types to Summary, we persist the entire tensor in a
+ binary file and store high level metrics in the summary object, such as min,
+ mean, variance, and 95th percentile.
+
+ Examples:
+ ```python
+ wandb.init(config=args)
+
+ best_accuracy = 0
+ for epoch in range(1, args.epochs + 1):
+ test_loss, test_accuracy = test()
+ if test_accuracy > best_accuracy:
+ wandb.run.summary["best_accuracy"] = test_accuracy
+ best_accuracy = test_accuracy
+ ```
+ """
+
+ _update_callback: t.Callable
+ _get_current_summary_callback: t.Callable
+
+ def __init__(self, get_current_summary_callback: t.Callable):
+ super().__init__()
+ object.__setattr__(self, "_update_callback", None)
+ object.__setattr__(
+ self, "_get_current_summary_callback", get_current_summary_callback
+ )
+
+ def _set_update_callback(self, update_callback: t.Callable):
+ object.__setattr__(self, "_update_callback", update_callback)
+
+ def _as_dict(self):
+ return self._get_current_summary_callback()
+
+ def _update(self, record: SummaryRecord):
+ if self._update_callback: # type: ignore
+ self._update_callback(record)
+
+
+class SummarySubDict(SummaryDict):
+ """Non-root node of the summary data structure.
+
+ Contains a path to itself from the root.
+ """
+
+ _items: t.Dict
+ _parent: SummaryDict
+ _parent_key: str
+
+ def __init__(self):
+ object.__setattr__(self, "_items", dict())
+ object.__setattr__(self, "_parent", None)
+ object.__setattr__(self, "_parent_key", None)
+
+ def _as_dict(self):
+ return self._items
+
+ def _update(self, record: SummaryRecord):
+ return self._parent._update(record._add_next_parent(self._parent_key))
diff --git a/wandb/sdk/wandb_sweep.py b/wandb/sdk/wandb_sweep.py
new file mode 100644
index 0000000000000000000000000000000000000000..cf4a51805b79c9e4ad3e5e684779ebe9f2af780d
--- /dev/null
+++ b/wandb/sdk/wandb_sweep.py
@@ -0,0 +1,114 @@
+import urllib.parse
+from typing import Callable, Dict, Optional, Union
+
+import wandb
+from wandb import env
+from wandb.apis import InternalApi
+from wandb.sdk.launch.sweeps.utils import handle_sweep_config_violations
+
+from . import wandb_login
+
+
+def _get_sweep_url(api, sweep_id):
+ """Return sweep url if we can figure it out."""
+ if api.api_key:
+ if api.settings("entity") is None:
+ viewer = api.viewer()
+ if viewer.get("entity"):
+ api.set_setting("entity", viewer["entity"])
+ project = api.settings("project")
+ if not project:
+ return
+ if api.settings("entity"):
+ return "{base}/{entity}/{project}/sweeps/{sweepid}".format(
+ base=api.app_url,
+ entity=urllib.parse.quote(api.settings("entity")),
+ project=urllib.parse.quote(project),
+ sweepid=urllib.parse.quote(sweep_id),
+ )
+
+
+def sweep(
+ sweep: Union[dict, Callable],
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+) -> str:
+ """Initialize a hyperparameter sweep.
+
+ Search for hyperparameters that optimizes a cost function
+ of a machine learning model by testing various combinations.
+
+ Make note the unique identifier, `sweep_id`, that is returned.
+ At a later step provide the `sweep_id` to a sweep agent.
+
+ Args:
+ sweep: The configuration of a hyperparameter search.
+ (or configuration generator). See
+ [Sweep configuration structure](https://docs.wandb.ai/guides/sweeps/define-sweep-configuration)
+ for information on how to define your sweep.
+ If you provide a callable, ensure that the callable does
+ not take arguments and that it returns a dictionary that
+ conforms to the W&B sweep config spec.
+ entity: The username or team name where you want to send W&B
+ runs created by the sweep to. Ensure that the entity you
+ specify already exists. If you don't specify an entity,
+ the run will be sent to your default entity,
+ which is usually your username.
+ project: The name of the project where W&B runs created from
+ the sweep are sent to. If the project is not specified, the
+ run is sent to a project labeled 'Uncategorized'.
+
+ Returns:
+ sweep_id: str. A unique identifier for the sweep.
+ """
+ if callable(sweep):
+ sweep = sweep()
+ """Sweep create for controller api and jupyter (eventually for cli)."""
+
+ # Project may be only found in the sweep config.
+ if project is None and isinstance(sweep, dict):
+ project = sweep.get("project", None)
+
+ if entity:
+ env.set_entity(entity)
+ if project:
+ env.set_project(project)
+
+ # Make sure we are logged in
+ if wandb.run is None:
+ wandb_login._login(_silent=True)
+ api = InternalApi()
+ sweep_id, warnings = api.upsert_sweep(sweep)
+ handle_sweep_config_violations(warnings)
+ print("Create sweep with ID:", sweep_id)
+ sweep_url = _get_sweep_url(api, sweep_id)
+ if sweep_url:
+ print("Sweep URL:", sweep_url)
+ return sweep_id
+
+
+def controller(
+ sweep_id_or_config: Optional[Union[str, Dict]] = None,
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+):
+ """Public sweep controller constructor.
+
+ Usage:
+ ```python
+ import wandb
+
+ tuner = wandb.controller(...)
+ print(tuner.sweep_config)
+ print(tuner.sweep_id)
+ tuner.configure_search(...)
+ tuner.configure_stopping(...)
+ ```
+
+ """
+ from ..wandb_controller import _WandbController
+
+ c = _WandbController(
+ sweep_id_or_config=sweep_id_or_config, entity=entity, project=project
+ )
+ return c
diff --git a/wandb/sdk/wandb_sync.py b/wandb/sdk/wandb_sync.py
new file mode 100644
index 0000000000000000000000000000000000000000..5654077e8fc8c30fa1a85a8c9d6df688ba5929d2
--- /dev/null
+++ b/wandb/sdk/wandb_sync.py
@@ -0,0 +1,75 @@
+import pathlib
+from typing import TYPE_CHECKING, Optional
+
+from ..errors.term import termerror, termlog
+from . import wandb_setup
+from .backend.backend import Backend
+from .lib.mailbox import Mailbox
+from .lib.runid import generate_id
+
+if TYPE_CHECKING:
+ from wandb.proto import wandb_internal_pb2
+
+
+def _sync(
+ path: str,
+ run_id: Optional[str] = None,
+ project: Optional[str] = None,
+ entity: Optional[str] = None,
+ mark_synced: Optional[bool] = None,
+ append: Optional[bool] = None,
+ skip_console: Optional[bool] = None,
+) -> "wandb_internal_pb2.SyncResponse":
+ p = pathlib.Path(path)
+
+ wl = wandb_setup.setup()
+ assert wl is not None
+
+ stream_id = generate_id()
+
+ settings = wl.settings.to_proto()
+ # update sync_file setting to point to the passed path
+ settings.sync_file.value = str(p.absolute())
+ settings.sync_dir.value = str(p.parent.absolute())
+ settings.files_dir.value = str(p.parent.absolute() / "files")
+ settings._sync.value = True
+ settings.run_id.value = stream_id # TODO: remove this
+ if append:
+ settings.resume.value = "allow"
+
+ manager = wl._get_manager()
+ manager._inform_init(settings=settings, run_id=stream_id)
+
+ mailbox = Mailbox()
+ backend = Backend(settings=wl.settings, manager=manager, mailbox=mailbox)
+ backend.ensure_launched()
+
+ assert backend.interface
+ backend.interface._stream_id = stream_id # type: ignore
+
+ mailbox.enable_keepalive()
+
+ # TODO: let's add extra sync messages here so we get the url in the beginning
+ handle = backend.interface.deliver_sync(
+ start_offset=0,
+ final_offset=-1,
+ entity=entity,
+ project=project,
+ run_id=run_id,
+ skip_output_raw=skip_console,
+ )
+ result = handle.wait(timeout=-1)
+ assert result and result.response
+ response = result.response.sync_response
+ if response.url:
+ termlog(f"Synced {p} to {response.url}")
+ # create a .synced file in the directory if mark_synced is true
+ if mark_synced:
+ with open(f"{p}.synced", "w"):
+ pass
+ else:
+ termerror(f"Failed to sync {p}")
+ if response.error and response.error.message:
+ termerror(response.error.message)
+
+ return response
diff --git a/wandb/sdk/wandb_watch.py b/wandb/sdk/wandb_watch.py
new file mode 100644
index 0000000000000000000000000000000000000000..e2eda84ed69dbbcbf3aaaa22addb32ac854d50bd
--- /dev/null
+++ b/wandb/sdk/wandb_watch.py
@@ -0,0 +1,128 @@
+"""watch."""
+
+import logging
+from typing import Optional
+
+try:
+ from typing import Literal
+except ImportError:
+ from typing_extensions import Literal # type: ignore
+
+import wandb
+
+from .lib import telemetry
+
+logger = logging.getLogger("wandb")
+
+_global_watch_idx = 0
+
+
+def watch(
+ models,
+ criterion=None,
+ log: Optional[Literal["gradients", "parameters", "all"]] = "gradients",
+ log_freq: int = 1000,
+ idx: Optional[int] = None,
+ log_graph: bool = False,
+):
+ """Hook into the torch model to collect gradients and the topology.
+
+ Should be extended to accept arbitrary ML models.
+
+ Args:
+ models: (torch.Module) The model to hook, can be a tuple
+ criterion: (torch.F) An optional loss value being optimized
+ log: (str) One of "gradients", "parameters", "all", or None
+ log_freq: (int) log gradients and parameters every N batches
+ idx: (int) an index to be used when calling wandb.watch on multiple models
+ log_graph: (boolean) log graph topology
+
+ Returns:
+ `wandb.Graph`: The graph object that will populate after the first backward pass
+
+ Raises:
+ ValueError: If called before `wandb.init` or if any of models is not a torch.nn.Module.
+ """
+ global _global_watch_idx
+
+ with telemetry.context() as tel:
+ tel.feature.watch = True
+
+ logger.info("Watching")
+
+ if wandb.run is None:
+ raise ValueError("You must call `wandb.init` before calling watch")
+
+ if log not in {"gradients", "parameters", "all", None}:
+ raise ValueError("log must be one of 'gradients', 'parameters', 'all', or None")
+
+ log_parameters = log in {"parameters", "all"}
+ log_gradients = log in {"gradients", "all"}
+
+ if not isinstance(models, (tuple, list)):
+ models = (models,)
+
+ torch = wandb.util.get_module(
+ "torch", required="wandb.watch only works with pytorch, couldn't import torch."
+ )
+
+ for model in models:
+ if not isinstance(model, torch.nn.Module):
+ raise ValueError(
+ "Expected a pytorch model (torch.nn.Module). Received "
+ + str(type(model))
+ )
+
+ graphs = []
+ prefix = ""
+
+ if idx is None:
+ idx = _global_watch_idx
+ for local_idx, model in enumerate(models):
+ global_idx = idx + local_idx
+ _global_watch_idx += 1
+ if global_idx > 0:
+ # TODO: this makes ugly chart names like gradients/graph_1conv1d.bias
+ prefix = "graph_%i" % global_idx
+
+ if log_parameters:
+ wandb.run._torch.add_log_parameters_hook(
+ model,
+ prefix=prefix,
+ log_freq=log_freq,
+ )
+
+ if log_gradients:
+ wandb.run._torch.add_log_gradients_hook(
+ model,
+ prefix=prefix,
+ log_freq=log_freq,
+ )
+
+ if log_graph:
+ graph = wandb.run._torch.hook_torch(model, criterion, graph_idx=global_idx)
+ graphs.append(graph)
+ # NOTE: the graph is set in run.summary by hook_torch on the backward pass
+ return graphs
+
+
+def unwatch(models=None):
+ """Remove pytorch model topology, gradient and parameter hooks.
+
+ Args:
+ models: (list) Optional list of pytorch models that have had watch called on them
+ """
+ if models:
+ if not isinstance(models, (tuple, list)):
+ models = (models,)
+ for model in models:
+ if not hasattr(model, "_wandb_hook_names"):
+ wandb.termwarn("%s model has not been watched" % model)
+ else:
+ for name in model._wandb_hook_names:
+ wandb.run._torch.unhook(name)
+ delattr(model, "_wandb_hook_names")
+ # TODO: we should also remove recursively model._wandb_watch_called
+
+ else:
+ wandb.run._torch.unhook_all()
diff --git a/wandb/sklearn/__init__.py b/wandb/sklearn/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..012c1ce9d2619e803921eb0da5da5d158120dd7d
--- /dev/null
+++ b/wandb/sklearn/__init__.py
@@ -0,0 +1,36 @@
+"""Create informative charts for scikit-learn models and log them to W&B."""
+from .plot import (
+ plot_calibration_curve,
+ plot_class_proportions,
+ plot_classifier,
+ plot_clusterer,
+ plot_confusion_matrix,
+ plot_elbow_curve,
+ plot_feature_importances,
+ plot_learning_curve,
+ plot_outlier_candidates,
+ plot_precision_recall,
+ plot_regressor,
+ plot_residuals,
+ plot_roc,
+ plot_silhouette,
+ plot_summary_metrics,
+)
+
+__all__ = [
+ "plot_classifier",
+ "plot_clusterer",
+ "plot_regressor",
+ "plot_summary_metrics",
+ "plot_learning_curve",
+ "plot_feature_importances",
+ "plot_class_proportions",
+ "plot_calibration_curve",
+ "plot_roc",
+ "plot_precision_recall",
+ "plot_confusion_matrix",
+ "plot_elbow_curve",
+ "plot_silhouette",
+ "plot_residuals",
+ "plot_outlier_candidates",
+]
diff --git a/wandb/sklearn/calculate/__init__.py b/wandb/sklearn/calculate/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d22d629bbc7dced6da390f885a4d327049491ff
--- /dev/null
+++ b/wandb/sklearn/calculate/__init__.py
@@ -0,0 +1,32 @@
+"""Calculates and formats metrics and charts for introspecting sklearn models.
+
+The functions in these modules are designed to be called by functions from the
+plot submodule that have been exported into the namespace of the wandb.sklearn
+submodule, rather than being called directly.
+"""
+
+from .calibration_curves import calibration_curves
+from .class_proportions import class_proportions
+from .confusion_matrix import confusion_matrix
+from .decision_boundaries import decision_boundaries
+from .elbow_curve import elbow_curve
+from .feature_importances import feature_importances
+from .learning_curve import learning_curve
+from .outlier_candidates import outlier_candidates
+from .residuals import residuals
+from .silhouette import silhouette
+from .summary_metrics import summary_metrics
+
+__all__ = [
+ "calibration_curves",
+ "class_proportions",
+ "confusion_matrix",
+ "decision_boundaries",
+ "elbow_curve",
+ "feature_importances",
+ "learning_curve",
+ "outlier_candidates",
+ "residuals",
+ "silhouette",
+ "summary_metrics",
+]
diff --git a/wandb/sklearn/calculate/calibration_curves.py b/wandb/sklearn/calculate/calibration_curves.py
new file mode 100644
index 0000000000000000000000000000000000000000..b177394aef988c951817d11d77ec359a13cbc048
--- /dev/null
+++ b/wandb/sklearn/calculate/calibration_curves.py
@@ -0,0 +1,125 @@
+from warnings import simplefilter
+
+import numpy as np
+import sklearn
+from sklearn import model_selection, naive_bayes
+from sklearn.calibration import CalibratedClassifierCV
+from sklearn.linear_model import LogisticRegression
+
+import wandb
+from wandb.sklearn import utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def calibration_curves(clf, X, y, clf_name):
+ # ComplementNB (introduced in 0.20.0) requires non-negative features
+ if int(sklearn.__version__.split(".")[1]) >= 20 and isinstance(
+ clf, naive_bayes.ComplementNB
+ ):
+ X = X - X.min()
+
+ # Calibrated with isotonic calibration
+ isotonic = CalibratedClassifierCV(clf, cv=2, method="isotonic")
+
+ # Calibrated with sigmoid calibration
+ sigmoid = CalibratedClassifierCV(clf, cv=2, method="sigmoid")
+
+ # Logistic regression with no calibration as baseline
+ lr = LogisticRegression(C=1.0)
+
+ model_column = [] # color
+ frac_positives_column = [] # y axis
+ mean_pred_value_column = [] # x axis
+ hist_column = [] # barchart y
+ edge_column = [] # barchart x
+
+ # Add curve for perfectly calibrated model
+ # format: model, fraction_of_positives, mean_predicted_value
+ model_column.append("Perfectly calibrated")
+ frac_positives_column.append(0)
+ mean_pred_value_column.append(0)
+ hist_column.append(0)
+ edge_column.append(0)
+ model_column.append("Perfectly calibrated")
+ hist_column.append(0)
+ edge_column.append(0)
+ frac_positives_column.append(1)
+ mean_pred_value_column.append(1)
+
+ X_train, X_test, y_train, y_test = model_selection.train_test_split(
+ X, y, test_size=0.9, random_state=42
+ )
+
+ # Add curve for LogisticRegression baseline and other models
+
+ models = [lr, isotonic, sigmoid]
+ names = ["Logistic", f"{clf_name} Isotonic", f"{clf_name} Sigmoid"]
+
+ for model, name in zip(models, names):
+ model.fit(X_train, y_train)
+ if hasattr(model, "predict_proba"):
+ prob_pos = model.predict_proba(X_test)[:, 1]
+ else: # use decision function
+ prob_pos = model.decision_function(X_test)
+ prob_pos = (prob_pos - prob_pos.min()) / (prob_pos.max() - prob_pos.min())
+
+ hist, edges = np.histogram(prob_pos, bins=10, density=False)
+ frac_positives, mean_pred_value = sklearn.calibration.calibration_curve(
+ y_test, prob_pos, n_bins=10
+ )
+
+ # format: model, fraction_of_positives, mean_predicted_value
+ num_entries = len(frac_positives)
+ for i in range(num_entries):
+ hist_column.append(hist[i])
+ edge_column.append(edges[i])
+ model_column.append(name)
+ frac_positives_column.append(utils.round_3(frac_positives[i]))
+ mean_pred_value_column.append(utils.round_3(mean_pred_value[i]))
+ if utils.check_against_limit(
+ i,
+ "calibration_curve",
+ utils.chart_limit - 2,
+ ):
+ break
+
+ table = make_table(
+ model_column,
+ frac_positives_column,
+ mean_pred_value_column,
+ hist_column,
+ edge_column,
+ )
+ chart = wandb.visualize("wandb/calibration/v1", table)
+
+ return chart
+
+
+def make_table(
+ model_column,
+ frac_positives_column,
+ mean_pred_value_column,
+ hist_column,
+ edge_column,
+):
+ columns = [
+ "model",
+ "fraction_of_positives",
+ "mean_predicted_value",
+ "hist_dict",
+ "edge_dict",
+ ]
+
+ data = list(
+ zip(
+ model_column,
+ frac_positives_column,
+ mean_pred_value_column,
+ hist_column,
+ edge_column,
+ )
+ )
+
+ return wandb.Table(columns=columns, data=data)
diff --git a/wandb/sklearn/calculate/class_proportions.py b/wandb/sklearn/calculate/class_proportions.py
new file mode 100644
index 0000000000000000000000000000000000000000..13e04ab37a9986ef6d00dfcea584efed2776e408
--- /dev/null
+++ b/wandb/sklearn/calculate/class_proportions.py
@@ -0,0 +1,68 @@
+from warnings import simplefilter
+
+import numpy as np
+from sklearn.utils.multiclass import unique_labels
+
+import wandb
+from wandb.sklearn import utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def class_proportions(y_train, y_test, labels):
+ # Get the unique values from the dataset
+ targets = (y_train,) if y_test is None else (y_train, y_test)
+ class_ids = np.array(unique_labels(*targets))
+
+ # Compute the class counts
+ counts_train = np.array([(y_train == c).sum() for c in class_ids])
+ counts_test = np.array([(y_test == c).sum() for c in class_ids])
+
+ class_column, dataset_column, count_column = make_columns(
+ class_ids, counts_train, counts_test
+ )
+
+ if labels is not None and (
+ isinstance(class_column[0], int) or isinstance(class_column[0], np.integer)
+ ):
+ class_column = get_named_labels(labels, class_column)
+
+ table = make_table(class_column, dataset_column, count_column)
+ chart = wandb.visualize("wandb/class_proportions/v1", table)
+
+ return chart
+
+
+def make_table(class_column, dataset_column, count_column):
+ columns = ["class", "dataset", "count"]
+ data = list(zip(class_column, dataset_column, count_column))
+
+ return wandb.Table(data=data, columns=columns)
+
+
+def make_columns(class_ids, counts_train, counts_test):
+ class_column, dataset_column, count_column = [], [], []
+
+ for i in range(len(class_ids)):
+ # add class counts from training set
+ class_column.append(class_ids[i])
+ dataset_column.append("train")
+ count_column.append(counts_train[i])
+ # add class counts from test set
+ class_column.append(class_ids[i])
+ dataset_column.append("test")
+ count_column.append(counts_test[i])
+
+ if utils.check_against_limit(
+ i,
+ "class_proportions",
+ utils.chart_limit,
+ ):
+ break
+
+ return class_column, dataset_column, count_column
+
+
+def get_named_labels(labels, numeric_labels):
+ return np.array([labels[num_label] for num_label in numeric_labels])
diff --git a/wandb/sklearn/calculate/confusion_matrix.py b/wandb/sklearn/calculate/confusion_matrix.py
new file mode 100644
index 0000000000000000000000000000000000000000..a2bc509848e27d874689d1455bf46cdc91ff7d9f
--- /dev/null
+++ b/wandb/sklearn/calculate/confusion_matrix.py
@@ -0,0 +1,92 @@
+import itertools
+from warnings import simplefilter
+
+import numpy as np
+from sklearn import metrics
+from sklearn.utils.multiclass import unique_labels
+
+import wandb
+from wandb.sklearn import utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def validate_labels(*args, **kwargs): # FIXME
+ assert False
+
+
+def confusion_matrix(
+ y_true=None,
+ y_pred=None,
+ labels=None,
+ true_labels=None,
+ pred_labels=None,
+ normalize=False,
+):
+ """Compute the confusion matrix to evaluate the performance of a classification.
+
+ Called by plot_confusion_matrix to visualize roc curves. Please use the function
+ plot_confusion_matrix() if you wish to visualize your confusion matrix.
+ """
+ cm = metrics.confusion_matrix(y_true, y_pred)
+
+ if labels is None:
+ classes = unique_labels(y_true, y_pred)
+ else:
+ classes = np.asarray(labels)
+
+ if normalize:
+ cm = cm.astype("float") / cm.sum(axis=1)[:, np.newaxis]
+ cm = np.around(cm, decimals=2)
+ cm[np.isnan(cm)] = 0.0
+
+ if true_labels is None:
+ true_classes = classes
+ else:
+ validate_labels(classes, true_labels, "true_labels")
+
+ true_label_indexes = np.in1d(classes, true_labels)
+
+ true_classes = classes[true_label_indexes]
+ cm = cm[true_label_indexes]
+
+ if pred_labels is None:
+ pred_classes = classes
+ else:
+ validate_labels(classes, pred_labels, "pred_labels")
+
+ pred_label_indexes = np.in1d(classes, pred_labels)
+
+ pred_classes = classes[pred_label_indexes]
+ cm = cm[:, pred_label_indexes]
+
+ table = make_table(cm, pred_classes, true_classes, labels)
+ chart = wandb.visualize("wandb/confusion_matrix/v1", table)
+
+ return chart
+
+
+def make_table(cm, pred_classes, true_classes, labels):
+ data, count = [], 0
+ for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
+ if labels is not None and (
+ isinstance(pred_classes[i], int) or isinstance(pred_classes[0], np.integer)
+ ):
+ pred = labels[pred_classes[i]]
+ true = labels[true_classes[j]]
+ else:
+ pred = pred_classes[i]
+ true = true_classes[j]
+ data.append([pred, true, cm[i, j]])
+ count += 1
+ if utils.check_against_limit(
+ count,
+ "confusion_matrix",
+ utils.chart_limit,
+ ):
+ break
+
+ table = wandb.Table(columns=["Predicted", "Actual", "Count"], data=data)
+
+ return table
diff --git a/wandb/sklearn/calculate/decision_boundaries.py b/wandb/sklearn/calculate/decision_boundaries.py
new file mode 100644
index 0000000000000000000000000000000000000000..a7a849b86b151270adf2437e4f2e72a03ce6c1a9
--- /dev/null
+++ b/wandb/sklearn/calculate/decision_boundaries.py
@@ -0,0 +1,40 @@
+from warnings import simplefilter
+
+import wandb
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def decision_boundaries(
+ decision_boundary_x,
+ decision_boundary_y,
+ decision_boundary_color,
+ train_x,
+ train_y,
+ train_color,
+ test_x,
+ test_y,
+ test_color,
+):
+ x_dict, y_dict, color_dict = [], [], []
+ for i in range(min(len(decision_boundary_x), 100)):
+ x_dict.append(decision_boundary_x[i])
+ y_dict.append(decision_boundary_y[i])
+ color_dict.append(decision_boundary_color)
+ for i in range(300):
+ x_dict.append(test_x[i])
+ y_dict.append(test_y[i])
+ color_dict.append(test_color[i])
+ for i in range(min(len(train_x), 600)):
+ x_dict.append(train_x[i])
+ y_dict.append(train_y[i])
+ color_dict.append(train_color[i])
+
+ return wandb.visualize(
+ "wandb/decision_boundaries/v1",
+ wandb.Table(
+ columns=["x", "y", "color"],
+ data=[[x_dict[i], y_dict[i], color_dict[i]] for i in range(len(x_dict))],
+ ),
+ )
diff --git a/wandb/sklearn/calculate/elbow_curve.py b/wandb/sklearn/calculate/elbow_curve.py
new file mode 100644
index 0000000000000000000000000000000000000000..f324684785fa96fc8f4e63922c871bffff8c5cfe
--- /dev/null
+++ b/wandb/sklearn/calculate/elbow_curve.py
@@ -0,0 +1,55 @@
+import time
+from warnings import simplefilter
+
+import numpy as np
+from joblib import Parallel, delayed
+from sklearn.base import clone
+
+import wandb
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def elbow_curve(clusterer, X, cluster_ranges, n_jobs, show_cluster_time):
+ if cluster_ranges is None:
+ cluster_ranges = range(1, 10, 2)
+ else:
+ cluster_ranges = sorted(cluster_ranges)
+
+ clfs, times = _compute_results_parallel(n_jobs, clusterer, X, cluster_ranges)
+
+ clfs = np.absolute(clfs)
+
+ table = make_table(cluster_ranges, clfs, times)
+ chart = wandb.visualize("wandb/elbow/v1", table)
+
+ return chart
+
+
+def make_table(cluster_ranges, clfs, times):
+ columns = ["cluster_ranges", "errors", "clustering_time"]
+
+ data = list(zip(cluster_ranges, clfs, times))
+
+ table = wandb.Table(columns=columns, data=data)
+
+ return table
+
+
+def _compute_results_parallel(n_jobs, clusterer, X, cluster_ranges):
+ parallel_runner = Parallel(n_jobs=n_jobs)
+ _cluster_scorer = delayed(_clone_and_score_clusterer)
+ results = parallel_runner(_cluster_scorer(clusterer, X, i) for i in cluster_ranges)
+
+ clfs, times = zip(*results)
+
+ return clfs, times
+
+
+def _clone_and_score_clusterer(clusterer, X, n_clusters):
+ start = time.time()
+ clusterer = clone(clusterer)
+ setattr(clusterer, "n_clusters", n_clusters)
+
+ return clusterer.fit(X).score(X), time.time() - start
diff --git a/wandb/sklearn/calculate/feature_importances.py b/wandb/sklearn/calculate/feature_importances.py
new file mode 100644
index 0000000000000000000000000000000000000000..fac0452b944fd31d3bf611df054f4f6c0dbc0895
--- /dev/null
+++ b/wandb/sklearn/calculate/feature_importances.py
@@ -0,0 +1,67 @@
+from warnings import simplefilter
+
+import numpy as np
+
+import wandb
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def feature_importances(model, feature_names):
+ attributes_to_check = ["feature_importances_", "feature_log_prob_", "coef_"]
+ found_attribute = check_for_attribute_on(model, attributes_to_check)
+ if found_attribute is None:
+ wandb.termwarn(
+ f"could not find any of attributes {', '.join(attributes_to_check)} on classifier. Cannot plot feature importances."
+ )
+ return
+ elif found_attribute == "feature_importances_":
+ importances = model.feature_importances_
+ elif found_attribute == "coef_": # ElasticNet-like models
+ importances = model.coef_
+ elif found_attribute == "feature_log_prob_":
+ # coef_ was deprecated in sklearn 0.24, replaced with
+ # feature_log_prob_
+ importances = model.feature_log_prob_
+
+ if len(importances.shape) > 1:
+ n_significant_dims = sum(i > 1 for i in importances.shape)
+ if n_significant_dims > 1:
+ nd = len(importances.shape)
+ wandb.termwarn(
+ f"{nd}-dimensional feature importances array passed to plot_feature_importances. "
+ f"{nd}-dimensional and higher feature importances arrays are not currently supported. "
+ f"These importances will not be plotted."
+ )
+ return
+ else:
+ importances = np.squeeze(importances)
+
+ indices = np.argsort(importances)[::-1]
+ importances = importances[indices]
+
+ if feature_names is None:
+ feature_names = indices
+ else:
+ feature_names = np.array(feature_names)[indices]
+
+ table = make_table(feature_names, importances)
+ chart = wandb.visualize("wandb/feature_importances/v1", table)
+
+ return chart
+
+
+def make_table(feature_names, importances):
+ table = wandb.Table(
+ columns=["feature_names", "importances"],
+ data=[[feature_names[i], importances[i]] for i in range(len(feature_names))],
+ )
+ return table
+
+
+def check_for_attribute_on(model, attributes_to_check):
+ for attr in attributes_to_check:
+ if hasattr(model, attr):
+ return attr
+ return None
diff --git a/wandb/sklearn/calculate/learning_curve.py b/wandb/sklearn/calculate/learning_curve.py
new file mode 100644
index 0000000000000000000000000000000000000000..b300462830e0a4cec9677d96958ebead677882f6
--- /dev/null
+++ b/wandb/sklearn/calculate/learning_curve.py
@@ -0,0 +1,64 @@
+from warnings import simplefilter
+
+import numpy as np
+from sklearn import model_selection
+
+import wandb
+from wandb.sklearn import utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def learning_curve(
+ model,
+ X,
+ y,
+ cv=None,
+ shuffle=False,
+ random_state=None,
+ train_sizes=None,
+ n_jobs=1,
+ scoring=None,
+):
+ """Train model on datasets of varying size and generates plot of score vs size.
+
+ Called by plot_learning_curve to visualize learning curve. Please use the function
+ plot_learning_curve() if you wish to visualize your learning curves.
+ """
+ train_sizes, train_scores, test_scores = model_selection.learning_curve(
+ model,
+ X,
+ y,
+ cv=cv,
+ n_jobs=n_jobs,
+ train_sizes=train_sizes,
+ scoring=scoring,
+ shuffle=shuffle,
+ random_state=random_state,
+ )
+ train_scores_mean = np.mean(train_scores, axis=1)
+ test_scores_mean = np.mean(test_scores, axis=1)
+
+ table = make_table(train_scores_mean, test_scores_mean, train_sizes)
+ chart = wandb.visualize("wandb/learning_curve/v1", table)
+
+ return chart
+
+
+def make_table(train, test, train_sizes):
+ data = []
+ for i in range(len(train)):
+ if utils.check_against_limit(
+ i,
+ "learning_curve",
+ utils.chart_limit / 2,
+ ):
+ break
+ train_set = ["train", utils.round_2(train[i]), train_sizes[i]]
+ test_set = ["test", utils.round_2(test[i]), train_sizes[i]]
+ data.append(train_set)
+ data.append(test_set)
+
+ table = wandb.Table(columns=["dataset", "score", "train_size"], data=data)
+ return table
diff --git a/wandb/sklearn/calculate/outlier_candidates.py b/wandb/sklearn/calculate/outlier_candidates.py
new file mode 100644
index 0000000000000000000000000000000000000000..db88e7dcbd0fbb66e375b6b07a48a87fed903e33
--- /dev/null
+++ b/wandb/sklearn/calculate/outlier_candidates.py
@@ -0,0 +1,69 @@
+from warnings import simplefilter
+
+import numpy as np
+
+import wandb
+from wandb.sklearn import utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def outlier_candidates(regressor, X, y):
+ # Fit a linear model to X and y to compute MSE
+ regressor.fit(X, y)
+
+ # Leverage is computed as the diagonal of the projection matrix of X
+ leverage = (X * np.linalg.pinv(X).T).sum(1)
+
+ # Compute the rank and the degrees of freedom of the OLS model
+ rank = np.linalg.matrix_rank(X)
+ df = X.shape[0] - rank
+
+ # Compute the MSE from the residuals
+ residuals = y - regressor.predict(X)
+ mse = np.dot(residuals, residuals) / df
+
+ # Compute Cook's distance
+ residuals_studentized = residuals / np.sqrt(mse) / np.sqrt(1 - leverage)
+ distance_ = residuals_studentized**2 / X.shape[1]
+ distance_ *= leverage / (1 - leverage)
+
+ # Compute the influence threshold rule of thumb
+ influence_threshold_ = 4 / X.shape[0]
+ outlier_percentage_ = sum(distance_ >= influence_threshold_) / X.shape[0]
+ outlier_percentage_ *= 100.0
+
+ distance_dict, count = [], 0
+ for d in distance_:
+ distance_dict.append(d)
+ count += 1
+ if utils.check_against_limit(
+ count,
+ "outlier_candidates",
+ utils.chart_limit,
+ ):
+ break
+
+ table = make_table(distance_dict, outlier_percentage_, influence_threshold_)
+ chart = wandb.visualize("wandb/outliers/v1", table)
+
+ return chart
+
+
+def make_table(distance, outlier_percentage, influence_threshold):
+ columns = [
+ "distance",
+ "instance_indicies",
+ "outlier_percentage",
+ "influence_threshold",
+ ]
+
+ data = [
+ [distance[i], i, utils.round_3(outlier_percentage), influence_threshold]
+ for i in range(len(distance))
+ ]
+
+ table = wandb.Table(columns=columns, data=data)
+
+ return table
diff --git a/wandb/sklearn/calculate/residuals.py b/wandb/sklearn/calculate/residuals.py
new file mode 100644
index 0000000000000000000000000000000000000000..ac2f4a4f0afa00b0883133653da0c30c6e1fcf1b
--- /dev/null
+++ b/wandb/sklearn/calculate/residuals.py
@@ -0,0 +1,86 @@
+from warnings import simplefilter
+
+from sklearn import model_selection
+
+import wandb
+from wandb.sklearn import utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def residuals(regressor, X, y):
+ # Create the train and test splits
+ X_train, X_test, y_train, y_test = model_selection.train_test_split(
+ X, y, test_size=0.2
+ )
+
+ # Store labels and colors for the legend ordered by call
+ regressor.fit(X_train, y_train)
+ train_score_ = regressor.score(X_train, y_train)
+ test_score_ = regressor.score(X_test, y_test)
+
+ y_pred_train = regressor.predict(X_train)
+ residuals_train = y_pred_train - y_train
+
+ y_pred_test = regressor.predict(X_test)
+ residuals_test = y_pred_test - y_test
+
+ table = make_table(
+ y_pred_train,
+ residuals_train,
+ y_pred_test,
+ residuals_test,
+ train_score_,
+ test_score_,
+ )
+ chart = wandb.visualize("wandb/residuals_plot/v1", table)
+
+ return chart
+
+
+def make_table(
+ y_pred_train,
+ residuals_train,
+ y_pred_test,
+ residuals_test,
+ train_score_,
+ test_score_,
+):
+ y_pred_column, dataset_column, residuals_column = [], [], []
+
+ datapoints, max_datapoints_train = 0, 100
+ for pred, residual in zip(y_pred_train, residuals_train):
+ # add class counts from training set
+ y_pred_column.append(pred)
+ dataset_column.append("train")
+ residuals_column.append(residual)
+ datapoints += 1
+ if utils.check_against_limit(datapoints, "residuals", max_datapoints_train):
+ break
+
+ datapoints = 0
+ for pred, residual in zip(y_pred_test, residuals_test):
+ # add class counts from training set
+ y_pred_column.append(pred)
+ dataset_column.append("test")
+ residuals_column.append(residual)
+ datapoints += 1
+ if utils.check_against_limit(datapoints, "residuals", max_datapoints_train):
+ break
+
+ columns = ["dataset", "y_pred", "residuals", "train_score", "test_score"]
+ data = [
+ [
+ dataset_column[i],
+ y_pred_column[i],
+ residuals_column[i],
+ train_score_,
+ test_score_,
+ ]
+ for i in range(len(y_pred_column))
+ ]
+
+ table = wandb.Table(columns=columns, data=data)
+
+ return table
diff --git a/wandb/sklearn/calculate/silhouette.py b/wandb/sklearn/calculate/silhouette.py
new file mode 100644
index 0000000000000000000000000000000000000000..a28d453c6cf918edd9f27b2378570419dd1aeade
--- /dev/null
+++ b/wandb/sklearn/calculate/silhouette.py
@@ -0,0 +1,118 @@
+from warnings import simplefilter
+
+import numpy as np
+from sklearn.metrics import silhouette_samples, silhouette_score
+from sklearn.preprocessing import LabelEncoder
+
+import wandb
+from wandb.sklearn import utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def silhouette(clusterer, X, cluster_labels, labels, metric, kmeans):
+ # Run clusterer for n_clusters in range(len(cluster_ranges), get cluster labels
+ # TODO - keep/delete once we decide if we should train clusterers
+ # or ask for trained models
+ # clusterer.set_params(n_clusters=n_clusters, random_state=42)
+ # cluster_labels = clusterer.fit_predict(X)
+ cluster_labels = np.asarray(cluster_labels)
+ labels = np.asarray(labels)
+
+ le = LabelEncoder()
+ _ = le.fit_transform(cluster_labels)
+ n_clusters = len(np.unique(cluster_labels))
+
+ # The silhouette_score gives the average value for all the samples.
+ # This gives a perspective into the density and separation of the formed
+ # clusters
+ silhouette_avg = silhouette_score(X, cluster_labels, metric=metric)
+
+ # Compute the silhouette scores for each sample
+ sample_silhouette_values = silhouette_samples(X, cluster_labels, metric=metric)
+
+ x_sil, y_sil, color_sil = [], [], []
+
+ count, y_lower = 0, 10
+ for i in range(n_clusters):
+ # Aggregate the silhouette scores for samples belonging to
+ # cluster i, and sort them
+ ith_cluster_silhouette_values = sample_silhouette_values[cluster_labels == i]
+
+ ith_cluster_silhouette_values.sort()
+
+ size_cluster_i = ith_cluster_silhouette_values.shape[0]
+ y_upper = y_lower + size_cluster_i
+
+ y_values = np.arange(y_lower, y_upper)
+
+ for j in range(len(y_values)):
+ y_sil.append(y_values[j])
+ x_sil.append(ith_cluster_silhouette_values[j])
+ color_sil.append(i)
+ count += 1
+ if utils.check_against_limit(count, "silhouette", utils.chart_limit):
+ break
+
+ # Compute the new y_lower for next plot
+ y_lower = y_upper + 10 # 10 for the 0 samples
+
+ if kmeans:
+ centers = clusterer.cluster_centers_
+ centerx = centers[:, 0]
+ centery = centers[:, 1]
+
+ else:
+ centerx = [None] * len(color_sil)
+ centery = [None] * len(color_sil)
+
+ table = make_table(
+ X[:, 0],
+ X[:, 1],
+ cluster_labels,
+ centerx,
+ centery,
+ y_sil,
+ x_sil,
+ color_sil,
+ silhouette_avg,
+ )
+ chart = wandb.visualize("wandb/silhouette_/v1", table)
+
+ return chart
+
+
+def make_table(x, y, colors, centerx, centery, y_sil, x_sil, color_sil, silhouette_avg):
+ columns = [
+ "x",
+ "y",
+ "colors",
+ "centerx",
+ "centery",
+ "y_sil",
+ "x1",
+ "x2",
+ "color_sil",
+ "silhouette_avg",
+ ]
+
+ data = [
+ [
+ x[i],
+ y[i],
+ colors[i],
+ centerx[colors[i]],
+ centery[colors[i]],
+ y_sil[i],
+ 0,
+ x_sil[i],
+ color_sil[i],
+ silhouette_avg,
+ ]
+ for i in range(len(color_sil))
+ ]
+
+ table = wandb.Table(data=data, columns=columns)
+
+ return table
diff --git a/wandb/sklearn/calculate/summary_metrics.py b/wandb/sklearn/calculate/summary_metrics.py
new file mode 100644
index 0000000000000000000000000000000000000000..1110053b89578dafaf63fb1ce1a149838dade298
--- /dev/null
+++ b/wandb/sklearn/calculate/summary_metrics.py
@@ -0,0 +1,62 @@
+from warnings import simplefilter
+
+import numpy as np
+import sklearn
+
+import wandb
+from wandb.sklearn import utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def summary_metrics(model=None, X=None, y=None, X_test=None, y_test=None):
+ """Calculate summary metrics for both regressors and classifiers.
+
+ Called by plot_summary_metrics to visualize metrics. Please use the function
+ plot_summary_metrics() if you wish to visualize your summary metrics.
+ """
+ y, y_test = np.asarray(y), np.asarray(y_test)
+ metrics = {}
+ model_name = model.__class__.__name__
+
+ y_pred = model.predict(X_test)
+
+ if sklearn.base.is_classifier(model):
+ accuracy_score = sklearn.metrics.accuracy_score(y_test, y_pred)
+ metrics["accuracy_score"] = accuracy_score
+
+ precision = sklearn.metrics.precision_score(y_test, y_pred, average="weighted")
+ metrics["precision"] = precision
+
+ recall = sklearn.metrics.recall_score(y_test, y_pred, average="weighted")
+ metrics["recall"] = recall
+
+ f1_score = sklearn.metrics.f1_score(y_test, y_pred, average="weighted")
+ metrics["f1_score"] = f1_score
+
+ elif sklearn.base.is_regressor(model):
+ mae = sklearn.metrics.mean_absolute_error(y_test, y_pred)
+ metrics["mae"] = mae
+
+ mse = sklearn.metrics.mean_squared_error(y_test, y_pred)
+ metrics["mse"] = mse
+
+ r2_score = sklearn.metrics.r2_score(y_test, y_pred)
+ metrics["r2_score"] = r2_score
+
+ metrics = {name: utils.round_2(metric) for name, metric in metrics.items()}
+
+ table = make_table(metrics, model_name)
+ chart = wandb.visualize("wandb/metrics/v1", table)
+
+ return chart
+
+
+def make_table(metrics, model_name):
+ columns = ["metric_name", "metric_value", "model_name"]
+ table_content = [[name, value, model_name] for name, value in metrics.items()]
+
+ table = wandb.Table(columns=columns, data=table_content)
+
+ return table
diff --git a/wandb/sklearn/plot/__init__.py b/wandb/sklearn/plot/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..6217fed62d392eb942951165350d9105fd271efd
--- /dev/null
+++ b/wandb/sklearn/plot/__init__.py
@@ -0,0 +1,34 @@
+"""Create and logs charts introspecting models built with scikit-learn to W&B."""
+from .classifier import calibration_curve as plot_calibration_curve
+from .classifier import class_proportions as plot_class_proportions
+from .classifier import classifier as plot_classifier
+from .classifier import confusion_matrix as plot_confusion_matrix
+from .classifier import feature_importances as plot_feature_importances
+from .classifier import precision_recall as plot_precision_recall
+from .classifier import roc as plot_roc
+from .clusterer import clusterer as plot_clusterer
+from .clusterer import elbow_curve as plot_elbow_curve
+from .clusterer import silhouette as plot_silhouette
+from .regressor import outlier_candidates as plot_outlier_candidates
+from .regressor import regressor as plot_regressor
+from .regressor import residuals as plot_residuals
+from .shared import learning_curve as plot_learning_curve
+from .shared import summary_metrics as plot_summary_metrics
+
+__all__ = [
+ "plot_classifier",
+ "plot_clusterer",
+ "plot_regressor",
+ "plot_summary_metrics",
+ "plot_learning_curve",
+ "plot_feature_importances",
+ "plot_class_proportions",
+ "plot_calibration_curve",
+ "plot_roc",
+ "plot_precision_recall",
+ "plot_confusion_matrix",
+ "plot_elbow_curve",
+ "plot_silhouette",
+ "plot_residuals",
+ "plot_outlier_candidates",
+]
diff --git a/wandb/sklearn/plot/classifier.py b/wandb/sklearn/plot/classifier.py
new file mode 100644
index 0000000000000000000000000000000000000000..bb142046ab0d2a7490abd8ca531e8ae247dfac63
--- /dev/null
+++ b/wandb/sklearn/plot/classifier.py
@@ -0,0 +1,330 @@
+"""Define plots for classification models built with scikit-learn."""
+from warnings import simplefilter
+
+import numpy as np
+from sklearn import naive_bayes
+
+import wandb
+import wandb.plots
+from wandb.sklearn import calculate, utils
+
+from . import shared
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def classifier(
+ model,
+ X_train,
+ X_test,
+ y_train,
+ y_test,
+ y_pred,
+ y_probas,
+ labels,
+ is_binary=False,
+ model_name="Classifier",
+ feature_names=None,
+ log_learning_curve=False,
+):
+ """Generate all sklearn classifier plots supported by W&B.
+
+ The following plots are generated:
+ feature importances, confusion matrix, summary metrics,
+ class propotions, calibration curve, roc curve, precision-recall curve.
+
+ Should only be called with a fitted classifer (otherwise an error is thrown).
+
+ Arguments:
+ model: (classifier) Takes in a fitted classifier.
+ X_train: (arr) Training set features.
+ y_train: (arr) Training set labels.
+ X_test: (arr) Test set features.
+ y_test: (arr) Test set labels.
+ y_pred: (arr) Test set predictions by the model passed.
+ y_probas: (arr) Test set predicted probabilities by the model passed.
+ labels: (list) Named labels for target varible (y). Makes plots easier to
+ read by replacing target values with corresponding index.
+ For example if `labels=['dog', 'cat', 'owl']` all 0s are
+ replaced by dog, 1s by cat.
+ is_binary: (bool) Is the model passed a binary classifier? Defaults to False
+ model_name: (str) Model name. Defaults to 'Classifier'
+ feature_names: (list) Names for features. Makes plots easier to read by
+ replacing feature indexes with corresponding names.
+ log_learning_curve: (bool) Whether or not to log the learning curve.
+ Defaults to False.
+
+ Returns:
+ None: To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```python
+ wandb.sklearn.plot_classifier(
+ model,
+ X_train,
+ X_test,
+ y_train,
+ y_test,
+ y_pred,
+ y_probas,
+ ["cat", "dog"],
+ False,
+ "RandomForest",
+ ["barks", "drools", "plays_fetch", "breed"],
+ )
+ ```
+ """
+ wandb.termlog("\nPlotting %s." % model_name)
+
+ if not isinstance(model, naive_bayes.MultinomialNB):
+ feature_importances(model, feature_names)
+ wandb.termlog("Logged feature importances.")
+
+ if log_learning_curve:
+ shared.learning_curve(model, X_train, y_train)
+ wandb.termlog("Logged learning curve.")
+
+ confusion_matrix(y_test, y_pred, labels)
+ wandb.termlog("Logged confusion matrix.")
+
+ shared.summary_metrics(model, X=X_train, y=y_train, X_test=X_test, y_test=y_test)
+ wandb.termlog("Logged summary metrics.")
+
+ class_proportions(y_train, y_test, labels)
+ wandb.termlog("Logged class proportions.")
+
+ if not isinstance(model, naive_bayes.MultinomialNB):
+ calibration_curve(model, X_train, y_train, model_name)
+ wandb.termlog("Logged calibration curve.")
+
+ roc(y_test, y_probas, labels)
+ wandb.termlog("Logged roc curve.")
+
+ precision_recall(y_test, y_probas, labels)
+ wandb.termlog("Logged precision-recall curve.")
+
+
+def roc(
+ y_true=None,
+ y_probas=None,
+ labels=None,
+ plot_micro=True,
+ plot_macro=True,
+ classes_to_plot=None,
+):
+ """Log the receiver-operating characteristic curve.
+
+ Arguments:
+ y_true: (arr) Test set labels.
+ y_probas: (arr) Test set predicted probabilities.
+ labels: (list) Named labels for target variable (y). Makes plots easier to
+ read by replacing target values with corresponding index.
+ For example if `labels=['dog', 'cat', 'owl']` all 0s are
+ replaced by dog, 1s by cat.
+
+ Returns:
+ None: To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```python
+ wandb.sklearn.plot_roc(y_true, y_probas, labels)
+ ```
+ """
+ roc_chart = wandb.plots.roc.roc(
+ y_true, y_probas, labels, plot_micro, plot_macro, classes_to_plot
+ )
+ wandb.log({"roc": roc_chart})
+
+
+def confusion_matrix(
+ y_true=None,
+ y_pred=None,
+ labels=None,
+ true_labels=None,
+ pred_labels=None,
+ normalize=False,
+):
+ """Log a confusion matrix to W&B.
+
+ Confusion matrices depict the pattern of misclassifications by a model.
+
+ Arguments:
+ y_true: (arr) Test set labels.
+ y_probas: (arr) Test set predicted probabilities.
+ labels: (list) Named labels for target variable (y). Makes plots easier to
+ read by replacing target values with corresponding index.
+ For example if `labels=['dog', 'cat', 'owl']` all 0s are
+ replaced by dog, 1s by cat.
+
+ Returns:
+ None: To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```python
+ wandb.sklearn.plot_confusion_matrix(y_true, y_probas, labels)
+ ```
+ """
+ y_true = np.asarray(y_true)
+ y_pred = np.asarray(y_pred)
+
+ not_missing = utils.test_missing(y_true=y_true, y_pred=y_pred)
+ correct_types = utils.test_types(y_true=y_true, y_pred=y_pred)
+
+ if not_missing and correct_types:
+ confusion_matrix_chart = calculate.confusion_matrix(
+ y_true,
+ y_pred,
+ labels,
+ true_labels,
+ pred_labels,
+ normalize,
+ )
+
+ wandb.log({"confusion_matrix": confusion_matrix_chart})
+
+
+def precision_recall(
+ y_true=None, y_probas=None, labels=None, plot_micro=True, classes_to_plot=None
+):
+ """Log a precision-recall curve to W&B.
+
+ Precision-recall curves depict the tradeoff between positive predictive value (precision)
+ and true positive rate (recall) as the threshold of a classifier is shifted.
+
+ Arguments:
+ y_true: (arr) Test set labels.
+ y_probas: (arr) Test set predicted probabilities.
+ labels: (list) Named labels for target variable (y). Makes plots easier to
+ read by replacing target values with corresponding index.
+ For example if `labels=['dog', 'cat', 'owl']` all 0s are
+ replaced by dog, 1s by cat.
+
+ Returns:
+ None: To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```python
+ wandb.sklearn.plot_precision_recall(y_true, y_probas, labels)
+ ```
+ """
+ precision_recall_chart = wandb.plots.precision_recall(
+ y_true, y_probas, labels, plot_micro, classes_to_plot
+ )
+
+ wandb.log({"precision_recall": precision_recall_chart})
+
+
+def feature_importances(
+ model=None, feature_names=None, title="Feature Importance", max_num_features=50
+):
+ """Log a plot depicting the relative importance of each feature for a classifier's decisions.
+
+ Should only be called with a fitted classifer (otherwise an error is thrown).
+ Only works with classifiers that have a feature_importances_ attribute, like trees.
+
+ Arguments:
+ model: (clf) Takes in a fitted classifier.
+ feature_names: (list) Names for features. Makes plots easier to read by
+ replacing feature indexes with corresponding names.
+
+ Returns:
+ None: To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```python
+ wandb.sklearn.plot_feature_importances(model, ["width", "height", "length"])
+ ```
+ """
+ not_missing = utils.test_missing(model=model)
+ correct_types = utils.test_types(model=model)
+ model_fitted = utils.test_fitted(model)
+
+ if not_missing and correct_types and model_fitted:
+ feature_importance_chart = calculate.feature_importances(model, feature_names)
+ wandb.log({"feature_importances": feature_importance_chart})
+
+
+def class_proportions(y_train=None, y_test=None, labels=None):
+ """Plot the distribution of target classses in training and test sets.
+
+ Useful for detecting imbalanced classes.
+
+ Arguments:
+ y_train: (arr) Training set labels.
+ y_test: (arr) Test set labels.
+ labels: (list) Named labels for target variable (y). Makes plots easier to
+ read by replacing target values with corresponding index.
+ For example if `labels=['dog', 'cat', 'owl']` all 0s are
+ replaced by dog, 1s by cat.
+
+ Returns:
+ None: To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```python
+ wandb.sklearn.plot_class_proportions(y_train, y_test, ["dog", "cat", "owl"])
+ ```
+ """
+ not_missing = utils.test_missing(y_train=y_train, y_test=y_test)
+ correct_types = utils.test_types(y_train=y_train, y_test=y_test)
+ if not_missing and correct_types:
+ y_train, y_test = np.array(y_train), np.array(y_test)
+ class_proportions_chart = calculate.class_proportions(y_train, y_test, labels)
+
+ wandb.log({"class_proportions": class_proportions_chart})
+
+
+def calibration_curve(clf=None, X=None, y=None, clf_name="Classifier"):
+ """Log a plot depicting how well-calibrated the predicted probabilities of a classifier are.
+
+ Also suggests how to calibrate an uncalibrated classifier. Compares estimated predicted
+ probabilities by a baseline logistic regression model, the model passed as
+ an argument, and by both its isotonic calibration and sigmoid calibrations.
+ The closer the calibration curves are to a diagonal the better.
+ A sine wave like curve represents an overfitted classifier, while a cosine
+ wave like curve represents an underfitted classifier.
+ By training isotonic and sigmoid calibrations of the model and comparing
+ their curves we can figure out whether the model is over or underfitting and
+ if so which calibration (sigmoid or isotonic) might help fix this.
+ For more details, see https://scikit-learn.org/stable/auto_examples/calibration/plot_calibration_curve.html.
+
+ Should only be called with a fitted classifer (otherwise an error is thrown).
+
+ Please note this function fits variations of the model on the training set when called.
+
+ Arguments:
+ clf: (clf) Takes in a fitted classifier.
+ X: (arr) Training set features.
+ y: (arr) Training set labels.
+ model_name: (str) Model name. Defaults to 'Classifier'
+
+ Returns:
+ None: To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```python
+ wandb.sklearn.plot_calibration_curve(clf, X, y, "RandomForestClassifier")
+ ```
+ """
+ not_missing = utils.test_missing(clf=clf, X=X, y=y)
+ correct_types = utils.test_types(clf=clf, X=X, y=y)
+ is_fitted = utils.test_fitted(clf)
+ if not_missing and correct_types and is_fitted:
+ y = np.asarray(y)
+ if y.dtype.char == "U" or not ((y == 0) | (y == 1)).all():
+ wandb.termwarn(
+ "This function only supports binary classification at the moment and therefore expects labels to be binary. Skipping calibration curve."
+ )
+ return
+
+ calibration_curve_chart = calculate.calibration_curves(clf, X, y, clf_name)
+
+ wandb.log({"calibration_curve": calibration_curve_chart})
diff --git a/wandb/sklearn/plot/clusterer.py b/wandb/sklearn/plot/clusterer.py
new file mode 100644
index 0000000000000000000000000000000000000000..4aaf5f43e0499bb8bb5c45e2d4fb0bc88e009c25
--- /dev/null
+++ b/wandb/sklearn/plot/clusterer.py
@@ -0,0 +1,141 @@
+"""Define plots for clustering models built with scikit-learn."""
+from warnings import simplefilter
+
+import pandas as pd
+import sklearn
+
+import wandb
+from wandb.sklearn import calculate, utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def clusterer(model, X_train, cluster_labels, labels=None, model_name="Clusterer"):
+ """Generates all sklearn clusterer plots supported by W&B.
+
+ The following plots are generated:
+ elbow curve, silhouette plot.
+
+ Should only be called with a fitted clusterer (otherwise an error is thrown).
+
+ Arguments:
+ model: (clusterer) Takes in a fitted clusterer.
+ X_train: (arr) Training set features.
+ cluster_labels: (list) Names for cluster labels. Makes plots easier to read
+ by replacing cluster indexes with corresponding names.
+ labels: (list) Named labels for target varible (y). Makes plots easier to
+ read by replacing target values with corresponding index.
+ For example if `labels=['dog', 'cat', 'owl']` all 0s are
+ replaced by dog, 1s by cat.
+ model_name: (str) Model name. Defaults to 'Clusterer'
+
+ Returns:
+ None: To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```python
+ wandb.sklearn.plot_clusterer(kmeans, X, cluster_labels, labels, "KMeans")
+ ```
+ """
+ wandb.termlog("\nPlotting %s." % model_name)
+ if isinstance(model, sklearn.cluster.KMeans):
+ elbow_curve(model, X_train)
+ wandb.termlog("Logged elbow curve.")
+
+ silhouette(model, X_train, cluster_labels, labels=labels, kmeans=True)
+
+ else:
+ silhouette(model, X_train, cluster_labels, kmeans=False)
+
+ wandb.termlog("Logged silhouette plot.")
+
+
+def elbow_curve(
+ clusterer=None, X=None, cluster_ranges=None, n_jobs=1, show_cluster_time=True
+):
+ """Measures and plots variance explained as a function of the number of clusters.
+
+ Useful in picking the optimal number of clusters.
+
+ Should only be called with a fitted clusterer (otherwise an error is thrown).
+
+ Please note this function fits the model on the training set when called.
+
+ Arguments:
+ model: (clusterer) Takes in a fitted clusterer.
+ X: (arr) Training set features.
+
+ Returns:
+ None: To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```python
+ wandb.sklearn.plot_elbow_curve(model, X_train)
+ ```
+ """
+ if not hasattr(clusterer, "n_clusters"):
+ wandb.termlog(
+ "n_clusters attribute not in classifier. Cannot plot elbow method."
+ )
+ return
+
+ not_missing = utils.test_missing(clusterer=clusterer)
+ correct_types = utils.test_types
+ is_fitted = utils.test_fitted(clusterer)
+
+ if not_missing and correct_types and is_fitted:
+ elbow_curve_chart = calculate.elbow_curve(
+ clusterer, X, cluster_ranges, n_jobs, show_cluster_time
+ )
+
+ wandb.log({"elbow_curve": elbow_curve_chart})
+
+
+def silhouette(
+ clusterer=None,
+ X=None,
+ cluster_labels=None,
+ labels=None,
+ metric="euclidean",
+ kmeans=True,
+):
+ """Measures & plots silhouette coefficients.
+
+ Silhouette coefficients near +1 indicate that the sample is far away from
+ the neighboring clusters. A value near 0 indicates that the sample is on or
+ very close to the decision boundary between two neighboring clusters and
+ negative values indicate that the samples might have been assigned to the wrong cluster.
+
+ Should only be called with a fitted clusterer (otherwise an error is thrown).
+
+ Please note this function fits the model on the training set when called.
+
+ Arguments:
+ model: (clusterer) Takes in a fitted clusterer.
+ X: (arr) Training set features.
+ cluster_labels: (list) Names for cluster labels. Makes plots easier to read
+ by replacing cluster indexes with corresponding names.
+
+ Returns:
+ None: To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```python
+ wandb.sklearn.plot_silhouette(model, X_train, ["spam", "not spam"])
+ ```
+ """
+ not_missing = utils.test_missing(clusterer=clusterer)
+ correct_types = utils.test_types(clusterer=clusterer)
+ is_fitted = utils.test_fitted(clusterer)
+
+ if not_missing and correct_types and is_fitted:
+ if isinstance(X, (pd.DataFrame)):
+ X = X.values
+ silhouette_chart = calculate.silhouette(
+ clusterer, X, cluster_labels, labels, metric, kmeans
+ )
+ wandb.log({"silhouette_plot": silhouette_chart})
diff --git a/wandb/sklearn/plot/regressor.py b/wandb/sklearn/plot/regressor.py
new file mode 100644
index 0000000000000000000000000000000000000000..94158d73a92abdec11ea1a077f43c20a4f7ac87e
--- /dev/null
+++ b/wandb/sklearn/plot/regressor.py
@@ -0,0 +1,120 @@
+"""Define plots for regression models built with scikit-learn."""
+from warnings import simplefilter
+
+import numpy as np
+
+import wandb
+from wandb.sklearn import calculate, utils
+
+from . import shared
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def regressor(model, X_train, X_test, y_train, y_test, model_name="Regressor"):
+ """Generates all sklearn regressor plots supported by W&B.
+
+ The following plots are generated:
+ learning curve, summary metrics, residuals plot, outlier candidates.
+
+ Should only be called with a fitted regressor (otherwise an error is thrown).
+
+ Arguments:
+ model: (regressor) Takes in a fitted regressor.
+ X_train: (arr) Training set features.
+ y_train: (arr) Training set labels.
+ X_test: (arr) Test set features.
+ y_test: (arr) Test set labels.
+ model_name: (str) Model name. Defaults to 'Regressor'
+
+ Returns:
+ None: To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```python
+ wandb.sklearn.plot_regressor(reg, X_train, X_test, y_train, y_test, "Ridge")
+ ```
+ """
+ wandb.termlog("\nPlotting %s." % model_name)
+
+ shared.summary_metrics(model, X_train, y_train, X_test, y_test)
+ wandb.termlog("Logged summary metrics.")
+
+ shared.learning_curve(model, X_train, y_train)
+ wandb.termlog("Logged learning curve.")
+
+ outlier_candidates(model, X_train, y_train)
+ wandb.termlog("Logged outlier candidates.")
+
+ residuals(model, X_train, y_train)
+ wandb.termlog("Logged residuals.")
+
+
+def outlier_candidates(regressor=None, X=None, y=None):
+ """Measures a datapoint's influence on regression model via cook's distance.
+
+ Instances with high influences could potentially be outliers.
+
+ Should only be called with a fitted regressor (otherwise an error is thrown).
+
+ Please note this function fits the model on the training set when called.
+
+ Arguments:
+ model: (regressor) Takes in a fitted regressor.
+ X: (arr) Training set features.
+ y: (arr) Training set labels.
+
+ Returns:
+ None: To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```python
+ wandb.sklearn.plot_outlier_candidates(model, X, y)
+ ```
+ """
+ is_missing = utils.test_missing(regressor=regressor, X=X, y=y)
+ correct_types = utils.test_types(regressor=regressor, X=X, y=y)
+ is_fitted = utils.test_fitted(regressor)
+
+ if is_missing and correct_types and is_fitted:
+ y = np.asarray(y)
+
+ outliers_chart = calculate.outlier_candidates(regressor, X, y)
+ wandb.log({"outlier_candidates": outliers_chart})
+
+
+def residuals(regressor=None, X=None, y=None):
+ """Measures and plots the regressor's predicted value against the residual.
+
+ The marginal distribution of residuals is also calculated and plotted.
+
+ Should only be called with a fitted regressor (otherwise an error is thrown).
+
+ Please note this function fits variations of the model on the training set when called.
+
+ Arguments:
+ regressor: (regressor) Takes in a fitted regressor.
+ X: (arr) Training set features.
+ y: (arr) Training set labels.
+
+ Returns:
+ None: To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```python
+ wandb.sklearn.plot_residuals(model, X, y)
+ ```
+ """
+ not_missing = utils.test_missing(regressor=regressor, X=X, y=y)
+ correct_types = utils.test_types(regressor=regressor, X=X, y=y)
+ is_fitted = utils.test_fitted(regressor)
+
+ if not_missing and correct_types and is_fitted:
+ y = np.asarray(y)
+
+ residuals_chart = calculate.residuals(regressor, X, y)
+ wandb.log({"residuals": residuals_chart})
diff --git a/wandb/sklearn/plot/shared.py b/wandb/sklearn/plot/shared.py
new file mode 100644
index 0000000000000000000000000000000000000000..09ccb36da88f5924b6ec51bff48d7ecc16764576
--- /dev/null
+++ b/wandb/sklearn/plot/shared.py
@@ -0,0 +1,90 @@
+"""Define plots used by multiple sklearn model classes."""
+from warnings import simplefilter
+
+import numpy as np
+
+import wandb
+from wandb.sklearn import calculate, utils
+
+# ignore all future warnings
+simplefilter(action="ignore", category=FutureWarning)
+
+
+def summary_metrics(model=None, X=None, y=None, X_test=None, y_test=None):
+ """Logs a chart depicting summary metrics for a model.
+
+ Should only be called with a fitted model (otherwise an error is thrown).
+
+ Arguments:
+ model: (clf or reg) Takes in a fitted regressor or classifier.
+ X: (arr) Training set features.
+ y: (arr) Training set labels.
+ X_test: (arr) Test set features.
+ y_test: (arr) Test set labels.
+
+ Returns:
+ None: To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```python
+ wandb.sklearn.plot_summary_metrics(model, X_train, y_train, X_test, y_test)
+ ```
+ """
+ not_missing = utils.test_missing(
+ model=model, X=X, y=y, X_test=X_test, y_test=y_test
+ )
+ correct_types = utils.test_types(
+ model=model, X=X, y=y, X_test=X_test, y_test=y_test
+ )
+ model_fitted = utils.test_fitted(model)
+
+ if not_missing and correct_types and model_fitted:
+ metrics_chart = calculate.summary_metrics(model, X, y, X_test, y_test)
+ wandb.log({"summary_metrics": metrics_chart})
+
+
+def learning_curve(
+ model=None,
+ X=None,
+ y=None,
+ cv=None,
+ shuffle=False,
+ random_state=None,
+ train_sizes=None,
+ n_jobs=1,
+ scoring=None,
+):
+ """Logs a plot depicting model performance against dataset size.
+
+ Please note this function fits the model to datasets of varying sizes when called.
+
+ Arguments:
+ model: (clf or reg) Takes in a fitted regressor or classifier.
+ X: (arr) Dataset features.
+ y: (arr) Dataset labels.
+
+ For details on the other keyword arguments, see the documentation for
+ `sklearn.model_selection.learning_curve`.
+
+ Returns:
+ None: To see plots, go to your W&B run page then expand the 'media' tab
+ under 'auto visualizations'.
+
+ Example:
+ ```python
+ wandb.sklearn.plot_learning_curve(model, X, y)
+ ```
+ """
+ not_missing = utils.test_missing(model=model, X=X, y=y)
+ correct_types = utils.test_types(model=model, X=X, y=y)
+ if not_missing and correct_types:
+ if train_sizes is None:
+ train_sizes = np.linspace(0.1, 1.0, 5)
+ y = np.asarray(y)
+
+ learning_curve_chart = calculate.learning_curve(
+ model, X, y, cv, shuffle, random_state, train_sizes, n_jobs, scoring
+ )
+
+ wandb.log({"learning_curve": learning_curve_chart})
diff --git a/wandb/sklearn/utils.py b/wandb/sklearn/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..31542de911109c88edda7333804a8dd379cf3fdc
--- /dev/null
+++ b/wandb/sklearn/utils.py
@@ -0,0 +1,182 @@
+"""Shared utilities for the modules in wandb.sklearn."""
+from collections.abc import Iterable, Sequence
+
+import numpy as np
+import pandas as pd
+import scipy
+import sklearn
+
+import wandb
+
+chart_limit = 1000
+
+
+def check_against_limit(count, chart, limit=None):
+ if limit is None:
+ limit = chart_limit
+ if count > limit:
+ warn_chart_limit(limit, chart)
+ return True
+ else:
+ return False
+
+
+def warn_chart_limit(limit, chart):
+ warning = f"using only the first {limit} datapoints to create chart {chart}"
+ wandb.termwarn(warning)
+
+
+def encode_labels(df):
+ le = sklearn.preprocessing.LabelEncoder()
+ # apply le on categorical feature columns
+ categorical_cols = df.select_dtypes(
+ exclude=["int", "float", "float64", "float32", "int32", "int64"]
+ ).columns
+ df[categorical_cols] = df[categorical_cols].apply(lambda col: le.fit_transform(col))
+
+
+def test_types(**kwargs):
+ test_passed = True
+ for k, v in kwargs.items():
+ # check for incorrect types
+ if (
+ (k == "X")
+ or (k == "X_test")
+ or (k == "y")
+ or (k == "y_test")
+ or (k == "y_true")
+ or (k == "y_probas")
+ ):
+ # FIXME: do this individually
+ if not isinstance(
+ v,
+ (
+ Sequence,
+ Iterable,
+ np.ndarray,
+ np.generic,
+ pd.DataFrame,
+ pd.Series,
+ list,
+ ),
+ ):
+ wandb.termerror("%s is not an array. Please try again." % (k))
+ test_passed = False
+ # check for classifier types
+ if k == "model":
+ if (not sklearn.base.is_classifier(v)) and (
+ not sklearn.base.is_regressor(v)
+ ):
+ wandb.termerror(
+ "%s is not a classifier or regressor. Please try again." % (k)
+ )
+ test_passed = False
+ elif k == "clf" or k == "binary_clf":
+ if not (sklearn.base.is_classifier(v)):
+ wandb.termerror("%s is not a classifier. Please try again." % (k))
+ test_passed = False
+ elif k == "regressor":
+ if not sklearn.base.is_regressor(v):
+ wandb.termerror("%s is not a regressor. Please try again." % (k))
+ test_passed = False
+ elif k == "clusterer":
+ if not (getattr(v, "_estimator_type", None) == "clusterer"):
+ wandb.termerror("%s is not a clusterer. Please try again." % (k))
+ test_passed = False
+ return test_passed
+
+
+def test_fitted(model):
+ try:
+ model.predict(np.zeros((7, 3)))
+ except sklearn.exceptions.NotFittedError:
+ wandb.termerror("Please fit the model before passing it in.")
+ return False
+ except AttributeError:
+ # Some clustering models (LDA, PCA, Agglomerative) don't implement ``predict``
+ try:
+ sklearn.utils.validation.check_is_fitted(
+ model,
+ [
+ "coef_",
+ "estimator_",
+ "labels_",
+ "n_clusters_",
+ "children_",
+ "components_",
+ "n_components_",
+ "n_iter_",
+ "n_batch_iter_",
+ "explained_variance_",
+ "singular_values_",
+ "mean_",
+ ],
+ all_or_any=any,
+ )
+ return True
+ except sklearn.exceptions.NotFittedError:
+ wandb.termerror("Please fit the model before passing it in.")
+ return False
+ except Exception:
+ # Assume it's fitted, since ``NotFittedError`` wasn't raised
+ return True
+
+
+# Test Asummptions for plotting parameters and datasets
+def test_missing(**kwargs):
+ test_passed = True
+ for k, v in kwargs.items():
+ # Missing/empty params/datapoint arrays
+ if v is None:
+ wandb.termerror("%s is None. Please try again." % (k))
+ test_passed = False
+ if (k == "X") or (k == "X_test"):
+ if isinstance(v, scipy.sparse.csr.csr_matrix):
+ v = v.toarray()
+ elif isinstance(v, (pd.DataFrame, pd.Series)):
+ v = v.to_numpy()
+ elif isinstance(v, list):
+ v = np.asarray(v)
+
+ # Warn the user about missing values
+ missing = 0
+ missing = np.count_nonzero(pd.isnull(v))
+ if missing > 0:
+ wandb.termwarn("%s contains %d missing values. " % (k, missing))
+ test_passed = False
+ # Ensure the dataset contains only integers
+ non_nums = 0
+ if v.ndim == 1:
+ non_nums = sum(
+ 1
+ for val in v
+ if (
+ not isinstance(val, (int, float, complex))
+ and not isinstance(val, np.number)
+ )
+ )
+ else:
+ non_nums = sum(
+ 1
+ for sl in v
+ for val in sl
+ if (
+ not isinstance(val, (int, float, complex))
+ and not isinstance(val, np.number)
+ )
+ )
+ if non_nums > 0:
+ wandb.termerror(
+ "%s contains values that are not numbers. Please vectorize, label encode or one hot encode %s and call the plotting function again."
+ % (k, k)
+ )
+ test_passed = False
+ return test_passed
+
+
+def round_3(n):
+ return round(n, 3)
+
+
+def round_2(n):
+ return round(n, 2)
diff --git a/wandb/sync/__init__.py b/wandb/sync/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..46f0496407a396880132aa943b33dcb37f15f7f3
--- /dev/null
+++ b/wandb/sync/__init__.py
@@ -0,0 +1,3 @@
+"""module sync."""
+
+from .sync import SyncManager, get_run_from_path, get_runs # noqa: F401
diff --git a/wandb/sync/sync.py b/wandb/sync/sync.py
new file mode 100644
index 0000000000000000000000000000000000000000..3e5c4ca61b71d7f79dfb44a7bf62c192b35e3718
--- /dev/null
+++ b/wandb/sync/sync.py
@@ -0,0 +1,443 @@
+"""sync."""
+
+import atexit
+import datetime
+import fnmatch
+import os
+import queue
+import sys
+import tempfile
+import threading
+import time
+from typing import List, Optional
+from urllib.parse import quote as url_quote
+
+import wandb
+from wandb.proto import wandb_internal_pb2 # type: ignore
+from wandb.sdk.interface.interface_queue import InterfaceQueue
+from wandb.sdk.internal import context, datastore, handler, sender, tb_watcher
+from wandb.sdk.internal.settings_static import SettingsStatic
+from wandb.sdk.lib import filesystem
+from wandb.util import check_and_warn_old
+
+WANDB_SUFFIX = ".wandb"
+SYNCED_SUFFIX = ".synced"
+TFEVENT_SUBSTRING = ".tfevents."
+
+
+class _LocalRun:
+ def __init__(self, path, synced=None):
+ self.path = path
+ self.synced = synced
+ self.offline = os.path.basename(path).startswith("offline-")
+ self.datetime = datetime.datetime.strptime(
+ os.path.basename(path).split("run-")[1].split("-")[0], "%Y%m%d_%H%M%S"
+ )
+
+ def __str__(self):
+ return self.path
+
+
+class SyncThread(threading.Thread):
+ def __init__(
+ self,
+ sync_list,
+ project=None,
+ entity=None,
+ run_id=None,
+ job_type=None,
+ view=None,
+ verbose=None,
+ mark_synced=None,
+ app_url=None,
+ sync_tensorboard=None,
+ log_path=None,
+ append=None,
+ skip_console=None,
+ ):
+ threading.Thread.__init__(self)
+ # mark this process as internal
+ wandb._set_internal_process(disable=True)
+ self._sync_list = sync_list
+ self._project = project
+ self._entity = entity
+ self._run_id = run_id
+ self._job_type = job_type
+ self._view = view
+ self._verbose = verbose
+ self._mark_synced = mark_synced
+ self._app_url = app_url
+ self._sync_tensorboard = sync_tensorboard
+ self._log_path = log_path
+ self._append = append
+ self._skip_console = skip_console
+
+ self._tmp_dir = tempfile.TemporaryDirectory()
+ atexit.register(self._tmp_dir.cleanup)
+
+ def _parse_pb(self, data, exit_pb=None):
+ pb = wandb_internal_pb2.Record()
+ pb.ParseFromString(data)
+ record_type = pb.WhichOneof("record_type")
+ if self._view:
+ if self._verbose:
+ print("Record:", pb)
+ else:
+ print("Record:", record_type)
+ return pb, exit_pb, True
+ if record_type == "run":
+ if self._run_id:
+ pb.run.run_id = self._run_id
+ if self._project:
+ pb.run.project = self._project
+ if self._entity:
+ pb.run.entity = self._entity
+ if self._job_type:
+ pb.run.job_type = self._job_type
+ pb.control.req_resp = True
+ elif record_type in ("output", "output_raw") and self._skip_console:
+ return pb, exit_pb, True
+ elif record_type == "exit":
+ exit_pb = pb
+ return pb, exit_pb, True
+ elif record_type == "final":
+ assert exit_pb, "final seen without exit"
+ pb = exit_pb
+ exit_pb = None
+ return pb, exit_pb, False
+
+ def _find_tfevent_files(self, sync_item):
+ tb_event_files = 0
+ tb_logdirs = []
+ tb_root = None
+ if self._sync_tensorboard:
+ if os.path.isdir(sync_item):
+ files = []
+ for dirpath, _, _files in os.walk(sync_item):
+ for f in _files:
+ if TFEVENT_SUBSTRING in f:
+ files.append(os.path.join(dirpath, f))
+ for tfevent in files:
+ tb_event_files += 1
+ tb_dir = os.path.dirname(os.path.abspath(tfevent))
+ if tb_dir not in tb_logdirs:
+ tb_logdirs.append(tb_dir)
+ if len(tb_logdirs) > 0:
+ tb_root = os.path.dirname(os.path.commonprefix(tb_logdirs))
+
+ elif TFEVENT_SUBSTRING in sync_item:
+ tb_root = os.path.dirname(os.path.abspath(sync_item))
+ tb_logdirs.append(tb_root)
+ tb_event_files = 1
+ return tb_event_files, tb_logdirs, tb_root
+
+ def _setup_tensorboard(self, tb_root, tb_logdirs, tb_event_files, sync_item):
+ """Return true if this sync item can be synced as tensorboard."""
+ if tb_root is not None:
+ if tb_event_files > 0 and sync_item.endswith(WANDB_SUFFIX):
+ wandb.termwarn("Found .wandb file, not streaming tensorboard metrics.")
+ else:
+ print(f"Found {tb_event_files} tfevent files in {tb_root}")
+ if len(tb_logdirs) > 3:
+ wandb.termwarn(
+ f"Found {len(tb_logdirs)} directories containing tfevent files. "
+ "If these represent multiple experiments, sync them "
+ "individually or pass a list of paths."
+ )
+ return True
+ return False
+
+ def _send_tensorboard(self, tb_root, tb_logdirs, send_manager):
+ if self._entity is None:
+ viewer, _ = send_manager._api.viewer_server_info()
+ self._entity = viewer.get("entity")
+ proto_run = wandb_internal_pb2.RunRecord()
+ proto_run.run_id = self._run_id or wandb.util.generate_id()
+ proto_run.project = self._project or wandb.util.auto_project_name(None)
+ proto_run.entity = self._entity
+ proto_run.telemetry.feature.sync_tfevents = True
+
+ url = "{}/{}/{}/runs/{}".format(
+ self._app_url,
+ url_quote(proto_run.entity),
+ url_quote(proto_run.project),
+ url_quote(proto_run.run_id),
+ )
+ print("Syncing: %s ..." % url)
+ sys.stdout.flush()
+ # using a handler here automatically handles the step
+ # logic, adds summaries to the run, and handles different
+ # file types (like images)... but we need to remake the send_manager
+ record_q = queue.Queue()
+ sender_record_q = queue.Queue()
+ new_interface = InterfaceQueue(record_q)
+ context_keeper = context.ContextKeeper()
+ send_manager = sender.SendManager(
+ settings=send_manager._settings,
+ record_q=sender_record_q,
+ result_q=queue.Queue(),
+ interface=new_interface,
+ context_keeper=context_keeper,
+ )
+ record = send_manager._interface._make_record(run=proto_run)
+ settings = wandb.Settings(
+ root_dir=self._tmp_dir.name,
+ run_id=proto_run.run_id,
+ _start_datetime=datetime.datetime.now(),
+ _start_time=time.time(),
+ )
+
+ settings_static = SettingsStatic(settings.to_proto())
+
+ handle_manager = handler.HandleManager(
+ settings=settings_static,
+ record_q=record_q,
+ result_q=None,
+ stopped=False,
+ writer_q=sender_record_q,
+ interface=new_interface,
+ context_keeper=context_keeper,
+ )
+
+ filesystem.mkdir_exists_ok(settings.files_dir)
+ send_manager.send_run(record, file_dir=settings.files_dir)
+ watcher = tb_watcher.TBWatcher(settings, proto_run, new_interface, True)
+
+ for tb in tb_logdirs:
+ watcher.add(tb, True, tb_root)
+ sys.stdout.flush()
+ watcher.finish()
+
+ # send all of our records like a boss
+ progress_step = 0
+ spinner_states = ["-", "\\", "|", "/"]
+ line = " Uploading data to wandb\r"
+ while len(handle_manager) > 0:
+ data = next(handle_manager)
+ handle_manager.handle(data)
+ while len(send_manager) > 0:
+ data = next(send_manager)
+ send_manager.send(data)
+
+ print_line = spinner_states[progress_step % 4] + line
+ wandb.termlog(print_line, newline=False, prefix=True)
+ progress_step += 1
+
+ # finish sending any data
+ while len(send_manager) > 0:
+ data = next(send_manager)
+ send_manager.send(data)
+ sys.stdout.flush()
+ handle_manager.finish()
+ send_manager.finish()
+
+ def _robust_scan(self, ds):
+ """Attempt to scan data, handling incomplete files."""
+ try:
+ return ds.scan_data()
+ except AssertionError as e:
+ if ds.in_last_block():
+ wandb.termwarn(
+ f".wandb file is incomplete ({e}), be sure to sync this run again once it's finished"
+ )
+ return None
+ else:
+ raise e
+
+ def run(self):
+ if self._log_path is not None:
+ print(f"Find logs at: {self._log_path}")
+ for sync_item in self._sync_list:
+ tb_event_files, tb_logdirs, tb_root = self._find_tfevent_files(sync_item)
+ if os.path.isdir(sync_item):
+ files = os.listdir(sync_item)
+ filtered_files = list(filter(lambda f: f.endswith(WANDB_SUFFIX), files))
+ if tb_root is None and (
+ check_and_warn_old(files) or len(filtered_files) != 1
+ ):
+ print(f"Skipping directory: {sync_item}")
+ continue
+ if len(filtered_files) > 0:
+ sync_item = os.path.join(sync_item, filtered_files[0])
+ sync_tb = self._setup_tensorboard(
+ tb_root, tb_logdirs, tb_event_files, sync_item
+ )
+ # If we're syncing tensorboard, let's use a tmp dir for images etc.
+ root_dir = self._tmp_dir.name if sync_tb else os.path.dirname(sync_item)
+
+ # When appending we are allowing a possible resume, ie the run
+ # doesnt have to exist already
+ resume = "allow" if self._append else None
+
+ sm = sender.SendManager.setup(root_dir, resume=resume)
+ if sync_tb:
+ self._send_tensorboard(tb_root, tb_logdirs, sm)
+ continue
+
+ ds = datastore.DataStore()
+ try:
+ ds.open_for_scan(sync_item)
+ except AssertionError as e:
+ print(f".wandb file is empty ({e}), skipping: {sync_item}")
+ continue
+
+ # save exit for final send
+ exit_pb = None
+ finished = False
+ shown = False
+ while True:
+ data = self._robust_scan(ds)
+ if data is None:
+ break
+ pb, exit_pb, cont = self._parse_pb(data, exit_pb)
+ if exit_pb is not None:
+ finished = True
+ if cont:
+ continue
+ sm.send(pb)
+ # send any records that were added in previous send
+ while not sm._record_q.empty():
+ data = sm._record_q.get(block=True)
+ sm.send(data)
+
+ if pb.control.req_resp:
+ result = sm._result_q.get(block=True)
+ result_type = result.WhichOneof("result_type")
+ if not shown and result_type == "run_result":
+ r = result.run_result.run
+ # TODO(jhr): hardcode until we have settings in sync
+ url = "{}/{}/{}/runs/{}".format(
+ self._app_url,
+ url_quote(r.entity),
+ url_quote(r.project),
+ url_quote(r.run_id),
+ )
+ print("Syncing: %s ... " % url, end="")
+ sys.stdout.flush()
+ shown = True
+ sm.finish()
+ # Only mark synced if the run actually finished
+ if self._mark_synced and not self._view and finished:
+ synced_file = f"{sync_item}{SYNCED_SUFFIX}"
+ with open(synced_file, "w"):
+ pass
+ print("done.")
+
+
+class SyncManager:
+ def __init__(
+ self,
+ project=None,
+ entity=None,
+ run_id=None,
+ job_type=None,
+ mark_synced=None,
+ app_url=None,
+ view=None,
+ verbose=None,
+ sync_tensorboard=None,
+ log_path=None,
+ append=None,
+ skip_console=None,
+ ):
+ self._sync_list = []
+ self._thread = None
+ self._project = project
+ self._entity = entity
+ self._run_id = run_id
+ self._job_type = job_type
+ self._mark_synced = mark_synced
+ self._app_url = app_url
+ self._view = view
+ self._verbose = verbose
+ self._sync_tensorboard = sync_tensorboard
+ self._log_path = log_path
+ self._append = append
+ self._skip_console = skip_console
+
+ def status(self):
+ pass
+
+ def add(self, p):
+ self._sync_list.append(os.path.abspath(str(p)))
+
+ def start(self):
+ # create a thread for each file?
+ self._thread = SyncThread(
+ sync_list=self._sync_list,
+ project=self._project,
+ entity=self._entity,
+ run_id=self._run_id,
+ job_type=self._job_type,
+ view=self._view,
+ verbose=self._verbose,
+ mark_synced=self._mark_synced,
+ app_url=self._app_url,
+ sync_tensorboard=self._sync_tensorboard,
+ log_path=self._log_path,
+ append=self._append,
+ skip_console=self._skip_console,
+ )
+ self._thread.start()
+
+ def is_done(self):
+ return not self._thread.is_alive()
+
+ def poll(self):
+ time.sleep(1)
+ return False
+
+
+def get_runs(
+ include_offline: bool = True,
+ include_online: bool = True,
+ include_synced: bool = False,
+ include_unsynced: bool = True,
+ exclude_globs: Optional[List[str]] = None,
+ include_globs: Optional[List[str]] = None,
+):
+ # TODO(jhr): grab dir info from settings
+ base = ".wandb" if os.path.exists(".wandb") else "wandb"
+ if not os.path.exists(base):
+ return ()
+ all_dirs = os.listdir(base)
+ dirs = []
+ if include_offline:
+ dirs += filter(lambda _d: _d.startswith("offline-run-"), all_dirs)
+ if include_online:
+ dirs += filter(lambda _d: _d.startswith("run-"), all_dirs)
+ # find run file in each dir
+ fnames = []
+ dirs.sort()
+ for d in dirs:
+ paths = os.listdir(os.path.join(base, d))
+ if exclude_globs:
+ paths = set(paths)
+ for g in exclude_globs:
+ paths = paths - set(fnmatch.filter(paths, g))
+ paths = list(paths)
+ if include_globs:
+ new_paths = set()
+ for g in include_globs:
+ new_paths = new_paths.union(fnmatch.filter(paths, g))
+ paths = list(new_paths)
+ for f in paths:
+ if f.endswith(WANDB_SUFFIX):
+ fnames.append(os.path.join(base, d, f))
+ filtered = []
+ for f in fnames:
+ dname = os.path.dirname(f)
+ # TODO(frz): online runs are assumed to be synced, verify from binary log.
+ if os.path.exists(f"{f}{SYNCED_SUFFIX}") or os.path.basename(dname).startswith(
+ "run-"
+ ):
+ if include_synced:
+ filtered.append(_LocalRun(dname, True))
+ else:
+ if include_unsynced:
+ filtered.append(_LocalRun(dname, False))
+ return tuple(filtered)
+
+
+def get_run_from_path(path):
+ return _LocalRun(path)
diff --git a/wandb/testing/relay.py b/wandb/testing/relay.py
new file mode 100644
index 0000000000000000000000000000000000000000..66bfcf4502b520d647099c508b5f354a9e712111
--- /dev/null
+++ b/wandb/testing/relay.py
@@ -0,0 +1,860 @@
+import dataclasses
+import json
+import logging
+import socket
+import sys
+import threading
+import traceback
+import urllib.parse
+from collections import defaultdict, deque
+from copy import deepcopy
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Callable,
+ Dict,
+ Iterable,
+ List,
+ Mapping,
+ Optional,
+ Union,
+)
+
+import flask
+import pandas as pd
+import requests
+import responses
+
+import wandb
+import wandb.util
+from wandb.sdk.lib.timer import Timer
+
+try:
+ from typing import Literal, TypedDict
+except ImportError:
+ from typing_extensions import Literal, TypedDict
+
+if sys.version_info >= (3, 8):
+ from typing import Protocol
+else:
+ from typing_extensions import Protocol
+
+if TYPE_CHECKING:
+ from typing import Deque
+
+ class RawRequestResponse(TypedDict):
+ url: str
+ request: Optional[Any]
+ response: Dict[str, Any]
+ time_elapsed: float # seconds
+
+ ResolverName = Literal[
+ "upsert_bucket",
+ "upload_files",
+ "uploaded_files",
+ "preempting",
+ "upsert_sweep",
+ ]
+
+ class Resolver(TypedDict):
+ name: ResolverName
+ resolver: Callable[[Any], Optional[Dict[str, Any]]]
+
+
+class DeliberateHTTPError(Exception):
+ def __init__(self, message, status_code: int = 500):
+ Exception.__init__(self)
+ self.message = message
+ self.status_code = status_code
+
+ def get_response(self):
+ return flask.Response(self.message, status=self.status_code)
+
+ def __repr__(self):
+ return f"DeliberateHTTPError({self.message!r}, {self.status_code!r})"
+
+
+@dataclasses.dataclass
+class RunAttrs:
+ """Simple data class for run attributes."""
+
+ name: str
+ display_name: str
+ description: str
+ sweep_name: str
+ project: Dict[str, Any]
+ config: Dict[str, Any]
+ remote: Optional[str] = None
+ commit: Optional[str] = None
+
+
+class Context:
+ """A container used to store the snooped state/data of a test.
+
+ Includes raw requests and responses, parsed and processed data, and a number of
+ convenience methods and properties for accessing the data.
+ """
+
+ def __init__(self) -> None:
+ # parsed/merged data. keys are the individual wandb run id's.
+ self._entries = defaultdict(dict)
+ # container for raw requests and responses:
+ self.raw_data: List[RawRequestResponse] = []
+ # concatenated file contents for all runs:
+ self._history: Optional[pd.DataFrame] = None
+ self._events: Optional[pd.DataFrame] = None
+ self._summary: Optional[pd.DataFrame] = None
+ self._config: Optional[Dict[str, Any]] = None
+ self._output: Optional[Any] = None
+
+ def upsert(self, entry: Dict[str, Any]) -> None:
+ try:
+ entry_id: str = entry["name"]
+ except KeyError:
+ entry_id = entry["id"]
+ self._entries[entry_id] = wandb.util.merge_dicts(entry, self._entries[entry_id])
+
+ # mapping interface
+ def __getitem__(self, key: str) -> Any:
+ return self._entries[key]
+
+ def keys(self) -> Iterable[str]:
+ return self._entries.keys()
+
+ def get_file_contents(self, file_name: str) -> pd.DataFrame:
+ dfs = []
+
+ for entry_id in self._entries:
+ # - extract the content from `file_name`
+ # - sort by offset (will be useful when relay server goes async)
+ # - extract data, merge into a list of dicts and convert to a pandas dataframe
+ content_list = self._entries[entry_id].get("files", {}).get(file_name, [])
+ content_list.sort(key=lambda x: x["offset"])
+ content_list = [item["content"] for item in content_list]
+ # merge list of lists content_list:
+ content_list = [item for sublist in content_list for item in sublist]
+ df = pd.DataFrame.from_records(content_list)
+ df["__run_id"] = entry_id
+ dfs.append(df)
+
+ return pd.concat(dfs)
+
+ # attributes to use in assertions
+ @property
+ def entries(self) -> Dict[str, Any]:
+ return deepcopy(self._entries)
+
+ @property
+ def history(self) -> pd.DataFrame:
+ # todo: caveat: this assumes that all assertions happen at the end of a test
+ if self._history is not None:
+ return deepcopy(self._history)
+
+ self._history = self.get_file_contents("wandb-history.jsonl")
+ return deepcopy(self._history)
+
+ @property
+ def events(self) -> pd.DataFrame:
+ if self._events is not None:
+ return deepcopy(self._events)
+
+ self._events = self.get_file_contents("wandb-events.jsonl")
+ return deepcopy(self._events)
+
+ @property
+ def summary(self) -> pd.DataFrame:
+ if self._summary is not None:
+ return deepcopy(self._summary)
+
+ _summary = self.get_file_contents("wandb-summary.json")
+
+ # run summary may be updated multiple times,
+ # but we are only interested in the last one.
+ # we can have multiple runs saved to context,
+ # so we need to group by run id and take the
+ # last one for each run.
+ self._summary = (
+ _summary.groupby("__run_id").last().reset_index(level=["__run_id"])
+ )
+
+ return deepcopy(self._summary)
+
+ @property
+ def output(self) -> pd.DataFrame:
+ if self._output is not None:
+ return deepcopy(self._output)
+
+ self._output = self.get_file_contents("output.log")
+ return deepcopy(self._output)
+
+ @property
+ def config(self) -> Dict[str, Any]:
+ if self._config is not None:
+ return deepcopy(self._config)
+
+ self._config = {k: v["config"] for (k, v) in self._entries.items()}
+ return deepcopy(self._config)
+
+ # @property
+ # def telemetry(self) -> pd.DataFrame:
+ # telemetry = pd.DataFrame.from_records(
+ # [
+ # {
+ # "__run_id": run_id,
+ # "telemetry": config.get("_wandb", {}).get("value", {}).get("t")
+ # }
+ # for (run_id, config) in self.config.items()
+ # ]
+ # )
+ # return telemetry
+
+ # convenience data access methods
+ def get_run_telemetry(self, run_id: str) -> Dict[str, Any]:
+ return self.config.get(run_id, {}).get("_wandb", {}).get("value", {}).get("t")
+
+ def get_run_metrics(self, run_id: str) -> Dict[str, Any]:
+ return self.config.get(run_id, {}).get("_wandb", {}).get("value", {}).get("m")
+
+ def get_run_summary(
+ self, run_id: str, include_private: bool = False
+ ) -> Dict[str, Any]:
+ # run summary dataframe must have only one row
+ # for the given run id, so we convert it to dict
+ # and extract the first (and only) row.
+ mask_run = self.summary["__run_id"] == run_id
+ run_summary = self.summary[mask_run]
+ ret = (
+ run_summary.filter(regex="^[^_]", axis=1)
+ if not include_private
+ else run_summary
+ ).to_dict(orient="records")
+ return ret[0] if len(ret) > 0 else {}
+
+ def get_run_history(
+ self, run_id: str, include_private: bool = False
+ ) -> pd.DataFrame:
+ mask_run = self.history["__run_id"] == run_id
+ run_history = self.history[mask_run]
+ return (
+ run_history.filter(regex="^[^_]", axis=1)
+ if not include_private
+ else run_history
+ )
+
+ def get_run_uploaded_files(self, run_id: str) -> Dict[str, Any]:
+ return self.entries.get(run_id, {}).get("uploaded", [])
+
+ def get_run_stats(self, run_id: str) -> pd.DataFrame:
+ mask_run = self.events["__run_id"] == run_id
+ run_stats = self.events[mask_run]
+ return run_stats
+
+ def get_run_attrs(self, run_id: str) -> Optional[RunAttrs]:
+ run_entry = self._entries.get(run_id)
+ if not run_entry:
+ return None
+
+ return RunAttrs(
+ name=run_entry["name"],
+ display_name=run_entry["displayName"],
+ description=run_entry["description"],
+ sweep_name=run_entry["sweepName"],
+ project=run_entry["project"],
+ config=run_entry["config"],
+ remote=run_entry.get("repo"),
+ commit=run_entry.get("commit"),
+ )
+
+ def get_run(self, run_id: str) -> Dict[str, Any]:
+ return self._entries.get(run_id, {})
+
+ # todo: add getter (by run_id) utilities for other properties
+
+
+class QueryResolver:
+ """Resolve request/response pairs against a set of known patterns.
+
+ This extracts and processes useful data to be later stored in a Context object.
+ """
+
+ def __init__(self):
+ self.resolvers: List[Resolver] = [
+ {
+ "name": "upsert_bucket",
+ "resolver": self.resolve_upsert_bucket,
+ },
+ {
+ "name": "upload_files",
+ "resolver": self.resolve_upload_files,
+ },
+ {
+ "name": "uploaded_files",
+ "resolver": self.resolve_uploaded_files,
+ },
+ {
+ "name": "uploaded_files_legacy",
+ "resolver": self.resolve_uploaded_files_legacy,
+ },
+ {
+ "name": "preempting",
+ "resolver": self.resolve_preempting,
+ },
+ {
+ "name": "upsert_sweep",
+ "resolver": self.resolve_upsert_sweep,
+ },
+ {
+ "name": "create_artifact",
+ "resolver": self.resolve_create_artifact,
+ },
+ {
+ "name": "delete_run",
+ "resolver": self.resolve_delete_run,
+ },
+ ]
+
+ @staticmethod
+ def resolve_upsert_bucket(
+ request_data: Dict[str, Any], response_data: Dict[str, Any], **kwargs: Any
+ ) -> Optional[Dict[str, Any]]:
+ if not isinstance(request_data, dict) or not isinstance(response_data, dict):
+ return None
+ query = response_data.get("data", {}).get("upsertBucket") is not None
+ if query:
+ data = {
+ k: v for (k, v) in request_data["variables"].items() if v is not None
+ }
+ data.update(response_data["data"]["upsertBucket"].get("bucket"))
+ if "config" in data:
+ data["config"] = json.loads(data["config"])
+ return data
+ return None
+
+ @staticmethod
+ def resolve_delete_run(
+ request_data: Dict[str, Any], response_data: Dict[str, Any], **kwargs: Any
+ ) -> Optional[Dict[str, Any]]:
+ if not isinstance(request_data, dict) or not isinstance(response_data, dict):
+ return None
+ query = "query" in request_data and "deleteRun" in request_data["query"]
+ if query:
+ data = {
+ k: v for (k, v) in request_data["variables"].items() if v is not None
+ }
+ data.update(response_data["data"]["deleteRun"])
+ return data
+ return None
+
+ @staticmethod
+ def resolve_upload_files(
+ request_data: Dict[str, Any], response_data: Dict[str, Any], **kwargs: Any
+ ) -> Optional[Dict[str, Any]]:
+ if not isinstance(request_data, dict):
+ return None
+
+ query = request_data.get("files") is not None
+ if query:
+ # todo: refactor this 🤮🤮🤮🤮🤮 eventually?
+ name = kwargs.get("path").split("/")[2]
+ files = defaultdict(list)
+ for file_name, file_value in request_data["files"].items():
+ content = []
+ for k in file_value.get("content", []):
+ try:
+ content.append(json.loads(k))
+ except json.decoder.JSONDecodeError:
+ content.append([k])
+
+ files[file_name].append(
+ {"offset": file_value.get("offset"), "content": content}
+ )
+
+ post_processed_data = {
+ "name": name,
+ "dropped": [request_data["dropped"]]
+ if "dropped" in request_data
+ else [],
+ "files": files,
+ }
+ return post_processed_data
+ return None
+
+ @staticmethod
+ def resolve_uploaded_files(
+ request_data: Dict[str, Any], response_data: Dict[str, Any], **kwargs: Any
+ ) -> Optional[Dict[str, Any]]:
+ if not isinstance(request_data, dict) or not isinstance(response_data, dict):
+ return None
+
+ query = "CreateRunFiles" in request_data.get("query", "")
+ if query:
+ run_name = request_data["variables"]["run"]
+ files = ((response_data.get("data") or {}).get("createRunFiles") or {}).get(
+ "files", {}
+ )
+ post_processed_data = {
+ "name": run_name,
+ "uploaded": [file["name"] for file in files] if files else [""],
+ }
+ return post_processed_data
+ return None
+
+ @staticmethod
+ def resolve_uploaded_files_legacy(
+ request_data: Dict[str, Any], response_data: Dict[str, Any], **kwargs: Any
+ ) -> Optional[Dict[str, Any]]:
+ # This is a legacy resolver for uploaded files
+ # No longer used by tests but leaving it here in case we need it in the future
+ # Please refer to upload_urls() in internal_api.py for more details
+ if not isinstance(request_data, dict) or not isinstance(response_data, dict):
+ return None
+
+ query = "RunUploadUrls" in request_data.get("query", "")
+ if query:
+ # todo: refactor this 🤮🤮🤮🤮🤮 eventually?
+ name = request_data["variables"]["run"]
+ files = (
+ response_data.get("data", {})
+ .get("model", {})
+ .get("bucket", {})
+ .get("files", {})
+ .get("edges", [])
+ )
+ # note: we count all attempts to upload files
+ post_processed_data = {
+ "name": name,
+ "uploaded": [files[0].get("node", {}).get("name")] if files else [""],
+ }
+ return post_processed_data
+ return None
+
+ @staticmethod
+ def resolve_preempting(
+ request_data: Dict[str, Any], response_data: Dict[str, Any], **kwargs: Any
+ ) -> Optional[Dict[str, Any]]:
+ if not isinstance(request_data, dict):
+ return None
+ query = "preempting" in request_data
+ if query:
+ name = kwargs.get("path").split("/")[2]
+ post_processed_data = {
+ "name": name,
+ "preempting": [request_data["preempting"]],
+ }
+ return post_processed_data
+ return None
+
+ @staticmethod
+ def resolve_upsert_sweep(
+ request_data: Dict[str, Any], response_data: Dict[str, Any], **kwargs: Any
+ ) -> Optional[Dict[str, Any]]:
+ if not isinstance(response_data, dict):
+ return None
+ query = response_data.get("data", {}).get("upsertSweep") is not None
+ if query:
+ data = response_data["data"]["upsertSweep"].get("sweep")
+ return data
+ return None
+
+ def resolve_create_artifact(
+ self, request_data: Dict[str, Any], response_data: Dict[str, Any], **kwargs: Any
+ ) -> Optional[Dict[str, Any]]:
+ if not isinstance(request_data, dict):
+ return None
+ query = (
+ "createArtifact(" in request_data.get("query", "")
+ and request_data.get("variables") is not None
+ and response_data is not None
+ )
+ if query:
+ name = request_data["variables"]["runName"]
+ post_processed_data = {
+ "name": name,
+ "create_artifact": [
+ {
+ "variables": request_data["variables"],
+ "response": response_data["data"]["createArtifact"]["artifact"],
+ }
+ ],
+ }
+ return post_processed_data
+ return None
+
+ def resolve(
+ self,
+ request_data: Dict[str, Any],
+ response_data: Dict[str, Any],
+ **kwargs: Any,
+ ) -> Optional[Dict[str, Any]]:
+ for resolver in self.resolvers:
+ result = resolver.get("resolver")(request_data, response_data, **kwargs)
+ if result is not None:
+ return result
+ return None
+
+
+class TokenizedCircularPattern:
+ APPLY_TOKEN = "1"
+ PASS_TOKEN = "0"
+ STOP_TOKEN = "2"
+
+ def __init__(self, pattern: str):
+ known_tokens = {self.APPLY_TOKEN, self.PASS_TOKEN, self.STOP_TOKEN}
+ if not pattern:
+ raise ValueError("Pattern cannot be empty")
+
+ if set(pattern) - known_tokens:
+ raise ValueError(f"Pattern can only contain {known_tokens}")
+ self.pattern: Deque[str] = deque(pattern)
+
+ def next(self):
+ if self.pattern[0] == self.STOP_TOKEN:
+ return
+ self.pattern.rotate(-1)
+
+ def should_apply(self) -> bool:
+ return self.pattern[0] == self.APPLY_TOKEN
+
+
+@dataclasses.dataclass
+class InjectedResponse:
+ method: str
+ url: str
+ body: Union[str, Exception]
+ status: int = 200
+ content_type: str = "text/plain"
+ # todo: add more fields for other types of responses?
+ custom_match_fn: Optional[Callable[..., bool]] = None
+ application_pattern: TokenizedCircularPattern = TokenizedCircularPattern("1")
+
+ # application_pattern defines the pattern of the response injection
+ # as the requests come in.
+ # 0 == do not inject the response
+ # 1 == inject the response
+ # 2 == stop using the response (END token)
+ #
+ # - when no END token is present, the pattern is repeated indefinitely
+ # - when END token is present, the pattern is applied until the END token is reached
+ # - to replicate the current behavior:
+ # - use application_pattern = "1" if wanting to apply the pattern to all requests
+ # - use application_pattern = "1" * COUNTER + "2" to apply the pattern to the first COUNTER requests
+ #
+ # Examples of application_pattern:
+ # 1. application_pattern = "1012"
+ # - inject the response for the first request
+ # - do not inject the response for the second request
+ # - inject the response for the third request
+ # - stop using the response starting from the fourth request onwards
+ # 2. application_pattern = "110"
+ # repeat the following pattern indefinitely:
+ # - inject the response for the first request
+ # - inject the response for the second request
+ # - stop using the response for the third request
+
+ def __eq__(
+ self,
+ other: Union["InjectedResponse", requests.Request, requests.PreparedRequest],
+ ):
+ """Check InjectedResponse object equality.
+
+ We use this to check if this response should be injected as a replacement of
+ `other`.
+
+ :param other:
+ :return:
+ """
+ if not isinstance(
+ other, (InjectedResponse, requests.Request, requests.PreparedRequest)
+ ):
+ return False
+
+ # always check the method and url
+ ret = self.method == other.method and self.url == other.url
+ # use custom_match_fn to check, e.g. the request body content
+ if ret and self.custom_match_fn is not None:
+ ret = self.custom_match_fn(self, other)
+ return ret
+
+ def to_dict(self):
+ excluded_fields = {"application_pattern", "custom_match_fn"}
+ return {
+ k: self.__getattribute__(k)
+ for k in self.__dict__
+ if (not k.startswith("_") and k not in excluded_fields)
+ }
+
+
+class RelayControlProtocol(Protocol):
+ def process(self, request: "flask.Request") -> None:
+ ... # pragma: no cover
+
+ def control(self, request: "flask.Request") -> Mapping[str, str]:
+ ... # pragma: no cover
+
+
+class RelayServer:
+ def __init__(
+ self,
+ base_url: str,
+ inject: Optional[List[InjectedResponse]] = None,
+ control: Optional[RelayControlProtocol] = None,
+ verbose: bool = False,
+ ) -> None:
+ # todo for the future:
+ # - consider switching from Flask to Quart
+ # - async app will allow for better failure injection/poor network perf
+ self.relay_control = control
+ self.app = flask.Flask(__name__)
+ self.app.logger.setLevel(logging.INFO)
+ self.app.register_error_handler(DeliberateHTTPError, self.handle_http_exception)
+ self.app.add_url_rule(
+ rule="/graphql",
+ endpoint="graphql",
+ view_func=self.graphql,
+ methods=["POST"],
+ )
+ self.app.add_url_rule(
+ rule="/files/<path:path>",
+ endpoint="files",
+ view_func=self.file_stream,
+ methods=["POST"],
+ )
+ self.app.add_url_rule(
+ rule="/storage",
+ endpoint="storage",
+ view_func=self.storage,
+ methods=["PUT", "GET"],
+ )
+ self.app.add_url_rule(
+ rule="/storage/<path:path>",
+ endpoint="storage_file",
+ view_func=self.storage_file,
+ methods=["PUT", "GET"],
+ )
+ if control:
+ self.app.add_url_rule(
+ rule="/_control",
+ endpoint="_control",
+ view_func=self.control,
+ methods=["POST"],
+ )
+ # @app.route("/artifacts/<entity>/<digest>", methods=["GET", "POST"])
+ self.port = self._get_free_port()
+ self.base_url = urllib.parse.urlparse(base_url)
+ self.session = requests.Session()
+ self.relay_url = f"http://127.0.0.1:{self.port}"
+
+ # todo: add an option to add custom resolvers
+ self.resolver = QueryResolver()
+ # recursively merge-able object to store state
+ self.context = Context()
+
+ # injected responses
+ self.inject = inject or []
+
+ # useful when debugging:
+ # self.after_request_fn = self.app.after_request(self.after_request_fn)
+ self.verbose = verbose
+
+ @staticmethod
+ def handle_http_exception(e):
+ response = e.get_response()
+ return response
+
+ @staticmethod
+ def _get_free_port() -> int:
+ sock = socket.socket()
+ sock.bind(("", 0))
+
+ _, port = sock.getsockname()
+ return port
+
+ def start(self) -> None:
+ # run server in a separate thread
+ relay_server_thread = threading.Thread(
+ target=self.app.run,
+ kwargs={"port": self.port},
+ daemon=True,
+ )
+ relay_server_thread.start()
+
+ def after_request_fn(self, response: "requests.Response") -> "requests.Response":
+ # todo: this is useful for debugging, but should be removed in the future
+ # flask.request.url = self.relay_url + flask.request.url
+ print(flask.request)
+ print(flask.request.get_json())
+ print(response)
+ print(response.json())
+ return response
+
+ def relay(
+ self,
+ request: "flask.Request",
+ ) -> Union["responses.Response", "requests.Response"]:
+ # replace the relay url with the real backend url (self.base_url)
+ url = (
+ urllib.parse.urlparse(request.url)
+ ._replace(netloc=self.base_url.netloc, scheme=self.base_url.scheme)
+ .geturl()
+ )
+ headers = {key: value for (key, value) in request.headers if key != "Host"}
+ prepared_relayed_request = requests.Request(
+ method=request.method,
+ url=url,
+ headers=headers,
+ data=request.get_data(),
+ json=request.get_json(),
+ ).prepare()
+
+ if self.verbose:
+ print("*****************")
+ print("RELAY REQUEST:")
+ print(prepared_relayed_request.url)
+ print(prepared_relayed_request.method)
+ print(prepared_relayed_request.headers)
+ print(prepared_relayed_request.body)
+ print("*****************")
+
+ for injected_response in self.inject:
+ # where are we in the application pattern?
+ should_apply = injected_response.application_pattern.should_apply()
+ # check if an injected response matches the request
+ if injected_response == prepared_relayed_request:
+ if self.verbose:
+ print("*****************")
+ print("INJECTING RESPONSE:")
+ print(injected_response.to_dict())
+ print("*****************")
+ # rotate the injection pattern
+ injected_response.application_pattern.next()
+ # TODO: allow access to the request object when making the mocked response
+ if should_apply:
+ with responses.RequestsMock() as mocked_responses:
+ # do the actual injection
+ mocked_responses.add(**injected_response.to_dict())
+ relayed_response = self.session.send(prepared_relayed_request)
+
+ return relayed_response
+
+ # normal case: no injected response matches the request
+ relayed_response = self.session.send(prepared_relayed_request)
+ return relayed_response
+
+ def snoop_context(
+ self,
+ request: "flask.Request",
+ response: "requests.Response",
+ time_elapsed: float,
+ **kwargs: Any,
+ ) -> None:
+ request_data = request.get_json()
+ response_data = response.json() or {}
+
+ if self.relay_control:
+ self.relay_control.process(request)
+
+ # store raw data
+ raw_data: RawRequestResponse = {
+ "url": request.url,
+ "request": request_data,
+ "response": response_data,
+ "time_elapsed": time_elapsed,
+ }
+ self.context.raw_data.append(raw_data)
+
+ try:
+ snooped_context = self.resolver.resolve(
+ request_data,
+ response_data,
+ **kwargs,
+ )
+ except Exception as e:
+ print("Failed to resolve context: ", e)
+ traceback.print_exc()
+ snooped_context = None
+
+ if snooped_context is not None:
+ self.context.upsert(snooped_context)
+
+ return None
+
+ def graphql(self) -> Mapping[str, str]:
+ request = flask.request
+ with Timer() as timer:
+ relayed_response = self.relay(request)
+ if self.verbose:
+ print("*****************")
+ print("GRAPHQL REQUEST:")
+ print(request.get_json())
+ print("GRAPHQL RESPONSE:")
+ print(relayed_response.status_code, relayed_response.json())
+ print("*****************")
+ # snoop work to extract the context
+ self.snoop_context(request, relayed_response, timer.elapsed)
+ if self.verbose:
+ print("*****************")
+ print("SNOOPED CONTEXT:")
+ print(self.context.entries)
+ print(len(self.context.raw_data))
+ print("*****************")
+
+ return relayed_response.json()
+
+ def file_stream(self, path) -> Mapping[str, str]:
+ request = flask.request
+ with Timer() as timer:
+ relayed_response = self.relay(request)
+ if self.verbose:
+ print("*****************")
+ print("FILE STREAM REQUEST:")
+ print("********PATH*********")
+ print(path)
+ print("********ENDPATH*********")
+ print(request.get_json())
+ print("FILE STREAM RESPONSE:")
+ print(relayed_response)
+ print(relayed_response.status_code, relayed_response.json())
+ print("*****************")
+
+ self.snoop_context(request, relayed_response, timer.elapsed, path=path)
+
+ return relayed_response.json()
+
+ def storage(self) -> Mapping[str, str]:
+ request = flask.request
+ with Timer() as timer:
+ relayed_response = self.relay(request)
+ if self.verbose:
+ print("*****************")
+ print("STORAGE REQUEST:")
+ print(request.get_json())
+ print("STORAGE RESPONSE:")
+ print(relayed_response.status_code, relayed_response.json())
+ print("*****************")
+
+ self.snoop_context(request, relayed_response, timer.elapsed)
+
+ return relayed_response.json()
+
+ def storage_file(self, path) -> Mapping[str, str]:
+ request = flask.request
+ with Timer() as timer:
+ relayed_response = self.relay(request)
+ if self.verbose:
+ print("*****************")
+ print("STORAGE FILE REQUEST:")
+ print("********PATH*********")
+ print(path)
+ print("********ENDPATH*********")
+ print(request.get_json())
+ print("STORAGE FILE RESPONSE:")
+ print(relayed_response.json())
+ print("*****************")
+
+ self.snoop_context(request, relayed_response, timer.elapsed, path=path)
+
+ return relayed_response.json()
+
+ def control(self) -> Mapping[str, str]:
+ assert self.relay_control
+ return self.relay_control.control(flask.request)
diff --git a/wandb/trigger.py b/wandb/trigger.py
new file mode 100644
index 0000000000000000000000000000000000000000..eb1333861abacf44fc1ded6a03d0a11caec2eca1
--- /dev/null
+++ b/wandb/trigger.py
@@ -0,0 +1,28 @@
+"""Module to facilitate adding hooks to wandb actions.
+
+Usage:
+ import trigger
+ trigger.register('on_something', func)
+ trigger.call('on_something', *args, **kwargs)
+ trigger.unregister('on_something', func)
+"""
+from typing import Any, Callable
+
+_triggers = {}
+
+
+def reset():
+ _triggers.clear()
+
+
+def register(event: str, func: Callable):
+ _triggers.setdefault(event, []).append(func)
+
+
+def call(event_str: str, *args: Any, **kwargs: Any):
+ for func in _triggers.get(event_str, []):
+ func(*args, **kwargs)
+
+
+def unregister(event: str, func: Callable):
+ _triggers[event].remove(func)
diff --git a/wandb/util.py b/wandb/util.py
new file mode 100644
index 0000000000000000000000000000000000000000..3fa018782ce7513eb74a898519c13cb0d3ef3e39
--- /dev/null
+++ b/wandb/util.py
@@ -0,0 +1,1859 @@
+import colorsys
+import contextlib
+import functools
+import gzip
+import importlib
+import importlib.util
+import itertools
+import json
+import logging
+import math
+import numbers
+import os
+import platform
+import queue
+import random
+import re
+import secrets
+import shlex
+import socket
+import string
+import sys
+import tarfile
+import tempfile
+import threading
+import time
+import types
+import urllib
+from dataclasses import asdict, is_dataclass
+from datetime import date, datetime, timedelta
+from importlib import import_module
+from sys import getsizeof
+from types import ModuleType
+from typing import (
+ IO,
+ TYPE_CHECKING,
+ Any,
+ Callable,
+ Dict,
+ Generator,
+ Iterable,
+ List,
+ Mapping,
+ Optional,
+ Sequence,
+ Set,
+ TextIO,
+ Tuple,
+ TypeVar,
+ Union,
+)
+
+import requests
+import yaml
+
+import wandb
+import wandb.env
+from wandb.errors import AuthenticationError, CommError, UsageError, term
+from wandb.sdk.internal.thread_local_settings import _thread_local_api_settings
+from wandb.sdk.lib import filesystem, runid
+from wandb.sdk.lib.json_util import dump, dumps
+from wandb.sdk.lib.paths import FilePathStr, StrPath
+
+if TYPE_CHECKING:
+ import wandb.sdk.internal.settings_static
+ import wandb.sdk.wandb_settings
+ from wandb.sdk.artifacts.artifact import Artifact
+
+CheckRetryFnType = Callable[[Exception], Union[bool, timedelta]]
+T = TypeVar("T")
+
+
+logger = logging.getLogger(__name__)
+_not_importable = set()
+
+MAX_LINE_BYTES = (10 << 20) - (100 << 10) # imposed by back end
+IS_GIT = os.path.exists(os.path.join(os.path.dirname(__file__), "..", ".git"))
+RE_WINFNAMES = re.compile(r'[<>:"\\?*]')
+
+# From https://docs.docker.com/engine/reference/commandline/tag/
+# "Name components may contain lowercase letters, digits and separators.
+# A separator is defined as a period, one or two underscores, or one or more dashes.
+# A name component may not start or end with a separator."
+DOCKER_IMAGE_NAME_SEPARATOR = "(?:__|[._]|[-]+)"
+RE_DOCKER_IMAGE_NAME_SEPARATOR_START = re.compile("^" + DOCKER_IMAGE_NAME_SEPARATOR)
+RE_DOCKER_IMAGE_NAME_SEPARATOR_END = re.compile(DOCKER_IMAGE_NAME_SEPARATOR + "$")
+RE_DOCKER_IMAGE_NAME_SEPARATOR_REPEAT = re.compile(DOCKER_IMAGE_NAME_SEPARATOR + "{2,}")
+RE_DOCKER_IMAGE_NAME_CHARS = re.compile(r"[^a-z0-9._\-]")
+
+# these match the environments for gorilla
+if IS_GIT:
+ SENTRY_ENV = "development"
+else:
+ SENTRY_ENV = "production"
+
+
+PLATFORM_WINDOWS = "windows"
+PLATFORM_LINUX = "linux"
+PLATFORM_BSD = "bsd"
+PLATFORM_DARWIN = "darwin"
+PLATFORM_UNKNOWN = "unknown"
+
+LAUNCH_JOB_ARTIFACT_SLOT_NAME = "_wandb_job"
+
+
+def get_platform_name() -> str:
+ if sys.platform.startswith("win"):
+ return PLATFORM_WINDOWS
+ elif sys.platform.startswith("darwin"):
+ return PLATFORM_DARWIN
+ elif sys.platform.startswith("linux"):
+ return PLATFORM_LINUX
+ elif sys.platform.startswith(
+ (
+ "dragonfly",
+ "freebsd",
+ "netbsd",
+ "openbsd",
+ )
+ ):
+ return PLATFORM_BSD
+ else:
+ return PLATFORM_UNKNOWN
+
+
+POW_10_BYTES = [
+ ("B", 10**0),
+ ("KB", 10**3),
+ ("MB", 10**6),
+ ("GB", 10**9),
+ ("TB", 10**12),
+ ("PB", 10**15),
+ ("EB", 10**18),
+]
+
+POW_2_BYTES = [
+ ("B", 2**0),
+ ("KiB", 2**10),
+ ("MiB", 2**20),
+ ("GiB", 2**30),
+ ("TiB", 2**40),
+ ("PiB", 2**50),
+ ("EiB", 2**60),
+]
+
+
+def vendor_setup() -> Callable:
+ """Create a function that restores user paths after vendor imports.
+
+ This enables us to use the vendor directory for packages we don't depend on. Call
+ the returned function after imports are complete. If you don't you may modify the
+ user's path which is never good.
+
+ Usage:
+
+ ```python
+ reset_path = vendor_setup()
+ # do any vendor imports...
+ reset_path()
+ ```
+ """
+ original_path = [directory for directory in sys.path]
+
+ def reset_import_path() -> None:
+ sys.path = original_path
+
+ parent_dir = os.path.abspath(os.path.dirname(__file__))
+ vendor_dir = os.path.join(parent_dir, "vendor")
+ vendor_packages = (
+ "gql-0.2.0",
+ "graphql-core-1.1",
+ "watchdog_0_9_0",
+ "promise-2.3.0",
+ )
+ package_dirs = [os.path.join(vendor_dir, p) for p in vendor_packages]
+ for p in [vendor_dir] + package_dirs:
+ if p not in sys.path:
+ sys.path.insert(1, p)
+
+ return reset_import_path
+
+
+def vendor_import(name: str) -> Any:
+ reset_path = vendor_setup()
+ module = import_module(name)
+ reset_path()
+ return module
+
+
+class LazyModuleState:
+ def __init__(self, module: types.ModuleType) -> None:
+ self.module = module
+ self.load_started = False
+ self.lock = threading.RLock()
+
+ def load(self) -> None:
+ with self.lock:
+ if self.load_started:
+ return
+ self.load_started = True
+ assert self.module.__spec__ is not None
+ assert self.module.__spec__.loader is not None
+ self.module.__spec__.loader.exec_module(self.module)
+ self.module.__class__ = types.ModuleType
+
+
+class LazyModule(types.ModuleType):
+ def __getattribute__(self, name: str) -> Any:
+ state = object.__getattribute__(self, "__lazy_module_state__")
+ state.load()
+ return object.__getattribute__(self, name)
+
+ def __setattr__(self, name: str, value: Any) -> None:
+ state = object.__getattribute__(self, "__lazy_module_state__")
+ state.load()
+ object.__setattr__(self, name, value)
+
+ def __delattr__(self, name: str) -> None:
+ state = object.__getattribute__(self, "__lazy_module_state__")
+ state.load()
+ object.__delattr__(self, name)
+
+
+def import_module_lazy(name: str) -> types.ModuleType:
+ """Import a module lazily, only when it is used.
+
+ Inspired by importlib.util.LazyLoader, but improved so that the module loading is
+ thread-safe. Circular dependency between modules can lead to a deadlock if the two
+ modules are loaded from different threads.
+
+ :param (str) name: Dot-separated module path. E.g., 'scipy.stats'.
+ """
+ try:
+ return sys.modules[name]
+ except KeyError:
+ spec = importlib.util.find_spec(name)
+ if spec is None:
+ raise ModuleNotFoundError
+ module = importlib.util.module_from_spec(spec)
+ module.__lazy_module_state__ = LazyModuleState(module) # type: ignore
+ module.__class__ = LazyModule
+ sys.modules[name] = module
+ return module
+
+
+def get_module(
+ name: str,
+ required: Optional[Union[str, bool]] = None,
+ lazy: bool = True,
+) -> Any:
+ """Return module or None. Absolute import is required.
+
+ :param (str) name: Dot-separated module path. E.g., 'scipy.stats'.
+ :param (str) required: A string to raise a ValueError if missing
+ :param (bool) lazy: If True, return a lazy loader for the module.
+ :return: (module|None) If import succeeds, the module will be returned.
+ """
+ if name not in _not_importable:
+ try:
+ if not lazy:
+ return import_module(name)
+ else:
+ return import_module_lazy(name)
+ except Exception:
+ _not_importable.add(name)
+ msg = f"Error importing optional module {name}"
+ if required:
+ logger.exception(msg)
+ if required and name in _not_importable:
+ raise wandb.Error(required)
+
+
+def get_optional_module(name) -> Optional["importlib.ModuleInterface"]: # type: ignore
+ return get_module(name)
+
+
+np = get_module("numpy")
+
+pd_available = False
+pandas_spec = importlib.util.find_spec("pandas")
+if pandas_spec is not None:
+ pd_available = True
+
+# TODO: Revisit these limits
+VALUE_BYTES_LIMIT = 100000
+
+
+def app_url(api_url: str) -> str:
+ """Return the frontend app url without a trailing slash."""
+ # TODO: move me to settings
+ app_url = wandb.env.get_app_url()
+ if app_url is not None:
+ return str(app_url.strip("/"))
+ if "://api.wandb.test" in api_url:
+ # dev mode
+ return api_url.replace("://api.", "://app.").strip("/")
+ elif "://api.wandb." in api_url:
+ # cloud
+ return api_url.replace("://api.", "://").strip("/")
+ elif "://api." in api_url:
+ # onprem cloud
+ return api_url.replace("://api.", "://app.").strip("/")
+ # wandb/local
+ return api_url
+
+
+def get_full_typename(o: Any) -> Any:
+ """Determine types based on type names.
+
+ Avoids needing to to import (and therefore depend on) PyTorch, TensorFlow, etc.
+ """
+ instance_name = o.__class__.__module__ + "." + o.__class__.__name__
+ if instance_name in ["builtins.module", "__builtin__.module"]:
+ return o.__name__
+ else:
+ return instance_name
+
+
+def get_h5_typename(o: Any) -> Any:
+ typename = get_full_typename(o)
+ if is_tf_tensor_typename(typename):
+ return "tensorflow.Tensor"
+ elif is_pytorch_tensor_typename(typename):
+ return "torch.Tensor"
+ else:
+ return o.__class__.__module__.split(".")[0] + "." + o.__class__.__name__
+
+
+def is_uri(string: str) -> bool:
+ parsed_uri = urllib.parse.urlparse(string)
+ return len(parsed_uri.scheme) > 0
+
+
+def local_file_uri_to_path(uri: str) -> str:
+ """Convert URI to local filesystem path.
+
+ No-op if the uri does not have the expected scheme.
+ """
+ path = urllib.parse.urlparse(uri).path if uri.startswith("file:") else uri
+ return urllib.request.url2pathname(path)
+
+
+def get_local_path_or_none(path_or_uri: str) -> Optional[str]:
+ """Return path if local, None otherwise.
+
+ Return None if the argument is a local path (not a scheme or file:///). Otherwise
+ return `path_or_uri`.
+ """
+ parsed_uri = urllib.parse.urlparse(path_or_uri)
+ if (
+ len(parsed_uri.scheme) == 0
+ or parsed_uri.scheme == "file"
+ and len(parsed_uri.netloc) == 0
+ ):
+ return local_file_uri_to_path(path_or_uri)
+ else:
+ return None
+
+
+def make_tarfile(
+ output_filename: str,
+ source_dir: str,
+ archive_name: str,
+ custom_filter: Optional[Callable] = None,
+) -> None:
+ # Helper for filtering out modification timestamps
+ def _filter_timestamps(tar_info: "tarfile.TarInfo") -> Optional["tarfile.TarInfo"]:
+ tar_info.mtime = 0
+ return tar_info if custom_filter is None else custom_filter(tar_info)
+
+ descriptor, unzipped_filename = tempfile.mkstemp()
+ try:
+ with tarfile.open(unzipped_filename, "w") as tar:
+ tar.add(source_dir, arcname=archive_name, filter=_filter_timestamps)
+ # When gzipping the tar, don't include the tar's filename or modification time in the
+ # zipped archive (see https://docs.python.org/3/library/gzip.html#gzip.GzipFile)
+ with gzip.GzipFile(
+ filename="", fileobj=open(output_filename, "wb"), mode="wb", mtime=0
+ ) as gzipped_tar, open(unzipped_filename, "rb") as tar_file:
+ gzipped_tar.write(tar_file.read())
+ finally:
+ os.close(descriptor)
+ os.remove(unzipped_filename)
+
+
+def is_tf_tensor(obj: Any) -> bool:
+ import tensorflow # type: ignore
+
+ return isinstance(obj, tensorflow.Tensor)
+
+
+def is_tf_tensor_typename(typename: str) -> bool:
+ return typename.startswith("tensorflow.") and (
+ "Tensor" in typename or "Variable" in typename
+ )
+
+
+def is_tf_eager_tensor_typename(typename: str) -> bool:
+ return typename.startswith("tensorflow.") and ("EagerTensor" in typename)
+
+
+def is_pytorch_tensor(obj: Any) -> bool:
+ import torch # type: ignore
+
+ return isinstance(obj, torch.Tensor)
+
+
+def is_pytorch_tensor_typename(typename: str) -> bool:
+ return typename.startswith("torch.") and (
+ "Tensor" in typename or "Variable" in typename
+ )
+
+
+def is_jax_tensor_typename(typename: str) -> bool:
+ return typename.startswith("jaxlib.") and "Array" in typename
+
+
+def get_jax_tensor(obj: Any) -> Optional[Any]:
+ import jax # type: ignore
+
+ return jax.device_get(obj)
+
+
+def is_fastai_tensor_typename(typename: str) -> bool:
+ return typename.startswith("fastai.") and ("Tensor" in typename)
+
+
+def is_pandas_data_frame_typename(typename: str) -> bool:
+ return typename.startswith("pandas.") and "DataFrame" in typename
+
+
+def is_matplotlib_typename(typename: str) -> bool:
+ return typename.startswith("matplotlib.")
+
+
+def is_plotly_typename(typename: str) -> bool:
+ return typename.startswith("plotly.")
+
+
+def is_plotly_figure_typename(typename: str) -> bool:
+ return typename.startswith("plotly.") and typename.endswith(".Figure")
+
+
+def is_numpy_array(obj: Any) -> bool:
+ return np and isinstance(obj, np.ndarray)
+
+
+def is_pandas_data_frame(obj: Any) -> bool:
+ if pd_available:
+ import pandas as pd
+
+ return isinstance(obj, pd.DataFrame)
+ else:
+ return is_pandas_data_frame_typename(get_full_typename(obj))
+
+
+def ensure_matplotlib_figure(obj: Any) -> Any:
+ """Extract the current figure from a matplotlib object.
+
+ Return the object itself if it's a figure.
+ Raises ValueError if the object can't be converted.
+ """
+ import matplotlib # type: ignore
+ from matplotlib.figure import Figure # type: ignore
+
+ # there are combinations of plotly and matplotlib versions that don't work well together,
+ # this patches matplotlib to add a removed method that plotly assumes exists
+ from matplotlib.spines import Spine # type: ignore
+
+ def is_frame_like(self: Any) -> bool:
+ """Return True if directly on axes frame.
+
+ This is useful for determining if a spine is the edge of an
+ old style MPL plot. If so, this function will return True.
+ """
+ position = self._position or ("outward", 0.0)
+ if isinstance(position, str):
+ if position == "center":
+ position = ("axes", 0.5)
+ elif position == "zero":
+ position = ("data", 0)
+ if len(position) != 2:
+ raise ValueError("position should be 2-tuple")
+ position_type, amount = position # type: ignore
+ if position_type == "outward" and amount == 0:
+ return True
+ else:
+ return False
+
+ Spine.is_frame_like = is_frame_like
+
+ if obj == matplotlib.pyplot:
+ obj = obj.gcf()
+ elif not isinstance(obj, Figure):
+ if hasattr(obj, "figure"):
+ obj = obj.figure
+ # Some matplotlib objects have a figure function
+ if not isinstance(obj, Figure):
+ raise ValueError(
+ "Only matplotlib.pyplot or matplotlib.pyplot.Figure objects are accepted."
+ )
+ return obj
+
+
+def matplotlib_to_plotly(obj: Any) -> Any:
+ obj = ensure_matplotlib_figure(obj)
+ tools = get_module(
+ "plotly.tools",
+ required=(
+ "plotly is required to log interactive plots, install with: "
+ "`pip install plotly` or convert the plot to an image with `wandb.Image(plt)`"
+ ),
+ )
+ return tools.mpl_to_plotly(obj)
+
+
+def matplotlib_contains_images(obj: Any) -> bool:
+ obj = ensure_matplotlib_figure(obj)
+ return any(len(ax.images) > 0 for ax in obj.axes)
+
+
+def _numpy_generic_convert(obj: Any) -> Any:
+ obj = obj.item()
+ if isinstance(obj, float) and math.isnan(obj):
+ obj = None
+ elif isinstance(obj, np.generic) and (
+ obj.dtype.kind == "f" or obj.dtype == "bfloat16"
+ ):
+ # obj is a numpy float with precision greater than that of native python float
+ # (i.e., float96 or float128) or it is of custom type such as bfloat16.
+ # in these cases, obj.item() does not return a native
+ # python float (in the first case - to avoid loss of precision,
+ # so we need to explicitly cast this down to a 64bit float)
+ obj = float(obj)
+ return obj
+
+
+def _find_all_matching_keys(
+ d: Dict,
+ match_fn: Callable[[Any], bool],
+ visited: Optional[Set[int]] = None,
+ key_path: Tuple[Any, ...] = (),
+) -> Generator[Tuple[Tuple[Any, ...], Any], None, None]:
+ """Recursively find all keys that satisfies a match function.
+
+ Args:
+ d: The dict to search.
+ match_fn: The function to determine if the key is a match.
+ visited: Keep track of visited nodes so we dont recurse forever.
+ key_path: Keep track of all the keys to get to the current node.
+
+ Yields:
+ (key_path, key): The location where the key was found, and the key
+ """
+ if visited is None:
+ visited = set()
+ me = id(d)
+ if me not in visited:
+ visited.add(me)
+ for key, value in d.items():
+ if match_fn(key):
+ yield key_path, key
+ if isinstance(value, dict):
+ yield from _find_all_matching_keys(
+ value,
+ match_fn,
+ visited=visited,
+ key_path=tuple(list(key_path) + [key]),
+ )
+
+
+def _sanitize_numpy_keys(d: Dict) -> Tuple[Dict, bool]:
+ np_keys = list(_find_all_matching_keys(d, lambda k: isinstance(k, np.generic)))
+ if not np_keys:
+ return d, False
+ for key_path, key in np_keys:
+ ptr = d
+ for k in key_path:
+ ptr = ptr[k]
+ ptr[_numpy_generic_convert(key)] = ptr.pop(key)
+ return d, True
+
+
+def json_friendly( # noqa: C901
+ obj: Any,
+) -> Union[Tuple[Any, bool], Tuple[Union[None, str, float], bool]]:
+ """Convert an object into something that's more becoming of JSON."""
+ converted = True
+ typename = get_full_typename(obj)
+
+ if is_tf_eager_tensor_typename(typename):
+ obj = obj.numpy()
+ elif is_tf_tensor_typename(typename):
+ try:
+ obj = obj.eval()
+ except RuntimeError:
+ obj = obj.numpy()
+ elif is_pytorch_tensor_typename(typename) or is_fastai_tensor_typename(typename):
+ try:
+ if obj.requires_grad:
+ obj = obj.detach()
+ except AttributeError:
+ pass # before 0.4 is only present on variables
+
+ try:
+ obj = obj.data
+ except RuntimeError:
+ pass # happens for Tensors before 0.4
+
+ if obj.size():
+ obj = obj.cpu().detach().numpy()
+ else:
+ return obj.item(), True
+ elif is_jax_tensor_typename(typename):
+ obj = get_jax_tensor(obj)
+
+ if is_numpy_array(obj):
+ if obj.size == 1:
+ obj = obj.flatten()[0]
+ elif obj.size <= 32:
+ obj = obj.tolist()
+ elif np and isinstance(obj, np.generic):
+ obj = _numpy_generic_convert(obj)
+ elif isinstance(obj, bytes):
+ obj = obj.decode("utf-8")
+ elif isinstance(obj, (datetime, date)):
+ obj = obj.isoformat()
+ elif callable(obj):
+ obj = (
+ f"{obj.__module__}.{obj.__qualname__}"
+ if hasattr(obj, "__qualname__") and hasattr(obj, "__module__")
+ else str(obj)
+ )
+ elif isinstance(obj, float) and math.isnan(obj):
+ obj = None
+ elif isinstance(obj, dict) and np:
+ obj, converted = _sanitize_numpy_keys(obj)
+ elif isinstance(obj, set):
+ # set is not json serializable, so we convert it to tuple
+ obj = tuple(obj)
+ else:
+ converted = False
+ if getsizeof(obj) > VALUE_BYTES_LIMIT:
+ wandb.termwarn(
+ "Serializing object of type {} that is {} bytes".format(
+ type(obj).__name__, getsizeof(obj)
+ )
+ )
+ return obj, converted
+
+
+def json_friendly_val(val: Any) -> Any:
+ """Make any value (including dict, slice, sequence, dataclass) JSON friendly."""
+ converted: Union[dict, list]
+ if isinstance(val, dict):
+ converted = {}
+ for key, value in val.items():
+ converted[key] = json_friendly_val(value)
+ return converted
+ if isinstance(val, slice):
+ converted = dict(
+ slice_start=val.start, slice_step=val.step, slice_stop=val.stop
+ )
+ return converted
+ val, _ = json_friendly(val)
+ if isinstance(val, Sequence) and not isinstance(val, str):
+ converted = []
+ for value in val:
+ converted.append(json_friendly_val(value))
+ return converted
+ if is_dataclass(val) and not isinstance(val, type):
+ converted = asdict(val)
+ return converted
+ else:
+ if val.__class__.__module__ not in ("builtins", "__builtin__"):
+ val = str(val)
+ return val
+
+
+def alias_is_version_index(alias: str) -> bool:
+ return len(alias) >= 2 and alias[0] == "v" and alias[1:].isnumeric()
+
+
+def convert_plots(obj: Any) -> Any:
+ if is_matplotlib_typename(get_full_typename(obj)):
+ tools = get_module(
+ "plotly.tools",
+ required=(
+ "plotly is required to log interactive plots, install with: "
+ "`pip install plotly` or convert the plot to an image with `wandb.Image(plt)`"
+ ),
+ )
+ obj = tools.mpl_to_plotly(obj)
+
+ if is_plotly_typename(get_full_typename(obj)):
+ return {"_type": "plotly", "plot": obj.to_plotly_json()}
+ else:
+ return obj
+
+
+def maybe_compress_history(obj: Any) -> Tuple[Any, bool]:
+ if np and isinstance(obj, np.ndarray) and obj.size > 32:
+ return wandb.Histogram(obj, num_bins=32).to_json(), True
+ else:
+ return obj, False
+
+
+def maybe_compress_summary(obj: Any, h5_typename: str) -> Tuple[Any, bool]:
+ if np and isinstance(obj, np.ndarray) and obj.size > 32:
+ return (
+ {
+ "_type": h5_typename, # may not be ndarray
+ "var": np.var(obj).item(),
+ "mean": np.mean(obj).item(),
+ "min": np.amin(obj).item(),
+ "max": np.amax(obj).item(),
+ "10%": np.percentile(obj, 10),
+ "25%": np.percentile(obj, 25),
+ "75%": np.percentile(obj, 75),
+ "90%": np.percentile(obj, 90),
+ "size": obj.size,
+ },
+ True,
+ )
+ else:
+ return obj, False
+
+
+def launch_browser(attempt_launch_browser: bool = True) -> bool:
+ """Decide if we should launch a browser."""
+ _display_variables = ["DISPLAY", "WAYLAND_DISPLAY", "MIR_SOCKET"]
+ _webbrowser_names_blocklist = ["www-browser", "lynx", "links", "elinks", "w3m"]
+
+ import webbrowser
+
+ launch_browser = attempt_launch_browser
+ if launch_browser:
+ if "linux" in sys.platform and not any(
+ os.getenv(var) for var in _display_variables
+ ):
+ launch_browser = False
+ try:
+ browser = webbrowser.get()
+ if hasattr(browser, "name") and browser.name in _webbrowser_names_blocklist:
+ launch_browser = False
+ except webbrowser.Error:
+ launch_browser = False
+
+ return launch_browser
+
+
+def generate_id(length: int = 8) -> str:
+ # Do not use this; use wandb.sdk.lib.runid.generate_id instead.
+ # This is kept only for legacy code.
+ return runid.generate_id(length)
+
+
+def parse_tfjob_config() -> Any:
+ """Attempt to parse TFJob config, returning False if it can't find it."""
+ if os.getenv("TF_CONFIG"):
+ try:
+ return json.loads(os.environ["TF_CONFIG"])
+ except ValueError:
+ return False
+ else:
+ return False
+
+
+class WandBJSONEncoder(json.JSONEncoder):
+ """A JSON Encoder that handles some extra types."""
+
+ def default(self, obj: Any) -> Any:
+ if hasattr(obj, "json_encode"):
+ return obj.json_encode()
+ # if hasattr(obj, 'to_json'):
+ # return obj.to_json()
+ tmp_obj, converted = json_friendly(obj)
+ if converted:
+ return tmp_obj
+ return json.JSONEncoder.default(self, obj)
+
+
+class WandBJSONEncoderOld(json.JSONEncoder):
+ """A JSON Encoder that handles some extra types."""
+
+ def default(self, obj: Any) -> Any:
+ tmp_obj, converted = json_friendly(obj)
+ tmp_obj, compressed = maybe_compress_summary(tmp_obj, get_h5_typename(obj))
+ if converted:
+ return tmp_obj
+ return json.JSONEncoder.default(self, tmp_obj)
+
+
+class WandBHistoryJSONEncoder(json.JSONEncoder):
+ """A JSON Encoder that handles some extra types.
+
+ This encoder turns numpy like objects with a size > 32 into histograms.
+ """
+
+ def default(self, obj: Any) -> Any:
+ obj, converted = json_friendly(obj)
+ obj, compressed = maybe_compress_history(obj)
+ if converted:
+ return obj
+ return json.JSONEncoder.default(self, obj)
+
+
+class JSONEncoderUncompressed(json.JSONEncoder):
+ """A JSON Encoder that handles some extra types.
+
+ This encoder turns numpy like objects with a size > 32 into histograms.
+ """
+
+ def default(self, obj: Any) -> Any:
+ if is_numpy_array(obj):
+ return obj.tolist()
+ elif np and isinstance(obj, np.generic):
+ obj = obj.item()
+ return json.JSONEncoder.default(self, obj)
+
+
+def json_dump_safer(obj: Any, fp: IO[str], **kwargs: Any) -> None:
+ """Convert obj to json, with some extra encodable types."""
+ return dump(obj, fp, cls=WandBJSONEncoder, **kwargs)
+
+
+def json_dumps_safer(obj: Any, **kwargs: Any) -> str:
+ """Convert obj to json, with some extra encodable types."""
+ return dumps(obj, cls=WandBJSONEncoder, **kwargs)
+
+
+# This is used for dumping raw json into files
+def json_dump_uncompressed(obj: Any, fp: IO[str], **kwargs: Any) -> None:
+ """Convert obj to json, with some extra encodable types."""
+ return dump(obj, fp, cls=JSONEncoderUncompressed, **kwargs)
+
+
+def json_dumps_safer_history(obj: Any, **kwargs: Any) -> str:
+ """Convert obj to json, with some extra encodable types, including histograms."""
+ return dumps(obj, cls=WandBHistoryJSONEncoder, **kwargs)
+
+
+def make_json_if_not_number(
+ v: Union[int, float, str, Mapping, Sequence]
+) -> Union[int, float, str]:
+ """If v is not a basic type convert it to json."""
+ if isinstance(v, (float, int)):
+ return v
+ return json_dumps_safer(v)
+
+
+def make_safe_for_json(obj: Any) -> Any:
+ """Replace invalid json floats with strings. Also converts to lists and dicts."""
+ if isinstance(obj, Mapping):
+ return {k: make_safe_for_json(v) for k, v in obj.items()}
+ elif isinstance(obj, str):
+ # str's are Sequence, so we need to short-circuit
+ return obj
+ elif isinstance(obj, Sequence):
+ return [make_safe_for_json(v) for v in obj]
+ elif isinstance(obj, float):
+ # W&B backend and UI handle these strings
+ if obj != obj: # standard way to check for NaN
+ return "NaN"
+ elif obj == float("+inf"):
+ return "Infinity"
+ elif obj == float("-inf"):
+ return "-Infinity"
+ return obj
+
+
+def no_retry_4xx(e: Exception) -> bool:
+ if not isinstance(e, requests.HTTPError):
+ return True
+ assert e.response is not None
+ if not (400 <= e.response.status_code < 500) or e.response.status_code == 429:
+ return True
+ body = json.loads(e.response.content)
+ raise UsageError(body["errors"][0]["message"])
+
+
+def no_retry_auth(e: Any) -> bool:
+ if hasattr(e, "exception"):
+ e = e.exception
+ if not isinstance(e, requests.HTTPError):
+ return True
+ if e.response is None:
+ return True
+ # Don't retry bad request errors; raise immediately
+ if e.response.status_code in (400, 409):
+ return False
+ # Retry all non-forbidden/unauthorized/not-found errors.
+ if e.response.status_code not in (401, 403, 404):
+ return True
+ # Crash w/message on forbidden/unauthorized errors.
+ if e.response.status_code == 401:
+ raise AuthenticationError(
+ "The API key you provided is either invalid or missing. "
+ f"If the `{wandb.env.API_KEY}` environment variable is set, make sure it is correct. "
+ "Otherwise, to resolve this issue, you may try running the 'wandb login --relogin' command. "
+ "If you are using a local server, make sure that you're using the correct hostname. "
+ "If you're not sure, you can try logging in again using the 'wandb login --relogin --host [hostname]' command."
+ f"(Error {e.response.status_code}: {e.response.reason})"
+ )
+ elif wandb.run:
+ raise CommError(f"Permission denied to access {wandb.run.path}")
+ else:
+ raise CommError(
+ "It appears that you do not have permission to access the requested resource. "
+ "Please reach out to the project owner to grant you access. "
+ "If you have the correct permissions, verify that there are no issues with your networking setup."
+ f"(Error {e.response.status_code}: {e.response.reason})"
+ )
+
+
+def check_retry_conflict(e: Any) -> Optional[bool]:
+ """Check if the exception is a conflict type so it can be retried.
+
+ Returns:
+ True - Should retry this operation
+ False - Should not retry this operation
+ None - No decision, let someone else decide
+ """
+ if hasattr(e, "exception"):
+ e = e.exception
+ if isinstance(e, requests.HTTPError) and e.response is not None:
+ if e.response.status_code == 409:
+ return True
+ return None
+
+
+def check_retry_conflict_or_gone(e: Any) -> Optional[bool]:
+ """Check if the exception is a conflict or gone type, so it can be retried or not.
+
+ Returns:
+ True - Should retry this operation
+ False - Should not retry this operation
+ None - No decision, let someone else decide
+ """
+ if hasattr(e, "exception"):
+ e = e.exception
+ if isinstance(e, requests.HTTPError) and e.response is not None:
+ if e.response.status_code == 409:
+ return True
+ if e.response.status_code == 410:
+ return False
+ return None
+
+
+def make_check_retry_fn(
+ fallback_retry_fn: CheckRetryFnType,
+ check_fn: Callable[[Exception], Optional[bool]],
+ check_timedelta: Optional[timedelta] = None,
+) -> CheckRetryFnType:
+ """Return a check_retry_fn which can be used by lib.Retry().
+
+ Arguments:
+ fallback_fn: Use this function if check_fn didn't decide if a retry should happen.
+ check_fn: Function which returns bool if retry should happen or None if unsure.
+ check_timedelta: Optional retry timeout if we check_fn matches the exception
+ """
+
+ def check_retry_fn(e: Exception) -> Union[bool, timedelta]:
+ check = check_fn(e)
+ if check is None:
+ return fallback_retry_fn(e)
+ if check is False:
+ return False
+ if check_timedelta:
+ return check_timedelta
+ return True
+
+ return check_retry_fn
+
+
+def find_runner(program: str) -> Union[None, list, List[str]]:
+ """Return a command that will run program.
+
+ Arguments:
+ program: The string name of the program to try to run.
+
+ Returns:
+ commandline list of strings to run the program (eg. with subprocess.call()) or None
+ """
+ if os.path.isfile(program) and not os.access(program, os.X_OK):
+ # program is a path to a non-executable file
+ try:
+ opened = open(program)
+ except OSError: # PermissionError doesn't exist in 2.7
+ return None
+ first_line = opened.readline().strip()
+ if first_line.startswith("#!"):
+ return shlex.split(first_line[2:])
+ if program.endswith(".py"):
+ return [sys.executable]
+ return None
+
+
+def downsample(values: Sequence, target_length: int) -> list:
+ """Downsample 1d values to target_length, including start and end.
+
+ Algorithm just rounds index down.
+
+ Values can be any sequence, including a generator.
+ """
+ if not target_length > 1:
+ raise UsageError("target_length must be > 1")
+ values = list(values)
+ if len(values) < target_length:
+ return values
+ ratio = float(len(values) - 1) / (target_length - 1)
+ result = []
+ for i in range(target_length):
+ result.append(values[int(i * ratio)])
+ return result
+
+
+def has_num(dictionary: Mapping, key: Any) -> bool:
+ return key in dictionary and isinstance(dictionary[key], numbers.Number)
+
+
+def get_log_file_path() -> str:
+ """Log file path used in error messages.
+
+ It would probably be better if this pointed to a log file in a
+ run directory.
+ """
+ # TODO(jhr, cvp): refactor
+ if wandb.run is not None:
+ return wandb.run._settings.log_internal
+ return os.path.join("wandb", "debug-internal.log")
+
+
+def docker_image_regex(image: str) -> Any:
+ """Regex match for valid docker image names."""
+ if image:
+ return re.match(
+ r"^(?:(?=[^:\/]{1,253})(?!-)[a-zA-Z0-9-]{1,63}(?<!-)(?:\.(?!-)[a-zA-Z0-9-]{1,63}(?<!-))*(?::[0-9]{1,5})?/)?((?![._-])(?:[a-z0-9._-]*)(?<![._-])(?:/(?![._-])[a-z0-9._-]*(?<![._-]))*)(?::(?![.-])[a-zA-Z0-9_.-]{1,128})?$",
+ image,
+ )
+ return None
+
+
+def image_from_docker_args(args: List[str]) -> Optional[str]:
+ """Scan docker run args and attempt to find the most likely docker image argument.
+
+ It excludes any arguments that start with a dash, and the argument after it if it
+ isn't a boolean switch. This can be improved, we currently fallback gracefully when
+ this fails.
+ """
+ bool_args = [
+ "-t",
+ "--tty",
+ "--rm",
+ "--privileged",
+ "--oom-kill-disable",
+ "--no-healthcheck",
+ "-i",
+ "--interactive",
+ "--init",
+ "--help",
+ "--detach",
+ "-d",
+ "--sig-proxy",
+ "-it",
+ "-itd",
+ ]
+ last_flag = -2
+ last_arg = ""
+ possible_images = []
+ if len(args) > 0 and args[0] == "run":
+ args.pop(0)
+ for i, arg in enumerate(args):
+ if arg.startswith("-"):
+ last_flag = i
+ last_arg = arg
+ elif "@sha256:" in arg:
+ # Because our regex doesn't match digests
+ possible_images.append(arg)
+ elif docker_image_regex(arg):
+ if last_flag == i - 2:
+ possible_images.append(arg)
+ elif "=" in last_arg:
+ possible_images.append(arg)
+ elif last_arg in bool_args and last_flag == i - 1:
+ possible_images.append(arg)
+ most_likely = None
+ for img in possible_images:
+ if ":" in img or "@" in img or "/" in img:
+ most_likely = img
+ break
+ if most_likely is None and len(possible_images) > 0:
+ most_likely = possible_images[0]
+ return most_likely
+
+
+def load_yaml(file: Any) -> Any:
+ return yaml.safe_load(file)
+
+
+def image_id_from_k8s() -> Optional[str]:
+ """Ping the k8s metadata service for the image id.
+
+ Specify the KUBERNETES_NAMESPACE environment variable if your pods are not in the
+ default namespace:
+
+ - name: KUBERNETES_NAMESPACE valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ """
+ token_path = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+
+ if not os.path.exists(token_path):
+ return None
+
+ try:
+ with open(token_path) as token_file:
+ token = token_file.read()
+ except FileNotFoundError:
+ logger.warning(f"Token file not found at {token_path}.")
+ return None
+ except PermissionError as e:
+ current_uid = os.getuid()
+ warning = (
+ f"Unable to read the token file at {token_path} due to permission error ({e})."
+ f"The current user id is {current_uid}. "
+ "Consider changing the securityContext to run the container as the current user."
+ )
+ logger.warning(warning)
+ wandb.termwarn(warning)
+ return None
+
+ if not token:
+ return None
+
+ k8s_server = "https://{}:{}/api/v1/namespaces/{}/pods/{}".format(
+ os.getenv("KUBERNETES_SERVICE_HOST"),
+ os.getenv("KUBERNETES_PORT_443_TCP_PORT"),
+ os.getenv("KUBERNETES_NAMESPACE", "default"),
+ os.getenv("HOSTNAME"),
+ )
+ try:
+ res = requests.get(
+ k8s_server,
+ verify="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
+ timeout=3,
+ headers={"Authorization": f"Bearer {token}"},
+ )
+ res.raise_for_status()
+ except requests.RequestException:
+ return None
+ try:
+ return str( # noqa: B005
+ res.json()["status"]["containerStatuses"][0]["imageID"]
+ ).strip("docker-pullable://")
+ except (ValueError, KeyError, IndexError):
+ logger.exception("Error checking kubernetes for image id")
+ return None
+
+
+def async_call(
+ target: Callable, timeout: Optional[Union[int, float]] = None
+) -> Callable:
+ """Wrap a method to run in the background with an optional timeout.
+
+ Returns a new method that will call the original with any args, waiting for upto
+ timeout seconds. This new method blocks on the original and returns the result or
+ None if timeout was reached, along with the thread. You can check thread.is_alive()
+ to determine if a timeout was reached. If an exception is thrown in the thread, we
+ reraise it.
+ """
+ q: queue.Queue = queue.Queue()
+
+ def wrapped_target(q: "queue.Queue", *args: Any, **kwargs: Any) -> Any:
+ try:
+ q.put(target(*args, **kwargs))
+ except Exception as e:
+ q.put(e)
+
+ def wrapper(
+ *args: Any, **kwargs: Any
+ ) -> Union[Tuple[Exception, "threading.Thread"], Tuple[None, "threading.Thread"]]:
+ thread = threading.Thread(
+ target=wrapped_target, args=(q,) + args, kwargs=kwargs
+ )
+ thread.daemon = True
+ thread.start()
+ try:
+ result = q.get(True, timeout)
+ if isinstance(result, Exception):
+ raise result.with_traceback(sys.exc_info()[2])
+ return result, thread
+ except queue.Empty:
+ return None, thread
+
+ return wrapper
+
+
+def read_many_from_queue(
+ q: "queue.Queue", max_items: int, queue_timeout: Union[int, float]
+) -> list:
+ try:
+ item = q.get(True, queue_timeout)
+ except queue.Empty:
+ return []
+ items = [item]
+ for _ in range(max_items):
+ try:
+ item = q.get_nowait()
+ except queue.Empty:
+ return items
+ items.append(item)
+ return items
+
+
+def stopwatch_now() -> float:
+ """Get a time value for interval comparisons.
+
+ When possible it is a monotonic clock to prevent backwards time issues.
+ """
+ return time.monotonic()
+
+
+def class_colors(class_count: int) -> List[List[int]]:
+ # make class 0 black, and the rest equally spaced fully saturated hues
+ return [[0, 0, 0]] + [
+ colorsys.hsv_to_rgb(i / (class_count - 1.0), 1.0, 1.0) # type: ignore
+ for i in range(class_count - 1)
+ ]
+
+
+def _prompt_choice(
+ input_timeout: Union[int, float, None] = None,
+ jupyter: bool = False,
+) -> str:
+ input_fn: Callable = input
+ prompt = term.LOG_STRING
+ if input_timeout is not None:
+ # delayed import to mitigate risk of timed_input complexity
+ from wandb.sdk.lib import timed_input
+
+ input_fn = functools.partial(timed_input.timed_input, timeout=input_timeout)
+ # timed_input doesn't handle enhanced prompts
+ if platform.system() == "Windows":
+ prompt = "wandb"
+
+ text = f"{prompt}: Enter your choice: "
+ if input_fn == input:
+ choice = input_fn(text)
+ else:
+ choice = input_fn(text, jupyter=jupyter)
+ return choice # type: ignore
+
+
+def prompt_choices(
+ choices: Sequence[str],
+ input_timeout: Union[int, float, None] = None,
+ jupyter: bool = False,
+) -> str:
+ """Allow a user to choose from a list of options."""
+ for i, choice in enumerate(choices):
+ wandb.termlog(f"({i+1}) {choice}")
+
+ idx = -1
+ while idx < 0 or idx > len(choices) - 1:
+ choice = _prompt_choice(input_timeout=input_timeout, jupyter=jupyter)
+ if not choice:
+ continue
+ idx = -1
+ try:
+ idx = int(choice) - 1
+ except ValueError:
+ pass
+ if idx < 0 or idx > len(choices) - 1:
+ wandb.termwarn("Invalid choice")
+ result = choices[idx]
+ wandb.termlog(f"You chose {result!r}")
+ return result
+
+
+def guess_data_type(shape: Sequence[int], risky: bool = False) -> Optional[str]:
+ """Infer the type of data based on the shape of the tensors.
+
+ Arguments:
+ shape (Sequence[int]): The shape of the data
+ risky(bool): some guesses are more likely to be wrong.
+ """
+ # (samples,) or (samples,logits)
+ if len(shape) in (1, 2):
+ return "label"
+ # Assume image mask like fashion mnist: (no color channel)
+ # This is risky because RNNs often have 3 dim tensors: batch, time, channels
+ if risky and len(shape) == 3:
+ return "image"
+ if len(shape) == 4:
+ if shape[-1] in (1, 3, 4):
+ # (samples, height, width, Y \ RGB \ RGBA)
+ return "image"
+ else:
+ # (samples, height, width, logits)
+ return "segmentation_mask"
+ return None
+
+
+def download_file_from_url(
+ dest_path: str, source_url: str, api_key: Optional[str] = None
+) -> None:
+ auth = None
+ if not _thread_local_api_settings.cookies:
+ auth = ("api", api_key or "")
+ response = requests.get(
+ source_url,
+ auth=auth,
+ headers=_thread_local_api_settings.headers,
+ cookies=_thread_local_api_settings.cookies,
+ stream=True,
+ timeout=5,
+ )
+ response.raise_for_status()
+
+ if os.sep in dest_path:
+ filesystem.mkdir_exists_ok(os.path.dirname(dest_path))
+ with fsync_open(dest_path, "wb") as file:
+ for data in response.iter_content(chunk_size=1024):
+ file.write(data)
+
+
+def download_file_into_memory(source_url: str, api_key: Optional[str] = None) -> bytes:
+ auth = None
+ if not _thread_local_api_settings.cookies:
+ auth = ("api", api_key or "")
+ response = requests.get(
+ source_url,
+ auth=auth,
+ headers=_thread_local_api_settings.headers,
+ cookies=_thread_local_api_settings.cookies,
+ stream=True,
+ timeout=5,
+ )
+ response.raise_for_status()
+ return response.content
+
+
+def isatty(ob: IO) -> bool:
+ return hasattr(ob, "isatty") and ob.isatty()
+
+
+def to_human_size(size: int, units: Optional[List[Tuple[str, Any]]] = None) -> str:
+ units = units or POW_10_BYTES
+ unit, value = units[0]
+ factor = round(float(size) / value, 1)
+ return (
+ f"{factor}{unit}"
+ if factor < 1024 or len(units) == 1
+ else to_human_size(size, units[1:])
+ )
+
+
+def from_human_size(size: str, units: Optional[List[Tuple[str, Any]]] = None) -> int:
+ units = units or POW_10_BYTES
+ units_dict = {unit.upper(): value for (unit, value) in units}
+ regex = re.compile(
+ r"(\d+\.?\d*)\s*({})?".format("|".join(units_dict.keys())), re.IGNORECASE
+ )
+ match = re.match(regex, size)
+ if not match:
+ raise ValueError("size must be of the form `10`, `10B` or `10 B`.")
+ factor, unit = (
+ float(match.group(1)),
+ units_dict[match.group(2).upper()] if match.group(2) else 1,
+ )
+ return int(factor * unit)
+
+
+def auto_project_name(program: Optional[str]) -> str:
+ # if we're in git, set project name to git repo name + relative path within repo
+ from wandb.sdk.lib.gitlib import GitRepo
+
+ root_dir = GitRepo().root_dir
+ if root_dir is None:
+ return "uncategorized"
+ # On windows, GitRepo returns paths in unix style, but os.path is windows
+ # style. Coerce here.
+ root_dir = to_native_slash_path(root_dir)
+ repo_name = os.path.basename(root_dir)
+ if program is None:
+ return str(repo_name)
+ if not os.path.isabs(program):
+ program = os.path.join(os.curdir, program)
+ prog_dir = os.path.dirname(os.path.abspath(program))
+ if not prog_dir.startswith(root_dir):
+ return str(repo_name)
+ project = repo_name
+ sub_path = os.path.relpath(prog_dir, root_dir)
+ if sub_path != ".":
+ project += "-" + sub_path
+ return str(project.replace(os.sep, "_"))
+
+
+# TODO(hugh): Deprecate version here and use wandb/sdk/lib/paths.py
+def to_forward_slash_path(path: str) -> str:
+ if platform.system() == "Windows":
+ path = path.replace("\\", "/")
+ return path
+
+
+# TODO(hugh): Deprecate version here and use wandb/sdk/lib/paths.py
+def to_native_slash_path(path: str) -> FilePathStr:
+ return FilePathStr(path.replace("/", os.sep))
+
+
+def check_and_warn_old(files: List[str]) -> bool:
+ if "wandb-metadata.json" in files:
+ wandb.termwarn("These runs were logged with a previous version of wandb.")
+ wandb.termwarn(
+ "Run pip install wandb<0.10.0 to get the old library and sync your runs."
+ )
+ return True
+ return False
+
+
+class ImportMetaHook:
+ def __init__(self) -> None:
+ self.modules: Dict[str, ModuleType] = dict()
+ self.on_import: Dict[str, list] = dict()
+
+ def add(self, fullname: str, on_import: Callable) -> None:
+ self.on_import.setdefault(fullname, []).append(on_import)
+
+ def install(self) -> None:
+ sys.meta_path.insert(0, self) # type: ignore
+
+ def uninstall(self) -> None:
+ sys.meta_path.remove(self) # type: ignore
+
+ def find_module(
+ self, fullname: str, path: Optional[str] = None
+ ) -> Optional["ImportMetaHook"]:
+ if fullname in self.on_import:
+ return self
+ return None
+
+ def load_module(self, fullname: str) -> ModuleType:
+ self.uninstall()
+ mod = importlib.import_module(fullname)
+ self.install()
+ self.modules[fullname] = mod
+ on_imports = self.on_import.get(fullname)
+ if on_imports:
+ for f in on_imports:
+ f()
+ return mod
+
+ def get_modules(self) -> Tuple[str, ...]:
+ return tuple(self.modules)
+
+ def get_module(self, module: str) -> ModuleType:
+ return self.modules[module]
+
+
+_import_hook: Optional[ImportMetaHook] = None
+
+
+def add_import_hook(fullname: str, on_import: Callable) -> None:
+ global _import_hook
+ if _import_hook is None:
+ _import_hook = ImportMetaHook()
+ _import_hook.install()
+ _import_hook.add(fullname, on_import)
+
+
+def host_from_path(path: Optional[str]) -> str:
+ """Return the host of the path."""
+ url = urllib.parse.urlparse(path)
+ return str(url.netloc)
+
+
+def uri_from_path(path: Optional[str]) -> str:
+ """Return the URI of the path."""
+ url = urllib.parse.urlparse(path)
+ uri = url.path if url.path[0] != "/" else url.path[1:]
+ return str(uri)
+
+
+def is_unicode_safe(stream: TextIO) -> bool:
+ """Return True if the stream supports UTF-8."""
+ encoding = getattr(stream, "encoding", None)
+ return encoding.lower() in {"utf-8", "utf_8"} if encoding else False
+
+
+def _has_internet() -> bool:
+ """Attempt to open a DNS connection to Googles root servers."""
+ try:
+ s = socket.create_connection(("8.8.8.8", 53), 0.5)
+ s.close()
+ return True
+ except OSError:
+ return False
+
+
+def rand_alphanumeric(
+ length: int = 8, rand: Optional[Union[ModuleType, random.Random]] = None
+) -> str:
+ wandb.termerror("rand_alphanumeric is deprecated, use 'secrets.token_hex'")
+ rand = rand or random
+ return "".join(rand.choice("0123456789ABCDEF") for _ in range(length))
+
+
+@contextlib.contextmanager
+def fsync_open(
+ path: StrPath, mode: str = "w", encoding: Optional[str] = None
+) -> Generator[IO[Any], None, None]:
+ """Open a path for I/O and guarantee that the file is flushed and synced."""
+ with open(path, mode, encoding=encoding) as f:
+ yield f
+
+ f.flush()
+ os.fsync(f.fileno())
+
+
+def _is_kaggle() -> bool:
+ return (
+ os.getenv("KAGGLE_KERNEL_RUN_TYPE") is not None
+ or "kaggle_environments" in sys.modules
+ )
+
+
+def _is_likely_kaggle() -> bool:
+ # Telemetry to mark first runs from Kagglers.
+ return (
+ _is_kaggle()
+ or os.path.exists(
+ os.path.expanduser(os.path.join("~", ".kaggle", "kaggle.json"))
+ )
+ or "kaggle" in sys.modules
+ )
+
+
+def _is_databricks() -> bool:
+ # check if we are running inside a databricks notebook by
+ # inspecting sys.modules, searching for dbutils and verifying that
+ # it has the appropriate structure
+
+ if "dbutils" in sys.modules:
+ dbutils = sys.modules["dbutils"]
+ if hasattr(dbutils, "shell"):
+ shell = dbutils.shell
+ if hasattr(shell, "sc"):
+ sc = shell.sc
+ if hasattr(sc, "appName"):
+ return bool(sc.appName == "Databricks Shell")
+ return False
+
+
+def _is_py_or_dockerfile(path: str) -> bool:
+ file = os.path.basename(path)
+ return file.endswith(".py") or file.startswith("Dockerfile")
+
+
+def check_windows_valid_filename(path: Union[int, str]) -> bool:
+ return not bool(re.search(RE_WINFNAMES, path)) # type: ignore
+
+
+def artifact_to_json(artifact: "Artifact") -> Dict[str, Any]:
+ return {
+ "_type": "artifactVersion",
+ "_version": "v0",
+ "id": artifact.id,
+ "version": artifact.source_version,
+ "sequenceName": artifact.source_name.split(":")[0],
+ "usedAs": artifact.use_as,
+ }
+
+
+def check_dict_contains_nested_artifact(d: dict, nested: bool = False) -> bool:
+ for item in d.values():
+ if isinstance(item, dict):
+ contains_artifacts = check_dict_contains_nested_artifact(item, True)
+ if contains_artifacts:
+ return True
+ elif (isinstance(item, wandb.Artifact) or _is_artifact_string(item)) and nested:
+ return True
+ return False
+
+
+def load_json_yaml_dict(config: str) -> Any:
+ ext = os.path.splitext(config)[-1]
+ if ext == ".json":
+ with open(config) as f:
+ return json.load(f)
+ elif ext == ".yaml":
+ with open(config) as f:
+ return yaml.safe_load(f)
+ else:
+ try:
+ return json.loads(config)
+ except ValueError:
+ return None
+
+
+def _parse_entity_project_item(path: str) -> tuple:
+ """Parse paths with the following formats: {item}, {project}/{item}, & {entity}/{project}/{item}.
+
+ Args:
+ path: `str`, input path; must be between 0 and 3 in length.
+
+ Returns:
+ tuple of length 3 - (item, project, entity)
+
+ Example:
+ alias, project, entity = _parse_entity_project_item("myproj/mymodel:best")
+
+ assert entity == ""
+ assert project == "myproj"
+ assert alias == "mymodel:best"
+
+ """
+ words = path.split("/")
+ if len(words) > 3:
+ raise ValueError(
+ "Invalid path: must be str the form {item}, {project}/{item}, or {entity}/{project}/{item}"
+ )
+ padded_words = [""] * (3 - len(words)) + words
+ return tuple(reversed(padded_words))
+
+
+def _resolve_aliases(aliases: Optional[Union[str, Iterable[str]]]) -> List[str]:
+ """Add the 'latest' alias and ensure that all aliases are unique.
+
+ Takes in `aliases` which can be None, str, or List[str] and returns List[str].
+ Ensures that "latest" is always present in the returned list.
+
+ Args:
+ aliases: `Optional[Union[str, List[str]]]`
+
+ Returns:
+ List[str], with "latest" always present.
+
+ Usage:
+
+ ```python
+ aliases = _resolve_aliases(["best", "dev"])
+ assert aliases == ["best", "dev", "latest"]
+
+ aliases = _resolve_aliases("boom")
+ assert aliases == ["boom", "latest"]
+ ```
+ """
+ aliases = aliases or ["latest"]
+
+ if isinstance(aliases, str):
+ aliases = [aliases]
+
+ try:
+ return list(set(aliases) | {"latest"})
+ except TypeError as exc:
+ raise ValueError("`aliases` must be Iterable or None") from exc
+
+
+def _is_artifact_object(v: Any) -> bool:
+ return isinstance(v, wandb.Artifact)
+
+
+def _is_artifact_string(v: Any) -> bool:
+ return isinstance(v, str) and v.startswith("wandb-artifact://")
+
+
+def _is_artifact_version_weave_dict(v: Any) -> bool:
+ return isinstance(v, dict) and v.get("_type") == "artifactVersion"
+
+
+def _is_artifact_representation(v: Any) -> bool:
+ return (
+ _is_artifact_object(v)
+ or _is_artifact_string(v)
+ or _is_artifact_version_weave_dict(v)
+ )
+
+
+def parse_artifact_string(v: str) -> Tuple[str, Optional[str], bool]:
+ if not v.startswith("wandb-artifact://"):
+ raise ValueError(f"Invalid artifact string: {v}")
+ parsed_v = v[len("wandb-artifact://") :]
+ base_uri = None
+ url_info = urllib.parse.urlparse(parsed_v)
+ if url_info.scheme != "":
+ base_uri = f"{url_info.scheme}://{url_info.netloc}"
+ parts = url_info.path.split("/")[1:]
+ else:
+ parts = parsed_v.split("/")
+ if parts[0] == "_id":
+ # for now can't fetch paths but this will be supported in the future
+ # when we allow passing typed media objects, this can be extended
+ # to include paths
+ return parts[1], base_uri, True
+
+ if len(parts) < 3:
+ raise ValueError(f"Invalid artifact string: {v}")
+
+ # for now can't fetch paths but this will be supported in the future
+ # when we allow passing typed media objects, this can be extended
+ # to include paths
+ entity, project, name_and_alias_or_version = parts[:3]
+ return f"{entity}/{project}/{name_and_alias_or_version}", base_uri, False
+
+
+def _get_max_cli_version() -> Union[str, None]:
+ max_cli_version = wandb.api.max_cli_version()
+ return str(max_cli_version) if max_cli_version is not None else None
+
+
+def _is_offline() -> bool:
+ return ( # type: ignore[no-any-return]
+ wandb.run is not None and wandb.run.settings._offline
+ ) or wandb.setup().settings._offline # type: ignore
+
+
+def ensure_text(
+ string: Union[str, bytes], encoding: str = "utf-8", errors: str = "strict"
+) -> str:
+ """Coerce s to str."""
+ if isinstance(string, bytes):
+ return string.decode(encoding, errors)
+ elif isinstance(string, str):
+ return string
+ else:
+ raise TypeError(f"not expecting type {type(string)!r}")
+
+
+def make_artifact_name_safe(name: str) -> str:
+ """Make an artifact name safe for use in artifacts."""
+ # artifact names may only contain alphanumeric characters, dashes, underscores, and dots.
+ cleaned = re.sub(r"[^a-zA-Z0-9_\-.]", "_", name)
+ if len(cleaned) <= 128:
+ return cleaned
+ # truncate with dots in the middle using regex
+ return re.sub(r"(^.{63}).*(.{63}$)", r"\g<1>..\g<2>", cleaned)
+
+
+def make_docker_image_name_safe(name: str) -> str:
+ """Make a docker image name safe for use in artifacts."""
+ safe_chars = RE_DOCKER_IMAGE_NAME_CHARS.sub("__", name.lower())
+ deduped = RE_DOCKER_IMAGE_NAME_SEPARATOR_REPEAT.sub("__", safe_chars)
+ trimmed_start = RE_DOCKER_IMAGE_NAME_SEPARATOR_START.sub("", deduped)
+ trimmed = RE_DOCKER_IMAGE_NAME_SEPARATOR_END.sub("", trimmed_start)
+ return trimmed if trimmed else "image"
+
+
+def merge_dicts(source: Dict[str, Any], destination: Dict[str, Any]) -> Dict[str, Any]:
+ """Recursively merge two dictionaries."""
+ for key, value in source.items():
+ if isinstance(value, dict):
+ # get node or create one
+ node = destination.setdefault(key, {})
+ merge_dicts(value, node)
+ else:
+ if isinstance(value, list):
+ if key in destination:
+ destination[key].extend(value)
+ else:
+ destination[key] = value
+ else:
+ destination[key] = value
+ return destination
+
+
+def coalesce(*arg: Any) -> Any:
+ """Return the first non-none value in the list of arguments.
+
+ Similar to ?? in C#.
+ """
+ return next((a for a in arg if a is not None), None)
+
+
+def cast_dictlike_to_dict(d: Dict[str, Any]) -> Dict[str, Any]:
+ for k, v in d.items():
+ if isinstance(v, dict):
+ cast_dictlike_to_dict(v)
+ elif hasattr(v, "keys"):
+ d[k] = dict(v)
+ cast_dictlike_to_dict(d[k])
+ return d
+
+
+def remove_keys_with_none_values(
+ d: Union[Dict[str, Any], Any]
+) -> Union[Dict[str, Any], Any]:
+ # otherwise iterrows will create a bunch of ugly charts
+ if not isinstance(d, dict):
+ return d
+
+ if isinstance(d, dict):
+ new_dict = {}
+ for k, v in d.items():
+ new_v = remove_keys_with_none_values(v)
+ if new_v is not None and not (isinstance(new_v, dict) and len(new_v) == 0):
+ new_dict[k] = new_v
+ return new_dict if new_dict else None
+
+
+def batched(n: int, iterable: Iterable[T]) -> Generator[List[T], None, None]:
+ i = iter(iterable)
+ batch = list(itertools.islice(i, n))
+ while batch:
+ yield batch
+ batch = list(itertools.islice(i, n))
+
+
+def random_string(length: int = 12) -> str:
+ """Generate a random string of a given length.
+
+ :param length: Length of the string to generate.
+ :return: Random string.
+ """
+ return "".join(
+ secrets.choice(string.ascii_lowercase + string.digits) for _ in range(length)
+ )
+
+
+def sample_with_exponential_decay_weights(
+ xs: Union[Iterable, Iterable[Iterable]],
+ ys: Iterable[Iterable],
+ keys: Optional[Iterable] = None,
+ sample_size: int = 1500,
+) -> Tuple[List, List, Optional[List]]:
+ """Sample from a list of lists with weights that decay exponentially.
+
+ May be used with the wandb.plot.line_series function.
+ """
+ xs_array = np.array(xs)
+ ys_array = np.array(ys)
+ keys_array = np.array(keys) if keys else None
+ weights = np.exp(-np.arange(len(xs_array)) / len(xs_array))
+ weights /= np.sum(weights)
+ sampled_indices = np.random.choice(len(xs_array), size=sample_size, p=weights)
+ sampled_xs = xs_array[sampled_indices].tolist()
+ sampled_ys = ys_array[sampled_indices].tolist()
+ sampled_keys = keys_array[sampled_indices].tolist() if keys else None
+
+ return sampled_xs, sampled_ys, sampled_keys
+
+
+def get_core_path() -> str:
+ core_path: str = os.environ.get("_WANDB_CORE_PATH", "")
+ wandb_core = get_module("wandb_core")
+ if not core_path and wandb_core:
+ _check_wandb_core_version_compatibility(wandb_core.__version__)
+ core_path = wandb_core.get_core_path()
+ return core_path
+
+
+def _check_wandb_core_version_compatibility(core_version: str) -> None:
+ """Checks if the installed wandb-core version is compatible with the wandb version."""
+ from pkg_resources import parse_version
+
+ if parse_version(core_version) < parse_version(wandb._minimum_core_version):
+ raise ImportError(
+ f"Requires wandb-core version {wandb._minimum_core_version} or later, "
+ f"but you have {core_version}. Run `pip install --upgrade wandb-core` to upgrade."
+ )
diff --git a/wandb/vendor/__init__.py b/wandb/vendor/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/vendor/gql-0.2.0/CODEOWNERS b/wandb/vendor/gql-0.2.0/CODEOWNERS
new file mode 100644
index 0000000000000000000000000000000000000000..44f0fc46985e8a67d232f76f1d9741ed07a5d8eb
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/CODEOWNERS
@@ -0,0 +1 @@
+/ @syrusakbary @ekampf @cito
\ No newline at end of file
diff --git a/wandb/vendor/gql-0.2.0/LICENSE b/wandb/vendor/gql-0.2.0/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..141776c3546b0a7ed487f1cdedffc5a14c14a34d
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/LICENSE
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 GraphQL Python
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/wandb/vendor/gql-0.2.0/MANIFEST.in b/wandb/vendor/gql-0.2.0/MANIFEST.in
new file mode 100644
index 0000000000000000000000000000000000000000..e9e681eb5abd54f2422514852c9c570b474a89b3
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/MANIFEST.in
@@ -0,0 +1,15 @@
+include MANIFEST.in
+
+include CODEOWNERS
+include LICENSE
+include README.md
+
+include dev_requirements.txt
+
+include tox.ini
+
+recursive-include tests *.py *.yaml
+
+prune gql-checker
+
+global-exclude *.py[co] __pycache__
diff --git a/wandb/vendor/gql-0.2.0/PKG-INFO b/wandb/vendor/gql-0.2.0/PKG-INFO
new file mode 100644
index 0000000000000000000000000000000000000000..2bd1708f5f7e08da9e6f5c9513029a54bc5708b7
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/PKG-INFO
@@ -0,0 +1,67 @@
+Metadata-Version: 2.1
+Name: gql
+Version: 0.2.0
+Summary: GraphQL client for Python
+Home-page: https://github.com/graphql-python/gql
+Author: Syrus Akbary
+Author-email: me@syrusakbary.com
+License: MIT
+Description: # GQL
+
+ This is a GraphQL client for Python.
+ Plays nicely with `graphene`, `graphql-core`, `graphql-js` and any other GraphQL implementation compatible with the spec.
+
+ GQL architecture is inspired by `React-Relay` and `Apollo-Client`.
+
+ [![travis][travis-image]][travis-url]
+ [![pypi][pypi-image]][pypi-url]
+ [![coveralls][coveralls-image]][coveralls-url]
+
+ [travis-image]: https://img.shields.io/travis/graphql-python/gql.svg?style=flat
+ [travis-url]: https://travis-ci.org/graphql-python/gql
+ [pypi-image]: https://img.shields.io/pypi/v/gql.svg?style=flat
+ [pypi-url]: https://pypi.python.org/pypi/gql
+ [coveralls-image]: https://coveralls.io/repos/graphql-python/gql/badge.svg?branch=master&service=github
+ [coveralls-url]: https://coveralls.io/github/graphql-python/gql?branch=master
+
+ ## Installation
+
+ $ pip install gql
+
+
+ ## Usage
+
+ The example below shows how you can execute queries against a local schema.
+
+
+ ```python
+ from gql import gql, Client
+
+ client = Client(schema=schema)
+ query = gql('''
+ {
+ hello
+ }
+ ''')
+
+ client.execute(query)
+ ```
+
+ ## License
+
+ [MIT License](https://github.com/graphql-python/gql/blob/master/LICENSE)
+
+Keywords: api graphql protocol rest relay gql client
+Platform: UNKNOWN
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Description-Content-Type: text/markdown
diff --git a/wandb/vendor/gql-0.2.0/README.md b/wandb/vendor/gql-0.2.0/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..69df1e992e2d2a81cca7ac156d5e1c6b987d263e
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/README.md
@@ -0,0 +1,44 @@
+# GQL
+
+This is a GraphQL client for Python.
+Plays nicely with `graphene`, `graphql-core`, `graphql-js` and any other GraphQL implementation compatible with the spec.
+
+GQL architecture is inspired by `React-Relay` and `Apollo-Client`.
+
+[![travis][travis-image]][travis-url]
+[![pypi][pypi-image]][pypi-url]
+[![coveralls][coveralls-image]][coveralls-url]
+
+[travis-image]: https://img.shields.io/travis/graphql-python/gql.svg?style=flat
+[travis-url]: https://travis-ci.org/graphql-python/gql
+[pypi-image]: https://img.shields.io/pypi/v/gql.svg?style=flat
+[pypi-url]: https://pypi.python.org/pypi/gql
+[coveralls-image]: https://coveralls.io/repos/graphql-python/gql/badge.svg?branch=master&service=github
+[coveralls-url]: https://coveralls.io/github/graphql-python/gql?branch=master
+
+## Installation
+
+ $ pip install gql
+
+
+## Usage
+
+The example below shows how you can execute queries against a local schema.
+
+
+```python
+from gql import gql, Client
+
+client = Client(schema=schema)
+query = gql('''
+{
+ hello
+}
+''')
+
+client.execute(query)
+```
+
+## License
+
+[MIT License](https://github.com/graphql-python/gql/blob/master/LICENSE)
diff --git a/wandb/vendor/gql-0.2.0/dev_requirements.txt b/wandb/vendor/gql-0.2.0/dev_requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ef3756e35822270d8a26ff02ad3acb38ead5ae48
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/dev_requirements.txt
@@ -0,0 +1,6 @@
+pytest>=3,<4
+pytest-cov>=2.8,<3
+coveralls>=1.8,<2
+flake8>=3.7,<4
+mock>=3,<4
+vcrpy>=2.1,<3
diff --git a/wandb/vendor/gql-0.2.0/gql.egg-info/PKG-INFO b/wandb/vendor/gql-0.2.0/gql.egg-info/PKG-INFO
new file mode 100644
index 0000000000000000000000000000000000000000..2bd1708f5f7e08da9e6f5c9513029a54bc5708b7
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/gql.egg-info/PKG-INFO
@@ -0,0 +1,67 @@
+Metadata-Version: 2.1
+Name: gql
+Version: 0.2.0
+Summary: GraphQL client for Python
+Home-page: https://github.com/graphql-python/gql
+Author: Syrus Akbary
+Author-email: me@syrusakbary.com
+License: MIT
+Description: # GQL
+
+ This is a GraphQL client for Python.
+ Plays nicely with `graphene`, `graphql-core`, `graphql-js` and any other GraphQL implementation compatible with the spec.
+
+ GQL architecture is inspired by `React-Relay` and `Apollo-Client`.
+
+ [![travis][travis-image]][travis-url]
+ [![pypi][pypi-image]][pypi-url]
+ [![coveralls][coveralls-image]][coveralls-url]
+
+ [travis-image]: https://img.shields.io/travis/graphql-python/gql.svg?style=flat
+ [travis-url]: https://travis-ci.org/graphql-python/gql
+ [pypi-image]: https://img.shields.io/pypi/v/gql.svg?style=flat
+ [pypi-url]: https://pypi.python.org/pypi/gql
+ [coveralls-image]: https://coveralls.io/repos/graphql-python/gql/badge.svg?branch=master&service=github
+ [coveralls-url]: https://coveralls.io/github/graphql-python/gql?branch=master
+
+ ## Installation
+
+ $ pip install gql
+
+
+ ## Usage
+
+ The example below shows how you can execute queries against a local schema.
+
+
+ ```python
+ from gql import gql, Client
+
+ client = Client(schema=schema)
+ query = gql('''
+ {
+ hello
+ }
+ ''')
+
+ client.execute(query)
+ ```
+
+ ## License
+
+ [MIT License](https://github.com/graphql-python/gql/blob/master/LICENSE)
+
+Keywords: api graphql protocol rest relay gql client
+Platform: UNKNOWN
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Description-Content-Type: text/markdown
diff --git a/wandb/vendor/gql-0.2.0/gql.egg-info/SOURCES.txt b/wandb/vendor/gql-0.2.0/gql.egg-info/SOURCES.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c4df8efa9dc31bc6ec515341433c83ce7b3ba8c6
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/gql.egg-info/SOURCES.txt
@@ -0,0 +1,33 @@
+CODEOWNERS
+LICENSE
+MANIFEST.in
+README.md
+dev_requirements.txt
+setup.cfg
+setup.py
+tox.ini
+gql/__init__.py
+gql/client.py
+gql/dsl.py
+gql/gql.py
+gql/utils.py
+gql.egg-info/PKG-INFO
+gql.egg-info/SOURCES.txt
+gql.egg-info/dependency_links.txt
+gql.egg-info/requires.txt
+gql.egg-info/top_level.txt
+gql/transport/__init__.py
+gql/transport/http.py
+gql/transport/local_schema.py
+gql/transport/requests.py
+tests/__init__.py
+tests/test_client.py
+tests/test_transport.py
+tests/fixtures/vcr_cassettes/client.yaml
+tests/fixtures/vcr_cassettes/execute.yaml
+tests/starwars/__init__.py
+tests/starwars/fixtures.py
+tests/starwars/schema.py
+tests/starwars/test_dsl.py
+tests/starwars/test_query.py
+tests/starwars/test_validation.py
\ No newline at end of file
diff --git a/wandb/vendor/gql-0.2.0/gql.egg-info/dependency_links.txt b/wandb/vendor/gql-0.2.0/gql.egg-info/dependency_links.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/gql.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
diff --git a/wandb/vendor/gql-0.2.0/gql.egg-info/requires.txt b/wandb/vendor/gql-0.2.0/gql.egg-info/requires.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6c4d4da4934ae26644bf147ff7a0159116c8f3f3
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/gql.egg-info/requires.txt
@@ -0,0 +1,4 @@
+six>=1.10.0
+graphql-core<2,>=0.5.0
+promise<3,>=2.0
+requests<3,>=2.12
diff --git a/wandb/vendor/gql-0.2.0/gql.egg-info/top_level.txt b/wandb/vendor/gql-0.2.0/gql.egg-info/top_level.txt
new file mode 100644
index 0000000000000000000000000000000000000000..113881889a7fc8fccc23e6dca8456a457a679030
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/gql.egg-info/top_level.txt
@@ -0,0 +1 @@
+gql
diff --git a/wandb/vendor/gql-0.2.0/setup.cfg b/wandb/vendor/gql-0.2.0/setup.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..b76a3311a1be64841d6aa6513c63e775758c155e
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/setup.cfg
@@ -0,0 +1,10 @@
+[flake8]
+max-line-length = 120
+
+[isort]
+known_first_party = gql
+
+[egg_info]
+tag_build =
+tag_date = 0
+
diff --git a/wandb/vendor/gql-0.2.0/setup.py b/wandb/vendor/gql-0.2.0/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..92482c03536227f47dcab140914feba539ff4d70
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/setup.py
@@ -0,0 +1,40 @@
+from setuptools import setup, find_packages
+
+setup(
+ name='gql',
+ version='0.2.0',
+ description='GraphQL client for Python',
+ long_description=open('README.md').read(),
+ long_description_content_type="text/markdown",
+ url='https://github.com/graphql-python/gql',
+ author='Syrus Akbary',
+ author_email='me@syrusakbary.com',
+ license='MIT',
+ classifiers=[
+ 'Development Status :: 3 - Alpha',
+ 'Intended Audience :: Developers',
+ 'Topic :: Software Development :: Libraries',
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.5',
+ 'Programming Language :: Python :: 3.6',
+ 'Programming Language :: Python :: 3.7',
+ 'Programming Language :: Python :: 3.8',
+ 'Programming Language :: Python :: Implementation :: PyPy',
+ ],
+ keywords='api graphql protocol rest relay gql client',
+ packages=find_packages(include=["gql*"]),
+ install_requires=[
+ 'six>=1.10.0',
+ 'graphql-core>=0.5.0,<2',
+ 'promise>=2.0,<3',
+ 'requests>=2.12,<3'
+ ],
+ tests_require=[
+ 'pytest>=3,<4',
+ 'pytest-cov>=2.8,<3',
+ 'mock>=3,<4',
+ 'vcrpy>=2.1,<3'
+ ],
+)
diff --git a/wandb/vendor/gql-0.2.0/tests/__init__.py b/wandb/vendor/gql-0.2.0/tests/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/vendor/gql-0.2.0/tests/fixtures/vcr_cassettes/client.yaml b/wandb/vendor/gql-0.2.0/tests/fixtures/vcr_cassettes/client.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b9b550f4ba4a7bc78416a2127b3a8306a5b1a01c
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/tests/fixtures/vcr_cassettes/client.yaml
@@ -0,0 +1,274 @@
+interactions:
+- request:
+ body: null
+ headers:
+ Accept:
+ - text/html
+ Accept-Encoding:
+ - gzip, deflate
+ Connection:
+ - keep-alive
+ Host:
+ - swapi.graphene-python.org
+ User-Agent:
+ - python-requests/2.22.0
+ method: GET
+ uri: http://127.0.0.1:8000/graphql
+ response:
+ body:
+ string: "<!--\nThe request to this GraphQL server provided the header \"Accept:
+ text/html\"\nand as a result has been presented GraphiQL - an in-browser IDE
+ for\nexploring GraphQL.\nIf you wish to receive JSON, provide the header \"Accept:
+ application/json\" or\nadd \"&raw\" to the end of the URL within a browser.\n-->\n<!DOCTYPE
+ html>\n<html>\n<head>\n <style>\n html, body {\n height: 100%;\n
+ \ margin: 0;\n overflow: hidden;\n width: 100%;\n }\n </style>\n
+ \ <link href=\"//cdn.jsdelivr.net/npm/graphiql@0.11.10/graphiql.css\" rel=\"stylesheet\"
+ />\n <script src=\"//cdn.jsdelivr.net/npm/whatwg-fetch@2.0.3/fetch.min.js\"></script>\n
+ \ <script src=\"//cdn.jsdelivr.net/npm/react@16.2.0/umd/react.production.min.js\"></script>\n
+ \ <script src=\"//cdn.jsdelivr.net/npm/react-dom@16.2.0/umd/react-dom.production.min.js\"></script>\n
+ \ <script src=\"//cdn.jsdelivr.net/npm/graphiql@0.11.10/graphiql.min.js\"></script>\n</head>\n<body>\n
+ \ <script>\n // Parse the cookie value for a CSRF token\n var csrftoken;\n
+ \ var cookies = ('; ' + document.cookie).split('; csrftoken=');\n if
+ (cookies.length == 2)\n csrftoken = cookies.pop().split(';').shift();\n\n
+ \ // Collect the URL parameters\n var parameters = {};\n window.location.search.substr(1).split('&').forEach(function
+ (entry) {\n var eq = entry.indexOf('=');\n if (eq >= 0) {\n parameters[decodeURIComponent(entry.slice(0,
+ eq))] =\n decodeURIComponent(entry.slice(eq + 1));\n }\n });\n
+ \ // Produce a Location query string from a parameter object.\n function
+ locationQuery(params) {\n return '?' + Object.keys(params).map(function
+ (key) {\n return encodeURIComponent(key) + '=' +\n encodeURIComponent(params[key]);\n
+ \ }).join('&');\n }\n // Derive a fetch URL from the current URL,
+ sans the GraphQL parameters.\n var graphqlParamNames = {\n query:
+ true,\n variables: true,\n operationName: true\n };\n var
+ otherParams = {};\n for (var k in parameters) {\n if (parameters.hasOwnProperty(k)
+ && graphqlParamNames[k] !== true) {\n otherParams[k] = parameters[k];\n
+ \ }\n }\n var fetchURL = locationQuery(otherParams);\n // Defines
+ a GraphQL fetcher using the fetch API.\n function graphQLFetcher(graphQLParams)
+ {\n var headers = {\n 'Accept': 'application/json',\n 'Content-Type':
+ 'application/json'\n };\n if (csrftoken) {\n headers['X-CSRFToken']
+ = csrftoken;\n }\n return fetch(fetchURL, {\n method: 'post',\n
+ \ headers: headers,\n body: JSON.stringify(graphQLParams),\n
+ \ credentials: 'include',\n }).then(function (response) {\n return
+ response.text();\n }).then(function (responseBody) {\n try {\n
+ \ return JSON.parse(responseBody);\n } catch (error) {\n return
+ responseBody;\n }\n });\n }\n // When the query and variables
+ string is edited, update the URL bar so\n // that it can be easily shared.\n
+ \ function onEditQuery(newQuery) {\n parameters.query = newQuery;\n
+ \ updateURL();\n }\n function onEditVariables(newVariables) {\n
+ \ parameters.variables = newVariables;\n updateURL();\n }\n function
+ onEditOperationName(newOperationName) {\n parameters.operationName =
+ newOperationName;\n updateURL();\n }\n function updateURL() {\n
+ \ history.replaceState(null, null, locationQuery(parameters));\n }\n
+ \ // Render <GraphiQL /> into the body.\n ReactDOM.render(\n React.createElement(GraphiQL,
+ {\n fetcher: graphQLFetcher,\n onEditQuery: onEditQuery,\n onEditVariables:
+ onEditVariables,\n onEditOperationName: onEditOperationName,\n query:
+ '',\n response: '',\n \n variables: 'null',\n \n
+ \ \n }),\n document.body\n );\n </script>\n</body>\n</html>\n"
+ headers:
+ Content-Length:
+ - '3808'
+ Content-Type:
+ - text/html; charset=utf-8
+ Date:
+ - Tue, 03 Dec 2019 08:22:54 GMT
+ Server:
+ - WSGIServer/0.1 Python/2.7.16
+ Set-Cookie:
+ - csrftoken=hRIez34v4hg2Wbl8XhrbshvDIB3HmLR2L9WNTJ3SdrIQHxAKtoukxiuwQlwRJewz;
+ expires=Tue, 01-Dec-2020 08:22:54 GMT; Max-Age=31449600; Path=/
+ Vary:
+ - Cookie
+ X-Frame-Options:
+ - SAMEORIGIN
+ status:
+ code: 200
+ message: OK
+- request:
+ body: query=query+IntrospectionQuery+%7B%0A++__schema+%7B%0A++++queryType+%7B%0A++++++name%0A++++%7D%0A++++mutationType+%7B%0A++++++name%0A++++%7D%0A++++subscriptionType+%7B%0A++++++name%0A++++%7D%0A++++types+%7B%0A++++++...FullType%0A++++%7D%0A++++directives+%7B%0A++++++name%0A++++++description%0A++++++locations%0A++++++args+%7B%0A++++++++...InputValue%0A++++++%7D%0A++++%7D%0A++%7D%0A%7D%0A%0Afragment+FullType+on+__Type+%7B%0A++kind%0A++name%0A++description%0A++fields%28includeDeprecated%3A+true%29+%7B%0A++++name%0A++++description%0A++++args+%7B%0A++++++...InputValue%0A++++%7D%0A++++type+%7B%0A++++++...TypeRef%0A++++%7D%0A++++isDeprecated%0A++++deprecationReason%0A++%7D%0A++inputFields+%7B%0A++++...InputValue%0A++%7D%0A++interfaces+%7B%0A++++...TypeRef%0A++%7D%0A++enumValues%28includeDeprecated%3A+true%29+%7B%0A++++name%0A++++description%0A++++isDeprecated%0A++++deprecationReason%0A++%7D%0A++possibleTypes+%7B%0A++++...TypeRef%0A++%7D%0A%7D%0A%0Afragment+InputValue+on+__InputValue+%7B%0A++name%0A++description%0A++type+%7B%0A++++...TypeRef%0A++%7D%0A++defaultValue%0A%7D%0A%0Afragment+TypeRef+on+__Type+%7B%0A++kind%0A++name%0A++ofType+%7B%0A++++kind%0A++++name%0A++++ofType+%7B%0A++++++kind%0A++++++name%0A++++++ofType+%7B%0A++++++++kind%0A++++++++name%0A++++++++ofType+%7B%0A++++++++++kind%0A++++++++++name%0A++++++++++ofType+%7B%0A++++++++++++kind%0A++++++++++++name%0A++++++++++++ofType+%7B%0A++++++++++++++kind%0A++++++++++++++name%0A++++++++++++++ofType+%7B%0A++++++++++++++++kind%0A++++++++++++++++name%0A++++++++++++++%7D%0A++++++++++++%7D%0A++++++++++%7D%0A++++++++%7D%0A++++++%7D%0A++++%7D%0A++%7D%0A%7D%0A
+ headers:
+ Accept:
+ - '*/*'
+ Accept-Encoding:
+ - gzip, deflate
+ Connection:
+ - keep-alive
+ Content-Length:
+ - '1625'
+ Content-Type:
+ - application/x-www-form-urlencoded
+ Cookie:
+ - csrftoken=hRIez34v4hg2Wbl8XhrbshvDIB3HmLR2L9WNTJ3SdrIQHxAKtoukxiuwQlwRJewz
+ User-Agent:
+ - python-requests/2.22.0
+ x-csrftoken:
+ - hRIez34v4hg2Wbl8XhrbshvDIB3HmLR2L9WNTJ3SdrIQHxAKtoukxiuwQlwRJewz
+ method: POST
+ uri: http://127.0.0.1:8000/graphql
+ response:
+ body:
+ string: '{"data":{"__schema":{"queryType":{"name":"Query"},"mutationType":{"name":"Mutation"},"subscriptionType":null,"types":[{"kind":"OBJECT","name":"Query","description":null,"fields":[{"name":"allFilms","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"episodeId_Gt","description":null,"type":{"kind":"SCALAR","name":"Float","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"FilmConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"allSpecies","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name_Startswith","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"name_Contains","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"SpecieConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"allCharacters","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"PersonConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"allVehicles","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name_Startswith","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"VehicleConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"allPlanets","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"PlanetConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"allStarships","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name_Startswith","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"name_Contains","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"StarshipConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"allHeroes","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name_Startswith","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"name_Contains","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"HeroConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"film","description":"The
+ ID of the object","args":[{"name":"id","description":null,"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"defaultValue":null}],"type":{"kind":"OBJECT","name":"Film","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"specie","description":"The
+ ID of the object","args":[{"name":"id","description":null,"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"defaultValue":null}],"type":{"kind":"OBJECT","name":"Specie","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"character","description":"The
+ ID of the object","args":[{"name":"id","description":null,"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"defaultValue":null}],"type":{"kind":"OBJECT","name":"Person","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"vehicle","description":"The
+ ID of the object","args":[{"name":"id","description":null,"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"defaultValue":null}],"type":{"kind":"OBJECT","name":"Vehicle","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"planet","description":"The
+ ID of the object","args":[{"name":"id","description":null,"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"defaultValue":null}],"type":{"kind":"OBJECT","name":"Planet","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"starship","description":"The
+ ID of the object","args":[{"name":"id","description":null,"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"defaultValue":null}],"type":{"kind":"OBJECT","name":"Starship","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"hero","description":"The
+ ID of the object","args":[{"name":"id","description":null,"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"defaultValue":null}],"type":{"kind":"OBJECT","name":"Hero","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"node","description":"The
+ ID of the object","args":[{"name":"id","description":null,"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"defaultValue":null}],"type":{"kind":"INTERFACE","name":"Node","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"viewer","description":null,"args":[],"type":{"kind":"OBJECT","name":"Query","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"__debug","description":null,"args":[],"type":{"kind":"OBJECT","name":"DjangoDebug","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"FilmConnection","description":null,"fields":[{"name":"pageInfo","description":"Pagination
+ data for this connection.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"PageInfo","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"edges","description":"Contains
+ the nodes in this connection.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"LIST","name":null,"ofType":{"kind":"OBJECT","name":"FilmEdge","ofType":null}}},"isDeprecated":false,"deprecationReason":null},{"name":"totalCount","description":null,"args":[],"type":{"kind":"SCALAR","name":"Int","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"PageInfo","description":"The
+ Relay compliant `PageInfo` type, containing data necessary to paginate this
+ connection.","fields":[{"name":"hasNextPage","description":"When paginating
+ forwards, are there more items?","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"Boolean","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"hasPreviousPage","description":"When
+ paginating backwards, are there more items?","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"Boolean","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"startCursor","description":"When
+ paginating backwards, the cursor to continue.","args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"endCursor","description":"When
+ paginating forwards, the cursor to continue.","args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"SCALAR","name":"Boolean","description":"The
+ `Boolean` scalar type represents `true` or `false`.","fields":null,"inputFields":null,"interfaces":null,"enumValues":null,"possibleTypes":null},{"kind":"SCALAR","name":"String","description":"The
+ `String` scalar type represents textual data, represented as UTF-8 character
+ sequences. The String type is most often used by GraphQL to represent free-form
+ human-readable text.","fields":null,"inputFields":null,"interfaces":null,"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"FilmEdge","description":"A
+ Relay edge containing a `Film` and its cursor.","fields":[{"name":"node","description":"The
+ item at the end of the edge","args":[],"type":{"kind":"OBJECT","name":"Film","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"cursor","description":"A
+ cursor for use in pagination","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"Film","description":"A
+ single film.","fields":[{"name":"id","description":"The ID of the object.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"title","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"episodeId","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"Int","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"openingCrawl","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"director","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"releaseDate","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"Date","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"characters","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"PersonConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"planets","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"PlanetConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"starships","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name_Startswith","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"name_Contains","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"StarshipConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"vehicles","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name_Startswith","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"VehicleConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"species","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name_Startswith","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"name_Contains","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"SpecieConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"producers","description":null,"args":[],"type":{"kind":"LIST","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[{"kind":"INTERFACE","name":"Node","ofType":null}],"enumValues":null,"possibleTypes":null},{"kind":"INTERFACE","name":"Node","description":"An
+ object with an ID","fields":[{"name":"id","description":"The ID of the object.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":null,"enumValues":null,"possibleTypes":[{"kind":"OBJECT","name":"Film","ofType":null},{"kind":"OBJECT","name":"Person","ofType":null},{"kind":"OBJECT","name":"Planet","ofType":null},{"kind":"OBJECT","name":"Specie","ofType":null},{"kind":"OBJECT","name":"Hero","ofType":null},{"kind":"OBJECT","name":"Starship","ofType":null},{"kind":"OBJECT","name":"Vehicle","ofType":null}]},{"kind":"SCALAR","name":"ID","description":"The
+ `ID` scalar type represents a unique identifier, often used to refetch an
+ object or as key for a cache. The ID type appears in a JSON response as a
+ String; however, it is not intended to be human-readable. When expected as
+ an input type, any string (such as `\"4\"`) or integer (such as `4`) input
+ value will be accepted as an ID.","fields":null,"inputFields":null,"interfaces":null,"enumValues":null,"possibleTypes":null},{"kind":"SCALAR","name":"Int","description":"The
+ `Int` scalar type represents non-fractional signed whole numeric values. Int
+ can represent values between -(2^31 - 1) and 2^31 - 1 since represented in
+ JSON as double-precision floating point numbers specifiedby [IEEE 754](http://en.wikipedia.org/wiki/IEEE_floating_point).","fields":null,"inputFields":null,"interfaces":null,"enumValues":null,"possibleTypes":null},{"kind":"SCALAR","name":"Date","description":"The
+ `Date` scalar type represents a Date\nvalue as specified by\n[iso8601](https://en.wikipedia.org/wiki/ISO_8601).","fields":null,"inputFields":null,"interfaces":null,"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"PersonConnection","description":null,"fields":[{"name":"pageInfo","description":"Pagination
+ data for this connection.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"PageInfo","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"edges","description":"Contains
+ the nodes in this connection.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"LIST","name":null,"ofType":{"kind":"OBJECT","name":"PersonEdge","ofType":null}}},"isDeprecated":false,"deprecationReason":null},{"name":"totalCount","description":null,"args":[],"type":{"kind":"SCALAR","name":"Int","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"PersonEdge","description":"A
+ Relay edge containing a `Person` and its cursor.","fields":[{"name":"node","description":"The
+ item at the end of the edge","args":[],"type":{"kind":"OBJECT","name":"Person","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"cursor","description":"A
+ cursor for use in pagination","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"Person","description":"An
+ individual person or character within the Star Wars universe.","fields":[{"name":"id","description":"The
+ ID of the object.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"name","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"height","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"mass","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"hairColor","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"skinColor","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"eyeColor","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"birthYear","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"gender","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"homeworld","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"Planet","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"species","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name_Startswith","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"name_Contains","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"SpecieConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"films","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"episodeId_Gt","description":null,"type":{"kind":"SCALAR","name":"Float","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"FilmConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"starships","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name_Startswith","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"name_Contains","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"StarshipConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"vehicles","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name_Startswith","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"VehicleConnection","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[{"kind":"INTERFACE","name":"Node","ofType":null}],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"Planet","description":"A
+ large mass, planet or planetoid in the Star Wars Universe,\nat the time of
+ 0 ABY.","fields":[{"name":"id","description":"The ID of the object.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"name","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"rotationPeriod","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"orbitalPeriod","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"diameter","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"gravity","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"surfaceWater","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"population","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"speciesSet","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name_Startswith","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"name_Contains","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"SpecieConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"films","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"episodeId_Gt","description":null,"type":{"kind":"SCALAR","name":"Float","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"FilmConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"heroes","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name_Startswith","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"name_Contains","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"HeroConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"residents","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"PersonConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"climates","description":null,"args":[],"type":{"kind":"LIST","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"terrains","description":null,"args":[],"type":{"kind":"LIST","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[{"kind":"INTERFACE","name":"Node","ofType":null}],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"SpecieConnection","description":null,"fields":[{"name":"pageInfo","description":"Pagination
+ data for this connection.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"PageInfo","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"edges","description":"Contains
+ the nodes in this connection.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"LIST","name":null,"ofType":{"kind":"OBJECT","name":"SpecieEdge","ofType":null}}},"isDeprecated":false,"deprecationReason":null},{"name":"totalCount","description":null,"args":[],"type":{"kind":"SCALAR","name":"Int","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"SpecieEdge","description":"A
+ Relay edge containing a `Specie` and its cursor.","fields":[{"name":"node","description":"The
+ item at the end of the edge","args":[],"type":{"kind":"OBJECT","name":"Specie","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"cursor","description":"A
+ cursor for use in pagination","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"Specie","description":"A
+ type of person or character within the Star Wars Universe.","fields":[{"name":"id","description":"The
+ ID of the object.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"name","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"classification","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"designation","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"averageHeight","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"averageLifespan","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"homeworld","description":"","args":[],"type":{"kind":"OBJECT","name":"Planet","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"language","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"people","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"PersonConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"films","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"episodeId_Gt","description":null,"type":{"kind":"SCALAR","name":"Float","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"FilmConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"eyeColors","description":null,"args":[],"type":{"kind":"LIST","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"hairColors","description":null,"args":[],"type":{"kind":"LIST","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"skinColors","description":null,"args":[],"type":{"kind":"LIST","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[{"kind":"INTERFACE","name":"Node","ofType":null}],"enumValues":null,"possibleTypes":null},{"kind":"SCALAR","name":"Float","description":"The
+ `Float` scalar type represents signed double-precision fractional values as
+ specified by [IEEE 754](http://en.wikipedia.org/wiki/IEEE_floating_point).
+ ","fields":null,"inputFields":null,"interfaces":null,"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"HeroConnection","description":null,"fields":[{"name":"pageInfo","description":"Pagination
+ data for this connection.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"PageInfo","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"edges","description":"Contains
+ the nodes in this connection.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"LIST","name":null,"ofType":{"kind":"OBJECT","name":"HeroEdge","ofType":null}}},"isDeprecated":false,"deprecationReason":null},{"name":"totalCount","description":null,"args":[],"type":{"kind":"SCALAR","name":"Int","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"HeroEdge","description":"A
+ Relay edge containing a `Hero` and its cursor.","fields":[{"name":"node","description":"The
+ item at the end of the edge","args":[],"type":{"kind":"OBJECT","name":"Hero","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"cursor","description":"A
+ cursor for use in pagination","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"Hero","description":"A
+ hero created by fans","fields":[{"name":"id","description":"The ID of the
+ object.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"name","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"homeworld","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"Planet","ofType":null}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[{"kind":"INTERFACE","name":"Node","ofType":null}],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"StarshipConnection","description":null,"fields":[{"name":"pageInfo","description":"Pagination
+ data for this connection.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"PageInfo","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"edges","description":"Contains
+ the nodes in this connection.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"LIST","name":null,"ofType":{"kind":"OBJECT","name":"StarshipEdge","ofType":null}}},"isDeprecated":false,"deprecationReason":null},{"name":"totalCount","description":null,"args":[],"type":{"kind":"SCALAR","name":"Int","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"StarshipEdge","description":"A
+ Relay edge containing a `Starship` and its cursor.","fields":[{"name":"node","description":"The
+ item at the end of the edge","args":[],"type":{"kind":"OBJECT","name":"Starship","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"cursor","description":"A
+ cursor for use in pagination","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"Starship","description":"A
+ single transport craft that has hyperdrive capability.","fields":[{"name":"id","description":"The
+ ID of the object.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"name","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"model","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"manufacturer","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"costInCredits","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"length","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"maxAtmospheringSpeed","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"crew","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"passengers","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"cargoCapacity","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"consumables","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"hyperdriveRating","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"MGLT","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"starshipClass","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"pilots","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"PersonConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"films","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"episodeId_Gt","description":null,"type":{"kind":"SCALAR","name":"Float","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"FilmConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"manufacturers","description":null,"args":[],"type":{"kind":"LIST","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[{"kind":"INTERFACE","name":"Node","ofType":null}],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"VehicleConnection","description":null,"fields":[{"name":"pageInfo","description":"Pagination
+ data for this connection.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"PageInfo","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"edges","description":"Contains
+ the nodes in this connection.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"LIST","name":null,"ofType":{"kind":"OBJECT","name":"VehicleEdge","ofType":null}}},"isDeprecated":false,"deprecationReason":null},{"name":"totalCount","description":null,"args":[],"type":{"kind":"SCALAR","name":"Int","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"VehicleEdge","description":"A
+ Relay edge containing a `Vehicle` and its cursor.","fields":[{"name":"node","description":"The
+ item at the end of the edge","args":[],"type":{"kind":"OBJECT","name":"Vehicle","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"cursor","description":"A
+ cursor for use in pagination","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"Vehicle","description":"A
+ single transport craft that does not have hyperdrive capability","fields":[{"name":"id","description":"The
+ ID of the object.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"ID","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"name","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"model","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"manufacturer","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"costInCredits","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"length","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"maxAtmospheringSpeed","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"crew","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"passengers","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"cargoCapacity","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"consumables","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"vehicleClass","description":"","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"pilots","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"name","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"PersonConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"films","description":null,"args":[{"name":"before","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"after","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null},{"name":"first","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"last","description":null,"type":{"kind":"SCALAR","name":"Int","ofType":null},"defaultValue":null},{"name":"episodeId_Gt","description":null,"type":{"kind":"SCALAR","name":"Float","ofType":null},"defaultValue":null}],"type":{"kind":"OBJECT","name":"FilmConnection","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"manufacturers","description":null,"args":[],"type":{"kind":"LIST","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[{"kind":"INTERFACE","name":"Node","ofType":null}],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"PlanetConnection","description":null,"fields":[{"name":"pageInfo","description":"Pagination
+ data for this connection.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"PageInfo","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"edges","description":"Contains
+ the nodes in this connection.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"LIST","name":null,"ofType":{"kind":"OBJECT","name":"PlanetEdge","ofType":null}}},"isDeprecated":false,"deprecationReason":null},{"name":"totalCount","description":null,"args":[],"type":{"kind":"SCALAR","name":"Int","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"PlanetEdge","description":"A
+ Relay edge containing a `Planet` and its cursor.","fields":[{"name":"node","description":"The
+ item at the end of the edge","args":[],"type":{"kind":"OBJECT","name":"Planet","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"cursor","description":"A
+ cursor for use in pagination","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"DjangoDebug","description":null,"fields":[{"name":"sql","description":null,"args":[],"type":{"kind":"LIST","name":null,"ofType":{"kind":"OBJECT","name":"DjangoDebugSQL","ofType":null}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"DjangoDebugSQL","description":null,"fields":[{"name":"vendor","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"alias","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"sql","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"duration","description":null,"args":[],"type":{"kind":"SCALAR","name":"Float","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"rawSql","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"params","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"startTime","description":null,"args":[],"type":{"kind":"SCALAR","name":"Float","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"stopTime","description":null,"args":[],"type":{"kind":"SCALAR","name":"Float","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"isSlow","description":null,"args":[],"type":{"kind":"SCALAR","name":"Boolean","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"isSelect","description":null,"args":[],"type":{"kind":"SCALAR","name":"Boolean","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"transId","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"transStatus","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"isoLevel","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"encoding","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"Mutation","description":null,"fields":[{"name":"createHero","description":null,"args":[{"name":"input","description":null,"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"INPUT_OBJECT","name":"CreateHeroInput","ofType":null}},"defaultValue":null}],"type":{"kind":"OBJECT","name":"CreateHeroPayload","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"CreateHeroPayload","description":null,"fields":[{"name":"hero","description":null,"args":[],"type":{"kind":"OBJECT","name":"Hero","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"ok","description":null,"args":[],"type":{"kind":"SCALAR","name":"Boolean","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"clientMutationId","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"INPUT_OBJECT","name":"CreateHeroInput","description":null,"fields":null,"inputFields":[{"name":"name","description":null,"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"defaultValue":null},{"name":"homeworldId","description":null,"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"defaultValue":null},{"name":"clientMutationId","description":null,"type":{"kind":"SCALAR","name":"String","ofType":null},"defaultValue":null}],"interfaces":null,"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"__Schema","description":"A
+ GraphQL Schema defines the capabilities of a GraphQL server. It exposes all
+ available types and directives on the server, as well as the entry points
+ for query, mutation and subscription operations.","fields":[{"name":"types","description":"A
+ list of all types supported by this server.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"LIST","name":null,"ofType":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"__Type","ofType":null}}}},"isDeprecated":false,"deprecationReason":null},{"name":"queryType","description":"The
+ type that query operations will be rooted at.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"__Type","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"mutationType","description":"If
+ this server supports mutation, the type that mutation operations will be rooted
+ at.","args":[],"type":{"kind":"OBJECT","name":"__Type","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"subscriptionType","description":"If
+ this server support subscription, the type that subscription operations will
+ be rooted at.","args":[],"type":{"kind":"OBJECT","name":"__Type","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"directives","description":"A
+ list of all directives supported by this server.","args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"LIST","name":null,"ofType":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"__Directive","ofType":null}}}},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"__Type","description":"The
+ fundamental unit of any GraphQL Schema is the type. There are many kinds of
+ types in GraphQL as represented by the `__TypeKind` enum.\n\nDepending on
+ the kind of a type, certain fields describe information about that type. Scalar
+ types provide no information beyond a name and description, while Enum types
+ provide their values. Object and Interface types provide the fields they describe.
+ Abstract types, Union and Interface, provide the Object types possible at
+ runtime. List and NonNull types compose other types.","fields":[{"name":"kind","description":null,"args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"ENUM","name":"__TypeKind","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"name","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"description","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"fields","description":null,"args":[{"name":"includeDeprecated","description":null,"type":{"kind":"SCALAR","name":"Boolean","ofType":null},"defaultValue":"false"}],"type":{"kind":"LIST","name":null,"ofType":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"__Field","ofType":null}}},"isDeprecated":false,"deprecationReason":null},{"name":"interfaces","description":null,"args":[],"type":{"kind":"LIST","name":null,"ofType":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"__Type","ofType":null}}},"isDeprecated":false,"deprecationReason":null},{"name":"possibleTypes","description":null,"args":[],"type":{"kind":"LIST","name":null,"ofType":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"__Type","ofType":null}}},"isDeprecated":false,"deprecationReason":null},{"name":"enumValues","description":null,"args":[{"name":"includeDeprecated","description":null,"type":{"kind":"SCALAR","name":"Boolean","ofType":null},"defaultValue":"false"}],"type":{"kind":"LIST","name":null,"ofType":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"__EnumValue","ofType":null}}},"isDeprecated":false,"deprecationReason":null},{"name":"inputFields","description":null,"args":[],"type":{"kind":"LIST","name":null,"ofType":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"__InputValue","ofType":null}}},"isDeprecated":false,"deprecationReason":null},{"name":"ofType","description":null,"args":[],"type":{"kind":"OBJECT","name":"__Type","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"ENUM","name":"__TypeKind","description":"An
+ enum describing what kind of type a given `__Type` is","fields":null,"inputFields":null,"interfaces":null,"enumValues":[{"name":"SCALAR","description":"Indicates
+ this type is a scalar.","isDeprecated":false,"deprecationReason":null},{"name":"OBJECT","description":"Indicates
+ this type is an object. `fields` and `interfaces` are valid fields.","isDeprecated":false,"deprecationReason":null},{"name":"INTERFACE","description":"Indicates
+ this type is an interface. `fields` and `possibleTypes` are valid fields.","isDeprecated":false,"deprecationReason":null},{"name":"UNION","description":"Indicates
+ this type is a union. `possibleTypes` is a valid field.","isDeprecated":false,"deprecationReason":null},{"name":"ENUM","description":"Indicates
+ this type is an enum. `enumValues` is a valid field.","isDeprecated":false,"deprecationReason":null},{"name":"INPUT_OBJECT","description":"Indicates
+ this type is an input object. `inputFields` is a valid field.","isDeprecated":false,"deprecationReason":null},{"name":"LIST","description":"Indicates
+ this type is a list. `ofType` is a valid field.","isDeprecated":false,"deprecationReason":null},{"name":"NON_NULL","description":"Indicates
+ this type is a non-null. `ofType` is a valid field.","isDeprecated":false,"deprecationReason":null}],"possibleTypes":null},{"kind":"OBJECT","name":"__Field","description":"Object
+ and Interface types are described by a list of Fields, each of which has a
+ name, potentially a list of arguments, and a return type.","fields":[{"name":"name","description":null,"args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"description","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"args","description":null,"args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"LIST","name":null,"ofType":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"__InputValue","ofType":null}}}},"isDeprecated":false,"deprecationReason":null},{"name":"type","description":null,"args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"__Type","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"isDeprecated","description":null,"args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"Boolean","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"deprecationReason","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"__InputValue","description":"Arguments
+ provided to Fields or Directives and the input fields of an InputObject are
+ represented as Input Values which describe their type and optionally a default
+ value.","fields":[{"name":"name","description":null,"args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"description","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"type","description":null,"args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"__Type","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"defaultValue","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"__EnumValue","description":"One
+ possible value for a given Enum. Enum values are unique values, not a placeholder
+ for a string or numeric value. However an Enum value is returned in a JSON
+ response as a string.","fields":[{"name":"name","description":null,"args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"description","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"isDeprecated","description":null,"args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"Boolean","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"deprecationReason","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"OBJECT","name":"__Directive","description":"A
+ Directive provides a way to describe alternate runtime execution and type
+ validation behavior in a GraphQL document.\n\nIn some cases, you need to provide
+ options to alter GraphQL''s execution behavior in ways field arguments will
+ not suffice, such as conditionally including or skipping a field. Directives
+ provide this by describing additional information to the executor.","fields":[{"name":"name","description":null,"args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"String","ofType":null}},"isDeprecated":false,"deprecationReason":null},{"name":"description","description":null,"args":[],"type":{"kind":"SCALAR","name":"String","ofType":null},"isDeprecated":false,"deprecationReason":null},{"name":"locations","description":null,"args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"LIST","name":null,"ofType":{"kind":"NON_NULL","name":null,"ofType":{"kind":"ENUM","name":"__DirectiveLocation","ofType":null}}}},"isDeprecated":false,"deprecationReason":null},{"name":"args","description":null,"args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"LIST","name":null,"ofType":{"kind":"NON_NULL","name":null,"ofType":{"kind":"OBJECT","name":"__InputValue","ofType":null}}}},"isDeprecated":false,"deprecationReason":null},{"name":"onOperation","description":null,"args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"Boolean","ofType":null}},"isDeprecated":true,"deprecationReason":"Use
+ `locations`."},{"name":"onFragment","description":null,"args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"Boolean","ofType":null}},"isDeprecated":true,"deprecationReason":"Use
+ `locations`."},{"name":"onField","description":null,"args":[],"type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"Boolean","ofType":null}},"isDeprecated":true,"deprecationReason":"Use
+ `locations`."}],"inputFields":null,"interfaces":[],"enumValues":null,"possibleTypes":null},{"kind":"ENUM","name":"__DirectiveLocation","description":"A
+ Directive can be adjacent to many parts of the GraphQL language, a __DirectiveLocation
+ describes one such possible adjacencies.","fields":null,"inputFields":null,"interfaces":null,"enumValues":[{"name":"QUERY","description":"Location
+ adjacent to a query operation.","isDeprecated":false,"deprecationReason":null},{"name":"MUTATION","description":"Location
+ adjacent to a mutation operation.","isDeprecated":false,"deprecationReason":null},{"name":"SUBSCRIPTION","description":"Location
+ adjacent to a subscription operation.","isDeprecated":false,"deprecationReason":null},{"name":"FIELD","description":"Location
+ adjacent to a field.","isDeprecated":false,"deprecationReason":null},{"name":"FRAGMENT_DEFINITION","description":"Location
+ adjacent to a fragment definition.","isDeprecated":false,"deprecationReason":null},{"name":"FRAGMENT_SPREAD","description":"Location
+ adjacent to a fragment spread.","isDeprecated":false,"deprecationReason":null},{"name":"INLINE_FRAGMENT","description":"Location
+ adjacent to an inline fragment.","isDeprecated":false,"deprecationReason":null},{"name":"SCHEMA","description":"Location
+ adjacent to a schema definition.","isDeprecated":false,"deprecationReason":null},{"name":"SCALAR","description":"Location
+ adjacent to a scalar definition.","isDeprecated":false,"deprecationReason":null},{"name":"OBJECT","description":"Location
+ adjacent to an object definition.","isDeprecated":false,"deprecationReason":null},{"name":"FIELD_DEFINITION","description":"Location
+ adjacent to a field definition.","isDeprecated":false,"deprecationReason":null},{"name":"ARGUMENT_DEFINITION","description":"Location
+ adjacent to an argument definition.","isDeprecated":false,"deprecationReason":null},{"name":"INTERFACE","description":"Location
+ adjacent to an interface definition.","isDeprecated":false,"deprecationReason":null},{"name":"UNION","description":"Location
+ adjacent to a union definition.","isDeprecated":false,"deprecationReason":null},{"name":"ENUM","description":"Location
+ adjacent to an enum definition.","isDeprecated":false,"deprecationReason":null},{"name":"ENUM_VALUE","description":"Location
+ adjacent to an enum value definition.","isDeprecated":false,"deprecationReason":null},{"name":"INPUT_OBJECT","description":"Location
+ adjacent to an input object definition.","isDeprecated":false,"deprecationReason":null},{"name":"INPUT_FIELD_DEFINITION","description":"Location
+ adjacent to an input object field definition.","isDeprecated":false,"deprecationReason":null}],"possibleTypes":null}],"directives":[{"name":"include","description":"Directs
+ the executor to include this field or fragment only when the `if` argument
+ is true.","locations":["FIELD","FRAGMENT_SPREAD","INLINE_FRAGMENT"],"args":[{"name":"if","description":"Included
+ when true.","type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"Boolean","ofType":null}},"defaultValue":null}]},{"name":"skip","description":"Directs
+ the executor to skip this field or fragment when the `if` argument is true.","locations":["FIELD","FRAGMENT_SPREAD","INLINE_FRAGMENT"],"args":[{"name":"if","description":"Skipped
+ when true.","type":{"kind":"NON_NULL","name":null,"ofType":{"kind":"SCALAR","name":"Boolean","ofType":null}},"defaultValue":null}]}]}}}'
+ headers:
+ Content-Length:
+ - '69554'
+ Content-Type:
+ - application/json
+ Date:
+ - Tue, 03 Dec 2019 08:22:54 GMT
+ Server:
+ - WSGIServer/0.1 Python/2.7.16
+ Set-Cookie:
+ - csrftoken=hRIez34v4hg2Wbl8XhrbshvDIB3HmLR2L9WNTJ3SdrIQHxAKtoukxiuwQlwRJewz;
+ expires=Tue, 01-Dec-2020 08:22:54 GMT; Max-Age=31449600; Path=/
+ Vary:
+ - Cookie
+ X-Frame-Options:
+ - SAMEORIGIN
+ status:
+ code: 200
+ message: OK
+version: 1
diff --git a/wandb/vendor/gql-0.2.0/tests/fixtures/vcr_cassettes/execute.yaml b/wandb/vendor/gql-0.2.0/tests/fixtures/vcr_cassettes/execute.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a6728a334ad1233fbcaec45312298cac47a6722f
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/tests/fixtures/vcr_cassettes/execute.yaml
@@ -0,0 +1,47 @@
+interactions:
+- request:
+ body: query=%7B%0A++myFavoriteFilm%3A+film%28id%3A+%22RmlsbToz%22%29+%7B%0A++++id%0A++++title%0A++++episodeId%0A++++characters%28first%3A+5%29+%7B%0A++++++edges+%7B%0A++++++++node+%7B%0A++++++++++name%0A++++++++%7D%0A++++++%7D%0A++++%7D%0A++%7D%0A%7D%0A
+ headers:
+ Accept:
+ - '*/*'
+ Accept-Encoding:
+ - gzip, deflate
+ Connection:
+ - keep-alive
+ Content-Length:
+ - '247'
+ Content-Type:
+ - application/x-www-form-urlencoded
+ Cookie:
+ - csrftoken=hRIez34v4hg2Wbl8XhrbshvDIB3HmLR2L9WNTJ3SdrIQHxAKtoukxiuwQlwRJewz
+ User-Agent:
+ - python-requests/2.22.0
+ x-csrftoken:
+ - hRIez34v4hg2Wbl8XhrbshvDIB3HmLR2L9WNTJ3SdrIQHxAKtoukxiuwQlwRJewz
+ method: POST
+ uri: http://127.0.0.1:8000/graphql
+ response:
+ body:
+ string: '{"data":{"myFavoriteFilm":{"id":"RmlsbToz","title":"Return of the Jedi","episodeId":6,"characters":{"edges":[{"node":{"name":"Luke
+ Skywalker"}},{"node":{"name":"C-3PO"}},{"node":{"name":"R2-D2"}},{"node":{"name":"Darth
+ Vader"}},{"node":{"name":"Leia Organa"}}]}}}}'
+ headers:
+ Content-Length:
+ - '264'
+ Content-Type:
+ - application/json
+ Date:
+ - Tue, 03 Dec 2019 08:23:58 GMT
+ Server:
+ - WSGIServer/0.1 Python/2.7.16
+ Set-Cookie:
+ - csrftoken=hRIez34v4hg2Wbl8XhrbshvDIB3HmLR2L9WNTJ3SdrIQHxAKtoukxiuwQlwRJewz;
+ expires=Tue, 01-Dec-2020 08:23:58 GMT; Max-Age=31449600; Path=/
+ Vary:
+ - Cookie
+ X-Frame-Options:
+ - SAMEORIGIN
+ status:
+ code: 200
+ message: OK
+version: 1
diff --git a/wandb/vendor/gql-0.2.0/tests/starwars/__init__.py b/wandb/vendor/gql-0.2.0/tests/starwars/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/vendor/gql-0.2.0/tests/starwars/fixtures.py b/wandb/vendor/gql-0.2.0/tests/starwars/fixtures.py
new file mode 100644
index 0000000000000000000000000000000000000000..51f29a59248cdeee13a7fc34b1c92154a0977e81
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/tests/starwars/fixtures.py
@@ -0,0 +1,96 @@
+from collections import namedtuple
+
+Human = namedtuple('Human', 'id name friends appearsIn homePlanet')
+
+luke = Human(
+ id='1000',
+ name='Luke Skywalker',
+ friends=['1002', '1003', '2000', '2001'],
+ appearsIn=[4, 5, 6],
+ homePlanet='Tatooine',
+)
+
+vader = Human(
+ id='1001',
+ name='Darth Vader',
+ friends=['1004'],
+ appearsIn=[4, 5, 6],
+ homePlanet='Tatooine',
+)
+
+han = Human(
+ id='1002',
+ name='Han Solo',
+ friends=['1000', '1003', '2001'],
+ appearsIn=[4, 5, 6],
+ homePlanet=None,
+)
+
+leia = Human(
+ id='1003',
+ name='Leia Organa',
+ friends=['1000', '1002', '2000', '2001'],
+ appearsIn=[4, 5, 6],
+ homePlanet='Alderaan',
+)
+
+tarkin = Human(
+ id='1004',
+ name='Wilhuff Tarkin',
+ friends=['1001'],
+ appearsIn=[4],
+ homePlanet=None,
+)
+
+humanData = {
+ '1000': luke,
+ '1001': vader,
+ '1002': han,
+ '1003': leia,
+ '1004': tarkin,
+}
+
+Droid = namedtuple('Droid', 'id name friends appearsIn primaryFunction')
+
+threepio = Droid(
+ id='2000',
+ name='C-3PO',
+ friends=['1000', '1002', '1003', '2001'],
+ appearsIn=[4, 5, 6],
+ primaryFunction='Protocol',
+)
+
+artoo = Droid(
+ id='2001',
+ name='R2-D2',
+ friends=['1000', '1002', '1003'],
+ appearsIn=[4, 5, 6],
+ primaryFunction='Astromech',
+)
+
+droidData = {
+ '2000': threepio,
+ '2001': artoo,
+}
+
+
+def getCharacter(id):
+ return humanData.get(id) or droidData.get(id)
+
+
+def getFriends(character):
+ return map(getCharacter, character.friends)
+
+
+def getHero(episode):
+ if episode == 5:
+ return luke
+ return artoo
+
+
+def getHuman(id):
+ return humanData.get(id)
+
+
+def getDroid(id):
+ return droidData.get(id)
diff --git a/wandb/vendor/gql-0.2.0/tests/starwars/schema.py b/wandb/vendor/gql-0.2.0/tests/starwars/schema.py
new file mode 100644
index 0000000000000000000000000000000000000000..da2d88d8cb674ddde76b593420e2405d39c53c4d
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/tests/starwars/schema.py
@@ -0,0 +1,146 @@
+from graphql.type import (GraphQLArgument, GraphQLEnumType, GraphQLEnumValue,
+ GraphQLField, GraphQLInterfaceType, GraphQLList,
+ GraphQLNonNull, GraphQLObjectType, GraphQLSchema,
+ GraphQLString)
+
+from .fixtures import getDroid, getFriends, getHero, getHuman
+
+episodeEnum = GraphQLEnumType(
+ 'Episode',
+ description='One of the films in the Star Wars Trilogy',
+ values={
+ 'NEWHOPE': GraphQLEnumValue(
+ 4,
+ description='Released in 1977.',
+ ),
+ 'EMPIRE': GraphQLEnumValue(
+ 5,
+ description='Released in 1980.',
+ ),
+ 'JEDI': GraphQLEnumValue(
+ 6,
+ description='Released in 1983.',
+ )
+ }
+)
+
+characterInterface = GraphQLInterfaceType(
+ 'Character',
+ description='A character in the Star Wars Trilogy',
+ fields=lambda: {
+ 'id': GraphQLField(
+ GraphQLNonNull(GraphQLString),
+ description='The id of the character.'
+ ),
+ 'name': GraphQLField(
+ GraphQLString,
+ description='The name of the character.'
+ ),
+ 'friends': GraphQLField(
+ GraphQLList(characterInterface),
+ description='The friends of the character, or an empty list if they have none.'
+ ),
+ 'appearsIn': GraphQLField(
+ GraphQLList(episodeEnum),
+ description='Which movies they appear in.'
+ ),
+ },
+ resolve_type=lambda character, *_: humanType if getHuman(character.id) else droidType,
+)
+
+humanType = GraphQLObjectType(
+ 'Human',
+ description='A humanoid creature in the Star Wars universe.',
+ fields=lambda: {
+ 'id': GraphQLField(
+ GraphQLNonNull(GraphQLString),
+ description='The id of the human.',
+ ),
+ 'name': GraphQLField(
+ GraphQLString,
+ description='The name of the human.',
+ ),
+ 'friends': GraphQLField(
+ GraphQLList(characterInterface),
+ description='The friends of the human, or an empty list if they have none.',
+ resolver=lambda human, *_: getFriends(human),
+ ),
+ 'appearsIn': GraphQLField(
+ GraphQLList(episodeEnum),
+ description='Which movies they appear in.',
+ ),
+ 'homePlanet': GraphQLField(
+ GraphQLString,
+ description='The home planet of the human, or null if unknown.',
+ )
+ },
+ interfaces=[characterInterface]
+)
+
+droidType = GraphQLObjectType(
+ 'Droid',
+ description='A mechanical creature in the Star Wars universe.',
+ fields=lambda: {
+ 'id': GraphQLField(
+ GraphQLNonNull(GraphQLString),
+ description='The id of the droid.',
+ ),
+ 'name': GraphQLField(
+ GraphQLString,
+ description='The name of the droid.',
+ ),
+ 'friends': GraphQLField(
+ GraphQLList(characterInterface),
+ description='The friends of the droid, or an empty list if they have none.',
+ resolver=lambda droid, *_: getFriends(droid),
+ ),
+ 'appearsIn': GraphQLField(
+ GraphQLList(episodeEnum),
+ description='Which movies they appear in.',
+ ),
+ 'primaryFunction': GraphQLField(
+ GraphQLString,
+ description='The primary function of the droid.',
+ )
+ },
+ interfaces=[characterInterface]
+)
+
+queryType = GraphQLObjectType(
+ 'Query',
+ fields=lambda: {
+ 'hero': GraphQLField(
+ characterInterface,
+ args={
+ 'episode': GraphQLArgument(
+ description='If omitted, returns the hero of the whole saga. If '
+ 'provided, returns the hero of that particular episode.',
+ type=episodeEnum,
+ )
+ },
+ resolver=lambda root, args, *_: getHero(args.get('episode')),
+ ),
+ 'human': GraphQLField(
+ humanType,
+ args={
+ 'id': GraphQLArgument(
+ description='id of the human',
+ type=GraphQLNonNull(GraphQLString),
+ )
+ },
+ resolver=lambda root, args, *_: getHuman(args['id']),
+ ),
+ 'droid': GraphQLField(
+ droidType,
+ args={
+ 'id': GraphQLArgument(
+ description='id of the droid',
+ type=GraphQLNonNull(GraphQLString),
+ )
+ },
+ resolver=lambda root, args, *_: getDroid(args['id']),
+ ),
+ }
+)
+
+StarWarsSchema = GraphQLSchema(query=queryType, types=[humanType, droidType])
diff --git a/wandb/vendor/gql-0.2.0/tests/starwars/test_dsl.py b/wandb/vendor/gql-0.2.0/tests/starwars/test_dsl.py
new file mode 100644
index 0000000000000000000000000000000000000000..e8fa1b6196f8eb8f35cf202b4017841ab63e8a74
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/tests/starwars/test_dsl.py
@@ -0,0 +1,293 @@
+import pytest
+
+from gql import Client
+from gql.dsl import DSLSchema
+
+from .schema import StarWarsSchema
+
+
+@pytest.fixture
+def ds():
+ client = Client(schema=StarWarsSchema)
+ ds = DSLSchema(client)
+ return ds
+
+
+def test_hero_name_query(ds):
+ query = '''
+hero {
+ name
+}
+ '''.strip()
+ query_dsl = ds.Query.hero.select(
+ ds.Character.name
+ )
+ assert query == str(query_dsl)
+
+
+def test_hero_name_and_friends_query(ds):
+ query = '''
+hero {
+ id
+ name
+ friends {
+ name
+ }
+}
+ '''.strip()
+ query_dsl = ds.Query.hero.select(
+ ds.Character.id,
+ ds.Character.name,
+ ds.Character.friends.select(
+ ds.Character.name,
+ )
+ )
+ assert query == str(query_dsl)
+
+
+def test_nested_query(ds):
+ query = '''
+hero {
+ name
+ friends {
+ name
+ appearsIn
+ friends {
+ name
+ }
+ }
+}
+ '''.strip()
+ query_dsl = ds.Query.hero.select(
+ ds.Character.name,
+ ds.Character.friends.select(
+ ds.Character.name,
+ ds.Character.appears_in,
+ ds.Character.friends.select(
+ ds.Character.name
+ )
+ )
+ )
+ assert query == str(query_dsl)
+
+
+def test_fetch_luke_query(ds):
+ query = '''
+human(id: "1000") {
+ name
+}
+ '''.strip()
+ query_dsl = ds.Query.human(id="1000").select(
+ ds.Human.name,
+ )
+
+ assert query == str(query_dsl)
+
+
+# def test_fetch_some_id_query():
+# query = '''
+# query FetchSomeIDQuery($someId: String!) {
+# human(id: $someId) {
+# name
+# }
+# }
+# '''
+# params = {
+# 'someId': '1000',
+# }
+# expected = {
+# 'human': {
+# 'name': 'Luke Skywalker',
+# }
+# }
+# result = schema.execute(query, None, params)
+# assert not result.errors
+# assert result.data == expected
+
+
+# def test_fetch_some_id_query2():
+# query = '''
+# query FetchSomeIDQuery($someId: String!) {
+# human(id: $someId) {
+# name
+# }
+# }
+# '''
+# params = {
+# 'someId': '1002',
+# }
+# expected = {
+# 'human': {
+# 'name': 'Han Solo',
+# }
+# }
+# result = schema.execute(query, None, params)
+# assert not result.errors
+# assert result.data == expected
+
+
+# def test_invalid_id_query():
+# query = '''
+# query humanQuery($id: String!) {
+# human(id: $id) {
+# name
+# }
+# }
+# '''
+# params = {
+# 'id': 'not a valid id',
+# }
+# expected = {
+# 'human': None
+# }
+# result = schema.execute(query, None, params)
+# assert not result.errors
+# assert result.data == expected
+
+
+def test_fetch_luke_aliased(ds):
+ query = '''
+luke: human(id: "1000") {
+ name
+}
+ '''.strip()
+ query_dsl = ds.Query.human.args(id=1000).alias('luke').select(
+ ds.Character.name,
+ )
+ assert query == str(query_dsl)
+
+
+# def test_fetch_luke_and_leia_aliased():
+# query = '''
+# query FetchLukeAndLeiaAliased {
+# luke: human(id: "1000") {
+# name
+# }
+# leia: human(id: "1003") {
+# name
+# }
+# }
+# '''
+# expected = {
+# 'luke': {
+# 'name': 'Luke Skywalker',
+# },
+# 'leia': {
+# 'name': 'Leia Organa',
+# }
+# }
+# result = schema.execute(query)
+# assert not result.errors
+# assert result.data == expected
+
+
+# def test_duplicate_fields():
+# query = '''
+# query DuplicateFields {
+# luke: human(id: "1000") {
+# name
+# homePlanet
+# }
+# leia: human(id: "1003") {
+# name
+# homePlanet
+# }
+# }
+# '''
+# expected = {
+# 'luke': {
+# 'name': 'Luke Skywalker',
+# 'homePlanet': 'Tatooine',
+# },
+# 'leia': {
+# 'name': 'Leia Organa',
+# 'homePlanet': 'Alderaan',
+# }
+# }
+# result = schema.execute(query)
+# assert not result.errors
+# assert result.data == expected
+
+
+# def test_use_fragment():
+# query = '''
+# query UseFragment {
+# luke: human(id: "1000") {
+# ...HumanFragment
+# }
+# leia: human(id: "1003") {
+# ...HumanFragment
+# }
+# }
+# fragment HumanFragment on Human {
+# name
+# homePlanet
+# }
+# '''
+# expected = {
+# 'luke': {
+# 'name': 'Luke Skywalker',
+# 'homePlanet': 'Tatooine',
+# },
+# 'leia': {
+# 'name': 'Leia Organa',
+# 'homePlanet': 'Alderaan',
+# }
+# }
+# result = schema.execute(query)
+# assert not result.errors
+# assert result.data == expected
+
+
+# def test_check_type_of_r2():
+# query = '''
+# query CheckTypeOfR2 {
+# hero {
+# __typename
+# name
+# }
+# }
+# '''
+# expected = {
+# 'hero': {
+# '__typename': 'Droid',
+# 'name': 'R2-D2',
+# }
+# }
+# result = schema.execute(query)
+# assert not result.errors
+# assert result.data == expected
+
+
+# def test_check_type_of_luke():
+# query = '''
+# query CheckTypeOfLuke {
+# hero(episode: EMPIRE) {
+# __typename
+# name
+# }
+# }
+# '''
+# expected = {
+# 'hero': {
+# '__typename': 'Human',
+# 'name': 'Luke Skywalker',
+# }
+# }
+# result = schema.execute(query)
+# assert not result.errors
+# assert result.data == expected
+
+
+def test_hero_name_query_result(ds):
+ result = ds.query(
+ ds.Query.hero.select(
+ ds.Character.name
+ )
+ )
+ expected = {
+ 'hero': {
+ 'name': 'R2-D2'
+ }
+ }
+ assert result == expected
diff --git a/wandb/vendor/gql-0.2.0/tests/starwars/test_query.py b/wandb/vendor/gql-0.2.0/tests/starwars/test_query.py
new file mode 100644
index 0000000000000000000000000000000000000000..6e3ebaf379aac98e2691cfd0ac1e438c25d7ed1d
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/tests/starwars/test_query.py
@@ -0,0 +1,355 @@
+import pytest
+from graphql.error import format_error
+
+from gql import Client, gql
+
+from .schema import StarWarsSchema
+
+
+@pytest.fixture
+def client():
+ return Client(schema=StarWarsSchema)
+
+
+def test_hero_name_query(client):
+ query = gql('''
+ query HeroNameQuery {
+ hero {
+ name
+ }
+ }
+ ''')
+ expected = {
+ 'hero': {
+ 'name': 'R2-D2'
+ }
+ }
+ result = client.execute(query)
+ assert result == expected
+
+
+def test_hero_name_and_friends_query(client):
+ query = gql('''
+ query HeroNameAndFriendsQuery {
+ hero {
+ id
+ name
+ friends {
+ name
+ }
+ }
+ }
+ ''')
+ expected = {
+ 'hero': {
+ 'id': '2001',
+ 'name': 'R2-D2',
+ 'friends': [
+ {'name': 'Luke Skywalker'},
+ {'name': 'Han Solo'},
+ {'name': 'Leia Organa'},
+ ]
+ }
+ }
+ result = client.execute(query)
+ assert result == expected
+
+
+def test_nested_query(client):
+ query = gql('''
+ query NestedQuery {
+ hero {
+ name
+ friends {
+ name
+ appearsIn
+ friends {
+ name
+ }
+ }
+ }
+ }
+ ''')
+ expected = {
+ 'hero': {
+ 'name': 'R2-D2',
+ 'friends': [
+ {
+ 'name': 'Luke Skywalker',
+ 'appearsIn': ['NEWHOPE', 'EMPIRE', 'JEDI'],
+ 'friends': [
+ {
+ 'name': 'Han Solo',
+ },
+ {
+ 'name': 'Leia Organa',
+ },
+ {
+ 'name': 'C-3PO',
+ },
+ {
+ 'name': 'R2-D2',
+ },
+ ]
+ },
+ {
+ 'name': 'Han Solo',
+ 'appearsIn': ['NEWHOPE', 'EMPIRE', 'JEDI'],
+ 'friends': [
+ {
+ 'name': 'Luke Skywalker',
+ },
+ {
+ 'name': 'Leia Organa',
+ },
+ {
+ 'name': 'R2-D2',
+ },
+ ]
+ },
+ {
+ 'name': 'Leia Organa',
+ 'appearsIn': ['NEWHOPE', 'EMPIRE', 'JEDI'],
+ 'friends': [
+ {
+ 'name': 'Luke Skywalker',
+ },
+ {
+ 'name': 'Han Solo',
+ },
+ {
+ 'name': 'C-3PO',
+ },
+ {
+ 'name': 'R2-D2',
+ },
+ ]
+ },
+ ]
+ }
+ }
+ result = client.execute(query)
+ assert result == expected
+
+
+def test_fetch_luke_query(client):
+ query = gql('''
+ query FetchLukeQuery {
+ human(id: "1000") {
+ name
+ }
+ }
+ ''')
+ expected = {
+ 'human': {
+ 'name': 'Luke Skywalker',
+ }
+ }
+ result = client.execute(query)
+ assert result == expected
+
+
+def test_fetch_some_id_query(client):
+ query = gql('''
+ query FetchSomeIDQuery($someId: String!) {
+ human(id: $someId) {
+ name
+ }
+ }
+ ''')
+ params = {
+ 'someId': '1000',
+ }
+ expected = {
+ 'human': {
+ 'name': 'Luke Skywalker',
+ }
+ }
+ result = client.execute(query, variable_values=params)
+ assert result == expected
+
+
+def test_fetch_some_id_query2(client):
+ query = gql('''
+ query FetchSomeIDQuery($someId: String!) {
+ human(id: $someId) {
+ name
+ }
+ }
+ ''')
+ params = {
+ 'someId': '1002',
+ }
+ expected = {
+ 'human': {
+ 'name': 'Han Solo',
+ }
+ }
+ result = client.execute(query, variable_values=params)
+ assert result == expected
+
+
+def test_invalid_id_query(client):
+ query = gql('''
+ query humanQuery($id: String!) {
+ human(id: $id) {
+ name
+ }
+ }
+ ''')
+ params = {
+ 'id': 'not a valid id',
+ }
+ expected = {
+ 'human': None
+ }
+ result = client.execute(query, variable_values=params)
+ assert result == expected
+
+
+def test_fetch_luke_aliased(client):
+ query = gql('''
+ query FetchLukeAliased {
+ luke: human(id: "1000") {
+ name
+ }
+ }
+ ''')
+ expected = {
+ 'luke': {
+ 'name': 'Luke Skywalker',
+ }
+ }
+ result = client.execute(query)
+ assert result == expected
+
+
+def test_fetch_luke_and_leia_aliased(client):
+ query = gql('''
+ query FetchLukeAndLeiaAliased {
+ luke: human(id: "1000") {
+ name
+ }
+ leia: human(id: "1003") {
+ name
+ }
+ }
+ ''')
+ expected = {
+ 'luke': {
+ 'name': 'Luke Skywalker',
+ },
+ 'leia': {
+ 'name': 'Leia Organa',
+ }
+ }
+ result = client.execute(query)
+ assert result == expected
+
+
+def test_duplicate_fields(client):
+ query = gql('''
+ query DuplicateFields {
+ luke: human(id: "1000") {
+ name
+ homePlanet
+ }
+ leia: human(id: "1003") {
+ name
+ homePlanet
+ }
+ }
+ ''')
+ expected = {
+ 'luke': {
+ 'name': 'Luke Skywalker',
+ 'homePlanet': 'Tatooine',
+ },
+ 'leia': {
+ 'name': 'Leia Organa',
+ 'homePlanet': 'Alderaan',
+ }
+ }
+ result = client.execute(query)
+ assert result == expected
+
+
+def test_use_fragment(client):
+ query = gql('''
+ query UseFragment {
+ luke: human(id: "1000") {
+ ...HumanFragment
+ }
+ leia: human(id: "1003") {
+ ...HumanFragment
+ }
+ }
+ fragment HumanFragment on Human {
+ name
+ homePlanet
+ }
+ ''')
+ expected = {
+ 'luke': {
+ 'name': 'Luke Skywalker',
+ 'homePlanet': 'Tatooine',
+ },
+ 'leia': {
+ 'name': 'Leia Organa',
+ 'homePlanet': 'Alderaan',
+ }
+ }
+ result = client.execute(query)
+ assert result == expected
+
+
+def test_check_type_of_r2(client):
+ query = gql('''
+ query CheckTypeOfR2 {
+ hero {
+ __typename
+ name
+ }
+ }
+ ''')
+ expected = {
+ 'hero': {
+ '__typename': 'Droid',
+ 'name': 'R2-D2',
+ }
+ }
+ result = client.execute(query)
+ assert result == expected
+
+
+def test_check_type_of_luke(client):
+ query = gql('''
+ query CheckTypeOfLuke {
+ hero(episode: EMPIRE) {
+ __typename
+ name
+ }
+ }
+ ''')
+ expected = {
+ 'hero': {
+ '__typename': 'Human',
+ 'name': 'Luke Skywalker',
+ }
+ }
+ result = client.execute(query)
+ assert result == expected
+
+
+def test_parse_error(client):
+ result = None
+ with pytest.raises(Exception) as excinfo:
+ query = gql('''
+ qeury
+ ''')
+ result = client.execute(query)
+ error = excinfo.value
+ formatted_error = format_error(error)
+ assert formatted_error['locations'] == [{'column': 13, 'line': 2}]
+ assert 'Syntax Error GraphQL request (2:13) Unexpected Name "qeury"' in formatted_error['message']
+ assert not result
diff --git a/wandb/vendor/gql-0.2.0/tests/starwars/test_validation.py b/wandb/vendor/gql-0.2.0/tests/starwars/test_validation.py
new file mode 100644
index 0000000000000000000000000000000000000000..e2699e95f16072e579a54572fdc7f391de1f43bb
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/tests/starwars/test_validation.py
@@ -0,0 +1,171 @@
+import pytest
+from graphql import graphql
+from graphql.utils.introspection_query import introspection_query
+
+from gql import Client, gql
+
+from .schema import StarWarsSchema
+
+introspection = graphql(StarWarsSchema, introspection_query).data
+
+
+@pytest.fixture
+def local_schema():
+ return Client(schema=StarWarsSchema)
+
+
+@pytest.fixture
+def typedef_schema():
+ return Client(type_def='''
+schema {
+ query: Query
+}
+
+interface Character {
+ appearsIn: [Episode]
+ friends: [Character]
+ id: String!
+ name: String
+}
+
+type Droid implements Character {
+ appearsIn: [Episode]
+ friends: [Character]
+ id: String!
+ name: String
+ primaryFunction: String
+}
+
+enum Episode {
+ EMPIRE
+ JEDI
+ NEWHOPE
+}
+
+type Human implements Character {
+ appearsIn: [Episode]
+ friends: [Character]
+ homePlanet: String
+ id: String!
+ name: String
+}
+
+type Query {
+ droid(id: String!): Droid
+ hero(episode: Episode): Character
+ human(id: String!): Human
+}''')
+
+
+@pytest.fixture
+def introspection_schema():
+ return Client(introspection=introspection)
+
+
+@pytest.fixture(params=['local_schema', 'typedef_schema', 'introspection_schema'])
+def client(request):
+ return request.getfixturevalue(request.param)
+
+
+def validation_errors(client, query):
+ query = gql(query)
+ try:
+ client.validate(query)
+ return False
+ except Exception:
+ return True
+
+
+def test_nested_query_with_fragment(client):
+ query = '''
+ query NestedQueryWithFragment {
+ hero {
+ ...NameAndAppearances
+ friends {
+ ...NameAndAppearances
+ friends {
+ ...NameAndAppearances
+ }
+ }
+ }
+ }
+ fragment NameAndAppearances on Character {
+ name
+ appearsIn
+ }
+ '''
+ assert not validation_errors(client, query)
+
+
+def test_non_existent_fields(client):
+ query = '''
+ query HeroSpaceshipQuery {
+ hero {
+ favoriteSpaceship
+ }
+ }
+ '''
+ assert validation_errors(client, query)
+
+
+def test_require_fields_on_object(client):
+ query = '''
+ query HeroNoFieldsQuery {
+ hero
+ }
+ '''
+ assert validation_errors(client, query)
+
+
+def test_disallows_fields_on_scalars(client):
+ query = '''
+ query HeroFieldsOnScalarQuery {
+ hero {
+ name {
+ firstCharacterOfName
+ }
+ }
+ }
+ '''
+ assert validation_errors(client, query)
+
+
+def test_disallows_object_fields_on_interfaces(client):
+ query = '''
+ query DroidFieldOnCharacter {
+ hero {
+ name
+ primaryFunction
+ }
+ }
+ '''
+ assert validation_errors(client, query)
+
+
+def test_allows_object_fields_in_fragments(client):
+ query = '''
+ query DroidFieldInFragment {
+ hero {
+ name
+ ...DroidFields
+ }
+ }
+ fragment DroidFields on Droid {
+ primaryFunction
+ }
+ '''
+ assert not validation_errors(client, query)
+
+
+def test_allows_object_fields_in_inline_fragments(client):
+ query = '''
+ query DroidFieldInFragment {
+ hero {
+ name
+ ... on Droid {
+ primaryFunction
+ }
+ }
+ }
+ '''
+ assert not validation_errors(client, query)
diff --git a/wandb/vendor/gql-0.2.0/tests/test_client.py b/wandb/vendor/gql-0.2.0/tests/test_client.py
new file mode 100644
index 0000000000000000000000000000000000000000..14f06c431aed8a4744fb3a0e941b6abe484817dd
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/tests/test_client.py
@@ -0,0 +1,31 @@
+import pytest
+import mock
+
+from gql import Client, gql
+from gql.transport.requests import RequestsHTTPTransport
+
+
+@mock.patch('gql.transport.requests.RequestsHTTPTransport.execute')
+def test_retries(execute_mock):
+ expected_retries = 3
+ execute_mock.side_effect = Exception("fail")
+
+ client = Client(
+ retries=expected_retries,
+ transport=RequestsHTTPTransport(url='http://swapi.graphene-python.org/graphql')
+ )
+
+ query = gql('''
+ {
+ myFavoriteFilm: film(id:"RmlsbToz") {
+ id
+ title
+ episodeId
+ }
+ }
+ ''')
+
+ with pytest.raises(Exception):
+ client.execute(query)
+
+ assert execute_mock.call_count == expected_retries
diff --git a/wandb/vendor/gql-0.2.0/tests/test_transport.py b/wandb/vendor/gql-0.2.0/tests/test_transport.py
new file mode 100644
index 0000000000000000000000000000000000000000..93334ba091334175a79a4cf9fb37a2f7b21371fd
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/tests/test_transport.py
@@ -0,0 +1,89 @@
+import pytest
+import requests
+import vcr
+
+from gql import Client, gql
+from gql.transport.requests import RequestsHTTPTransport
+
+# https://github.com/graphql-python/swapi-graphene
+URL = 'http://127.0.0.1:8000/graphql'
+
+
+@pytest.fixture
+def client():
+ with vcr.use_cassette('tests/fixtures/vcr_cassettes/client.yaml'):
+ request = requests.get(
+ URL,
+ headers={
+ 'Host': 'swapi.graphene-python.org',
+ 'Accept': 'text/html',
+ }
+ )
+ request.raise_for_status()
+ csrf = request.cookies['csrftoken']
+
+ return Client(
+ transport=RequestsHTTPTransport(
+ url=URL,
+ cookies={"csrftoken": csrf},
+ headers={'x-csrftoken': csrf}),
+ fetch_schema_from_transport=True
+ )
+
+
+def test_hero_name_query(client):
+ query = gql('''
+ {
+ myFavoriteFilm: film(id:"RmlsbToz") {
+ id
+ title
+ episodeId
+ characters(first:5) {
+ edges {
+ node {
+ name
+ }
+ }
+ }
+ }
+ }
+ ''')
+ expected = {
+ "myFavoriteFilm": {
+ "id": "RmlsbToz",
+ "title": "Return of the Jedi",
+ "episodeId": 6,
+ "characters": {
+ "edges": [
+ {
+ "node": {
+ "name": "Luke Skywalker"
+ }
+ },
+ {
+ "node": {
+ "name": "C-3PO"
+ }
+ },
+ {
+ "node": {
+ "name": "R2-D2"
+ }
+ },
+ {
+ "node": {
+ "name": "Darth Vader"
+ }
+ },
+ {
+ "node": {
+ "name": "Leia Organa"
+ }
+ }
+ ]
+ }
+ }
+ }
+ with vcr.use_cassette('tests/fixtures/vcr_cassettes/execute.yaml'):
+ result = client.execute(query)
+ assert result == expected
diff --git a/wandb/vendor/gql-0.2.0/tox.ini b/wandb/vendor/gql-0.2.0/tox.ini
new file mode 100644
index 0000000000000000000000000000000000000000..31073214424458637fde9ed91e4ecde4a0885b85
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/tox.ini
@@ -0,0 +1,27 @@
+[tox]
+envlist = py{27,35,36,37,38}, flake8, manifest
+
+[testenv:flake8]
+basepython = python3.7
+deps = flake8>=3.7,<4
+commands =
+ flake8 gql tests
+
+[testenv:manifest]
+basepython = python3.7
+deps = check-manifest>=0.40,<1
+commands =
+ check-manifest -v
+
+[testenv]
+setenv =
+ PYTHONPATH = {toxinidir}
+ MULTIDICT_NO_EXTENSIONS = 1
+ YARL_NO_EXTENSIONS = 1
+deps =
+ pytest>=3,<4
+ pytest-cov>=2.8,<3
+ mock>=3,<4
+ vcrpy>=2.1,<3
+commands =
+ pytest --cov=gql tests {posargs}
diff --git a/wandb/vendor/gql-0.2.0/wandb-vendor.diff b/wandb/vendor/gql-0.2.0/wandb-vendor.diff
new file mode 100644
index 0000000000000000000000000000000000000000..62a9bd7132473bb2922cc2c83881e51e29f380b2
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/wandb-vendor.diff
@@ -0,0 +1,95 @@
+diff --git a/wandb/vendor/gql-0.2.0/wandb_gql/client.py b/wandb/vendor/gql-0.2.0/wandb_gql/client.py
+index 95c565c..ab3ed7f 100644
+--- a/wandb/vendor/gql-0.2.0/wandb_gql/client.py
++++ b/wandb/vendor/gql-0.2.0/wandb_gql/client.py
+@@ -1,7 +1,7 @@
+ import logging
+
+-from graphql import parse, introspection_query, build_ast_schema, build_client_schema
+-from graphql.validation import validate
++from wandb_graphql import parse, introspection_query, build_ast_schema, build_client_schema
++from wandb_graphql.validation import validate
+
+ from .transport.local_schema import LocalSchemaTransport
+
+diff --git a/wandb/vendor/gql-0.2.0/wandb_gql/dsl.py b/wandb/vendor/gql-0.2.0/wandb_gql/dsl.py
+index 135d808..052e1eb 100644
+--- a/wandb/vendor/gql-0.2.0/wandb_gql/dsl.py
++++ b/wandb/vendor/gql-0.2.0/wandb_gql/dsl.py
+@@ -1,11 +1,10 @@
+-import collections
++from collections.abc import Iterable
+ import decimal
+ from functools import partial
+
+-import six
+-from graphql.language import ast
+-from graphql.language.printer import print_ast
+-from graphql.type import (GraphQLField, GraphQLList,
++from wandb_graphql.language import ast
++from wandb_graphql.language.printer import print_ast
++from wandb_graphql.type import (GraphQLField, GraphQLList,
+ GraphQLNonNull, GraphQLEnumType)
+
+ from .utils import to_camel_case
+@@ -61,7 +60,7 @@ def selections(*fields):
+ def get_ast_value(value):
+ if isinstance(value, ast.Node):
+ return value
+- if isinstance(value, six.string_types):
++ if isinstance(value, str):
+ return ast.StringValue(value=value)
+ elif isinstance(value, bool):
+ return ast.BooleanValue(value=value)
+@@ -134,7 +133,7 @@ def query(*fields):
+
+
+ def serialize_list(serializer, values):
+- assert isinstance(values, collections.Iterable), 'Expected iterable, received "{}"'.format(repr(values))
++ assert isinstance(values, Iterable), 'Expected iterable, received "{}"'.format(repr(values))
+ return [serializer(v) for v in values]
+
+
+diff --git a/wandb/vendor/gql-0.2.0/wandb_gql/gql.py b/wandb/vendor/gql-0.2.0/wandb_gql/gql.py
+index 782943f..21edd39 100644
+--- a/wandb/vendor/gql-0.2.0/wandb_gql/gql.py
++++ b/wandb/vendor/gql-0.2.0/wandb_gql/gql.py
+@@ -1,10 +1,9 @@
+-import six
+-from graphql.language.parser import parse
+-from graphql.language.source import Source
++from wandb_graphql.language.parser import parse
++from wandb_graphql.language.source import Source
+
+
+ def gql(request_string):
+- if isinstance(request_string, six.string_types):
++ if isinstance(request_string, str):
+ source = Source(request_string, 'GraphQL request')
+ return parse(source)
+ else:
+diff --git a/wandb/vendor/gql-0.2.0/wandb_gql/transport/local_schema.py b/wandb/vendor/gql-0.2.0/wandb_gql/transport/local_schema.py
+index 30d577e..5bc7d33 100644
+--- a/wandb/vendor/gql-0.2.0/wandb_gql/transport/local_schema.py
++++ b/wandb/vendor/gql-0.2.0/wandb_gql/transport/local_schema.py
+@@ -1,4 +1,4 @@
+-from graphql.execution import execute
++from wandb_graphql.execution import execute
+
+
+ class LocalSchemaTransport(object):
+diff --git a/wandb/vendor/gql-0.2.0/wandb_gql/transport/requests.py b/wandb/vendor/gql-0.2.0/wandb_gql/transport/requests.py
+index 71399a5..305ca8a 100644
+--- a/wandb/vendor/gql-0.2.0/wandb_gql/transport/requests.py
++++ b/wandb/vendor/gql-0.2.0/wandb_gql/transport/requests.py
+@@ -1,8 +1,8 @@
+ from __future__ import absolute_import
+
+ import requests
+-from graphql.execution import ExecutionResult
+-from graphql.language.printer import print_ast
++from wandb_graphql.execution import ExecutionResult
++from wandb_graphql.language.printer import print_ast
+
+ from .http import HTTPTransport
+
diff --git a/wandb/vendor/gql-0.2.0/wandb-vendor.md b/wandb/vendor/gql-0.2.0/wandb-vendor.md
new file mode 100644
index 0000000000000000000000000000000000000000..9a3291bad2e80e488e425e7e0e9d491b5fb800c0
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/wandb-vendor.md
@@ -0,0 +1,7 @@
+
+# Modification steps
+
+```shell
+mv gql wandb_gql
+patch -p4 < wandb-vendor.diff
+```
diff --git a/wandb/vendor/gql-0.2.0/wandb_gql/__init__.py b/wandb/vendor/gql-0.2.0/wandb_gql/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..f24fce1295c27bf2575c3a41c8569ce257dd7ffa
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/wandb_gql/__init__.py
@@ -0,0 +1,4 @@
+from .gql import gql
+from .client import Client
+
+__all__ = ['gql', 'Client']
diff --git a/wandb/vendor/gql-0.2.0/wandb_gql/client.py b/wandb/vendor/gql-0.2.0/wandb_gql/client.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab3ed7f0e121a908e892ed0537f215f0aa942f93
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/wandb_gql/client.py
@@ -0,0 +1,75 @@
+import logging
+
+from wandb_graphql import parse, introspection_query, build_ast_schema, build_client_schema
+from wandb_graphql.validation import validate
+
+from .transport.local_schema import LocalSchemaTransport
+
+log = logging.getLogger(__name__)
+
+
+class RetryError(Exception):
+ """Custom exception thrown when retry logic fails"""
+ def __init__(self, retries_count, last_exception):
+ message = "Failed %s retries: %s" % (retries_count, last_exception)
+ super(RetryError, self).__init__(message)
+ self.last_exception = last_exception
+
+
+class Client(object):
+ def __init__(self, schema=None, introspection=None, type_def=None, transport=None,
+ fetch_schema_from_transport=False, retries=0):
+ assert not(type_def and introspection), 'Cant provide introspection type definition at the same time'
+ if transport and fetch_schema_from_transport:
+ assert not schema, 'Cant fetch the schema from transport if is already provided'
+ introspection = transport.execute(parse(introspection_query)).data
+ if introspection:
+ assert not schema, 'Cant provide introspection and schema at the same time'
+ schema = build_client_schema(introspection)
+ elif type_def:
+ assert not schema, 'Cant provide Type definition and schema at the same time'
+ type_def_ast = parse(type_def)
+ schema = build_ast_schema(type_def_ast)
+ elif schema and not transport:
+ transport = LocalSchemaTransport(schema)
+
+ self.schema = schema
+ self.introspection = introspection
+ self.transport = transport
+ self.retries = retries
+
+ def validate(self, document):
+ if not self.schema:
+ raise Exception("Cannot validate locally the document, you need to pass a schema.")
+ validation_errors = validate(self.schema, document)
+ if validation_errors:
+ raise validation_errors[0]
+
+ def execute(self, document, *args, **kwargs):
+ if self.schema:
+ self.validate(document)
+
+ result = self._get_result(document, *args, **kwargs)
+ if result.errors:
+ raise Exception(str(result.errors[0]))
+
+ return result.data
+
+ def _get_result(self, document, *args, **kwargs):
+ if not self.retries:
+ return self.transport.execute(document, *args, **kwargs)
+
+ last_exception = None
+ retries_count = 0
+ while retries_count < self.retries:
+ try:
+ result = self.transport.execute(document, *args, **kwargs)
+ return result
+ except Exception as e:
+ last_exception = e
+ log.warning("Request failed with exception %s. Retrying for the %s time...",
+ e, retries_count + 1, exc_info=True)
+ finally:
+ retries_count += 1
+
+ raise RetryError(retries_count, last_exception)
diff --git a/wandb/vendor/gql-0.2.0/wandb_gql/dsl.py b/wandb/vendor/gql-0.2.0/wandb_gql/dsl.py
new file mode 100644
index 0000000000000000000000000000000000000000..052e1ebd277b98cf80dec0d8e2313593fa04c3d4
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/wandb_gql/dsl.py
@@ -0,0 +1,152 @@
+from collections.abc import Iterable
+import decimal
+from functools import partial
+
+from wandb_graphql.language import ast
+from wandb_graphql.language.printer import print_ast
+from wandb_graphql.type import (GraphQLField, GraphQLList,
+ GraphQLNonNull, GraphQLEnumType)
+
+from .utils import to_camel_case
+
+
+class DSLSchema(object):
+ def __init__(self, client):
+ self.client = client
+
+ @property
+ def schema(self):
+ return self.client.schema
+
+ def __getattr__(self, name):
+ type_def = self.schema.get_type(name)
+ return DSLType(type_def)
+
+ def query(self, *args, **kwargs):
+ return self.execute(query(*args, **kwargs))
+
+ def mutate(self, *args, **kwargs):
+ return self.query(*args, operation='mutate', **kwargs)
+
+ def execute(self, document):
+ return self.client.execute(document)
+
+
+class DSLType(object):
+ def __init__(self, type):
+ self.type = type
+
+ def __getattr__(self, name):
+ formatted_name, field_def = self.get_field(name)
+ return DSLField(formatted_name, field_def)
+
+ def get_field(self, name):
+ camel_cased_name = to_camel_case(name)
+
+ if name in self.type.fields:
+ return name, self.type.fields[name]
+
+ if camel_cased_name in self.type.fields:
+ return camel_cased_name, self.type.fields[camel_cased_name]
+
+ raise KeyError('Field {} doesnt exist in type {}.'.format(name, self.type.name))
+
+
+def selections(*fields):
+ for _field in fields:
+ yield field(_field).ast
+
+
+def get_ast_value(value):
+ if isinstance(value, ast.Node):
+ return value
+ if isinstance(value, str):
+ return ast.StringValue(value=value)
+ elif isinstance(value, bool):
+ return ast.BooleanValue(value=value)
+ elif isinstance(value, (float, decimal.Decimal)):
+ return ast.FloatValue(value=value)
+ elif isinstance(value, int):
+ return ast.IntValue(value=value)
+ return None
+
+
+class DSLField(object):
+
+ def __init__(self, name, field):
+ self.field = field
+ self.ast_field = ast.Field(name=ast.Name(value=name), arguments=[])
+ self.selection_set = None
+
+ def select(self, *fields):
+ if not self.ast_field.selection_set:
+ self.ast_field.selection_set = ast.SelectionSet(selections=[])
+ self.ast_field.selection_set.selections.extend(selections(*fields))
+ return self
+
+ def __call__(self, *args, **kwargs):
+ return self.args(*args, **kwargs)
+
+ def alias(self, alias):
+ self.ast_field.alias = ast.Name(value=alias)
+ return self
+
+ def args(self, **args):
+ for name, value in args.items():
+ arg = self.field.args.get(name)
+ arg_type_serializer = get_arg_serializer(arg.type)
+ value = arg_type_serializer(value)
+ self.ast_field.arguments.append(
+ ast.Argument(
+ name=ast.Name(value=name),
+ value=get_ast_value(value)
+ )
+ )
+ return self
+
+ @property
+ def ast(self):
+ return self.ast_field
+
+ def __str__(self):
+ return print_ast(self.ast_field)
+
+
+def field(field, **args):
+ if isinstance(field, GraphQLField):
+ return DSLField(field).args(**args)
+ elif isinstance(field, DSLField):
+ return field
+
+ raise Exception('Received incompatible query field: "{}".'.format(field))
+
+
+def query(*fields):
+ return ast.Document(
+ definitions=[ast.OperationDefinition(
+ operation='query',
+ selection_set=ast.SelectionSet(
+ selections=list(selections(*fields))
+ )
+ )]
+ )
+
+
+def serialize_list(serializer, values):
+ assert isinstance(values, Iterable), 'Expected iterable, received "{}"'.format(repr(values))
+ return [serializer(v) for v in values]
+
+
+def get_arg_serializer(arg_type):
+ if isinstance(arg_type, GraphQLNonNull):
+ return get_arg_serializer(arg_type.of_type)
+ if isinstance(arg_type, GraphQLList):
+ inner_serializer = get_arg_serializer(arg_type.of_type)
+ return partial(serialize_list, inner_serializer)
+ if isinstance(arg_type, GraphQLEnumType):
+ return lambda value: ast.EnumValue(value=arg_type.serialize(value))
+ return arg_type.serialize
+
+
+def var(name):
+ return ast.Variable(name=name)
diff --git a/wandb/vendor/gql-0.2.0/wandb_gql/gql.py b/wandb/vendor/gql-0.2.0/wandb_gql/gql.py
new file mode 100644
index 0000000000000000000000000000000000000000..21edd394ad7dec411676982f285a977cb4d07752
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/wandb_gql/gql.py
@@ -0,0 +1,10 @@
+from wandb_graphql.language.parser import parse
+from wandb_graphql.language.source import Source
+
+
+def gql(request_string):
+ if isinstance(request_string, str):
+ source = Source(request_string, 'GraphQL request')
+ return parse(source)
+ else:
+ raise Exception('Received incompatible request "{}".'.format(request_string))
diff --git a/wandb/vendor/gql-0.2.0/wandb_gql/transport/__init__.py b/wandb/vendor/gql-0.2.0/wandb_gql/transport/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/vendor/gql-0.2.0/wandb_gql/transport/http.py b/wandb/vendor/gql-0.2.0/wandb_gql/transport/http.py
new file mode 100644
index 0000000000000000000000000000000000000000..bbd0e04758f0db007b316041518b0db14c53eb12
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/wandb_gql/transport/http.py
@@ -0,0 +1,6 @@
+class HTTPTransport(object):
+
+ def __init__(self, url, headers=None, cookies=None):
+ self.url = url
+ self.headers = headers
+ self.cookies = cookies
diff --git a/wandb/vendor/gql-0.2.0/wandb_gql/transport/local_schema.py b/wandb/vendor/gql-0.2.0/wandb_gql/transport/local_schema.py
new file mode 100644
index 0000000000000000000000000000000000000000..5bc7d33dc589a1cd1a0410bfd9465f9cb4a7a50a
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/wandb_gql/transport/local_schema.py
@@ -0,0 +1,15 @@
+from wandb_graphql.execution import execute
+
+
+class LocalSchemaTransport(object):
+
+ def __init__(self, schema):
+ self.schema = schema
+
+ def execute(self, document, *args, **kwargs):
+ return execute(
+ self.schema,
+ document,
+ *args,
+ **kwargs
+ )
diff --git a/wandb/vendor/gql-0.2.0/wandb_gql/transport/requests.py b/wandb/vendor/gql-0.2.0/wandb_gql/transport/requests.py
new file mode 100644
index 0000000000000000000000000000000000000000..305ca8af968f65e172e6ba262febaef02d70cf0e
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/wandb_gql/transport/requests.py
@@ -0,0 +1,46 @@
+from __future__ import absolute_import
+
+import requests
+from wandb_graphql.execution import ExecutionResult
+from wandb_graphql.language.printer import print_ast
+
+from .http import HTTPTransport
+
+
+class RequestsHTTPTransport(HTTPTransport):
+ def __init__(self, url, auth=None, use_json=False, timeout=None, **kwargs):
+ """
+ :param url: The GraphQL URL
+ :param auth: Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth
+ :param use_json: Send request body as JSON instead of form-urlencoded
+ :param timeout: Specifies a default timeout for requests (Default: None)
+ """
+ super(RequestsHTTPTransport, self).__init__(url, **kwargs)
+ self.auth = auth
+ self.default_timeout = timeout
+ self.use_json = use_json
+
+ def execute(self, document, variable_values=None, timeout=None):
+ query_str = print_ast(document)
+ payload = {
+ 'query': query_str,
+ 'variables': variable_values or {}
+ }
+
+ data_key = 'json' if self.use_json else 'data'
+ post_args = {
+ 'headers': self.headers,
+ 'auth': self.auth,
+ 'cookies': self.cookies,
+ 'timeout': timeout or self.default_timeout,
+ data_key: payload
+ }
+ request = requests.post(self.url, **post_args)
+ request.raise_for_status()
+
+ result = request.json()
+ assert 'errors' in result or 'data' in result, 'Received non-compatible response "{}"'.format(result)
+ return ExecutionResult(
+ errors=result.get('errors'),
+ data=result.get('data')
+ )
diff --git a/wandb/vendor/gql-0.2.0/wandb_gql/utils.py b/wandb/vendor/gql-0.2.0/wandb_gql/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..d59964977fda9da9c877841f5cfd17b219e6bbb4
--- /dev/null
+++ b/wandb/vendor/gql-0.2.0/wandb_gql/utils.py
@@ -0,0 +1,21 @@
+import re
+
+
+# From this response in Stackoverflow
+# http://stackoverflow.com/a/19053800/1072990
+def to_camel_case(snake_str):
+ components = snake_str.split('_')
+ # We capitalize the first letter of each component except the first one
+ # with the 'title' method and join them together.
+ return components[0] + "".join(x.title() if x else '_' for x in components[1:])
+
+
+# From this response in Stackoverflow
+# http://stackoverflow.com/a/1176023/1072990
+def to_snake_case(name):
+ s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
+ return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
+
+
+def to_const(string):
+ return re.sub(r'[\W|^]+', '_', string).upper()
diff --git a/wandb/vendor/graphql-core-1.1/MANIFEST.in b/wandb/vendor/graphql-core-1.1/MANIFEST.in
new file mode 100644
index 0000000000000000000000000000000000000000..a24c614f977a7a3f9fd7ee3928a94425211ca413
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/MANIFEST.in
@@ -0,0 +1,4 @@
+global-exclude tests/*
+recursive-exclude tests *
+recursive-exclude tests_py35 *
+recursive-exclude examples *
diff --git a/wandb/vendor/graphql-core-1.1/PKG-INFO b/wandb/vendor/graphql-core-1.1/PKG-INFO
new file mode 100644
index 0000000000000000000000000000000000000000..9ffb00892feb06c0a811537202f77a01a2406db3
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/PKG-INFO
@@ -0,0 +1,25 @@
+Metadata-Version: 1.1
+Name: graphql-core
+Version: 1.1
+Summary: GraphQL implementation for Python
+Home-page: https://github.com/graphql-python/graphql-core
+Author: Syrus Akbary, Jake Heinz, Taeho Kim
+Author-email: Syrus Akbary <me@syrusakbary.com>, Jake Heinz <me@jh.gg>, Taeho Kim <dittos@gmail.com>
+License: MIT
+Download-URL: https://github.com/graphql-python/graphql-core/releases
+Description: UNKNOWN
+Keywords: api graphql protocol rest
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Database :: Front-Ends
+Classifier: Topic :: Internet :: WWW/HTTP
diff --git a/wandb/vendor/graphql-core-1.1/graphql_core.egg-info/PKG-INFO b/wandb/vendor/graphql-core-1.1/graphql_core.egg-info/PKG-INFO
new file mode 100644
index 0000000000000000000000000000000000000000..9ffb00892feb06c0a811537202f77a01a2406db3
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/graphql_core.egg-info/PKG-INFO
@@ -0,0 +1,25 @@
+Metadata-Version: 1.1
+Name: graphql-core
+Version: 1.1
+Summary: GraphQL implementation for Python
+Home-page: https://github.com/graphql-python/graphql-core
+Author: Syrus Akbary, Jake Heinz, Taeho Kim
+Author-email: Syrus Akbary <me@syrusakbary.com>, Jake Heinz <me@jh.gg>, Taeho Kim <dittos@gmail.com>
+License: MIT
+Download-URL: https://github.com/graphql-python/graphql-core/releases
+Description: UNKNOWN
+Keywords: api graphql protocol rest
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Database :: Front-Ends
+Classifier: Topic :: Internet :: WWW/HTTP
diff --git a/wandb/vendor/graphql-core-1.1/graphql_core.egg-info/SOURCES.txt b/wandb/vendor/graphql-core-1.1/graphql_core.egg-info/SOURCES.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4e2e327dad66f94a82ddf8364e6ad3a4a26ecafb
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/graphql_core.egg-info/SOURCES.txt
@@ -0,0 +1,106 @@
+MANIFEST.in
+setup.cfg
+setup.py
+graphql/__init__.py
+graphql/graphql.py
+graphql/error/__init__.py
+graphql/error/base.py
+graphql/error/format_error.py
+graphql/error/located_error.py
+graphql/error/syntax_error.py
+graphql/execution/__init__.py
+graphql/execution/base.py
+graphql/execution/executor.py
+graphql/execution/middleware.py
+graphql/execution/values.py
+graphql/execution/executors/__init__.py
+graphql/execution/executors/asyncio.py
+graphql/execution/executors/gevent.py
+graphql/execution/executors/process.py
+graphql/execution/executors/sync.py
+graphql/execution/executors/thread.py
+graphql/execution/executors/utils.py
+graphql/execution/experimental/__init__.py
+graphql/execution/experimental/executor.py
+graphql/execution/experimental/fragment.py
+graphql/execution/experimental/resolver.py
+graphql/execution/experimental/utils.py
+graphql/language/__init__.py
+graphql/language/ast.py
+graphql/language/base.py
+graphql/language/lexer.py
+graphql/language/location.py
+graphql/language/parser.py
+graphql/language/printer.py
+graphql/language/source.py
+graphql/language/visitor.py
+graphql/language/visitor_meta.py
+graphql/pyutils/__init__.py
+graphql/pyutils/cached_property.py
+graphql/pyutils/contain_subset.py
+graphql/pyutils/default_ordered_dict.py
+graphql/pyutils/ordereddict.py
+graphql/pyutils/pair_set.py
+graphql/pyutils/version.py
+graphql/type/__init__.py
+graphql/type/definition.py
+graphql/type/directives.py
+graphql/type/introspection.py
+graphql/type/scalars.py
+graphql/type/schema.py
+graphql/type/typemap.py
+graphql/utils/__init__.py
+graphql/utils/assert_valid_name.py
+graphql/utils/ast_from_value.py
+graphql/utils/ast_to_code.py
+graphql/utils/ast_to_dict.py
+graphql/utils/base.py
+graphql/utils/build_ast_schema.py
+graphql/utils/build_client_schema.py
+graphql/utils/concat_ast.py
+graphql/utils/extend_schema.py
+graphql/utils/get_field_def.py
+graphql/utils/get_operation_ast.py
+graphql/utils/introspection_query.py
+graphql/utils/is_valid_literal_value.py
+graphql/utils/is_valid_value.py
+graphql/utils/quoted_or_list.py
+graphql/utils/schema_printer.py
+graphql/utils/suggestion_list.py
+graphql/utils/type_comparators.py
+graphql/utils/type_from_ast.py
+graphql/utils/type_info.py
+graphql/utils/value_from_ast.py
+graphql/validation/__init__.py
+graphql/validation/validation.py
+graphql/validation/rules/__init__.py
+graphql/validation/rules/arguments_of_correct_type.py
+graphql/validation/rules/base.py
+graphql/validation/rules/default_values_of_correct_type.py
+graphql/validation/rules/fields_on_correct_type.py
+graphql/validation/rules/fragments_on_composite_types.py
+graphql/validation/rules/known_argument_names.py
+graphql/validation/rules/known_directives.py
+graphql/validation/rules/known_fragment_names.py
+graphql/validation/rules/known_type_names.py
+graphql/validation/rules/lone_anonymous_operation.py
+graphql/validation/rules/no_fragment_cycles.py
+graphql/validation/rules/no_undefined_variables.py
+graphql/validation/rules/no_unused_fragments.py
+graphql/validation/rules/no_unused_variables.py
+graphql/validation/rules/overlapping_fields_can_be_merged.py
+graphql/validation/rules/possible_fragment_spreads.py
+graphql/validation/rules/provided_non_null_arguments.py
+graphql/validation/rules/scalar_leafs.py
+graphql/validation/rules/unique_argument_names.py
+graphql/validation/rules/unique_fragment_names.py
+graphql/validation/rules/unique_input_field_names.py
+graphql/validation/rules/unique_operation_names.py
+graphql/validation/rules/unique_variable_names.py
+graphql/validation/rules/variables_are_input_types.py
+graphql/validation/rules/variables_in_allowed_position.py
+graphql_core.egg-info/PKG-INFO
+graphql_core.egg-info/SOURCES.txt
+graphql_core.egg-info/dependency_links.txt
+graphql_core.egg-info/requires.txt
+graphql_core.egg-info/top_level.txt
\ No newline at end of file
diff --git a/wandb/vendor/graphql-core-1.1/graphql_core.egg-info/dependency_links.txt b/wandb/vendor/graphql-core-1.1/graphql_core.egg-info/dependency_links.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/graphql_core.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
diff --git a/wandb/vendor/graphql-core-1.1/graphql_core.egg-info/requires.txt b/wandb/vendor/graphql-core-1.1/graphql_core.egg-info/requires.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d5226f418ca8edf088f4e9576b658e9c8b15bd98
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/graphql_core.egg-info/requires.txt
@@ -0,0 +1,15 @@
+six>=1.10.0
+promise>=2.0
+
+[gevent]
+gevent==1.1rc1
+
+[test]
+pytest==3.0.2
+pytest-django==2.9.1
+pytest-cov==2.3.1
+coveralls
+gevent==1.1rc1
+six>=1.10.0
+pytest-benchmark==3.0.0
+pytest-mock==1.2
diff --git a/wandb/vendor/graphql-core-1.1/graphql_core.egg-info/top_level.txt b/wandb/vendor/graphql-core-1.1/graphql_core.egg-info/top_level.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1a71fca28d4f9d56e1d23ca24230d8c3db42d12d
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/graphql_core.egg-info/top_level.txt
@@ -0,0 +1,3 @@
+graphql
+tests
+tests_py35
diff --git a/wandb/vendor/graphql-core-1.1/setup.cfg b/wandb/vendor/graphql-core-1.1/setup.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..e16c1a9c5f625cfdbcb46761134c708f6fc8cf89
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/setup.cfg
@@ -0,0 +1,9 @@
+[flake8]
+exclude = tests,scripts,setup.py,docs
+max-line-length = 160
+
+[egg_info]
+tag_build =
+tag_svn_revision = 0
+tag_date = 0
+
diff --git a/wandb/vendor/graphql-core-1.1/setup.py b/wandb/vendor/graphql-core-1.1/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..16ecb33816b535c767a272e7e06eb9799e011e8b
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/setup.py
@@ -0,0 +1,86 @@
+from setuptools import setup, find_packages
+from setuptools.command.test import test as TestCommand
+import sys
+
+if sys.version_info[0] < 3:
+ import __builtin__ as builtins
+else:
+ import builtins
+
+# This is a bit (!) hackish: we are setting a global variable so that the main
+# graphql __init__ can detect if it is being loaded by the setup routine, to
+# avoid attempting to load components that aren't built yet:
+# the numpy distutils extensions that are used by scikit-learn to recursively
+# build the compiled extensions in sub-packages is based on the Python import
+# machinery.
+if 'test' not in sys.argv:
+ builtins.__GRAPHQL_SETUP__ = True
+
+version = __import__('graphql').get_version()
+
+install_requires = [
+ 'six>=1.10.0',
+ 'promise>=2.0'
+]
+
+tests_requires = [
+ 'pytest==3.0.2',
+ 'pytest-django==2.9.1',
+ 'pytest-cov==2.3.1',
+ 'coveralls',
+ 'gevent==1.1rc1',
+ 'six>=1.10.0',
+ 'pytest-benchmark==3.0.0',
+ 'pytest-mock==1.2',
+]
+
+class PyTest(TestCommand):
+ def finalize_options(self):
+ TestCommand.finalize_options(self)
+ self.test_args = ['graphql', '-vrsx']
+ self.test_suite = True
+
+ def run_tests(self):
+ #import here, cause outside the eggs aren't loaded
+ import pytest
+ errno = pytest.main(self.test_args)
+ sys.exit(errno)
+
+
+setup(
+ name='graphql-core',
+ version=version,
+ description='GraphQL implementation for Python',
+ url='https://github.com/graphql-python/graphql-core',
+ download_url='https://github.com/graphql-python/graphql-core/releases',
+ author='Syrus Akbary, Jake Heinz, Taeho Kim',
+ author_email='Syrus Akbary <me@syrusakbary.com>, Jake Heinz <me@jh.gg>, Taeho Kim <dittos@gmail.com>',
+ license='MIT',
+ classifiers=[
+ 'Development Status :: 5 - Production/Stable',
+ 'Intended Audience :: Developers',
+ 'Topic :: Software Development :: Libraries',
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.3',
+ 'Programming Language :: Python :: 3.4',
+ 'Programming Language :: Python :: 3.5',
+ 'Programming Language :: Python :: Implementation :: PyPy',
+ 'License :: OSI Approved :: MIT License',
+ 'Topic :: Database :: Front-Ends',
+ 'Topic :: Internet :: WWW/HTTP',
+ ],
+
+ keywords='api graphql protocol rest',
+ packages=find_packages(exclude=['tests', 'tests_py35']),
+ install_requires=install_requires,
+ tests_require=tests_requires,
+ cmdclass = {'test': PyTest},
+ extras_require={
+ 'gevent': [
+ 'gevent==1.1rc1'
+ ],
+ 'test': tests_requires
+ }
+)
diff --git a/wandb/vendor/graphql-core-1.1/wandb-vendor.diff b/wandb/vendor/graphql-core-1.1/wandb-vendor.diff
new file mode 100644
index 0000000000000000000000000000000000000000..ef2e67e270e0776cc763f7270a79041670ed9eb8
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb-vendor.diff
@@ -0,0 +1,416 @@
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/error/base.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/error/base.py
+index 44126cd..74d6447 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/error/base.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/error/base.py
+@@ -1,4 +1,3 @@
+-import six
+ from ..language.location import get_location
+
+
+@@ -32,7 +31,7 @@ class GraphQLError(Exception):
+
+ def reraise(self):
+ if self.stack:
+- six.reraise(type(self), self, self.stack)
++ raise self.with_traceback(self.stack)
+ else:
+ raise self
+
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executor.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executor.py
+index 202ba98..53a9f35 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executor.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executor.py
+@@ -1,9 +1,9 @@
+ import collections
++from collections.abc import Iterable
+ import functools
+ import logging
+ import sys
+
+-from six import string_types
+ from promise import Promise, promise_for_dict, is_thenable
+
+ from ..error import GraphQLError, GraphQLLocatedError
+@@ -294,7 +294,7 @@ def complete_list_value(exe_context, return_type, field_asts, info, result):
+ """
+ Complete a list value by completing each item in the list with the inner type
+ """
+- assert isinstance(result, collections.Iterable), \
++ assert isinstance(result, Iterable), \
+ ('User Error: expected iterable, but did not find one ' +
+ 'for field {}.{}.').format(info.parent_type, info.field_name)
+
+@@ -335,7 +335,7 @@ def complete_abstract_value(exe_context, return_type, field_asts, info, result):
+ else:
+ runtime_type = get_default_resolve_type_fn(result, exe_context.context_value, info, return_type)
+
+- if isinstance(runtime_type, string_types):
++ if isinstance(runtime_type, str):
+ runtime_type = info.schema.get_type(runtime_type)
+
+ if not isinstance(runtime_type, GraphQLObjectType):
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/resolver.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/resolver.py
+index 75f5b7d..ab12110 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/resolver.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/resolver.py
+@@ -1,5 +1,5 @@
+ import sys
+-import collections
++from collections.abc import Iterable
+ from functools import partial
+
+ from promise import Promise, is_thenable
+@@ -37,7 +37,7 @@ def complete_list_value(inner_resolver, exe_context, info, on_error, result):
+ if result is None:
+ return None
+
+- assert isinstance(result, collections.Iterable), \
++ assert isinstance(result, Iterable), \
+ ('User Error: expected iterable, but did not find one ' +
+ 'for field {}.{}.').format(info.parent_type, info.field_name)
+
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/values.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/values.py
+index 5600846..473ada1 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/values.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/values.py
+@@ -1,8 +1,6 @@
+-import collections
++from collections.abc import Iterable
+ import json
+
+-from six import string_types
+-
+ from ..error import GraphQLError
+ from ..language.printer import print_ast
+ from ..type import (GraphQLEnumType, GraphQLInputObjectType, GraphQLList,
+@@ -121,7 +119,7 @@ def coerce_value(type, value):
+
+ if isinstance(type, GraphQLList):
+ item_type = type.of_type
+- if not isinstance(value, string_types) and isinstance(value, collections.Iterable):
++ if not isinstance(value, str) and isinstance(value, Iterable):
+ return [coerce_value(item_type, item) for item in value]
+ else:
+ return [coerce_value(item_type, value)]
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/lexer.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/lexer.py
+index 711525e..c6909fb 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/lexer.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/lexer.py
+@@ -1,7 +1,5 @@
+ import json
+
+-from six import unichr
+-
+ from ..error import GraphQLSyntaxError
+
+ __all__ = ['Token', 'Lexer', 'TokenKind',
+@@ -134,7 +132,7 @@ def print_char_code(code):
+ return '<EOF>'
+
+ if code < 0x007F:
+- return json.dumps(unichr(code))
++ return json.dumps(chr(code))
+
+ return '"\\u%04X"' % code
+
+@@ -368,12 +366,12 @@ def read_string(source, start):
+ u'Invalid character escape sequence: \\u{}.'.format(body[position + 1: position + 5])
+ )
+
+- append(unichr(char_code))
++ append(chr(char_code))
+ position += 4
+ else:
+ raise GraphQLSyntaxError(
+ source, position,
+- u'Invalid character escape sequence: \\{}.'.format(unichr(code))
++ u'Invalid character escape sequence: \\{}.'.format(chr(code))
+ )
+
+ position += 1
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/parser.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/parser.py
+index 21adac9..d2854a3 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/parser.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/parser.py
+@@ -1,5 +1,3 @@
+-from six import string_types
+-
+ from . import ast
+ from ..error import GraphQLSyntaxError
+ from .lexer import Lexer, TokenKind, get_token_desc, get_token_kind_desc
+@@ -14,7 +12,7 @@ def parse(source, **kwargs):
+ options.update(kwargs)
+ source_obj = source
+
+- if isinstance(source, string_types):
++ if isinstance(source, str):
+ source_obj = Source(source)
+
+ parser = Parser(source_obj, options)
+@@ -26,7 +24,7 @@ def parse_value(source, **kwargs):
+ options.update(kwargs)
+ source_obj = source
+
+- if isinstance(source, string_types):
++ if isinstance(source, str):
+ source_obj = Source(source)
+
+ parser = Parser(source_obj, options)
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor.py
+index a77c36e..95cd69a 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor.py
+@@ -1,7 +1,5 @@
+ from copy import copy
+
+-import six
+-
+ from . import ast
+ from .visitor_meta import QUERY_DOCUMENT_KEYS, VisitorMeta
+
+@@ -158,8 +156,7 @@ def visit(root, visitor, key_map=None):
+ return new_root
+
+
+-@six.add_metaclass(VisitorMeta)
+-class Visitor(object):
++class Visitor(metaclass=VisitorMeta):
+ __slots__ = ()
+
+ def enter(self, node, key, parent, path, ancestors):
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/version.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/version.py
+index f733911..614df9f 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/version.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/version.py
+@@ -42,7 +42,7 @@ def get_complete_version(version=None):
+ then checks for correctness of the tuple provided.
+ """
+ if version is None:
+- from graphql import VERSION as version
++ from wandb_graphql import VERSION as version
+ else:
+ assert len(version) == 5
+ assert version[3] in ('alpha', 'beta', 'rc', 'final')
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/definition.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/definition.py
+index 65e3a90..1d0c4b4 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/definition.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/definition.py
+@@ -1,3 +1,4 @@
++from collections.abc import Mapping, Hashable
+ import collections
+ import copy
+
+@@ -188,7 +189,7 @@ def define_field_map(type, field_map):
+ if callable(field_map):
+ field_map = field_map()
+
+- assert isinstance(field_map, collections.Mapping) and len(field_map) > 0, (
++ assert isinstance(field_map, Mapping) and len(field_map) > 0, (
+ '{} fields must be a mapping (dict / OrderedDict) with field names as keys or a '
+ 'function which returns such a mapping.'
+ ).format(type)
+@@ -198,7 +199,7 @@ def define_field_map(type, field_map):
+ field_args = getattr(field, 'args', None)
+
+ if field_args:
+- assert isinstance(field_args, collections.Mapping), (
++ assert isinstance(field_args, Mapping), (
+ '{}.{} args must be a mapping (dict / OrderedDict) with argument names as keys.'.format(type,
+ field_name)
+ )
+@@ -407,7 +408,7 @@ class GraphQLEnumType(GraphQLType):
+ self.values = define_enum_values(self, values)
+
+ def serialize(self, value):
+- if isinstance(value, collections.Hashable):
++ if isinstance(value, Hashable):
+ enum_value = self._value_lookup.get(value)
+
+ if enum_value:
+@@ -416,7 +417,7 @@ class GraphQLEnumType(GraphQLType):
+ return None
+
+ def parse_value(self, value):
+- if isinstance(value, collections.Hashable):
++ if isinstance(value, Hashable):
+ enum_value = self._name_lookup.get(value)
+
+ if enum_value:
+@@ -441,7 +442,7 @@ class GraphQLEnumType(GraphQLType):
+
+
+ def define_enum_values(type, value_map):
+- assert isinstance(value_map, collections.Mapping) and len(value_map) > 0, (
++ assert isinstance(value_map, Mapping) and len(value_map) > 0, (
+ '{} values must be a mapping (dict / OrderedDict) with value names as keys.'.format(type)
+ )
+
+@@ -522,7 +523,7 @@ class GraphQLInputObjectType(GraphQLType):
+ if callable(fields):
+ fields = fields()
+
+- assert isinstance(fields, collections.Mapping) and len(fields) > 0, (
++ assert isinstance(fields, Mapping) and len(fields) > 0, (
+ '{} fields must be a mapping (dict / OrderedDict) with field names as keys or a '
+ 'function which returns such a mapping.'
+ ).format(self)
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/directives.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/directives.py
+index 06c920d..c566b4d 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/directives.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/directives.py
+@@ -1,4 +1,4 @@
+-import collections
++from collections.abc import Iterable, Mapping
+
+ from ..pyutils.ordereddict import OrderedDict
+ from ..utils.assert_valid_name import assert_valid_name
+@@ -52,14 +52,14 @@ class GraphQLDirective(object):
+ def __init__(self, name, description=None, args=None, locations=None):
+ assert name, 'Directive must be named.'
+ assert_valid_name(name)
+- assert isinstance(locations, collections.Iterable), 'Must provide locations for directive.'
++ assert isinstance(locations, Iterable), 'Must provide locations for directive.'
+
+ self.name = name
+ self.description = description
+ self.locations = locations
+
+ if args:
+- assert isinstance(args, collections.Mapping), '{} args must be a dict with argument names as keys.'.format(name)
++ assert isinstance(args, Mapping), '{} args must be a dict with argument names as keys.'.format(name)
+ for arg_name, _arg in args.items():
+ assert_valid_name(arg_name)
+ assert is_input_type(_arg.type), '{}({}) argument type must be Input Type but got {}.'.format(
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/scalars.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/scalars.py
+index bae7a9c..62955b5 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/scalars.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/scalars.py
+@@ -1,5 +1,3 @@
+-from six import string_types, text_type
+-
+ from ..language.ast import BooleanValue, FloatValue, IntValue, StringValue
+ from .definition import GraphQLScalarType
+
+@@ -68,20 +66,20 @@ GraphQLFloat = GraphQLScalarType(
+
+
+ def coerce_string(value):
+- if isinstance(value, string_types):
++ if isinstance(value, str):
+ return value
+
+ if isinstance(value, bool):
+ return u'true' if value else u'false'
+
+- return text_type(value)
++ return str(value)
+
+
+ def coerce_str(value):
+- if isinstance(value, string_types):
++ if isinstance(value, str):
+ return value
+
+- return text_type(value)
++ return str(value)
+
+
+ def parse_string_literal(ast):
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/schema.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/schema.py
+index 088183f..7f29b8f 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/schema.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/schema.py
+@@ -1,4 +1,4 @@
+-from collections import Iterable
++from collections.abc import Iterable
+
+ from .definition import GraphQLObjectType
+ from .directives import GraphQLDirective, specified_directives
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/typemap.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/typemap.py
+index 577a111..12733a6 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/typemap.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/typemap.py
+@@ -1,4 +1,5 @@
+-from collections import OrderedDict, Sequence, defaultdict
++from collections import OrderedDict, defaultdict
++from collections.abc import Sequence
+ from functools import reduce
+
+ from ..utils.type_comparators import is_equal_type, is_type_sub_type_of
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_from_value.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_from_value.py
+index d41a91d..1355ef5 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_from_value.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_from_value.py
+@@ -2,8 +2,6 @@ import json
+ import re
+ import sys
+
+-from six import string_types
+-
+ from ..language import ast
+ from ..type.definition import (GraphQLEnumType, GraphQLInputObjectType,
+ GraphQLList, GraphQLNonNull)
+@@ -40,7 +38,7 @@ def ast_from_value(value, type=None):
+
+ return ast.FloatValue(string_num)
+
+- if isinstance(value, string_types):
++ if isinstance(value, str):
+ if isinstance(type, GraphQLEnumType) and re.match(r'^[_a-zA-Z][_a-zA-Z0-9]*$', value):
+ return ast.EnumValue(value)
+
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_value.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_value.py
+index b3d5d05..eddce4f 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_value.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_value.py
+@@ -2,11 +2,9 @@
+ Implementation of isValidJSValue from graphql.s
+ """
+
+-import collections
++from collections.abc import Iterable, Mapping
+ import json
+
+-from six import string_types
+-
+ from ..type import (GraphQLEnumType, GraphQLInputObjectType, GraphQLList,
+ GraphQLNonNull, GraphQLScalarType)
+
+@@ -27,7 +25,7 @@ def is_valid_value(value, type):
+
+ if isinstance(type, GraphQLList):
+ item_type = type.of_type
+- if not isinstance(value, string_types) and isinstance(value, collections.Iterable):
++ if not isinstance(value, str) and isinstance(value, Iterable):
+ errors = []
+ for i, item in enumerate(value):
+ item_errors = is_valid_value(item, item_type)
+@@ -40,7 +38,7 @@ def is_valid_value(value, type):
+ return is_valid_value(value, item_type)
+
+ if isinstance(type, GraphQLInputObjectType):
+- if not isinstance(value, collections.Mapping):
++ if not isinstance(value, Mapping):
+ return [u'Expected "{}", found not an object.'.format(type)]
+
+ fields = type.fields
+diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_info.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_info.py
+index d81fd0d..6baf417 100644
+--- a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_info.py
++++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_info.py
+@@ -1,5 +1,3 @@
+-import six
+-
+ from ..language import visitor_meta
+ from ..type.definition import (GraphQLInputObjectType, GraphQLList,
+ get_named_type, get_nullable_type,
+@@ -14,8 +12,7 @@ def pop(lst):
+
+
+ # noinspection PyPep8Naming
+-@six.add_metaclass(visitor_meta.VisitorMeta)
+-class TypeInfo(object):
++class TypeInfo(metaclass=visitor_meta.VisitorMeta):
+ __slots__ = '_schema', '_type_stack', '_parent_type_stack', '_input_type_stack', '_field_def_stack', '_directive', \
+ '_argument', '_get_field_def_fn'
+
diff --git a/wandb/vendor/graphql-core-1.1/wandb-vendor.md b/wandb/vendor/graphql-core-1.1/wandb-vendor.md
new file mode 100644
index 0000000000000000000000000000000000000000..3c7c4099a2283d85e84a804afb3baa6b3ce7e009
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb-vendor.md
@@ -0,0 +1,7 @@
+
+# Modification steps
+
+```shell
+mv graphql wandb_graphql
+patch -p4 < wandb-vendor.diff
+```
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/__init__.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..d53457d40101cf306e08a2918db023290081fa28
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/__init__.py
@@ -0,0 +1,287 @@
+'''
+GraphQL.js provides a reference implementation for the GraphQL specification
+but is also a useful utility for operating on GraphQL files and building
+sophisticated tools.
+
+This primary module exports a general purpose function for fulfilling all
+steps of the GraphQL specification in a single operation, but also includes
+utilities for every part of the GraphQL specification:
+
+ - Parsing the GraphQL language.
+ - Building a GraphQL type schema.
+ - Validating a GraphQL request against a type schema.
+ - Executing a GraphQL request against a type schema.
+
+This also includes utility functions for operating on GraphQL types and
+GraphQL documents to facilitate building tools.
+
+You may also import from each sub-directory directly. For example, the
+following two import statements are equivalent:
+
+ from graphql import parse
+ from graphql.language.base import parse
+'''
+from .pyutils.version import get_version
+
+
+try:
+ # This variable is injected in the __builtins__ by the build
+ # process. It used to enable importing subpackages when
+ # the required packages are not installed
+ __GRAPHQL_SETUP__
+except NameError:
+ __GRAPHQL_SETUP__ = False
+
+
+VERSION = (1, 1, 0, 'final', 0)
+
+__version__ = get_version(VERSION)
+
+
+if not __GRAPHQL_SETUP__:
+ # The primary entry point into fulfilling a GraphQL request.
+ from .graphql import (
+ graphql
+ )
+
+ # Create and operate on GraphQL type definitions and schema.
+ from .type import ( # no import order
+ GraphQLSchema,
+
+ # Definitions
+ GraphQLScalarType,
+ GraphQLObjectType,
+ GraphQLInterfaceType,
+ GraphQLUnionType,
+ GraphQLEnumType,
+ GraphQLInputObjectType,
+ GraphQLList,
+ GraphQLNonNull,
+ GraphQLField,
+ GraphQLInputObjectField,
+ GraphQLArgument,
+
+ # "Enum" of Type Kinds
+ TypeKind,
+
+ # "Enum" of Directive locations
+ DirectiveLocation,
+
+ # Scalars
+ GraphQLInt,
+ GraphQLFloat,
+ GraphQLString,
+ GraphQLBoolean,
+ GraphQLID,
+
+ # Directive definition
+ GraphQLDirective,
+
+ # Built-in directives defined by the Spec
+ specified_directives,
+ GraphQLSkipDirective,
+ GraphQLIncludeDirective,
+ GraphQLDeprecatedDirective,
+
+ # Constant Deprecation Reason
+ DEFAULT_DEPRECATION_REASON,
+
+ # GraphQL Types for introspection.
+ __Schema,
+ __Directive,
+ __DirectiveLocation,
+ __Type,
+ __Field,
+ __InputValue,
+ __EnumValue,
+ __TypeKind,
+
+ # Meta-field definitions.
+ SchemaMetaFieldDef,
+ TypeMetaFieldDef,
+ TypeNameMetaFieldDef,
+
+ # Predicates
+ is_type,
+ is_input_type,
+ is_output_type,
+ is_leaf_type,
+ is_composite_type,
+ is_abstract_type,
+
+ # Un-modifiers
+ get_nullable_type,
+ get_named_type,
+ )
+
+ # Parse and operate on GraphQL language source files.
+ from .language.base import ( # no import order
+ Source,
+ get_location,
+
+ # Parse
+ parse,
+ parse_value,
+
+ # Print
+ print_ast,
+
+ # Visit
+ visit,
+ ParallelVisitor,
+ TypeInfoVisitor,
+ BREAK,
+ )
+
+ # Execute GraphQL queries.
+ from .execution import ( # no import order
+ execute,
+ MiddlewareManager,
+ middlewares
+ )
+
+ # Validate GraphQL queries.
+ from .validation import ( # no import order
+ validate,
+ specified_rules,
+ )
+
+ # Create and format GraphQL errors.
+ from .error import (
+ GraphQLError,
+ format_error,
+ )
+
+ # Utilities for operating on GraphQL type schema and parsed sources.
+ from .utils.base import (
+ # The GraphQL query recommended for a full schema introspection.
+ introspection_query,
+
+ # Gets the target Operation from a Document
+ get_operation_ast,
+
+ # Build a GraphQLSchema from an introspection result.
+ build_client_schema,
+
+ # Build a GraphQLSchema from a parsed GraphQL Schema language AST.
+ build_ast_schema,
+
+ # Extends an existing GraphQLSchema from a parsed GraphQL Schema
+ # language AST.
+ extend_schema,
+
+ # Print a GraphQLSchema to GraphQL Schema language.
+ print_schema,
+
+ # Create a GraphQLType from a GraphQL language AST.
+ type_from_ast,
+
+ # Create a JavaScript value from a GraphQL language AST.
+ value_from_ast,
+
+ # Create a GraphQL language AST from a JavaScript value.
+ ast_from_value,
+
+ # A helper to use within recursive-descent visitors which need to be aware of
+ # the GraphQL type system.
+ TypeInfo,
+
+ # Determine if JavaScript values adhere to a GraphQL type.
+ is_valid_value,
+
+ # Determine if AST values adhere to a GraphQL type.
+ is_valid_literal_value,
+
+ # Concatenates multiple AST together.
+ concat_ast,
+
+ # Comparators for types
+ is_equal_type,
+ is_type_sub_type_of,
+ do_types_overlap,
+
+ # Asserts a string is a valid GraphQL name.
+ assert_valid_name,
+ )
+
+ __all__ = (
+ 'graphql',
+ 'GraphQLBoolean',
+ 'GraphQLEnumType',
+ 'GraphQLFloat',
+ 'GraphQLID',
+ 'GraphQLInputObjectType',
+ 'GraphQLInt',
+ 'GraphQLInterfaceType',
+ 'GraphQLList',
+ 'GraphQLNonNull',
+ 'GraphQLField',
+ 'GraphQLInputObjectField',
+ 'GraphQLArgument',
+ 'GraphQLObjectType',
+ 'GraphQLScalarType',
+ 'GraphQLSchema',
+ 'GraphQLString',
+ 'GraphQLUnionType',
+ 'GraphQLDirective',
+ 'specified_directives',
+ 'GraphQLSkipDirective',
+ 'GraphQLIncludeDirective',
+ 'GraphQLDeprecatedDirective',
+ 'DEFAULT_DEPRECATION_REASON',
+ 'TypeKind',
+ 'DirectiveLocation',
+ '__Schema',
+ '__Directive',
+ '__DirectiveLocation',
+ '__Type',
+ '__Field',
+ '__InputValue',
+ '__EnumValue',
+ '__TypeKind',
+ 'SchemaMetaFieldDef',
+ 'TypeMetaFieldDef',
+ 'TypeNameMetaFieldDef',
+ 'get_named_type',
+ 'get_nullable_type',
+ 'is_abstract_type',
+ 'is_composite_type',
+ 'is_input_type',
+ 'is_leaf_type',
+ 'is_output_type',
+ 'is_type',
+ 'BREAK',
+ 'ParallelVisitor',
+ 'Source',
+ 'TypeInfoVisitor',
+ 'get_location',
+ 'parse',
+ 'parse_value',
+ 'print_ast',
+ 'visit',
+ 'execute',
+ 'MiddlewareManager',
+ 'middlewares',
+ 'specified_rules',
+ 'validate',
+ 'GraphQLError',
+ 'format_error',
+ 'TypeInfo',
+ 'assert_valid_name',
+ 'ast_from_value',
+ 'build_ast_schema',
+ 'build_client_schema',
+ 'concat_ast',
+ 'do_types_overlap',
+ 'extend_schema',
+ 'get_operation_ast',
+ 'introspection_query',
+ 'is_equal_type',
+ 'is_type_sub_type_of',
+ 'is_valid_literal_value',
+ 'is_valid_value',
+ 'print_schema',
+ 'type_from_ast',
+ 'value_from_ast',
+ 'get_version',
+ )
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/error/__init__.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/error/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..fdcf81493f1248b2c8bd503d1f0cf6ec83df7599
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/error/__init__.py
@@ -0,0 +1,6 @@
+from .base import GraphQLError
+from .located_error import GraphQLLocatedError
+from .syntax_error import GraphQLSyntaxError
+from .format_error import format_error
+
+__all__ = ['GraphQLError', 'GraphQLLocatedError', 'GraphQLSyntaxError', 'format_error']
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/error/base.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/error/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..74d644705fbe732c0688f6acec6042c07f84f59e
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/error/base.py
@@ -0,0 +1,42 @@
+from ..language.location import get_location
+
+
+class GraphQLError(Exception):
+ __slots__ = 'message', 'nodes', 'stack', 'original_error', '_source', '_positions'
+
+ def __init__(self, message, nodes=None, stack=None, source=None, positions=None):
+ super(GraphQLError, self).__init__(message)
+ self.message = message
+ self.nodes = nodes
+ self.stack = stack
+ self._source = source
+ self._positions = positions
+
+ @property
+ def source(self):
+ if self._source:
+ return self._source
+ if self.nodes:
+ node = self.nodes[0]
+ return node and node.loc and node.loc.source
+
+ @property
+ def positions(self):
+ if self._positions:
+ return self._positions
+ if self.nodes is not None:
+ node_positions = [node.loc and node.loc.start for node in self.nodes]
+ if any(node_positions):
+ return node_positions
+
+ def reraise(self):
+ if self.stack:
+ raise self.with_traceback(self.stack)
+ else:
+ raise self
+
+ @property
+ def locations(self):
+ source = self.source
+ if self.positions and source:
+ return [get_location(source, pos) for pos in self.positions]
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/error/format_error.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/error/format_error.py
new file mode 100644
index 0000000000000000000000000000000000000000..040955030421d78f887682e8251ab8f64b3426d4
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/error/format_error.py
@@ -0,0 +1,11 @@
+def format_error(error):
+ formatted_error = {
+ 'message': error.message,
+ }
+ if error.locations is not None:
+ formatted_error['locations'] = [
+ {'line': loc.line, 'column': loc.column}
+ for loc in error.locations
+ ]
+
+ return formatted_error
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/error/located_error.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/error/located_error.py
new file mode 100644
index 0000000000000000000000000000000000000000..9111f764ddc03487eb656a9c761a4453acd8d803
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/error/located_error.py
@@ -0,0 +1,29 @@
+import sys
+
+from .base import GraphQLError
+
+__all__ = ['GraphQLLocatedError']
+
+
+class GraphQLLocatedError(GraphQLError):
+
+ def __init__(self, nodes, original_error=None):
+ if original_error:
+ try:
+ message = str(original_error)
+ except UnicodeEncodeError:
+ message = original_error.message.encode('utf-8')
+ else:
+ message = 'An unknown error occurred.'
+
+ if hasattr(original_error, 'stack'):
+ stack = original_error.stack
+ else:
+ stack = sys.exc_info()[2]
+
+ super(GraphQLLocatedError, self).__init__(
+ message=message,
+ nodes=nodes,
+ stack=stack
+ )
+ self.original_error = original_error
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/error/syntax_error.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/error/syntax_error.py
new file mode 100644
index 0000000000000000000000000000000000000000..16eb487fb2dd9f29532ec30299f9ea486a1a90d9
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/error/syntax_error.py
@@ -0,0 +1,36 @@
+from ..language.location import get_location
+from .base import GraphQLError
+
+__all__ = ['GraphQLSyntaxError']
+
+
+class GraphQLSyntaxError(GraphQLError):
+
+ def __init__(self, source, position, description):
+ location = get_location(source, position)
+ super(GraphQLSyntaxError, self).__init__(
+ message=u'Syntax Error {} ({}:{}) {}\n\n{}'.format(
+ source.name,
+ location.line,
+ location.column,
+ description,
+ highlight_source_at_location(source, location),
+ ),
+ source=source,
+ positions=[position],
+ )
+
+
+def highlight_source_at_location(source, location):
+ line = location.line
+ lines = source.body.splitlines()
+ pad_len = len(str(line + 1))
+ result = u''
+ format = (u'{:>' + str(pad_len) + '}: {}\n').format
+ if line >= 2:
+ result += format(line - 1, lines[line - 2])
+ result += format(line, lines[line - 1])
+ result += ' ' * (1 + pad_len + location.column) + '^\n'
+ if line < len(lines):
+ result += format(line + 1, lines[line])
+ return result
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/__init__.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..546f425ce95a4c8b129dbd27d53afb56e2c37579
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/__init__.py
@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+"""
+Terminology
+
+"Definitions" are the generic name for top-level statements in the document.
+Examples of this include:
+1) Operations (such as a query)
+2) Fragments
+
+"Operations" are a generic name for requests in the document.
+Examples of this include:
+1) query,
+2) mutation
+
+"Selections" are the statements that can appear legally and at
+single level of the query. These include:
+1) field references e.g "a"
+2) fragment "spreads" e.g. "...c"
+3) inline fragment "spreads" e.g. "...on Type { a }"
+"""
+from .executor import execute
+from .base import ExecutionResult
+from .middleware import middlewares, MiddlewareManager
+
+
+__all__ = ['execute', 'ExecutionResult', 'MiddlewareManager', 'middlewares']
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/base.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..7929cd5a9b151cc2e65758e62cb2f8dfabae4a7f
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/base.py
@@ -0,0 +1,311 @@
+# -*- coding: utf-8 -*-
+from ..error import GraphQLError
+from ..language import ast
+from ..pyutils.default_ordered_dict import DefaultOrderedDict
+from ..type.definition import GraphQLInterfaceType, GraphQLUnionType
+from ..type.directives import GraphQLIncludeDirective, GraphQLSkipDirective
+from ..type.introspection import (SchemaMetaFieldDef, TypeMetaFieldDef,
+ TypeNameMetaFieldDef)
+from ..utils.type_from_ast import type_from_ast
+from .values import get_argument_values, get_variable_values
+
+Undefined = object()
+
+
+class ExecutionContext(object):
+ """Data that must be available at all points during query execution.
+
+ Namely, schema of the type system that is currently executing,
+ and the fragments defined in the query document"""
+
+ __slots__ = 'schema', 'fragments', 'root_value', 'operation', 'variable_values', 'errors', 'context_value', \
+ 'argument_values_cache', 'executor', 'middleware', '_subfields_cache'
+
+ def __init__(self, schema, document_ast, root_value, context_value, variable_values, operation_name, executor, middleware):
+ """Constructs a ExecutionContext object from the arguments passed
+ to execute, which we will pass throughout the other execution
+ methods."""
+ errors = []
+ operation = None
+ fragments = {}
+
+ for definition in document_ast.definitions:
+ if isinstance(definition, ast.OperationDefinition):
+ if not operation_name and operation:
+ raise GraphQLError('Must provide operation name if query contains multiple operations.')
+
+ if not operation_name or definition.name and definition.name.value == operation_name:
+ operation = definition
+
+ elif isinstance(definition, ast.FragmentDefinition):
+ fragments[definition.name.value] = definition
+
+ else:
+ raise GraphQLError(
+ u'GraphQL cannot execute a request containing a {}.'.format(definition.__class__.__name__),
+ definition
+ )
+
+ if not operation:
+ if operation_name:
+ raise GraphQLError(u'Unknown operation named "{}".'.format(operation_name))
+
+ else:
+ raise GraphQLError('Must provide an operation.')
+
+ variable_values = get_variable_values(schema, operation.variable_definitions or [], variable_values)
+
+ self.schema = schema
+ self.fragments = fragments
+ self.root_value = root_value
+ self.operation = operation
+ self.variable_values = variable_values
+ self.errors = errors
+ self.context_value = context_value
+ self.argument_values_cache = {}
+ self.executor = executor
+ self.middleware = middleware
+ self._subfields_cache = {}
+
+ def get_field_resolver(self, field_resolver):
+ if not self.middleware:
+ return field_resolver
+ return self.middleware.get_field_resolver(field_resolver)
+
+ def get_argument_values(self, field_def, field_ast):
+ k = field_def, field_ast
+ result = self.argument_values_cache.get(k)
+
+ if not result:
+ result = self.argument_values_cache[k] = get_argument_values(field_def.args, field_ast.arguments,
+ self.variable_values)
+
+ return result
+
+ def get_sub_fields(self, return_type, field_asts):
+ k = return_type, tuple(field_asts)
+ if k not in self._subfields_cache:
+ subfield_asts = DefaultOrderedDict(list)
+ visited_fragment_names = set()
+ for field_ast in field_asts:
+ selection_set = field_ast.selection_set
+ if selection_set:
+ subfield_asts = collect_fields(
+ self, return_type, selection_set,
+ subfield_asts, visited_fragment_names
+ )
+ self._subfields_cache[k] = subfield_asts
+ return self._subfields_cache[k]
+
+
+class ExecutionResult(object):
+ """The result of execution. `data` is the result of executing the
+ query, `errors` is null if no errors occurred, and is a
+ non-empty array if an error occurred."""
+
+ __slots__ = 'data', 'errors', 'invalid'
+
+ def __init__(self, data=None, errors=None, invalid=False):
+ self.data = data
+ self.errors = errors
+
+ if invalid:
+ assert data is None
+
+ self.invalid = invalid
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, ExecutionResult) and
+ self.data == other.data and
+ self.errors == other.errors and
+ self.invalid == other.invalid
+ )
+ )
+
+
+def get_operation_root_type(schema, operation):
+ op = operation.operation
+ if op == 'query':
+ return schema.get_query_type()
+
+ elif op == 'mutation':
+ mutation_type = schema.get_mutation_type()
+
+ if not mutation_type:
+ raise GraphQLError(
+ 'Schema is not configured for mutations',
+ [operation]
+ )
+
+ return mutation_type
+
+ elif op == 'subscription':
+ subscription_type = schema.get_subscription_type()
+
+ if not subscription_type:
+ raise GraphQLError(
+ 'Schema is not configured for subscriptions',
+ [operation]
+ )
+
+ return subscription_type
+
+ raise GraphQLError(
+ 'Can only execute queries, mutations and subscriptions',
+ [operation]
+ )
+
+
+def collect_fields(ctx, runtime_type, selection_set, fields, prev_fragment_names):
+ """
+ Given a selectionSet, adds all of the fields in that selection to
+ the passed in map of fields, and returns it at the end.
+
+ collect_fields requires the "runtime type" of an object. For a field which
+ returns and Interface or Union type, the "runtime type" will be the actual
+ Object type returned by that field.
+ """
+ for selection in selection_set.selections:
+ directives = selection.directives
+
+ if isinstance(selection, ast.Field):
+ if not should_include_node(ctx, directives):
+ continue
+
+ name = get_field_entry_key(selection)
+ fields[name].append(selection)
+
+ elif isinstance(selection, ast.InlineFragment):
+ if not should_include_node(
+ ctx, directives) or not does_fragment_condition_match(
+ ctx, selection, runtime_type):
+ continue
+
+ collect_fields(ctx, runtime_type, selection.selection_set, fields, prev_fragment_names)
+
+ elif isinstance(selection, ast.FragmentSpread):
+ frag_name = selection.name.value
+
+ if frag_name in prev_fragment_names or not should_include_node(ctx, directives):
+ continue
+
+ prev_fragment_names.add(frag_name)
+ fragment = ctx.fragments.get(frag_name)
+ frag_directives = fragment.directives
+ if not fragment or not \
+ should_include_node(ctx, frag_directives) or not \
+ does_fragment_condition_match(ctx, fragment, runtime_type):
+ continue
+
+ collect_fields(ctx, runtime_type, fragment.selection_set, fields, prev_fragment_names)
+
+ return fields
+
+
+def should_include_node(ctx, directives):
+ """Determines if a field should be included based on the @include and
+ @skip directives, where @skip has higher precidence than @include."""
+ # TODO: Refactor based on latest code
+ if directives:
+ skip_ast = None
+
+ for directive in directives:
+ if directive.name.value == GraphQLSkipDirective.name:
+ skip_ast = directive
+ break
+
+ if skip_ast:
+ args = get_argument_values(
+ GraphQLSkipDirective.args,
+ skip_ast.arguments,
+ ctx.variable_values,
+ )
+ if args.get('if') is True:
+ return False
+
+ include_ast = None
+
+ for directive in directives:
+ if directive.name.value == GraphQLIncludeDirective.name:
+ include_ast = directive
+ break
+
+ if include_ast:
+ args = get_argument_values(
+ GraphQLIncludeDirective.args,
+ include_ast.arguments,
+ ctx.variable_values,
+ )
+
+ if args.get('if') is False:
+ return False
+
+ return True
+
+
+def does_fragment_condition_match(ctx, fragment, type_):
+ type_condition_ast = fragment.type_condition
+ if not type_condition_ast:
+ return True
+
+ conditional_type = type_from_ast(ctx.schema, type_condition_ast)
+ if conditional_type.is_same_type(type_):
+ return True
+
+ if isinstance(conditional_type, (GraphQLInterfaceType, GraphQLUnionType)):
+ return ctx.schema.is_possible_type(conditional_type, type_)
+
+ return False
+
+
+def get_field_entry_key(node):
+ """Implements the logic to compute the key of a given field's entry"""
+ if node.alias:
+ return node.alias.value
+ return node.name.value
+
+
+class ResolveInfo(object):
+ __slots__ = ('field_name', 'field_asts', 'return_type', 'parent_type',
+ 'schema', 'fragments', 'root_value', 'operation', 'variable_values')
+
+ def __init__(self, field_name, field_asts, return_type, parent_type,
+ schema, fragments, root_value, operation, variable_values):
+ self.field_name = field_name
+ self.field_asts = field_asts
+ self.return_type = return_type
+ self.parent_type = parent_type
+ self.schema = schema
+ self.fragments = fragments
+ self.root_value = root_value
+ self.operation = operation
+ self.variable_values = variable_values
+
+
+def default_resolve_fn(source, args, context, info):
+ """If a resolve function is not given, then a default resolve behavior is used which takes the property of the source object
+ of the same name as the field and returns it as the result, or if it's a function, returns the result of calling that function."""
+ name = info.field_name
+ property = getattr(source, name, None)
+ if callable(property):
+ return property()
+ return property
+
+
+def get_field_def(schema, parent_type, field_name):
+ """This method looks up the field on the given type defintion.
+ It has special casing for the two introspection fields, __schema
+ and __typename. __typename is special because it can always be
+ queried as a field, even in situations where no other fields
+ are allowed, like on a Union. __schema could get automatically
+ added to the query type, but that would require mutating type
+ definitions, which would cause issues."""
+ if field_name == '__schema' and schema.get_query_type() == parent_type:
+ return SchemaMetaFieldDef
+ elif field_name == '__type' and schema.get_query_type() == parent_type:
+ return TypeMetaFieldDef
+ elif field_name == '__typename':
+ return TypeNameMetaFieldDef
+ return parent_type.fields.get(field_name)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executor.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executor.py
new file mode 100644
index 0000000000000000000000000000000000000000..e8e9fd293cd375cb50c158d9a8e2a1fd9d4cb566
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executor.py
@@ -0,0 +1,398 @@
+import collections
+from collections.abc import Iterable
+import functools
+import logging
+import sys
+
+from wandb_promise import Promise, promise_for_dict, is_thenable
+
+from ..error import GraphQLError, GraphQLLocatedError
+from ..pyutils.default_ordered_dict import DefaultOrderedDict
+from ..pyutils.ordereddict import OrderedDict
+from ..type import (GraphQLEnumType, GraphQLInterfaceType, GraphQLList,
+ GraphQLNonNull, GraphQLObjectType, GraphQLScalarType,
+ GraphQLSchema, GraphQLUnionType)
+from .base import (ExecutionContext, ExecutionResult, ResolveInfo, Undefined,
+ collect_fields, default_resolve_fn, get_field_def,
+ get_operation_root_type)
+from .executors.sync import SyncExecutor
+from .experimental.executor import execute as experimental_execute
+from .middleware import MiddlewareManager
+
+logger = logging.getLogger(__name__)
+
+
+use_experimental_executor = False
+
+
+def execute(schema, document_ast, root_value=None, context_value=None,
+ variable_values=None, operation_name=None, executor=None,
+ return_promise=False, middleware=None):
+ if use_experimental_executor:
+ return experimental_execute(
+ schema, document_ast, root_value, context_value,
+ variable_values, operation_name, executor,
+ return_promise, middleware
+ )
+
+ assert schema, 'Must provide schema'
+ assert isinstance(schema, GraphQLSchema), (
+ 'Schema must be an instance of GraphQLSchema. Also ensure that there are ' +
+ 'not multiple versions of GraphQL installed in your node_modules directory.'
+ )
+ if middleware:
+ if not isinstance(middleware, MiddlewareManager):
+ middleware = MiddlewareManager(*middleware)
+
+ assert isinstance(middleware, MiddlewareManager), (
+ 'middlewares have to be an instance'
+ ' of MiddlewareManager. Received "{}".'.format(middleware)
+ )
+
+ if executor is None:
+ executor = SyncExecutor()
+
+ context = ExecutionContext(
+ schema,
+ document_ast,
+ root_value,
+ context_value,
+ variable_values,
+ operation_name,
+ executor,
+ middleware
+ )
+
+ def executor(resolve, reject):
+ return resolve(execute_operation(context, context.operation, root_value))
+
+ def on_rejected(error):
+ context.errors.append(error)
+ return None
+
+ def on_resolve(data):
+ if not context.errors:
+ return ExecutionResult(data=data)
+ return ExecutionResult(data=data, errors=context.errors)
+
+ promise = Promise(executor).catch(on_rejected).then(on_resolve)
+ if return_promise:
+ return promise
+ context.executor.wait_until_finished()
+ return promise.get()
+
+
+def execute_operation(exe_context, operation, root_value):
+ type = get_operation_root_type(exe_context.schema, operation)
+ fields = collect_fields(
+ exe_context,
+ type,
+ operation.selection_set,
+ DefaultOrderedDict(list),
+ set()
+ )
+
+ if operation.operation == 'mutation':
+ return execute_fields_serially(exe_context, type, root_value, fields)
+
+ return execute_fields(exe_context, type, root_value, fields)
+
+
+def execute_fields_serially(exe_context, parent_type, source_value, fields):
+ def execute_field_callback(results, response_name):
+ field_asts = fields[response_name]
+ result = resolve_field(
+ exe_context,
+ parent_type,
+ source_value,
+ field_asts
+ )
+ if result is Undefined:
+ return results
+
+ if is_thenable(result):
+ def collect_result(resolved_result):
+ results[response_name] = resolved_result
+ return results
+
+ return result.then(collect_result, None)
+
+ results[response_name] = result
+ return results
+
+ def execute_field(prev_promise, response_name):
+ return prev_promise.then(lambda results: execute_field_callback(results, response_name))
+
+ return functools.reduce(execute_field, fields.keys(), Promise.resolve(collections.OrderedDict()))
+
+
+def execute_fields(exe_context, parent_type, source_value, fields):
+ contains_promise = False
+
+ final_results = OrderedDict()
+
+ for response_name, field_asts in fields.items():
+ result = resolve_field(exe_context, parent_type, source_value, field_asts)
+ if result is Undefined:
+ continue
+
+ final_results[response_name] = result
+ if is_thenable(result):
+ contains_promise = True
+
+ if not contains_promise:
+ return final_results
+
+ return promise_for_dict(final_results)
+
+
+def resolve_field(exe_context, parent_type, source, field_asts):
+ field_ast = field_asts[0]
+ field_name = field_ast.name.value
+
+ field_def = get_field_def(exe_context.schema, parent_type, field_name)
+ if not field_def:
+ return Undefined
+
+ return_type = field_def.type
+ resolve_fn = field_def.resolver or default_resolve_fn
+
+ # We wrap the resolve_fn from the middleware
+ resolve_fn_middleware = exe_context.get_field_resolver(resolve_fn)
+
+ # Build a dict of arguments from the field.arguments AST, using the variables scope to
+ # fulfill any variable references.
+ args = exe_context.get_argument_values(field_def, field_ast)
+
+ # The resolve function's optional third argument is a context value that
+ # is provided to every resolve function within an execution. It is commonly
+ # used to represent an authenticated user, or request-specific caches.
+ context = exe_context.context_value
+
+ # The resolve function's optional third argument is a collection of
+ # information about the current execution state.
+ info = ResolveInfo(
+ field_name,
+ field_asts,
+ return_type,
+ parent_type,
+ schema=exe_context.schema,
+ fragments=exe_context.fragments,
+ root_value=exe_context.root_value,
+ operation=exe_context.operation,
+ variable_values=exe_context.variable_values,
+ )
+
+ executor = exe_context.executor
+ result = resolve_or_error(resolve_fn_middleware, source, args, context, info, executor)
+
+ return complete_value_catching_error(
+ exe_context,
+ return_type,
+ field_asts,
+ info,
+ result
+ )
+
+
+def resolve_or_error(resolve_fn, source, args, context, info, executor):
+ try:
+ return executor.execute(resolve_fn, source, args, context, info)
+ except Exception as e:
+ logger.exception("An error occurred while resolving field {}.{}".format(
+ info.parent_type.name, info.field_name
+ ))
+ e.stack = sys.exc_info()[2]
+ return e
+
+
+def complete_value_catching_error(exe_context, return_type, field_asts, info, result):
+ # If the field type is non-nullable, then it is resolved without any
+ # protection from errors.
+ if isinstance(return_type, GraphQLNonNull):
+ return complete_value(exe_context, return_type, field_asts, info, result)
+
+ # Otherwise, error protection is applied, logging the error and
+ # resolving a null value for this field if one is encountered.
+ try:
+ completed = complete_value(exe_context, return_type, field_asts, info, result)
+ if is_thenable(completed):
+ def handle_error(error):
+ exe_context.errors.append(error)
+ return None
+
+ return completed.catch(handle_error)
+
+ return completed
+ except Exception as e:
+ exe_context.errors.append(e)
+ return None
+
+
+def complete_value(exe_context, return_type, field_asts, info, result):
+ """
+ Implements the instructions for completeValue as defined in the
+ "Field entries" section of the spec.
+
+ If the field type is Non-Null, then this recursively completes the value for the inner type. It throws a field
+ error if that completion returns null, as per the "Nullability" section of the spec.
+
+ If the field type is a List, then this recursively completes the value for the inner type on each item in the
+ list.
+
+ If the field type is a Scalar or Enum, ensures the completed value is a legal value of the type by calling the
+ `serialize` method of GraphQL type definition.
+
+ If the field is an abstract type, determine the runtime type of the value and then complete based on that type.
+
+ Otherwise, the field type expects a sub-selection set, and will complete the value by evaluating all
+ sub-selections.
+ """
+ # If field type is NonNull, complete for inner type, and throw field error if result is null.
+
+ if is_thenable(result):
+ return Promise.resolve(result).then(
+ lambda resolved: complete_value(
+ exe_context,
+ return_type,
+ field_asts,
+ info,
+ resolved
+ ),
+ lambda error: Promise.rejected(GraphQLLocatedError(field_asts, original_error=error))
+ )
+
+ # print return_type, type(result)
+ if isinstance(result, Exception):
+ raise GraphQLLocatedError(field_asts, original_error=result)
+
+ if isinstance(return_type, GraphQLNonNull):
+ return complete_nonnull_value(exe_context, return_type, field_asts, info, result)
+
+ # If result is null-like, return null.
+ if result is None:
+ return None
+
+ # If field type is List, complete each item in the list with the inner type
+ if isinstance(return_type, GraphQLList):
+ return complete_list_value(exe_context, return_type, field_asts, info, result)
+
+ # If field type is Scalar or Enum, serialize to a valid value, returning null if coercion is not possible.
+ if isinstance(return_type, (GraphQLScalarType, GraphQLEnumType)):
+ return complete_leaf_value(return_type, result)
+
+ if isinstance(return_type, (GraphQLInterfaceType, GraphQLUnionType)):
+ return complete_abstract_value(exe_context, return_type, field_asts, info, result)
+
+ if isinstance(return_type, GraphQLObjectType):
+ return complete_object_value(exe_context, return_type, field_asts, info, result)
+
+ assert False, u'Cannot complete value of unexpected type "{}".'.format(return_type)
+
+
+def complete_list_value(exe_context, return_type, field_asts, info, result):
+ """
+ Complete a list value by completing each item in the list with the inner type
+ """
+ assert isinstance(result, Iterable), \
+ ('User Error: expected iterable, but did not find one ' +
+ 'for field {}.{}.').format(info.parent_type, info.field_name)
+
+ item_type = return_type.of_type
+ completed_results = []
+ contains_promise = False
+ for item in result:
+ completed_item = complete_value_catching_error(exe_context, item_type, field_asts, info, item)
+ if not contains_promise and is_thenable(completed_item):
+ contains_promise = True
+
+ completed_results.append(completed_item)
+
+ return Promise.all(completed_results) if contains_promise else completed_results
+
+
+def complete_leaf_value(return_type, result):
+ """
+ Complete a Scalar or Enum by serializing to a valid value, returning null if serialization is not possible.
+ """
+ # serialize = getattr(return_type, 'serialize', None)
+ # assert serialize, 'Missing serialize method on type'
+
+ return return_type.serialize(result)
+
+
+def complete_abstract_value(exe_context, return_type, field_asts, info, result):
+ """
+ Complete an value of an abstract type by determining the runtime type of that value, then completing based
+ on that type.
+ """
+ runtime_type = None
+
+ # Field type must be Object, Interface or Union and expect sub-selections.
+ if isinstance(return_type, (GraphQLInterfaceType, GraphQLUnionType)):
+ if return_type.resolve_type:
+ runtime_type = return_type.resolve_type(result, exe_context.context_value, info)
+ else:
+ runtime_type = get_default_resolve_type_fn(result, exe_context.context_value, info, return_type)
+
+ if isinstance(runtime_type, str):
+ runtime_type = info.schema.get_type(runtime_type)
+
+ if not isinstance(runtime_type, GraphQLObjectType):
+ raise GraphQLError(
+ ('Abstract type {} must resolve to an Object type at runtime ' +
+ 'for field {}.{} with value "{}", received "{}".').format(
+ return_type,
+ info.parent_type,
+ info.field_name,
+ result,
+ runtime_type,
+ ),
+ field_asts
+ )
+
+ if not exe_context.schema.is_possible_type(return_type, runtime_type):
+ raise GraphQLError(
+ u'Runtime Object type "{}" is not a possible type for "{}".'.format(runtime_type, return_type),
+ field_asts
+ )
+
+ return complete_object_value(exe_context, runtime_type, field_asts, info, result)
+
+
+def get_default_resolve_type_fn(value, context, info, abstract_type):
+ possible_types = info.schema.get_possible_types(abstract_type)
+ for type in possible_types:
+ if callable(type.is_type_of) and type.is_type_of(value, context, info):
+ return type
+
+
+def complete_object_value(exe_context, return_type, field_asts, info, result):
+ """
+ Complete an Object value by evaluating all sub-selections.
+ """
+ if return_type.is_type_of and not return_type.is_type_of(result, exe_context.context_value, info):
+ raise GraphQLError(
+ u'Expected value of type "{}" but got: {}.'.format(return_type, type(result).__name__),
+ field_asts
+ )
+
+ # Collect sub-fields to execute to complete this value.
+ subfield_asts = exe_context.get_sub_fields(return_type, field_asts)
+ return execute_fields(exe_context, return_type, result, subfield_asts)
+
+
+def complete_nonnull_value(exe_context, return_type, field_asts, info, result):
+ """
+ Complete a NonNull value by completing the inner type
+ """
+ completed = complete_value(
+ exe_context, return_type.of_type, field_asts, info, result
+ )
+ if completed is None:
+ raise GraphQLError(
+ 'Cannot return null for non-nullable field {}.{}.'.format(info.parent_type, info.field_name),
+ field_asts
+ )
+
+ return completed
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/__init__.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/asyncio.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/asyncio.py
new file mode 100644
index 0000000000000000000000000000000000000000..eabeca1461d486cffacbc2d83bfe68315146c19d
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/asyncio.py
@@ -0,0 +1,53 @@
+from __future__ import absolute_import
+
+from asyncio import Future, get_event_loop, iscoroutine, wait
+
+from wandb_promise import Promise
+
+try:
+ from asyncio import ensure_future
+except ImportError:
+ # ensure_future is only implemented in Python 3.4.4+
+ def ensure_future(coro_or_future, loop=None):
+ """Wrap a coroutine or an awaitable in a future.
+
+ If the argument is a Future, it is returned directly.
+ """
+ if isinstance(coro_or_future, Future):
+ if loop is not None and loop is not coro_or_future._loop:
+ raise ValueError('loop argument must agree with Future')
+ return coro_or_future
+ elif iscoroutine(coro_or_future):
+ if loop is None:
+ loop = get_event_loop()
+ task = loop.create_task(coro_or_future)
+ if task._source_traceback:
+ del task._source_traceback[-1]
+ return task
+ else:
+ raise TypeError('A Future, a coroutine or an awaitable is required')
+
+
+class AsyncioExecutor(object):
+
+ def __init__(self, loop=None):
+ if loop is None:
+ loop = get_event_loop()
+ self.loop = loop
+ self.futures = []
+
+ def wait_until_finished(self):
+ # if there are futures to wait for
+ while self.futures:
+ # wait for the futures to finish
+ futures = self.futures
+ self.futures = []
+ self.loop.run_until_complete(wait(futures))
+
+ def execute(self, fn, *args, **kwargs):
+ result = fn(*args, **kwargs)
+ if isinstance(result, Future) or iscoroutine(result):
+ future = ensure_future(result, loop=self.loop)
+ self.futures.append(future)
+ return Promise.resolve(future)
+ return result
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/gevent.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/gevent.py
new file mode 100644
index 0000000000000000000000000000000000000000..67395d3f7f359c14899c229ded73f0e6883c79b3
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/gevent.py
@@ -0,0 +1,22 @@
+from __future__ import absolute_import
+
+import gevent
+from wandb_promise import Promise
+
+from .utils import process
+
+
+class GeventExecutor(object):
+
+ def __init__(self):
+ self.jobs = []
+
+ def wait_until_finished(self):
+ [j.join() for j in self.jobs]
+ # gevent.joinall(self.jobs)
+
+ def execute(self, fn, *args, **kwargs):
+ promise = Promise()
+ job = gevent.spawn(process, promise, fn, args, kwargs)
+ self.jobs.append(job)
+ return promise
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/process.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/process.py
new file mode 100644
index 0000000000000000000000000000000000000000..51301c864a5579c6a93df07da92e2ef520264c0c
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/process.py
@@ -0,0 +1,32 @@
+from multiprocessing import Process, Queue
+
+from wandb_promise import Promise
+
+from .utils import process
+
+
+def queue_process(q):
+ promise, fn, args, kwargs = q.get()
+ process(promise, fn, args, kwargs)
+
+
+class ProcessExecutor(object):
+
+ def __init__(self):
+ self.processes = []
+ self.q = Queue()
+
+ def wait_until_finished(self):
+ for _process in self.processes:
+ _process.join()
+ self.q.close()
+ self.q.join_thread()
+
+ def execute(self, fn, *args, **kwargs):
+ promise = Promise()
+
+ self.q.put([promise, fn, args, kwargs], False)
+ _process = Process(target=queue_process, args=(self.q))
+ _process.start()
+ self.processes.append(_process)
+ return promise
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/sync.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/sync.py
new file mode 100644
index 0000000000000000000000000000000000000000..85f8471b6a148bf22184b5b0d9ea06bff78c0b2e
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/sync.py
@@ -0,0 +1,7 @@
+class SyncExecutor(object):
+
+ def wait_until_finished(self):
+ pass
+
+ def execute(self, fn, *args, **kwargs):
+ return fn(*args, **kwargs)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/thread.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/thread.py
new file mode 100644
index 0000000000000000000000000000000000000000..28e8883608849e6012e936086f3f6131c0407678
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/thread.py
@@ -0,0 +1,35 @@
+from multiprocessing.pool import ThreadPool
+from threading import Thread
+
+from wandb_promise import Promise
+
+from .utils import process
+
+
+class ThreadExecutor(object):
+
+ pool = None
+
+ def __init__(self, pool=False):
+ self.threads = []
+ if pool:
+ self.execute = self.execute_in_pool
+ self.pool = ThreadPool(processes=pool)
+ else:
+ self.execute = self.execute_in_thread
+
+ def wait_until_finished(self):
+ for thread in self.threads:
+ thread.join()
+
+ def execute_in_thread(self, fn, *args, **kwargs):
+ promise = Promise()
+ thread = Thread(target=process, args=(promise, fn, args, kwargs))
+ thread.start()
+ self.threads.append(thread)
+ return promise
+
+ def execute_in_pool(self, fn, *args, **kwargs):
+ promise = Promise()
+ self.pool.map(lambda input: process(*input), [(promise, fn, args, kwargs)])
+ return promise
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/utils.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..4fc44875b8e42a4e08bce82c5826c2979572e389
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/utils.py
@@ -0,0 +1,6 @@
+def process(p, f, args, kwargs):
+ try:
+ val = f(*args, **kwargs)
+ p.do_resolve(val)
+ except Exception as e:
+ p.do_reject(e)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/__init__.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/executor.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/executor.py
new file mode 100644
index 0000000000000000000000000000000000000000..a0d9d99c4e89e576caa51f1070f133a2bcebc832
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/executor.py
@@ -0,0 +1,66 @@
+from wandb_promise import Promise
+
+from ...type import GraphQLSchema
+from ..base import ExecutionContext, ExecutionResult, get_operation_root_type
+from ..executors.sync import SyncExecutor
+from ..middleware import MiddlewareManager
+from .fragment import Fragment
+
+
+def execute(schema, document_ast, root_value=None, context_value=None,
+ variable_values=None, operation_name=None, executor=None,
+ return_promise=False, middleware=None):
+ assert schema, 'Must provide schema'
+ assert isinstance(schema, GraphQLSchema), (
+ 'Schema must be an instance of GraphQLSchema. Also ensure that there are ' +
+ 'not multiple versions of GraphQL installed in your node_modules directory.'
+ )
+ if middleware:
+ if not isinstance(middleware, MiddlewareManager):
+ middleware = MiddlewareManager(*middleware)
+ assert isinstance(middleware, MiddlewareManager), (
+ 'middlewares have to be an instance'
+ ' of MiddlewareManager. Received "{}".'.format(middleware)
+ )
+
+ if executor is None:
+ executor = SyncExecutor()
+
+ context = ExecutionContext(
+ schema,
+ document_ast,
+ root_value,
+ context_value,
+ variable_values,
+ operation_name,
+ executor,
+ middleware
+ )
+
+ def executor(resolve, reject):
+ return resolve(execute_operation(context, context.operation, root_value))
+
+ def on_rejected(error):
+ context.errors.append(error)
+ return None
+
+ def on_resolve(data):
+ if not context.errors:
+ return ExecutionResult(data=data)
+ return ExecutionResult(data=data, errors=context.errors)
+
+ promise = Promise(executor).catch(on_rejected).then(on_resolve)
+ if return_promise:
+ return promise
+ context.executor.wait_until_finished()
+ return promise.get()
+
+
+def execute_operation(exe_context, operation, root_value):
+ type = get_operation_root_type(exe_context.schema, operation)
+ execute_serially = operation.operation == 'mutation'
+
+ fragment = Fragment(type=type, field_asts=[operation], context=exe_context)
+ if execute_serially:
+ return fragment.resolve_serially(root_value)
+ return fragment.resolve(root_value)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/fragment.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/fragment.py
new file mode 100644
index 0000000000000000000000000000000000000000..ccef4b405066a3e77a923152e46b9cad4c2dcee1
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/fragment.py
@@ -0,0 +1,252 @@
+import functools
+
+from wandb_promise import Promise, is_thenable, promise_for_dict
+
+from ...pyutils.cached_property import cached_property
+from ...pyutils.default_ordered_dict import DefaultOrderedDict
+from ...type import (GraphQLInterfaceType, GraphQLList, GraphQLNonNull,
+ GraphQLObjectType, GraphQLUnionType)
+from ..base import ResolveInfo, Undefined, collect_fields, get_field_def
+from ..values import get_argument_values
+from ...error import GraphQLError
+try:
+ from itertools import izip as zip
+except:
+ pass
+
+
+def get_base_type(type):
+ if isinstance(type, (GraphQLList, GraphQLNonNull)):
+ return get_base_type(type.of_type)
+ return type
+
+
+def get_subfield_asts(context, return_type, field_asts):
+ subfield_asts = DefaultOrderedDict(list)
+ visited_fragment_names = set()
+ for field_ast in field_asts:
+ selection_set = field_ast.selection_set
+ if selection_set:
+ subfield_asts = collect_fields(
+ context, return_type, selection_set,
+ subfield_asts, visited_fragment_names
+ )
+ return subfield_asts
+
+
+def get_resolvers(context, type, field_asts):
+ from .resolver import field_resolver
+ subfield_asts = get_subfield_asts(context, type, field_asts)
+
+ for response_name, field_asts in subfield_asts.items():
+ field_ast = field_asts[0]
+ field_name = field_ast.name.value
+ field_def = get_field_def(context and context.schema, type, field_name)
+ if not field_def:
+ continue
+ field_base_type = get_base_type(field_def.type)
+ field_fragment = None
+ info = ResolveInfo(
+ field_name,
+ field_asts,
+ field_base_type,
+ parent_type=type,
+ schema=context and context.schema,
+ fragments=context and context.fragments,
+ root_value=context and context.root_value,
+ operation=context and context.operation,
+ variable_values=context and context.variable_values,
+ )
+ if isinstance(field_base_type, GraphQLObjectType):
+ field_fragment = Fragment(
+ type=field_base_type,
+ field_asts=field_asts,
+ info=info,
+ context=context
+ )
+ elif isinstance(field_base_type, (GraphQLInterfaceType, GraphQLUnionType)):
+ field_fragment = AbstractFragment(
+ abstract_type=field_base_type,
+ field_asts=field_asts,
+ info=info,
+ context=context
+ )
+ resolver = field_resolver(field_def, exe_context=context, info=info, fragment=field_fragment)
+ args = get_argument_values(
+ field_def.args,
+ field_ast.arguments,
+ context and context.variable_values
+ )
+ yield (response_name, Field(resolver, args, context and context.context_value, info))
+
+
+class Field(object):
+ __slots__ = ('fn', 'args', 'context', 'info')
+
+ def __init__(self, fn, args, context, info):
+ self.fn = fn
+ self.args = args
+ self.context = context
+ self.info = info
+
+ def execute(self, root):
+ return self.fn(root, self.args, self.context, self.info)
+
+
+class Fragment(object):
+
+ def __init__(self, type, field_asts, context=None, info=None):
+ self.type = type
+ self.field_asts = field_asts
+ self.context = context
+ self.info = info
+
+ @cached_property
+ def partial_resolvers(self):
+ return list(get_resolvers(
+ self.context,
+ self.type,
+ self.field_asts
+ ))
+
+ @cached_property
+ def fragment_container(self):
+ try:
+ fields = next(zip(*self.partial_resolvers))
+ except StopIteration:
+ fields = tuple()
+
+ class FragmentInstance(dict):
+ # def __init__(self):
+ # self.fields = fields
+ # _fields = ('c','b','a')
+ set = dict.__setitem__
+ # def set(self, name, value):
+ # self[name] = value
+
+ def __iter__(self):
+ return iter(fields)
+
+ return FragmentInstance
+
+ def have_type(self, root):
+ return not self.type.is_type_of or self.type.is_type_of(root, self.context.context_value, self.info)
+
+ def resolve(self, root):
+ if root and not self.have_type(root):
+ raise GraphQLError(
+ u'Expected value of type "{}" but got: {}.'.format(self.type, type(root).__name__),
+ self.info.field_asts
+ )
+
+ contains_promise = False
+
+ final_results = self.fragment_container()
+ # return OrderedDict(
+ # ((field_name, field_resolver(root, field_args, context, info))
+ # for field_name, field_resolver, field_args, context, info in self.partial_resolvers)
+ # )
+ for response_name, field_resolver in self.partial_resolvers:
+
+ result = field_resolver.execute(root)
+ if result is Undefined:
+ continue
+
+ if not contains_promise and is_thenable(result):
+ contains_promise = True
+
+ final_results[response_name] = result
+
+ if not contains_promise:
+ return final_results
+
+ return promise_for_dict(final_results)
+ # return {
+ # field_name: field_resolver(root, field_args, context, info)
+ # for field_name, field_resolver, field_args, context, info in self.partial_resolvers
+ # }
+
+ def resolve_serially(self, root):
+ def execute_field_callback(results, resolver):
+ response_name, field_resolver = resolver
+
+ result = field_resolver.execute(root)
+
+ if result is Undefined:
+ return results
+
+ if is_thenable(result):
+ def collect_result(resolved_result):
+ results[response_name] = resolved_result
+ return results
+
+ return result.then(collect_result)
+
+ results[response_name] = result
+ return results
+
+ def execute_field(prev_promise, resolver):
+ return prev_promise.then(lambda results: execute_field_callback(results, resolver))
+
+ return functools.reduce(execute_field, self.partial_resolvers, Promise.resolve(self.fragment_container()))
+
+ def __eq__(self, other):
+ return isinstance(other, Fragment) and (
+ other.type == self.type and
+ other.field_asts == self.field_asts and
+ other.context == self.context and
+ other.info == self.info
+ )
+
+
+class AbstractFragment(object):
+
+ def __init__(self, abstract_type, field_asts, context=None, info=None):
+ self.abstract_type = abstract_type
+ self.field_asts = field_asts
+ self.context = context
+ self.info = info
+ self._fragments = {}
+
+ @cached_property
+ def possible_types(self):
+ return self.context.schema.get_possible_types(self.abstract_type)
+
+ @cached_property
+ def possible_types_with_is_type_of(self):
+ return [
+ (type, type.is_type_of) for type in self.possible_types if callable(type.is_type_of)
+ ]
+
+ def get_fragment(self, type):
+ if isinstance(type, str):
+ type = self.context.schema.get_type(type)
+
+ if type not in self._fragments:
+ assert type in self.possible_types, (
+ 'Runtime Object type "{}" is not a possible type for "{}".'
+ ).format(type, self.abstract_type)
+ self._fragments[type] = Fragment(
+ type,
+ self.field_asts,
+ self.context,
+ self.info
+ )
+
+ return self._fragments[type]
+
+ def resolve_type(self, result):
+ return_type = self.abstract_type
+ context = self.context.context_value
+
+ if return_type.resolve_type:
+ return return_type.resolve_type(result, context, self.info)
+
+ for type, is_type_of in self.possible_types_with_is_type_of:
+ if is_type_of(result, context, self.info):
+ return type
+
+ def resolve(self, root):
+ _type = self.resolve_type(root)
+ fragment = self.get_fragment(_type)
+ return fragment.resolve(root)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/resolver.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/resolver.py
new file mode 100644
index 0000000000000000000000000000000000000000..4d39cacd1dc810ec6a4de2fa709077c248760983
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/resolver.py
@@ -0,0 +1,151 @@
+import sys
+from collections.abc import Iterable
+from functools import partial
+
+from wandb_promise import Promise, is_thenable
+
+from ...error import GraphQLError, GraphQLLocatedError
+from ...type import (GraphQLEnumType, GraphQLInterfaceType, GraphQLList,
+ GraphQLNonNull, GraphQLObjectType, GraphQLScalarType,
+ GraphQLUnionType)
+from ..base import default_resolve_fn
+from ...execution import executor
+from .utils import imap, normal_map
+
+
+def on_complete_resolver(on_error, __func, exe_context, info, __resolver, *args, **kwargs):
+ try:
+ result = __resolver(*args, **kwargs)
+ if isinstance(result, Exception):
+ return on_error(result)
+ # return Promise.resolve(result).then(__func).catch(on_error)
+ if is_thenable(result):
+ # TODO: Remove this, if a promise is resolved with an Exception,
+ # it should raise by default. This is fixing an old behavior
+ # in the Promise package
+ def on_resolve(value):
+ if isinstance(value, Exception):
+ return on_error(value)
+ return value
+ return result.then(on_resolve).then(__func).catch(on_error)
+ return __func(result)
+ except Exception as e:
+ return on_error(e)
+
+
+def complete_list_value(inner_resolver, exe_context, info, on_error, result):
+ if result is None:
+ return None
+
+ assert isinstance(result, Iterable), \
+ ('User Error: expected iterable, but did not find one ' +
+ 'for field {}.{}.').format(info.parent_type, info.field_name)
+
+ completed_results = normal_map(inner_resolver, result)
+
+ if not any(imap(is_thenable, completed_results)):
+ return completed_results
+
+ return Promise.all(completed_results).catch(on_error)
+
+
+def complete_nonnull_value(exe_context, info, result):
+ if result is None:
+ raise GraphQLError(
+ 'Cannot return null for non-nullable field {}.{}.'.format(info.parent_type, info.field_name),
+ info.field_asts
+ )
+ return result
+
+
+def complete_leaf_value(serialize, result):
+ if result is None:
+ return None
+ return serialize(result)
+
+
+def complete_object_value(fragment_resolve, exe_context, on_error, result):
+ if result is None:
+ return None
+
+ result = fragment_resolve(result)
+ if is_thenable(result):
+ return result.catch(on_error)
+ return result
+
+
+def field_resolver(field, fragment=None, exe_context=None, info=None):
+ # resolver = exe_context.get_field_resolver(field.resolver or default_resolve_fn)
+ resolver = field.resolver or default_resolve_fn
+ if exe_context:
+ # We decorate the resolver with the middleware
+ resolver = exe_context.get_field_resolver(resolver)
+ return type_resolver(field.type, resolver,
+ fragment, exe_context, info, catch_error=True)
+
+
+def type_resolver(return_type, resolver, fragment=None, exe_context=None, info=None, catch_error=False):
+ if isinstance(return_type, GraphQLNonNull):
+ return type_resolver_non_null(return_type, resolver, fragment, exe_context, info)
+
+ if isinstance(return_type, (GraphQLScalarType, GraphQLEnumType)):
+ return type_resolver_leaf(return_type, resolver, exe_context, info, catch_error)
+
+ if isinstance(return_type, (GraphQLList)):
+ return type_resolver_list(return_type, resolver, fragment, exe_context, info, catch_error)
+
+ if isinstance(return_type, (GraphQLObjectType)):
+ assert fragment and fragment.type == return_type, 'Fragment and return_type dont match'
+ return type_resolver_fragment(return_type, resolver, fragment, exe_context, info, catch_error)
+
+ if isinstance(return_type, (GraphQLInterfaceType, GraphQLUnionType)):
+ assert fragment, 'You need to pass a fragment to resolve a Interface or Union'
+ return type_resolver_fragment(return_type, resolver, fragment, exe_context, info, catch_error)
+
+ raise Exception("The resolver have to be created for a fragment")
+
+
+def on_error(exe_context, info, catch_error, e):
+ error = e
+ if not isinstance(e, (GraphQLLocatedError, GraphQLError)):
+ error = GraphQLLocatedError(info.field_asts, original_error=e)
+ if catch_error:
+ exe_context.errors.append(error)
+ executor.logger.exception("An error occurred while resolving field {}.{}".format(
+ info.parent_type.name, info.field_name
+ ))
+ error.stack = sys.exc_info()[2]
+ return None
+ raise error
+
+
+def type_resolver_fragment(return_type, resolver, fragment, exe_context, info, catch_error):
+ on_complete_type_error = partial(on_error, exe_context, info, catch_error)
+ complete_object_value_resolve = partial(
+ complete_object_value,
+ fragment.resolve,
+ exe_context,
+ on_complete_type_error)
+ on_resolve_error = partial(on_error, exe_context, info, catch_error)
+ return partial(on_complete_resolver, on_resolve_error, complete_object_value_resolve, exe_context, info, resolver)
+
+
+def type_resolver_non_null(return_type, resolver, fragment, exe_context, info): # no catch_error
+ resolver = type_resolver(return_type.of_type, resolver, fragment, exe_context, info)
+ nonnull_complete = partial(complete_nonnull_value, exe_context, info)
+ on_resolve_error = partial(on_error, exe_context, info, False)
+ return partial(on_complete_resolver, on_resolve_error, nonnull_complete, exe_context, info, resolver)
+
+
+def type_resolver_leaf(return_type, resolver, exe_context, info, catch_error):
+ leaf_complete = partial(complete_leaf_value, return_type.serialize)
+ on_resolve_error = partial(on_error, exe_context, info, catch_error)
+ return partial(on_complete_resolver, on_resolve_error, leaf_complete, exe_context, info, resolver)
+
+
+def type_resolver_list(return_type, resolver, fragment, exe_context, info, catch_error):
+ item_type = return_type.of_type
+ inner_resolver = type_resolver(item_type, lambda item: item, fragment, exe_context, info, catch_error=True)
+ on_resolve_error = partial(on_error, exe_context, info, catch_error)
+ list_complete = partial(complete_list_value, inner_resolver, exe_context, info, on_resolve_error)
+ return partial(on_complete_resolver, on_resolve_error, list_complete, exe_context, info, resolver)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/utils.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..8aef421d9b89f3d7853e68603768af4e1bfd5271
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/utils.py
@@ -0,0 +1,7 @@
+try:
+ from itertools import imap
+ normal_map = map
+except:
+ def normal_map(func, iter):
+ return list(map(func, iter))
+ imap = map
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/middleware.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/middleware.py
new file mode 100644
index 0000000000000000000000000000000000000000..7a18f2988857c62e23b66fb17e53cadfa70a6460
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/middleware.py
@@ -0,0 +1,57 @@
+import inspect
+from functools import partial
+from itertools import chain
+
+from wandb_promise import Promise
+
+MIDDLEWARE_RESOLVER_FUNCTION = 'resolve'
+
+
+class MiddlewareManager(object):
+
+ def __init__(self, *middlewares, **kwargs):
+ self.middlewares = middlewares
+ self.wrap_in_promise = kwargs.get('wrap_in_promise', True)
+ self._middleware_resolvers = list(get_middleware_resolvers(middlewares))
+ self._cached_resolvers = {}
+
+ def get_field_resolver(self, field_resolver):
+ if field_resolver not in self._cached_resolvers:
+ self._cached_resolvers[field_resolver] = middleware_chain(
+ field_resolver,
+ self._middleware_resolvers,
+ wrap_in_promise=self.wrap_in_promise,
+ )
+
+ return self._cached_resolvers[field_resolver]
+
+
+middlewares = MiddlewareManager
+
+
+def get_middleware_resolvers(middlewares):
+ for middleware in middlewares:
+ # If the middleware is a function instead of a class
+ if inspect.isfunction(middleware):
+ yield middleware
+ if not hasattr(middleware, MIDDLEWARE_RESOLVER_FUNCTION):
+ continue
+ yield getattr(middleware, MIDDLEWARE_RESOLVER_FUNCTION)
+
+
+def middleware_chain(func, middlewares, wrap_in_promise):
+ if not middlewares:
+ return func
+ if wrap_in_promise:
+ middlewares = chain((func, make_it_promise), middlewares)
+ else:
+ middlewares = chain((func,), middlewares)
+ last_func = None
+ for middleware in middlewares:
+ last_func = partial(middleware, last_func) if last_func else middleware
+
+ return last_func
+
+
+def make_it_promise(next, *a, **b):
+ return Promise.resolve(next(*a, **b))
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/values.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/values.py
new file mode 100644
index 0000000000000000000000000000000000000000..473ada13c2d8f9ab9029cad419cd0f195380969b
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/execution/values.py
@@ -0,0 +1,145 @@
+from collections.abc import Iterable
+import json
+
+from ..error import GraphQLError
+from ..language.printer import print_ast
+from ..type import (GraphQLEnumType, GraphQLInputObjectType, GraphQLList,
+ GraphQLNonNull, GraphQLScalarType, is_input_type)
+from ..utils.is_valid_value import is_valid_value
+from ..utils.type_from_ast import type_from_ast
+from ..utils.value_from_ast import value_from_ast
+
+__all__ = ['get_variable_values', 'get_argument_values']
+
+
+def get_variable_values(schema, definition_asts, inputs):
+ """Prepares an object map of variables of the correct type based on the provided variable definitions and arbitrary input.
+ If the input cannot be parsed to match the variable definitions, a GraphQLError will be thrown."""
+ if inputs is None:
+ inputs = {}
+
+ values = {}
+ for def_ast in definition_asts:
+ var_name = def_ast.variable.name.value
+ value = get_variable_value(schema, def_ast, inputs.get(var_name))
+ values[var_name] = value
+
+ return values
+
+
+def get_argument_values(arg_defs, arg_asts, variables=None):
+ """Prepares an object map of argument values given a list of argument
+ definitions and list of argument AST nodes."""
+ if not arg_defs:
+ return {}
+
+ if arg_asts:
+ arg_ast_map = {arg.name.value: arg for arg in arg_asts}
+ else:
+ arg_ast_map = {}
+
+ result = {}
+ for name, arg_def in arg_defs.items():
+ value_ast = arg_ast_map.get(name)
+ if value_ast:
+ value_ast = value_ast.value
+
+ value = value_from_ast(
+ value_ast,
+ arg_def.type,
+ variables
+ )
+
+ if value is None:
+ value = arg_def.default_value
+
+ if value is not None:
+ # We use out_name as the output name for the
+ # dict if exists
+ result[arg_def.out_name or name] = value
+
+ return result
+
+
+def get_variable_value(schema, definition_ast, input):
+ """Given a variable definition, and any value of input, return a value which adheres to the variable definition,
+ or throw an error."""
+ type = type_from_ast(schema, definition_ast.type)
+ variable = definition_ast.variable
+
+ if not type or not is_input_type(type):
+ raise GraphQLError(
+ 'Variable "${}" expected value of type "{}" which cannot be used as an input type.'.format(
+ variable.name.value,
+ print_ast(definition_ast.type),
+ ),
+ [definition_ast]
+ )
+
+ input_type = type
+ errors = is_valid_value(input, input_type)
+ if not errors:
+ if input is None:
+ default_value = definition_ast.default_value
+ if default_value:
+ return value_from_ast(default_value, input_type)
+
+ return coerce_value(input_type, input)
+
+ if input is None:
+ raise GraphQLError(
+ 'Variable "${}" of required type "{}" was not provided.'.format(
+ variable.name.value,
+ print_ast(definition_ast.type)
+ ),
+ [definition_ast]
+ )
+
+ message = (u'\n' + u'\n'.join(errors)) if errors else u''
+ raise GraphQLError(
+ 'Variable "${}" got invalid value {}.{}'.format(
+ variable.name.value,
+ json.dumps(input, sort_keys=True),
+ message
+ ),
+ [definition_ast]
+ )
+
+
+def coerce_value(type, value):
+ """Given a type and any value, return a runtime value coerced to match the type."""
+ if isinstance(type, GraphQLNonNull):
+ # Note: we're not checking that the result of coerceValue is
+ # non-null.
+ # We only call this function after calling isValidValue.
+ return coerce_value(type.of_type, value)
+
+ if value is None:
+ return None
+
+ if isinstance(type, GraphQLList):
+ item_type = type.of_type
+ if not isinstance(value, str) and isinstance(value, Iterable):
+ return [coerce_value(item_type, item) for item in value]
+ else:
+ return [coerce_value(item_type, value)]
+
+ if isinstance(type, GraphQLInputObjectType):
+ fields = type.fields
+ obj = {}
+ for field_name, field in fields.items():
+ field_value = coerce_value(field.type, value.get(field_name))
+ if field_value is None:
+ field_value = field.default_value
+
+ if field_value is not None:
+ # We use out_name as the output name for the
+ # dict if exists
+ obj[field.out_name or field_name] = field_value
+
+ return obj
+
+ assert isinstance(type, (GraphQLScalarType, GraphQLEnumType)), \
+ 'Must be input type'
+
+ return type.parse_value(value)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/graphql.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/graphql.py
new file mode 100644
index 0000000000000000000000000000000000000000..e8d311c94adec29f221ae2af13efa4a0e84816e2
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/graphql.py
@@ -0,0 +1,60 @@
+from .execution import ExecutionResult, execute
+from .language.ast import Document
+from .language.parser import parse
+from .language.source import Source
+from .validation import validate
+
+
+# This is the primary entry point function for fulfilling GraphQL operations
+# by parsing, validating, and executing a GraphQL document along side a
+# GraphQL schema.
+
+# More sophisticated GraphQL servers, such as those which persist queries,
+# may wish to separate the validation and execution phases to a static time
+# tooling step, and a server runtime step.
+
+# schema:
+# The GraphQL type system to use when validating and executing a query.
+# requestString:
+# A GraphQL language formatted string representing the requested operation.
+# rootValue:
+# The value provided as the first argument to resolver functions on the top
+# level type (e.g. the query object type).
+# variableValues:
+# A mapping of variable name to runtime value to use for all variables
+# defined in the requestString.
+# operationName:
+# The name of the operation to use if requestString contains multiple
+# possible operations. Can be omitted if requestString contains only
+# one operation.
+def graphql(schema, request_string='', root_value=None, context_value=None,
+ variable_values=None, operation_name=None, executor=None,
+ return_promise=False, middleware=None):
+ try:
+ if isinstance(request_string, Document):
+ ast = request_string
+ else:
+ source = Source(request_string, 'GraphQL request')
+ ast = parse(source)
+ validation_errors = validate(schema, ast)
+ if validation_errors:
+ return ExecutionResult(
+ errors=validation_errors,
+ invalid=True,
+ )
+ return execute(
+ schema,
+ ast,
+ root_value,
+ context_value,
+ operation_name=operation_name,
+ variable_values=variable_values or {},
+ executor=executor,
+ return_promise=return_promise,
+ middleware=middleware,
+ )
+ except Exception as e:
+ return ExecutionResult(
+ errors=[e],
+ invalid=True,
+ )
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/__init__.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/ast.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/ast.py
new file mode 100644
index 0000000000000000000000000000000000000000..6fffae84b8835e5d6a71e0b15534de730b607c91
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/ast.py
@@ -0,0 +1,1349 @@
+# This is autogenerated code. DO NOT change this manually.
+# Run scripts/generate_ast.py to generate this file.
+
+
+class Node(object):
+ __slots__ = ()
+
+
+class Definition(Node):
+ __slots__ = ()
+
+
+class Document(Node):
+ __slots__ = ('loc', 'definitions',)
+ _fields = ('definitions',)
+
+ def __init__(self, definitions, loc=None):
+ self.loc = loc
+ self.definitions = definitions
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, Document) and
+ # self.loc == other.loc and
+ self.definitions == other.definitions
+ )
+ )
+
+ def __repr__(self):
+ return ('Document('
+ 'definitions={self.definitions!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.definitions,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class OperationDefinition(Definition):
+ __slots__ = ('loc', 'operation', 'name', 'variable_definitions', 'directives', 'selection_set',)
+ _fields = ('operation', 'name', 'variable_definitions', 'directives', 'selection_set',)
+
+ def __init__(self, operation, selection_set, name=None, variable_definitions=None, directives=None, loc=None):
+ self.loc = loc
+ self.operation = operation
+ self.name = name
+ self.variable_definitions = variable_definitions
+ self.directives = directives
+ self.selection_set = selection_set
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, OperationDefinition) and
+ # self.loc == other.loc and
+ self.operation == other.operation and
+ self.name == other.name and
+ self.variable_definitions == other.variable_definitions and
+ self.directives == other.directives and
+ self.selection_set == other.selection_set
+ )
+ )
+
+ def __repr__(self):
+ return ('OperationDefinition('
+ 'operation={self.operation!r}'
+ ', name={self.name!r}'
+ ', variable_definitions={self.variable_definitions!r}'
+ ', directives={self.directives!r}'
+ ', selection_set={self.selection_set!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.operation,
+ self.selection_set,
+ self.name,
+ self.variable_definitions,
+ self.directives,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class VariableDefinition(Node):
+ __slots__ = ('loc', 'variable', 'type', 'default_value',)
+ _fields = ('variable', 'type', 'default_value',)
+
+ def __init__(self, variable, type, default_value=None, loc=None):
+ self.loc = loc
+ self.variable = variable
+ self.type = type
+ self.default_value = default_value
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, VariableDefinition) and
+ # self.loc == other.loc and
+ self.variable == other.variable and
+ self.type == other.type and
+ self.default_value == other.default_value
+ )
+ )
+
+ def __repr__(self):
+ return ('VariableDefinition('
+ 'variable={self.variable!r}'
+ ', type={self.type!r}'
+ ', default_value={self.default_value!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.variable,
+ self.type,
+ self.default_value,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class SelectionSet(Node):
+ __slots__ = ('loc', 'selections',)
+ _fields = ('selections',)
+
+ def __init__(self, selections, loc=None):
+ self.loc = loc
+ self.selections = selections
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, SelectionSet) and
+ # self.loc == other.loc and
+ self.selections == other.selections
+ )
+ )
+
+ def __repr__(self):
+ return ('SelectionSet('
+ 'selections={self.selections!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.selections,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class Selection(Node):
+ __slots__ = ()
+
+
+class Field(Selection):
+ __slots__ = ('loc', 'alias', 'name', 'arguments', 'directives', 'selection_set',)
+ _fields = ('alias', 'name', 'arguments', 'directives', 'selection_set',)
+
+ def __init__(self, name, alias=None, arguments=None, directives=None, selection_set=None, loc=None):
+ self.loc = loc
+ self.alias = alias
+ self.name = name
+ self.arguments = arguments
+ self.directives = directives
+ self.selection_set = selection_set
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, Field) and
+ # self.loc == other.loc and
+ self.alias == other.alias and
+ self.name == other.name and
+ self.arguments == other.arguments and
+ self.directives == other.directives and
+ self.selection_set == other.selection_set
+ )
+ )
+
+ def __repr__(self):
+ return ('Field('
+ 'alias={self.alias!r}'
+ ', name={self.name!r}'
+ ', arguments={self.arguments!r}'
+ ', directives={self.directives!r}'
+ ', selection_set={self.selection_set!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.alias,
+ self.arguments,
+ self.directives,
+ self.selection_set,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class Argument(Node):
+ __slots__ = ('loc', 'name', 'value',)
+ _fields = ('name', 'value',)
+
+ def __init__(self, name, value, loc=None):
+ self.loc = loc
+ self.name = name
+ self.value = value
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, Argument) and
+ # self.loc == other.loc and
+ self.name == other.name and
+ self.value == other.value
+ )
+ )
+
+ def __repr__(self):
+ return ('Argument('
+ 'name={self.name!r}'
+ ', value={self.value!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.value,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class FragmentSpread(Selection):
+ __slots__ = ('loc', 'name', 'directives',)
+ _fields = ('name', 'directives',)
+
+ def __init__(self, name, directives=None, loc=None):
+ self.loc = loc
+ self.name = name
+ self.directives = directives
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, FragmentSpread) and
+ # self.loc == other.loc and
+ self.name == other.name and
+ self.directives == other.directives
+ )
+ )
+
+ def __repr__(self):
+ return ('FragmentSpread('
+ 'name={self.name!r}'
+ ', directives={self.directives!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.directives,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class InlineFragment(Selection):
+ __slots__ = ('loc', 'type_condition', 'directives', 'selection_set',)
+ _fields = ('type_condition', 'directives', 'selection_set',)
+
+ def __init__(self, type_condition, selection_set, directives=None, loc=None):
+ self.loc = loc
+ self.type_condition = type_condition
+ self.directives = directives
+ self.selection_set = selection_set
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, InlineFragment) and
+ # self.loc == other.loc and
+ self.type_condition == other.type_condition and
+ self.directives == other.directives and
+ self.selection_set == other.selection_set
+ )
+ )
+
+ def __repr__(self):
+ return ('InlineFragment('
+ 'type_condition={self.type_condition!r}'
+ ', directives={self.directives!r}'
+ ', selection_set={self.selection_set!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.type_condition,
+ self.selection_set,
+ self.directives,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class FragmentDefinition(Definition):
+ __slots__ = ('loc', 'name', 'type_condition', 'directives', 'selection_set',)
+ _fields = ('name', 'type_condition', 'directives', 'selection_set',)
+
+ def __init__(self, name, type_condition, selection_set, directives=None, loc=None):
+ self.loc = loc
+ self.name = name
+ self.type_condition = type_condition
+ self.directives = directives
+ self.selection_set = selection_set
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, FragmentDefinition) and
+ # self.loc == other.loc and
+ self.name == other.name and
+ self.type_condition == other.type_condition and
+ self.directives == other.directives and
+ self.selection_set == other.selection_set
+ )
+ )
+
+ def __repr__(self):
+ return ('FragmentDefinition('
+ 'name={self.name!r}'
+ ', type_condition={self.type_condition!r}'
+ ', directives={self.directives!r}'
+ ', selection_set={self.selection_set!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.type_condition,
+ self.selection_set,
+ self.directives,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class Value(Node):
+ __slots__ = ()
+
+
+class Variable(Value):
+ __slots__ = ('loc', 'name',)
+ _fields = ('name',)
+
+ def __init__(self, name, loc=None):
+ self.loc = loc
+ self.name = name
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, Variable) and
+ # self.loc == other.loc and
+ self.name == other.name
+ )
+ )
+
+ def __repr__(self):
+ return ('Variable('
+ 'name={self.name!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class IntValue(Value):
+ __slots__ = ('loc', 'value',)
+ _fields = ('value',)
+
+ def __init__(self, value, loc=None):
+ self.loc = loc
+ self.value = value
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, IntValue) and
+ # self.loc == other.loc and
+ self.value == other.value
+ )
+ )
+
+ def __repr__(self):
+ return ('IntValue('
+ 'value={self.value!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.value,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class FloatValue(Value):
+ __slots__ = ('loc', 'value',)
+ _fields = ('value',)
+
+ def __init__(self, value, loc=None):
+ self.loc = loc
+ self.value = value
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, FloatValue) and
+ # self.loc == other.loc and
+ self.value == other.value
+ )
+ )
+
+ def __repr__(self):
+ return ('FloatValue('
+ 'value={self.value!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.value,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class StringValue(Value):
+ __slots__ = ('loc', 'value',)
+ _fields = ('value',)
+
+ def __init__(self, value, loc=None):
+ self.loc = loc
+ self.value = value
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, StringValue) and
+ # self.loc == other.loc and
+ self.value == other.value
+ )
+ )
+
+ def __repr__(self):
+ return ('StringValue('
+ 'value={self.value!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.value,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class BooleanValue(Value):
+ __slots__ = ('loc', 'value',)
+ _fields = ('value',)
+
+ def __init__(self, value, loc=None):
+ self.loc = loc
+ self.value = value
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, BooleanValue) and
+ # self.loc == other.loc and
+ self.value == other.value
+ )
+ )
+
+ def __repr__(self):
+ return ('BooleanValue('
+ 'value={self.value!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.value,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class EnumValue(Value):
+ __slots__ = ('loc', 'value',)
+ _fields = ('value',)
+
+ def __init__(self, value, loc=None):
+ self.loc = loc
+ self.value = value
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, EnumValue) and
+ # self.loc == other.loc and
+ self.value == other.value
+ )
+ )
+
+ def __repr__(self):
+ return ('EnumValue('
+ 'value={self.value!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.value,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class ListValue(Value):
+ __slots__ = ('loc', 'values',)
+ _fields = ('values',)
+
+ def __init__(self, values, loc=None):
+ self.loc = loc
+ self.values = values
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, ListValue) and
+ # self.loc == other.loc and
+ self.values == other.values
+ )
+ )
+
+ def __repr__(self):
+ return ('ListValue('
+ 'values={self.values!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.values,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class ObjectValue(Value):
+ __slots__ = ('loc', 'fields',)
+ _fields = ('fields',)
+
+ def __init__(self, fields, loc=None):
+ self.loc = loc
+ self.fields = fields
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, ObjectValue) and
+ # self.loc == other.loc and
+ self.fields == other.fields
+ )
+ )
+
+ def __repr__(self):
+ return ('ObjectValue('
+ 'fields={self.fields!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.fields,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class ObjectField(Node):
+ __slots__ = ('loc', 'name', 'value',)
+ _fields = ('name', 'value',)
+
+ def __init__(self, name, value, loc=None):
+ self.loc = loc
+ self.name = name
+ self.value = value
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, ObjectField) and
+ # self.loc == other.loc and
+ self.name == other.name and
+ self.value == other.value
+ )
+ )
+
+ def __repr__(self):
+ return ('ObjectField('
+ 'name={self.name!r}'
+ ', value={self.value!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.value,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class Directive(Node):
+ __slots__ = ('loc', 'name', 'arguments',)
+ _fields = ('name', 'arguments',)
+
+ def __init__(self, name, arguments=None, loc=None):
+ self.loc = loc
+ self.name = name
+ self.arguments = arguments
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, Directive) and
+ # self.loc == other.loc and
+ self.name == other.name and
+ self.arguments == other.arguments
+ )
+ )
+
+ def __repr__(self):
+ return ('Directive('
+ 'name={self.name!r}'
+ ', arguments={self.arguments!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.arguments,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class Type(Node):
+ __slots__ = ()
+
+
+class NamedType(Type):
+ __slots__ = ('loc', 'name',)
+ _fields = ('name',)
+
+ def __init__(self, name, loc=None):
+ self.loc = loc
+ self.name = name
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, NamedType) and
+ # self.loc == other.loc and
+ self.name == other.name
+ )
+ )
+
+ def __repr__(self):
+ return ('NamedType('
+ 'name={self.name!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class ListType(Type):
+ __slots__ = ('loc', 'type',)
+ _fields = ('type',)
+
+ def __init__(self, type, loc=None):
+ self.loc = loc
+ self.type = type
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, ListType) and
+ # self.loc == other.loc and
+ self.type == other.type
+ )
+ )
+
+ def __repr__(self):
+ return ('ListType('
+ 'type={self.type!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.type,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class NonNullType(Type):
+ __slots__ = ('loc', 'type',)
+ _fields = ('type',)
+
+ def __init__(self, type, loc=None):
+ self.loc = loc
+ self.type = type
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, NonNullType) and
+ # self.loc == other.loc and
+ self.type == other.type
+ )
+ )
+
+ def __repr__(self):
+ return ('NonNullType('
+ 'type={self.type!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.type,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class Name(Node):
+ __slots__ = ('loc', 'value',)
+ _fields = ('value',)
+
+ def __init__(self, value, loc=None):
+ self.loc = loc
+ self.value = value
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, Name) and
+ # self.loc == other.loc and
+ self.value == other.value
+ )
+ )
+
+ def __repr__(self):
+ return ('Name('
+ 'value={self.value!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.value,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+# Type System Definition
+
+class TypeDefinition(Node):
+ pass
+
+
+class TypeSystemDefinition(TypeDefinition):
+ pass
+
+
+class SchemaDefinition(TypeSystemDefinition):
+ __slots__ = ('loc', 'directives', 'operation_types',)
+ _fields = ('operation_types',)
+
+ def __init__(self, operation_types, loc=None, directives=None):
+ self.operation_types = operation_types
+ self.loc = loc
+ self.directives = directives
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, SchemaDefinition) and
+ self.operation_types == other.operation_types and
+ self.directives == other.directives
+ )
+ )
+
+ def __repr__(self):
+ return ('SchemaDefinition('
+ 'operation_types={self.operation_types!r}'
+ ', directives={self.directives!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.operation_types,
+ self.loc,
+ self.directives,
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class OperationTypeDefinition(Node):
+ __slots__ = ('loc', 'operation', 'type',)
+ _fields = ('operation', 'type',)
+
+ def __init__(self, operation, type, loc=None):
+ self.operation = operation
+ self.type = type
+ self.loc = loc
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, OperationTypeDefinition) and
+ self.operation == other.operation and
+ self.type == other.type
+ )
+ )
+
+ def __repr__(self):
+ return ('OperationTypeDefinition('
+ 'operation={self.operation!r}'
+ ', type={self.type!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.operation,
+ self.type,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class ObjectTypeDefinition(TypeDefinition):
+ __slots__ = ('loc', 'name', 'interfaces', 'directives', 'fields',)
+ _fields = ('name', 'interfaces', 'fields',)
+
+ def __init__(self, name, fields, interfaces=None, loc=None, directives=None):
+ self.loc = loc
+ self.name = name
+ self.interfaces = interfaces
+ self.fields = fields
+ self.directives = directives
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, ObjectTypeDefinition) and
+ # self.loc == other.loc and
+ self.name == other.name and
+ self.interfaces == other.interfaces and
+ self.fields == other.fields and
+ self.directives == other.directives
+ )
+ )
+
+ def __repr__(self):
+ return ('ObjectTypeDefinition('
+ 'name={self.name!r}'
+ ', interfaces={self.interfaces!r}'
+ ', fields={self.fields!r}'
+ ', directives={self.directives!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.fields,
+ self.interfaces,
+ self.loc,
+ self.directives,
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class FieldDefinition(Node):
+ __slots__ = ('loc', 'name', 'arguments', 'type', 'directives',)
+ _fields = ('name', 'arguments', 'type',)
+
+ def __init__(self, name, arguments, type, loc=None, directives=None):
+ self.loc = loc
+ self.name = name
+ self.arguments = arguments
+ self.type = type
+ self.directives = directives
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, FieldDefinition) and
+ # self.loc == other.loc and
+ self.name == other.name and
+ self.arguments == other.arguments and
+ self.type == other.type and
+ self.directives == other.directives
+ )
+ )
+
+ def __repr__(self):
+ return ('FieldDefinition('
+ 'name={self.name!r}'
+ ', arguments={self.arguments!r}'
+ ', type={self.type!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.arguments,
+ self.type,
+ self.loc,
+ self.directives,
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class InputValueDefinition(Node):
+ __slots__ = ('loc', 'name', 'type', 'default_value', 'directives')
+ _fields = ('name', 'type', 'default_value',)
+
+ def __init__(self, name, type, default_value=None, loc=None,
+ directives=None):
+ self.loc = loc
+ self.name = name
+ self.type = type
+ self.default_value = default_value
+ self.directives = directives
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, InputValueDefinition) and
+ # self.loc == other.loc and
+ self.name == other.name and
+ self.type == other.type and
+ self.default_value == other.default_value and
+ self.directives == other.directives
+ )
+ )
+
+ def __repr__(self):
+ return ('InputValueDefinition('
+ 'name={self.name!r}'
+ ', type={self.type!r}'
+ ', default_value={self.default_value!r}'
+ ', directives={self.directives!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.type,
+ self.default_value,
+ self.loc,
+ self.directives,
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class InterfaceTypeDefinition(TypeDefinition):
+ __slots__ = ('loc', 'name', 'fields', 'directives',)
+ _fields = ('name', 'fields',)
+
+ def __init__(self, name, fields, loc=None, directives=None):
+ self.loc = loc
+ self.name = name
+ self.fields = fields
+ self.directives = directives
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, InterfaceTypeDefinition) and
+ # self.loc == other.loc and
+ self.name == other.name and
+ self.fields == other.fields and
+ self.directives == other.directives
+ )
+ )
+
+ def __repr__(self):
+ return ('InterfaceTypeDefinition('
+ 'name={self.name!r}'
+ ', fields={self.fields!r}'
+ ', directives={self.directives!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.fields,
+ self.loc,
+ self.directives,
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class UnionTypeDefinition(TypeDefinition):
+ __slots__ = ('loc', 'name', 'types', 'directives',)
+ _fields = ('name', 'types',)
+
+ def __init__(self, name, types, loc=None, directives=None):
+ self.loc = loc
+ self.name = name
+ self.types = types
+ self.directives = directives
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, UnionTypeDefinition) and
+ # self.loc == other.loc and
+ self.name == other.name and
+ self.types == other.types and
+ self.directives == other.directives
+ )
+ )
+
+ def __repr__(self):
+ return ('UnionTypeDefinition('
+ 'name={self.name!r}'
+ ', types={self.types!r}'
+ ', directives={self.directives!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.types,
+ self.loc,
+ self.directives,
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class ScalarTypeDefinition(TypeDefinition):
+ __slots__ = ('loc', 'name', 'directives',)
+ _fields = ('name',)
+
+ def __init__(self, name, loc=None, directives=None):
+ self.loc = loc
+ self.name = name
+ self.directives = directives
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, ScalarTypeDefinition) and
+ # self.loc == other.loc and
+ self.name == other.name and
+ self.directives == other.directives
+ )
+ )
+
+ def __repr__(self):
+ return ('ScalarTypeDefinition('
+ 'name={self.name!r}'
+ 'directives={self.directives!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.loc,
+ self.directives
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class EnumTypeDefinition(TypeDefinition):
+ __slots__ = ('loc', 'name', 'values', 'directives',)
+ _fields = ('name', 'values',)
+
+ def __init__(self, name, values, loc=None, directives=None):
+ self.loc = loc
+ self.name = name
+ self.values = values
+ self.directives = directives
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, EnumTypeDefinition) and
+ # self.loc == other.loc and
+ self.name == other.name and
+ self.values == other.values and
+ self.directives == other.directives
+ )
+ )
+
+ def __repr__(self):
+ return ('EnumTypeDefinition('
+ 'name={self.name!r}'
+ ', values={self.values!r}'
+ ', directives={self.directives!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.values,
+ self.loc,
+ self.directives,
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class EnumValueDefinition(Node):
+ __slots__ = ('loc', 'name', 'directives',)
+ _fields = ('name',)
+
+ def __init__(self, name, loc=None, directives=None):
+ self.loc = loc
+ self.name = name
+ self.directives = directives
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, EnumValueDefinition) and
+ # self.loc == other.loc and
+ self.name == other.name and
+ self.directives == other.directives
+ )
+ )
+
+ def __repr__(self):
+ return ('EnumValueDefinition('
+ 'name={self.name!r}'
+ ', directives={self.directives!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.loc,
+ self.directives,
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class InputObjectTypeDefinition(TypeDefinition):
+ __slots__ = ('loc', 'name', 'fields', 'directives',)
+ _fields = ('name', 'fields',)
+
+ def __init__(self, name, fields, loc=None, directives=None):
+ self.loc = loc
+ self.name = name
+ self.fields = fields
+ self.directives = directives
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, InputObjectTypeDefinition) and
+ # self.loc == other.loc and
+ self.name == other.name and
+ self.fields == other.fields and
+ self.directives == other.directives
+ )
+ )
+
+ def __repr__(self):
+ return ('InputObjectTypeDefinition('
+ 'name={self.name!r}'
+ ', fields={self.fields!r}'
+ ', directives={self.directives!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.fields,
+ self.loc,
+ self.directives,
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class TypeExtensionDefinition(TypeSystemDefinition):
+ __slots__ = ('loc', 'definition',)
+ _fields = ('definition',)
+
+ def __init__(self, definition, loc=None):
+ self.loc = loc
+ self.definition = definition
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, TypeExtensionDefinition) and
+ # self.loc == other.loc and
+ self.definition == other.definition
+ )
+ )
+
+ def __repr__(self):
+ return ('TypeExtensionDefinition('
+ 'definition={self.definition!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.definition,
+ self.loc
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class DirectiveDefinition(TypeSystemDefinition):
+ __slots__ = ('loc', 'name', 'arguments', 'locations')
+ _fields = ('name', 'locations')
+
+ def __init__(self, name, locations, arguments=None, loc=None):
+ self.name = name
+ self.locations = locations
+ self.loc = loc
+ self.arguments = arguments
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, DirectiveDefinition) and
+ self.name == other.name and
+ self.locations == other.locations and
+ # self.loc == other.loc and
+ self.arguments == other.arguments
+ )
+ )
+
+ def __repr__(self):
+ return ('DirectiveDefinition('
+ 'name={self.name!r}, '
+ 'locations={self.locations!r}'
+ ')').format(self=self)
+
+ def __copy__(self):
+ return type(self)(
+ self.name,
+ self.locations,
+ self.arguments,
+ self.loc,
+ )
+
+ def __hash__(self):
+ return id(self)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/base.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..f6d9d91b0b66764daeabb1bade9200bf8142b701
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/base.py
@@ -0,0 +1,19 @@
+from .lexer import Lexer
+from .location import get_location
+from .parser import parse, parse_value
+from .printer import print_ast
+from .source import Source
+from .visitor import BREAK, ParallelVisitor, TypeInfoVisitor, visit
+
+__all__ = [
+ 'Lexer',
+ 'get_location',
+ 'parse',
+ 'parse_value',
+ 'print_ast',
+ 'Source',
+ 'BREAK',
+ 'ParallelVisitor',
+ 'TypeInfoVisitor',
+ 'visit',
+]
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/lexer.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/lexer.py
new file mode 100644
index 0000000000000000000000000000000000000000..3bdad3c586195bce43ae5d5a94a2a3c74f94a376
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/lexer.py
@@ -0,0 +1,435 @@
+import json
+
+from ..error import GraphQLSyntaxError
+
+__all__ = ['Token', 'Lexer', 'TokenKind',
+ 'get_token_desc', 'get_token_kind_desc']
+
+
+class Token(object):
+ __slots__ = 'kind', 'start', 'end', 'value'
+
+ def __init__(self, kind, start, end, value=None):
+ self.kind = kind
+ self.start = start
+ self.end = end
+ self.value = value
+
+ def __repr__(self):
+ return u'<Token kind={} at {}..{} value={}>'.format(
+ get_token_kind_desc(self.kind),
+ self.start,
+ self.end,
+ repr(self.value)
+ )
+
+ def __eq__(self, other):
+ return (self.kind == other.kind and
+ self.start == other.start and
+ self.end == other.end and
+ self.value == other.value)
+
+
+class Lexer(object):
+ __slots__ = 'source', 'prev_position'
+
+ def __init__(self, source):
+ self.source = source
+ self.prev_position = 0
+
+ def next_token(self, reset_position=None):
+ if reset_position is None:
+ reset_position = self.prev_position
+ token = read_token(self.source, reset_position)
+ self.prev_position = token.end
+ return token
+
+
+class TokenKind(object):
+ EOF = 1
+ BANG = 2
+ DOLLAR = 3
+ PAREN_L = 4
+ PAREN_R = 5
+ SPREAD = 6
+ COLON = 7
+ EQUALS = 8
+ AT = 9
+ BRACKET_L = 10
+ BRACKET_R = 11
+ BRACE_L = 12
+ PIPE = 13
+ BRACE_R = 14
+ NAME = 15
+ VARIABLE = 16
+ INT = 17
+ FLOAT = 18
+ STRING = 19
+
+
+def get_token_desc(token):
+ if token.value:
+ return u'{} "{}"'.format(
+ get_token_kind_desc(token.kind),
+ token.value
+ )
+ else:
+ return get_token_kind_desc(token.kind)
+
+
+def get_token_kind_desc(kind):
+ return TOKEN_DESCRIPTION[kind]
+
+
+TOKEN_DESCRIPTION = {
+ TokenKind.EOF: 'EOF',
+ TokenKind.BANG: '!',
+ TokenKind.DOLLAR: '$',
+ TokenKind.PAREN_L: '(',
+ TokenKind.PAREN_R: ')',
+ TokenKind.SPREAD: '...',
+ TokenKind.COLON: ':',
+ TokenKind.EQUALS: '=',
+ TokenKind.AT: '@',
+ TokenKind.BRACKET_L: '[',
+ TokenKind.BRACKET_R: ']',
+ TokenKind.BRACE_L: '{',
+ TokenKind.PIPE: '|',
+ TokenKind.BRACE_R: '}',
+ TokenKind.NAME: 'Name',
+ TokenKind.VARIABLE: 'Variable',
+ TokenKind.INT: 'Int',
+ TokenKind.FLOAT: 'Float',
+ TokenKind.STRING: 'String',
+}
+
+
+def char_code_at(s, pos):
+ if 0 <= pos < len(s):
+ return ord(s[pos])
+
+ return None
+
+
+PUNCT_CODE_TO_KIND = {
+ ord('!'): TokenKind.BANG,
+ ord('$'): TokenKind.DOLLAR,
+ ord('('): TokenKind.PAREN_L,
+ ord(')'): TokenKind.PAREN_R,
+ ord(':'): TokenKind.COLON,
+ ord('='): TokenKind.EQUALS,
+ ord('@'): TokenKind.AT,
+ ord('['): TokenKind.BRACKET_L,
+ ord(']'): TokenKind.BRACKET_R,
+ ord('{'): TokenKind.BRACE_L,
+ ord('|'): TokenKind.PIPE,
+ ord('}'): TokenKind.BRACE_R,
+}
+
+
+def print_char_code(code):
+ if code is None:
+ return '<EOF>'
+
+ if code < 0x007F:
+ return json.dumps(chr(code))
+
+ return '"\\u%04X"' % code
+
+
+def read_token(source, from_position):
+ """Gets the next token from the source starting at the given position.
+
+ This skips over whitespace and comments until it finds the next lexable
+ token, then lexes punctuators immediately or calls the appropriate
+ helper fucntion for more complicated tokens."""
+ body = source.body
+ body_length = len(body)
+
+ position = position_after_whitespace(body, from_position)
+
+ if position >= body_length:
+ return Token(TokenKind.EOF, position, position)
+
+ code = char_code_at(body, position)
+
+ if code < 0x0020 and code not in (0x0009, 0x000A, 0x000D):
+ raise GraphQLSyntaxError(
+ source, position,
+ u'Invalid character {}.'.format(print_char_code(code))
+ )
+
+ kind = PUNCT_CODE_TO_KIND.get(code)
+ if kind is not None:
+ return Token(kind, position, position + 1)
+
+ if code == 46: # .
+ if char_code_at(body, position + 1) == char_code_at(body, position + 2) == 46:
+ return Token(TokenKind.SPREAD, position, position + 3)
+
+ elif 65 <= code <= 90 or code == 95 or 97 <= code <= 122:
+ # A-Z, _, a-z
+ return read_name(source, position)
+
+ elif code == 45 or 48 <= code <= 57: # -, 0-9
+ return read_number(source, position, code)
+
+ elif code == 34: # "
+ return read_string(source, position)
+
+ raise GraphQLSyntaxError(
+ source, position,
+ u'Unexpected character {}.'.format(print_char_code(code)))
+
+
+ignored_whitespace_characters = frozenset([
+ # BOM
+ 0xFEFF,
+ # White Space
+ 0x0009, # tab
+ 0x0020, # space
+ # Line Terminator
+ 0x000A, # new line
+ 0x000D, # carriage return
+ # Comma
+ 0x002C
+])
+
+
+def position_after_whitespace(body, start_position):
+ """Reads from body starting at start_position until it finds a
+ non-whitespace or commented character, then returns the position of
+ that character for lexing."""
+ body_length = len(body)
+ position = start_position
+ while position < body_length:
+ code = char_code_at(body, position)
+ if code in ignored_whitespace_characters:
+ position += 1
+
+ elif code == 35: # #, skip comments
+ position += 1
+ while position < body_length:
+ code = char_code_at(body, position)
+ if not (code is not None and (code > 0x001F or code == 0x0009) and code not in (0x000A, 0x000D)):
+ break
+
+ position += 1
+ else:
+ break
+ return position
+
+
+def read_number(source, start, first_code):
+ """Reads a number token from the source file, either a float
+ or an int depending on whether a decimal point appears.
+ """
+ code = first_code
+ body = source.body
+ position = start
+ is_float = False
+
+ if code == 45: # -
+ position += 1
+ code = char_code_at(body, position)
+
+ if code == 48: # 0
+ position += 1
+ code = char_code_at(body, position)
+
+ if code is not None and 48 <= code <= 57:
+ raise GraphQLSyntaxError(
+ source,
+ position,
+ u'Invalid number, unexpected digit after 0: {}.'.format(print_char_code(code))
+ )
+ else:
+ position = read_digits(source, position, code)
+ code = char_code_at(body, position)
+
+ if code == 46: # .
+ is_float = True
+
+ position += 1
+ code = char_code_at(body, position)
+ position = read_digits(source, position, code)
+ code = char_code_at(body, position)
+
+ if code in (69, 101): # E e
+ is_float = True
+ position += 1
+ code = char_code_at(body, position)
+ if code in (43, 45): # + -
+ position += 1
+ code = char_code_at(body, position)
+
+ position = read_digits(source, position, code)
+
+ return Token(
+ TokenKind.FLOAT if is_float else TokenKind.INT,
+ start,
+ position,
+ body[start:position]
+ )
+
+
+def read_digits(source, start, first_code):
+ body = source.body
+ position = start
+ code = first_code
+
+ if code is not None and 48 <= code <= 57: # 0 - 9
+ while True:
+ position += 1
+ code = char_code_at(body, position)
+
+ if not (code is not None and 48 <= code <= 57):
+ break
+
+ return position
+
+ raise GraphQLSyntaxError(
+ source,
+ position,
+ u'Invalid number, expected digit but got: {}.'.format(print_char_code(code))
+ )
+
+
+ESCAPED_CHAR_CODES = {
+ 34: '"',
+ 47: '/',
+ 92: '\\',
+ 98: '\b',
+ 102: '\f',
+ 110: '\n',
+ 114: '\r',
+ 116: '\t',
+}
+
+
+def read_string(source, start):
+ """Reads a string token from the source file.
+
+ "([^"\\\u000A\u000D\u2028\u2029]|(\\(u[0-9a-fA-F]{4}|["\\/bfnrt])))*"
+ """
+ body = source.body
+ body_length = len(body)
+
+ position = start + 1
+ chunk_start = position
+ code = 0
+ value = []
+ append = value.append
+
+ while position < body_length:
+ code = char_code_at(body, position)
+ if not (
+ code is not None and
+ code not in (
+ # LineTerminator
+ 0x000A, 0x000D,
+ # Quote
+ 34
+ )
+ ):
+ break
+
+ if code < 0x0020 and code != 0x0009:
+ raise GraphQLSyntaxError(
+ source,
+ position,
+ u'Invalid character within String: {}.'.format(print_char_code(code))
+ )
+
+ position += 1
+ if code == 92: # \
+ append(body[chunk_start:position - 1])
+
+ code = char_code_at(body, position)
+ escaped = ESCAPED_CHAR_CODES.get(code)
+ if escaped is not None:
+ append(escaped)
+
+ elif code == 117: # u
+ char_code = uni_char_code(
+ char_code_at(body, position + 1) or 0,
+ char_code_at(body, position + 2) or 0,
+ char_code_at(body, position + 3) or 0,
+ char_code_at(body, position + 4) or 0,
+ )
+
+ if char_code < 0:
+ raise GraphQLSyntaxError(
+ source, position,
+ u'Invalid character escape sequence: \\u{}.'.format(body[position + 1: position + 5])
+ )
+
+ append(chr(char_code))
+ position += 4
+ else:
+ raise GraphQLSyntaxError(
+ source, position,
+ u'Invalid character escape sequence: \\{}.'.format(chr(code))
+ )
+
+ position += 1
+ chunk_start = position
+
+ if code != 34: # Quote (")
+ raise GraphQLSyntaxError(source, position, 'Unterminated string')
+
+ append(body[chunk_start:position])
+ return Token(TokenKind.STRING, start, position + 1, u''.join(value))
+
+
+def uni_char_code(a, b, c, d):
+ """Converts four hexidecimal chars to the integer that the
+ string represents. For example, uniCharCode('0','0','0','f')
+ will return 15, and uniCharCode('0','0','f','f') returns 255.
+
+ Returns a negative number on error, if a char was invalid.
+
+ This is implemented by noting that char2hex() returns -1 on error,
+ which means the result of ORing the char2hex() will also be negative.
+ """
+ return (char2hex(a) << 12 | char2hex(b) << 8 |
+ char2hex(c) << 4 | char2hex(d))
+
+
+def char2hex(a):
+ """Converts a hex character to its integer value.
+ '0' becomes 0, '9' becomes 9
+ 'A' becomes 10, 'F' becomes 15
+ 'a' becomes 10, 'f' becomes 15
+
+ Returns -1 on error."""
+ if 48 <= a <= 57: # 0-9
+ return a - 48
+ elif 65 <= a <= 70: # A-F
+ return a - 55
+ elif 97 <= a <= 102: # a-f
+ return a - 87
+ return -1
+
+
+def read_name(source, position):
+ """Reads an alphanumeric + underscore name from the source.
+
+ [_A-Za-z][_0-9A-Za-z]*"""
+ body = source.body
+ body_length = len(body)
+ end = position + 1
+
+ while end != body_length:
+ code = char_code_at(body, end)
+ if not (code is not None and (
+ code == 95 or # _
+ 48 <= code <= 57 or # 0-9
+ 65 <= code <= 90 or # A-Z
+ 97 <= code <= 122 # a-z
+ )):
+ break
+
+ end += 1
+
+ return Token(TokenKind.NAME, position, end, body[position:end])
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/location.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/location.py
new file mode 100644
index 0000000000000000000000000000000000000000..c478dcbd94d1db7d25f936a3784063e8b9c273b0
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/location.py
@@ -0,0 +1,30 @@
+__all__ = ['get_location', 'SourceLocation']
+
+
+class SourceLocation(object):
+ __slots__ = 'line', 'column'
+
+ def __init__(self, line, column):
+ self.line = line
+ self.column = column
+
+ def __repr__(self):
+ return '<SourceLocation line={} column={}>'.format(self.line, self.column)
+
+ def __eq__(self, other):
+ return (
+ isinstance(other, SourceLocation) and
+ self.line == other.line and
+ self.column == other.column
+ )
+
+
+def get_location(source, position):
+ lines = source.body[:position].splitlines()
+ if lines:
+ line = len(lines)
+ column = len(lines[-1]) + 1
+ else:
+ line = 1
+ column = 1
+ return SourceLocation(line, column)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/parser.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/parser.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2854a303095d4fcbbaed9f29a9e4dcca5e06db4
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/parser.py
@@ -0,0 +1,779 @@
+from . import ast
+from ..error import GraphQLSyntaxError
+from .lexer import Lexer, TokenKind, get_token_desc, get_token_kind_desc
+from .source import Source
+
+__all__ = ['parse']
+
+
+def parse(source, **kwargs):
+ """Given a GraphQL source, parses it into a Document."""
+ options = {'no_location': False, 'no_source': False}
+ options.update(kwargs)
+ source_obj = source
+
+ if isinstance(source, str):
+ source_obj = Source(source)
+
+ parser = Parser(source_obj, options)
+ return parse_document(parser)
+
+
+def parse_value(source, **kwargs):
+ options = {'no_location': False, 'no_source': False}
+ options.update(kwargs)
+ source_obj = source
+
+ if isinstance(source, str):
+ source_obj = Source(source)
+
+ parser = Parser(source_obj, options)
+ return parse_value_literal(parser, False)
+
+
+class Parser(object):
+ __slots__ = 'lexer', 'source', 'options', 'prev_end', 'token'
+
+ def __init__(self, source, options):
+ self.lexer = Lexer(source)
+ self.source = source
+ self.options = options
+ self.prev_end = 0
+ self.token = self.lexer.next_token()
+
+
+class Loc(object):
+ __slots__ = 'start', 'end', 'source'
+
+ def __init__(self, start, end, source=None):
+ self.start = start
+ self.end = end
+ self.source = source
+
+ def __repr__(self):
+ source = ' source={}'.format(self.source) if self.source else ''
+ return '<Loc start={} end={}{}>'.format(self.start, self.end, source)
+
+ def __eq__(self, other):
+ return (
+ isinstance(other, Loc) and
+ self.start == other.start and
+ self.end == other.end and
+ self.source == other.source
+ )
+
+
+def loc(parser, start):
+ """Returns a location object, used to identify the place in
+ the source that created a given parsed object."""
+ if parser.options['no_location']:
+ return None
+
+ if parser.options['no_source']:
+ return Loc(start, parser.prev_end)
+
+ return Loc(start, parser.prev_end, parser.source)
+
+
+def advance(parser):
+ """Moves the internal parser object to the next lexed token."""
+ prev_end = parser.token.end
+ parser.prev_end = prev_end
+ parser.token = parser.lexer.next_token(prev_end)
+
+
+def peek(parser, kind):
+ """Determines if the next token is of a given kind"""
+ return parser.token.kind == kind
+
+
+def skip(parser, kind):
+ """If the next token is of the given kind, return true after advancing
+ the parser. Otherwise, do not change the parser state
+ and throw an error."""
+ match = parser.token.kind == kind
+ if match:
+ advance(parser)
+
+ return match
+
+
+def expect(parser, kind):
+ """If the next token is of the given kind, return that token after
+ advancing the parser. Otherwise, do not change the parser state and
+ return False."""
+ token = parser.token
+ if token.kind == kind:
+ advance(parser)
+ return token
+
+ raise GraphQLSyntaxError(
+ parser.source,
+ token.start,
+ u'Expected {}, found {}'.format(
+ get_token_kind_desc(kind),
+ get_token_desc(token)
+ )
+ )
+
+
+def expect_keyword(parser, value):
+ """If the next token is a keyword with the given value, return that
+ token after advancing the parser. Otherwise, do not change the parser
+ state and return False."""
+ token = parser.token
+ if token.kind == TokenKind.NAME and token.value == value:
+ advance(parser)
+ return token
+
+ raise GraphQLSyntaxError(
+ parser.source,
+ token.start,
+ u'Expected "{}", found {}'.format(value, get_token_desc(token))
+ )
+
+
+def unexpected(parser, at_token=None):
+ """Helper function for creating an error when an unexpected lexed token
+ is encountered."""
+ token = at_token or parser.token
+ return GraphQLSyntaxError(
+ parser.source,
+ token.start,
+ u'Unexpected {}'.format(get_token_desc(token))
+ )
+
+
+def any(parser, open_kind, parse_fn, close_kind):
+ """Returns a possibly empty list of parse nodes, determined by
+ the parse_fn. This list begins with a lex token of openKind
+ and ends with a lex token of closeKind. Advances the parser
+ to the next lex token after the closing token."""
+ expect(parser, open_kind)
+ nodes = []
+ while not skip(parser, close_kind):
+ nodes.append(parse_fn(parser))
+
+ return nodes
+
+
+def many(parser, open_kind, parse_fn, close_kind):
+ """Returns a non-empty list of parse nodes, determined by
+ the parse_fn. This list begins with a lex token of openKind
+ and ends with a lex token of closeKind. Advances the parser
+ to the next lex token after the closing token."""
+ expect(parser, open_kind)
+ nodes = [parse_fn(parser)]
+ while not skip(parser, close_kind):
+ nodes.append(parse_fn(parser))
+
+ return nodes
+
+
+def parse_name(parser):
+ """Converts a name lex token into a name parse node."""
+ token = expect(parser, TokenKind.NAME)
+ return ast.Name(
+ value=token.value,
+ loc=loc(parser, token.start)
+ )
+
+
+# Implements the parsing rules in the Document section.
+
+def parse_document(parser):
+ start = parser.token.start
+ definitions = []
+ while True:
+ definitions.append(parse_definition(parser))
+
+ if skip(parser, TokenKind.EOF):
+ break
+
+ return ast.Document(
+ definitions=definitions,
+ loc=loc(parser, start)
+ )
+
+
+def parse_definition(parser):
+ if peek(parser, TokenKind.BRACE_L):
+ return parse_operation_definition(parser)
+
+ if peek(parser, TokenKind.NAME):
+ name = parser.token.value
+
+ if name in ('query', 'mutation', 'subscription'):
+ return parse_operation_definition(parser)
+ elif name == 'fragment':
+ return parse_fragment_definition(parser)
+ elif name in ('schema', 'scalar', 'type', 'interface', 'union', 'enum', 'input', 'extend', 'directive'):
+ return parse_type_system_definition(parser)
+
+ raise unexpected(parser)
+
+
+# Implements the parsing rules in the Operations section.
+def parse_operation_definition(parser):
+ start = parser.token.start
+ if peek(parser, TokenKind.BRACE_L):
+ return ast.OperationDefinition(
+ operation='query',
+ name=None,
+ variable_definitions=None,
+ directives=[],
+ selection_set=parse_selection_set(parser),
+ loc=loc(parser, start)
+ )
+
+ operation = parse_operation_type(parser)
+
+ name = None
+ if peek(parser, TokenKind.NAME):
+ name = parse_name(parser)
+
+ return ast.OperationDefinition(
+ operation=operation,
+ name=name,
+ variable_definitions=parse_variable_definitions(parser),
+ directives=parse_directives(parser),
+ selection_set=parse_selection_set(parser),
+ loc=loc(parser, start)
+ )
+
+
+def parse_operation_type(parser):
+ operation_token = expect(parser, TokenKind.NAME)
+ operation = operation_token.value
+ if operation == 'query':
+ return 'query'
+ elif operation == 'mutation':
+ return 'mutation'
+ elif operation == 'subscription':
+ return 'subscription'
+
+ raise unexpected(parser, operation_token)
+
+
+def parse_variable_definitions(parser):
+ if peek(parser, TokenKind.PAREN_L):
+ return many(
+ parser,
+ TokenKind.PAREN_L,
+ parse_variable_definition,
+ TokenKind.PAREN_R
+ )
+
+ return []
+
+
+def parse_variable_definition(parser):
+ start = parser.token.start
+
+ return ast.VariableDefinition(
+ variable=parse_variable(parser),
+ type=expect(parser, TokenKind.COLON) and parse_type(parser),
+ default_value=parse_value_literal(parser, True) if skip(parser, TokenKind.EQUALS) else None,
+ loc=loc(parser, start)
+ )
+
+
+def parse_variable(parser):
+ start = parser.token.start
+ expect(parser, TokenKind.DOLLAR)
+
+ return ast.Variable(
+ name=parse_name(parser),
+ loc=loc(parser, start)
+ )
+
+
+def parse_selection_set(parser):
+ start = parser.token.start
+ return ast.SelectionSet(
+ selections=many(parser, TokenKind.BRACE_L, parse_selection, TokenKind.BRACE_R),
+ loc=loc(parser, start)
+ )
+
+
+def parse_selection(parser):
+ if peek(parser, TokenKind.SPREAD):
+ return parse_fragment(parser)
+ else:
+ return parse_field(parser)
+
+
+def parse_field(parser):
+ # Corresponds to both Field and Alias in the spec
+ start = parser.token.start
+
+ name_or_alias = parse_name(parser)
+ if skip(parser, TokenKind.COLON):
+ alias = name_or_alias
+ name = parse_name(parser)
+ else:
+ alias = None
+ name = name_or_alias
+
+ return ast.Field(
+ alias=alias,
+ name=name,
+ arguments=parse_arguments(parser),
+ directives=parse_directives(parser),
+ selection_set=parse_selection_set(parser) if peek(parser, TokenKind.BRACE_L) else None,
+ loc=loc(parser, start)
+ )
+
+
+def parse_arguments(parser):
+ if peek(parser, TokenKind.PAREN_L):
+ return many(
+ parser, TokenKind.PAREN_L,
+ parse_argument, TokenKind.PAREN_R)
+
+ return []
+
+
+def parse_argument(parser):
+ start = parser.token.start
+
+ return ast.Argument(
+ name=parse_name(parser),
+ value=expect(parser, TokenKind.COLON) and parse_value_literal(parser, False),
+ loc=loc(parser, start)
+ )
+
+
+# Implements the parsing rules in the Fragments section.
+
+def parse_fragment(parser):
+ # Corresponds to both FragmentSpread and InlineFragment in the spec
+ start = parser.token.start
+ expect(parser, TokenKind.SPREAD)
+
+ if peek(parser, TokenKind.NAME) and parser.token.value != 'on':
+ return ast.FragmentSpread(
+ name=parse_fragment_name(parser),
+ directives=parse_directives(parser),
+ loc=loc(parser, start)
+ )
+
+ type_condition = None
+ if parser.token.value == 'on':
+ advance(parser)
+ type_condition = parse_named_type(parser)
+
+ return ast.InlineFragment(
+ type_condition=type_condition,
+ directives=parse_directives(parser),
+ selection_set=parse_selection_set(parser),
+ loc=loc(parser, start)
+ )
+
+
+def parse_fragment_definition(parser):
+ start = parser.token.start
+ expect_keyword(parser, 'fragment')
+
+ return ast.FragmentDefinition(
+ name=parse_fragment_name(parser),
+ type_condition=expect_keyword(parser, 'on') and parse_named_type(parser),
+ directives=parse_directives(parser),
+ selection_set=parse_selection_set(parser),
+ loc=loc(parser, start)
+ )
+
+
+def parse_fragment_name(parser):
+ if parser.token.value == 'on':
+ raise unexpected(parser)
+
+ return parse_name(parser)
+
+
+def parse_value_literal(parser, is_const):
+ token = parser.token
+ if token.kind == TokenKind.BRACKET_L:
+ return parse_list(parser, is_const)
+
+ elif token.kind == TokenKind.BRACE_L:
+ return parse_object(parser, is_const)
+
+ elif token.kind == TokenKind.INT:
+ advance(parser)
+ return ast.IntValue(value=token.value, loc=loc(parser, token.start))
+
+ elif token.kind == TokenKind.FLOAT:
+ advance(parser)
+ return ast.FloatValue(value=token.value, loc=loc(parser, token.start))
+
+ elif token.kind == TokenKind.STRING:
+ advance(parser)
+ return ast.StringValue(value=token.value, loc=loc(parser, token.start))
+
+ elif token.kind == TokenKind.NAME:
+ if token.value in ('true', 'false'):
+ advance(parser)
+ return ast.BooleanValue(value=token.value == 'true', loc=loc(parser, token.start))
+
+ if token.value != 'null':
+ advance(parser)
+ return ast.EnumValue(value=token.value, loc=loc(parser, token.start))
+
+ elif token.kind == TokenKind.DOLLAR:
+ if not is_const:
+ return parse_variable(parser)
+
+ raise unexpected(parser)
+
+
+# Implements the parsing rules in the Values section.
+def parse_variable_value(parser):
+ return parse_value_literal(parser, False)
+
+
+def parse_const_value(parser):
+ return parse_value_literal(parser, True)
+
+
+def parse_list(parser, is_const):
+ start = parser.token.start
+ item = parse_const_value if is_const else parse_variable_value
+
+ return ast.ListValue(
+ values=any(
+ parser, TokenKind.BRACKET_L,
+ item, TokenKind.BRACKET_R),
+ loc=loc(parser, start)
+ )
+
+
+def parse_object(parser, is_const):
+ start = parser.token.start
+ expect(parser, TokenKind.BRACE_L)
+ fields = []
+
+ while not skip(parser, TokenKind.BRACE_R):
+ fields.append(parse_object_field(parser, is_const))
+
+ return ast.ObjectValue(fields=fields, loc=loc(parser, start))
+
+
+def parse_object_field(parser, is_const):
+ start = parser.token.start
+ return ast.ObjectField(
+ name=parse_name(parser),
+ value=expect(parser, TokenKind.COLON) and parse_value_literal(parser, is_const),
+ loc=loc(parser, start)
+ )
+
+
+# Implements the parsing rules in the Directives section.
+
+def parse_directives(parser):
+ directives = []
+ while peek(parser, TokenKind.AT):
+ directives.append(parse_directive(parser))
+ return directives
+
+
+def parse_directive(parser):
+ start = parser.token.start
+ expect(parser, TokenKind.AT)
+
+ return ast.Directive(
+ name=parse_name(parser),
+ arguments=parse_arguments(parser),
+ loc=loc(parser, start),
+ )
+
+
+# Implements the parsing rules in the Types section.
+def parse_type(parser):
+ """Handles the 'Type': TypeName, ListType, and NonNullType
+ parsing rules."""
+ start = parser.token.start
+ if skip(parser, TokenKind.BRACKET_L):
+ ast_type = parse_type(parser)
+ expect(parser, TokenKind.BRACKET_R)
+ ast_type = ast.ListType(type=ast_type, loc=loc(parser, start))
+
+ else:
+ ast_type = parse_named_type(parser)
+
+ if skip(parser, TokenKind.BANG):
+ return ast.NonNullType(type=ast_type, loc=loc(parser, start))
+
+ return ast_type
+
+
+def parse_named_type(parser):
+ start = parser.token.start
+ return ast.NamedType(
+ name=parse_name(parser),
+ loc=loc(parser, start),
+ )
+
+
+def parse_type_system_definition(parser):
+ '''
+ TypeSystemDefinition :
+ - SchemaDefinition
+ - TypeDefinition
+ - TypeExtensionDefinition
+ - DirectiveDefinition
+
+ TypeDefinition :
+ - ScalarTypeDefinition
+ - ObjectTypeDefinition
+ - InterfaceTypeDefinition
+ - UnionTypeDefinition
+ - EnumTypeDefinition
+ - InputObjectTypeDefinition
+ '''
+ if not peek(parser, TokenKind.NAME):
+ raise unexpected(parser)
+
+ name = parser.token.value
+
+ if name == 'schema':
+ return parse_schema_definition(parser)
+
+ elif name == 'scalar':
+ return parse_scalar_type_definition(parser)
+
+ elif name == 'type':
+ return parse_object_type_definition(parser)
+
+ elif name == 'interface':
+ return parse_interface_type_definition(parser)
+
+ elif name == 'union':
+ return parse_union_type_definition(parser)
+
+ elif name == 'enum':
+ return parse_enum_type_definition(parser)
+
+ elif name == 'input':
+ return parse_input_object_type_definition(parser)
+
+ elif name == 'extend':
+ return parse_type_extension_definition(parser)
+
+ elif name == 'directive':
+ return parse_directive_definition(parser)
+
+ raise unexpected(parser)
+
+
+def parse_schema_definition(parser):
+ start = parser.token.start
+ expect_keyword(parser, 'schema')
+ directives = parse_directives(parser)
+ operation_types = many(
+ parser,
+ TokenKind.BRACE_L,
+ parse_operation_type_definition,
+ TokenKind.BRACE_R
+ )
+
+ return ast.SchemaDefinition(
+ directives=directives,
+ operation_types=operation_types,
+ loc=loc(parser, start)
+ )
+
+
+def parse_operation_type_definition(parser):
+ start = parser.token.start
+ operation = parse_operation_type(parser)
+ expect(parser, TokenKind.COLON)
+
+ return ast.OperationTypeDefinition(
+ operation=operation,
+ type=parse_named_type(parser),
+ loc=loc(parser, start)
+ )
+
+
+def parse_scalar_type_definition(parser):
+ start = parser.token.start
+ expect_keyword(parser, 'scalar')
+
+ return ast.ScalarTypeDefinition(
+ name=parse_name(parser),
+ directives=parse_directives(parser),
+ loc=loc(parser, start),
+ )
+
+
+def parse_object_type_definition(parser):
+ start = parser.token.start
+ expect_keyword(parser, 'type')
+ return ast.ObjectTypeDefinition(
+ name=parse_name(parser),
+ interfaces=parse_implements_interfaces(parser),
+ directives=parse_directives(parser),
+ fields=any(
+ parser,
+ TokenKind.BRACE_L,
+ parse_field_definition,
+ TokenKind.BRACE_R
+ ),
+ loc=loc(parser, start),
+ )
+
+
+def parse_implements_interfaces(parser):
+ types = []
+ if parser.token.value == 'implements':
+ advance(parser)
+
+ while True:
+ types.append(parse_named_type(parser))
+
+ if not peek(parser, TokenKind.NAME):
+ break
+
+ return types
+
+
+def parse_field_definition(parser):
+ start = parser.token.start
+
+ return ast.FieldDefinition(
+ name=parse_name(parser),
+ arguments=parse_argument_defs(parser),
+ type=expect(parser, TokenKind.COLON) and parse_type(parser),
+ directives=parse_directives(parser),
+ loc=loc(parser, start),
+ )
+
+
+def parse_argument_defs(parser):
+ if not peek(parser, TokenKind.PAREN_L):
+ return []
+
+ return many(parser, TokenKind.PAREN_L, parse_input_value_def, TokenKind.PAREN_R)
+
+
+def parse_input_value_def(parser):
+ start = parser.token.start
+
+ return ast.InputValueDefinition(
+ name=parse_name(parser),
+ type=expect(parser, TokenKind.COLON) and parse_type(parser),
+ default_value=parse_const_value(parser) if skip(parser, TokenKind.EQUALS) else None,
+ directives=parse_directives(parser),
+ loc=loc(parser, start),
+ )
+
+
+def parse_interface_type_definition(parser):
+ start = parser.token.start
+ expect_keyword(parser, 'interface')
+
+ return ast.InterfaceTypeDefinition(
+ name=parse_name(parser),
+ directives=parse_directives(parser),
+ fields=any(parser, TokenKind.BRACE_L, parse_field_definition, TokenKind.BRACE_R),
+ loc=loc(parser, start),
+ )
+
+
+def parse_union_type_definition(parser):
+ start = parser.token.start
+ expect_keyword(parser, 'union')
+
+ return ast.UnionTypeDefinition(
+ name=parse_name(parser),
+ directives=parse_directives(parser),
+ types=expect(parser, TokenKind.EQUALS) and parse_union_members(parser),
+ loc=loc(parser, start),
+ )
+
+
+def parse_union_members(parser):
+ members = []
+
+ while True:
+ members.append(parse_named_type(parser))
+
+ if not skip(parser, TokenKind.PIPE):
+ break
+
+ return members
+
+
+def parse_enum_type_definition(parser):
+ start = parser.token.start
+ expect_keyword(parser, 'enum')
+
+ return ast.EnumTypeDefinition(
+ name=parse_name(parser),
+ directives=parse_directives(parser),
+ values=many(parser, TokenKind.BRACE_L, parse_enum_value_definition, TokenKind.BRACE_R),
+ loc=loc(parser, start),
+ )
+
+
+def parse_enum_value_definition(parser):
+ start = parser.token.start
+
+ return ast.EnumValueDefinition(
+ name=parse_name(parser),
+ directives=parse_directives(parser),
+ loc=loc(parser, start),
+ )
+
+
+def parse_input_object_type_definition(parser):
+ start = parser.token.start
+ expect_keyword(parser, 'input')
+
+ return ast.InputObjectTypeDefinition(
+ name=parse_name(parser),
+ directives=parse_directives(parser),
+ fields=any(parser, TokenKind.BRACE_L, parse_input_value_def, TokenKind.BRACE_R),
+ loc=loc(parser, start),
+ )
+
+
+def parse_type_extension_definition(parser):
+ start = parser.token.start
+ expect_keyword(parser, 'extend')
+
+ return ast.TypeExtensionDefinition(
+ definition=parse_object_type_definition(parser),
+ loc=loc(parser, start)
+ )
+
+
+def parse_directive_definition(parser):
+ start = parser.token.start
+ expect_keyword(parser, 'directive')
+ expect(parser, TokenKind.AT)
+
+ name = parse_name(parser)
+ args = parse_argument_defs(parser)
+ expect_keyword(parser, 'on')
+
+ locations = parse_directive_locations(parser)
+ return ast.DirectiveDefinition(
+ name=name,
+ locations=locations,
+ arguments=args,
+ loc=loc(parser, start)
+ )
+
+
+def parse_directive_locations(parser):
+ locations = []
+
+ while True:
+ locations.append(parse_name(parser))
+
+ if not skip(parser, TokenKind.PIPE):
+ break
+
+ return locations
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/printer.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/printer.py
new file mode 100644
index 0000000000000000000000000000000000000000..a1a6dd360d51afff285d7f2822611dbbd3b5a510
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/printer.py
@@ -0,0 +1,193 @@
+import json
+
+from .visitor import Visitor, visit
+
+__all__ = ['print_ast']
+
+
+def print_ast(ast):
+ return visit(ast, PrintingVisitor())
+
+
+class PrintingVisitor(Visitor):
+ __slots__ = ()
+
+ def leave_Name(self, node, *args):
+ return node.value
+
+ def leave_Variable(self, node, *args):
+ return '$' + node.name
+
+ def leave_Document(self, node, *args):
+ return join(node.definitions, '\n\n') + '\n'
+
+ def leave_OperationDefinition(self, node, *args):
+ name = node.name
+ selection_set = node.selection_set
+ op = node.operation
+ var_defs = wrap('(', join(node.variable_definitions, ', '), ')')
+ directives = join(node.directives, ' ')
+
+ if not name and not directives and not var_defs and op == 'query':
+ return selection_set
+
+ return join([op, join([name, var_defs]), directives, selection_set], ' ')
+
+ def leave_VariableDefinition(self, node, *args):
+ return node.variable + ': ' + node.type + wrap(' = ', node.default_value)
+
+ def leave_SelectionSet(self, node, *args):
+ return block(node.selections)
+
+ def leave_Field(self, node, *args):
+ return join([
+ wrap('', node.alias, ': ') + node.name + wrap('(', join(node.arguments, ', '), ')'),
+ join(node.directives, ' '),
+ node.selection_set
+ ], ' ')
+
+ def leave_Argument(self, node, *args):
+ return node.name + ': ' + node.value
+
+ # Fragments
+
+ def leave_FragmentSpread(self, node, *args):
+ return '...' + node.name + wrap(' ', join(node.directives, ' '))
+
+ def leave_InlineFragment(self, node, *args):
+ return join([
+ '...',
+ wrap('on ', node.type_condition),
+ join(node.directives, ''),
+ node.selection_set
+ ], ' ')
+
+ def leave_FragmentDefinition(self, node, *args):
+ return ('fragment {} on {} '.format(node.name, node.type_condition) +
+ wrap('', join(node.directives, ' '), ' ') +
+ node.selection_set)
+
+ # Value
+
+ def leave_IntValue(self, node, *args):
+ return node.value
+
+ def leave_FloatValue(self, node, *args):
+ return node.value
+
+ def leave_StringValue(self, node, *args):
+ return json.dumps(node.value)
+
+ def leave_BooleanValue(self, node, *args):
+ return json.dumps(node.value)
+
+ def leave_EnumValue(self, node, *args):
+ return node.value
+
+ def leave_ListValue(self, node, *args):
+ return '[' + join(node.values, ', ') + ']'
+
+ def leave_ObjectValue(self, node, *args):
+ return '{' + join(node.fields, ', ') + '}'
+
+ def leave_ObjectField(self, node, *args):
+ return node.name + ': ' + node.value
+
+ # Directive
+
+ def leave_Directive(self, node, *args):
+ return '@' + node.name + wrap('(', join(node.arguments, ', '), ')')
+
+ # Type
+
+ def leave_NamedType(self, node, *args):
+ return node.name
+
+ def leave_ListType(self, node, *args):
+ return '[' + node.type + ']'
+
+ def leave_NonNullType(self, node, *args):
+ return node.type + '!'
+
+ # Type Definitions:
+
+ def leave_SchemaDefinition(self, node, *args):
+ return join([
+ 'schema',
+ join(node.directives, ' '),
+ block(node.operation_types),
+ ], ' ')
+
+ def leave_OperationTypeDefinition(self, node, *args):
+ return '{}: {}'.format(node.operation, node.type)
+
+ def leave_ScalarTypeDefinition(self, node, *args):
+ return 'scalar ' + node.name + wrap(' ', join(node.directives, ' '))
+
+ def leave_ObjectTypeDefinition(self, node, *args):
+ return join([
+ 'type',
+ node.name,
+ wrap('implements ', join(node.interfaces, ', ')),
+ join(node.directives, ' '),
+ block(node.fields)
+ ], ' ')
+
+ def leave_FieldDefinition(self, node, *args):
+ return (
+ node.name +
+ wrap('(', join(node.arguments, ', '), ')') +
+ ': ' +
+ node.type +
+ wrap(' ', join(node.directives, ' '))
+ )
+
+ def leave_InputValueDefinition(self, node, *args):
+ return node.name + ': ' + node.type + wrap(' = ', node.default_value) + wrap(' ', join(node.directives, ' '))
+
+ def leave_InterfaceTypeDefinition(self, node, *args):
+ return 'interface ' + node.name + wrap(' ', join(node.directives, ' ')) + ' ' + block(node.fields)
+
+ def leave_UnionTypeDefinition(self, node, *args):
+ return 'union ' + node.name + wrap(' ', join(node.directives, ' ')) + ' = ' + join(node.types, ' | ')
+
+ def leave_EnumTypeDefinition(self, node, *args):
+ return 'enum ' + node.name + wrap(' ', join(node.directives, ' ')) + ' ' + block(node.values)
+
+ def leave_EnumValueDefinition(self, node, *args):
+ return node.name + wrap(' ', join(node.directives, ' '))
+
+ def leave_InputObjectTypeDefinition(self, node, *args):
+ return 'input ' + node.name + wrap(' ', join(node.directives, ' ')) + ' ' + block(node.fields)
+
+ def leave_TypeExtensionDefinition(self, node, *args):
+ return 'extend ' + node.definition
+
+ def leave_DirectiveDefinition(self, node, *args):
+ return 'directive @{}{} on {}'.format(node.name, wrap(
+ '(', join(node.arguments, ', '), ')'), ' | '.join(node.locations))
+
+
+def join(maybe_list, separator=''):
+ if maybe_list:
+ return separator.join(filter(None, maybe_list))
+ return ''
+
+
+def block(_list):
+ '''Given a list, print each item on its own line, wrapped in an indented "{ }" block.'''
+ if _list:
+ return indent('{\n' + join(_list, '\n')) + '\n}'
+ return '{}'
+
+
+def wrap(start, maybe_str, end=''):
+ if maybe_str:
+ return start + maybe_str + end
+ return ''
+
+
+def indent(maybe_str):
+ if maybe_str:
+ return maybe_str.replace('\n', '\n ')
+ return maybe_str
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/source.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/source.py
new file mode 100644
index 0000000000000000000000000000000000000000..14e22fac7cc24237418e7845c3ca2ad6b76e1eb3
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/source.py
@@ -0,0 +1,18 @@
+__all__ = ['Source']
+
+
+class Source(object):
+ __slots__ = 'body', 'name'
+
+ def __init__(self, body, name='GraphQL'):
+ self.body = body
+ self.name = name
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, Source) and
+ self.body == other.body and
+ self.name == other.name
+ )
+ )
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor.py
new file mode 100644
index 0000000000000000000000000000000000000000..95cd69a04af1abb7161e6b3926fbe16bb0f1ac3c
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor.py
@@ -0,0 +1,222 @@
+from copy import copy
+
+from . import ast
+from .visitor_meta import QUERY_DOCUMENT_KEYS, VisitorMeta
+
+
+class Falsey(object):
+
+ def __nonzero__(self):
+ return False
+
+ def __bool__(self):
+ return False
+
+
+BREAK = object()
+REMOVE = Falsey()
+
+
+class Stack(object):
+ __slots__ = 'in_array', 'index', 'keys', 'edits', 'prev'
+
+ def __init__(self, in_array, index, keys, edits, prev):
+ self.in_array = in_array
+ self.index = index
+ self.keys = keys
+ self.edits = edits
+ self.prev = prev
+
+
+def visit(root, visitor, key_map=None):
+ visitor_keys = key_map or QUERY_DOCUMENT_KEYS
+
+ stack = None
+ in_array = isinstance(root, list)
+ keys = [root]
+ index = -1
+ edits = []
+ parent = None
+ path = []
+ ancestors = []
+ new_root = root
+ leave = visitor.leave
+ enter = visitor.enter
+ path_pop = path.pop
+ ancestors_pop = ancestors.pop
+ path_append = path.append
+ ancestors_append = ancestors.append
+
+ while True:
+ index += 1
+ is_leaving = index == len(keys)
+ is_edited = is_leaving and edits
+ if is_leaving:
+ key = path_pop() if ancestors else None
+ node = parent
+ parent = ancestors_pop() if ancestors else None
+
+ if is_edited:
+ if in_array:
+ node = list(node)
+
+ else:
+ node = copy(node)
+ edit_offset = 0
+ for edit_key, edit_value in edits:
+ if in_array:
+ edit_key -= edit_offset
+
+ if in_array and edit_value is REMOVE:
+ node.pop(edit_key)
+ edit_offset += 1
+
+ else:
+ if isinstance(node, list):
+ node[edit_key] = edit_value
+
+ else:
+ setattr(node, edit_key, edit_value)
+
+ index = stack.index
+ keys = stack.keys
+ edits = stack.edits
+ in_array = stack.in_array
+ stack = stack.prev
+
+ else:
+ if parent:
+ key = index if in_array else keys[index]
+ if isinstance(parent, list):
+ node = parent[key]
+
+ else:
+ node = getattr(parent, key, None)
+
+ else:
+ key = None
+ node = new_root
+
+ if node is REMOVE or node is None:
+ continue
+
+ if parent:
+ path_append(key)
+
+ result = None
+
+ if not isinstance(node, list):
+ assert isinstance(node, ast.Node), 'Invalid AST Node: ' + repr(node)
+
+ if is_leaving:
+ result = leave(node, key, parent, path, ancestors)
+
+ else:
+ result = enter(node, key, parent, path, ancestors)
+
+ if result is BREAK:
+ break
+
+ if result is False:
+ if not is_leaving:
+ path_pop()
+ continue
+
+ elif result is not None:
+ edits.append((key, result))
+ if not is_leaving:
+ if isinstance(result, ast.Node):
+ node = result
+
+ else:
+ path_pop()
+ continue
+
+ if result is None and is_edited:
+ edits.append((key, node))
+
+ if not is_leaving:
+ stack = Stack(in_array, index, keys, edits, stack)
+ in_array = isinstance(node, list)
+ keys = node if in_array else visitor_keys.get(type(node), None) or []
+ index = -1
+ edits = []
+
+ if parent:
+ ancestors_append(parent)
+
+ parent = node
+
+ if not stack:
+ break
+
+ if edits:
+ new_root = edits[-1][1]
+
+ return new_root
+
+
+class Visitor(metaclass=VisitorMeta):
+ __slots__ = ()
+
+ def enter(self, node, key, parent, path, ancestors):
+ method = self._get_enter_handler(type(node))
+ if method:
+ return method(self, node, key, parent, path, ancestors)
+
+ def leave(self, node, key, parent, path, ancestors):
+ method = self._get_leave_handler(type(node))
+ if method:
+ return method(self, node, key, parent, path, ancestors)
+
+
+class ParallelVisitor(Visitor):
+ __slots__ = 'skipping', 'visitors'
+
+ def __init__(self, visitors):
+ self.visitors = visitors
+ self.skipping = [None] * len(visitors)
+
+ def enter(self, node, key, parent, path, ancestors):
+ for i, visitor in enumerate(self.visitors):
+ if not self.skipping[i]:
+ result = visitor.enter(node, key, parent, path, ancestors)
+ if result is False:
+ self.skipping[i] = node
+ elif result is BREAK:
+ self.skipping[i] = BREAK
+ elif result is not None:
+ return result
+
+ def leave(self, node, key, parent, path, ancestors):
+ for i, visitor in enumerate(self.visitors):
+ if not self.skipping[i]:
+ result = visitor.leave(node, key, parent, path, ancestors)
+ if result is BREAK:
+ self.skipping[i] = BREAK
+ elif result is not None and result is not False:
+ return result
+ elif self.skipping[i] == node:
+ self.skipping[i] = REMOVE
+
+
+class TypeInfoVisitor(Visitor):
+ __slots__ = 'visitor', 'type_info'
+
+ def __init__(self, type_info, visitor):
+ self.type_info = type_info
+ self.visitor = visitor
+
+ def enter(self, node, key, parent, path, ancestors):
+ self.type_info.enter(node)
+ result = self.visitor.enter(node, key, parent, path, ancestors)
+ if result is not None:
+ self.type_info.leave(node)
+ if isinstance(result, ast.Node):
+ self.type_info.enter(result)
+ return result
+
+ def leave(self, node, key, parent, path, ancestors):
+ result = self.visitor.leave(node, key, parent, path, ancestors)
+ self.type_info.leave(node)
+ return result
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor_meta.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor_meta.py
new file mode 100644
index 0000000000000000000000000000000000000000..db2e640931abe178a2f0f58d11f0d96442fc1249
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor_meta.py
@@ -0,0 +1,82 @@
+from . import ast
+
+QUERY_DOCUMENT_KEYS = {
+ ast.Name: (),
+
+ ast.Document: ('definitions',),
+ ast.OperationDefinition: ('name', 'variable_definitions', 'directives', 'selection_set'),
+ ast.VariableDefinition: ('variable', 'type', 'default_value'),
+ ast.Variable: ('name',),
+ ast.SelectionSet: ('selections',),
+ ast.Field: ('alias', 'name', 'arguments', 'directives', 'selection_set'),
+ ast.Argument: ('name', 'value'),
+
+ ast.FragmentSpread: ('name', 'directives'),
+ ast.InlineFragment: ('type_condition', 'directives', 'selection_set'),
+ ast.FragmentDefinition: ('name', 'type_condition', 'directives', 'selection_set'),
+
+ ast.IntValue: (),
+ ast.FloatValue: (),
+ ast.StringValue: (),
+ ast.BooleanValue: (),
+ ast.EnumValue: (),
+ ast.ListValue: ('values',),
+ ast.ObjectValue: ('fields',),
+ ast.ObjectField: ('name', 'value'),
+
+ ast.Directive: ('name', 'arguments'),
+
+ ast.NamedType: ('name',),
+ ast.ListType: ('type',),
+ ast.NonNullType: ('type',),
+
+ ast.SchemaDefinition: ('directives', 'operation_types',),
+ ast.OperationTypeDefinition: ('type',),
+
+ ast.ScalarTypeDefinition: ('name', 'directives',),
+ ast.ObjectTypeDefinition: ('name', 'interfaces', 'directives', 'fields'),
+ ast.FieldDefinition: ('name', 'arguments', 'directives', 'type'),
+ ast.InputValueDefinition: ('name', 'type', 'directives', 'default_value'),
+ ast.InterfaceTypeDefinition: ('name', 'directives', 'fields'),
+ ast.UnionTypeDefinition: ('name', 'directives', 'types'),
+ ast.EnumTypeDefinition: ('name', 'directives', 'values'),
+ ast.EnumValueDefinition: ('name', 'directives',),
+ ast.InputObjectTypeDefinition: ('name', 'directives', 'fields'),
+
+ ast.TypeExtensionDefinition: ('definition',),
+
+ ast.DirectiveDefinition: ('name', 'arguments', 'locations'),
+}
+
+AST_KIND_TO_TYPE = {c.__name__: c for c in QUERY_DOCUMENT_KEYS.keys()}
+
+
+class VisitorMeta(type):
+
+ def __new__(cls, name, bases, attrs):
+ enter_handlers = {}
+ leave_handlers = {}
+
+ for base in bases:
+ if hasattr(base, '_enter_handlers'):
+ enter_handlers.update(base._enter_handlers)
+
+ if hasattr(base, '_leave_handlers'):
+ leave_handlers.update(base._leave_handlers)
+
+ for attr, val in attrs.items():
+ if attr.startswith('enter_'):
+ ast_kind = attr[6:]
+ ast_type = AST_KIND_TO_TYPE.get(ast_kind)
+ enter_handlers[ast_type] = val
+
+ elif attr.startswith('leave_'):
+ ast_kind = attr[6:]
+ ast_type = AST_KIND_TO_TYPE.get(ast_kind)
+ leave_handlers[ast_type] = val
+
+ attrs['_enter_handlers'] = enter_handlers
+ attrs['_leave_handlers'] = leave_handlers
+ attrs['_get_enter_handler'] = enter_handlers.get
+ attrs['_get_leave_handler'] = leave_handlers.get
+ return super(VisitorMeta, cls).__new__(cls, name, bases, attrs)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/__init__.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/cached_property.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/cached_property.py
new file mode 100644
index 0000000000000000000000000000000000000000..b5db6d48a9300e45cd68ff5a80f5d6085d5d4fca
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/cached_property.py
@@ -0,0 +1,17 @@
+class cached_property(object):
+ """ A property that is only computed once per instance and then replaces
+ itself with an ordinary attribute. Deleting the attribute resets the
+ property.
+
+ Source: https://github.com/bottlepy/bottle/commit/fa7733e075da0d790d809aa3d2f53071897e6f76
+ """
+
+ def __init__(self, func):
+ self.__doc__ = getattr(func, '__doc__')
+ self.func = func
+
+ def __get__(self, obj, cls):
+ if obj is None:
+ return self
+ value = obj.__dict__[self.func.__name__] = self.func(obj)
+ return value
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/contain_subset.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/contain_subset.py
new file mode 100644
index 0000000000000000000000000000000000000000..6c34936d4adff92254e4625fe12a55285378e79b
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/contain_subset.py
@@ -0,0 +1,28 @@
+obj = (dict, list, tuple)
+
+
+def contain_subset(expected, actual):
+ t_actual = type(actual)
+ t_expected = type(expected)
+ actual_is_dict = issubclass(t_actual, dict)
+ expected_is_dict = issubclass(t_expected, dict)
+ both_dicts = actual_is_dict and expected_is_dict
+ if not(both_dicts) and not(issubclass(t_actual, t_expected) or issubclass(t_expected, t_actual)):
+ return False
+ if not isinstance(expected, obj) or expected is None:
+ return expected == actual
+ if expected and not actual:
+ return False
+ if isinstance(expected, list):
+ aa = actual[:]
+ return all([any([contain_subset(exp, act) for act in aa]) for exp in expected])
+ for key in expected.keys():
+ eo = expected[key]
+ ao = actual.get(key)
+ if isinstance(eo, obj) and eo is not None and ao is not None:
+ if not contain_subset(eo, ao):
+ return False
+ continue
+ if ao != eo:
+ return False
+ return True
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/default_ordered_dict.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/default_ordered_dict.py
new file mode 100644
index 0000000000000000000000000000000000000000..e82a1be12fa1b876271c880abdd8391c316bca2c
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/default_ordered_dict.py
@@ -0,0 +1,40 @@
+import copy
+from collections import OrderedDict
+
+
+class DefaultOrderedDict(OrderedDict):
+ __slots__ = 'default_factory',
+
+ # Source: http://stackoverflow.com/a/6190500/562769
+ def __init__(self, default_factory=None, *a, **kw):
+ if default_factory is not None and not callable(default_factory):
+ raise TypeError('first argument must be callable')
+
+ OrderedDict.__init__(self, *a, **kw)
+ self.default_factory = default_factory
+
+ def __missing__(self, key):
+ if self.default_factory is None:
+ raise KeyError(key)
+ self[key] = value = self.default_factory()
+ return value
+
+ def __reduce__(self):
+ if self.default_factory is None:
+ args = tuple()
+ else:
+ args = self.default_factory,
+ return type(self), args, None, None, iter(self.items())
+
+ def copy(self):
+ return self.__copy__()
+
+ def __copy__(self):
+ return type(self)(self.default_factory, self)
+
+ def __deepcopy__(self, memo):
+ return self.__class__(self.default_factory, copy.deepcopy(list(self.items())))
+
+ def __repr__(self):
+ return 'DefaultOrderedDict(%s, %s)' % (self.default_factory,
+ OrderedDict.__repr__(self)[19:-1])
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/ordereddict.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/ordereddict.py
new file mode 100644
index 0000000000000000000000000000000000000000..ff341a4c86b8857fabf22fe2ef644b470e9586c6
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/ordereddict.py
@@ -0,0 +1,8 @@
+try:
+ # Try to load the Cython performant OrderedDict (C)
+ # as is more performant than collections.OrderedDict (Python)
+ from cyordereddict import OrderedDict
+except ImportError:
+ from collections import OrderedDict
+
+__all__ = ['OrderedDict']
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/pair_set.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/pair_set.py
new file mode 100644
index 0000000000000000000000000000000000000000..0af547a9d93aa3a2a6af356bd3d25d56104aa977
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/pair_set.py
@@ -0,0 +1,43 @@
+class PairSet(object):
+ __slots__ = '_data',
+
+ def __init__(self):
+ self._data = {}
+
+ def __contains__(self, item):
+ return self.has(item[0], item[1], item[2])
+
+ def __str__(self):
+ return str(self._data)
+
+ def __repr__(self):
+ return str(self._data)
+
+ def has(self, a, b, are_mutually_exclusive):
+ first = self._data.get(a)
+ result = first and first.get(b)
+ if result is None:
+ return False
+
+ # are_mutually_exclusive being false is a superset of being true,
+ # hence if we want to know if this PairSet "has" these two with no
+ # exclusivity, we have to ensure it was added as such.
+ if not are_mutually_exclusive:
+ return not result
+
+ return True
+
+ def add(self, a, b, are_mutually_exclusive):
+ _pair_set_add(self._data, a, b, are_mutually_exclusive)
+ _pair_set_add(self._data, b, a, are_mutually_exclusive)
+ return self
+
+
+def _pair_set_add(data, a, b, are_mutually_exclusive):
+ sub_dict = data.get(a)
+
+ if not sub_dict:
+ sub_dict = {}
+ data[a] = sub_dict
+
+ sub_dict[b] = are_mutually_exclusive
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/version.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/version.py
new file mode 100644
index 0000000000000000000000000000000000000000..614df9f570dc8c92101cc0126de6ef49c3841615
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/version.py
@@ -0,0 +1,78 @@
+from __future__ import unicode_literals
+
+import datetime
+import os
+import subprocess
+
+
+def get_version(version=None):
+ "Returns a PEP 440-compliant version number from VERSION."
+ version = get_complete_version(version)
+
+ # Now build the two parts of the version number:
+ # main = X.Y[.Z]
+ # sub = .devN - for pre-alpha releases
+ # | {a|b|rc}N - for alpha, beta, and rc releases
+
+ main = get_main_version(version)
+
+ sub = ''
+ if version[3] == 'alpha' and version[4] == 0:
+ git_changeset = get_git_changeset()
+ if git_changeset:
+ sub = '.dev%s' % git_changeset
+ else:
+ sub = '.dev'
+ elif version[3] != 'final':
+ mapping = {'alpha': 'a', 'beta': 'b', 'rc': 'rc'}
+ sub = mapping[version[3]] + str(version[4])
+
+ return str(main + sub)
+
+
+def get_main_version(version=None):
+ "Returns main version (X.Y[.Z]) from VERSION."
+ version = get_complete_version(version)
+ parts = 2 if version[2] == 0 else 3
+ return '.'.join(str(x) for x in version[:parts])
+
+
+def get_complete_version(version=None):
+ """Returns a tuple of the graphql version. If version argument is non-empty,
+ then checks for correctness of the tuple provided.
+ """
+ if version is None:
+ from wandb_graphql import VERSION as version
+ else:
+ assert len(version) == 5
+ assert version[3] in ('alpha', 'beta', 'rc', 'final')
+
+ return version
+
+
+def get_docs_version(version=None):
+ version = get_complete_version(version)
+ if version[3] != 'final':
+ return 'dev'
+ else:
+ return '%d.%d' % version[:2]
+
+
+def get_git_changeset():
+ """Returns a numeric identifier of the latest git changeset.
+ The result is the UTC timestamp of the changeset in YYYYMMDDHHMMSS format.
+ This value isn't guaranteed to be unique, but collisions are very unlikely,
+ so it's sufficient for generating the development version numbers.
+ """
+ repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ try:
+ git_log = subprocess.Popen(
+ 'git log --pretty=format:%ct --quiet -1 HEAD',
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ shell=True, cwd=repo_dir, universal_newlines=True,
+ )
+ timestamp = git_log.communicate()[0]
+ timestamp = datetime.datetime.utcfromtimestamp(int(timestamp))
+ except:
+ return None
+ return timestamp.strftime('%Y%m%d%H%M%S')
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/__init__.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..153c1b5e333723de1379049da0c1d0250e94dfc1
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/__init__.py
@@ -0,0 +1,67 @@
+# flake8: noqa
+from .definition import ( # no import order
+ GraphQLScalarType,
+ GraphQLObjectType,
+ GraphQLField,
+ GraphQLArgument,
+ GraphQLInterfaceType,
+ GraphQLUnionType,
+ GraphQLEnumType,
+ GraphQLEnumValue,
+ GraphQLInputObjectType,
+ GraphQLInputObjectField,
+ GraphQLList,
+ GraphQLNonNull,
+ get_named_type,
+ is_abstract_type,
+ is_composite_type,
+ is_input_type,
+ is_leaf_type,
+ is_type,
+ get_nullable_type,
+ is_output_type
+)
+from .directives import (
+ # "Enum" of Directive locations
+ DirectiveLocation,
+
+ # Directive definition
+ GraphQLDirective,
+
+ # Built-in directives defined by the Spec
+ specified_directives,
+ GraphQLSkipDirective,
+ GraphQLIncludeDirective,
+ GraphQLDeprecatedDirective,
+
+ # Constant Deprecation Reason
+ DEFAULT_DEPRECATION_REASON,
+)
+from .scalars import ( # no import order
+ GraphQLInt,
+ GraphQLFloat,
+ GraphQLString,
+ GraphQLBoolean,
+ GraphQLID,
+)
+from .schema import GraphQLSchema
+
+from .introspection import (
+ # "Enum" of Type Kinds
+ TypeKind,
+
+ # GraphQL Types for introspection.
+ __Schema,
+ __Directive,
+ __DirectiveLocation,
+ __Type,
+ __Field,
+ __InputValue,
+ __EnumValue,
+ __TypeKind,
+
+ # Meta-field definitions.
+ SchemaMetaFieldDef,
+ TypeMetaFieldDef,
+ TypeNameMetaFieldDef
+)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/definition.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/definition.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d0c4b4c182372fdf5b411dd4f1337c796ec6d14
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/definition.py
@@ -0,0 +1,619 @@
+from collections.abc import Mapping, Hashable
+import collections
+import copy
+
+from ..language import ast
+from ..pyutils.cached_property import cached_property
+from ..pyutils.ordereddict import OrderedDict
+from ..utils.assert_valid_name import assert_valid_name
+
+
+def is_type(type):
+ return isinstance(type, (
+ GraphQLScalarType,
+ GraphQLObjectType,
+ GraphQLInterfaceType,
+ GraphQLUnionType,
+ GraphQLEnumType,
+ GraphQLInputObjectType,
+ GraphQLList,
+ GraphQLNonNull
+ ))
+
+
+def is_input_type(type):
+ named_type = get_named_type(type)
+ return isinstance(named_type, (
+ GraphQLScalarType,
+ GraphQLEnumType,
+ GraphQLInputObjectType,
+ ))
+
+
+def is_output_type(type):
+ named_type = get_named_type(type)
+ return isinstance(named_type, (
+ GraphQLScalarType,
+ GraphQLObjectType,
+ GraphQLInterfaceType,
+ GraphQLUnionType,
+ GraphQLEnumType
+ ))
+
+
+def is_leaf_type(type):
+ return isinstance(type, (
+ GraphQLScalarType,
+ GraphQLEnumType,
+ ))
+
+
+def is_composite_type(type):
+ named_type = get_named_type(type)
+ return isinstance(named_type, (
+ GraphQLObjectType,
+ GraphQLInterfaceType,
+ GraphQLUnionType,
+ ))
+
+
+def is_abstract_type(type):
+ return isinstance(type, (
+ GraphQLInterfaceType,
+ GraphQLUnionType
+ ))
+
+
+def get_nullable_type(type):
+ if isinstance(type, GraphQLNonNull):
+ return type.of_type
+ return type
+
+
+def get_named_type(type):
+ unmodified_type = type
+ while isinstance(unmodified_type, (GraphQLList, GraphQLNonNull)):
+ unmodified_type = unmodified_type.of_type
+
+ return unmodified_type
+
+
+class GraphQLType(object):
+ __slots__ = 'name',
+
+ def __str__(self):
+ return self.name
+
+ def is_same_type(self, other):
+ return self.__class__ is other.__class__ and self.name == other.name
+
+
+def none_func(x):
+ None
+
+
+class GraphQLScalarType(GraphQLType):
+ """Scalar Type Definition
+
+ The leaf values of any request and input values to arguments are
+ Scalars (or Enums) and are defined with a name and a series of coercion
+ functions used to ensure validity.
+
+ Example:
+
+ def coerce_odd(value):
+ if value % 2 == 1:
+ return value
+ return None
+
+ OddType = GraphQLScalarType(name='Odd', serialize=coerce_odd)
+ """
+
+ __slots__ = 'name', 'description', 'serialize', 'parse_value', 'parse_literal'
+
+ def __init__(self, name, description=None, serialize=None, parse_value=None, parse_literal=None):
+ assert name, 'Type must be named.'
+ assert_valid_name(name)
+ self.name = name
+ self.description = description
+
+ assert callable(serialize), (
+ '{} must provide "serialize" function. If this custom Scalar is '
+ 'also used as an input type, ensure "parse_value" and "parse_literal" '
+ 'functions are also provided.'
+ ).format(self)
+
+ if parse_value is not None or parse_literal is not None:
+ assert callable(parse_value) and callable(parse_literal), (
+ '{} must provide both "parse_value" and "parse_literal" functions.'.format(self)
+ )
+
+ self.serialize = serialize
+ self.parse_value = parse_value or none_func
+ self.parse_literal = parse_literal or none_func
+
+ def __str__(self):
+ return self.name
+
+
+class GraphQLObjectType(GraphQLType):
+ """Object Type Definition
+
+ Almost all of the GraphQL types you define will be object types.
+ Object types have a name, but most importantly describe their fields.
+
+ Example:
+
+ AddressType = GraphQLObjectType('Address', {
+ 'street': GraphQLField(GraphQLString),
+ 'number': GraphQLField(GraphQLInt),
+ 'formatted': GraphQLField(GraphQLString,
+ resolver=lambda obj, args, context, info: obj.number + ' ' + obj.street),
+ })
+
+ When two types need to refer to each other, or a type needs to refer to
+ itself in a field, you can use a static method to supply the fields
+ lazily.
+
+ Example:
+
+ PersonType = GraphQLObjectType('Person', lambda: {
+ 'name': GraphQLField(GraphQLString),
+ 'bestFriend': GraphQLField(PersonType)
+ })
+ """
+ def __init__(self, name, fields, interfaces=None, is_type_of=None, description=None):
+ assert name, 'Type must be named.'
+ assert_valid_name(name)
+ self.name = name
+ self.description = description
+
+ if is_type_of is not None:
+ assert callable(is_type_of), '{} must provide "is_type_of" as a function.'.format(self)
+
+ self.is_type_of = is_type_of
+ self._fields = fields
+ self._provided_interfaces = interfaces
+ self._interfaces = None
+
+ @cached_property
+ def fields(self):
+ return define_field_map(self, self._fields)
+
+ @cached_property
+ def interfaces(self):
+ return define_interfaces(self, self._provided_interfaces)
+
+
+def define_field_map(type, field_map):
+ if callable(field_map):
+ field_map = field_map()
+
+ assert isinstance(field_map, Mapping) and len(field_map) > 0, (
+ '{} fields must be a mapping (dict / OrderedDict) with field names as keys or a '
+ 'function which returns such a mapping.'
+ ).format(type)
+
+ for field_name, field in field_map.items():
+ assert_valid_name(field_name)
+ field_args = getattr(field, 'args', None)
+
+ if field_args:
+ assert isinstance(field_args, Mapping), (
+ '{}.{} args must be a mapping (dict / OrderedDict) with argument names as keys.'.format(type,
+ field_name)
+ )
+
+ for arg_name, arg in field_args.items():
+ assert_valid_name(arg_name)
+
+ return OrderedDict(field_map)
+
+
+def define_interfaces(type, interfaces):
+ if callable(interfaces):
+ interfaces = interfaces()
+
+ if interfaces is None:
+ interfaces = []
+
+ assert isinstance(interfaces, (list, tuple)), (
+ '{} interfaces must be a list/tuple or a function which returns a list/tuple.'.format(type)
+ )
+
+ for interface in interfaces:
+ assert isinstance(interface, GraphQLInterfaceType), (
+ '{} may only implement Interface types, it cannot implement: {}.'.format(type, interface)
+ )
+
+ if not callable(interface.resolve_type):
+ assert callable(type.is_type_of), (
+ 'Interface Type {} does not provide a "resolve_type" function '
+ 'and implementing Type {} does not provide a "is_type_of" '
+ 'function. There is no way to resolve this implementing type '
+ 'during execution.'
+ ).format(interface, type)
+
+ return interfaces
+
+
+class GraphQLField(object):
+ __slots__ = 'type', 'args', 'resolver', 'deprecation_reason', 'description'
+
+ def __init__(self, type, args=None, resolver=None, deprecation_reason=None, description=None):
+ self.type = type
+ self.args = args or OrderedDict()
+ self.resolver = resolver
+ self.deprecation_reason = deprecation_reason
+ self.description = description
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, GraphQLField) and
+ self.type == other.type and
+ self.args == other.args and
+ self.resolver == other.resolver and
+ self.deprecation_reason == other.deprecation_reason and
+ self.description == other.description
+ )
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class GraphQLArgument(object):
+ __slots__ = 'type', 'default_value', 'description', 'out_name'
+
+ def __init__(self, type, default_value=None, description=None, out_name=None):
+ self.type = type
+ self.default_value = default_value
+ self.description = description
+ self.out_name = out_name
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, GraphQLArgument) and
+ self.type == other.type and
+ self.default_value == other.default_value and
+ self.description == other.description and
+ self.out_name == other.out_name
+ )
+ )
+
+ def __hash__(self):
+ return id(self)
+
+
+class GraphQLInterfaceType(GraphQLType):
+ """Interface Type Definition
+
+ When a field can return one of a heterogeneous set of types, a Interface type is used to describe what types are possible,
+ what fields are in common across all types, as well as a function to determine which type is actually used when the field is resolved.
+
+ Example:
+
+ EntityType = GraphQLInterfaceType(
+ name='Entity',
+ fields={
+ 'name': GraphQLField(GraphQLString),
+ })
+ """
+
+ def __init__(self, name, fields=None, resolve_type=None, description=None):
+ assert name, 'Type must be named.'
+ assert_valid_name(name)
+ self.name = name
+ self.description = description
+
+ if resolve_type is not None:
+ assert callable(resolve_type), '{} must provide "resolve_type" as a function.'.format(self)
+
+ self.resolve_type = resolve_type
+ self._fields = fields
+
+ @cached_property
+ def fields(self):
+ return define_field_map(self, self._fields)
+
+
+class GraphQLUnionType(GraphQLType):
+ """Union Type Definition
+
+ When a field can return one of a heterogeneous set of types, a Union type is used to describe what types are possible
+ as well as providing a function to determine which type is actually used when the field is resolved.
+
+ Example:
+
+ class PetType(GraphQLUnionType):
+ name = 'Pet'
+ types = [DogType, CatType]
+
+ def resolve_type(self, value):
+ if isinstance(value, Dog):
+ return DogType()
+ if isinstance(value, Cat):
+ return CatType()
+ """
+
+ def __init__(self, name, types=None, resolve_type=None, description=None):
+ assert name, 'Type must be named.'
+ assert_valid_name(name)
+ self.name = name
+ self.description = description
+
+ if resolve_type is not None:
+ assert callable(resolve_type), '{} must provide "resolve_type" as a function.'.format(self)
+
+ self.resolve_type = resolve_type
+ self._types = types
+
+ @cached_property
+ def types(self):
+ return define_types(self, self._types)
+
+
+def define_types(union_type, types):
+ if callable(types):
+ types = types()
+
+ assert isinstance(types, (list, tuple)) and len(
+ types) > 0, 'Must provide types for Union {}.'.format(union_type.name)
+ has_resolve_type_fn = callable(union_type.resolve_type)
+
+ for type in types:
+ assert isinstance(type, GraphQLObjectType), (
+ '{} may only contain Object types, it cannot contain: {}.'.format(union_type, type)
+ )
+
+ if not has_resolve_type_fn:
+ assert callable(type.is_type_of), (
+ 'Union Type {} does not provide a "resolve_type" function '
+ 'and possible Type {} does not provide a "is_type_of" '
+ 'function. There is no way to resolve this possible type '
+ 'during execution.'
+ ).format(union_type, type)
+
+ return types
+
+
+class GraphQLEnumType(GraphQLType):
+ """Enum Type Definition
+
+ Some leaf values of requests and input values are Enums. GraphQL serializes Enum values as strings,
+ however internally Enums can be represented by any kind of type, often integers.
+
+ Example:
+
+ RGBType = GraphQLEnumType(
+ name='RGB',
+ values=OrderedDict([
+ ('RED', GraphQLEnumValue(0)),
+ ('GREEN', GraphQLEnumValue(1)),
+ ('BLUE', GraphQLEnumValue(2))
+ ])
+ )
+
+ Note: If a value is not provided in a definition, the name of the enum value will be used as it's internal value.
+ """
+
+ def __init__(self, name, values, description=None):
+ assert name, 'Type must provide name.'
+ assert_valid_name(name)
+ self.name = name
+ self.description = description
+
+ self.values = define_enum_values(self, values)
+
+ def serialize(self, value):
+ if isinstance(value, Hashable):
+ enum_value = self._value_lookup.get(value)
+
+ if enum_value:
+ return enum_value.name
+
+ return None
+
+ def parse_value(self, value):
+ if isinstance(value, Hashable):
+ enum_value = self._name_lookup.get(value)
+
+ if enum_value:
+ return enum_value.value
+
+ return None
+
+ def parse_literal(self, value_ast):
+ if isinstance(value_ast, ast.EnumValue):
+ enum_value = self._name_lookup.get(value_ast.value)
+
+ if enum_value:
+ return enum_value.value
+
+ @cached_property
+ def _value_lookup(self):
+ return {value.value: value for value in self.values}
+
+ @cached_property
+ def _name_lookup(self):
+ return {value.name: value for value in self.values}
+
+
+def define_enum_values(type, value_map):
+ assert isinstance(value_map, Mapping) and len(value_map) > 0, (
+ '{} values must be a mapping (dict / OrderedDict) with value names as keys.'.format(type)
+ )
+
+ values = []
+ if not isinstance(value_map, (collections.OrderedDict, OrderedDict)):
+ value_map = OrderedDict(sorted(list(value_map.items())))
+
+ for value_name, value in value_map.items():
+ assert_valid_name(value_name)
+ assert isinstance(value, GraphQLEnumValue), (
+ '{}.{} must be an instance of GraphQLEnumValue, but got: {}'.format(type, value_name, value)
+ )
+ value = copy.copy(value)
+ value.name = value_name
+ if value.value is None:
+ value.value = value_name
+
+ values.append(value)
+
+ return values
+
+
+class GraphQLEnumValue(object):
+ __slots__ = 'name', 'value', 'deprecation_reason', 'description'
+
+ def __init__(self, value=None, deprecation_reason=None, description=None, name=None):
+ self.name = name
+ self.value = value
+ self.deprecation_reason = deprecation_reason
+ self.description = description
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, GraphQLEnumValue) and
+ self.name == other.name and
+ self.value == other.value and
+ self.deprecation_reason == other.deprecation_reason and
+ self.description == other.description
+ )
+ )
+
+
+class GraphQLInputObjectType(GraphQLType):
+ """Input Object Type Definition
+
+ An input object defines a structured collection of fields which may be
+ supplied to a field argument.
+
+ Using `NonNull` will ensure that a value must be provided by the query
+
+ Example:
+
+ NonNullFloat = GraphQLNonNull(GraphQLFloat())
+
+ class GeoPoint(GraphQLInputObjectType):
+ name = 'GeoPoint'
+ fields = {
+ 'lat': GraphQLInputObjectField(NonNullFloat),
+ 'lon': GraphQLInputObjectField(NonNullFloat),
+ 'alt': GraphQLInputObjectField(GraphQLFloat(),
+ default_value=0)
+ }
+ """
+ def __init__(self, name, fields, description=None):
+ assert name, 'Type must be named.'
+ self.name = name
+ self.description = description
+
+ self._fields = fields
+
+ @cached_property
+ def fields(self):
+ return self._define_field_map()
+
+ def _define_field_map(self):
+ fields = self._fields
+ if callable(fields):
+ fields = fields()
+
+ assert isinstance(fields, Mapping) and len(fields) > 0, (
+ '{} fields must be a mapping (dict / OrderedDict) with field names as keys or a '
+ 'function which returns such a mapping.'
+ ).format(self)
+ if not isinstance(fields, (collections.OrderedDict, OrderedDict)):
+ fields = OrderedDict(sorted(list(fields.items())))
+
+ for field_name, field in fields.items():
+ assert_valid_name(field_name)
+
+ return fields
+
+
+class GraphQLInputObjectField(object):
+ __slots__ = 'type', 'default_value', 'description', 'out_name'
+
+ def __init__(self, type, default_value=None, description=None, out_name=None):
+ self.type = type
+ self.default_value = default_value
+ self.description = description
+ self.out_name = out_name
+
+ def __eq__(self, other):
+ return (
+ self is other or (
+ isinstance(other, GraphQLInputObjectField) and
+ self.type == other.type and
+ self.description == other.description and
+ self.out_name == other.out_name
+ )
+ )
+
+
+class GraphQLList(GraphQLType):
+ """List Modifier
+
+ A list is a kind of type marker, a wrapping type which points to another
+ type. Lists are often created within the context of defining the fields
+ of an object type.
+
+ Example:
+
+ class PersonType(GraphQLObjectType):
+ name = 'Person'
+
+ def get_fields(self):
+ return {
+ 'parents': GraphQLField(GraphQLList(PersonType())),
+ 'children': GraphQLField(GraphQLList(PersonType())),
+ }
+ """
+ __slots__ = 'of_type',
+
+ def __init__(self, type):
+ assert is_type(type), 'Can only create List of a GraphQLType but got: {}.'.format(type)
+ self.of_type = type
+
+ def __str__(self):
+ return '[' + str(self.of_type) + ']'
+
+ def is_same_type(self, other):
+ return isinstance(other, GraphQLList) and self.of_type.is_same_type(other.of_type)
+
+
+class GraphQLNonNull(GraphQLType):
+ """Non-Null Modifier
+
+ A non-null is a kind of type marker, a wrapping type which points to another type. Non-null types enforce that their values are never null
+ and can ensure an error is raised if this ever occurs during a request. It is useful for fields which you can make a strong guarantee on
+ non-nullability, for example usually the id field of a database row will never be null.
+
+ Example:
+
+ class RowType(GraphQLObjectType):
+ name = 'Row'
+ fields = {
+ 'id': GraphQLField(type=GraphQLNonNull(GraphQLString()))
+ }
+
+ Note: the enforcement of non-nullability occurs within the executor.
+ """
+ __slots__ = 'of_type',
+
+ def __init__(self, type):
+ assert is_type(type) and not isinstance(type, GraphQLNonNull), (
+ 'Can only create NonNull of a Nullable GraphQLType but got: {}.'.format(type)
+ )
+ self.of_type = type
+
+ def __str__(self):
+ return str(self.of_type) + '!'
+
+ def is_same_type(self, other):
+ return isinstance(other, GraphQLNonNull) and self.of_type.is_same_type(other.of_type)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/directives.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/directives.py
new file mode 100644
index 0000000000000000000000000000000000000000..c566b4d5d8ebdc43a12ae4648f01074eab0b1ae8
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/directives.py
@@ -0,0 +1,132 @@
+from collections.abc import Iterable, Mapping
+
+from ..pyutils.ordereddict import OrderedDict
+from ..utils.assert_valid_name import assert_valid_name
+from .definition import GraphQLArgument, GraphQLNonNull, is_input_type
+from .scalars import GraphQLBoolean, GraphQLString
+
+
+class DirectiveLocation(object):
+ # Operations
+ QUERY = 'QUERY'
+ MUTATION = 'MUTATION'
+ SUBSCRIPTION = 'SUBSCRIPTION'
+ FIELD = 'FIELD'
+ FRAGMENT_DEFINITION = 'FRAGMENT_DEFINITION'
+ FRAGMENT_SPREAD = 'FRAGMENT_SPREAD'
+ INLINE_FRAGMENT = 'INLINE_FRAGMENT'
+
+ # Schema Definitions
+ SCHEMA = 'SCHEMA'
+ SCALAR = 'SCALAR'
+ OBJECT = 'OBJECT'
+ FIELD_DEFINITION = 'FIELD_DEFINITION'
+ ARGUMENT_DEFINITION = 'ARGUMENT_DEFINITION'
+ INTERFACE = 'INTERFACE'
+ UNION = 'UNION'
+ ENUM = 'ENUM'
+ ENUM_VALUE = 'ENUM_VALUE'
+ INPUT_OBJECT = 'INPUT_OBJECT'
+ INPUT_FIELD_DEFINITION = 'INPUT_FIELD_DEFINITION'
+
+ OPERATION_LOCATIONS = [
+ QUERY,
+ MUTATION,
+ SUBSCRIPTION
+ ]
+
+ FRAGMENT_LOCATIONS = [
+ FRAGMENT_DEFINITION,
+ FRAGMENT_SPREAD,
+ INLINE_FRAGMENT
+ ]
+
+ FIELD_LOCATIONS = [
+ FIELD
+ ]
+
+
+class GraphQLDirective(object):
+ __slots__ = 'name', 'args', 'description', 'locations'
+
+ def __init__(self, name, description=None, args=None, locations=None):
+ assert name, 'Directive must be named.'
+ assert_valid_name(name)
+ assert isinstance(locations, Iterable), 'Must provide locations for directive.'
+
+ self.name = name
+ self.description = description
+ self.locations = locations
+
+ if args:
+ assert isinstance(args, Mapping), '{} args must be a dict with argument names as keys.'.format(name)
+ for arg_name, _arg in args.items():
+ assert_valid_name(arg_name)
+ assert is_input_type(_arg.type), '{}({}) argument type must be Input Type but got {}.'.format(
+ name,
+ arg_name,
+ _arg.type)
+ self.args = args or OrderedDict()
+
+
+"""Used to conditionally include fields or fragments."""
+GraphQLIncludeDirective = GraphQLDirective(
+ name='include',
+ description='Directs the executor to include this field or fragment only when the `if` argument is true.',
+ args={
+ 'if': GraphQLArgument(
+ type=GraphQLNonNull(GraphQLBoolean),
+ description='Included when true.',
+ ),
+ },
+ locations=[
+ DirectiveLocation.FIELD,
+ DirectiveLocation.FRAGMENT_SPREAD,
+ DirectiveLocation.INLINE_FRAGMENT,
+ ]
+)
+
+"""Used to conditionally skip (exclude) fields or fragments."""
+GraphQLSkipDirective = GraphQLDirective(
+ name='skip',
+ description='Directs the executor to skip this field or fragment when the `if` argument is true.',
+ args={
+ 'if': GraphQLArgument(
+ type=GraphQLNonNull(GraphQLBoolean),
+ description='Skipped when true.',
+ ),
+ },
+ locations=[
+ DirectiveLocation.FIELD,
+ DirectiveLocation.FRAGMENT_SPREAD,
+ DirectiveLocation.INLINE_FRAGMENT,
+ ]
+)
+
+"""Constant string used for default reason for a deprecation."""
+DEFAULT_DEPRECATION_REASON = 'No longer supported'
+
+"""Used to declare element of a GraphQL schema as deprecated."""
+GraphQLDeprecatedDirective = GraphQLDirective(
+ name='deprecated',
+ description='Marks an element of a GraphQL schema as no longer supported.',
+ args={
+ 'reason': GraphQLArgument(
+ type=GraphQLString,
+ description=('Explains why this element was deprecated, usually also including a suggestion for how to'
+ 'access supported similar data. Formatted in [Markdown]'
+ '(https://daringfireball.net/projects/markdown/).'),
+ default_value=DEFAULT_DEPRECATION_REASON
+ ),
+ },
+ locations=[
+ DirectiveLocation.FIELD_DEFINITION,
+ DirectiveLocation.ENUM_VALUE,
+ ]
+)
+
+specified_directives = [
+ GraphQLIncludeDirective,
+ GraphQLSkipDirective,
+ GraphQLDeprecatedDirective
+]
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/introspection.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/introspection.py
new file mode 100644
index 0000000000000000000000000000000000000000..b2732fc82b0888598e82b06b91ecebc2e16980df
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/introspection.py
@@ -0,0 +1,440 @@
+from collections import OrderedDict, namedtuple
+
+from ..language.printer import print_ast
+from ..utils.ast_from_value import ast_from_value
+from .definition import (GraphQLArgument, GraphQLEnumType, GraphQLEnumValue,
+ GraphQLField, GraphQLInputObjectType,
+ GraphQLInterfaceType, GraphQLList, GraphQLNonNull,
+ GraphQLObjectType, GraphQLScalarType,
+ GraphQLUnionType)
+from .directives import DirectiveLocation
+from .scalars import GraphQLBoolean, GraphQLString
+
+InputField = namedtuple('InputField', ['name', 'description', 'type', 'default_value'])
+Field = namedtuple('Field', ['name', 'type', 'description', 'args', 'deprecation_reason'])
+
+
+def input_fields_to_list(input_fields):
+ fields = []
+ for field_name, field in input_fields.items():
+ fields.append(InputField(
+ name=field_name,
+ description=field.description,
+ type=field.type,
+ default_value=field.default_value))
+ return fields
+
+
+__Schema = GraphQLObjectType(
+ '__Schema',
+ description='A GraphQL Schema defines the capabilities of a GraphQL server. It '
+ 'exposes all available types and directives on the server, as well as '
+ 'the entry points for query, mutation and subscription operations.',
+ fields=lambda: OrderedDict([
+ ('types', GraphQLField(
+ description='A list of all types supported by this server.',
+ type=GraphQLNonNull(GraphQLList(GraphQLNonNull(__Type))),
+ resolver=lambda schema, *_: schema.get_type_map().values(),
+ )),
+ ('queryType', GraphQLField(
+ description='The type that query operations will be rooted at.',
+ type=GraphQLNonNull(__Type),
+ resolver=lambda schema, *_: schema.get_query_type(),
+ )),
+ ('mutationType', GraphQLField(
+ description='If this server supports mutation, the type that '
+ 'mutation operations will be rooted at.',
+ type=__Type,
+ resolver=lambda schema, *_: schema.get_mutation_type(),
+ )),
+ ('subscriptionType', GraphQLField(
+ description='If this server support subscription, the type '
+ 'that subscription operations will be rooted at.',
+ type=__Type,
+ resolver=lambda schema, *_: schema.get_subscription_type(),
+ )),
+ ('directives', GraphQLField(
+ description='A list of all directives supported by this server.',
+ type=GraphQLNonNull(GraphQLList(GraphQLNonNull(__Directive))),
+ resolver=lambda schema, *_: schema.get_directives(),
+ )),
+ ]))
+
+_on_operation_locations = set(DirectiveLocation.OPERATION_LOCATIONS)
+_on_fragment_locations = set(DirectiveLocation.FRAGMENT_LOCATIONS)
+_on_field_locations = set(DirectiveLocation.FIELD_LOCATIONS)
+
+__Directive = GraphQLObjectType(
+ '__Directive',
+ description='A Directive provides a way to describe alternate runtime execution and '
+ 'type validation behavior in a GraphQL document.'
+ '\n\nIn some cases, you need to provide options to alter GraphQL\'s '
+ 'execution behavior in ways field arguments will not suffice, such as '
+ 'conditionally including or skipping a field. Directives provide this by '
+ 'describing additional information to the executor.',
+ fields=lambda: OrderedDict([
+ ('name', GraphQLField(GraphQLNonNull(GraphQLString))),
+ ('description', GraphQLField(GraphQLString)),
+ ('locations', GraphQLField(
+ type=GraphQLNonNull(GraphQLList(GraphQLNonNull(__DirectiveLocation))),
+ )),
+ ('args', GraphQLField(
+ type=GraphQLNonNull(GraphQLList(GraphQLNonNull(__InputValue))),
+ resolver=lambda directive, *args: input_fields_to_list(directive.args),
+ )),
+ ('onOperation', GraphQLField(
+ type=GraphQLNonNull(GraphQLBoolean),
+ deprecation_reason='Use `locations`.',
+ resolver=lambda directive, *args: set(directive.locations) & _on_operation_locations,
+ )),
+ ('onFragment', GraphQLField(
+ type=GraphQLNonNull(GraphQLBoolean),
+ deprecation_reason='Use `locations`.',
+ resolver=lambda directive, *args: set(directive.locations) & _on_fragment_locations,
+ )),
+ ('onField', GraphQLField(
+ type=GraphQLNonNull(GraphQLBoolean),
+ deprecation_reason='Use `locations`.',
+ resolver=lambda directive, *args: set(directive.locations) & _on_field_locations,
+ ))
+ ]))
+
+__DirectiveLocation = GraphQLEnumType(
+ '__DirectiveLocation',
+ description=(
+ 'A Directive can be adjacent to many parts of the GraphQL language, a ' +
+ '__DirectiveLocation describes one such possible adjacencies.'
+ ),
+ values=OrderedDict([
+ ('QUERY', GraphQLEnumValue(
+ DirectiveLocation.QUERY,
+ description='Location adjacent to a query operation.'
+ )),
+ ('MUTATION', GraphQLEnumValue(
+ DirectiveLocation.MUTATION,
+ description='Location adjacent to a mutation operation.'
+ )),
+ ('SUBSCRIPTION', GraphQLEnumValue(
+ DirectiveLocation.SUBSCRIPTION,
+ description='Location adjacent to a subscription operation.'
+ )),
+ ('FIELD', GraphQLEnumValue(
+ DirectiveLocation.FIELD,
+ description='Location adjacent to a field.'
+ )),
+ ('FRAGMENT_DEFINITION', GraphQLEnumValue(
+ DirectiveLocation.FRAGMENT_DEFINITION,
+ description='Location adjacent to a fragment definition.'
+ )),
+ ('FRAGMENT_SPREAD', GraphQLEnumValue(
+ DirectiveLocation.FRAGMENT_SPREAD,
+ description='Location adjacent to a fragment spread.'
+ )),
+ ('INLINE_FRAGMENT', GraphQLEnumValue(
+ DirectiveLocation.INLINE_FRAGMENT,
+ description='Location adjacent to an inline fragment.'
+ )),
+ ('SCHEMA', GraphQLEnumValue(
+ DirectiveLocation.SCHEMA,
+ description='Location adjacent to a schema definition.'
+ )),
+ ('SCALAR', GraphQLEnumValue(
+ DirectiveLocation.SCALAR,
+ description='Location adjacent to a scalar definition.'
+ )),
+ ('OBJECT', GraphQLEnumValue(
+ DirectiveLocation.OBJECT,
+ description='Location adjacent to an object definition.'
+ )),
+ ('FIELD_DEFINITION', GraphQLEnumValue(
+ DirectiveLocation.FIELD_DEFINITION,
+ description='Location adjacent to a field definition.'
+ )),
+ ('ARGUMENT_DEFINITION', GraphQLEnumValue(
+ DirectiveLocation.ARGUMENT_DEFINITION,
+ description='Location adjacent to an argument definition.'
+ )),
+ ('INTERFACE', GraphQLEnumValue(
+ DirectiveLocation.INTERFACE,
+ description='Location adjacent to an interface definition.'
+ )),
+ ('UNION', GraphQLEnumValue(
+ DirectiveLocation.UNION,
+ description='Location adjacent to a union definition.'
+ )),
+ ('ENUM', GraphQLEnumValue(
+ DirectiveLocation.ENUM,
+ description='Location adjacent to an enum definition.'
+ )),
+ ('ENUM_VALUE', GraphQLEnumValue(
+ DirectiveLocation.ENUM_VALUE,
+ description='Location adjacent to an enum value definition.'
+ )),
+ ('INPUT_OBJECT', GraphQLEnumValue(
+ DirectiveLocation.INPUT_OBJECT,
+ description='Location adjacent to an input object definition.'
+ )),
+ ('INPUT_FIELD_DEFINITION', GraphQLEnumValue(
+ DirectiveLocation.INPUT_FIELD_DEFINITION,
+ description='Location adjacent to an input object field definition.'
+ )),
+ ]))
+
+
+class TypeKind(object):
+ SCALAR = 'SCALAR'
+ OBJECT = 'OBJECT'
+ INTERFACE = 'INTERFACE'
+ UNION = 'UNION'
+ ENUM = 'ENUM'
+ INPUT_OBJECT = 'INPUT_OBJECT'
+ LIST = 'LIST'
+ NON_NULL = 'NON_NULL'
+
+
+class TypeFieldResolvers(object):
+ _kinds = (
+ (GraphQLScalarType, TypeKind.SCALAR),
+ (GraphQLObjectType, TypeKind.OBJECT),
+ (GraphQLInterfaceType, TypeKind.INTERFACE),
+ (GraphQLUnionType, TypeKind.UNION),
+ (GraphQLEnumType, TypeKind.ENUM),
+ (GraphQLInputObjectType, TypeKind.INPUT_OBJECT),
+ (GraphQLList, TypeKind.LIST),
+ (GraphQLNonNull, TypeKind.NON_NULL),
+ )
+
+ @classmethod
+ def kind(cls, type, *_):
+ for klass, kind in cls._kinds:
+ if isinstance(type, klass):
+ return kind
+
+ raise Exception('Unknown kind of type: {}'.format(type))
+
+ @staticmethod
+ def fields(type, args, *_):
+ if isinstance(type, (GraphQLObjectType, GraphQLInterfaceType)):
+ fields = []
+ include_deprecated = args.get('includeDeprecated')
+ for field_name, field in type.fields.items():
+ if field.deprecation_reason and not include_deprecated:
+ continue
+ fields.append(Field(
+ name=field_name,
+ description=field.description,
+ type=field.type,
+ args=field.args,
+ deprecation_reason=field.deprecation_reason
+ ))
+ return fields
+ return None
+
+ @staticmethod
+ def interfaces(type, *_):
+ if isinstance(type, GraphQLObjectType):
+ return type.interfaces
+
+ @staticmethod
+ def possible_types(type, args, context, info):
+ if isinstance(type, (GraphQLInterfaceType, GraphQLUnionType)):
+ return info.schema.get_possible_types(type)
+
+ @staticmethod
+ def enum_values(type, args, *_):
+ if isinstance(type, GraphQLEnumType):
+ values = type.values
+ if not args.get('includeDeprecated'):
+ values = [v for v in values if not v.deprecation_reason]
+
+ return values
+
+ @staticmethod
+ def input_fields(type, *_):
+ if isinstance(type, GraphQLInputObjectType):
+ return input_fields_to_list(type.fields)
+
+
+__Type = GraphQLObjectType(
+ '__Type',
+ description='The fundamental unit of any GraphQL Schema is the type. There are '
+ 'many kinds of types in GraphQL as represented by the `__TypeKind` enum.'
+ '\n\nDepending on the kind of a type, certain fields describe '
+ 'information about that type. Scalar types provide no information '
+ 'beyond a name and description, while Enum types provide their values. '
+ 'Object and Interface types provide the fields they describe. Abstract '
+ 'types, Union and Interface, provide the Object types possible '
+ 'at runtime. List and NonNull types compose other types.',
+ fields=lambda: OrderedDict([
+ ('kind', GraphQLField(
+ type=GraphQLNonNull(__TypeKind),
+ resolver=TypeFieldResolvers.kind
+ )),
+ ('name', GraphQLField(GraphQLString)),
+ ('description', GraphQLField(GraphQLString)),
+ ('fields', GraphQLField(
+ type=GraphQLList(GraphQLNonNull(__Field)),
+ args={
+ 'includeDeprecated': GraphQLArgument(
+ GraphQLBoolean,
+ default_value=False
+ )
+ },
+ resolver=TypeFieldResolvers.fields
+ )),
+ ('interfaces', GraphQLField(
+ type=GraphQLList(GraphQLNonNull(__Type)),
+ resolver=TypeFieldResolvers.interfaces
+ )),
+ ('possibleTypes', GraphQLField(
+ type=GraphQLList(GraphQLNonNull(__Type)),
+ resolver=TypeFieldResolvers.possible_types
+ )),
+ ('enumValues', GraphQLField(
+ type=GraphQLList(GraphQLNonNull(__EnumValue)),
+ args={
+ 'includeDeprecated': GraphQLArgument(
+ GraphQLBoolean,
+ default_value=False
+ )
+ },
+ resolver=TypeFieldResolvers.enum_values
+ )),
+ ('inputFields', GraphQLField(
+ type=GraphQLList(GraphQLNonNull(__InputValue)),
+ resolver=TypeFieldResolvers.input_fields
+ )),
+ ('ofType', GraphQLField(
+ type=__Type,
+ resolver=lambda type, *_: getattr(type, 'of_type', None)
+ )),
+ ]))
+
+__Field = GraphQLObjectType(
+ '__Field',
+ description='Object and Interface types are described by a list of Fields, each of '
+ 'which has a name, potentially a list of arguments, and a return type.',
+ fields=lambda: OrderedDict([
+ ('name', GraphQLField(GraphQLNonNull(GraphQLString))),
+ ('description', GraphQLField(GraphQLString)),
+ ('args', GraphQLField(
+ type=GraphQLNonNull(GraphQLList(GraphQLNonNull(__InputValue))),
+ resolver=lambda field, *_: input_fields_to_list(field.args)
+ )),
+ ('type', GraphQLField(GraphQLNonNull(__Type))),
+ ('isDeprecated', GraphQLField(
+ type=GraphQLNonNull(GraphQLBoolean),
+ resolver=lambda field, *_: bool(field.deprecation_reason)
+ )),
+ ('deprecationReason', GraphQLField(
+ type=GraphQLString,
+ resolver=lambda field, *_: field.deprecation_reason
+ ))
+ ])
+)
+
+__InputValue = GraphQLObjectType(
+ '__InputValue',
+ description='Arguments provided to Fields or Directives and the input fields of an '
+ 'InputObject are represented as Input Values which describe their type '
+ 'and optionally a default value.',
+ fields=lambda: OrderedDict([
+ ('name', GraphQLField(GraphQLNonNull(GraphQLString))),
+ ('description', GraphQLField(GraphQLString)),
+ ('type', GraphQLField(GraphQLNonNull(__Type))),
+ ('defaultValue', GraphQLField(
+ type=GraphQLString,
+ resolver=lambda input_val, *_:
+ None if input_val.default_value is None
+ else print_ast(ast_from_value(input_val.default_value, input_val))
+ ))
+ ]))
+
+__EnumValue = GraphQLObjectType(
+ '__EnumValue',
+ description='One possible value for a given Enum. Enum values are unique values, not '
+ 'a placeholder for a string or numeric value. However an Enum value is '
+ 'returned in a JSON response as a string.',
+ fields=lambda: OrderedDict([
+ ('name', GraphQLField(GraphQLNonNull(GraphQLString))),
+ ('description', GraphQLField(GraphQLString)),
+ ('isDeprecated', GraphQLField(
+ type=GraphQLNonNull(GraphQLBoolean),
+ resolver=lambda field, *_: bool(field.deprecation_reason)
+ )),
+ ('deprecationReason', GraphQLField(
+ type=GraphQLString,
+ resolver=lambda enum_value, *_: enum_value.deprecation_reason,
+ ))
+ ]))
+
+__TypeKind = GraphQLEnumType(
+ '__TypeKind',
+ description='An enum describing what kind of type a given `__Type` is',
+ values=OrderedDict([
+ ('SCALAR', GraphQLEnumValue(
+ TypeKind.SCALAR,
+ description='Indicates this type is a scalar.'
+ )),
+ ('OBJECT', GraphQLEnumValue(
+ TypeKind.OBJECT,
+ description='Indicates this type is an object. '
+ '`fields` and `interfaces` are valid fields.'
+ )),
+ ('INTERFACE', GraphQLEnumValue(
+ TypeKind.INTERFACE,
+ description='Indicates this type is an interface. '
+ '`fields` and `possibleTypes` are valid fields.'
+ )),
+ ('UNION', GraphQLEnumValue(
+ TypeKind.UNION,
+ description='Indicates this type is a union. '
+ '`possibleTypes` is a valid field.'
+ )),
+ ('ENUM', GraphQLEnumValue(
+ TypeKind.ENUM,
+ description='Indicates this type is an enum. '
+ '`enumValues` is a valid field.'
+ )),
+ ('INPUT_OBJECT', GraphQLEnumValue(
+ TypeKind.INPUT_OBJECT,
+ description='Indicates this type is an input object. '
+ '`inputFields` is a valid field.'
+ )),
+ ('LIST', GraphQLEnumValue(
+ TypeKind.LIST,
+ description='Indicates this type is a list. '
+ '`ofType` is a valid field.'
+ )),
+ ('NON_NULL', GraphQLEnumValue(
+ TypeKind.NON_NULL,
+ description='Indicates this type is a non-null. '
+ '`ofType` is a valid field.'
+ )),
+ ]))
+
+IntrospectionSchema = __Schema
+
+SchemaMetaFieldDef = GraphQLField(
+ # name='__schema',
+ type=GraphQLNonNull(__Schema),
+ description='Access the current type schema of this server.',
+ resolver=lambda source, args, context, info: info.schema,
+ args={}
+)
+
+TypeMetaFieldDef = GraphQLField(
+ type=__Type,
+ # name='__type',
+ description='Request the type information of a single type.',
+ args={'name': GraphQLArgument(GraphQLNonNull(GraphQLString))},
+ resolver=lambda source, args, context, info: info.schema.get_type(args['name'])
+)
+
+TypeNameMetaFieldDef = GraphQLField(
+ type=GraphQLNonNull(GraphQLString),
+ # name='__typename',
+ description='The name of the current Object type at runtime.',
+ resolver=lambda source, args, context, info: info.parent_type.name,
+ args={}
+)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/scalars.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/scalars.py
new file mode 100644
index 0000000000000000000000000000000000000000..62955b53d3f89cda049a07ff2b3b269b290aaff2
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/scalars.py
@@ -0,0 +1,131 @@
+from ..language.ast import BooleanValue, FloatValue, IntValue, StringValue
+from .definition import GraphQLScalarType
+
+# As per the GraphQL Spec, Integers are only treated as valid when a valid
+# 32-bit signed integer, providing the broadest support across platforms.
+#
+# n.b. JavaScript's integers are safe between -(2^53 - 1) and 2^53 - 1 because
+# they are internally represented as IEEE 754 doubles.
+MAX_INT = 2147483647
+MIN_INT = -2147483648
+
+
+def coerce_int(value):
+ if isinstance(value, int):
+ num = value
+ else:
+ try:
+ num = int(value)
+ except ValueError:
+ num = int(float(value))
+ if MIN_INT <= num <= MAX_INT:
+ return num
+ raise Exception((
+ "Int cannot represent non 32-bit signed integer value: {}"
+ ).format(value))
+
+
+def parse_int_literal(ast):
+ if isinstance(ast, IntValue):
+ num = int(ast.value)
+ if MIN_INT <= num <= MAX_INT:
+ return num
+
+
+GraphQLInt = GraphQLScalarType(
+ name='Int',
+ description='The `Int` scalar type represents non-fractional signed whole numeric '
+ 'values. Int can represent values between -(2^53 - 1) and 2^53 - 1 since '
+ 'represented in JSON as double-precision floating point numbers specified'
+ 'by [IEEE 754](http://en.wikipedia.org/wiki/IEEE_floating_point).',
+ serialize=coerce_int,
+ parse_value=coerce_int,
+ parse_literal=parse_int_literal)
+
+
+def coerce_float(value):
+ if isinstance(value, float):
+ return value
+ return float(value)
+
+
+def parse_float_literal(ast):
+ if isinstance(ast, (FloatValue, IntValue)):
+ return float(ast.value)
+ return None
+
+
+GraphQLFloat = GraphQLScalarType(
+ name='Float',
+ description='The `Float` scalar type represents signed double-precision fractional '
+ 'values as specified by '
+ '[IEEE 754](http://en.wikipedia.org/wiki/IEEE_floating_point). ',
+ serialize=coerce_float,
+ parse_value=coerce_float,
+ parse_literal=parse_float_literal)
+
+
+def coerce_string(value):
+ if isinstance(value, str):
+ return value
+
+ if isinstance(value, bool):
+ return u'true' if value else u'false'
+
+ return str(value)
+
+
+def coerce_str(value):
+ if isinstance(value, str):
+ return value
+
+ return str(value)
+
+
+def parse_string_literal(ast):
+ if isinstance(ast, StringValue):
+ return ast.value
+
+ return None
+
+
+GraphQLString = GraphQLScalarType(
+ name='String',
+ description='The `String` scalar type represents textual data, represented as UTF-8 '
+ 'character sequences. The String type is most often used by GraphQL to '
+ 'represent free-form human-readable text.',
+ serialize=coerce_string,
+ parse_value=coerce_string,
+ parse_literal=parse_string_literal)
+
+
+def parse_boolean_literal(ast):
+ if isinstance(ast, BooleanValue):
+ return ast.value
+ return None
+
+
+GraphQLBoolean = GraphQLScalarType(
+ name='Boolean',
+ description='The `Boolean` scalar type represents `true` or `false`.',
+ serialize=bool,
+ parse_value=bool,
+ parse_literal=parse_boolean_literal)
+
+
+def parse_id_literal(ast):
+ if isinstance(ast, (StringValue, IntValue)):
+ return ast.value
+ return None
+
+
+GraphQLID = GraphQLScalarType(
+ name='ID',
+ description='The `ID` scalar type represents a unique identifier, often used to '
+ 'refetch an object or as key for a cache. The ID type appears in a JSON '
+ 'response as a String; however, it is not intended to be human-readable. '
+ 'When expected as an input type, any string (such as `"4"`) or integer '
+ '(such as `4`) input value will be accepted as an ID.',
+ serialize=coerce_str,
+ parse_value=coerce_str,
+ parse_literal=parse_id_literal)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/schema.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/schema.py
new file mode 100644
index 0000000000000000000000000000000000000000..7f29b8fe176d1777edd4ba2a971a659169fcc543
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/schema.py
@@ -0,0 +1,100 @@
+from collections.abc import Iterable
+
+from .definition import GraphQLObjectType
+from .directives import GraphQLDirective, specified_directives
+from .introspection import IntrospectionSchema
+from .typemap import GraphQLTypeMap
+
+
+class GraphQLSchema(object):
+ """Schema Definition
+
+ A Schema is created by supplying the root types of each type of operation, query and mutation (optional).
+ A schema definition is then supplied to the validator and executor.
+
+ Example:
+
+ MyAppSchema = GraphQLSchema(
+ query=MyAppQueryRootType,
+ mutation=MyAppMutationRootType,
+ )
+
+ Note: If an array of `directives` are provided to GraphQLSchema, that will be
+ the exact list of directives represented and allowed. If `directives` is not
+ provided then a default set of the specified directives (e.g. @include and
+ @skip) will be used. If you wish to provide *additional* directives to these
+ specified directives, you must explicitly declare them. Example:
+
+ MyAppSchema = GraphQLSchema(
+ ...
+ directives=specified_directives.extend([MyCustomerDirective]),
+ )
+ """
+ __slots__ = '_query', '_mutation', '_subscription', '_type_map', '_directives', '_implementations', '_possible_type_map'
+
+ def __init__(self, query, mutation=None, subscription=None, directives=None, types=None):
+ assert isinstance(query, GraphQLObjectType), 'Schema query must be Object Type but got: {}.'.format(query)
+ if mutation:
+ assert isinstance(mutation, GraphQLObjectType), \
+ 'Schema mutation must be Object Type but got: {}.'.format(mutation)
+
+ if subscription:
+ assert isinstance(subscription, GraphQLObjectType), \
+ 'Schema subscription must be Object Type but got: {}.'.format(subscription)
+
+ if types:
+ assert isinstance(types, Iterable), \
+ 'Schema types must be iterable if provided but got: {}.'.format(types)
+
+ self._query = query
+ self._mutation = mutation
+ self._subscription = subscription
+ if directives is None:
+ directives = specified_directives
+
+ assert all(isinstance(d, GraphQLDirective) for d in directives), \
+ 'Schema directives must be List[GraphQLDirective] if provided but got: {}.'.format(
+ directives
+ )
+ self._directives = directives
+
+ initial_types = [
+ query,
+ mutation,
+ subscription,
+ IntrospectionSchema
+ ]
+ if types:
+ initial_types += types
+ self._type_map = GraphQLTypeMap(initial_types)
+
+ def get_query_type(self):
+ return self._query
+
+ def get_mutation_type(self):
+ return self._mutation
+
+ def get_subscription_type(self):
+ return self._subscription
+
+ def get_type_map(self):
+ return self._type_map
+
+ def get_type(self, name):
+ return self._type_map.get(name)
+
+ def get_directives(self):
+ return self._directives
+
+ def get_directive(self, name):
+ for directive in self.get_directives():
+ if directive.name == name:
+ return directive
+
+ return None
+
+ def get_possible_types(self, abstract_type):
+ return self._type_map.get_possible_types(abstract_type)
+
+ def is_possible_type(self, abstract_type, possible_type):
+ return self._type_map.is_possible_type(abstract_type, possible_type)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/type/typemap.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/typemap.py
new file mode 100644
index 0000000000000000000000000000000000000000..12733a661371c44ee63f7edaeca9eb27c04c2a80
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/type/typemap.py
@@ -0,0 +1,145 @@
+from collections import OrderedDict, defaultdict
+from collections.abc import Sequence
+from functools import reduce
+
+from ..utils.type_comparators import is_equal_type, is_type_sub_type_of
+from .definition import (GraphQLArgument, GraphQLField,
+ GraphQLInputObjectField, GraphQLInputObjectType,
+ GraphQLInterfaceType, GraphQLList, GraphQLNonNull,
+ GraphQLObjectType, GraphQLUnionType, is_input_type,
+ is_output_type)
+
+
+class GraphQLTypeMap(OrderedDict):
+
+ def __init__(self, types):
+ super(GraphQLTypeMap, self).__init__()
+ self.update(reduce(self.reducer, types, OrderedDict()))
+ self._possible_type_map = defaultdict(set)
+
+ # Keep track of all implementations by interface name.
+ self._implementations = {}
+ for gql_type in self.values():
+ if isinstance(gql_type, GraphQLObjectType):
+ for interface in gql_type.interfaces:
+ self._implementations.setdefault(interface.name, []).append(gql_type)
+
+ # Enforce correct interface implementations.
+ for type in self.values():
+ if isinstance(type, GraphQLObjectType):
+ for interface in type.interfaces:
+ self.assert_object_implements_interface(self, type, interface)
+
+ def get_possible_types(self, abstract_type):
+ if isinstance(abstract_type, GraphQLUnionType):
+ return abstract_type.types
+ assert isinstance(abstract_type, GraphQLInterfaceType)
+ return self._implementations.get(abstract_type.name, None)
+
+ def is_possible_type(self, abstract_type, possible_type):
+ possible_types = self.get_possible_types(abstract_type)
+ assert isinstance(possible_types, Sequence), (
+ 'Could not find possible implementing types for ${} in ' +
+ 'schema. Check that schema.types is defined and is an array of' +
+ 'all possible types in the schema.'
+ ).format(abstract_type)
+
+ if not self._possible_type_map[abstract_type.name]:
+ self._possible_type_map[abstract_type.name].update([p.name for p in possible_types])
+
+ return possible_type.name in self._possible_type_map[abstract_type.name]
+
+ @classmethod
+ def reducer(cls, map, type):
+ if not type:
+ return map
+
+ if isinstance(type, GraphQLList) or isinstance(type, GraphQLNonNull):
+ return cls.reducer(map, type.of_type)
+
+ if type.name in map:
+ assert map[type.name] == type, (
+ 'Schema must contain unique named types but contains multiple types named "{}".'
+ ).format(type.name)
+
+ return map
+
+ map[type.name] = type
+
+ reduced_map = map
+
+ if isinstance(type, (GraphQLUnionType)):
+ for t in type.types:
+ reduced_map = cls.reducer(reduced_map, t)
+
+ if isinstance(type, GraphQLObjectType):
+ for t in type.interfaces:
+ reduced_map = cls.reducer(reduced_map, t)
+
+ if isinstance(type, (GraphQLObjectType, GraphQLInterfaceType, GraphQLInputObjectType)):
+ field_map = type.fields
+ type_is_input = isinstance(type, GraphQLInputObjectType)
+ for field_name, field in field_map.items():
+ if type_is_input:
+ assert isinstance(field, GraphQLInputObjectField), (
+ '{}.{} must be an instance of GraphQLInputObjectField.'.format(type, field_name)
+ )
+ assert is_input_type(field.type), (
+ '{}.{} field type must be Input Type but got: {}.'.format(type, field_name, field.type)
+ )
+ else:
+ assert isinstance(field, (GraphQLField, GraphQLField)), (
+ '{}.{} must be an instance of GraphQLField.'.format(type, field_name)
+ )
+ assert is_output_type(field.type), (
+ '{}.{} field type must be Output Type but got: {}.'.format(type, field_name, field.type)
+ )
+ for arg_name, arg in field.args.items():
+ assert isinstance(arg, (GraphQLArgument, GraphQLArgument)), (
+ '{}.{}({}:) argument must be an instance of GraphQLArgument.'.format(type, field_name, arg_name)
+ )
+ assert is_input_type(arg.type), (
+ '{}.{}({}:) argument type must be Input Type but got: {}.'.format(type, field_name, arg_name,
+ arg.type)
+ )
+ reduced_map = cls.reducer(reduced_map, arg.type)
+
+ reduced_map = cls.reducer(reduced_map, getattr(field, 'type', None))
+
+ return reduced_map
+
+ @classmethod
+ def assert_object_implements_interface(cls, schema, object, interface):
+ object_field_map = object.fields
+ interface_field_map = interface.fields
+
+ for field_name, interface_field in interface_field_map.items():
+ object_field = object_field_map.get(field_name)
+
+ assert object_field, '"{}" expects field "{}" but "{}" does not provide it.'.format(
+ interface, field_name, object
+ )
+
+ assert is_type_sub_type_of(schema, object_field.type, interface_field.type), (
+ '{}.{} expects type "{}" but {}.{} provides type "{}".'
+ ).format(interface, field_name, interface_field.type, object, field_name, object_field.type)
+
+ for arg_name, interface_arg in interface_field.args.items():
+ object_arg = object_field.args.get(arg_name)
+
+ assert object_arg, (
+ '{}.{} expects argument "{}" but {}.{} does not provide it.'
+ ).format(interface, field_name, arg_name, object, field_name)
+
+ assert is_equal_type(interface_arg.type, object_arg.type), (
+ '{}.{}({}:) expects type "{}" but {}.{}({}:) provides type "{}".'
+ ).format(interface, field_name, arg_name, interface_arg.type, object, field_name, arg_name, object_arg.type)
+
+ for arg_name, object_arg in object_field.args.items():
+ interface_arg = interface_field.args.get(arg_name)
+ if not interface_arg:
+ assert not isinstance(object_arg.type, GraphQLNonNull), (
+ '{}.{}({}:) is of required type '
+ '"{}" but is not also provided by the '
+ 'interface {}.{}.'
+ ).format(object, field_name, arg_name, object_arg.type, interface, field_name)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/__init__.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/assert_valid_name.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/assert_valid_name.py
new file mode 100644
index 0000000000000000000000000000000000000000..40afe59622232f4fc82e0ebeba623a138a02f4c6
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/assert_valid_name.py
@@ -0,0 +1,9 @@
+import re
+
+NAME_PATTERN = r'^[_a-zA-Z][_a-zA-Z0-9]*$'
+COMPILED_NAME_PATTERN = re.compile(NAME_PATTERN)
+
+
+def assert_valid_name(name):
+ '''Helper to assert that provided names are valid.'''
+ assert COMPILED_NAME_PATTERN.match(name), 'Names must match /{}/ but "{}" does not.'.format(NAME_PATTERN, name)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_from_value.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_from_value.py
new file mode 100644
index 0000000000000000000000000000000000000000..1355ef565184335ffe7f483da8be6234e766e27d
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_from_value.py
@@ -0,0 +1,65 @@
+import json
+import re
+import sys
+
+from ..language import ast
+from ..type.definition import (GraphQLEnumType, GraphQLInputObjectType,
+ GraphQLList, GraphQLNonNull)
+from ..type.scalars import GraphQLFloat
+
+
+def ast_from_value(value, type=None):
+ if isinstance(type, GraphQLNonNull):
+ return ast_from_value(value, type.of_type)
+
+ if value is None:
+ return None
+
+ if isinstance(value, list):
+ item_type = type.of_type if isinstance(type, GraphQLList) else None
+ return ast.ListValue([ast_from_value(item, item_type) for item in value])
+
+ elif isinstance(type, GraphQLList):
+ return ast_from_value(value, type.of_type)
+
+ if isinstance(value, bool):
+ return ast.BooleanValue(value)
+
+ if isinstance(value, (int, float)):
+ string_num = str(value)
+ int_value = int(value)
+ is_int_value = string_num.isdigit()
+
+ if is_int_value or (int_value == value and value < sys.maxsize):
+ if type == GraphQLFloat:
+ return ast.FloatValue(str(float(value)))
+
+ return ast.IntValue(str(int(value)))
+
+ return ast.FloatValue(string_num)
+
+ if isinstance(value, str):
+ if isinstance(type, GraphQLEnumType) and re.match(r'^[_a-zA-Z][_a-zA-Z0-9]*$', value):
+ return ast.EnumValue(value)
+
+ return ast.StringValue(json.dumps(value)[1:-1])
+
+ assert isinstance(value, dict)
+
+ fields = []
+ is_graph_ql_input_object_type = isinstance(type, GraphQLInputObjectType)
+
+ for field_name, field_value in value.items():
+ field_type = None
+ if is_graph_ql_input_object_type:
+ field_def = type.fields.get(field_name)
+ field_type = field_def and field_def.type
+
+ field_value = ast_from_value(field_value, field_type)
+ if field_value:
+ fields.append(ast.ObjectField(
+ ast.Name(field_name),
+ field_value
+ ))
+
+ return ast.ObjectValue(fields)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_to_code.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_to_code.py
new file mode 100644
index 0000000000000000000000000000000000000000..8c307eead92e9a533a01f36b80c7187d060df9ae
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_to_code.py
@@ -0,0 +1,49 @@
+from ..language.ast import Node
+from ..language.parser import Loc
+
+
+def ast_to_code(ast, indent=0):
+ """
+ Converts an ast into a python code representation of the AST.
+ """
+ code = []
+
+ def append(line):
+ code.append((' ' * indent) + line)
+
+ if isinstance(ast, Node):
+ append('ast.{}('.format(ast.__class__.__name__))
+ indent += 1
+ for i, k in enumerate(ast._fields, 1):
+ v = getattr(ast, k)
+ append('{}={},'.format(
+ k,
+ ast_to_code(v, indent),
+ ))
+ if ast.loc:
+ append('loc={}'.format(ast_to_code(ast.loc, indent)))
+
+ indent -= 1
+ append(')')
+
+ elif isinstance(ast, Loc):
+ append('loc({}, {})'.format(ast.start, ast.end))
+
+ elif isinstance(ast, list):
+ if ast:
+ append('[')
+ indent += 1
+
+ for i, it in enumerate(ast, 1):
+ is_last = i == len(ast)
+ append(ast_to_code(it, indent) + (',' if not is_last else ''))
+
+ indent -= 1
+ append(']')
+ else:
+ append('[]')
+
+ else:
+ append(repr(ast))
+
+ return '\n'.join(code).strip()
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_to_dict.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_to_dict.py
new file mode 100644
index 0000000000000000000000000000000000000000..a2b5cac5a134902ef98a7caeea8a4cc802f22cc8
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_to_dict.py
@@ -0,0 +1,24 @@
+from ..language.ast import Node
+
+
+def ast_to_dict(node, include_loc=False):
+ if isinstance(node, Node):
+ d = {
+ 'kind': node.__class__.__name__
+ }
+ if hasattr(node, '_fields'):
+ for field in node._fields:
+ d[field] = ast_to_dict(getattr(node, field), include_loc)
+
+ if include_loc and hasattr(node, 'loc') and node.loc:
+ d['loc'] = {
+ 'start': node.loc.start,
+ 'end': node.loc.end
+ }
+
+ return d
+
+ elif isinstance(node, list):
+ return [ast_to_dict(item, include_loc) for item in node]
+
+ return node
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/base.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e8958539b88c2637596e1ca03cc2195632376b7
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/base.py
@@ -0,0 +1,75 @@
+"""
+ Base GraphQL utilities
+ isort:skip_file
+"""
+
+# The GraphQL query recommended for a full schema introspection.
+from .introspection_query import introspection_query
+
+# Gets the target Operation from a Document
+from .get_operation_ast import get_operation_ast
+
+# Build a GraphQLSchema from an introspection result.
+from .build_client_schema import build_client_schema
+
+# Build a GraphQLSchema from a parsed GraphQL Schema language AST.
+from .build_ast_schema import build_ast_schema
+
+# Extends an existing GraphQLSchema from a parsed GraphQL Schema language AST.
+from .extend_schema import extend_schema
+
+# Print a GraphQLSchema to GraphQL Schema language.
+from .schema_printer import print_schema, print_introspection_schema
+
+# Create a GraphQLType from a GraphQL language AST.
+from .type_from_ast import type_from_ast
+
+# Create a JavaScript value from a GraphQL language AST.
+from .value_from_ast import value_from_ast
+
+# Create a GraphQL language AST from a JavaScript value.
+from .ast_from_value import ast_from_value
+
+# A helper to use within recursive-descent visitors which need to be aware of
+# the GraphQL type system.
+from .type_info import TypeInfo
+
+# Determine if JavaScript values adhere to a GraphQL type.
+from .is_valid_value import is_valid_value
+
+# Determine if AST values adhere to a GraphQL type.
+from .is_valid_literal_value import is_valid_literal_value
+
+# Concatenates multiple AST together.
+from .concat_ast import concat_ast
+
+# Comparators for types
+from .type_comparators import (
+ is_equal_type,
+ is_type_sub_type_of,
+ do_types_overlap
+)
+
+# Asserts that a string is a valid GraphQL name
+from .assert_valid_name import assert_valid_name
+
+__all__ = [
+ 'introspection_query',
+ 'get_operation_ast',
+ 'build_client_schema',
+ 'build_ast_schema',
+ 'extend_schema',
+ 'print_introspection_schema',
+ 'print_schema',
+ 'type_from_ast',
+ 'value_from_ast',
+ 'ast_from_value',
+ 'TypeInfo',
+ 'is_valid_value',
+ 'is_valid_literal_value',
+ 'concat_ast',
+ 'do_types_overlap',
+ 'is_equal_type',
+ 'is_type_sub_type_of',
+ 'assert_valid_name',
+]
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/build_ast_schema.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/build_ast_schema.py
new file mode 100644
index 0000000000000000000000000000000000000000..e0c632b24bce9fe37227dc7b631c5ddcae90ee1d
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/build_ast_schema.py
@@ -0,0 +1,291 @@
+from ..execution.values import get_argument_values
+from ..language import ast
+from ..pyutils.ordereddict import OrderedDict
+from ..type import (GraphQLArgument, GraphQLBoolean,
+ GraphQLDeprecatedDirective, GraphQLDirective,
+ GraphQLEnumType, GraphQLEnumValue, GraphQLField,
+ GraphQLFloat, GraphQLID, GraphQLIncludeDirective,
+ GraphQLInputObjectField, GraphQLInputObjectType,
+ GraphQLInt, GraphQLInterfaceType, GraphQLList,
+ GraphQLNonNull, GraphQLObjectType, GraphQLScalarType,
+ GraphQLSchema, GraphQLSkipDirective, GraphQLString,
+ GraphQLUnionType)
+from ..type.introspection import (__Directive, __DirectiveLocation,
+ __EnumValue, __Field, __InputValue, __Schema,
+ __Type, __TypeKind)
+from ..utils.value_from_ast import value_from_ast
+
+
+def _build_wrapped_type(inner_type, input_type_ast):
+ if isinstance(input_type_ast, ast.ListType):
+ return GraphQLList(_build_wrapped_type(inner_type, input_type_ast.type))
+
+ if isinstance(input_type_ast, ast.NonNullType):
+ return GraphQLNonNull(_build_wrapped_type(inner_type, input_type_ast.type))
+
+ return inner_type
+
+
+def _get_inner_type_name(type_ast):
+ if isinstance(type_ast, (ast.ListType, ast.NonNullType)):
+ return _get_inner_type_name(type_ast.type)
+
+ return type_ast.name.value
+
+
+def _get_named_type_ast(type_ast):
+ named_type = type_ast
+ while isinstance(named_type, (ast.ListType, ast.NonNullType)):
+ named_type = named_type.type
+
+ return named_type
+
+
+def _false(*_):
+ return False
+
+
+def _none(*_):
+ return None
+
+
+def build_ast_schema(document):
+ assert isinstance(document, ast.Document), 'must pass in Document ast.'
+
+ schema_def = None
+
+ type_asts = (
+ ast.ScalarTypeDefinition,
+ ast.ObjectTypeDefinition,
+ ast.InterfaceTypeDefinition,
+ ast.EnumTypeDefinition,
+ ast.UnionTypeDefinition,
+ ast.InputObjectTypeDefinition,
+ )
+
+ type_defs = []
+ directive_defs = []
+
+ for d in document.definitions:
+ if isinstance(d, ast.SchemaDefinition):
+ if schema_def:
+ raise Exception('Must provide only one schema definition.')
+ schema_def = d
+ if isinstance(d, type_asts):
+ type_defs.append(d)
+ elif isinstance(d, ast.DirectiveDefinition):
+ directive_defs.append(d)
+
+ if not schema_def:
+ raise Exception('Must provide a schema definition.')
+
+ query_type_name = None
+ mutation_type_name = None
+ subscription_type_name = None
+
+ for operation_type in schema_def.operation_types:
+ type_name = operation_type.type.name.value
+ if operation_type.operation == 'query':
+ if query_type_name:
+ raise Exception('Must provide only one query type in schema.')
+ query_type_name = type_name
+ elif operation_type.operation == 'mutation':
+ if mutation_type_name:
+ raise Exception('Must provide only one mutation type in schema.')
+ mutation_type_name = type_name
+ elif operation_type.operation == 'subscription':
+ if subscription_type_name:
+ raise Exception('Must provide only one subscription type in schema.')
+ subscription_type_name = type_name
+
+ if not query_type_name:
+ raise Exception('Must provide schema definition with query type.')
+
+ ast_map = {d.name.value: d for d in type_defs}
+
+ if query_type_name not in ast_map:
+ raise Exception('Specified query type "{}" not found in document.'.format(query_type_name))
+
+ if mutation_type_name and mutation_type_name not in ast_map:
+ raise Exception('Specified mutation type "{}" not found in document.'.format(mutation_type_name))
+
+ if subscription_type_name and subscription_type_name not in ast_map:
+ raise Exception('Specified subscription type "{}" not found in document.'.format(subscription_type_name))
+
+ inner_type_map = OrderedDict([
+ ('String', GraphQLString),
+ ('Int', GraphQLInt),
+ ('Float', GraphQLFloat),
+ ('Boolean', GraphQLBoolean),
+ ('ID', GraphQLID),
+ ('__Schema', __Schema),
+ ('__Directive', __Directive),
+ ('__DirectiveLocation', __DirectiveLocation),
+ ('__Type', __Type),
+ ('__Field', __Field),
+ ('__InputValue', __InputValue),
+ ('__EnumValue', __EnumValue),
+ ('__TypeKind', __TypeKind),
+ ])
+
+ def get_directive(directive_ast):
+ return GraphQLDirective(
+ name=directive_ast.name.value,
+ locations=[node.value for node in directive_ast.locations],
+ args=make_input_values(directive_ast.arguments, GraphQLArgument),
+ )
+
+ def get_object_type(type_ast):
+ type = type_def_named(type_ast.name.value)
+ assert isinstance(type, GraphQLObjectType), 'AST must provide object type'
+ return type
+
+ def produce_type_def(type_ast):
+ type_name = _get_named_type_ast(type_ast).name.value
+ type_def = type_def_named(type_name)
+ return _build_wrapped_type(type_def, type_ast)
+
+ def type_def_named(type_name):
+ if type_name in inner_type_map:
+ return inner_type_map[type_name]
+
+ if type_name not in ast_map:
+ raise Exception('Type "{}" not found in document'.format(type_name))
+
+ inner_type_def = make_schema_def(ast_map[type_name])
+ if not inner_type_def:
+ raise Exception('Nothing constructed for "{}".'.format(type_name))
+
+ inner_type_map[type_name] = inner_type_def
+ return inner_type_def
+
+ def make_schema_def(definition):
+ if not definition:
+ raise Exception('def must be defined.')
+
+ handler = _schema_def_handlers.get(type(definition))
+ if not handler:
+ raise Exception('Type kind "{}" not supported.'.format(type(definition).__name__))
+
+ return handler(definition)
+
+ def make_type_def(definition):
+ return GraphQLObjectType(
+ name=definition.name.value,
+ fields=lambda: make_field_def_map(definition),
+ interfaces=make_implemented_interfaces(definition)
+ )
+
+ def make_field_def_map(definition):
+ return OrderedDict(
+ (f.name.value, GraphQLField(
+ type=produce_type_def(f.type),
+ args=make_input_values(f.arguments, GraphQLArgument),
+ deprecation_reason=get_deprecation_reason(f.directives),
+ ))
+ for f in definition.fields
+ )
+
+ def make_implemented_interfaces(definition):
+ return [produce_type_def(i) for i in definition.interfaces]
+
+ def make_input_values(values, cls):
+ return OrderedDict(
+ (value.name.value, cls(
+ type=produce_type_def(value.type),
+ default_value=value_from_ast(value.default_value, produce_type_def(value.type))
+ ))
+ for value in values
+ )
+
+ def make_interface_def(definition):
+ return GraphQLInterfaceType(
+ name=definition.name.value,
+ resolve_type=_none,
+ fields=lambda: make_field_def_map(definition)
+ )
+
+ def make_enum_def(definition):
+ values = OrderedDict((v.name.value, GraphQLEnumValue(deprecation_reason=get_deprecation_reason(v.directives)))
+ for v in definition.values)
+ return GraphQLEnumType(
+ name=definition.name.value,
+ values=values
+ )
+
+ def make_union_def(definition):
+ return GraphQLUnionType(
+ name=definition.name.value,
+ resolve_type=_none,
+ types=[produce_type_def(t) for t in definition.types]
+ )
+
+ def make_scalar_def(definition):
+ return GraphQLScalarType(
+ name=definition.name.value,
+ serialize=_none,
+ # Validation calls the parse functions to determine if a literal value is correct.
+ # Returning none, however would cause the scalar to fail validation. Returning false,
+ # will cause them to pass.
+ parse_literal=_false,
+ parse_value=_false
+ )
+
+ def make_input_object_def(definition):
+ return GraphQLInputObjectType(
+ name=definition.name.value,
+ fields=make_input_values(definition.fields, GraphQLInputObjectField)
+ )
+
+ _schema_def_handlers = {
+ ast.ObjectTypeDefinition: make_type_def,
+ ast.InterfaceTypeDefinition: make_interface_def,
+ ast.EnumTypeDefinition: make_enum_def,
+ ast.UnionTypeDefinition: make_union_def,
+ ast.ScalarTypeDefinition: make_scalar_def,
+ ast.InputObjectTypeDefinition: make_input_object_def
+ }
+ types = [type_def_named(definition.name.value) for definition in type_defs]
+ directives = [get_directive(d) for d in directive_defs]
+
+ # If specified directive were not explicitly declared, add them.
+ find_skip_directive = (directive.name for directive in directives if directive.name == 'skip')
+ find_include_directive = (directive.name for directive in directives if directive.name == 'include')
+ find_deprecated_directive = (directive.name for directive in directives if directive.name == 'deprecated')
+
+ if not next(find_skip_directive, None):
+ directives.append(GraphQLSkipDirective)
+
+ if not next(find_include_directive, None):
+ directives.append(GraphQLIncludeDirective)
+
+ if not next(find_deprecated_directive, None):
+ directives.append(GraphQLDeprecatedDirective)
+
+ schema_kwargs = {'query': get_object_type(ast_map[query_type_name])}
+
+ if mutation_type_name:
+ schema_kwargs['mutation'] = get_object_type(ast_map[mutation_type_name])
+
+ if subscription_type_name:
+ schema_kwargs['subscription'] = get_object_type(ast_map[subscription_type_name])
+
+ if directive_defs:
+ schema_kwargs['directives'] = directives
+
+ if types:
+ schema_kwargs['types'] = types
+
+ return GraphQLSchema(**schema_kwargs)
+
+
+def get_deprecation_reason(directives):
+ deprecated_ast = next((directive for directive in directives
+ if directive.name.value == GraphQLDeprecatedDirective.name),
+ None)
+
+ if deprecated_ast:
+ args = get_argument_values(GraphQLDeprecatedDirective.args, deprecated_ast.arguments)
+ return args['reason']
+ else:
+ return None
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/build_client_schema.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/build_client_schema.py
new file mode 100644
index 0000000000000000000000000000000000000000..116b77715eef61327bd5856165ab4f9f67e42627
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/build_client_schema.py
@@ -0,0 +1,250 @@
+from ..language.parser import parse_value
+from ..pyutils.ordereddict import OrderedDict
+from ..type import (GraphQLArgument, GraphQLBoolean, GraphQLEnumType,
+ GraphQLEnumValue, GraphQLField, GraphQLFloat, GraphQLID,
+ GraphQLInputObjectField, GraphQLInputObjectType,
+ GraphQLInt, GraphQLInterfaceType, GraphQLList,
+ GraphQLNonNull, GraphQLObjectType, GraphQLScalarType,
+ GraphQLSchema, GraphQLString, GraphQLUnionType,
+ is_input_type, is_output_type)
+from ..type.directives import DirectiveLocation, GraphQLDirective
+from ..type.introspection import (TypeKind, __Directive, __DirectiveLocation,
+ __EnumValue, __Field, __InputValue, __Schema,
+ __Type, __TypeKind)
+from .value_from_ast import value_from_ast
+
+
+def _false(*_):
+ return False
+
+
+def _none(*_):
+ return None
+
+
+def no_execution(*args):
+ raise Exception('Client Schema cannot be used for execution.')
+
+
+def build_client_schema(introspection):
+ schema_introspection = introspection['__schema']
+
+ type_introspection_map = {t['name']: t for t in schema_introspection['types']}
+
+ type_def_cache = {
+ 'String': GraphQLString,
+ 'Int': GraphQLInt,
+ 'Float': GraphQLFloat,
+ 'Boolean': GraphQLBoolean,
+ 'ID': GraphQLID,
+ '__Schema': __Schema,
+ '__Directive': __Directive,
+ '__DirectiveLocation': __DirectiveLocation,
+ '__Type': __Type,
+ '__Field': __Field,
+ '__InputValue': __InputValue,
+ '__EnumValue': __EnumValue,
+ '__TypeKind': __TypeKind,
+
+ }
+
+ def get_type(type_ref):
+ kind = type_ref.get('kind')
+
+ if kind == TypeKind.LIST:
+ item_ref = type_ref.get('ofType')
+
+ if not item_ref:
+ raise Exception('Decorated type deeper than introspection query.')
+
+ return GraphQLList(get_type(item_ref))
+
+ elif kind == TypeKind.NON_NULL:
+ nullable_ref = type_ref.get('ofType')
+ if not nullable_ref:
+ raise Exception('Decorated type deeper than introspection query.')
+
+ return GraphQLNonNull(get_type(nullable_ref))
+
+ return get_named_type(type_ref['name'])
+
+ def get_named_type(type_name):
+ if type_name in type_def_cache:
+ return type_def_cache[type_name]
+
+ type_introspection = type_introspection_map.get(type_name)
+ if not type_introspection:
+ raise Exception(
+ 'Invalid or incomplete schema, unknown type: {}. Ensure that a full introspection query '
+ 'is used in order to build a client schema.'.format(type_name)
+ )
+
+ type_def = type_def_cache[type_name] = build_type(type_introspection)
+ return type_def
+
+ def get_input_type(type_ref):
+ input_type = get_type(type_ref)
+ assert is_input_type(input_type), 'Introspection must provide input type for arguments.'
+ return input_type
+
+ def get_output_type(type_ref):
+ output_type = get_type(type_ref)
+ assert is_output_type(output_type), 'Introspection must provide output type for fields.'
+ return output_type
+
+ def get_object_type(type_ref):
+ object_type = get_type(type_ref)
+ assert isinstance(object_type, GraphQLObjectType), 'Introspection must provide object type for possibleTypes.'
+ return object_type
+
+ def get_interface_type(type_ref):
+ interface_type = get_type(type_ref)
+ assert isinstance(interface_type, GraphQLInterfaceType), \
+ 'Introspection must provide interface type for interfaces.'
+ return interface_type
+
+ def build_type(type):
+ type_kind = type.get('kind')
+ handler = type_builders.get(type_kind)
+ if not handler:
+ raise Exception(
+ 'Invalid or incomplete schema, unknown kind: {}. Ensure that a full introspection query '
+ 'is used in order to build a client schema.'.format(type_kind)
+ )
+
+ return handler(type)
+
+ def build_scalar_def(scalar_introspection):
+ return GraphQLScalarType(
+ name=scalar_introspection['name'],
+ description=scalar_introspection.get('description'),
+ serialize=_none,
+ parse_value=_false,
+ parse_literal=_false
+ )
+
+ def build_object_def(object_introspection):
+ return GraphQLObjectType(
+ name=object_introspection['name'],
+ description=object_introspection.get('description'),
+ interfaces=[get_interface_type(i) for i in object_introspection.get('interfaces', [])],
+ fields=lambda: build_field_def_map(object_introspection)
+ )
+
+ def build_interface_def(interface_introspection):
+ return GraphQLInterfaceType(
+ name=interface_introspection['name'],
+ description=interface_introspection.get('description'),
+ fields=lambda: build_field_def_map(interface_introspection),
+ resolve_type=no_execution
+ )
+
+ def build_union_def(union_introspection):
+ return GraphQLUnionType(
+ name=union_introspection['name'],
+ description=union_introspection.get('description'),
+ types=[get_object_type(t) for t in union_introspection.get('possibleTypes', [])],
+ resolve_type=no_execution
+ )
+
+ def build_enum_def(enum_introspection):
+ return GraphQLEnumType(
+ name=enum_introspection['name'],
+ description=enum_introspection.get('description'),
+ values=OrderedDict([(value_introspection['name'],
+ GraphQLEnumValue(description=value_introspection.get('description'),
+ deprecation_reason=value_introspection.get('deprecationReason')))
+ for value_introspection in enum_introspection.get('enumValues', [])
+ ])
+ )
+
+ def build_input_object_def(input_object_introspection):
+ return GraphQLInputObjectType(
+ name=input_object_introspection['name'],
+ description=input_object_introspection.get('description'),
+ fields=lambda: build_input_value_def_map(
+ input_object_introspection.get('inputFields'), GraphQLInputObjectField
+ )
+ )
+
+ type_builders = {
+ TypeKind.SCALAR: build_scalar_def,
+ TypeKind.OBJECT: build_object_def,
+ TypeKind.INTERFACE: build_interface_def,
+ TypeKind.UNION: build_union_def,
+ TypeKind.ENUM: build_enum_def,
+ TypeKind.INPUT_OBJECT: build_input_object_def
+ }
+
+ def build_field_def_map(type_introspection):
+ return OrderedDict([
+ (f['name'], GraphQLField(
+ type=get_output_type(f['type']),
+ description=f.get('description'),
+ resolver=no_execution,
+ deprecation_reason=f.get('deprecationReason'),
+ args=build_input_value_def_map(f.get('args'), GraphQLArgument)))
+ for f in type_introspection.get('fields', [])
+ ])
+
+ def build_default_value(f):
+ default_value = f.get('defaultValue')
+ if default_value is None:
+ return None
+
+ return value_from_ast(parse_value(default_value), get_input_type(f['type']))
+
+ def build_input_value_def_map(input_value_introspection, argument_type):
+ return OrderedDict([
+ (f['name'], build_input_value(f, argument_type)) for f in input_value_introspection
+ ])
+
+ def build_input_value(input_value_introspection, argument_type):
+ input_value = argument_type(
+ description=input_value_introspection['description'],
+ type=get_input_type(input_value_introspection['type']),
+ default_value=build_default_value(input_value_introspection)
+ )
+ return input_value
+
+ def build_directive(directive_introspection):
+ # Support deprecated `on****` fields for building `locations`, as this
+ # is used by GraphiQL which may need to support outdated servers.
+ locations = list(directive_introspection.get('locations', []))
+ if not locations:
+ locations = []
+ if directive_introspection.get('onField', False):
+ locations += list(DirectiveLocation.FIELD_LOCATIONS)
+ if directive_introspection.get('onOperation', False):
+ locations += list(DirectiveLocation.OPERATION_LOCATIONS)
+ if directive_introspection.get('onFragment', False):
+ locations += list(DirectiveLocation.FRAGMENT_LOCATIONS)
+
+ return GraphQLDirective(
+ name=directive_introspection['name'],
+ description=directive_introspection.get('description'),
+ # TODO: {} ?
+ args=build_input_value_def_map(directive_introspection.get('args', {}), GraphQLArgument),
+ locations=locations
+ )
+
+ # Iterate through all types, getting the type definition for each, ensuring
+ # that any type not directly referenced by a field will get created.
+ types = [get_named_type(type_introspection_name) for type_introspection_name in type_introspection_map.keys()]
+
+ query_type = get_object_type(schema_introspection['queryType'])
+ mutation_type = get_object_type(
+ schema_introspection['mutationType']) if schema_introspection.get('mutationType') else None
+ subscription_type = get_object_type(schema_introspection['subscriptionType']) if \
+ schema_introspection.get('subscriptionType') else None
+
+ directives = [build_directive(d) for d in schema_introspection['directives']] \
+ if schema_introspection['directives'] else []
+
+ return GraphQLSchema(
+ query=query_type,
+ mutation=mutation_type,
+ subscription=subscription_type,
+ directives=directives,
+ types=types
+ )
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/concat_ast.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/concat_ast.py
new file mode 100644
index 0000000000000000000000000000000000000000..9abebe9245f4d34ef5af35d5cf47d412d0d29398
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/concat_ast.py
@@ -0,0 +1,9 @@
+import itertools
+
+from ..language.ast import Document
+
+
+def concat_ast(asts):
+ return Document(definitions=list(itertools.chain.from_iterable(
+ document.definitions for document in asts
+ )))
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/extend_schema.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/extend_schema.py
new file mode 100644
index 0000000000000000000000000000000000000000..e1bd3451e6e879a6f1cb192e0a9acbf085cf8706
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/extend_schema.py
@@ -0,0 +1,357 @@
+from collections import defaultdict
+
+from ..error import GraphQLError
+from ..language import ast
+from ..pyutils.ordereddict import OrderedDict
+from ..type.definition import (GraphQLArgument, GraphQLEnumType,
+ GraphQLEnumValue, GraphQLField,
+ GraphQLInputObjectField, GraphQLInputObjectType,
+ GraphQLInterfaceType, GraphQLList,
+ GraphQLNonNull, GraphQLObjectType,
+ GraphQLScalarType, GraphQLUnionType)
+from ..type.introspection import (__Directive, __DirectiveLocation,
+ __EnumValue, __Field, __InputValue, __Schema,
+ __Type, __TypeKind)
+from ..type.scalars import (GraphQLBoolean, GraphQLFloat, GraphQLID,
+ GraphQLInt, GraphQLString)
+from ..type.schema import GraphQLSchema
+from .value_from_ast import value_from_ast
+
+
+def extend_schema(schema, documentAST=None):
+ """Produces a new schema given an existing schema and a document which may
+ contain GraphQL type extensions and definitions. The original schema will
+ remain unaltered.
+
+ Because a schema represents a graph of references, a schema cannot be
+ extended without effectively making an entire copy. We do not know until it's
+ too late if subgraphs remain unchanged.
+
+ This algorithm copies the provided schema, applying extensions while
+ producing the copy. The original schema remains unaltered."""
+
+ assert isinstance(
+ schema, GraphQLSchema), 'Must provide valid GraphQLSchema'
+ assert documentAST and isinstance(
+ documentAST, ast.Document), 'Must provide valid Document AST'
+
+ # Collect the type definitions and extensions found in the document.
+ type_definition_map = {}
+ type_extensions_map = defaultdict(list)
+
+ for _def in documentAST.definitions:
+ if isinstance(_def, (
+ ast.ObjectTypeDefinition,
+ ast.InterfaceTypeDefinition,
+ ast.EnumTypeDefinition,
+ ast.UnionTypeDefinition,
+ ast.ScalarTypeDefinition,
+ ast.InputObjectTypeDefinition,
+ )):
+ # Sanity check that none of the defined types conflict with the
+ # schema's existing types.
+ type_name = _def.name.value
+ if schema.get_type(type_name):
+ raise GraphQLError(
+ ('Type "{}" already exists in the schema. It cannot also ' +
+ 'be defined in this type definition.').format(type_name),
+ [_def]
+ )
+
+ type_definition_map[type_name] = _def
+ elif isinstance(_def, ast.TypeExtensionDefinition):
+ # Sanity check that this type extension exists within the
+ # schema's existing types.
+ extended_type_name = _def.definition.name.value
+ existing_type = schema.get_type(extended_type_name)
+ if not existing_type:
+ raise GraphQLError(
+ ('Cannot extend type "{}" because it does not ' +
+ 'exist in the existing schema.').format(extended_type_name),
+ [_def.definition]
+ )
+ if not isinstance(existing_type, GraphQLObjectType):
+ raise GraphQLError(
+ 'Cannot extend non-object type "{}".'.format(
+ extended_type_name),
+ [_def.definition]
+ )
+
+ type_extensions_map[extended_type_name].append(_def)
+
+ # Below are functions used for producing this schema that have closed over
+ # this scope and have access to the schema, cache, and newly defined types.
+
+ def get_type_from_def(type_def):
+ type = _get_named_type(type_def.name)
+ assert type, 'Invalid schema'
+ return type
+
+ def get_type_from_AST(astNode):
+ type = _get_named_type(astNode.name.value)
+ if not type:
+ raise GraphQLError(
+ ('Unknown type: "{}". Ensure that this type exists ' +
+ 'either in the original schema, or is added in a type definition.').format(
+ astNode.name.value),
+ [astNode]
+ )
+ return type
+
+ # Given a name, returns a type from either the existing schema or an
+ # added type.
+ def _get_named_type(typeName):
+ cached_type_def = type_def_cache.get(typeName)
+ if cached_type_def:
+ return cached_type_def
+
+ existing_type = schema.get_type(typeName)
+ if existing_type:
+ type_def = extend_type(existing_type)
+ type_def_cache[typeName] = type_def
+ return type_def
+
+ type_ast = type_definition_map.get(typeName)
+ if type_ast:
+ type_def = build_type(type_ast)
+ type_def_cache[typeName] = type_def
+ return type_def
+
+ # Given a type's introspection result, construct the correct
+ # GraphQLType instance.
+ def extend_type(type):
+ if isinstance(type, GraphQLObjectType):
+ return extend_object_type(type)
+ if isinstance(type, GraphQLInterfaceType):
+ return extend_interface_type(type)
+ if isinstance(type, GraphQLUnionType):
+ return extend_union_type(type)
+ return type
+
+ def extend_object_type(type):
+ return GraphQLObjectType(
+ name=type.name,
+ description=type.description,
+ interfaces=lambda: extend_implemented_interfaces(type),
+ fields=lambda: extend_field_map(type),
+ )
+
+ def extend_interface_type(type):
+ return GraphQLInterfaceType(
+ name=type.name,
+ description=type.description,
+ fields=lambda: extend_field_map(type),
+ resolve_type=cannot_execute_client_schema,
+ )
+
+ def extend_union_type(type):
+ return GraphQLUnionType(
+ name=type.name,
+ description=type.description,
+ types=list(map(get_type_from_def, type.types)),
+ resolve_type=cannot_execute_client_schema,
+ )
+
+ def extend_implemented_interfaces(type):
+ interfaces = list(map(get_type_from_def, type.interfaces))
+
+ # If there are any extensions to the interfaces, apply those here.
+ extensions = type_extensions_map[type.name]
+ for extension in extensions:
+ for namedType in extension.definition.interfaces:
+ interface_name = namedType.name.value
+ if any([_def.name == interface_name for _def in interfaces]):
+ raise GraphQLError(
+ ('Type "{}" already implements "{}". ' +
+ 'It cannot also be implemented in this type extension.').format(
+ type.name, interface_name),
+ [namedType]
+ )
+ interfaces.append(get_type_from_AST(namedType))
+
+ return interfaces
+
+ def extend_field_map(type):
+ new_field_map = OrderedDict()
+ old_field_map = type.fields
+ for field_name, field in old_field_map.items():
+ new_field_map[field_name] = GraphQLField(
+ extend_field_type(field.type),
+ description=field.description,
+ deprecation_reason=field.deprecation_reason,
+ args=field.args,
+ resolver=cannot_execute_client_schema,
+ )
+
+ # If there are any extensions to the fields, apply those here.
+ extensions = type_extensions_map[type.name]
+ for extension in extensions:
+ for field in extension.definition.fields:
+ field_name = field.name.value
+ if field_name in old_field_map:
+ raise GraphQLError(
+ ('Field "{}.{}" already exists in the ' +
+ 'schema. It cannot also be defined in this type extension.').format(
+ type.name, field_name),
+ [field]
+ )
+ new_field_map[field_name] = GraphQLField(
+ build_field_type(field.type),
+ args=build_input_values(field.arguments),
+ resolver=cannot_execute_client_schema,
+ )
+
+ return new_field_map
+
+ def extend_field_type(type):
+ if isinstance(type, GraphQLList):
+ return GraphQLList(extend_field_type(type.of_type))
+ if isinstance(type, GraphQLNonNull):
+ return GraphQLNonNull(extend_field_type(type.of_type))
+ return get_type_from_def(type)
+
+ def build_type(type_ast):
+ _type_build = {
+ ast.ObjectTypeDefinition: build_object_type,
+ ast.InterfaceTypeDefinition: build_interface_type,
+ ast.UnionTypeDefinition: build_union_type,
+ ast.ScalarTypeDefinition: build_scalar_type,
+ ast.EnumTypeDefinition: build_enum_type,
+ ast.InputObjectTypeDefinition: build_input_object_type
+ }
+ func = _type_build.get(type(type_ast))
+ if func:
+ return func(type_ast)
+
+ def build_object_type(type_ast):
+ return GraphQLObjectType(
+ type_ast.name.value,
+ interfaces=lambda: build_implemented_interfaces(type_ast),
+ fields=lambda: build_field_map(type_ast),
+ )
+
+ def build_interface_type(type_ast):
+ return GraphQLInterfaceType(
+ type_ast.name.value,
+ fields=lambda: build_field_map(type_ast),
+ resolve_type=cannot_execute_client_schema,
+ )
+
+ def build_union_type(type_ast):
+ return GraphQLUnionType(
+ type_ast.name.value,
+ types=list(map(get_type_from_AST, type_ast.types)),
+ resolve_type=cannot_execute_client_schema,
+ )
+
+ def build_scalar_type(type_ast):
+ return GraphQLScalarType(
+ type_ast.name.value,
+ serialize=lambda *args, **kwargs: None,
+ # Note: validation calls the parse functions to determine if a
+ # literal value is correct. Returning null would cause use of custom
+ # scalars to always fail validation. Returning false causes them to
+ # always pass validation.
+ parse_value=lambda *args, **kwargs: False,
+ parse_literal=lambda *args, **kwargs: False,
+ )
+
+ def build_enum_type(type_ast):
+ return GraphQLEnumType(
+ type_ast.name.value,
+ values={v.name.value: GraphQLEnumValue() for v in type_ast.values},
+ )
+
+ def build_input_object_type(type_ast):
+ return GraphQLInputObjectType(
+ type_ast.name.value,
+ fields=lambda: build_input_values(
+ type_ast.fields, GraphQLInputObjectField),
+ )
+
+ def build_implemented_interfaces(type_ast):
+ return list(map(get_type_from_AST, type_ast.interfaces))
+
+ def build_field_map(type_ast):
+ return {
+ field.name.value: GraphQLField(
+ build_field_type(field.type),
+ args=build_input_values(field.arguments),
+ resolver=cannot_execute_client_schema,
+ ) for field in type_ast.fields
+ }
+
+ def build_input_values(values, input_type=GraphQLArgument):
+ input_values = OrderedDict()
+ for value in values:
+ type = build_field_type(value.type)
+ input_values[value.name.value] = input_type(
+ type,
+ default_value=value_from_ast(value.default_value, type)
+ )
+ return input_values
+
+ def build_field_type(type_ast):
+ if isinstance(type_ast, ast.ListType):
+ return GraphQLList(build_field_type(type_ast.type))
+ if isinstance(type_ast, ast.NonNullType):
+ return GraphQLNonNull(build_field_type(type_ast.type))
+ return get_type_from_AST(type_ast)
+
+ # If this document contains no new types, then return the same unmodified
+ # GraphQLSchema instance.
+ if not type_extensions_map and not type_definition_map:
+ return schema
+
+ # A cache to use to store the actual GraphQLType definition objects by name.
+ # Initialize to the GraphQL built in scalars and introspection types. All
+ # functions below are inline so that this type def cache is within the scope
+ # of the closure.
+
+ type_def_cache = {
+ 'String': GraphQLString,
+ 'Int': GraphQLInt,
+ 'Float': GraphQLFloat,
+ 'Boolean': GraphQLBoolean,
+ 'ID': GraphQLID,
+ '__Schema': __Schema,
+ '__Directive': __Directive,
+ '__DirectiveLocation': __DirectiveLocation,
+ '__Type': __Type,
+ '__Field': __Field,
+ '__InputValue': __InputValue,
+ '__EnumValue': __EnumValue,
+ '__TypeKind': __TypeKind,
+ }
+
+ # Get the root Query, Mutation, and Subscription types.
+ query_type = get_type_from_def(schema.get_query_type())
+
+ existing_mutation_type = schema.get_mutation_type()
+ mutationType = existing_mutation_type and get_type_from_def(
+ existing_mutation_type) or None
+
+ existing_subscription_type = schema.get_subscription_type()
+ subscription_type = existing_subscription_type and get_type_from_def(
+ existing_subscription_type) or None
+
+ # Iterate through all types, getting the type definition for each, ensuring
+ # that any type not directly referenced by a field will get created.
+ types = [get_type_from_def(_def) for _def in schema.get_type_map().values()]
+
+ # Do the same with new types, appending to the list of defined types.
+ types += [get_type_from_AST(_def) for _def in type_definition_map.values()]
+
+ # Then produce and return a Schema with these types.
+ return GraphQLSchema(
+ query=query_type,
+ mutation=mutationType,
+ subscription=subscription_type,
+ # Copy directives.
+ directives=schema.get_directives(),
+ types=types
+ )
+
+
+def cannot_execute_client_schema(*args, **kwargs):
+ raise Exception('Client Schema cannot be used for execution.')
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/get_field_def.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/get_field_def.py
new file mode 100644
index 0000000000000000000000000000000000000000..5dbf4e18fb8d0c64681ead618fbde00cd900f1b6
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/get_field_def.py
@@ -0,0 +1,27 @@
+from ..type.definition import (GraphQLInterfaceType, GraphQLObjectType,
+ GraphQLUnionType)
+from ..type.introspection import (SchemaMetaFieldDef, TypeMetaFieldDef,
+ TypeNameMetaFieldDef)
+
+
+def get_field_def(schema, parent_type, field_ast):
+ """Not exactly the same as the executor's definition of get_field_def, in this
+ statically evaluated environment we do not always have an Object type,
+ and need to handle Interface and Union types."""
+ name = field_ast.name.value
+ if name == '__schema' and schema.get_query_type() == parent_type:
+ return SchemaMetaFieldDef
+
+ elif name == '__type' and schema.get_query_type() == parent_type:
+ return TypeMetaFieldDef
+
+ elif name == '__typename' and \
+ isinstance(parent_type, (
+ GraphQLObjectType,
+ GraphQLInterfaceType,
+ GraphQLUnionType,
+ )):
+ return TypeNameMetaFieldDef
+
+ elif isinstance(parent_type, (GraphQLObjectType, GraphQLInterfaceType)):
+ return parent_type.fields.get(name)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/get_operation_ast.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/get_operation_ast.py
new file mode 100644
index 0000000000000000000000000000000000000000..899e907ceee8631ba1a31d10718dbcd7dade680d
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/get_operation_ast.py
@@ -0,0 +1,21 @@
+from ..language import ast
+
+
+def get_operation_ast(document_ast, operation_name=None):
+ operation = None
+
+ for definition in document_ast.definitions:
+ if isinstance(definition, ast.OperationDefinition):
+ if not operation_name:
+ # If no operation name is provided, only return an Operation if it is the only one present in the
+ # document. This means that if we've encountered a second operation as we were iterating over the
+ # definitions in the document, there are more than one Operation defined, and we should return None.
+ if operation:
+ return None
+
+ operation = definition
+
+ elif definition.name and definition.name.value == operation_name:
+ return definition
+
+ return operation
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/introspection_query.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/introspection_query.py
new file mode 100644
index 0000000000000000000000000000000000000000..2b87ec13cc0f2bdf728262ed4b39833b982605c3
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/introspection_query.py
@@ -0,0 +1,90 @@
+introspection_query = '''
+ query IntrospectionQuery {
+ __schema {
+ queryType { name }
+ mutationType { name }
+ subscriptionType { name }
+ types {
+ ...FullType
+ }
+ directives {
+ name
+ description
+ locations
+ args {
+ ...InputValue
+ }
+ }
+ }
+ }
+ fragment FullType on __Type {
+ kind
+ name
+ description
+ fields(includeDeprecated: true) {
+ name
+ description
+ args {
+ ...InputValue
+ }
+ type {
+ ...TypeRef
+ }
+ isDeprecated
+ deprecationReason
+ }
+ inputFields {
+ ...InputValue
+ }
+ interfaces {
+ ...TypeRef
+ }
+ enumValues(includeDeprecated: true) {
+ name
+ description
+ isDeprecated
+ deprecationReason
+ }
+ possibleTypes {
+ ...TypeRef
+ }
+ }
+ fragment InputValue on __InputValue {
+ name
+ description
+ type { ...TypeRef }
+ defaultValue
+ }
+ fragment TypeRef on __Type {
+ kind
+ name
+ ofType {
+ kind
+ name
+ ofType {
+ kind
+ name
+ ofType {
+ kind
+ name
+ ofType {
+ kind
+ name
+ ofType {
+ kind
+ name
+ ofType {
+ kind
+ name
+ ofType {
+ kind
+ name
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+'''
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_literal_value.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_literal_value.py
new file mode 100644
index 0000000000000000000000000000000000000000..d329d3b29f252789954ddbb31e64c39aaf42d3a7
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_literal_value.py
@@ -0,0 +1,67 @@
+from ..language import ast
+from ..language.printer import print_ast
+from ..type.definition import (GraphQLEnumType, GraphQLInputObjectType,
+ GraphQLList, GraphQLNonNull, GraphQLScalarType)
+
+_empty_list = []
+
+
+def is_valid_literal_value(type, value_ast):
+ if isinstance(type, GraphQLNonNull):
+ of_type = type.of_type
+ if not value_ast:
+ return [u'Expected "{}", found null.'.format(type)]
+
+ return is_valid_literal_value(of_type, value_ast)
+
+ if not value_ast:
+ return _empty_list
+
+ if isinstance(value_ast, ast.Variable):
+ return _empty_list
+
+ if isinstance(type, GraphQLList):
+ item_type = type.of_type
+ if isinstance(value_ast, ast.ListValue):
+ errors = []
+
+ for i, item_ast in enumerate(value_ast.values):
+ item_errors = is_valid_literal_value(item_type, item_ast)
+ for error in item_errors:
+ errors.append(u'In element #{}: {}'.format(i, error))
+
+ return errors
+
+ return is_valid_literal_value(item_type, value_ast)
+
+ if isinstance(type, GraphQLInputObjectType):
+ if not isinstance(value_ast, ast.ObjectValue):
+ return [u'Expected "{}", found not an object.'.format(type)]
+
+ fields = type.fields
+ field_asts = value_ast.fields
+
+ errors = []
+ for provided_field_ast in field_asts:
+ if provided_field_ast.name.value not in fields:
+ errors.append(u'In field "{}": Unknown field.'.format(provided_field_ast.name.value))
+
+ field_ast_map = {field_ast.name.value: field_ast for field_ast in field_asts}
+
+ def get_field_ast_value(field_name):
+ if field_name in field_ast_map:
+ return field_ast_map[field_name].value
+
+ for field_name, field in fields.items():
+ subfield_errors = is_valid_literal_value(field.type, get_field_ast_value(field_name))
+ errors.extend(u'In field "{}": {}'.format(field_name, e) for e in subfield_errors)
+
+ return errors
+
+ assert isinstance(type, (GraphQLScalarType, GraphQLEnumType)), 'Must be input type'
+
+ parse_result = type.parse_literal(value_ast)
+ if parse_result is None:
+ return [u'Expected type "{}", found {}.'.format(type.name, print_ast(value_ast))]
+
+ return _empty_list
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_value.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_value.py
new file mode 100644
index 0000000000000000000000000000000000000000..eddce4f5d9dc119776fd11dd8a1971e34c3063d4
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_value.py
@@ -0,0 +1,66 @@
+"""
+ Implementation of isValidJSValue from graphql.s
+"""
+
+from collections.abc import Iterable, Mapping
+import json
+
+from ..type import (GraphQLEnumType, GraphQLInputObjectType, GraphQLList,
+ GraphQLNonNull, GraphQLScalarType)
+
+_empty_list = []
+
+
+def is_valid_value(value, type):
+ """Given a type and any value, return True if that value is valid."""
+ if isinstance(type, GraphQLNonNull):
+ of_type = type.of_type
+ if value is None:
+ return [u'Expected "{}", found null.'.format(type)]
+
+ return is_valid_value(value, of_type)
+
+ if value is None:
+ return _empty_list
+
+ if isinstance(type, GraphQLList):
+ item_type = type.of_type
+ if not isinstance(value, str) and isinstance(value, Iterable):
+ errors = []
+ for i, item in enumerate(value):
+ item_errors = is_valid_value(item, item_type)
+ for error in item_errors:
+ errors.append(u'In element #{}: {}'.format(i, error))
+
+ return errors
+
+ else:
+ return is_valid_value(value, item_type)
+
+ if isinstance(type, GraphQLInputObjectType):
+ if not isinstance(value, Mapping):
+ return [u'Expected "{}", found not an object.'.format(type)]
+
+ fields = type.fields
+ errors = []
+
+ for provided_field in sorted(value.keys()):
+ if provided_field not in fields:
+ errors.append(u'In field "{}": Unknown field.'.format(provided_field))
+
+ for field_name, field in fields.items():
+ subfield_errors = is_valid_value(value.get(field_name), field.type)
+ errors.extend(u'In field "{}": {}'.format(field_name, e) for e in subfield_errors)
+
+ return errors
+
+ assert isinstance(type, (GraphQLScalarType, GraphQLEnumType)), \
+ 'Must be input type'
+
+ # Scalar/Enum input checks to ensure the type can parse the value to
+ # a non-null value.
+ parse_result = type.parse_value(value)
+ if parse_result is None:
+ return [u'Expected type "{}", found {}.'.format(type, json.dumps(value))]
+
+ return _empty_list
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/quoted_or_list.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/quoted_or_list.py
new file mode 100644
index 0000000000000000000000000000000000000000..9f98bcd8bf97d4392dede1b936cdbef2a6e5dddf
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/quoted_or_list.py
@@ -0,0 +1,21 @@
+import functools
+
+MAX_LENGTH = 5
+
+
+def quoted_or_list(items):
+ '''Given [ A, B, C ] return '"A", "B" or "C"'.'''
+ selected = items[:MAX_LENGTH]
+ quoted_items = ('"{}"'.format(t) for t in selected)
+
+ def quoted_or_text(text, quoted_and_index):
+ index = quoted_and_index[0]
+ quoted_item = quoted_and_index[1]
+ text += ((', ' if len(selected) > 2 and not index == len(selected) - 1 else ' ') +
+ ('or ' if index == len(selected) - 1 else '') +
+ quoted_item)
+ return text
+
+ enumerated_items = enumerate(quoted_items)
+ first_item = next(enumerated_items)[1]
+ return functools.reduce(quoted_or_text, enumerated_items, first_item)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/schema_printer.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/schema_printer.py
new file mode 100644
index 0000000000000000000000000000000000000000..168a17ecc304afbd4cda4daaad9495fba476b9ae
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/schema_printer.py
@@ -0,0 +1,168 @@
+from ..language.printer import print_ast
+from ..type.definition import (GraphQLEnumType, GraphQLInputObjectType,
+ GraphQLInterfaceType, GraphQLObjectType,
+ GraphQLScalarType, GraphQLUnionType)
+from ..type.directives import DEFAULT_DEPRECATION_REASON
+from .ast_from_value import ast_from_value
+
+
+def print_schema(schema):
+ return _print_filtered_schema(schema, lambda n: not(is_spec_directive(n)), _is_defined_type)
+
+
+def print_introspection_schema(schema):
+ return _print_filtered_schema(schema, is_spec_directive, _is_introspection_type)
+
+
+def is_spec_directive(directive_name):
+ return directive_name in ('skip', 'include', 'deprecated')
+
+
+def _is_defined_type(typename):
+ return not _is_introspection_type(typename) and not _is_builtin_scalar(typename)
+
+
+def _is_introspection_type(typename):
+ return typename.startswith('__')
+
+
+_builtin_scalars = frozenset(['String', 'Boolean', 'Int', 'Float', 'ID'])
+
+
+def _is_builtin_scalar(typename):
+ return typename in _builtin_scalars
+
+
+def _print_filtered_schema(schema, directive_filter, type_filter):
+ return '\n\n'.join([
+ _print_schema_definition(schema)
+ ] + [
+ _print_directive(directive)
+ for directive in schema.get_directives()
+ if directive_filter(directive.name)
+ ] + [
+ _print_type(type)
+ for typename, type in sorted(schema.get_type_map().items())
+ if type_filter(typename)
+ ]) + '\n'
+
+
+def _print_schema_definition(schema):
+ operation_types = []
+
+ query_type = schema.get_query_type()
+ if query_type:
+ operation_types.append(' query: {}'.format(query_type))
+
+ mutation_type = schema.get_mutation_type()
+ if mutation_type:
+ operation_types.append(' mutation: {}'.format(mutation_type))
+
+ subscription_type = schema.get_subscription_type()
+ if subscription_type:
+ operation_types.append(' subscription: {}'.format(subscription_type))
+
+ return 'schema {{\n{}\n}}'.format('\n'.join(operation_types))
+
+
+def _print_type(type):
+ if isinstance(type, GraphQLScalarType):
+ return _print_scalar(type)
+
+ elif isinstance(type, GraphQLObjectType):
+ return _print_object(type)
+
+ elif isinstance(type, GraphQLInterfaceType):
+ return _print_interface(type)
+
+ elif isinstance(type, GraphQLUnionType):
+ return _print_union(type)
+
+ elif isinstance(type, GraphQLEnumType):
+ return _print_enum(type)
+
+ assert isinstance(type, GraphQLInputObjectType)
+ return _print_input_object(type)
+
+
+def _print_scalar(type):
+ return 'scalar {}'.format(type.name)
+
+
+def _print_object(type):
+ interfaces = type.interfaces
+ implemented_interfaces = \
+ ' implements {}'.format(', '.join(i.name for i in interfaces)) if interfaces else ''
+
+ return (
+ 'type {}{} {{\n'
+ '{}\n'
+ '}}'
+ ).format(type.name, implemented_interfaces, _print_fields(type))
+
+
+def _print_interface(type):
+ return (
+ 'interface {} {{\n'
+ '{}\n'
+ '}}'
+ ).format(type.name, _print_fields(type))
+
+
+def _print_union(type):
+ return 'union {} = {}'.format(type.name, ' | '.join(str(t) for t in type.types))
+
+
+def _print_enum(type):
+ return (
+ 'enum {} {{\n'
+ '{}\n'
+ '}}'
+ ).format(type.name, '\n'.join(' ' + v.name + _print_deprecated(v) for v in type.values))
+
+
+def _print_input_object(type):
+ return (
+ 'input {} {{\n'
+ '{}\n'
+ '}}'
+ ).format(type.name, '\n'.join(' ' + _print_input_value(name, field) for name, field in type.fields.items()))
+
+
+def _print_fields(type):
+ return '\n'.join(' {}{}: {}{}'.format(f_name, _print_args(f), f.type, _print_deprecated(f))
+ for f_name, f in type.fields.items())
+
+
+def _print_deprecated(field_or_enum_value):
+ reason = field_or_enum_value.deprecation_reason
+
+ if reason is None:
+ return ''
+ elif reason in ('', DEFAULT_DEPRECATION_REASON):
+ return ' @deprecated'
+ else:
+ return ' @deprecated(reason: {})'.format(print_ast(ast_from_value(reason)))
+
+
+def _print_args(field_or_directives):
+ if not field_or_directives.args:
+ return ''
+
+ return '({})'.format(', '.join(_print_input_value(arg_name, arg) for arg_name, arg in field_or_directives.args.items()))
+
+
+def _print_input_value(name, arg):
+ if arg.default_value is not None:
+ default_value = ' = ' + print_ast(ast_from_value(arg.default_value, arg.type))
+ else:
+ default_value = ''
+
+ return '{}: {}{}'.format(name, arg.type, default_value)
+
+
+def _print_directive(directive):
+ return 'directive @{}{} on {}'.format(directive.name, _print_args(directive), ' | '.join(directive.locations))
+
+
+__all__ = ['print_schema', 'print_introspection_schema']
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/suggestion_list.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/suggestion_list.py
new file mode 100644
index 0000000000000000000000000000000000000000..208f8e31e08f5940eefe40fa19aad86c00891cc6
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/suggestion_list.py
@@ -0,0 +1,56 @@
+from collections import OrderedDict
+
+
+def suggestion_list(inp, options):
+ '''
+ Given an invalid input string and a list of valid options, returns a filtered
+ list of valid options sorted based on their similarity with the input.
+ '''
+ options_by_distance = OrderedDict()
+ input_threshold = len(inp) / 2
+
+ for option in options:
+ distance = lexical_distance(inp, option)
+ threshold = max(input_threshold, len(option) / 2, 1)
+ if distance <= threshold:
+ options_by_distance[option] = distance
+
+ return sorted(list(options_by_distance.keys()), key=lambda k: options_by_distance[k])
+
+
+def lexical_distance(a, b):
+ '''
+ Computes the lexical distance between strings A and B.
+ The "distance" between two strings is given by counting the minimum number
+ of edits needed to transform string A into string B. An edit can be an
+ insertion, deletion, or substitution of a single character, or a swap of two
+ adjacent characters.
+ This distance can be useful for detecting typos in input or sorting
+ @returns distance in number of edits
+ '''
+
+ d = [[i] for i in range(len(a) + 1)] or []
+ d_len = len(d) or 1
+ for i in range(d_len):
+ for j in range(1, len(b) + 1):
+ if i == 0:
+ d[i].append(j)
+ else:
+ d[i].append(0)
+
+ for i in range(1, len(a) + 1):
+ for j in range(1, len(b) + 1):
+ cost = 0 if a[i - 1] == b[j - 1] else 1
+
+ d[i][j] = min(
+ d[i - 1][j] + 1,
+ d[i][j - 1] + 1,
+ d[i - 1][j - 1] + cost
+ )
+
+ if (i > 1 and j < 1 and
+ a[i - 1] == b[j - 2] and
+ a[i - 2] == b[j - 1]):
+ d[i][j] = min(d[i][j], d[i - 2][j - 2] + cost)
+
+ return d[len(a)][len(b)]
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_comparators.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_comparators.py
new file mode 100644
index 0000000000000000000000000000000000000000..93ebb0455761a16c5d90952c906c1aeac3134397
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_comparators.py
@@ -0,0 +1,69 @@
+from ..type.definition import (GraphQLInterfaceType, GraphQLList,
+ GraphQLNonNull, GraphQLObjectType,
+ GraphQLUnionType, is_abstract_type)
+
+
+def is_equal_type(type_a, type_b):
+ if type_a is type_b:
+ return True
+
+ if isinstance(type_a, GraphQLNonNull) and isinstance(type_b, GraphQLNonNull):
+ return is_equal_type(type_a.of_type, type_b.of_type)
+
+ if isinstance(type_a, GraphQLList) and isinstance(type_b, GraphQLList):
+ return is_equal_type(type_a.of_type, type_b.of_type)
+
+ return False
+
+
+def is_type_sub_type_of(schema, maybe_subtype, super_type):
+ if maybe_subtype is super_type:
+ return True
+
+ if isinstance(super_type, GraphQLNonNull):
+ if isinstance(maybe_subtype, GraphQLNonNull):
+ return is_type_sub_type_of(schema, maybe_subtype.of_type, super_type.of_type)
+ return False
+ elif isinstance(maybe_subtype, GraphQLNonNull):
+ return is_type_sub_type_of(schema, maybe_subtype.of_type, super_type)
+
+ if isinstance(super_type, GraphQLList):
+ if isinstance(maybe_subtype, GraphQLList):
+ return is_type_sub_type_of(schema, maybe_subtype.of_type, super_type.of_type)
+ return False
+ elif isinstance(maybe_subtype, GraphQLList):
+ return False
+
+ if is_abstract_type(super_type) and isinstance(
+ maybe_subtype, GraphQLObjectType) and schema.is_possible_type(
+ super_type, maybe_subtype):
+ return True
+
+ return False
+
+
+def do_types_overlap(schema, t1, t2):
+ # print 'do_types_overlap', t1, t2
+ if t1 == t2:
+ # print '1'
+ return True
+
+ if isinstance(t1, (GraphQLInterfaceType, GraphQLUnionType)):
+ if isinstance(t2, (GraphQLInterfaceType, GraphQLUnionType)):
+ # If both types are abstract, then determine if there is any intersection
+ # between possible concrete types of each.
+ s = any([schema.is_possible_type(t2, type) for type in schema.get_possible_types(t1)])
+ # print '2',s
+ return s
+ # Determine if the latter type is a possible concrete type of the former.
+ r = schema.is_possible_type(t1, t2)
+ # print '3', r
+ return r
+
+ if isinstance(t2, (GraphQLInterfaceType, GraphQLUnionType)):
+ t = schema.is_possible_type(t2, t1)
+ # print '4', t
+ return t
+
+ # print '5'
+ return False
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_from_ast.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_from_ast.py
new file mode 100644
index 0000000000000000000000000000000000000000..8689f27adf0527887a1029d6a3730d42568244df
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_from_ast.py
@@ -0,0 +1,21 @@
+from ..language import ast
+from ..type.definition import GraphQLList, GraphQLNonNull
+
+
+def type_from_ast(schema, input_type_ast):
+ if isinstance(input_type_ast, ast.ListType):
+ inner_type = type_from_ast(schema, input_type_ast.type)
+ if inner_type:
+ return GraphQLList(inner_type)
+ else:
+ return None
+
+ if isinstance(input_type_ast, ast.NonNullType):
+ inner_type = type_from_ast(schema, input_type_ast.type)
+ if inner_type:
+ return GraphQLNonNull(inner_type)
+ else:
+ return None
+
+ assert isinstance(input_type_ast, ast.NamedType), 'Must be a type name.'
+ return schema.get_type(input_type_ast.name.value)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_info.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_info.py
new file mode 100644
index 0000000000000000000000000000000000000000..6baf41760a71ae37be2d068e0390220af1d2b92f
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_info.py
@@ -0,0 +1,149 @@
+from ..language import visitor_meta
+from ..type.definition import (GraphQLInputObjectType, GraphQLList,
+ get_named_type, get_nullable_type,
+ is_composite_type)
+from .get_field_def import get_field_def
+from .type_from_ast import type_from_ast
+
+
+def pop(lst):
+ if lst:
+ lst.pop()
+
+
+# noinspection PyPep8Naming
+class TypeInfo(metaclass=visitor_meta.VisitorMeta):
+ __slots__ = '_schema', '_type_stack', '_parent_type_stack', '_input_type_stack', '_field_def_stack', '_directive', \
+ '_argument', '_get_field_def_fn'
+
+ def __init__(self, schema, get_field_def_fn=get_field_def):
+ self._schema = schema
+ self._type_stack = []
+ self._parent_type_stack = []
+ self._input_type_stack = []
+ self._field_def_stack = []
+ self._directive = None
+ self._argument = None
+ self._get_field_def_fn = get_field_def_fn
+
+ def get_type(self):
+ if self._type_stack:
+ return self._type_stack[-1]
+
+ def get_parent_type(self):
+ if self._parent_type_stack:
+ return self._parent_type_stack[-1]
+
+ def get_input_type(self):
+ if self._input_type_stack:
+ return self._input_type_stack[-1]
+
+ def get_field_def(self):
+ if self._field_def_stack:
+ return self._field_def_stack[-1]
+
+ def get_directive(self):
+ return self._directive
+
+ def get_argument(self):
+ return self._argument
+
+ def leave(self, node):
+ method = self._get_leave_handler(type(node))
+ if method:
+ return method(self)
+
+ def enter(self, node):
+ method = self._get_enter_handler(type(node))
+ if method:
+ return method(self, node)
+
+ def enter_SelectionSet(self, node):
+ named_type = get_named_type(self.get_type())
+ composite_type = None
+ if is_composite_type(named_type):
+ composite_type = named_type
+ self._parent_type_stack.append(composite_type)
+
+ def enter_Field(self, node):
+ parent_type = self.get_parent_type()
+ field_def = None
+ if parent_type:
+ field_def = self._get_field_def_fn(self._schema, parent_type, node)
+ self._field_def_stack.append(field_def)
+ self._type_stack.append(field_def and field_def.type)
+
+ def enter_Directive(self, node):
+ self._directive = self._schema.get_directive(node.name.value)
+
+ def enter_OperationDefinition(self, node):
+ definition_type = None
+ if node.operation == 'query':
+ definition_type = self._schema.get_query_type()
+ elif node.operation == 'mutation':
+ definition_type = self._schema.get_mutation_type()
+
+ self._type_stack.append(definition_type)
+
+ def enter_InlineFragment(self, node):
+ type_condition_ast = node.type_condition
+ type = type_from_ast(self._schema, type_condition_ast) if type_condition_ast else self.get_type()
+ self._type_stack.append(type)
+
+ enter_FragmentDefinition = enter_InlineFragment
+
+ def enter_VariableDefinition(self, node):
+ self._input_type_stack.append(type_from_ast(self._schema, node.type))
+
+ def enter_Argument(self, node):
+ arg_def = None
+ arg_type = None
+ field_or_directive = self.get_directive() or self.get_field_def()
+ if field_or_directive:
+ arg_def = field_or_directive.args.get(node.name.value)
+ if arg_def:
+ arg_type = arg_def.type
+ self._argument = arg_def
+ self._input_type_stack.append(arg_type)
+
+ def enter_ListValue(self, node):
+ list_type = get_nullable_type(self.get_input_type())
+ self._input_type_stack.append(
+ list_type.of_type if isinstance(list_type, GraphQLList) else None
+ )
+
+ def enter_ObjectField(self, node):
+ object_type = get_named_type(self.get_input_type())
+ field_type = None
+ if isinstance(object_type, GraphQLInputObjectType):
+ input_field = object_type.fields.get(node.name.value)
+ field_type = input_field.type if input_field else None
+ self._input_type_stack.append(field_type)
+
+ def leave_SelectionSet(self):
+ pop(self._parent_type_stack)
+
+ def leave_Field(self):
+ pop(self._field_def_stack)
+ pop(self._type_stack)
+
+ def leave_Directive(self):
+ self._directive = None
+
+ def leave_OperationDefinition(self):
+ pop(self._type_stack)
+
+ leave_InlineFragment = leave_OperationDefinition
+ leave_FragmentDefinition = leave_OperationDefinition
+
+ def leave_VariableDefinition(self):
+ pop(self._input_type_stack)
+
+ def leave_Argument(self):
+ self._argument = None
+ pop(self._input_type_stack)
+
+ def leave_ListType(self):
+ pop(self._input_type_stack)
+
+ leave_ObjectField = leave_ListType
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/value_from_ast.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/value_from_ast.py
new file mode 100644
index 0000000000000000000000000000000000000000..ff7486bef40dc9687fb95d93333eda2c416855b0
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/utils/value_from_ast.py
@@ -0,0 +1,69 @@
+from ..language import ast
+from ..type import (GraphQLEnumType, GraphQLInputObjectType, GraphQLList,
+ GraphQLNonNull, GraphQLScalarType)
+
+
+def value_from_ast(value_ast, type, variables=None):
+ """Given a type and a value AST node known to match this type, build a
+ runtime value."""
+ if isinstance(type, GraphQLNonNull):
+ # Note: we're not checking that the result of coerceValueAST is non-null.
+ # We're assuming that this query has been validated and the value used here is of the correct type.
+ return value_from_ast(value_ast, type.of_type, variables)
+
+ if not value_ast:
+ return None
+
+ if isinstance(value_ast, ast.Variable):
+ variable_name = value_ast.name.value
+ if not variables or variable_name not in variables:
+ return None
+
+ # Note: we're not doing any checking that this variable is correct. We're assuming that this query
+ # has been validated and the variable usage here is of the correct type.
+ return variables[variable_name]
+
+ if isinstance(type, GraphQLList):
+ item_type = type.of_type
+ if isinstance(value_ast, ast.ListValue):
+ return [value_from_ast(item_ast, item_type, variables)
+ for item_ast in value_ast.values]
+
+ else:
+ return [value_from_ast(value_ast, item_type, variables)]
+
+ if isinstance(type, GraphQLInputObjectType):
+ fields = type.fields
+ if not isinstance(value_ast, ast.ObjectValue):
+ return None
+
+ field_asts = {}
+
+ for field in value_ast.fields:
+ field_asts[field.name.value] = field
+
+ obj = {}
+ for field_name, field in fields.items():
+ field_ast = field_asts.get(field_name)
+ field_value_ast = None
+
+ if field_ast:
+ field_value_ast = field_ast.value
+
+ field_value = value_from_ast(
+ field_value_ast, field.type, variables
+ )
+ if field_value is None:
+ field_value = field.default_value
+
+ if field_value is not None:
+ # We use out_name as the output name for the
+ # dict if exists
+ obj[field.out_name or field_name] = field_value
+
+ return obj
+
+ assert isinstance(type, (GraphQLScalarType, GraphQLEnumType)), \
+ 'Must be input type'
+
+ return type.parse_literal(value_ast)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/__init__.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..893af3e790576776013bbd60f9db943d5d4094ba
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/__init__.py
@@ -0,0 +1,4 @@
+from .validation import validate
+from .rules import specified_rules
+
+__all__ = ['validate', 'specified_rules']
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/__init__.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..b1ebd7d6ad8d9e7e49eaa2232d3050f087e923b9
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/__init__.py
@@ -0,0 +1,79 @@
+from .arguments_of_correct_type import ArgumentsOfCorrectType
+from .default_values_of_correct_type import DefaultValuesOfCorrectType
+from .fields_on_correct_type import FieldsOnCorrectType
+from .fragments_on_composite_types import FragmentsOnCompositeTypes
+from .known_argument_names import KnownArgumentNames
+from .known_directives import KnownDirectives
+from .known_fragment_names import KnownFragmentNames
+from .known_type_names import KnownTypeNames
+from .lone_anonymous_operation import LoneAnonymousOperation
+from .no_fragment_cycles import NoFragmentCycles
+from .no_undefined_variables import NoUndefinedVariables
+from .no_unused_fragments import NoUnusedFragments
+from .no_unused_variables import NoUnusedVariables
+from .overlapping_fields_can_be_merged import OverlappingFieldsCanBeMerged
+from .possible_fragment_spreads import PossibleFragmentSpreads
+from .provided_non_null_arguments import ProvidedNonNullArguments
+from .scalar_leafs import ScalarLeafs
+from .unique_argument_names import UniqueArgumentNames
+from .unique_fragment_names import UniqueFragmentNames
+from .unique_input_field_names import UniqueInputFieldNames
+from .unique_operation_names import UniqueOperationNames
+from .unique_variable_names import UniqueVariableNames
+from .variables_are_input_types import VariablesAreInputTypes
+from .variables_in_allowed_position import VariablesInAllowedPosition
+
+specified_rules = [
+ UniqueOperationNames,
+ LoneAnonymousOperation,
+ KnownTypeNames,
+ FragmentsOnCompositeTypes,
+ VariablesAreInputTypes,
+ ScalarLeafs,
+ FieldsOnCorrectType,
+ UniqueFragmentNames,
+ KnownFragmentNames,
+ NoUnusedFragments,
+ PossibleFragmentSpreads,
+ NoFragmentCycles,
+ NoUndefinedVariables,
+ NoUnusedVariables,
+ KnownDirectives,
+ KnownArgumentNames,
+ UniqueArgumentNames,
+ ArgumentsOfCorrectType,
+ ProvidedNonNullArguments,
+ DefaultValuesOfCorrectType,
+ VariablesInAllowedPosition,
+ OverlappingFieldsCanBeMerged,
+ UniqueInputFieldNames,
+ UniqueVariableNames
+]
+
+__all__ = [
+ 'ArgumentsOfCorrectType',
+ 'DefaultValuesOfCorrectType',
+ 'FieldsOnCorrectType',
+ 'FragmentsOnCompositeTypes',
+ 'KnownArgumentNames',
+ 'KnownDirectives',
+ 'KnownFragmentNames',
+ 'KnownTypeNames',
+ 'LoneAnonymousOperation',
+ 'NoFragmentCycles',
+ 'UniqueVariableNames',
+ 'NoUndefinedVariables',
+ 'NoUnusedFragments',
+ 'NoUnusedVariables',
+ 'OverlappingFieldsCanBeMerged',
+ 'PossibleFragmentSpreads',
+ 'ProvidedNonNullArguments',
+ 'ScalarLeafs',
+ 'UniqueArgumentNames',
+ 'UniqueFragmentNames',
+ 'UniqueInputFieldNames',
+ 'UniqueOperationNames',
+ 'VariablesAreInputTypes',
+ 'VariablesInAllowedPosition',
+ 'specified_rules'
+]
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/arguments_of_correct_type.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/arguments_of_correct_type.py
new file mode 100644
index 0000000000000000000000000000000000000000..011fae79b59d229c156afa812c78d427879a99d6
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/arguments_of_correct_type.py
@@ -0,0 +1,24 @@
+from ...error import GraphQLError
+from ...language.printer import print_ast
+from ...utils.is_valid_literal_value import is_valid_literal_value
+from .base import ValidationRule
+
+
+class ArgumentsOfCorrectType(ValidationRule):
+
+ def enter_Argument(self, node, key, parent, path, ancestors):
+ arg_def = self.context.get_argument()
+ if arg_def:
+ errors = is_valid_literal_value(arg_def.type, node.value)
+ if errors:
+ self.context.report_error(GraphQLError(
+ self.bad_value_message(node.name.value, arg_def.type,
+ print_ast(node.value), errors),
+ [node.value]
+ ))
+ return False
+
+ @staticmethod
+ def bad_value_message(arg_name, type, value, verbose_errors):
+ message = (u'\n' + u'\n'.join(verbose_errors)) if verbose_errors else ''
+ return 'Argument "{}" has invalid value {}.{}'.format(arg_name, value, message)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/base.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..43bb53b7493ad3711fb0d7d99e195e200cfa51c6
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/base.py
@@ -0,0 +1,8 @@
+from ...language.visitor import Visitor
+
+
+class ValidationRule(Visitor):
+ __slots__ = 'context',
+
+ def __init__(self, context):
+ self.context = context
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/default_values_of_correct_type.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/default_values_of_correct_type.py
new file mode 100644
index 0000000000000000000000000000000000000000..ad6346b46573289f43040425938cb327007e2393
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/default_values_of_correct_type.py
@@ -0,0 +1,44 @@
+from ...error import GraphQLError
+from ...language.printer import print_ast
+from ...type.definition import GraphQLNonNull
+from ...utils.is_valid_literal_value import is_valid_literal_value
+from .base import ValidationRule
+
+
+class DefaultValuesOfCorrectType(ValidationRule):
+
+ def enter_VariableDefinition(self, node, key, parent, path, ancestors):
+ name = node.variable.name.value
+ default_value = node.default_value
+ type = self.context.get_input_type()
+
+ if isinstance(type, GraphQLNonNull) and default_value:
+ self.context.report_error(GraphQLError(
+ self.default_for_non_null_arg_message(name, type, type.of_type),
+ [default_value]
+ ))
+
+ if type and default_value:
+ errors = is_valid_literal_value(type, default_value)
+ if errors:
+ self.context.report_error(GraphQLError(
+ self.bad_value_for_default_arg_message(name, type, print_ast(default_value), errors),
+ [default_value]
+ ))
+ return False
+
+ def enter_SelectionSet(self, node, key, parent, path, ancestors):
+ return False
+
+ def enter_FragmentDefinition(self, node, key, parent, path, ancestors):
+ return False
+
+ @staticmethod
+ def default_for_non_null_arg_message(var_name, type, guess_type):
+ return u'Variable "${}" of type "{}" is required and will not use the default value. ' \
+ u'Perhaps you meant to use type "{}".'.format(var_name, type, guess_type)
+
+ @staticmethod
+ def bad_value_for_default_arg_message(var_name, type, value, verbose_errors):
+ message = (u'\n' + u'\n'.join(verbose_errors)) if verbose_errors else u''
+ return u'Variable "${}" of type "{}" has invalid default value: {}.{}'.format(var_name, type, value, message)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/fields_on_correct_type.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/fields_on_correct_type.py
new file mode 100644
index 0000000000000000000000000000000000000000..55bb2221c9e4df059215227dcecacf9ec884e6e4
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/fields_on_correct_type.py
@@ -0,0 +1,113 @@
+from collections import Counter
+
+from ...error import GraphQLError
+from ...pyutils.ordereddict import OrderedDict
+from ...type.definition import (GraphQLInterfaceType, GraphQLObjectType,
+ GraphQLUnionType)
+from ...utils.quoted_or_list import quoted_or_list
+from ...utils.suggestion_list import suggestion_list
+from .base import ValidationRule
+
+try:
+ # Python 2
+ from itertools import izip
+except ImportError:
+ # Python 3
+ izip = zip
+
+
+def _undefined_field_message(field_name, type, suggested_types,
+ suggested_fields):
+ message = 'Cannot query field "{}" on type "{}".'.format(field_name, type)
+
+ if suggested_types:
+ suggestions = quoted_or_list(suggested_types)
+ message += " Did you mean to use an inline fragment on {}?".format(suggestions)
+ elif suggested_fields:
+ suggestions = quoted_or_list(suggested_fields)
+ message += " Did you mean {}?".format(suggestions)
+
+ return message
+
+
+class OrderedCounter(Counter, OrderedDict):
+ pass
+
+
+class FieldsOnCorrectType(ValidationRule):
+ '''Fields on correct type
+
+ A GraphQL document is only valid if all fields selected are defined by the
+ parent type, or are an allowed meta field such as __typenamme
+ '''
+
+ def enter_Field(self, node, key, parent, path, ancestors):
+ parent_type = self.context.get_parent_type()
+ if not parent_type:
+ return
+
+ field_def = self.context.get_field_def()
+ if not field_def:
+ # This field doesn't exist, lets look for suggestions.
+ schema = self.context.get_schema()
+ field_name = node.name.value
+
+ # First determine if there are any suggested types to condition on.
+ suggested_type_names = get_suggested_type_names(schema, parent_type, field_name)
+ # if there are no suggested types perhaps it was a typo?
+ suggested_field_names = [] if suggested_type_names else get_suggested_field_names(schema, parent_type, field_name)
+
+ # report an error including helpful suggestions.
+ self.context.report_error(GraphQLError(
+ _undefined_field_message(field_name, parent_type.name, suggested_type_names, suggested_field_names),
+ [node]
+ ))
+
+
+def get_suggested_type_names(schema, output_type, field_name):
+ '''Go through all of the implementations of type, as well as the interfaces
+ that they implement. If any of those types include the provided field,
+ suggest them, sorted by how often the type is referenced, starting
+ with Interfaces.'''
+
+ if isinstance(output_type, (GraphQLInterfaceType, GraphQLUnionType)):
+ suggested_object_types = []
+ interface_usage_count = OrderedDict()
+ for possible_type in schema.get_possible_types(output_type):
+ if not possible_type.fields.get(field_name):
+ return
+
+ # This object type defines this field.
+ suggested_object_types.append(possible_type.name)
+
+ for possible_interface in possible_type.interfaces:
+ if not possible_interface.fields.get(field_name):
+ continue
+
+ # This interface type defines this field.
+ interface_usage_count[possible_interface.name] = (
+ interface_usage_count.get(possible_interface.name, 0) + 1)
+
+ # Suggest interface types based on how common they are.
+ suggested_interface_types = sorted(list(interface_usage_count.keys()), key=lambda k: interface_usage_count[k],
+ reverse=True)
+
+ # Suggest both interface and object types.
+ suggested_interface_types.extend(suggested_object_types)
+ return suggested_interface_types
+
+ # Otherwise, must be an Object type, which does not have possible fields.
+ return []
+
+
+def get_suggested_field_names(schema, graphql_type, field_name):
+ '''For the field name provided, determine if there are any similar field names
+ that may be the result of a typo.'''
+
+ if isinstance(graphql_type, (GraphQLInterfaceType, GraphQLObjectType)):
+ possible_field_names = list(graphql_type.fields.keys())
+
+ return suggestion_list(field_name, possible_field_names)
+
+ # Otherwise, must be a Union type, which does not define fields.
+ return []
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/fragments_on_composite_types.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/fragments_on_composite_types.py
new file mode 100644
index 0000000000000000000000000000000000000000..a95e247c08fa14846786520ad9a4720bce0dba32
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/fragments_on_composite_types.py
@@ -0,0 +1,33 @@
+from ...error import GraphQLError
+from ...language.printer import print_ast
+from ...type.definition import is_composite_type
+from .base import ValidationRule
+
+
+class FragmentsOnCompositeTypes(ValidationRule):
+
+ def enter_InlineFragment(self, node, key, parent, path, ancestors):
+ type = self.context.get_type()
+
+ if node.type_condition and type and not is_composite_type(type):
+ self.context.report_error(GraphQLError(
+ self.inline_fragment_on_non_composite_error_message(print_ast(node.type_condition)),
+ [node.type_condition]
+ ))
+
+ def enter_FragmentDefinition(self, node, key, parent, path, ancestors):
+ type = self.context.get_type()
+
+ if type and not is_composite_type(type):
+ self.context.report_error(GraphQLError(
+ self.fragment_on_non_composite_error_message(node.name.value, print_ast(node.type_condition)),
+ [node.type_condition]
+ ))
+
+ @staticmethod
+ def inline_fragment_on_non_composite_error_message(type):
+ return 'Fragment cannot condition on non composite type "{}".'.format(type)
+
+ @staticmethod
+ def fragment_on_non_composite_error_message(frag_name, type):
+ return 'Fragment "{}" cannot condition on non composite type "{}".'.format(frag_name, type)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_argument_names.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_argument_names.py
new file mode 100644
index 0000000000000000000000000000000000000000..2b5860397ec2c58d9f45fee3e3eebdc0358fad80
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_argument_names.py
@@ -0,0 +1,70 @@
+from ...error import GraphQLError
+from ...language import ast
+from ...utils.quoted_or_list import quoted_or_list
+from ...utils.suggestion_list import suggestion_list
+from .base import ValidationRule
+
+
+def _unknown_arg_message(arg_name, field_name, type, suggested_args):
+ message = 'Unknown argument "{}" on field "{}" of type "{}".'.format(arg_name, field_name, type)
+ if suggested_args:
+ message += ' Did you mean {}?'.format(quoted_or_list(suggested_args))
+
+ return message
+
+
+def _unknown_directive_arg_message(arg_name, directive_name, suggested_args):
+ message = 'Unknown argument "{}" on directive "@{}".'.format(arg_name, directive_name)
+ if suggested_args:
+ message += ' Did you mean {}?'.format(quoted_or_list(suggested_args))
+
+ return message
+
+
+class KnownArgumentNames(ValidationRule):
+
+ def enter_Argument(self, node, key, parent, path, ancestors):
+ argument_of = ancestors[-1]
+
+ if isinstance(argument_of, ast.Field):
+ field_def = self.context.get_field_def()
+ if not field_def:
+ return
+
+ field_arg_def = field_def.args.get(node.name.value)
+
+ if not field_arg_def:
+ parent_type = self.context.get_parent_type()
+ assert parent_type
+ self.context.report_error(GraphQLError(
+ _unknown_arg_message(
+ node.name.value,
+ argument_of.name.value,
+ parent_type.name,
+ suggestion_list(
+ node.name.value,
+ (arg_name for arg_name in field_def.args.keys())
+ )
+ ),
+ [node]
+ ))
+
+ elif isinstance(argument_of, ast.Directive):
+ directive = self.context.get_directive()
+ if not directive:
+ return
+
+ directive_arg_def = directive.args.get(node.name.value)
+
+ if not directive_arg_def:
+ self.context.report_error(GraphQLError(
+ _unknown_directive_arg_message(
+ node.name.value,
+ directive.name,
+ suggestion_list(
+ node.name.value,
+ (arg_name for arg_name in directive.args.keys())
+ )
+ ),
+ [node]
+ ))
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_directives.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_directives.py
new file mode 100644
index 0000000000000000000000000000000000000000..f672105df350babee2dd9b03fb11b5609dded655
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_directives.py
@@ -0,0 +1,97 @@
+from ...error import GraphQLError
+from ...language import ast
+from ...type.directives import DirectiveLocation
+from .base import ValidationRule
+
+
+class KnownDirectives(ValidationRule):
+
+ def enter_Directive(self, node, key, parent, path, ancestors):
+ directive_def = next((
+ definition for definition in self.context.get_schema().get_directives()
+ if definition.name == node.name.value
+ ), None)
+
+ if not directive_def:
+ return self.context.report_error(GraphQLError(
+ self.unknown_directive_message(node.name.value),
+ [node]
+ ))
+
+ candidate_location = get_directive_location_for_ast_path(ancestors)
+ if not candidate_location:
+ self.context.report_error(GraphQLError(
+ self.misplaced_directive_message(node.name.value, node.type),
+ [node]
+ ))
+ elif candidate_location not in directive_def.locations:
+ self.context.report_error(GraphQLError(
+ self.misplaced_directive_message(node.name.value, candidate_location),
+ [node]
+ ))
+
+ @staticmethod
+ def unknown_directive_message(directive_name):
+ return 'Unknown directive "{}".'.format(directive_name)
+
+ @staticmethod
+ def misplaced_directive_message(directive_name, location):
+ return 'Directive "{}" may not be used on "{}".'.format(directive_name, location)
+
+
+_operation_definition_map = {
+ 'query': DirectiveLocation.QUERY,
+ 'mutation': DirectiveLocation.MUTATION,
+ 'subscription': DirectiveLocation.SUBSCRIPTION,
+}
+
+
+def get_directive_location_for_ast_path(ancestors):
+ applied_to = ancestors[-1]
+ if isinstance(applied_to, ast.OperationDefinition):
+ return _operation_definition_map.get(applied_to.operation)
+
+ elif isinstance(applied_to, ast.Field):
+ return DirectiveLocation.FIELD
+
+ elif isinstance(applied_to, ast.FragmentSpread):
+ return DirectiveLocation.FRAGMENT_SPREAD
+
+ elif isinstance(applied_to, ast.InlineFragment):
+ return DirectiveLocation.INLINE_FRAGMENT
+
+ elif isinstance(applied_to, ast.FragmentDefinition):
+ return DirectiveLocation.FRAGMENT_DEFINITION
+
+ elif isinstance(applied_to, ast.SchemaDefinition):
+ return DirectiveLocation.SCHEMA
+
+ elif isinstance(applied_to, ast.ScalarTypeDefinition):
+ return DirectiveLocation.SCALAR
+
+ elif isinstance(applied_to, ast.ObjectTypeDefinition):
+ return DirectiveLocation.OBJECT
+
+ elif isinstance(applied_to, ast.FieldDefinition):
+ return DirectiveLocation.FIELD_DEFINITION
+
+ elif isinstance(applied_to, ast.InterfaceTypeDefinition):
+ return DirectiveLocation.INTERFACE
+
+ elif isinstance(applied_to, ast.UnionTypeDefinition):
+ return DirectiveLocation.UNION
+
+ elif isinstance(applied_to, ast.EnumTypeDefinition):
+ return DirectiveLocation.ENUM
+
+ elif isinstance(applied_to, ast.EnumValueDefinition):
+ return DirectiveLocation.ENUM_VALUE
+
+ elif isinstance(applied_to, ast.InputObjectTypeDefinition):
+ return DirectiveLocation.INPUT_OBJECT
+
+ elif isinstance(applied_to, ast.InputValueDefinition):
+ parent_node = ancestors[-3]
+ return (DirectiveLocation.INPUT_FIELD_DEFINITION
+ if isinstance(parent_node, ast.InputObjectTypeDefinition)
+ else DirectiveLocation.ARGUMENT_DEFINITION)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_fragment_names.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_fragment_names.py
new file mode 100644
index 0000000000000000000000000000000000000000..6c7375e3c4c9d8b083b3873643114bbf7a8625b3
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_fragment_names.py
@@ -0,0 +1,19 @@
+from ...error import GraphQLError
+from .base import ValidationRule
+
+
+class KnownFragmentNames(ValidationRule):
+
+ def enter_FragmentSpread(self, node, key, parent, path, ancestors):
+ fragment_name = node.name.value
+ fragment = self.context.get_fragment(fragment_name)
+
+ if not fragment:
+ self.context.report_error(GraphQLError(
+ self.unknown_fragment_message(fragment_name),
+ [node.name]
+ ))
+
+ @staticmethod
+ def unknown_fragment_message(fragment_name):
+ return 'Unknown fragment "{}".'.format(fragment_name)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_type_names.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_type_names.py
new file mode 100644
index 0000000000000000000000000000000000000000..d7d1369933238e797f3efced67c7a38c5dbea689
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_type_names.py
@@ -0,0 +1,43 @@
+from ...error import GraphQLError
+from ...utils.quoted_or_list import quoted_or_list
+from ...utils.suggestion_list import suggestion_list
+from .base import ValidationRule
+
+
+def _unknown_type_message(type, suggested_types):
+ message = 'Unknown type "{}".'.format(type)
+ if suggested_types:
+ message += ' Perhaps you meant {}?'.format(quoted_or_list(suggested_types))
+
+ return message
+
+
+class KnownTypeNames(ValidationRule):
+
+ def enter_ObjectTypeDefinition(self, node, *args):
+ return False
+
+ def enter_InterfaceTypeDefinition(self, node, *args):
+ return False
+
+ def enter_UnionTypeDefinition(self, node, *args):
+ return False
+
+ def enter_InputObjectTypeDefinition(self, node, *args):
+ return False
+
+ def enter_NamedType(self, node, *args):
+ schema = self.context.get_schema()
+ type_name = node.name.value
+ type = schema.get_type(type_name)
+
+ if not type:
+ self.context.report_error(
+ GraphQLError(
+ _unknown_type_message(
+ type_name,
+ suggestion_list(type_name, list(schema.get_type_map().keys()))
+ ),
+ [node]
+ )
+ )
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/lone_anonymous_operation.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/lone_anonymous_operation.py
new file mode 100644
index 0000000000000000000000000000000000000000..462a67f31174753baac5b421a0142c0c12c04e08
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/lone_anonymous_operation.py
@@ -0,0 +1,23 @@
+from ...error import GraphQLError
+from ...language import ast
+from .base import ValidationRule
+
+
+class LoneAnonymousOperation(ValidationRule):
+ __slots__ = 'operation_count',
+
+ def __init__(self, context):
+ self.operation_count = 0
+ super(LoneAnonymousOperation, self).__init__(context)
+
+ def enter_Document(self, node, key, parent, path, ancestors):
+ self.operation_count = \
+ sum(1 for definition in node.definitions if isinstance(definition, ast.OperationDefinition))
+
+ def enter_OperationDefinition(self, node, key, parent, path, ancestors):
+ if not node.name and self.operation_count > 1:
+ self.context.report_error(GraphQLError(self.anonymous_operation_not_alone_message(), [node]))
+
+ @staticmethod
+ def anonymous_operation_not_alone_message():
+ return 'This anonymous operation must be the only defined operation.'
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_fragment_cycles.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_fragment_cycles.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2e0d79f0e4bc0130180a44ab40fa6102f8d68b3
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_fragment_cycles.py
@@ -0,0 +1,59 @@
+from ...error import GraphQLError
+from .base import ValidationRule
+
+
+class NoFragmentCycles(ValidationRule):
+ __slots__ = 'errors', 'visited_frags', 'spread_path', 'spread_path_index_by_name'
+
+ def __init__(self, context):
+ super(NoFragmentCycles, self).__init__(context)
+ self.errors = []
+ self.visited_frags = set()
+ self.spread_path = []
+ self.spread_path_index_by_name = {}
+
+ def enter_OperationDefinition(self, node, key, parent, path, ancestors):
+ return False
+
+ def enter_FragmentDefinition(self, node, key, parent, path, ancestors):
+ if node.name.value not in self.visited_frags:
+ self.detect_cycle_recursive(node)
+ return False
+
+ def detect_cycle_recursive(self, fragment):
+ fragment_name = fragment.name.value
+ self.visited_frags.add(fragment_name)
+
+ spread_nodes = self.context.get_fragment_spreads(fragment.selection_set)
+ if not spread_nodes:
+ return
+
+ self.spread_path_index_by_name[fragment_name] = len(self.spread_path)
+
+ for spread_node in spread_nodes:
+ spread_name = spread_node.name.value
+ cycle_index = self.spread_path_index_by_name.get(spread_name)
+
+ if cycle_index is None:
+ self.spread_path.append(spread_node)
+ if spread_name not in self.visited_frags:
+ spread_fragment = self.context.get_fragment(spread_name)
+ if spread_fragment:
+ self.detect_cycle_recursive(spread_fragment)
+ self.spread_path.pop()
+ else:
+ cycle_path = self.spread_path[cycle_index:]
+ self.context.report_error(GraphQLError(
+ self.cycle_error_message(
+ spread_name,
+ [s.name.value for s in cycle_path]
+ ),
+ cycle_path + [spread_node]
+ ))
+
+ self.spread_path_index_by_name[fragment_name] = None
+
+ @staticmethod
+ def cycle_error_message(fragment_name, spread_names):
+ via = ' via {}'.format(', '.join(spread_names)) if spread_names else ''
+ return 'Cannot spread fragment "{}" within itself{}.'.format(fragment_name, via)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_undefined_variables.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_undefined_variables.py
new file mode 100644
index 0000000000000000000000000000000000000000..81c9c4953cb57cb91b30e2488693f350704012c3
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_undefined_variables.py
@@ -0,0 +1,36 @@
+from ...error import GraphQLError
+from .base import ValidationRule
+
+
+class NoUndefinedVariables(ValidationRule):
+ __slots__ = 'defined_variable_names',
+
+ def __init__(self, context):
+ self.defined_variable_names = set()
+ super(NoUndefinedVariables, self).__init__(context)
+
+ @staticmethod
+ def undefined_var_message(var_name, op_name=None):
+ if op_name:
+ return 'Variable "${}" is not defined by operation "{}".'.format(
+ var_name, op_name
+ )
+ return 'Variable "${}" is not defined.'.format(var_name)
+
+ def enter_OperationDefinition(self, operation, key, parent, path, ancestors):
+ self.defined_variable_names = set()
+
+ def leave_OperationDefinition(self, operation, key, parent, path, ancestors):
+ usages = self.context.get_recursive_variable_usages(operation)
+
+ for variable_usage in usages:
+ node = variable_usage.node
+ var_name = node.name.value
+ if var_name not in self.defined_variable_names:
+ self.context.report_error(GraphQLError(
+ self.undefined_var_message(var_name, operation.name and operation.name.value),
+ [node, operation]
+ ))
+
+ def enter_VariableDefinition(self, node, key, parent, path, ancestors):
+ self.defined_variable_names.add(node.variable.name.value)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_unused_fragments.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_unused_fragments.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d35f483612600227fa158ceb39eebd7eb8cdd6f
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_unused_fragments.py
@@ -0,0 +1,38 @@
+from ...error import GraphQLError
+from .base import ValidationRule
+
+
+class NoUnusedFragments(ValidationRule):
+ __slots__ = 'fragment_definitions', 'operation_definitions', 'fragment_adjacencies', 'spread_names'
+
+ def __init__(self, context):
+ super(NoUnusedFragments, self).__init__(context)
+ self.operation_definitions = []
+ self.fragment_definitions = []
+
+ def enter_OperationDefinition(self, node, key, parent, path, ancestors):
+ self.operation_definitions.append(node)
+ return False
+
+ def enter_FragmentDefinition(self, node, key, parent, path, ancestors):
+ self.fragment_definitions.append(node)
+ return False
+
+ def leave_Document(self, node, key, parent, path, ancestors):
+ fragment_names_used = set()
+
+ for operation in self.operation_definitions:
+ fragments = self.context.get_recursively_referenced_fragments(operation)
+ for fragment in fragments:
+ fragment_names_used.add(fragment.name.value)
+
+ for fragment_definition in self.fragment_definitions:
+ if fragment_definition.name.value not in fragment_names_used:
+ self.context.report_error(GraphQLError(
+ self.unused_fragment_message(fragment_definition.name.value),
+ [fragment_definition]
+ ))
+
+ @staticmethod
+ def unused_fragment_message(fragment_name):
+ return 'Fragment "{}" is never used.'.format(fragment_name)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_unused_variables.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_unused_variables.py
new file mode 100644
index 0000000000000000000000000000000000000000..a799e826472214019ab8b9037954165f4a299749
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_unused_variables.py
@@ -0,0 +1,37 @@
+from ...error import GraphQLError
+from .base import ValidationRule
+
+
+class NoUnusedVariables(ValidationRule):
+ __slots__ = 'variable_definitions'
+
+ def __init__(self, context):
+ self.variable_definitions = []
+ super(NoUnusedVariables, self).__init__(context)
+
+ def enter_OperationDefinition(self, node, key, parent, path, ancestors):
+ self.variable_definitions = []
+
+ def leave_OperationDefinition(self, operation, key, parent, path, ancestors):
+ variable_name_used = set()
+ usages = self.context.get_recursive_variable_usages(operation)
+ op_name = operation.name and operation.name.value or None
+
+ for variable_usage in usages:
+ variable_name_used.add(variable_usage.node.name.value)
+
+ for variable_definition in self.variable_definitions:
+ if variable_definition.variable.name.value not in variable_name_used:
+ self.context.report_error(GraphQLError(
+ self.unused_variable_message(variable_definition.variable.name.value, op_name),
+ [variable_definition]
+ ))
+
+ def enter_VariableDefinition(self, node, key, parent, path, ancestors):
+ self.variable_definitions.append(node)
+
+ @staticmethod
+ def unused_variable_message(variable_name, op_name):
+ if op_name:
+ return 'Variable "${}" is never used in operation "{}".'.format(variable_name, op_name)
+ return 'Variable "${}" is never used.'.format(variable_name)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/overlapping_fields_can_be_merged.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/overlapping_fields_can_be_merged.py
new file mode 100644
index 0000000000000000000000000000000000000000..bc7f51acfc431a563cf5c3f7aeef1f4e9bacedf9
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/overlapping_fields_can_be_merged.py
@@ -0,0 +1,529 @@
+import itertools
+from collections import OrderedDict
+
+from ...error import GraphQLError
+from ...language import ast
+from ...language.printer import print_ast
+from ...pyutils.pair_set import PairSet
+from ...type.definition import (GraphQLInterfaceType, GraphQLList,
+ GraphQLNonNull, GraphQLObjectType,
+ get_named_type, is_leaf_type)
+from ...utils.type_comparators import is_equal_type
+from ...utils.type_from_ast import type_from_ast
+from .base import ValidationRule
+
+
+class OverlappingFieldsCanBeMerged(ValidationRule):
+ __slots__ = ('_compared_fragments', '_cached_fields_and_fragment_names', )
+
+ def __init__(self, context):
+ super(OverlappingFieldsCanBeMerged, self).__init__(context)
+ # A memoization for when two fragments are compared "between" each other for
+ # conflicts. Two fragments may be compared many times, so memoizing this can
+ # dramatically improve the performance of this validator.
+ self._compared_fragments = PairSet()
+
+ # A cache for the "field map" and list of fragment names found in any given
+ # selection set. Selection sets may be asked for this information multiple
+ # times, so this improves the performance of this validator.
+ self._cached_fields_and_fragment_names = {}
+
+ def leave_SelectionSet(self, node, key, parent, path, ancestors):
+ # Note: we validate on the reverse traversal so deeper conflicts will be
+ # caught first, for correct calculation of mutual exclusivity and for
+ # clearer error messages.
+ # field_map = _collect_field_asts_and_defs(
+ # self.context,
+ # self.context.get_parent_type(),
+ # node
+ # )
+
+ # conflicts = _find_conflicts(self.context, False, field_map, self.compared_set)
+ conflicts = _find_conflicts_within_selection_set(self.context, self._cached_fields_and_fragment_names,
+ self._compared_fragments, self.context.get_parent_type(),
+ node)
+
+ for (reason_name, reason), fields1, fields2 in conflicts:
+ self.context.report_error(GraphQLError(
+ self.fields_conflict_message(reason_name, reason),
+ list(fields1) + list(fields2)
+ ))
+
+ @staticmethod
+ def same_type(type1, type2):
+ return is_equal_type(type1, type2)
+ # return type1.is_same_type(type2)
+
+ @classmethod
+ def fields_conflict_message(cls, reason_name, reason):
+ return (
+ 'Fields "{}" conflict because {}. '
+ 'Use different aliases on the fields to fetch both if this was '
+ 'intentional.'
+ ).format(reason_name, cls.reason_message(reason))
+
+ @classmethod
+ def reason_message(cls, reason):
+ if isinstance(reason, list):
+ return ' and '.join('subfields "{}" conflict because {}'.format(reason_name, cls.reason_message(sub_reason))
+ for reason_name, sub_reason in reason)
+
+ return reason
+
+
+# Algorithm:
+#
+# Conflicts occur when two fields exist in a query which will produce the same
+# response name, but represent differing values, thus creating a conflict.
+# The algorithm below finds all conflicts via making a series of comparisons
+# between fields. In order to compare as few fields as possible, this makes
+# a series of comparisons "within" sets of fields and "between" sets of fields.
+#
+# Given any selection set, a collection produces both a set of fields by
+# also including all inline fragments, as well as a list of fragments
+# referenced by fragment spreads.
+#
+# A) Each selection set represented in the document first compares "within" its
+# collected set of fields, finding any conflicts between every pair of
+# overlapping fields.
+# Note: This is the only time that a the fields "within" a set are compared
+# to each other. After this only fields "between" sets are compared.
+#
+# B) Also, if any fragment is referenced in a selection set, then a
+# comparison is made "between" the original set of fields and the
+# referenced fragment.
+#
+# C) Also, if multiple fragments are referenced, then comparisons
+# are made "between" each referenced fragment.
+#
+# D) When comparing "between" a set of fields and a referenced fragment, first
+# a comparison is made between each field in the original set of fields and
+# each field in the the referenced set of fields.
+#
+# E) Also, if any fragment is referenced in the referenced selection set,
+# then a comparison is made "between" the original set of fields and the
+# referenced fragment (recursively referring to step D).
+#
+# F) When comparing "between" two fragments, first a comparison is made between
+# each field in the first referenced set of fields and each field in the the
+# second referenced set of fields.
+#
+# G) Also, any fragments referenced by the first must be compared to the
+# second, and any fragments referenced by the second must be compared to the
+# first (recursively referring to step F).
+#
+# H) When comparing two fields, if both have selection sets, then a comparison
+# is made "between" both selection sets, first comparing the set of fields in
+# the first selection set with the set of fields in the second.
+#
+# I) Also, if any fragment is referenced in either selection set, then a
+# comparison is made "between" the other set of fields and the
+# referenced fragment.
+#
+# J) Also, if two fragments are referenced in both selection sets, then a
+# comparison is made "between" the two fragments.
+
+def _find_conflicts_within_selection_set(context, cached_fields_and_fragment_names, compared_fragments, parent_type,
+ selection_set):
+ """Find all conflicts found "within" a selection set, including those found via spreading in fragments.
+
+ Called when visiting each SelectionSet in the GraphQL Document.
+ """
+ conflicts = []
+
+ field_map, fragment_names = _get_fields_and_fragments_names(context, cached_fields_and_fragment_names, parent_type,
+ selection_set)
+
+ # (A) Find all conflicts "within" the fields of this selection set.
+ # Note: this is the *only place* `collect_conflicts_within` is called.
+ _collect_conflicts_within(
+ context,
+ conflicts,
+ cached_fields_and_fragment_names,
+ compared_fragments,
+ field_map
+ )
+
+ # (B) Then collect conflicts between these fields and those represented by
+ # each spread fragment name found.
+ for i, fragment_name in enumerate(fragment_names):
+ _collect_conflicts_between_fields_and_fragment(
+ context,
+ conflicts,
+ cached_fields_and_fragment_names,
+ compared_fragments,
+ False,
+ field_map,
+ fragment_name,
+ )
+
+ # (C) Then compare this fragment with all other fragments found in this
+ # selection set to collect conflicts within fragments spread together.
+ # This compares each item in the list of fragment names to every other item
+ # in that same list (except for itself).
+ for other_fragment_name in fragment_names[i+1:]:
+ _collect_conflicts_between_fragments(
+ context,
+ conflicts,
+ cached_fields_and_fragment_names,
+ compared_fragments,
+ False,
+ fragment_name,
+ other_fragment_name,
+ )
+
+ return conflicts
+
+
+def _collect_conflicts_between_fields_and_fragment(context, conflicts, cached_fields_and_fragment_names,
+ compared_fragments, are_mutually_exclusive, field_map,
+ fragment_name):
+
+ fragment = context.get_fragment(fragment_name)
+
+ if not fragment:
+ return None
+
+ field_map2, fragment_names2 = _get_referenced_fields_and_fragment_names(context, cached_fields_and_fragment_names,
+ fragment)
+
+ # (D) First collect any conflicts between the provided collection of fields
+ # and the collection of fields represented by the given fragment.
+ _collect_conflicts_between(context, conflicts, cached_fields_and_fragment_names, compared_fragments,
+ are_mutually_exclusive, field_map, field_map2)
+
+ # (E) Then collect any conflicts between the provided collection of fields
+ # and any fragment names found in the given fragment.
+ for fragment_name2 in fragment_names2:
+ _collect_conflicts_between_fields_and_fragment(context, conflicts, cached_fields_and_fragment_names,
+ compared_fragments, are_mutually_exclusive, field_map,
+ fragment_name2)
+
+
+# Collect all conflicts found between two fragments, including via spreading in
+# any nested fragments
+def _collect_conflicts_between_fragments(context, conflicts, cached_fields_and_fragment_names, compared_fragments,
+ are_mutually_exclusive, fragment_name1, fragment_name2):
+
+ fragment1 = context.get_fragment(fragment_name1)
+ fragment2 = context.get_fragment(fragment_name2)
+
+ if not fragment1 or not fragment2:
+ return None
+
+ # No need to compare a fragment to itself.
+ if fragment1 == fragment2:
+ return None
+
+ # Memoize so two fragments are not compared for conflicts more than once.
+ if compared_fragments.has(fragment_name1, fragment_name2, are_mutually_exclusive):
+ return None
+
+ compared_fragments.add(fragment_name1, fragment_name2, are_mutually_exclusive)
+
+ field_map1, fragment_names1 = _get_referenced_fields_and_fragment_names(context, cached_fields_and_fragment_names,
+ fragment1)
+
+ field_map2, fragment_names2 = _get_referenced_fields_and_fragment_names(context, cached_fields_and_fragment_names,
+ fragment2)
+
+ # (F) First, collect all conflicts between these two collections of fields
+ # (not including any nested fragments)
+ _collect_conflicts_between(context, conflicts, cached_fields_and_fragment_names, compared_fragments,
+ are_mutually_exclusive, field_map1, field_map2)
+
+ # (G) Then collect conflicts between the first fragment and any nested
+ # fragments spread in the second fragment.
+ for _fragment_name2 in fragment_names2:
+ _collect_conflicts_between_fragments(context, conflicts, cached_fields_and_fragment_names, compared_fragments,
+ are_mutually_exclusive, fragment_name1, _fragment_name2)
+
+ # (G) Then collect conflicts between the second fragment and any nested
+ # fragments spread in the first fragment.
+ for _fragment_name1 in fragment_names1:
+ _collect_conflicts_between_fragments(context, conflicts, cached_fields_and_fragment_names, compared_fragments,
+ are_mutually_exclusive, _fragment_name1, fragment_name2)
+
+
+def _find_conflicts_between_sub_selection_sets(context, cached_fields_and_fragment_names, compared_fragments,
+ are_mutually_exclusive, parent_type1, selection_set1,
+ parent_type2, selection_set2):
+ """Find all conflicts found between two selection sets.
+
+ Includes those found via spreading in fragments. Called when determining if conflicts exist
+ between the sub-fields of two overlapping fields.
+ """
+
+ conflicts = []
+
+ field_map1, fragment_names1 = _get_fields_and_fragments_names(context, cached_fields_and_fragment_names,
+ parent_type1, selection_set1)
+
+ field_map2, fragment_names2 = _get_fields_and_fragments_names(context, cached_fields_and_fragment_names,
+ parent_type2, selection_set2)
+
+ # (H) First, collect all conflicts between these two collections of field.
+ _collect_conflicts_between(context, conflicts, cached_fields_and_fragment_names, compared_fragments,
+ are_mutually_exclusive, field_map1, field_map2)
+
+ # (I) Then collect conflicts between the first collection of fields and
+ # those referenced by each fragment name associated with the second.
+ for fragment_name2 in fragment_names2:
+ _collect_conflicts_between_fields_and_fragment(context, conflicts, cached_fields_and_fragment_names,
+ compared_fragments, are_mutually_exclusive, field_map1,
+ fragment_name2)
+
+ # (I) Then collect conflicts between the second collection of fields and
+ # those referenced by each fragment name associated with the first.
+ for fragment_name1 in fragment_names1:
+ _collect_conflicts_between_fields_and_fragment(context, conflicts, cached_fields_and_fragment_names,
+ compared_fragments, are_mutually_exclusive, field_map2,
+ fragment_name1)
+
+ # (J) Also collect conflicts between any fragment names by the first and
+ # fragment names by the second. This compares each item in the first set of
+ # names to each item in the second set of names.
+ for fragment_name1 in fragment_names1:
+ for fragment_name2 in fragment_names2:
+ _collect_conflicts_between_fragments(context, conflicts, cached_fields_and_fragment_names,
+ compared_fragments, are_mutually_exclusive,
+ fragment_name1, fragment_name2)
+
+ return conflicts
+
+
+def _collect_conflicts_within(context, conflicts, cached_fields_and_fragment_names, compared_fragments, field_map):
+ """Collect all Conflicts "within" one collection of fields."""
+
+ # field map is a keyed collection, where each key represents a response
+ # name and the value at that key is a list of all fields which provide that
+ # response name. For every response name, if there are multiple fields, they
+ # must be compared to find a potential conflict.
+ for response_name, fields in list(field_map.items()):
+ # This compares every field in the list to every other field in this list
+ # (except to itself). If the list only has one item, nothing needs to
+ # be compared.
+ for i, field in enumerate(fields):
+ for other_field in fields[i+1:]:
+ # within one collection is never mutually exclusive
+ conflict = _find_conflict(context, cached_fields_and_fragment_names, compared_fragments, False,
+ response_name, field, other_field)
+ if conflict:
+ conflicts.append(conflict)
+
+
+def _collect_conflicts_between(context, conflicts, cached_fields_and_fragment_names, compared_fragments,
+ parent_fields_are_mutually_exclusive, field_map1, field_map2):
+ """Collect all Conflicts between two collections of fields.
+
+ This is similar to, but different from the `collect_conflicts_within` function above. This check assumes that
+ `collect_conflicts_within` has already been called on each provided collection of fields.
+ This is true because this validator traverses each individual selection set.
+ """
+ # A field map is a keyed collection, where each key represents a response
+ # name and the value at that key is a list of all fields which provide that
+ # response name. For any response name which appears in both provided field
+ # maps, each field from the first field map must be compared to every field
+ # in the second field map to find potential conflicts.
+ for response_name, fields1 in list(field_map1.items()):
+ fields2 = field_map2.get(response_name)
+
+ if fields2:
+ for field1 in fields1:
+ for field2 in fields2:
+ conflict = _find_conflict(context, cached_fields_and_fragment_names, compared_fragments,
+ parent_fields_are_mutually_exclusive, response_name, field1, field2)
+
+ if conflict:
+ conflicts.append(conflict)
+
+
+def _find_conflict(context, cached_fields_and_fragment_names, compared_fragments, parent_fields_are_mutually_exclusive,
+ response_name, field1, field2):
+ """Determines if there is a conflict between two particular fields."""
+ parent_type1, ast1, def1 = field1
+ parent_type2, ast2, def2 = field2
+
+ # If it is known that two fields could not possibly apply at the same
+ # time, due to the parent types, then it is safe to permit them to diverge
+ # in aliased field or arguments used as they will not present any ambiguity
+ # by differing.
+ # It is known that two parent types could never overlap if they are
+ # different Object types. Interface or Union types might overlap - if not
+ # in the current state of the schema, then perhaps in some future version,
+ # thus may not safely diverge.
+
+ are_mutually_exclusive = (
+ parent_fields_are_mutually_exclusive or (
+ parent_type1 != parent_type2 and
+ isinstance(parent_type1, GraphQLObjectType) and
+ isinstance(parent_type2, GraphQLObjectType)
+ )
+ )
+
+ # The return type for each field.
+ type1 = def1 and def1.type
+ type2 = def2 and def2.type
+
+ if not are_mutually_exclusive:
+ # Two aliases must refer to the same field.
+ name1 = ast1.name.value
+ name2 = ast2.name.value
+
+ if name1 != name2:
+ return (
+ (response_name, '{} and {} are different fields'.format(name1, name2)),
+ [ast1],
+ [ast2]
+ )
+
+ # Two field calls must have the same arguments.
+ if not _same_arguments(ast1.arguments, ast2.arguments):
+ return (
+ (response_name, 'they have differing arguments'),
+ [ast1],
+ [ast2]
+ )
+
+ if type1 and type2 and do_types_conflict(type1, type2):
+ return (
+ (response_name, 'they return conflicting types {} and {}'.format(type1, type2)),
+ [ast1],
+ [ast2]
+ )
+
+ # Collect and compare sub-fields. Use the same "visited fragment names" list
+ # for both collections so fields in a fragment reference are never
+ # compared to themselves.
+ selection_set1 = ast1.selection_set
+ selection_set2 = ast2.selection_set
+
+ if selection_set1 and selection_set2:
+ conflicts = _find_conflicts_between_sub_selection_sets(context, cached_fields_and_fragment_names,
+ compared_fragments, are_mutually_exclusive,
+ get_named_type(type1), selection_set1,
+ get_named_type(type2), selection_set2)
+
+ return _subfield_conflicts(conflicts, response_name, ast1, ast2)
+
+
+def _get_fields_and_fragments_names(context, cached_fields_and_fragment_names, parent_type, selection_set):
+ cached = cached_fields_and_fragment_names.get(selection_set)
+
+ if not cached:
+ ast_and_defs = OrderedDict()
+ fragment_names = OrderedDict()
+ _collect_fields_and_fragment_names(context, parent_type, selection_set, ast_and_defs, fragment_names)
+ cached = [ast_and_defs, list(fragment_names.keys())]
+ cached_fields_and_fragment_names[selection_set] = cached
+
+ return cached
+
+
+def _get_referenced_fields_and_fragment_names(context, cached_fields_and_fragment_names, fragment):
+ """Given a reference to a fragment, return the represented collection of fields as well as a list of
+ nested fragment names referenced via fragment spreads."""
+
+ # Short-circuit building a type from the AST if possible.
+ cached = cached_fields_and_fragment_names.get(fragment.selection_set)
+
+ if cached:
+ return cached
+
+ fragment_type = type_from_ast(context.get_schema(), fragment.type_condition)
+
+ return _get_fields_and_fragments_names(context, cached_fields_and_fragment_names,
+ fragment_type, fragment.selection_set)
+
+
+def _collect_fields_and_fragment_names(context, parent_type, selection_set, ast_and_defs, fragment_names):
+
+ for selection in selection_set.selections:
+ if isinstance(selection, ast.Field):
+ field_name = selection.name.value
+ if isinstance(parent_type, (GraphQLObjectType, GraphQLInterfaceType)):
+ field_def = parent_type.fields.get(field_name)
+ else:
+ field_def = None
+
+ response_name = selection.alias.value if selection.alias else field_name
+
+ if not ast_and_defs.get(response_name):
+ ast_and_defs[response_name] = []
+
+ ast_and_defs[response_name].append([parent_type, selection, field_def])
+
+ elif isinstance(selection, ast.FragmentSpread):
+ fragment_names[selection.name.value] = True
+ elif isinstance(selection, ast.InlineFragment):
+ type_condition = selection.type_condition
+ if type_condition:
+ inline_fragment_type = type_from_ast(context.get_schema(), selection.type_condition)
+ else:
+ inline_fragment_type = parent_type
+
+ _collect_fields_and_fragment_names(context, inline_fragment_type, selection.selection_set, ast_and_defs,
+ fragment_names)
+
+
+def _subfield_conflicts(conflicts, response_name, ast1, ast2):
+ """Given a series of Conflicts which occurred between two sub-fields, generate a single Conflict."""
+ if conflicts:
+ return (
+ (response_name, [conflict[0] for conflict in conflicts]),
+ tuple(itertools.chain([ast1], *[conflict[1] for conflict in conflicts])),
+ tuple(itertools.chain([ast2], *[conflict[2] for conflict in conflicts]))
+ )
+
+
+def do_types_conflict(type1, type2):
+ if isinstance(type1, GraphQLList):
+ if isinstance(type2, GraphQLList):
+ return do_types_conflict(type1.of_type, type2.of_type)
+ return True
+
+ if isinstance(type2, GraphQLList):
+ if isinstance(type1, GraphQLList):
+ return do_types_conflict(type1.of_type, type2.of_type)
+ return True
+
+ if isinstance(type1, GraphQLNonNull):
+ if isinstance(type2, GraphQLNonNull):
+ return do_types_conflict(type1.of_type, type2.of_type)
+ return True
+
+ if isinstance(type2, GraphQLNonNull):
+ if isinstance(type1, GraphQLNonNull):
+ return do_types_conflict(type1.of_type, type2.of_type)
+ return True
+
+ if is_leaf_type(type1) or is_leaf_type(type2):
+ return type1 != type2
+
+ return False
+
+
+def _same_value(value1, value2):
+ return (not value1 and not value2) or print_ast(value1) == print_ast(value2)
+
+
+def _same_arguments(arguments1, arguments2):
+ # Check to see if they are empty arguments or nones. If they are, we can
+ # bail out early.
+ if not (arguments1 or arguments2):
+ return True
+
+ if len(arguments1) != len(arguments2):
+ return False
+
+ arguments2_values_to_arg = {a.name.value: a for a in arguments2}
+
+ for argument1 in arguments1:
+ argument2 = arguments2_values_to_arg.get(argument1.name.value)
+ if not argument2:
+ return False
+
+ if not _same_value(argument1.value, argument2.value):
+ return False
+
+ return True
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/possible_fragment_spreads.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/possible_fragment_spreads.py
new file mode 100644
index 0000000000000000000000000000000000000000..b9cc4165867f59e042cd6233340dfa40a568a568
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/possible_fragment_spreads.py
@@ -0,0 +1,44 @@
+from ...error import GraphQLError
+from ...utils.type_comparators import do_types_overlap
+from ...utils.type_from_ast import type_from_ast
+from .base import ValidationRule
+
+
+class PossibleFragmentSpreads(ValidationRule):
+
+ def enter_InlineFragment(self, node, key, parent, path, ancestors):
+ frag_type = self.context.get_type()
+ parent_type = self.context.get_parent_type()
+ schema = self.context.get_schema()
+ if frag_type and parent_type and not do_types_overlap(schema, frag_type, parent_type):
+ self.context.report_error(GraphQLError(
+ self.type_incompatible_anon_spread_message(parent_type, frag_type),
+ [node]
+ ))
+
+ def enter_FragmentSpread(self, node, key, parent, path, ancestors):
+ frag_name = node.name.value
+ frag_type = self.get_fragment_type(self.context, frag_name)
+ parent_type = self.context.get_parent_type()
+ schema = self.context.get_schema()
+ if frag_type and parent_type and not do_types_overlap(schema, frag_type, parent_type):
+ self.context.report_error(GraphQLError(
+ self.type_incompatible_spread_message(frag_name, parent_type, frag_type),
+ [node]
+ ))
+
+ @staticmethod
+ def get_fragment_type(context, name):
+ frag = context.get_fragment(name)
+ return frag and type_from_ast(context.get_schema(), frag.type_condition)
+
+ @staticmethod
+ def type_incompatible_spread_message(frag_name, parent_type, frag_type):
+ return 'Fragment {} cannot be spread here as objects of type {} can never be of type {}'.format(frag_name,
+ parent_type,
+ frag_type)
+
+ @staticmethod
+ def type_incompatible_anon_spread_message(parent_type, frag_type):
+ return 'Fragment cannot be spread here as objects of type {} can never be of type {}'.format(parent_type,
+ frag_type)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/provided_non_null_arguments.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/provided_non_null_arguments.py
new file mode 100644
index 0000000000000000000000000000000000000000..09507277d922740bbc2d16b8e1dbc6401b31fc4b
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/provided_non_null_arguments.py
@@ -0,0 +1,46 @@
+from ...error import GraphQLError
+from ...type.definition import GraphQLNonNull
+from .base import ValidationRule
+
+
+class ProvidedNonNullArguments(ValidationRule):
+
+ def leave_Field(self, node, key, parent, path, ancestors):
+ field_def = self.context.get_field_def()
+ if not field_def:
+ return False
+
+ arg_asts = node.arguments or []
+ arg_ast_map = {arg.name.value: arg for arg in arg_asts}
+
+ for arg_name, arg_def in field_def.args.items():
+ arg_ast = arg_ast_map.get(arg_name, None)
+ if not arg_ast and isinstance(arg_def.type, GraphQLNonNull):
+ self.context.report_error(GraphQLError(
+ self.missing_field_arg_message(node.name.value, arg_name, arg_def.type),
+ [node]
+ ))
+
+ def leave_Directive(self, node, key, parent, path, ancestors):
+ directive_def = self.context.get_directive()
+ if not directive_def:
+ return False
+
+ arg_asts = node.arguments or []
+ arg_ast_map = {arg.name.value: arg for arg in arg_asts}
+
+ for arg_name, arg_def in directive_def.args.items():
+ arg_ast = arg_ast_map.get(arg_name, None)
+ if not arg_ast and isinstance(arg_def.type, GraphQLNonNull):
+ self.context.report_error(GraphQLError(
+ self.missing_directive_arg_message(node.name.value, arg_name, arg_def.type),
+ [node]
+ ))
+
+ @staticmethod
+ def missing_field_arg_message(name, arg_name, type):
+ return 'Field "{}" argument "{}" of type "{}" is required but not provided.'.format(name, arg_name, type)
+
+ @staticmethod
+ def missing_directive_arg_message(name, arg_name, type):
+ return 'Directive "{}" argument "{}" of type "{}" is required but not provided.'.format(name, arg_name, type)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/scalar_leafs.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/scalar_leafs.py
new file mode 100644
index 0000000000000000000000000000000000000000..f03efbc7af3ab1116894e589cb52f87ae9813d3e
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/scalar_leafs.py
@@ -0,0 +1,33 @@
+from ...error import GraphQLError
+from ...type.definition import get_named_type, is_leaf_type
+from .base import ValidationRule
+
+
+class ScalarLeafs(ValidationRule):
+
+ def enter_Field(self, node, key, parent, path, ancestors):
+ type = self.context.get_type()
+
+ if not type:
+ return
+
+ if is_leaf_type(get_named_type(type)):
+ if node.selection_set:
+ self.context.report_error(GraphQLError(
+ self.no_subselection_allowed_message(node.name.value, type),
+ [node.selection_set]
+ ))
+
+ elif not node.selection_set:
+ self.context.report_error(GraphQLError(
+ self.required_subselection_message(node.name.value, type),
+ [node]
+ ))
+
+ @staticmethod
+ def no_subselection_allowed_message(field, type):
+ return 'Field "{}" of type "{}" must not have a sub selection.'.format(field, type)
+
+ @staticmethod
+ def required_subselection_message(field, type):
+ return 'Field "{}" of type "{}" must have a sub selection.'.format(field, type)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_argument_names.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_argument_names.py
new file mode 100644
index 0000000000000000000000000000000000000000..9e25c75c8e5cfbe6e4853f806de68e7f6bbd97a4
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_argument_names.py
@@ -0,0 +1,32 @@
+from ...error import GraphQLError
+from .base import ValidationRule
+
+
+class UniqueArgumentNames(ValidationRule):
+ __slots__ = 'known_arg_names',
+
+ def __init__(self, context):
+ super(UniqueArgumentNames, self).__init__(context)
+ self.known_arg_names = {}
+
+ def enter_Field(self, node, key, parent, path, ancestors):
+ self.known_arg_names = {}
+
+ def enter_Directive(self, node, key, parent, path, ancestors):
+ self.known_arg_names = {}
+
+ def enter_Argument(self, node, key, parent, path, ancestors):
+ arg_name = node.name.value
+
+ if arg_name in self.known_arg_names:
+ self.context.report_error(GraphQLError(
+ self.duplicate_arg_message(arg_name),
+ [self.known_arg_names[arg_name], node.name]
+ ))
+ else:
+ self.known_arg_names[arg_name] = node.name
+ return False
+
+ @staticmethod
+ def duplicate_arg_message(field):
+ return 'There can only be one argument named "{}".'.format(field)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_fragment_names.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_fragment_names.py
new file mode 100644
index 0000000000000000000000000000000000000000..91de32714570b977bc96164f941af1a5f22764b9
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_fragment_names.py
@@ -0,0 +1,28 @@
+from ...error import GraphQLError
+from .base import ValidationRule
+
+
+class UniqueFragmentNames(ValidationRule):
+ __slots__ = 'known_fragment_names',
+
+ def __init__(self, context):
+ super(UniqueFragmentNames, self).__init__(context)
+ self.known_fragment_names = {}
+
+ def enter_OperationDefinition(self, node, key, parent, path, ancestors):
+ return False
+
+ def enter_FragmentDefinition(self, node, key, parent, path, ancestors):
+ fragment_name = node.name.value
+ if fragment_name in self.known_fragment_names:
+ self.context.report_error(GraphQLError(
+ self.duplicate_fragment_name_message(fragment_name),
+ [self.known_fragment_names[fragment_name], node.name]
+ ))
+ else:
+ self.known_fragment_names[fragment_name] = node.name
+ return False
+
+ @staticmethod
+ def duplicate_fragment_name_message(field):
+ return 'There can only be one fragment named "{}".'.format(field)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_input_field_names.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_input_field_names.py
new file mode 100644
index 0000000000000000000000000000000000000000..6fe18bab577d8b1f81eebda4b60b1181fd9d93a3
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_input_field_names.py
@@ -0,0 +1,33 @@
+from ...error import GraphQLError
+from .base import ValidationRule
+
+
+class UniqueInputFieldNames(ValidationRule):
+ __slots__ = 'known_names', 'known_names_stack'
+
+ def __init__(self, context):
+ super(UniqueInputFieldNames, self).__init__(context)
+ self.known_names = {}
+ self.known_names_stack = []
+
+ def enter_ObjectValue(self, node, key, parent, path, ancestors):
+ self.known_names_stack.append(self.known_names)
+ self.known_names = {}
+
+ def leave_ObjectValue(self, node, key, parent, path, ancestors):
+ self.known_names = self.known_names_stack.pop()
+
+ def enter_ObjectField(self, node, key, parent, path, ancestors):
+ field_name = node.name.value
+ if field_name in self.known_names:
+ self.context.report_error(GraphQLError(
+ self.duplicate_input_field_message(field_name),
+ [self.known_names[field_name], node.name]
+ ))
+ else:
+ self.known_names[field_name] = node.name
+ return False
+
+ @staticmethod
+ def duplicate_input_field_message(field_name):
+ return 'There can only be one input field named "{}".'.format(field_name)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_operation_names.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_operation_names.py
new file mode 100644
index 0000000000000000000000000000000000000000..1fccbb9b2230fb708bed7ee42d5ff96ce4083b20
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_operation_names.py
@@ -0,0 +1,31 @@
+from ...error import GraphQLError
+from .base import ValidationRule
+
+
+class UniqueOperationNames(ValidationRule):
+ __slots__ = 'known_operation_names',
+
+ def __init__(self, context):
+ super(UniqueOperationNames, self).__init__(context)
+ self.known_operation_names = {}
+
+ def enter_OperationDefinition(self, node, key, parent, path, ancestors):
+ operation_name = node.name
+ if not operation_name:
+ return
+
+ if operation_name.value in self.known_operation_names:
+ self.context.report_error(GraphQLError(
+ self.duplicate_operation_name_message(operation_name.value),
+ [self.known_operation_names[operation_name.value], operation_name]
+ ))
+ else:
+ self.known_operation_names[operation_name.value] = operation_name
+ return False
+
+ def enter_FragmentDefinition(self, node, key, parent, path, ancestors):
+ return False
+
+ @staticmethod
+ def duplicate_operation_name_message(operation_name):
+ return 'There can only be one operation named "{}".'.format(operation_name)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_variable_names.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_variable_names.py
new file mode 100644
index 0000000000000000000000000000000000000000..f471e3464343d8d304e8d62341daba736a2c4ab7
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_variable_names.py
@@ -0,0 +1,27 @@
+from ...error import GraphQLError
+from .base import ValidationRule
+
+
+class UniqueVariableNames(ValidationRule):
+ __slots__ = 'known_variable_names',
+
+ def __init__(self, context):
+ super(UniqueVariableNames, self).__init__(context)
+ self.known_variable_names = {}
+
+ def enter_OperationDefinition(self, node, key, parent, path, ancestors):
+ self.known_variable_names = {}
+
+ def enter_VariableDefinition(self, node, key, parent, path, ancestors):
+ variable_name = node.variable.name.value
+ if variable_name in self.known_variable_names:
+ self.context.report_error(GraphQLError(
+ self.duplicate_variable_message(variable_name),
+ [self.known_variable_names[variable_name], node.variable.name]
+ ))
+ else:
+ self.known_variable_names[variable_name] = node.variable.name
+
+ @staticmethod
+ def duplicate_variable_message(operation_name):
+ return 'There can be only one variable named "{}".'.format(operation_name)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/variables_are_input_types.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/variables_are_input_types.py
new file mode 100644
index 0000000000000000000000000000000000000000..f510fbba0ad1584ea5101e0869c5133f14dbb708
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/variables_are_input_types.py
@@ -0,0 +1,21 @@
+from ...error import GraphQLError
+from ...language.printer import print_ast
+from ...type.definition import is_input_type
+from ...utils.type_from_ast import type_from_ast
+from .base import ValidationRule
+
+
+class VariablesAreInputTypes(ValidationRule):
+
+ def enter_VariableDefinition(self, node, key, parent, path, ancestors):
+ type = type_from_ast(self.context.get_schema(), node.type)
+
+ if type and not is_input_type(type):
+ self.context.report_error(GraphQLError(
+ self.non_input_type_on_variable_message(node.variable.name.value, print_ast(node.type)),
+ [node.type]
+ ))
+
+ @staticmethod
+ def non_input_type_on_variable_message(variable_name, type_name):
+ return 'Variable "${}" cannot be non-input type "{}".'.format(variable_name, type_name)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/variables_in_allowed_position.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/variables_in_allowed_position.py
new file mode 100644
index 0000000000000000000000000000000000000000..4117e0f8217f3d7ddee4fc6850d4899a130c0602
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/variables_in_allowed_position.py
@@ -0,0 +1,53 @@
+from ...error import GraphQLError
+from ...type.definition import GraphQLNonNull
+from ...utils.type_comparators import is_type_sub_type_of
+from ...utils.type_from_ast import type_from_ast
+from .base import ValidationRule
+
+
+class VariablesInAllowedPosition(ValidationRule):
+ __slots__ = 'var_def_map'
+
+ def __init__(self, context):
+ super(VariablesInAllowedPosition, self).__init__(context)
+ self.var_def_map = {}
+
+ def enter_OperationDefinition(self, node, key, parent, path, ancestors):
+ self.var_def_map = {}
+
+ def leave_OperationDefinition(self, operation, key, parent, path, ancestors):
+ usages = self.context.get_recursive_variable_usages(operation)
+
+ for usage in usages:
+ node = usage.node
+ type = usage.type
+ var_name = node.name.value
+ var_def = self.var_def_map.get(var_name)
+ if var_def and type:
+ # A var type is allowed if it is the same or more strict (e.g. is
+ # a subtype of) than the expected type. It can be more strict if
+ # the variable type is non-null when the expected type is nullable.
+ # If both are list types, the variable item type can be more strict
+ # than the expected item type (contravariant).
+ schema = self.context.get_schema()
+ var_type = type_from_ast(schema, var_def.type)
+ if var_type and not is_type_sub_type_of(schema, self.effective_type(var_type, var_def), type):
+ self.context.report_error(GraphQLError(
+ self.bad_var_pos_message(var_name, var_type, type),
+ [var_def, node]
+ ))
+
+ def enter_VariableDefinition(self, node, key, parent, path, ancestors):
+ self.var_def_map[node.variable.name.value] = node
+
+ @staticmethod
+ def effective_type(var_type, var_def):
+ if not var_def.default_value or isinstance(var_type, GraphQLNonNull):
+ return var_type
+
+ return GraphQLNonNull(var_type)
+
+ @staticmethod
+ def bad_var_pos_message(var_name, var_type, expected_type):
+ return 'Variable "{}" of type "{}" used in position expecting type "{}".'.format(var_name, var_type,
+ expected_type)
diff --git a/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/validation.py b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/validation.py
new file mode 100644
index 0000000000000000000000000000000000000000..3610dbf4812ff5c7971a326d2a4e68fd3805c6c7
--- /dev/null
+++ b/wandb/vendor/graphql-core-1.1/wandb_graphql/validation/validation.py
@@ -0,0 +1,158 @@
+from ..language.ast import (FragmentDefinition, FragmentSpread,
+ OperationDefinition)
+from ..language.visitor import ParallelVisitor, TypeInfoVisitor, Visitor, visit
+from ..type import GraphQLSchema
+from ..utils.type_info import TypeInfo
+from .rules import specified_rules
+
+
+def validate(schema, ast, rules=specified_rules):
+ assert schema, 'Must provide schema'
+ assert ast, 'Must provide document'
+ assert isinstance(schema, GraphQLSchema)
+ type_info = TypeInfo(schema)
+ return visit_using_rules(schema, type_info, ast, rules)
+
+
+def visit_using_rules(schema, type_info, ast, rules):
+ context = ValidationContext(schema, ast, type_info)
+ visitors = [rule(context) for rule in rules]
+ visit(ast, TypeInfoVisitor(type_info, ParallelVisitor(visitors)))
+ return context.get_errors()
+
+
+class VariableUsage(object):
+ __slots__ = 'node', 'type'
+
+ def __init__(self, node, type):
+ self.node = node
+ self.type = type
+
+
+class UsageVisitor(Visitor):
+ __slots__ = 'usages', 'type_info'
+
+ def __init__(self, usages, type_info):
+ self.usages = usages
+ self.type_info = type_info
+
+ def enter_VariableDefinition(self, node, key, parent, path, ancestors):
+ return False
+
+ def enter_Variable(self, node, key, parent, path, ancestors):
+ usage = VariableUsage(node, type=self.type_info.get_input_type())
+ self.usages.append(usage)
+
+
+class ValidationContext(object):
+ __slots__ = ('_schema', '_ast', '_type_info', '_errors', '_fragments', '_fragment_spreads',
+ '_recursively_referenced_fragments', '_variable_usages', '_recursive_variable_usages')
+
+ def __init__(self, schema, ast, type_info):
+ self._schema = schema
+ self._ast = ast
+ self._type_info = type_info
+ self._errors = []
+ self._fragments = None
+ self._fragment_spreads = {}
+ self._recursively_referenced_fragments = {}
+ self._variable_usages = {}
+ self._recursive_variable_usages = {}
+
+ def report_error(self, error):
+ self._errors.append(error)
+
+ def get_errors(self):
+ return self._errors
+
+ def get_schema(self):
+ return self._schema
+
+ def get_variable_usages(self, node):
+ usages = self._variable_usages.get(node)
+ if usages is None:
+ usages = []
+ sub_visitor = UsageVisitor(usages, self._type_info)
+ visit(node, TypeInfoVisitor(self._type_info, sub_visitor))
+ self._variable_usages[node] = usages
+
+ return usages
+
+ def get_recursive_variable_usages(self, operation):
+ assert isinstance(operation, OperationDefinition)
+ usages = self._recursive_variable_usages.get(operation)
+ if usages is None:
+ usages = self.get_variable_usages(operation)
+ fragments = self.get_recursively_referenced_fragments(operation)
+ for fragment in fragments:
+ usages.extend(self.get_variable_usages(fragment))
+ self._recursive_variable_usages[operation] = usages
+
+ return usages
+
+ def get_recursively_referenced_fragments(self, operation):
+ assert isinstance(operation, OperationDefinition)
+ fragments = self._recursively_referenced_fragments.get(operation)
+ if not fragments:
+ fragments = []
+ collected_names = set()
+ nodes_to_visit = [operation.selection_set]
+ while nodes_to_visit:
+ node = nodes_to_visit.pop()
+ spreads = self.get_fragment_spreads(node)
+ for spread in spreads:
+ frag_name = spread.name.value
+ if frag_name not in collected_names:
+ collected_names.add(frag_name)
+ fragment = self.get_fragment(frag_name)
+ if fragment:
+ fragments.append(fragment)
+ nodes_to_visit.append(fragment.selection_set)
+ self._recursively_referenced_fragments[operation] = fragments
+ return fragments
+
+ def get_fragment_spreads(self, node):
+ spreads = self._fragment_spreads.get(node)
+ if not spreads:
+ spreads = []
+ sets_to_visit = [node]
+ while sets_to_visit:
+ _set = sets_to_visit.pop()
+ for selection in _set.selections:
+ if isinstance(selection, FragmentSpread):
+ spreads.append(selection)
+ elif selection.selection_set:
+ sets_to_visit.append(selection.selection_set)
+
+ self._fragment_spreads[node] = spreads
+ return spreads
+
+ def get_ast(self):
+ return self._ast
+
+ def get_fragment(self, name):
+ fragments = self._fragments
+ if fragments is None:
+ self._fragments = fragments = {}
+ for statement in self.get_ast().definitions:
+ if isinstance(statement, FragmentDefinition):
+ fragments[statement.name.value] = statement
+ return fragments.get(name)
+
+ def get_type(self):
+ return self._type_info.get_type()
+
+ def get_parent_type(self):
+ return self._type_info.get_parent_type()
+
+ def get_input_type(self):
+ return self._type_info.get_input_type()
+
+ def get_field_def(self):
+ return self._type_info.get_field_def()
+
+ def get_directive(self):
+ return self._type_info.get_directive()
+
+ def get_argument(self):
+ return self._type_info.get_argument()
diff --git a/wandb/vendor/promise-2.3.0/.coveragerc b/wandb/vendor/promise-2.3.0/.coveragerc
new file mode 100644
index 0000000000000000000000000000000000000000..46d6196f0cebb15f69e4de94843919a4d1ed03e6
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/.coveragerc
@@ -0,0 +1,2 @@
+[run]
+omit = promise/compat.py,promise/iterate_promise.py,promise/utils.py
diff --git a/wandb/vendor/promise-2.3.0/.gitignore b/wandb/vendor/promise-2.3.0/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..4ff83cd2b99d2408d6c02608acf65f5f12d1bad7
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/.gitignore
@@ -0,0 +1,71 @@
+# Created by https://www.gitignore.io
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IntelliJ
+.idea
+
+# OS X
+.DS_Store
+/.vscode
+/.pytest_cache
+/.pyre
+/.mypy_cache
+/type_info.json
diff --git a/wandb/vendor/promise-2.3.0/.travis.yml b/wandb/vendor/promise-2.3.0/.travis.yml
new file mode 100644
index 0000000000000000000000000000000000000000..81a125ff0a0728cdd8d17cea0c7fdccdc1b0de4c
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/.travis.yml
@@ -0,0 +1,27 @@
+language: python
+sudo: false
+python:
+- 2.7
+- 3.5
+- 3.6
+- 3.7
+- 3.8
+- pypy
+cache: pip
+install:
+- pip install -e .[test]
+script:
+- py.test --cov=promise tests
+after_success:
+- coveralls
+matrix:
+ fast_finish: true
+ include:
+ - python: '3.8'
+ script: |
+ # pip install pyre-check
+ # pyre --source-directory promise check
+ pip install flake8
+ flake8
+ pip install mypy
+ mypy promise --ignore-missing-imports
diff --git a/wandb/vendor/promise-2.3.0/LICENSE b/wandb/vendor/promise-2.3.0/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..a3c843c2f839d1d62f429ed5eeec550d38eea7c0
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/LICENSE
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Syrus Akbary
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/wandb/vendor/promise-2.3.0/MANIFEST.in b/wandb/vendor/promise-2.3.0/MANIFEST.in
new file mode 100644
index 0000000000000000000000000000000000000000..51530e6e06c4cc5367b4aa9f6be8778dc5a0a4a0
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/MANIFEST.in
@@ -0,0 +1,3 @@
+global-exclude tests/*
+recursive-exclude tests *
+include LICENSE
diff --git a/wandb/vendor/promise-2.3.0/README.md b/wandb/vendor/promise-2.3.0/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..33828a391c5cb5b749f87d32cb502e5150bb1ec7
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/README.md
@@ -0,0 +1,160 @@
+# Promise
+
+This is a implementation of Promises in Python.
+It is a super set of Promises/A+ designed to have readable, performant code and to provide just the extensions that are absolutely necessary for using promises in Python.
+
+Its fully compatible with the [Promises/A+ spec](http://promises-aplus.github.io/promises-spec/)
+
+[![travis][travis-image]][travis-url]
+[![pypi][pypi-image]][pypi-url]
+[![coveralls][coveralls-image]][coveralls-url]
+
+[travis-image]: https://img.shields.io/travis/syrusakbary/promise.svg?style=flat
+[travis-url]: https://travis-ci.org/syrusakbary/promise
+[pypi-image]: https://img.shields.io/pypi/v/promise.svg?style=flat
+[pypi-url]: https://pypi.python.org/pypi/promise
+[coveralls-image]: https://coveralls.io/repos/syrusakbary/promise/badge.svg?branch=master&service=github
+[coveralls-url]: https://coveralls.io/github/syrusakbary/promise?branch=master
+
+## Installation
+
+ $ pip install promise
+
+## Usage
+
+The example below shows how you can load the promise library. It then demonstrates creating a promise from scratch. You simply call `Promise(fn)`. There is a complete specification for what is returned by this method in [Promises/A+](http://promises-aplus.github.com/promises-spec/).
+
+```python
+from promise import Promise
+
+promise = Promise(
+ lambda resolve, reject: resolve('RESOLVED!')
+)
+```
+
+## API
+
+Before all examples, you will need:
+
+```python
+from promise import Promise
+```
+
+### Promise(resolver)
+
+This creates and returns a new promise. `resolver` must be a function. The `resolver` function is passed two arguments:
+
+1. `resolve` should be called with a single argument. If it is called with a non-promise value then the promise is fulfilled with that value. If it is called with a promise (A) then the returned promise takes on the state of that new promise (A).
+2. `reject` should be called with a single argument. The returned promise will be rejected with that argument.
+
+### Class Methods
+
+These methods are invoked by calling `Promise.methodName`.
+
+#### Promise.resolve(value)
+
+Converts values and foreign promises into Promises/A+ promises. If you pass it a value then it returns a Promise for that value. If you pass it something that is close to a promise (such as a jQuery attempt at a promise) it returns a Promise that takes on the state of `value` (rejected or fulfilled).
+
+#### Promise.reject(value)
+
+Returns a rejected promise with the given value.
+
+#### Promise.all(list)
+
+Returns a promise for a list. If it is called with a single argument then this returns a promise for a copy of that list with any promises replaced by their fulfilled values. e.g.
+
+```python
+p = Promise.all([Promise.resolve('a'), 'b', Promise.resolve('c')]) \
+ .then(lambda res: res == ['a', 'b', 'c'])
+
+assert p.get() is True
+```
+
+#### Promise.cast(obj)
+
+This function wraps the `obj` ect as a `Promise` if possible.
+Python `Future`s are supported, with a callback to `promise.done` when resolved.
+Has the same effects as `Promise.resolve(obj)`.
+
+#### Promise.for_dict(d)
+
+A special function that takes a dictionary of promises and turns them
+into a promise for a dictionary of values. In other words, this turns
+a dictionary of promises for values into a promise for a dictionary
+of values.
+
+#### Promise.is_thenable(obj)
+
+This function checks if the `obj` is a `Promise`, or could be `cast`ed.
+
+#### Promise.promisify(func)
+
+This function wraps the result of calling `func` in a `Promise` instance.
+
+### Instance Methods
+
+These methods are invoked on a promise instance by calling `myPromise.methodName`
+
+### promise.then(did_fulfill, did_reject)
+
+This method follows the [Promises/A+ spec](http://promises-aplus.github.io/promises-spec/). It explains things very clearly so I recommend you read it.
+
+Either `did_fulfill` or `did_reject` will be called and they will not be called more than once. They will be passed a single argument and will always be called asynchronously (in the next turn of the event loop).
+
+If the promise is fulfilled then `did_fulfill` is called. If the promise is rejected then `did_reject` is called.
+
+The call to `.then` also returns a promise. If the handler that is called returns a promise, the promise returned by `.then` takes on the state of that returned promise. If the handler that is called returns a value that is not a promise, the promise returned by `.then` will be fulfilled with that value. If the handler that is called throws an exception then the promise returned by `.then` is rejected with that exception.
+
+#### promise.catch(did_reject)
+
+Sugar for `promise.then(None, did_reject)`, to mirror `catch` in synchronous code.
+
+#### promise.done(did_fulfill, did_reject)
+
+The same semantics as `.then` except that it does not return a promise and any exceptions are re-thrown so that they can be logged (crashing the application in non-browser environments)
+
+# Contributing
+
+After cloning this repo, ensure dependencies are installed by running:
+
+```sh
+pip install -e ".[test]"
+```
+
+After developing, the full test suite can be evaluated by running:
+
+```sh
+py.test tests --cov=promise --benchmark-skip # Use -v -s for verbose mode
+```
+
+You can also run the benchmarks with:
+
+```sh
+py.test tests --benchmark-only
+```
+
+## Static type checking
+
+Python type annotations are very useful for making sure we use the libary the way is intended.
+
+You can run `mypy` static type checker:
+
+```sh
+pip install mypy
+mypy promise --ignore-missing-imports
+```
+
+Or `pyre`:
+
+```sh
+pip install pyre-check
+pyre --source-directory promise check
+```
+
+# Notes
+
+This package is heavily insipired in [aplus](https://github.com/xogeny/aplus).
+
+## License
+
+[MIT License](https://github.com/syrusakbary/promise/blob/master/LICENSE)
diff --git a/wandb/vendor/promise-2.3.0/README.rst b/wandb/vendor/promise-2.3.0/README.rst
new file mode 100644
index 0000000000000000000000000000000000000000..5a0595c6737240aeb2137e7e3d81e94d116ade15
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/README.rst
@@ -0,0 +1,220 @@
+Promise
+=======
+
+This is a implementation of Promises in Python. It is a super set of
+Promises/A+ designed to have readable, performant code and to provide
+just the extensions that are absolutely necessary for using promises in
+Python.
+
+Its fully compatible with the `Promises/A+
+spec <http://promises-aplus.github.io/promises-spec/>`__
+
+|travis| |pypi| |coveralls|
+
+Installation
+------------
+
+::
+
+ $ pip install promise
+
+Usage
+-----
+
+The example below shows how you can load the promise library. It then
+demonstrates creating a promise from scratch. You simply call
+``Promise(fn)``. There is a complete specification for what is returned
+by this method in
+`Promises/A+ <http://promises-aplus.github.com/promises-spec/>`__.
+
+.. code:: python
+
+ from promise import Promise
+
+ promise = Promise(
+ lambda resolve, reject: resolve('RESOLVED!')
+ )
+
+API
+---
+
+Before all examples, you will need:
+
+.. code:: python
+
+ from promise import Promise
+
+Promise(resolver)
+~~~~~~~~~~~~~~~~~
+
+This creates and returns a new promise. ``resolver`` must be a function.
+The ``resolver`` function is passed two arguments:
+
+1. ``resolve`` should be called with a single argument. If it is called
+ with a non-promise value then the promise is fulfilled with that
+ value. If it is called with a promise (A) then the returned promise
+ takes on the state of that new promise (A).
+2. ``reject`` should be called with a single argument. The returned
+ promise will be rejected with that argument.
+
+Class Methods
+~~~~~~~~~~~~~
+
+These methods are invoked by calling ``Promise.methodName``.
+
+Promise.resolve(value)
+^^^^^^^^^^^^^^^^^^^^^^
+
+Converts values and foreign promises into Promises/A+ promises. If you
+pass it a value then it returns a Promise for that value. If you pass it
+something that is close to a promise (such as a jQuery attempt at a
+promise) it returns a Promise that takes on the state of ``value``
+(rejected or fulfilled).
+
+Promise.reject(value)
+^^^^^^^^^^^^^^^^^^^^^
+
+Returns a rejected promise with the given value.
+
+Promise.all(list)
+^^^^^^^^^^^^^^^^^
+
+Returns a promise for a list. If it is called with a single argument
+then this returns a promise for a copy of that list with any promises
+replaced by their fulfilled values. e.g.
+
+.. code:: python
+
+ p = Promise.all([Promise.resolve('a'), 'b', Promise.resolve('c')]) \
+ .then(lambda res: res == ['a', 'b', 'c'])
+
+ assert p.get() is True
+
+Promise.cast(obj)
+^^^^^^^^^^^^^^^^^
+
+This function wraps the ``obj`` act as a ``Promise`` if possible. Python
+``Future``\ s are supported, with a callback to ``promise.done`` when
+resolved. Have the same effects as ``Promise.resolve(obj)``.
+
+Promise.for\_dict(d)
+^^^^^^^^^^^^^^^^^^^^
+
+A special function that takes a dictionary of promises and turns them
+into a promise for a dictionary of values. In other words, this turns an
+dictionary of promises for values into a promise for a dictionary of
+values.
+
+Promise.is\_thenable(obj)
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This function checks if the ``obj`` is a ``Promise``, or could be
+``cast``\ ed.
+
+Promise.promisify(func)
+^^^^^^^^^^^^^^^^^^^^^^^
+
+This function wraps the result of calling ``func`` in a ``Promise``
+instance.
+
+Instance Methods
+~~~~~~~~~~~~~~~~
+
+These methods are invoked on a promise instance by calling
+``myPromise.methodName``
+
+promise.then(did\_fulfill, did\_reject)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This method follows the `Promises/A+
+spec <http://promises-aplus.github.io/promises-spec/>`__. It explains
+things very clearly so I recommend you read it.
+
+Either ``did_fulfill`` or ``did_reject`` will be called and they will
+not be called more than once. They will be passed a single argument and
+will always be called asynchronously (in the next turn of the event
+loop).
+
+If the promise is fulfilled then ``did_fulfill`` is called. If the
+promise is rejected then ``did_reject`` is called.
+
+The call to ``.then`` also returns a promise. If the handler that is
+called returns a promise, the promise returned by ``.then`` takes on the
+state of that returned promise. If the handler that is called returns a
+value that is not a promise, the promise returned by ``.then`` will be
+fulfilled with that value. If the handler that is called throws an
+exception then the promise returned by ``.then`` is rejected with that
+exception.
+
+promise.catch(did\_reject)
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Sugar for ``promise.then(None, did_reject)``, to mirror ``catch`` in
+synchronous code.
+
+promise.done(did\_fulfill, did\_reject)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The same semantics as ``.then`` except that it does not return a promise
+and any exceptions are re-thrown so that they can be logged (crashing
+the application in non-browser environments)
+
+Contributing
+============
+
+After cloning this repo, ensure dependencies are installed by running:
+
+.. code:: sh
+
+ pip install -e ".[test]"
+
+After developing, the full test suite can be evaluated by running:
+
+.. code:: sh
+
+ py.test tests --cov=promise --benchmark-skip # Use -v -s for verbose mode
+
+You can also run the benchmarks with:
+
+.. code:: sh
+
+ py.test tests --benchmark-only
+
+Static type checking
+--------------------
+
+Python type annotations are very useful for making sure we use the
+libary the way is intended.
+
+You can run ``mypy`` static type checker:
+
+.. code:: sh
+
+ pip install mypy
+ mypy promise --ignore-missing-imports
+
+Or ``pyre``:
+
+.. code:: sh
+
+ pip install pyre-check
+ pyre --source-directory promise check
+
+Notes
+=====
+
+This package is heavily insipired in
+`aplus <https://github.com/xogeny/aplus>`__.
+
+License
+-------
+
+`MIT
+License <https://github.com/syrusakbary/promise/blob/master/LICENSE>`__
+
+.. |travis| image:: https://img.shields.io/travis/syrusakbary/promise.svg?style=flat
+ :target: https://travis-ci.org/syrusakbary/promise
+.. |pypi| image:: https://img.shields.io/pypi/v/promise.svg?style=flat
+ :target: https://pypi.python.org/pypi/promise
+.. |coveralls| image:: https://coveralls.io/repos/syrusakbary/promise/badge.svg?branch=master&service=github
+ :target: https://coveralls.io/github/syrusakbary/promise?branch=master
diff --git a/wandb/vendor/promise-2.3.0/conftest.py b/wandb/vendor/promise-2.3.0/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..a36e6ba1ffed579764055782646373de3b9ed488
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/conftest.py
@@ -0,0 +1,30 @@
+# Configuration for pytest to automatically collect types.
+# Thanks to Guilherme Salgado.
+import pytest
+
+try:
+ import pyannotate_runtime
+ PYANOTATE_PRESENT = True
+except ImportError:
+ PYANOTATE_PRESENT = False
+
+if PYANOTATE_PRESENT:
+ def pytest_collection_finish(session):
+ """Handle the pytest collection finish hook: configure pyannotate.
+ Explicitly delay importing `collect_types` until all tests have
+ been collected. This gives gevent a chance to monkey patch the
+ world before importing pyannotate.
+ """
+ from pyannotate_runtime import collect_types
+ collect_types.init_types_collection()
+
+ @pytest.fixture(autouse=True)
+ def collect_types_fixture():
+ from pyannotate_runtime import collect_types
+ collect_types.resume()
+ yield
+ collect_types.pause()
+
+ def pytest_sessionfinish(session, exitstatus):
+ from pyannotate_runtime import collect_types
+ collect_types.dump_stats("type_info.json")
diff --git a/wandb/vendor/promise-2.3.0/setup.cfg b/wandb/vendor/promise-2.3.0/setup.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..acb94498119eea87982b6fba70119369de0e5750
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/setup.cfg
@@ -0,0 +1,3 @@
+[flake8]
+exclude = tests,scripts,setup.py,docs,promise/utils.py,conftest.py
+max-line-length = 120
diff --git a/wandb/vendor/promise-2.3.0/setup.py b/wandb/vendor/promise-2.3.0/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..9c13695c9d197a77ecbb069ae0bf2dd33d8b5fef
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/setup.py
@@ -0,0 +1,64 @@
+import sys
+from setuptools import setup, find_packages
+
+if sys.version_info[0] < 3:
+ import __builtin__ as builtins
+else:
+ import builtins
+
+builtins.__SETUP__ = True
+
+version = __import__("promise").get_version()
+
+
+IS_PY3 = sys.hexversion >= 0x03000000
+
+tests_require = [
+ "pytest>=2.7.3",
+ "pytest-cov",
+ "coveralls",
+ "futures",
+ "pytest-benchmark",
+ "mock",
+]
+if IS_PY3:
+ tests_require += ["pytest-asyncio"]
+
+
+setup(
+ name="promise",
+ version=version,
+ description="Promises/A+ implementation for Python",
+ long_description=open("README.rst").read(),
+ url="https://github.com/syrusakbary/promise",
+ download_url="https://github.com/syrusakbary/promise/releases",
+ author="Syrus Akbary",
+ author_email="me@syrusakbary.com",
+ license="MIT",
+ classifiers=[
+ "Development Status :: 5 - Production/Stable",
+ "Intended Audience :: Developers",
+ "Topic :: Software Development :: Libraries",
+ "Programming Language :: Python :: 2",
+ "Programming Language :: Python :: 2.7",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.3",
+ "Programming Language :: Python :: 3.4",
+ "Programming Language :: Python :: 3.5",
+ "Programming Language :: Python :: 3.6",
+ "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: Implementation :: PyPy",
+ "License :: OSI Approved :: MIT License",
+ ],
+ keywords="concurrent future deferred promise",
+ packages=find_packages(exclude=["tests"]),
+ # PEP-561: https://www.python.org/dev/peps/pep-0561/
+ package_data={"promise": ["py.typed"]},
+ extras_require={"test": tests_require},
+ install_requires=[
+ "typing>=3.6.4; python_version < '3.5'",
+ "six"
+ ],
+ tests_require=tests_require,
+)
diff --git a/wandb/vendor/promise-2.3.0/tests/__init__.py b/wandb/vendor/promise-2.3.0/tests/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/vendor/promise-2.3.0/tests/conftest.py b/wandb/vendor/promise-2.3.0/tests/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..26122664faf386b96dbcffc63dcb2ea41b57c049
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/tests/conftest.py
@@ -0,0 +1,8 @@
+from sys import version_info
+
+collect_ignore = []
+if version_info[:2] < (3, 4):
+ collect_ignore.append("test_awaitable.py")
+if version_info[:2] < (3, 5):
+ collect_ignore.append("test_awaitable_35.py")
+ collect_ignore.append("test_dataloader_awaitable_35.py")
diff --git a/wandb/vendor/promise-2.3.0/tests/test_awaitable.py b/wandb/vendor/promise-2.3.0/tests/test_awaitable.py
new file mode 100644
index 0000000000000000000000000000000000000000..aad7f2ce41eb1420aa19fba3f0af656dc0668586
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/tests/test_awaitable.py
@@ -0,0 +1,32 @@
+from asyncio import coroutine
+from pytest import mark
+from time import sleep
+from promise import Promise
+
+
+@mark.asyncio
+@coroutine
+def test_await():
+ yield from Promise.resolve(True)
+
+
+@mark.asyncio
+@coroutine
+def test_await_time():
+ def resolve_or_reject(resolve, reject):
+ sleep(.1)
+ resolve(True)
+
+ p = Promise(resolve_or_reject)
+ assert p.get() is True
+
+
+@mark.asyncio
+@coroutine
+def test_promise_coroutine():
+ @coroutine
+ def my_coro():
+ yield from Promise.resolve(True)
+
+ promise = Promise.resolve(my_coro())
+ assert isinstance(promise, Promise)
diff --git a/wandb/vendor/promise-2.3.0/tests/test_awaitable_35.py b/wandb/vendor/promise-2.3.0/tests/test_awaitable_35.py
new file mode 100644
index 0000000000000000000000000000000000000000..4aa70503a74554f63fc1bb86928ac3e66e7857f5
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/tests/test_awaitable_35.py
@@ -0,0 +1,47 @@
+from asyncio import sleep, Future, wait, FIRST_COMPLETED
+from pytest import mark
+from promise import Promise, is_thenable
+
+
+@mark.asyncio
+async def test_await():
+ assert await Promise.resolve(True)
+
+
+@mark.asyncio
+async def test_promisify_coroutine():
+ async def my_coroutine():
+ await sleep(.01)
+ return True
+
+ assert await Promise.resolve(my_coroutine())
+
+
+@mark.asyncio
+async def test_coroutine_is_thenable():
+ async def my_coroutine():
+ await sleep(.01)
+ return True
+
+ assert is_thenable(my_coroutine())
+
+
+@mark.asyncio
+async def test_promisify_future():
+ future = Future()
+ future.set_result(True)
+ assert await Promise.resolve(future)
+
+
+@mark.asyncio
+async def test_await_in_safe_promise():
+ async def inner():
+ @Promise.safe
+ def x():
+ promise = Promise.resolve(True).then(lambda x: x)
+ return promise
+
+ return await x()
+
+ result = await inner()
+ assert result == True
diff --git a/wandb/vendor/promise-2.3.0/tests/test_benchmark.py b/wandb/vendor/promise-2.3.0/tests/test_benchmark.py
new file mode 100644
index 0000000000000000000000000000000000000000..eb30f24e2f587ab7d806013a3fd7af077abc1c2b
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/tests/test_benchmark.py
@@ -0,0 +1,116 @@
+from pytest import raises
+import time
+from promise import Promise, promisify, is_thenable
+
+
+def test_benchmark_promise_creation(benchmark):
+ @benchmark
+ def create_promise(): # unnecessary function call
+ p = Promise()
+
+
+def test_benchmark_promise_resolve(benchmark):
+ def create_promise():
+ return Promise.resolve(True)
+
+ result = benchmark(create_promise).get()
+ assert result == True
+
+
+def test_benchmark_is_thenable_basic_type(benchmark):
+ def create_promise():
+ return is_thenable(True)
+
+ result = benchmark(create_promise)
+ assert result == False
+
+
+def test_benchmark_is_thenable_custom_type(benchmark):
+ class MyType(object):
+ pass
+
+ my_type_instance = MyType()
+
+ def create_promise():
+ return is_thenable(my_type_instance)
+
+ result = benchmark(create_promise)
+ assert result == False
+
+
+def test_benchmark_promise_creation_with_resolve(benchmark):
+ do_resolve = lambda resolve, reject: resolve(True)
+
+ def create_promise(): # unnecessary function call
+ p = Promise(do_resolve)
+ # p._wait()
+ return p
+
+ result = benchmark(create_promise).get()
+ assert result == True
+
+
+def test_benchmark_promise_creation_with_reject(benchmark):
+ do_resolve = lambda resolve, reject: reject(Exception("Error"))
+
+ def create_promise(): # unnecessary function call
+ p = Promise(do_resolve)
+ # p._wait()
+ return p
+
+ with raises(Exception) as exc_info:
+ result = benchmark(create_promise).get()
+
+ assert str(exc_info.value) == "Error"
+
+
+# def test_benchmark_promisify_promise(benchmark):
+# instance = Promise()
+
+# def create_promise(): # unnecessary function call
+# return promisify(instance)
+
+# result = benchmark(create_promise)
+
+# assert isinstance(result, Promise)
+
+
+def test_benchmark_promisify_custom_type(benchmark):
+ class CustomThenable(object):
+ pass
+ # def then(self, resolve, reject):
+ # return resolve(True)
+
+ instance = CustomThenable()
+
+ def create_promise(): # unnecessary function call
+ return Promise.resolve(instance)
+
+ result = benchmark(create_promise)
+
+ assert isinstance(result, Promise)
+ assert result.get() == instance
+
+
+def test_benchmark_promise_all(benchmark):
+ values = range(1000)
+
+ def create_promise(): # unnecessary function call
+ return Promise.all(values)
+
+ result = benchmark(create_promise)
+
+ assert isinstance(result, Promise)
+ assert result.get() == list(range(1000))
+
+
+def test_benchmark_promise_all_promise(benchmark):
+ values = [Promise.resolve(i) for i in range(100000)]
+
+ def create_promise(): # unnecessary function call
+ return Promise.all(values)
+
+ result = benchmark(create_promise)
+
+ assert isinstance(result, Promise)
+ assert result.get() == list(range(100000))
diff --git a/wandb/vendor/promise-2.3.0/tests/test_complex_threads.py b/wandb/vendor/promise-2.3.0/tests/test_complex_threads.py
new file mode 100644
index 0000000000000000000000000000000000000000..6cddfaac1e9900c9351c50eeaacbda6f93cef477
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/tests/test_complex_threads.py
@@ -0,0 +1,23 @@
+from time import sleep
+from concurrent.futures import ThreadPoolExecutor
+from promise import Promise
+from operator import mul
+
+executor = ThreadPoolExecutor(max_workers=40000)
+
+
+def promise_factorial(n):
+ if n < 2:
+ return 1
+ sleep(.02)
+ a = executor.submit(promise_factorial, n - 1)
+
+ def promise_then(r):
+ return mul(r, n)
+
+ return Promise.resolve(a).then(promise_then)
+
+
+def test_factorial():
+ p = promise_factorial(10)
+ assert p.get() == 3628800
diff --git a/wandb/vendor/promise-2.3.0/tests/test_dataloader.py b/wandb/vendor/promise-2.3.0/tests/test_dataloader.py
new file mode 100644
index 0000000000000000000000000000000000000000..a8352fc6e3d5f7b6c71d949d87a59bb9ecdd79cd
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/tests/test_dataloader.py
@@ -0,0 +1,452 @@
+from pytest import raises
+
+from promise import Promise, async_instance
+from promise.dataloader import DataLoader
+
+
+def id_loader(**options):
+ load_calls = []
+
+ resolve = options.pop("resolve", Promise.resolve)
+
+ def fn(keys):
+ load_calls.append(keys)
+ return resolve(keys)
+
+ identity_loader = DataLoader(fn, **options)
+ return identity_loader, load_calls
+
+
+def test_build_a_simple_data_loader():
+ def call_fn(keys):
+ return Promise.resolve(keys)
+
+ identity_loader = DataLoader(call_fn)
+
+ promise1 = identity_loader.load(1)
+ assert isinstance(promise1, Promise)
+
+ value1 = promise1.get()
+ assert value1 == 1
+
+
+def test_supports_loading_multiple_keys_in_one_call():
+ def call_fn(keys):
+ return Promise.resolve(keys)
+
+ identity_loader = DataLoader(call_fn)
+
+ promise_all = identity_loader.load_many([1, 2])
+ assert isinstance(promise_all, Promise)
+
+ values = promise_all.get()
+ assert values == [1, 2]
+
+ promise_all = identity_loader.load_many([])
+ assert isinstance(promise_all, Promise)
+
+ values = promise_all.get()
+ assert values == []
+
+
+def test_batches_multiple_requests():
+ @Promise.safe
+ def do():
+ identity_loader, load_calls = id_loader()
+
+ promise1 = identity_loader.load(1)
+ promise2 = identity_loader.load(2)
+
+ p = Promise.all([promise1, promise2])
+
+ value1, value2 = p.get()
+
+ assert value1 == 1
+ assert value2 == 2
+
+ assert load_calls == [[1, 2]]
+
+ do().get()
+
+
+def test_batches_multiple_requests_with_max_batch_sizes():
+ @Promise.safe
+ def do():
+ identity_loader, load_calls = id_loader(max_batch_size=2)
+
+ promise1 = identity_loader.load(1)
+ promise2 = identity_loader.load(2)
+ promise3 = identity_loader.load(3)
+
+ p = Promise.all([promise1, promise2, promise3])
+
+ value1, value2, value3 = p.get()
+
+ assert value1 == 1
+ assert value2 == 2
+ assert value3 == 3
+
+ assert load_calls == [[1, 2], [3]]
+
+ do().get()
+
+
+def test_coalesces_identical_requests():
+ @Promise.safe
+ def do():
+ identity_loader, load_calls = id_loader()
+
+ promise1 = identity_loader.load(1)
+ promise2 = identity_loader.load(1)
+
+ assert promise1 == promise2
+ p = Promise.all([promise1, promise2])
+
+ value1, value2 = p.get()
+
+ assert value1 == 1
+ assert value2 == 1
+
+ assert load_calls == [[1]]
+
+ do().get()
+
+
+def test_caches_repeated_requests():
+ @Promise.safe
+ def do():
+ identity_loader, load_calls = id_loader()
+
+ a, b = Promise.all([identity_loader.load("A"), identity_loader.load("B")]).get()
+
+ assert a == "A"
+ assert b == "B"
+
+ assert load_calls == [["A", "B"]]
+
+ a2, c = Promise.all(
+ [identity_loader.load("A"), identity_loader.load("C")]
+ ).get()
+
+ assert a2 == "A"
+ assert c == "C"
+
+ assert load_calls == [["A", "B"], ["C"]]
+
+ a3, b2, c2 = Promise.all(
+ [
+ identity_loader.load("A"),
+ identity_loader.load("B"),
+ identity_loader.load("C"),
+ ]
+ ).get()
+
+ assert a3 == "A"
+ assert b2 == "B"
+ assert c2 == "C"
+
+ assert load_calls == [["A", "B"], ["C"]]
+
+ do().get()
+
+
+def test_clears_single_value_in_loader():
+ @Promise.safe
+ def do():
+ identity_loader, load_calls = id_loader()
+
+ a, b = Promise.all([identity_loader.load("A"), identity_loader.load("B")]).get()
+
+ assert a == "A"
+ assert b == "B"
+
+ assert load_calls == [["A", "B"]]
+
+ identity_loader.clear("A")
+
+ a2, b2 = Promise.all(
+ [identity_loader.load("A"), identity_loader.load("B")]
+ ).get()
+
+ assert a2 == "A"
+ assert b2 == "B"
+
+ assert load_calls == [["A", "B"], ["A"]]
+
+ do().get()
+
+
+def test_clears_all_values_in_loader():
+ @Promise.safe
+ def do():
+ identity_loader, load_calls = id_loader()
+
+ a, b = Promise.all([identity_loader.load("A"), identity_loader.load("B")]).get()
+
+ assert a == "A"
+ assert b == "B"
+
+ assert load_calls == [["A", "B"]]
+
+ identity_loader.clear_all()
+
+ a2, b2 = Promise.all(
+ [identity_loader.load("A"), identity_loader.load("B")]
+ ).get()
+
+ assert a2 == "A"
+ assert b2 == "B"
+
+ assert load_calls == [["A", "B"], ["A", "B"]]
+
+ do().get()
+
+
+def test_does_not_replace_cache_map():
+ @Promise.safe
+ def do():
+ identity_loader, _ = id_loader()
+ a, b = Promise.all([identity_loader.load("A"), identity_loader.load("B")]).get()
+
+ assert a == "A"
+ assert b == "B"
+
+ cache_map = identity_loader._promise_cache
+
+ identity_loader.clear_all()
+
+ assert id(identity_loader._promise_cache) == id(cache_map)
+
+ do().get()
+
+
+def test_allows_priming_the_cache():
+ @Promise.safe
+ def do():
+ identity_loader, load_calls = id_loader()
+
+ identity_loader.prime("A", "A")
+
+ a, b = Promise.all([identity_loader.load("A"), identity_loader.load("B")]).get()
+
+ assert a == "A"
+ assert b == "B"
+
+ assert load_calls == [["B"]]
+
+ do().get()
+
+
+def test_does_not_prime_keys_that_already_exist():
+ @Promise.safe
+ def do():
+ identity_loader, load_calls = id_loader()
+
+ identity_loader.prime("A", "X")
+
+ a1 = identity_loader.load("A").get()
+ b1 = identity_loader.load("B").get()
+
+ assert a1 == "X"
+ assert b1 == "B"
+
+ identity_loader.prime("A", "Y")
+ identity_loader.prime("B", "Y")
+
+ a2 = identity_loader.load("A").get()
+ b2 = identity_loader.load("B").get()
+
+ assert a2 == "X"
+ assert b2 == "B"
+
+ assert load_calls == [["B"]]
+
+ do().get()
+
+
+# Represents Errors
+
+
+def test_resolves_to_error_to_indicate_failure():
+ @Promise.safe
+ def do():
+ def resolve(keys):
+ mapped_keys = [
+ key if key % 2 == 0 else Exception("Odd: {}".format(key))
+ for key in keys
+ ]
+ return Promise.resolve(mapped_keys)
+
+ even_loader, load_calls = id_loader(resolve=resolve)
+
+ with raises(Exception) as exc_info:
+ even_loader.load(1).get()
+
+ assert str(exc_info.value) == "Odd: 1"
+
+ value2 = even_loader.load(2).get()
+ assert value2 == 2
+ assert load_calls == [[1], [2]]
+
+ do().get()
+
+
+def test_can_represent_failures_and_successes_simultaneously():
+ @Promise.safe
+ def do():
+ def resolve(keys):
+ mapped_keys = [
+ key if key % 2 == 0 else Exception("Odd: {}".format(key))
+ for key in keys
+ ]
+ return Promise.resolve(mapped_keys)
+
+ even_loader, load_calls = id_loader(resolve=resolve)
+
+ promise1 = even_loader.load(1)
+ promise2 = even_loader.load(2)
+
+ with raises(Exception) as exc_info:
+ promise1.get()
+
+ assert str(exc_info.value) == "Odd: 1"
+ value2 = promise2.get()
+ assert value2 == 2
+ assert load_calls == [[1, 2]]
+
+ do().get()
+
+
+def test_caches_failed_fetches():
+ @Promise.safe
+ def do():
+ def resolve(keys):
+ mapped_keys = [Exception("Error: {}".format(key)) for key in keys]
+ return Promise.resolve(mapped_keys)
+
+ error_loader, load_calls = id_loader(resolve=resolve)
+
+ with raises(Exception) as exc_info:
+ error_loader.load(1).get()
+
+ assert str(exc_info.value) == "Error: 1"
+
+ with raises(Exception) as exc_info:
+ error_loader.load(1).get()
+
+ assert str(exc_info.value) == "Error: 1"
+
+ assert load_calls == [[1]]
+
+ do().get()
+
+
+def test_caches_failed_fetches():
+ @Promise.safe
+ def do():
+ identity_loader, load_calls = id_loader()
+
+ identity_loader.prime(1, Exception("Error: 1"))
+
+ with raises(Exception) as exc_info:
+ identity_loader.load(1).get()
+
+ assert load_calls == []
+
+ do().get()
+
+
+# It is resilient to job queue ordering
+# def test_batches_loads_occuring_within_promises():
+# @Promise.safe
+# def do():
+# identity_loader, load_calls = id_loader()
+# values = Promise.all([
+# identity_loader.load('A'),
+# Promise.resolve(None).then(lambda v: Promise.resolve(None)).then(
+# lambda v: identity_loader.load('B')
+# )
+# ]).get()
+
+# assert values == ['A', 'B']
+# assert load_calls == [['A', 'B']]
+
+# do().get()
+
+
+def test_catches_error_if_loader_resolver_fails():
+ @Promise.safe
+ def do():
+ def do_resolve(x):
+ raise Exception("AOH!")
+
+ a_loader, a_load_calls = id_loader(resolve=do_resolve)
+
+ with raises(Exception) as exc_info:
+ a_loader.load("A1").get()
+
+ assert str(exc_info.value) == "AOH!"
+
+ do().get()
+
+
+def test_can_call_a_loader_from_a_loader():
+ @Promise.safe
+ def do():
+ deep_loader, deep_load_calls = id_loader()
+ a_loader, a_load_calls = id_loader(
+ resolve=lambda keys: deep_loader.load(tuple(keys))
+ )
+ b_loader, b_load_calls = id_loader(
+ resolve=lambda keys: deep_loader.load(tuple(keys))
+ )
+
+ a1, b1, a2, b2 = Promise.all(
+ [
+ a_loader.load("A1"),
+ b_loader.load("B1"),
+ a_loader.load("A2"),
+ b_loader.load("B2"),
+ ]
+ ).get()
+
+ assert a1 == "A1"
+ assert b1 == "B1"
+ assert a2 == "A2"
+ assert b2 == "B2"
+
+ assert a_load_calls == [["A1", "A2"]]
+ assert b_load_calls == [["B1", "B2"]]
+ assert deep_load_calls == [[("A1", "A2"), ("B1", "B2")]]
+
+ do().get()
+
+
+def test_dataloader_clear_with_missing_key_works():
+ @Promise.safe
+ def do():
+ def do_resolve(x):
+ return x
+
+ a_loader, a_load_calls = id_loader(resolve=do_resolve)
+ assert a_loader.clear("A1") == a_loader
+
+ do().get()
+
+
+def test_wrong_loader_return_type_does_not_block_async_instance():
+ @Promise.safe
+ def do():
+ def do_resolve(x):
+ return x
+
+ a_loader, a_load_calls = id_loader(resolve=do_resolve)
+
+ with raises(Exception):
+ a_loader.load("A1").get()
+ assert async_instance.have_drained_queues
+ with raises(Exception):
+ a_loader.load("A2").get()
+ assert async_instance.have_drained_queues
+
+ do().get()
diff --git a/wandb/vendor/promise-2.3.0/tests/test_dataloader_awaitable_35.py b/wandb/vendor/promise-2.3.0/tests/test_dataloader_awaitable_35.py
new file mode 100644
index 0000000000000000000000000000000000000000..88161dd595ea19b608ca27a76b03621457469ad2
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/tests/test_dataloader_awaitable_35.py
@@ -0,0 +1,99 @@
+from pytest import mark
+from promise import Promise
+from promise.dataloader import DataLoader
+
+
+def id_loader(**options):
+ load_calls = []
+
+ resolve = options.pop("resolve", Promise.resolve)
+
+ def fn(keys):
+ load_calls.append(keys)
+ return resolve(keys)
+
+ identity_loader = DataLoader(fn, **options)
+ return identity_loader, load_calls
+
+
+@mark.asyncio
+async def test_await_dataloader():
+ identity_loader, load_calls = id_loader()
+
+ async def load_multiple(identity_loader):
+ one = identity_loader.load("load1")
+ two = identity_loader.load("load2")
+ return await Promise.all([one, two])
+
+ result = await load_multiple(identity_loader)
+ assert result == ["load1", "load2"]
+ assert load_calls == [["load1"], ["load2"]]
+
+
+@mark.asyncio
+async def test_await_dataloader_safe_promise():
+ identity_loader, load_calls = id_loader()
+
+ @Promise.safe
+ async def load_multiple(identity_loader):
+ one = identity_loader.load("load1")
+ two = identity_loader.load("load2")
+ return await Promise.all([one, two])
+
+ result = await load_multiple(identity_loader)
+ assert result == ["load1", "load2"]
+ assert load_calls == [["load1"], ["load2"]]
+
+
+@mark.asyncio
+async def test_await_dataloader_individual():
+ identity_loader, load_calls = id_loader()
+
+ async def load_one_then_two(identity_loader):
+ one = await identity_loader.load("load1")
+ two = await identity_loader.load("load2")
+ return [one, two]
+
+ result = await load_one_then_two(identity_loader)
+ assert result == ["load1", "load2"]
+ assert load_calls == [["load1"], ["load2"]]
+
+
+@mark.asyncio
+async def test_await_dataloader_individual_safe_promise():
+ identity_loader, load_calls = id_loader()
+
+ @Promise.safe
+ async def load_one_then_two(identity_loader):
+ one = await identity_loader.load("load1")
+ two = await identity_loader.load("load2")
+ return [one, two]
+
+ result = await load_one_then_two(identity_loader)
+ assert result == ["load1", "load2"]
+ assert load_calls == [["load1"], ["load2"]]
+
+
+@mark.asyncio
+async def test_await_dataloader_two():
+ identity_loader, load_calls = id_loader()
+
+ async def load_one_then_two(identity_loader):
+ one = await identity_loader.load("load1")
+ two = await identity_loader.load("load2")
+ return (one, two)
+
+ result12 = await Promise.all([load_one_then_two(identity_loader)])
+
+
+@mark.asyncio
+async def test_await_dataloader_two_safe_promise():
+ identity_loader, load_calls = id_loader()
+
+ @Promise.safe
+ async def load_one_then_two(identity_loader):
+ one = await identity_loader.load("load1")
+ two = await identity_loader.load("load2")
+ return (one, two)
+
+ result12 = await Promise.all([load_one_then_two(identity_loader)])
diff --git a/wandb/vendor/promise-2.3.0/tests/test_dataloader_extra.py b/wandb/vendor/promise-2.3.0/tests/test_dataloader_extra.py
new file mode 100644
index 0000000000000000000000000000000000000000..f24fc18a954e020afa6f8bc0dcc7e9219c76af02
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/tests/test_dataloader_extra.py
@@ -0,0 +1,65 @@
+# from promise import Promise
+# from promise.dataloader import DataLoader
+
+
+# def id_loader(**options):
+# load_calls = []
+
+# def fn(keys):
+# load_calls.append(keys)
+# return Promise.resolve(keys)
+
+# identity_loader = DataLoader(fn, **options)
+# return identity_loader, load_calls
+
+
+# def test_batches_multiple_requests():
+# identity_loader, load_calls = id_loader()
+
+# @Promise.safe
+# def safe():
+# promise1 = identity_loader.load(1)
+# promise2 = identity_loader.load(2)
+# return promise1, promise2
+
+# promise1, promise2 = safe()
+# value1, value2 = Promise.all([promise1, promise2]).get()
+# assert value1 == 1
+# assert value2 == 2
+
+# assert load_calls == [[1, 2]]
+
+
+# def test_batches_multiple_requests_two():
+# identity_loader, load_calls = id_loader()
+
+# @Promise.safe
+# def safe():
+# promise1 = identity_loader.load(1)
+# promise2 = identity_loader.load(2)
+# return Promise.all([promise1, promise2])
+
+# p = safe()
+# value1, value2 = p.get()
+
+# assert value1 == 1
+# assert value2 == 2
+
+# assert load_calls == [[1, 2]]
+
+
+# @Promise.safe
+# def test_batches_multiple_requests_safe():
+# identity_loader, load_calls = id_loader()
+
+# promise1 = identity_loader.load(1)
+# promise2 = identity_loader.load(2)
+
+# p = Promise.all([promise1, promise2])
+
+# value1, value2 = p.get()
+
+# assert value1 == 1
+# assert value2 == 2
+
+# assert load_calls == [[1, 2]]
diff --git a/wandb/vendor/promise-2.3.0/tests/test_extra.py b/wandb/vendor/promise-2.3.0/tests/test_extra.py
new file mode 100644
index 0000000000000000000000000000000000000000..4a083718d134ede122d5a761c592318a9fa5748e
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/tests/test_extra.py
@@ -0,0 +1,670 @@
+# This exercises some capabilities above and beyond
+# the Promises/A+ test suite
+from time import sleep
+from pytest import raises, fixture
+
+from threading import Event
+from promise import (
+ Promise,
+ is_thenable,
+ promisify,
+ promise_for_dict as free_promise_for_dict,
+)
+from concurrent.futures import Future
+from threading import Thread
+
+from .utils import assert_exception
+
+
+class DelayedFulfill(Thread):
+ def __init__(self, d, p, v):
+ self.delay = d
+ self.promise = p
+ self.value = v
+ Thread.__init__(self)
+
+ def run(self):
+ sleep(self.delay)
+ self.promise.do_resolve(self.value)
+
+
+class DelayedRejection(Thread):
+ def __init__(self, d, p, r):
+ self.delay = d
+ self.promise = p
+ self.reason = r
+ Thread.__init__(self)
+
+ def run(self):
+ sleep(self.delay)
+ self.promise.do_reject(self.reason)
+
+
+class FakeThenPromise:
+ def __init__(self, raises=True):
+ self.raises = raises
+
+ def then(self, s=None, f=None):
+ if self.raises:
+ raise Exception("FakeThenPromise raises in 'then'")
+
+
+def df(value, dtime):
+ p = Promise()
+ t = DelayedFulfill(dtime, p, value)
+ t.start()
+
+ return p
+
+
+def dr(reason, dtime):
+ p = Promise()
+ t = DelayedRejection(dtime, p, reason)
+ t.start()
+
+ return p
+
+
+# Static methods
+def test_fulfilled():
+ p = Promise.fulfilled(4)
+ assert p.is_fulfilled
+ assert p.get() == 4
+
+
+def test_rejected():
+ p = Promise.rejected(Exception("Static rejected"))
+ assert p.is_rejected
+ with raises(Exception) as exc_info:
+ p.get()
+ assert str(exc_info.value) == "Static rejected"
+
+
+# Fulfill
+def test_fulfill_self():
+ p = Promise()
+ with raises(TypeError) as excinfo:
+ p.do_resolve(p)
+ p.get()
+
+
+# Exceptions
+def test_exceptions():
+ def throws(v):
+ assert False
+
+ p1 = Promise()
+ p1.then(throws)
+ p1.do_resolve(5)
+
+ p2 = Promise()
+ p2.catch(throws)
+ p2.do_reject(Exception())
+
+ with raises(Exception) as excinfo:
+ p2.get()
+
+
+def test_thrown_exceptions_have_stacktrace():
+ def throws(v):
+ assert False
+
+ p3 = Promise.resolve("a").then(throws)
+ with raises(AssertionError) as assert_exc:
+ p3.get()
+
+ assert assert_exc.traceback[-1].path.strpath == __file__
+
+
+def test_thrown_exceptions_preserve_stacktrace():
+ def throws(v):
+ assert False
+
+ def after_throws(v):
+ pass
+
+ p3 = Promise.resolve("a").then(throws).then(after_throws)
+ with raises(AssertionError) as assert_exc:
+ p3.get()
+
+ assert assert_exc.traceback[-1].path.strpath == __file__
+
+
+# WAIT
+# def test_wait_when():
+# p1 = df(5, 0.01)
+# assert p1.is_pending
+# p1._wait()
+# assert p1.is_fulfilled
+
+
+def test_wait_if():
+ p1 = Promise()
+ p1.do_resolve(5)
+ p1._wait()
+ assert p1.is_fulfilled
+
+
+# def test_wait_timeout():
+# p1 = df(5, 0.1)
+# assert p1.is_pending
+# with raises(Exception) as exc_info:
+# p1._wait(timeout=0.05)
+# assert str(exc_info.value) == "Timeout"
+# assert p1.is_pending
+# p1._wait()
+# assert p1.is_fulfilled
+
+
+# # GET
+# def test_get_when():
+# p1 = df(5, 0.01)
+# assert p1.is_pending
+# v = p1.get()
+# assert p1.is_fulfilled
+# assert 5 == v
+
+
+def test_get_if():
+ p1 = Promise()
+ p1.do_resolve(5)
+ v = p1.get()
+ assert p1.is_fulfilled
+ assert 5 == v
+
+
+# def test_get_timeout():
+# p1 = df(5, 0.1)
+# assert p1.is_pending
+# with raises(Exception) as exc_info:
+# p1._wait(timeout=0.05)
+# assert str(exc_info.value) == "Timeout"
+# assert p1.is_pending
+# v = p1.get()
+# assert p1.is_fulfilled
+# assert 5 == v
+
+
+# Promise.all
+def test_promise_all_when():
+ p1 = Promise()
+ p2 = Promise()
+ pl = Promise.all([p1, p2])
+ assert p1.is_pending
+ assert p2.is_pending
+ assert pl.is_pending
+ p1.do_resolve(5)
+ p1._wait()
+ assert p1.is_fulfilled
+ assert p2.is_pending
+ assert pl.is_pending
+ p2.do_resolve(10)
+ p2._wait()
+ pl._wait()
+ assert p1.is_fulfilled
+ assert p2.is_fulfilled
+ assert pl.is_fulfilled
+ assert 5 == p1.get()
+ assert 10 == p2.get()
+ assert 5 == pl.get()[0]
+ assert 10 == pl.get()[1]
+
+
+def test_promise_all_when_mixed_promises():
+ p1 = Promise()
+ p2 = Promise()
+ pl = Promise.all([p1, 32, p2, False, True])
+ assert p1.is_pending
+ assert p2.is_pending
+ assert pl.is_pending
+ p1.do_resolve(5)
+ p1._wait()
+ assert p1.is_fulfilled
+ assert p2.is_pending
+ assert pl.is_pending
+ p2.do_resolve(10)
+ p2._wait()
+ pl._wait()
+ assert p1.is_fulfilled
+ assert p2.is_fulfilled
+ assert pl.is_fulfilled
+ assert 5 == p1.get()
+ assert 10 == p2.get()
+ assert pl.get() == [5, 32, 10, False, True]
+
+
+def test_promise_all_when_if_no_promises():
+ pl = Promise.all([10, 32, False, True])
+ assert pl.get() == [10, 32, False, True]
+
+
+def test_promise_all_if():
+ p1 = Promise()
+ p2 = Promise()
+ pd1 = Promise.all([p1, p2])
+ pd2 = Promise.all([p1])
+ pd3 = Promise.all([])
+ pd3._wait()
+ assert p1.is_pending
+ assert p2.is_pending
+ assert pd1.is_pending
+ assert pd2.is_pending
+ assert pd3.is_fulfilled
+ p1.do_resolve(5)
+ p1._wait()
+ pd2._wait()
+ assert p1.is_fulfilled
+ assert p2.is_pending
+ assert pd1.is_pending
+ assert pd2.is_fulfilled
+ p2.do_resolve(10)
+ p2._wait()
+ pd1._wait()
+ pd2._wait()
+ assert p1.is_fulfilled
+ assert p2.is_fulfilled
+ assert pd1.is_fulfilled
+ assert pd2.is_fulfilled
+ assert 5 == p1.get()
+ assert 10 == p2.get()
+ assert 5 == pd1.get()[0]
+ assert 5 == pd2.get()[0]
+ assert 10 == pd1.get()[1]
+ assert [] == pd3.get()
+
+
+# promise_for_dict
+@fixture(params=[Promise.for_dict, free_promise_for_dict])
+def promise_for_dict(request):
+ return request.param
+
+
+def test_dict_promise_when(promise_for_dict):
+ p1 = Promise()
+ p2 = Promise()
+ d = {"a": p1, "b": p2}
+ pd1 = promise_for_dict(d)
+ pd2 = promise_for_dict({"a": p1})
+ pd3 = promise_for_dict({})
+ assert p1.is_pending
+ assert p2.is_pending
+ assert pd1.is_pending
+ assert pd2.is_pending
+ pd3._wait()
+ assert pd3.is_fulfilled
+ p1.do_resolve(5)
+ p1._wait()
+ pd2._wait()
+ assert p1.is_fulfilled
+ assert p2.is_pending
+ assert pd1.is_pending
+ assert pd2.is_fulfilled
+ p2.do_resolve(10)
+ p2._wait()
+ pd1._wait()
+ assert p1.is_fulfilled
+ assert p2.is_fulfilled
+ assert pd1.is_fulfilled
+ assert pd2.is_fulfilled
+ assert 5 == p1.get()
+ assert 10 == p2.get()
+ assert 5 == pd1.get()["a"]
+ assert 5 == pd2.get()["a"]
+ assert 10 == pd1.get()["b"]
+ assert {} == pd3.get()
+
+
+def test_dict_promise_if(promise_for_dict):
+ p1 = Promise()
+ p2 = Promise()
+ d = {"a": p1, "b": p2}
+ pd = promise_for_dict(d)
+ assert p1.is_pending
+ assert p2.is_pending
+ assert pd.is_pending
+ p1.do_resolve(5)
+ p1._wait()
+ assert p1.is_fulfilled
+ assert p2.is_pending
+ assert pd.is_pending
+ p2.do_resolve(10)
+ p2._wait()
+ assert p1.is_fulfilled
+ assert p2.is_fulfilled
+ # pd._wait()
+ # assert pd.is_fulfilled
+ # assert 5 == p1.get()
+ # assert 10 == p2.get()
+ # assert 5 == pd.get()["a"]
+ # assert 10 == pd.get()["b"]
+
+
+def test_done():
+ counter = [0]
+ r = Promise()
+
+ def inc(_):
+ counter[0] += 1
+
+ def dec(_):
+ counter[0] -= 1
+
+ def end(_):
+ r.do_resolve(None)
+
+ p = Promise()
+ p.done(inc, dec)
+ p.done(inc, dec)
+ p.done(end)
+ p.do_resolve(4)
+
+ Promise.wait(r)
+ assert counter[0] == 2
+
+ r = Promise()
+
+ counter = [0]
+ p = Promise()
+ p.done(inc, dec)
+ p.done(inc, dec)
+ p.done(None, end)
+ p.do_reject(Exception())
+
+ Promise.wait(r)
+ assert counter[0] == -2
+
+
+def test_done_all():
+ counter = [0]
+
+ def inc(_):
+ counter[0] += 1
+
+ def dec(_):
+ counter[0] -= 1
+
+ p = Promise()
+ r = Promise()
+ p.done_all()
+ p.done_all([(inc, dec)])
+ p.done_all(
+ [
+ (inc, dec),
+ (inc, dec),
+ {"success": inc, "failure": dec},
+ lambda _: r.do_resolve(None),
+ ]
+ )
+ p.do_resolve(4)
+ Promise.wait(r)
+ assert counter[0] == 4
+
+ p = Promise()
+ r = Promise()
+ p.done_all()
+ p.done_all([inc])
+ p.done_all([(inc, dec)])
+ p.done_all(
+ [
+ (inc, dec),
+ {"success": inc, "failure": dec},
+ (None, lambda _: r.do_resolve(None)),
+ ]
+ )
+ p.do_reject(Exception("Uh oh!"))
+ Promise.wait(r)
+ assert counter[0] == 1
+
+
+def test_then_all():
+ p = Promise()
+
+ handlers = [
+ ((lambda x: x * x), (lambda r: 1)),
+ {"success": (lambda x: x + x), "failure": (lambda r: 2)},
+ ]
+
+ results = (
+ p.then_all()
+ + p.then_all([lambda x: x])
+ + p.then_all([(lambda x: x * x, lambda r: 1)])
+ + p.then_all(handlers)
+ )
+
+ p.do_resolve(4)
+
+ assert [r.get() for r in results] == [4, 16, 16, 8]
+
+ p = Promise()
+
+ handlers = [
+ ((lambda x: x * x), (lambda r: 1)),
+ {"success": (lambda x: x + x), "failure": (lambda r: 2)},
+ ]
+
+ results = (
+ p.then_all()
+ + p.then_all([(lambda x: x * x, lambda r: 1)])
+ + p.then_all(handlers)
+ )
+
+ p.do_reject(Exception())
+
+ assert [r.get() for r in results] == [1, 1, 2]
+
+
+def test_do_resolve():
+ p1 = Promise(lambda resolve, reject: resolve(0))
+ assert p1.get() == 0
+ assert p1.is_fulfilled
+
+
+def test_do_resolve_fail_on_call():
+ def raises(resolve, reject):
+ raise Exception("Fails")
+
+ p1 = Promise(raises)
+ assert not p1.is_fulfilled
+ assert str(p1.reason) == "Fails"
+
+
+def test_catch():
+ p1 = Promise(lambda resolve, reject: resolve(0))
+ p2 = p1.then(lambda value: 1 / value).catch(lambda e: e).then(lambda e: type(e))
+ assert p2.get() == ZeroDivisionError
+ assert p2.is_fulfilled
+
+
+def test_is_thenable_promise():
+ promise = Promise()
+ assert is_thenable(promise)
+
+
+def test_is_thenable_then_object():
+ promise = FakeThenPromise()
+ assert not is_thenable(promise)
+
+
+def test_is_thenable_future():
+ promise = Future()
+ assert is_thenable(promise)
+
+
+def test_is_thenable_simple_object():
+ assert not is_thenable(object())
+
+
+@fixture(params=[Promise.resolve])
+def resolve(request):
+ return request.param
+
+
+def test_resolve_promise(resolve):
+ promise = Promise()
+ assert resolve(promise) == promise
+
+
+def test_resolve_then_object(resolve):
+ promise = FakeThenPromise(raises=False)
+ p = resolve(promise)
+ assert isinstance(p, Promise)
+
+
+def test_resolve_future(resolve):
+ future = Future()
+ promise = resolve(future)
+ assert promise.is_pending
+ future.set_result(1)
+ assert promise.get() == 1
+ assert promise.is_fulfilled
+
+
+def test_resolve_future_rejected(resolve):
+ future = Future()
+ promise = resolve(future)
+ assert promise.is_pending
+ future.set_exception(Exception("Future rejected"))
+ assert promise.is_rejected
+ assert_exception(promise.reason, Exception, "Future rejected")
+
+
+def test_resolve_object(resolve):
+ val = object()
+ promised = resolve(val)
+ assert isinstance(promised, Promise)
+ assert promised.get() == val
+
+
+def test_resolve_promise_subclass():
+ class MyPromise(Promise):
+ pass
+
+ p = Promise()
+ p.do_resolve(10)
+ m_p = MyPromise.resolve(p)
+
+ assert isinstance(m_p, MyPromise)
+ assert m_p.get() == p.get()
+
+
+def test_promise_repr_pending():
+ promise = Promise()
+ assert repr(promise) == "<Promise at {} pending>".format(hex(id(promise)))
+
+
+def test_promise_repr_pending():
+ val = {1: 2}
+ promise = Promise.fulfilled(val)
+ promise._wait()
+ assert repr(promise) == "<Promise at {} fulfilled with {}>".format(
+ hex(id(promise)), repr(val)
+ )
+
+
+def test_promise_repr_fulfilled():
+ val = {1: 2}
+ promise = Promise.fulfilled(val)
+ promise._wait()
+ assert repr(promise) == "<Promise at {} fulfilled with {}>".format(
+ hex(id(promise)), repr(val)
+ )
+
+
+def test_promise_repr_rejected():
+ err = Exception("Error!")
+ promise = Promise.rejected(err)
+ promise._wait()
+ assert repr(promise) == "<Promise at {} rejected with {}>".format(
+ hex(id(promise)), repr(err)
+ )
+
+
+def test_promise_loop():
+ def by_two(result):
+ return result * 2
+
+ def executor(resolve, reject):
+ resolve(Promise.resolve(1).then(lambda v: Promise.resolve(v).then(by_two)))
+
+ p = Promise(executor)
+ assert p.get(.1) == 2
+
+
+def test_resolve_future_like(resolve):
+ class CustomThenable(object):
+ def add_done_callback(self, f):
+ f(True)
+
+ def done(self):
+ return True
+
+ def exception(self):
+ pass
+
+ def result(self):
+ return True
+
+ instance = CustomThenable()
+
+ promise = resolve(instance)
+ assert promise.get() == True
+
+
+def sum_function(a, b):
+ return a + b
+
+
+def test_promisify_function_resolved(resolve):
+ promisified_func = promisify(sum_function)
+
+ result = promisified_func(1, 2)
+ assert isinstance(result, Promise)
+ assert result.get() == 3
+
+
+def test_promisify_function_rejected(resolve):
+ promisified_func = promisify(sum_function)
+
+ result = promisified_func(None, None)
+ assert isinstance(result, Promise)
+ with raises(Exception) as exc_info_promise:
+ result.get()
+
+ with raises(Exception) as exc_info:
+ sum_function(None, None)
+
+ assert str(exc_info_promise.value) == str(exc_info.value)
+
+
+def test_promises_with_only_then():
+ context = {"success": False}
+ error = RuntimeError("Ooops!")
+ promise1 = Promise(
+ lambda resolve, reject: context.update({"promise1_reject": reject})
+ )
+ promise2 = promise1.then(lambda x: None)
+ promise3 = promise1.then(lambda x: None)
+ context["promise1_reject"](error)
+
+ promise2._wait()
+ promise3._wait()
+ assert promise2.reason == error
+ assert promise3.reason == error
+
+
+def test_promises_promisify_still_works_but_deprecated_for_non_callables():
+ x = promisify(1)
+ assert isinstance(x, Promise)
+ assert x.get() == 1
+
+
+# def test_promise_loop():
+# values = Promise.resolve([1, None, 2])
+# def on_error(error):
+# error
+
+# def executor(resolve, reject):
+# resolve(Promise.resolve(values).then(lambda values: Promise.all([Promise.resolve(values[0])]).catch(on_error)))
+
+# p = Promise(executor)
+# assert p.get(.1) == 2
diff --git a/wandb/vendor/promise-2.3.0/tests/test_issues.py b/wandb/vendor/promise-2.3.0/tests/test_issues.py
new file mode 100644
index 0000000000000000000000000000000000000000..91974090443f6600bcee908b7412dd41c8049b03
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/tests/test_issues.py
@@ -0,0 +1,132 @@
+# This tests reported issues in the Promise package
+from concurrent.futures import ThreadPoolExecutor
+from promise import Promise
+import time
+import weakref
+import gc
+
+executor = ThreadPoolExecutor(max_workers=40000)
+
+
+def test_issue_11():
+ # https://github.com/syrusakbary/promise/issues/11
+ def test(x):
+ def my(resolve, reject):
+ if x > 0:
+ resolve(x)
+ else:
+ reject(Exception(x))
+
+ return Promise(my)
+
+ promise_resolved = test(42).then(lambda x: x)
+ assert promise_resolved.get() == 42
+
+ promise_rejected = test(-42).then(lambda x: x, lambda e: str(e))
+ assert promise_rejected.get() == "-42"
+
+
+def identity(x, wait):
+ if wait:
+ time.sleep(wait)
+ return x
+
+
+def promise_with_wait(x, wait):
+ return Promise.resolve(identity(x, wait))
+
+
+def test_issue_9():
+ no_wait = Promise.all(
+ [promise_with_wait(x1, None).then(lambda y: x1 * y) for x1 in (0, 1, 2, 3)]
+ ).get()
+ wait_a_bit = Promise.all(
+ [promise_with_wait(x2, 0.05).then(lambda y: x2 * y) for x2 in (0, 1, 2, 3)]
+ ).get()
+ wait_longer = Promise.all(
+ [promise_with_wait(x3, 0.1).then(lambda y: x3 * y) for x3 in (0, 1, 2, 3)]
+ ).get()
+
+ assert no_wait == wait_a_bit
+ assert no_wait == wait_longer
+
+
+@Promise.safe
+def test_issue_9_safe():
+ no_wait = Promise.all(
+ [promise_with_wait(x1, None).then(lambda y: x1 * y) for x1 in (0, 1, 2, 3)]
+ ).get()
+ wait_a_bit = Promise.all(
+ [promise_with_wait(x2, 0.05).then(lambda y: x2 * y) for x2 in (0, 1, 2, 3)]
+ ).get()
+ wait_longer = Promise.all(
+ [promise_with_wait(x3, 0.1).then(lambda y: x3 * y) for x3 in (0, 1, 2, 3)]
+ ).get()
+
+ assert no_wait == [0, 3, 6, 9]
+ assert no_wait == wait_a_bit
+ assert no_wait == wait_longer
+
+
+def test_issue_26():
+ context = {"success": False}
+ promise1 = Promise(
+ lambda resolve, reject: context.update({"promise1_reject": reject})
+ )
+ promise1.then(lambda x: None)
+ promise1.then(lambda x: None)
+ context["promise1_reject"](RuntimeError("Ooops!"))
+
+ promise2 = Promise(
+ lambda resolve, reject: context.update({"promise2_resolve": resolve})
+ )
+ promise3 = promise2.then(lambda x: context.update({"success": True}))
+ context["promise2_resolve"](None)
+
+ # We wait so it works in asynchronous envs
+ promise3._wait(timeout=.1)
+ assert context["success"]
+
+
+# def promise_in_executor(x, wait):
+# return Promise.promisify(executor.submit(identity, x, wait))
+
+
+# @Promise.safe
+# def test_issue_9_extra():
+# no_wait = Promise.all([promise_in_executor(x1, None).then(lambda y: x1*y) for x1 in (0,1,2,3)]).get()
+# wait_a_bit = Promise.all([promise_in_executor(x2, 0.1).then(lambda y: x2*y) for x2 in (0,1,2,3)]).get()
+# wait_longer = Promise.all([promise_in_executor(x3, 0.5).then(lambda y: x3*y) for x3 in (0,1,2,3)]).get()
+
+# assert no_wait == [0, 3, 6, 9]
+# assert no_wait == wait_a_bit
+# assert no_wait == wait_longer
+
+
+def test_issue_33():
+ def do(x):
+ v = Promise.resolve("ok").then(lambda x: x).get()
+ return v
+
+ p = Promise.resolve(None).then(do)
+ assert p.get() == "ok"
+
+
+def test_issue_75():
+ def function_with_local_type():
+ class A:
+ pass
+
+ a = A()
+ assert a == Promise.resolve(a).get()
+
+ return weakref.ref(A)
+
+ weak_reference = function_with_local_type()
+
+ # The local type 'A' from the function is still kept alive by reference cycles.
+ gc.collect()
+
+ # Now the local type should have been garbage collected,
+ # such that the weak reference should be invalid.
+ assert not weak_reference()
diff --git a/wandb/vendor/promise-2.3.0/tests/test_promise_list.py b/wandb/vendor/promise-2.3.0/tests/test_promise_list.py
new file mode 100644
index 0000000000000000000000000000000000000000..e8dc35a99a92f083ce92552d6bc395f197475e93
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/tests/test_promise_list.py
@@ -0,0 +1,70 @@
+from pytest import raises
+
+from promise import Promise
+from promise.promise_list import PromiseList
+
+
+def all(promises):
+ return PromiseList(promises, Promise).promise
+
+
+def test_empty_promises():
+ all_promises = all([])
+ assert all_promises.get() == []
+
+
+def test_bad_promises():
+ all_promises = all(None)
+
+ with raises(Exception) as exc_info:
+ all_promises.get()
+
+ assert str(exc_info.value) == "PromiseList requires an iterable. Received None."
+
+
+def test_promise_basic():
+ all_promises = all([1, 2])
+ assert all_promises.get() == [1, 2]
+
+
+def test_promise_mixed():
+ all_promises = all([1, 2, Promise.resolve(3)])
+ assert all_promises.get() == [1, 2, 3]
+
+
+def test_promise_rejected():
+ e = Exception("Error")
+ all_promises = all([1, 2, Promise.reject(e)])
+
+ with raises(Exception) as exc_info:
+ all_promises.get()
+
+ assert str(exc_info.value) == "Error"
+
+
+def test_promise_reject_skip_all_other_values():
+ e1 = Exception("Error1")
+ e2 = Exception("Error2")
+ p = Promise()
+ all_promises = all([1, Promise.reject(e1), Promise.reject(e2)])
+
+ with raises(Exception) as exc_info:
+ all_promises.get()
+
+ assert str(exc_info.value) == "Error1"
+
+
+def test_promise_lazy_promise():
+ p = Promise()
+ all_promises = all([1, 2, p])
+ assert not all_promises.is_fulfilled
+ p.do_resolve(3)
+ assert all_promises.get() == [1, 2, 3]
+
+
+def test_promise_contained_promise():
+ p = Promise()
+ all_promises = all([1, 2, Promise.resolve(None).then(lambda v: p)])
+ assert not all_promises.is_fulfilled
+ p.do_resolve(3)
+ assert all_promises.get() == [1, 2, 3]
diff --git a/wandb/vendor/promise-2.3.0/tests/test_spec.py b/wandb/vendor/promise-2.3.0/tests/test_spec.py
new file mode 100644
index 0000000000000000000000000000000000000000..dec812fbd7324b58cc24800a5f1aeceeb5d7dadc
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/tests/test_spec.py
@@ -0,0 +1,584 @@
+# Tests the spec based on:
+# https://github.com/promises-aplus/promises-tests
+
+from promise import Promise
+from .utils import assert_exception
+
+from threading import Event
+
+
+class Counter:
+ """
+ A helper class with some side effects
+ we can test.
+ """
+
+ def __init__(self):
+ self.count = 0
+
+ def tick(self):
+ self.count += 1
+
+ def value(self):
+ return self.count
+
+
+def test_3_2_1():
+ """
+ Test that the arguments to 'then' are optional.
+ """
+
+ p1 = Promise()
+ p2 = p1.then()
+ p3 = Promise()
+ p4 = p3.then()
+ p1.do_resolve(5)
+ p3.do_reject(Exception("How dare you!"))
+
+
+def test_3_2_1_1():
+ """
+ That that the first argument to 'then' is ignored if it
+ is not a function.
+ """
+ results = {}
+ nonFunctions = [None, False, 5, {}, []]
+
+ def testNonFunction(nonFunction):
+ def foo(k, r):
+ results[k] = r
+
+ p1 = Promise.reject(Exception("Error: " + str(nonFunction)))
+ p2 = p1.then(nonFunction, lambda r: foo(str(nonFunction), r))
+ p2._wait()
+
+ for v in nonFunctions:
+ testNonFunction(v)
+
+ for v in nonFunctions:
+ assert_exception(results[str(v)], Exception, "Error: " + str(v))
+
+
+def test_3_2_1_2():
+ """
+ That that the second argument to 'then' is ignored if it
+ is not a function.
+ """
+ results = {}
+ nonFunctions = [None, False, 5, {}, []]
+
+ def testNonFunction(nonFunction):
+ def foo(k, r):
+ results[k] = r
+
+ p1 = Promise.resolve("Error: " + str(nonFunction))
+ p2 = p1.then(lambda r: foo(str(nonFunction), r), nonFunction)
+ p2._wait()
+
+ for v in nonFunctions:
+ testNonFunction(v)
+
+ for v in nonFunctions:
+ assert "Error: " + str(v) == results[str(v)]
+
+
+def test_3_2_2_1():
+ """
+ The first argument to 'then' must be called when a promise is
+ fulfilled.
+ """
+
+ c = Counter()
+
+ def check(v, c):
+ assert v == 5
+ c.tick()
+
+ p1 = Promise.resolve(5)
+ p2 = p1.then(lambda v: check(v, c))
+ p2._wait()
+ assert 1 == c.value()
+
+
+def test_3_2_2_2():
+ """
+ Make sure callbacks are never called more than once.
+ """
+
+ c = Counter()
+ p1 = Promise.resolve(5)
+ p2 = p1.then(lambda v: c.tick())
+ p2._wait()
+ try:
+ # I throw an exception
+ p1.do_resolve(5)
+ assert False # Should not get here!
+ except AssertionError:
+ # This is expected
+ pass
+ assert 1 == c.value()
+
+
+def test_3_2_2_3():
+ """
+ Make sure fulfilled callback never called if promise is rejected
+ """
+
+ cf = Counter()
+ cr = Counter()
+ p1 = Promise.reject(Exception("Error"))
+ p2 = p1.then(lambda v: cf.tick(), lambda r: cr.tick())
+ p2._wait()
+ assert 0 == cf.value()
+ assert 1 == cr.value()
+
+
+def test_3_2_3_1():
+ """
+ The second argument to 'then' must be called when a promise is
+ rejected.
+ """
+
+ c = Counter()
+
+ def check(r, c):
+ assert_exception(r, Exception, "Error")
+ c.tick()
+
+ p1 = Promise.reject(Exception("Error"))
+ p2 = p1.then(None, lambda r: check(r, c))
+ p2._wait()
+ assert 1 == c.value()
+
+
+def test_3_2_3_2():
+ """
+ Make sure callbacks are never called more than once.
+ """
+
+ c = Counter()
+ p1 = Promise.reject(Exception("Error"))
+ p2 = p1.then(None, lambda v: c.tick())
+ p2._wait()
+ try:
+ # I throw an exception
+ p1.do_reject(Exception("Error"))
+ assert False # Should not get here!
+ except AssertionError:
+ # This is expected
+ pass
+ assert 1 == c.value()
+
+
+def test_3_2_3_3():
+ """
+ Make sure rejected callback never called if promise is fulfilled
+ """
+
+ cf = Counter()
+ cr = Counter()
+ p1 = Promise.resolve(5)
+ p2 = p1.then(lambda v: cf.tick(), lambda r: cr.tick())
+ p2._wait()
+ assert 0 == cr.value()
+ assert 1 == cf.value()
+
+
+def test_3_2_5_1_when():
+ """
+ Then can be called multiple times on the same promise
+ and callbacks must be called in the order of the
+ then calls.
+ """
+
+ def add(l, v):
+ l.append(v)
+
+ p1 = Promise.resolve(2)
+ order = []
+ p2 = p1.then(lambda v: add(order, "p2"))
+ p3 = p1.then(lambda v: add(order, "p3"))
+ p2._wait()
+ p3._wait()
+ assert 2 == len(order)
+ assert "p2" == order[0]
+ assert "p3" == order[1]
+
+
+def test_3_2_5_1_if():
+ """
+ Then can be called multiple times on the same promise
+ and callbacks must be called in the order of the
+ then calls.
+ """
+
+ def add(l, v):
+ l.append(v)
+
+ p1 = Promise.resolve(2)
+ order = []
+ p2 = p1.then(lambda v: add(order, "p2"))
+ p3 = p1.then(lambda v: add(order, "p3"))
+ p2._wait()
+ p3._wait()
+ assert 2 == len(order)
+ assert "p2" == order[0]
+ assert "p3" == order[1]
+
+
+def test_3_2_5_2_when():
+ """
+ Then can be called multiple times on the same promise
+ and callbacks must be called in the order of the
+ then calls.
+ """
+
+ def add(l, v):
+ l.append(v)
+
+ p1 = Promise.reject(Exception("Error"))
+ order = []
+ p2 = p1.then(None, lambda v: add(order, "p2"))
+ p3 = p1.then(None, lambda v: add(order, "p3"))
+ p2._wait()
+ p3._wait()
+ assert 2 == len(order)
+ assert "p2" == order[0]
+ assert "p3" == order[1]
+
+
+def test_3_2_5_2_if():
+ """
+ Then can be called multiple times on the same promise
+ and callbacks must be called in the order of the
+ then calls.
+ """
+
+ def add(l, v):
+ l.append(v)
+
+ p1 = Promise.reject(Exception("Error"))
+ order = []
+ p2 = p1.then(None, lambda v: add(order, "p2"))
+ p3 = p1.then(None, lambda v: add(order, "p3"))
+ p2._wait()
+ p3._wait()
+ assert 2 == len(order)
+ assert "p2" == order[0]
+ assert "p3" == order[1]
+
+
+def test_3_2_6_1():
+ """
+ Promises returned by then must be fulfilled when the promise
+ they are chained from is fulfilled IF the fulfillment value
+ is not a promise.
+ """
+
+ p1 = Promise.resolve(5)
+ pf = p1.then(lambda v: v * v)
+ assert pf.get() == 25
+
+ p2 = Promise.reject(Exception("Error"))
+ pr = p2.then(None, lambda r: 5)
+ assert 5 == pr.get()
+
+
+def test_3_2_6_2_when():
+ """
+ Promises returned by then must be rejected when any of their
+ callbacks throw an exception.
+ """
+
+ def fail(v):
+ raise AssertionError("Exception Message")
+
+ p1 = Promise.resolve(5)
+ pf = p1.then(fail)
+ pf._wait()
+ assert pf.is_rejected
+ assert_exception(pf.reason, AssertionError, "Exception Message")
+
+ p2 = Promise.reject(Exception("Error"))
+ pr = p2.then(None, fail)
+ pr._wait()
+ assert pr.is_rejected
+ assert_exception(pr.reason, AssertionError, "Exception Message")
+
+
+def test_3_2_6_2_if():
+ """
+ Promises returned by then must be rejected when any of their
+ callbacks throw an exception.
+ """
+
+ def fail(v):
+ raise AssertionError("Exception Message")
+
+ p1 = Promise.resolve(5)
+ pf = p1.then(fail)
+ pf._wait()
+ assert pf.is_rejected
+ assert_exception(pf.reason, AssertionError, "Exception Message")
+
+ p2 = Promise.reject(Exception("Error"))
+ pr = p2.then(None, fail)
+ pr._wait()
+ assert pr.is_rejected
+ assert_exception(pr.reason, AssertionError, "Exception Message")
+
+
+def test_3_2_6_3_when_fulfilled():
+ """
+ Testing return of pending promises to make
+ sure they are properly chained.
+ This covers the case where the root promise
+ is fulfilled after the chaining is defined.
+ """
+
+ p1 = Promise()
+ pending = Promise()
+
+ def p1_resolved(v):
+ return pending
+
+ pf = p1.then(p1_resolved)
+
+ assert pending.is_pending
+ assert pf.is_pending
+ p1.do_resolve(10)
+ pending.do_resolve(5)
+ pending._wait()
+ assert pending.is_fulfilled
+ assert 5 == pending.get()
+ pf._wait()
+ assert pf.is_fulfilled
+ assert 5 == pf.get()
+
+ p2 = Promise()
+ bad = Promise()
+ pr = p2.then(lambda r: bad)
+ assert bad.is_pending
+ assert pr.is_pending
+ p2.do_resolve(10)
+ bad._reject_callback(Exception("Error"))
+ bad._wait()
+ assert bad.is_rejected
+ assert_exception(bad.reason, Exception, "Error")
+ pr._wait()
+ assert pr.is_rejected
+ assert_exception(pr.reason, Exception, "Error")
+
+
+def test_3_2_6_3_if_fulfilled():
+ """
+ Testing return of pending promises to make
+ sure they are properly chained.
+ This covers the case where the root promise
+ is fulfilled before the chaining is defined.
+ """
+
+ p1 = Promise()
+ p1.do_resolve(10)
+ pending = Promise()
+ pending.do_resolve(5)
+ pf = p1.then(lambda r: pending)
+ pending._wait()
+ assert pending.is_fulfilled
+ assert 5 == pending.get()
+ pf._wait()
+ assert pf.is_fulfilled
+ assert 5 == pf.get()
+
+ p2 = Promise()
+ p2.do_resolve(10)
+ bad = Promise()
+ bad.do_reject(Exception("Error"))
+ pr = p2.then(lambda r: bad)
+ bad._wait()
+ assert_exception(bad.reason, Exception, "Error")
+ pr._wait()
+ assert pr.is_rejected
+ assert_exception(pr.reason, Exception, "Error")
+
+
+def test_3_2_6_3_when_rejected():
+ """
+ Testing return of pending promises to make
+ sure they are properly chained.
+ This covers the case where the root promise
+ is rejected after the chaining is defined.
+ """
+
+ p1 = Promise()
+ pending = Promise()
+ pr = p1.then(None, lambda r: pending)
+ assert pending.is_pending
+ assert pr.is_pending
+ p1.do_reject(Exception("Error"))
+ pending.do_resolve(10)
+ pending._wait()
+ assert pending.is_fulfilled
+ assert 10 == pending.get()
+ assert 10 == pr.get()
+
+ p2 = Promise()
+ bad = Promise()
+ pr = p2.then(None, lambda r: bad)
+ assert bad.is_pending
+ assert pr.is_pending
+ p2.do_reject(Exception("Error"))
+ bad.do_reject(Exception("Assertion"))
+ bad._wait()
+ assert bad.is_rejected
+ assert_exception(bad.reason, Exception, "Assertion")
+ pr._wait()
+ assert pr.is_rejected
+ assert_exception(pr.reason, Exception, "Assertion")
+
+
+def test_3_2_6_3_if_rejected():
+ """
+ Testing return of pending promises to make
+ sure they are properly chained.
+ This covers the case where the root promise
+ is rejected before the chaining is defined.
+ """
+
+ p1 = Promise()
+ p1.do_reject(Exception("Error"))
+ pending = Promise()
+ pending.do_resolve(10)
+ pr = p1.then(None, lambda r: pending)
+ pending._wait()
+ assert pending.is_fulfilled
+ assert 10 == pending.get()
+ pr._wait()
+ assert pr.is_fulfilled
+ assert 10 == pr.get()
+
+ p2 = Promise()
+ p2.do_reject(Exception("Error"))
+ bad = Promise()
+ bad.do_reject(Exception("Assertion"))
+ pr = p2.then(None, lambda r: bad)
+ bad._wait()
+ assert bad.is_rejected
+ assert_exception(bad.reason, Exception, "Assertion")
+ pr._wait()
+ assert pr.is_rejected
+ assert_exception(pr.reason, Exception, "Assertion")
+
+
+def test_3_2_6_4_pending():
+ """
+ Handles the case where the arguments to then
+ are not functions or promises.
+ """
+ p1 = Promise()
+ p2 = p1.then(5)
+ p1.do_resolve(10)
+ assert 10 == p1.get()
+ p2._wait()
+ assert p2.is_fulfilled
+ assert 10 == p2.get()
+
+
+def test_3_2_6_4_fulfilled():
+ """
+ Handles the case where the arguments to then
+ are values, not functions or promises.
+ """
+ p1 = Promise()
+ p1.do_resolve(10)
+ p2 = p1.then(5)
+ assert 10 == p1.get()
+ p2._wait()
+ assert p2.is_fulfilled
+ assert 10 == p2.get()
+
+
+def test_3_2_6_5_pending():
+ """
+ Handles the case where the arguments to then
+ are values, not functions or promises.
+ """
+ p1 = Promise()
+ p2 = p1.then(None, 5)
+ p1.do_reject(Exception("Error"))
+ assert_exception(p1.reason, Exception, "Error")
+ p2._wait()
+ assert p2.is_rejected
+ assert_exception(p2.reason, Exception, "Error")
+
+
+def test_3_2_6_5_rejected():
+ """
+ Handles the case where the arguments to then
+ are values, not functions or promises.
+ """
+ p1 = Promise()
+ p1.do_reject(Exception("Error"))
+ p2 = p1.then(None, 5)
+ assert_exception(p1.reason, Exception, "Error")
+ p2._wait()
+ assert p2.is_rejected
+ assert_exception(p2.reason, Exception, "Error")
+
+
+def test_chained_promises():
+ """
+ Handles the case where the arguments to then
+ are values, not functions or promises.
+ """
+ p1 = Promise(lambda resolve, reject: resolve(Promise.resolve(True)))
+ assert p1.get() == True
+
+
+def test_promise_resolved_after():
+ """
+ The first argument to 'then' must be called when a promise is
+ fulfilled.
+ """
+
+ c = Counter()
+
+ def check(v, c):
+ assert v == 5
+ c.tick()
+
+ p1 = Promise()
+ p2 = p1.then(lambda v: check(v, c))
+ p1.do_resolve(5)
+ Promise.wait(p2)
+
+ assert 1 == c.value()
+
+
+def test_promise_follows_indifentely():
+ a = Promise.resolve(None)
+ b = a.then(lambda x: Promise.resolve("X"))
+ e = Event()
+
+ def b_then(v):
+
+ c = Promise.resolve(None)
+ d = c.then(lambda v: Promise.resolve("B"))
+ return d
+
+ promise = b.then(b_then)
+
+ assert promise.get() == "B"
+
+
+def test_promise_all_follows_indifentely():
+ promises = Promise.all(
+ [
+ Promise.resolve("A"),
+ Promise.resolve(None)
+ .then(Promise.resolve)
+ .then(lambda v: Promise.resolve(None).then(lambda v: Promise.resolve("B"))),
+ ]
+ )
+
+ assert promises.get() == ["A", "B"]
diff --git a/wandb/vendor/promise-2.3.0/tests/test_thread_safety.py b/wandb/vendor/promise-2.3.0/tests/test_thread_safety.py
new file mode 100644
index 0000000000000000000000000000000000000000..ed55a84ff70b71f4a3b465a9c2cc5c9646f7c210
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/tests/test_thread_safety.py
@@ -0,0 +1,115 @@
+from promise import Promise
+from promise.dataloader import DataLoader
+import threading
+
+
+
+def test_promise_thread_safety():
+ """
+ Promise tasks should never be executed in a different thread from the one they are scheduled from,
+ unless the ThreadPoolExecutor is used.
+
+ Here we assert that the pending promise tasks on thread 1 are not executed on thread 2 as thread 2
+ resolves its own promise tasks.
+ """
+ event_1 = threading.Event()
+ event_2 = threading.Event()
+
+ assert_object = {'is_same_thread': True}
+
+ def task_1():
+ thread_name = threading.current_thread().getName()
+
+ def then_1(value):
+ # Enqueue tasks to run later.
+ # This relies on the fact that `then` does not execute the function synchronously when called from
+ # within another `then` callback function.
+ promise = Promise.resolve(None).then(then_2)
+ assert promise.is_pending
+ event_1.set() # Unblock main thread
+ event_2.wait() # Wait for thread 2
+
+ def then_2(value):
+ assert_object['is_same_thread'] = (thread_name == threading.current_thread().getName())
+
+ promise = Promise.resolve(None).then(then_1)
+
+ def task_2():
+ promise = Promise.resolve(None).then(lambda v: None)
+ promise.get() # Drain task queue
+ event_2.set() # Unblock thread 1
+
+ thread_1 = threading.Thread(target=task_1)
+ thread_1.start()
+
+ event_1.wait() # Wait for Thread 1 to enqueue promise tasks
+
+ thread_2 = threading.Thread(target=task_2)
+ thread_2.start()
+
+ for thread in (thread_1, thread_2):
+ thread.join()
+
+ assert assert_object['is_same_thread']
+
+
+def test_dataloader_thread_safety():
+ """
+ Dataloader should only batch `load` calls that happened on the same thread.
+
+ Here we assert that `load` calls on thread 2 are not batched on thread 1 as
+ thread 1 batches its own `load` calls.
+ """
+ def load_many(keys):
+ thead_name = threading.current_thread().getName()
+ return Promise.resolve([thead_name for key in keys])
+
+ thread_name_loader = DataLoader(load_many)
+
+ event_1 = threading.Event()
+ event_2 = threading.Event()
+ event_3 = threading.Event()
+
+ assert_object = {
+ 'is_same_thread_1': True,
+ 'is_same_thread_2': True,
+ }
+
+ def task_1():
+ @Promise.safe
+ def do():
+ promise = thread_name_loader.load(1)
+ event_1.set()
+ event_2.wait() # Wait for thread 2 to call `load`
+ assert_object['is_same_thread_1'] = (
+ promise.get() == threading.current_thread().getName()
+ )
+ event_3.set() # Unblock thread 2
+
+ do().get()
+
+ def task_2():
+ @Promise.safe
+ def do():
+ promise = thread_name_loader.load(2)
+ event_2.set()
+ event_3.wait() # Wait for thread 1 to run `dispatch_queue_batch`
+ assert_object['is_same_thread_2'] = (
+ promise.get() == threading.current_thread().getName()
+ )
+
+ do().get()
+
+ thread_1 = threading.Thread(target=task_1)
+ thread_1.start()
+
+ event_1.wait() # Wait for thread 1 to call `load`
+
+ thread_2 = threading.Thread(target=task_2)
+ thread_2.start()
+
+ for thread in (thread_1, thread_2):
+ thread.join()
+
+ assert assert_object['is_same_thread_1']
+ assert assert_object['is_same_thread_2']
diff --git a/wandb/vendor/promise-2.3.0/tests/utils.py b/wandb/vendor/promise-2.3.0/tests/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..73dd1619c250ecb2b42680afffad6884ae97282f
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/tests/utils.py
@@ -0,0 +1,3 @@
+def assert_exception(exception, expected_exception_cls, expected_message):
+ assert isinstance(exception, expected_exception_cls)
+ assert str(exception) == expected_message
diff --git a/wandb/vendor/promise-2.3.0/wandb-vendor.diff b/wandb/vendor/promise-2.3.0/wandb-vendor.diff
new file mode 100644
index 0000000000000000000000000000000000000000..a74c255e58a9ba1843d907406f3fa5649a72ee95
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/wandb-vendor.diff
@@ -0,0 +1,21 @@
+diff --git a/wandb_promise/promise.py b/wandb_promise/promise.py
+index ef9a45f..5a8a7b3 100644
+--- a/wandb_promise/promise.py
++++ b/wandb_promise/promise.py
+@@ -5,7 +5,6 @@ from threading import RLock
+ from types import TracebackType
+ from weakref import WeakKeyDictionary
+
+-from six import reraise # type: ignore
+ from .async_ import Async
+ from .compat import (
+ Future,
+@@ -223,7 +222,7 @@ class Promise(Generic[T]):
+ elif self._state == STATE_REJECTED:
+ if _raise:
+ raise_val = self._fulfillment_handler0
+- reraise(type(raise_val), raise_val, self._traceback)
++ raise raise_val.with_traceback(self._traceback)
+ return self._fulfillment_handler0
+
+ def _fulfill(self, value):
diff --git a/wandb/vendor/promise-2.3.0/wandb-vendor.md b/wandb/vendor/promise-2.3.0/wandb-vendor.md
new file mode 100644
index 0000000000000000000000000000000000000000..aada0eba0e66b060d23f18cc7d741f41cdf17a85
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/wandb-vendor.md
@@ -0,0 +1,9 @@
+
+# Modification steps
+
+```shell
+git checkout v2.3.0
+mv promise wandb_promise
+rm -rf .git
+patch -p4 < wandb-vendor.diff
+```
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/__init__.py b/wandb/vendor/promise-2.3.0/wandb_promise/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..ed495d0fda9acc8ed0cb6a94ec93485d68781b9e
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/wandb_promise/__init__.py
@@ -0,0 +1,38 @@
+from .pyutils.version import get_version
+
+
+try:
+ # This variable is injected in the __builtins__ by the build
+ # process. It used to enable importing subpackages when
+ # the required packages are not installed
+ __SETUP__ # type: ignore
+except NameError:
+ __SETUP__ = False
+
+
+VERSION = (2, 3, 0, "final", 0)
+
+__version__ = get_version(VERSION)
+
+if not __SETUP__:
+ from .promise import (
+ Promise,
+ promise_for_dict,
+ promisify,
+ is_thenable,
+ async_instance,
+ get_default_scheduler,
+ set_default_scheduler,
+ )
+ from .schedulers.immediate import ImmediateScheduler
+
+ __all__ = [
+ "Promise",
+ "promise_for_dict",
+ "promisify",
+ "is_thenable",
+ "async_instance",
+ "get_default_scheduler",
+ "set_default_scheduler",
+ "ImmediateScheduler",
+ ]
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/async_.py b/wandb/vendor/promise-2.3.0/wandb_promise/async_.py
new file mode 100644
index 0000000000000000000000000000000000000000..21ac6e2ce6e6cd855921ed8accd3c4e9e0291fdf
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/wandb_promise/async_.py
@@ -0,0 +1,135 @@
+# Based on https://github.com/petkaantonov/bluebird/blob/master/src/promise.js
+from collections import deque
+from threading import local
+
+if False:
+ from .promise import Promise
+ from typing import Any, Callable, Optional, Union # flake8: noqa
+
+
+class Async(local):
+ def __init__(self, trampoline_enabled=True):
+ self.is_tick_used = False
+ self.late_queue = deque() # type: ignore
+ self.normal_queue = deque() # type: ignore
+ self.have_drained_queues = False
+ self.trampoline_enabled = trampoline_enabled
+
+ def enable_trampoline(self):
+ self.trampoline_enabled = True
+
+ def disable_trampoline(self):
+ self.trampoline_enabled = False
+
+ def have_items_queued(self):
+ return self.is_tick_used or self.have_drained_queues
+
+ def _async_invoke_later(self, fn, scheduler):
+ self.late_queue.append(fn)
+ self.queue_tick(scheduler)
+
+ def _async_invoke(self, fn, scheduler):
+ # type: (Callable, Any) -> None
+ self.normal_queue.append(fn)
+ self.queue_tick(scheduler)
+
+ def _async_settle_promise(self, promise):
+ # type: (Promise) -> None
+ self.normal_queue.append(promise)
+ self.queue_tick(promise.scheduler)
+
+ def invoke_later(self, fn):
+ if self.trampoline_enabled:
+ self._async_invoke_later(fn, scheduler)
+ else:
+ scheduler.call_later(0.1, fn)
+
+ def invoke(self, fn, scheduler):
+ # type: (Callable, Any) -> None
+ if self.trampoline_enabled:
+ self._async_invoke(fn, scheduler)
+ else:
+ scheduler.call(fn)
+
+ def settle_promises(self, promise):
+ # type: (Promise) -> None
+ if self.trampoline_enabled:
+ self._async_settle_promise(promise)
+ else:
+ promise.scheduler.call(promise._settle_promises)
+
+ def throw_later(self, reason, scheduler):
+ # type: (Exception, Any) -> None
+ def fn():
+ # type: () -> None
+ raise reason
+
+ scheduler.call(fn)
+
+ fatal_error = throw_later
+
+ def drain_queue(self, queue):
+ # type: (deque) -> None
+ from .promise import Promise
+
+ while queue:
+ fn = queue.popleft()
+ if isinstance(fn, Promise):
+ fn._settle_promises()
+ continue
+ fn()
+
+ def drain_queue_until_resolved(self, promise):
+ # type: (Promise) -> None
+ from .promise import Promise
+
+ queue = self.normal_queue
+ while queue:
+ if not promise.is_pending:
+ return
+ fn = queue.popleft()
+ if isinstance(fn, Promise):
+ fn._settle_promises()
+ continue
+ fn()
+
+ self.reset()
+ self.have_drained_queues = True
+ self.drain_queue(self.late_queue)
+
+ def wait(self, promise, timeout=None):
+ # type: (Promise, Optional[float]) -> None
+ if not promise.is_pending:
+ # We return if the promise is already
+ # fulfilled or rejected
+ return
+
+ target = promise._target()
+
+ if self.trampoline_enabled:
+ if self.is_tick_used:
+ self.drain_queue_until_resolved(target)
+
+ if not promise.is_pending:
+ # We return if the promise is already
+ # fulfilled or rejected
+ return
+ target.scheduler.wait(target, timeout)
+
+ def drain_queues(self):
+ # type: () -> None
+ assert self.is_tick_used
+ self.drain_queue(self.normal_queue)
+ self.reset()
+ self.have_drained_queues = True
+ self.drain_queue(self.late_queue)
+
+ def queue_tick(self, scheduler):
+ # type: (Any) -> None
+ if not self.is_tick_used:
+ self.is_tick_used = True
+ scheduler.call(self.drain_queues)
+
+ def reset(self):
+ # type: () -> None
+ self.is_tick_used = False
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/compat.py b/wandb/vendor/promise-2.3.0/wandb_promise/compat.py
new file mode 100644
index 0000000000000000000000000000000000000000..21b091989858e0588a4dc5e1ad10baa677cd7e0b
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/wandb_promise/compat.py
@@ -0,0 +1,32 @@
+try:
+ from inspect import iscoroutine
+except ImportError:
+
+ def iscoroutine(obj): # type: ignore
+ return False
+
+
+try:
+ from asyncio import Future, ensure_future # type: ignore
+except ImportError:
+
+ class Future: # type: ignore
+ def __init__(self):
+ raise Exception("You need asyncio for using Futures")
+
+ def set_result(self):
+ raise Exception("You need asyncio for using Futures")
+
+ def set_exception(self):
+ raise Exception("You need asyncio for using Futures")
+
+ def ensure_future(): # type: ignore
+ raise Exception("ensure_future needs asyncio for executing")
+
+
+try:
+ from .iterate_promise import iterate_promise
+except (SyntaxError, ImportError):
+
+ def iterate_promise(promise): # type: ignore
+ raise Exception('You need "yield from" syntax for iterate in a Promise.')
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/dataloader.py b/wandb/vendor/promise-2.3.0/wandb_promise/dataloader.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb1dd0dd472671749cd26c15bc2fa82cd6facb73
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/wandb_promise/dataloader.py
@@ -0,0 +1,326 @@
+from collections import namedtuple
+try:
+ from collections.abc import Iterable
+except ImportError:
+ from collections import Iterable
+from functools import partial
+from threading import local
+
+from .promise import Promise, async_instance, get_default_scheduler
+
+if False:
+ from typing import (
+ Any,
+ List,
+ Sized,
+ Callable,
+ Optional,
+ Tuple,
+ Union,
+ Iterator,
+ Hashable,
+ ) # flake8: noqa
+
+
+def get_chunks(iterable_obj, chunk_size=1):
+ # type: (List[Loader], int) -> Iterator
+ chunk_size = max(1, chunk_size)
+ return (
+ iterable_obj[i : i + chunk_size]
+ for i in range(0, len(iterable_obj), chunk_size)
+ )
+
+
+Loader = namedtuple("Loader", "key,resolve,reject")
+
+
+class DataLoader(local):
+
+ batch = True
+ max_batch_size = None # type: int
+ cache = True
+
+ def __init__(
+ self,
+ batch_load_fn=None, # type: Callable
+ batch=None, # type: Optional[Any]
+ max_batch_size=None, # type: Optional[int]
+ cache=None, # type: Optional[Any]
+ get_cache_key=None, # type: Optional[Any]
+ cache_map=None, # type: Optional[Any]
+ scheduler=None, # type: Optional[Any]
+ ):
+ # type: (...) -> None
+
+ if batch_load_fn is not None:
+ self.batch_load_fn = batch_load_fn
+
+ if not callable(self.batch_load_fn):
+ raise TypeError(
+ (
+ "DataLoader must be have a batch_load_fn which accepts "
+ "List<key> and returns Promise<List<value>>, but got: {}."
+ ).format(batch_load_fn)
+ )
+
+ if batch is not None:
+ self.batch = batch
+
+ if max_batch_size is not None:
+ self.max_batch_size = max_batch_size
+
+ if cache is not None:
+ self.cache = cache
+
+ self.get_cache_key = get_cache_key or (lambda x: x)
+ self._promise_cache = cache_map or {}
+ self._queue = [] # type: List[Loader]
+ self._scheduler = scheduler
+
+ def load(self, key=None):
+ # type: (Hashable) -> Promise
+ """
+ Loads a key, returning a `Promise` for the value represented by that key.
+ """
+ if key is None:
+ raise TypeError(
+ (
+ "The loader.load() function must be called with a value,"
+ + "but got: {}."
+ ).format(key)
+ )
+
+ cache_key = self.get_cache_key(key)
+
+ # If caching and there is a cache-hit, return cached Promise.
+ if self.cache:
+ cached_promise = self._promise_cache.get(cache_key)
+ if cached_promise:
+ return cached_promise
+
+ # Otherwise, produce a new Promise for this value.
+
+ promise = Promise(partial(self.do_resolve_reject, key)) # type: ignore
+
+ # If caching, cache this promise.
+ if self.cache:
+ self._promise_cache[cache_key] = promise
+
+ return promise
+
+ def do_resolve_reject(self, key, resolve, reject):
+ # type: (Hashable, Callable, Callable) -> None
+ # Enqueue this Promise to be dispatched.
+ self._queue.append(Loader(key=key, resolve=resolve, reject=reject))
+ # Determine if a dispatch of this queue should be scheduled.
+ # A single dispatch should be scheduled per queue at the time when the
+ # queue changes from "empty" to "full".
+ if len(self._queue) == 1:
+ if self.batch:
+ # If batching, schedule a task to dispatch the queue.
+ enqueue_post_promise_job(partial(dispatch_queue, self), self._scheduler)
+ else:
+ # Otherwise dispatch the (queue of one) immediately.
+ dispatch_queue(self)
+
+ def load_many(self, keys):
+ # type: (Iterable[Hashable]) -> Promise
+ """
+ Loads multiple keys, promising an array of values
+
+ >>> a, b = await my_loader.load_many([ 'a', 'b' ])
+
+ This is equivalent to the more verbose:
+
+ >>> a, b = await Promise.all([
+ >>> my_loader.load('a'),
+ >>> my_loader.load('b')
+ >>> ])
+ """
+ if not isinstance(keys, Iterable):
+ raise TypeError(
+ (
+ "The loader.loadMany() function must be called with Array<key> "
+ + "but got: {}."
+ ).format(keys)
+ )
+
+ return Promise.all([self.load(key) for key in keys])
+
+ def clear(self, key):
+ # type: (Hashable) -> DataLoader
+ """
+ Clears the value at `key` from the cache, if it exists. Returns itself for
+ method chaining.
+ """
+ cache_key = self.get_cache_key(key)
+ self._promise_cache.pop(cache_key, None)
+ return self
+
+ def clear_all(self):
+ # type: () -> DataLoader
+ """
+ Clears the entire cache. To be used when some event results in unknown
+ invalidations across this particular `DataLoader`. Returns itself for
+ method chaining.
+ """
+ self._promise_cache.clear()
+ return self
+
+ def prime(self, key, value):
+ # type: (Hashable, Any) -> DataLoader
+ """
+ Adds the provied key and value to the cache. If the key already exists, no
+ change is made. Returns itself for method chaining.
+ """
+ cache_key = self.get_cache_key(key)
+
+ # Only add the key if it does not already exist.
+ if cache_key not in self._promise_cache:
+ # Cache a rejected promise if the value is an Error, in order to match
+ # the behavior of load(key).
+ if isinstance(value, Exception):
+ promise = Promise.reject(value)
+ else:
+ promise = Promise.resolve(value)
+
+ self._promise_cache[cache_key] = promise
+
+ return self
+
+
+# Private: Enqueue a Job to be executed after all "PromiseJobs" Jobs.
+#
+# ES6 JavaScript uses the concepts Job and JobQueue to schedule work to occur
+# after the current execution context has completed:
+# http://www.ecma-international.org/ecma-262/6.0/#sec-jobs-and-job-queues
+#
+# Node.js uses the `process.nextTick` mechanism to implement the concept of a
+# Job, maintaining a global FIFO JobQueue for all Jobs, which is flushed after
+# the current call stack ends.
+#
+# When calling `then` on a Promise, it enqueues a Job on a specific
+# "PromiseJobs" JobQueue which is flushed in Node as a single Job on the
+# global JobQueue.
+#
+# DataLoader batches all loads which occur in a single frame of execution, but
+# should include in the batch all loads which occur during the flushing of the
+# "PromiseJobs" JobQueue after that same execution frame.
+#
+# In order to avoid the DataLoader dispatch Job occuring before "PromiseJobs",
+# A Promise Job is created with the sole purpose of enqueuing a global Job,
+# ensuring that it always occurs after "PromiseJobs" ends.
+
+# Private: cached resolved Promise instance
+cache = local()
+
+def enqueue_post_promise_job(fn, scheduler):
+ # type: (Callable, Any) -> None
+ global cache
+ if not hasattr(cache, 'resolved_promise'):
+ cache.resolved_promise = Promise.resolve(None)
+ if not scheduler:
+ scheduler = get_default_scheduler()
+
+ def on_promise_resolve(v):
+ # type: (Any) -> None
+ async_instance.invoke(fn, scheduler)
+
+ cache.resolved_promise.then(on_promise_resolve)
+
+
+def dispatch_queue(loader):
+ # type: (DataLoader) -> None
+ """
+ Given the current state of a Loader instance, perform a batch load
+ from its current queue.
+ """
+ # Take the current loader queue, replacing it with an empty queue.
+ queue = loader._queue
+ loader._queue = []
+
+ # If a maxBatchSize was provided and the queue is longer, then segment the
+ # queue into multiple batches, otherwise treat the queue as a single batch.
+ max_batch_size = loader.max_batch_size
+
+ if max_batch_size and max_batch_size < len(queue):
+ chunks = get_chunks(queue, max_batch_size)
+ for chunk in chunks:
+ dispatch_queue_batch(loader, chunk)
+ else:
+ dispatch_queue_batch(loader, queue)
+
+
+def dispatch_queue_batch(loader, queue):
+ # type: (DataLoader, List[Loader]) -> None
+ # Collect all keys to be loaded in this dispatch
+ keys = [l.key for l in queue]
+
+ # Call the provided batch_load_fn for this loader with the loader queue's keys.
+ try:
+ batch_promise = loader.batch_load_fn(keys)
+ except Exception as e:
+ failed_dispatch(loader, queue, e)
+ return None
+
+ # Assert the expected response from batch_load_fn
+ if not batch_promise or not isinstance(batch_promise, Promise):
+ failed_dispatch(
+ loader,
+ queue,
+ TypeError(
+ (
+ "DataLoader must be constructed with a function which accepts "
+ "Array<key> and returns Promise<Array<value>>, but the function did "
+ "not return a Promise: {}."
+ ).format(batch_promise)
+ ),
+ )
+ return None
+
+ def batch_promise_resolved(values):
+ # type: (Sized) -> None
+ # Assert the expected resolution from batchLoadFn.
+ if not isinstance(values, Iterable):
+ raise TypeError(
+ (
+ "DataLoader must be constructed with a function which accepts "
+ "Array<key> and returns Promise<Array<value>>, but the function did "
+ "not return a Promise of an Array: {}."
+ ).format(values)
+ )
+
+ if len(values) != len(keys):
+ raise TypeError(
+ (
+ "DataLoader must be constructed with a function which accepts "
+ "Array<key> and returns Promise<Array<value>>, but the function did "
+ "not return a Promise of an Array of the same length as the Array "
+ "of keys."
+ "\n\nKeys:\n{}"
+ "\n\nValues:\n{}"
+ ).format(keys, values)
+ )
+
+ # Step through the values, resolving or rejecting each Promise in the
+ # loaded queue.
+ for l, value in zip(queue, values):
+ if isinstance(value, Exception):
+ l.reject(value)
+ else:
+ l.resolve(value)
+
+ batch_promise.then(batch_promise_resolved).catch(
+ partial(failed_dispatch, loader, queue)
+ )
+
+
+def failed_dispatch(loader, queue, error):
+ # type: (DataLoader, Iterable[Loader], Exception) -> None
+ """
+ Do not cache individual loads if the entire batch dispatch fails,
+ but still reject each request so they do not hang.
+ """
+ for l in queue:
+ loader.clear(l.key)
+ l.reject(error)
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/iterate_promise.py b/wandb/vendor/promise-2.3.0/wandb_promise/iterate_promise.py
new file mode 100644
index 0000000000000000000000000000000000000000..ba64f9c02fb23d0fc2196825a478360e7ab3d941
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/wandb_promise/iterate_promise.py
@@ -0,0 +1,12 @@
+# flake8: noqa
+if False:
+ from .promise import Promise
+ from typing import Iterator
+
+
+def iterate_promise(promise):
+ # type: (Promise) -> Iterator
+ if not promise.is_fulfilled:
+ yield from promise.future # type: ignore
+ assert promise.is_fulfilled
+ return promise.get()
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/promise.py b/wandb/vendor/promise-2.3.0/wandb_promise/promise.py
new file mode 100644
index 0000000000000000000000000000000000000000..5a8a7b3645e696212d1afaf5908f11fb3e6901de
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/wandb_promise/promise.py
@@ -0,0 +1,848 @@
+from collections import namedtuple
+from functools import partial, wraps
+from sys import version_info, exc_info
+from threading import RLock
+from types import TracebackType
+from weakref import WeakKeyDictionary
+
+from .async_ import Async
+from .compat import (
+ Future,
+ ensure_future,
+ iscoroutine, # type: ignore
+ iterate_promise,
+) # type: ignore
+from .utils import deprecated, integer_types, string_types, text_type, binary_type, warn
+from .promise_list import PromiseList
+from .schedulers.immediate import ImmediateScheduler
+from typing import TypeVar, Generic
+
+# from .schedulers.gevent import GeventScheduler
+# from .schedulers.asyncio import AsyncioScheduler
+# from .schedulers.thread import ThreadScheduler
+
+if False:
+ from typing import (
+ Type,
+ List,
+ Any,
+ Callable,
+ Dict,
+ Iterator,
+ Optional, # flake8: noqa
+ Tuple,
+ Union,
+ Generic,
+ Hashable,
+ MutableMapping,
+ )
+
+
+default_scheduler = ImmediateScheduler()
+
+async_instance = Async()
+
+
+def get_default_scheduler():
+ # type: () -> ImmediateScheduler
+ return default_scheduler
+
+
+def set_default_scheduler(scheduler):
+ global default_scheduler
+ default_scheduler = scheduler
+
+
+IS_PYTHON2 = version_info[0] == 2
+DEFAULT_TIMEOUT = None # type: Optional[float]
+
+MAX_LENGTH = 0xFFFF | 0
+CALLBACK_SIZE = 3
+
+CALLBACK_FULFILL_OFFSET = 0
+CALLBACK_REJECT_OFFSET = 1
+CALLBACK_PROMISE_OFFSET = 2
+
+BASE_TYPES = set(
+ integer_types
+ + string_types
+ + (bool, float, complex, tuple, list, dict, text_type, binary_type)
+)
+
+# These are the potential states of a promise
+STATE_PENDING = -1
+STATE_REJECTED = 0
+STATE_FULFILLED = 1
+
+
+def make_self_resolution_error():
+ # type: () -> TypeError
+ return TypeError("Promise is self")
+
+
+def try_catch(handler, *args, **kwargs):
+ # type: (Callable, Any, Any) -> Union[Tuple[Any, None], Tuple[None, Tuple[Exception, Optional[TracebackType]]]]
+ try:
+ return (handler(*args, **kwargs), None)
+ except Exception as e:
+ tb = exc_info()[2]
+ return (None, (e, tb))
+
+
+T = TypeVar("T")
+S = TypeVar("S", contravariant=True)
+
+
+class Promise(Generic[T]):
+ """
+ This is the Promise class that complies
+ Promises/A+ specification.
+ """
+
+ # __slots__ = ('_state', '_is_final', '_is_bound', '_is_following', '_is_async_guaranteed',
+ # '_length', '_handlers', '_fulfillment_handler0', '_rejection_handler0', '_promise0',
+ # '_is_waiting', '_future', '_trace', '_event_instance'
+ # )
+
+ _state = STATE_PENDING # type: int
+ _is_final = False
+ _is_bound = False
+ _is_following = False
+ _is_async_guaranteed = False
+ _length = 0
+ _handlers = None # type: Dict[int, Union[Callable, Promise, None]]
+ _fulfillment_handler0 = None # type: Any
+ _rejection_handler0 = None # type: Any
+ _promise0 = None # type: Optional[Promise]
+ _future = None # type: Future
+ _traceback = None # type: Optional[TracebackType]
+ # _trace = None
+ _is_waiting = False
+ _scheduler = None
+
+ def __init__(self, executor=None, scheduler=None):
+ # type: (Optional[Callable[[Callable[[T], None], Callable[[Exception], None]], None]], Any) -> None
+ """
+ Initialize the Promise into a pending state.
+ """
+ # self._state = STATE_PENDING # type: int
+ # self._is_final = False
+ # self._is_bound = False
+ # self._is_following = False
+ # self._is_async_guaranteed = False
+ # self._length = 0
+ # self._handlers = None # type: Dict[int, Union[Callable, None]]
+ # self._fulfillment_handler0 = None # type: Union[Callable, partial]
+ # self._rejection_handler0 = None # type: Union[Callable, partial]
+ # self._promise0 = None # type: Promise
+ # self._future = None # type: Future
+ # self._event_instance = None # type: Event
+
+ # self._is_waiting = False
+ self._scheduler = scheduler
+
+ if executor is not None:
+ self._resolve_from_executor(executor)
+
+ # For compatibility reasons
+ # self.reject = self._deprecated_reject
+ # self.resolve = self._deprecated_resolve
+
+ @property
+ def scheduler(self):
+ # type: () -> ImmediateScheduler
+ return self._scheduler or default_scheduler
+
+ @property
+ def future(self):
+ # type: (Promise) -> Future
+ if not self._future:
+ self._future = Future() # type: ignore
+ self._then( # type: ignore
+ self._future.set_result, self._future.set_exception
+ )
+ return self._future
+
+ def __iter__(self):
+ # type: () -> Iterator
+ return iterate_promise(self._target()) # type: ignore
+
+ __await__ = __iter__
+
+ @deprecated(
+ "Rejecting directly in a Promise instance is deprecated, as Promise.reject() is now a class method. "
+ "Please use promise.do_reject() instead.",
+ name="reject",
+ )
+ def _deprecated_reject(self, e):
+ self.do_reject(e)
+
+ @deprecated(
+ "Resolving directly in a Promise instance is deprecated, as Promise.resolve() is now a class method. "
+ "Please use promise.do_resolve() instead.",
+ name="resolve",
+ )
+ def _deprecated_resolve(self, value):
+ self.do_resolve(value)
+
+ def _resolve_callback(self, value):
+ # type: (T) -> None
+ if value is self:
+ return self._reject_callback(make_self_resolution_error(), False)
+
+ if not self.is_thenable(value):
+ return self._fulfill(value)
+
+ promise = self._try_convert_to_promise(value)._target()
+ if promise == self:
+ self._reject(make_self_resolution_error())
+ return
+
+ if promise._state == STATE_PENDING:
+ len = self._length
+ if len > 0:
+ promise._migrate_callback0(self)
+ for i in range(1, len):
+ promise._migrate_callback_at(self, i)
+
+ self._is_following = True
+ self._length = 0
+ self._set_followee(promise)
+ elif promise._state == STATE_FULFILLED:
+ self._fulfill(promise._value())
+ elif promise._state == STATE_REJECTED:
+ self._reject(promise._reason(), promise._target()._traceback)
+
+ def _settled_value(self, _raise=False):
+ # type: (bool) -> Any
+ assert not self._is_following
+
+ if self._state == STATE_FULFILLED:
+ return self._rejection_handler0
+ elif self._state == STATE_REJECTED:
+ if _raise:
+ raise_val = self._fulfillment_handler0
+ raise raise_val.with_traceback(self._traceback)
+ return self._fulfillment_handler0
+
+ def _fulfill(self, value):
+ # type: (T) -> None
+ if value is self:
+ err = make_self_resolution_error()
+ # self._attach_extratrace(err)
+ return self._reject(err)
+ self._state = STATE_FULFILLED
+ self._rejection_handler0 = value
+
+ if self._length > 0:
+ if self._is_async_guaranteed:
+ self._settle_promises()
+ else:
+ async_instance.settle_promises(self)
+
+ def _reject(self, reason, traceback=None):
+ # type: (Exception, Optional[TracebackType]) -> None
+ self._state = STATE_REJECTED
+ self._fulfillment_handler0 = reason
+ self._traceback = traceback
+
+ if self._is_final:
+ assert self._length == 0
+ async_instance.fatal_error(reason, self.scheduler)
+ return
+
+ if self._length > 0:
+ async_instance.settle_promises(self)
+ else:
+ self._ensure_possible_rejection_handled()
+
+ if self._is_async_guaranteed:
+ self._settle_promises()
+ else:
+ async_instance.settle_promises(self)
+
+ def _ensure_possible_rejection_handled(self):
+ # type: () -> None
+ # self._rejection_is_unhandled = True
+ # async_instance.invoke_later(self._notify_unhandled_rejection, self)
+ pass
+
+ def _reject_callback(self, reason, synchronous=False, traceback=None):
+ # type: (Exception, bool, Optional[TracebackType]) -> None
+ assert isinstance(
+ reason, Exception
+ ), "A promise was rejected with a non-error: {}".format(reason)
+ # trace = ensure_error_object(reason)
+ # has_stack = trace is reason
+ # self._attach_extratrace(trace, synchronous and has_stack)
+ self._reject(reason, traceback)
+
+ def _clear_callback_data_index_at(self, index):
+ # type: (int) -> None
+ assert not self._is_following
+ assert index > 0
+ base = index * CALLBACK_SIZE - CALLBACK_SIZE
+ self._handlers[base + CALLBACK_PROMISE_OFFSET] = None
+ self._handlers[base + CALLBACK_FULFILL_OFFSET] = None
+ self._handlers[base + CALLBACK_REJECT_OFFSET] = None
+
+ def _fulfill_promises(self, length, value):
+ # type: (int, T) -> None
+ for i in range(1, length):
+ handler = self._fulfillment_handler_at(i)
+ promise = self._promise_at(i)
+ self._clear_callback_data_index_at(i)
+ self._settle_promise(promise, handler, value, None)
+
+ def _reject_promises(self, length, reason):
+ # type: (int, Exception) -> None
+ for i in range(1, length):
+ handler = self._rejection_handler_at(i)
+ promise = self._promise_at(i)
+ self._clear_callback_data_index_at(i)
+ self._settle_promise(promise, handler, reason, None)
+
+ def _settle_promise(
+ self,
+ promise, # type: Optional[Promise]
+ handler, # type: Optional[Callable]
+ value, # type: Union[T, Exception]
+ traceback, # type: Optional[TracebackType]
+ ):
+ # type: (...) -> None
+ assert not self._is_following
+ is_promise = isinstance(promise, self.__class__)
+ async_guaranteed = self._is_async_guaranteed
+ if callable(handler):
+ if not is_promise:
+ handler(value) # , promise
+ else:
+ if async_guaranteed:
+ promise._is_async_guaranteed = True # type: ignore
+ self._settle_promise_from_handler( # type: ignore
+ handler, value, promise # type: ignore
+ ) # type: ignore
+ elif is_promise:
+ if async_guaranteed:
+ promise._is_async_guaranteed = True # type: ignore
+ if self._state == STATE_FULFILLED:
+ promise._fulfill(value) # type: ignore
+ else:
+ promise._reject(value, self._traceback) # type: ignore
+
+ def _settle_promise0(
+ self,
+ handler, # type: Optional[Callable]
+ value, # type: Any
+ traceback, # type: Optional[TracebackType]
+ ):
+ # type: (...) -> None
+ promise = self._promise0
+ self._promise0 = None
+ self._settle_promise(promise, handler, value, traceback) # type: ignore
+
+ def _settle_promise_from_handler(self, handler, value, promise):
+ # type: (Callable, Any, Promise) -> None
+ value, error_with_tb = try_catch(handler, value) # , promise
+
+ if error_with_tb:
+ error, tb = error_with_tb
+ promise._reject_callback(error, False, tb)
+ else:
+ promise._resolve_callback(value)
+
+ def _promise_at(self, index):
+ # type: (int) -> Optional[Promise]
+ assert index > 0
+ assert not self._is_following
+ return self._handlers.get( # type: ignore
+ index * CALLBACK_SIZE - CALLBACK_SIZE + CALLBACK_PROMISE_OFFSET
+ )
+
+ def _fulfillment_handler_at(self, index):
+ # type: (int) -> Optional[Callable]
+ assert not self._is_following
+ assert index > 0
+ return self._handlers.get( # type: ignore
+ index * CALLBACK_SIZE - CALLBACK_SIZE + CALLBACK_FULFILL_OFFSET
+ )
+
+ def _rejection_handler_at(self, index):
+ # type: (int) -> Optional[Callable]
+ assert not self._is_following
+ assert index > 0
+ return self._handlers.get( # type: ignore
+ index * CALLBACK_SIZE - CALLBACK_SIZE + CALLBACK_REJECT_OFFSET
+ )
+
+ def _migrate_callback0(self, follower):
+ # type: (Promise) -> None
+ self._add_callbacks(
+ follower._fulfillment_handler0,
+ follower._rejection_handler0,
+ follower._promise0,
+ )
+
+ def _migrate_callback_at(self, follower, index):
+ self._add_callbacks(
+ follower._fulfillment_handler_at(index),
+ follower._rejection_handler_at(index),
+ follower._promise_at(index),
+ )
+
+ def _add_callbacks(
+ self,
+ fulfill, # type: Optional[Callable]
+ reject, # type: Optional[Callable]
+ promise, # type: Optional[Promise]
+ ):
+ # type: (...) -> int
+ assert not self._is_following
+
+ if self._handlers is None:
+ self._handlers = {}
+
+ index = self._length
+ if index > MAX_LENGTH - CALLBACK_SIZE:
+ index = 0
+ self._length = 0
+
+ if index == 0:
+ assert not self._promise0
+ assert not self._fulfillment_handler0
+ assert not self._rejection_handler0
+
+ self._promise0 = promise
+ if callable(fulfill):
+ self._fulfillment_handler0 = fulfill
+ if callable(reject):
+ self._rejection_handler0 = reject
+
+ else:
+ base = index * CALLBACK_SIZE - CALLBACK_SIZE
+
+ assert (base + CALLBACK_PROMISE_OFFSET) not in self._handlers
+ assert (base + CALLBACK_FULFILL_OFFSET) not in self._handlers
+ assert (base + CALLBACK_REJECT_OFFSET) not in self._handlers
+
+ self._handlers[base + CALLBACK_PROMISE_OFFSET] = promise
+ if callable(fulfill):
+ self._handlers[base + CALLBACK_FULFILL_OFFSET] = fulfill
+ if callable(reject):
+ self._handlers[base + CALLBACK_REJECT_OFFSET] = reject
+
+ self._length = index + 1
+ return index
+
+ def _target(self):
+ # type: () -> Promise
+ ret = self
+ while ret._is_following:
+ ret = ret._followee()
+ return ret
+
+ def _followee(self):
+ # type: () -> Promise
+ assert self._is_following
+ assert isinstance(self._rejection_handler0, Promise)
+ return self._rejection_handler0
+
+ def _set_followee(self, promise):
+ # type: (Promise) -> None
+ assert self._is_following
+ assert not isinstance(self._rejection_handler0, Promise)
+ self._rejection_handler0 = promise
+
+ def _settle_promises(self):
+ # type: () -> None
+ length = self._length
+ if length > 0:
+ if self._state == STATE_REJECTED:
+ reason = self._fulfillment_handler0
+ traceback = self._traceback
+ self._settle_promise0(self._rejection_handler0, reason, traceback)
+ self._reject_promises(length, reason)
+ else:
+ value = self._rejection_handler0
+ self._settle_promise0(self._fulfillment_handler0, value, None)
+ self._fulfill_promises(length, value)
+
+ self._length = 0
+
+ def _resolve_from_executor(self, executor):
+ # type: (Callable[[Callable[[T], None], Callable[[Exception], None]], None]) -> None
+ # self._capture_stacktrace()
+ synchronous = True
+
+ def resolve(value):
+ # type: (T) -> None
+ self._resolve_callback(value)
+
+ def reject(reason, traceback=None):
+ # type: (Exception, TracebackType) -> None
+ self._reject_callback(reason, synchronous, traceback)
+
+ error = None
+ traceback = None
+ try:
+ executor(resolve, reject)
+ except Exception as e:
+ traceback = exc_info()[2]
+ error = e
+
+ synchronous = False
+
+ if error is not None:
+ self._reject_callback(error, True, traceback)
+
+ @classmethod
+ def wait(cls, promise, timeout=None):
+ # type: (Promise, Optional[float]) -> None
+ async_instance.wait(promise, timeout)
+
+ def _wait(self, timeout=None):
+ # type: (Optional[float]) -> None
+ self.wait(self, timeout)
+
+ def get(self, timeout=None):
+ # type: (Optional[float]) -> T
+ target = self._target()
+ self._wait(timeout or DEFAULT_TIMEOUT)
+ return self._target_settled_value(_raise=True)
+
+ def _target_settled_value(self, _raise=False):
+ # type: (bool) -> Any
+ return self._target()._settled_value(_raise)
+
+ _value = _reason = _target_settled_value
+ value = reason = property(_target_settled_value)
+
+ def __repr__(self):
+ # type: () -> str
+ hex_id = hex(id(self))
+ if self._is_following:
+ return "<Promise at {} following {}>".format(hex_id, self._target())
+ state = self._state
+ if state == STATE_PENDING:
+ return "<Promise at {} pending>".format(hex_id)
+ elif state == STATE_FULFILLED:
+ return "<Promise at {} fulfilled with {}>".format(
+ hex_id, repr(self._rejection_handler0)
+ )
+ elif state == STATE_REJECTED:
+ return "<Promise at {} rejected with {}>".format(
+ hex_id, repr(self._fulfillment_handler0)
+ )
+
+ return "<Promise unknown>"
+
+ @property
+ def is_pending(self):
+ # type: (Promise) -> bool
+ """Indicate whether the Promise is still pending. Could be wrong the moment the function returns."""
+ return self._target()._state == STATE_PENDING
+
+ @property
+ def is_fulfilled(self):
+ # type: (Promise) -> bool
+ """Indicate whether the Promise has been fulfilled. Could be wrong the moment the function returns."""
+ return self._target()._state == STATE_FULFILLED
+
+ @property
+ def is_rejected(self):
+ # type: (Promise) -> bool
+ """Indicate whether the Promise has been rejected. Could be wrong the moment the function returns."""
+ return self._target()._state == STATE_REJECTED
+
+ def catch(self, on_rejection):
+ # type: (Promise, Callable[[Exception], Any]) -> Promise
+ """
+ This method returns a Promise and deals with rejected cases only.
+ It behaves the same as calling Promise.then(None, on_rejection).
+ """
+ return self.then(None, on_rejection)
+
+ def _then(
+ self,
+ did_fulfill=None, # type: Optional[Callable[[T], S]]
+ did_reject=None, # type: Optional[Callable[[Exception], S]]
+ ):
+ # type: (...) -> Promise[S]
+ promise = self.__class__() # type: Promise
+ target = self._target()
+
+ state = target._state
+ if state == STATE_PENDING:
+ target._add_callbacks(did_fulfill, did_reject, promise)
+ else:
+ traceback = None
+ if state == STATE_FULFILLED:
+ value = target._rejection_handler0
+ handler = did_fulfill
+ elif state == STATE_REJECTED:
+ value = target._fulfillment_handler0
+ traceback = target._traceback
+ handler = did_reject # type: ignore
+ # target._rejection_is_unhandled = False
+ async_instance.invoke(
+ partial(target._settle_promise, promise, handler, value, traceback),
+ promise.scheduler
+ # target._settle_promise instead?
+ # settler,
+ # target,
+ )
+
+ return promise
+
+ fulfill = _resolve_callback
+ do_resolve = _resolve_callback
+ do_reject = _reject_callback
+
+ def then(self, did_fulfill=None, did_reject=None):
+ # type: (Promise, Callable[[T], S], Optional[Callable[[Exception], S]]) -> Promise[S]
+ """
+ This method takes two optional arguments. The first argument
+ is used if the "self promise" is fulfilled and the other is
+ used if the "self promise" is rejected. In either case, this
+ method returns another promise that effectively represents
+ the result of either the first of the second argument (in the
+ case that the "self promise" is fulfilled or rejected,
+ respectively).
+ Each argument can be either:
+ * None - Meaning no action is taken
+ * A function - which will be called with either the value
+ of the "self promise" or the reason for rejection of
+ the "self promise". The function may return:
+ * A value - which will be used to fulfill the promise
+ returned by this method.
+ * A promise - which, when fulfilled or rejected, will
+ cascade its value or reason to the promise returned
+ by this method.
+ * A value - which will be assigned as either the value
+ or the reason for the promise returned by this method
+ when the "self promise" is either fulfilled or rejected,
+ respectively.
+ :type success: (Any) -> object
+ :type failure: (Any) -> object
+ :rtype : Promise
+ """
+ return self._then(did_fulfill, did_reject)
+
+ def done(self, did_fulfill=None, did_reject=None):
+ # type: (Optional[Callable], Optional[Callable]) -> None
+ promise = self._then(did_fulfill, did_reject)
+ promise._is_final = True
+
+ def done_all(self, handlers=None):
+ # type: (Promise, Optional[List[Union[Dict[str, Optional[Callable]], Tuple[Callable, Callable], Callable]]]) -> None
+ """
+ :type handlers: list[(Any) -> object] | list[((Any) -> object, (Any) -> object)]
+ """
+ if not handlers:
+ return
+
+ for handler in handlers:
+ if isinstance(handler, tuple):
+ s, f = handler
+ self.done(s, f)
+ elif isinstance(handler, dict):
+ s = handler.get("success") # type: ignore
+ f = handler.get("failure") # type: ignore
+
+ self.done(s, f)
+ else:
+ self.done(handler)
+
+ def then_all(self, handlers=None):
+ # type: (Promise, List[Callable]) -> List[Promise]
+ """
+ Utility function which calls 'then' for each handler provided. Handler can either
+ be a function in which case it is used as success handler, or a tuple containing
+ the success and the failure handler, where each of them could be None.
+ :type handlers: list[(Any) -> object] | list[((Any) -> object, (Any) -> object)]
+ :param handlers
+ :rtype : list[Promise]
+ """
+ if not handlers:
+ return []
+
+ promises = [] # type: List[Promise]
+
+ for handler in handlers:
+ if isinstance(handler, tuple):
+ s, f = handler
+
+ promises.append(self.then(s, f))
+ elif isinstance(handler, dict):
+ s = handler.get("success")
+ f = handler.get("failure")
+
+ promises.append(self.then(s, f))
+ else:
+ promises.append(self.then(handler))
+
+ return promises
+
+ @classmethod
+ def _try_convert_to_promise(cls, obj):
+ # type: (Any) -> Promise
+ _type = obj.__class__
+ if issubclass(_type, Promise):
+ if cls is not Promise:
+ return cls(obj.then, obj._scheduler)
+ return obj
+
+ if iscoroutine(obj): # type: ignore
+ obj = ensure_future(obj) # type: ignore
+ _type = obj.__class__
+
+ if is_future_like(_type):
+
+ def executor(resolve, reject):
+ # type: (Callable, Callable) -> None
+ if obj.done():
+ _process_future_result(resolve, reject)(obj)
+ else:
+ obj.add_done_callback(_process_future_result(resolve, reject))
+ # _process_future_result(resolve, reject)(obj)
+
+ promise = cls(executor) # type: Promise
+ promise._future = obj
+ return promise
+
+ return obj
+
+ @classmethod
+ def reject(cls, reason):
+ # type: (Exception) -> Promise
+ ret = cls() # type: Promise
+ # ret._capture_stacktrace();
+ # ret._rejectCallback(reason, true);
+ ret._reject_callback(reason, True)
+ return ret
+
+ rejected = reject
+
+ @classmethod
+ def resolve(cls, obj):
+ # type: (T) -> Promise[T]
+ if not cls.is_thenable(obj):
+ ret = cls() # type: Promise
+ # ret._capture_stacktrace()
+ ret._state = STATE_FULFILLED
+ ret._rejection_handler0 = obj
+ return ret
+
+ return cls._try_convert_to_promise(obj)
+
+ cast = resolve
+ fulfilled = cast
+
+ @classmethod
+ def promisify(cls, f):
+ # type: (Callable) -> Callable[..., Promise]
+ if not callable(f):
+ warn(
+ "Promise.promisify is now a function decorator, please use Promise.resolve instead."
+ )
+ return cls.resolve(f)
+
+ @wraps(f)
+ def wrapper(*args, **kwargs):
+ # type: (*Any, **Any) -> Promise
+ def executor(resolve, reject):
+ # type: (Callable, Callable) -> Optional[Any]
+ return resolve(f(*args, **kwargs))
+
+ return cls(executor)
+
+ return wrapper
+
+ _safe_resolved_promise = None # type: Promise
+
+ @classmethod
+ def safe(cls, fn):
+ # type: (Callable) -> Callable
+ from functools import wraps
+
+ if not cls._safe_resolved_promise:
+ cls._safe_resolved_promise = Promise.resolve(None)
+
+ @wraps(fn)
+ def wrapper(*args, **kwargs):
+ # type: (*Any, **Any) -> Promise
+ return cls._safe_resolved_promise.then(lambda v: fn(*args, **kwargs))
+
+ return wrapper
+
+ @classmethod
+ def all(cls, promises):
+ # type: (Any) -> Promise
+ return PromiseList(promises, promise_class=cls).promise
+
+ @classmethod
+ def for_dict(cls, m):
+ # type: (Dict[Hashable, Promise[S]]) -> Promise[Dict[Hashable, S]]
+ """
+ A special function that takes a dictionary of promises
+ and turns them into a promise for a dictionary of values.
+ In other words, this turns an dictionary of promises for values
+ into a promise for a dictionary of values.
+ """
+ dict_type = type(m) # type: Type[Dict]
+
+ if not m:
+ return cls.resolve(dict_type()) # type: ignore
+
+ def handle_success(resolved_values):
+ # type: (List[S]) -> Dict[Hashable, S]
+ return dict_type(zip(m.keys(), resolved_values))
+
+ return cls.all(m.values()).then(handle_success)
+
+ @classmethod
+ def is_thenable(cls, obj):
+ # type: (Any) -> bool
+ """
+ A utility function to determine if the specified
+ object is a promise using "duck typing".
+ """
+ _type = obj.__class__
+ if obj is None or _type in BASE_TYPES:
+ return False
+
+ return (
+ issubclass(_type, Promise)
+ or iscoroutine(obj) # type: ignore
+ or is_future_like(_type)
+ )
+
+
+_type_done_callbacks = WeakKeyDictionary() # type: MutableMapping[type, bool]
+
+
+def is_future_like(_type):
+ # type: (type) -> bool
+ if _type not in _type_done_callbacks:
+ _type_done_callbacks[_type] = callable(
+ getattr(_type, "add_done_callback", None)
+ )
+ return _type_done_callbacks[_type]
+
+
+promisify = Promise.promisify
+promise_for_dict = Promise.for_dict
+is_thenable = Promise.is_thenable
+
+
+def _process_future_result(resolve, reject):
+ # type: (Callable, Callable) -> Callable
+ def handle_future_result(future):
+ # type: (Any) -> None
+ try:
+ resolve(future.result())
+ except Exception as e:
+ tb = exc_info()[2]
+ reject(e, tb)
+
+ return handle_future_result
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/promise_list.py b/wandb/vendor/promise-2.3.0/wandb_promise/promise_list.py
new file mode 100644
index 0000000000000000000000000000000000000000..071c77fe01fed1f9ab9da0201a0bb654356488b6
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/wandb_promise/promise_list.py
@@ -0,0 +1,151 @@
+from functools import partial
+try:
+ from collections.abc import Iterable
+except ImportError:
+ from collections import Iterable
+
+if False:
+ from .promise import Promise
+ from typing import (
+ Any,
+ Optional,
+ Tuple,
+ Union,
+ List,
+ Type,
+ Collection,
+ ) # flake8: noqa
+
+
+class PromiseList(object):
+
+ __slots__ = ("_values", "_length", "_total_resolved", "promise", "_promise_class")
+
+ def __init__(self, values, promise_class):
+ # type: (Union[Collection, Promise[Collection]], Type[Promise]) -> None
+ self._promise_class = promise_class
+ self.promise = self._promise_class()
+
+ self._length = 0
+ self._total_resolved = 0
+ self._values = None # type: Optional[Collection]
+ Promise = self._promise_class
+ if Promise.is_thenable(values):
+ values_as_promise = Promise._try_convert_to_promise(
+ values
+ )._target() # type: ignore
+ self._init_promise(values_as_promise)
+ else:
+ self._init(values) # type: ignore
+
+ def __len__(self):
+ # type: () -> int
+ return self._length
+
+ def _init_promise(self, values):
+ # type: (Promise[Collection]) -> None
+ if values.is_fulfilled:
+ values = values._value()
+ elif values.is_rejected:
+ self._reject(values._reason())
+ return
+
+ self.promise._is_async_guaranteed = True
+ values._then(self._init, self._reject)
+ return
+
+ def _init(self, values):
+ # type: (Collection) -> None
+ self._values = values
+ if not isinstance(values, Iterable):
+ err = Exception(
+ "PromiseList requires an iterable. Received {}.".format(repr(values))
+ )
+ self.promise._reject_callback(err, False)
+ return
+
+ if not values:
+ self._resolve([])
+ return
+
+ self._iterate(values)
+ return
+
+ def _iterate(self, values):
+ # type: (Collection[Any]) -> None
+ Promise = self._promise_class
+ is_resolved = False
+
+ self._length = len(values)
+ self._values = [None] * self._length
+
+ result = self.promise
+
+ for i, val in enumerate(values):
+ if Promise.is_thenable(val):
+ maybe_promise = Promise._try_convert_to_promise(val)._target()
+ # if is_resolved:
+ # # maybe_promise.suppressUnhandledRejections
+ # pass
+ if maybe_promise.is_pending:
+ maybe_promise._add_callbacks(
+ partial(self._promise_fulfilled, i=i),
+ self._promise_rejected,
+ None,
+ )
+ self._values[i] = maybe_promise
+ elif maybe_promise.is_fulfilled:
+ is_resolved = self._promise_fulfilled(maybe_promise._value(), i)
+ elif maybe_promise.is_rejected:
+ is_resolved = self._promise_rejected(maybe_promise._reason())
+
+ else:
+ is_resolved = self._promise_fulfilled(val, i)
+
+ if is_resolved:
+ break
+
+ if not is_resolved:
+ result._is_async_guaranteed = True
+
+ def _promise_fulfilled(self, value, i):
+ # type: (Any, int) -> bool
+ if self.is_resolved:
+ return False
+ # assert not self.is_resolved
+ # assert isinstance(self._values, Iterable)
+ # assert isinstance(i, int)
+ self._values[i] = value # type: ignore
+ self._total_resolved += 1
+ if self._total_resolved >= self._length:
+ self._resolve(self._values) # type: ignore
+ return True
+ return False
+
+ def _promise_rejected(self, reason):
+ # type: (Exception) -> bool
+ if self.is_resolved:
+ return False
+ # assert not self.is_resolved
+ # assert isinstance(self._values, Iterable)
+ self._total_resolved += 1
+ self._reject(reason)
+ return True
+
+ @property
+ def is_resolved(self):
+ # type: () -> bool
+ return self._values is None
+
+ def _resolve(self, value):
+ # type: (Collection[Any]) -> None
+ assert not self.is_resolved
+ assert not isinstance(value, self._promise_class)
+ self._values = None
+ self.promise._fulfill(value)
+
+ def _reject(self, reason):
+ # type: (Exception) -> None
+ assert not self.is_resolved
+ self._values = None
+ self.promise._reject_callback(reason, False)
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/py.typed b/wandb/vendor/promise-2.3.0/wandb_promise/py.typed
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/pyutils/__init__.py b/wandb/vendor/promise-2.3.0/wandb_promise/pyutils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/pyutils/version.py b/wandb/vendor/promise-2.3.0/wandb_promise/pyutils/version.py
new file mode 100644
index 0000000000000000000000000000000000000000..47d439145ed903c0cf8f207c96cf2300917860a4
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/wandb_promise/pyutils/version.py
@@ -0,0 +1,83 @@
+from __future__ import unicode_literals
+
+import datetime
+import os
+import subprocess
+
+
+def get_version(version=None):
+ "Returns a PEP 440-compliant version number from VERSION."
+ version = get_complete_version(version)
+
+ # Now build the two parts of the version number:
+ # main = X.Y[.Z]
+ # sub = .devN - for pre-alpha releases
+ # | {a|b|rc}N - for alpha, beta, and rc releases
+
+ main = get_main_version(version)
+
+ sub = ""
+ if version[3] == "alpha" and version[4] == 0:
+ git_changeset = get_git_changeset()
+ if git_changeset:
+ sub = ".dev%s" % git_changeset
+ else:
+ sub = ".dev"
+ elif version[3] != "final":
+ mapping = {"alpha": "a", "beta": "b", "rc": "rc"}
+ sub = mapping[version[3]] + str(version[4])
+
+ return str(main + sub)
+
+
+def get_main_version(version=None):
+ "Returns main version (X.Y[.Z]) from VERSION."
+ version = get_complete_version(version)
+ parts = 2 if version[2] == 0 else 3
+ return ".".join(str(x) for x in version[:parts])
+
+
+def get_complete_version(version=None):
+ """Returns a tuple of the promise version. If version argument is non-empty,
+ then checks for correctness of the tuple provided.
+ """
+ if version is None:
+ from promise import VERSION
+
+ return VERSION
+ else:
+ assert len(version) == 5
+ assert version[3] in ("alpha", "beta", "rc", "final")
+
+ return version
+
+
+def get_docs_version(version=None):
+ version = get_complete_version(version)
+ if version[3] != "final":
+ return "dev"
+ else:
+ return "%d.%d" % version[:2]
+
+
+def get_git_changeset():
+ """Returns a numeric identifier of the latest git changeset.
+ The result is the UTC timestamp of the changeset in YYYYMMDDHHMMSS format.
+ This value isn't guaranteed to be unique, but collisions are very unlikely,
+ so it's sufficient for generating the development version numbers.
+ """
+ repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ try:
+ git_log = subprocess.Popen(
+ "git log --pretty=format:%ct --quiet -1 HEAD",
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ shell=True,
+ cwd=repo_dir,
+ universal_newlines=True,
+ )
+ timestamp = git_log.communicate()[0]
+ timestamp = datetime.datetime.utcfromtimestamp(int(timestamp))
+ except Exception:
+ return None
+ return timestamp.strftime("%Y%m%d%H%M%S")
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/schedulers/__init__.py b/wandb/vendor/promise-2.3.0/wandb_promise/schedulers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/schedulers/asyncio.py b/wandb/vendor/promise-2.3.0/wandb_promise/schedulers/asyncio.py
new file mode 100644
index 0000000000000000000000000000000000000000..34285fa5e250e9934bcf752ec670b77af55aa5ec
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/wandb_promise/schedulers/asyncio.py
@@ -0,0 +1,22 @@
+from __future__ import absolute_import
+
+from asyncio import get_event_loop, Event
+
+
+class AsyncioScheduler(object):
+ def __init__(self, loop=None):
+ self.loop = loop or get_event_loop()
+
+ def call(self, fn):
+ self.loop.call_soon(fn)
+
+ def wait(self, promise, timeout=None):
+ e = Event()
+
+ def on_resolve_or_reject(_):
+ e.set()
+
+ promise._then(on_resolve_or_reject, on_resolve_or_reject)
+
+ # We can't use the timeout in Asyncio event
+ e.wait()
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/schedulers/gevent.py b/wandb/vendor/promise-2.3.0/wandb_promise/schedulers/gevent.py
new file mode 100644
index 0000000000000000000000000000000000000000..6b84c4c8b579f9c6c663161ed3e7a5c6d6d83dac
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/wandb_promise/schedulers/gevent.py
@@ -0,0 +1,21 @@
+from __future__ import absolute_import
+
+from gevent.event import Event # type: ignore
+import gevent # type: ignore
+
+
+class GeventScheduler(object):
+ def call(self, fn):
+ # print fn
+ gevent.spawn(fn)
+
+ def wait(self, promise, timeout=None):
+ e = Event()
+
+ def on_resolve_or_reject(_):
+ e.set()
+
+ promise._then(on_resolve_or_reject, on_resolve_or_reject)
+ waited = e.wait(timeout)
+ if not waited:
+ raise Exception("Timeout")
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/schedulers/immediate.py b/wandb/vendor/promise-2.3.0/wandb_promise/schedulers/immediate.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e1dd4c11a1fb198e63e90eaa7f784544effebb8
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/wandb_promise/schedulers/immediate.py
@@ -0,0 +1,27 @@
+from threading import Event
+
+if False:
+ from ..promise import Promise
+ from typing import Callable, Any, Optional # flake8: noqa
+
+
+class ImmediateScheduler(object):
+ def call(self, fn):
+ # type: (Callable) -> None
+ try:
+ fn()
+ except:
+ pass
+
+ def wait(self, promise, timeout=None):
+ # type: (Promise, Optional[float]) -> None
+ e = Event()
+
+ def on_resolve_or_reject(_):
+ # type: (Any) -> None
+ e.set()
+
+ promise._then(on_resolve_or_reject, on_resolve_or_reject)
+ waited = e.wait(timeout)
+ if not waited:
+ raise Exception("Timeout")
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/schedulers/thread.py b/wandb/vendor/promise-2.3.0/wandb_promise/schedulers/thread.py
new file mode 100644
index 0000000000000000000000000000000000000000..a83f9a0a41f95fc0ed2066cee42d224f1aab8100
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/wandb_promise/schedulers/thread.py
@@ -0,0 +1,18 @@
+from threading import Thread, Event
+
+
+class ThreadScheduler(object):
+ def call(self, fn):
+ thread = Thread(target=fn)
+ thread.start()
+
+ def wait(self, promise, timeout=None):
+ e = Event()
+
+ def on_resolve_or_reject(_):
+ e.set()
+
+ promise._then(on_resolve_or_reject, on_resolve_or_reject)
+ waited = e.wait(timeout)
+ if not waited:
+ raise Exception("Timeout")
diff --git a/wandb/vendor/promise-2.3.0/wandb_promise/utils.py b/wandb/vendor/promise-2.3.0/wandb_promise/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..ca695776f5b1913993dd958097dc23c6f8f4e9c2
--- /dev/null
+++ b/wandb/vendor/promise-2.3.0/wandb_promise/utils.py
@@ -0,0 +1,56 @@
+import functools
+import inspect
+import types
+import warnings
+import sys
+
+
+def warn(msg):
+ # type: (str) -> None
+ warnings.simplefilter("always", DeprecationWarning) # turn off filter
+ warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
+ warnings.simplefilter("default", DeprecationWarning) # reset filter
+
+
+class deprecated(object):
+ def __init__(self, reason, name=None):
+ if inspect.isclass(reason) or inspect.isfunction(reason):
+ raise TypeError("Reason for deprecation must be supplied")
+ self.reason = reason
+ self.name = name
+
+ def __call__(self, cls_or_func):
+ if inspect.isfunction(cls_or_func):
+ fmt = "Call to deprecated function or method {name} ({reason})."
+
+ elif inspect.isclass(cls_or_func):
+ fmt = "Call to deprecated class {name} ({reason})."
+
+ else:
+ raise TypeError(type(cls_or_func))
+
+ msg = fmt.format(name=self.name or cls_or_func.__name__, reason=self.reason)
+
+ @functools.wraps(cls_or_func)
+ def new_func(*args, **kwargs):
+ warn(msg)
+ return cls_or_func(*args, **kwargs)
+
+ return new_func
+
+
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+ string_types = (str,) # type: tuple
+ integer_types = (int,) # type: tuple
+ class_types = (type,) # type: tuple
+ text_type = str
+ binary_type = bytes
+else:
+ string_types = (basestring,) # type: tuple
+ integer_types = (int, long) # type: tuple
+ class_types = (type, types.ClassType) # type: tuple
+ text_type = unicode
+ binary_type = str
diff --git a/wandb/vendor/pygments/__init__.py b/wandb/vendor/pygments/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..394a85f2a4c1c7f84b47f2e17420c5a1ab761b00
--- /dev/null
+++ b/wandb/vendor/pygments/__init__.py
@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+"""
+ Pygments
+ ~~~~~~~~
+
+ Pygments is a syntax highlighting package written in Python.
+
+ It is a generic syntax highlighter for general use in all kinds of software
+ such as forum systems, wikis or other applications that need to prettify
+ source code. Highlights are:
+
+ * a wide range of common languages and markup formats is supported
+ * special attention is paid to details, increasing quality by a fair amount
+ * support for new languages and formats are added easily
+ * a number of output formats, presently HTML, LaTeX, RTF, SVG, all image
+ formats that PIL supports, and ANSI sequences
+ * it is usable as a command-line tool and as a library
+ * ... and it highlights even Brainfuck!
+
+ The `Pygments tip`_ is installable with ``easy_install Pygments==dev``.
+
+ .. _Pygments tip:
+ http://bitbucket.org/birkenfeld/pygments-main/get/tip.zip#egg=Pygments-dev
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+import sys
+
+from pygments.util import StringIO, BytesIO
+
+__version__ = '2.2.0'
+__docformat__ = 'restructuredtext'
+
+__all__ = ['lex', 'format', 'highlight']
+
+
+def lex(code, lexer):
+ """
+ Lex ``code`` with ``lexer`` and return an iterable of tokens.
+ """
+ try:
+ return lexer.get_tokens(code)
+ except TypeError as err:
+ if (isinstance(err.args[0], str) and
+ ('unbound method get_tokens' in err.args[0] or
+ 'missing 1 required positional argument' in err.args[0])):
+ raise TypeError('lex() argument must be a lexer instance, '
+ 'not a class')
+ raise
+
+
+def format(tokens, formatter, outfile=None): # pylint: disable=redefined-builtin
+ """
+ Format a tokenlist ``tokens`` with the formatter ``formatter``.
+
+ If ``outfile`` is given and a valid file object (an object
+ with a ``write`` method), the result will be written to it, otherwise
+ it is returned as a string.
+ """
+ try:
+ if not outfile:
+ realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO()
+ formatter.format(tokens, realoutfile)
+ return realoutfile.getvalue()
+ else:
+ formatter.format(tokens, outfile)
+ except TypeError as err:
+ if (isinstance(err.args[0], str) and
+ ('unbound method format' in err.args[0] or
+ 'missing 1 required positional argument' in err.args[0])):
+ raise TypeError('format() argument must be a formatter instance, '
+ 'not a class')
+ raise
+
+
+def highlight(code, lexer, formatter, outfile=None):
+ """
+ Lex ``code`` with ``lexer`` and format it with the formatter ``formatter``.
+
+ If ``outfile`` is given and a valid file object (an object
+ with a ``write`` method), the result will be written to it, otherwise
+ it is returned as a string.
+ """
+ return format(lex(code, lexer), formatter, outfile)
+
+
+if __name__ == '__main__': # pragma: no cover
+ from pygments.cmdline import main
+ sys.exit(main(sys.argv))
diff --git a/wandb/vendor/pygments/cmdline.py b/wandb/vendor/pygments/cmdline.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e1f39e2aa4c1ca05d2e8a5ea1700890460e24e1
--- /dev/null
+++ b/wandb/vendor/pygments/cmdline.py
@@ -0,0 +1,568 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.cmdline
+ ~~~~~~~~~~~~~~~~
+
+ Command line interface.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import print_function
+
+import sys
+import getopt
+from textwrap import dedent
+
+from pygments import __version__, highlight
+from pygments.util import ClassNotFound, OptionError, docstring_headline, \
+ guess_decode, guess_decode_from_terminal, terminal_encoding
+from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \
+ load_lexer_from_file, get_lexer_for_filename, find_lexer_class_for_filename
+from pygments.lexers.special import TextLexer
+from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter
+from pygments.formatters import get_all_formatters, get_formatter_by_name, \
+ load_formatter_from_file, get_formatter_for_filename, find_formatter_class
+from pygments.formatters.terminal import TerminalFormatter
+from pygments.filters import get_all_filters, find_filter_class
+from pygments.styles import get_all_styles, get_style_by_name
+
+
+USAGE = """\
+Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
+ [-O <options>] [-P <option=value>] [-s] [-v] [-x] [-o <outfile>] [<infile>]
+
+ %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
+ %s -L [<which> ...]
+ %s -N <filename>
+ %s -H <type> <name>
+ %s -h | -V
+
+Highlight the input file and write the result to <outfile>.
+
+If no input file is given, use stdin, if -o is not given, use stdout.
+
+If -s is passed, lexing will be done in "streaming" mode, reading and
+highlighting one line at a time. This will only work properly with
+lexers that have no constructs spanning multiple lines!
+
+<lexer> is a lexer name (query all lexer names with -L). If -l is not
+given, the lexer is guessed from the extension of the input file name
+(this obviously doesn't work if the input is stdin). If -g is passed,
+attempt to guess the lexer from the file contents, or pass through as
+plain text if this fails (this can work for stdin).
+
+Likewise, <formatter> is a formatter name, and will be guessed from
+the extension of the output file name. If no output file is given,
+the terminal formatter will be used by default.
+
+The additional option -x allows custom lexers and formatters to be
+loaded from a .py file relative to the current working directory. For
+example, ``-l ./customlexer.py -x``. By default, this option expects a
+file with a class named CustomLexer or CustomFormatter; you can also
+specify your own class name with a colon (``-l ./lexer.py:MyLexer``).
+Users should be very careful not to use this option with untrusted files,
+because it will import and run them.
+
+With the -O option, you can give the lexer and formatter a comma-
+separated list of options, e.g. ``-O bg=light,python=cool``.
+
+The -P option adds lexer and formatter options like the -O option, but
+you can only give one option per -P. That way, the option value may
+contain commas and equals signs, which it can't with -O, e.g.
+``-P "heading=Pygments, the Python highlighter".
+
+With the -F option, you can add filters to the token stream, you can
+give options in the same way as for -O after a colon (note: there must
+not be spaces around the colon).
+
+The -O, -P and -F options can be given multiple times.
+
+With the -S option, print out style definitions for style <style>
+for formatter <formatter>. The argument given by -a is formatter
+dependent.
+
+The -L option lists lexers, formatters, styles or filters -- set
+`which` to the thing you want to list (e.g. "styles"), or omit it to
+list everything.
+
+The -N option guesses and prints out a lexer name based solely on
+the given filename. It does not take input or highlight anything.
+If no specific lexer can be determined "text" is returned.
+
+The -H option prints detailed help for the object <name> of type <type>,
+where <type> is one of "lexer", "formatter" or "filter".
+
+The -s option processes lines one at a time until EOF, rather than
+waiting to process the entire file. This only works for stdin, and
+is intended for streaming input such as you get from 'tail -f'.
+Example usage: "tail -f sql.log | pygmentize -s -l sql"
+
+The -v option prints a detailed traceback on unhandled exceptions,
+which is useful for debugging and bug reports.
+
+The -h option prints this help.
+The -V option prints the package version.
+"""
+
+
+def _parse_options(o_strs):
+ opts = {}
+ if not o_strs:
+ return opts
+ for o_str in o_strs:
+ if not o_str.strip():
+ continue
+ o_args = o_str.split(',')
+ for o_arg in o_args:
+ o_arg = o_arg.strip()
+ try:
+ o_key, o_val = o_arg.split('=', 1)
+ o_key = o_key.strip()
+ o_val = o_val.strip()
+ except ValueError:
+ opts[o_arg] = True
+ else:
+ opts[o_key] = o_val
+ return opts
+
+
+def _parse_filters(f_strs):
+ filters = []
+ if not f_strs:
+ return filters
+ for f_str in f_strs:
+ if ':' in f_str:
+ fname, fopts = f_str.split(':', 1)
+ filters.append((fname, _parse_options([fopts])))
+ else:
+ filters.append((f_str, {}))
+ return filters
+
+
+def _print_help(what, name):
+ try:
+ if what == 'lexer':
+ cls = get_lexer_by_name(name)
+ print("Help on the %s lexer:" % cls.name)
+ print(dedent(cls.__doc__))
+ elif what == 'formatter':
+ cls = find_formatter_class(name)
+ print("Help on the %s formatter:" % cls.name)
+ print(dedent(cls.__doc__))
+ elif what == 'filter':
+ cls = find_filter_class(name)
+ print("Help on the %s filter:" % name)
+ print(dedent(cls.__doc__))
+ return 0
+ except (AttributeError, ValueError):
+ print("%s not found!" % what, file=sys.stderr)
+ return 1
+
+
+def _print_list(what):
+ if what == 'lexer':
+ print()
+ print("Lexers:")
+ print("~~~~~~~")
+
+ info = []
+ for fullname, names, exts, _ in get_all_lexers():
+ tup = (', '.join(names)+':', fullname,
+ exts and '(filenames ' + ', '.join(exts) + ')' or '')
+ info.append(tup)
+ info.sort()
+ for i in info:
+ print(('* %s\n %s %s') % i)
+
+ elif what == 'formatter':
+ print()
+ print("Formatters:")
+ print("~~~~~~~~~~~")
+
+ info = []
+ for cls in get_all_formatters():
+ doc = docstring_headline(cls)
+ tup = (', '.join(cls.aliases) + ':', doc, cls.filenames and
+ '(filenames ' + ', '.join(cls.filenames) + ')' or '')
+ info.append(tup)
+ info.sort()
+ for i in info:
+ print(('* %s\n %s %s') % i)
+
+ elif what == 'filter':
+ print()
+ print("Filters:")
+ print("~~~~~~~~")
+
+ for name in get_all_filters():
+ cls = find_filter_class(name)
+ print("* " + name + ':')
+ print(" %s" % docstring_headline(cls))
+
+ elif what == 'style':
+ print()
+ print("Styles:")
+ print("~~~~~~~")
+
+ for name in get_all_styles():
+ cls = get_style_by_name(name)
+ print("* " + name + ':')
+ print(" %s" % docstring_headline(cls))
+
+
+def main_inner(popts, args, usage):
+ opts = {}
+ O_opts = []
+ P_opts = []
+ F_opts = []
+ for opt, arg in popts:
+ if opt == '-O':
+ O_opts.append(arg)
+ elif opt == '-P':
+ P_opts.append(arg)
+ elif opt == '-F':
+ F_opts.append(arg)
+ opts[opt] = arg
+
+ if opts.pop('-h', None) is not None:
+ print(usage)
+ return 0
+
+ if opts.pop('-V', None) is not None:
+ print('Pygments version %s, (c) 2006-2017 by Georg Brandl.' % __version__)
+ return 0
+
+ # handle ``pygmentize -L``
+ L_opt = opts.pop('-L', None)
+ if L_opt is not None:
+ if opts:
+ print(usage, file=sys.stderr)
+ return 2
+
+ # print version
+ main(['', '-V'])
+ if not args:
+ args = ['lexer', 'formatter', 'filter', 'style']
+ for arg in args:
+ _print_list(arg.rstrip('s'))
+ return 0
+
+ # handle ``pygmentize -H``
+ H_opt = opts.pop('-H', None)
+ if H_opt is not None:
+ if opts or len(args) != 2:
+ print(usage, file=sys.stderr)
+ return 2
+
+ what, name = args # pylint: disable=unbalanced-tuple-unpacking
+ if what not in ('lexer', 'formatter', 'filter'):
+ print(usage, file=sys.stderr)
+ return 2
+
+ return _print_help(what, name)
+
+ # parse -O options
+ parsed_opts = _parse_options(O_opts)
+ opts.pop('-O', None)
+
+ # parse -P options
+ for p_opt in P_opts:
+ try:
+ name, value = p_opt.split('=', 1)
+ except ValueError:
+ parsed_opts[p_opt] = True
+ else:
+ parsed_opts[name] = value
+ opts.pop('-P', None)
+
+ # encodings
+ inencoding = parsed_opts.get('inencoding', parsed_opts.get('encoding'))
+ outencoding = parsed_opts.get('outencoding', parsed_opts.get('encoding'))
+
+ # handle ``pygmentize -N``
+ infn = opts.pop('-N', None)
+ if infn is not None:
+ lexer = find_lexer_class_for_filename(infn)
+ if lexer is None:
+ lexer = TextLexer
+
+ print(lexer.aliases[0])
+ return 0
+
+ # handle ``pygmentize -S``
+ S_opt = opts.pop('-S', None)
+ a_opt = opts.pop('-a', None)
+ if S_opt is not None:
+ f_opt = opts.pop('-f', None)
+ if not f_opt:
+ print(usage, file=sys.stderr)
+ return 2
+ if opts or args:
+ print(usage, file=sys.stderr)
+ return 2
+
+ try:
+ parsed_opts['style'] = S_opt
+ fmter = get_formatter_by_name(f_opt, **parsed_opts)
+ except ClassNotFound as err:
+ print(err, file=sys.stderr)
+ return 1
+
+ print(fmter.get_style_defs(a_opt or ''))
+ return 0
+
+ # if no -S is given, -a is not allowed
+ if a_opt is not None:
+ print(usage, file=sys.stderr)
+ return 2
+
+ # parse -F options
+ F_opts = _parse_filters(F_opts)
+ opts.pop('-F', None)
+
+ allow_custom_lexer_formatter = False
+ # -x: allow custom (eXternal) lexers and formatters
+ if opts.pop('-x', None) is not None:
+ allow_custom_lexer_formatter = True
+
+ # select lexer
+ lexer = None
+
+ # given by name?
+ lexername = opts.pop('-l', None)
+ if lexername:
+ # custom lexer, located relative to user's cwd
+ if allow_custom_lexer_formatter and '.py' in lexername:
+ try:
+ if ':' in lexername:
+ filename, name = lexername.rsplit(':', 1)
+ lexer = load_lexer_from_file(filename, name,
+ **parsed_opts)
+ else:
+ lexer = load_lexer_from_file(lexername, **parsed_opts)
+ except ClassNotFound as err:
+ print('Error:', err, file=sys.stderr)
+ return 1
+ else:
+ try:
+ lexer = get_lexer_by_name(lexername, **parsed_opts)
+ except (OptionError, ClassNotFound) as err:
+ print('Error:', err, file=sys.stderr)
+ return 1
+
+ # read input code
+ code = None
+
+ if args:
+ if len(args) > 1:
+ print(usage, file=sys.stderr)
+ return 2
+
+ if '-s' in opts:
+ print('Error: -s option not usable when input file specified',
+ file=sys.stderr)
+ return 2
+
+ infn = args[0]
+ try:
+ with open(infn, 'rb') as infp:
+ code = infp.read()
+ except Exception as err:
+ print('Error: cannot read infile:', err, file=sys.stderr)
+ return 1
+ if not inencoding:
+ code, inencoding = guess_decode(code)
+
+ # do we have to guess the lexer?
+ if not lexer:
+ try:
+ lexer = get_lexer_for_filename(infn, code, **parsed_opts)
+ except ClassNotFound as err:
+ if '-g' in opts:
+ try:
+ lexer = guess_lexer(code, **parsed_opts)
+ except ClassNotFound:
+ lexer = TextLexer(**parsed_opts)
+ else:
+ print('Error:', err, file=sys.stderr)
+ return 1
+ except OptionError as err:
+ print('Error:', err, file=sys.stderr)
+ return 1
+
+ elif '-s' not in opts: # treat stdin as full file (-s support is later)
+ # read code from terminal, always in binary mode since we want to
+ # decode ourselves and be tolerant with it
+ if sys.version_info > (3,):
+ # Python 3: we have to use .buffer to get a binary stream
+ code = sys.stdin.buffer.read()
+ else:
+ code = sys.stdin.read()
+ if not inencoding:
+ code, inencoding = guess_decode_from_terminal(code, sys.stdin)
+ # else the lexer will do the decoding
+ if not lexer:
+ try:
+ lexer = guess_lexer(code, **parsed_opts)
+ except ClassNotFound:
+ lexer = TextLexer(**parsed_opts)
+
+ else: # -s option needs a lexer with -l
+ if not lexer:
+ print('Error: when using -s a lexer has to be selected with -l',
+ file=sys.stderr)
+ return 2
+
+ # process filters
+ for fname, fopts in F_opts:
+ try:
+ lexer.add_filter(fname, **fopts)
+ except ClassNotFound as err:
+ print('Error:', err, file=sys.stderr)
+ return 1
+
+ # select formatter
+ outfn = opts.pop('-o', None)
+ fmter = opts.pop('-f', None)
+ if fmter:
+ # custom formatter, located relative to user's cwd
+ if allow_custom_lexer_formatter and '.py' in fmter:
+ try:
+ if ':' in fmter:
+ file, fmtername = fmter.rsplit(':', 1)
+ fmter = load_formatter_from_file(file, fmtername,
+ **parsed_opts)
+ else:
+ fmter = load_formatter_from_file(fmter, **parsed_opts)
+ except ClassNotFound as err:
+ print('Error:', err, file=sys.stderr)
+ return 1
+ else:
+ try:
+ fmter = get_formatter_by_name(fmter, **parsed_opts)
+ except (OptionError, ClassNotFound) as err:
+ print('Error:', err, file=sys.stderr)
+ return 1
+
+ if outfn:
+ if not fmter:
+ try:
+ fmter = get_formatter_for_filename(outfn, **parsed_opts)
+ except (OptionError, ClassNotFound) as err:
+ print('Error:', err, file=sys.stderr)
+ return 1
+ try:
+ outfile = open(outfn, 'wb')
+ except Exception as err:
+ print('Error: cannot open outfile:', err, file=sys.stderr)
+ return 1
+ else:
+ if not fmter:
+ fmter = TerminalFormatter(**parsed_opts)
+ if sys.version_info > (3,):
+ # Python 3: we have to use .buffer to get a binary stream
+ outfile = sys.stdout.buffer
+ else:
+ outfile = sys.stdout
+
+ # determine output encoding if not explicitly selected
+ if not outencoding:
+ if outfn:
+ # output file? use lexer encoding for now (can still be None)
+ fmter.encoding = inencoding
+ else:
+ # else use terminal encoding
+ fmter.encoding = terminal_encoding(sys.stdout)
+
+ # provide coloring under Windows, if possible
+ if not outfn and sys.platform in ('win32', 'cygwin') and \
+ fmter.name in ('Terminal', 'Terminal256'): # pragma: no cover
+ # unfortunately colorama doesn't support binary streams on Py3
+ if sys.version_info > (3,):
+ from pygments.util import UnclosingTextIOWrapper
+ outfile = UnclosingTextIOWrapper(outfile, encoding=fmter.encoding)
+ fmter.encoding = None
+ try:
+ import colorama.initialise
+ except ImportError:
+ pass
+ else:
+ outfile = colorama.initialise.wrap_stream(
+ outfile, convert=None, strip=None, autoreset=False, wrap=True)
+
+ # When using the LaTeX formatter and the option `escapeinside` is
+ # specified, we need a special lexer which collects escaped text
+ # before running the chosen language lexer.
+ escapeinside = parsed_opts.get('escapeinside', '')
+ if len(escapeinside) == 2 and isinstance(fmter, LatexFormatter):
+ left = escapeinside[0]
+ right = escapeinside[1]
+ lexer = LatexEmbeddedLexer(left, right, lexer)
+
+ # ... and do it!
+ if '-s' not in opts:
+ # process whole input as per normal...
+ highlight(code, lexer, fmter, outfile)
+ return 0
+ else:
+ # line by line processing of stdin (eg: for 'tail -f')...
+ try:
+ while 1:
+ if sys.version_info > (3,):
+ # Python 3: we have to use .buffer to get a binary stream
+ line = sys.stdin.buffer.readline()
+ else:
+ line = sys.stdin.readline()
+ if not line:
+ break
+ if not inencoding:
+ line = guess_decode_from_terminal(line, sys.stdin)[0]
+ highlight(line, lexer, fmter, outfile)
+ if hasattr(outfile, 'flush'):
+ outfile.flush()
+ return 0
+ except KeyboardInterrupt: # pragma: no cover
+ return 0
+
+
+def main(args=sys.argv):
+ """
+ Main command line entry point.
+ """
+ usage = USAGE % ((args[0],) * 6)
+
+ try:
+ popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:vhVHgsx")
+ except getopt.GetoptError:
+ print(usage, file=sys.stderr)
+ return 2
+
+ try:
+ return main_inner(popts, args, usage)
+ except Exception:
+ if '-v' in dict(popts):
+ print(file=sys.stderr)
+ print('*' * 65, file=sys.stderr)
+ print('An unhandled exception occurred while highlighting.',
+ file=sys.stderr)
+ print('Please report the whole traceback to the issue tracker at',
+ file=sys.stderr)
+ print('<https://bitbucket.org/birkenfeld/pygments-main/issues>.',
+ file=sys.stderr)
+ print('*' * 65, file=sys.stderr)
+ print(file=sys.stderr)
+ raise
+ import traceback
+ info = traceback.format_exception(*sys.exc_info())
+ msg = info[-1].strip()
+ if len(info) >= 3:
+ # extract relevant file and position info
+ msg += '\n (f%s)' % info[-2].split('\n')[0].strip()[1:]
+ print(file=sys.stderr)
+ print('*** Error while highlighting:', file=sys.stderr)
+ print(msg, file=sys.stderr)
+ print('*** If this is a bug you want to report, please rerun with -v.',
+ file=sys.stderr)
+ return 1
diff --git a/wandb/vendor/pygments/console.py b/wandb/vendor/pygments/console.py
new file mode 100644
index 0000000000000000000000000000000000000000..31b6839d1c689ef1cd0ad78da60a61945089bc12
--- /dev/null
+++ b/wandb/vendor/pygments/console.py
@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.console
+ ~~~~~~~~~~~~~~~~
+
+ Format colored console output.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+esc = "\x1b["
+
+codes = {}
+codes[""] = ""
+codes["reset"] = esc + "39;49;00m"
+
+codes["bold"] = esc + "01m"
+codes["faint"] = esc + "02m"
+codes["standout"] = esc + "03m"
+codes["underline"] = esc + "04m"
+codes["blink"] = esc + "05m"
+codes["overline"] = esc + "06m"
+
+dark_colors = ["black", "darkred", "darkgreen", "brown", "darkblue",
+ "purple", "teal", "lightgray"]
+light_colors = ["darkgray", "red", "green", "yellow", "blue",
+ "fuchsia", "turquoise", "white"]
+
+x = 30
+for d, l in zip(dark_colors, light_colors):
+ codes[d] = esc + "%im" % x
+ codes[l] = esc + "%i;01m" % x
+ x += 1
+
+del d, l, x
+
+codes["darkteal"] = codes["turquoise"]
+codes["darkyellow"] = codes["brown"]
+codes["fuscia"] = codes["fuchsia"]
+codes["white"] = codes["bold"]
+
+
+def reset_color():
+ return codes["reset"]
+
+
+def colorize(color_key, text):
+ return codes[color_key] + text + codes["reset"]
+
+
+def ansiformat(attr, text):
+ """
+ Format ``text`` with a color and/or some attributes::
+
+ color normal color
+ *color* bold color
+ _color_ underlined color
+ +color+ blinking color
+ """
+ result = []
+ if attr[:1] == attr[-1:] == '+':
+ result.append(codes['blink'])
+ attr = attr[1:-1]
+ if attr[:1] == attr[-1:] == '*':
+ result.append(codes['bold'])
+ attr = attr[1:-1]
+ if attr[:1] == attr[-1:] == '_':
+ result.append(codes['underline'])
+ attr = attr[1:-1]
+ result.append(codes[attr])
+ result.append(text)
+ result.append(codes['reset'])
+ return ''.join(result)
diff --git a/wandb/vendor/pygments/filter.py b/wandb/vendor/pygments/filter.py
new file mode 100644
index 0000000000000000000000000000000000000000..fcf48657356a71e1c2f55bfc7b447fd6d9862197
--- /dev/null
+++ b/wandb/vendor/pygments/filter.py
@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.filter
+ ~~~~~~~~~~~~~~~
+
+ Module that implements the default filter.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+
+def apply_filters(stream, filters, lexer=None):
+ """
+ Use this method to apply an iterable of filters to
+ a stream. If lexer is given it's forwarded to the
+ filter, otherwise the filter receives `None`.
+ """
+ def _apply(filter_, stream):
+ for token in filter_.filter(lexer, stream):
+ yield token
+ for filter_ in filters:
+ stream = _apply(filter_, stream)
+ return stream
+
+
+def simplefilter(f):
+ """
+ Decorator that converts a function into a filter::
+
+ @simplefilter
+ def lowercase(self, lexer, stream, options):
+ for ttype, value in stream:
+ yield ttype, value.lower()
+ """
+ return type(f.__name__, (FunctionFilter,), {
+ '__module__': getattr(f, '__module__'),
+ '__doc__': f.__doc__,
+ 'function': f,
+ })
+
+
+class Filter(object):
+ """
+ Default filter. Subclass this class or use the `simplefilter`
+ decorator to create own filters.
+ """
+
+ def __init__(self, **options):
+ self.options = options
+
+ def filter(self, lexer, stream):
+ raise NotImplementedError
+
+
+class FunctionFilter(Filter):
+ """
+ Abstract class used by `simplefilter` to create simple
+ function filters on the fly. The `simplefilter` decorator
+ automatically creates subclasses of this class for
+ functions passed to it.
+ """
+ function = None
+
+ def __init__(self, **options):
+ if not hasattr(self, 'function'):
+ raise TypeError('%r used without bound function' %
+ self.__class__.__name__)
+ Filter.__init__(self, **options)
+
+ def filter(self, lexer, stream):
+ # pylint: disable=not-callable
+ for ttype, value in self.function(lexer, stream, self.options):
+ yield ttype, value
diff --git a/wandb/vendor/pygments/filters/__init__.py b/wandb/vendor/pygments/filters/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..45f9608c0afb7c19147f77bc864f2f896edb8a21
--- /dev/null
+++ b/wandb/vendor/pygments/filters/__init__.py
@@ -0,0 +1,350 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.filters
+ ~~~~~~~~~~~~~~~~
+
+ Module containing filter lookup functions and default
+ filters.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.token import String, Comment, Keyword, Name, Error, Whitespace, \
+ string_to_tokentype
+from pygments.filter import Filter
+from pygments.util import get_list_opt, get_int_opt, get_bool_opt, \
+ get_choice_opt, ClassNotFound, OptionError, text_type, string_types
+from pygments.plugin import find_plugin_filters
+
+
+def find_filter_class(filtername):
+ """Lookup a filter by name. Return None if not found."""
+ if filtername in FILTERS:
+ return FILTERS[filtername]
+ for name, cls in find_plugin_filters():
+ if name == filtername:
+ return cls
+ return None
+
+
+def get_filter_by_name(filtername, **options):
+ """Return an instantiated filter.
+
+ Options are passed to the filter initializer if wanted.
+ Raise a ClassNotFound if not found.
+ """
+ cls = find_filter_class(filtername)
+ if cls:
+ return cls(**options)
+ else:
+ raise ClassNotFound('filter %r not found' % filtername)
+
+
+def get_all_filters():
+ """Return a generator of all filter names."""
+ for name in FILTERS:
+ yield name
+ for name, _ in find_plugin_filters():
+ yield name
+
+
+def _replace_special(ttype, value, regex, specialttype,
+ replacefunc=lambda x: x):
+ last = 0
+ for match in regex.finditer(value):
+ start, end = match.start(), match.end()
+ if start != last:
+ yield ttype, value[last:start]
+ yield specialttype, replacefunc(value[start:end])
+ last = end
+ if last != len(value):
+ yield ttype, value[last:]
+
+
+class CodeTagFilter(Filter):
+ """Highlight special code tags in comments and docstrings.
+
+ Options accepted:
+
+ `codetags` : list of strings
+ A list of strings that are flagged as code tags. The default is to
+ highlight ``XXX``, ``TODO``, ``BUG`` and ``NOTE``.
+ """
+
+ def __init__(self, **options):
+ Filter.__init__(self, **options)
+ tags = get_list_opt(options, 'codetags',
+ ['XXX', 'TODO', 'BUG', 'NOTE'])
+ self.tag_re = re.compile(r'\b(%s)\b' % '|'.join([
+ re.escape(tag) for tag in tags if tag
+ ]))
+
+ def filter(self, lexer, stream):
+ regex = self.tag_re
+ for ttype, value in stream:
+ if ttype in String.Doc or \
+ ttype in Comment and \
+ ttype not in Comment.Preproc:
+ for sttype, svalue in _replace_special(ttype, value, regex,
+ Comment.Special):
+ yield sttype, svalue
+ else:
+ yield ttype, value
+
+
+class KeywordCaseFilter(Filter):
+ """Convert keywords to lowercase or uppercase or capitalize them, which
+ means first letter uppercase, rest lowercase.
+
+ This can be useful e.g. if you highlight Pascal code and want to adapt the
+ code to your styleguide.
+
+ Options accepted:
+
+ `case` : string
+ The casing to convert keywords to. Must be one of ``'lower'``,
+ ``'upper'`` or ``'capitalize'``. The default is ``'lower'``.
+ """
+
+ def __init__(self, **options):
+ Filter.__init__(self, **options)
+ case = get_choice_opt(options, 'case',
+ ['lower', 'upper', 'capitalize'], 'lower')
+ self.convert = getattr(text_type, case)
+
+ def filter(self, lexer, stream):
+ for ttype, value in stream:
+ if ttype in Keyword:
+ yield ttype, self.convert(value)
+ else:
+ yield ttype, value
+
+
+class NameHighlightFilter(Filter):
+ """Highlight a normal Name (and Name.*) token with a different token type.
+
+ Example::
+
+ filter = NameHighlightFilter(
+ names=['foo', 'bar', 'baz'],
+ tokentype=Name.Function,
+ )
+
+ This would highlight the names "foo", "bar" and "baz"
+ as functions. `Name.Function` is the default token type.
+
+ Options accepted:
+
+ `names` : list of strings
+ A list of names that should be given the different token type.
+ There is no default.
+ `tokentype` : TokenType or string
+ A token type or a string containing a token type name that is
+ used for highlighting the strings in `names`. The default is
+ `Name.Function`.
+ """
+
+ def __init__(self, **options):
+ Filter.__init__(self, **options)
+ self.names = set(get_list_opt(options, 'names', []))
+ tokentype = options.get('tokentype')
+ if tokentype:
+ self.tokentype = string_to_tokentype(tokentype)
+ else:
+ self.tokentype = Name.Function
+
+ def filter(self, lexer, stream):
+ for ttype, value in stream:
+ if ttype in Name and value in self.names:
+ yield self.tokentype, value
+ else:
+ yield ttype, value
+
+
+class ErrorToken(Exception):
+ pass
+
+
+class RaiseOnErrorTokenFilter(Filter):
+ """Raise an exception when the lexer generates an error token.
+
+ Options accepted:
+
+ `excclass` : Exception class
+ The exception class to raise.
+ The default is `pygments.filters.ErrorToken`.
+
+ .. versionadded:: 0.8
+ """
+
+ def __init__(self, **options):
+ Filter.__init__(self, **options)
+ self.exception = options.get('excclass', ErrorToken)
+ try:
+ # issubclass() will raise TypeError if first argument is not a class
+ if not issubclass(self.exception, Exception):
+ raise TypeError
+ except TypeError:
+ raise OptionError('excclass option is not an exception class')
+
+ def filter(self, lexer, stream):
+ for ttype, value in stream:
+ if ttype is Error:
+ raise self.exception(value)
+ yield ttype, value
+
+
+class VisibleWhitespaceFilter(Filter):
+ """Convert tabs, newlines and/or spaces to visible characters.
+
+ Options accepted:
+
+ `spaces` : string or bool
+ If this is a one-character string, spaces will be replaces by this string.
+ If it is another true value, spaces will be replaced by ``·`` (unicode
+ MIDDLE DOT). If it is a false value, spaces will not be replaced. The
+ default is ``False``.
+ `tabs` : string or bool
+ The same as for `spaces`, but the default replacement character is ``»``
+ (unicode RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK). The default value
+ is ``False``. Note: this will not work if the `tabsize` option for the
+ lexer is nonzero, as tabs will already have been expanded then.
+ `tabsize` : int
+ If tabs are to be replaced by this filter (see the `tabs` option), this
+ is the total number of characters that a tab should be expanded to.
+ The default is ``8``.
+ `newlines` : string or bool
+ The same as for `spaces`, but the default replacement character is ``¶``
+ (unicode PILCROW SIGN). The default value is ``False``.
+ `wstokentype` : bool
+ If true, give whitespace the special `Whitespace` token type. This allows
+ styling the visible whitespace differently (e.g. greyed out), but it can
+ disrupt background colors. The default is ``True``.
+
+ .. versionadded:: 0.8
+ """
+
+ def __init__(self, **options):
+ Filter.__init__(self, **options)
+ for name, default in [('spaces', u'·'),
+ ('tabs', u'»'),
+ ('newlines', u'¶')]:
+ opt = options.get(name, False)
+ if isinstance(opt, string_types) and len(opt) == 1:
+ setattr(self, name, opt)
+ else:
+ setattr(self, name, (opt and default or ''))
+ tabsize = get_int_opt(options, 'tabsize', 8)
+ if self.tabs:
+ self.tabs += ' ' * (tabsize - 1)
+ if self.newlines:
+ self.newlines += '\n'
+ self.wstt = get_bool_opt(options, 'wstokentype', True)
+
+ def filter(self, lexer, stream):
+ if self.wstt:
+ spaces = self.spaces or u' '
+ tabs = self.tabs or u'\t'
+ newlines = self.newlines or u'\n'
+ regex = re.compile(r'\s')
+ def replacefunc(wschar):
+ if wschar == ' ':
+ return spaces
+ elif wschar == '\t':
+ return tabs
+ elif wschar == '\n':
+ return newlines
+ return wschar
+
+ for ttype, value in stream:
+ for sttype, svalue in _replace_special(ttype, value, regex,
+ Whitespace, replacefunc):
+ yield sttype, svalue
+ else:
+ spaces, tabs, newlines = self.spaces, self.tabs, self.newlines
+ # simpler processing
+ for ttype, value in stream:
+ if spaces:
+ value = value.replace(' ', spaces)
+ if tabs:
+ value = value.replace('\t', tabs)
+ if newlines:
+ value = value.replace('\n', newlines)
+ yield ttype, value
+
+
+class GobbleFilter(Filter):
+ """Gobbles source code lines (eats initial characters).
+
+ This filter drops the first ``n`` characters off every line of code. This
+ may be useful when the source code fed to the lexer is indented by a fixed
+ amount of space that isn't desired in the output.
+
+ Options accepted:
+
+ `n` : int
+ The number of characters to gobble.
+
+ .. versionadded:: 1.2
+ """
+ def __init__(self, **options):
+ Filter.__init__(self, **options)
+ self.n = get_int_opt(options, 'n', 0)
+
+ def gobble(self, value, left):
+ if left < len(value):
+ return value[left:], 0
+ else:
+ return u'', left - len(value)
+
+ def filter(self, lexer, stream):
+ n = self.n
+ left = n # How many characters left to gobble.
+ for ttype, value in stream:
+ # Remove ``left`` tokens from first line, ``n`` from all others.
+ parts = value.split('\n')
+ (parts[0], left) = self.gobble(parts[0], left)
+ for i in range(1, len(parts)):
+ (parts[i], left) = self.gobble(parts[i], n)
+ value = u'\n'.join(parts)
+
+ if value != '':
+ yield ttype, value
+
+
+class TokenMergeFilter(Filter):
+ """Merges consecutive tokens with the same token type in the output
+ stream of a lexer.
+
+ .. versionadded:: 1.2
+ """
+ def __init__(self, **options):
+ Filter.__init__(self, **options)
+
+ def filter(self, lexer, stream):
+ current_type = None
+ current_value = None
+ for ttype, value in stream:
+ if ttype is current_type:
+ current_value += value
+ else:
+ if current_type is not None:
+ yield current_type, current_value
+ current_type = ttype
+ current_value = value
+ if current_type is not None:
+ yield current_type, current_value
+
+
+FILTERS = {
+ 'codetagify': CodeTagFilter,
+ 'keywordcase': KeywordCaseFilter,
+ 'highlight': NameHighlightFilter,
+ 'raiseonerror': RaiseOnErrorTokenFilter,
+ 'whitespace': VisibleWhitespaceFilter,
+ 'gobble': GobbleFilter,
+ 'tokenmerge': TokenMergeFilter,
+}
diff --git a/wandb/vendor/pygments/formatter.py b/wandb/vendor/pygments/formatter.py
new file mode 100644
index 0000000000000000000000000000000000000000..c0780f62a1ca099ee80055c0fd52a80c1d2a6b29
--- /dev/null
+++ b/wandb/vendor/pygments/formatter.py
@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.formatter
+ ~~~~~~~~~~~~~~~~~~
+
+ Base formatter class.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import codecs
+
+from pygments.util import get_bool_opt, string_types
+from pygments.styles import get_style_by_name
+
+__all__ = ['Formatter']
+
+
+def _lookup_style(style):
+ if isinstance(style, string_types):
+ return get_style_by_name(style)
+ return style
+
+
+class Formatter(object):
+ """
+ Converts a token stream to text.
+
+ Options accepted:
+
+ ``style``
+ The style to use, can be a string or a Style subclass
+ (default: "default"). Not used by e.g. the
+ TerminalFormatter.
+ ``full``
+ Tells the formatter to output a "full" document, i.e.
+ a complete self-contained document. This doesn't have
+ any effect for some formatters (default: false).
+ ``title``
+ If ``full`` is true, the title that should be used to
+ caption the document (default: '').
+ ``encoding``
+ If given, must be an encoding name. This will be used to
+ convert the Unicode token strings to byte strings in the
+ output. If it is "" or None, Unicode strings will be written
+ to the output file, which most file-like objects do not
+ support (default: None).
+ ``outencoding``
+ Overrides ``encoding`` if given.
+ """
+
+ #: Name of the formatter
+ name = None
+
+ #: Shortcuts for the formatter
+ aliases = []
+
+ #: fn match rules
+ filenames = []
+
+ #: If True, this formatter outputs Unicode strings when no encoding
+ #: option is given.
+ unicodeoutput = True
+
+ def __init__(self, **options):
+ self.style = _lookup_style(options.get('style', 'default'))
+ self.full = get_bool_opt(options, 'full', False)
+ self.title = options.get('title', '')
+ self.encoding = options.get('encoding', None) or None
+ if self.encoding in ('guess', 'chardet'):
+ # can happen for e.g. pygmentize -O encoding=guess
+ self.encoding = 'utf-8'
+ self.encoding = options.get('outencoding') or self.encoding
+ self.options = options
+
+ def get_style_defs(self, arg=''):
+ """
+ Return the style definitions for the current style as a string.
+
+ ``arg`` is an additional argument whose meaning depends on the
+ formatter used. Note that ``arg`` can also be a list or tuple
+ for some formatters like the html formatter.
+ """
+ return ''
+
+ def format(self, tokensource, outfile):
+ """
+ Format ``tokensource``, an iterable of ``(tokentype, tokenstring)``
+ tuples and write it into ``outfile``.
+ """
+ if self.encoding:
+ # wrap the outfile in a StreamWriter
+ outfile = codecs.lookup(self.encoding)[3](outfile)
+ return self.format_unencoded(tokensource, outfile)
diff --git a/wandb/vendor/pygments/formatters/__init__.py b/wandb/vendor/pygments/formatters/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..965c5f1ad795dfebb9a2b9f57348ceac8c80c9a4
--- /dev/null
+++ b/wandb/vendor/pygments/formatters/__init__.py
@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.formatters
+ ~~~~~~~~~~~~~~~~~~~
+
+ Pygments formatters.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+import sys
+import types
+import fnmatch
+from os.path import basename
+
+from pygments.formatters._mapping import FORMATTERS
+from pygments.plugin import find_plugin_formatters
+from pygments.util import ClassNotFound, itervalues
+
+__all__ = ['get_formatter_by_name', 'get_formatter_for_filename',
+ 'get_all_formatters', 'load_formatter_from_file'] + list(FORMATTERS)
+
+_formatter_cache = {} # classes by name
+_pattern_cache = {}
+
+
+def _fn_matches(fn, glob):
+ """Return whether the supplied file name fn matches pattern filename."""
+ if glob not in _pattern_cache:
+ pattern = _pattern_cache[glob] = re.compile(fnmatch.translate(glob))
+ return pattern.match(fn)
+ return _pattern_cache[glob].match(fn)
+
+
+def _load_formatters(module_name):
+ """Load a formatter (and all others in the module too)."""
+ mod = __import__(module_name, None, None, ['__all__'])
+ for formatter_name in mod.__all__:
+ cls = getattr(mod, formatter_name)
+ _formatter_cache[cls.name] = cls
+
+
+def get_all_formatters():
+ """Return a generator for all formatter classes."""
+ # NB: this returns formatter classes, not info like get_all_lexers().
+ for info in itervalues(FORMATTERS):
+ if info[1] not in _formatter_cache:
+ _load_formatters(info[0])
+ yield _formatter_cache[info[1]]
+ for _, formatter in find_plugin_formatters():
+ yield formatter
+
+
+def find_formatter_class(alias):
+ """Lookup a formatter by alias.
+
+ Returns None if not found.
+ """
+ for module_name, name, aliases, _, _ in itervalues(FORMATTERS):
+ if alias in aliases:
+ if name not in _formatter_cache:
+ _load_formatters(module_name)
+ return _formatter_cache[name]
+ for _, cls in find_plugin_formatters():
+ if alias in cls.aliases:
+ return cls
+
+
+def get_formatter_by_name(_alias, **options):
+ """Lookup and instantiate a formatter by alias.
+
+ Raises ClassNotFound if not found.
+ """
+ cls = find_formatter_class(_alias)
+ if cls is None:
+ raise ClassNotFound("no formatter found for name %r" % _alias)
+ return cls(**options)
+
+
+def load_formatter_from_file(filename, formattername="CustomFormatter",
+ **options):
+ """Load a formatter from a file.
+
+ This method expects a file located relative to the current working
+ directory, which contains a class named CustomFormatter. By default,
+ it expects the Formatter to be named CustomFormatter; you can specify
+ your own class name as the second argument to this function.
+
+ Users should be very careful with the input, because this method
+ is equivalent to running eval on the input file.
+
+ Raises ClassNotFound if there are any problems importing the Formatter.
+
+ .. versionadded:: 2.2
+ """
+ try:
+ # This empty dict will contain the namespace for the exec'd file
+ custom_namespace = {}
+ exec(open(filename, 'rb').read(), custom_namespace)
+ # Retrieve the class `formattername` from that namespace
+ if formattername not in custom_namespace:
+ raise ClassNotFound('no valid %s class found in %s' %
+ (formattername, filename))
+ formatter_class = custom_namespace[formattername]
+ # And finally instantiate it with the options
+ return formatter_class(**options)
+ except IOError as err:
+ raise ClassNotFound('cannot read %s' % filename)
+ except ClassNotFound as err:
+ raise
+ except Exception as err:
+ raise ClassNotFound('error when loading custom formatter: %s' % err)
+
+
+def get_formatter_for_filename(fn, **options):
+ """Lookup and instantiate a formatter by filename pattern.
+
+ Raises ClassNotFound if not found.
+ """
+ fn = basename(fn)
+ for modname, name, _, filenames, _ in itervalues(FORMATTERS):
+ for filename in filenames:
+ if _fn_matches(fn, filename):
+ if name not in _formatter_cache:
+ _load_formatters(modname)
+ return _formatter_cache[name](**options)
+ for cls in find_plugin_formatters():
+ for filename in cls.filenames:
+ if _fn_matches(fn, filename):
+ return cls(**options)
+ raise ClassNotFound("no formatter found for file name %r" % fn)
+
+
+class _automodule(types.ModuleType):
+ """Automatically import formatters."""
+
+ def __getattr__(self, name):
+ info = FORMATTERS.get(name)
+ if info:
+ _load_formatters(info[0])
+ cls = _formatter_cache[info[1]]
+ setattr(self, name, cls)
+ return cls
+ raise AttributeError(name)
+
+
+oldmod = sys.modules[__name__]
+newmod = _automodule(__name__)
+newmod.__dict__.update(oldmod.__dict__)
+sys.modules[__name__] = newmod
+del newmod.newmod, newmod.oldmod, newmod.sys, newmod.types
diff --git a/wandb/vendor/pygments/formatters/_mapping.py b/wandb/vendor/pygments/formatters/_mapping.py
new file mode 100644
index 0000000000000000000000000000000000000000..7bb3e71c1673ccbc0f3285f38edbfb455e5d0b6c
--- /dev/null
+++ b/wandb/vendor/pygments/formatters/_mapping.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.formatters._mapping
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Formatter mapping definitions. This file is generated by itself. Everytime
+ you change something on a builtin formatter definition, run this script from
+ the formatters folder to update it.
+
+ Do not alter the FORMATTERS dictionary by hand.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import print_function
+
+FORMATTERS = {
+ 'BBCodeFormatter': ('pygments.formatters.bbcode', 'BBCode', ('bbcode', 'bb'), (), 'Format tokens with BBcodes. These formatting codes are used by many bulletin boards, so you can highlight your sourcecode with pygments before posting it there.'),
+ 'BmpImageFormatter': ('pygments.formatters.img', 'img_bmp', ('bmp', 'bitmap'), ('*.bmp',), 'Create a bitmap image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+ 'GifImageFormatter': ('pygments.formatters.img', 'img_gif', ('gif',), ('*.gif',), 'Create a GIF image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+ 'HtmlFormatter': ('pygments.formatters.html', 'HTML', ('html',), ('*.html', '*.htm'), "Format tokens as HTML 4 ``<span>`` tags within a ``<pre>`` tag, wrapped in a ``<div>`` tag. The ``<div>``'s CSS class can be set by the `cssclass` option."),
+ 'IRCFormatter': ('pygments.formatters.irc', 'IRC', ('irc', 'IRC'), (), 'Format tokens with IRC color sequences'),
+ 'ImageFormatter': ('pygments.formatters.img', 'img', ('img', 'IMG', 'png'), ('*.png',), 'Create a PNG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+ 'JpgImageFormatter': ('pygments.formatters.img', 'img_jpg', ('jpg', 'jpeg'), ('*.jpg',), 'Create a JPEG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+ 'LatexFormatter': ('pygments.formatters.latex', 'LaTeX', ('latex', 'tex'), ('*.tex',), 'Format tokens as LaTeX code. This needs the `fancyvrb` and `color` standard packages.'),
+ 'NullFormatter': ('pygments.formatters.other', 'Text only', ('text', 'null'), ('*.txt',), 'Output the text unchanged without any formatting.'),
+ 'RawTokenFormatter': ('pygments.formatters.other', 'Raw tokens', ('raw', 'tokens'), ('*.raw',), 'Format tokens as a raw representation for storing token streams.'),
+ 'RtfFormatter': ('pygments.formatters.rtf', 'RTF', ('rtf',), ('*.rtf',), 'Format tokens as RTF markup. This formatter automatically outputs full RTF documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft(R) Word(R) documents.'),
+ 'SvgFormatter': ('pygments.formatters.svg', 'SVG', ('svg',), ('*.svg',), 'Format tokens as an SVG graphics file. This formatter is still experimental. Each line of code is a ``<text>`` element with explicit ``x`` and ``y`` coordinates containing ``<tspan>`` elements with the individual token styles.'),
+ 'Terminal256Formatter': ('pygments.formatters.terminal256', 'Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
+ 'TerminalFormatter': ('pygments.formatters.terminal', 'Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.'),
+ 'TerminalTrueColorFormatter': ('pygments.formatters.terminal256', 'TerminalTrueColor', ('terminal16m', 'console16m', '16m'), (), 'Format tokens with ANSI color sequences, for output in a true-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
+ 'TestcaseFormatter': ('pygments.formatters.other', 'Testcase', ('testcase',), (), 'Format tokens as appropriate for a new testcase.')
+}
+
+if __name__ == '__main__': # pragma: no cover
+ import sys
+ import os
+
+ # lookup formatters
+ found_formatters = []
+ imports = []
+ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
+ from pygments.util import docstring_headline
+
+ for root, dirs, files in os.walk('.'):
+ for filename in files:
+ if filename.endswith('.py') and not filename.startswith('_'):
+ module_name = 'pygments.formatters%s.%s' % (
+ root[1:].replace('/', '.'), filename[:-3])
+ print(module_name)
+ module = __import__(module_name, None, None, [''])
+ for formatter_name in module.__all__:
+ formatter = getattr(module, formatter_name)
+ found_formatters.append(
+ '%r: %r' % (formatter_name,
+ (module_name,
+ formatter.name,
+ tuple(formatter.aliases),
+ tuple(formatter.filenames),
+ docstring_headline(formatter))))
+ # sort them to make the diff minimal
+ found_formatters.sort()
+
+ # extract useful sourcecode from this file
+ with open(__file__) as fp:
+ content = fp.read()
+ # replace crnl to nl for Windows.
+ #
+ # Note that, originally, contributers should keep nl of master
+ # repository, for example by using some kind of automatic
+ # management EOL, like `EolExtension
+ # <https://www.mercurial-scm.org/wiki/EolExtension>`.
+ content = content.replace("\r\n", "\n")
+ header = content[:content.find('FORMATTERS = {')]
+ footer = content[content.find("if __name__ == '__main__':"):]
+
+ # write new file
+ with open(__file__, 'w') as fp:
+ fp.write(header)
+ fp.write('FORMATTERS = {\n %s\n}\n\n' % ',\n '.join(found_formatters))
+ fp.write(footer)
+
+ print ('=== %d formatters processed.' % len(found_formatters))
diff --git a/wandb/vendor/pygments/formatters/bbcode.py b/wandb/vendor/pygments/formatters/bbcode.py
new file mode 100644
index 0000000000000000000000000000000000000000..9fc9476dc6c68d622a103600e03ffb5907276add
--- /dev/null
+++ b/wandb/vendor/pygments/formatters/bbcode.py
@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.formatters.bbcode
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ BBcode formatter.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+
+from pygments.formatter import Formatter
+from pygments.util import get_bool_opt
+
+__all__ = ['BBCodeFormatter']
+
+
+class BBCodeFormatter(Formatter):
+ """
+ Format tokens with BBcodes. These formatting codes are used by many
+ bulletin boards, so you can highlight your sourcecode with pygments before
+ posting it there.
+
+ This formatter has no support for background colors and borders, as there
+ are no common BBcode tags for that.
+
+ Some board systems (e.g. phpBB) don't support colors in their [code] tag,
+ so you can't use the highlighting together with that tag.
+ Text in a [code] tag usually is shown with a monospace font (which this
+ formatter can do with the ``monofont`` option) and no spaces (which you
+ need for indentation) are removed.
+
+ Additional options accepted:
+
+ `style`
+ The style to use, can be a string or a Style subclass (default:
+ ``'default'``).
+
+ `codetag`
+ If set to true, put the output into ``[code]`` tags (default:
+ ``false``)
+
+ `monofont`
+ If set to true, add a tag to show the code with a monospace font
+ (default: ``false``).
+ """
+ name = 'BBCode'
+ aliases = ['bbcode', 'bb']
+ filenames = []
+
+ def __init__(self, **options):
+ Formatter.__init__(self, **options)
+ self._code = get_bool_opt(options, 'codetag', False)
+ self._mono = get_bool_opt(options, 'monofont', False)
+
+ self.styles = {}
+ self._make_styles()
+
+ def _make_styles(self):
+ for ttype, ndef in self.style:
+ start = end = ''
+ if ndef['color']:
+ start += '[color=#%s]' % ndef['color']
+ end = '[/color]' + end
+ if ndef['bold']:
+ start += '[b]'
+ end = '[/b]' + end
+ if ndef['italic']:
+ start += '[i]'
+ end = '[/i]' + end
+ if ndef['underline']:
+ start += '[u]'
+ end = '[/u]' + end
+ # there are no common BBcodes for background-color and border
+
+ self.styles[ttype] = start, end
+
+ def format_unencoded(self, tokensource, outfile):
+ if self._code:
+ outfile.write('[code]')
+ if self._mono:
+ outfile.write('[font=monospace]')
+
+ lastval = ''
+ lasttype = None
+
+ for ttype, value in tokensource:
+ while ttype not in self.styles:
+ ttype = ttype.parent
+ if ttype == lasttype:
+ lastval += value
+ else:
+ if lastval:
+ start, end = self.styles[lasttype]
+ outfile.write(''.join((start, lastval, end)))
+ lastval = value
+ lasttype = ttype
+
+ if lastval:
+ start, end = self.styles[lasttype]
+ outfile.write(''.join((start, lastval, end)))
+
+ if self._mono:
+ outfile.write('[/font]')
+ if self._code:
+ outfile.write('[/code]')
+ if self._code or self._mono:
+ outfile.write('\n')
diff --git a/wandb/vendor/pygments/formatters/html.py b/wandb/vendor/pygments/formatters/html.py
new file mode 100644
index 0000000000000000000000000000000000000000..2969d5026381e3d0acc21dae985ee24ef56223eb
--- /dev/null
+++ b/wandb/vendor/pygments/formatters/html.py
@@ -0,0 +1,851 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.formatters.html
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Formatter for HTML output.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import print_function
+
+import os
+import sys
+import os.path
+
+from pygments.formatter import Formatter
+from pygments.token import Token, Text, STANDARD_TYPES
+from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
+ StringIO, string_types, iteritems
+
+try:
+ import ctags
+except ImportError:
+ ctags = None
+
+__all__ = ['HtmlFormatter']
+
+
+_escape_html_table = {
+ ord('&'): u'&',
+ ord('<'): u'<',
+ ord('>'): u'>',
+ ord('"'): u'"',
+ ord("'"): u''',
+}
+
+
+def escape_html(text, table=_escape_html_table):
+ """Escape &, <, > as well as single and double quotes for HTML."""
+ return text.translate(table)
+
+
+def _get_ttype_class(ttype):
+ fname = STANDARD_TYPES.get(ttype)
+ if fname:
+ return fname
+ aname = ''
+ while fname is None:
+ aname = '-' + ttype[-1] + aname
+ ttype = ttype.parent
+ fname = STANDARD_TYPES.get(ttype)
+ return fname + aname
+
+
+CSSFILE_TEMPLATE = '''\
+td.linenos { background-color: #f0f0f0; padding-right: 10px; }
+span.lineno { background-color: #f0f0f0; padding: 0 5px 0 5px; }
+pre { line-height: 125%%; }
+%(styledefs)s
+'''
+
+DOC_HEADER = '''\
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
+ "http://www.w3.org/TR/html4/strict.dtd">
+
+<html>
+<head>
+ <title>%(title)s</title>
+ <meta http-equiv="content-type" content="text/html; charset=%(encoding)s">
+ <style type="text/css">
+''' + CSSFILE_TEMPLATE + '''
+ </style>
+</head>
+<body>
+<h2>%(title)s</h2>
+
+'''
+
+DOC_HEADER_EXTERNALCSS = '''\
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
+ "http://www.w3.org/TR/html4/strict.dtd">
+
+<html>
+<head>
+ <title>%(title)s</title>
+ <meta http-equiv="content-type" content="text/html; charset=%(encoding)s">
+ <link rel="stylesheet" href="%(cssfile)s" type="text/css">
+</head>
+<body>
+<h2>%(title)s</h2>
+
+'''
+
+DOC_FOOTER = '''\
+</body>
+</html>
+'''
+
+
+class HtmlFormatter(Formatter):
+ r"""
+ Format tokens as HTML 4 ``<span>`` tags within a ``<pre>`` tag, wrapped
+ in a ``<div>`` tag. The ``<div>``'s CSS class can be set by the `cssclass`
+ option.
+
+ If the `linenos` option is set to ``"table"``, the ``<pre>`` is
+ additionally wrapped inside a ``<table>`` which has one row and two
+ cells: one containing the line numbers and one containing the code.
+ Example:
+
+ .. sourcecode:: html
+
+ <div class="highlight" >
+ <table><tr>
+ <td class="linenos" title="click to toggle"
+ onclick="with (this.firstChild.style)
+ { display = (display == '') ? 'none' : '' }">
+ <pre>1
+ 2</pre>
+ </td>
+ <td class="code">
+ <pre><span class="Ke">def </span><span class="NaFu">foo</span>(bar):
+ <span class="Ke">pass</span>
+ </pre>
+ </td>
+ </tr></table></div>
+
+ (whitespace added to improve clarity).
+
+ Wrapping can be disabled using the `nowrap` option.
+
+ A list of lines can be specified using the `hl_lines` option to make these
+ lines highlighted (as of Pygments 0.11).
+
+ With the `full` option, a complete HTML 4 document is output, including
+ the style definitions inside a ``<style>`` tag, or in a separate file if
+ the `cssfile` option is given.
+
+ When `tagsfile` is set to the path of a ctags index file, it is used to
+ generate hyperlinks from names to their definition. You must enable
+ `lineanchors` and run ctags with the `-n` option for this to work. The
+ `python-ctags` module from PyPI must be installed to use this feature;
+ otherwise a `RuntimeError` will be raised.
+
+ The `get_style_defs(arg='')` method of a `HtmlFormatter` returns a string
+ containing CSS rules for the CSS classes used by the formatter. The
+ argument `arg` can be used to specify additional CSS selectors that
+ are prepended to the classes. A call `fmter.get_style_defs('td .code')`
+ would result in the following CSS classes:
+
+ .. sourcecode:: css
+
+ td .code .kw { font-weight: bold; color: #00FF00 }
+ td .code .cm { color: #999999 }
+ ...
+
+ If you have Pygments 0.6 or higher, you can also pass a list or tuple to the
+ `get_style_defs()` method to request multiple prefixes for the tokens:
+
+ .. sourcecode:: python
+
+ formatter.get_style_defs(['div.syntax pre', 'pre.syntax'])
+
+ The output would then look like this:
+
+ .. sourcecode:: css
+
+ div.syntax pre .kw,
+ pre.syntax .kw { font-weight: bold; color: #00FF00 }
+ div.syntax pre .cm,
+ pre.syntax .cm { color: #999999 }
+ ...
+
+ Additional options accepted:
+
+ `nowrap`
+ If set to ``True``, don't wrap the tokens at all, not even inside a ``<pre>``
+ tag. This disables most other options (default: ``False``).
+
+ `full`
+ Tells the formatter to output a "full" document, i.e. a complete
+ self-contained document (default: ``False``).
+
+ `title`
+ If `full` is true, the title that should be used to caption the
+ document (default: ``''``).
+
+ `style`
+ The style to use, can be a string or a Style subclass (default:
+ ``'default'``). This option has no effect if the `cssfile`
+ and `noclobber_cssfile` option are given and the file specified in
+ `cssfile` exists.
+
+ `noclasses`
+ If set to true, token ``<span>`` tags will not use CSS classes, but
+ inline styles. This is not recommended for larger pieces of code since
+ it increases output size by quite a bit (default: ``False``).
+
+ `classprefix`
+ Since the token types use relatively short class names, they may clash
+ with some of your own class names. In this case you can use the
+ `classprefix` option to give a string to prepend to all Pygments-generated
+ CSS class names for token types.
+ Note that this option also affects the output of `get_style_defs()`.
+
+ `cssclass`
+ CSS class for the wrapping ``<div>`` tag (default: ``'highlight'``).
+ If you set this option, the default selector for `get_style_defs()`
+ will be this class.
+
+ .. versionadded:: 0.9
+ If you select the ``'table'`` line numbers, the wrapping table will
+ have a CSS class of this string plus ``'table'``, the default is
+ accordingly ``'highlighttable'``.
+
+ `cssstyles`
+ Inline CSS styles for the wrapping ``<div>`` tag (default: ``''``).
+
+ `prestyles`
+ Inline CSS styles for the ``<pre>`` tag (default: ``''``).
+
+ .. versionadded:: 0.11
+
+ `cssfile`
+ If the `full` option is true and this option is given, it must be the
+ name of an external file. If the filename does not include an absolute
+ path, the file's path will be assumed to be relative to the main output
+ file's path, if the latter can be found. The stylesheet is then written
+ to this file instead of the HTML file.
+
+ .. versionadded:: 0.6
+
+ `noclobber_cssfile`
+ If `cssfile` is given and the specified file exists, the css file will
+ not be overwritten. This allows the use of the `full` option in
+ combination with a user specified css file. Default is ``False``.
+
+ .. versionadded:: 1.1
+
+ `linenos`
+ If set to ``'table'``, output line numbers as a table with two cells,
+ one containing the line numbers, the other the whole code. This is
+ copy-and-paste-friendly, but may cause alignment problems with some
+ browsers or fonts. If set to ``'inline'``, the line numbers will be
+ integrated in the ``<pre>`` tag that contains the code (that setting
+ is *new in Pygments 0.8*).
+
+ For compatibility with Pygments 0.7 and earlier, every true value
+ except ``'inline'`` means the same as ``'table'`` (in particular, that
+ means also ``True``).
+
+ The default value is ``False``, which means no line numbers at all.
+
+ **Note:** with the default ("table") line number mechanism, the line
+ numbers and code can have different line heights in Internet Explorer
+ unless you give the enclosing ``<pre>`` tags an explicit ``line-height``
+ CSS property (you get the default line spacing with ``line-height:
+ 125%``).
+
+ `hl_lines`
+ Specify a list of lines to be highlighted.
+
+ .. versionadded:: 0.11
+
+ `linenostart`
+ The line number for the first line (default: ``1``).
+
+ `linenostep`
+ If set to a number n > 1, only every nth line number is printed.
+
+ `linenospecial`
+ If set to a number n > 0, every nth line number is given the CSS
+ class ``"special"`` (default: ``0``).
+
+ `nobackground`
+ If set to ``True``, the formatter won't output the background color
+ for the wrapping element (this automatically defaults to ``False``
+ when there is no wrapping element [eg: no argument for the
+ `get_syntax_defs` method given]) (default: ``False``).
+
+ .. versionadded:: 0.6
+
+ `lineseparator`
+ This string is output between lines of code. It defaults to ``"\n"``,
+ which is enough to break a line inside ``<pre>`` tags, but you can
+ e.g. set it to ``"<br>"`` to get HTML line breaks.
+
+ .. versionadded:: 0.7
+
+ `lineanchors`
+ If set to a nonempty string, e.g. ``foo``, the formatter will wrap each
+ output line in an anchor tag with a ``name`` of ``foo-linenumber``.
+ This allows easy linking to certain lines.
+
+ .. versionadded:: 0.9
+
+ `linespans`
+ If set to a nonempty string, e.g. ``foo``, the formatter will wrap each
+ output line in a span tag with an ``id`` of ``foo-linenumber``.
+ This allows easy access to lines via javascript.
+
+ .. versionadded:: 1.6
+
+ `anchorlinenos`
+ If set to `True`, will wrap line numbers in <a> tags. Used in
+ combination with `linenos` and `lineanchors`.
+
+ `tagsfile`
+ If set to the path of a ctags file, wrap names in anchor tags that
+ link to their definitions. `lineanchors` should be used, and the
+ tags file should specify line numbers (see the `-n` option to ctags).
+
+ .. versionadded:: 1.6
+
+ `tagurlformat`
+ A string formatting pattern used to generate links to ctags definitions.
+ Available variables are `%(path)s`, `%(fname)s` and `%(fext)s`.
+ Defaults to an empty string, resulting in just `#prefix-number` links.
+
+ .. versionadded:: 1.6
+
+ `filename`
+ A string used to generate a filename when rendering <pre> blocks,
+ for example if displaying source code.
+
+ .. versionadded:: 2.1
+
+
+ **Subclassing the HTML formatter**
+
+ .. versionadded:: 0.7
+
+ The HTML formatter is now built in a way that allows easy subclassing, thus
+ customizing the output HTML code. The `format()` method calls
+ `self._format_lines()` which returns a generator that yields tuples of ``(1,
+ line)``, where the ``1`` indicates that the ``line`` is a line of the
+ formatted source code.
+
+ If the `nowrap` option is set, the generator is the iterated over and the
+ resulting HTML is output.
+
+ Otherwise, `format()` calls `self.wrap()`, which wraps the generator with
+ other generators. These may add some HTML code to the one generated by
+ `_format_lines()`, either by modifying the lines generated by the latter,
+ then yielding them again with ``(1, line)``, and/or by yielding other HTML
+ code before or after the lines, with ``(0, html)``. The distinction between
+ source lines and other code makes it possible to wrap the generator multiple
+ times.
+
+ The default `wrap()` implementation adds a ``<div>`` and a ``<pre>`` tag.
+
+ A custom `HtmlFormatter` subclass could look like this:
+
+ .. sourcecode:: python
+
+ class CodeHtmlFormatter(HtmlFormatter):
+
+ def wrap(self, source, outfile):
+ return self._wrap_code(source)
+
+ def _wrap_code(self, source):
+ yield 0, '<code>'
+ for i, t in source:
+ if i == 1:
+ # it's a line of formatted code
+ t += '<br>'
+ yield i, t
+ yield 0, '</code>'
+
+ This results in wrapping the formatted lines with a ``<code>`` tag, where the
+ source lines are broken using ``<br>`` tags.
+
+ After calling `wrap()`, the `format()` method also adds the "line numbers"
+ and/or "full document" wrappers if the respective options are set. Then, all
+ HTML yielded by the wrapped generator is output.
+ """
+
+ name = 'HTML'
+ aliases = ['html']
+ filenames = ['*.html', '*.htm']
+
+ def __init__(self, **options):
+ Formatter.__init__(self, **options)
+ self.title = self._decodeifneeded(self.title)
+ self.nowrap = get_bool_opt(options, 'nowrap', False)
+ self.noclasses = get_bool_opt(options, 'noclasses', False)
+ self.classprefix = options.get('classprefix', '')
+ self.cssclass = self._decodeifneeded(options.get('cssclass', 'highlight'))
+ self.cssstyles = self._decodeifneeded(options.get('cssstyles', ''))
+ self.prestyles = self._decodeifneeded(options.get('prestyles', ''))
+ self.cssfile = self._decodeifneeded(options.get('cssfile', ''))
+ self.noclobber_cssfile = get_bool_opt(options, 'noclobber_cssfile', False)
+ self.tagsfile = self._decodeifneeded(options.get('tagsfile', ''))
+ self.tagurlformat = self._decodeifneeded(options.get('tagurlformat', ''))
+ self.filename = self._decodeifneeded(options.get('filename', ''))
+
+ if self.tagsfile:
+ if not ctags:
+ raise RuntimeError('The "ctags" package must to be installed '
+ 'to be able to use the "tagsfile" feature.')
+ self._ctags = ctags.CTags(self.tagsfile)
+
+ linenos = options.get('linenos', False)
+ if linenos == 'inline':
+ self.linenos = 2
+ elif linenos:
+ # compatibility with <= 0.7
+ self.linenos = 1
+ else:
+ self.linenos = 0
+ self.linenostart = abs(get_int_opt(options, 'linenostart', 1))
+ self.linenostep = abs(get_int_opt(options, 'linenostep', 1))
+ self.linenospecial = abs(get_int_opt(options, 'linenospecial', 0))
+ self.nobackground = get_bool_opt(options, 'nobackground', False)
+ self.lineseparator = options.get('lineseparator', '\n')
+ self.lineanchors = options.get('lineanchors', '')
+ self.linespans = options.get('linespans', '')
+ self.anchorlinenos = options.get('anchorlinenos', False)
+ self.hl_lines = set()
+ for lineno in get_list_opt(options, 'hl_lines', []):
+ try:
+ self.hl_lines.add(int(lineno))
+ except ValueError:
+ pass
+
+ self._create_stylesheet()
+
+ def _get_css_class(self, ttype):
+ """Return the css class of this token type prefixed with
+ the classprefix option."""
+ ttypeclass = _get_ttype_class(ttype)
+ if ttypeclass:
+ return self.classprefix + ttypeclass
+ return ''
+
+ def _get_css_classes(self, ttype):
+ """Return the css classes of this token type prefixed with
+ the classprefix option."""
+ cls = self._get_css_class(ttype)
+ while ttype not in STANDARD_TYPES:
+ ttype = ttype.parent
+ cls = self._get_css_class(ttype) + ' ' + cls
+ return cls
+
+ def _create_stylesheet(self):
+ t2c = self.ttype2class = {Token: ''}
+ c2s = self.class2style = {}
+ for ttype, ndef in self.style:
+ name = self._get_css_class(ttype)
+ style = ''
+ if ndef['color']:
+ style += 'color: #%s; ' % ndef['color']
+ if ndef['bold']:
+ style += 'font-weight: bold; '
+ if ndef['italic']:
+ style += 'font-style: italic; '
+ if ndef['underline']:
+ style += 'text-decoration: underline; '
+ if ndef['bgcolor']:
+ style += 'background-color: #%s; ' % ndef['bgcolor']
+ if ndef['border']:
+ style += 'border: 1px solid #%s; ' % ndef['border']
+ if style:
+ t2c[ttype] = name
+ # save len(ttype) to enable ordering the styles by
+ # hierarchy (necessary for CSS cascading rules!)
+ c2s[name] = (style[:-2], ttype, len(ttype))
+
+ def get_style_defs(self, arg=None):
+ """
+ Return CSS style definitions for the classes produced by the current
+ highlighting style. ``arg`` can be a string or list of selectors to
+ insert before the token type classes.
+ """
+ if arg is None:
+ arg = ('cssclass' in self.options and '.'+self.cssclass or '')
+ if isinstance(arg, string_types):
+ args = [arg]
+ else:
+ args = list(arg)
+
+ def prefix(cls):
+ if cls:
+ cls = '.' + cls
+ tmp = []
+ for arg in args:
+ tmp.append((arg and arg + ' ' or '') + cls)
+ return ', '.join(tmp)
+
+ styles = [(level, ttype, cls, style)
+ for cls, (style, ttype, level) in iteritems(self.class2style)
+ if cls and style]
+ styles.sort()
+ lines = ['%s { %s } /* %s */' % (prefix(cls), style, repr(ttype)[6:])
+ for (level, ttype, cls, style) in styles]
+ if arg and not self.nobackground and \
+ self.style.background_color is not None:
+ text_style = ''
+ if Text in self.ttype2class:
+ text_style = ' ' + self.class2style[self.ttype2class[Text]][0]
+ lines.insert(0, '%s { background: %s;%s }' %
+ (prefix(''), self.style.background_color, text_style))
+ if self.style.highlight_color is not None:
+ lines.insert(0, '%s.hll { background-color: %s }' %
+ (prefix(''), self.style.highlight_color))
+ return '\n'.join(lines)
+
+ def _decodeifneeded(self, value):
+ if isinstance(value, bytes):
+ if self.encoding:
+ return value.decode(self.encoding)
+ return value.decode()
+ return value
+
+ def _wrap_full(self, inner, outfile):
+ if self.cssfile:
+ if os.path.isabs(self.cssfile):
+ # it's an absolute filename
+ cssfilename = self.cssfile
+ else:
+ try:
+ filename = outfile.name
+ if not filename or filename[0] == '<':
+ # pseudo files, e.g. name == '<fdopen>'
+ raise AttributeError
+ cssfilename = os.path.join(os.path.dirname(filename),
+ self.cssfile)
+ except AttributeError:
+ print('Note: Cannot determine output file name, '
+ 'using current directory as base for the CSS file name',
+ file=sys.stderr)
+ cssfilename = self.cssfile
+ # write CSS file only if noclobber_cssfile isn't given as an option.
+ try:
+ if not os.path.exists(cssfilename) or not self.noclobber_cssfile:
+ cf = open(cssfilename, "w")
+ cf.write(CSSFILE_TEMPLATE %
+ {'styledefs': self.get_style_defs('body')})
+ cf.close()
+ except IOError as err:
+ err.strerror = 'Error writing CSS file: ' + err.strerror
+ raise
+
+ yield 0, (DOC_HEADER_EXTERNALCSS %
+ dict(title=self.title,
+ cssfile=self.cssfile,
+ encoding=self.encoding))
+ else:
+ yield 0, (DOC_HEADER %
+ dict(title=self.title,
+ styledefs=self.get_style_defs('body'),
+ encoding=self.encoding))
+
+ for t, line in inner:
+ yield t, line
+ yield 0, DOC_FOOTER
+
+ def _wrap_tablelinenos(self, inner):
+ dummyoutfile = StringIO()
+ lncount = 0
+ for t, line in inner:
+ if t:
+ lncount += 1
+ dummyoutfile.write(line)
+
+ fl = self.linenostart
+ mw = len(str(lncount + fl - 1))
+ sp = self.linenospecial
+ st = self.linenostep
+ la = self.lineanchors
+ aln = self.anchorlinenos
+ nocls = self.noclasses
+ if sp:
+ lines = []
+
+ for i in range(fl, fl+lncount):
+ if i % st == 0:
+ if i % sp == 0:
+ if aln:
+ lines.append('<a href="#%s-%d" class="special">%*d</a>' %
+ (la, i, mw, i))
+ else:
+ lines.append('<span class="special">%*d</span>' % (mw, i))
+ else:
+ if aln:
+ lines.append('<a href="#%s-%d">%*d</a>' % (la, i, mw, i))
+ else:
+ lines.append('%*d' % (mw, i))
+ else:
+ lines.append('')
+ ls = '\n'.join(lines)
+ else:
+ lines = []
+ for i in range(fl, fl+lncount):
+ if i % st == 0:
+ if aln:
+ lines.append('<a href="#%s-%d">%*d</a>' % (la, i, mw, i))
+ else:
+ lines.append('%*d' % (mw, i))
+ else:
+ lines.append('')
+ ls = '\n'.join(lines)
+
+ # in case you wonder about the seemingly redundant <div> here: since the
+ # content in the other cell also is wrapped in a div, some browsers in
+ # some configurations seem to mess up the formatting...
+ if nocls:
+ yield 0, ('<table class="%stable">' % self.cssclass +
+ '<tr><td><div class="linenodiv" '
+ 'style="background-color: #f0f0f0; padding-right: 10px">'
+ '<pre style="line-height: 125%">' +
+ ls + '</pre></div></td><td class="code">')
+ else:
+ yield 0, ('<table class="%stable">' % self.cssclass +
+ '<tr><td class="linenos"><div class="linenodiv"><pre>' +
+ ls + '</pre></div></td><td class="code">')
+ yield 0, dummyoutfile.getvalue()
+ yield 0, '</td></tr></table>'
+
+ def _wrap_inlinelinenos(self, inner):
+ # need a list of lines since we need the width of a single number :(
+ lines = list(inner)
+ sp = self.linenospecial
+ st = self.linenostep
+ num = self.linenostart
+ mw = len(str(len(lines) + num - 1))
+
+ if self.noclasses:
+ if sp:
+ for t, line in lines:
+ if num % sp == 0:
+ style = 'background-color: #ffffc0; padding: 0 5px 0 5px'
+ else:
+ style = 'background-color: #f0f0f0; padding: 0 5px 0 5px'
+ yield 1, '<span style="%s">%*s </span>' % (
+ style, mw, (num % st and ' ' or num)) + line
+ num += 1
+ else:
+ for t, line in lines:
+ yield 1, ('<span style="background-color: #f0f0f0; '
+ 'padding: 0 5px 0 5px">%*s </span>' % (
+ mw, (num % st and ' ' or num)) + line)
+ num += 1
+ elif sp:
+ for t, line in lines:
+ yield 1, '<span class="lineno%s">%*s </span>' % (
+ num % sp == 0 and ' special' or '', mw,
+ (num % st and ' ' or num)) + line
+ num += 1
+ else:
+ for t, line in lines:
+ yield 1, '<span class="lineno">%*s </span>' % (
+ mw, (num % st and ' ' or num)) + line
+ num += 1
+
+ def _wrap_lineanchors(self, inner):
+ s = self.lineanchors
+ # subtract 1 since we have to increment i *before* yielding
+ i = self.linenostart - 1
+ for t, line in inner:
+ if t:
+ i += 1
+ yield 1, '<a name="%s-%d"></a>' % (s, i) + line
+ else:
+ yield 0, line
+
+ def _wrap_linespans(self, inner):
+ s = self.linespans
+ i = self.linenostart - 1
+ for t, line in inner:
+ if t:
+ i += 1
+ yield 1, '<span id="%s-%d">%s</span>' % (s, i, line)
+ else:
+ yield 0, line
+
+ def _wrap_div(self, inner):
+ style = []
+ if (self.noclasses and not self.nobackground and
+ self.style.background_color is not None):
+ style.append('background: %s' % (self.style.background_color,))
+ if self.cssstyles:
+ style.append(self.cssstyles)
+ style = '; '.join(style)
+
+ yield 0, ('<div' + (self.cssclass and ' class="%s"' % self.cssclass) +
+ (style and (' style="%s"' % style)) + '>')
+ for tup in inner:
+ yield tup
+ yield 0, '</div>\n'
+
+ def _wrap_pre(self, inner):
+ style = []
+ if self.prestyles:
+ style.append(self.prestyles)
+ if self.noclasses:
+ style.append('line-height: 125%')
+ style = '; '.join(style)
+
+ if self.filename:
+ yield 0, ('<span class="filename">' + self.filename + '</span>')
+
+ # the empty span here is to keep leading empty lines from being
+ # ignored by HTML parsers
+ yield 0, ('<pre' + (style and ' style="%s"' % style) + '><span></span>')
+ for tup in inner:
+ yield tup
+ yield 0, '</pre>'
+
+ def _format_lines(self, tokensource):
+ """
+ Just format the tokens, without any wrapping tags.
+ Yield individual lines.
+ """
+ nocls = self.noclasses
+ lsep = self.lineseparator
+ # for <span style=""> lookup only
+ getcls = self.ttype2class.get
+ c2s = self.class2style
+ escape_table = _escape_html_table
+ tagsfile = self.tagsfile
+
+ lspan = ''
+ line = []
+ for ttype, value in tokensource:
+ if nocls:
+ cclass = getcls(ttype)
+ while cclass is None:
+ ttype = ttype.parent
+ cclass = getcls(ttype)
+ cspan = cclass and '<span style="%s">' % c2s[cclass][0] or ''
+ else:
+ cls = self._get_css_classes(ttype)
+ cspan = cls and '<span class="%s">' % cls or ''
+
+ parts = value.translate(escape_table).split('\n')
+
+ if tagsfile and ttype in Token.Name:
+ filename, linenumber = self._lookup_ctag(value)
+ if linenumber:
+ base, filename = os.path.split(filename)
+ if base:
+ base += '/'
+ filename, extension = os.path.splitext(filename)
+ url = self.tagurlformat % {'path': base, 'fname': filename,
+ 'fext': extension}
+ parts[0] = "<a href=\"%s#%s-%d\">%s" % \
+ (url, self.lineanchors, linenumber, parts[0])
+ parts[-1] = parts[-1] + "</a>"
+
+ # for all but the last line
+ for part in parts[:-1]:
+ if line:
+ if lspan != cspan:
+ line.extend(((lspan and '</span>'), cspan, part,
+ (cspan and '</span>'), lsep))
+ else: # both are the same
+ line.extend((part, (lspan and '</span>'), lsep))
+ yield 1, ''.join(line)
+ line = []
+ elif part:
+ yield 1, ''.join((cspan, part, (cspan and '</span>'), lsep))
+ else:
+ yield 1, lsep
+ # for the last line
+ if line and parts[-1]:
+ if lspan != cspan:
+ line.extend(((lspan and '</span>'), cspan, parts[-1]))
+ lspan = cspan
+ else:
+ line.append(parts[-1])
+ elif parts[-1]:
+ line = [cspan, parts[-1]]
+ lspan = cspan
+ # else we neither have to open a new span nor set lspan
+
+ if line:
+ line.extend(((lspan and '</span>'), lsep))
+ yield 1, ''.join(line)
+
+ def _lookup_ctag(self, token):
+ entry = ctags.TagEntry()
+ if self._ctags.find(entry, token, 0):
+ return entry['file'], entry['lineNumber']
+ else:
+ return None, None
+
+ def _highlight_lines(self, tokensource):
+ """
+ Highlighted the lines specified in the `hl_lines` option by
+ post-processing the token stream coming from `_format_lines`.
+ """
+ hls = self.hl_lines
+
+ for i, (t, value) in enumerate(tokensource):
+ if t != 1:
+ yield t, value
+ if i + 1 in hls: # i + 1 because Python indexes start at 0
+ if self.noclasses:
+ style = ''
+ if self.style.highlight_color is not None:
+ style = (' style="background-color: %s"' %
+ (self.style.highlight_color,))
+ yield 1, '<span%s>%s</span>' % (style, value)
+ else:
+ yield 1, '<span class="hll">%s</span>' % value
+ else:
+ yield 1, value
+
+ def wrap(self, source, outfile):
+ """
+ Wrap the ``source``, which is a generator yielding
+ individual lines, in custom generators. See docstring
+ for `format`. Can be overridden.
+ """
+ return self._wrap_div(self._wrap_pre(source))
+
+ def format_unencoded(self, tokensource, outfile):
+ """
+ The formatting process uses several nested generators; which of
+ them are used is determined by the user's options.
+
+ Each generator should take at least one argument, ``inner``,
+ and wrap the pieces of text generated by this.
+
+ Always yield 2-tuples: (code, text). If "code" is 1, the text
+ is part of the original tokensource being highlighted, if it's
+ 0, the text is some piece of wrapping. This makes it possible to
+ use several different wrappers that process the original source
+ linewise, e.g. line number generators.
+ """
+ source = self._format_lines(tokensource)
+ if self.hl_lines:
+ source = self._highlight_lines(source)
+ if not self.nowrap:
+ if self.linenos == 2:
+ source = self._wrap_inlinelinenos(source)
+ if self.lineanchors:
+ source = self._wrap_lineanchors(source)
+ if self.linespans:
+ source = self._wrap_linespans(source)
+ source = self.wrap(source, outfile)
+ if self.linenos == 1:
+ source = self._wrap_tablelinenos(source)
+ if self.full:
+ source = self._wrap_full(source, outfile)
+
+ for t, piece in source:
+ outfile.write(piece)
diff --git a/wandb/vendor/pygments/formatters/img.py b/wandb/vendor/pygments/formatters/img.py
new file mode 100644
index 0000000000000000000000000000000000000000..2fb0dea5a8c97df5114f9db8c814c8415b56a2cb
--- /dev/null
+++ b/wandb/vendor/pygments/formatters/img.py
@@ -0,0 +1,600 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.formatters.img
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Formatter for Pixmap output.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import os
+import sys
+
+from pygments.formatter import Formatter
+from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
+ get_choice_opt, xrange
+
+import subprocess
+
+# Import this carefully
+try:
+ from PIL import Image, ImageDraw, ImageFont
+ pil_available = True
+except ImportError:
+ pil_available = False
+
+try:
+ import _winreg
+except ImportError:
+ try:
+ import winreg as _winreg
+ except ImportError:
+ _winreg = None
+
+__all__ = ['ImageFormatter', 'GifImageFormatter', 'JpgImageFormatter',
+ 'BmpImageFormatter']
+
+
+# For some unknown reason every font calls it something different
+STYLES = {
+ 'NORMAL': ['', 'Roman', 'Book', 'Normal', 'Regular', 'Medium'],
+ 'ITALIC': ['Oblique', 'Italic'],
+ 'BOLD': ['Bold'],
+ 'BOLDITALIC': ['Bold Oblique', 'Bold Italic'],
+}
+
+# A sane default for modern systems
+DEFAULT_FONT_NAME_NIX = 'Bitstream Vera Sans Mono'
+DEFAULT_FONT_NAME_WIN = 'Courier New'
+DEFAULT_FONT_NAME_MAC = 'Courier New'
+
+
+class PilNotAvailable(ImportError):
+ """When Python imaging library is not available"""
+
+
+class FontNotFound(Exception):
+ """When there are no usable fonts specified"""
+
+
+class FontManager(object):
+ """
+ Manages a set of fonts: normal, italic, bold, etc...
+ """
+
+ def __init__(self, font_name, font_size=14):
+ self.font_name = font_name
+ self.font_size = font_size
+ self.fonts = {}
+ self.encoding = None
+ if sys.platform.startswith('win'):
+ if not font_name:
+ self.font_name = DEFAULT_FONT_NAME_WIN
+ self._create_win()
+ elif sys.platform.startswith('darwin'):
+ if not font_name:
+ self.font_name = DEFAULT_FONT_NAME_MAC
+ self._create_mac()
+ else:
+ if not font_name:
+ self.font_name = DEFAULT_FONT_NAME_NIX
+ self._create_nix()
+
+ def _get_nix_font_path(self, name, style):
+ proc = subprocess.Popen(['fc-list', "%s:style=%s" % (name, style), 'file'],
+ stdout=subprocess.PIPE, stderr=None)
+ stdout, _ = proc.communicate()
+ if proc.returncode == 0:
+ lines = stdout.splitlines()
+ for line in lines:
+ if line.startswith(b'Fontconfig warning:'):
+ continue
+ path = line.decode().strip().strip(':')
+ if path:
+ return path
+ return None
+
+ def _create_nix(self):
+ for name in STYLES['NORMAL']:
+ path = self._get_nix_font_path(self.font_name, name)
+ if path is not None:
+ self.fonts['NORMAL'] = ImageFont.truetype(path, self.font_size)
+ break
+ else:
+ raise FontNotFound('No usable fonts named: "%s"' %
+ self.font_name)
+ for style in ('ITALIC', 'BOLD', 'BOLDITALIC'):
+ for stylename in STYLES[style]:
+ path = self._get_nix_font_path(self.font_name, stylename)
+ if path is not None:
+ self.fonts[style] = ImageFont.truetype(path, self.font_size)
+ break
+ else:
+ if style == 'BOLDITALIC':
+ self.fonts[style] = self.fonts['BOLD']
+ else:
+ self.fonts[style] = self.fonts['NORMAL']
+
+ def _get_mac_font_path(self, font_map, name, style):
+ return font_map.get((name + ' ' + style).strip().lower())
+
+ def _create_mac(self):
+ font_map = {}
+ for font_dir in (os.path.join(os.getenv("HOME"), 'Library/Fonts/'),
+ '/Library/Fonts/', '/System/Library/Fonts/'):
+ font_map.update(
+ ((os.path.splitext(f)[0].lower(), os.path.join(font_dir, f))
+ for f in os.listdir(font_dir) if f.lower().endswith('ttf')))
+
+ for name in STYLES['NORMAL']:
+ path = self._get_mac_font_path(font_map, self.font_name, name)
+ if path is not None:
+ self.fonts['NORMAL'] = ImageFont.truetype(path, self.font_size)
+ break
+ else:
+ raise FontNotFound('No usable fonts named: "%s"' %
+ self.font_name)
+ for style in ('ITALIC', 'BOLD', 'BOLDITALIC'):
+ for stylename in STYLES[style]:
+ path = self._get_mac_font_path(font_map, self.font_name, stylename)
+ if path is not None:
+ self.fonts[style] = ImageFont.truetype(path, self.font_size)
+ break
+ else:
+ if style == 'BOLDITALIC':
+ self.fonts[style] = self.fonts['BOLD']
+ else:
+ self.fonts[style] = self.fonts['NORMAL']
+
+ def _lookup_win(self, key, basename, styles, fail=False):
+ for suffix in ('', ' (TrueType)'):
+ for style in styles:
+ try:
+ valname = '%s%s%s' % (basename, style and ' '+style, suffix)
+ val, _ = _winreg.QueryValueEx(key, valname)
+ return val
+ except EnvironmentError:
+ continue
+ else:
+ if fail:
+ raise FontNotFound('Font %s (%s) not found in registry' %
+ (basename, styles[0]))
+ return None
+
+ def _create_win(self):
+ try:
+ key = _winreg.OpenKey(
+ _winreg.HKEY_LOCAL_MACHINE,
+ r'Software\Microsoft\Windows NT\CurrentVersion\Fonts')
+ except EnvironmentError:
+ try:
+ key = _winreg.OpenKey(
+ _winreg.HKEY_LOCAL_MACHINE,
+ r'Software\Microsoft\Windows\CurrentVersion\Fonts')
+ except EnvironmentError:
+ raise FontNotFound('Can\'t open Windows font registry key')
+ try:
+ path = self._lookup_win(key, self.font_name, STYLES['NORMAL'], True)
+ self.fonts['NORMAL'] = ImageFont.truetype(path, self.font_size)
+ for style in ('ITALIC', 'BOLD', 'BOLDITALIC'):
+ path = self._lookup_win(key, self.font_name, STYLES[style])
+ if path:
+ self.fonts[style] = ImageFont.truetype(path, self.font_size)
+ else:
+ if style == 'BOLDITALIC':
+ self.fonts[style] = self.fonts['BOLD']
+ else:
+ self.fonts[style] = self.fonts['NORMAL']
+ finally:
+ _winreg.CloseKey(key)
+
+ def get_char_size(self):
+ """
+ Get the character size.
+ """
+ return self.fonts['NORMAL'].getsize('M')
+
+ def get_font(self, bold, oblique):
+ """
+ Get the font based on bold and italic flags.
+ """
+ if bold and oblique:
+ return self.fonts['BOLDITALIC']
+ elif bold:
+ return self.fonts['BOLD']
+ elif oblique:
+ return self.fonts['ITALIC']
+ else:
+ return self.fonts['NORMAL']
+
+
+class ImageFormatter(Formatter):
+ """
+ Create a PNG image from source code. This uses the Python Imaging Library to
+ generate a pixmap from the source code.
+
+ .. versionadded:: 0.10
+
+ Additional options accepted:
+
+ `image_format`
+ An image format to output to that is recognised by PIL, these include:
+
+ * "PNG" (default)
+ * "JPEG"
+ * "BMP"
+ * "GIF"
+
+ `line_pad`
+ The extra spacing (in pixels) between each line of text.
+
+ Default: 2
+
+ `font_name`
+ The font name to be used as the base font from which others, such as
+ bold and italic fonts will be generated. This really should be a
+ monospace font to look sane.
+
+ Default: "Bitstream Vera Sans Mono" on Windows, Courier New on \*nix
+
+ `font_size`
+ The font size in points to be used.
+
+ Default: 14
+
+ `image_pad`
+ The padding, in pixels to be used at each edge of the resulting image.
+
+ Default: 10
+
+ `line_numbers`
+ Whether line numbers should be shown: True/False
+
+ Default: True
+
+ `line_number_start`
+ The line number of the first line.
+
+ Default: 1
+
+ `line_number_step`
+ The step used when printing line numbers.
+
+ Default: 1
+
+ `line_number_bg`
+ The background colour (in "#123456" format) of the line number bar, or
+ None to use the style background color.
+
+ Default: "#eed"
+
+ `line_number_fg`
+ The text color of the line numbers (in "#123456"-like format).
+
+ Default: "#886"
+
+ `line_number_chars`
+ The number of columns of line numbers allowable in the line number
+ margin.
+
+ Default: 2
+
+ `line_number_bold`
+ Whether line numbers will be bold: True/False
+
+ Default: False
+
+ `line_number_italic`
+ Whether line numbers will be italicized: True/False
+
+ Default: False
+
+ `line_number_separator`
+ Whether a line will be drawn between the line number area and the
+ source code area: True/False
+
+ Default: True
+
+ `line_number_pad`
+ The horizontal padding (in pixels) between the line number margin, and
+ the source code area.
+
+ Default: 6
+
+ `hl_lines`
+ Specify a list of lines to be highlighted.
+
+ .. versionadded:: 1.2
+
+ Default: empty list
+
+ `hl_color`
+ Specify the color for highlighting lines.
+
+ .. versionadded:: 1.2
+
+ Default: highlight color of the selected style
+ """
+
+ # Required by the pygments mapper
+ name = 'img'
+ aliases = ['img', 'IMG', 'png']
+ filenames = ['*.png']
+
+ unicodeoutput = False
+
+ default_image_format = 'png'
+
+ def __init__(self, **options):
+ """
+ See the class docstring for explanation of options.
+ """
+ if not pil_available:
+ raise PilNotAvailable(
+ 'Python Imaging Library is required for this formatter')
+ Formatter.__init__(self, **options)
+ self.encoding = 'latin1' # let pygments.format() do the right thing
+ # Read the style
+ self.styles = dict(self.style)
+ if self.style.background_color is None:
+ self.background_color = '#fff'
+ else:
+ self.background_color = self.style.background_color
+ # Image options
+ self.image_format = get_choice_opt(
+ options, 'image_format', ['png', 'jpeg', 'gif', 'bmp'],
+ self.default_image_format, normcase=True)
+ self.image_pad = get_int_opt(options, 'image_pad', 10)
+ self.line_pad = get_int_opt(options, 'line_pad', 2)
+ # The fonts
+ fontsize = get_int_opt(options, 'font_size', 14)
+ self.fonts = FontManager(options.get('font_name', ''), fontsize)
+ self.fontw, self.fonth = self.fonts.get_char_size()
+ # Line number options
+ self.line_number_fg = options.get('line_number_fg', '#886')
+ self.line_number_bg = options.get('line_number_bg', '#eed')
+ self.line_number_chars = get_int_opt(options,
+ 'line_number_chars', 2)
+ self.line_number_bold = get_bool_opt(options,
+ 'line_number_bold', False)
+ self.line_number_italic = get_bool_opt(options,
+ 'line_number_italic', False)
+ self.line_number_pad = get_int_opt(options, 'line_number_pad', 6)
+ self.line_numbers = get_bool_opt(options, 'line_numbers', True)
+ self.line_number_separator = get_bool_opt(options,
+ 'line_number_separator', True)
+ self.line_number_step = get_int_opt(options, 'line_number_step', 1)
+ self.line_number_start = get_int_opt(options, 'line_number_start', 1)
+ if self.line_numbers:
+ self.line_number_width = (self.fontw * self.line_number_chars +
+ self.line_number_pad * 2)
+ else:
+ self.line_number_width = 0
+ self.hl_lines = []
+ hl_lines_str = get_list_opt(options, 'hl_lines', [])
+ for line in hl_lines_str:
+ try:
+ self.hl_lines.append(int(line))
+ except ValueError:
+ pass
+ self.hl_color = options.get('hl_color',
+ self.style.highlight_color) or '#f90'
+ self.drawables = []
+
+ def get_style_defs(self, arg=''):
+ raise NotImplementedError('The -S option is meaningless for the image '
+ 'formatter. Use -O style=<stylename> instead.')
+
+ def _get_line_height(self):
+ """
+ Get the height of a line.
+ """
+ return self.fonth + self.line_pad
+
+ def _get_line_y(self, lineno):
+ """
+ Get the Y coordinate of a line number.
+ """
+ return lineno * self._get_line_height() + self.image_pad
+
+ def _get_char_width(self):
+ """
+ Get the width of a character.
+ """
+ return self.fontw
+
+ def _get_char_x(self, charno):
+ """
+ Get the X coordinate of a character position.
+ """
+ return charno * self.fontw + self.image_pad + self.line_number_width
+
+ def _get_text_pos(self, charno, lineno):
+ """
+ Get the actual position for a character and line position.
+ """
+ return self._get_char_x(charno), self._get_line_y(lineno)
+
+ def _get_linenumber_pos(self, lineno):
+ """
+ Get the actual position for the start of a line number.
+ """
+ return (self.image_pad, self._get_line_y(lineno))
+
+ def _get_text_color(self, style):
+ """
+ Get the correct color for the token from the style.
+ """
+ if style['color'] is not None:
+ fill = '#' + style['color']
+ else:
+ fill = '#000'
+ return fill
+
+ def _get_style_font(self, style):
+ """
+ Get the correct font for the style.
+ """
+ return self.fonts.get_font(style['bold'], style['italic'])
+
+ def _get_image_size(self, maxcharno, maxlineno):
+ """
+ Get the required image size.
+ """
+ return (self._get_char_x(maxcharno) + self.image_pad,
+ self._get_line_y(maxlineno + 0) + self.image_pad)
+
+ def _draw_linenumber(self, posno, lineno):
+ """
+ Remember a line number drawable to paint later.
+ """
+ self._draw_text(
+ self._get_linenumber_pos(posno),
+ str(lineno).rjust(self.line_number_chars),
+ font=self.fonts.get_font(self.line_number_bold,
+ self.line_number_italic),
+ fill=self.line_number_fg,
+ )
+
+ def _draw_text(self, pos, text, font, **kw):
+ """
+ Remember a single drawable tuple to paint later.
+ """
+ self.drawables.append((pos, text, font, kw))
+
+ def _create_drawables(self, tokensource):
+ """
+ Create drawables for the token content.
+ """
+ lineno = charno = maxcharno = 0
+ for ttype, value in tokensource:
+ while ttype not in self.styles:
+ ttype = ttype.parent
+ style = self.styles[ttype]
+ # TODO: make sure tab expansion happens earlier in the chain. It
+ # really ought to be done on the input, as to do it right here is
+ # quite complex.
+ value = value.expandtabs(4)
+ lines = value.splitlines(True)
+ # print lines
+ for i, line in enumerate(lines):
+ temp = line.rstrip('\n')
+ if temp:
+ self._draw_text(
+ self._get_text_pos(charno, lineno),
+ temp,
+ font = self._get_style_font(style),
+ fill = self._get_text_color(style)
+ )
+ charno += len(temp)
+ maxcharno = max(maxcharno, charno)
+ if line.endswith('\n'):
+ # add a line for each extra line in the value
+ charno = 0
+ lineno += 1
+ self.maxcharno = maxcharno
+ self.maxlineno = lineno
+
+ def _draw_line_numbers(self):
+ """
+ Create drawables for the line numbers.
+ """
+ if not self.line_numbers:
+ return
+ for p in xrange(self.maxlineno):
+ n = p + self.line_number_start
+ if (n % self.line_number_step) == 0:
+ self._draw_linenumber(p, n)
+
+ def _paint_line_number_bg(self, im):
+ """
+ Paint the line number background on the image.
+ """
+ if not self.line_numbers:
+ return
+ if self.line_number_fg is None:
+ return
+ draw = ImageDraw.Draw(im)
+ recth = im.size[-1]
+ rectw = self.image_pad + self.line_number_width - self.line_number_pad
+ draw.rectangle([(0, 0), (rectw, recth)],
+ fill=self.line_number_bg)
+ draw.line([(rectw, 0), (rectw, recth)], fill=self.line_number_fg)
+ del draw
+
+ def format(self, tokensource, outfile):
+ """
+ Format ``tokensource``, an iterable of ``(tokentype, tokenstring)``
+ tuples and write it into ``outfile``.
+
+ This implementation calculates where it should draw each token on the
+ pixmap, then calculates the required pixmap size and draws the items.
+ """
+ self._create_drawables(tokensource)
+ self._draw_line_numbers()
+ im = Image.new(
+ 'RGB',
+ self._get_image_size(self.maxcharno, self.maxlineno),
+ self.background_color
+ )
+ self._paint_line_number_bg(im)
+ draw = ImageDraw.Draw(im)
+ # Highlight
+ if self.hl_lines:
+ x = self.image_pad + self.line_number_width - self.line_number_pad + 1
+ recth = self._get_line_height()
+ rectw = im.size[0] - x
+ for linenumber in self.hl_lines:
+ y = self._get_line_y(linenumber - 1)
+ draw.rectangle([(x, y), (x + rectw, y + recth)],
+ fill=self.hl_color)
+ for pos, value, font, kw in self.drawables:
+ draw.text(pos, value, font=font, **kw)
+ im.save(outfile, self.image_format.upper())
+
+
+# Add one formatter per format, so that the "-f gif" option gives the correct result
+# when used in pygmentize.
+
+class GifImageFormatter(ImageFormatter):
+ """
+ Create a GIF image from source code. This uses the Python Imaging Library to
+ generate a pixmap from the source code.
+
+ .. versionadded:: 1.0
+ """
+
+ name = 'img_gif'
+ aliases = ['gif']
+ filenames = ['*.gif']
+ default_image_format = 'gif'
+
+
+class JpgImageFormatter(ImageFormatter):
+ """
+ Create a JPEG image from source code. This uses the Python Imaging Library to
+ generate a pixmap from the source code.
+
+ .. versionadded:: 1.0
+ """
+
+ name = 'img_jpg'
+ aliases = ['jpg', 'jpeg']
+ filenames = ['*.jpg']
+ default_image_format = 'jpeg'
+
+
+class BmpImageFormatter(ImageFormatter):
+ """
+ Create a bitmap image from source code. This uses the Python Imaging Library to
+ generate a pixmap from the source code.
+
+ .. versionadded:: 1.0
+ """
+
+ name = 'img_bmp'
+ aliases = ['bmp', 'bitmap']
+ filenames = ['*.bmp']
+ default_image_format = 'bmp'
diff --git a/wandb/vendor/pygments/formatters/irc.py b/wandb/vendor/pygments/formatters/irc.py
new file mode 100644
index 0000000000000000000000000000000000000000..eb744d74e7f0a62ff898af393d677191d3fc96af
--- /dev/null
+++ b/wandb/vendor/pygments/formatters/irc.py
@@ -0,0 +1,182 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.formatters.irc
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Formatter for IRC output
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import sys
+
+from pygments.formatter import Formatter
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Token, Whitespace
+from pygments.util import get_choice_opt
+
+
+__all__ = ['IRCFormatter']
+
+
+#: Map token types to a tuple of color values for light and dark
+#: backgrounds.
+IRC_COLORS = {
+ Token: ('', ''),
+
+ Whitespace: ('lightgray', 'darkgray'),
+ Comment: ('lightgray', 'darkgray'),
+ Comment.Preproc: ('teal', 'turquoise'),
+ Keyword: ('darkblue', 'blue'),
+ Keyword.Type: ('teal', 'turquoise'),
+ Operator.Word: ('purple', 'fuchsia'),
+ Name.Builtin: ('teal', 'turquoise'),
+ Name.Function: ('darkgreen', 'green'),
+ Name.Namespace: ('_teal_', '_turquoise_'),
+ Name.Class: ('_darkgreen_', '_green_'),
+ Name.Exception: ('teal', 'turquoise'),
+ Name.Decorator: ('darkgray', 'lightgray'),
+ Name.Variable: ('darkred', 'red'),
+ Name.Constant: ('darkred', 'red'),
+ Name.Attribute: ('teal', 'turquoise'),
+ Name.Tag: ('blue', 'blue'),
+ String: ('brown', 'brown'),
+ Number: ('darkblue', 'blue'),
+
+ Generic.Deleted: ('red', 'red'),
+ Generic.Inserted: ('darkgreen', 'green'),
+ Generic.Heading: ('**', '**'),
+ Generic.Subheading: ('*purple*', '*fuchsia*'),
+ Generic.Error: ('red', 'red'),
+
+ Error: ('_red_', '_red_'),
+}
+
+
+IRC_COLOR_MAP = {
+ 'white': 0,
+ 'black': 1,
+ 'darkblue': 2,
+ 'green': 3,
+ 'red': 4,
+ 'brown': 5,
+ 'purple': 6,
+ 'orange': 7,
+ 'darkgreen': 7, #compat w/ ansi
+ 'yellow': 8,
+ 'lightgreen': 9,
+ 'turquoise': 9, # compat w/ ansi
+ 'teal': 10,
+ 'lightblue': 11,
+ 'darkred': 11, # compat w/ ansi
+ 'blue': 12,
+ 'fuchsia': 13,
+ 'darkgray': 14,
+ 'lightgray': 15,
+}
+
+def ircformat(color, text):
+ if len(color) < 1:
+ return text
+ add = sub = ''
+ if '_' in color: # italic
+ add += '\x1D'
+ sub = '\x1D' + sub
+ color = color.strip('_')
+ if '*' in color: # bold
+ add += '\x02'
+ sub = '\x02' + sub
+ color = color.strip('*')
+ # underline (\x1F) not supported
+ # backgrounds (\x03FF,BB) not supported
+ if len(color) > 0: # actual color - may have issues with ircformat("red", "blah")+"10" type stuff
+ add += '\x03' + str(IRC_COLOR_MAP[color]).zfill(2)
+ sub = '\x03' + sub
+ return add + text + sub
+ return '<'+add+'>'+text+'</'+sub+'>'
+
+
+class IRCFormatter(Formatter):
+ r"""
+ Format tokens with IRC color sequences
+
+ The `get_style_defs()` method doesn't do anything special since there is
+ no support for common styles.
+
+ Options accepted:
+
+ `bg`
+ Set to ``"light"`` or ``"dark"`` depending on the terminal's background
+ (default: ``"light"``).
+
+ `colorscheme`
+ A dictionary mapping token types to (lightbg, darkbg) color names or
+ ``None`` (default: ``None`` = use builtin colorscheme).
+
+ `linenos`
+ Set to ``True`` to have line numbers in the output as well
+ (default: ``False`` = no line numbers).
+ """
+ name = 'IRC'
+ aliases = ['irc', 'IRC']
+ filenames = []
+
+ def __init__(self, **options):
+ Formatter.__init__(self, **options)
+ self.darkbg = get_choice_opt(options, 'bg',
+ ['light', 'dark'], 'light') == 'dark'
+ self.colorscheme = options.get('colorscheme', None) or IRC_COLORS
+ self.linenos = options.get('linenos', False)
+ self._lineno = 0
+
+ def _write_lineno(self, outfile):
+ self._lineno += 1
+ outfile.write("\n%04d: " % self._lineno)
+
+ def _format_unencoded_with_lineno(self, tokensource, outfile):
+ self._write_lineno(outfile)
+
+ for ttype, value in tokensource:
+ if value.endswith("\n"):
+ self._write_lineno(outfile)
+ value = value[:-1]
+ color = self.colorscheme.get(ttype)
+ while color is None:
+ ttype = ttype[:-1]
+ color = self.colorscheme.get(ttype)
+ if color:
+ color = color[self.darkbg]
+ spl = value.split('\n')
+ for line in spl[:-1]:
+ self._write_lineno(outfile)
+ if line:
+ outfile.write(ircformat(color, line[:-1]))
+ if spl[-1]:
+ outfile.write(ircformat(color, spl[-1]))
+ else:
+ outfile.write(value)
+
+ outfile.write("\n")
+
+ def format_unencoded(self, tokensource, outfile):
+ if self.linenos:
+ self._format_unencoded_with_lineno(tokensource, outfile)
+ return
+
+ for ttype, value in tokensource:
+ color = self.colorscheme.get(ttype)
+ while color is None:
+ ttype = ttype[:-1]
+ color = self.colorscheme.get(ttype)
+ if color:
+ color = color[self.darkbg]
+ spl = value.split('\n')
+ for line in spl[:-1]:
+ if line:
+ outfile.write(ircformat(color, line))
+ outfile.write('\n')
+ if spl[-1]:
+ outfile.write(ircformat(color, spl[-1]))
+ else:
+ outfile.write(value)
diff --git a/wandb/vendor/pygments/formatters/latex.py b/wandb/vendor/pygments/formatters/latex.py
new file mode 100644
index 0000000000000000000000000000000000000000..336b59ded43af27f36d29d2d4cf2ff24f7033e8b
--- /dev/null
+++ b/wandb/vendor/pygments/formatters/latex.py
@@ -0,0 +1,482 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.formatters.latex
+ ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Formatter for LaTeX fancyvrb output.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import division
+
+from pygments.formatter import Formatter
+from pygments.lexer import Lexer
+from pygments.token import Token, STANDARD_TYPES
+from pygments.util import get_bool_opt, get_int_opt, StringIO, xrange, \
+ iteritems
+
+
+__all__ = ['LatexFormatter']
+
+
+def escape_tex(text, commandprefix):
+ return text.replace('\\', '\x00'). \
+ replace('{', '\x01'). \
+ replace('}', '\x02'). \
+ replace('\x00', r'\%sZbs{}' % commandprefix). \
+ replace('\x01', r'\%sZob{}' % commandprefix). \
+ replace('\x02', r'\%sZcb{}' % commandprefix). \
+ replace('^', r'\%sZca{}' % commandprefix). \
+ replace('_', r'\%sZus{}' % commandprefix). \
+ replace('&', r'\%sZam{}' % commandprefix). \
+ replace('<', r'\%sZlt{}' % commandprefix). \
+ replace('>', r'\%sZgt{}' % commandprefix). \
+ replace('#', r'\%sZsh{}' % commandprefix). \
+ replace('%', r'\%sZpc{}' % commandprefix). \
+ replace('$', r'\%sZdl{}' % commandprefix). \
+ replace('-', r'\%sZhy{}' % commandprefix). \
+ replace("'", r'\%sZsq{}' % commandprefix). \
+ replace('"', r'\%sZdq{}' % commandprefix). \
+ replace('~', r'\%sZti{}' % commandprefix)
+
+
+DOC_TEMPLATE = r'''
+\documentclass{%(docclass)s}
+\usepackage{fancyvrb}
+\usepackage{color}
+\usepackage[%(encoding)s]{inputenc}
+%(preamble)s
+
+%(styledefs)s
+
+\begin{document}
+
+\section*{%(title)s}
+
+%(code)s
+\end{document}
+'''
+
+## Small explanation of the mess below :)
+#
+# The previous version of the LaTeX formatter just assigned a command to
+# each token type defined in the current style. That obviously is
+# problematic if the highlighted code is produced for a different style
+# than the style commands themselves.
+#
+# This version works much like the HTML formatter which assigns multiple
+# CSS classes to each <span> tag, from the most specific to the least
+# specific token type, thus falling back to the parent token type if one
+# is not defined. Here, the classes are there too and use the same short
+# forms given in token.STANDARD_TYPES.
+#
+# Highlighted code now only uses one custom command, which by default is
+# \PY and selectable by the commandprefix option (and in addition the
+# escapes \PYZat, \PYZlb and \PYZrb which haven't been renamed for
+# backwards compatibility purposes).
+#
+# \PY has two arguments: the classes, separated by +, and the text to
+# render in that style. The classes are resolved into the respective
+# style commands by magic, which serves to ignore unknown classes.
+#
+# The magic macros are:
+# * \PY@it, \PY@bf, etc. are unconditionally wrapped around the text
+# to render in \PY@do. Their definition determines the style.
+# * \PY@reset resets \PY@it etc. to do nothing.
+# * \PY@toks parses the list of classes, using magic inspired by the
+# keyval package (but modified to use plusses instead of commas
+# because fancyvrb redefines commas inside its environments).
+# * \PY@tok processes one class, calling the \PY@tok@classname command
+# if it exists.
+# * \PY@tok@classname sets the \PY@it etc. to reflect the chosen style
+# for its class.
+# * \PY resets the style, parses the classnames and then calls \PY@do.
+#
+# Tip: to read this code, print it out in substituted form using e.g.
+# >>> print STYLE_TEMPLATE % {'cp': 'PY'}
+
+STYLE_TEMPLATE = r'''
+\makeatletter
+\def\%(cp)s@reset{\let\%(cp)s@it=\relax \let\%(cp)s@bf=\relax%%
+ \let\%(cp)s@ul=\relax \let\%(cp)s@tc=\relax%%
+ \let\%(cp)s@bc=\relax \let\%(cp)s@ff=\relax}
+\def\%(cp)s@tok#1{\csname %(cp)s@tok@#1\endcsname}
+\def\%(cp)s@toks#1+{\ifx\relax#1\empty\else%%
+ \%(cp)s@tok{#1}\expandafter\%(cp)s@toks\fi}
+\def\%(cp)s@do#1{\%(cp)s@bc{\%(cp)s@tc{\%(cp)s@ul{%%
+ \%(cp)s@it{\%(cp)s@bf{\%(cp)s@ff{#1}}}}}}}
+\def\%(cp)s#1#2{\%(cp)s@reset\%(cp)s@toks#1+\relax+\%(cp)s@do{#2}}
+
+%(styles)s
+
+\def\%(cp)sZbs{\char`\\}
+\def\%(cp)sZus{\char`\_}
+\def\%(cp)sZob{\char`\{}
+\def\%(cp)sZcb{\char`\}}
+\def\%(cp)sZca{\char`\^}
+\def\%(cp)sZam{\char`\&}
+\def\%(cp)sZlt{\char`\<}
+\def\%(cp)sZgt{\char`\>}
+\def\%(cp)sZsh{\char`\#}
+\def\%(cp)sZpc{\char`\%%}
+\def\%(cp)sZdl{\char`\$}
+\def\%(cp)sZhy{\char`\-}
+\def\%(cp)sZsq{\char`\'}
+\def\%(cp)sZdq{\char`\"}
+\def\%(cp)sZti{\char`\~}
+%% for compatibility with earlier versions
+\def\%(cp)sZat{@}
+\def\%(cp)sZlb{[}
+\def\%(cp)sZrb{]}
+\makeatother
+'''
+
+
+def _get_ttype_name(ttype):
+ fname = STANDARD_TYPES.get(ttype)
+ if fname:
+ return fname
+ aname = ''
+ while fname is None:
+ aname = ttype[-1] + aname
+ ttype = ttype.parent
+ fname = STANDARD_TYPES.get(ttype)
+ return fname + aname
+
+
+class LatexFormatter(Formatter):
+ r"""
+ Format tokens as LaTeX code. This needs the `fancyvrb` and `color`
+ standard packages.
+
+ Without the `full` option, code is formatted as one ``Verbatim``
+ environment, like this:
+
+ .. sourcecode:: latex
+
+ \begin{Verbatim}[commandchars=\\\{\}]
+ \PY{k}{def }\PY{n+nf}{foo}(\PY{n}{bar}):
+ \PY{k}{pass}
+ \end{Verbatim}
+
+ The special command used here (``\PY``) and all the other macros it needs
+ are output by the `get_style_defs` method.
+
+ With the `full` option, a complete LaTeX document is output, including
+ the command definitions in the preamble.
+
+ The `get_style_defs()` method of a `LatexFormatter` returns a string
+ containing ``\def`` commands defining the macros needed inside the
+ ``Verbatim`` environments.
+
+ Additional options accepted:
+
+ `style`
+ The style to use, can be a string or a Style subclass (default:
+ ``'default'``).
+
+ `full`
+ Tells the formatter to output a "full" document, i.e. a complete
+ self-contained document (default: ``False``).
+
+ `title`
+ If `full` is true, the title that should be used to caption the
+ document (default: ``''``).
+
+ `docclass`
+ If the `full` option is enabled, this is the document class to use
+ (default: ``'article'``).
+
+ `preamble`
+ If the `full` option is enabled, this can be further preamble commands,
+ e.g. ``\usepackage`` (default: ``''``).
+
+ `linenos`
+ If set to ``True``, output line numbers (default: ``False``).
+
+ `linenostart`
+ The line number for the first line (default: ``1``).
+
+ `linenostep`
+ If set to a number n > 1, only every nth line number is printed.
+
+ `verboptions`
+ Additional options given to the Verbatim environment (see the *fancyvrb*
+ docs for possible values) (default: ``''``).
+
+ `commandprefix`
+ The LaTeX commands used to produce colored output are constructed
+ using this prefix and some letters (default: ``'PY'``).
+
+ .. versionadded:: 0.7
+ .. versionchanged:: 0.10
+ The default is now ``'PY'`` instead of ``'C'``.
+
+ `texcomments`
+ If set to ``True``, enables LaTeX comment lines. That is, LaTex markup
+ in comment tokens is not escaped so that LaTeX can render it (default:
+ ``False``).
+
+ .. versionadded:: 1.2
+
+ `mathescape`
+ If set to ``True``, enables LaTeX math mode escape in comments. That
+ is, ``'$...$'`` inside a comment will trigger math mode (default:
+ ``False``).
+
+ .. versionadded:: 1.2
+
+ `escapeinside`
+ If set to a string of length 2, enables escaping to LaTeX. Text
+ delimited by these 2 characters is read as LaTeX code and
+ typeset accordingly. It has no effect in string literals. It has
+ no effect in comments if `texcomments` or `mathescape` is
+ set. (default: ``''``).
+
+ .. versionadded:: 2.0
+
+ `envname`
+ Allows you to pick an alternative environment name replacing Verbatim.
+ The alternate environment still has to support Verbatim's option syntax.
+ (default: ``'Verbatim'``).
+
+ .. versionadded:: 2.0
+ """
+ name = 'LaTeX'
+ aliases = ['latex', 'tex']
+ filenames = ['*.tex']
+
+ def __init__(self, **options):
+ Formatter.__init__(self, **options)
+ self.docclass = options.get('docclass', 'article')
+ self.preamble = options.get('preamble', '')
+ self.linenos = get_bool_opt(options, 'linenos', False)
+ self.linenostart = abs(get_int_opt(options, 'linenostart', 1))
+ self.linenostep = abs(get_int_opt(options, 'linenostep', 1))
+ self.verboptions = options.get('verboptions', '')
+ self.nobackground = get_bool_opt(options, 'nobackground', False)
+ self.commandprefix = options.get('commandprefix', 'PY')
+ self.texcomments = get_bool_opt(options, 'texcomments', False)
+ self.mathescape = get_bool_opt(options, 'mathescape', False)
+ self.escapeinside = options.get('escapeinside', '')
+ if len(self.escapeinside) == 2:
+ self.left = self.escapeinside[0]
+ self.right = self.escapeinside[1]
+ else:
+ self.escapeinside = ''
+ self.envname = options.get('envname', u'Verbatim')
+
+ self._create_stylesheet()
+
+ def _create_stylesheet(self):
+ t2n = self.ttype2name = {Token: ''}
+ c2d = self.cmd2def = {}
+ cp = self.commandprefix
+
+ def rgbcolor(col):
+ if col:
+ return ','.join(['%.2f' % (int(col[i] + col[i + 1], 16) / 255.0)
+ for i in (0, 2, 4)])
+ else:
+ return '1,1,1'
+
+ for ttype, ndef in self.style:
+ name = _get_ttype_name(ttype)
+ cmndef = ''
+ if ndef['bold']:
+ cmndef += r'\let\$$@bf=\textbf'
+ if ndef['italic']:
+ cmndef += r'\let\$$@it=\textit'
+ if ndef['underline']:
+ cmndef += r'\let\$$@ul=\underline'
+ if ndef['roman']:
+ cmndef += r'\let\$$@ff=\textrm'
+ if ndef['sans']:
+ cmndef += r'\let\$$@ff=\textsf'
+ if ndef['mono']:
+ cmndef += r'\let\$$@ff=\textsf'
+ if ndef['color']:
+ cmndef += (r'\def\$$@tc##1{\textcolor[rgb]{%s}{##1}}' %
+ rgbcolor(ndef['color']))
+ if ndef['border']:
+ cmndef += (r'\def\$$@bc##1{\setlength{\fboxsep}{0pt}'
+ r'\fcolorbox[rgb]{%s}{%s}{\strut ##1}}' %
+ (rgbcolor(ndef['border']),
+ rgbcolor(ndef['bgcolor'])))
+ elif ndef['bgcolor']:
+ cmndef += (r'\def\$$@bc##1{\setlength{\fboxsep}{0pt}'
+ r'\colorbox[rgb]{%s}{\strut ##1}}' %
+ rgbcolor(ndef['bgcolor']))
+ if cmndef == '':
+ continue
+ cmndef = cmndef.replace('$$', cp)
+ t2n[ttype] = name
+ c2d[name] = cmndef
+
+ def get_style_defs(self, arg=''):
+ """
+ Return the command sequences needed to define the commands
+ used to format text in the verbatim environment. ``arg`` is ignored.
+ """
+ cp = self.commandprefix
+ styles = []
+ for name, definition in iteritems(self.cmd2def):
+ styles.append(r'\expandafter\def\csname %s@tok@%s\endcsname{%s}' %
+ (cp, name, definition))
+ return STYLE_TEMPLATE % {'cp': self.commandprefix,
+ 'styles': '\n'.join(styles)}
+
+ def format_unencoded(self, tokensource, outfile):
+ # TODO: add support for background colors
+ t2n = self.ttype2name
+ cp = self.commandprefix
+
+ if self.full:
+ realoutfile = outfile
+ outfile = StringIO()
+
+ outfile.write(u'\\begin{' + self.envname + u'}[commandchars=\\\\\\{\\}')
+ if self.linenos:
+ start, step = self.linenostart, self.linenostep
+ outfile.write(u',numbers=left' +
+ (start and u',firstnumber=%d' % start or u'') +
+ (step and u',stepnumber=%d' % step or u''))
+ if self.mathescape or self.texcomments or self.escapeinside:
+ outfile.write(u',codes={\\catcode`\\$=3\\catcode`\\^=7\\catcode`\\_=8}')
+ if self.verboptions:
+ outfile.write(u',' + self.verboptions)
+ outfile.write(u']\n')
+
+ for ttype, value in tokensource:
+ if ttype in Token.Comment:
+ if self.texcomments:
+ # Try to guess comment starting lexeme and escape it ...
+ start = value[0:1]
+ for i in xrange(1, len(value)):
+ if start[0] != value[i]:
+ break
+ start += value[i]
+
+ value = value[len(start):]
+ start = escape_tex(start, cp)
+
+ # ... but do not escape inside comment.
+ value = start + value
+ elif self.mathescape:
+ # Only escape parts not inside a math environment.
+ parts = value.split('$')
+ in_math = False
+ for i, part in enumerate(parts):
+ if not in_math:
+ parts[i] = escape_tex(part, cp)
+ in_math = not in_math
+ value = '$'.join(parts)
+ elif self.escapeinside:
+ text = value
+ value = ''
+ while text:
+ a, sep1, text = text.partition(self.left)
+ if sep1:
+ b, sep2, text = text.partition(self.right)
+ if sep2:
+ value += escape_tex(a, cp) + b
+ else:
+ value += escape_tex(a + sep1 + b, cp)
+ else:
+ value += escape_tex(a, cp)
+ else:
+ value = escape_tex(value, cp)
+ elif ttype not in Token.Escape:
+ value = escape_tex(value, cp)
+ styles = []
+ while ttype is not Token:
+ try:
+ styles.append(t2n[ttype])
+ except KeyError:
+ # not in current style
+ styles.append(_get_ttype_name(ttype))
+ ttype = ttype.parent
+ styleval = '+'.join(reversed(styles))
+ if styleval:
+ spl = value.split('\n')
+ for line in spl[:-1]:
+ if line:
+ outfile.write("\\%s{%s}{%s}" % (cp, styleval, line))
+ outfile.write('\n')
+ if spl[-1]:
+ outfile.write("\\%s{%s}{%s}" % (cp, styleval, spl[-1]))
+ else:
+ outfile.write(value)
+
+ outfile.write(u'\\end{' + self.envname + u'}\n')
+
+ if self.full:
+ encoding = self.encoding or 'utf8'
+ # map known existings encodings from LaTeX distribution
+ encoding = {
+ 'utf_8': 'utf8',
+ 'latin_1': 'latin1',
+ 'iso_8859_1': 'latin1',
+ }.get(encoding.replace('-', '_'), encoding)
+ realoutfile.write(DOC_TEMPLATE %
+ dict(docclass = self.docclass,
+ preamble = self.preamble,
+ title = self.title,
+ encoding = encoding,
+ styledefs = self.get_style_defs(),
+ code = outfile.getvalue()))
+
+
+class LatexEmbeddedLexer(Lexer):
+ """
+ This lexer takes one lexer as argument, the lexer for the language
+ being formatted, and the left and right delimiters for escaped text.
+
+ First everything is scanned using the language lexer to obtain
+ strings and comments. All other consecutive tokens are merged and
+ the resulting text is scanned for escaped segments, which are given
+ the Token.Escape type. Finally text that is not escaped is scanned
+ again with the language lexer.
+ """
+ def __init__(self, left, right, lang, **options):
+ self.left = left
+ self.right = right
+ self.lang = lang
+ Lexer.__init__(self, **options)
+
+ def get_tokens_unprocessed(self, text):
+ buf = ''
+ idx = 0
+ for i, t, v in self.lang.get_tokens_unprocessed(text):
+ if t in Token.Comment or t in Token.String:
+ if buf:
+ for x in self.get_tokens_aux(idx, buf):
+ yield x
+ buf = ''
+ yield i, t, v
+ else:
+ if not buf:
+ idx = i
+ buf += v
+ if buf:
+ for x in self.get_tokens_aux(idx, buf):
+ yield x
+
+ def get_tokens_aux(self, index, text):
+ while text:
+ a, sep1, text = text.partition(self.left)
+ if a:
+ for i, t, v in self.lang.get_tokens_unprocessed(a):
+ yield index + i, t, v
+ index += len(a)
+ if sep1:
+ b, sep2, text = text.partition(self.right)
+ if sep2:
+ yield index + len(sep1), Token.Escape, b
+ index += len(sep1) + len(b) + len(sep2)
+ else:
+ yield index, Token.Error, sep1
+ index += len(sep1)
+ text = b
diff --git a/wandb/vendor/pygments/formatters/other.py b/wandb/vendor/pygments/formatters/other.py
new file mode 100644
index 0000000000000000000000000000000000000000..d6bfcacf01cd7e817bf0a1853f545a00344f5e9d
--- /dev/null
+++ b/wandb/vendor/pygments/formatters/other.py
@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.formatters.other
+ ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Other formatters: NullFormatter, RawTokenFormatter.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.formatter import Formatter
+from pygments.util import OptionError, get_choice_opt
+from pygments.token import Token
+from pygments.console import colorize
+
+__all__ = ['NullFormatter', 'RawTokenFormatter', 'TestcaseFormatter']
+
+
+class NullFormatter(Formatter):
+ """
+ Output the text unchanged without any formatting.
+ """
+ name = 'Text only'
+ aliases = ['text', 'null']
+ filenames = ['*.txt']
+
+ def format(self, tokensource, outfile):
+ enc = self.encoding
+ for ttype, value in tokensource:
+ if enc:
+ outfile.write(value.encode(enc))
+ else:
+ outfile.write(value)
+
+
+class RawTokenFormatter(Formatter):
+ r"""
+ Format tokens as a raw representation for storing token streams.
+
+ The format is ``tokentype<TAB>repr(tokenstring)\n``. The output can later
+ be converted to a token stream with the `RawTokenLexer`, described in the
+ :doc:`lexer list <lexers>`.
+
+ Only two options are accepted:
+
+ `compress`
+ If set to ``'gz'`` or ``'bz2'``, compress the output with the given
+ compression algorithm after encoding (default: ``''``).
+ `error_color`
+ If set to a color name, highlight error tokens using that color. If
+ set but with no value, defaults to ``'red'``.
+
+ .. versionadded:: 0.11
+
+ """
+ name = 'Raw tokens'
+ aliases = ['raw', 'tokens']
+ filenames = ['*.raw']
+
+ unicodeoutput = False
+
+ def __init__(self, **options):
+ Formatter.__init__(self, **options)
+ # We ignore self.encoding if it is set, since it gets set for lexer
+ # and formatter if given with -Oencoding on the command line.
+ # The RawTokenFormatter outputs only ASCII. Override here.
+ self.encoding = 'ascii' # let pygments.format() do the right thing
+ self.compress = get_choice_opt(options, 'compress',
+ ['', 'none', 'gz', 'bz2'], '')
+ self.error_color = options.get('error_color', None)
+ if self.error_color is True:
+ self.error_color = 'red'
+ if self.error_color is not None:
+ try:
+ colorize(self.error_color, '')
+ except KeyError:
+ raise ValueError("Invalid color %r specified" %
+ self.error_color)
+
+ def format(self, tokensource, outfile):
+ try:
+ outfile.write(b'')
+ except TypeError:
+ raise TypeError('The raw tokens formatter needs a binary '
+ 'output file')
+ if self.compress == 'gz':
+ import gzip
+ outfile = gzip.GzipFile('', 'wb', 9, outfile)
+ def write(text):
+ outfile.write(text.encode())
+ flush = outfile.flush
+ elif self.compress == 'bz2':
+ import bz2
+ compressor = bz2.BZ2Compressor(9)
+ def write(text):
+ outfile.write(compressor.compress(text.encode()))
+ def flush():
+ outfile.write(compressor.flush())
+ outfile.flush()
+ else:
+ def write(text):
+ outfile.write(text.encode())
+ flush = outfile.flush
+
+ if self.error_color:
+ for ttype, value in tokensource:
+ line = "%s\t%r\n" % (ttype, value)
+ if ttype is Token.Error:
+ write(colorize(self.error_color, line))
+ else:
+ write(line)
+ else:
+ for ttype, value in tokensource:
+ write("%s\t%r\n" % (ttype, value))
+ flush()
+
+TESTCASE_BEFORE = u'''\
+ def testNeedsName(self):
+ fragment = %r
+ tokens = [
+'''
+TESTCASE_AFTER = u'''\
+ ]
+ self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+'''
+
+
+class TestcaseFormatter(Formatter):
+ """
+ Format tokens as appropriate for a new testcase.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Testcase'
+ aliases = ['testcase']
+
+ def __init__(self, **options):
+ Formatter.__init__(self, **options)
+ if self.encoding is not None and self.encoding != 'utf-8':
+ raise ValueError("Only None and utf-8 are allowed encodings.")
+
+ def format(self, tokensource, outfile):
+ indentation = ' ' * 12
+ rawbuf = []
+ outbuf = []
+ for ttype, value in tokensource:
+ rawbuf.append(value)
+ outbuf.append('%s(%s, %r),\n' % (indentation, ttype, value))
+
+ before = TESTCASE_BEFORE % (u''.join(rawbuf),)
+ during = u''.join(outbuf)
+ after = TESTCASE_AFTER
+ if self.encoding is None:
+ outfile.write(before + during + after)
+ else:
+ outfile.write(before.encode('utf-8'))
+ outfile.write(during.encode('utf-8'))
+ outfile.write(after.encode('utf-8'))
+ outfile.flush()
diff --git a/wandb/vendor/pygments/formatters/rtf.py b/wandb/vendor/pygments/formatters/rtf.py
new file mode 100644
index 0000000000000000000000000000000000000000..c6353c12dea8a855e36ac459ccc13f5e0346d410
--- /dev/null
+++ b/wandb/vendor/pygments/formatters/rtf.py
@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.formatters.rtf
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ A formatter that generates RTF files.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.formatter import Formatter
+from pygments.util import get_int_opt, _surrogatepair
+
+
+__all__ = ['RtfFormatter']
+
+
+class RtfFormatter(Formatter):
+ """
+ Format tokens as RTF markup. This formatter automatically outputs full RTF
+ documents with color information and other useful stuff. Perfect for Copy and
+ Paste into Microsoft(R) Word(R) documents.
+
+ Please note that ``encoding`` and ``outencoding`` options are ignored.
+ The RTF format is ASCII natively, but handles unicode characters correctly
+ thanks to escape sequences.
+
+ .. versionadded:: 0.6
+
+ Additional options accepted:
+
+ `style`
+ The style to use, can be a string or a Style subclass (default:
+ ``'default'``).
+
+ `fontface`
+ The used font famliy, for example ``Bitstream Vera Sans``. Defaults to
+ some generic font which is supposed to have fixed width.
+
+ `fontsize`
+ Size of the font used. Size is specified in half points. The
+ default is 24 half-points, giving a size 12 font.
+
+ .. versionadded:: 2.0
+ """
+ name = 'RTF'
+ aliases = ['rtf']
+ filenames = ['*.rtf']
+
+ def __init__(self, **options):
+ r"""
+ Additional options accepted:
+
+ ``fontface``
+ Name of the font used. Could for example be ``'Courier New'``
+ to further specify the default which is ``'\fmodern'``. The RTF
+ specification claims that ``\fmodern`` are "Fixed-pitch serif
+ and sans serif fonts". Hope every RTF implementation thinks
+ the same about modern...
+
+ """
+ Formatter.__init__(self, **options)
+ self.fontface = options.get('fontface') or ''
+ self.fontsize = get_int_opt(options, 'fontsize', 0)
+
+ def _escape(self, text):
+ return text.replace(u'\\', u'\\\\') \
+ .replace(u'{', u'\\{') \
+ .replace(u'}', u'\\}')
+
+ def _escape_text(self, text):
+ # empty strings, should give a small performance improvment
+ if not text:
+ return u''
+
+ # escape text
+ text = self._escape(text)
+
+ buf = []
+ for c in text:
+ cn = ord(c)
+ if cn < (2**7):
+ # ASCII character
+ buf.append(str(c))
+ elif (2**7) <= cn < (2**16):
+ # single unicode escape sequence
+ buf.append(u'{\\u%d}' % cn)
+ elif (2**16) <= cn:
+ # RTF limits unicode to 16 bits.
+ # Force surrogate pairs
+ buf.append(u'{\\u%d}{\\u%d}' % _surrogatepair(cn))
+
+ return u''.join(buf).replace(u'\n', u'\\par\n')
+
+ def format_unencoded(self, tokensource, outfile):
+ # rtf 1.8 header
+ outfile.write(u'{\\rtf1\\ansi\\uc0\\deff0'
+ u'{\\fonttbl{\\f0\\fmodern\\fprq1\\fcharset0%s;}}'
+ u'{\\colortbl;' % (self.fontface and
+ u' ' + self._escape(self.fontface) or
+ u''))
+
+ # convert colors and save them in a mapping to access them later.
+ color_mapping = {}
+ offset = 1
+ for _, style in self.style:
+ for color in style['color'], style['bgcolor'], style['border']:
+ if color and color not in color_mapping:
+ color_mapping[color] = offset
+ outfile.write(u'\\red%d\\green%d\\blue%d;' % (
+ int(color[0:2], 16),
+ int(color[2:4], 16),
+ int(color[4:6], 16)
+ ))
+ offset += 1
+ outfile.write(u'}\\f0 ')
+ if self.fontsize:
+ outfile.write(u'\\fs%d' % (self.fontsize))
+
+ # highlight stream
+ for ttype, value in tokensource:
+ while not self.style.styles_token(ttype) and ttype.parent:
+ ttype = ttype.parent
+ style = self.style.style_for_token(ttype)
+ buf = []
+ if style['bgcolor']:
+ buf.append(u'\\cb%d' % color_mapping[style['bgcolor']])
+ if style['color']:
+ buf.append(u'\\cf%d' % color_mapping[style['color']])
+ if style['bold']:
+ buf.append(u'\\b')
+ if style['italic']:
+ buf.append(u'\\i')
+ if style['underline']:
+ buf.append(u'\\ul')
+ if style['border']:
+ buf.append(u'\\chbrdr\\chcfpat%d' %
+ color_mapping[style['border']])
+ start = u''.join(buf)
+ if start:
+ outfile.write(u'{%s ' % start)
+ outfile.write(self._escape_text(value))
+ if start:
+ outfile.write(u'}')
+
+ outfile.write(u'}')
diff --git a/wandb/vendor/pygments/formatters/svg.py b/wandb/vendor/pygments/formatters/svg.py
new file mode 100644
index 0000000000000000000000000000000000000000..944b25e0173dbeecc80b5cf49ff72e44dd769117
--- /dev/null
+++ b/wandb/vendor/pygments/formatters/svg.py
@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.formatters.svg
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Formatter for SVG output.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.formatter import Formatter
+from pygments.util import get_bool_opt, get_int_opt
+
+__all__ = ['SvgFormatter']
+
+
+def escape_html(text):
+ """Escape &, <, > as well as single and double quotes for HTML."""
+ return text.replace('&', '&'). \
+ replace('<', '<'). \
+ replace('>', '>'). \
+ replace('"', '"'). \
+ replace("'", ''')
+
+
+class2style = {}
+
+class SvgFormatter(Formatter):
+ """
+ Format tokens as an SVG graphics file. This formatter is still experimental.
+ Each line of code is a ``<text>`` element with explicit ``x`` and ``y``
+ coordinates containing ``<tspan>`` elements with the individual token styles.
+
+ By default, this formatter outputs a full SVG document including doctype
+ declaration and the ``<svg>`` root element.
+
+ .. versionadded:: 0.9
+
+ Additional options accepted:
+
+ `nowrap`
+ Don't wrap the SVG ``<text>`` elements in ``<svg><g>`` elements and
+ don't add a XML declaration and a doctype. If true, the `fontfamily`
+ and `fontsize` options are ignored. Defaults to ``False``.
+
+ `fontfamily`
+ The value to give the wrapping ``<g>`` element's ``font-family``
+ attribute, defaults to ``"monospace"``.
+
+ `fontsize`
+ The value to give the wrapping ``<g>`` element's ``font-size``
+ attribute, defaults to ``"14px"``.
+
+ `xoffset`
+ Starting offset in X direction, defaults to ``0``.
+
+ `yoffset`
+ Starting offset in Y direction, defaults to the font size if it is given
+ in pixels, or ``20`` else. (This is necessary since text coordinates
+ refer to the text baseline, not the top edge.)
+
+ `ystep`
+ Offset to add to the Y coordinate for each subsequent line. This should
+ roughly be the text size plus 5. It defaults to that value if the text
+ size is given in pixels, or ``25`` else.
+
+ `spacehack`
+ Convert spaces in the source to `` ``, which are non-breaking
+ spaces. SVG provides the ``xml:space`` attribute to control how
+ whitespace inside tags is handled, in theory, the ``preserve`` value
+ could be used to keep all whitespace as-is. However, many current SVG
+ viewers don't obey that rule, so this option is provided as a workaround
+ and defaults to ``True``.
+ """
+ name = 'SVG'
+ aliases = ['svg']
+ filenames = ['*.svg']
+
+ def __init__(self, **options):
+ Formatter.__init__(self, **options)
+ self.nowrap = get_bool_opt(options, 'nowrap', False)
+ self.fontfamily = options.get('fontfamily', 'monospace')
+ self.fontsize = options.get('fontsize', '14px')
+ self.xoffset = get_int_opt(options, 'xoffset', 0)
+ fs = self.fontsize.strip()
+ if fs.endswith('px'): fs = fs[:-2].strip()
+ try:
+ int_fs = int(fs)
+ except:
+ int_fs = 20
+ self.yoffset = get_int_opt(options, 'yoffset', int_fs)
+ self.ystep = get_int_opt(options, 'ystep', int_fs + 5)
+ self.spacehack = get_bool_opt(options, 'spacehack', True)
+ self._stylecache = {}
+
+ def format_unencoded(self, tokensource, outfile):
+ """
+ Format ``tokensource``, an iterable of ``(tokentype, tokenstring)``
+ tuples and write it into ``outfile``.
+
+ For our implementation we put all lines in their own 'line group'.
+ """
+ x = self.xoffset
+ y = self.yoffset
+ if not self.nowrap:
+ if self.encoding:
+ outfile.write('<?xml version="1.0" encoding="%s"?>\n' %
+ self.encoding)
+ else:
+ outfile.write('<?xml version="1.0"?>\n')
+ outfile.write('<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" '
+ '"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/'
+ 'svg10.dtd">\n')
+ outfile.write('<svg xmlns="http://www.w3.org/2000/svg">\n')
+ outfile.write('<g font-family="%s" font-size="%s">\n' %
+ (self.fontfamily, self.fontsize))
+ outfile.write('<text x="%s" y="%s" xml:space="preserve">' % (x, y))
+ for ttype, value in tokensource:
+ style = self._get_style(ttype)
+ tspan = style and '<tspan' + style + '>' or ''
+ tspanend = tspan and '</tspan>' or ''
+ value = escape_html(value)
+ if self.spacehack:
+ value = value.expandtabs().replace(' ', ' ')
+ parts = value.split('\n')
+ for part in parts[:-1]:
+ outfile.write(tspan + part + tspanend)
+ y += self.ystep
+ outfile.write('</text>\n<text x="%s" y="%s" '
+ 'xml:space="preserve">' % (x, y))
+ outfile.write(tspan + parts[-1] + tspanend)
+ outfile.write('</text>')
+
+ if not self.nowrap:
+ outfile.write('</g></svg>\n')
+
+ def _get_style(self, tokentype):
+ if tokentype in self._stylecache:
+ return self._stylecache[tokentype]
+ otokentype = tokentype
+ while not self.style.styles_token(tokentype):
+ tokentype = tokentype.parent
+ value = self.style.style_for_token(tokentype)
+ result = ''
+ if value['color']:
+ result = ' fill="#' + value['color'] + '"'
+ if value['bold']:
+ result += ' font-weight="bold"'
+ if value['italic']:
+ result += ' font-style="italic"'
+ self._stylecache[otokentype] = result
+ return result
diff --git a/wandb/vendor/pygments/formatters/terminal.py b/wandb/vendor/pygments/formatters/terminal.py
new file mode 100644
index 0000000000000000000000000000000000000000..b8fec52e3e593b5fb7105393c7bd6eeab78a1133
--- /dev/null
+++ b/wandb/vendor/pygments/formatters/terminal.py
@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.formatters.terminal
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Formatter for terminal output with ANSI sequences.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import sys
+
+from pygments.formatter import Formatter
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Token, Whitespace
+from pygments.console import ansiformat
+from pygments.util import get_choice_opt
+
+
+__all__ = ['TerminalFormatter']
+
+
+#: Map token types to a tuple of color values for light and dark
+#: backgrounds.
+TERMINAL_COLORS = {
+ Token: ('', ''),
+
+ Whitespace: ('lightgray', 'darkgray'),
+ Comment: ('lightgray', 'darkgray'),
+ Comment.Preproc: ('teal', 'turquoise'),
+ Keyword: ('darkblue', 'blue'),
+ Keyword.Type: ('teal', 'turquoise'),
+ Operator.Word: ('purple', 'fuchsia'),
+ Name.Builtin: ('teal', 'turquoise'),
+ Name.Function: ('darkgreen', 'green'),
+ Name.Namespace: ('_teal_', '_turquoise_'),
+ Name.Class: ('_darkgreen_', '_green_'),
+ Name.Exception: ('teal', 'turquoise'),
+ Name.Decorator: ('darkgray', 'lightgray'),
+ Name.Variable: ('darkred', 'red'),
+ Name.Constant: ('darkred', 'red'),
+ Name.Attribute: ('teal', 'turquoise'),
+ Name.Tag: ('blue', 'blue'),
+ String: ('brown', 'brown'),
+ Number: ('darkblue', 'blue'),
+
+ Generic.Deleted: ('red', 'red'),
+ Generic.Inserted: ('darkgreen', 'green'),
+ Generic.Heading: ('**', '**'),
+ Generic.Subheading: ('*purple*', '*fuchsia*'),
+ Generic.Prompt: ('**', '**'),
+ Generic.Error: ('red', 'red'),
+
+ Error: ('_red_', '_red_'),
+}
+
+
+class TerminalFormatter(Formatter):
+ r"""
+ Format tokens with ANSI color sequences, for output in a text console.
+ Color sequences are terminated at newlines, so that paging the output
+ works correctly.
+
+ The `get_style_defs()` method doesn't do anything special since there is
+ no support for common styles.
+
+ Options accepted:
+
+ `bg`
+ Set to ``"light"`` or ``"dark"`` depending on the terminal's background
+ (default: ``"light"``).
+
+ `colorscheme`
+ A dictionary mapping token types to (lightbg, darkbg) color names or
+ ``None`` (default: ``None`` = use builtin colorscheme).
+
+ `linenos`
+ Set to ``True`` to have line numbers on the terminal output as well
+ (default: ``False`` = no line numbers).
+ """
+ name = 'Terminal'
+ aliases = ['terminal', 'console']
+ filenames = []
+
+ def __init__(self, **options):
+ Formatter.__init__(self, **options)
+ self.darkbg = get_choice_opt(options, 'bg',
+ ['light', 'dark'], 'light') == 'dark'
+ self.colorscheme = options.get('colorscheme', None) or TERMINAL_COLORS
+ self.linenos = options.get('linenos', False)
+ self._lineno = 0
+
+ def format(self, tokensource, outfile):
+ # hack: if the output is a terminal and has an encoding set,
+ # use that to avoid unicode encode problems
+ if not self.encoding and hasattr(outfile, "encoding") and \
+ hasattr(outfile, "isatty") and outfile.isatty() and \
+ sys.version_info < (3,):
+ self.encoding = outfile.encoding
+ return Formatter.format(self, tokensource, outfile)
+
+ def _write_lineno(self, outfile):
+ self._lineno += 1
+ outfile.write("%s%04d: " % (self._lineno != 1 and '\n' or '', self._lineno))
+
+ def _get_color(self, ttype):
+ # self.colorscheme is a dict containing usually generic types, so we
+ # have to walk the tree of dots. The base Token type must be a key,
+ # even if it's empty string, as in the default above.
+ colors = self.colorscheme.get(ttype)
+ while colors is None:
+ ttype = ttype.parent
+ colors = self.colorscheme.get(ttype)
+ return colors[self.darkbg]
+
+ def format_unencoded(self, tokensource, outfile):
+ if self.linenos:
+ self._write_lineno(outfile)
+
+ for ttype, value in tokensource:
+ color = self._get_color(ttype)
+
+ for line in value.splitlines(True):
+ if color:
+ outfile.write(ansiformat(color, line.rstrip('\n')))
+ else:
+ outfile.write(line.rstrip('\n'))
+ if line.endswith('\n'):
+ if self.linenos:
+ self._write_lineno(outfile)
+ else:
+ outfile.write('\n')
+
+ if self.linenos:
+ outfile.write("\n")
diff --git a/wandb/vendor/pygments/formatters/terminal256.py b/wandb/vendor/pygments/formatters/terminal256.py
new file mode 100644
index 0000000000000000000000000000000000000000..b80dc7dd639ecc9f10a9fbcea086600a229322a2
--- /dev/null
+++ b/wandb/vendor/pygments/formatters/terminal256.py
@@ -0,0 +1,309 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.formatters.terminal256
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Formatter for 256-color terminal output with ANSI sequences.
+
+ RGB-to-XTERM color conversion routines adapted from xterm256-conv
+ tool (http://frexx.de/xterm-256-notes/data/xterm256-conv2.tar.bz2)
+ by Wolfgang Frisch.
+
+ Formatter version 1.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+# TODO:
+# - Options to map style's bold/underline/italic/border attributes
+# to some ANSI attrbutes (something like 'italic=underline')
+# - An option to output "style RGB to xterm RGB/index" conversion table
+# - An option to indicate that we are running in "reverse background"
+# xterm. This means that default colors are white-on-black, not
+# black-on-while, so colors like "white background" need to be converted
+# to "white background, black foreground", etc...
+
+import sys
+
+from pygments.formatter import Formatter
+from pygments.console import codes
+from pygments.style import ansicolors
+
+
+__all__ = ['Terminal256Formatter', 'TerminalTrueColorFormatter']
+
+
+class EscapeSequence:
+ def __init__(self, fg=None, bg=None, bold=False, underline=False):
+ self.fg = fg
+ self.bg = bg
+ self.bold = bold
+ self.underline = underline
+
+ def escape(self, attrs):
+ if len(attrs):
+ return "\x1b[" + ";".join(attrs) + "m"
+ return ""
+
+ def color_string(self):
+ attrs = []
+ if self.fg is not None:
+ if self.fg in ansicolors:
+ esc = codes[self.fg[5:]]
+ if ';01m' in esc:
+ self.bold = True
+ # extract fg color code.
+ attrs.append(esc[2:4])
+ else:
+ attrs.extend(("38", "5", "%i" % self.fg))
+ if self.bg is not None:
+ if self.bg in ansicolors:
+ esc = codes[self.bg[5:]]
+ # extract fg color code, add 10 for bg.
+ attrs.append(str(int(esc[2:4])+10))
+ else:
+ attrs.extend(("48", "5", "%i" % self.bg))
+ if self.bold:
+ attrs.append("01")
+ if self.underline:
+ attrs.append("04")
+ return self.escape(attrs)
+
+ def true_color_string(self):
+ attrs = []
+ if self.fg:
+ attrs.extend(("38", "2", str(self.fg[0]), str(self.fg[1]), str(self.fg[2])))
+ if self.bg:
+ attrs.extend(("48", "2", str(self.bg[0]), str(self.bg[1]), str(self.bg[2])))
+ if self.bold:
+ attrs.append("01")
+ if self.underline:
+ attrs.append("04")
+ return self.escape(attrs)
+
+ def reset_string(self):
+ attrs = []
+ if self.fg is not None:
+ attrs.append("39")
+ if self.bg is not None:
+ attrs.append("49")
+ if self.bold or self.underline:
+ attrs.append("00")
+ return self.escape(attrs)
+
+
+class Terminal256Formatter(Formatter):
+ """
+ Format tokens with ANSI color sequences, for output in a 256-color
+ terminal or console. Like in `TerminalFormatter` color sequences
+ are terminated at newlines, so that paging the output works correctly.
+
+ The formatter takes colors from a style defined by the `style` option
+ and converts them to nearest ANSI 256-color escape sequences. Bold and
+ underline attributes from the style are preserved (and displayed).
+
+ .. versionadded:: 0.9
+
+ .. versionchanged:: 2.2
+ If the used style defines foreground colors in the form ``#ansi*``, then
+ `Terminal256Formatter` will map these to non extended foreground color.
+ See :ref:`AnsiTerminalStyle` for more information.
+
+ Options accepted:
+
+ `style`
+ The style to use, can be a string or a Style subclass (default:
+ ``'default'``).
+ """
+ name = 'Terminal256'
+ aliases = ['terminal256', 'console256', '256']
+ filenames = []
+
+ def __init__(self, **options):
+ Formatter.__init__(self, **options)
+
+ self.xterm_colors = []
+ self.best_match = {}
+ self.style_string = {}
+
+ self.usebold = 'nobold' not in options
+ self.useunderline = 'nounderline' not in options
+
+ self._build_color_table() # build an RGB-to-256 color conversion table
+ self._setup_styles() # convert selected style's colors to term. colors
+
+ def _build_color_table(self):
+ # colors 0..15: 16 basic colors
+
+ self.xterm_colors.append((0x00, 0x00, 0x00)) # 0
+ self.xterm_colors.append((0xcd, 0x00, 0x00)) # 1
+ self.xterm_colors.append((0x00, 0xcd, 0x00)) # 2
+ self.xterm_colors.append((0xcd, 0xcd, 0x00)) # 3
+ self.xterm_colors.append((0x00, 0x00, 0xee)) # 4
+ self.xterm_colors.append((0xcd, 0x00, 0xcd)) # 5
+ self.xterm_colors.append((0x00, 0xcd, 0xcd)) # 6
+ self.xterm_colors.append((0xe5, 0xe5, 0xe5)) # 7
+ self.xterm_colors.append((0x7f, 0x7f, 0x7f)) # 8
+ self.xterm_colors.append((0xff, 0x00, 0x00)) # 9
+ self.xterm_colors.append((0x00, 0xff, 0x00)) # 10
+ self.xterm_colors.append((0xff, 0xff, 0x00)) # 11
+ self.xterm_colors.append((0x5c, 0x5c, 0xff)) # 12
+ self.xterm_colors.append((0xff, 0x00, 0xff)) # 13
+ self.xterm_colors.append((0x00, 0xff, 0xff)) # 14
+ self.xterm_colors.append((0xff, 0xff, 0xff)) # 15
+
+ # colors 16..232: the 6x6x6 color cube
+
+ valuerange = (0x00, 0x5f, 0x87, 0xaf, 0xd7, 0xff)
+
+ for i in range(217):
+ r = valuerange[(i // 36) % 6]
+ g = valuerange[(i // 6) % 6]
+ b = valuerange[i % 6]
+ self.xterm_colors.append((r, g, b))
+
+ # colors 233..253: grayscale
+
+ for i in range(1, 22):
+ v = 8 + i * 10
+ self.xterm_colors.append((v, v, v))
+
+ def _closest_color(self, r, g, b):
+ distance = 257*257*3 # "infinity" (>distance from #000000 to #ffffff)
+ match = 0
+
+ for i in range(0, 254):
+ values = self.xterm_colors[i]
+
+ rd = r - values[0]
+ gd = g - values[1]
+ bd = b - values[2]
+ d = rd*rd + gd*gd + bd*bd
+
+ if d < distance:
+ match = i
+ distance = d
+ return match
+
+ def _color_index(self, color):
+ index = self.best_match.get(color, None)
+ if color in ansicolors:
+ # strip the `#ansi` part and look up code
+ index = color
+ self.best_match[color] = index
+ if index is None:
+ try:
+ rgb = int(str(color), 16)
+ except ValueError:
+ rgb = 0
+
+ r = (rgb >> 16) & 0xff
+ g = (rgb >> 8) & 0xff
+ b = rgb & 0xff
+ index = self._closest_color(r, g, b)
+ self.best_match[color] = index
+ return index
+
+ def _setup_styles(self):
+ for ttype, ndef in self.style:
+ escape = EscapeSequence()
+ # get foreground from ansicolor if set
+ if ndef['ansicolor']:
+ escape.fg = self._color_index(ndef['ansicolor'])
+ elif ndef['color']:
+ escape.fg = self._color_index(ndef['color'])
+ if ndef['bgansicolor']:
+ escape.bg = self._color_index(ndef['bgansicolor'])
+ elif ndef['bgcolor']:
+ escape.bg = self._color_index(ndef['bgcolor'])
+ if self.usebold and ndef['bold']:
+ escape.bold = True
+ if self.useunderline and ndef['underline']:
+ escape.underline = True
+ self.style_string[str(ttype)] = (escape.color_string(),
+ escape.reset_string())
+
+ def format(self, tokensource, outfile):
+ # hack: if the output is a terminal and has an encoding set,
+ # use that to avoid unicode encode problems
+ if not self.encoding and hasattr(outfile, "encoding") and \
+ hasattr(outfile, "isatty") and outfile.isatty() and \
+ sys.version_info < (3,):
+ self.encoding = outfile.encoding
+ return Formatter.format(self, tokensource, outfile)
+
+ def format_unencoded(self, tokensource, outfile):
+ for ttype, value in tokensource:
+ not_found = True
+ while ttype and not_found:
+ try:
+ # outfile.write( "<" + str(ttype) + ">" )
+ on, off = self.style_string[str(ttype)]
+
+ # Like TerminalFormatter, add "reset colors" escape sequence
+ # on newline.
+ spl = value.split('\n')
+ for line in spl[:-1]:
+ if line:
+ outfile.write(on + line + off)
+ outfile.write('\n')
+ if spl[-1]:
+ outfile.write(on + spl[-1] + off)
+
+ not_found = False
+ # outfile.write( '#' + str(ttype) + '#' )
+
+ except KeyError:
+ # ottype = ttype
+ ttype = ttype[:-1]
+ # outfile.write( '!' + str(ottype) + '->' + str(ttype) + '!' )
+
+ if not_found:
+ outfile.write(value)
+
+
+class TerminalTrueColorFormatter(Terminal256Formatter):
+ r"""
+ Format tokens with ANSI color sequences, for output in a true-color
+ terminal or console. Like in `TerminalFormatter` color sequences
+ are terminated at newlines, so that paging the output works correctly.
+
+ .. versionadded:: 2.1
+
+ Options accepted:
+
+ `style`
+ The style to use, can be a string or a Style subclass (default:
+ ``'default'``).
+ """
+ name = 'TerminalTrueColor'
+ aliases = ['terminal16m', 'console16m', '16m']
+ filenames = []
+
+ def _build_color_table(self):
+ pass
+
+ def _color_tuple(self, color):
+ try:
+ rgb = int(str(color), 16)
+ except ValueError:
+ return None
+ r = (rgb >> 16) & 0xff
+ g = (rgb >> 8) & 0xff
+ b = rgb & 0xff
+ return (r, g, b)
+
+ def _setup_styles(self):
+ for ttype, ndef in self.style:
+ escape = EscapeSequence()
+ if ndef['color']:
+ escape.fg = self._color_tuple(ndef['color'])
+ if ndef['bgcolor']:
+ escape.bg = self._color_tuple(ndef['bgcolor'])
+ if self.usebold and ndef['bold']:
+ escape.bold = True
+ if self.useunderline and ndef['underline']:
+ escape.underline = True
+ self.style_string[str(ttype)] = (escape.true_color_string(),
+ escape.reset_string())
diff --git a/wandb/vendor/pygments/lexer.py b/wandb/vendor/pygments/lexer.py
new file mode 100644
index 0000000000000000000000000000000000000000..90905ba51bf23d6f8aa5f5033d0528e3ed80bc0e
--- /dev/null
+++ b/wandb/vendor/pygments/lexer.py
@@ -0,0 +1,871 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexer
+ ~~~~~~~~~~~~~~
+
+ Base lexer classes.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import print_function
+
+import re
+import sys
+import time
+
+from pygments.filter import apply_filters, Filter
+from pygments.filters import get_filter_by_name
+from pygments.token import Error, Text, Other, _TokenType
+from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
+ make_analysator, text_type, add_metaclass, iteritems, Future, guess_decode
+from pygments.regexopt import regex_opt
+
+__all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer',
+ 'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this',
+ 'default', 'words']
+
+
+_encoding_map = [(b'\xef\xbb\xbf', 'utf-8'),
+ (b'\xff\xfe\0\0', 'utf-32'),
+ (b'\0\0\xfe\xff', 'utf-32be'),
+ (b'\xff\xfe', 'utf-16'),
+ (b'\xfe\xff', 'utf-16be')]
+
+_default_analyse = staticmethod(lambda x: 0.0)
+
+
+class LexerMeta(type):
+ """
+ This metaclass automagically converts ``analyse_text`` methods into
+ static methods which always return float values.
+ """
+
+ def __new__(mcs, name, bases, d):
+ if 'analyse_text' in d:
+ d['analyse_text'] = make_analysator(d['analyse_text'])
+ return type.__new__(mcs, name, bases, d)
+
+
+@add_metaclass(LexerMeta)
+class Lexer(object):
+ """
+ Lexer for a specific language.
+
+ Basic options recognized:
+ ``stripnl``
+ Strip leading and trailing newlines from the input (default: True).
+ ``stripall``
+ Strip all leading and trailing whitespace from the input
+ (default: False).
+ ``ensurenl``
+ Make sure that the input ends with a newline (default: True). This
+ is required for some lexers that consume input linewise.
+
+ .. versionadded:: 1.3
+
+ ``tabsize``
+ If given and greater than 0, expand tabs in the input (default: 0).
+ ``encoding``
+ If given, must be an encoding name. This encoding will be used to
+ convert the input string to Unicode, if it is not already a Unicode
+ string (default: ``'guess'``, which uses a simple UTF-8 / Locale /
+ Latin1 detection. Can also be ``'chardet'`` to use the chardet
+ library, if it is installed.
+ ``inencoding``
+ Overrides the ``encoding`` if given.
+ """
+
+ #: Name of the lexer
+ name = None
+
+ #: Shortcuts for the lexer
+ aliases = []
+
+ #: File name globs
+ filenames = []
+
+ #: Secondary file name globs
+ alias_filenames = []
+
+ #: MIME types
+ mimetypes = []
+
+ #: Priority, should multiple lexers match and no content is provided
+ priority = 0
+
+ def __init__(self, **options):
+ self.options = options
+ self.stripnl = get_bool_opt(options, 'stripnl', True)
+ self.stripall = get_bool_opt(options, 'stripall', False)
+ self.ensurenl = get_bool_opt(options, 'ensurenl', True)
+ self.tabsize = get_int_opt(options, 'tabsize', 0)
+ self.encoding = options.get('encoding', 'guess')
+ self.encoding = options.get('inencoding') or self.encoding
+ self.filters = []
+ for filter_ in get_list_opt(options, 'filters', ()):
+ self.add_filter(filter_)
+
+ def __repr__(self):
+ if self.options:
+ return '<pygments.lexers.%s with %r>' % (self.__class__.__name__,
+ self.options)
+ else:
+ return '<pygments.lexers.%s>' % self.__class__.__name__
+
+ def add_filter(self, filter_, **options):
+ """
+ Add a new stream filter to this lexer.
+ """
+ if not isinstance(filter_, Filter):
+ filter_ = get_filter_by_name(filter_, **options)
+ self.filters.append(filter_)
+
+ def analyse_text(text):
+ """
+ Has to return a float between ``0`` and ``1`` that indicates
+ if a lexer wants to highlight this text. Used by ``guess_lexer``.
+ If this method returns ``0`` it won't highlight it in any case, if
+ it returns ``1`` highlighting with this lexer is guaranteed.
+
+ The `LexerMeta` metaclass automatically wraps this function so
+ that it works like a static method (no ``self`` or ``cls``
+ parameter) and the return value is automatically converted to
+ `float`. If the return value is an object that is boolean `False`
+ it's the same as if the return values was ``0.0``.
+ """
+
+ def get_tokens(self, text, unfiltered=False):
+ """
+ Return an iterable of (tokentype, value) pairs generated from
+ `text`. If `unfiltered` is set to `True`, the filtering mechanism
+ is bypassed even if filters are defined.
+
+ Also preprocess the text, i.e. expand tabs and strip it if
+ wanted and applies registered filters.
+ """
+ if not isinstance(text, text_type):
+ if self.encoding == 'guess':
+ text, _ = guess_decode(text)
+ elif self.encoding == 'chardet':
+ try:
+ import chardet
+ except ImportError:
+ raise ImportError('To enable chardet encoding guessing, '
+ 'please install the chardet library '
+ 'from http://chardet.feedparser.org/')
+ # check for BOM first
+ decoded = None
+ for bom, encoding in _encoding_map:
+ if text.startswith(bom):
+ decoded = text[len(bom):].decode(encoding, 'replace')
+ break
+ # no BOM found, so use chardet
+ if decoded is None:
+ enc = chardet.detect(text[:1024]) # Guess using first 1KB
+ decoded = text.decode(enc.get('encoding') or 'utf-8',
+ 'replace')
+ text = decoded
+ else:
+ text = text.decode(self.encoding)
+ if text.startswith(u'\ufeff'):
+ text = text[len(u'\ufeff'):]
+ else:
+ if text.startswith(u'\ufeff'):
+ text = text[len(u'\ufeff'):]
+
+ # text now *is* a unicode string
+ text = text.replace('\r\n', '\n')
+ text = text.replace('\r', '\n')
+ if self.stripall:
+ text = text.strip()
+ elif self.stripnl:
+ text = text.strip('\n')
+ if self.tabsize > 0:
+ text = text.expandtabs(self.tabsize)
+ if self.ensurenl and not text.endswith('\n'):
+ text += '\n'
+
+ def streamer():
+ for _, t, v in self.get_tokens_unprocessed(text):
+ yield t, v
+ stream = streamer()
+ if not unfiltered:
+ stream = apply_filters(stream, self.filters, self)
+ return stream
+
+ def get_tokens_unprocessed(self, text):
+ """
+ Return an iterable of (index, tokentype, value) pairs where "index"
+ is the starting position of the token within the input text.
+
+ In subclasses, implement this method as a generator to
+ maximize effectiveness.
+ """
+ raise NotImplementedError
+
+
+class DelegatingLexer(Lexer):
+ """
+ This lexer takes two lexer as arguments. A root lexer and
+ a language lexer. First everything is scanned using the language
+ lexer, afterwards all ``Other`` tokens are lexed using the root
+ lexer.
+
+ The lexers from the ``template`` lexer package use this base lexer.
+ """
+
+ def __init__(self, _root_lexer, _language_lexer, _needle=Other, **options):
+ self.root_lexer = _root_lexer(**options)
+ self.language_lexer = _language_lexer(**options)
+ self.needle = _needle
+ Lexer.__init__(self, **options)
+
+ def get_tokens_unprocessed(self, text):
+ buffered = ''
+ insertions = []
+ lng_buffer = []
+ for i, t, v in self.language_lexer.get_tokens_unprocessed(text):
+ if t is self.needle:
+ if lng_buffer:
+ insertions.append((len(buffered), lng_buffer))
+ lng_buffer = []
+ buffered += v
+ else:
+ lng_buffer.append((i, t, v))
+ if lng_buffer:
+ insertions.append((len(buffered), lng_buffer))
+ return do_insertions(insertions,
+ self.root_lexer.get_tokens_unprocessed(buffered))
+
+
+# ------------------------------------------------------------------------------
+# RegexLexer and ExtendedRegexLexer
+#
+
+
+class include(str): # pylint: disable=invalid-name
+ """
+ Indicates that a state should include rules from another state.
+ """
+ pass
+
+
+class _inherit(object):
+ """
+ Indicates the a state should inherit from its superclass.
+ """
+ def __repr__(self):
+ return 'inherit'
+
+inherit = _inherit() # pylint: disable=invalid-name
+
+
+class combined(tuple): # pylint: disable=invalid-name
+ """
+ Indicates a state combined from multiple states.
+ """
+
+ def __new__(cls, *args):
+ return tuple.__new__(cls, args)
+
+ def __init__(self, *args):
+ # tuple.__init__ doesn't do anything
+ pass
+
+
+class _PseudoMatch(object):
+ """
+ A pseudo match object constructed from a string.
+ """
+
+ def __init__(self, start, text):
+ self._text = text
+ self._start = start
+
+ def start(self, arg=None):
+ return self._start
+
+ def end(self, arg=None):
+ return self._start + len(self._text)
+
+ def group(self, arg=None):
+ if arg:
+ raise IndexError('No such group')
+ return self._text
+
+ def groups(self):
+ return (self._text,)
+
+ def groupdict(self):
+ return {}
+
+
+def bygroups(*args):
+ """
+ Callback that yields multiple actions for each group in the match.
+ """
+ def callback(lexer, match, ctx=None):
+ for i, action in enumerate(args):
+ if action is None:
+ continue
+ elif type(action) is _TokenType:
+ data = match.group(i + 1)
+ if data:
+ yield match.start(i + 1), action, data
+ else:
+ data = match.group(i + 1)
+ if data is not None:
+ if ctx:
+ ctx.pos = match.start(i + 1)
+ for item in action(lexer,
+ _PseudoMatch(match.start(i + 1), data), ctx):
+ if item:
+ yield item
+ if ctx:
+ ctx.pos = match.end()
+ return callback
+
+
+class _This(object):
+ """
+ Special singleton used for indicating the caller class.
+ Used by ``using``.
+ """
+this = _This()
+
+
+def using(_other, **kwargs):
+ """
+ Callback that processes the match with a different lexer.
+
+ The keyword arguments are forwarded to the lexer, except `state` which
+ is handled separately.
+
+ `state` specifies the state that the new lexer will start in, and can
+ be an enumerable such as ('root', 'inline', 'string') or a simple
+ string which is assumed to be on top of the root state.
+
+ Note: For that to work, `_other` must not be an `ExtendedRegexLexer`.
+ """
+ gt_kwargs = {}
+ if 'state' in kwargs:
+ s = kwargs.pop('state')
+ if isinstance(s, (list, tuple)):
+ gt_kwargs['stack'] = s
+ else:
+ gt_kwargs['stack'] = ('root', s)
+
+ if _other is this:
+ def callback(lexer, match, ctx=None):
+ # if keyword arguments are given the callback
+ # function has to create a new lexer instance
+ if kwargs:
+ # XXX: cache that somehow
+ kwargs.update(lexer.options)
+ lx = lexer.__class__(**kwargs)
+ else:
+ lx = lexer
+ s = match.start()
+ for i, t, v in lx.get_tokens_unprocessed(match.group(), **gt_kwargs):
+ yield i + s, t, v
+ if ctx:
+ ctx.pos = match.end()
+ else:
+ def callback(lexer, match, ctx=None):
+ # XXX: cache that somehow
+ kwargs.update(lexer.options)
+ lx = _other(**kwargs)
+
+ s = match.start()
+ for i, t, v in lx.get_tokens_unprocessed(match.group(), **gt_kwargs):
+ yield i + s, t, v
+ if ctx:
+ ctx.pos = match.end()
+ return callback
+
+
+class default:
+ """
+ Indicates a state or state action (e.g. #pop) to apply.
+ For example default('#pop') is equivalent to ('', Token, '#pop')
+ Note that state tuples may be used as well.
+
+ .. versionadded:: 2.0
+ """
+ def __init__(self, state):
+ self.state = state
+
+
+class words(Future):
+ """
+ Indicates a list of literal words that is transformed into an optimized
+ regex that matches any of the words.
+
+ .. versionadded:: 2.0
+ """
+ def __init__(self, words, prefix='', suffix=''):
+ self.words = words
+ self.prefix = prefix
+ self.suffix = suffix
+
+ def get(self):
+ return regex_opt(self.words, prefix=self.prefix, suffix=self.suffix)
+
+
+class RegexLexerMeta(LexerMeta):
+ """
+ Metaclass for RegexLexer, creates the self._tokens attribute from
+ self.tokens on the first instantiation.
+ """
+
+ def _process_regex(cls, regex, rflags, state):
+ """Preprocess the regular expression component of a token definition."""
+ if isinstance(regex, Future):
+ regex = regex.get()
+ return re.compile(regex, rflags).match
+
+ def _process_token(cls, token):
+ """Preprocess the token component of a token definition."""
+ assert type(token) is _TokenType or callable(token), \
+ 'token type must be simple type or callable, not %r' % (token,)
+ return token
+
+ def _process_new_state(cls, new_state, unprocessed, processed):
+ """Preprocess the state transition action of a token definition."""
+ if isinstance(new_state, str):
+ # an existing state
+ if new_state == '#pop':
+ return -1
+ elif new_state in unprocessed:
+ return (new_state,)
+ elif new_state == '#push':
+ return new_state
+ elif new_state[:5] == '#pop:':
+ return -int(new_state[5:])
+ else:
+ assert False, 'unknown new state %r' % new_state
+ elif isinstance(new_state, combined):
+ # combine a new state from existing ones
+ tmp_state = '_tmp_%d' % cls._tmpname
+ cls._tmpname += 1
+ itokens = []
+ for istate in new_state:
+ assert istate != new_state, 'circular state ref %r' % istate
+ itokens.extend(cls._process_state(unprocessed,
+ processed, istate))
+ processed[tmp_state] = itokens
+ return (tmp_state,)
+ elif isinstance(new_state, tuple):
+ # push more than one state
+ for istate in new_state:
+ assert (istate in unprocessed or
+ istate in ('#pop', '#push')), \
+ 'unknown new state ' + istate
+ return new_state
+ else:
+ assert False, 'unknown new state def %r' % new_state
+
+ def _process_state(cls, unprocessed, processed, state):
+ """Preprocess a single state definition."""
+ assert type(state) is str, "wrong state name %r" % state
+ assert state[0] != '#', "invalid state name %r" % state
+ if state in processed:
+ return processed[state]
+ tokens = processed[state] = []
+ rflags = cls.flags
+ for tdef in unprocessed[state]:
+ if isinstance(tdef, include):
+ # it's a state reference
+ assert tdef != state, "circular state reference %r" % state
+ tokens.extend(cls._process_state(unprocessed, processed,
+ str(tdef)))
+ continue
+ if isinstance(tdef, _inherit):
+ # should be processed already, but may not in the case of:
+ # 1. the state has no counterpart in any parent
+ # 2. the state includes more than one 'inherit'
+ continue
+ if isinstance(tdef, default):
+ new_state = cls._process_new_state(tdef.state, unprocessed, processed)
+ tokens.append((re.compile('').match, None, new_state))
+ continue
+
+ assert type(tdef) is tuple, "wrong rule def %r" % tdef
+
+ try:
+ rex = cls._process_regex(tdef[0], rflags, state)
+ except Exception as err:
+ raise ValueError("uncompilable regex %r in state %r of %r: %s" %
+ (tdef[0], state, cls, err))
+
+ token = cls._process_token(tdef[1])
+
+ if len(tdef) == 2:
+ new_state = None
+ else:
+ new_state = cls._process_new_state(tdef[2],
+ unprocessed, processed)
+
+ tokens.append((rex, token, new_state))
+ return tokens
+
+ def process_tokendef(cls, name, tokendefs=None):
+ """Preprocess a dictionary of token definitions."""
+ processed = cls._all_tokens[name] = {}
+ tokendefs = tokendefs or cls.tokens[name]
+ for state in list(tokendefs):
+ cls._process_state(tokendefs, processed, state)
+ return processed
+
+ def get_tokendefs(cls):
+ """
+ Merge tokens from superclasses in MRO order, returning a single tokendef
+ dictionary.
+
+ Any state that is not defined by a subclass will be inherited
+ automatically. States that *are* defined by subclasses will, by
+ default, override that state in the superclass. If a subclass wishes to
+ inherit definitions from a superclass, it can use the special value
+ "inherit", which will cause the superclass' state definition to be
+ included at that point in the state.
+ """
+ tokens = {}
+ inheritable = {}
+ for c in cls.__mro__:
+ toks = c.__dict__.get('tokens', {})
+
+ for state, items in iteritems(toks):
+ curitems = tokens.get(state)
+ if curitems is None:
+ # N.b. because this is assigned by reference, sufficiently
+ # deep hierarchies are processed incrementally (e.g. for
+ # A(B), B(C), C(RegexLexer), B will be premodified so X(B)
+ # will not see any inherits in B).
+ tokens[state] = items
+ try:
+ inherit_ndx = items.index(inherit)
+ except ValueError:
+ continue
+ inheritable[state] = inherit_ndx
+ continue
+
+ inherit_ndx = inheritable.pop(state, None)
+ if inherit_ndx is None:
+ continue
+
+ # Replace the "inherit" value with the items
+ curitems[inherit_ndx:inherit_ndx+1] = items
+ try:
+ # N.b. this is the index in items (that is, the superclass
+ # copy), so offset required when storing below.
+ new_inh_ndx = items.index(inherit)
+ except ValueError:
+ pass
+ else:
+ inheritable[state] = inherit_ndx + new_inh_ndx
+
+ return tokens
+
+ def __call__(cls, *args, **kwds):
+ """Instantiate cls after preprocessing its token definitions."""
+ if '_tokens' not in cls.__dict__:
+ cls._all_tokens = {}
+ cls._tmpname = 0
+ if hasattr(cls, 'token_variants') and cls.token_variants:
+ # don't process yet
+ pass
+ else:
+ cls._tokens = cls.process_tokendef('', cls.get_tokendefs())
+
+ return type.__call__(cls, *args, **kwds)
+
+
+@add_metaclass(RegexLexerMeta)
+class RegexLexer(Lexer):
+ """
+ Base for simple stateful regular expression-based lexers.
+ Simplifies the lexing process so that you need only
+ provide a list of states and regular expressions.
+ """
+
+ #: Flags for compiling the regular expressions.
+ #: Defaults to MULTILINE.
+ flags = re.MULTILINE
+
+ #: Dict of ``{'state': [(regex, tokentype, new_state), ...], ...}``
+ #:
+ #: The initial state is 'root'.
+ #: ``new_state`` can be omitted to signify no state transition.
+ #: If it is a string, the state is pushed on the stack and changed.
+ #: If it is a tuple of strings, all states are pushed on the stack and
+ #: the current state will be the topmost.
+ #: It can also be ``combined('state1', 'state2', ...)``
+ #: to signify a new, anonymous state combined from the rules of two
+ #: or more existing ones.
+ #: Furthermore, it can be '#pop' to signify going back one step in
+ #: the state stack, or '#push' to push the current state on the stack
+ #: again.
+ #:
+ #: The tuple can also be replaced with ``include('state')``, in which
+ #: case the rules from the state named by the string are included in the
+ #: current one.
+ tokens = {}
+
+ def get_tokens_unprocessed(self, text, stack=('root',)):
+ """
+ Split ``text`` into (tokentype, text) pairs.
+
+ ``stack`` is the inital stack (default: ``['root']``)
+ """
+ pos = 0
+ tokendefs = self._tokens
+ statestack = list(stack)
+ statetokens = tokendefs[statestack[-1]]
+ while 1:
+ for rexmatch, action, new_state in statetokens:
+ m = rexmatch(text, pos)
+ if m:
+ if action is not None:
+ if type(action) is _TokenType:
+ yield pos, action, m.group()
+ else:
+ for item in action(self, m):
+ yield item
+ pos = m.end()
+ if new_state is not None:
+ # state transition
+ if isinstance(new_state, tuple):
+ for state in new_state:
+ if state == '#pop':
+ statestack.pop()
+ elif state == '#push':
+ statestack.append(statestack[-1])
+ else:
+ statestack.append(state)
+ elif isinstance(new_state, int):
+ # pop
+ del statestack[new_state:]
+ elif new_state == '#push':
+ statestack.append(statestack[-1])
+ else:
+ assert False, "wrong state def: %r" % new_state
+ statetokens = tokendefs[statestack[-1]]
+ break
+ else:
+ # We are here only if all state tokens have been considered
+ # and there was not a match on any of them.
+ try:
+ if text[pos] == '\n':
+ # at EOL, reset state to "root"
+ statestack = ['root']
+ statetokens = tokendefs['root']
+ yield pos, Text, u'\n'
+ pos += 1
+ continue
+ yield pos, Error, text[pos]
+ pos += 1
+ except IndexError:
+ break
+
+
+class LexerContext(object):
+ """
+ A helper object that holds lexer position data.
+ """
+
+ def __init__(self, text, pos, stack=None, end=None):
+ self.text = text
+ self.pos = pos
+ self.end = end or len(text) # end=0 not supported ;-)
+ self.stack = stack or ['root']
+
+ def __repr__(self):
+ return 'LexerContext(%r, %r, %r)' % (
+ self.text, self.pos, self.stack)
+
+
+class ExtendedRegexLexer(RegexLexer):
+ """
+ A RegexLexer that uses a context object to store its state.
+ """
+
+ def get_tokens_unprocessed(self, text=None, context=None):
+ """
+ Split ``text`` into (tokentype, text) pairs.
+ If ``context`` is given, use this lexer context instead.
+ """
+ tokendefs = self._tokens
+ if not context:
+ ctx = LexerContext(text, 0)
+ statetokens = tokendefs['root']
+ else:
+ ctx = context
+ statetokens = tokendefs[ctx.stack[-1]]
+ text = ctx.text
+ while 1:
+ for rexmatch, action, new_state in statetokens:
+ m = rexmatch(text, ctx.pos, ctx.end)
+ if m:
+ if action is not None:
+ if type(action) is _TokenType:
+ yield ctx.pos, action, m.group()
+ ctx.pos = m.end()
+ else:
+ for item in action(self, m, ctx):
+ yield item
+ if not new_state:
+ # altered the state stack?
+ statetokens = tokendefs[ctx.stack[-1]]
+ # CAUTION: callback must set ctx.pos!
+ if new_state is not None:
+ # state transition
+ if isinstance(new_state, tuple):
+ for state in new_state:
+ if state == '#pop':
+ ctx.stack.pop()
+ elif state == '#push':
+ ctx.stack.append(ctx.stack[-1])
+ else:
+ ctx.stack.append(state)
+ elif isinstance(new_state, int):
+ # pop
+ del ctx.stack[new_state:]
+ elif new_state == '#push':
+ ctx.stack.append(ctx.stack[-1])
+ else:
+ assert False, "wrong state def: %r" % new_state
+ statetokens = tokendefs[ctx.stack[-1]]
+ break
+ else:
+ try:
+ if ctx.pos >= ctx.end:
+ break
+ if text[ctx.pos] == '\n':
+ # at EOL, reset state to "root"
+ ctx.stack = ['root']
+ statetokens = tokendefs['root']
+ yield ctx.pos, Text, u'\n'
+ ctx.pos += 1
+ continue
+ yield ctx.pos, Error, text[ctx.pos]
+ ctx.pos += 1
+ except IndexError:
+ break
+
+
+def do_insertions(insertions, tokens):
+ """
+ Helper for lexers which must combine the results of several
+ sublexers.
+
+ ``insertions`` is a list of ``(index, itokens)`` pairs.
+ Each ``itokens`` iterable should be inserted at position
+ ``index`` into the token stream given by the ``tokens``
+ argument.
+
+ The result is a combined token stream.
+
+ TODO: clean up the code here.
+ """
+ insertions = iter(insertions)
+ try:
+ index, itokens = next(insertions)
+ except StopIteration:
+ # no insertions
+ for item in tokens:
+ yield item
+ return
+
+ realpos = None
+ insleft = True
+
+ # iterate over the token stream where we want to insert
+ # the tokens from the insertion list.
+ for i, t, v in tokens:
+ # first iteration. store the postition of first item
+ if realpos is None:
+ realpos = i
+ oldi = 0
+ while insleft and i + len(v) >= index:
+ tmpval = v[oldi:index - i]
+ yield realpos, t, tmpval
+ realpos += len(tmpval)
+ for it_index, it_token, it_value in itokens:
+ yield realpos, it_token, it_value
+ realpos += len(it_value)
+ oldi = index - i
+ try:
+ index, itokens = next(insertions)
+ except StopIteration:
+ insleft = False
+ break # not strictly necessary
+ yield realpos, t, v[oldi:]
+ realpos += len(v) - oldi
+
+ # leftover tokens
+ while insleft:
+ # no normal tokens, set realpos to zero
+ realpos = realpos or 0
+ for p, t, v in itokens:
+ yield realpos, t, v
+ realpos += len(v)
+ try:
+ index, itokens = next(insertions)
+ except StopIteration:
+ insleft = False
+ break # not strictly necessary
+
+
+class ProfilingRegexLexerMeta(RegexLexerMeta):
+ """Metaclass for ProfilingRegexLexer, collects regex timing info."""
+
+ def _process_regex(cls, regex, rflags, state):
+ if isinstance(regex, words):
+ rex = regex_opt(regex.words, prefix=regex.prefix,
+ suffix=regex.suffix)
+ else:
+ rex = regex
+ compiled = re.compile(rex, rflags)
+
+ def match_func(text, pos, endpos=sys.maxsize):
+ info = cls._prof_data[-1].setdefault((state, rex), [0, 0.0])
+ t0 = time.time()
+ res = compiled.match(text, pos, endpos)
+ t1 = time.time()
+ info[0] += 1
+ info[1] += t1 - t0
+ return res
+ return match_func
+
+
+@add_metaclass(ProfilingRegexLexerMeta)
+class ProfilingRegexLexer(RegexLexer):
+ """Drop-in replacement for RegexLexer that does profiling of its regexes."""
+
+ _prof_data = []
+ _prof_sort_index = 4 # defaults to time per call
+
+ def get_tokens_unprocessed(self, text, stack=('root',)):
+ # this needs to be a stack, since using(this) will produce nested calls
+ self.__class__._prof_data.append({})
+ for tok in RegexLexer.get_tokens_unprocessed(self, text, stack):
+ yield tok
+ rawdata = self.__class__._prof_data.pop()
+ data = sorted(((s, repr(r).strip('u\'').replace('\\\\', '\\')[:65],
+ n, 1000 * t, 1000 * t / n)
+ for ((s, r), (n, t)) in rawdata.items()),
+ key=lambda x: x[self._prof_sort_index],
+ reverse=True)
+ sum_total = sum(x[3] for x in data)
+
+ print()
+ print('Profiling result for %s lexing %d chars in %.3f ms' %
+ (self.__class__.__name__, len(text), sum_total))
+ print('=' * 110)
+ print('%-20s %-64s ncalls tottime percall' % ('state', 'regex'))
+ print('-' * 110)
+ for d in data:
+ print('%-20s %-65s %5d %8.4f %8.4f' % d)
+ print('=' * 110)
diff --git a/wandb/vendor/pygments/lexers/__init__.py b/wandb/vendor/pygments/lexers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..328e072c919026a88ecd1b4b38a7c99163680332
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/__init__.py
@@ -0,0 +1,329 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers
+ ~~~~~~~~~~~~~~~
+
+ Pygments lexers.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+import sys
+import types
+import fnmatch
+from os.path import basename
+
+from pygments.lexers._mapping import LEXERS
+from pygments.modeline import get_filetype_from_buffer
+from pygments.plugin import find_plugin_lexers
+from pygments.util import ClassNotFound, itervalues, guess_decode
+
+
+__all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class',
+ 'guess_lexer', 'load_lexer_from_file'] + list(LEXERS)
+
+_lexer_cache = {}
+_pattern_cache = {}
+
+
+def _fn_matches(fn, glob):
+ """Return whether the supplied file name fn matches pattern filename."""
+ if glob not in _pattern_cache:
+ pattern = _pattern_cache[glob] = re.compile(fnmatch.translate(glob))
+ return pattern.match(fn)
+ return _pattern_cache[glob].match(fn)
+
+
+def _load_lexers(module_name):
+ """Load a lexer (and all others in the module too)."""
+ mod = __import__(module_name, None, None, ['__all__'])
+ for lexer_name in mod.__all__:
+ cls = getattr(mod, lexer_name)
+ _lexer_cache[cls.name] = cls
+
+
+def get_all_lexers():
+ """Return a generator of tuples in the form ``(name, aliases,
+ filenames, mimetypes)`` of all know lexers.
+ """
+ for item in itervalues(LEXERS):
+ yield item[1:]
+ for lexer in find_plugin_lexers():
+ yield lexer.name, lexer.aliases, lexer.filenames, lexer.mimetypes
+
+
+def find_lexer_class(name):
+ """Lookup a lexer class by name.
+
+ Return None if not found.
+ """
+ if name in _lexer_cache:
+ return _lexer_cache[name]
+ # lookup builtin lexers
+ for module_name, lname, aliases, _, _ in itervalues(LEXERS):
+ if name == lname:
+ _load_lexers(module_name)
+ return _lexer_cache[name]
+ # continue with lexers from setuptools entrypoints
+ for cls in find_plugin_lexers():
+ if cls.name == name:
+ return cls
+
+
+def find_lexer_class_by_name(_alias):
+ """Lookup a lexer class by alias.
+
+ Like `get_lexer_by_name`, but does not instantiate the class.
+
+ .. versionadded:: 2.2
+ """
+ if not _alias:
+ raise ClassNotFound('no lexer for alias %r found' % _alias)
+ # lookup builtin lexers
+ for module_name, name, aliases, _, _ in itervalues(LEXERS):
+ if _alias.lower() in aliases:
+ if name not in _lexer_cache:
+ _load_lexers(module_name)
+ return _lexer_cache[name]
+ # continue with lexers from setuptools entrypoints
+ for cls in find_plugin_lexers():
+ if _alias.lower() in cls.aliases:
+ return cls
+ raise ClassNotFound('no lexer for alias %r found' % _alias)
+
+
+def get_lexer_by_name(_alias, **options):
+ """Get a lexer by an alias.
+
+ Raises ClassNotFound if not found.
+ """
+ if not _alias:
+ raise ClassNotFound('no lexer for alias %r found' % _alias)
+
+ # lookup builtin lexers
+ for module_name, name, aliases, _, _ in itervalues(LEXERS):
+ if _alias.lower() in aliases:
+ if name not in _lexer_cache:
+ _load_lexers(module_name)
+ return _lexer_cache[name](**options)
+ # continue with lexers from setuptools entrypoints
+ for cls in find_plugin_lexers():
+ if _alias.lower() in cls.aliases:
+ return cls(**options)
+ raise ClassNotFound('no lexer for alias %r found' % _alias)
+
+
+def load_lexer_from_file(filename, lexername="CustomLexer", **options):
+ """Load a lexer from a file.
+
+ This method expects a file located relative to the current working
+ directory, which contains a Lexer class. By default, it expects the
+ Lexer to be name CustomLexer; you can specify your own class name
+ as the second argument to this function.
+
+ Users should be very careful with the input, because this method
+ is equivalent to running eval on the input file.
+
+ Raises ClassNotFound if there are any problems importing the Lexer.
+
+ .. versionadded:: 2.2
+ """
+ try:
+ # This empty dict will contain the namespace for the exec'd file
+ custom_namespace = {}
+ exec(open(filename, 'rb').read(), custom_namespace)
+ # Retrieve the class `lexername` from that namespace
+ if lexername not in custom_namespace:
+ raise ClassNotFound('no valid %s class found in %s' %
+ (lexername, filename))
+ lexer_class = custom_namespace[lexername]
+ # And finally instantiate it with the options
+ return lexer_class(**options)
+ except IOError as err:
+ raise ClassNotFound('cannot read %s' % filename)
+ except ClassNotFound as err:
+ raise
+ except Exception as err:
+ raise ClassNotFound('error when loading custom lexer: %s' % err)
+
+
+def find_lexer_class_for_filename(_fn, code=None):
+ """Get a lexer for a filename.
+
+ If multiple lexers match the filename pattern, use ``analyse_text()`` to
+ figure out which one is more appropriate.
+
+ Returns None if not found.
+ """
+ matches = []
+ fn = basename(_fn)
+ for modname, name, _, filenames, _ in itervalues(LEXERS):
+ for filename in filenames:
+ if _fn_matches(fn, filename):
+ if name not in _lexer_cache:
+ _load_lexers(modname)
+ matches.append((_lexer_cache[name], filename))
+ for cls in find_plugin_lexers():
+ for filename in cls.filenames:
+ if _fn_matches(fn, filename):
+ matches.append((cls, filename))
+
+ if sys.version_info > (3,) and isinstance(code, bytes):
+ # decode it, since all analyse_text functions expect unicode
+ code = guess_decode(code)
+
+ def get_rating(info):
+ cls, filename = info
+ # explicit patterns get a bonus
+ bonus = '*' not in filename and 0.5 or 0
+ # The class _always_ defines analyse_text because it's included in
+ # the Lexer class. The default implementation returns None which
+ # gets turned into 0.0. Run scripts/detect_missing_analyse_text.py
+ # to find lexers which need it overridden.
+ if code:
+ return cls.analyse_text(code) + bonus, cls.__name__
+ return cls.priority + bonus, cls.__name__
+
+ if matches:
+ matches.sort(key=get_rating)
+ # print "Possible lexers, after sort:", matches
+ return matches[-1][0]
+
+
+def get_lexer_for_filename(_fn, code=None, **options):
+ """Get a lexer for a filename.
+
+ If multiple lexers match the filename pattern, use ``analyse_text()`` to
+ figure out which one is more appropriate.
+
+ Raises ClassNotFound if not found.
+ """
+ res = find_lexer_class_for_filename(_fn, code)
+ if not res:
+ raise ClassNotFound('no lexer for filename %r found' % _fn)
+ return res(**options)
+
+
+def get_lexer_for_mimetype(_mime, **options):
+ """Get a lexer for a mimetype.
+
+ Raises ClassNotFound if not found.
+ """
+ for modname, name, _, _, mimetypes in itervalues(LEXERS):
+ if _mime in mimetypes:
+ if name not in _lexer_cache:
+ _load_lexers(modname)
+ return _lexer_cache[name](**options)
+ for cls in find_plugin_lexers():
+ if _mime in cls.mimetypes:
+ return cls(**options)
+ raise ClassNotFound('no lexer for mimetype %r found' % _mime)
+
+
+def _iter_lexerclasses(plugins=True):
+ """Return an iterator over all lexer classes."""
+ for key in sorted(LEXERS):
+ module_name, name = LEXERS[key][:2]
+ if name not in _lexer_cache:
+ _load_lexers(module_name)
+ yield _lexer_cache[name]
+ if plugins:
+ for lexer in find_plugin_lexers():
+ yield lexer
+
+
+def guess_lexer_for_filename(_fn, _text, **options):
+ """
+ Lookup all lexers that handle those filenames primary (``filenames``)
+ or secondary (``alias_filenames``). Then run a text analysis for those
+ lexers and choose the best result.
+
+ usage::
+
+ >>> from pygments.lexers import guess_lexer_for_filename
+ >>> guess_lexer_for_filename('hello.html', '<%= @foo %>')
+ <pygments.lexers.templates.RhtmlLexer object at 0xb7d2f32c>
+ >>> guess_lexer_for_filename('hello.html', '<h1>{{ title|e }}</h1>')
+ <pygments.lexers.templates.HtmlDjangoLexer object at 0xb7d2f2ac>
+ >>> guess_lexer_for_filename('style.css', 'a { color: <?= $link ?> }')
+ <pygments.lexers.templates.CssPhpLexer object at 0xb7ba518c>
+ """
+ fn = basename(_fn)
+ primary = {}
+ matching_lexers = set()
+ for lexer in _iter_lexerclasses():
+ for filename in lexer.filenames:
+ if _fn_matches(fn, filename):
+ matching_lexers.add(lexer)
+ primary[lexer] = True
+ for filename in lexer.alias_filenames:
+ if _fn_matches(fn, filename):
+ matching_lexers.add(lexer)
+ primary[lexer] = False
+ if not matching_lexers:
+ raise ClassNotFound('no lexer for filename %r found' % fn)
+ if len(matching_lexers) == 1:
+ return matching_lexers.pop()(**options)
+ result = []
+ for lexer in matching_lexers:
+ rv = lexer.analyse_text(_text)
+ if rv == 1.0:
+ return lexer(**options)
+ result.append((rv, lexer))
+
+ def type_sort(t):
+ # sort by:
+ # - analyse score
+ # - is primary filename pattern?
+ # - priority
+ # - last resort: class name
+ return (t[0], primary[t[1]], t[1].priority, t[1].__name__)
+ result.sort(key=type_sort)
+
+ return result[-1][1](**options)
+
+
+def guess_lexer(_text, **options):
+ """Guess a lexer by strong distinctions in the text (eg, shebang)."""
+
+ # try to get a vim modeline first
+ ft = get_filetype_from_buffer(_text)
+
+ if ft is not None:
+ try:
+ return get_lexer_by_name(ft, **options)
+ except ClassNotFound:
+ pass
+
+ best_lexer = [0.0, None]
+ for lexer in _iter_lexerclasses():
+ rv = lexer.analyse_text(_text)
+ if rv == 1.0:
+ return lexer(**options)
+ if rv > best_lexer[0]:
+ best_lexer[:] = (rv, lexer)
+ if not best_lexer[0] or best_lexer[1] is None:
+ raise ClassNotFound('no lexer matching the text found')
+ return best_lexer[1](**options)
+
+
+class _automodule(types.ModuleType):
+ """Automatically import lexers."""
+
+ def __getattr__(self, name):
+ info = LEXERS.get(name)
+ if info:
+ _load_lexers(info[0])
+ cls = _lexer_cache[info[1]]
+ setattr(self, name, cls)
+ return cls
+ raise AttributeError(name)
+
+
+oldmod = sys.modules[__name__]
+newmod = _automodule(__name__)
+newmod.__dict__.update(oldmod.__dict__)
+sys.modules[__name__] = newmod
+del newmod.newmod, newmod.oldmod, newmod.sys, newmod.types
diff --git a/wandb/vendor/pygments/lexers/_asy_builtins.py b/wandb/vendor/pygments/lexers/_asy_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..1f831cdb78f95ce28078959241be7bff86582cbe
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_asy_builtins.py
@@ -0,0 +1,1645 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._asy_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ This file contains the asy-function names and asy-variable names of
+ Asymptote.
+
+ Do not edit the ASYFUNCNAME and ASYVARNAME sets by hand.
+ TODO: perl/python script in Asymptote SVN similar to asy-list.pl but only
+ for function and variable names.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+ASYFUNCNAME = set((
+ 'AND',
+ 'Arc',
+ 'ArcArrow',
+ 'ArcArrows',
+ 'Arrow',
+ 'Arrows',
+ 'Automatic',
+ 'AvantGarde',
+ 'BBox',
+ 'BWRainbow',
+ 'BWRainbow2',
+ 'Bar',
+ 'Bars',
+ 'BeginArcArrow',
+ 'BeginArrow',
+ 'BeginBar',
+ 'BeginDotMargin',
+ 'BeginMargin',
+ 'BeginPenMargin',
+ 'Blank',
+ 'Bookman',
+ 'Bottom',
+ 'BottomTop',
+ 'Bounds',
+ 'Break',
+ 'Broken',
+ 'BrokenLog',
+ 'Ceil',
+ 'Circle',
+ 'CircleBarIntervalMarker',
+ 'Cos',
+ 'Courier',
+ 'CrossIntervalMarker',
+ 'DefaultFormat',
+ 'DefaultLogFormat',
+ 'Degrees',
+ 'Dir',
+ 'DotMargin',
+ 'DotMargins',
+ 'Dotted',
+ 'Draw',
+ 'Drawline',
+ 'Embed',
+ 'EndArcArrow',
+ 'EndArrow',
+ 'EndBar',
+ 'EndDotMargin',
+ 'EndMargin',
+ 'EndPenMargin',
+ 'Fill',
+ 'FillDraw',
+ 'Floor',
+ 'Format',
+ 'Full',
+ 'Gaussian',
+ 'Gaussrand',
+ 'Gaussrandpair',
+ 'Gradient',
+ 'Grayscale',
+ 'Helvetica',
+ 'Hermite',
+ 'HookHead',
+ 'InOutTicks',
+ 'InTicks',
+ 'J',
+ 'Label',
+ 'Landscape',
+ 'Left',
+ 'LeftRight',
+ 'LeftTicks',
+ 'Legend',
+ 'Linear',
+ 'Link',
+ 'Log',
+ 'LogFormat',
+ 'Margin',
+ 'Margins',
+ 'Mark',
+ 'MidArcArrow',
+ 'MidArrow',
+ 'NOT',
+ 'NewCenturySchoolBook',
+ 'NoBox',
+ 'NoMargin',
+ 'NoModifier',
+ 'NoTicks',
+ 'NoTicks3',
+ 'NoZero',
+ 'NoZeroFormat',
+ 'None',
+ 'OR',
+ 'OmitFormat',
+ 'OmitTick',
+ 'OutTicks',
+ 'Ox',
+ 'Oy',
+ 'Palatino',
+ 'PaletteTicks',
+ 'Pen',
+ 'PenMargin',
+ 'PenMargins',
+ 'Pentype',
+ 'Portrait',
+ 'RadialShade',
+ 'Rainbow',
+ 'Range',
+ 'Relative',
+ 'Right',
+ 'RightTicks',
+ 'Rotate',
+ 'Round',
+ 'SQR',
+ 'Scale',
+ 'ScaleX',
+ 'ScaleY',
+ 'ScaleZ',
+ 'Seascape',
+ 'Shift',
+ 'Sin',
+ 'Slant',
+ 'Spline',
+ 'StickIntervalMarker',
+ 'Straight',
+ 'Symbol',
+ 'Tan',
+ 'TeXify',
+ 'Ticks',
+ 'Ticks3',
+ 'TildeIntervalMarker',
+ 'TimesRoman',
+ 'Top',
+ 'TrueMargin',
+ 'UnFill',
+ 'UpsideDown',
+ 'Wheel',
+ 'X',
+ 'XEquals',
+ 'XOR',
+ 'XY',
+ 'XYEquals',
+ 'XYZero',
+ 'XYgrid',
+ 'XZEquals',
+ 'XZZero',
+ 'XZero',
+ 'XZgrid',
+ 'Y',
+ 'YEquals',
+ 'YXgrid',
+ 'YZ',
+ 'YZEquals',
+ 'YZZero',
+ 'YZero',
+ 'YZgrid',
+ 'Z',
+ 'ZX',
+ 'ZXgrid',
+ 'ZYgrid',
+ 'ZapfChancery',
+ 'ZapfDingbats',
+ '_cputime',
+ '_draw',
+ '_eval',
+ '_image',
+ '_labelpath',
+ '_projection',
+ '_strokepath',
+ '_texpath',
+ 'aCos',
+ 'aSin',
+ 'aTan',
+ 'abort',
+ 'abs',
+ 'accel',
+ 'acos',
+ 'acosh',
+ 'acot',
+ 'acsc',
+ 'add',
+ 'addArrow',
+ 'addMargins',
+ 'addSaveFunction',
+ 'addnode',
+ 'addnodes',
+ 'addpenarc',
+ 'addpenline',
+ 'addseg',
+ 'adjust',
+ 'alias',
+ 'align',
+ 'all',
+ 'altitude',
+ 'angabscissa',
+ 'angle',
+ 'angpoint',
+ 'animate',
+ 'annotate',
+ 'anticomplementary',
+ 'antipedal',
+ 'apply',
+ 'approximate',
+ 'arc',
+ 'arcarrowsize',
+ 'arccircle',
+ 'arcdir',
+ 'arcfromcenter',
+ 'arcfromfocus',
+ 'arclength',
+ 'arcnodesnumber',
+ 'arcpoint',
+ 'arcsubtended',
+ 'arcsubtendedcenter',
+ 'arctime',
+ 'arctopath',
+ 'array',
+ 'arrow',
+ 'arrow2',
+ 'arrowbase',
+ 'arrowbasepoints',
+ 'arrowsize',
+ 'asec',
+ 'asin',
+ 'asinh',
+ 'ask',
+ 'assert',
+ 'asy',
+ 'asycode',
+ 'asydir',
+ 'asyfigure',
+ 'asyfilecode',
+ 'asyinclude',
+ 'asywrite',
+ 'atan',
+ 'atan2',
+ 'atanh',
+ 'atbreakpoint',
+ 'atexit',
+ 'atime',
+ 'attach',
+ 'attract',
+ 'atupdate',
+ 'autoformat',
+ 'autoscale',
+ 'autoscale3',
+ 'axes',
+ 'axes3',
+ 'axialshade',
+ 'axis',
+ 'axiscoverage',
+ 'azimuth',
+ 'babel',
+ 'background',
+ 'bangles',
+ 'bar',
+ 'barmarksize',
+ 'barsize',
+ 'basealign',
+ 'baseline',
+ 'bbox',
+ 'beep',
+ 'begin',
+ 'beginclip',
+ 'begingroup',
+ 'beginpoint',
+ 'between',
+ 'bevel',
+ 'bezier',
+ 'bezierP',
+ 'bezierPP',
+ 'bezierPPP',
+ 'bezulate',
+ 'bibliography',
+ 'bibliographystyle',
+ 'binarytree',
+ 'binarytreeNode',
+ 'binomial',
+ 'binput',
+ 'bins',
+ 'bisector',
+ 'bisectorpoint',
+ 'blend',
+ 'boutput',
+ 'box',
+ 'bqe',
+ 'breakpoint',
+ 'breakpoints',
+ 'brick',
+ 'buildRestoreDefaults',
+ 'buildRestoreThunk',
+ 'buildcycle',
+ 'bulletcolor',
+ 'canonical',
+ 'canonicalcartesiansystem',
+ 'cartesiansystem',
+ 'case1',
+ 'case2',
+ 'case3',
+ 'cbrt',
+ 'cd',
+ 'ceil',
+ 'center',
+ 'centerToFocus',
+ 'centroid',
+ 'cevian',
+ 'change2',
+ 'changecoordsys',
+ 'checkSegment',
+ 'checkconditionlength',
+ 'checker',
+ 'checklengths',
+ 'checkposition',
+ 'checktriangle',
+ 'choose',
+ 'circle',
+ 'circlebarframe',
+ 'circlemarkradius',
+ 'circlenodesnumber',
+ 'circumcenter',
+ 'circumcircle',
+ 'clamped',
+ 'clear',
+ 'clip',
+ 'clipdraw',
+ 'close',
+ 'cmyk',
+ 'code',
+ 'colatitude',
+ 'collect',
+ 'collinear',
+ 'color',
+ 'colorless',
+ 'colors',
+ 'colorspace',
+ 'comma',
+ 'compassmark',
+ 'complement',
+ 'complementary',
+ 'concat',
+ 'concurrent',
+ 'cone',
+ 'conic',
+ 'conicnodesnumber',
+ 'conictype',
+ 'conj',
+ 'connect',
+ 'containmentTree',
+ 'contains',
+ 'contour',
+ 'contour3',
+ 'controlSpecifier',
+ 'convert',
+ 'coordinates',
+ 'coordsys',
+ 'copy',
+ 'cos',
+ 'cosh',
+ 'cot',
+ 'countIntersections',
+ 'cputime',
+ 'crop',
+ 'cropcode',
+ 'cross',
+ 'crossframe',
+ 'crosshatch',
+ 'crossmarksize',
+ 'csc',
+ 'cubicroots',
+ 'curabscissa',
+ 'curlSpecifier',
+ 'curpoint',
+ 'currentarrow',
+ 'currentexitfunction',
+ 'currentmomarrow',
+ 'currentpolarconicroutine',
+ 'curve',
+ 'cut',
+ 'cutafter',
+ 'cutbefore',
+ 'cyclic',
+ 'cylinder',
+ 'debugger',
+ 'deconstruct',
+ 'defaultdir',
+ 'defaultformat',
+ 'defaultpen',
+ 'defined',
+ 'degenerate',
+ 'degrees',
+ 'delete',
+ 'deletepreamble',
+ 'determinant',
+ 'diagonal',
+ 'diamond',
+ 'diffdiv',
+ 'dir',
+ 'dirSpecifier',
+ 'dirtime',
+ 'display',
+ 'distance',
+ 'divisors',
+ 'do_overpaint',
+ 'dot',
+ 'dotframe',
+ 'dotsize',
+ 'downcase',
+ 'draw',
+ 'drawAll',
+ 'drawDoubleLine',
+ 'drawFermion',
+ 'drawGhost',
+ 'drawGluon',
+ 'drawMomArrow',
+ 'drawPhoton',
+ 'drawScalar',
+ 'drawVertex',
+ 'drawVertexBox',
+ 'drawVertexBoxO',
+ 'drawVertexBoxX',
+ 'drawVertexO',
+ 'drawVertexOX',
+ 'drawVertexTriangle',
+ 'drawVertexTriangleO',
+ 'drawVertexX',
+ 'drawarrow',
+ 'drawarrow2',
+ 'drawline',
+ 'drawtick',
+ 'duplicate',
+ 'elle',
+ 'ellipse',
+ 'ellipsenodesnumber',
+ 'embed',
+ 'embed3',
+ 'empty',
+ 'enclose',
+ 'end',
+ 'endScript',
+ 'endclip',
+ 'endgroup',
+ 'endl',
+ 'endpoint',
+ 'endpoints',
+ 'eof',
+ 'eol',
+ 'equation',
+ 'equations',
+ 'erase',
+ 'erasestep',
+ 'erf',
+ 'erfc',
+ 'error',
+ 'errorbar',
+ 'errorbars',
+ 'eval',
+ 'excenter',
+ 'excircle',
+ 'exit',
+ 'exitXasyMode',
+ 'exitfunction',
+ 'exp',
+ 'expfactors',
+ 'expi',
+ 'expm1',
+ 'exradius',
+ 'extend',
+ 'extension',
+ 'extouch',
+ 'fabs',
+ 'factorial',
+ 'fermat',
+ 'fft',
+ 'fhorner',
+ 'figure',
+ 'file',
+ 'filecode',
+ 'fill',
+ 'filldraw',
+ 'filloutside',
+ 'fillrule',
+ 'filltype',
+ 'find',
+ 'finite',
+ 'finiteDifferenceJacobian',
+ 'firstcut',
+ 'firstframe',
+ 'fit',
+ 'fit2',
+ 'fixedscaling',
+ 'floor',
+ 'flush',
+ 'fmdefaults',
+ 'fmod',
+ 'focusToCenter',
+ 'font',
+ 'fontcommand',
+ 'fontsize',
+ 'foot',
+ 'format',
+ 'frac',
+ 'frequency',
+ 'fromCenter',
+ 'fromFocus',
+ 'fspline',
+ 'functionshade',
+ 'gamma',
+ 'generate_random_backtrace',
+ 'generateticks',
+ 'gergonne',
+ 'getc',
+ 'getint',
+ 'getpair',
+ 'getreal',
+ 'getstring',
+ 'gettriple',
+ 'gluon',
+ 'gouraudshade',
+ 'graph',
+ 'graphic',
+ 'gray',
+ 'grestore',
+ 'grid',
+ 'grid3',
+ 'gsave',
+ 'halfbox',
+ 'hatch',
+ 'hdiffdiv',
+ 'hermite',
+ 'hex',
+ 'histogram',
+ 'history',
+ 'hline',
+ 'hprojection',
+ 'hsv',
+ 'hyperbola',
+ 'hyperbolanodesnumber',
+ 'hyperlink',
+ 'hypot',
+ 'identity',
+ 'image',
+ 'incenter',
+ 'incentral',
+ 'incircle',
+ 'increasing',
+ 'incrementposition',
+ 'indexedTransform',
+ 'indexedfigure',
+ 'initXasyMode',
+ 'initdefaults',
+ 'input',
+ 'inradius',
+ 'insert',
+ 'inside',
+ 'integrate',
+ 'interactive',
+ 'interior',
+ 'interp',
+ 'interpolate',
+ 'intersect',
+ 'intersection',
+ 'intersectionpoint',
+ 'intersectionpoints',
+ 'intersections',
+ 'intouch',
+ 'inverse',
+ 'inversion',
+ 'invisible',
+ 'is3D',
+ 'isDuplicate',
+ 'isogonal',
+ 'isogonalconjugate',
+ 'isotomic',
+ 'isotomicconjugate',
+ 'isparabola',
+ 'italic',
+ 'item',
+ 'key',
+ 'kurtosis',
+ 'kurtosisexcess',
+ 'label',
+ 'labelaxis',
+ 'labelmargin',
+ 'labelpath',
+ 'labels',
+ 'labeltick',
+ 'labelx',
+ 'labelx3',
+ 'labely',
+ 'labely3',
+ 'labelz',
+ 'labelz3',
+ 'lastcut',
+ 'latex',
+ 'latitude',
+ 'latticeshade',
+ 'layer',
+ 'layout',
+ 'ldexp',
+ 'leastsquares',
+ 'legend',
+ 'legenditem',
+ 'length',
+ 'lift',
+ 'light',
+ 'limits',
+ 'line',
+ 'linear',
+ 'linecap',
+ 'lineinversion',
+ 'linejoin',
+ 'linemargin',
+ 'lineskip',
+ 'linetype',
+ 'linewidth',
+ 'link',
+ 'list',
+ 'lm_enorm',
+ 'lm_evaluate_default',
+ 'lm_lmdif',
+ 'lm_lmpar',
+ 'lm_minimize',
+ 'lm_print_default',
+ 'lm_print_quiet',
+ 'lm_qrfac',
+ 'lm_qrsolv',
+ 'locale',
+ 'locate',
+ 'locatefile',
+ 'location',
+ 'log',
+ 'log10',
+ 'log1p',
+ 'logaxiscoverage',
+ 'longitude',
+ 'lookup',
+ 'magnetize',
+ 'makeNode',
+ 'makedraw',
+ 'makepen',
+ 'map',
+ 'margin',
+ 'markangle',
+ 'markangleradius',
+ 'markanglespace',
+ 'markarc',
+ 'marker',
+ 'markinterval',
+ 'marknodes',
+ 'markrightangle',
+ 'markuniform',
+ 'mass',
+ 'masscenter',
+ 'massformat',
+ 'math',
+ 'max',
+ 'max3',
+ 'maxbezier',
+ 'maxbound',
+ 'maxcoords',
+ 'maxlength',
+ 'maxratio',
+ 'maxtimes',
+ 'mean',
+ 'medial',
+ 'median',
+ 'midpoint',
+ 'min',
+ 'min3',
+ 'minbezier',
+ 'minbound',
+ 'minipage',
+ 'minratio',
+ 'mintimes',
+ 'miterlimit',
+ 'momArrowPath',
+ 'momarrowsize',
+ 'monotonic',
+ 'multifigure',
+ 'nativeformat',
+ 'natural',
+ 'needshipout',
+ 'newl',
+ 'newpage',
+ 'newslide',
+ 'newton',
+ 'newtree',
+ 'nextframe',
+ 'nextnormal',
+ 'nextpage',
+ 'nib',
+ 'nodabscissa',
+ 'none',
+ 'norm',
+ 'normalvideo',
+ 'notaknot',
+ 'nowarn',
+ 'numberpage',
+ 'nurb',
+ 'object',
+ 'offset',
+ 'onpath',
+ 'opacity',
+ 'opposite',
+ 'orientation',
+ 'orig_circlenodesnumber',
+ 'orig_circlenodesnumber1',
+ 'orig_draw',
+ 'orig_ellipsenodesnumber',
+ 'orig_ellipsenodesnumber1',
+ 'orig_hyperbolanodesnumber',
+ 'orig_parabolanodesnumber',
+ 'origin',
+ 'orthic',
+ 'orthocentercenter',
+ 'outformat',
+ 'outline',
+ 'outprefix',
+ 'output',
+ 'overloadedMessage',
+ 'overwrite',
+ 'pack',
+ 'pad',
+ 'pairs',
+ 'palette',
+ 'parabola',
+ 'parabolanodesnumber',
+ 'parallel',
+ 'partialsum',
+ 'path',
+ 'path3',
+ 'pattern',
+ 'pause',
+ 'pdf',
+ 'pedal',
+ 'periodic',
+ 'perp',
+ 'perpendicular',
+ 'perpendicularmark',
+ 'phantom',
+ 'phi1',
+ 'phi2',
+ 'phi3',
+ 'photon',
+ 'piecewisestraight',
+ 'point',
+ 'polar',
+ 'polarconicroutine',
+ 'polargraph',
+ 'polygon',
+ 'postcontrol',
+ 'postscript',
+ 'pow10',
+ 'ppoint',
+ 'prc',
+ 'prc0',
+ 'precision',
+ 'precontrol',
+ 'prepend',
+ 'print_random_addresses',
+ 'project',
+ 'projection',
+ 'purge',
+ 'pwhermite',
+ 'quadrant',
+ 'quadraticroots',
+ 'quantize',
+ 'quarticroots',
+ 'quotient',
+ 'radialshade',
+ 'radians',
+ 'radicalcenter',
+ 'radicalline',
+ 'radius',
+ 'rand',
+ 'randompath',
+ 'rd',
+ 'readline',
+ 'realmult',
+ 'realquarticroots',
+ 'rectangle',
+ 'rectangular',
+ 'rectify',
+ 'reflect',
+ 'relabscissa',
+ 'relative',
+ 'relativedistance',
+ 'reldir',
+ 'relpoint',
+ 'reltime',
+ 'remainder',
+ 'remark',
+ 'removeDuplicates',
+ 'rename',
+ 'replace',
+ 'report',
+ 'resetdefaultpen',
+ 'restore',
+ 'restoredefaults',
+ 'reverse',
+ 'reversevideo',
+ 'rf',
+ 'rfind',
+ 'rgb',
+ 'rgba',
+ 'rgbint',
+ 'rms',
+ 'rotate',
+ 'rotateO',
+ 'rotation',
+ 'round',
+ 'roundbox',
+ 'roundedpath',
+ 'roundrectangle',
+ 'samecoordsys',
+ 'sameside',
+ 'sample',
+ 'save',
+ 'savedefaults',
+ 'saveline',
+ 'scale',
+ 'scale3',
+ 'scaleO',
+ 'scaleT',
+ 'scaleless',
+ 'scientific',
+ 'search',
+ 'searchtree',
+ 'sec',
+ 'secondaryX',
+ 'secondaryY',
+ 'seconds',
+ 'section',
+ 'sector',
+ 'seek',
+ 'seekeof',
+ 'segment',
+ 'sequence',
+ 'setpens',
+ 'sgn',
+ 'sgnd',
+ 'sharpangle',
+ 'sharpdegrees',
+ 'shift',
+ 'shiftless',
+ 'shipout',
+ 'shipout3',
+ 'show',
+ 'side',
+ 'simeq',
+ 'simpson',
+ 'sin',
+ 'single',
+ 'sinh',
+ 'size',
+ 'size3',
+ 'skewness',
+ 'skip',
+ 'slant',
+ 'sleep',
+ 'slope',
+ 'slopefield',
+ 'solve',
+ 'solveBVP',
+ 'sort',
+ 'sourceline',
+ 'sphere',
+ 'split',
+ 'sqrt',
+ 'square',
+ 'srand',
+ 'standardizecoordsys',
+ 'startScript',
+ 'startTrembling',
+ 'stdev',
+ 'step',
+ 'stickframe',
+ 'stickmarksize',
+ 'stickmarkspace',
+ 'stop',
+ 'straight',
+ 'straightness',
+ 'string',
+ 'stripdirectory',
+ 'stripextension',
+ 'stripfile',
+ 'strokepath',
+ 'subdivide',
+ 'subitem',
+ 'subpath',
+ 'substr',
+ 'sum',
+ 'surface',
+ 'symmedial',
+ 'symmedian',
+ 'system',
+ 'tab',
+ 'tableau',
+ 'tan',
+ 'tangent',
+ 'tangential',
+ 'tangents',
+ 'tanh',
+ 'tell',
+ 'tensionSpecifier',
+ 'tensorshade',
+ 'tex',
+ 'texcolor',
+ 'texify',
+ 'texpath',
+ 'texpreamble',
+ 'texreset',
+ 'texshipout',
+ 'texsize',
+ 'textpath',
+ 'thick',
+ 'thin',
+ 'tick',
+ 'tickMax',
+ 'tickMax3',
+ 'tickMin',
+ 'tickMin3',
+ 'ticklabelshift',
+ 'ticklocate',
+ 'tildeframe',
+ 'tildemarksize',
+ 'tile',
+ 'tiling',
+ 'time',
+ 'times',
+ 'title',
+ 'titlepage',
+ 'topbox',
+ 'transform',
+ 'transformation',
+ 'transpose',
+ 'tremble',
+ 'trembleFuzz',
+ 'tremble_circlenodesnumber',
+ 'tremble_circlenodesnumber1',
+ 'tremble_draw',
+ 'tremble_ellipsenodesnumber',
+ 'tremble_ellipsenodesnumber1',
+ 'tremble_hyperbolanodesnumber',
+ 'tremble_marknodes',
+ 'tremble_markuniform',
+ 'tremble_parabolanodesnumber',
+ 'triangle',
+ 'triangleAbc',
+ 'triangleabc',
+ 'triangulate',
+ 'tricoef',
+ 'tridiagonal',
+ 'trilinear',
+ 'trim',
+ 'trueMagnetize',
+ 'truepoint',
+ 'tube',
+ 'uncycle',
+ 'unfill',
+ 'uniform',
+ 'unit',
+ 'unitrand',
+ 'unitsize',
+ 'unityroot',
+ 'unstraighten',
+ 'upcase',
+ 'updatefunction',
+ 'uperiodic',
+ 'upscale',
+ 'uptodate',
+ 'usepackage',
+ 'usersetting',
+ 'usetypescript',
+ 'usleep',
+ 'value',
+ 'variance',
+ 'variancebiased',
+ 'vbox',
+ 'vector',
+ 'vectorfield',
+ 'verbatim',
+ 'view',
+ 'vline',
+ 'vperiodic',
+ 'vprojection',
+ 'warn',
+ 'warning',
+ 'windingnumber',
+ 'write',
+ 'xaxis',
+ 'xaxis3',
+ 'xaxis3At',
+ 'xaxisAt',
+ 'xequals',
+ 'xinput',
+ 'xlimits',
+ 'xoutput',
+ 'xpart',
+ 'xscale',
+ 'xscaleO',
+ 'xtick',
+ 'xtick3',
+ 'xtrans',
+ 'yaxis',
+ 'yaxis3',
+ 'yaxis3At',
+ 'yaxisAt',
+ 'yequals',
+ 'ylimits',
+ 'ypart',
+ 'yscale',
+ 'yscaleO',
+ 'ytick',
+ 'ytick3',
+ 'ytrans',
+ 'zaxis3',
+ 'zaxis3At',
+ 'zero',
+ 'zero3',
+ 'zlimits',
+ 'zpart',
+ 'ztick',
+ 'ztick3',
+ 'ztrans'
+))
+
+ASYVARNAME = set((
+ 'AliceBlue',
+ 'Align',
+ 'Allow',
+ 'AntiqueWhite',
+ 'Apricot',
+ 'Aqua',
+ 'Aquamarine',
+ 'Aspect',
+ 'Azure',
+ 'BeginPoint',
+ 'Beige',
+ 'Bisque',
+ 'Bittersweet',
+ 'Black',
+ 'BlanchedAlmond',
+ 'Blue',
+ 'BlueGreen',
+ 'BlueViolet',
+ 'Both',
+ 'Break',
+ 'BrickRed',
+ 'Brown',
+ 'BurlyWood',
+ 'BurntOrange',
+ 'CCW',
+ 'CW',
+ 'CadetBlue',
+ 'CarnationPink',
+ 'Center',
+ 'Centered',
+ 'Cerulean',
+ 'Chartreuse',
+ 'Chocolate',
+ 'Coeff',
+ 'Coral',
+ 'CornflowerBlue',
+ 'Cornsilk',
+ 'Crimson',
+ 'Crop',
+ 'Cyan',
+ 'Dandelion',
+ 'DarkBlue',
+ 'DarkCyan',
+ 'DarkGoldenrod',
+ 'DarkGray',
+ 'DarkGreen',
+ 'DarkKhaki',
+ 'DarkMagenta',
+ 'DarkOliveGreen',
+ 'DarkOrange',
+ 'DarkOrchid',
+ 'DarkRed',
+ 'DarkSalmon',
+ 'DarkSeaGreen',
+ 'DarkSlateBlue',
+ 'DarkSlateGray',
+ 'DarkTurquoise',
+ 'DarkViolet',
+ 'DeepPink',
+ 'DeepSkyBlue',
+ 'DefaultHead',
+ 'DimGray',
+ 'DodgerBlue',
+ 'Dotted',
+ 'Draw',
+ 'E',
+ 'ENE',
+ 'EPS',
+ 'ESE',
+ 'E_Euler',
+ 'E_PC',
+ 'E_RK2',
+ 'E_RK3BS',
+ 'Emerald',
+ 'EndPoint',
+ 'Euler',
+ 'Fill',
+ 'FillDraw',
+ 'FireBrick',
+ 'FloralWhite',
+ 'ForestGreen',
+ 'Fuchsia',
+ 'Gainsboro',
+ 'GhostWhite',
+ 'Gold',
+ 'Goldenrod',
+ 'Gray',
+ 'Green',
+ 'GreenYellow',
+ 'Honeydew',
+ 'HookHead',
+ 'Horizontal',
+ 'HotPink',
+ 'I',
+ 'IgnoreAspect',
+ 'IndianRed',
+ 'Indigo',
+ 'Ivory',
+ 'JOIN_IN',
+ 'JOIN_OUT',
+ 'JungleGreen',
+ 'Khaki',
+ 'LM_DWARF',
+ 'LM_MACHEP',
+ 'LM_SQRT_DWARF',
+ 'LM_SQRT_GIANT',
+ 'LM_USERTOL',
+ 'Label',
+ 'Lavender',
+ 'LavenderBlush',
+ 'LawnGreen',
+ 'LeftJustified',
+ 'LeftSide',
+ 'LemonChiffon',
+ 'LightBlue',
+ 'LightCoral',
+ 'LightCyan',
+ 'LightGoldenrodYellow',
+ 'LightGreen',
+ 'LightGrey',
+ 'LightPink',
+ 'LightSalmon',
+ 'LightSeaGreen',
+ 'LightSkyBlue',
+ 'LightSlateGray',
+ 'LightSteelBlue',
+ 'LightYellow',
+ 'Lime',
+ 'LimeGreen',
+ 'Linear',
+ 'Linen',
+ 'Log',
+ 'Logarithmic',
+ 'Magenta',
+ 'Mahogany',
+ 'Mark',
+ 'MarkFill',
+ 'Maroon',
+ 'Max',
+ 'MediumAquamarine',
+ 'MediumBlue',
+ 'MediumOrchid',
+ 'MediumPurple',
+ 'MediumSeaGreen',
+ 'MediumSlateBlue',
+ 'MediumSpringGreen',
+ 'MediumTurquoise',
+ 'MediumVioletRed',
+ 'Melon',
+ 'MidPoint',
+ 'MidnightBlue',
+ 'Min',
+ 'MintCream',
+ 'MistyRose',
+ 'Moccasin',
+ 'Move',
+ 'MoveQuiet',
+ 'Mulberry',
+ 'N',
+ 'NE',
+ 'NNE',
+ 'NNW',
+ 'NW',
+ 'NavajoWhite',
+ 'Navy',
+ 'NavyBlue',
+ 'NoAlign',
+ 'NoCrop',
+ 'NoFill',
+ 'NoSide',
+ 'OldLace',
+ 'Olive',
+ 'OliveDrab',
+ 'OliveGreen',
+ 'Orange',
+ 'OrangeRed',
+ 'Orchid',
+ 'Ox',
+ 'Oy',
+ 'PC',
+ 'PaleGoldenrod',
+ 'PaleGreen',
+ 'PaleTurquoise',
+ 'PaleVioletRed',
+ 'PapayaWhip',
+ 'Peach',
+ 'PeachPuff',
+ 'Periwinkle',
+ 'Peru',
+ 'PineGreen',
+ 'Pink',
+ 'Plum',
+ 'PowderBlue',
+ 'ProcessBlue',
+ 'Purple',
+ 'RK2',
+ 'RK3',
+ 'RK3BS',
+ 'RK4',
+ 'RK5',
+ 'RK5DP',
+ 'RK5F',
+ 'RawSienna',
+ 'Red',
+ 'RedOrange',
+ 'RedViolet',
+ 'Rhodamine',
+ 'RightJustified',
+ 'RightSide',
+ 'RosyBrown',
+ 'RoyalBlue',
+ 'RoyalPurple',
+ 'RubineRed',
+ 'S',
+ 'SE',
+ 'SSE',
+ 'SSW',
+ 'SW',
+ 'SaddleBrown',
+ 'Salmon',
+ 'SandyBrown',
+ 'SeaGreen',
+ 'Seashell',
+ 'Sepia',
+ 'Sienna',
+ 'Silver',
+ 'SimpleHead',
+ 'SkyBlue',
+ 'SlateBlue',
+ 'SlateGray',
+ 'Snow',
+ 'SpringGreen',
+ 'SteelBlue',
+ 'Suppress',
+ 'SuppressQuiet',
+ 'Tan',
+ 'TeXHead',
+ 'Teal',
+ 'TealBlue',
+ 'Thistle',
+ 'Ticksize',
+ 'Tomato',
+ 'Turquoise',
+ 'UnFill',
+ 'VERSION',
+ 'Value',
+ 'Vertical',
+ 'Violet',
+ 'VioletRed',
+ 'W',
+ 'WNW',
+ 'WSW',
+ 'Wheat',
+ 'White',
+ 'WhiteSmoke',
+ 'WildStrawberry',
+ 'XYAlign',
+ 'YAlign',
+ 'Yellow',
+ 'YellowGreen',
+ 'YellowOrange',
+ 'addpenarc',
+ 'addpenline',
+ 'align',
+ 'allowstepping',
+ 'angularsystem',
+ 'animationdelay',
+ 'appendsuffix',
+ 'arcarrowangle',
+ 'arcarrowfactor',
+ 'arrow2sizelimit',
+ 'arrowangle',
+ 'arrowbarb',
+ 'arrowdir',
+ 'arrowfactor',
+ 'arrowhookfactor',
+ 'arrowlength',
+ 'arrowsizelimit',
+ 'arrowtexfactor',
+ 'authorpen',
+ 'axis',
+ 'axiscoverage',
+ 'axislabelfactor',
+ 'background',
+ 'backgroundcolor',
+ 'backgroundpen',
+ 'barfactor',
+ 'barmarksizefactor',
+ 'basealign',
+ 'baselinetemplate',
+ 'beveljoin',
+ 'bigvertexpen',
+ 'bigvertexsize',
+ 'black',
+ 'blue',
+ 'bm',
+ 'bottom',
+ 'bp',
+ 'brown',
+ 'bullet',
+ 'byfoci',
+ 'byvertices',
+ 'camerafactor',
+ 'chartreuse',
+ 'circlemarkradiusfactor',
+ 'circlenodesnumberfactor',
+ 'circleprecision',
+ 'circlescale',
+ 'cm',
+ 'codefile',
+ 'codepen',
+ 'codeskip',
+ 'colorPen',
+ 'coloredNodes',
+ 'coloredSegments',
+ 'conditionlength',
+ 'conicnodesfactor',
+ 'count',
+ 'cputimeformat',
+ 'crossmarksizefactor',
+ 'currentcoordsys',
+ 'currentlight',
+ 'currentpatterns',
+ 'currentpen',
+ 'currentpicture',
+ 'currentposition',
+ 'currentprojection',
+ 'curvilinearsystem',
+ 'cuttings',
+ 'cyan',
+ 'darkblue',
+ 'darkbrown',
+ 'darkcyan',
+ 'darkgray',
+ 'darkgreen',
+ 'darkgrey',
+ 'darkmagenta',
+ 'darkolive',
+ 'darkred',
+ 'dashdotted',
+ 'dashed',
+ 'datepen',
+ 'dateskip',
+ 'debuggerlines',
+ 'debugging',
+ 'deepblue',
+ 'deepcyan',
+ 'deepgray',
+ 'deepgreen',
+ 'deepgrey',
+ 'deepmagenta',
+ 'deepred',
+ 'default',
+ 'defaultControl',
+ 'defaultS',
+ 'defaultbackpen',
+ 'defaultcoordsys',
+ 'defaultfilename',
+ 'defaultformat',
+ 'defaultmassformat',
+ 'defaultpen',
+ 'diagnostics',
+ 'differentlengths',
+ 'dot',
+ 'dotfactor',
+ 'dotframe',
+ 'dotted',
+ 'doublelinepen',
+ 'doublelinespacing',
+ 'down',
+ 'duplicateFuzz',
+ 'ellipsenodesnumberfactor',
+ 'eps',
+ 'epsgeo',
+ 'epsilon',
+ 'evenodd',
+ 'extendcap',
+ 'fermionpen',
+ 'figureborder',
+ 'figuremattpen',
+ 'firstnode',
+ 'firststep',
+ 'foregroundcolor',
+ 'fuchsia',
+ 'fuzz',
+ 'gapfactor',
+ 'ghostpen',
+ 'gluonamplitude',
+ 'gluonpen',
+ 'gluonratio',
+ 'gray',
+ 'green',
+ 'grey',
+ 'hatchepsilon',
+ 'havepagenumber',
+ 'heavyblue',
+ 'heavycyan',
+ 'heavygray',
+ 'heavygreen',
+ 'heavygrey',
+ 'heavymagenta',
+ 'heavyred',
+ 'hline',
+ 'hwratio',
+ 'hyperbolanodesnumberfactor',
+ 'identity4',
+ 'ignore',
+ 'inXasyMode',
+ 'inch',
+ 'inches',
+ 'includegraphicscommand',
+ 'inf',
+ 'infinity',
+ 'institutionpen',
+ 'intMax',
+ 'intMin',
+ 'invert',
+ 'invisible',
+ 'itempen',
+ 'itemskip',
+ 'itemstep',
+ 'labelmargin',
+ 'landscape',
+ 'lastnode',
+ 'left',
+ 'legendhskip',
+ 'legendlinelength',
+ 'legendmargin',
+ 'legendmarkersize',
+ 'legendmaxrelativewidth',
+ 'legendvskip',
+ 'lightblue',
+ 'lightcyan',
+ 'lightgray',
+ 'lightgreen',
+ 'lightgrey',
+ 'lightmagenta',
+ 'lightolive',
+ 'lightred',
+ 'lightyellow',
+ 'linemargin',
+ 'lm_infmsg',
+ 'lm_shortmsg',
+ 'longdashdotted',
+ 'longdashed',
+ 'magenta',
+ 'magneticPoints',
+ 'magneticRadius',
+ 'mantissaBits',
+ 'markangleradius',
+ 'markangleradiusfactor',
+ 'markanglespace',
+ 'markanglespacefactor',
+ 'mediumblue',
+ 'mediumcyan',
+ 'mediumgray',
+ 'mediumgreen',
+ 'mediumgrey',
+ 'mediummagenta',
+ 'mediumred',
+ 'mediumyellow',
+ 'middle',
+ 'minDistDefault',
+ 'minblockheight',
+ 'minblockwidth',
+ 'mincirclediameter',
+ 'minipagemargin',
+ 'minipagewidth',
+ 'minvertexangle',
+ 'miterjoin',
+ 'mm',
+ 'momarrowfactor',
+ 'momarrowlength',
+ 'momarrowmargin',
+ 'momarrowoffset',
+ 'momarrowpen',
+ 'monoPen',
+ 'morepoints',
+ 'nCircle',
+ 'newbulletcolor',
+ 'ngraph',
+ 'nil',
+ 'nmesh',
+ 'nobasealign',
+ 'nodeMarginDefault',
+ 'nodesystem',
+ 'nomarker',
+ 'nopoint',
+ 'noprimary',
+ 'nullpath',
+ 'nullpen',
+ 'numarray',
+ 'ocgindex',
+ 'oldbulletcolor',
+ 'olive',
+ 'orange',
+ 'origin',
+ 'overpaint',
+ 'page',
+ 'pageheight',
+ 'pagemargin',
+ 'pagenumberalign',
+ 'pagenumberpen',
+ 'pagenumberposition',
+ 'pagewidth',
+ 'paleblue',
+ 'palecyan',
+ 'palegray',
+ 'palegreen',
+ 'palegrey',
+ 'palemagenta',
+ 'palered',
+ 'paleyellow',
+ 'parabolanodesnumberfactor',
+ 'perpfactor',
+ 'phi',
+ 'photonamplitude',
+ 'photonpen',
+ 'photonratio',
+ 'pi',
+ 'pink',
+ 'plain',
+ 'plus',
+ 'preamblenodes',
+ 'pt',
+ 'purple',
+ 'r3',
+ 'r4a',
+ 'r4b',
+ 'randMax',
+ 'realDigits',
+ 'realEpsilon',
+ 'realMax',
+ 'realMin',
+ 'red',
+ 'relativesystem',
+ 'reverse',
+ 'right',
+ 'roundcap',
+ 'roundjoin',
+ 'royalblue',
+ 'salmon',
+ 'saveFunctions',
+ 'scalarpen',
+ 'sequencereal',
+ 'settings',
+ 'shipped',
+ 'signedtrailingzero',
+ 'solid',
+ 'springgreen',
+ 'sqrtEpsilon',
+ 'squarecap',
+ 'squarepen',
+ 'startposition',
+ 'stdin',
+ 'stdout',
+ 'stepfactor',
+ 'stepfraction',
+ 'steppagenumberpen',
+ 'stepping',
+ 'stickframe',
+ 'stickmarksizefactor',
+ 'stickmarkspacefactor',
+ 'textpen',
+ 'ticksize',
+ 'tildeframe',
+ 'tildemarksizefactor',
+ 'tinv',
+ 'titlealign',
+ 'titlepagepen',
+ 'titlepageposition',
+ 'titlepen',
+ 'titleskip',
+ 'top',
+ 'trailingzero',
+ 'treeLevelStep',
+ 'treeMinNodeWidth',
+ 'treeNodeStep',
+ 'trembleAngle',
+ 'trembleFrequency',
+ 'trembleRandom',
+ 'tremblingMode',
+ 'undefined',
+ 'unitcircle',
+ 'unitsquare',
+ 'up',
+ 'urlpen',
+ 'urlskip',
+ 'version',
+ 'vertexpen',
+ 'vertexsize',
+ 'viewportmargin',
+ 'viewportsize',
+ 'vline',
+ 'white',
+ 'wye',
+ 'xformStack',
+ 'yellow',
+ 'ylabelwidth',
+ 'zerotickfuzz',
+ 'zerowinding'
+))
diff --git a/wandb/vendor/pygments/lexers/_cl_builtins.py b/wandb/vendor/pygments/lexers/_cl_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..ce5ad48e2c2f8db0254db89668191f1e0e0e6dcf
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_cl_builtins.py
@@ -0,0 +1,232 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._cl_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ ANSI Common Lisp builtins.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+BUILTIN_FUNCTIONS = set(( # 638 functions
+ '<', '<=', '=', '>', '>=', '-', '/', '/=', '*', '+', '1-', '1+',
+ 'abort', 'abs', 'acons', 'acos', 'acosh', 'add-method', 'adjoin',
+ 'adjustable-array-p', 'adjust-array', 'allocate-instance',
+ 'alpha-char-p', 'alphanumericp', 'append', 'apply', 'apropos',
+ 'apropos-list', 'aref', 'arithmetic-error-operands',
+ 'arithmetic-error-operation', 'array-dimension', 'array-dimensions',
+ 'array-displacement', 'array-element-type', 'array-has-fill-pointer-p',
+ 'array-in-bounds-p', 'arrayp', 'array-rank', 'array-row-major-index',
+ 'array-total-size', 'ash', 'asin', 'asinh', 'assoc', 'assoc-if',
+ 'assoc-if-not', 'atan', 'atanh', 'atom', 'bit', 'bit-and', 'bit-andc1',
+ 'bit-andc2', 'bit-eqv', 'bit-ior', 'bit-nand', 'bit-nor', 'bit-not',
+ 'bit-orc1', 'bit-orc2', 'bit-vector-p', 'bit-xor', 'boole',
+ 'both-case-p', 'boundp', 'break', 'broadcast-stream-streams',
+ 'butlast', 'byte', 'byte-position', 'byte-size', 'caaaar', 'caaadr',
+ 'caaar', 'caadar', 'caaddr', 'caadr', 'caar', 'cadaar', 'cadadr',
+ 'cadar', 'caddar', 'cadddr', 'caddr', 'cadr', 'call-next-method', 'car',
+ 'cdaaar', 'cdaadr', 'cdaar', 'cdadar', 'cdaddr', 'cdadr', 'cdar',
+ 'cddaar', 'cddadr', 'cddar', 'cdddar', 'cddddr', 'cdddr', 'cddr', 'cdr',
+ 'ceiling', 'cell-error-name', 'cerror', 'change-class', 'char', 'char<',
+ 'char<=', 'char=', 'char>', 'char>=', 'char/=', 'character',
+ 'characterp', 'char-code', 'char-downcase', 'char-equal',
+ 'char-greaterp', 'char-int', 'char-lessp', 'char-name',
+ 'char-not-equal', 'char-not-greaterp', 'char-not-lessp', 'char-upcase',
+ 'cis', 'class-name', 'class-of', 'clear-input', 'clear-output',
+ 'close', 'clrhash', 'code-char', 'coerce', 'compile',
+ 'compiled-function-p', 'compile-file', 'compile-file-pathname',
+ 'compiler-macro-function', 'complement', 'complex', 'complexp',
+ 'compute-applicable-methods', 'compute-restarts', 'concatenate',
+ 'concatenated-stream-streams', 'conjugate', 'cons', 'consp',
+ 'constantly', 'constantp', 'continue', 'copy-alist', 'copy-list',
+ 'copy-pprint-dispatch', 'copy-readtable', 'copy-seq', 'copy-structure',
+ 'copy-symbol', 'copy-tree', 'cos', 'cosh', 'count', 'count-if',
+ 'count-if-not', 'decode-float', 'decode-universal-time', 'delete',
+ 'delete-duplicates', 'delete-file', 'delete-if', 'delete-if-not',
+ 'delete-package', 'denominator', 'deposit-field', 'describe',
+ 'describe-object', 'digit-char', 'digit-char-p', 'directory',
+ 'directory-namestring', 'disassemble', 'documentation', 'dpb',
+ 'dribble', 'echo-stream-input-stream', 'echo-stream-output-stream',
+ 'ed', 'eighth', 'elt', 'encode-universal-time', 'endp',
+ 'enough-namestring', 'ensure-directories-exist',
+ 'ensure-generic-function', 'eq', 'eql', 'equal', 'equalp', 'error',
+ 'eval', 'evenp', 'every', 'exp', 'export', 'expt', 'fboundp',
+ 'fceiling', 'fdefinition', 'ffloor', 'fifth', 'file-author',
+ 'file-error-pathname', 'file-length', 'file-namestring',
+ 'file-position', 'file-string-length', 'file-write-date',
+ 'fill', 'fill-pointer', 'find', 'find-all-symbols', 'find-class',
+ 'find-if', 'find-if-not', 'find-method', 'find-package', 'find-restart',
+ 'find-symbol', 'finish-output', 'first', 'float', 'float-digits',
+ 'floatp', 'float-precision', 'float-radix', 'float-sign', 'floor',
+ 'fmakunbound', 'force-output', 'format', 'fourth', 'fresh-line',
+ 'fround', 'ftruncate', 'funcall', 'function-keywords',
+ 'function-lambda-expression', 'functionp', 'gcd', 'gensym', 'gentemp',
+ 'get', 'get-decoded-time', 'get-dispatch-macro-character', 'getf',
+ 'gethash', 'get-internal-real-time', 'get-internal-run-time',
+ 'get-macro-character', 'get-output-stream-string', 'get-properties',
+ 'get-setf-expansion', 'get-universal-time', 'graphic-char-p',
+ 'hash-table-count', 'hash-table-p', 'hash-table-rehash-size',
+ 'hash-table-rehash-threshold', 'hash-table-size', 'hash-table-test',
+ 'host-namestring', 'identity', 'imagpart', 'import',
+ 'initialize-instance', 'input-stream-p', 'inspect',
+ 'integer-decode-float', 'integer-length', 'integerp',
+ 'interactive-stream-p', 'intern', 'intersection',
+ 'invalid-method-error', 'invoke-debugger', 'invoke-restart',
+ 'invoke-restart-interactively', 'isqrt', 'keywordp', 'last', 'lcm',
+ 'ldb', 'ldb-test', 'ldiff', 'length', 'lisp-implementation-type',
+ 'lisp-implementation-version', 'list', 'list*', 'list-all-packages',
+ 'listen', 'list-length', 'listp', 'load',
+ 'load-logical-pathname-translations', 'log', 'logand', 'logandc1',
+ 'logandc2', 'logbitp', 'logcount', 'logeqv', 'logical-pathname',
+ 'logical-pathname-translations', 'logior', 'lognand', 'lognor',
+ 'lognot', 'logorc1', 'logorc2', 'logtest', 'logxor', 'long-site-name',
+ 'lower-case-p', 'machine-instance', 'machine-type', 'machine-version',
+ 'macroexpand', 'macroexpand-1', 'macro-function', 'make-array',
+ 'make-broadcast-stream', 'make-concatenated-stream', 'make-condition',
+ 'make-dispatch-macro-character', 'make-echo-stream', 'make-hash-table',
+ 'make-instance', 'make-instances-obsolete', 'make-list',
+ 'make-load-form', 'make-load-form-saving-slots', 'make-package',
+ 'make-pathname', 'make-random-state', 'make-sequence', 'make-string',
+ 'make-string-input-stream', 'make-string-output-stream', 'make-symbol',
+ 'make-synonym-stream', 'make-two-way-stream', 'makunbound', 'map',
+ 'mapc', 'mapcan', 'mapcar', 'mapcon', 'maphash', 'map-into', 'mapl',
+ 'maplist', 'mask-field', 'max', 'member', 'member-if', 'member-if-not',
+ 'merge', 'merge-pathnames', 'method-combination-error',
+ 'method-qualifiers', 'min', 'minusp', 'mismatch', 'mod',
+ 'muffle-warning', 'name-char', 'namestring', 'nbutlast', 'nconc',
+ 'next-method-p', 'nintersection', 'ninth', 'no-applicable-method',
+ 'no-next-method', 'not', 'notany', 'notevery', 'nreconc', 'nreverse',
+ 'nset-difference', 'nset-exclusive-or', 'nstring-capitalize',
+ 'nstring-downcase', 'nstring-upcase', 'nsublis', 'nsubst', 'nsubst-if',
+ 'nsubst-if-not', 'nsubstitute', 'nsubstitute-if', 'nsubstitute-if-not',
+ 'nth', 'nthcdr', 'null', 'numberp', 'numerator', 'nunion', 'oddp',
+ 'open', 'open-stream-p', 'output-stream-p', 'package-error-package',
+ 'package-name', 'package-nicknames', 'packagep',
+ 'package-shadowing-symbols', 'package-used-by-list', 'package-use-list',
+ 'pairlis', 'parse-integer', 'parse-namestring', 'pathname',
+ 'pathname-device', 'pathname-directory', 'pathname-host',
+ 'pathname-match-p', 'pathname-name', 'pathnamep', 'pathname-type',
+ 'pathname-version', 'peek-char', 'phase', 'plusp', 'position',
+ 'position-if', 'position-if-not', 'pprint', 'pprint-dispatch',
+ 'pprint-fill', 'pprint-indent', 'pprint-linear', 'pprint-newline',
+ 'pprint-tab', 'pprint-tabular', 'prin1', 'prin1-to-string', 'princ',
+ 'princ-to-string', 'print', 'print-object', 'probe-file', 'proclaim',
+ 'provide', 'random', 'random-state-p', 'rassoc', 'rassoc-if',
+ 'rassoc-if-not', 'rational', 'rationalize', 'rationalp', 'read',
+ 'read-byte', 'read-char', 'read-char-no-hang', 'read-delimited-list',
+ 'read-from-string', 'read-line', 'read-preserving-whitespace',
+ 'read-sequence', 'readtable-case', 'readtablep', 'realp', 'realpart',
+ 'reduce', 'reinitialize-instance', 'rem', 'remhash', 'remove',
+ 'remove-duplicates', 'remove-if', 'remove-if-not', 'remove-method',
+ 'remprop', 'rename-file', 'rename-package', 'replace', 'require',
+ 'rest', 'restart-name', 'revappend', 'reverse', 'room', 'round',
+ 'row-major-aref', 'rplaca', 'rplacd', 'sbit', 'scale-float', 'schar',
+ 'search', 'second', 'set', 'set-difference',
+ 'set-dispatch-macro-character', 'set-exclusive-or',
+ 'set-macro-character', 'set-pprint-dispatch', 'set-syntax-from-char',
+ 'seventh', 'shadow', 'shadowing-import', 'shared-initialize',
+ 'short-site-name', 'signal', 'signum', 'simple-bit-vector-p',
+ 'simple-condition-format-arguments', 'simple-condition-format-control',
+ 'simple-string-p', 'simple-vector-p', 'sin', 'sinh', 'sixth', 'sleep',
+ 'slot-boundp', 'slot-exists-p', 'slot-makunbound', 'slot-missing',
+ 'slot-unbound', 'slot-value', 'software-type', 'software-version',
+ 'some', 'sort', 'special-operator-p', 'sqrt', 'stable-sort',
+ 'standard-char-p', 'store-value', 'stream-element-type',
+ 'stream-error-stream', 'stream-external-format', 'streamp', 'string',
+ 'string<', 'string<=', 'string=', 'string>', 'string>=', 'string/=',
+ 'string-capitalize', 'string-downcase', 'string-equal',
+ 'string-greaterp', 'string-left-trim', 'string-lessp',
+ 'string-not-equal', 'string-not-greaterp', 'string-not-lessp',
+ 'stringp', 'string-right-trim', 'string-trim', 'string-upcase',
+ 'sublis', 'subseq', 'subsetp', 'subst', 'subst-if', 'subst-if-not',
+ 'substitute', 'substitute-if', 'substitute-if-not', 'subtypep','svref',
+ 'sxhash', 'symbol-function', 'symbol-name', 'symbolp', 'symbol-package',
+ 'symbol-plist', 'symbol-value', 'synonym-stream-symbol', 'syntax:',
+ 'tailp', 'tan', 'tanh', 'tenth', 'terpri', 'third',
+ 'translate-logical-pathname', 'translate-pathname', 'tree-equal',
+ 'truename', 'truncate', 'two-way-stream-input-stream',
+ 'two-way-stream-output-stream', 'type-error-datum',
+ 'type-error-expected-type', 'type-of', 'typep', 'unbound-slot-instance',
+ 'unexport', 'unintern', 'union', 'unread-char', 'unuse-package',
+ 'update-instance-for-different-class',
+ 'update-instance-for-redefined-class', 'upgraded-array-element-type',
+ 'upgraded-complex-part-type', 'upper-case-p', 'use-package',
+ 'user-homedir-pathname', 'use-value', 'values', 'values-list', 'vector',
+ 'vectorp', 'vector-pop', 'vector-push', 'vector-push-extend', 'warn',
+ 'wild-pathname-p', 'write', 'write-byte', 'write-char', 'write-line',
+ 'write-sequence', 'write-string', 'write-to-string', 'yes-or-no-p',
+ 'y-or-n-p', 'zerop',
+))
+
+SPECIAL_FORMS = set((
+ 'block', 'catch', 'declare', 'eval-when', 'flet', 'function', 'go', 'if',
+ 'labels', 'lambda', 'let', 'let*', 'load-time-value', 'locally', 'macrolet',
+ 'multiple-value-call', 'multiple-value-prog1', 'progn', 'progv', 'quote',
+ 'return-from', 'setq', 'symbol-macrolet', 'tagbody', 'the', 'throw',
+ 'unwind-protect',
+))
+
+MACROS = set((
+ 'and', 'assert', 'call-method', 'case', 'ccase', 'check-type', 'cond',
+ 'ctypecase', 'decf', 'declaim', 'defclass', 'defconstant', 'defgeneric',
+ 'define-compiler-macro', 'define-condition', 'define-method-combination',
+ 'define-modify-macro', 'define-setf-expander', 'define-symbol-macro',
+ 'defmacro', 'defmethod', 'defpackage', 'defparameter', 'defsetf',
+ 'defstruct', 'deftype', 'defun', 'defvar', 'destructuring-bind', 'do',
+ 'do*', 'do-all-symbols', 'do-external-symbols', 'dolist', 'do-symbols',
+ 'dotimes', 'ecase', 'etypecase', 'formatter', 'handler-bind',
+ 'handler-case', 'ignore-errors', 'incf', 'in-package', 'lambda', 'loop',
+ 'loop-finish', 'make-method', 'multiple-value-bind', 'multiple-value-list',
+ 'multiple-value-setq', 'nth-value', 'or', 'pop',
+ 'pprint-exit-if-list-exhausted', 'pprint-logical-block', 'pprint-pop',
+ 'print-unreadable-object', 'prog', 'prog*', 'prog1', 'prog2', 'psetf',
+ 'psetq', 'push', 'pushnew', 'remf', 'restart-bind', 'restart-case',
+ 'return', 'rotatef', 'setf', 'shiftf', 'step', 'time', 'trace', 'typecase',
+ 'unless', 'untrace', 'when', 'with-accessors', 'with-compilation-unit',
+ 'with-condition-restarts', 'with-hash-table-iterator',
+ 'with-input-from-string', 'with-open-file', 'with-open-stream',
+ 'with-output-to-string', 'with-package-iterator', 'with-simple-restart',
+ 'with-slots', 'with-standard-io-syntax',
+))
+
+LAMBDA_LIST_KEYWORDS = set((
+ '&allow-other-keys', '&aux', '&body', '&environment', '&key', '&optional',
+ '&rest', '&whole',
+))
+
+DECLARATIONS = set((
+ 'dynamic-extent', 'ignore', 'optimize', 'ftype', 'inline', 'special',
+ 'ignorable', 'notinline', 'type',
+))
+
+BUILTIN_TYPES = set((
+ 'atom', 'boolean', 'base-char', 'base-string', 'bignum', 'bit',
+ 'compiled-function', 'extended-char', 'fixnum', 'keyword', 'nil',
+ 'signed-byte', 'short-float', 'single-float', 'double-float', 'long-float',
+ 'simple-array', 'simple-base-string', 'simple-bit-vector', 'simple-string',
+ 'simple-vector', 'standard-char', 'unsigned-byte',
+
+ # Condition Types
+ 'arithmetic-error', 'cell-error', 'condition', 'control-error',
+ 'division-by-zero', 'end-of-file', 'error', 'file-error',
+ 'floating-point-inexact', 'floating-point-overflow',
+ 'floating-point-underflow', 'floating-point-invalid-operation',
+ 'parse-error', 'package-error', 'print-not-readable', 'program-error',
+ 'reader-error', 'serious-condition', 'simple-condition', 'simple-error',
+ 'simple-type-error', 'simple-warning', 'stream-error', 'storage-condition',
+ 'style-warning', 'type-error', 'unbound-variable', 'unbound-slot',
+ 'undefined-function', 'warning',
+))
+
+BUILTIN_CLASSES = set((
+ 'array', 'broadcast-stream', 'bit-vector', 'built-in-class', 'character',
+ 'class', 'complex', 'concatenated-stream', 'cons', 'echo-stream',
+ 'file-stream', 'float', 'function', 'generic-function', 'hash-table',
+ 'integer', 'list', 'logical-pathname', 'method-combination', 'method',
+ 'null', 'number', 'package', 'pathname', 'ratio', 'rational', 'readtable',
+ 'real', 'random-state', 'restart', 'sequence', 'standard-class',
+ 'standard-generic-function', 'standard-method', 'standard-object',
+ 'string-stream', 'stream', 'string', 'structure-class', 'structure-object',
+ 'symbol', 'synonym-stream', 't', 'two-way-stream', 'vector',
+))
diff --git a/wandb/vendor/pygments/lexers/_cocoa_builtins.py b/wandb/vendor/pygments/lexers/_cocoa_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..064167ffd4f4a0d30111106cade6ec67876d6032
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_cocoa_builtins.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._cocoa_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ This file defines a set of types used across Cocoa frameworks from Apple.
+ There is a list of @interfaces, @protocols and some other (structs, unions)
+
+ File may be also used as standalone generator for aboves.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import print_function
+
+COCOA_INTERFACES = set(['UITableViewCell', 'HKCorrelationQuery', 'NSURLSessionDataTask', 'PHFetchOptions', 'NSLinguisticTagger', 'NSStream', 'AVAudioUnitDelay', 'GCMotion', 'SKPhysicsWorld', 'NSString', 'CMAttitude', 'AVAudioEnvironmentDistanceAttenuationParameters', 'HKStatisticsCollection', 'SCNPlane', 'CBPeer', 'JSContext', 'SCNTransaction', 'SCNTorus', 'AVAudioUnitEffect', 'UICollectionReusableView', 'MTLSamplerDescriptor', 'AVAssetReaderSampleReferenceOutput', 'AVMutableCompositionTrack', 'GKLeaderboard', 'NSFetchedResultsController', 'SKRange', 'MKTileOverlayRenderer', 'MIDINetworkSession', 'UIVisualEffectView', 'CIWarpKernel', 'PKObject', 'MKRoute', 'MPVolumeView', 'UIPrintInfo', 'SCNText', 'ADClient', 'PKPayment', 'AVMutableAudioMix', 'GLKEffectPropertyLight', 'WKScriptMessage', 'AVMIDIPlayer', 'PHCollectionListChangeRequest', 'UICollectionViewLayout', 'NSMutableCharacterSet', 'SKPaymentTransaction', 'NEOnDemandRuleConnect', 'NSShadow', 'SCNView', 'NSURLSessionConfiguration', 'MTLVertexAttributeDescriptor', 'CBCharacteristic', 'HKQuantityType', 'CKLocationSortDescriptor', 'NEVPNIKEv2SecurityAssociationParameters', 'CMStepCounter', 'NSNetService', 'AVAssetWriterInputMetadataAdaptor', 'UICollectionView', 'UIViewPrintFormatter', 'SCNLevelOfDetail', 'CAShapeLayer', 'MCPeerID', 'MPRatingCommand', 'WKNavigation', 'NSDictionary', 'NSFileVersion', 'CMGyroData', 'AVAudioUnitDistortion', 'CKFetchRecordsOperation', 'SKPhysicsJointSpring', 'SCNHitTestResult', 'AVAudioTime', 'CIFilter', 'UIView', 'SCNConstraint', 'CAPropertyAnimation', 'MKMapItem', 'MPRemoteCommandCenter', 'PKPaymentSummaryItem', 'UICollectionViewFlowLayoutInvalidationContext', 'UIInputViewController', 'PKPass', 'SCNPhysicsBehavior', 'MTLRenderPassColorAttachmentDescriptor', 'MKPolygonRenderer', 'CKNotification', 'JSValue', 'PHCollectionList', 'CLGeocoder', 'NSByteCountFormatter', 'AVCaptureScreenInput', 'MPFeedbackCommand', 'CAAnimation', 'MKOverlayPathView', 'UIActionSheet', 'UIMotionEffectGroup', 'NSLengthFormatter', 'UIBarItem', 'SKProduct', 'AVAssetExportSession', 'NSKeyedUnarchiver', 'NSMutableSet', 'SCNPyramid', 'PHAssetCollection', 'MKMapView', 'HMHomeManager', 'CATransition', 'MTLCompileOptions', 'UIVibrancyEffect', 'CLCircularRegion', 'MKTileOverlay', 'SCNShape', 'ACAccountCredential', 'SKPhysicsJointLimit', 'MKMapSnapshotter', 'AVMediaSelectionGroup', 'NSIndexSet', 'CBPeripheralManager', 'CKRecordZone', 'AVAudioRecorder', 'NSURL', 'CBCentral', 'NSNumber', 'AVAudioOutputNode', 'MTLVertexAttributeDescriptorArray', 'MKETAResponse', 'SKTransition', 'SSReadingList', 'HKSourceQuery', 'UITableViewRowAction', 'UITableView', 'SCNParticlePropertyController', 'AVCaptureStillImageOutput', 'GCController', 'AVAudioPlayerNode', 'AVAudioSessionPortDescription', 'NSHTTPURLResponse', 'NEOnDemandRuleEvaluateConnection', 'SKEffectNode', 'HKQuantity', 'GCControllerElement', 'AVPlayerItemAccessLogEvent', 'SCNBox', 'NSExtensionContext', 'MKOverlayRenderer', 'SCNPhysicsVehicle', 'NSDecimalNumber', 'EKReminder', 'MKPolylineView', 'CKQuery', 'AVAudioMixerNode', 'GKAchievementDescription', 'EKParticipant', 'NSBlockOperation', 'UIActivityItemProvider', 'CLLocation', 'NSBatchUpdateRequest', 'PHContentEditingOutput', 'PHObjectChangeDetails', 'HKWorkoutType', 'MPMoviePlayerController', 'AVAudioFormat', 'HMTrigger', 'MTLRenderPassDepthAttachmentDescriptor', 'SCNRenderer', 'GKScore', 'UISplitViewController', 'HKSource', 'NSURLConnection', 'ABUnknownPersonViewController', 'SCNTechnique', 'UIMenuController', 'NSEvent', 'SKTextureAtlas', 'NSKeyedArchiver', 'GKLeaderboardSet', 'NSSimpleCString', 'AVAudioPCMBuffer', 'CBATTRequest', 'GKMatchRequest', 'AVMetadataObject', 'SKProductsRequest', 'UIAlertView', 'NSIncrementalStore', 'MFMailComposeViewController', 'SCNFloor', 'NSSortDescriptor', 'CKFetchNotificationChangesOperation', 'MPMovieAccessLog', 'NSManagedObjectContext', 'AVAudioUnitGenerator', 'WKBackForwardList', 'SKMutableTexture', 'AVCaptureAudioDataOutput', 'ACAccount', 'AVMetadataItem', 'MPRatingCommandEvent', 'AVCaptureDeviceInputSource', 'CLLocationManager', 'MPRemoteCommand', 'AVCaptureSession', 'UIStepper', 'UIRefreshControl', 'NEEvaluateConnectionRule', 'CKModifyRecordsOperation', 'UICollectionViewTransitionLayout', 'CBCentralManager', 'NSPurgeableData', 'PKShippingMethod', 'SLComposeViewController', 'NSHashTable', 'MKUserTrackingBarButtonItem', 'UILexiconEntry', 'CMMotionActivity', 'SKAction', 'SKShader', 'AVPlayerItemOutput', 'MTLRenderPassAttachmentDescriptor', 'UIDocumentInteractionController', 'UIDynamicItemBehavior', 'NSMutableDictionary', 'UILabel', 'AVCaptureInputPort', 'NSExpression', 'CAInterAppAudioTransportView', 'SKMutablePayment', 'UIImage', 'PHCachingImageManager', 'SCNTransformConstraint', 'HKCorrelationType', 'UIColor', 'SCNGeometrySource', 'AVCaptureAutoExposureBracketedStillImageSettings', 'UIPopoverBackgroundView', 'UIToolbar', 'NSNotificationCenter', 'UICollectionViewLayoutAttributes', 'AVAssetReaderOutputMetadataAdaptor', 'NSEntityMigrationPolicy', 'HMUser', 'NSLocale', 'NSURLSession', 'SCNCamera', 'NSTimeZone', 'UIManagedDocument', 'AVMutableVideoCompositionLayerInstruction', 'AVAssetTrackGroup', 'NSInvocationOperation', 'ALAssetRepresentation', 'AVQueuePlayer', 'HMServiceGroup', 'UIPasteboard', 'PHContentEditingInput', 'NSLayoutManager', 'EKCalendarChooser', 'EKObject', 'CATiledLayer', 'GLKReflectionMapEffect', 'NSManagedObjectID', 'NSEnergyFormatter', 'SLRequest', 'HMCharacteristic', 'AVPlayerLayer', 'MTLRenderPassDescriptor', 'SKPayment', 'NSPointerArray', 'AVAudioMix', 'SCNLight', 'MCAdvertiserAssistant', 'MKMapSnapshotOptions', 'HKCategorySample', 'AVAudioEnvironmentReverbParameters', 'SCNMorpher', 'AVTimedMetadataGroup', 'CBMutableCharacteristic', 'NSFetchRequest', 'UIDevice', 'NSManagedObject', 'NKAssetDownload', 'AVOutputSettingsAssistant', 'SKPhysicsJointPin', 'UITabBar', 'UITextInputMode', 'NSFetchRequestExpression', 'HMActionSet', 'CTSubscriber', 'PHAssetChangeRequest', 'NSPersistentStoreRequest', 'UITabBarController', 'HKQuantitySample', 'AVPlayerItem', 'AVSynchronizedLayer', 'MKDirectionsRequest', 'NSMetadataItem', 'UIPresentationController', 'UINavigationItem', 'PHFetchResultChangeDetails', 'PHImageManager', 'AVCaptureManualExposureBracketedStillImageSettings', 'UIStoryboardPopoverSegue', 'SCNLookAtConstraint', 'UIGravityBehavior', 'UIWindow', 'CBMutableDescriptor', 'NEOnDemandRuleDisconnect', 'UIBezierPath', 'UINavigationController', 'ABPeoplePickerNavigationController', 'EKSource', 'AVAssetWriterInput', 'AVPlayerItemTrack', 'GLKEffectPropertyTexture', 'NSHTTPCookie', 'NSURLResponse', 'SKPaymentQueue', 'NSAssertionHandler', 'MKReverseGeocoder', 'GCControllerAxisInput', 'NSArray', 'NSOrthography', 'NSURLSessionUploadTask', 'NSCharacterSet', 'AVMutableVideoCompositionInstruction', 'AVAssetReaderOutput', 'EAGLContext', 'WKFrameInfo', 'CMPedometer', 'MyClass', 'CKModifyBadgeOperation', 'AVCaptureAudioFileOutput', 'SKEmitterNode', 'NSMachPort', 'AVVideoCompositionCoreAnimationTool', 'PHCollection', 'SCNPhysicsWorld', 'NSURLRequest', 'CMAccelerometerData', 'NSNetServiceBrowser', 'CLFloor', 'AVAsynchronousVideoCompositionRequest', 'SCNGeometry', 'SCNIKConstraint', 'CIKernel', 'CAGradientLayer', 'HKCharacteristicType', 'NSFormatter', 'SCNAction', 'CATransaction', 'CBUUID', 'UIStoryboard', 'MPMediaLibrary', 'UITapGestureRecognizer', 'MPMediaItemArtwork', 'NSURLSessionTask', 'AVAudioUnit', 'MCBrowserViewController', 'UIFontDescriptor', 'NSRelationshipDescription', 'HKSample', 'WKWebView', 'NSMutableAttributedString', 'NSPersistentStoreAsynchronousResult', 'MPNowPlayingInfoCenter', 'MKLocalSearch', 'EAAccessory', 'HKCorrelation', 'CATextLayer', 'NSNotificationQueue', 'UINib', 'GLKTextureLoader', 'HKObjectType', 'NSValue', 'NSMutableIndexSet', 'SKPhysicsContact', 'NSProgress', 'AVPlayerViewController', 'CAScrollLayer', 'GKSavedGame', 'NSTextCheckingResult', 'PHObjectPlaceholder', 'SKConstraint', 'EKEventEditViewController', 'NSEntityDescription', 'NSURLCredentialStorage', 'UIApplication', 'SKDownload', 'SCNNode', 'MKLocalSearchRequest', 'SKScene', 'UISearchDisplayController', 'NEOnDemandRule', 'MTLRenderPassStencilAttachmentDescriptor', 'CAReplicatorLayer', 'UIPrintPageRenderer', 'EKCalendarItem', 'NSUUID', 'EAAccessoryManager', 'NEOnDemandRuleIgnore', 'SKRegion', 'AVAssetResourceLoader', 'EAWiFiUnconfiguredAccessoryBrowser', 'NSUserActivity', 'CTCall', 'UIPrinterPickerController', 'CIVector', 'UINavigationBar', 'UIPanGestureRecognizer', 'MPMediaQuery', 'ABNewPersonViewController', 'CKRecordZoneID', 'HKAnchoredObjectQuery', 'CKFetchRecordZonesOperation', 'UIStoryboardSegue', 'ACAccountType', 'GKSession', 'SKVideoNode', 'PHChange', 'SKReceiptRefreshRequest', 'GCExtendedGamepadSnapshot', 'MPSeekCommandEvent', 'GCExtendedGamepad', 'CAValueFunction', 'SCNCylinder', 'NSNotification', 'NSBatchUpdateResult', 'PKPushCredentials', 'SCNPhysicsSliderJoint', 'AVCaptureDeviceFormat', 'AVPlayerItemErrorLog', 'NSMapTable', 'NSSet', 'CMMotionManager', 'GKVoiceChatService', 'UIPageControl', 'UILexicon', 'MTLArrayType', 'AVAudioUnitReverb', 'MKGeodesicPolyline', 'AVMutableComposition', 'NSLayoutConstraint', 'UIPrinter', 'NSOrderedSet', 'CBAttribute', 'PKPushPayload', 'NSIncrementalStoreNode', 'EKEventStore', 'MPRemoteCommandEvent', 'UISlider', 'UIBlurEffect', 'CKAsset', 'AVCaptureInput', 'AVAudioEngine', 'MTLVertexDescriptor', 'SKPhysicsBody', 'NSOperation', 'PKPaymentPass', 'UIImageAsset', 'MKMapCamera', 'SKProductsResponse', 'GLKEffectPropertyMaterial', 'AVCaptureDevice', 'CTCallCenter', 'CABTMIDILocalPeripheralViewController', 'NEVPNManager', 'HKQuery', 'SCNPhysicsContact', 'CBMutableService', 'AVSampleBufferDisplayLayer', 'SCNSceneSource', 'SKLightNode', 'CKDiscoveredUserInfo', 'NSMutableArray', 'MTLDepthStencilDescriptor', 'MTLArgument', 'NSMassFormatter', 'CIRectangleFeature', 'PKPushRegistry', 'NEVPNConnection', 'MCNearbyServiceBrowser', 'NSOperationQueue', 'MKPolylineRenderer', 'HKWorkout', 'NSValueTransformer', 'UICollectionViewFlowLayout', 'MPChangePlaybackRateCommandEvent', 'NSEntityMapping', 'SKTexture', 'NSMergePolicy', 'UITextInputStringTokenizer', 'NSRecursiveLock', 'AVAsset', 'NSUndoManager', 'AVAudioUnitSampler', 'NSItemProvider', 'SKUniform', 'MPMediaPickerController', 'CKOperation', 'MTLRenderPipelineDescriptor', 'EAWiFiUnconfiguredAccessory', 'NSFileCoordinator', 'SKRequest', 'NSFileHandle', 'NSConditionLock', 'UISegmentedControl', 'NSManagedObjectModel', 'UITabBarItem', 'SCNCone', 'MPMediaItem', 'SCNMaterial', 'EKRecurrenceRule', 'UIEvent', 'UITouch', 'UIPrintInteractionController', 'CMDeviceMotion', 'NEVPNProtocol', 'NSCompoundPredicate', 'HKHealthStore', 'MKMultiPoint', 'HKSampleType', 'UIPrintFormatter', 'AVAudioUnitEQFilterParameters', 'SKView', 'NSConstantString', 'UIPopoverController', 'CKDatabase', 'AVMetadataFaceObject', 'UIAccelerometer', 'EKEventViewController', 'CMAltitudeData', 'MTLStencilDescriptor', 'UISwipeGestureRecognizer', 'NSPort', 'MKCircleRenderer', 'AVCompositionTrack', 'NSAsynchronousFetchRequest', 'NSUbiquitousKeyValueStore', 'NSMetadataQueryResultGroup', 'AVAssetResourceLoadingDataRequest', 'UITableViewHeaderFooterView', 'CKNotificationID', 'AVAudioSession', 'HKUnit', 'NSNull', 'NSPersistentStoreResult', 'MKCircleView', 'AVAudioChannelLayout', 'NEVPNProtocolIKEv2', 'WKProcessPool', 'UIAttachmentBehavior', 'CLBeacon', 'NSInputStream', 'NSURLCache', 'GKPlayer', 'NSMappingModel', 'CIQRCodeFeature', 'AVMutableVideoComposition', 'PHFetchResult', 'NSAttributeDescription', 'AVPlayer', 'MKAnnotationView', 'PKPaymentRequest', 'NSTimer', 'CBDescriptor', 'MKOverlayView', 'AVAudioUnitTimePitch', 'NSSaveChangesRequest', 'UIReferenceLibraryViewController', 'SKPhysicsJointFixed', 'UILocalizedIndexedCollation', 'UIInterpolatingMotionEffect', 'UIDocumentPickerViewController', 'AVAssetWriter', 'NSBundle', 'SKStoreProductViewController', 'GLKViewController', 'NSMetadataQueryAttributeValueTuple', 'GKTurnBasedMatch', 'AVAudioFile', 'UIActivity', 'NSPipe', 'MKShape', 'NSMergeConflict', 'CIImage', 'HKObject', 'UIRotationGestureRecognizer', 'AVPlayerItemLegibleOutput', 'AVAssetImageGenerator', 'GCControllerButtonInput', 'CKMarkNotificationsReadOperation', 'CKSubscription', 'MPTimedMetadata', 'NKIssue', 'UIScreenMode', 'HMAccessoryBrowser', 'GKTurnBasedEventHandler', 'UIWebView', 'MKPolyline', 'JSVirtualMachine', 'AVAssetReader', 'NSAttributedString', 'GKMatchmakerViewController', 'NSCountedSet', 'UIButton', 'WKNavigationResponse', 'GKLocalPlayer', 'MPMovieErrorLog', 'AVSpeechUtterance', 'HKStatistics', 'UILocalNotification', 'HKBiologicalSexObject', 'AVURLAsset', 'CBPeripheral', 'NSDateComponentsFormatter', 'SKSpriteNode', 'UIAccessibilityElement', 'AVAssetWriterInputGroup', 'HMZone', 'AVAssetReaderAudioMixOutput', 'NSEnumerator', 'UIDocument', 'MKLocalSearchResponse', 'UISimpleTextPrintFormatter', 'PHPhotoLibrary', 'CBService', 'UIDocumentMenuViewController', 'MCSession', 'QLPreviewController', 'CAMediaTimingFunction', 'UITextPosition', 'ASIdentifierManager', 'AVAssetResourceLoadingRequest', 'SLComposeServiceViewController', 'UIPinchGestureRecognizer', 'PHObject', 'NSExtensionItem', 'HKSampleQuery', 'MTLRenderPipelineColorAttachmentDescriptorArray', 'MKRouteStep', 'SCNCapsule', 'NSMetadataQuery', 'AVAssetResourceLoadingContentInformationRequest', 'UITraitCollection', 'CTCarrier', 'NSFileSecurity', 'UIAcceleration', 'UIMotionEffect', 'MTLRenderPipelineReflection', 'CLHeading', 'CLVisit', 'MKDirectionsResponse', 'HMAccessory', 'MTLStructType', 'UITextView', 'CMMagnetometerData', 'UICollisionBehavior', 'UIProgressView', 'CKServerChangeToken', 'UISearchBar', 'MKPlacemark', 'AVCaptureConnection', 'NSPropertyMapping', 'ALAssetsFilter', 'SK3DNode', 'AVPlayerItemErrorLogEvent', 'NSJSONSerialization', 'AVAssetReaderVideoCompositionOutput', 'ABPersonViewController', 'CIDetector', 'GKTurnBasedMatchmakerViewController', 'MPMediaItemCollection', 'SCNSphere', 'NSCondition', 'NSURLCredential', 'MIDINetworkConnection', 'NSFileProviderExtension', 'NSDecimalNumberHandler', 'NSAtomicStoreCacheNode', 'NSAtomicStore', 'EKAlarm', 'CKNotificationInfo', 'AVAudioUnitEQ', 'UIPercentDrivenInteractiveTransition', 'MKPolygon', 'AVAssetTrackSegment', 'MTLVertexAttribute', 'NSExpressionDescription', 'HKStatisticsCollectionQuery', 'NSURLAuthenticationChallenge', 'NSDirectoryEnumerator', 'MKDistanceFormatter', 'UIAlertAction', 'NSPropertyListSerialization', 'GKPeerPickerController', 'UIUserNotificationSettings', 'UITableViewController', 'GKNotificationBanner', 'MKPointAnnotation', 'MTLRenderPassColorAttachmentDescriptorArray', 'NSCache', 'SKPhysicsJoint', 'NSXMLParser', 'UIViewController', 'PKPaymentToken', 'MFMessageComposeViewController', 'AVAudioInputNode', 'NSDataDetector', 'CABTMIDICentralViewController', 'AVAudioUnitMIDIInstrument', 'AVCaptureVideoPreviewLayer', 'AVAssetWriterInputPassDescription', 'MPChangePlaybackRateCommand', 'NSURLComponents', 'CAMetalLayer', 'UISnapBehavior', 'AVMetadataMachineReadableCodeObject', 'CKDiscoverUserInfosOperation', 'NSTextAttachment', 'NSException', 'UIMenuItem', 'CMMotionActivityManager', 'SCNGeometryElement', 'NCWidgetController', 'CAEmitterLayer', 'MKUserLocation', 'UIImagePickerController', 'CIFeature', 'AVCaptureDeviceInput', 'ALAsset', 'NSURLSessionDownloadTask', 'SCNPhysicsHingeJoint', 'MPMoviePlayerViewController', 'NSMutableOrderedSet', 'SCNMaterialProperty', 'UIFont', 'AVCaptureVideoDataOutput', 'NSCachedURLResponse', 'ALAssetsLibrary', 'NSInvocation', 'UILongPressGestureRecognizer', 'NSTextStorage', 'WKWebViewConfiguration', 'CIFaceFeature', 'MKMapSnapshot', 'GLKEffectPropertyFog', 'AVComposition', 'CKDiscoverAllContactsOperation', 'AVAudioMixInputParameters', 'CAEmitterBehavior', 'PKPassLibrary', 'UIMutableUserNotificationCategory', 'NSLock', 'NEVPNProtocolIPSec', 'ADBannerView', 'UIDocumentPickerExtensionViewController', 'UIActivityIndicatorView', 'AVPlayerMediaSelectionCriteria', 'CALayer', 'UIAccessibilityCustomAction', 'UIBarButtonItem', 'AVAudioSessionRouteDescription', 'CLBeaconRegion', 'HKBloodTypeObject', 'MTLVertexBufferLayoutDescriptorArray', 'CABasicAnimation', 'AVVideoCompositionInstruction', 'AVMutableTimedMetadataGroup', 'EKRecurrenceEnd', 'NSTextContainer', 'TWTweetComposeViewController', 'PKPaymentAuthorizationViewController', 'UIScrollView', 'WKNavigationAction', 'AVPlayerItemMetadataOutput', 'EKRecurrenceDayOfWeek', 'NSNumberFormatter', 'MTLComputePipelineReflection', 'UIScreen', 'CLRegion', 'NSProcessInfo', 'GLKTextureInfo', 'SCNSkinner', 'AVCaptureMetadataOutput', 'SCNAnimationEvent', 'NSTextTab', 'JSManagedValue', 'NSDate', 'UITextChecker', 'WKBackForwardListItem', 'NSData', 'NSParagraphStyle', 'AVMutableMetadataItem', 'EKCalendar', 'HKWorkoutEvent', 'NSMutableURLRequest', 'UIVideoEditorController', 'HMTimerTrigger', 'AVAudioUnitVarispeed', 'UIDynamicAnimator', 'AVCompositionTrackSegment', 'GCGamepadSnapshot', 'MPMediaEntity', 'GLKSkyboxEffect', 'UISwitch', 'EKStructuredLocation', 'UIGestureRecognizer', 'NSProxy', 'GLKBaseEffect', 'UIPushBehavior', 'GKScoreChallenge', 'NSCoder', 'MPMediaPlaylist', 'NSDateComponents', 'WKUserScript', 'EKEvent', 'NSDateFormatter', 'NSAsynchronousFetchResult', 'AVAssetWriterInputPixelBufferAdaptor', 'UIVisualEffect', 'UICollectionViewCell', 'UITextField', 'CLPlacemark', 'MPPlayableContentManager', 'AVCaptureOutput', 'HMCharacteristicWriteAction', 'CKModifySubscriptionsOperation', 'NSPropertyDescription', 'GCGamepad', 'UIMarkupTextPrintFormatter', 'SCNTube', 'NSPersistentStoreCoordinator', 'AVAudioEnvironmentNode', 'GKMatchmaker', 'CIContext', 'NSThread', 'SLComposeSheetConfigurationItem', 'SKPhysicsJointSliding', 'NSPredicate', 'GKVoiceChat', 'SKCropNode', 'AVCaptureAudioPreviewOutput', 'NSStringDrawingContext', 'GKGameCenterViewController', 'UIPrintPaper', 'SCNPhysicsBallSocketJoint', 'UICollectionViewLayoutInvalidationContext', 'GLKEffectPropertyTransform', 'AVAudioIONode', 'UIDatePicker', 'MKDirections', 'ALAssetsGroup', 'CKRecordZoneNotification', 'SCNScene', 'MPMovieAccessLogEvent', 'CKFetchSubscriptionsOperation', 'CAEmitterCell', 'AVAudioUnitTimeEffect', 'HMCharacteristicMetadata', 'MKPinAnnotationView', 'UIPickerView', 'UIImageView', 'UIUserNotificationCategory', 'SCNPhysicsVehicleWheel', 'HKCategoryType', 'MPMediaQuerySection', 'GKFriendRequestComposeViewController', 'NSError', 'MTLRenderPipelineColorAttachmentDescriptor', 'SCNPhysicsShape', 'UISearchController', 'SCNPhysicsBody', 'CTSubscriberInfo', 'AVPlayerItemAccessLog', 'MPMediaPropertyPredicate', 'CMLogItem', 'NSAutoreleasePool', 'NSSocketPort', 'AVAssetReaderTrackOutput', 'SKNode', 'UIMutableUserNotificationAction', 'SCNProgram', 'AVSpeechSynthesisVoice', 'CMAltimeter', 'AVCaptureAudioChannel', 'GKTurnBasedExchangeReply', 'AVVideoCompositionLayerInstruction', 'AVSpeechSynthesizer', 'GKChallengeEventHandler', 'AVCaptureFileOutput', 'UIControl', 'SCNPhysicsField', 'CKReference', 'LAContext', 'CKRecordID', 'ADInterstitialAd', 'AVAudioSessionDataSourceDescription', 'AVAudioBuffer', 'CIColorKernel', 'GCControllerDirectionPad', 'NSFileManager', 'AVMutableAudioMixInputParameters', 'UIScreenEdgePanGestureRecognizer', 'CAKeyframeAnimation', 'CKQueryNotification', 'PHAdjustmentData', 'EASession', 'AVAssetResourceRenewalRequest', 'UIInputView', 'NSFileWrapper', 'UIResponder', 'NSPointerFunctions', 'UIKeyCommand', 'NSHTTPCookieStorage', 'AVMediaSelectionOption', 'NSRunLoop', 'NSFileAccessIntent', 'CAAnimationGroup', 'MKCircle', 'UIAlertController', 'NSMigrationManager', 'NSDateIntervalFormatter', 'UICollectionViewUpdateItem', 'CKDatabaseOperation', 'PHImageRequestOptions', 'SKReachConstraints', 'CKRecord', 'CAInterAppAudioSwitcherView', 'WKWindowFeatures', 'GKInvite', 'NSMutableData', 'PHAssetCollectionChangeRequest', 'NSMutableParagraphStyle', 'UIDynamicBehavior', 'GLKEffectProperty', 'CKFetchRecordChangesOperation', 'SKShapeNode', 'MPMovieErrorLogEvent', 'MKPolygonView', 'MPContentItem', 'HMAction', 'NSScanner', 'GKAchievementChallenge', 'AVAudioPlayer', 'CKContainer', 'AVVideoComposition', 'NKLibrary', 'NSPersistentStore', 'AVCaptureMovieFileOutput', 'HMRoom', 'GKChallenge', 'UITextRange', 'NSURLProtectionSpace', 'ACAccountStore', 'MPSkipIntervalCommand', 'NSComparisonPredicate', 'HMHome', 'PHVideoRequestOptions', 'NSOutputStream', 'MPSkipIntervalCommandEvent', 'PKAddPassesViewController', 'UITextSelectionRect', 'CTTelephonyNetworkInfo', 'AVTextStyleRule', 'NSFetchedPropertyDescription', 'UIPageViewController', 'CATransformLayer', 'UICollectionViewController', 'AVAudioNode', 'MCNearbyServiceAdvertiser', 'NSObject', 'PHAsset', 'GKLeaderboardViewController', 'CKQueryCursor', 'MPMusicPlayerController', 'MKOverlayPathRenderer', 'CMPedometerData', 'HMService', 'SKFieldNode', 'GKAchievement', 'WKUserContentController', 'AVAssetTrack', 'TWRequest', 'SKLabelNode', 'AVCaptureBracketedStillImageSettings', 'MIDINetworkHost', 'MPMediaPredicate', 'AVFrameRateRange', 'MTLTextureDescriptor', 'MTLVertexBufferLayoutDescriptor', 'MPFeedbackCommandEvent', 'UIUserNotificationAction', 'HKStatisticsQuery', 'SCNParticleSystem', 'NSIndexPath', 'AVVideoCompositionRenderContext', 'CADisplayLink', 'HKObserverQuery', 'UIPopoverPresentationController', 'CKQueryOperation', 'CAEAGLLayer', 'NSMutableString', 'NSMessagePort', 'NSURLQueryItem', 'MTLStructMember', 'AVAudioSessionChannelDescription', 'GLKView', 'UIActivityViewController', 'GKAchievementViewController', 'GKTurnBasedParticipant', 'NSURLProtocol', 'NSUserDefaults', 'NSCalendar', 'SKKeyframeSequence', 'AVMetadataItemFilter', 'CKModifyRecordZonesOperation', 'WKPreferences', 'NSMethodSignature', 'NSRegularExpression', 'EAGLSharegroup', 'AVPlayerItemVideoOutput', 'PHContentEditingInputRequestOptions', 'GKMatch', 'CIColor', 'UIDictationPhrase'])
+COCOA_PROTOCOLS = set(['SKStoreProductViewControllerDelegate', 'AVVideoCompositionInstruction', 'AVAudioSessionDelegate', 'GKMatchDelegate', 'NSFileManagerDelegate', 'UILayoutSupport', 'NSCopying', 'UIPrintInteractionControllerDelegate', 'QLPreviewControllerDataSource', 'SKProductsRequestDelegate', 'NSTextStorageDelegate', 'MCBrowserViewControllerDelegate', 'MTLComputeCommandEncoder', 'SCNSceneExportDelegate', 'UISearchResultsUpdating', 'MFMailComposeViewControllerDelegate', 'MTLBlitCommandEncoder', 'NSDecimalNumberBehaviors', 'PHContentEditingController', 'NSMutableCopying', 'UIActionSheetDelegate', 'UIViewControllerTransitioningDelegate', 'UIAlertViewDelegate', 'AVAudioPlayerDelegate', 'MKReverseGeocoderDelegate', 'NSCoding', 'UITextInputTokenizer', 'GKFriendRequestComposeViewControllerDelegate', 'UIActivityItemSource', 'NSCacheDelegate', 'UIAdaptivePresentationControllerDelegate', 'GKAchievementViewControllerDelegate', 'UIViewControllerTransitionCoordinator', 'EKEventEditViewDelegate', 'NSURLConnectionDelegate', 'UITableViewDelegate', 'GKPeerPickerControllerDelegate', 'UIGuidedAccessRestrictionDelegate', 'AVSpeechSynthesizerDelegate', 'AVAudio3DMixing', 'AVPlayerItemLegibleOutputPushDelegate', 'ADInterstitialAdDelegate', 'HMAccessoryBrowserDelegate', 'AVAssetResourceLoaderDelegate', 'UITabBarControllerDelegate', 'CKRecordValue', 'SKPaymentTransactionObserver', 'AVCaptureAudioDataOutputSampleBufferDelegate', 'UIInputViewAudioFeedback', 'GKChallengeListener', 'SKSceneDelegate', 'UIPickerViewDelegate', 'UIWebViewDelegate', 'UIApplicationDelegate', 'GKInviteEventListener', 'MPMediaPlayback', 'MyClassJavaScriptMethods', 'AVAsynchronousKeyValueLoading', 'QLPreviewItem', 'SCNBoundingVolume', 'NSPortDelegate', 'UIContentContainer', 'SCNNodeRendererDelegate', 'SKRequestDelegate', 'SKPhysicsContactDelegate', 'HMAccessoryDelegate', 'UIPageViewControllerDataSource', 'SCNSceneRendererDelegate', 'SCNPhysicsContactDelegate', 'MKMapViewDelegate', 'AVPlayerItemOutputPushDelegate', 'UICollectionViewDelegate', 'UIImagePickerControllerDelegate', 'MTLRenderCommandEncoder', 'PKPaymentAuthorizationViewControllerDelegate', 'UIToolbarDelegate', 'WKUIDelegate', 'SCNActionable', 'NSURLConnectionDataDelegate', 'MKOverlay', 'CBCentralManagerDelegate', 'JSExport', 'NSTextLayoutOrientationProvider', 'UIPickerViewDataSource', 'PKPushRegistryDelegate', 'UIViewControllerTransitionCoordinatorContext', 'NSLayoutManagerDelegate', 'MTLLibrary', 'NSFetchedResultsControllerDelegate', 'ABPeoplePickerNavigationControllerDelegate', 'MTLResource', 'NSDiscardableContent', 'UITextFieldDelegate', 'MTLBuffer', 'MTLSamplerState', 'GKGameCenterControllerDelegate', 'MPMediaPickerControllerDelegate', 'UISplitViewControllerDelegate', 'UIAppearance', 'UIPickerViewAccessibilityDelegate', 'UITraitEnvironment', 'UIScrollViewAccessibilityDelegate', 'ADBannerViewDelegate', 'MPPlayableContentDataSource', 'MTLComputePipelineState', 'NSURLSessionDelegate', 'MTLCommandBuffer', 'NSXMLParserDelegate', 'UIViewControllerRestoration', 'UISearchBarDelegate', 'UIBarPositioning', 'CBPeripheralDelegate', 'UISearchDisplayDelegate', 'CAAction', 'PKAddPassesViewControllerDelegate', 'MCNearbyServiceAdvertiserDelegate', 'MTLDepthStencilState', 'GKTurnBasedMatchmakerViewControllerDelegate', 'MPPlayableContentDelegate', 'AVCaptureVideoDataOutputSampleBufferDelegate', 'UIAppearanceContainer', 'UIStateRestoring', 'UITextDocumentProxy', 'MTLDrawable', 'NSURLSessionTaskDelegate', 'NSFilePresenter', 'AVAudioStereoMixing', 'UIViewControllerContextTransitioning', 'UITextInput', 'CBPeripheralManagerDelegate', 'UITextInputDelegate', 'NSFastEnumeration', 'NSURLAuthenticationChallengeSender', 'SCNProgramDelegate', 'AVVideoCompositing', 'SCNAnimatable', 'NSSecureCoding', 'MCAdvertiserAssistantDelegate', 'GKLocalPlayerListener', 'GLKNamedEffect', 'UIPopoverControllerDelegate', 'AVCaptureMetadataOutputObjectsDelegate', 'NSExtensionRequestHandling', 'UITextSelecting', 'UIPrinterPickerControllerDelegate', 'NCWidgetProviding', 'MTLCommandEncoder', 'NSURLProtocolClient', 'MFMessageComposeViewControllerDelegate', 'UIVideoEditorControllerDelegate', 'WKNavigationDelegate', 'GKSavedGameListener', 'UITableViewDataSource', 'MTLFunction', 'EKCalendarChooserDelegate', 'NSUserActivityDelegate', 'UICollisionBehaviorDelegate', 'NSStreamDelegate', 'MCNearbyServiceBrowserDelegate', 'HMHomeDelegate', 'UINavigationControllerDelegate', 'MCSessionDelegate', 'UIDocumentPickerDelegate', 'UIViewControllerInteractiveTransitioning', 'GKTurnBasedEventListener', 'SCNSceneRenderer', 'MTLTexture', 'GLKViewDelegate', 'EAAccessoryDelegate', 'WKScriptMessageHandler', 'PHPhotoLibraryChangeObserver', 'NSKeyedUnarchiverDelegate', 'AVPlayerItemMetadataOutputPushDelegate', 'NSMachPortDelegate', 'SCNShadable', 'UIPopoverBackgroundViewMethods', 'UIDocumentMenuDelegate', 'UIBarPositioningDelegate', 'ABPersonViewControllerDelegate', 'NSNetServiceBrowserDelegate', 'EKEventViewDelegate', 'UIScrollViewDelegate', 'NSURLConnectionDownloadDelegate', 'UIGestureRecognizerDelegate', 'UINavigationBarDelegate', 'AVAudioMixing', 'NSFetchedResultsSectionInfo', 'UIDocumentInteractionControllerDelegate', 'MTLParallelRenderCommandEncoder', 'QLPreviewControllerDelegate', 'UIAccessibilityReadingContent', 'ABUnknownPersonViewControllerDelegate', 'GLKViewControllerDelegate', 'UICollectionViewDelegateFlowLayout', 'UIPopoverPresentationControllerDelegate', 'UIDynamicAnimatorDelegate', 'NSTextAttachmentContainer', 'MKAnnotation', 'UIAccessibilityIdentification', 'UICoordinateSpace', 'ABNewPersonViewControllerDelegate', 'MTLDevice', 'CAMediaTiming', 'AVCaptureFileOutputRecordingDelegate', 'HMHomeManagerDelegate', 'UITextViewDelegate', 'UITabBarDelegate', 'GKLeaderboardViewControllerDelegate', 'UISearchControllerDelegate', 'EAWiFiUnconfiguredAccessoryBrowserDelegate', 'UITextInputTraits', 'MTLRenderPipelineState', 'GKVoiceChatClient', 'UIKeyInput', 'UICollectionViewDataSource', 'SCNTechniqueSupport', 'NSLocking', 'AVCaptureFileOutputDelegate', 'GKChallengeEventHandlerDelegate', 'UIObjectRestoration', 'CIFilterConstructor', 'AVPlayerItemOutputPullDelegate', 'EAGLDrawable', 'AVVideoCompositionValidationHandling', 'UIViewControllerAnimatedTransitioning', 'NSURLSessionDownloadDelegate', 'UIAccelerometerDelegate', 'UIPageViewControllerDelegate', 'MTLCommandQueue', 'UIDataSourceModelAssociation', 'AVAudioRecorderDelegate', 'GKSessionDelegate', 'NSKeyedArchiverDelegate', 'CAMetalDrawable', 'UIDynamicItem', 'CLLocationManagerDelegate', 'NSMetadataQueryDelegate', 'NSNetServiceDelegate', 'GKMatchmakerViewControllerDelegate', 'NSURLSessionDataDelegate'])
+COCOA_PRIMITIVES = set(['ROTAHeader', '__CFBundle', 'MortSubtable', 'AudioFilePacketTableInfo', 'CGPDFOperatorTable', 'KerxStateEntry', 'ExtendedTempoEvent', 'CTParagraphStyleSetting', 'OpaqueMIDIPort', '_GLKMatrix3', '_GLKMatrix2', '_GLKMatrix4', 'ExtendedControlEvent', 'CAFAudioDescription', 'OpaqueCMBlockBuffer', 'CGTextDrawingMode', 'EKErrorCode', 'gss_buffer_desc_struct', 'AudioUnitParameterInfo', '__SCPreferences', '__CTFrame', '__CTLine', 'AudioFile_SMPTE_Time', 'gss_krb5_lucid_context_v1', 'OpaqueJSValue', 'TrakTableEntry', 'AudioFramePacketTranslation', 'CGImageSource', 'OpaqueJSPropertyNameAccumulator', 'JustPCGlyphRepeatAddAction', '__CFBinaryHeap', 'OpaqueMIDIThruConnection', 'opaqueCMBufferQueue', 'OpaqueMusicSequence', 'MortRearrangementSubtable', 'MixerDistanceParams', 'MorxSubtable', 'MIDIObjectPropertyChangeNotification', 'SFNTLookupSegment', 'CGImageMetadataErrors', 'CGPath', 'OpaqueMIDIEndpoint', 'AudioComponentPlugInInterface', 'gss_ctx_id_t_desc_struct', 'sfntFontFeatureSetting', 'OpaqueJSContextGroup', '__SCNetworkConnection', 'AudioUnitParameterValueTranslation', 'CGImageMetadataType', 'CGPattern', 'AudioFileTypeAndFormatID', 'CGContext', 'AUNodeInteraction', 'SFNTLookupTable', 'JustPCDecompositionAction', 'KerxControlPointHeader', 'AudioStreamPacketDescription', 'KernSubtableHeader', '__SecCertificate', 'AUMIDIOutputCallbackStruct', 'MIDIMetaEvent', 'AudioQueueChannelAssignment', 'AnchorPoint', 'JustTable', '__CFNetService', 'CF_BRIDGED_TYPE', 'gss_krb5_lucid_key', 'CGPDFDictionary', 'KerxSubtableHeader', 'CAF_UUID_ChunkHeader', 'gss_krb5_cfx_keydata', 'OpaqueJSClass', 'CGGradient', 'OpaqueMIDISetup', 'JustPostcompTable', '__CTParagraphStyle', 'AudioUnitParameterHistoryInfo', 'OpaqueJSContext', 'CGShading', 'MIDIThruConnectionParams', 'BslnFormat0Part', 'SFNTLookupSingle', '__CFHost', '__SecRandom', '__CTFontDescriptor', '_NSRange', 'sfntDirectory', 'AudioQueueLevelMeterState', 'CAFPositionPeak', 'PropLookupSegment', '__CVOpenGLESTextureCache', 'sfntInstance', '_GLKQuaternion', 'AnkrTable', '__SCNetworkProtocol', 'CAFFileHeader', 'KerxOrderedListHeader', 'CGBlendMode', 'STXEntryOne', 'CAFRegion', 'SFNTLookupTrimmedArrayHeader', 'SCNMatrix4', 'KerxControlPointEntry', 'OpaqueMusicTrack', '_GLKVector4', 'gss_OID_set_desc_struct', 'OpaqueMusicPlayer', '_CFHTTPAuthentication', 'CGAffineTransform', 'CAFMarkerChunk', 'AUHostIdentifier', 'ROTAGlyphEntry', 'BslnTable', 'gss_krb5_lucid_context_version', '_GLKMatrixStack', 'CGImage', 'KernStateEntry', 'SFNTLookupSingleHeader', 'MortLigatureSubtable', 'CAFUMIDChunk', 'SMPTETime', 'CAFDataChunk', 'CGPDFStream', 'AudioFileRegionList', 'STEntryTwo', 'SFNTLookupBinarySearchHeader', 'OpbdTable', '__CTGlyphInfo', 'BslnFormat2Part', 'KerxIndexArrayHeader', 'TrakTable', 'KerxKerningPair', '__CFBitVector', 'KernVersion0SubtableHeader', 'OpaqueAudioComponentInstance', 'AudioChannelLayout', '__CFUUID', 'MIDISysexSendRequest', '__CFNumberFormatter', 'CGImageSourceStatus', 'AudioFileMarkerList', 'AUSamplerBankPresetData', 'CGDataProvider', 'AudioFormatInfo', '__SecIdentity', 'sfntCMapExtendedSubHeader', 'MIDIChannelMessage', 'KernOffsetTable', 'CGColorSpaceModel', 'MFMailComposeErrorCode', 'CGFunction', '__SecTrust', 'AVAudio3DAngularOrientation', 'CGFontPostScriptFormat', 'KernStateHeader', 'AudioUnitCocoaViewInfo', 'CGDataConsumer', 'OpaqueMIDIDevice', 'KernVersion0Header', 'AnchorPointTable', 'CGImageDestination', 'CAFInstrumentChunk', 'AudioUnitMeterClipping', 'MorxChain', '__CTFontCollection', 'STEntryOne', 'STXEntryTwo', 'ExtendedNoteOnEvent', 'CGColorRenderingIntent', 'KerxSimpleArrayHeader', 'MorxTable', '_GLKVector3', '_GLKVector2', 'MortTable', 'CGPDFBox', 'AudioUnitParameterValueFromString', '__CFSocket', 'ALCdevice_struct', 'MIDINoteMessage', 'sfntFeatureHeader', 'CGRect', '__SCNetworkInterface', '__CFTree', 'MusicEventUserData', 'TrakTableData', 'GCQuaternion', 'MortContextualSubtable', '__CTRun', 'AudioUnitFrequencyResponseBin', 'MortChain', 'MorxInsertionSubtable', 'CGImageMetadata', 'gss_auth_identity', 'AudioUnitMIDIControlMapping', 'CAFChunkHeader', 'CGImagePropertyOrientation', 'CGPDFScanner', 'OpaqueMusicEventIterator', 'sfntDescriptorHeader', 'AudioUnitNodeConnection', 'OpaqueMIDIDeviceList', 'ExtendedAudioFormatInfo', 'BslnFormat1Part', 'sfntFontDescriptor', 'KernSimpleArrayHeader', '__CFRunLoopObserver', 'CGPatternTiling', 'MIDINotification', 'MorxLigatureSubtable', 'MessageComposeResult', 'MIDIThruConnectionEndpoint', 'MusicDeviceStdNoteParams', 'opaqueCMSimpleQueue', 'ALCcontext_struct', 'OpaqueAudioQueue', 'PropLookupSingle', 'CGInterpolationQuality', 'CGColor', 'AudioOutputUnitStartAtTimeParams', 'gss_name_t_desc_struct', 'CGFunctionCallbacks', 'CAFPacketTableHeader', 'AudioChannelDescription', 'sfntFeatureName', 'MorxContextualSubtable', 'CVSMPTETime', 'AudioValueRange', 'CGTextEncoding', 'AudioStreamBasicDescription', 'AUNodeRenderCallback', 'AudioPanningInfo', 'KerxOrderedListEntry', '__CFAllocator', 'OpaqueJSPropertyNameArray', '__SCDynamicStore', 'OpaqueMIDIEntity', '__CTRubyAnnotation', 'SCNVector4', 'CFHostClientContext', 'CFNetServiceClientContext', 'AudioUnitPresetMAS_SettingData', 'opaqueCMBufferQueueTriggerToken', 'AudioUnitProperty', 'CAFRegionChunk', 'CGPDFString', '__GLsync', '__CFStringTokenizer', 'JustWidthDeltaEntry', 'sfntVariationAxis', '__CFNetDiagnostic', 'CAFOverviewSample', 'sfntCMapEncoding', 'CGVector', '__SCNetworkService', 'opaqueCMSampleBuffer', 'AUHostVersionIdentifier', 'AudioBalanceFade', 'sfntFontRunFeature', 'KerxCoordinateAction', 'sfntCMapSubHeader', 'CVPlanarPixelBufferInfo', 'AUNumVersion', 'AUSamplerInstrumentData', 'AUPreset', '__CTRunDelegate', 'OpaqueAudioQueueProcessingTap', 'KerxTableHeader', '_NSZone', 'OpaqueExtAudioFile', '__CFRunLoopSource', '__CVMetalTextureCache', 'KerxAnchorPointAction', 'OpaqueJSString', 'AudioQueueParameterEvent', '__CFHTTPMessage', 'OpaqueCMClock', 'ScheduledAudioFileRegion', 'STEntryZero', 'AVAudio3DPoint', 'gss_channel_bindings_struct', 'sfntVariationHeader', 'AUChannelInfo', 'UIOffset', 'GLKEffectPropertyPrv', 'KerxStateHeader', 'CGLineJoin', 'CGPDFDocument', '__CFBag', 'KernOrderedListHeader', '__SCNetworkSet', '__SecKey', 'MIDIObjectAddRemoveNotification', 'AudioUnitParameter', 'JustPCActionSubrecord', 'AudioComponentDescription', 'AudioUnitParameterValueName', 'AudioUnitParameterEvent', 'KerxControlPointAction', 'AudioTimeStamp', 'KernKerningPair', 'gss_buffer_set_desc_struct', 'MortFeatureEntry', 'FontVariation', 'CAFStringID', 'LcarCaretClassEntry', 'AudioUnitParameterStringFromValue', 'ACErrorCode', 'ALMXGlyphEntry', 'LtagTable', '__CTTypesetter', 'AuthorizationOpaqueRef', 'UIEdgeInsets', 'CGPathElement', 'CAFMarker', 'KernTableHeader', 'NoteParamsControlValue', 'SSLContext', 'gss_cred_id_t_desc_struct', 'AudioUnitParameterNameInfo', 'CGDataConsumerCallbacks', 'ALMXHeader', 'CGLineCap', 'MIDIControlTransform', 'CGPDFArray', '__SecPolicy', 'AudioConverterPrimeInfo', '__CTTextTab', '__CFNetServiceMonitor', 'AUInputSamplesInOutputCallbackStruct', '__CTFramesetter', 'CGPDFDataFormat', 'STHeader', 'CVPlanarPixelBufferInfo_YCbCrPlanar', 'MIDIValueMap', 'JustDirectionTable', '__SCBondStatus', 'SFNTLookupSegmentHeader', 'OpaqueCMMemoryPool', 'CGPathDrawingMode', 'CGFont', '__SCNetworkReachability', 'AudioClassDescription', 'CGPoint', 'AVAudio3DVectorOrientation', 'CAFStrings', '__CFNetServiceBrowser', 'opaqueMTAudioProcessingTap', 'sfntNameRecord', 'CGPDFPage', 'CGLayer', 'ComponentInstanceRecord', 'CAFInfoStrings', 'HostCallbackInfo', 'MusicDeviceNoteParams', 'OpaqueVTCompressionSession', 'KernIndexArrayHeader', 'CVPlanarPixelBufferInfo_YCbCrBiPlanar', 'MusicTrackLoopInfo', 'opaqueCMFormatDescription', 'STClassTable', 'sfntDirectoryEntry', 'OpaqueCMTimebase', 'CGDataProviderDirectCallbacks', 'MIDIPacketList', 'CAFOverviewChunk', 'MIDIPacket', 'ScheduledAudioSlice', 'CGDataProviderSequentialCallbacks', 'AudioBuffer', 'MorxRearrangementSubtable', 'CGPatternCallbacks', 'AUDistanceAttenuationData', 'MIDIIOErrorNotification', 'CGPDFContentStream', 'IUnknownVTbl', 'MIDITransform', 'MortInsertionSubtable', 'CABarBeatTime', 'AudioBufferList', '__CVBuffer', 'AURenderCallbackStruct', 'STXEntryZero', 'JustPCDuctilityAction', 'OpaqueAudioQueueTimeline', 'VTDecompressionOutputCallbackRecord', 'OpaqueMIDIClient', '__CFPlugInInstance', 'AudioQueueBuffer', '__CFFileDescriptor', 'AudioUnitConnection', '_GKTurnBasedExchangeStatus', 'LcarCaretTable', 'CVPlanarComponentInfo', 'JustWidthDeltaGroup', 'OpaqueAudioComponent', 'ParameterEvent', '__CVPixelBufferPool', '__CTFont', 'CGColorSpace', 'CGSize', 'AUDependentParameter', 'MIDIDriverInterface', 'gss_krb5_rfc1964_keydata', '__CFDateFormatter', 'LtagStringRange', 'OpaqueVTDecompressionSession', 'gss_iov_buffer_desc_struct', 'AUPresetEvent', 'PropTable', 'KernOrderedListEntry', 'CF_BRIDGED_MUTABLE_TYPE', 'gss_OID_desc_struct', 'AudioUnitPresetMAS_Settings', 'AudioFileMarker', 'JustPCConditionalAddAction', 'BslnFormat3Part', '__CFNotificationCenter', 'MortSwashSubtable', 'AUParameterMIDIMapping', 'SCNVector3', 'OpaqueAudioConverter', 'MIDIRawData', 'sfntNameHeader', '__CFRunLoop', 'MFMailComposeResult', 'CATransform3D', 'OpbdSideValues', 'CAF_SMPTE_Time', '__SecAccessControl', 'JustPCAction', 'OpaqueVTFrameSilo', 'OpaqueVTMultiPassStorage', 'CGPathElementType', 'AudioFormatListItem', 'AudioUnitExternalBuffer', 'AudioFileRegion', 'AudioValueTranslation', 'CGImageMetadataTag', 'CAFPeakChunk', 'AudioBytePacketTranslation', 'sfntCMapHeader', '__CFURLEnumerator', 'STXHeader', 'CGPDFObjectType', 'SFNTLookupArrayHeader'])
+
+if __name__ == '__main__': # pragma: no cover
+ import os
+ import re
+
+ FRAMEWORKS_PATH = '/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS8.1.sdk/System/Library/Frameworks/'
+ frameworks = os.listdir(FRAMEWORKS_PATH)
+
+ all_interfaces = set()
+ all_protocols = set()
+ all_primitives = set()
+ for framework in frameworks:
+ frameworkHeadersDir = FRAMEWORKS_PATH + framework + '/Headers/'
+ if not os.path.exists(frameworkHeadersDir):
+ continue
+
+ headerFilenames = os.listdir(frameworkHeadersDir)
+
+ for f in headerFilenames:
+ if not f.endswith('.h'):
+ continue
+
+ headerFilePath = frameworkHeadersDir + f
+ content = open(headerFilePath).read()
+ res = re.findall('(?<=@interface )\w+', content)
+ for r in res:
+ all_interfaces.add(r)
+
+ res = re.findall('(?<=@protocol )\w+', content)
+ for r in res:
+ all_protocols.add(r)
+
+ res = re.findall('(?<=typedef enum )\w+', content)
+ for r in res:
+ all_primitives.add(r)
+
+ res = re.findall('(?<=typedef struct )\w+', content)
+ for r in res:
+ all_primitives.add(r)
+
+ res = re.findall('(?<=typedef const struct )\w+', content)
+ for r in res:
+ all_primitives.add(r)
+
+
+ print("ALL interfaces: \n")
+ print(all_interfaces)
+
+ print("\nALL protocols: \n")
+ print(all_protocols)
+
+ print("\nALL primitives: \n")
+ print(all_primitives)
diff --git a/wandb/vendor/pygments/lexers/_csound_builtins.py b/wandb/vendor/pygments/lexers/_csound_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..e5a9aaf758621ee11c092d427f546b401f77383c
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_csound_builtins.py
@@ -0,0 +1,1346 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._csound_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+# Opcodes in Csound 6.05 from
+# csound --list-opcodes
+# except
+# cggoto <http://www.csounds.com/manual/html/cggoto.html>
+# cigoto <http://www.csounds.com/manual/html/cigoto.html>
+# cingoto (undocumented)
+# ckgoto <http://www.csounds.com/manual/html/ckgoto.html>
+# cngoto <http://www.csounds.com/manual/html/cngoto.html>
+# endin <http://www.csounds.com/manual/html/endin.html
+# endop <http://www.csounds.com/manual/html/endop.html
+# goto <http://www.csounds.com/manual/html/goto.html>
+# igoto <http://www.csounds.com/manual/html/igoto.html>
+# instr <http://www.csounds.com/manual/html/instr.html>
+# kgoto <http://www.csounds.com/manual/html/kgoto.html>
+# loop_ge <http://www.csounds.com/manual/html/loop_ge.html>
+# loop_gt <http://www.csounds.com/manual/html/loop_gt.html>
+# loop_le <http://www.csounds.com/manual/html/loop_le.html>
+# loop_lt <http://www.csounds.com/manual/html/loop_lt.html>
+# opcode <http://www.csounds.com/manual/html/opcode.html>
+# return <http://www.csounds.com/manual/html/return.html>
+# rigoto <http://www.csounds.com/manual/html/rigoto.html>
+# tigoto <http://www.csounds.com/manual/html/tigoto.html>
+# timout <http://www.csounds.com/manual/html/timout.html>
+# which are treated as keywords; the scoreline opcodes
+# scoreline <http://www.csounds.com/manual/html/scoreline.html>
+# scoreline_i <http://www.csounds.com/manual/html/scoreline_i.html>
+# which allow Csound Score highlighting; the pyrun opcodes
+# <http://www.csounds.com/manual/html/pyrun.html>
+# pylrun
+# pylruni
+# pylrunt
+# pyrun
+# pyruni
+# pyrunt
+# which allow Python highlighting; and the Lua opcodes
+# lua_exec <http://www.csounds.com/manual/html/lua_exec.html>
+# lua_opdef <http://www.csounds.com/manual/html/lua_opdef.html>
+# which allow Lua highlighting.
+OPCODES = set((
+ 'ATSadd',
+ 'ATSaddnz',
+ 'ATSbufread',
+ 'ATScross',
+ 'ATSinfo',
+ 'ATSinterpread',
+ 'ATSpartialtap',
+ 'ATSread',
+ 'ATSreadnz',
+ 'ATSsinnoi',
+ 'FLbox',
+ 'FLbutBank',
+ 'FLbutton',
+ 'FLcloseButton',
+ 'FLcolor',
+ 'FLcolor2',
+ 'FLcount',
+ 'FLexecButton',
+ 'FLgetsnap',
+ 'FLgroup',
+ 'FLgroupEnd',
+ 'FLgroup_end',
+ 'FLhide',
+ 'FLhvsBox',
+ 'FLhvsBoxSetValue',
+ 'FLjoy',
+ 'FLkeyIn',
+ 'FLknob',
+ 'FLlabel',
+ 'FLloadsnap',
+ 'FLmouse',
+ 'FLpack',
+ 'FLpackEnd',
+ 'FLpack_end',
+ 'FLpanel',
+ 'FLpanelEnd',
+ 'FLpanel_end',
+ 'FLprintk',
+ 'FLprintk2',
+ 'FLroller',
+ 'FLrun',
+ 'FLsavesnap',
+ 'FLscroll',
+ 'FLscrollEnd',
+ 'FLscroll_end',
+ 'FLsetAlign',
+ 'FLsetBox',
+ 'FLsetColor',
+ 'FLsetColor2',
+ 'FLsetFont',
+ 'FLsetPosition',
+ 'FLsetSize',
+ 'FLsetSnapGroup',
+ 'FLsetText',
+ 'FLsetTextColor',
+ 'FLsetTextSize',
+ 'FLsetTextType',
+ 'FLsetVal',
+ 'FLsetVal_i',
+ 'FLsetVali',
+ 'FLsetsnap',
+ 'FLshow',
+ 'FLslidBnk',
+ 'FLslidBnk2',
+ 'FLslidBnk2Set',
+ 'FLslidBnk2Setk',
+ 'FLslidBnkGetHandle',
+ 'FLslidBnkSet',
+ 'FLslidBnkSetk',
+ 'FLslider',
+ 'FLtabs',
+ 'FLtabsEnd',
+ 'FLtabs_end',
+ 'FLtext',
+ 'FLupdate',
+ 'FLvalue',
+ 'FLvkeybd',
+ 'FLvslidBnk',
+ 'FLvslidBnk2',
+ 'FLxyin',
+ 'MixerClear',
+ 'MixerGetLevel',
+ 'MixerReceive',
+ 'MixerSend',
+ 'MixerSetLevel',
+ 'MixerSetLevel_i',
+ 'OSCinit',
+ 'OSClisten',
+ 'OSCsend',
+ 'a',
+ 'abs',
+ 'active',
+ 'adsr',
+ 'adsyn',
+ 'adsynt',
+ 'adsynt2',
+ 'aftouch',
+ 'alpass',
+ 'alwayson',
+ 'ampdb',
+ 'ampdbfs',
+ 'ampmidi',
+ 'ampmidid',
+ 'areson',
+ 'aresonk',
+ 'array',
+ 'atone',
+ 'atonek',
+ 'atonex',
+ 'babo',
+ 'balance',
+ 'bamboo',
+ 'barmodel',
+ 'bbcutm',
+ 'bbcuts',
+ 'betarand',
+ 'bexprnd',
+ 'bformdec',
+ 'bformdec1',
+ 'bformenc',
+ 'bformenc1',
+ 'binit',
+ 'biquad',
+ 'biquada',
+ 'birnd',
+ 'bqrez',
+ 'buchla',
+ 'butbp',
+ 'butbr',
+ 'buthp',
+ 'butlp',
+ 'butterbp',
+ 'butterbr',
+ 'butterhp',
+ 'butterlp',
+ 'button',
+ 'buzz',
+ 'c2r',
+ 'cabasa',
+ 'cauchy',
+ 'cauchyi',
+ 'ceil',
+ 'cell',
+ 'cent',
+ 'centroid',
+ 'ceps',
+ #'cggoto',
+ 'chanctrl',
+ 'changed',
+ 'chani',
+ 'chano',
+ 'chebyshevpoly',
+ 'checkbox',
+ 'chn_S',
+ 'chn_a',
+ 'chn_k',
+ 'chnclear',
+ 'chnexport',
+ 'chnget',
+ 'chnmix',
+ 'chnparams',
+ 'chnset',
+ 'chuap',
+ #'cigoto',
+ #'cingoto',
+ #'ckgoto',
+ 'clear',
+ 'clfilt',
+ 'clip',
+ 'clockoff',
+ 'clockon',
+ 'cmplxprod',
+ #'cngoto',
+ 'comb',
+ 'combinv',
+ 'compilecsd',
+ 'compileorc',
+ 'compilestr',
+ 'compress',
+ 'connect',
+ 'control',
+ 'convle',
+ 'convolve',
+ 'copy2ftab',
+ 'copy2ttab',
+ 'copya2ftab',
+ 'copyf2array',
+ 'cos',
+ 'cosh',
+ 'cosinv',
+ 'cosseg',
+ 'cossegb',
+ 'cossegr',
+ 'cps2pch',
+ 'cpsmidi',
+ 'cpsmidib',
+ 'cpsmidinn',
+ 'cpsoct',
+ 'cpspch',
+ 'cpstmid',
+ 'cpstun',
+ 'cpstuni',
+ 'cpsxpch',
+ 'cpuprc',
+ 'cross2',
+ 'crossfm',
+ 'crossfmi',
+ 'crossfmpm',
+ 'crossfmpmi',
+ 'crosspm',
+ 'crosspmi',
+ 'crunch',
+ 'ctlchn',
+ 'ctrl14',
+ 'ctrl21',
+ 'ctrl7',
+ 'ctrlinit',
+ 'cuserrnd',
+ 'dam',
+ 'date',
+ 'dates',
+ 'db',
+ 'dbamp',
+ 'dbfsamp',
+ 'dcblock',
+ 'dcblock2',
+ 'dconv',
+ 'delay',
+ 'delay1',
+ 'delayk',
+ 'delayr',
+ 'delayw',
+ 'deltap',
+ 'deltap3',
+ 'deltapi',
+ 'deltapn',
+ 'deltapx',
+ 'deltapxw',
+ 'denorm',
+ 'diff',
+ 'diskgrain',
+ 'diskin',
+ 'diskin2',
+ 'dispfft',
+ 'display',
+ 'distort',
+ 'distort1',
+ 'divz',
+ 'doppler',
+ 'downsamp',
+ 'dripwater',
+ 'dumpk',
+ 'dumpk2',
+ 'dumpk3',
+ 'dumpk4',
+ 'duserrnd',
+ 'dust',
+ 'dust2',
+ #'endin',
+ #'endop',
+ 'envlpx',
+ 'envlpxr',
+ 'ephasor',
+ 'eqfil',
+ 'evalstr',
+ 'event',
+ 'event_i',
+ 'exciter',
+ 'exitnow',
+ 'exp',
+ 'expcurve',
+ 'expon',
+ 'exprand',
+ 'exprandi',
+ 'expseg',
+ 'expsega',
+ 'expsegb',
+ 'expsegba',
+ 'expsegr',
+ 'fareylen',
+ 'fareyleni',
+ 'faustaudio',
+ 'faustcompile',
+ 'faustctl',
+ 'faustgen',
+ 'fft',
+ 'fftinv',
+ 'ficlose',
+ 'filebit',
+ 'filelen',
+ 'filenchnls',
+ 'filepeak',
+ 'filesr',
+ 'filevalid',
+ 'fillarray',
+ 'filter2',
+ 'fin',
+ 'fini',
+ 'fink',
+ 'fiopen',
+ 'flanger',
+ 'flashtxt',
+ 'flooper',
+ 'flooper2',
+ 'floor',
+ 'fluidAllOut',
+ 'fluidCCi',
+ 'fluidCCk',
+ 'fluidControl',
+ 'fluidEngine',
+ 'fluidLoad',
+ 'fluidNote',
+ 'fluidOut',
+ 'fluidProgramSelect',
+ 'fluidSetInterpMethod',
+ 'fmb3',
+ 'fmbell',
+ 'fmmetal',
+ 'fmpercfl',
+ 'fmrhode',
+ 'fmvoice',
+ 'fmwurlie',
+ 'fof',
+ 'fof2',
+ 'fofilter',
+ 'fog',
+ 'fold',
+ 'follow',
+ 'follow2',
+ 'foscil',
+ 'foscili',
+ 'fout',
+ 'fouti',
+ 'foutir',
+ 'foutk',
+ 'fprintks',
+ 'fprints',
+ 'frac',
+ 'fractalnoise',
+ 'freeverb',
+ 'ftchnls',
+ 'ftconv',
+ 'ftcps',
+ 'ftfree',
+ 'ftgen',
+ 'ftgenonce',
+ 'ftgentmp',
+ 'ftlen',
+ 'ftload',
+ 'ftloadk',
+ 'ftlptim',
+ 'ftmorf',
+ 'ftresize',
+ 'ftresizei',
+ 'ftsave',
+ 'ftsavek',
+ 'ftsr',
+ 'gain',
+ 'gainslider',
+ 'gauss',
+ 'gaussi',
+ 'gausstrig',
+ 'gbuzz',
+ 'genarray',
+ 'genarray_i',
+ 'gendy',
+ 'gendyc',
+ 'gendyx',
+ 'getcfg',
+ 'getcol',
+ 'getrow',
+ 'gogobel',
+ #'goto',
+ 'grain',
+ 'grain2',
+ 'grain3',
+ 'granule',
+ 'guiro',
+ 'harmon',
+ 'harmon2',
+ 'harmon3',
+ 'harmon4',
+ 'hdf5read',
+ 'hdf5write',
+ 'hilbert',
+ 'hrtfearly',
+ 'hrtfer',
+ 'hrtfmove',
+ 'hrtfmove2',
+ 'hrtfreverb',
+ 'hrtfstat',
+ 'hsboscil',
+ 'hvs1',
+ 'hvs2',
+ 'hvs3',
+ 'i',
+ 'iceps',
+ #'igoto',
+ 'ihold',
+ 'imagecreate',
+ 'imagefree',
+ 'imagegetpixel',
+ 'imageload',
+ 'imagesave',
+ 'imagesetpixel',
+ 'imagesize',
+ 'in',
+ 'in32',
+ 'inch',
+ 'inh',
+ 'init',
+ 'initc14',
+ 'initc21',
+ 'initc7',
+ 'inleta',
+ 'inletf',
+ 'inletk',
+ 'inletkid',
+ 'inletv',
+ 'ino',
+ 'inq',
+ 'inrg',
+ 'ins',
+ 'insglobal',
+ 'insremot',
+ #'instr',
+ 'int',
+ 'integ',
+ 'interp',
+ 'invalue',
+ 'inx',
+ 'inz',
+ 'jitter',
+ 'jitter2',
+ 'jspline',
+ 'k',
+ #'kgoto',
+ 'ktableseg',
+ 'lenarray',
+ 'lentab',
+ 'lfo',
+ 'limit',
+ 'line',
+ 'linen',
+ 'linenr',
+ 'lineto',
+ 'linrand',
+ 'linseg',
+ 'linsegb',
+ 'linsegr',
+ 'locsend',
+ 'locsig',
+ 'log',
+ 'log10',
+ 'log2',
+ 'logbtwo',
+ 'logcurve',
+ #'loop_ge',
+ #'loop_gt',
+ #'loop_le',
+ #'loop_lt',
+ 'loopseg',
+ 'loopsegp',
+ 'looptseg',
+ 'loopxseg',
+ 'lorenz',
+ 'loscil',
+ 'loscil3',
+ 'loscilx',
+ 'lowpass2',
+ 'lowres',
+ 'lowresx',
+ 'lpf18',
+ 'lpform',
+ 'lpfreson',
+ 'lphasor',
+ 'lpinterp',
+ 'lposcil',
+ 'lposcil3',
+ 'lposcila',
+ 'lposcilsa',
+ 'lposcilsa2',
+ 'lpread',
+ 'lpreson',
+ 'lpshold',
+ 'lpsholdp',
+ 'lpslot',
+ #'lua_exec',
+ 'lua_ikopcall',
+ #'lua_opdef',
+ 'mac',
+ 'maca',
+ 'madsr',
+ 'mags',
+ 'mandel',
+ 'mandol',
+ 'maparray',
+ 'maparray_i',
+ 'marimba',
+ 'massign',
+ 'max',
+ 'max_k',
+ 'maxabs',
+ 'maxabsaccum',
+ 'maxaccum',
+ 'maxalloc',
+ 'maxarray',
+ 'maxtab',
+ 'mclock',
+ 'mdelay',
+ 'median',
+ 'mediank',
+ 'metro',
+ 'midglobal',
+ 'midic14',
+ 'midic21',
+ 'midic7',
+ 'midichannelaftertouch',
+ 'midichn',
+ 'midicontrolchange',
+ 'midictrl',
+ 'mididefault',
+ 'midifilestatus',
+ 'midiin',
+ 'midinoteoff',
+ 'midinoteoncps',
+ 'midinoteonkey',
+ 'midinoteonoct',
+ 'midinoteonpch',
+ 'midion',
+ 'midion2',
+ 'midiout',
+ 'midipgm',
+ 'midipitchbend',
+ 'midipolyaftertouch',
+ 'midiprogramchange',
+ 'miditempo',
+ 'midremot',
+ 'min',
+ 'minabs',
+ 'minabsaccum',
+ 'minaccum',
+ 'minarray',
+ 'mincer',
+ 'mintab',
+ 'mirror',
+ 'mode',
+ 'modmatrix',
+ 'monitor',
+ 'moog',
+ 'moogladder',
+ 'moogvcf',
+ 'moogvcf2',
+ 'moscil',
+ 'mp3bitrate',
+ 'mp3in',
+ 'mp3len',
+ 'mp3nchnls',
+ 'mp3sr',
+ 'mpulse',
+ 'mrtmsg',
+ 'multitap',
+ 'mute',
+ 'mxadsr',
+ 'nestedap',
+ 'nlalp',
+ 'nlfilt',
+ 'nlfilt2',
+ 'noise',
+ 'noteoff',
+ 'noteon',
+ 'noteondur',
+ 'noteondur2',
+ 'notnum',
+ 'nreverb',
+ 'nrpn',
+ 'nsamp',
+ 'nstance',
+ 'nstrnum',
+ 'ntrpol',
+ 'octave',
+ 'octcps',
+ 'octmidi',
+ 'octmidib',
+ 'octmidinn',
+ 'octpch',
+ #'opcode',
+ 'oscbnk',
+ 'oscil',
+ 'oscil1',
+ 'oscil1i',
+ 'oscil3',
+ 'oscili',
+ 'oscilikt',
+ 'osciliktp',
+ 'oscilikts',
+ 'osciln',
+ 'oscils',
+ 'oscilx',
+ 'out',
+ 'out32',
+ 'outc',
+ 'outch',
+ 'outh',
+ 'outiat',
+ 'outic',
+ 'outic14',
+ 'outipat',
+ 'outipb',
+ 'outipc',
+ 'outkat',
+ 'outkc',
+ 'outkc14',
+ 'outkpat',
+ 'outkpb',
+ 'outkpc',
+ 'outleta',
+ 'outletf',
+ 'outletk',
+ 'outletkid',
+ 'outletv',
+ 'outo',
+ 'outq',
+ 'outq1',
+ 'outq2',
+ 'outq3',
+ 'outq4',
+ 'outrg',
+ 'outs',
+ 'outs1',
+ 'outs2',
+ 'outvalue',
+ 'outx',
+ 'outz',
+ 'p',
+ 'pan',
+ 'pan2',
+ 'pareq',
+ 'partials',
+ 'partikkel',
+ 'partikkelget',
+ 'partikkelset',
+ 'partikkelsync',
+ 'passign',
+ 'pcauchy',
+ 'pchbend',
+ 'pchmidi',
+ 'pchmidib',
+ 'pchmidinn',
+ 'pchoct',
+ 'pconvolve',
+ 'pcount',
+ 'pdclip',
+ 'pdhalf',
+ 'pdhalfy',
+ 'peak',
+ 'pgmassign',
+ 'pgmchn',
+ 'phaser1',
+ 'phaser2',
+ 'phasor',
+ 'phasorbnk',
+ 'phs',
+ 'pindex',
+ 'pinker',
+ 'pinkish',
+ 'pitch',
+ 'pitchac',
+ 'pitchamdf',
+ 'planet',
+ 'platerev',
+ 'plltrack',
+ 'pluck',
+ 'poisson',
+ 'pol2rect',
+ 'polyaft',
+ 'polynomial',
+ 'pop',
+ 'pop_f',
+ 'port',
+ 'portk',
+ 'poscil',
+ 'poscil3',
+ 'pow',
+ 'powershape',
+ 'powoftwo',
+ 'prealloc',
+ 'prepiano',
+ 'print',
+ 'print_type',
+ 'printf',
+ 'printf_i',
+ 'printk',
+ 'printk2',
+ 'printks',
+ 'printks2',
+ 'prints',
+ 'product',
+ 'pset',
+ 'ptable',
+ 'ptable3',
+ 'ptablei',
+ 'ptableiw',
+ 'ptablew',
+ 'ptrack',
+ 'push',
+ 'push_f',
+ 'puts',
+ 'pvadd',
+ 'pvbufread',
+ 'pvcross',
+ 'pvinterp',
+ 'pvoc',
+ 'pvread',
+ 'pvs2array',
+ 'pvs2tab',
+ 'pvsadsyn',
+ 'pvsanal',
+ 'pvsarp',
+ 'pvsbandp',
+ 'pvsbandr',
+ 'pvsbin',
+ 'pvsblur',
+ 'pvsbuffer',
+ 'pvsbufread',
+ 'pvsbufread2',
+ 'pvscale',
+ 'pvscent',
+ 'pvsceps',
+ 'pvscross',
+ 'pvsdemix',
+ 'pvsdiskin',
+ 'pvsdisp',
+ 'pvsenvftw',
+ 'pvsfilter',
+ 'pvsfread',
+ 'pvsfreeze',
+ 'pvsfromarray',
+ 'pvsftr',
+ 'pvsftw',
+ 'pvsfwrite',
+ 'pvsgain',
+ 'pvsgendy',
+ 'pvshift',
+ 'pvsifd',
+ 'pvsin',
+ 'pvsinfo',
+ 'pvsinit',
+ 'pvslock',
+ 'pvsmaska',
+ 'pvsmix',
+ 'pvsmooth',
+ 'pvsmorph',
+ 'pvsosc',
+ 'pvsout',
+ 'pvspitch',
+ 'pvstanal',
+ 'pvstencil',
+ 'pvsvoc',
+ 'pvswarp',
+ 'pvsynth',
+ 'pwd',
+ 'pyassign',
+ 'pyassigni',
+ 'pyassignt',
+ 'pycall',
+ 'pycall1',
+ 'pycall1i',
+ 'pycall1t',
+ 'pycall2',
+ 'pycall2i',
+ 'pycall2t',
+ 'pycall3',
+ 'pycall3i',
+ 'pycall3t',
+ 'pycall4',
+ 'pycall4i',
+ 'pycall4t',
+ 'pycall5',
+ 'pycall5i',
+ 'pycall5t',
+ 'pycall6',
+ 'pycall6i',
+ 'pycall6t',
+ 'pycall7',
+ 'pycall7i',
+ 'pycall7t',
+ 'pycall8',
+ 'pycall8i',
+ 'pycall8t',
+ 'pycalli',
+ 'pycalln',
+ 'pycallni',
+ 'pycallt',
+ 'pyeval',
+ 'pyevali',
+ 'pyevalt',
+ 'pyexec',
+ 'pyexeci',
+ 'pyexect',
+ 'pyinit',
+ 'pylassign',
+ 'pylassigni',
+ 'pylassignt',
+ 'pylcall',
+ 'pylcall1',
+ 'pylcall1i',
+ 'pylcall1t',
+ 'pylcall2',
+ 'pylcall2i',
+ 'pylcall2t',
+ 'pylcall3',
+ 'pylcall3i',
+ 'pylcall3t',
+ 'pylcall4',
+ 'pylcall4i',
+ 'pylcall4t',
+ 'pylcall5',
+ 'pylcall5i',
+ 'pylcall5t',
+ 'pylcall6',
+ 'pylcall6i',
+ 'pylcall6t',
+ 'pylcall7',
+ 'pylcall7i',
+ 'pylcall7t',
+ 'pylcall8',
+ 'pylcall8i',
+ 'pylcall8t',
+ 'pylcalli',
+ 'pylcalln',
+ 'pylcallni',
+ 'pylcallt',
+ 'pyleval',
+ 'pylevali',
+ 'pylevalt',
+ 'pylexec',
+ 'pylexeci',
+ 'pylexect',
+ #'pylrun',
+ #'pylruni',
+ #'pylrunt',
+ #'pyrun',
+ #'pyruni',
+ #'pyrunt',
+ 'qinf',
+ 'qnan',
+ 'r2c',
+ 'rand',
+ 'randh',
+ 'randi',
+ 'random',
+ 'randomh',
+ 'randomi',
+ 'rbjeq',
+ 'readclock',
+ 'readf',
+ 'readfi',
+ 'readk',
+ 'readk2',
+ 'readk3',
+ 'readk4',
+ 'readks',
+ 'readscore',
+ 'readscratch',
+ 'rect2pol',
+ 'reinit',
+ 'release',
+ 'remoteport',
+ 'remove',
+ 'repluck',
+ 'reson',
+ 'resonk',
+ 'resonr',
+ 'resonx',
+ 'resonxk',
+ 'resony',
+ 'resonz',
+ 'resyn',
+ #'return',
+ 'reverb',
+ 'reverb2',
+ 'reverbsc',
+ 'rewindscore',
+ 'rezzy',
+ 'rfft',
+ 'rifft',
+ #'rigoto',
+ 'rireturn',
+ 'rms',
+ 'rnd',
+ 'rnd31',
+ 'round',
+ 'rspline',
+ 'rtclock',
+ 's16b14',
+ 's32b14',
+ 'samphold',
+ 'sandpaper',
+ 'scale',
+ 'scalearray',
+ 'scalet',
+ 'scanhammer',
+ 'scans',
+ 'scantable',
+ 'scanu',
+ 'schedkwhen',
+ 'schedkwhennamed',
+ 'schedule',
+ 'schedwhen',
+ #'scoreline',
+ #'scoreline_i',
+ 'seed',
+ 'sekere',
+ 'semitone',
+ 'sense',
+ 'sensekey',
+ 'seqtime',
+ 'seqtime2',
+ 'serialBegin',
+ 'serialEnd',
+ 'serialFlush',
+ 'serialPrint',
+ 'serialRead',
+ 'serialWrite',
+ 'serialWrite_i',
+ 'setcol',
+ 'setctrl',
+ 'setksmps',
+ 'setrow',
+ 'setscorepos',
+ 'sfilist',
+ 'sfinstr',
+ 'sfinstr3',
+ 'sfinstr3m',
+ 'sfinstrm',
+ 'sfload',
+ 'sflooper',
+ 'sfpassign',
+ 'sfplay',
+ 'sfplay3',
+ 'sfplay3m',
+ 'sfplaym',
+ 'sfplist',
+ 'sfpreset',
+ 'shaker',
+ 'shiftin',
+ 'shiftout',
+ 'signalflowgraph',
+ 'signum',
+ 'sin',
+ 'sinh',
+ 'sininv',
+ 'sinsyn',
+ 'sleighbells',
+ 'slicearray',
+ 'slider16',
+ 'slider16f',
+ 'slider16table',
+ 'slider16tablef',
+ 'slider32',
+ 'slider32f',
+ 'slider32table',
+ 'slider32tablef',
+ 'slider64',
+ 'slider64f',
+ 'slider64table',
+ 'slider64tablef',
+ 'slider8',
+ 'slider8f',
+ 'slider8table',
+ 'slider8tablef',
+ 'sliderKawai',
+ 'sndload',
+ 'sndloop',
+ 'sndwarp',
+ 'sndwarpst',
+ 'sockrecv',
+ 'sockrecvs',
+ 'socksend',
+ 'socksends',
+ 'soundin',
+ 'soundout',
+ 'soundouts',
+ 'space',
+ 'spat3d',
+ 'spat3di',
+ 'spat3dt',
+ 'spdist',
+ 'specaddm',
+ 'specdiff',
+ 'specdisp',
+ 'specfilt',
+ 'spechist',
+ 'specptrk',
+ 'specscal',
+ 'specsum',
+ 'spectrum',
+ 'splitrig',
+ 'sprintf',
+ 'sprintfk',
+ 'spsend',
+ 'sqrt',
+ 'stack',
+ 'statevar',
+ 'stix',
+ 'strcat',
+ 'strcatk',
+ 'strchar',
+ 'strchark',
+ 'strcmp',
+ 'strcmpk',
+ 'strcpy',
+ 'strcpyk',
+ 'strecv',
+ 'streson',
+ 'strfromurl',
+ 'strget',
+ 'strindex',
+ 'strindexk',
+ 'strlen',
+ 'strlenk',
+ 'strlower',
+ 'strlowerk',
+ 'strrindex',
+ 'strrindexk',
+ 'strset',
+ 'strsub',
+ 'strsubk',
+ 'strtod',
+ 'strtodk',
+ 'strtol',
+ 'strtolk',
+ 'strupper',
+ 'strupperk',
+ 'stsend',
+ 'subinstr',
+ 'subinstrinit',
+ 'sum',
+ 'sumarray',
+ 'sumtab',
+ 'svfilter',
+ 'syncgrain',
+ 'syncloop',
+ 'syncphasor',
+ 'system',
+ 'system_i',
+ 'tab',
+ 'tab2pvs',
+ 'tab_i',
+ 'tabgen',
+ 'table',
+ 'table3',
+ 'table3kt',
+ 'tablecopy',
+ 'tablefilter',
+ 'tablefilteri',
+ 'tablegpw',
+ 'tablei',
+ 'tableicopy',
+ 'tableigpw',
+ 'tableikt',
+ 'tableimix',
+ 'tableiw',
+ 'tablekt',
+ 'tablemix',
+ 'tableng',
+ 'tablera',
+ 'tableseg',
+ 'tableshuffle',
+ 'tableshufflei',
+ 'tablew',
+ 'tablewa',
+ 'tablewkt',
+ 'tablexkt',
+ 'tablexseg',
+ 'tabmap',
+ 'tabmap_i',
+ 'tabmorph',
+ 'tabmorpha',
+ 'tabmorphak',
+ 'tabmorphi',
+ 'tabplay',
+ 'tabrec',
+ 'tabslice',
+ 'tabsum',
+ 'tabw',
+ 'tabw_i',
+ 'tambourine',
+ 'tan',
+ 'tanh',
+ 'taninv',
+ 'taninv2',
+ 'tb0',
+ 'tb0_init',
+ 'tb1',
+ 'tb10',
+ 'tb10_init',
+ 'tb11',
+ 'tb11_init',
+ 'tb12',
+ 'tb12_init',
+ 'tb13',
+ 'tb13_init',
+ 'tb14',
+ 'tb14_init',
+ 'tb15',
+ 'tb15_init',
+ 'tb1_init',
+ 'tb2',
+ 'tb2_init',
+ 'tb3',
+ 'tb3_init',
+ 'tb4',
+ 'tb4_init',
+ 'tb5',
+ 'tb5_init',
+ 'tb6',
+ 'tb6_init',
+ 'tb7',
+ 'tb7_init',
+ 'tb8',
+ 'tb8_init',
+ 'tb9',
+ 'tb9_init',
+ 'tbvcf',
+ 'tempest',
+ 'tempo',
+ 'temposcal',
+ 'tempoval',
+ #'tigoto',
+ 'timedseq',
+ 'timeinstk',
+ 'timeinsts',
+ 'timek',
+ 'times',
+ #'timout',
+ 'tival',
+ 'tlineto',
+ 'tone',
+ 'tonek',
+ 'tonex',
+ 'tradsyn',
+ 'trandom',
+ 'transeg',
+ 'transegb',
+ 'transegr',
+ 'trcross',
+ 'trfilter',
+ 'trhighest',
+ 'trigger',
+ 'trigseq',
+ 'trirand',
+ 'trlowest',
+ 'trmix',
+ 'trscale',
+ 'trshift',
+ 'trsplit',
+ 'turnoff',
+ 'turnoff2',
+ 'turnon',
+ 'unirand',
+ 'unwrap',
+ 'upsamp',
+ 'urd',
+ 'vactrol',
+ 'vadd',
+ 'vadd_i',
+ 'vaddv',
+ 'vaddv_i',
+ 'vaget',
+ 'valpass',
+ 'vaset',
+ 'vbap',
+ 'vbap16',
+ 'vbap4',
+ 'vbap4move',
+ 'vbap8',
+ 'vbap8move',
+ 'vbapg',
+ 'vbapgmove',
+ 'vbaplsinit',
+ 'vbapmove',
+ 'vbapz',
+ 'vbapzmove',
+ 'vcella',
+ 'vco',
+ 'vco2',
+ 'vco2ft',
+ 'vco2ift',
+ 'vco2init',
+ 'vcomb',
+ 'vcopy',
+ 'vcopy_i',
+ 'vdel_k',
+ 'vdelay',
+ 'vdelay3',
+ 'vdelayk',
+ 'vdelayx',
+ 'vdelayxq',
+ 'vdelayxs',
+ 'vdelayxw',
+ 'vdelayxwq',
+ 'vdelayxws',
+ 'vdivv',
+ 'vdivv_i',
+ 'vecdelay',
+ 'veloc',
+ 'vexp',
+ 'vexp_i',
+ 'vexpseg',
+ 'vexpv',
+ 'vexpv_i',
+ 'vibes',
+ 'vibr',
+ 'vibrato',
+ 'vincr',
+ 'vlimit',
+ 'vlinseg',
+ 'vlowres',
+ 'vmap',
+ 'vmirror',
+ 'vmult',
+ 'vmult_i',
+ 'vmultv',
+ 'vmultv_i',
+ 'voice',
+ 'vosim',
+ 'vphaseseg',
+ 'vport',
+ 'vpow',
+ 'vpow_i',
+ 'vpowv',
+ 'vpowv_i',
+ 'vpvoc',
+ 'vrandh',
+ 'vrandi',
+ 'vsubv',
+ 'vsubv_i',
+ 'vtaba',
+ 'vtabi',
+ 'vtabk',
+ 'vtable1k',
+ 'vtablea',
+ 'vtablei',
+ 'vtablek',
+ 'vtablewa',
+ 'vtablewi',
+ 'vtablewk',
+ 'vtabwa',
+ 'vtabwi',
+ 'vtabwk',
+ 'vwrap',
+ 'waveset',
+ 'weibull',
+ 'wgbow',
+ 'wgbowedbar',
+ 'wgbrass',
+ 'wgclar',
+ 'wgflute',
+ 'wgpluck',
+ 'wgpluck2',
+ 'wguide1',
+ 'wguide2',
+ 'wiiconnect',
+ 'wiidata',
+ 'wiirange',
+ 'wiisend',
+ 'window',
+ 'wrap',
+ 'writescratch',
+ 'wterrain',
+ 'xadsr',
+ 'xin',
+ 'xout',
+ 'xscanmap',
+ 'xscans',
+ 'xscansmap',
+ 'xscanu',
+ 'xtratim',
+ 'xyin',
+ 'zacl',
+ 'zakinit',
+ 'zamod',
+ 'zar',
+ 'zarg',
+ 'zaw',
+ 'zawm',
+ 'zfilter2',
+ 'zir',
+ 'ziw',
+ 'ziwm',
+ 'zkcl',
+ 'zkmod',
+ 'zkr',
+ 'zkw',
+ 'zkwm'
+))
diff --git a/wandb/vendor/pygments/lexers/_lasso_builtins.py b/wandb/vendor/pygments/lexers/_lasso_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..d950cbe859f20088e2765f8717f1a9b2930ceb83
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_lasso_builtins.py
@@ -0,0 +1,5327 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._lasso_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Built-in Lasso types, traits, methods, and members.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+BUILTINS = {
+ 'Types': (
+ 'array',
+ 'atbegin',
+ 'boolean',
+ 'bson_iter',
+ 'bson',
+ 'bytes_document_body',
+ 'bytes',
+ 'cache_server_element',
+ 'cache_server',
+ 'capture',
+ 'client_address',
+ 'client_ip',
+ 'component_container',
+ 'component_render_state',
+ 'component',
+ 'curl',
+ 'curltoken',
+ 'currency',
+ 'custom',
+ 'data_document',
+ 'database_registry',
+ 'date',
+ 'dateandtime',
+ 'dbgp_packet',
+ 'dbgp_server',
+ 'debugging_stack',
+ 'decimal',
+ 'delve',
+ 'dir',
+ 'dirdesc',
+ 'dns_response',
+ 'document_base',
+ 'document_body',
+ 'document_header',
+ 'dsinfo',
+ 'duration',
+ 'eacher',
+ 'email_compose',
+ 'email_parse',
+ 'email_pop',
+ 'email_queue_impl_base',
+ 'email_queue_impl',
+ 'email_smtp',
+ 'email_stage_impl_base',
+ 'email_stage_impl',
+ 'fastcgi_each_fcgi_param',
+ 'fastcgi_server',
+ 'fcgi_record',
+ 'fcgi_request',
+ 'file',
+ 'filedesc',
+ 'filemaker_datasource',
+ 'generateforeachkeyed',
+ 'generateforeachunkeyed',
+ 'generateseries',
+ 'hash_map',
+ 'html_atomic_element',
+ 'html_attr',
+ 'html_base',
+ 'html_binary',
+ 'html_br',
+ 'html_cdata',
+ 'html_container_element',
+ 'html_div',
+ 'html_document_body',
+ 'html_document_head',
+ 'html_eol',
+ 'html_fieldset',
+ 'html_form',
+ 'html_h1',
+ 'html_h2',
+ 'html_h3',
+ 'html_h4',
+ 'html_h5',
+ 'html_h6',
+ 'html_hr',
+ 'html_img',
+ 'html_input',
+ 'html_json',
+ 'html_label',
+ 'html_legend',
+ 'html_link',
+ 'html_meta',
+ 'html_object',
+ 'html_option',
+ 'html_raw',
+ 'html_script',
+ 'html_select',
+ 'html_span',
+ 'html_style',
+ 'html_table',
+ 'html_td',
+ 'html_text',
+ 'html_th',
+ 'html_tr',
+ 'http_document_header',
+ 'http_document',
+ 'http_error',
+ 'http_header_field',
+ 'http_server_connection_handler_globals',
+ 'http_server_connection_handler',
+ 'http_server_request_logger_thread',
+ 'http_server_web_connection',
+ 'http_server',
+ 'image',
+ 'include_cache',
+ 'inline_type',
+ 'integer',
+ 'java_jnienv',
+ 'jbyte',
+ 'jbytearray',
+ 'jchar',
+ 'jchararray',
+ 'jfieldid',
+ 'jfloat',
+ 'jint',
+ 'jmethodid',
+ 'jobject',
+ 'jshort',
+ 'json_decode',
+ 'json_encode',
+ 'json_literal',
+ 'json_object',
+ 'keyword',
+ 'lassoapp_compiledsrc_appsource',
+ 'lassoapp_compiledsrc_fileresource',
+ 'lassoapp_content_rep_halt',
+ 'lassoapp_dirsrc_appsource',
+ 'lassoapp_dirsrc_fileresource',
+ 'lassoapp_installer',
+ 'lassoapp_livesrc_appsource',
+ 'lassoapp_livesrc_fileresource',
+ 'lassoapp_long_expiring_bytes',
+ 'lassoapp_manualsrc_appsource',
+ 'lassoapp_zip_file_server',
+ 'lassoapp_zipsrc_appsource',
+ 'lassoapp_zipsrc_fileresource',
+ 'ldap',
+ 'library_thread_loader',
+ 'list_node',
+ 'list',
+ 'locale',
+ 'log_impl_base',
+ 'log_impl',
+ 'magick_image',
+ 'map_node',
+ 'map',
+ 'memberstream',
+ 'memory_session_driver_impl_entry',
+ 'memory_session_driver_impl',
+ 'memory_session_driver',
+ 'mime_reader',
+ 'mongo_client',
+ 'mongo_collection',
+ 'mongo_cursor',
+ 'mustache_ctx',
+ 'mysql_session_driver_impl',
+ 'mysql_session_driver',
+ 'net_named_pipe',
+ 'net_tcp_ssl',
+ 'net_tcp',
+ 'net_udp_packet',
+ 'net_udp',
+ 'null',
+ 'odbc_session_driver_impl',
+ 'odbc_session_driver',
+ 'opaque',
+ 'os_process',
+ 'pair_compare',
+ 'pair',
+ 'pairup',
+ 'pdf_barcode',
+ 'pdf_chunk',
+ 'pdf_color',
+ 'pdf_doc',
+ 'pdf_font',
+ 'pdf_hyphenator',
+ 'pdf_image',
+ 'pdf_list',
+ 'pdf_paragraph',
+ 'pdf_phrase',
+ 'pdf_read',
+ 'pdf_table',
+ 'pdf_text',
+ 'pdf_typebase',
+ 'percent',
+ 'portal_impl',
+ 'queriable_groupby',
+ 'queriable_grouping',
+ 'queriable_groupjoin',
+ 'queriable_join',
+ 'queriable_orderby',
+ 'queriable_orderbydescending',
+ 'queriable_select',
+ 'queriable_selectmany',
+ 'queriable_skip',
+ 'queriable_take',
+ 'queriable_thenby',
+ 'queriable_thenbydescending',
+ 'queriable_where',
+ 'queue',
+ 'raw_document_body',
+ 'regexp',
+ 'repeat',
+ 'scientific',
+ 'security_registry',
+ 'serialization_element',
+ 'serialization_object_identity_compare',
+ 'serialization_reader',
+ 'serialization_writer_ref',
+ 'serialization_writer_standin',
+ 'serialization_writer',
+ 'session_delete_expired_thread',
+ 'set',
+ 'signature',
+ 'sourcefile',
+ 'sqlite_column',
+ 'sqlite_currentrow',
+ 'sqlite_db',
+ 'sqlite_results',
+ 'sqlite_session_driver_impl_entry',
+ 'sqlite_session_driver_impl',
+ 'sqlite_session_driver',
+ 'sqlite_table',
+ 'sqlite3_stmt',
+ 'sqlite3',
+ 'staticarray',
+ 'string',
+ 'sys_process',
+ 'tag',
+ 'text_document',
+ 'tie',
+ 'timeonly',
+ 'trait',
+ 'tree_base',
+ 'tree_node',
+ 'tree_nullnode',
+ 'ucal',
+ 'usgcpu',
+ 'usgvm',
+ 'void',
+ 'web_error_atend',
+ 'web_node_base',
+ 'web_node_content_representation_css_specialized',
+ 'web_node_content_representation_html_specialized',
+ 'web_node_content_representation_js_specialized',
+ 'web_node_content_representation_xhr_container',
+ 'web_node_echo',
+ 'web_node_root',
+ 'web_request_impl',
+ 'web_request',
+ 'web_response_impl',
+ 'web_response',
+ 'web_router',
+ 'websocket_handler',
+ 'worker_pool',
+ 'xml_attr',
+ 'xml_cdatasection',
+ 'xml_characterdata',
+ 'xml_comment',
+ 'xml_document',
+ 'xml_documentfragment',
+ 'xml_documenttype',
+ 'xml_domimplementation',
+ 'xml_element',
+ 'xml_entity',
+ 'xml_entityreference',
+ 'xml_namednodemap_attr',
+ 'xml_namednodemap_ht',
+ 'xml_namednodemap',
+ 'xml_node',
+ 'xml_nodelist',
+ 'xml_notation',
+ 'xml_processinginstruction',
+ 'xml_text',
+ 'xmlstream',
+ 'zip_file_impl',
+ 'zip_file',
+ 'zip_impl',
+ 'zip',
+ ),
+ 'Traits': (
+ 'any',
+ 'formattingbase',
+ 'html_attributed',
+ 'html_element_coreattrs',
+ 'html_element_eventsattrs',
+ 'html_element_i18nattrs',
+ 'lassoapp_capabilities',
+ 'lassoapp_resource',
+ 'lassoapp_source',
+ 'queriable_asstring',
+ 'session_driver',
+ 'trait_array',
+ 'trait_asstring',
+ 'trait_backcontractible',
+ 'trait_backended',
+ 'trait_backexpandable',
+ 'trait_close',
+ 'trait_contractible',
+ 'trait_decompose_assignment',
+ 'trait_doubleended',
+ 'trait_each_sub',
+ 'trait_encodeurl',
+ 'trait_endedfullymutable',
+ 'trait_expandable',
+ 'trait_file',
+ 'trait_finite',
+ 'trait_finiteforeach',
+ 'trait_foreach',
+ 'trait_foreachtextelement',
+ 'trait_frontcontractible',
+ 'trait_frontended',
+ 'trait_frontexpandable',
+ 'trait_fullymutable',
+ 'trait_generator',
+ 'trait_generatorcentric',
+ 'trait_hashable',
+ 'trait_json_serialize',
+ 'trait_keyed',
+ 'trait_keyedfinite',
+ 'trait_keyedforeach',
+ 'trait_keyedmutable',
+ 'trait_list',
+ 'trait_map',
+ 'trait_net',
+ 'trait_pathcomponents',
+ 'trait_positionallykeyed',
+ 'trait_positionallysearchable',
+ 'trait_queriable',
+ 'trait_queriablelambda',
+ 'trait_readbytes',
+ 'trait_readstring',
+ 'trait_scalar',
+ 'trait_searchable',
+ 'trait_serializable',
+ 'trait_setencoding',
+ 'trait_setoperations',
+ 'trait_stack',
+ 'trait_treenode',
+ 'trait_writebytes',
+ 'trait_writestring',
+ 'trait_xml_elementcompat',
+ 'trait_xml_nodecompat',
+ 'web_connection',
+ 'web_node_container',
+ 'web_node_content_css_specialized',
+ 'web_node_content_document',
+ 'web_node_content_html_specialized',
+ 'web_node_content_js_specialized',
+ 'web_node_content_json_specialized',
+ 'web_node_content_representation',
+ 'web_node_content',
+ 'web_node_postable',
+ 'web_node',
+ ),
+ 'Unbound Methods': (
+ 'abort_clear',
+ 'abort_now',
+ 'abort',
+ 'action_param',
+ 'action_params',
+ 'action_statement',
+ 'admin_authorization',
+ 'admin_currentgroups',
+ 'admin_currentuserid',
+ 'admin_currentusername',
+ 'admin_getpref',
+ 'admin_initialize',
+ 'admin_lassoservicepath',
+ 'admin_removepref',
+ 'admin_setpref',
+ 'admin_userexists',
+ 'all',
+ 'auth_admin',
+ 'auth_check',
+ 'auth_custom',
+ 'auth_group',
+ 'auth_prompt',
+ 'auth_user',
+ 'bom_utf16be',
+ 'bom_utf16le',
+ 'bom_utf32be',
+ 'bom_utf32le',
+ 'bom_utf8',
+ 'bw',
+ 'capture_nearestloopabort',
+ 'capture_nearestloopcontinue',
+ 'capture_nearestloopcount',
+ 'checked',
+ 'cipher_decrypt_private',
+ 'cipher_decrypt_public',
+ 'cipher_decrypt',
+ 'cipher_digest',
+ 'cipher_encrypt_private',
+ 'cipher_encrypt_public',
+ 'cipher_encrypt',
+ 'cipher_generate_key',
+ 'cipher_hmac',
+ 'cipher_keylength',
+ 'cipher_list',
+ 'cipher_open',
+ 'cipher_seal',
+ 'cipher_sign',
+ 'cipher_verify',
+ 'client_addr',
+ 'client_authorization',
+ 'client_browser',
+ 'client_contentlength',
+ 'client_contenttype',
+ 'client_cookielist',
+ 'client_cookies',
+ 'client_encoding',
+ 'client_formmethod',
+ 'client_getargs',
+ 'client_getparam',
+ 'client_getparams',
+ 'client_headers',
+ 'client_integertoip',
+ 'client_iptointeger',
+ 'client_password',
+ 'client_postargs',
+ 'client_postparam',
+ 'client_postparams',
+ 'client_type',
+ 'client_url',
+ 'client_username',
+ 'cn',
+ 'column_name',
+ 'column_names',
+ 'column_type',
+ 'column',
+ 'compress',
+ 'content_addheader',
+ 'content_body',
+ 'content_encoding',
+ 'content_header',
+ 'content_replaceheader',
+ 'content_type',
+ 'cookie_set',
+ 'cookie',
+ 'curl_easy_cleanup',
+ 'curl_easy_duphandle',
+ 'curl_easy_getinfo',
+ 'curl_easy_init',
+ 'curl_easy_reset',
+ 'curl_easy_setopt',
+ 'curl_easy_strerror',
+ 'curl_getdate',
+ 'curl_http_version_1_0',
+ 'curl_http_version_1_1',
+ 'curl_http_version_none',
+ 'curl_ipresolve_v4',
+ 'curl_ipresolve_v6',
+ 'curl_ipresolve_whatever',
+ 'curl_multi_perform',
+ 'curl_multi_result',
+ 'curl_netrc_ignored',
+ 'curl_netrc_optional',
+ 'curl_netrc_required',
+ 'curl_sslversion_default',
+ 'curl_sslversion_sslv2',
+ 'curl_sslversion_sslv3',
+ 'curl_sslversion_tlsv1',
+ 'curl_version_asynchdns',
+ 'curl_version_debug',
+ 'curl_version_gssnegotiate',
+ 'curl_version_idn',
+ 'curl_version_info',
+ 'curl_version_ipv6',
+ 'curl_version_kerberos4',
+ 'curl_version_largefile',
+ 'curl_version_libz',
+ 'curl_version_ntlm',
+ 'curl_version_spnego',
+ 'curl_version_ssl',
+ 'curl_version',
+ 'curlauth_any',
+ 'curlauth_anysafe',
+ 'curlauth_basic',
+ 'curlauth_digest',
+ 'curlauth_gssnegotiate',
+ 'curlauth_none',
+ 'curlauth_ntlm',
+ 'curle_aborted_by_callback',
+ 'curle_bad_calling_order',
+ 'curle_bad_content_encoding',
+ 'curle_bad_download_resume',
+ 'curle_bad_function_argument',
+ 'curle_bad_password_entered',
+ 'curle_couldnt_connect',
+ 'curle_couldnt_resolve_host',
+ 'curle_couldnt_resolve_proxy',
+ 'curle_failed_init',
+ 'curle_file_couldnt_read_file',
+ 'curle_filesize_exceeded',
+ 'curle_ftp_access_denied',
+ 'curle_ftp_cant_get_host',
+ 'curle_ftp_cant_reconnect',
+ 'curle_ftp_couldnt_get_size',
+ 'curle_ftp_couldnt_retr_file',
+ 'curle_ftp_couldnt_set_ascii',
+ 'curle_ftp_couldnt_set_binary',
+ 'curle_ftp_couldnt_use_rest',
+ 'curle_ftp_port_failed',
+ 'curle_ftp_quote_error',
+ 'curle_ftp_ssl_failed',
+ 'curle_ftp_user_password_incorrect',
+ 'curle_ftp_weird_227_format',
+ 'curle_ftp_weird_pass_reply',
+ 'curle_ftp_weird_pasv_reply',
+ 'curle_ftp_weird_server_reply',
+ 'curle_ftp_weird_user_reply',
+ 'curle_ftp_write_error',
+ 'curle_function_not_found',
+ 'curle_got_nothing',
+ 'curle_http_post_error',
+ 'curle_http_range_error',
+ 'curle_http_returned_error',
+ 'curle_interface_failed',
+ 'curle_ldap_cannot_bind',
+ 'curle_ldap_invalid_url',
+ 'curle_ldap_search_failed',
+ 'curle_library_not_found',
+ 'curle_login_denied',
+ 'curle_malformat_user',
+ 'curle_obsolete',
+ 'curle_ok',
+ 'curle_operation_timeouted',
+ 'curle_out_of_memory',
+ 'curle_partial_file',
+ 'curle_read_error',
+ 'curle_recv_error',
+ 'curle_send_error',
+ 'curle_send_fail_rewind',
+ 'curle_share_in_use',
+ 'curle_ssl_cacert',
+ 'curle_ssl_certproblem',
+ 'curle_ssl_cipher',
+ 'curle_ssl_connect_error',
+ 'curle_ssl_engine_initfailed',
+ 'curle_ssl_engine_notfound',
+ 'curle_ssl_engine_setfailed',
+ 'curle_ssl_peer_certificate',
+ 'curle_telnet_option_syntax',
+ 'curle_too_many_redirects',
+ 'curle_unknown_telnet_option',
+ 'curle_unsupported_protocol',
+ 'curle_url_malformat_user',
+ 'curle_url_malformat',
+ 'curle_write_error',
+ 'curlftpauth_default',
+ 'curlftpauth_ssl',
+ 'curlftpauth_tls',
+ 'curlftpssl_all',
+ 'curlftpssl_control',
+ 'curlftpssl_last',
+ 'curlftpssl_none',
+ 'curlftpssl_try',
+ 'curlinfo_connect_time',
+ 'curlinfo_content_length_download',
+ 'curlinfo_content_length_upload',
+ 'curlinfo_content_type',
+ 'curlinfo_effective_url',
+ 'curlinfo_filetime',
+ 'curlinfo_header_size',
+ 'curlinfo_http_connectcode',
+ 'curlinfo_httpauth_avail',
+ 'curlinfo_namelookup_time',
+ 'curlinfo_num_connects',
+ 'curlinfo_os_errno',
+ 'curlinfo_pretransfer_time',
+ 'curlinfo_proxyauth_avail',
+ 'curlinfo_redirect_count',
+ 'curlinfo_redirect_time',
+ 'curlinfo_request_size',
+ 'curlinfo_response_code',
+ 'curlinfo_size_download',
+ 'curlinfo_size_upload',
+ 'curlinfo_speed_download',
+ 'curlinfo_speed_upload',
+ 'curlinfo_ssl_engines',
+ 'curlinfo_ssl_verifyresult',
+ 'curlinfo_starttransfer_time',
+ 'curlinfo_total_time',
+ 'curlmsg_done',
+ 'curlopt_autoreferer',
+ 'curlopt_buffersize',
+ 'curlopt_cainfo',
+ 'curlopt_capath',
+ 'curlopt_connecttimeout',
+ 'curlopt_cookie',
+ 'curlopt_cookiefile',
+ 'curlopt_cookiejar',
+ 'curlopt_cookiesession',
+ 'curlopt_crlf',
+ 'curlopt_customrequest',
+ 'curlopt_dns_use_global_cache',
+ 'curlopt_egdsocket',
+ 'curlopt_encoding',
+ 'curlopt_failonerror',
+ 'curlopt_filetime',
+ 'curlopt_followlocation',
+ 'curlopt_forbid_reuse',
+ 'curlopt_fresh_connect',
+ 'curlopt_ftp_account',
+ 'curlopt_ftp_create_missing_dirs',
+ 'curlopt_ftp_response_timeout',
+ 'curlopt_ftp_ssl',
+ 'curlopt_ftp_use_eprt',
+ 'curlopt_ftp_use_epsv',
+ 'curlopt_ftpappend',
+ 'curlopt_ftplistonly',
+ 'curlopt_ftpport',
+ 'curlopt_ftpsslauth',
+ 'curlopt_header',
+ 'curlopt_http_version',
+ 'curlopt_http200aliases',
+ 'curlopt_httpauth',
+ 'curlopt_httpget',
+ 'curlopt_httpheader',
+ 'curlopt_httppost',
+ 'curlopt_httpproxytunnel',
+ 'curlopt_infilesize_large',
+ 'curlopt_infilesize',
+ 'curlopt_interface',
+ 'curlopt_ipresolve',
+ 'curlopt_krb4level',
+ 'curlopt_low_speed_limit',
+ 'curlopt_low_speed_time',
+ 'curlopt_mail_from',
+ 'curlopt_mail_rcpt',
+ 'curlopt_maxconnects',
+ 'curlopt_maxfilesize_large',
+ 'curlopt_maxfilesize',
+ 'curlopt_maxredirs',
+ 'curlopt_netrc_file',
+ 'curlopt_netrc',
+ 'curlopt_nobody',
+ 'curlopt_noprogress',
+ 'curlopt_port',
+ 'curlopt_post',
+ 'curlopt_postfields',
+ 'curlopt_postfieldsize_large',
+ 'curlopt_postfieldsize',
+ 'curlopt_postquote',
+ 'curlopt_prequote',
+ 'curlopt_proxy',
+ 'curlopt_proxyauth',
+ 'curlopt_proxyport',
+ 'curlopt_proxytype',
+ 'curlopt_proxyuserpwd',
+ 'curlopt_put',
+ 'curlopt_quote',
+ 'curlopt_random_file',
+ 'curlopt_range',
+ 'curlopt_readdata',
+ 'curlopt_referer',
+ 'curlopt_resume_from_large',
+ 'curlopt_resume_from',
+ 'curlopt_ssl_cipher_list',
+ 'curlopt_ssl_verifyhost',
+ 'curlopt_ssl_verifypeer',
+ 'curlopt_sslcert',
+ 'curlopt_sslcerttype',
+ 'curlopt_sslengine_default',
+ 'curlopt_sslengine',
+ 'curlopt_sslkey',
+ 'curlopt_sslkeypasswd',
+ 'curlopt_sslkeytype',
+ 'curlopt_sslversion',
+ 'curlopt_tcp_nodelay',
+ 'curlopt_timecondition',
+ 'curlopt_timeout',
+ 'curlopt_timevalue',
+ 'curlopt_transfertext',
+ 'curlopt_unrestricted_auth',
+ 'curlopt_upload',
+ 'curlopt_url',
+ 'curlopt_use_ssl',
+ 'curlopt_useragent',
+ 'curlopt_userpwd',
+ 'curlopt_verbose',
+ 'curlopt_writedata',
+ 'curlproxy_http',
+ 'curlproxy_socks4',
+ 'curlproxy_socks5',
+ 'database_adddefaultsqlitehost',
+ 'database_database',
+ 'database_initialize',
+ 'database_name',
+ 'database_qs',
+ 'database_table_database_tables',
+ 'database_table_datasource_databases',
+ 'database_table_datasource_hosts',
+ 'database_table_datasources',
+ 'database_table_table_fields',
+ 'database_util_cleanpath',
+ 'dbgp_stop_stack_name',
+ 'debugging_break',
+ 'debugging_breakpoint_get',
+ 'debugging_breakpoint_list',
+ 'debugging_breakpoint_remove',
+ 'debugging_breakpoint_set',
+ 'debugging_breakpoint_update',
+ 'debugging_context_locals',
+ 'debugging_context_self',
+ 'debugging_context_vars',
+ 'debugging_detach',
+ 'debugging_enabled',
+ 'debugging_get_context',
+ 'debugging_get_stack',
+ 'debugging_run',
+ 'debugging_step_in',
+ 'debugging_step_out',
+ 'debugging_step_over',
+ 'debugging_stop',
+ 'debugging_terminate',
+ 'decimal_random',
+ 'decompress',
+ 'decrypt_blowfish',
+ 'define_atbegin',
+ 'define_atend',
+ 'dns_default',
+ 'dns_lookup',
+ 'document',
+ 'email_attachment_mime_type',
+ 'email_batch',
+ 'email_digestchallenge',
+ 'email_digestresponse',
+ 'email_extract',
+ 'email_findemails',
+ 'email_fix_address_list',
+ 'email_fix_address',
+ 'email_fs_error_clean',
+ 'email_immediate',
+ 'email_initialize',
+ 'email_merge',
+ 'email_mxlookup',
+ 'email_pop_priv_extract',
+ 'email_pop_priv_quote',
+ 'email_pop_priv_substring',
+ 'email_queue',
+ 'email_result',
+ 'email_safeemail',
+ 'email_send',
+ 'email_status',
+ 'email_token',
+ 'email_translatebreakstocrlf',
+ 'encode_qheader',
+ 'encoding_iso88591',
+ 'encoding_utf8',
+ 'encrypt_blowfish',
+ 'encrypt_crammd5',
+ 'encrypt_hmac',
+ 'encrypt_md5',
+ 'eol',
+ 'eq',
+ 'error_code_aborted',
+ 'error_code_dividebyzero',
+ 'error_code_filenotfound',
+ 'error_code_invalidparameter',
+ 'error_code_methodnotfound',
+ 'error_code_networkerror',
+ 'error_code_noerror',
+ 'error_code_resnotfound',
+ 'error_code_runtimeassertion',
+ 'error_code',
+ 'error_msg_aborted',
+ 'error_msg_dividebyzero',
+ 'error_msg_filenotfound',
+ 'error_msg_invalidparameter',
+ 'error_msg_methodnotfound',
+ 'error_msg_networkerror',
+ 'error_msg_noerror',
+ 'error_msg_resnotfound',
+ 'error_msg_runtimeassertion',
+ 'error_msg',
+ 'error_obj',
+ 'error_pop',
+ 'error_push',
+ 'error_reset',
+ 'error_stack',
+ 'escape_tag',
+ 'evdns_resolve_ipv4',
+ 'evdns_resolve_ipv6',
+ 'evdns_resolve_reverse_ipv6',
+ 'evdns_resolve_reverse',
+ 'ew',
+ 'fail_if',
+ 'fail_ifnot',
+ 'fail_now',
+ 'fail',
+ 'failure_clear',
+ 'fastcgi_createfcgirequest',
+ 'fastcgi_handlecon',
+ 'fastcgi_handlereq',
+ 'fastcgi_initialize',
+ 'fastcgi_initiate_request',
+ 'fcgi_abort_request',
+ 'fcgi_authorize',
+ 'fcgi_begin_request',
+ 'fcgi_bodychunksize',
+ 'fcgi_cant_mpx_conn',
+ 'fcgi_data',
+ 'fcgi_end_request',
+ 'fcgi_filter',
+ 'fcgi_get_values_result',
+ 'fcgi_get_values',
+ 'fcgi_keep_conn',
+ 'fcgi_makeendrequestbody',
+ 'fcgi_makestdoutbody',
+ 'fcgi_max_conns',
+ 'fcgi_max_reqs',
+ 'fcgi_mpxs_conns',
+ 'fcgi_null_request_id',
+ 'fcgi_overloaded',
+ 'fcgi_params',
+ 'fcgi_read_timeout_seconds',
+ 'fcgi_readparam',
+ 'fcgi_request_complete',
+ 'fcgi_responder',
+ 'fcgi_stderr',
+ 'fcgi_stdin',
+ 'fcgi_stdout',
+ 'fcgi_unknown_role',
+ 'fcgi_unknown_type',
+ 'fcgi_version_1',
+ 'fcgi_x_stdin',
+ 'field_name',
+ 'field_names',
+ 'field',
+ 'file_copybuffersize',
+ 'file_defaultencoding',
+ 'file_forceroot',
+ 'file_modechar',
+ 'file_modeline',
+ 'file_stderr',
+ 'file_stdin',
+ 'file_stdout',
+ 'file_tempfile',
+ 'filemakerds_initialize',
+ 'filemakerds',
+ 'found_count',
+ 'ft',
+ 'ftp_deletefile',
+ 'ftp_getdata',
+ 'ftp_getfile',
+ 'ftp_getlisting',
+ 'ftp_putdata',
+ 'ftp_putfile',
+ 'full',
+ 'generateforeach',
+ 'gt',
+ 'gte',
+ 'handle_failure',
+ 'handle',
+ 'hash_primes',
+ 'html_comment',
+ 'http_char_colon',
+ 'http_char_cr',
+ 'http_char_htab',
+ 'http_char_lf',
+ 'http_char_question',
+ 'http_char_space',
+ 'http_default_files',
+ 'http_read_headers',
+ 'http_read_timeout_secs',
+ 'http_server_apps_path',
+ 'http_server_request_logger',
+ 'if_empty',
+ 'if_false',
+ 'if_null',
+ 'if_true',
+ 'include_cache_compare',
+ 'include_currentpath',
+ 'include_filepath',
+ 'include_localpath',
+ 'include_once',
+ 'include_path',
+ 'include_raw',
+ 'include_url',
+ 'include',
+ 'includes',
+ 'inline_colinfo_name_pos',
+ 'inline_colinfo_type_pos',
+ 'inline_colinfo_valuelist_pos',
+ 'inline_columninfo_pos',
+ 'inline_foundcount_pos',
+ 'inline_namedget',
+ 'inline_namedput',
+ 'inline_resultrows_pos',
+ 'inline_scopeget',
+ 'inline_scopepop',
+ 'inline_scopepush',
+ 'inline',
+ 'integer_bitor',
+ 'integer_random',
+ 'io_dir_dt_blk',
+ 'io_dir_dt_chr',
+ 'io_dir_dt_dir',
+ 'io_dir_dt_fifo',
+ 'io_dir_dt_lnk',
+ 'io_dir_dt_reg',
+ 'io_dir_dt_sock',
+ 'io_dir_dt_unknown',
+ 'io_dir_dt_wht',
+ 'io_file_access',
+ 'io_file_chdir',
+ 'io_file_chmod',
+ 'io_file_chown',
+ 'io_file_dirname',
+ 'io_file_f_dupfd',
+ 'io_file_f_getfd',
+ 'io_file_f_getfl',
+ 'io_file_f_getlk',
+ 'io_file_f_rdlck',
+ 'io_file_f_setfd',
+ 'io_file_f_setfl',
+ 'io_file_f_setlk',
+ 'io_file_f_setlkw',
+ 'io_file_f_test',
+ 'io_file_f_tlock',
+ 'io_file_f_ulock',
+ 'io_file_f_unlck',
+ 'io_file_f_wrlck',
+ 'io_file_fd_cloexec',
+ 'io_file_fioasync',
+ 'io_file_fioclex',
+ 'io_file_fiodtype',
+ 'io_file_fiogetown',
+ 'io_file_fionbio',
+ 'io_file_fionclex',
+ 'io_file_fionread',
+ 'io_file_fiosetown',
+ 'io_file_getcwd',
+ 'io_file_lchown',
+ 'io_file_link',
+ 'io_file_lockf',
+ 'io_file_lstat_atime',
+ 'io_file_lstat_mode',
+ 'io_file_lstat_mtime',
+ 'io_file_lstat_size',
+ 'io_file_mkdir',
+ 'io_file_mkfifo',
+ 'io_file_mkstemp',
+ 'io_file_o_append',
+ 'io_file_o_async',
+ 'io_file_o_creat',
+ 'io_file_o_excl',
+ 'io_file_o_exlock',
+ 'io_file_o_fsync',
+ 'io_file_o_nofollow',
+ 'io_file_o_nonblock',
+ 'io_file_o_rdonly',
+ 'io_file_o_rdwr',
+ 'io_file_o_shlock',
+ 'io_file_o_sync',
+ 'io_file_o_trunc',
+ 'io_file_o_wronly',
+ 'io_file_pipe',
+ 'io_file_readlink',
+ 'io_file_realpath',
+ 'io_file_remove',
+ 'io_file_rename',
+ 'io_file_rmdir',
+ 'io_file_s_ifblk',
+ 'io_file_s_ifchr',
+ 'io_file_s_ifdir',
+ 'io_file_s_ififo',
+ 'io_file_s_iflnk',
+ 'io_file_s_ifmt',
+ 'io_file_s_ifreg',
+ 'io_file_s_ifsock',
+ 'io_file_s_irgrp',
+ 'io_file_s_iroth',
+ 'io_file_s_irusr',
+ 'io_file_s_irwxg',
+ 'io_file_s_irwxo',
+ 'io_file_s_irwxu',
+ 'io_file_s_isgid',
+ 'io_file_s_isuid',
+ 'io_file_s_isvtx',
+ 'io_file_s_iwgrp',
+ 'io_file_s_iwoth',
+ 'io_file_s_iwusr',
+ 'io_file_s_ixgrp',
+ 'io_file_s_ixoth',
+ 'io_file_s_ixusr',
+ 'io_file_seek_cur',
+ 'io_file_seek_end',
+ 'io_file_seek_set',
+ 'io_file_stat_atime',
+ 'io_file_stat_mode',
+ 'io_file_stat_mtime',
+ 'io_file_stat_size',
+ 'io_file_stderr',
+ 'io_file_stdin',
+ 'io_file_stdout',
+ 'io_file_symlink',
+ 'io_file_tempnam',
+ 'io_file_truncate',
+ 'io_file_umask',
+ 'io_file_unlink',
+ 'io_net_accept',
+ 'io_net_af_inet',
+ 'io_net_af_inet6',
+ 'io_net_af_unix',
+ 'io_net_bind',
+ 'io_net_connect',
+ 'io_net_getpeername',
+ 'io_net_getsockname',
+ 'io_net_ipproto_ip',
+ 'io_net_ipproto_udp',
+ 'io_net_listen',
+ 'io_net_msg_oob',
+ 'io_net_msg_peek',
+ 'io_net_msg_waitall',
+ 'io_net_recv',
+ 'io_net_recvfrom',
+ 'io_net_send',
+ 'io_net_sendto',
+ 'io_net_shut_rd',
+ 'io_net_shut_rdwr',
+ 'io_net_shut_wr',
+ 'io_net_shutdown',
+ 'io_net_so_acceptconn',
+ 'io_net_so_broadcast',
+ 'io_net_so_debug',
+ 'io_net_so_dontroute',
+ 'io_net_so_error',
+ 'io_net_so_keepalive',
+ 'io_net_so_linger',
+ 'io_net_so_oobinline',
+ 'io_net_so_rcvbuf',
+ 'io_net_so_rcvlowat',
+ 'io_net_so_rcvtimeo',
+ 'io_net_so_reuseaddr',
+ 'io_net_so_sndbuf',
+ 'io_net_so_sndlowat',
+ 'io_net_so_sndtimeo',
+ 'io_net_so_timestamp',
+ 'io_net_so_type',
+ 'io_net_so_useloopback',
+ 'io_net_sock_dgram',
+ 'io_net_sock_raw',
+ 'io_net_sock_rdm',
+ 'io_net_sock_seqpacket',
+ 'io_net_sock_stream',
+ 'io_net_socket',
+ 'io_net_sol_socket',
+ 'io_net_ssl_accept',
+ 'io_net_ssl_begin',
+ 'io_net_ssl_connect',
+ 'io_net_ssl_end',
+ 'io_net_ssl_error',
+ 'io_net_ssl_errorstring',
+ 'io_net_ssl_funcerrorstring',
+ 'io_net_ssl_liberrorstring',
+ 'io_net_ssl_read',
+ 'io_net_ssl_reasonerrorstring',
+ 'io_net_ssl_setacceptstate',
+ 'io_net_ssl_setconnectstate',
+ 'io_net_ssl_setverifylocations',
+ 'io_net_ssl_shutdown',
+ 'io_net_ssl_usecertificatechainfile',
+ 'io_net_ssl_useprivatekeyfile',
+ 'io_net_ssl_write',
+ 'java_jvm_create',
+ 'java_jvm_getenv',
+ 'jdbc_initialize',
+ 'json_back_slash',
+ 'json_back_space',
+ 'json_close_array',
+ 'json_close_object',
+ 'json_colon',
+ 'json_comma',
+ 'json_consume_array',
+ 'json_consume_object',
+ 'json_consume_string',
+ 'json_consume_token',
+ 'json_cr',
+ 'json_debug',
+ 'json_deserialize',
+ 'json_e_lower',
+ 'json_e_upper',
+ 'json_f_lower',
+ 'json_form_feed',
+ 'json_forward_slash',
+ 'json_lf',
+ 'json_n_lower',
+ 'json_negative',
+ 'json_open_array',
+ 'json_open_object',
+ 'json_period',
+ 'json_positive',
+ 'json_quote_double',
+ 'json_rpccall',
+ 'json_serialize',
+ 'json_t_lower',
+ 'json_tab',
+ 'json_white_space',
+ 'keycolumn_name',
+ 'keycolumn_value',
+ 'keyfield_name',
+ 'keyfield_value',
+ 'lasso_currentaction',
+ 'lasso_errorreporting',
+ 'lasso_executiontimelimit',
+ 'lasso_methodexists',
+ 'lasso_tagexists',
+ 'lasso_uniqueid',
+ 'lasso_version',
+ 'lassoapp_current_app',
+ 'lassoapp_current_include',
+ 'lassoapp_do_with_include',
+ 'lassoapp_exists',
+ 'lassoapp_find_missing_file',
+ 'lassoapp_format_mod_date',
+ 'lassoapp_get_capabilities_name',
+ 'lassoapp_include_current',
+ 'lassoapp_include',
+ 'lassoapp_initialize_db',
+ 'lassoapp_initialize',
+ 'lassoapp_invoke_resource',
+ 'lassoapp_issourcefileextension',
+ 'lassoapp_link',
+ 'lassoapp_load_module',
+ 'lassoapp_mime_get',
+ 'lassoapp_mime_type_appcache',
+ 'lassoapp_mime_type_css',
+ 'lassoapp_mime_type_csv',
+ 'lassoapp_mime_type_doc',
+ 'lassoapp_mime_type_docx',
+ 'lassoapp_mime_type_eof',
+ 'lassoapp_mime_type_eot',
+ 'lassoapp_mime_type_gif',
+ 'lassoapp_mime_type_html',
+ 'lassoapp_mime_type_ico',
+ 'lassoapp_mime_type_jpg',
+ 'lassoapp_mime_type_js',
+ 'lassoapp_mime_type_lasso',
+ 'lassoapp_mime_type_map',
+ 'lassoapp_mime_type_pdf',
+ 'lassoapp_mime_type_png',
+ 'lassoapp_mime_type_ppt',
+ 'lassoapp_mime_type_rss',
+ 'lassoapp_mime_type_svg',
+ 'lassoapp_mime_type_swf',
+ 'lassoapp_mime_type_tif',
+ 'lassoapp_mime_type_ttf',
+ 'lassoapp_mime_type_txt',
+ 'lassoapp_mime_type_woff',
+ 'lassoapp_mime_type_xaml',
+ 'lassoapp_mime_type_xap',
+ 'lassoapp_mime_type_xbap',
+ 'lassoapp_mime_type_xhr',
+ 'lassoapp_mime_type_xml',
+ 'lassoapp_mime_type_zip',
+ 'lassoapp_path_to_method_name',
+ 'lassoapp_settingsdb',
+ 'layout_name',
+ 'lcapi_datasourceadd',
+ 'lcapi_datasourcecloseconnection',
+ 'lcapi_datasourcedelete',
+ 'lcapi_datasourceduplicate',
+ 'lcapi_datasourceexecsql',
+ 'lcapi_datasourcefindall',
+ 'lcapi_datasourceimage',
+ 'lcapi_datasourceinfo',
+ 'lcapi_datasourceinit',
+ 'lcapi_datasourcematchesname',
+ 'lcapi_datasourcenames',
+ 'lcapi_datasourcenothing',
+ 'lcapi_datasourceopand',
+ 'lcapi_datasourceopany',
+ 'lcapi_datasourceopbw',
+ 'lcapi_datasourceopct',
+ 'lcapi_datasourceopeq',
+ 'lcapi_datasourceopew',
+ 'lcapi_datasourceopft',
+ 'lcapi_datasourceopgt',
+ 'lcapi_datasourceopgteq',
+ 'lcapi_datasourceopin',
+ 'lcapi_datasourceoplt',
+ 'lcapi_datasourceoplteq',
+ 'lcapi_datasourceopnbw',
+ 'lcapi_datasourceopnct',
+ 'lcapi_datasourceopneq',
+ 'lcapi_datasourceopnew',
+ 'lcapi_datasourceopnin',
+ 'lcapi_datasourceopno',
+ 'lcapi_datasourceopnot',
+ 'lcapi_datasourceopnrx',
+ 'lcapi_datasourceopor',
+ 'lcapi_datasourceoprx',
+ 'lcapi_datasourcepreparesql',
+ 'lcapi_datasourceprotectionnone',
+ 'lcapi_datasourceprotectionreadonly',
+ 'lcapi_datasourcerandom',
+ 'lcapi_datasourceschemanames',
+ 'lcapi_datasourcescripts',
+ 'lcapi_datasourcesearch',
+ 'lcapi_datasourcesortascending',
+ 'lcapi_datasourcesortcustom',
+ 'lcapi_datasourcesortdescending',
+ 'lcapi_datasourcetablenames',
+ 'lcapi_datasourceterm',
+ 'lcapi_datasourcetickle',
+ 'lcapi_datasourcetypeblob',
+ 'lcapi_datasourcetypeboolean',
+ 'lcapi_datasourcetypedate',
+ 'lcapi_datasourcetypedecimal',
+ 'lcapi_datasourcetypeinteger',
+ 'lcapi_datasourcetypestring',
+ 'lcapi_datasourceunpreparesql',
+ 'lcapi_datasourceupdate',
+ 'lcapi_fourchartointeger',
+ 'lcapi_listdatasources',
+ 'lcapi_loadmodule',
+ 'lcapi_loadmodules',
+ 'lcapi_updatedatasourceslist',
+ 'ldap_scope_base',
+ 'ldap_scope_children',
+ 'ldap_scope_onelevel',
+ 'ldap_scope_subtree',
+ 'library_once',
+ 'library',
+ 'ljapi_initialize',
+ 'locale_availablelocales',
+ 'locale_canada',
+ 'locale_canadafrench',
+ 'locale_china',
+ 'locale_chinese',
+ 'locale_default',
+ 'locale_english',
+ 'locale_format_style_date_time',
+ 'locale_format_style_default',
+ 'locale_format_style_full',
+ 'locale_format_style_long',
+ 'locale_format_style_medium',
+ 'locale_format_style_none',
+ 'locale_format_style_short',
+ 'locale_format',
+ 'locale_france',
+ 'locale_french',
+ 'locale_german',
+ 'locale_germany',
+ 'locale_isocountries',
+ 'locale_isolanguages',
+ 'locale_italian',
+ 'locale_italy',
+ 'locale_japan',
+ 'locale_japanese',
+ 'locale_korea',
+ 'locale_korean',
+ 'locale_prc',
+ 'locale_setdefault',
+ 'locale_simplifiedchinese',
+ 'locale_taiwan',
+ 'locale_traditionalchinese',
+ 'locale_uk',
+ 'locale_us',
+ 'log_always',
+ 'log_critical',
+ 'log_deprecated',
+ 'log_destination_console',
+ 'log_destination_database',
+ 'log_destination_file',
+ 'log_detail',
+ 'log_initialize',
+ 'log_level_critical',
+ 'log_level_deprecated',
+ 'log_level_detail',
+ 'log_level_sql',
+ 'log_level_warning',
+ 'log_max_file_size',
+ 'log_setdestination',
+ 'log_sql',
+ 'log_trim_file_size',
+ 'log_warning',
+ 'log',
+ 'loop_abort',
+ 'loop_continue',
+ 'loop_count',
+ 'loop_key_pop',
+ 'loop_key_push',
+ 'loop_key',
+ 'loop_pop',
+ 'loop_push',
+ 'loop_value_pop',
+ 'loop_value_push',
+ 'loop_value',
+ 'loop',
+ 'lt',
+ 'lte',
+ 'main_thread_only',
+ 'max',
+ 'maxrecords_value',
+ 'median',
+ 'method_name',
+ 'micros',
+ 'millis',
+ 'min',
+ 'minimal',
+ 'mongo_insert_continue_on_error',
+ 'mongo_insert_no_validate',
+ 'mongo_insert_none',
+ 'mongo_query_await_data',
+ 'mongo_query_exhaust',
+ 'mongo_query_no_cursor_timeout',
+ 'mongo_query_none',
+ 'mongo_query_oplog_replay',
+ 'mongo_query_partial',
+ 'mongo_query_slave_ok',
+ 'mongo_query_tailable_cursor',
+ 'mongo_remove_none',
+ 'mongo_remove_single_remove',
+ 'mongo_update_multi_update',
+ 'mongo_update_no_validate',
+ 'mongo_update_none',
+ 'mongo_update_upsert',
+ 'mustache_compile_file',
+ 'mustache_compile_string',
+ 'mustache_include',
+ 'mysqlds',
+ 'namespace_global',
+ 'namespace_import',
+ 'namespace_using',
+ 'nbw',
+ 'ncn',
+ 'neq',
+ 'net_connectinprogress',
+ 'net_connectok',
+ 'net_typessl',
+ 'net_typessltcp',
+ 'net_typessludp',
+ 'net_typetcp',
+ 'net_typeudp',
+ 'net_waitread',
+ 'net_waittimeout',
+ 'net_waitwrite',
+ 'new',
+ 'none',
+ 'nrx',
+ 'nslookup',
+ 'odbc_session_driver_mssql',
+ 'odbc',
+ 'output_none',
+ 'output',
+ 'pdf_package',
+ 'pdf_rectangle',
+ 'pdf_serve',
+ 'pi',
+ 'portal',
+ 'postgresql',
+ 'process',
+ 'protect_now',
+ 'protect',
+ 'queriable_average',
+ 'queriable_defaultcompare',
+ 'queriable_do',
+ 'queriable_internal_combinebindings',
+ 'queriable_max',
+ 'queriable_min',
+ 'queriable_qsort',
+ 'queriable_reversecompare',
+ 'queriable_sum',
+ 'random_seed',
+ 'range',
+ 'records_array',
+ 'records_map',
+ 'records',
+ 'redirect_url',
+ 'referer_url',
+ 'referrer_url',
+ 'register_thread',
+ 'register',
+ 'response_filepath',
+ 'response_localpath',
+ 'response_path',
+ 'response_realm',
+ 'response_root',
+ 'resultset_count',
+ 'resultset',
+ 'resultsets',
+ 'rows_array',
+ 'rows_impl',
+ 'rows',
+ 'rx',
+ 'schema_name',
+ 'security_database',
+ 'security_default_realm',
+ 'security_initialize',
+ 'security_table_groups',
+ 'security_table_ug_map',
+ 'security_table_users',
+ 'selected',
+ 'series',
+ 'server_admin',
+ 'server_ip',
+ 'server_name',
+ 'server_port',
+ 'server_protocol',
+ 'server_push',
+ 'server_signature',
+ 'server_software',
+ 'session_abort',
+ 'session_addvar',
+ 'session_decorate',
+ 'session_deleteexpired',
+ 'session_end',
+ 'session_getdefaultdriver',
+ 'session_id',
+ 'session_initialize',
+ 'session_removevar',
+ 'session_result',
+ 'session_setdefaultdriver',
+ 'session_start',
+ 'shown_count',
+ 'shown_first',
+ 'shown_last',
+ 'site_id',
+ 'site_name',
+ 'skiprecords_value',
+ 'sleep',
+ 'split_thread',
+ 'sqlite_abort',
+ 'sqlite_auth',
+ 'sqlite_blob',
+ 'sqlite_busy',
+ 'sqlite_cantopen',
+ 'sqlite_constraint',
+ 'sqlite_corrupt',
+ 'sqlite_createdb',
+ 'sqlite_done',
+ 'sqlite_empty',
+ 'sqlite_error',
+ 'sqlite_float',
+ 'sqlite_format',
+ 'sqlite_full',
+ 'sqlite_integer',
+ 'sqlite_internal',
+ 'sqlite_interrupt',
+ 'sqlite_ioerr',
+ 'sqlite_locked',
+ 'sqlite_mismatch',
+ 'sqlite_misuse',
+ 'sqlite_nolfs',
+ 'sqlite_nomem',
+ 'sqlite_notadb',
+ 'sqlite_notfound',
+ 'sqlite_null',
+ 'sqlite_ok',
+ 'sqlite_perm',
+ 'sqlite_protocol',
+ 'sqlite_range',
+ 'sqlite_readonly',
+ 'sqlite_row',
+ 'sqlite_schema',
+ 'sqlite_setsleepmillis',
+ 'sqlite_setsleeptries',
+ 'sqlite_text',
+ 'sqlite_toobig',
+ 'sqliteconnector',
+ 'staticarray_join',
+ 'stdout',
+ 'stdoutnl',
+ 'string_validcharset',
+ 'suspend',
+ 'sys_appspath',
+ 'sys_chroot',
+ 'sys_clock',
+ 'sys_clockspersec',
+ 'sys_credits',
+ 'sys_databasespath',
+ 'sys_detach_exec',
+ 'sys_difftime',
+ 'sys_dll_ext',
+ 'sys_drand48',
+ 'sys_environ',
+ 'sys_eol',
+ 'sys_erand48',
+ 'sys_errno',
+ 'sys_exec_pid_to_os_pid',
+ 'sys_exec',
+ 'sys_exit',
+ 'sys_fork',
+ 'sys_garbagecollect',
+ 'sys_getbytessincegc',
+ 'sys_getchar',
+ 'sys_getegid',
+ 'sys_getenv',
+ 'sys_geteuid',
+ 'sys_getgid',
+ 'sys_getgrnam',
+ 'sys_getheapfreebytes',
+ 'sys_getheapsize',
+ 'sys_getlogin',
+ 'sys_getpid',
+ 'sys_getppid',
+ 'sys_getpwnam',
+ 'sys_getpwuid',
+ 'sys_getstartclock',
+ 'sys_getthreadcount',
+ 'sys_getuid',
+ 'sys_growheapby',
+ 'sys_homepath',
+ 'sys_is_full_path',
+ 'sys_is_windows',
+ 'sys_isfullpath',
+ 'sys_iswindows',
+ 'sys_iterate',
+ 'sys_jrand48',
+ 'sys_kill_exec',
+ 'sys_kill',
+ 'sys_lcong48',
+ 'sys_librariespath',
+ 'sys_listtraits',
+ 'sys_listtypes',
+ 'sys_listunboundmethods',
+ 'sys_loadlibrary',
+ 'sys_lrand48',
+ 'sys_masterhomepath',
+ 'sys_mrand48',
+ 'sys_nrand48',
+ 'sys_pid_exec',
+ 'sys_pointersize',
+ 'sys_rand',
+ 'sys_random',
+ 'sys_seed48',
+ 'sys_setenv',
+ 'sys_setgid',
+ 'sys_setsid',
+ 'sys_setuid',
+ 'sys_sigabrt',
+ 'sys_sigalrm',
+ 'sys_sigbus',
+ 'sys_sigchld',
+ 'sys_sigcont',
+ 'sys_sigfpe',
+ 'sys_sighup',
+ 'sys_sigill',
+ 'sys_sigint',
+ 'sys_sigkill',
+ 'sys_sigpipe',
+ 'sys_sigprof',
+ 'sys_sigquit',
+ 'sys_sigsegv',
+ 'sys_sigstop',
+ 'sys_sigsys',
+ 'sys_sigterm',
+ 'sys_sigtrap',
+ 'sys_sigtstp',
+ 'sys_sigttin',
+ 'sys_sigttou',
+ 'sys_sigurg',
+ 'sys_sigusr1',
+ 'sys_sigusr2',
+ 'sys_sigvtalrm',
+ 'sys_sigxcpu',
+ 'sys_sigxfsz',
+ 'sys_srand',
+ 'sys_srand48',
+ 'sys_srandom',
+ 'sys_strerror',
+ 'sys_supportpath',
+ 'sys_test_exec',
+ 'sys_time',
+ 'sys_uname',
+ 'sys_unsetenv',
+ 'sys_usercapimodulepath',
+ 'sys_userstartuppath',
+ 'sys_version',
+ 'sys_wait_exec',
+ 'sys_waitpid',
+ 'sys_wcontinued',
+ 'sys_while',
+ 'sys_wnohang',
+ 'sys_wuntraced',
+ 'table_name',
+ 'tag_exists',
+ 'tag_name',
+ 'thread_var_get',
+ 'thread_var_pop',
+ 'thread_var_push',
+ 'threadvar_find',
+ 'threadvar_get',
+ 'threadvar_set_asrt',
+ 'threadvar_set',
+ 'timer',
+ 'token_value',
+ 'treemap',
+ 'u_lb_alphabetic',
+ 'u_lb_ambiguous',
+ 'u_lb_break_after',
+ 'u_lb_break_before',
+ 'u_lb_break_both',
+ 'u_lb_break_symbols',
+ 'u_lb_carriage_return',
+ 'u_lb_close_punctuation',
+ 'u_lb_combining_mark',
+ 'u_lb_complex_context',
+ 'u_lb_contingent_break',
+ 'u_lb_exclamation',
+ 'u_lb_glue',
+ 'u_lb_h2',
+ 'u_lb_h3',
+ 'u_lb_hyphen',
+ 'u_lb_ideographic',
+ 'u_lb_infix_numeric',
+ 'u_lb_inseparable',
+ 'u_lb_jl',
+ 'u_lb_jt',
+ 'u_lb_jv',
+ 'u_lb_line_feed',
+ 'u_lb_mandatory_break',
+ 'u_lb_next_line',
+ 'u_lb_nonstarter',
+ 'u_lb_numeric',
+ 'u_lb_open_punctuation',
+ 'u_lb_postfix_numeric',
+ 'u_lb_prefix_numeric',
+ 'u_lb_quotation',
+ 'u_lb_space',
+ 'u_lb_surrogate',
+ 'u_lb_unknown',
+ 'u_lb_word_joiner',
+ 'u_lb_zwspace',
+ 'u_nt_decimal',
+ 'u_nt_digit',
+ 'u_nt_none',
+ 'u_nt_numeric',
+ 'u_sb_aterm',
+ 'u_sb_close',
+ 'u_sb_format',
+ 'u_sb_lower',
+ 'u_sb_numeric',
+ 'u_sb_oletter',
+ 'u_sb_other',
+ 'u_sb_sep',
+ 'u_sb_sp',
+ 'u_sb_sterm',
+ 'u_sb_upper',
+ 'u_wb_aletter',
+ 'u_wb_extendnumlet',
+ 'u_wb_format',
+ 'u_wb_katakana',
+ 'u_wb_midletter',
+ 'u_wb_midnum',
+ 'u_wb_numeric',
+ 'u_wb_other',
+ 'ucal_ampm',
+ 'ucal_dayofmonth',
+ 'ucal_dayofweek',
+ 'ucal_dayofweekinmonth',
+ 'ucal_dayofyear',
+ 'ucal_daysinfirstweek',
+ 'ucal_dowlocal',
+ 'ucal_dstoffset',
+ 'ucal_era',
+ 'ucal_extendedyear',
+ 'ucal_firstdayofweek',
+ 'ucal_hour',
+ 'ucal_hourofday',
+ 'ucal_julianday',
+ 'ucal_lenient',
+ 'ucal_listtimezones',
+ 'ucal_millisecond',
+ 'ucal_millisecondsinday',
+ 'ucal_minute',
+ 'ucal_month',
+ 'ucal_second',
+ 'ucal_weekofmonth',
+ 'ucal_weekofyear',
+ 'ucal_year',
+ 'ucal_yearwoy',
+ 'ucal_zoneoffset',
+ 'uchar_age',
+ 'uchar_alphabetic',
+ 'uchar_ascii_hex_digit',
+ 'uchar_bidi_class',
+ 'uchar_bidi_control',
+ 'uchar_bidi_mirrored',
+ 'uchar_bidi_mirroring_glyph',
+ 'uchar_block',
+ 'uchar_canonical_combining_class',
+ 'uchar_case_folding',
+ 'uchar_case_sensitive',
+ 'uchar_dash',
+ 'uchar_decomposition_type',
+ 'uchar_default_ignorable_code_point',
+ 'uchar_deprecated',
+ 'uchar_diacritic',
+ 'uchar_east_asian_width',
+ 'uchar_extender',
+ 'uchar_full_composition_exclusion',
+ 'uchar_general_category_mask',
+ 'uchar_general_category',
+ 'uchar_grapheme_base',
+ 'uchar_grapheme_cluster_break',
+ 'uchar_grapheme_extend',
+ 'uchar_grapheme_link',
+ 'uchar_hangul_syllable_type',
+ 'uchar_hex_digit',
+ 'uchar_hyphen',
+ 'uchar_id_continue',
+ 'uchar_ideographic',
+ 'uchar_ids_binary_operator',
+ 'uchar_ids_trinary_operator',
+ 'uchar_iso_comment',
+ 'uchar_join_control',
+ 'uchar_joining_group',
+ 'uchar_joining_type',
+ 'uchar_lead_canonical_combining_class',
+ 'uchar_line_break',
+ 'uchar_logical_order_exception',
+ 'uchar_lowercase_mapping',
+ 'uchar_lowercase',
+ 'uchar_math',
+ 'uchar_name',
+ 'uchar_nfc_inert',
+ 'uchar_nfc_quick_check',
+ 'uchar_nfd_inert',
+ 'uchar_nfd_quick_check',
+ 'uchar_nfkc_inert',
+ 'uchar_nfkc_quick_check',
+ 'uchar_nfkd_inert',
+ 'uchar_nfkd_quick_check',
+ 'uchar_noncharacter_code_point',
+ 'uchar_numeric_type',
+ 'uchar_numeric_value',
+ 'uchar_pattern_syntax',
+ 'uchar_pattern_white_space',
+ 'uchar_posix_alnum',
+ 'uchar_posix_blank',
+ 'uchar_posix_graph',
+ 'uchar_posix_print',
+ 'uchar_posix_xdigit',
+ 'uchar_quotation_mark',
+ 'uchar_radical',
+ 'uchar_s_term',
+ 'uchar_script',
+ 'uchar_segment_starter',
+ 'uchar_sentence_break',
+ 'uchar_simple_case_folding',
+ 'uchar_simple_lowercase_mapping',
+ 'uchar_simple_titlecase_mapping',
+ 'uchar_simple_uppercase_mapping',
+ 'uchar_soft_dotted',
+ 'uchar_terminal_punctuation',
+ 'uchar_titlecase_mapping',
+ 'uchar_trail_canonical_combining_class',
+ 'uchar_unicode_1_name',
+ 'uchar_unified_ideograph',
+ 'uchar_uppercase_mapping',
+ 'uchar_uppercase',
+ 'uchar_variation_selector',
+ 'uchar_white_space',
+ 'uchar_word_break',
+ 'uchar_xid_continue',
+ 'uncompress',
+ 'usage',
+ 'uuid_compare',
+ 'uuid_copy',
+ 'uuid_generate_random',
+ 'uuid_generate_time',
+ 'uuid_generate',
+ 'uuid_is_null',
+ 'uuid_parse',
+ 'uuid_unparse_lower',
+ 'uuid_unparse_upper',
+ 'uuid_unparse',
+ 'value_list',
+ 'value_listitem',
+ 'valuelistitem',
+ 'var_keys',
+ 'var_values',
+ 'wap_isenabled',
+ 'wap_maxbuttons',
+ 'wap_maxcolumns',
+ 'wap_maxhorzpixels',
+ 'wap_maxrows',
+ 'wap_maxvertpixels',
+ 'web_handlefcgirequest',
+ 'web_node_content_representation_css',
+ 'web_node_content_representation_html',
+ 'web_node_content_representation_js',
+ 'web_node_content_representation_xhr',
+ 'web_node_forpath',
+ 'web_nodes_initialize',
+ 'web_nodes_normalizeextension',
+ 'web_nodes_processcontentnode',
+ 'web_nodes_requesthandler',
+ 'web_response_nodesentry',
+ 'web_router_database',
+ 'web_router_initialize',
+ 'websocket_handler_timeout',
+ 'wexitstatus',
+ 'wifcontinued',
+ 'wifexited',
+ 'wifsignaled',
+ 'wifstopped',
+ 'wstopsig',
+ 'wtermsig',
+ 'xml_transform',
+ 'xml',
+ 'zip_add_dir',
+ 'zip_add',
+ 'zip_checkcons',
+ 'zip_close',
+ 'zip_cm_bzip2',
+ 'zip_cm_default',
+ 'zip_cm_deflate',
+ 'zip_cm_deflate64',
+ 'zip_cm_implode',
+ 'zip_cm_pkware_implode',
+ 'zip_cm_reduce_1',
+ 'zip_cm_reduce_2',
+ 'zip_cm_reduce_3',
+ 'zip_cm_reduce_4',
+ 'zip_cm_shrink',
+ 'zip_cm_store',
+ 'zip_create',
+ 'zip_delete',
+ 'zip_em_3des_112',
+ 'zip_em_3des_168',
+ 'zip_em_aes_128',
+ 'zip_em_aes_192',
+ 'zip_em_aes_256',
+ 'zip_em_des',
+ 'zip_em_none',
+ 'zip_em_rc2_old',
+ 'zip_em_rc2',
+ 'zip_em_rc4',
+ 'zip_em_trad_pkware',
+ 'zip_em_unknown',
+ 'zip_er_changed',
+ 'zip_er_close',
+ 'zip_er_compnotsupp',
+ 'zip_er_crc',
+ 'zip_er_deleted',
+ 'zip_er_eof',
+ 'zip_er_exists',
+ 'zip_er_incons',
+ 'zip_er_internal',
+ 'zip_er_inval',
+ 'zip_er_memory',
+ 'zip_er_multidisk',
+ 'zip_er_noent',
+ 'zip_er_nozip',
+ 'zip_er_ok',
+ 'zip_er_open',
+ 'zip_er_read',
+ 'zip_er_remove',
+ 'zip_er_rename',
+ 'zip_er_seek',
+ 'zip_er_tmpopen',
+ 'zip_er_write',
+ 'zip_er_zipclosed',
+ 'zip_er_zlib',
+ 'zip_error_get_sys_type',
+ 'zip_error_get',
+ 'zip_error_to_str',
+ 'zip_et_none',
+ 'zip_et_sys',
+ 'zip_et_zlib',
+ 'zip_excl',
+ 'zip_fclose',
+ 'zip_file_error_get',
+ 'zip_file_strerror',
+ 'zip_fl_compressed',
+ 'zip_fl_nocase',
+ 'zip_fl_nodir',
+ 'zip_fl_unchanged',
+ 'zip_fopen_index',
+ 'zip_fopen',
+ 'zip_fread',
+ 'zip_get_archive_comment',
+ 'zip_get_file_comment',
+ 'zip_get_name',
+ 'zip_get_num_files',
+ 'zip_name_locate',
+ 'zip_open',
+ 'zip_rename',
+ 'zip_replace',
+ 'zip_set_archive_comment',
+ 'zip_set_file_comment',
+ 'zip_stat_index',
+ 'zip_stat',
+ 'zip_strerror',
+ 'zip_unchange_all',
+ 'zip_unchange_archive',
+ 'zip_unchange',
+ 'zlib_version',
+ ),
+ 'Lasso 8 Tags': (
+ '__char',
+ '__sync_timestamp__',
+ '_admin_addgroup',
+ '_admin_adduser',
+ '_admin_defaultconnector',
+ '_admin_defaultconnectornames',
+ '_admin_defaultdatabase',
+ '_admin_defaultfield',
+ '_admin_defaultgroup',
+ '_admin_defaulthost',
+ '_admin_defaulttable',
+ '_admin_defaultuser',
+ '_admin_deleteconnector',
+ '_admin_deletedatabase',
+ '_admin_deletefield',
+ '_admin_deletegroup',
+ '_admin_deletehost',
+ '_admin_deletetable',
+ '_admin_deleteuser',
+ '_admin_duplicategroup',
+ '_admin_internaldatabase',
+ '_admin_listconnectors',
+ '_admin_listdatabases',
+ '_admin_listfields',
+ '_admin_listgroups',
+ '_admin_listhosts',
+ '_admin_listtables',
+ '_admin_listusers',
+ '_admin_refreshconnector',
+ '_admin_refreshsecurity',
+ '_admin_servicepath',
+ '_admin_updateconnector',
+ '_admin_updatedatabase',
+ '_admin_updatefield',
+ '_admin_updategroup',
+ '_admin_updatehost',
+ '_admin_updatetable',
+ '_admin_updateuser',
+ '_chartfx_activation_string',
+ '_chartfx_getchallengestring',
+ '_chop_args',
+ '_chop_mimes',
+ '_client_addr_old',
+ '_client_address_old',
+ '_client_ip_old',
+ '_database_names',
+ '_datasource_reload',
+ '_date_current',
+ '_date_format',
+ '_date_msec',
+ '_date_parse',
+ '_execution_timelimit',
+ '_file_chmod',
+ '_initialize',
+ '_jdbc_acceptsurl',
+ '_jdbc_debug',
+ '_jdbc_deletehost',
+ '_jdbc_driverclasses',
+ '_jdbc_driverinfo',
+ '_jdbc_metainfo',
+ '_jdbc_propertyinfo',
+ '_jdbc_setdriver',
+ '_lasso_param',
+ '_log_helper',
+ '_proc_noparam',
+ '_proc_withparam',
+ '_recursion_limit',
+ '_request_param',
+ '_security_binaryexpiration',
+ '_security_flushcaches',
+ '_security_isserialized',
+ '_security_serialexpiration',
+ '_srand',
+ '_strict_literals',
+ '_substring',
+ '_xmlrpc_exconverter',
+ '_xmlrpc_inconverter',
+ '_xmlrpc_xmlinconverter',
+ 'abort',
+ 'action_addinfo',
+ 'action_addrecord',
+ 'action_param',
+ 'action_params',
+ 'action_setfoundcount',
+ 'action_setrecordid',
+ 'action_settotalcount',
+ 'action_statement',
+ 'admin_allowedfileroots',
+ 'admin_changeuser',
+ 'admin_createuser',
+ 'admin_currentgroups',
+ 'admin_currentuserid',
+ 'admin_currentusername',
+ 'admin_getpref',
+ 'admin_groupassignuser',
+ 'admin_grouplistusers',
+ 'admin_groupremoveuser',
+ 'admin_lassoservicepath',
+ 'admin_listgroups',
+ 'admin_refreshlicensing',
+ 'admin_refreshsecurity',
+ 'admin_reloaddatasource',
+ 'admin_removepref',
+ 'admin_setpref',
+ 'admin_userexists',
+ 'admin_userlistgroups',
+ 'all',
+ 'and',
+ 'array',
+ 'array_iterator',
+ 'auth',
+ 'auth_admin',
+ 'auth_auth',
+ 'auth_custom',
+ 'auth_group',
+ 'auth_prompt',
+ 'auth_user',
+ 'base64',
+ 'bean',
+ 'bigint',
+ 'bom_utf16be',
+ 'bom_utf16le',
+ 'bom_utf32be',
+ 'bom_utf32le',
+ 'bom_utf8',
+ 'boolean',
+ 'bw',
+ 'bytes',
+ 'cache',
+ 'cache_delete',
+ 'cache_empty',
+ 'cache_exists',
+ 'cache_fetch',
+ 'cache_internal',
+ 'cache_maintenance',
+ 'cache_object',
+ 'cache_preferences',
+ 'cache_store',
+ 'case',
+ 'chartfx',
+ 'chartfx_records',
+ 'chartfx_serve',
+ 'checked',
+ 'choice_list',
+ 'choice_listitem',
+ 'choicelistitem',
+ 'cipher_decrypt',
+ 'cipher_digest',
+ 'cipher_encrypt',
+ 'cipher_hmac',
+ 'cipher_keylength',
+ 'cipher_list',
+ 'click_text',
+ 'client_addr',
+ 'client_address',
+ 'client_authorization',
+ 'client_browser',
+ 'client_contentlength',
+ 'client_contenttype',
+ 'client_cookielist',
+ 'client_cookies',
+ 'client_encoding',
+ 'client_formmethod',
+ 'client_getargs',
+ 'client_getparams',
+ 'client_headers',
+ 'client_ip',
+ 'client_ipfrominteger',
+ 'client_iptointeger',
+ 'client_password',
+ 'client_postargs',
+ 'client_postparams',
+ 'client_type',
+ 'client_url',
+ 'client_username',
+ 'cn',
+ 'column',
+ 'column_name',
+ 'column_names',
+ 'compare_beginswith',
+ 'compare_contains',
+ 'compare_endswith',
+ 'compare_equalto',
+ 'compare_greaterthan',
+ 'compare_greaterthanorequals',
+ 'compare_greaterthanorequls',
+ 'compare_lessthan',
+ 'compare_lessthanorequals',
+ 'compare_notbeginswith',
+ 'compare_notcontains',
+ 'compare_notendswith',
+ 'compare_notequalto',
+ 'compare_notregexp',
+ 'compare_regexp',
+ 'compare_strictequalto',
+ 'compare_strictnotequalto',
+ 'compiler_removecacheddoc',
+ 'compiler_setdefaultparserflags',
+ 'compress',
+ 'content_body',
+ 'content_encoding',
+ 'content_header',
+ 'content_type',
+ 'cookie',
+ 'cookie_set',
+ 'curl_ftp_getfile',
+ 'curl_ftp_getlisting',
+ 'curl_ftp_putfile',
+ 'curl_include_url',
+ 'currency',
+ 'database_changecolumn',
+ 'database_changefield',
+ 'database_createcolumn',
+ 'database_createfield',
+ 'database_createtable',
+ 'database_fmcontainer',
+ 'database_hostinfo',
+ 'database_inline',
+ 'database_name',
+ 'database_nameitem',
+ 'database_names',
+ 'database_realname',
+ 'database_removecolumn',
+ 'database_removefield',
+ 'database_removetable',
+ 'database_repeating',
+ 'database_repeating_valueitem',
+ 'database_repeatingvalueitem',
+ 'database_schemanameitem',
+ 'database_schemanames',
+ 'database_tablecolumn',
+ 'database_tablenameitem',
+ 'database_tablenames',
+ 'datasource_name',
+ 'datasource_register',
+ 'date',
+ 'date__date_current',
+ 'date__date_format',
+ 'date__date_msec',
+ 'date__date_parse',
+ 'date_add',
+ 'date_date',
+ 'date_difference',
+ 'date_duration',
+ 'date_format',
+ 'date_getcurrentdate',
+ 'date_getday',
+ 'date_getdayofweek',
+ 'date_gethour',
+ 'date_getlocaltimezone',
+ 'date_getminute',
+ 'date_getmonth',
+ 'date_getsecond',
+ 'date_gettime',
+ 'date_getyear',
+ 'date_gmttolocal',
+ 'date_localtogmt',
+ 'date_maximum',
+ 'date_minimum',
+ 'date_msec',
+ 'date_setformat',
+ 'date_subtract',
+ 'db_layoutnameitem',
+ 'db_layoutnames',
+ 'db_nameitem',
+ 'db_names',
+ 'db_tablenameitem',
+ 'db_tablenames',
+ 'dbi_column_names',
+ 'dbi_field_names',
+ 'decimal',
+ 'decimal_setglobaldefaultprecision',
+ 'decode_base64',
+ 'decode_bheader',
+ 'decode_hex',
+ 'decode_html',
+ 'decode_json',
+ 'decode_qheader',
+ 'decode_quotedprintable',
+ 'decode_quotedprintablebytes',
+ 'decode_url',
+ 'decode_xml',
+ 'decompress',
+ 'decrypt_blowfish',
+ 'decrypt_blowfish2',
+ 'default',
+ 'define_atbegin',
+ 'define_atend',
+ 'define_constant',
+ 'define_prototype',
+ 'define_tag',
+ 'define_tagp',
+ 'define_type',
+ 'define_typep',
+ 'deserialize',
+ 'directory_directorynameitem',
+ 'directory_lister',
+ 'directory_nameitem',
+ 'directorynameitem',
+ 'dns_default',
+ 'dns_lookup',
+ 'dns_response',
+ 'duration',
+ 'else',
+ 'email_batch',
+ 'email_compose',
+ 'email_digestchallenge',
+ 'email_digestresponse',
+ 'email_extract',
+ 'email_findemails',
+ 'email_immediate',
+ 'email_merge',
+ 'email_mxerror',
+ 'email_mxlookup',
+ 'email_parse',
+ 'email_pop',
+ 'email_queue',
+ 'email_result',
+ 'email_safeemail',
+ 'email_send',
+ 'email_smtp',
+ 'email_status',
+ 'email_token',
+ 'email_translatebreakstocrlf',
+ 'encode_base64',
+ 'encode_bheader',
+ 'encode_break',
+ 'encode_breaks',
+ 'encode_crc32',
+ 'encode_hex',
+ 'encode_html',
+ 'encode_htmltoxml',
+ 'encode_json',
+ 'encode_qheader',
+ 'encode_quotedprintable',
+ 'encode_quotedprintablebytes',
+ 'encode_set',
+ 'encode_smart',
+ 'encode_sql',
+ 'encode_sql92',
+ 'encode_stricturl',
+ 'encode_url',
+ 'encode_xml',
+ 'encrypt_blowfish',
+ 'encrypt_blowfish2',
+ 'encrypt_crammd5',
+ 'encrypt_hmac',
+ 'encrypt_md5',
+ 'eq',
+ 'error_adderror',
+ 'error_code',
+ 'error_code_aborted',
+ 'error_code_assert',
+ 'error_code_bof',
+ 'error_code_connectioninvalid',
+ 'error_code_couldnotclosefile',
+ 'error_code_couldnotcreateoropenfile',
+ 'error_code_couldnotdeletefile',
+ 'error_code_couldnotdisposememory',
+ 'error_code_couldnotlockmemory',
+ 'error_code_couldnotreadfromfile',
+ 'error_code_couldnotunlockmemory',
+ 'error_code_couldnotwritetofile',
+ 'error_code_criterianotmet',
+ 'error_code_datasourceerror',
+ 'error_code_directoryfull',
+ 'error_code_diskfull',
+ 'error_code_dividebyzero',
+ 'error_code_eof',
+ 'error_code_failure',
+ 'error_code_fieldrestriction',
+ 'error_code_file',
+ 'error_code_filealreadyexists',
+ 'error_code_filecorrupt',
+ 'error_code_fileinvalid',
+ 'error_code_fileinvalidaccessmode',
+ 'error_code_fileisclosed',
+ 'error_code_fileisopen',
+ 'error_code_filelocked',
+ 'error_code_filenotfound',
+ 'error_code_fileunlocked',
+ 'error_code_httpfilenotfound',
+ 'error_code_illegalinstruction',
+ 'error_code_illegaluseoffrozeninstance',
+ 'error_code_invaliddatabase',
+ 'error_code_invalidfilename',
+ 'error_code_invalidmemoryobject',
+ 'error_code_invalidparameter',
+ 'error_code_invalidpassword',
+ 'error_code_invalidpathname',
+ 'error_code_invalidusername',
+ 'error_code_ioerror',
+ 'error_code_loopaborted',
+ 'error_code_memory',
+ 'error_code_network',
+ 'error_code_nilpointer',
+ 'error_code_noerr',
+ 'error_code_nopermission',
+ 'error_code_outofmemory',
+ 'error_code_outofstackspace',
+ 'error_code_overflow',
+ 'error_code_postconditionfailed',
+ 'error_code_preconditionfailed',
+ 'error_code_resnotfound',
+ 'error_code_resource',
+ 'error_code_streamreaderror',
+ 'error_code_streamwriteerror',
+ 'error_code_syntaxerror',
+ 'error_code_tagnotfound',
+ 'error_code_unknownerror',
+ 'error_code_varnotfound',
+ 'error_code_volumedoesnotexist',
+ 'error_code_webactionnotsupported',
+ 'error_code_webadderror',
+ 'error_code_webdeleteerror',
+ 'error_code_webmodulenotfound',
+ 'error_code_webnosuchobject',
+ 'error_code_webrepeatingrelatedfield',
+ 'error_code_webrequiredfieldmissing',
+ 'error_code_webtimeout',
+ 'error_code_webupdateerror',
+ 'error_columnrestriction',
+ 'error_currenterror',
+ 'error_databaseconnectionunavailable',
+ 'error_databasetimeout',
+ 'error_deleteerror',
+ 'error_fieldrestriction',
+ 'error_filenotfound',
+ 'error_invaliddatabase',
+ 'error_invalidpassword',
+ 'error_invalidusername',
+ 'error_modulenotfound',
+ 'error_msg',
+ 'error_msg_aborted',
+ 'error_msg_assert',
+ 'error_msg_bof',
+ 'error_msg_connectioninvalid',
+ 'error_msg_couldnotclosefile',
+ 'error_msg_couldnotcreateoropenfile',
+ 'error_msg_couldnotdeletefile',
+ 'error_msg_couldnotdisposememory',
+ 'error_msg_couldnotlockmemory',
+ 'error_msg_couldnotreadfromfile',
+ 'error_msg_couldnotunlockmemory',
+ 'error_msg_couldnotwritetofile',
+ 'error_msg_criterianotmet',
+ 'error_msg_datasourceerror',
+ 'error_msg_directoryfull',
+ 'error_msg_diskfull',
+ 'error_msg_dividebyzero',
+ 'error_msg_eof',
+ 'error_msg_failure',
+ 'error_msg_fieldrestriction',
+ 'error_msg_file',
+ 'error_msg_filealreadyexists',
+ 'error_msg_filecorrupt',
+ 'error_msg_fileinvalid',
+ 'error_msg_fileinvalidaccessmode',
+ 'error_msg_fileisclosed',
+ 'error_msg_fileisopen',
+ 'error_msg_filelocked',
+ 'error_msg_filenotfound',
+ 'error_msg_fileunlocked',
+ 'error_msg_httpfilenotfound',
+ 'error_msg_illegalinstruction',
+ 'error_msg_illegaluseoffrozeninstance',
+ 'error_msg_invaliddatabase',
+ 'error_msg_invalidfilename',
+ 'error_msg_invalidmemoryobject',
+ 'error_msg_invalidparameter',
+ 'error_msg_invalidpassword',
+ 'error_msg_invalidpathname',
+ 'error_msg_invalidusername',
+ 'error_msg_ioerror',
+ 'error_msg_loopaborted',
+ 'error_msg_memory',
+ 'error_msg_network',
+ 'error_msg_nilpointer',
+ 'error_msg_noerr',
+ 'error_msg_nopermission',
+ 'error_msg_outofmemory',
+ 'error_msg_outofstackspace',
+ 'error_msg_overflow',
+ 'error_msg_postconditionfailed',
+ 'error_msg_preconditionfailed',
+ 'error_msg_resnotfound',
+ 'error_msg_resource',
+ 'error_msg_streamreaderror',
+ 'error_msg_streamwriteerror',
+ 'error_msg_syntaxerror',
+ 'error_msg_tagnotfound',
+ 'error_msg_unknownerror',
+ 'error_msg_varnotfound',
+ 'error_msg_volumedoesnotexist',
+ 'error_msg_webactionnotsupported',
+ 'error_msg_webadderror',
+ 'error_msg_webdeleteerror',
+ 'error_msg_webmodulenotfound',
+ 'error_msg_webnosuchobject',
+ 'error_msg_webrepeatingrelatedfield',
+ 'error_msg_webrequiredfieldmissing',
+ 'error_msg_webtimeout',
+ 'error_msg_webupdateerror',
+ 'error_noerror',
+ 'error_nopermission',
+ 'error_norecordsfound',
+ 'error_outofmemory',
+ 'error_pop',
+ 'error_push',
+ 'error_reqcolumnmissing',
+ 'error_reqfieldmissing',
+ 'error_requiredcolumnmissing',
+ 'error_requiredfieldmissing',
+ 'error_reset',
+ 'error_seterrorcode',
+ 'error_seterrormessage',
+ 'error_updateerror',
+ 'euro',
+ 'event_schedule',
+ 'ew',
+ 'fail',
+ 'fail_if',
+ 'false',
+ 'field',
+ 'field_name',
+ 'field_names',
+ 'file',
+ 'file_autoresolvefullpaths',
+ 'file_chmod',
+ 'file_control',
+ 'file_copy',
+ 'file_create',
+ 'file_creationdate',
+ 'file_currenterror',
+ 'file_delete',
+ 'file_exists',
+ 'file_getlinecount',
+ 'file_getsize',
+ 'file_isdirectory',
+ 'file_listdirectory',
+ 'file_moddate',
+ 'file_modechar',
+ 'file_modeline',
+ 'file_move',
+ 'file_openread',
+ 'file_openreadwrite',
+ 'file_openwrite',
+ 'file_openwriteappend',
+ 'file_openwritetruncate',
+ 'file_probeeol',
+ 'file_processuploads',
+ 'file_read',
+ 'file_readline',
+ 'file_rename',
+ 'file_serve',
+ 'file_setsize',
+ 'file_stream',
+ 'file_streamcopy',
+ 'file_uploads',
+ 'file_waitread',
+ 'file_waittimeout',
+ 'file_waitwrite',
+ 'file_write',
+ 'find_soap_ops',
+ 'form_param',
+ 'found_count',
+ 'ft',
+ 'ftp_getfile',
+ 'ftp_getlisting',
+ 'ftp_putfile',
+ 'full',
+ 'global',
+ 'global_defined',
+ 'global_remove',
+ 'global_reset',
+ 'globals',
+ 'gt',
+ 'gte',
+ 'handle',
+ 'handle_error',
+ 'header',
+ 'html_comment',
+ 'http_getfile',
+ 'ical_alarm',
+ 'ical_attribute',
+ 'ical_calendar',
+ 'ical_daylight',
+ 'ical_event',
+ 'ical_freebusy',
+ 'ical_item',
+ 'ical_journal',
+ 'ical_parse',
+ 'ical_standard',
+ 'ical_timezone',
+ 'ical_todo',
+ 'if',
+ 'if_empty',
+ 'if_false',
+ 'if_null',
+ 'if_true',
+ 'image',
+ 'image_url',
+ 'img',
+ 'include',
+ 'include_cgi',
+ 'include_currentpath',
+ 'include_once',
+ 'include_raw',
+ 'include_url',
+ 'inline',
+ 'integer',
+ 'iterate',
+ 'iterator',
+ 'java',
+ 'java_bean',
+ 'json_records',
+ 'json_rpccall',
+ 'keycolumn_name',
+ 'keycolumn_value',
+ 'keyfield_name',
+ 'keyfield_value',
+ 'lasso_comment',
+ 'lasso_currentaction',
+ 'lasso_datasourceis',
+ 'lasso_datasourceis4d',
+ 'lasso_datasourceisfilemaker',
+ 'lasso_datasourceisfilemaker7',
+ 'lasso_datasourceisfilemaker9',
+ 'lasso_datasourceisfilemakersa',
+ 'lasso_datasourceisjdbc',
+ 'lasso_datasourceislassomysql',
+ 'lasso_datasourceismysql',
+ 'lasso_datasourceisodbc',
+ 'lasso_datasourceisopenbase',
+ 'lasso_datasourceisoracle',
+ 'lasso_datasourceispostgresql',
+ 'lasso_datasourceisspotlight',
+ 'lasso_datasourceissqlite',
+ 'lasso_datasourceissqlserver',
+ 'lasso_datasourcemodulename',
+ 'lasso_datatype',
+ 'lasso_disableondemand',
+ 'lasso_errorreporting',
+ 'lasso_executiontimelimit',
+ 'lasso_parser',
+ 'lasso_process',
+ 'lasso_sessionid',
+ 'lasso_siteid',
+ 'lasso_siteisrunning',
+ 'lasso_sitename',
+ 'lasso_siterestart',
+ 'lasso_sitestart',
+ 'lasso_sitestop',
+ 'lasso_tagexists',
+ 'lasso_tagmodulename',
+ 'lasso_uniqueid',
+ 'lasso_updatecheck',
+ 'lasso_uptime',
+ 'lasso_version',
+ 'lassoapp_create',
+ 'lassoapp_dump',
+ 'lassoapp_flattendir',
+ 'lassoapp_getappdata',
+ 'lassoapp_link',
+ 'lassoapp_list',
+ 'lassoapp_process',
+ 'lassoapp_unitize',
+ 'layout_name',
+ 'ldap',
+ 'ldap_scope_base',
+ 'ldap_scope_onelevel',
+ 'ldap_scope_subtree',
+ 'ldml',
+ 'ldml_ldml',
+ 'library',
+ 'library_once',
+ 'link',
+ 'link_currentaction',
+ 'link_currentactionparams',
+ 'link_currentactionurl',
+ 'link_currentgroup',
+ 'link_currentgroupparams',
+ 'link_currentgroupurl',
+ 'link_currentrecord',
+ 'link_currentrecordparams',
+ 'link_currentrecordurl',
+ 'link_currentsearch',
+ 'link_currentsearchparams',
+ 'link_currentsearchurl',
+ 'link_detail',
+ 'link_detailparams',
+ 'link_detailurl',
+ 'link_firstgroup',
+ 'link_firstgroupparams',
+ 'link_firstgroupurl',
+ 'link_firstrecord',
+ 'link_firstrecordparams',
+ 'link_firstrecordurl',
+ 'link_lastgroup',
+ 'link_lastgroupparams',
+ 'link_lastgroupurl',
+ 'link_lastrecord',
+ 'link_lastrecordparams',
+ 'link_lastrecordurl',
+ 'link_nextgroup',
+ 'link_nextgroupparams',
+ 'link_nextgroupurl',
+ 'link_nextrecord',
+ 'link_nextrecordparams',
+ 'link_nextrecordurl',
+ 'link_params',
+ 'link_prevgroup',
+ 'link_prevgroupparams',
+ 'link_prevgroupurl',
+ 'link_prevrecord',
+ 'link_prevrecordparams',
+ 'link_prevrecordurl',
+ 'link_setformat',
+ 'link_url',
+ 'list',
+ 'list_additem',
+ 'list_fromlist',
+ 'list_fromstring',
+ 'list_getitem',
+ 'list_itemcount',
+ 'list_iterator',
+ 'list_removeitem',
+ 'list_replaceitem',
+ 'list_reverseiterator',
+ 'list_tostring',
+ 'literal',
+ 'ljax_end',
+ 'ljax_hastarget',
+ 'ljax_include',
+ 'ljax_start',
+ 'ljax_target',
+ 'local',
+ 'local_defined',
+ 'local_remove',
+ 'local_reset',
+ 'locale_format',
+ 'locals',
+ 'log',
+ 'log_always',
+ 'log_critical',
+ 'log_deprecated',
+ 'log_destination_console',
+ 'log_destination_database',
+ 'log_destination_file',
+ 'log_detail',
+ 'log_level_critical',
+ 'log_level_deprecated',
+ 'log_level_detail',
+ 'log_level_sql',
+ 'log_level_warning',
+ 'log_setdestination',
+ 'log_sql',
+ 'log_warning',
+ 'logicalop_value',
+ 'logicaloperator_value',
+ 'loop',
+ 'loop_abort',
+ 'loop_continue',
+ 'loop_count',
+ 'lt',
+ 'lte',
+ 'magick_image',
+ 'map',
+ 'map_iterator',
+ 'match_comparator',
+ 'match_notrange',
+ 'match_notregexp',
+ 'match_range',
+ 'match_regexp',
+ 'math_abs',
+ 'math_acos',
+ 'math_add',
+ 'math_asin',
+ 'math_atan',
+ 'math_atan2',
+ 'math_ceil',
+ 'math_converteuro',
+ 'math_cos',
+ 'math_div',
+ 'math_exp',
+ 'math_floor',
+ 'math_internal_rand',
+ 'math_internal_randmax',
+ 'math_internal_srand',
+ 'math_ln',
+ 'math_log',
+ 'math_log10',
+ 'math_max',
+ 'math_min',
+ 'math_mod',
+ 'math_mult',
+ 'math_pow',
+ 'math_random',
+ 'math_range',
+ 'math_rint',
+ 'math_roman',
+ 'math_round',
+ 'math_sin',
+ 'math_sqrt',
+ 'math_sub',
+ 'math_tan',
+ 'maxrecords_value',
+ 'memory_session_driver',
+ 'mime_type',
+ 'minimal',
+ 'misc__srand',
+ 'misc_randomnumber',
+ 'misc_roman',
+ 'misc_valid_creditcard',
+ 'mysql_session_driver',
+ 'named_param',
+ 'namespace_current',
+ 'namespace_delimiter',
+ 'namespace_exists',
+ 'namespace_file_fullpathexists',
+ 'namespace_global',
+ 'namespace_import',
+ 'namespace_load',
+ 'namespace_page',
+ 'namespace_unload',
+ 'namespace_using',
+ 'neq',
+ 'net',
+ 'net_connectinprogress',
+ 'net_connectok',
+ 'net_typessl',
+ 'net_typessltcp',
+ 'net_typessludp',
+ 'net_typetcp',
+ 'net_typeudp',
+ 'net_waitread',
+ 'net_waittimeout',
+ 'net_waitwrite',
+ 'no_default_output',
+ 'none',
+ 'noprocess',
+ 'not',
+ 'nrx',
+ 'nslookup',
+ 'null',
+ 'object',
+ 'once',
+ 'oneoff',
+ 'op_logicalvalue',
+ 'operator_logicalvalue',
+ 'option',
+ 'or',
+ 'os_process',
+ 'output',
+ 'output_none',
+ 'pair',
+ 'params_up',
+ 'pdf_barcode',
+ 'pdf_color',
+ 'pdf_doc',
+ 'pdf_font',
+ 'pdf_image',
+ 'pdf_list',
+ 'pdf_read',
+ 'pdf_serve',
+ 'pdf_table',
+ 'pdf_text',
+ 'percent',
+ 'portal',
+ 'postcondition',
+ 'precondition',
+ 'prettyprintingnsmap',
+ 'prettyprintingtypemap',
+ 'priorityqueue',
+ 'private',
+ 'proc_convert',
+ 'proc_convertbody',
+ 'proc_convertone',
+ 'proc_extract',
+ 'proc_extractone',
+ 'proc_find',
+ 'proc_first',
+ 'proc_foreach',
+ 'proc_get',
+ 'proc_join',
+ 'proc_lasso',
+ 'proc_last',
+ 'proc_map_entry',
+ 'proc_null',
+ 'proc_regexp',
+ 'proc_xml',
+ 'proc_xslt',
+ 'process',
+ 'protect',
+ 'queue',
+ 'rand',
+ 'randomnumber',
+ 'raw',
+ 'recid_value',
+ 'record_count',
+ 'recordcount',
+ 'recordid_value',
+ 'records',
+ 'records_array',
+ 'records_map',
+ 'redirect_url',
+ 'reference',
+ 'referer',
+ 'referer_url',
+ 'referrer',
+ 'referrer_url',
+ 'regexp',
+ 'repeating',
+ 'repeating_valueitem',
+ 'repeatingvalueitem',
+ 'repetition',
+ 'req_column',
+ 'req_field',
+ 'required_column',
+ 'required_field',
+ 'response_fileexists',
+ 'response_filepath',
+ 'response_localpath',
+ 'response_path',
+ 'response_realm',
+ 'resultset',
+ 'resultset_count',
+ 'return',
+ 'return_value',
+ 'reverseiterator',
+ 'roman',
+ 'row_count',
+ 'rows',
+ 'rows_array',
+ 'run_children',
+ 'rx',
+ 'schema_name',
+ 'scientific',
+ 'search_args',
+ 'search_arguments',
+ 'search_columnitem',
+ 'search_fielditem',
+ 'search_operatoritem',
+ 'search_opitem',
+ 'search_valueitem',
+ 'searchfielditem',
+ 'searchoperatoritem',
+ 'searchopitem',
+ 'searchvalueitem',
+ 'select',
+ 'selected',
+ 'self',
+ 'serialize',
+ 'series',
+ 'server_date',
+ 'server_day',
+ 'server_ip',
+ 'server_name',
+ 'server_port',
+ 'server_push',
+ 'server_siteisrunning',
+ 'server_sitestart',
+ 'server_sitestop',
+ 'server_time',
+ 'session_abort',
+ 'session_addoutputfilter',
+ 'session_addvar',
+ 'session_addvariable',
+ 'session_deleteexpired',
+ 'session_driver',
+ 'session_end',
+ 'session_id',
+ 'session_removevar',
+ 'session_removevariable',
+ 'session_result',
+ 'session_setdriver',
+ 'session_start',
+ 'set',
+ 'set_iterator',
+ 'set_reverseiterator',
+ 'shown_count',
+ 'shown_first',
+ 'shown_last',
+ 'site_atbegin',
+ 'site_id',
+ 'site_name',
+ 'site_restart',
+ 'skiprecords_value',
+ 'sleep',
+ 'soap_convertpartstopairs',
+ 'soap_definetag',
+ 'soap_info',
+ 'soap_lastrequest',
+ 'soap_lastresponse',
+ 'soap_stub',
+ 'sort_args',
+ 'sort_arguments',
+ 'sort_columnitem',
+ 'sort_fielditem',
+ 'sort_orderitem',
+ 'sortcolumnitem',
+ 'sortfielditem',
+ 'sortorderitem',
+ 'sqlite_createdb',
+ 'sqlite_session_driver',
+ 'sqlite_setsleepmillis',
+ 'sqlite_setsleeptries',
+ 'srand',
+ 'stack',
+ 'stock_quote',
+ 'string',
+ 'string_charfromname',
+ 'string_concatenate',
+ 'string_countfields',
+ 'string_endswith',
+ 'string_extract',
+ 'string_findposition',
+ 'string_findregexp',
+ 'string_fordigit',
+ 'string_getfield',
+ 'string_getunicodeversion',
+ 'string_insert',
+ 'string_isalpha',
+ 'string_isalphanumeric',
+ 'string_isdigit',
+ 'string_ishexdigit',
+ 'string_islower',
+ 'string_isnumeric',
+ 'string_ispunctuation',
+ 'string_isspace',
+ 'string_isupper',
+ 'string_length',
+ 'string_lowercase',
+ 'string_remove',
+ 'string_removeleading',
+ 'string_removetrailing',
+ 'string_replace',
+ 'string_replaceregexp',
+ 'string_todecimal',
+ 'string_tointeger',
+ 'string_uppercase',
+ 'string_validcharset',
+ 'table_name',
+ 'table_realname',
+ 'tag',
+ 'tag_name',
+ 'tags',
+ 'tags_find',
+ 'tags_list',
+ 'tcp_close',
+ 'tcp_open',
+ 'tcp_send',
+ 'tcp_tcp_close',
+ 'tcp_tcp_open',
+ 'tcp_tcp_send',
+ 'thread_abort',
+ 'thread_atomic',
+ 'thread_event',
+ 'thread_exists',
+ 'thread_getcurrentid',
+ 'thread_getpriority',
+ 'thread_info',
+ 'thread_list',
+ 'thread_lock',
+ 'thread_pipe',
+ 'thread_priority_default',
+ 'thread_priority_high',
+ 'thread_priority_low',
+ 'thread_rwlock',
+ 'thread_semaphore',
+ 'thread_setpriority',
+ 'token_value',
+ 'total_records',
+ 'treemap',
+ 'treemap_iterator',
+ 'true',
+ 'url_rewrite',
+ 'valid_creditcard',
+ 'valid_date',
+ 'valid_email',
+ 'valid_url',
+ 'value_list',
+ 'value_listitem',
+ 'valuelistitem',
+ 'var',
+ 'var_defined',
+ 'var_remove',
+ 'var_reset',
+ 'var_set',
+ 'variable',
+ 'variable_defined',
+ 'variable_set',
+ 'variables',
+ 'variant_count',
+ 'vars',
+ 'wap_isenabled',
+ 'wap_maxbuttons',
+ 'wap_maxcolumns',
+ 'wap_maxhorzpixels',
+ 'wap_maxrows',
+ 'wap_maxvertpixels',
+ 'while',
+ 'wsdl_extract',
+ 'wsdl_getbinding',
+ 'wsdl_getbindingforoperation',
+ 'wsdl_getbindingoperations',
+ 'wsdl_getmessagenamed',
+ 'wsdl_getmessageparts',
+ 'wsdl_getmessagetriofromporttype',
+ 'wsdl_getopbodystyle',
+ 'wsdl_getopbodyuse',
+ 'wsdl_getoperation',
+ 'wsdl_getoplocation',
+ 'wsdl_getopmessagetypes',
+ 'wsdl_getopsoapaction',
+ 'wsdl_getportaddress',
+ 'wsdl_getportsforservice',
+ 'wsdl_getporttype',
+ 'wsdl_getporttypeoperation',
+ 'wsdl_getservicedocumentation',
+ 'wsdl_getservices',
+ 'wsdl_gettargetnamespace',
+ 'wsdl_issoapoperation',
+ 'wsdl_listoperations',
+ 'wsdl_maketest',
+ 'xml',
+ 'xml_extract',
+ 'xml_rpc',
+ 'xml_rpccall',
+ 'xml_rw',
+ 'xml_serve',
+ 'xml_transform',
+ 'xml_xml',
+ 'xml_xmlstream',
+ 'xmlstream',
+ 'xsd_attribute',
+ 'xsd_blankarraybase',
+ 'xsd_blankbase',
+ 'xsd_buildtype',
+ 'xsd_cache',
+ 'xsd_checkcardinality',
+ 'xsd_continueall',
+ 'xsd_continueannotation',
+ 'xsd_continueany',
+ 'xsd_continueanyattribute',
+ 'xsd_continueattribute',
+ 'xsd_continueattributegroup',
+ 'xsd_continuechoice',
+ 'xsd_continuecomplexcontent',
+ 'xsd_continuecomplextype',
+ 'xsd_continuedocumentation',
+ 'xsd_continueextension',
+ 'xsd_continuegroup',
+ 'xsd_continuekey',
+ 'xsd_continuelist',
+ 'xsd_continuerestriction',
+ 'xsd_continuesequence',
+ 'xsd_continuesimplecontent',
+ 'xsd_continuesimpletype',
+ 'xsd_continueunion',
+ 'xsd_deserialize',
+ 'xsd_fullyqualifyname',
+ 'xsd_generate',
+ 'xsd_generateblankfromtype',
+ 'xsd_generateblanksimpletype',
+ 'xsd_generatetype',
+ 'xsd_getschematype',
+ 'xsd_issimpletype',
+ 'xsd_loadschema',
+ 'xsd_lookupnamespaceuri',
+ 'xsd_lookuptype',
+ 'xsd_processany',
+ 'xsd_processattribute',
+ 'xsd_processattributegroup',
+ 'xsd_processcomplextype',
+ 'xsd_processelement',
+ 'xsd_processgroup',
+ 'xsd_processimport',
+ 'xsd_processinclude',
+ 'xsd_processschema',
+ 'xsd_processsimpletype',
+ 'xsd_ref',
+ 'xsd_type',
+ )
+}
+MEMBERS = {
+ 'Member Methods': (
+ 'abort',
+ 'abs',
+ 'accept_charset',
+ 'accept',
+ 'acceptconnections',
+ 'acceptdeserializedelement',
+ 'acceptnossl',
+ 'acceptpost',
+ 'accesskey',
+ 'acos',
+ 'acosh',
+ 'action',
+ 'actionparams',
+ 'active_tick',
+ 'add',
+ 'addatend',
+ 'addattachment',
+ 'addbarcode',
+ 'addchapter',
+ 'addcheckbox',
+ 'addcolumninfo',
+ 'addcombobox',
+ 'addcomment',
+ 'addcomponent',
+ 'addcomponents',
+ 'addcss',
+ 'adddatabasetable',
+ 'adddatasource',
+ 'adddatasourcedatabase',
+ 'adddatasourcehost',
+ 'adddir',
+ 'adddirpath',
+ 'addendjs',
+ 'addendjstext',
+ 'adderror',
+ 'addfavicon',
+ 'addfile',
+ 'addgroup',
+ 'addheader',
+ 'addhiddenfield',
+ 'addhtmlpart',
+ 'addimage',
+ 'addjavascript',
+ 'addjs',
+ 'addjstext',
+ 'addlist',
+ 'addmathfunctions',
+ 'addmember',
+ 'addoneheaderline',
+ 'addpage',
+ 'addparagraph',
+ 'addpart',
+ 'addpasswordfield',
+ 'addphrase',
+ 'addpostdispatch',
+ 'addpredispatch',
+ 'addradiobutton',
+ 'addradiogroup',
+ 'addresetbutton',
+ 'addrow',
+ 'addsection',
+ 'addselectlist',
+ 'addset',
+ 'addsubmitbutton',
+ 'addsubnode',
+ 'addtable',
+ 'addtask',
+ 'addtext',
+ 'addtextarea',
+ 'addtextfield',
+ 'addtextpart',
+ 'addtobuffer',
+ 'addtrait',
+ 'adduser',
+ 'addusertogroup',
+ 'addwarning',
+ 'addzip',
+ 'allocobject',
+ 'am',
+ 'ampm',
+ 'annotate',
+ 'answer',
+ 'apop',
+ 'append',
+ 'appendarray',
+ 'appendarraybegin',
+ 'appendarrayend',
+ 'appendbool',
+ 'appendbytes',
+ 'appendchar',
+ 'appendchild',
+ 'appendcolon',
+ 'appendcomma',
+ 'appenddata',
+ 'appenddatetime',
+ 'appenddbpointer',
+ 'appenddecimal',
+ 'appenddocument',
+ 'appendimagetolist',
+ 'appendinteger',
+ 'appendnowutc',
+ 'appendnull',
+ 'appendoid',
+ 'appendregex',
+ 'appendreplacement',
+ 'appendstring',
+ 'appendtail',
+ 'appendtime',
+ 'applyheatcolors',
+ 'appmessage',
+ 'appname',
+ 'appprefix',
+ 'appstatus',
+ 'arc',
+ 'archive',
+ 'arguments',
+ 'argumentvalue',
+ 'asarray',
+ 'asarraystring',
+ 'asasync',
+ 'asbytes',
+ 'ascopy',
+ 'ascopydeep',
+ 'asdecimal',
+ 'asgenerator',
+ 'asin',
+ 'asinh',
+ 'asinteger',
+ 'askeyedgenerator',
+ 'aslazystring',
+ 'aslist',
+ 'asraw',
+ 'asstaticarray',
+ 'asstring',
+ 'asstringhex',
+ 'asstringoct',
+ 'asxml',
+ 'atan',
+ 'atan2',
+ 'atanh',
+ 'atend',
+ 'atends',
+ 'atime',
+ 'attributecount',
+ 'attributes',
+ 'attrs',
+ 'auth',
+ 'authenticate',
+ 'authorize',
+ 'autocollectbuffer',
+ 'average',
+ 'back',
+ 'basename',
+ 'basepaths',
+ 'baseuri',
+ 'bcc',
+ 'beginssl',
+ 'beginswith',
+ 'begintls',
+ 'bestcharset',
+ 'bind_blob',
+ 'bind_double',
+ 'bind_int',
+ 'bind_null',
+ 'bind_parameter_index',
+ 'bind_text',
+ 'bind',
+ 'bindcount',
+ 'bindone',
+ 'bindparam',
+ 'bitand',
+ 'bitclear',
+ 'bitflip',
+ 'bitformat',
+ 'bitnot',
+ 'bitor',
+ 'bitset',
+ 'bitshiftleft',
+ 'bitshiftright',
+ 'bittest',
+ 'bitxor',
+ 'blur',
+ 'body',
+ 'bodybytes',
+ 'boundary',
+ 'bptoxml',
+ 'bptypetostr',
+ 'bucketnumber',
+ 'buff',
+ 'buildquery',
+ 'businessdaysbetween',
+ 'by',
+ 'bytes',
+ 'cachedappprefix',
+ 'cachedroot',
+ 'callboolean',
+ 'callbooleanmethod',
+ 'callbytemethod',
+ 'callcharmethod',
+ 'calldoublemethod',
+ 'calledname',
+ 'callfirst',
+ 'callfloat',
+ 'callfloatmethod',
+ 'callint',
+ 'callintmethod',
+ 'calllongmethod',
+ 'callnonvirtualbooleanmethod',
+ 'callnonvirtualbytemethod',
+ 'callnonvirtualcharmethod',
+ 'callnonvirtualdoublemethod',
+ 'callnonvirtualfloatmethod',
+ 'callnonvirtualintmethod',
+ 'callnonvirtuallongmethod',
+ 'callnonvirtualobjectmethod',
+ 'callnonvirtualshortmethod',
+ 'callnonvirtualvoidmethod',
+ 'callobject',
+ 'callobjectmethod',
+ 'callshortmethod',
+ 'callsite_col',
+ 'callsite_file',
+ 'callsite_line',
+ 'callstack',
+ 'callstaticboolean',
+ 'callstaticbooleanmethod',
+ 'callstaticbytemethod',
+ 'callstaticcharmethod',
+ 'callstaticdoublemethod',
+ 'callstaticfloatmethod',
+ 'callstaticint',
+ 'callstaticintmethod',
+ 'callstaticlongmethod',
+ 'callstaticobject',
+ 'callstaticobjectmethod',
+ 'callstaticshortmethod',
+ 'callstaticstring',
+ 'callstaticvoidmethod',
+ 'callstring',
+ 'callvoid',
+ 'callvoidmethod',
+ 'cancel',
+ 'cap',
+ 'capa',
+ 'capabilities',
+ 'capi',
+ 'cbrt',
+ 'cc',
+ 'ceil',
+ 'chardigitvalue',
+ 'charname',
+ 'charset',
+ 'chartype',
+ 'checkdebugging',
+ 'checked',
+ 'checkuser',
+ 'childnodes',
+ 'chk',
+ 'chmod',
+ 'choosecolumntype',
+ 'chown',
+ 'chunked',
+ 'circle',
+ 'class',
+ 'classid',
+ 'clear',
+ 'clonenode',
+ 'close',
+ 'closepath',
+ 'closeprepared',
+ 'closewrite',
+ 'code',
+ 'codebase',
+ 'codetype',
+ 'colmap',
+ 'colorspace',
+ 'column_blob',
+ 'column_count',
+ 'column_decltype',
+ 'column_double',
+ 'column_int64',
+ 'column_name',
+ 'column_text',
+ 'column_type',
+ 'command',
+ 'comments',
+ 'compare',
+ 'comparecodepointorder',
+ 'componentdelimiter',
+ 'components',
+ 'composite',
+ 'compress',
+ 'concat',
+ 'condtoint',
+ 'configureds',
+ 'configuredskeys',
+ 'connect',
+ 'connection',
+ 'connectionhandler',
+ 'connhandler',
+ 'consume_domain',
+ 'consume_label',
+ 'consume_message',
+ 'consume_rdata',
+ 'consume_string',
+ 'contains',
+ 'content_disposition',
+ 'content_transfer_encoding',
+ 'content_type',
+ 'content',
+ 'contentlength',
+ 'contents',
+ 'contenttype',
+ 'continuation',
+ 'continuationpacket',
+ 'continuationpoint',
+ 'continuationstack',
+ 'continue',
+ 'contrast',
+ 'conventionaltop',
+ 'convert',
+ 'cookie',
+ 'cookies',
+ 'cookiesarray',
+ 'cookiesary',
+ 'copyto',
+ 'cos',
+ 'cosh',
+ 'count',
+ 'countkeys',
+ 'country',
+ 'countusersbygroup',
+ 'crc',
+ 'create',
+ 'createattribute',
+ 'createattributens',
+ 'createcdatasection',
+ 'createcomment',
+ 'createdocument',
+ 'createdocumentfragment',
+ 'createdocumenttype',
+ 'createelement',
+ 'createelementns',
+ 'createentityreference',
+ 'createindex',
+ 'createprocessinginstruction',
+ 'createtable',
+ 'createtextnode',
+ 'criteria',
+ 'crop',
+ 'csscontent',
+ 'curl',
+ 'current',
+ 'currentfile',
+ 'curveto',
+ 'd',
+ 'data',
+ 'databasecolumnnames',
+ 'databasecolumns',
+ 'databasemap',
+ 'databasename',
+ 'datasourcecolumnnames',
+ 'datasourcecolumns',
+ 'datasourcemap',
+ 'date',
+ 'day',
+ 'dayofmonth',
+ 'dayofweek',
+ 'dayofweekinmonth',
+ 'dayofyear',
+ 'days',
+ 'daysbetween',
+ 'db',
+ 'dbtablestable',
+ 'debug',
+ 'declare',
+ 'decodebase64',
+ 'decodehex',
+ 'decodehtml',
+ 'decodeqp',
+ 'decodeurl',
+ 'decodexml',
+ 'decompose',
+ 'decomposeassignment',
+ 'defaultcontentrepresentation',
+ 'defer',
+ 'deg2rad',
+ 'dele',
+ 'delete',
+ 'deletedata',
+ 'deleteglobalref',
+ 'deletelocalref',
+ 'delim',
+ 'depth',
+ 'dereferencepointer',
+ 'describe',
+ 'description',
+ 'deserialize',
+ 'detach',
+ 'detectcharset',
+ 'didinclude',
+ 'difference',
+ 'digit',
+ 'dir',
+ 'displaycountry',
+ 'displaylanguage',
+ 'displayname',
+ 'displayscript',
+ 'displayvariant',
+ 'div',
+ 'dns_response',
+ 'do',
+ 'doatbegins',
+ 'doatends',
+ 'doccomment',
+ 'doclose',
+ 'doctype',
+ 'document',
+ 'documentelement',
+ 'documentroot',
+ 'domainbody',
+ 'done',
+ 'dosessions',
+ 'dowithclose',
+ 'dowlocal',
+ 'download',
+ 'drawtext',
+ 'drop',
+ 'dropindex',
+ 'dsdbtable',
+ 'dshoststable',
+ 'dsinfo',
+ 'dst',
+ 'dstable',
+ 'dstoffset',
+ 'dtdid',
+ 'dup',
+ 'dup2',
+ 'each',
+ 'eachbyte',
+ 'eachcharacter',
+ 'eachchild',
+ 'eachcomponent',
+ 'eachdir',
+ 'eachdirpath',
+ 'eachdirpathrecursive',
+ 'eachentry',
+ 'eachfile',
+ 'eachfilename',
+ 'eachfilepath',
+ 'eachfilepathrecursive',
+ 'eachkey',
+ 'eachline',
+ 'eachlinebreak',
+ 'eachmatch',
+ 'eachnode',
+ 'eachpair',
+ 'eachpath',
+ 'eachpathrecursive',
+ 'eachrow',
+ 'eachsub',
+ 'eachword',
+ 'eachwordbreak',
+ 'element',
+ 'eligiblepath',
+ 'eligiblepaths',
+ 'encodebase64',
+ 'encodehex',
+ 'encodehtml',
+ 'encodehtmltoxml',
+ 'encodemd5',
+ 'encodepassword',
+ 'encodeqp',
+ 'encodesql',
+ 'encodesql92',
+ 'encodeurl',
+ 'encodevalue',
+ 'encodexml',
+ 'encoding',
+ 'enctype',
+ 'end',
+ 'endjs',
+ 'endssl',
+ 'endswith',
+ 'endtls',
+ 'enhance',
+ 'ensurestopped',
+ 'entities',
+ 'entry',
+ 'env',
+ 'equals',
+ 'era',
+ 'erf',
+ 'erfc',
+ 'err',
+ 'errcode',
+ 'errmsg',
+ 'error',
+ 'errors',
+ 'errstack',
+ 'escape_member',
+ 'establisherrorstate',
+ 'exceptioncheck',
+ 'exceptionclear',
+ 'exceptiondescribe',
+ 'exceptionoccurred',
+ 'exchange',
+ 'execinits',
+ 'execinstalls',
+ 'execute',
+ 'executelazy',
+ 'executenow',
+ 'exists',
+ 'exit',
+ 'exitcode',
+ 'exp',
+ 'expire',
+ 'expireminutes',
+ 'expiresminutes',
+ 'expm1',
+ 'export16bits',
+ 'export32bits',
+ 'export64bits',
+ 'export8bits',
+ 'exportas',
+ 'exportbytes',
+ 'exportfdf',
+ 'exportpointerbits',
+ 'exportsigned16bits',
+ 'exportsigned32bits',
+ 'exportsigned64bits',
+ 'exportsigned8bits',
+ 'exportstring',
+ 'expose',
+ 'extendedyear',
+ 'extensiondelimiter',
+ 'extensions',
+ 'extract',
+ 'extractfast',
+ 'extractfastone',
+ 'extractimage',
+ 'extractone',
+ 'f',
+ 'fabs',
+ 'fail',
+ 'failnoconnectionhandler',
+ 'family',
+ 'fatalerror',
+ 'fcgireq',
+ 'fchdir',
+ 'fchmod',
+ 'fchown',
+ 'fd',
+ 'features',
+ 'fetchdata',
+ 'fieldnames',
+ 'fieldposition',
+ 'fieldstable',
+ 'fieldtype',
+ 'fieldvalue',
+ 'file',
+ 'filename',
+ 'filenames',
+ 'filequeue',
+ 'fileuploads',
+ 'fileuploadsary',
+ 'filterinputcolumn',
+ 'finalize',
+ 'find',
+ 'findall',
+ 'findandmodify',
+ 'findbucket',
+ 'findcase',
+ 'findclass',
+ 'findcount',
+ 'finddescendant',
+ 'findfirst',
+ 'findinclude',
+ 'findinctx',
+ 'findindex',
+ 'findlast',
+ 'findpattern',
+ 'findposition',
+ 'findsymbols',
+ 'first',
+ 'firstchild',
+ 'firstcomponent',
+ 'firstdayofweek',
+ 'firstnode',
+ 'fixformat',
+ 'flags',
+ 'fliph',
+ 'flipv',
+ 'floor',
+ 'flush',
+ 'foldcase',
+ 'foo',
+ 'for',
+ 'forcedrowid',
+ 'foreach',
+ 'foreachaccept',
+ 'foreachbyte',
+ 'foreachcharacter',
+ 'foreachchild',
+ 'foreachday',
+ 'foreachentry',
+ 'foreachfile',
+ 'foreachfilename',
+ 'foreachkey',
+ 'foreachline',
+ 'foreachlinebreak',
+ 'foreachmatch',
+ 'foreachnode',
+ 'foreachpair',
+ 'foreachpathcomponent',
+ 'foreachrow',
+ 'foreachspool',
+ 'foreachsub',
+ 'foreachwordbreak',
+ 'form',
+ 'format',
+ 'formatas',
+ 'formatcontextelement',
+ 'formatcontextelements',
+ 'formatnumber',
+ 'free',
+ 'frexp',
+ 'from',
+ 'fromname',
+ 'fromport',
+ 'fromreflectedfield',
+ 'fromreflectedmethod',
+ 'front',
+ 'fsync',
+ 'ftpdeletefile',
+ 'ftpgetlisting',
+ 'ftruncate',
+ 'fullpath',
+ 'fx',
+ 'gamma',
+ 'gatewayinterface',
+ 'gen',
+ 'generatechecksum',
+ 'get',
+ 'getabswidth',
+ 'getalignment',
+ 'getappsource',
+ 'getarraylength',
+ 'getattr',
+ 'getattribute',
+ 'getattributenamespace',
+ 'getattributenode',
+ 'getattributenodens',
+ 'getattributens',
+ 'getbarheight',
+ 'getbarmultiplier',
+ 'getbarwidth',
+ 'getbaseline',
+ 'getbold',
+ 'getbooleanarrayelements',
+ 'getbooleanarrayregion',
+ 'getbooleanfield',
+ 'getbordercolor',
+ 'getborderwidth',
+ 'getbytearrayelements',
+ 'getbytearrayregion',
+ 'getbytefield',
+ 'getchararrayelements',
+ 'getchararrayregion',
+ 'getcharfield',
+ 'getclass',
+ 'getcode',
+ 'getcolor',
+ 'getcolumn',
+ 'getcolumncount',
+ 'getcolumns',
+ 'getdatabasebyalias',
+ 'getdatabasebyid',
+ 'getdatabasebyname',
+ 'getdatabasehost',
+ 'getdatabasetable',
+ 'getdatabasetablebyalias',
+ 'getdatabasetablebyid',
+ 'getdatabasetablepart',
+ 'getdatasource',
+ 'getdatasourcedatabase',
+ 'getdatasourcedatabasebyid',
+ 'getdatasourcehost',
+ 'getdatasourceid',
+ 'getdatasourcename',
+ 'getdefaultstorage',
+ 'getdoublearrayelements',
+ 'getdoublearrayregion',
+ 'getdoublefield',
+ 'getelementbyid',
+ 'getelementsbytagname',
+ 'getelementsbytagnamens',
+ 'getencoding',
+ 'getface',
+ 'getfield',
+ 'getfieldid',
+ 'getfile',
+ 'getfloatarrayelements',
+ 'getfloatarrayregion',
+ 'getfloatfield',
+ 'getfont',
+ 'getformat',
+ 'getfullfontname',
+ 'getgroup',
+ 'getgroupid',
+ 'getheader',
+ 'getheaders',
+ 'gethostdatabase',
+ 'gethtmlattr',
+ 'gethtmlattrstring',
+ 'getinclude',
+ 'getintarrayelements',
+ 'getintarrayregion',
+ 'getintfield',
+ 'getisocomment',
+ 'getitalic',
+ 'getlasterror',
+ 'getlcapitype',
+ 'getlibrary',
+ 'getlongarrayelements',
+ 'getlongarrayregion',
+ 'getlongfield',
+ 'getmargins',
+ 'getmethodid',
+ 'getmode',
+ 'getnameditem',
+ 'getnameditemns',
+ 'getnode',
+ 'getnumericvalue',
+ 'getobjectarrayelement',
+ 'getobjectclass',
+ 'getobjectfield',
+ 'getpadding',
+ 'getpagenumber',
+ 'getparts',
+ 'getprefs',
+ 'getpropertyvalue',
+ 'getprowcount',
+ 'getpsfontname',
+ 'getrange',
+ 'getrowcount',
+ 'getset',
+ 'getshortarrayelements',
+ 'getshortarrayregion',
+ 'getshortfield',
+ 'getsize',
+ 'getsortfieldspart',
+ 'getspacing',
+ 'getstaticbooleanfield',
+ 'getstaticbytefield',
+ 'getstaticcharfield',
+ 'getstaticdoublefield',
+ 'getstaticfieldid',
+ 'getstaticfloatfield',
+ 'getstaticintfield',
+ 'getstaticlongfield',
+ 'getstaticmethodid',
+ 'getstaticobjectfield',
+ 'getstaticshortfield',
+ 'getstatus',
+ 'getstringchars',
+ 'getstringlength',
+ 'getstyle',
+ 'getsupportedencodings',
+ 'gettablebyid',
+ 'gettext',
+ 'gettextalignment',
+ 'gettextsize',
+ 'gettrigger',
+ 'gettype',
+ 'getunderline',
+ 'getuniquealiasname',
+ 'getuser',
+ 'getuserbykey',
+ 'getuserid',
+ 'getversion',
+ 'getzipfilebytes',
+ 'givenblock',
+ 'gmt',
+ 'gotconnection',
+ 'gotfileupload',
+ 'groupby',
+ 'groupcolumns',
+ 'groupcount',
+ 'groupjoin',
+ 'handlebreakpointget',
+ 'handlebreakpointlist',
+ 'handlebreakpointremove',
+ 'handlebreakpointset',
+ 'handlebreakpointupdate',
+ 'handlecontextget',
+ 'handlecontextnames',
+ 'handlecontinuation',
+ 'handledefinitionbody',
+ 'handledefinitionhead',
+ 'handledefinitionresource',
+ 'handledevconnection',
+ 'handleevalexpired',
+ 'handlefeatureget',
+ 'handlefeatureset',
+ 'handlelassoappcontent',
+ 'handlelassoappresponse',
+ 'handlenested',
+ 'handlenormalconnection',
+ 'handlepop',
+ 'handleresource',
+ 'handlesource',
+ 'handlestackget',
+ 'handlestderr',
+ 'handlestdin',
+ 'handlestdout',
+ 'handshake',
+ 'hasattribute',
+ 'hasattributens',
+ 'hasattributes',
+ 'hasbinaryproperty',
+ 'haschildnodes',
+ 'hasexpired',
+ 'hasfeature',
+ 'hasfield',
+ 'hash',
+ 'hashtmlattr',
+ 'hasmethod',
+ 'hastable',
+ 'hastrailingcomponent',
+ 'hasvalue',
+ 'head',
+ 'header',
+ 'headerbytes',
+ 'headers',
+ 'headersarray',
+ 'headersmap',
+ 'height',
+ 'histogram',
+ 'home',
+ 'host',
+ 'hostcolumnnames',
+ 'hostcolumnnames2',
+ 'hostcolumns',
+ 'hostcolumns2',
+ 'hostdatasource',
+ 'hostextra',
+ 'hostid',
+ 'hostisdynamic',
+ 'hostmap',
+ 'hostmap2',
+ 'hostname',
+ 'hostpassword',
+ 'hostport',
+ 'hostschema',
+ 'hosttableencoding',
+ 'hosttonet16',
+ 'hosttonet32',
+ 'hosttonet64',
+ 'hostusername',
+ 'hour',
+ 'hourofampm',
+ 'hourofday',
+ 'hoursbetween',
+ 'href',
+ 'hreflang',
+ 'htmlcontent',
+ 'htmlizestacktrace',
+ 'htmlizestacktracelink',
+ 'httpaccept',
+ 'httpacceptencoding',
+ 'httpacceptlanguage',
+ 'httpauthorization',
+ 'httpcachecontrol',
+ 'httpconnection',
+ 'httpcookie',
+ 'httpequiv',
+ 'httphost',
+ 'httpreferer',
+ 'httpreferrer',
+ 'httpuseragent',
+ 'hypot',
+ 'id',
+ 'idealinmemory',
+ 'idle',
+ 'idmap',
+ 'ifempty',
+ 'ifkey',
+ 'ifnotempty',
+ 'ifnotkey',
+ 'ignorecase',
+ 'ilogb',
+ 'imgptr',
+ 'implementation',
+ 'import16bits',
+ 'import32bits',
+ 'import64bits',
+ 'import8bits',
+ 'importas',
+ 'importbytes',
+ 'importfdf',
+ 'importnode',
+ 'importpointer',
+ 'importstring',
+ 'in',
+ 'include',
+ 'includebytes',
+ 'includelibrary',
+ 'includelibraryonce',
+ 'includeonce',
+ 'includes',
+ 'includestack',
+ 'indaylighttime',
+ 'index',
+ 'init',
+ 'initialize',
+ 'initrequest',
+ 'inits',
+ 'inneroncompare',
+ 'input',
+ 'inputcolumns',
+ 'inputtype',
+ 'insert',
+ 'insertback',
+ 'insertbefore',
+ 'insertdata',
+ 'insertfirst',
+ 'insertfrom',
+ 'insertfront',
+ 'insertinternal',
+ 'insertlast',
+ 'insertpage',
+ 'install',
+ 'installs',
+ 'integer',
+ 'internalsubset',
+ 'interrupt',
+ 'intersection',
+ 'inttocond',
+ 'invoke',
+ 'invokeautocollect',
+ 'invokeuntil',
+ 'invokewhile',
+ 'ioctl',
+ 'isa',
+ 'isalive',
+ 'isallof',
+ 'isalnum',
+ 'isalpha',
+ 'isanyof',
+ 'isbase',
+ 'isblank',
+ 'iscntrl',
+ 'isdigit',
+ 'isdir',
+ 'isdirectory',
+ 'isempty',
+ 'isemptyelement',
+ 'isfirststep',
+ 'isfullpath',
+ 'isgraph',
+ 'ishttps',
+ 'isidle',
+ 'isinstanceof',
+ 'islink',
+ 'islower',
+ 'ismultipart',
+ 'isnan',
+ 'isnota',
+ 'isnotempty',
+ 'isnothing',
+ 'iso3country',
+ 'iso3language',
+ 'isopen',
+ 'isprint',
+ 'ispunct',
+ 'issameobject',
+ 'isset',
+ 'issourcefile',
+ 'isspace',
+ 'isssl',
+ 'issupported',
+ 'istitle',
+ 'istruetype',
+ 'istype',
+ 'isualphabetic',
+ 'isulowercase',
+ 'isupper',
+ 'isuuppercase',
+ 'isuwhitespace',
+ 'isvalid',
+ 'iswhitespace',
+ 'isxdigit',
+ 'isxhr',
+ 'item',
+ 'j0',
+ 'j1',
+ 'javascript',
+ 'jbarcode',
+ 'jcolor',
+ 'jfont',
+ 'jimage',
+ 'jlist',
+ 'jn',
+ 'jobjectisa',
+ 'join',
+ 'jread',
+ 'jscontent',
+ 'jsonfornode',
+ 'jsonhtml',
+ 'jsonisleaf',
+ 'jsonlabel',
+ 'jtable',
+ 'jtext',
+ 'julianday',
+ 'kernel',
+ 'key',
+ 'keycolumns',
+ 'keys',
+ 'keywords',
+ 'kill',
+ 'label',
+ 'lang',
+ 'language',
+ 'last_insert_rowid',
+ 'last',
+ 'lastaccessdate',
+ 'lastaccesstime',
+ 'lastchild',
+ 'lastcomponent',
+ 'lasterror',
+ 'lastinsertid',
+ 'lastnode',
+ 'lastpoint',
+ 'lasttouched',
+ 'lazyvalue',
+ 'ldexp',
+ 'leaveopen',
+ 'left',
+ 'length',
+ 'lgamma',
+ 'line',
+ 'linediffers',
+ 'linkto',
+ 'linktype',
+ 'list',
+ 'listactivedatasources',
+ 'listalldatabases',
+ 'listalltables',
+ 'listdatabasetables',
+ 'listdatasourcedatabases',
+ 'listdatasourcehosts',
+ 'listdatasources',
+ 'listen',
+ 'listgroups',
+ 'listgroupsbyuser',
+ 'listhostdatabases',
+ 'listhosts',
+ 'listmethods',
+ 'listnode',
+ 'listusers',
+ 'listusersbygroup',
+ 'loadcerts',
+ 'loaddatasourcehostinfo',
+ 'loaddatasourceinfo',
+ 'loadlibrary',
+ 'localaddress',
+ 'localname',
+ 'locals',
+ 'lock',
+ 'log',
+ 'log10',
+ 'log1p',
+ 'logb',
+ 'lookupnamespace',
+ 'lop',
+ 'lowagiefont',
+ 'lowercase',
+ 'makecolor',
+ 'makecolumnlist',
+ 'makecolumnmap',
+ 'makecookieyumyum',
+ 'makefullpath',
+ 'makeinheritedcopy',
+ 'makenonrelative',
+ 'makeurl',
+ 'map',
+ 'marker',
+ 'matches',
+ 'matchesstart',
+ 'matchposition',
+ 'matchstring',
+ 'matchtriggers',
+ 'max',
+ 'maxinmemory',
+ 'maxlength',
+ 'maxrows',
+ 'maxworkers',
+ 'maybeslash',
+ 'maybevalue',
+ 'md5hex',
+ 'media',
+ 'members',
+ 'merge',
+ 'meta',
+ 'method',
+ 'methodname',
+ 'millisecond',
+ 'millisecondsinday',
+ 'mime_boundary',
+ 'mime_contenttype',
+ 'mime_hdrs',
+ 'mime',
+ 'mimes',
+ 'min',
+ 'minute',
+ 'minutesbetween',
+ 'moddatestr',
+ 'mode',
+ 'modf',
+ 'modificationdate',
+ 'modificationtime',
+ 'modulate',
+ 'monitorenter',
+ 'monitorexit',
+ 'month',
+ 'moveto',
+ 'movetoattribute',
+ 'movetoattributenamespace',
+ 'movetoelement',
+ 'movetofirstattribute',
+ 'movetonextattribute',
+ 'msg',
+ 'mtime',
+ 'multiple',
+ 'n',
+ 'name',
+ 'named',
+ 'namespaceuri',
+ 'needinitialization',
+ 'net',
+ 'nettohost16',
+ 'nettohost32',
+ 'nettohost64',
+ 'new',
+ 'newbooleanarray',
+ 'newbytearray',
+ 'newchararray',
+ 'newdoublearray',
+ 'newfloatarray',
+ 'newglobalref',
+ 'newintarray',
+ 'newlongarray',
+ 'newobject',
+ 'newobjectarray',
+ 'newshortarray',
+ 'newstring',
+ 'next',
+ 'nextafter',
+ 'nextnode',
+ 'nextprime',
+ 'nextprune',
+ 'nextprunedelta',
+ 'nextsibling',
+ 'nodeforpath',
+ 'nodelist',
+ 'nodename',
+ 'nodetype',
+ 'nodevalue',
+ 'noop',
+ 'normalize',
+ 'notationname',
+ 'notations',
+ 'novaluelists',
+ 'numsets',
+ 'object',
+ 'objects',
+ 'objecttype',
+ 'onclick',
+ 'oncompare',
+ 'oncomparestrict',
+ 'onconvert',
+ 'oncreate',
+ 'ondblclick',
+ 'onkeydown',
+ 'onkeypress',
+ 'onkeyup',
+ 'onmousedown',
+ 'onmousemove',
+ 'onmouseout',
+ 'onmouseover',
+ 'onmouseup',
+ 'onreset',
+ 'onsubmit',
+ 'ontop',
+ 'open',
+ 'openappend',
+ 'openread',
+ 'opentruncate',
+ 'openwith',
+ 'openwrite',
+ 'openwriteonly',
+ 'orderby',
+ 'orderbydescending',
+ 'out',
+ 'output',
+ 'outputencoding',
+ 'ownerdocument',
+ 'ownerelement',
+ 'padleading',
+ 'padtrailing',
+ 'padzero',
+ 'pagecount',
+ 'pagerotation',
+ 'pagesize',
+ 'param',
+ 'paramdescs',
+ 'params',
+ 'parent',
+ 'parentdir',
+ 'parentnode',
+ 'parse_body',
+ 'parse_boundary',
+ 'parse_charset',
+ 'parse_content_disposition',
+ 'parse_content_transfer_encoding',
+ 'parse_content_type',
+ 'parse_hdrs',
+ 'parse_mode',
+ 'parse_msg',
+ 'parse_parts',
+ 'parse_rawhdrs',
+ 'parse',
+ 'parseas',
+ 'parsedocument',
+ 'parsenumber',
+ 'parseoneheaderline',
+ 'pass',
+ 'path',
+ 'pathinfo',
+ 'pathtouri',
+ 'pathtranslated',
+ 'pause',
+ 'payload',
+ 'pdifference',
+ 'perform',
+ 'performonce',
+ 'perms',
+ 'pid',
+ 'pixel',
+ 'pm',
+ 'polldbg',
+ 'pollide',
+ 'pop_capa',
+ 'pop_cmd',
+ 'pop_debug',
+ 'pop_err',
+ 'pop_get',
+ 'pop_ids',
+ 'pop_index',
+ 'pop_log',
+ 'pop_mode',
+ 'pop_net',
+ 'pop_res',
+ 'pop_server',
+ 'pop_timeout',
+ 'pop_token',
+ 'pop',
+ 'popctx',
+ 'popinclude',
+ 'populate',
+ 'port',
+ 'position',
+ 'postdispatch',
+ 'postparam',
+ 'postparams',
+ 'postparamsary',
+ 'poststring',
+ 'pow',
+ 'predispatch',
+ 'prefix',
+ 'preflight',
+ 'prepare',
+ 'prepared',
+ 'pretty',
+ 'prev',
+ 'previoussibling',
+ 'printsimplemsg',
+ 'private_compare',
+ 'private_find',
+ 'private_findlast',
+ 'private_merge',
+ 'private_rebalanceforinsert',
+ 'private_rebalanceforremove',
+ 'private_replaceall',
+ 'private_replacefirst',
+ 'private_rotateleft',
+ 'private_rotateright',
+ 'private_setrange',
+ 'private_split',
+ 'probemimetype',
+ 'provides',
+ 'proxying',
+ 'prune',
+ 'publicid',
+ 'pullhttpheader',
+ 'pullmimepost',
+ 'pulloneheaderline',
+ 'pullpost',
+ 'pullrawpost',
+ 'pullrawpostchunks',
+ 'pullrequest',
+ 'pullrequestline',
+ 'push',
+ 'pushctx',
+ 'pushinclude',
+ 'qdarray',
+ 'qdcount',
+ 'queryparam',
+ 'queryparams',
+ 'queryparamsary',
+ 'querystring',
+ 'queue_maintenance',
+ 'queue_messages',
+ 'queue_status',
+ 'queue',
+ 'quit',
+ 'r',
+ 'raw',
+ 'rawcontent',
+ 'rawdiff',
+ 'rawheader',
+ 'rawheaders',
+ 'rawinvokable',
+ 'read',
+ 'readattributevalue',
+ 'readbytes',
+ 'readbytesfully',
+ 'readdestinations',
+ 'readerror',
+ 'readidobjects',
+ 'readline',
+ 'readmessage',
+ 'readnumber',
+ 'readobject',
+ 'readobjecttcp',
+ 'readpacket',
+ 'readsomebytes',
+ 'readstring',
+ 'ready',
+ 'realdoc',
+ 'realpath',
+ 'receivefd',
+ 'recipients',
+ 'recover',
+ 'rect',
+ 'rectype',
+ 'red',
+ 'redirectto',
+ 'referrals',
+ 'refid',
+ 'refobj',
+ 'refresh',
+ 'rel',
+ 'remainder',
+ 'remoteaddr',
+ 'remoteaddress',
+ 'remoteport',
+ 'remove',
+ 'removeall',
+ 'removeattribute',
+ 'removeattributenode',
+ 'removeattributens',
+ 'removeback',
+ 'removechild',
+ 'removedatabasetable',
+ 'removedatasource',
+ 'removedatasourcedatabase',
+ 'removedatasourcehost',
+ 'removefield',
+ 'removefirst',
+ 'removefront',
+ 'removegroup',
+ 'removelast',
+ 'removeleading',
+ 'removenameditem',
+ 'removenameditemns',
+ 'removenode',
+ 'removesubnode',
+ 'removetrailing',
+ 'removeuser',
+ 'removeuserfromallgroups',
+ 'removeuserfromgroup',
+ 'rename',
+ 'renderbytes',
+ 'renderdocumentbytes',
+ 'renderstring',
+ 'replace',
+ 'replaceall',
+ 'replacechild',
+ 'replacedata',
+ 'replacefirst',
+ 'replaceheader',
+ 'replacepattern',
+ 'representnode',
+ 'representnoderesult',
+ 'reqid',
+ 'requestid',
+ 'requestmethod',
+ 'requestparams',
+ 'requesturi',
+ 'requires',
+ 'reserve',
+ 'reset',
+ 'resize',
+ 'resolutionh',
+ 'resolutionv',
+ 'resolvelinks',
+ 'resourcedata',
+ 'resourceinvokable',
+ 'resourcename',
+ 'resources',
+ 'respond',
+ 'restart',
+ 'restname',
+ 'result',
+ 'results',
+ 'resume',
+ 'retr',
+ 'retrieve',
+ 'returncolumns',
+ 'returntype',
+ 'rev',
+ 'reverse',
+ 'rewind',
+ 'right',
+ 'rint',
+ 'roll',
+ 'root',
+ 'rootmap',
+ 'rotate',
+ 'route',
+ 'rowsfound',
+ 'rset',
+ 'rule',
+ 'rules',
+ 'run',
+ 'running',
+ 'runonce',
+ 's',
+ 'sa',
+ 'safeexport8bits',
+ 'sameas',
+ 'save',
+ 'savedata',
+ 'scalb',
+ 'scale',
+ 'scanfordatasource',
+ 'scantasks',
+ 'scanworkers',
+ 'schemaname',
+ 'scheme',
+ 'script',
+ 'scriptextensions',
+ 'scriptfilename',
+ 'scriptname',
+ 'scripttype',
+ 'scripturi',
+ 'scripturl',
+ 'scrubkeywords',
+ 'search',
+ 'searchinbucket',
+ 'searchurl',
+ 'second',
+ 'secondsbetween',
+ 'seek',
+ 'select',
+ 'selected',
+ 'selectmany',
+ 'self',
+ 'send',
+ 'sendchunk',
+ 'sendfd',
+ 'sendfile',
+ 'sendpacket',
+ 'sendresponse',
+ 'separator',
+ 'serializationelements',
+ 'serialize',
+ 'serveraddr',
+ 'serveradmin',
+ 'servername',
+ 'serverport',
+ 'serverprotocol',
+ 'serversignature',
+ 'serversoftware',
+ 'sessionsdump',
+ 'sessionsmap',
+ 'set',
+ 'setalignment',
+ 'setattr',
+ 'setattribute',
+ 'setattributenode',
+ 'setattributenodens',
+ 'setattributens',
+ 'setbarheight',
+ 'setbarmultiplier',
+ 'setbarwidth',
+ 'setbaseline',
+ 'setbold',
+ 'setbooleanarrayregion',
+ 'setbooleanfield',
+ 'setbordercolor',
+ 'setborderwidth',
+ 'setbytearrayregion',
+ 'setbytefield',
+ 'setchararrayregion',
+ 'setcharfield',
+ 'setcode',
+ 'setcolor',
+ 'setcolorspace',
+ 'setcookie',
+ 'setcwd',
+ 'setdefaultstorage',
+ 'setdestination',
+ 'setdoublearrayregion',
+ 'setdoublefield',
+ 'setencoding',
+ 'setface',
+ 'setfieldvalue',
+ 'setfindpattern',
+ 'setfloatarrayregion',
+ 'setfloatfield',
+ 'setfont',
+ 'setformat',
+ 'setgeneratechecksum',
+ 'setheaders',
+ 'sethtmlattr',
+ 'setignorecase',
+ 'setinput',
+ 'setintarrayregion',
+ 'setintfield',
+ 'setitalic',
+ 'setlinewidth',
+ 'setlongarrayregion',
+ 'setlongfield',
+ 'setmarker',
+ 'setmaxfilesize',
+ 'setmode',
+ 'setname',
+ 'setnameditem',
+ 'setnameditemns',
+ 'setobjectarrayelement',
+ 'setobjectfield',
+ 'setpadding',
+ 'setpagenumber',
+ 'setpagerange',
+ 'setposition',
+ 'setrange',
+ 'setreplacepattern',
+ 'setshortarrayregion',
+ 'setshortfield',
+ 'setshowchecksum',
+ 'setsize',
+ 'setspacing',
+ 'setstaticbooleanfield',
+ 'setstaticbytefield',
+ 'setstaticcharfield',
+ 'setstaticdoublefield',
+ 'setstaticfloatfield',
+ 'setstaticintfield',
+ 'setstaticlongfield',
+ 'setstaticobjectfield',
+ 'setstaticshortfield',
+ 'setstatus',
+ 'settextalignment',
+ 'settextsize',
+ 'settimezone',
+ 'settrait',
+ 'setunderline',
+ 'sharpen',
+ 'shouldabort',
+ 'shouldclose',
+ 'showchecksum',
+ 'showcode39startstop',
+ 'showeanguardbars',
+ 'shutdownrd',
+ 'shutdownrdwr',
+ 'shutdownwr',
+ 'sin',
+ 'sinh',
+ 'size',
+ 'skip',
+ 'skiprows',
+ 'sort',
+ 'sortcolumns',
+ 'source',
+ 'sourcecolumn',
+ 'sourcefile',
+ 'sourceline',
+ 'specified',
+ 'split',
+ 'splitconnection',
+ 'splitdebuggingthread',
+ 'splitextension',
+ 'splittext',
+ 'splitthread',
+ 'splittoprivatedev',
+ 'splituppath',
+ 'sql',
+ 'sqlite3',
+ 'sqrt',
+ 'src',
+ 'srcpath',
+ 'sslerrfail',
+ 'stack',
+ 'standby',
+ 'start',
+ 'startone',
+ 'startup',
+ 'stat',
+ 'statement',
+ 'statementonly',
+ 'stats',
+ 'status',
+ 'statuscode',
+ 'statusmsg',
+ 'stdin',
+ 'step',
+ 'stls',
+ 'stop',
+ 'stoprunning',
+ 'storedata',
+ 'stripfirstcomponent',
+ 'striplastcomponent',
+ 'style',
+ 'styletype',
+ 'sub',
+ 'subject',
+ 'subnode',
+ 'subnodes',
+ 'substringdata',
+ 'subtract',
+ 'subtraits',
+ 'sum',
+ 'supportscontentrepresentation',
+ 'swapbytes',
+ 'systemid',
+ 't',
+ 'tabindex',
+ 'table',
+ 'tablecolumnnames',
+ 'tablecolumns',
+ 'tablehascolumn',
+ 'tableizestacktrace',
+ 'tableizestacktracelink',
+ 'tablemap',
+ 'tablename',
+ 'tables',
+ 'tabs',
+ 'tabstr',
+ 'tag',
+ 'tagname',
+ 'take',
+ 'tan',
+ 'tanh',
+ 'target',
+ 'tasks',
+ 'tb',
+ 'tell',
+ 'testexitcode',
+ 'testlock',
+ 'textwidth',
+ 'thenby',
+ 'thenbydescending',
+ 'threadreaddesc',
+ 'throw',
+ 'thrownew',
+ 'time',
+ 'timezone',
+ 'title',
+ 'titlecase',
+ 'to',
+ 'token',
+ 'tolower',
+ 'top',
+ 'toreflectedfield',
+ 'toreflectedmethod',
+ 'total_changes',
+ 'totitle',
+ 'touch',
+ 'toupper',
+ 'toxmlstring',
+ 'trace',
+ 'trackingid',
+ 'trait',
+ 'transform',
+ 'trigger',
+ 'trim',
+ 'trunk',
+ 'tryfinderrorfile',
+ 'trylock',
+ 'tryreadobject',
+ 'type',
+ 'typename',
+ 'uidl',
+ 'uncompress',
+ 'unescape',
+ 'union',
+ 'uniqueid',
+ 'unlock',
+ 'unspool',
+ 'up',
+ 'update',
+ 'updategroup',
+ 'upload',
+ 'uppercase',
+ 'url',
+ 'used',
+ 'usemap',
+ 'user',
+ 'usercolumns',
+ 'valid',
+ 'validate',
+ 'validatesessionstable',
+ 'value',
+ 'values',
+ 'valuetype',
+ 'variant',
+ 'version',
+ 'wait',
+ 'waitforcompletion',
+ 'warnings',
+ 'week',
+ 'weekofmonth',
+ 'weekofyear',
+ 'where',
+ 'width',
+ 'workers',
+ 'workinginputcolumns',
+ 'workingkeycolumns',
+ 'workingkeyfield_name',
+ 'workingreturncolumns',
+ 'workingsortcolumns',
+ 'write',
+ 'writebodybytes',
+ 'writebytes',
+ 'writeheader',
+ 'writeheaderbytes',
+ 'writeheaderline',
+ 'writeid',
+ 'writemessage',
+ 'writeobject',
+ 'writeobjecttcp',
+ 'writestring',
+ 'wroteheaders',
+ 'xhtml',
+ 'xmllang',
+ 'y0',
+ 'y1',
+ 'year',
+ 'yearwoy',
+ 'yn',
+ 'z',
+ 'zip',
+ 'zipfile',
+ 'zipfilename',
+ 'zipname',
+ 'zips',
+ 'zoneoffset',
+ ),
+ 'Lasso 8 Member Tags': (
+ 'accept',
+ 'add',
+ 'addattachment',
+ 'addattribute',
+ 'addbarcode',
+ 'addchapter',
+ 'addcheckbox',
+ 'addchild',
+ 'addcombobox',
+ 'addcomment',
+ 'addcontent',
+ 'addhiddenfield',
+ 'addhtmlpart',
+ 'addimage',
+ 'addjavascript',
+ 'addlist',
+ 'addnamespace',
+ 'addnextsibling',
+ 'addpage',
+ 'addparagraph',
+ 'addparenttype',
+ 'addpart',
+ 'addpasswordfield',
+ 'addphrase',
+ 'addprevsibling',
+ 'addradiobutton',
+ 'addradiogroup',
+ 'addresetbutton',
+ 'addsection',
+ 'addselectlist',
+ 'addsibling',
+ 'addsubmitbutton',
+ 'addtable',
+ 'addtext',
+ 'addtextarea',
+ 'addtextfield',
+ 'addtextpart',
+ 'alarms',
+ 'annotate',
+ 'answer',
+ 'append',
+ 'appendreplacement',
+ 'appendtail',
+ 'arc',
+ 'asasync',
+ 'astype',
+ 'atbegin',
+ 'atbottom',
+ 'atend',
+ 'atfarleft',
+ 'atfarright',
+ 'attop',
+ 'attributecount',
+ 'attributes',
+ 'authenticate',
+ 'authorize',
+ 'backward',
+ 'baseuri',
+ 'bcc',
+ 'beanproperties',
+ 'beginswith',
+ 'bind',
+ 'bitand',
+ 'bitclear',
+ 'bitflip',
+ 'bitformat',
+ 'bitnot',
+ 'bitor',
+ 'bitset',
+ 'bitshiftleft',
+ 'bitshiftright',
+ 'bittest',
+ 'bitxor',
+ 'blur',
+ 'body',
+ 'boundary',
+ 'bytes',
+ 'call',
+ 'cancel',
+ 'capabilities',
+ 'cc',
+ 'chardigitvalue',
+ 'charname',
+ 'charset',
+ 'chartype',
+ 'children',
+ 'circle',
+ 'close',
+ 'closepath',
+ 'closewrite',
+ 'code',
+ 'colorspace',
+ 'command',
+ 'comments',
+ 'compare',
+ 'comparecodepointorder',
+ 'compile',
+ 'composite',
+ 'connect',
+ 'contains',
+ 'content_disposition',
+ 'content_transfer_encoding',
+ 'content_type',
+ 'contents',
+ 'contrast',
+ 'convert',
+ 'crop',
+ 'curveto',
+ 'data',
+ 'date',
+ 'day',
+ 'daylights',
+ 'dayofweek',
+ 'dayofyear',
+ 'decrement',
+ 'delete',
+ 'depth',
+ 'describe',
+ 'description',
+ 'deserialize',
+ 'detach',
+ 'detachreference',
+ 'difference',
+ 'digit',
+ 'document',
+ 'down',
+ 'drawtext',
+ 'dst',
+ 'dump',
+ 'endswith',
+ 'enhance',
+ 'equals',
+ 'errors',
+ 'eval',
+ 'events',
+ 'execute',
+ 'export16bits',
+ 'export32bits',
+ 'export64bits',
+ 'export8bits',
+ 'exportfdf',
+ 'exportstring',
+ 'extract',
+ 'extractone',
+ 'fieldnames',
+ 'fieldtype',
+ 'fieldvalue',
+ 'file',
+ 'find',
+ 'findindex',
+ 'findnamespace',
+ 'findnamespacebyhref',
+ 'findpattern',
+ 'findposition',
+ 'first',
+ 'firstchild',
+ 'fliph',
+ 'flipv',
+ 'flush',
+ 'foldcase',
+ 'foreach',
+ 'format',
+ 'forward',
+ 'freebusies',
+ 'freezetype',
+ 'freezevalue',
+ 'from',
+ 'fulltype',
+ 'generatechecksum',
+ 'get',
+ 'getabswidth',
+ 'getalignment',
+ 'getattribute',
+ 'getattributenamespace',
+ 'getbarheight',
+ 'getbarmultiplier',
+ 'getbarwidth',
+ 'getbaseline',
+ 'getbordercolor',
+ 'getborderwidth',
+ 'getcode',
+ 'getcolor',
+ 'getcolumncount',
+ 'getencoding',
+ 'getface',
+ 'getfont',
+ 'getformat',
+ 'getfullfontname',
+ 'getheaders',
+ 'getmargins',
+ 'getmethod',
+ 'getnumericvalue',
+ 'getpadding',
+ 'getpagenumber',
+ 'getparams',
+ 'getproperty',
+ 'getpsfontname',
+ 'getrange',
+ 'getrowcount',
+ 'getsize',
+ 'getspacing',
+ 'getsupportedencodings',
+ 'gettextalignment',
+ 'gettextsize',
+ 'gettype',
+ 'gmt',
+ 'groupcount',
+ 'hasattribute',
+ 'haschildren',
+ 'hasvalue',
+ 'header',
+ 'headers',
+ 'height',
+ 'histogram',
+ 'hosttonet16',
+ 'hosttonet32',
+ 'hour',
+ 'id',
+ 'ignorecase',
+ 'import16bits',
+ 'import32bits',
+ 'import64bits',
+ 'import8bits',
+ 'importfdf',
+ 'importstring',
+ 'increment',
+ 'input',
+ 'insert',
+ 'insertatcurrent',
+ 'insertfirst',
+ 'insertfrom',
+ 'insertlast',
+ 'insertpage',
+ 'integer',
+ 'intersection',
+ 'invoke',
+ 'isa',
+ 'isalnum',
+ 'isalpha',
+ 'isbase',
+ 'iscntrl',
+ 'isdigit',
+ 'isemptyelement',
+ 'islower',
+ 'isopen',
+ 'isprint',
+ 'isspace',
+ 'istitle',
+ 'istruetype',
+ 'isualphabetic',
+ 'isulowercase',
+ 'isupper',
+ 'isuuppercase',
+ 'isuwhitespace',
+ 'iswhitespace',
+ 'iterator',
+ 'javascript',
+ 'join',
+ 'journals',
+ 'key',
+ 'keys',
+ 'last',
+ 'lastchild',
+ 'lasterror',
+ 'left',
+ 'length',
+ 'line',
+ 'listen',
+ 'localaddress',
+ 'localname',
+ 'lock',
+ 'lookupnamespace',
+ 'lowercase',
+ 'marker',
+ 'matches',
+ 'matchesstart',
+ 'matchposition',
+ 'matchstring',
+ 'merge',
+ 'millisecond',
+ 'minute',
+ 'mode',
+ 'modulate',
+ 'month',
+ 'moveto',
+ 'movetoattributenamespace',
+ 'movetoelement',
+ 'movetofirstattribute',
+ 'movetonextattribute',
+ 'name',
+ 'namespaces',
+ 'namespaceuri',
+ 'nettohost16',
+ 'nettohost32',
+ 'newchild',
+ 'next',
+ 'nextsibling',
+ 'nodetype',
+ 'open',
+ 'output',
+ 'padleading',
+ 'padtrailing',
+ 'pagecount',
+ 'pagesize',
+ 'paraminfo',
+ 'params',
+ 'parent',
+ 'path',
+ 'pixel',
+ 'position',
+ 'prefix',
+ 'previoussibling',
+ 'properties',
+ 'rawheaders',
+ 'read',
+ 'readattributevalue',
+ 'readerror',
+ 'readfrom',
+ 'readline',
+ 'readlock',
+ 'readstring',
+ 'readunlock',
+ 'recipients',
+ 'rect',
+ 'refcount',
+ 'referrals',
+ 'remoteaddress',
+ 'remove',
+ 'removeall',
+ 'removeattribute',
+ 'removechild',
+ 'removecurrent',
+ 'removefirst',
+ 'removelast',
+ 'removeleading',
+ 'removenamespace',
+ 'removetrailing',
+ 'render',
+ 'replace',
+ 'replaceall',
+ 'replacefirst',
+ 'replacepattern',
+ 'replacewith',
+ 'reserve',
+ 'reset',
+ 'resolutionh',
+ 'resolutionv',
+ 'response',
+ 'results',
+ 'retrieve',
+ 'returntype',
+ 'reverse',
+ 'reverseiterator',
+ 'right',
+ 'rotate',
+ 'run',
+ 'save',
+ 'scale',
+ 'search',
+ 'second',
+ 'send',
+ 'serialize',
+ 'set',
+ 'setalignment',
+ 'setbarheight',
+ 'setbarmultiplier',
+ 'setbarwidth',
+ 'setbaseline',
+ 'setblocking',
+ 'setbordercolor',
+ 'setborderwidth',
+ 'setbytes',
+ 'setcode',
+ 'setcolor',
+ 'setcolorspace',
+ 'setdatatype',
+ 'setencoding',
+ 'setface',
+ 'setfieldvalue',
+ 'setfont',
+ 'setformat',
+ 'setgeneratechecksum',
+ 'setheight',
+ 'setlassodata',
+ 'setlinewidth',
+ 'setmarker',
+ 'setmode',
+ 'setname',
+ 'setpadding',
+ 'setpagenumber',
+ 'setpagerange',
+ 'setposition',
+ 'setproperty',
+ 'setrange',
+ 'setshowchecksum',
+ 'setsize',
+ 'setspacing',
+ 'settemplate',
+ 'settemplatestr',
+ 'settextalignment',
+ 'settextdata',
+ 'settextsize',
+ 'settype',
+ 'setunderline',
+ 'setwidth',
+ 'setxmldata',
+ 'sharpen',
+ 'showchecksum',
+ 'showcode39startstop',
+ 'showeanguardbars',
+ 'signal',
+ 'signalall',
+ 'size',
+ 'smooth',
+ 'sort',
+ 'sortwith',
+ 'split',
+ 'standards',
+ 'steal',
+ 'subject',
+ 'substring',
+ 'subtract',
+ 'swapbytes',
+ 'textwidth',
+ 'time',
+ 'timezones',
+ 'titlecase',
+ 'to',
+ 'todos',
+ 'tolower',
+ 'totitle',
+ 'toupper',
+ 'transform',
+ 'trim',
+ 'type',
+ 'unescape',
+ 'union',
+ 'uniqueid',
+ 'unlock',
+ 'unserialize',
+ 'up',
+ 'uppercase',
+ 'value',
+ 'values',
+ 'valuetype',
+ 'wait',
+ 'waskeyword',
+ 'week',
+ 'width',
+ 'write',
+ 'writelock',
+ 'writeto',
+ 'writeunlock',
+ 'xmllang',
+ 'xmlschematype',
+ 'year',
+ )
+}
diff --git a/wandb/vendor/pygments/lexers/_lua_builtins.py b/wandb/vendor/pygments/lexers/_lua_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..c60bf5a2d021f5d922c464bed59d7d492ea159f8
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_lua_builtins.py
@@ -0,0 +1,295 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._lua_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ This file contains the names and modules of lua functions
+ It is able to re-generate itself, but for adding new functions you
+ probably have to add some callbacks (see function module_callbacks).
+
+ Do not edit the MODULES dict by hand.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import print_function
+
+MODULES = {'basic': ('_G',
+ '_VERSION',
+ 'assert',
+ 'collectgarbage',
+ 'dofile',
+ 'error',
+ 'getmetatable',
+ 'ipairs',
+ 'load',
+ 'loadfile',
+ 'next',
+ 'pairs',
+ 'pcall',
+ 'print',
+ 'rawequal',
+ 'rawget',
+ 'rawlen',
+ 'rawset',
+ 'select',
+ 'setmetatable',
+ 'tonumber',
+ 'tostring',
+ 'type',
+ 'xpcall'),
+ 'bit32': ('bit32.arshift',
+ 'bit32.band',
+ 'bit32.bnot',
+ 'bit32.bor',
+ 'bit32.btest',
+ 'bit32.bxor',
+ 'bit32.extract',
+ 'bit32.lrotate',
+ 'bit32.lshift',
+ 'bit32.replace',
+ 'bit32.rrotate',
+ 'bit32.rshift'),
+ 'coroutine': ('coroutine.create',
+ 'coroutine.isyieldable',
+ 'coroutine.resume',
+ 'coroutine.running',
+ 'coroutine.status',
+ 'coroutine.wrap',
+ 'coroutine.yield'),
+ 'debug': ('debug.debug',
+ 'debug.gethook',
+ 'debug.getinfo',
+ 'debug.getlocal',
+ 'debug.getmetatable',
+ 'debug.getregistry',
+ 'debug.getupvalue',
+ 'debug.getuservalue',
+ 'debug.sethook',
+ 'debug.setlocal',
+ 'debug.setmetatable',
+ 'debug.setupvalue',
+ 'debug.setuservalue',
+ 'debug.traceback',
+ 'debug.upvalueid',
+ 'debug.upvaluejoin'),
+ 'io': ('io.close',
+ 'io.flush',
+ 'io.input',
+ 'io.lines',
+ 'io.open',
+ 'io.output',
+ 'io.popen',
+ 'io.read',
+ 'io.stderr',
+ 'io.stdin',
+ 'io.stdout',
+ 'io.tmpfile',
+ 'io.type',
+ 'io.write'),
+ 'math': ('math.abs',
+ 'math.acos',
+ 'math.asin',
+ 'math.atan',
+ 'math.atan2',
+ 'math.ceil',
+ 'math.cos',
+ 'math.cosh',
+ 'math.deg',
+ 'math.exp',
+ 'math.floor',
+ 'math.fmod',
+ 'math.frexp',
+ 'math.huge',
+ 'math.ldexp',
+ 'math.log',
+ 'math.max',
+ 'math.maxinteger',
+ 'math.min',
+ 'math.mininteger',
+ 'math.modf',
+ 'math.pi',
+ 'math.pow',
+ 'math.rad',
+ 'math.random',
+ 'math.randomseed',
+ 'math.sin',
+ 'math.sinh',
+ 'math.sqrt',
+ 'math.tan',
+ 'math.tanh',
+ 'math.tointeger',
+ 'math.type',
+ 'math.ult'),
+ 'modules': ('package.config',
+ 'package.cpath',
+ 'package.loaded',
+ 'package.loadlib',
+ 'package.path',
+ 'package.preload',
+ 'package.searchers',
+ 'package.searchpath',
+ 'require'),
+ 'os': ('os.clock',
+ 'os.date',
+ 'os.difftime',
+ 'os.execute',
+ 'os.exit',
+ 'os.getenv',
+ 'os.remove',
+ 'os.rename',
+ 'os.setlocale',
+ 'os.time',
+ 'os.tmpname'),
+ 'string': ('string.byte',
+ 'string.char',
+ 'string.dump',
+ 'string.find',
+ 'string.format',
+ 'string.gmatch',
+ 'string.gsub',
+ 'string.len',
+ 'string.lower',
+ 'string.match',
+ 'string.pack',
+ 'string.packsize',
+ 'string.rep',
+ 'string.reverse',
+ 'string.sub',
+ 'string.unpack',
+ 'string.upper'),
+ 'table': ('table.concat',
+ 'table.insert',
+ 'table.move',
+ 'table.pack',
+ 'table.remove',
+ 'table.sort',
+ 'table.unpack'),
+ 'utf8': ('utf8.char',
+ 'utf8.charpattern',
+ 'utf8.codepoint',
+ 'utf8.codes',
+ 'utf8.len',
+ 'utf8.offset')}
+
+if __name__ == '__main__': # pragma: no cover
+ import re
+ import sys
+
+ # urllib ends up wanting to import a module called 'math' -- if
+ # pygments/lexers is in the path, this ends badly.
+ for i in range(len(sys.path)-1, -1, -1):
+ if sys.path[i].endswith('/lexers'):
+ del sys.path[i]
+
+ try:
+ from urllib import urlopen
+ except ImportError:
+ from urllib.request import urlopen
+ import pprint
+
+ # you can't generally find out what module a function belongs to if you
+ # have only its name. Because of this, here are some callback functions
+ # that recognize if a gioven function belongs to a specific module
+ def module_callbacks():
+ def is_in_coroutine_module(name):
+ return name.startswith('coroutine.')
+
+ def is_in_modules_module(name):
+ if name in ['require', 'module'] or name.startswith('package'):
+ return True
+ else:
+ return False
+
+ def is_in_string_module(name):
+ return name.startswith('string.')
+
+ def is_in_table_module(name):
+ return name.startswith('table.')
+
+ def is_in_math_module(name):
+ return name.startswith('math')
+
+ def is_in_io_module(name):
+ return name.startswith('io.')
+
+ def is_in_os_module(name):
+ return name.startswith('os.')
+
+ def is_in_debug_module(name):
+ return name.startswith('debug.')
+
+ return {'coroutine': is_in_coroutine_module,
+ 'modules': is_in_modules_module,
+ 'string': is_in_string_module,
+ 'table': is_in_table_module,
+ 'math': is_in_math_module,
+ 'io': is_in_io_module,
+ 'os': is_in_os_module,
+ 'debug': is_in_debug_module}
+
+
+
+ def get_newest_version():
+ f = urlopen('http://www.lua.org/manual/')
+ r = re.compile(r'^<A HREF="(\d\.\d)/">(Lua )?\1</A>')
+ for line in f:
+ m = r.match(line)
+ if m is not None:
+ return m.groups()[0]
+
+ def get_lua_functions(version):
+ f = urlopen('http://www.lua.org/manual/%s/' % version)
+ r = re.compile(r'^<A HREF="manual.html#pdf-(?!lua|LUA)([^:]+)">\1</A>')
+ functions = []
+ for line in f:
+ m = r.match(line)
+ if m is not None:
+ functions.append(m.groups()[0])
+ return functions
+
+ def get_function_module(name):
+ for mod, cb in module_callbacks().items():
+ if cb(name):
+ return mod
+ if '.' in name:
+ return name.split('.')[0]
+ else:
+ return 'basic'
+
+ def regenerate(filename, modules):
+ with open(filename) as fp:
+ content = fp.read()
+
+ header = content[:content.find('MODULES = {')]
+ footer = content[content.find("if __name__ == '__main__':"):]
+
+
+ with open(filename, 'w') as fp:
+ fp.write(header)
+ fp.write('MODULES = %s\n\n' % pprint.pformat(modules))
+ fp.write(footer)
+
+ def run():
+ version = get_newest_version()
+ functions = set()
+ for v in ('5.2', version):
+ print('> Downloading function index for Lua %s' % v)
+ f = get_lua_functions(v)
+ print('> %d functions found, %d new:' %
+ (len(f), len(set(f) - functions)))
+ functions |= set(f)
+
+ functions = sorted(functions)
+
+ modules = {}
+ for full_function_name in functions:
+ print('>> %s' % full_function_name)
+ m = get_function_module(full_function_name)
+ modules.setdefault(m, []).append(full_function_name)
+ modules = {k: tuple(v) for k, v in modules.iteritems()}
+
+ regenerate(__file__, modules)
+
+ run()
diff --git a/wandb/vendor/pygments/lexers/_mapping.py b/wandb/vendor/pygments/lexers/_mapping.py
new file mode 100644
index 0000000000000000000000000000000000000000..ea54241ca0fd006b3d41a0c546dcaa175b7ba34c
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_mapping.py
@@ -0,0 +1,500 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._mapping
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer mapping definitions. This file is generated by itself. Everytime
+ you change something on a builtin lexer definition, run this script from
+ the lexers folder to update it.
+
+ Do not alter the LEXERS dictionary by hand.
+
+ :copyright: Copyright 2006-2014, 2016 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import print_function
+
+LEXERS = {
+ 'ABAPLexer': ('pygments.lexers.business', 'ABAP', ('abap',), ('*.abap', '*.ABAP'), ('text/x-abap',)),
+ 'APLLexer': ('pygments.lexers.apl', 'APL', ('apl',), ('*.apl',), ()),
+ 'AbnfLexer': ('pygments.lexers.grammar_notation', 'ABNF', ('abnf',), ('*.abnf',), ('text/x-abnf',)),
+ 'ActionScript3Lexer': ('pygments.lexers.actionscript', 'ActionScript 3', ('as3', 'actionscript3'), ('*.as',), ('application/x-actionscript3', 'text/x-actionscript3', 'text/actionscript3')),
+ 'ActionScriptLexer': ('pygments.lexers.actionscript', 'ActionScript', ('as', 'actionscript'), ('*.as',), ('application/x-actionscript', 'text/x-actionscript', 'text/actionscript')),
+ 'AdaLexer': ('pygments.lexers.pascal', 'Ada', ('ada', 'ada95', 'ada2005'), ('*.adb', '*.ads', '*.ada'), ('text/x-ada',)),
+ 'AdlLexer': ('pygments.lexers.archetype', 'ADL', ('adl',), ('*.adl', '*.adls', '*.adlf', '*.adlx'), ()),
+ 'AgdaLexer': ('pygments.lexers.haskell', 'Agda', ('agda',), ('*.agda',), ('text/x-agda',)),
+ 'AheuiLexer': ('pygments.lexers.esoteric', 'Aheui', ('aheui',), ('*.aheui',), ()),
+ 'AlloyLexer': ('pygments.lexers.dsls', 'Alloy', ('alloy',), ('*.als',), ('text/x-alloy',)),
+ 'AmbientTalkLexer': ('pygments.lexers.ambient', 'AmbientTalk', ('at', 'ambienttalk', 'ambienttalk/2'), ('*.at',), ('text/x-ambienttalk',)),
+ 'AmplLexer': ('pygments.lexers.ampl', 'Ampl', ('ampl',), ('*.run',), ()),
+ 'Angular2HtmlLexer': ('pygments.lexers.templates', 'HTML + Angular2', ('html+ng2',), ('*.ng2',), ()),
+ 'Angular2Lexer': ('pygments.lexers.templates', 'Angular2', ('ng2',), (), ()),
+ 'AntlrActionScriptLexer': ('pygments.lexers.parsers', 'ANTLR With ActionScript Target', ('antlr-as', 'antlr-actionscript'), ('*.G', '*.g'), ()),
+ 'AntlrCSharpLexer': ('pygments.lexers.parsers', 'ANTLR With C# Target', ('antlr-csharp', 'antlr-c#'), ('*.G', '*.g'), ()),
+ 'AntlrCppLexer': ('pygments.lexers.parsers', 'ANTLR With CPP Target', ('antlr-cpp',), ('*.G', '*.g'), ()),
+ 'AntlrJavaLexer': ('pygments.lexers.parsers', 'ANTLR With Java Target', ('antlr-java',), ('*.G', '*.g'), ()),
+ 'AntlrLexer': ('pygments.lexers.parsers', 'ANTLR', ('antlr',), (), ()),
+ 'AntlrObjectiveCLexer': ('pygments.lexers.parsers', 'ANTLR With ObjectiveC Target', ('antlr-objc',), ('*.G', '*.g'), ()),
+ 'AntlrPerlLexer': ('pygments.lexers.parsers', 'ANTLR With Perl Target', ('antlr-perl',), ('*.G', '*.g'), ()),
+ 'AntlrPythonLexer': ('pygments.lexers.parsers', 'ANTLR With Python Target', ('antlr-python',), ('*.G', '*.g'), ()),
+ 'AntlrRubyLexer': ('pygments.lexers.parsers', 'ANTLR With Ruby Target', ('antlr-ruby', 'antlr-rb'), ('*.G', '*.g'), ()),
+ 'ApacheConfLexer': ('pygments.lexers.configs', 'ApacheConf', ('apacheconf', 'aconf', 'apache'), ('.htaccess', 'apache.conf', 'apache2.conf'), ('text/x-apacheconf',)),
+ 'AppleScriptLexer': ('pygments.lexers.scripting', 'AppleScript', ('applescript',), ('*.applescript',), ()),
+ 'ArduinoLexer': ('pygments.lexers.c_like', 'Arduino', ('arduino',), ('*.ino',), ('text/x-arduino',)),
+ 'AspectJLexer': ('pygments.lexers.jvm', 'AspectJ', ('aspectj',), ('*.aj',), ('text/x-aspectj',)),
+ 'AsymptoteLexer': ('pygments.lexers.graphics', 'Asymptote', ('asy', 'asymptote'), ('*.asy',), ('text/x-asymptote',)),
+ 'AutoItLexer': ('pygments.lexers.automation', 'AutoIt', ('autoit',), ('*.au3',), ('text/x-autoit',)),
+ 'AutohotkeyLexer': ('pygments.lexers.automation', 'autohotkey', ('ahk', 'autohotkey'), ('*.ahk', '*.ahkl'), ('text/x-autohotkey',)),
+ 'AwkLexer': ('pygments.lexers.textedit', 'Awk', ('awk', 'gawk', 'mawk', 'nawk'), ('*.awk',), ('application/x-awk',)),
+ 'BBCodeLexer': ('pygments.lexers.markup', 'BBCode', ('bbcode',), (), ('text/x-bbcode',)),
+ 'BCLexer': ('pygments.lexers.algebra', 'BC', ('bc',), ('*.bc',), ()),
+ 'BSTLexer': ('pygments.lexers.bibtex', 'BST', ('bst', 'bst-pybtex'), ('*.bst',), ()),
+ 'BaseMakefileLexer': ('pygments.lexers.make', 'Base Makefile', ('basemake',), (), ()),
+ 'BashLexer': ('pygments.lexers.shell', 'Bash', ('bash', 'sh', 'ksh', 'zsh', 'shell'), ('*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass', '*.exheres-0', '*.exlib', '*.zsh', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc', 'PKGBUILD'), ('application/x-sh', 'application/x-shellscript')),
+ 'BashSessionLexer': ('pygments.lexers.shell', 'Bash Session', ('console', 'shell-session'), ('*.sh-session', '*.shell-session'), ('application/x-shell-session', 'application/x-sh-session')),
+ 'BatchLexer': ('pygments.lexers.shell', 'Batchfile', ('bat', 'batch', 'dosbatch', 'winbatch'), ('*.bat', '*.cmd'), ('application/x-dos-batch',)),
+ 'BefungeLexer': ('pygments.lexers.esoteric', 'Befunge', ('befunge',), ('*.befunge',), ('application/x-befunge',)),
+ 'BibTeXLexer': ('pygments.lexers.bibtex', 'BibTeX', ('bib', 'bibtex'), ('*.bib',), ('text/x-bibtex',)),
+ 'BlitzBasicLexer': ('pygments.lexers.basic', 'BlitzBasic', ('blitzbasic', 'b3d', 'bplus'), ('*.bb', '*.decls'), ('text/x-bb',)),
+ 'BlitzMaxLexer': ('pygments.lexers.basic', 'BlitzMax', ('blitzmax', 'bmax'), ('*.bmx',), ('text/x-bmx',)),
+ 'BnfLexer': ('pygments.lexers.grammar_notation', 'BNF', ('bnf',), ('*.bnf',), ('text/x-bnf',)),
+ 'BooLexer': ('pygments.lexers.dotnet', 'Boo', ('boo',), ('*.boo',), ('text/x-boo',)),
+ 'BoogieLexer': ('pygments.lexers.verification', 'Boogie', ('boogie',), ('*.bpl',), ()),
+ 'BrainfuckLexer': ('pygments.lexers.esoteric', 'Brainfuck', ('brainfuck', 'bf'), ('*.bf', '*.b'), ('application/x-brainfuck',)),
+ 'BroLexer': ('pygments.lexers.dsls', 'Bro', ('bro',), ('*.bro',), ()),
+ 'BugsLexer': ('pygments.lexers.modeling', 'BUGS', ('bugs', 'winbugs', 'openbugs'), ('*.bug',), ()),
+ 'CAmkESLexer': ('pygments.lexers.esoteric', 'CAmkES', ('camkes', 'idl4'), ('*.camkes', '*.idl4'), ()),
+ 'CLexer': ('pygments.lexers.c_cpp', 'C', ('c',), ('*.c', '*.h', '*.idc'), ('text/x-chdr', 'text/x-csrc')),
+ 'CMakeLexer': ('pygments.lexers.make', 'CMake', ('cmake',), ('*.cmake', 'CMakeLists.txt'), ('text/x-cmake',)),
+ 'CObjdumpLexer': ('pygments.lexers.asm', 'c-objdump', ('c-objdump',), ('*.c-objdump',), ('text/x-c-objdump',)),
+ 'CPSALexer': ('pygments.lexers.lisp', 'CPSA', ('cpsa',), ('*.cpsa',), ()),
+ 'CSharpAspxLexer': ('pygments.lexers.dotnet', 'aspx-cs', ('aspx-cs',), ('*.aspx', '*.asax', '*.ascx', '*.ashx', '*.asmx', '*.axd'), ()),
+ 'CSharpLexer': ('pygments.lexers.dotnet', 'C#', ('csharp', 'c#'), ('*.cs',), ('text/x-csharp',)),
+ 'Ca65Lexer': ('pygments.lexers.asm', 'ca65 assembler', ('ca65',), ('*.s',), ()),
+ 'CadlLexer': ('pygments.lexers.archetype', 'cADL', ('cadl',), ('*.cadl',), ()),
+ 'CapDLLexer': ('pygments.lexers.esoteric', 'CapDL', ('capdl',), ('*.cdl',), ()),
+ 'CapnProtoLexer': ('pygments.lexers.capnproto', "Cap'n Proto", ('capnp',), ('*.capnp',), ()),
+ 'CbmBasicV2Lexer': ('pygments.lexers.basic', 'CBM BASIC V2', ('cbmbas',), ('*.bas',), ()),
+ 'CeylonLexer': ('pygments.lexers.jvm', 'Ceylon', ('ceylon',), ('*.ceylon',), ('text/x-ceylon',)),
+ 'Cfengine3Lexer': ('pygments.lexers.configs', 'CFEngine3', ('cfengine3', 'cf3'), ('*.cf',), ()),
+ 'ChaiscriptLexer': ('pygments.lexers.scripting', 'ChaiScript', ('chai', 'chaiscript'), ('*.chai',), ('text/x-chaiscript', 'application/x-chaiscript')),
+ 'ChapelLexer': ('pygments.lexers.chapel', 'Chapel', ('chapel', 'chpl'), ('*.chpl',), ()),
+ 'CheetahHtmlLexer': ('pygments.lexers.templates', 'HTML+Cheetah', ('html+cheetah', 'html+spitfire', 'htmlcheetah'), (), ('text/html+cheetah', 'text/html+spitfire')),
+ 'CheetahJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Cheetah', ('js+cheetah', 'javascript+cheetah', 'js+spitfire', 'javascript+spitfire'), (), ('application/x-javascript+cheetah', 'text/x-javascript+cheetah', 'text/javascript+cheetah', 'application/x-javascript+spitfire', 'text/x-javascript+spitfire', 'text/javascript+spitfire')),
+ 'CheetahLexer': ('pygments.lexers.templates', 'Cheetah', ('cheetah', 'spitfire'), ('*.tmpl', '*.spt'), ('application/x-cheetah', 'application/x-spitfire')),
+ 'CheetahXmlLexer': ('pygments.lexers.templates', 'XML+Cheetah', ('xml+cheetah', 'xml+spitfire'), (), ('application/xml+cheetah', 'application/xml+spitfire')),
+ 'CirruLexer': ('pygments.lexers.webmisc', 'Cirru', ('cirru',), ('*.cirru',), ('text/x-cirru',)),
+ 'ClayLexer': ('pygments.lexers.c_like', 'Clay', ('clay',), ('*.clay',), ('text/x-clay',)),
+ 'CleanLexer': ('pygments.lexers.clean', 'Clean', ('clean',), ('*.icl', '*.dcl'), ()),
+ 'ClojureLexer': ('pygments.lexers.jvm', 'Clojure', ('clojure', 'clj'), ('*.clj',), ('text/x-clojure', 'application/x-clojure')),
+ 'ClojureScriptLexer': ('pygments.lexers.jvm', 'ClojureScript', ('clojurescript', 'cljs'), ('*.cljs',), ('text/x-clojurescript', 'application/x-clojurescript')),
+ 'CobolFreeformatLexer': ('pygments.lexers.business', 'COBOLFree', ('cobolfree',), ('*.cbl', '*.CBL'), ()),
+ 'CobolLexer': ('pygments.lexers.business', 'COBOL', ('cobol',), ('*.cob', '*.COB', '*.cpy', '*.CPY'), ('text/x-cobol',)),
+ 'CoffeeScriptLexer': ('pygments.lexers.javascript', 'CoffeeScript', ('coffee-script', 'coffeescript', 'coffee'), ('*.coffee',), ('text/coffeescript',)),
+ 'ColdfusionCFCLexer': ('pygments.lexers.templates', 'Coldfusion CFC', ('cfc',), ('*.cfc',), ()),
+ 'ColdfusionHtmlLexer': ('pygments.lexers.templates', 'Coldfusion HTML', ('cfm',), ('*.cfm', '*.cfml'), ('application/x-coldfusion',)),
+ 'ColdfusionLexer': ('pygments.lexers.templates', 'cfstatement', ('cfs',), (), ()),
+ 'CommonLispLexer': ('pygments.lexers.lisp', 'Common Lisp', ('common-lisp', 'cl', 'lisp'), ('*.cl', '*.lisp'), ('text/x-common-lisp',)),
+ 'ComponentPascalLexer': ('pygments.lexers.oberon', 'Component Pascal', ('componentpascal', 'cp'), ('*.cp', '*.cps'), ('text/x-component-pascal',)),
+ 'CoqLexer': ('pygments.lexers.theorem', 'Coq', ('coq',), ('*.v',), ('text/x-coq',)),
+ 'CppLexer': ('pygments.lexers.c_cpp', 'C++', ('cpp', 'c++'), ('*.cpp', '*.hpp', '*.c++', '*.h++', '*.cc', '*.hh', '*.cxx', '*.hxx', '*.C', '*.H', '*.cp', '*.CPP'), ('text/x-c++hdr', 'text/x-c++src')),
+ 'CppObjdumpLexer': ('pygments.lexers.asm', 'cpp-objdump', ('cpp-objdump', 'c++-objdumb', 'cxx-objdump'), ('*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump'), ('text/x-cpp-objdump',)),
+ 'CrmshLexer': ('pygments.lexers.dsls', 'Crmsh', ('crmsh', 'pcmk'), ('*.crmsh', '*.pcmk'), ()),
+ 'CrocLexer': ('pygments.lexers.d', 'Croc', ('croc',), ('*.croc',), ('text/x-crocsrc',)),
+ 'CryptolLexer': ('pygments.lexers.haskell', 'Cryptol', ('cryptol', 'cry'), ('*.cry',), ('text/x-cryptol',)),
+ 'CrystalLexer': ('pygments.lexers.crystal', 'Crystal', ('cr', 'crystal'), ('*.cr',), ('text/x-crystal',)),
+ 'CsoundDocumentLexer': ('pygments.lexers.csound', 'Csound Document', ('csound-document', 'csound-csd'), ('*.csd',), ()),
+ 'CsoundOrchestraLexer': ('pygments.lexers.csound', 'Csound Orchestra', ('csound', 'csound-orc'), ('*.orc',), ()),
+ 'CsoundScoreLexer': ('pygments.lexers.csound', 'Csound Score', ('csound-score', 'csound-sco'), ('*.sco',), ()),
+ 'CssDjangoLexer': ('pygments.lexers.templates', 'CSS+Django/Jinja', ('css+django', 'css+jinja'), (), ('text/css+django', 'text/css+jinja')),
+ 'CssErbLexer': ('pygments.lexers.templates', 'CSS+Ruby', ('css+erb', 'css+ruby'), (), ('text/css+ruby',)),
+ 'CssGenshiLexer': ('pygments.lexers.templates', 'CSS+Genshi Text', ('css+genshitext', 'css+genshi'), (), ('text/css+genshi',)),
+ 'CssLexer': ('pygments.lexers.css', 'CSS', ('css',), ('*.css',), ('text/css',)),
+ 'CssPhpLexer': ('pygments.lexers.templates', 'CSS+PHP', ('css+php',), (), ('text/css+php',)),
+ 'CssSmartyLexer': ('pygments.lexers.templates', 'CSS+Smarty', ('css+smarty',), (), ('text/css+smarty',)),
+ 'CudaLexer': ('pygments.lexers.c_like', 'CUDA', ('cuda', 'cu'), ('*.cu', '*.cuh'), ('text/x-cuda',)),
+ 'CypherLexer': ('pygments.lexers.graph', 'Cypher', ('cypher',), ('*.cyp', '*.cypher'), ()),
+ 'CythonLexer': ('pygments.lexers.python', 'Cython', ('cython', 'pyx', 'pyrex'), ('*.pyx', '*.pxd', '*.pxi'), ('text/x-cython', 'application/x-cython')),
+ 'DLexer': ('pygments.lexers.d', 'D', ('d',), ('*.d', '*.di'), ('text/x-dsrc',)),
+ 'DObjdumpLexer': ('pygments.lexers.asm', 'd-objdump', ('d-objdump',), ('*.d-objdump',), ('text/x-d-objdump',)),
+ 'DarcsPatchLexer': ('pygments.lexers.diff', 'Darcs Patch', ('dpatch',), ('*.dpatch', '*.darcspatch'), ()),
+ 'DartLexer': ('pygments.lexers.javascript', 'Dart', ('dart',), ('*.dart',), ('text/x-dart',)),
+ 'DebianControlLexer': ('pygments.lexers.installers', 'Debian Control file', ('control', 'debcontrol'), ('control',), ()),
+ 'DelphiLexer': ('pygments.lexers.pascal', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas', '*.dpr'), ('text/x-pascal',)),
+ 'DgLexer': ('pygments.lexers.python', 'dg', ('dg',), ('*.dg',), ('text/x-dg',)),
+ 'DiffLexer': ('pygments.lexers.diff', 'Diff', ('diff', 'udiff'), ('*.diff', '*.patch'), ('text/x-diff', 'text/x-patch')),
+ 'DjangoLexer': ('pygments.lexers.templates', 'Django/Jinja', ('django', 'jinja'), (), ('application/x-django-templating', 'application/x-jinja')),
+ 'DockerLexer': ('pygments.lexers.configs', 'Docker', ('docker', 'dockerfile'), ('Dockerfile', '*.docker'), ('text/x-dockerfile-config',)),
+ 'DtdLexer': ('pygments.lexers.html', 'DTD', ('dtd',), ('*.dtd',), ('application/xml-dtd',)),
+ 'DuelLexer': ('pygments.lexers.webmisc', 'Duel', ('duel', 'jbst', 'jsonml+bst'), ('*.duel', '*.jbst'), ('text/x-duel', 'text/x-jbst')),
+ 'DylanConsoleLexer': ('pygments.lexers.dylan', 'Dylan session', ('dylan-console', 'dylan-repl'), ('*.dylan-console',), ('text/x-dylan-console',)),
+ 'DylanLexer': ('pygments.lexers.dylan', 'Dylan', ('dylan',), ('*.dylan', '*.dyl', '*.intr'), ('text/x-dylan',)),
+ 'DylanLidLexer': ('pygments.lexers.dylan', 'DylanLID', ('dylan-lid', 'lid'), ('*.lid', '*.hdp'), ('text/x-dylan-lid',)),
+ 'ECLLexer': ('pygments.lexers.ecl', 'ECL', ('ecl',), ('*.ecl',), ('application/x-ecl',)),
+ 'ECLexer': ('pygments.lexers.c_like', 'eC', ('ec',), ('*.ec', '*.eh'), ('text/x-echdr', 'text/x-ecsrc')),
+ 'EarlGreyLexer': ('pygments.lexers.javascript', 'Earl Grey', ('earl-grey', 'earlgrey', 'eg'), ('*.eg',), ('text/x-earl-grey',)),
+ 'EasytrieveLexer': ('pygments.lexers.scripting', 'Easytrieve', ('easytrieve',), ('*.ezt', '*.mac'), ('text/x-easytrieve',)),
+ 'EbnfLexer': ('pygments.lexers.parsers', 'EBNF', ('ebnf',), ('*.ebnf',), ('text/x-ebnf',)),
+ 'EiffelLexer': ('pygments.lexers.eiffel', 'Eiffel', ('eiffel',), ('*.e',), ('text/x-eiffel',)),
+ 'ElixirConsoleLexer': ('pygments.lexers.erlang', 'Elixir iex session', ('iex',), (), ('text/x-elixir-shellsession',)),
+ 'ElixirLexer': ('pygments.lexers.erlang', 'Elixir', ('elixir', 'ex', 'exs'), ('*.ex', '*.exs'), ('text/x-elixir',)),
+ 'ElmLexer': ('pygments.lexers.elm', 'Elm', ('elm',), ('*.elm',), ('text/x-elm',)),
+ 'EmacsLispLexer': ('pygments.lexers.lisp', 'EmacsLisp', ('emacs', 'elisp', 'emacs-lisp'), ('*.el',), ('text/x-elisp', 'application/x-elisp')),
+ 'ErbLexer': ('pygments.lexers.templates', 'ERB', ('erb',), (), ('application/x-ruby-templating',)),
+ 'ErlangLexer': ('pygments.lexers.erlang', 'Erlang', ('erlang',), ('*.erl', '*.hrl', '*.es', '*.escript'), ('text/x-erlang',)),
+ 'ErlangShellLexer': ('pygments.lexers.erlang', 'Erlang erl session', ('erl',), ('*.erl-sh',), ('text/x-erl-shellsession',)),
+ 'EvoqueHtmlLexer': ('pygments.lexers.templates', 'HTML+Evoque', ('html+evoque',), ('*.html',), ('text/html+evoque',)),
+ 'EvoqueLexer': ('pygments.lexers.templates', 'Evoque', ('evoque',), ('*.evoque',), ('application/x-evoque',)),
+ 'EvoqueXmlLexer': ('pygments.lexers.templates', 'XML+Evoque', ('xml+evoque',), ('*.xml',), ('application/xml+evoque',)),
+ 'EzhilLexer': ('pygments.lexers.ezhil', 'Ezhil', ('ezhil',), ('*.n',), ('text/x-ezhil',)),
+ 'FSharpLexer': ('pygments.lexers.dotnet', 'FSharp', ('fsharp',), ('*.fs', '*.fsi'), ('text/x-fsharp',)),
+ 'FactorLexer': ('pygments.lexers.factor', 'Factor', ('factor',), ('*.factor',), ('text/x-factor',)),
+ 'FancyLexer': ('pygments.lexers.ruby', 'Fancy', ('fancy', 'fy'), ('*.fy', '*.fancypack'), ('text/x-fancysrc',)),
+ 'FantomLexer': ('pygments.lexers.fantom', 'Fantom', ('fan',), ('*.fan',), ('application/x-fantom',)),
+ 'FelixLexer': ('pygments.lexers.felix', 'Felix', ('felix', 'flx'), ('*.flx', '*.flxh'), ('text/x-felix',)),
+ 'FishShellLexer': ('pygments.lexers.shell', 'Fish', ('fish', 'fishshell'), ('*.fish', '*.load'), ('application/x-fish',)),
+ 'FlatlineLexer': ('pygments.lexers.dsls', 'Flatline', ('flatline',), (), ('text/x-flatline',)),
+ 'ForthLexer': ('pygments.lexers.forth', 'Forth', ('forth',), ('*.frt', '*.fs'), ('application/x-forth',)),
+ 'FortranFixedLexer': ('pygments.lexers.fortran', 'FortranFixed', ('fortranfixed',), ('*.f', '*.F'), ()),
+ 'FortranLexer': ('pygments.lexers.fortran', 'Fortran', ('fortran',), ('*.f03', '*.f90', '*.F03', '*.F90'), ('text/x-fortran',)),
+ 'FoxProLexer': ('pygments.lexers.foxpro', 'FoxPro', ('foxpro', 'vfp', 'clipper', 'xbase'), ('*.PRG', '*.prg'), ()),
+ 'GAPLexer': ('pygments.lexers.algebra', 'GAP', ('gap',), ('*.g', '*.gd', '*.gi', '*.gap'), ()),
+ 'GLShaderLexer': ('pygments.lexers.graphics', 'GLSL', ('glsl',), ('*.vert', '*.frag', '*.geo'), ('text/x-glslsrc',)),
+ 'GasLexer': ('pygments.lexers.asm', 'GAS', ('gas', 'asm'), ('*.s', '*.S'), ('text/x-gas',)),
+ 'GenshiLexer': ('pygments.lexers.templates', 'Genshi', ('genshi', 'kid', 'xml+genshi', 'xml+kid'), ('*.kid',), ('application/x-genshi', 'application/x-kid')),
+ 'GenshiTextLexer': ('pygments.lexers.templates', 'Genshi Text', ('genshitext',), (), ('application/x-genshi-text', 'text/x-genshi')),
+ 'GettextLexer': ('pygments.lexers.textfmts', 'Gettext Catalog', ('pot', 'po'), ('*.pot', '*.po'), ('application/x-gettext', 'text/x-gettext', 'text/gettext')),
+ 'GherkinLexer': ('pygments.lexers.testing', 'Gherkin', ('cucumber', 'gherkin'), ('*.feature',), ('text/x-gherkin',)),
+ 'GnuplotLexer': ('pygments.lexers.graphics', 'Gnuplot', ('gnuplot',), ('*.plot', '*.plt'), ('text/x-gnuplot',)),
+ 'GoLexer': ('pygments.lexers.go', 'Go', ('go',), ('*.go',), ('text/x-gosrc',)),
+ 'GoloLexer': ('pygments.lexers.jvm', 'Golo', ('golo',), ('*.golo',), ()),
+ 'GoodDataCLLexer': ('pygments.lexers.business', 'GoodData-CL', ('gooddata-cl',), ('*.gdc',), ('text/x-gooddata-cl',)),
+ 'GosuLexer': ('pygments.lexers.jvm', 'Gosu', ('gosu',), ('*.gs', '*.gsx', '*.gsp', '*.vark'), ('text/x-gosu',)),
+ 'GosuTemplateLexer': ('pygments.lexers.jvm', 'Gosu Template', ('gst',), ('*.gst',), ('text/x-gosu-template',)),
+ 'GroffLexer': ('pygments.lexers.markup', 'Groff', ('groff', 'nroff', 'man'), ('*.[1234567]', '*.man'), ('application/x-troff', 'text/troff')),
+ 'GroovyLexer': ('pygments.lexers.jvm', 'Groovy', ('groovy',), ('*.groovy', '*.gradle'), ('text/x-groovy',)),
+ 'HamlLexer': ('pygments.lexers.html', 'Haml', ('haml',), ('*.haml',), ('text/x-haml',)),
+ 'HandlebarsHtmlLexer': ('pygments.lexers.templates', 'HTML+Handlebars', ('html+handlebars',), ('*.handlebars', '*.hbs'), ('text/html+handlebars', 'text/x-handlebars-template')),
+ 'HandlebarsLexer': ('pygments.lexers.templates', 'Handlebars', ('handlebars',), (), ()),
+ 'HaskellLexer': ('pygments.lexers.haskell', 'Haskell', ('haskell', 'hs'), ('*.hs',), ('text/x-haskell',)),
+ 'HaxeLexer': ('pygments.lexers.haxe', 'Haxe', ('hx', 'haxe', 'hxsl'), ('*.hx', '*.hxsl'), ('text/haxe', 'text/x-haxe', 'text/x-hx')),
+ 'HexdumpLexer': ('pygments.lexers.hexdump', 'Hexdump', ('hexdump',), (), ()),
+ 'HsailLexer': ('pygments.lexers.asm', 'HSAIL', ('hsail', 'hsa'), ('*.hsail',), ('text/x-hsail',)),
+ 'HtmlDjangoLexer': ('pygments.lexers.templates', 'HTML+Django/Jinja', ('html+django', 'html+jinja', 'htmldjango'), (), ('text/html+django', 'text/html+jinja')),
+ 'HtmlGenshiLexer': ('pygments.lexers.templates', 'HTML+Genshi', ('html+genshi', 'html+kid'), (), ('text/html+genshi',)),
+ 'HtmlLexer': ('pygments.lexers.html', 'HTML', ('html',), ('*.html', '*.htm', '*.xhtml', '*.xslt'), ('text/html', 'application/xhtml+xml')),
+ 'HtmlPhpLexer': ('pygments.lexers.templates', 'HTML+PHP', ('html+php',), ('*.phtml',), ('application/x-php', 'application/x-httpd-php', 'application/x-httpd-php3', 'application/x-httpd-php4', 'application/x-httpd-php5')),
+ 'HtmlSmartyLexer': ('pygments.lexers.templates', 'HTML+Smarty', ('html+smarty',), (), ('text/html+smarty',)),
+ 'HttpLexer': ('pygments.lexers.textfmts', 'HTTP', ('http',), (), ()),
+ 'HxmlLexer': ('pygments.lexers.haxe', 'Hxml', ('haxeml', 'hxml'), ('*.hxml',), ()),
+ 'HyLexer': ('pygments.lexers.lisp', 'Hy', ('hylang',), ('*.hy',), ('text/x-hy', 'application/x-hy')),
+ 'HybrisLexer': ('pygments.lexers.scripting', 'Hybris', ('hybris', 'hy'), ('*.hy', '*.hyb'), ('text/x-hybris', 'application/x-hybris')),
+ 'IDLLexer': ('pygments.lexers.idl', 'IDL', ('idl',), ('*.pro',), ('text/idl',)),
+ 'IdrisLexer': ('pygments.lexers.haskell', 'Idris', ('idris', 'idr'), ('*.idr',), ('text/x-idris',)),
+ 'IgorLexer': ('pygments.lexers.igor', 'Igor', ('igor', 'igorpro'), ('*.ipf',), ('text/ipf',)),
+ 'Inform6Lexer': ('pygments.lexers.int_fiction', 'Inform 6', ('inform6', 'i6'), ('*.inf',), ()),
+ 'Inform6TemplateLexer': ('pygments.lexers.int_fiction', 'Inform 6 template', ('i6t',), ('*.i6t',), ()),
+ 'Inform7Lexer': ('pygments.lexers.int_fiction', 'Inform 7', ('inform7', 'i7'), ('*.ni', '*.i7x'), ()),
+ 'IniLexer': ('pygments.lexers.configs', 'INI', ('ini', 'cfg', 'dosini'), ('*.ini', '*.cfg', '*.inf'), ('text/x-ini', 'text/inf')),
+ 'IoLexer': ('pygments.lexers.iolang', 'Io', ('io',), ('*.io',), ('text/x-iosrc',)),
+ 'IokeLexer': ('pygments.lexers.jvm', 'Ioke', ('ioke', 'ik'), ('*.ik',), ('text/x-iokesrc',)),
+ 'IrcLogsLexer': ('pygments.lexers.textfmts', 'IRC logs', ('irc',), ('*.weechatlog',), ('text/x-irclog',)),
+ 'IsabelleLexer': ('pygments.lexers.theorem', 'Isabelle', ('isabelle',), ('*.thy',), ('text/x-isabelle',)),
+ 'JLexer': ('pygments.lexers.j', 'J', ('j',), ('*.ijs',), ('text/x-j',)),
+ 'JagsLexer': ('pygments.lexers.modeling', 'JAGS', ('jags',), ('*.jag', '*.bug'), ()),
+ 'JasminLexer': ('pygments.lexers.jvm', 'Jasmin', ('jasmin', 'jasminxt'), ('*.j',), ()),
+ 'JavaLexer': ('pygments.lexers.jvm', 'Java', ('java',), ('*.java',), ('text/x-java',)),
+ 'JavascriptDjangoLexer': ('pygments.lexers.templates', 'JavaScript+Django/Jinja', ('js+django', 'javascript+django', 'js+jinja', 'javascript+jinja'), (), ('application/x-javascript+django', 'application/x-javascript+jinja', 'text/x-javascript+django', 'text/x-javascript+jinja', 'text/javascript+django', 'text/javascript+jinja')),
+ 'JavascriptErbLexer': ('pygments.lexers.templates', 'JavaScript+Ruby', ('js+erb', 'javascript+erb', 'js+ruby', 'javascript+ruby'), (), ('application/x-javascript+ruby', 'text/x-javascript+ruby', 'text/javascript+ruby')),
+ 'JavascriptGenshiLexer': ('pygments.lexers.templates', 'JavaScript+Genshi Text', ('js+genshitext', 'js+genshi', 'javascript+genshitext', 'javascript+genshi'), (), ('application/x-javascript+genshi', 'text/x-javascript+genshi', 'text/javascript+genshi')),
+ 'JavascriptLexer': ('pygments.lexers.javascript', 'JavaScript', ('js', 'javascript'), ('*.js', '*.jsm'), ('application/javascript', 'application/x-javascript', 'text/x-javascript', 'text/javascript')),
+ 'JavascriptPhpLexer': ('pygments.lexers.templates', 'JavaScript+PHP', ('js+php', 'javascript+php'), (), ('application/x-javascript+php', 'text/x-javascript+php', 'text/javascript+php')),
+ 'JavascriptSmartyLexer': ('pygments.lexers.templates', 'JavaScript+Smarty', ('js+smarty', 'javascript+smarty'), (), ('application/x-javascript+smarty', 'text/x-javascript+smarty', 'text/javascript+smarty')),
+ 'JclLexer': ('pygments.lexers.scripting', 'JCL', ('jcl',), ('*.jcl',), ('text/x-jcl',)),
+ 'JsgfLexer': ('pygments.lexers.grammar_notation', 'JSGF', ('jsgf',), ('*.jsgf',), ('application/jsgf', 'application/x-jsgf', 'text/jsgf')),
+ 'JsonBareObjectLexer': ('pygments.lexers.data', 'JSONBareObject', ('json-object',), (), ('application/json-object',)),
+ 'JsonLdLexer': ('pygments.lexers.data', 'JSON-LD', ('jsonld', 'json-ld'), ('*.jsonld',), ('application/ld+json',)),
+ 'JsonLexer': ('pygments.lexers.data', 'JSON', ('json',), ('*.json',), ('application/json',)),
+ 'JspLexer': ('pygments.lexers.templates', 'Java Server Page', ('jsp',), ('*.jsp',), ('application/x-jsp',)),
+ 'JuliaConsoleLexer': ('pygments.lexers.julia', 'Julia console', ('jlcon',), (), ()),
+ 'JuliaLexer': ('pygments.lexers.julia', 'Julia', ('julia', 'jl'), ('*.jl',), ('text/x-julia', 'application/x-julia')),
+ 'JuttleLexer': ('pygments.lexers.javascript', 'Juttle', ('juttle', 'juttle'), ('*.juttle',), ('application/juttle', 'application/x-juttle', 'text/x-juttle', 'text/juttle')),
+ 'KalLexer': ('pygments.lexers.javascript', 'Kal', ('kal',), ('*.kal',), ('text/kal', 'application/kal')),
+ 'KconfigLexer': ('pygments.lexers.configs', 'Kconfig', ('kconfig', 'menuconfig', 'linux-config', 'kernel-config'), ('Kconfig', '*Config.in*', 'external.in*', 'standard-modules.in'), ('text/x-kconfig',)),
+ 'KokaLexer': ('pygments.lexers.haskell', 'Koka', ('koka',), ('*.kk', '*.kki'), ('text/x-koka',)),
+ 'KotlinLexer': ('pygments.lexers.jvm', 'Kotlin', ('kotlin',), ('*.kt',), ('text/x-kotlin',)),
+ 'LSLLexer': ('pygments.lexers.scripting', 'LSL', ('lsl',), ('*.lsl',), ('text/x-lsl',)),
+ 'LassoCssLexer': ('pygments.lexers.templates', 'CSS+Lasso', ('css+lasso',), (), ('text/css+lasso',)),
+ 'LassoHtmlLexer': ('pygments.lexers.templates', 'HTML+Lasso', ('html+lasso',), (), ('text/html+lasso', 'application/x-httpd-lasso', 'application/x-httpd-lasso[89]')),
+ 'LassoJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Lasso', ('js+lasso', 'javascript+lasso'), (), ('application/x-javascript+lasso', 'text/x-javascript+lasso', 'text/javascript+lasso')),
+ 'LassoLexer': ('pygments.lexers.javascript', 'Lasso', ('lasso', 'lassoscript'), ('*.lasso', '*.lasso[89]'), ('text/x-lasso',)),
+ 'LassoXmlLexer': ('pygments.lexers.templates', 'XML+Lasso', ('xml+lasso',), (), ('application/xml+lasso',)),
+ 'LeanLexer': ('pygments.lexers.theorem', 'Lean', ('lean',), ('*.lean',), ('text/x-lean',)),
+ 'LessCssLexer': ('pygments.lexers.css', 'LessCss', ('less',), ('*.less',), ('text/x-less-css',)),
+ 'LighttpdConfLexer': ('pygments.lexers.configs', 'Lighttpd configuration file', ('lighty', 'lighttpd'), (), ('text/x-lighttpd-conf',)),
+ 'LimboLexer': ('pygments.lexers.inferno', 'Limbo', ('limbo',), ('*.b',), ('text/limbo',)),
+ 'LiquidLexer': ('pygments.lexers.templates', 'liquid', ('liquid',), ('*.liquid',), ()),
+ 'LiterateAgdaLexer': ('pygments.lexers.haskell', 'Literate Agda', ('lagda', 'literate-agda'), ('*.lagda',), ('text/x-literate-agda',)),
+ 'LiterateCryptolLexer': ('pygments.lexers.haskell', 'Literate Cryptol', ('lcry', 'literate-cryptol', 'lcryptol'), ('*.lcry',), ('text/x-literate-cryptol',)),
+ 'LiterateHaskellLexer': ('pygments.lexers.haskell', 'Literate Haskell', ('lhs', 'literate-haskell', 'lhaskell'), ('*.lhs',), ('text/x-literate-haskell',)),
+ 'LiterateIdrisLexer': ('pygments.lexers.haskell', 'Literate Idris', ('lidr', 'literate-idris', 'lidris'), ('*.lidr',), ('text/x-literate-idris',)),
+ 'LiveScriptLexer': ('pygments.lexers.javascript', 'LiveScript', ('live-script', 'livescript'), ('*.ls',), ('text/livescript',)),
+ 'LlvmLexer': ('pygments.lexers.asm', 'LLVM', ('llvm',), ('*.ll',), ('text/x-llvm',)),
+ 'LogosLexer': ('pygments.lexers.objective', 'Logos', ('logos',), ('*.x', '*.xi', '*.xm', '*.xmi'), ('text/x-logos',)),
+ 'LogtalkLexer': ('pygments.lexers.prolog', 'Logtalk', ('logtalk',), ('*.lgt', '*.logtalk'), ('text/x-logtalk',)),
+ 'LuaLexer': ('pygments.lexers.scripting', 'Lua', ('lua',), ('*.lua', '*.wlua'), ('text/x-lua', 'application/x-lua')),
+ 'MOOCodeLexer': ('pygments.lexers.scripting', 'MOOCode', ('moocode', 'moo'), ('*.moo',), ('text/x-moocode',)),
+ 'MSDOSSessionLexer': ('pygments.lexers.shell', 'MSDOS Session', ('doscon',), (), ()),
+ 'MakefileLexer': ('pygments.lexers.make', 'Makefile', ('make', 'makefile', 'mf', 'bsdmake'), ('*.mak', '*.mk', 'Makefile', 'makefile', 'Makefile.*', 'GNUmakefile'), ('text/x-makefile',)),
+ 'MakoCssLexer': ('pygments.lexers.templates', 'CSS+Mako', ('css+mako',), (), ('text/css+mako',)),
+ 'MakoHtmlLexer': ('pygments.lexers.templates', 'HTML+Mako', ('html+mako',), (), ('text/html+mako',)),
+ 'MakoJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Mako', ('js+mako', 'javascript+mako'), (), ('application/x-javascript+mako', 'text/x-javascript+mako', 'text/javascript+mako')),
+ 'MakoLexer': ('pygments.lexers.templates', 'Mako', ('mako',), ('*.mao',), ('application/x-mako',)),
+ 'MakoXmlLexer': ('pygments.lexers.templates', 'XML+Mako', ('xml+mako',), (), ('application/xml+mako',)),
+ 'MaqlLexer': ('pygments.lexers.business', 'MAQL', ('maql',), ('*.maql',), ('text/x-gooddata-maql', 'application/x-gooddata-maql')),
+ 'MarkdownLexer': ('pygments.lexers.markup', 'markdown', ('md',), ('*.md',), ('text/x-markdown',)),
+ 'MaskLexer': ('pygments.lexers.javascript', 'Mask', ('mask',), ('*.mask',), ('text/x-mask',)),
+ 'MasonLexer': ('pygments.lexers.templates', 'Mason', ('mason',), ('*.m', '*.mhtml', '*.mc', '*.mi', 'autohandler', 'dhandler'), ('application/x-mason',)),
+ 'MathematicaLexer': ('pygments.lexers.algebra', 'Mathematica', ('mathematica', 'mma', 'nb'), ('*.nb', '*.cdf', '*.nbp', '*.ma'), ('application/mathematica', 'application/vnd.wolfram.mathematica', 'application/vnd.wolfram.mathematica.package', 'application/vnd.wolfram.cdf')),
+ 'MatlabLexer': ('pygments.lexers.matlab', 'Matlab', ('matlab',), ('*.m',), ('text/matlab',)),
+ 'MatlabSessionLexer': ('pygments.lexers.matlab', 'Matlab session', ('matlabsession',), (), ()),
+ 'MiniDLexer': ('pygments.lexers.d', 'MiniD', ('minid',), (), ('text/x-minidsrc',)),
+ 'ModelicaLexer': ('pygments.lexers.modeling', 'Modelica', ('modelica',), ('*.mo',), ('text/x-modelica',)),
+ 'Modula2Lexer': ('pygments.lexers.modula2', 'Modula-2', ('modula2', 'm2'), ('*.def', '*.mod'), ('text/x-modula2',)),
+ 'MoinWikiLexer': ('pygments.lexers.markup', 'MoinMoin/Trac Wiki markup', ('trac-wiki', 'moin'), (), ('text/x-trac-wiki',)),
+ 'MonkeyLexer': ('pygments.lexers.basic', 'Monkey', ('monkey',), ('*.monkey',), ('text/x-monkey',)),
+ 'MonteLexer': ('pygments.lexers.monte', 'Monte', ('monte',), ('*.mt',), ()),
+ 'MoonScriptLexer': ('pygments.lexers.scripting', 'MoonScript', ('moon', 'moonscript'), ('*.moon',), ('text/x-moonscript', 'application/x-moonscript')),
+ 'MozPreprocCssLexer': ('pygments.lexers.markup', 'CSS+mozpreproc', ('css+mozpreproc',), ('*.css.in',), ()),
+ 'MozPreprocHashLexer': ('pygments.lexers.markup', 'mozhashpreproc', ('mozhashpreproc',), (), ()),
+ 'MozPreprocJavascriptLexer': ('pygments.lexers.markup', 'Javascript+mozpreproc', ('javascript+mozpreproc',), ('*.js.in',), ()),
+ 'MozPreprocPercentLexer': ('pygments.lexers.markup', 'mozpercentpreproc', ('mozpercentpreproc',), (), ()),
+ 'MozPreprocXulLexer': ('pygments.lexers.markup', 'XUL+mozpreproc', ('xul+mozpreproc',), ('*.xul.in',), ()),
+ 'MqlLexer': ('pygments.lexers.c_like', 'MQL', ('mql', 'mq4', 'mq5', 'mql4', 'mql5'), ('*.mq4', '*.mq5', '*.mqh'), ('text/x-mql',)),
+ 'MscgenLexer': ('pygments.lexers.dsls', 'Mscgen', ('mscgen', 'msc'), ('*.msc',), ()),
+ 'MuPADLexer': ('pygments.lexers.algebra', 'MuPAD', ('mupad',), ('*.mu',), ()),
+ 'MxmlLexer': ('pygments.lexers.actionscript', 'MXML', ('mxml',), ('*.mxml',), ()),
+ 'MySqlLexer': ('pygments.lexers.sql', 'MySQL', ('mysql',), (), ('text/x-mysql',)),
+ 'MyghtyCssLexer': ('pygments.lexers.templates', 'CSS+Myghty', ('css+myghty',), (), ('text/css+myghty',)),
+ 'MyghtyHtmlLexer': ('pygments.lexers.templates', 'HTML+Myghty', ('html+myghty',), (), ('text/html+myghty',)),
+ 'MyghtyJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Myghty', ('js+myghty', 'javascript+myghty'), (), ('application/x-javascript+myghty', 'text/x-javascript+myghty', 'text/javascript+mygthy')),
+ 'MyghtyLexer': ('pygments.lexers.templates', 'Myghty', ('myghty',), ('*.myt', 'autodelegate'), ('application/x-myghty',)),
+ 'MyghtyXmlLexer': ('pygments.lexers.templates', 'XML+Myghty', ('xml+myghty',), (), ('application/xml+myghty',)),
+ 'NCLLexer': ('pygments.lexers.ncl', 'NCL', ('ncl',), ('*.ncl',), ('text/ncl',)),
+ 'NSISLexer': ('pygments.lexers.installers', 'NSIS', ('nsis', 'nsi', 'nsh'), ('*.nsi', '*.nsh'), ('text/x-nsis',)),
+ 'NasmLexer': ('pygments.lexers.asm', 'NASM', ('nasm',), ('*.asm', '*.ASM'), ('text/x-nasm',)),
+ 'NasmObjdumpLexer': ('pygments.lexers.asm', 'objdump-nasm', ('objdump-nasm',), ('*.objdump-intel',), ('text/x-nasm-objdump',)),
+ 'NemerleLexer': ('pygments.lexers.dotnet', 'Nemerle', ('nemerle',), ('*.n',), ('text/x-nemerle',)),
+ 'NesCLexer': ('pygments.lexers.c_like', 'nesC', ('nesc',), ('*.nc',), ('text/x-nescsrc',)),
+ 'NewLispLexer': ('pygments.lexers.lisp', 'NewLisp', ('newlisp',), ('*.lsp', '*.nl', '*.kif'), ('text/x-newlisp', 'application/x-newlisp')),
+ 'NewspeakLexer': ('pygments.lexers.smalltalk', 'Newspeak', ('newspeak',), ('*.ns2',), ('text/x-newspeak',)),
+ 'NginxConfLexer': ('pygments.lexers.configs', 'Nginx configuration file', ('nginx',), ('nginx.conf',), ('text/x-nginx-conf',)),
+ 'NimrodLexer': ('pygments.lexers.nimrod', 'Nimrod', ('nim', 'nimrod'), ('*.nim', '*.nimrod'), ('text/x-nim',)),
+ 'NitLexer': ('pygments.lexers.nit', 'Nit', ('nit',), ('*.nit',), ()),
+ 'NixLexer': ('pygments.lexers.nix', 'Nix', ('nixos', 'nix'), ('*.nix',), ('text/x-nix',)),
+ 'NuSMVLexer': ('pygments.lexers.smv', 'NuSMV', ('nusmv',), ('*.smv',), ()),
+ 'NumPyLexer': ('pygments.lexers.python', 'NumPy', ('numpy',), (), ()),
+ 'ObjdumpLexer': ('pygments.lexers.asm', 'objdump', ('objdump',), ('*.objdump',), ('text/x-objdump',)),
+ 'ObjectiveCLexer': ('pygments.lexers.objective', 'Objective-C', ('objective-c', 'objectivec', 'obj-c', 'objc'), ('*.m', '*.h'), ('text/x-objective-c',)),
+ 'ObjectiveCppLexer': ('pygments.lexers.objective', 'Objective-C++', ('objective-c++', 'objectivec++', 'obj-c++', 'objc++'), ('*.mm', '*.hh'), ('text/x-objective-c++',)),
+ 'ObjectiveJLexer': ('pygments.lexers.javascript', 'Objective-J', ('objective-j', 'objectivej', 'obj-j', 'objj'), ('*.j',), ('text/x-objective-j',)),
+ 'OcamlLexer': ('pygments.lexers.ml', 'OCaml', ('ocaml',), ('*.ml', '*.mli', '*.mll', '*.mly'), ('text/x-ocaml',)),
+ 'OctaveLexer': ('pygments.lexers.matlab', 'Octave', ('octave',), ('*.m',), ('text/octave',)),
+ 'OdinLexer': ('pygments.lexers.archetype', 'ODIN', ('odin',), ('*.odin',), ('text/odin',)),
+ 'OocLexer': ('pygments.lexers.ooc', 'Ooc', ('ooc',), ('*.ooc',), ('text/x-ooc',)),
+ 'OpaLexer': ('pygments.lexers.ml', 'Opa', ('opa',), ('*.opa',), ('text/x-opa',)),
+ 'OpenEdgeLexer': ('pygments.lexers.business', 'OpenEdge ABL', ('openedge', 'abl', 'progress'), ('*.p', '*.cls'), ('text/x-openedge', 'application/x-openedge')),
+ 'PacmanConfLexer': ('pygments.lexers.configs', 'PacmanConf', ('pacmanconf',), ('pacman.conf',), ()),
+ 'PanLexer': ('pygments.lexers.dsls', 'Pan', ('pan',), ('*.pan',), ()),
+ 'ParaSailLexer': ('pygments.lexers.parasail', 'ParaSail', ('parasail',), ('*.psi', '*.psl'), ('text/x-parasail',)),
+ 'PawnLexer': ('pygments.lexers.pawn', 'Pawn', ('pawn',), ('*.p', '*.pwn', '*.inc'), ('text/x-pawn',)),
+ 'Perl6Lexer': ('pygments.lexers.perl', 'Perl6', ('perl6', 'pl6'), ('*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6', '*.6pm', '*.p6m', '*.pm6', '*.t'), ('text/x-perl6', 'application/x-perl6')),
+ 'PerlLexer': ('pygments.lexers.perl', 'Perl', ('perl', 'pl'), ('*.pl', '*.pm', '*.t'), ('text/x-perl', 'application/x-perl')),
+ 'PhpLexer': ('pygments.lexers.php', 'PHP', ('php', 'php3', 'php4', 'php5'), ('*.php', '*.php[345]', '*.inc'), ('text/x-php',)),
+ 'PigLexer': ('pygments.lexers.jvm', 'Pig', ('pig',), ('*.pig',), ('text/x-pig',)),
+ 'PikeLexer': ('pygments.lexers.c_like', 'Pike', ('pike',), ('*.pike', '*.pmod'), ('text/x-pike',)),
+ 'PkgConfigLexer': ('pygments.lexers.configs', 'PkgConfig', ('pkgconfig',), ('*.pc',), ()),
+ 'PlPgsqlLexer': ('pygments.lexers.sql', 'PL/pgSQL', ('plpgsql',), (), ('text/x-plpgsql',)),
+ 'PostScriptLexer': ('pygments.lexers.graphics', 'PostScript', ('postscript', 'postscr'), ('*.ps', '*.eps'), ('application/postscript',)),
+ 'PostgresConsoleLexer': ('pygments.lexers.sql', 'PostgreSQL console (psql)', ('psql', 'postgresql-console', 'postgres-console'), (), ('text/x-postgresql-psql',)),
+ 'PostgresLexer': ('pygments.lexers.sql', 'PostgreSQL SQL dialect', ('postgresql', 'postgres'), (), ('text/x-postgresql',)),
+ 'PovrayLexer': ('pygments.lexers.graphics', 'POVRay', ('pov',), ('*.pov', '*.inc'), ('text/x-povray',)),
+ 'PowerShellLexer': ('pygments.lexers.shell', 'PowerShell', ('powershell', 'posh', 'ps1', 'psm1'), ('*.ps1', '*.psm1'), ('text/x-powershell',)),
+ 'PowerShellSessionLexer': ('pygments.lexers.shell', 'PowerShell Session', ('ps1con',), (), ()),
+ 'PraatLexer': ('pygments.lexers.praat', 'Praat', ('praat',), ('*.praat', '*.proc', '*.psc'), ()),
+ 'PrologLexer': ('pygments.lexers.prolog', 'Prolog', ('prolog',), ('*.ecl', '*.prolog', '*.pro', '*.pl'), ('text/x-prolog',)),
+ 'PropertiesLexer': ('pygments.lexers.configs', 'Properties', ('properties', 'jproperties'), ('*.properties',), ('text/x-java-properties',)),
+ 'ProtoBufLexer': ('pygments.lexers.dsls', 'Protocol Buffer', ('protobuf', 'proto'), ('*.proto',), ()),
+ 'PugLexer': ('pygments.lexers.html', 'Pug', ('pug', 'jade'), ('*.pug', '*.jade'), ('text/x-pug', 'text/x-jade')),
+ 'PuppetLexer': ('pygments.lexers.dsls', 'Puppet', ('puppet',), ('*.pp',), ()),
+ 'PyPyLogLexer': ('pygments.lexers.console', 'PyPy Log', ('pypylog', 'pypy'), ('*.pypylog',), ('application/x-pypylog',)),
+ 'Python3Lexer': ('pygments.lexers.python', 'Python 3', ('python3', 'py3'), (), ('text/x-python3', 'application/x-python3')),
+ 'Python3TracebackLexer': ('pygments.lexers.python', 'Python 3.0 Traceback', ('py3tb',), ('*.py3tb',), ('text/x-python3-traceback',)),
+ 'PythonConsoleLexer': ('pygments.lexers.python', 'Python console session', ('pycon',), (), ('text/x-python-doctest',)),
+ 'PythonLexer': ('pygments.lexers.python', 'Python', ('python', 'py', 'sage'), ('*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac', '*.sage'), ('text/x-python', 'application/x-python')),
+ 'PythonTracebackLexer': ('pygments.lexers.python', 'Python Traceback', ('pytb',), ('*.pytb',), ('text/x-python-traceback',)),
+ 'QBasicLexer': ('pygments.lexers.basic', 'QBasic', ('qbasic', 'basic'), ('*.BAS', '*.bas'), ('text/basic',)),
+ 'QVToLexer': ('pygments.lexers.qvt', 'QVTO', ('qvto', 'qvt'), ('*.qvto',), ()),
+ 'QmlLexer': ('pygments.lexers.webmisc', 'QML', ('qml', 'qbs'), ('*.qml', '*.qbs'), ('application/x-qml', 'application/x-qt.qbs+qml')),
+ 'RConsoleLexer': ('pygments.lexers.r', 'RConsole', ('rconsole', 'rout'), ('*.Rout',), ()),
+ 'RNCCompactLexer': ('pygments.lexers.rnc', 'Relax-NG Compact', ('rnc', 'rng-compact'), ('*.rnc',), ()),
+ 'RPMSpecLexer': ('pygments.lexers.installers', 'RPMSpec', ('spec',), ('*.spec',), ('text/x-rpm-spec',)),
+ 'RacketLexer': ('pygments.lexers.lisp', 'Racket', ('racket', 'rkt'), ('*.rkt', '*.rktd', '*.rktl'), ('text/x-racket', 'application/x-racket')),
+ 'RagelCLexer': ('pygments.lexers.parsers', 'Ragel in C Host', ('ragel-c',), ('*.rl',), ()),
+ 'RagelCppLexer': ('pygments.lexers.parsers', 'Ragel in CPP Host', ('ragel-cpp',), ('*.rl',), ()),
+ 'RagelDLexer': ('pygments.lexers.parsers', 'Ragel in D Host', ('ragel-d',), ('*.rl',), ()),
+ 'RagelEmbeddedLexer': ('pygments.lexers.parsers', 'Embedded Ragel', ('ragel-em',), ('*.rl',), ()),
+ 'RagelJavaLexer': ('pygments.lexers.parsers', 'Ragel in Java Host', ('ragel-java',), ('*.rl',), ()),
+ 'RagelLexer': ('pygments.lexers.parsers', 'Ragel', ('ragel',), (), ()),
+ 'RagelObjectiveCLexer': ('pygments.lexers.parsers', 'Ragel in Objective C Host', ('ragel-objc',), ('*.rl',), ()),
+ 'RagelRubyLexer': ('pygments.lexers.parsers', 'Ragel in Ruby Host', ('ragel-ruby', 'ragel-rb'), ('*.rl',), ()),
+ 'RawTokenLexer': ('pygments.lexers.special', 'Raw token data', ('raw',), (), ('application/x-pygments-tokens',)),
+ 'RdLexer': ('pygments.lexers.r', 'Rd', ('rd',), ('*.Rd',), ('text/x-r-doc',)),
+ 'RebolLexer': ('pygments.lexers.rebol', 'REBOL', ('rebol',), ('*.r', '*.r3', '*.reb'), ('text/x-rebol',)),
+ 'RedLexer': ('pygments.lexers.rebol', 'Red', ('red', 'red/system'), ('*.red', '*.reds'), ('text/x-red', 'text/x-red-system')),
+ 'RedcodeLexer': ('pygments.lexers.esoteric', 'Redcode', ('redcode',), ('*.cw',), ()),
+ 'RegeditLexer': ('pygments.lexers.configs', 'reg', ('registry',), ('*.reg',), ('text/x-windows-registry',)),
+ 'ResourceLexer': ('pygments.lexers.resource', 'ResourceBundle', ('resource', 'resourcebundle'), ('*.txt',), ()),
+ 'RexxLexer': ('pygments.lexers.scripting', 'Rexx', ('rexx', 'arexx'), ('*.rexx', '*.rex', '*.rx', '*.arexx'), ('text/x-rexx',)),
+ 'RhtmlLexer': ('pygments.lexers.templates', 'RHTML', ('rhtml', 'html+erb', 'html+ruby'), ('*.rhtml',), ('text/html+ruby',)),
+ 'RoboconfGraphLexer': ('pygments.lexers.roboconf', 'Roboconf Graph', ('roboconf-graph',), ('*.graph',), ()),
+ 'RoboconfInstancesLexer': ('pygments.lexers.roboconf', 'Roboconf Instances', ('roboconf-instances',), ('*.instances',), ()),
+ 'RobotFrameworkLexer': ('pygments.lexers.robotframework', 'RobotFramework', ('robotframework',), ('*.txt', '*.robot'), ('text/x-robotframework',)),
+ 'RqlLexer': ('pygments.lexers.sql', 'RQL', ('rql',), ('*.rql',), ('text/x-rql',)),
+ 'RslLexer': ('pygments.lexers.dsls', 'RSL', ('rsl',), ('*.rsl',), ('text/rsl',)),
+ 'RstLexer': ('pygments.lexers.markup', 'reStructuredText', ('rst', 'rest', 'restructuredtext'), ('*.rst', '*.rest'), ('text/x-rst', 'text/prs.fallenstein.rst')),
+ 'RtsLexer': ('pygments.lexers.trafficscript', 'TrafficScript', ('rts', 'trafficscript'), ('*.rts',), ()),
+ 'RubyConsoleLexer': ('pygments.lexers.ruby', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)),
+ 'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby', 'Gemfile'), ('text/x-ruby', 'application/x-ruby')),
+ 'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust',), ('*.rs', '*.rs.in'), ('text/rust',)),
+ 'SASLexer': ('pygments.lexers.sas', 'SAS', ('sas',), ('*.SAS', '*.sas'), ('text/x-sas', 'text/sas', 'application/x-sas')),
+ 'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')),
+ 'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')),
+ 'SassLexer': ('pygments.lexers.css', 'Sass', ('sass',), ('*.sass',), ('text/x-sass',)),
+ 'ScalaLexer': ('pygments.lexers.jvm', 'Scala', ('scala',), ('*.scala',), ('text/x-scala',)),
+ 'ScamlLexer': ('pygments.lexers.html', 'Scaml', ('scaml',), ('*.scaml',), ('text/x-scaml',)),
+ 'SchemeLexer': ('pygments.lexers.lisp', 'Scheme', ('scheme', 'scm'), ('*.scm', '*.ss'), ('text/x-scheme', 'application/x-scheme')),
+ 'ScilabLexer': ('pygments.lexers.matlab', 'Scilab', ('scilab',), ('*.sci', '*.sce', '*.tst'), ('text/scilab',)),
+ 'ScssLexer': ('pygments.lexers.css', 'SCSS', ('scss',), ('*.scss',), ('text/x-scss',)),
+ 'ShenLexer': ('pygments.lexers.lisp', 'Shen', ('shen',), ('*.shen',), ('text/x-shen', 'application/x-shen')),
+ 'SilverLexer': ('pygments.lexers.verification', 'Silver', ('silver',), ('*.sil', '*.vpr'), ()),
+ 'SlimLexer': ('pygments.lexers.webmisc', 'Slim', ('slim',), ('*.slim',), ('text/x-slim',)),
+ 'SmaliLexer': ('pygments.lexers.dalvik', 'Smali', ('smali',), ('*.smali',), ('text/smali',)),
+ 'SmalltalkLexer': ('pygments.lexers.smalltalk', 'Smalltalk', ('smalltalk', 'squeak', 'st'), ('*.st',), ('text/x-smalltalk',)),
+ 'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ('application/x-smarty',)),
+ 'SnobolLexer': ('pygments.lexers.snobol', 'Snobol', ('snobol',), ('*.snobol',), ('text/x-snobol',)),
+ 'SnowballLexer': ('pygments.lexers.dsls', 'Snowball', ('snowball',), ('*.sbl',), ()),
+ 'SourcePawnLexer': ('pygments.lexers.pawn', 'SourcePawn', ('sp',), ('*.sp',), ('text/x-sourcepawn',)),
+ 'SourcesListLexer': ('pygments.lexers.installers', 'Debian Sourcelist', ('sourceslist', 'sources.list', 'debsources'), ('sources.list',), ()),
+ 'SparqlLexer': ('pygments.lexers.rdf', 'SPARQL', ('sparql',), ('*.rq', '*.sparql'), ('application/sparql-query',)),
+ 'SqlLexer': ('pygments.lexers.sql', 'SQL', ('sql',), ('*.sql',), ('text/x-sql',)),
+ 'SqliteConsoleLexer': ('pygments.lexers.sql', 'sqlite3con', ('sqlite3',), ('*.sqlite3-console',), ('text/x-sqlite3-console',)),
+ 'SquidConfLexer': ('pygments.lexers.configs', 'SquidConf', ('squidconf', 'squid.conf', 'squid'), ('squid.conf',), ('text/x-squidconf',)),
+ 'SspLexer': ('pygments.lexers.templates', 'Scalate Server Page', ('ssp',), ('*.ssp',), ('application/x-ssp',)),
+ 'StanLexer': ('pygments.lexers.modeling', 'Stan', ('stan',), ('*.stan',), ()),
+ 'StataLexer': ('pygments.lexers.stata', 'Stata', ('stata', 'do'), ('*.do', '*.ado'), ('text/x-stata', 'text/stata', 'application/x-stata')),
+ 'SuperColliderLexer': ('pygments.lexers.supercollider', 'SuperCollider', ('sc', 'supercollider'), ('*.sc', '*.scd'), ('application/supercollider', 'text/supercollider')),
+ 'SwiftLexer': ('pygments.lexers.objective', 'Swift', ('swift',), ('*.swift',), ('text/x-swift',)),
+ 'SwigLexer': ('pygments.lexers.c_like', 'SWIG', ('swig',), ('*.swg', '*.i'), ('text/swig',)),
+ 'SystemVerilogLexer': ('pygments.lexers.hdl', 'systemverilog', ('systemverilog', 'sv'), ('*.sv', '*.svh'), ('text/x-systemverilog',)),
+ 'TAPLexer': ('pygments.lexers.testing', 'TAP', ('tap',), ('*.tap',), ()),
+ 'Tads3Lexer': ('pygments.lexers.int_fiction', 'TADS 3', ('tads3',), ('*.t',), ()),
+ 'TasmLexer': ('pygments.lexers.asm', 'TASM', ('tasm',), ('*.asm', '*.ASM', '*.tasm'), ('text/x-tasm',)),
+ 'TclLexer': ('pygments.lexers.tcl', 'Tcl', ('tcl',), ('*.tcl', '*.rvt'), ('text/x-tcl', 'text/x-script.tcl', 'application/x-tcl')),
+ 'TcshLexer': ('pygments.lexers.shell', 'Tcsh', ('tcsh', 'csh'), ('*.tcsh', '*.csh'), ('application/x-csh',)),
+ 'TcshSessionLexer': ('pygments.lexers.shell', 'Tcsh Session', ('tcshcon',), (), ()),
+ 'TeaTemplateLexer': ('pygments.lexers.templates', 'Tea', ('tea',), ('*.tea',), ('text/x-tea',)),
+ 'TermcapLexer': ('pygments.lexers.configs', 'Termcap', ('termcap',), ('termcap', 'termcap.src'), ()),
+ 'TerminfoLexer': ('pygments.lexers.configs', 'Terminfo', ('terminfo',), ('terminfo', 'terminfo.src'), ()),
+ 'TerraformLexer': ('pygments.lexers.configs', 'Terraform', ('terraform', 'tf'), ('*.tf',), ('application/x-tf', 'application/x-terraform')),
+ 'TexLexer': ('pygments.lexers.markup', 'TeX', ('tex', 'latex'), ('*.tex', '*.aux', '*.toc'), ('text/x-tex', 'text/x-latex')),
+ 'TextLexer': ('pygments.lexers.special', 'Text only', ('text',), ('*.txt',), ('text/plain',)),
+ 'ThriftLexer': ('pygments.lexers.dsls', 'Thrift', ('thrift',), ('*.thrift',), ('application/x-thrift',)),
+ 'TodotxtLexer': ('pygments.lexers.textfmts', 'Todotxt', ('todotxt',), ('todo.txt', '*.todotxt'), ('text/x-todo',)),
+ 'TransactSqlLexer': ('pygments.lexers.sql', 'Transact-SQL', ('tsql', 't-sql'), ('*.sql',), ('text/x-tsql',)),
+ 'TreetopLexer': ('pygments.lexers.parsers', 'Treetop', ('treetop',), ('*.treetop', '*.tt'), ()),
+ 'TurtleLexer': ('pygments.lexers.rdf', 'Turtle', ('turtle',), ('*.ttl',), ('text/turtle', 'application/x-turtle')),
+ 'TwigHtmlLexer': ('pygments.lexers.templates', 'HTML+Twig', ('html+twig',), ('*.twig',), ('text/html+twig',)),
+ 'TwigLexer': ('pygments.lexers.templates', 'Twig', ('twig',), (), ('application/x-twig',)),
+ 'TypeScriptLexer': ('pygments.lexers.javascript', 'TypeScript', ('ts', 'typescript'), ('*.ts',), ('text/x-typescript',)),
+ 'TypoScriptCssDataLexer': ('pygments.lexers.typoscript', 'TypoScriptCssData', ('typoscriptcssdata',), (), ()),
+ 'TypoScriptHtmlDataLexer': ('pygments.lexers.typoscript', 'TypoScriptHtmlData', ('typoscripthtmldata',), (), ()),
+ 'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.ts', '*.txt'), ('text/x-typoscript',)),
+ 'UrbiscriptLexer': ('pygments.lexers.urbi', 'UrbiScript', ('urbiscript',), ('*.u',), ('application/x-urbiscript',)),
+ 'VCLLexer': ('pygments.lexers.varnish', 'VCL', ('vcl',), ('*.vcl',), ('text/x-vclsrc',)),
+ 'VCLSnippetLexer': ('pygments.lexers.varnish', 'VCLSnippets', ('vclsnippets', 'vclsnippet'), (), ('text/x-vclsnippet',)),
+ 'VCTreeStatusLexer': ('pygments.lexers.console', 'VCTreeStatus', ('vctreestatus',), (), ()),
+ 'VGLLexer': ('pygments.lexers.dsls', 'VGL', ('vgl',), ('*.rpf',), ()),
+ 'ValaLexer': ('pygments.lexers.c_like', 'Vala', ('vala', 'vapi'), ('*.vala', '*.vapi'), ('text/x-vala',)),
+ 'VbNetAspxLexer': ('pygments.lexers.dotnet', 'aspx-vb', ('aspx-vb',), ('*.aspx', '*.asax', '*.ascx', '*.ashx', '*.asmx', '*.axd'), ()),
+ 'VbNetLexer': ('pygments.lexers.dotnet', 'VB.net', ('vb.net', 'vbnet'), ('*.vb', '*.bas'), ('text/x-vbnet', 'text/x-vba')),
+ 'VelocityHtmlLexer': ('pygments.lexers.templates', 'HTML+Velocity', ('html+velocity',), (), ('text/html+velocity',)),
+ 'VelocityLexer': ('pygments.lexers.templates', 'Velocity', ('velocity',), ('*.vm', '*.fhtml'), ()),
+ 'VelocityXmlLexer': ('pygments.lexers.templates', 'XML+Velocity', ('xml+velocity',), (), ('application/xml+velocity',)),
+ 'VerilogLexer': ('pygments.lexers.hdl', 'verilog', ('verilog', 'v'), ('*.v',), ('text/x-verilog',)),
+ 'VhdlLexer': ('pygments.lexers.hdl', 'vhdl', ('vhdl',), ('*.vhdl', '*.vhd'), ('text/x-vhdl',)),
+ 'VimLexer': ('pygments.lexers.textedit', 'VimL', ('vim',), ('*.vim', '.vimrc', '.exrc', '.gvimrc', '_vimrc', '_exrc', '_gvimrc', 'vimrc', 'gvimrc'), ('text/x-vim',)),
+ 'WDiffLexer': ('pygments.lexers.diff', 'WDiff', ('wdiff',), ('*.wdiff',), ()),
+ 'WhileyLexer': ('pygments.lexers.whiley', 'Whiley', ('whiley',), ('*.whiley',), ('text/x-whiley',)),
+ 'X10Lexer': ('pygments.lexers.x10', 'X10', ('x10', 'xten'), ('*.x10',), ('text/x-x10',)),
+ 'XQueryLexer': ('pygments.lexers.webmisc', 'XQuery', ('xquery', 'xqy', 'xq', 'xql', 'xqm'), ('*.xqy', '*.xquery', '*.xq', '*.xql', '*.xqm'), ('text/xquery', 'application/xquery')),
+ 'XmlDjangoLexer': ('pygments.lexers.templates', 'XML+Django/Jinja', ('xml+django', 'xml+jinja'), (), ('application/xml+django', 'application/xml+jinja')),
+ 'XmlErbLexer': ('pygments.lexers.templates', 'XML+Ruby', ('xml+erb', 'xml+ruby'), (), ('application/xml+ruby',)),
+ 'XmlLexer': ('pygments.lexers.html', 'XML', ('xml',), ('*.xml', '*.xsl', '*.rss', '*.xslt', '*.xsd', '*.wsdl', '*.wsf'), ('text/xml', 'application/xml', 'image/svg+xml', 'application/rss+xml', 'application/atom+xml')),
+ 'XmlPhpLexer': ('pygments.lexers.templates', 'XML+PHP', ('xml+php',), (), ('application/xml+php',)),
+ 'XmlSmartyLexer': ('pygments.lexers.templates', 'XML+Smarty', ('xml+smarty',), (), ('application/xml+smarty',)),
+ 'XsltLexer': ('pygments.lexers.html', 'XSLT', ('xslt',), ('*.xsl', '*.xslt', '*.xpl'), ('application/xsl+xml', 'application/xslt+xml')),
+ 'XtendLexer': ('pygments.lexers.jvm', 'Xtend', ('xtend',), ('*.xtend',), ('text/x-xtend',)),
+ 'XtlangLexer': ('pygments.lexers.lisp', 'xtlang', ('extempore',), ('*.xtm',), ()),
+ 'YamlJinjaLexer': ('pygments.lexers.templates', 'YAML+Jinja', ('yaml+jinja', 'salt', 'sls'), ('*.sls',), ('text/x-yaml+jinja', 'text/x-sls')),
+ 'YamlLexer': ('pygments.lexers.data', 'YAML', ('yaml',), ('*.yaml', '*.yml'), ('text/x-yaml',)),
+ 'ZephirLexer': ('pygments.lexers.php', 'Zephir', ('zephir',), ('*.zep',), ()),
+}
+
+if __name__ == '__main__': # pragma: no cover
+ import sys
+ import os
+
+ # lookup lexers
+ found_lexers = []
+ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
+ for root, dirs, files in os.walk('.'):
+ for filename in files:
+ if filename.endswith('.py') and not filename.startswith('_'):
+ module_name = 'pygments.lexers%s.%s' % (
+ root[1:].replace('/', '.'), filename[:-3])
+ print(module_name)
+ module = __import__(module_name, None, None, [''])
+ for lexer_name in module.__all__:
+ lexer = getattr(module, lexer_name)
+ found_lexers.append(
+ '%r: %r' % (lexer_name,
+ (module_name,
+ lexer.name,
+ tuple(lexer.aliases),
+ tuple(lexer.filenames),
+ tuple(lexer.mimetypes))))
+ # sort them to make the diff minimal
+ found_lexers.sort()
+
+ # extract useful sourcecode from this file
+ with open(__file__) as fp:
+ content = fp.read()
+ # replace crnl to nl for Windows.
+ #
+ # Note that, originally, contributers should keep nl of master
+ # repository, for example by using some kind of automatic
+ # management EOL, like `EolExtension
+ # <https://www.mercurial-scm.org/wiki/EolExtension>`.
+ content = content.replace("\r\n", "\n")
+ header = content[:content.find('LEXERS = {')]
+ footer = content[content.find("if __name__ == '__main__':"):]
+
+ # write new file
+ with open(__file__, 'w') as fp:
+ fp.write(header)
+ fp.write('LEXERS = {\n %s,\n}\n\n' % ',\n '.join(found_lexers))
+ fp.write(footer)
+
+ print ('=== %d lexers processed.' % len(found_lexers))
diff --git a/wandb/vendor/pygments/lexers/_mql_builtins.py b/wandb/vendor/pygments/lexers/_mql_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..6eb600c446780ada9ec9042cbcce9373e646134f
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_mql_builtins.py
@@ -0,0 +1,1172 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._mql_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Builtins for the MqlLexer.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+types = (
+ 'AccountBalance',
+ 'AccountCompany',
+ 'AccountCredit',
+ 'AccountCurrency',
+ 'AccountEquity',
+ 'AccountFreeMarginCheck',
+ 'AccountFreeMarginMode',
+ 'AccountFreeMargin',
+ 'AccountInfoDouble',
+ 'AccountInfoInteger',
+ 'AccountInfoString',
+ 'AccountLeverage',
+ 'AccountMargin',
+ 'AccountName',
+ 'AccountNumber',
+ 'AccountProfit',
+ 'AccountServer',
+ 'AccountStopoutLevel',
+ 'AccountStopoutMode',
+ 'Alert',
+ 'ArrayBsearch',
+ 'ArrayCompare',
+ 'ArrayCopyRates',
+ 'ArrayCopySeries',
+ 'ArrayCopy',
+ 'ArrayDimension',
+ 'ArrayFill',
+ 'ArrayFree',
+ 'ArrayGetAsSeries',
+ 'ArrayInitialize',
+ 'ArrayIsDynamic',
+ 'ArrayIsSeries',
+ 'ArrayMaximum',
+ 'ArrayMinimum',
+ 'ArrayRange',
+ 'ArrayResize',
+ 'ArraySetAsSeries',
+ 'ArraySize',
+ 'ArraySort',
+ 'CharArrayToString',
+ 'CharToString',
+ 'CharToStr',
+ 'CheckPointer',
+ 'ColorToARGB',
+ 'ColorToString',
+ 'Comment',
+ 'CopyClose',
+ 'CopyHigh',
+ 'CopyLow',
+ 'CopyOpen',
+ 'CopyRates',
+ 'CopyRealVolume',
+ 'CopySpread',
+ 'CopyTickVolume',
+ 'CopyTime',
+ 'DayOfWeek',
+ 'DayOfYear',
+ 'Day',
+ 'DebugBreak',
+ 'Digits',
+ 'DoubleToString',
+ 'DoubleToStr',
+ 'EnumToString',
+ 'EventChartCustom',
+ 'EventKillTimer',
+ 'EventSetMillisecondTimer',
+ 'EventSetTimer',
+ 'ExpertRemove',
+ 'FileClose',
+ 'FileCopy',
+ 'FileDelete',
+ 'FileFindClose',
+ 'FileFindFirst',
+ 'FileFindNext',
+ 'FileFlush',
+ 'FileGetInteger',
+ 'FileIsEnding',
+ 'FileIsExist',
+ 'FileIsLineEnding',
+ 'FileMove',
+ 'FileOpenHistory',
+ 'FileOpen',
+ 'FileReadArray',
+ 'FileReadBool',
+ 'FileReadDatetime',
+ 'FileReadDouble',
+ 'FileReadFloat',
+ 'FileReadInteger',
+ 'FileReadLong',
+ 'FileReadNumber',
+ 'FileReadString',
+ 'FileReadStruct',
+ 'FileSeek',
+ 'FileSize',
+ 'FileTell',
+ 'FileWriteArray',
+ 'FileWriteDouble',
+ 'FileWriteFloat',
+ 'FileWriteInteger',
+ 'FileWriteLong',
+ 'FileWriteString',
+ 'FileWriteStruct',
+ 'FileWrite',
+ 'FolderClean',
+ 'FolderCreate',
+ 'FolderDelete',
+ 'GetLastError',
+ 'GetPointer',
+ 'GetTickCount',
+ 'GlobalVariableCheck',
+ 'GlobalVariableDel',
+ 'GlobalVariableGet',
+ 'GlobalVariableName',
+ 'GlobalVariableSetOnCondition',
+ 'GlobalVariableSet',
+ 'GlobalVariableTemp',
+ 'GlobalVariableTime',
+ 'GlobalVariablesDeleteAll',
+ 'GlobalVariablesFlush',
+ 'GlobalVariablesTotal',
+ 'HideTestIndicators',
+ 'Hour',
+ 'IndicatorBuffers',
+ 'IndicatorCounted',
+ 'IndicatorDigits',
+ 'IndicatorSetDouble',
+ 'IndicatorSetInteger',
+ 'IndicatorSetString',
+ 'IndicatorShortName',
+ 'IntegerToString',
+ 'IsConnected',
+ 'IsDemo',
+ 'IsDllsAllowed',
+ 'IsExpertEnabled',
+ 'IsLibrariesAllowed',
+ 'IsOptimization',
+ 'IsStopped',
+ 'IsTesting',
+ 'IsTradeAllowed',
+ 'IsTradeContextBusy',
+ 'IsVisualMode',
+ 'MQLInfoInteger',
+ 'MQLInfoString',
+ 'MarketInfo',
+ 'MathAbs',
+ 'MathArccos',
+ 'MathArcsin',
+ 'MathArctan',
+ 'MathCeil',
+ 'MathCos',
+ 'MathExp',
+ 'MathFloor',
+ 'MathIsValidNumber',
+ 'MathLog',
+ 'MathMax',
+ 'MathMin',
+ 'MathMod',
+ 'MathPow',
+ 'MathRand',
+ 'MathRound',
+ 'MathSin',
+ 'MathSqrt',
+ 'MathSrand',
+ 'MathTan',
+ 'MessageBox',
+ 'Minute',
+ 'Month',
+ 'NormalizeDouble',
+ 'ObjectCreate',
+ 'ObjectDelete',
+ 'ObjectDescription',
+ 'ObjectFind',
+ 'ObjectGetDouble',
+ 'ObjectGetFiboDescription',
+ 'ObjectGetInteger',
+ 'ObjectGetShiftByValue',
+ 'ObjectGetString',
+ 'ObjectGetTimeByValue',
+ 'ObjectGetValueByShift',
+ 'ObjectGetValueByTime',
+ 'ObjectGet',
+ 'ObjectMove',
+ 'ObjectName',
+ 'ObjectSetDouble',
+ 'ObjectSetFiboDescription',
+ 'ObjectSetInteger',
+ 'ObjectSetString',
+ 'ObjectSetText',
+ 'ObjectSet',
+ 'ObjectType',
+ 'ObjectsDeleteAll',
+ 'ObjectsTotal',
+ 'OrderCloseBy',
+ 'OrderClosePrice',
+ 'OrderCloseTime',
+ 'OrderClose',
+ 'OrderComment',
+ 'OrderCommission',
+ 'OrderDelete',
+ 'OrderExpiration',
+ 'OrderLots',
+ 'OrderMagicNumber',
+ 'OrderModify',
+ 'OrderOpenPrice',
+ 'OrderOpenTime',
+ 'OrderPrint',
+ 'OrderProfit',
+ 'OrderSelect',
+ 'OrderSend',
+ 'OrderStopLoss',
+ 'OrderSwap',
+ 'OrderSymbol',
+ 'OrderTakeProfit',
+ 'OrderTicket',
+ 'OrderType',
+ 'OrdersHistoryTotal',
+ 'OrdersTotal',
+ 'PeriodSeconds',
+ 'Period',
+ 'PlaySound',
+ 'Point',
+ 'PrintFormat',
+ 'Print',
+ 'RefreshRates',
+ 'ResetLastError',
+ 'ResourceCreate',
+ 'ResourceFree',
+ 'ResourceReadImage',
+ 'ResourceSave',
+ 'Seconds',
+ 'SendFTP',
+ 'SendMail',
+ 'SendNotification',
+ 'SeriesInfoInteger',
+ 'SetIndexArrow',
+ 'SetIndexBuffer',
+ 'SetIndexDrawBegin',
+ 'SetIndexEmptyValue',
+ 'SetIndexLabel',
+ 'SetIndexShift',
+ 'SetIndexStyle',
+ 'SetLevelStyle',
+ 'SetLevelValue',
+ 'ShortArrayToString',
+ 'ShortToString',
+ 'Sleep',
+ 'StrToDouble',
+ 'StrToInteger',
+ 'StrToTime',
+ 'StringAdd',
+ 'StringBufferLen',
+ 'StringCompare',
+ 'StringConcatenate',
+ 'StringFill',
+ 'StringFind',
+ 'StringFormat',
+ 'StringGetCharacter',
+ 'StringGetChar',
+ 'StringInit',
+ 'StringLen',
+ 'StringReplace',
+ 'StringSetCharacter',
+ 'StringSetChar',
+ 'StringSplit',
+ 'StringSubstr',
+ 'StringToCharArray',
+ 'StringToColor',
+ 'StringToDouble',
+ 'StringToInteger',
+ 'StringToLower',
+ 'StringToShortArray',
+ 'StringToTime',
+ 'StringToUpper',
+ 'StringTrimLeft',
+ 'StringTrimRight',
+ 'StructToTime',
+ 'SymbolInfoDouble',
+ 'SymbolInfoInteger',
+ 'SymbolInfoSessionQuote',
+ 'SymbolInfoSessionTrade',
+ 'SymbolInfoString',
+ 'SymbolInfoTick',
+ 'SymbolIsSynchronized',
+ 'SymbolName',
+ 'SymbolSelect',
+ 'SymbolsTotal',
+ 'Symbol',
+ 'TerminalClose',
+ 'TerminalCompany',
+ 'TerminalName',
+ 'TerminalPath',
+ 'TesterStatistics',
+ 'TextGetSize',
+ 'TextOut',
+ 'TextSetFont',
+ 'TimeCurrent',
+ 'TimeDayOfWeek',
+ 'TimeDayOfYear',
+ 'TimeDaylightSavings',
+ 'TimeDay',
+ 'TimeGMTOffset',
+ 'TimeGMT',
+ 'TimeHour',
+ 'TimeLocal',
+ 'TimeMinute',
+ 'TimeMonth',
+ 'TimeSeconds',
+ 'TimeToString',
+ 'TimeToStruct',
+ 'TimeToStr',
+ 'TimeTradeServer',
+ 'TimeYear',
+ 'UninitializeReason',
+ 'WindowBarsPerChart',
+ 'WindowExpertName',
+ 'WindowFind',
+ 'WindowFirstVisibleBar',
+ 'WindowHandle',
+ 'WindowIsVisible',
+ 'WindowOnDropped',
+ 'WindowPriceMax',
+ 'WindowPriceMin',
+ 'WindowPriceOnDropped',
+ 'WindowRedraw',
+ 'WindowScreenShot',
+ 'WindowTimeOnDropped',
+ 'WindowXOnDropped',
+ 'WindowYOnDropped',
+ 'WindowsTotal',
+ 'Year',
+ 'ZeroMemory',
+ 'iAC',
+ 'iADX',
+ 'iAD',
+ 'iAO',
+ 'iATR',
+ 'iAlligator',
+ 'iBWMFI',
+ 'iBandsOnArray',
+ 'iBands',
+ 'iBarShift',
+ 'iBars',
+ 'iBearsPower',
+ 'iBullsPower',
+ 'iCCIOnArray',
+ 'iCCI',
+ 'iClose',
+ 'iCustom',
+ 'iDeMarker',
+ 'iEnvelopesOnArray',
+ 'iEnvelopes',
+ 'iForce',
+ 'iFractals',
+ 'iGator',
+ 'iHighest',
+ 'iHigh',
+ 'iIchimoku',
+ 'iLowest',
+ 'iLow',
+ 'iMACD',
+ 'iMAOnArray',
+ 'iMA',
+ 'iMFI',
+ 'iMomentumOnArray',
+ 'iMomentum',
+ 'iOBV',
+ 'iOpen',
+ 'iOsMA',
+ 'iRSIOnArray',
+ 'iRSI',
+ 'iRVI',
+ 'iSAR',
+ 'iStdDevOnArray',
+ 'iStdDev',
+ 'iStochastic',
+ 'iTime',
+ 'iVolume',
+ 'iWPR',
+)
+
+constants = (
+ 'ACCOUNT_BALANCE',
+ 'ACCOUNT_COMPANY',
+ 'ACCOUNT_CREDIT',
+ 'ACCOUNT_CURRENCY',
+ 'ACCOUNT_EQUITY',
+ 'ACCOUNT_FREEMARGIN',
+ 'ACCOUNT_LEVERAGE',
+ 'ACCOUNT_LIMIT_ORDERS',
+ 'ACCOUNT_LOGIN',
+ 'ACCOUNT_MARGIN',
+ 'ACCOUNT_MARGIN_LEVEL',
+ 'ACCOUNT_MARGIN_SO_CALL',
+ 'ACCOUNT_MARGIN_SO_MODE',
+ 'ACCOUNT_MARGIN_SO_SO',
+ 'ACCOUNT_NAME',
+ 'ACCOUNT_PROFIT',
+ 'ACCOUNT_SERVER',
+ 'ACCOUNT_STOPOUT_MODE_MONEY',
+ 'ACCOUNT_STOPOUT_MODE_PERCENT',
+ 'ACCOUNT_TRADE_ALLOWED',
+ 'ACCOUNT_TRADE_EXPERT',
+ 'ACCOUNT_TRADE_MODE',
+ 'ACCOUNT_TRADE_MODE_CONTEST',
+ 'ACCOUNT_TRADE_MODE_DEMO',
+ 'ACCOUNT_TRADE_MODE_REAL',
+ 'ALIGN_CENTER',
+ 'ALIGN_LEFT',
+ 'ALIGN_RIGHT',
+ 'ANCHOR_BOTTOM',
+ 'ANCHOR_CENTER',
+ 'ANCHOR_LEFT',
+ 'ANCHOR_LEFT_LOWER',
+ 'ANCHOR_LEFT_UPPER',
+ 'ANCHOR_LOWER',
+ 'ANCHOR_RIGHT',
+ 'ANCHOR_RIGHT_LOWER',
+ 'ANCHOR_RIGHT_UPPER',
+ 'ANCHOR_TOP',
+ 'ANCHOR_UPPER',
+ 'BORDER_FLAT',
+ 'BORDER_RAISED',
+ 'BORDER_SUNKEN',
+ 'CHARTEVENT_CHART_CHANGE',
+ 'CHARTEVENT_CLICK',
+ 'CHARTEVENT_CUSTOM',
+ 'CHARTEVENT_CUSTOM_LAST',
+ 'CHARTEVENT_KEYDOWN',
+ 'CHARTEVENT_MOUSE_MOVE',
+ 'CHARTEVENT_OBJECT_CHANGE',
+ 'CHARTEVENT_OBJECT_CLICK',
+ 'CHARTEVENT_OBJECT_CREATE',
+ 'CHARTEVENT_OBJECT_DELETE',
+ 'CHARTEVENT_OBJECT_DRAG',
+ 'CHARTEVENT_OBJECT_ENDEDIT',
+ 'CHARTS_MAX',
+ 'CHART_AUTOSCROLL',
+ 'CHART_BARS',
+ 'CHART_BEGIN',
+ 'CHART_BRING_TO_TOP',
+ 'CHART_CANDLES',
+ 'CHART_COLOR_ASK',
+ 'CHART_COLOR_BACKGROUND',
+ 'CHART_COLOR_BID',
+ 'CHART_COLOR_CANDLE_BEAR',
+ 'CHART_COLOR_CANDLE_BULL',
+ 'CHART_COLOR_CHART_DOWN',
+ 'CHART_COLOR_CHART_LINE',
+ 'CHART_COLOR_CHART_UP',
+ 'CHART_COLOR_FOREGROUND',
+ 'CHART_COLOR_GRID',
+ 'CHART_COLOR_LAST',
+ 'CHART_COLOR_STOP_LEVEL',
+ 'CHART_COLOR_VOLUME',
+ 'CHART_COMMENT',
+ 'CHART_CURRENT_POS',
+ 'CHART_DRAG_TRADE_LEVELS',
+ 'CHART_END',
+ 'CHART_EVENT_MOUSE_MOVE',
+ 'CHART_EVENT_OBJECT_CREATE',
+ 'CHART_EVENT_OBJECT_DELETE',
+ 'CHART_FIRST_VISIBLE_BAR',
+ 'CHART_FIXED_MAX',
+ 'CHART_FIXED_MIN',
+ 'CHART_FIXED_POSITION',
+ 'CHART_FOREGROUND',
+ 'CHART_HEIGHT_IN_PIXELS',
+ 'CHART_IS_OBJECT',
+ 'CHART_LINE',
+ 'CHART_MODE',
+ 'CHART_MOUSE_SCROLL',
+ 'CHART_POINTS_PER_BAR',
+ 'CHART_PRICE_MAX',
+ 'CHART_PRICE_MIN',
+ 'CHART_SCALEFIX',
+ 'CHART_SCALEFIX_11',
+ 'CHART_SCALE',
+ 'CHART_SCALE_PT_PER_BAR',
+ 'CHART_SHIFT',
+ 'CHART_SHIFT_SIZE',
+ 'CHART_SHOW_ASK_LINE',
+ 'CHART_SHOW_BID_LINE',
+ 'CHART_SHOW_DATE_SCALE',
+ 'CHART_SHOW_GRID',
+ 'CHART_SHOW_LAST_LINE',
+ 'CHART_SHOW_OBJECT_DESCR',
+ 'CHART_SHOW_OHLC',
+ 'CHART_SHOW_PERIOD_SEP',
+ 'CHART_SHOW_PRICE_SCALE',
+ 'CHART_SHOW_TRADE_LEVELS',
+ 'CHART_SHOW_VOLUMES',
+ 'CHART_VISIBLE_BARS',
+ 'CHART_VOLUME_HIDE',
+ 'CHART_VOLUME_REAL',
+ 'CHART_VOLUME_TICK',
+ 'CHART_WIDTH_IN_BARS',
+ 'CHART_WIDTH_IN_PIXELS',
+ 'CHART_WINDOWS_TOTAL',
+ 'CHART_WINDOW_HANDLE',
+ 'CHART_WINDOW_IS_VISIBLE',
+ 'CHART_WINDOW_YDISTANCE',
+ 'CHAR_MAX',
+ 'CHAR_MIN',
+ 'CLR_NONE',
+ 'CORNER_LEFT_LOWER',
+ 'CORNER_LEFT_UPPER',
+ 'CORNER_RIGHT_LOWER',
+ 'CORNER_RIGHT_UPPER',
+ 'CP_ACP',
+ 'CP_MACCP',
+ 'CP_OEMCP',
+ 'CP_SYMBOL',
+ 'CP_THREAD_ACP',
+ 'CP_UTF7',
+ 'CP_UTF8',
+ 'DBL_DIG',
+ 'DBL_EPSILON',
+ 'DBL_MANT_DIG',
+ 'DBL_MAX',
+ 'DBL_MAX_10_EXP',
+ 'DBL_MAX_EXP',
+ 'DBL_MIN',
+ 'DBL_MIN_10_EXP',
+ 'DBL_MIN_EXP',
+ 'DRAW_ARROW',
+ 'DRAW_FILLING',
+ 'DRAW_HISTOGRAM',
+ 'DRAW_LINE',
+ 'DRAW_NONE',
+ 'DRAW_SECTION',
+ 'DRAW_ZIGZAG',
+ 'EMPTY',
+ 'EMPTY_VALUE',
+ 'ERR_ACCOUNT_DISABLED',
+ 'ERR_BROKER_BUSY',
+ 'ERR_COMMON_ERROR',
+ 'ERR_INVALID_ACCOUNT',
+ 'ERR_INVALID_PRICE',
+ 'ERR_INVALID_STOPS',
+ 'ERR_INVALID_TRADE_PARAMETERS',
+ 'ERR_INVALID_TRADE_VOLUME',
+ 'ERR_LONG_POSITIONS_ONLY_ALLOWED',
+ 'ERR_MALFUNCTIONAL_TRADE',
+ 'ERR_MARKET_CLOSED',
+ 'ERR_NOT_ENOUGH_MONEY',
+ 'ERR_NOT_ENOUGH_RIGHTS',
+ 'ERR_NO_CONNECTION',
+ 'ERR_NO_ERROR',
+ 'ERR_NO_RESULT',
+ 'ERR_OFF_QUOTES',
+ 'ERR_OLD_VERSION',
+ 'ERR_ORDER_LOCKED',
+ 'ERR_PRICE_CHANGED',
+ 'ERR_REQUOTE',
+ 'ERR_SERVER_BUSY',
+ 'ERR_TOO_FREQUENT_REQUESTS',
+ 'ERR_TOO_MANY_REQUESTS',
+ 'ERR_TRADE_CONTEXT_BUSY',
+ 'ERR_TRADE_DISABLED',
+ 'ERR_TRADE_EXPIRATION_DENIED',
+ 'ERR_TRADE_HEDGE_PROHIBITED',
+ 'ERR_TRADE_MODIFY_DENIED',
+ 'ERR_TRADE_PROHIBITED_BY_FIFO',
+ 'ERR_TRADE_TIMEOUT',
+ 'ERR_TRADE_TOO_MANY_ORDERS',
+ 'FILE_ACCESS_DATE',
+ 'FILE_ANSI',
+ 'FILE_BIN',
+ 'FILE_COMMON',
+ 'FILE_CREATE_DATE',
+ 'FILE_CSV',
+ 'FILE_END',
+ 'FILE_EXISTS',
+ 'FILE_IS_ANSI',
+ 'FILE_IS_BINARY',
+ 'FILE_IS_COMMON',
+ 'FILE_IS_CSV',
+ 'FILE_IS_READABLE',
+ 'FILE_IS_TEXT',
+ 'FILE_IS_WRITABLE',
+ 'FILE_LINE_END',
+ 'FILE_MODIFY_DATE',
+ 'FILE_POSITION',
+ 'FILE_READ',
+ 'FILE_REWRITE',
+ 'FILE_SHARE_READ',
+ 'FILE_SHARE_WRITE',
+ 'FILE_SIZE',
+ 'FILE_TXT',
+ 'FILE_UNICODE',
+ 'FILE_WRITE',
+ 'FLT_DIG',
+ 'FLT_EPSILON',
+ 'FLT_MANT_DIG',
+ 'FLT_MAX',
+ 'FLT_MAX_10_EXP',
+ 'FLT_MAX_EXP',
+ 'FLT_MIN',
+ 'FLT_MIN_10_EXP',
+ 'FLT_MIN_EXP',
+ 'FRIDAY',
+ 'GANN_DOWN_TREND',
+ 'GANN_UP_TREND',
+ 'IDABORT',
+ 'IDCANCEL',
+ 'IDCONTINUE',
+ 'IDIGNORE',
+ 'IDNO',
+ 'IDOK',
+ 'IDRETRY',
+ 'IDTRYAGAIN',
+ 'IDYES',
+ 'INDICATOR_CALCULATIONS',
+ 'INDICATOR_COLOR_INDEX',
+ 'INDICATOR_DATA',
+ 'INDICATOR_DIGITS',
+ 'INDICATOR_HEIGHT',
+ 'INDICATOR_LEVELCOLOR',
+ 'INDICATOR_LEVELSTYLE',
+ 'INDICATOR_LEVELS',
+ 'INDICATOR_LEVELTEXT',
+ 'INDICATOR_LEVELVALUE',
+ 'INDICATOR_LEVELWIDTH',
+ 'INDICATOR_MAXIMUM',
+ 'INDICATOR_MINIMUM',
+ 'INDICATOR_SHORTNAME',
+ 'INT_MAX',
+ 'INT_MIN',
+ 'INVALID_HANDLE',
+ 'IS_DEBUG_MODE',
+ 'IS_PROFILE_MODE',
+ 'LICENSE_DEMO',
+ 'LICENSE_FREE',
+ 'LICENSE_FULL',
+ 'LICENSE_TIME',
+ 'LONG_MAX',
+ 'LONG_MIN',
+ 'MB_ABORTRETRYIGNORE',
+ 'MB_CANCELTRYCONTINUE',
+ 'MB_DEFBUTTON1',
+ 'MB_DEFBUTTON2',
+ 'MB_DEFBUTTON3',
+ 'MB_DEFBUTTON4',
+ 'MB_ICONASTERISK',
+ 'MB_ICONERROR',
+ 'MB_ICONEXCLAMATION',
+ 'MB_ICONHAND',
+ 'MB_ICONINFORMATION',
+ 'MB_ICONQUESTION',
+ 'MB_ICONSTOP',
+ 'MB_ICONWARNING',
+ 'MB_OKCANCEL',
+ 'MB_OK',
+ 'MB_RETRYCANCEL',
+ 'MB_YESNOCANCEL',
+ 'MB_YESNO',
+ 'MODE_ASK',
+ 'MODE_BID',
+ 'MODE_CHINKOUSPAN',
+ 'MODE_CLOSE',
+ 'MODE_DIGITS',
+ 'MODE_EMA',
+ 'MODE_EXPIRATION',
+ 'MODE_FREEZELEVEL',
+ 'MODE_GATORJAW',
+ 'MODE_GATORLIPS',
+ 'MODE_GATORTEETH',
+ 'MODE_HIGH',
+ 'MODE_KIJUNSEN',
+ 'MODE_LOTSIZE',
+ 'MODE_LOTSTEP',
+ 'MODE_LOWER',
+ 'MODE_LOW',
+ 'MODE_LWMA',
+ 'MODE_MAIN',
+ 'MODE_MARGINCALCMODE',
+ 'MODE_MARGINHEDGED',
+ 'MODE_MARGININIT',
+ 'MODE_MARGINMAINTENANCE',
+ 'MODE_MARGINREQUIRED',
+ 'MODE_MAXLOT',
+ 'MODE_MINLOT',
+ 'MODE_MINUSDI',
+ 'MODE_OPEN',
+ 'MODE_PLUSDI',
+ 'MODE_POINT',
+ 'MODE_PROFITCALCMODE',
+ 'MODE_SENKOUSPANA',
+ 'MODE_SENKOUSPANB',
+ 'MODE_SIGNAL',
+ 'MODE_SMA',
+ 'MODE_SMMA',
+ 'MODE_SPREAD',
+ 'MODE_STARTING',
+ 'MODE_STOPLEVEL',
+ 'MODE_SWAPLONG',
+ 'MODE_SWAPSHORT',
+ 'MODE_SWAPTYPE',
+ 'MODE_TENKANSEN',
+ 'MODE_TICKSIZE',
+ 'MODE_TICKVALUE',
+ 'MODE_TIME',
+ 'MODE_TRADEALLOWED',
+ 'MODE_UPPER',
+ 'MODE_VOLUME',
+ 'MONDAY',
+ 'MQL_DEBUG',
+ 'MQL_DLLS_ALLOWED',
+ 'MQL_FRAME_MODE',
+ 'MQL_LICENSE_TYPE',
+ 'MQL_OPTIMIZATION',
+ 'MQL_PROFILER',
+ 'MQL_PROGRAM_NAME',
+ 'MQL_PROGRAM_PATH',
+ 'MQL_PROGRAM_TYPE',
+ 'MQL_TESTER',
+ 'MQL_TRADE_ALLOWED',
+ 'MQL_VISUAL_MODE',
+ 'M_1_PI',
+ 'M_2_PI',
+ 'M_2_SQRTPI',
+ 'M_E',
+ 'M_LN2',
+ 'M_LN10',
+ 'M_LOG2E',
+ 'M_LOG10E',
+ 'M_PI',
+ 'M_PI_2',
+ 'M_PI_4',
+ 'M_SQRT1_2',
+ 'M_SQRT2',
+ 'NULL',
+ 'OBJPROP_ALIGN',
+ 'OBJPROP_ANCHOR',
+ 'OBJPROP_ANGLE',
+ 'OBJPROP_ARROWCODE',
+ 'OBJPROP_BACK',
+ 'OBJPROP_BGCOLOR',
+ 'OBJPROP_BMPFILE',
+ 'OBJPROP_BORDER_COLOR',
+ 'OBJPROP_BORDER_TYPE',
+ 'OBJPROP_CHART_ID',
+ 'OBJPROP_CHART_SCALE',
+ 'OBJPROP_COLOR',
+ 'OBJPROP_CORNER',
+ 'OBJPROP_CREATETIME',
+ 'OBJPROP_DATE_SCALE',
+ 'OBJPROP_DEVIATION',
+ 'OBJPROP_DRAWLINES',
+ 'OBJPROP_ELLIPSE',
+ 'OBJPROP_FIBOLEVELS',
+ 'OBJPROP_FILL',
+ 'OBJPROP_FIRSTLEVEL',
+ 'OBJPROP_FONTSIZE',
+ 'OBJPROP_FONT',
+ 'OBJPROP_HIDDEN',
+ 'OBJPROP_LEVELCOLOR',
+ 'OBJPROP_LEVELSTYLE',
+ 'OBJPROP_LEVELS',
+ 'OBJPROP_LEVELTEXT',
+ 'OBJPROP_LEVELVALUE',
+ 'OBJPROP_LEVELWIDTH',
+ 'OBJPROP_NAME',
+ 'OBJPROP_PERIOD',
+ 'OBJPROP_PRICE1',
+ 'OBJPROP_PRICE2',
+ 'OBJPROP_PRICE3',
+ 'OBJPROP_PRICE',
+ 'OBJPROP_PRICE_SCALE',
+ 'OBJPROP_RAY',
+ 'OBJPROP_RAY_RIGHT',
+ 'OBJPROP_READONLY',
+ 'OBJPROP_SCALE',
+ 'OBJPROP_SELECTABLE',
+ 'OBJPROP_SELECTED',
+ 'OBJPROP_STATE',
+ 'OBJPROP_STYLE',
+ 'OBJPROP_SYMBOL',
+ 'OBJPROP_TEXT',
+ 'OBJPROP_TIME1',
+ 'OBJPROP_TIME2',
+ 'OBJPROP_TIME3',
+ 'OBJPROP_TIMEFRAMES',
+ 'OBJPROP_TIME',
+ 'OBJPROP_TOOLTIP',
+ 'OBJPROP_TYPE',
+ 'OBJPROP_WIDTH',
+ 'OBJPROP_XDISTANCE',
+ 'OBJPROP_XOFFSET',
+ 'OBJPROP_XSIZE',
+ 'OBJPROP_YDISTANCE',
+ 'OBJPROP_YOFFSET',
+ 'OBJPROP_YSIZE',
+ 'OBJPROP_ZORDER',
+ 'OBJ_ALL_PERIODS',
+ 'OBJ_ARROW',
+ 'OBJ_ARROW_BUY',
+ 'OBJ_ARROW_CHECK',
+ 'OBJ_ARROW_DOWN',
+ 'OBJ_ARROW_LEFT_PRICE',
+ 'OBJ_ARROW_RIGHT_PRICE',
+ 'OBJ_ARROW_SELL',
+ 'OBJ_ARROW_STOP',
+ 'OBJ_ARROW_THUMB_DOWN',
+ 'OBJ_ARROW_THUMB_UP',
+ 'OBJ_ARROW_UP',
+ 'OBJ_BITMAP',
+ 'OBJ_BITMAP_LABEL',
+ 'OBJ_BUTTON',
+ 'OBJ_CHANNEL',
+ 'OBJ_CYCLES',
+ 'OBJ_EDIT',
+ 'OBJ_ELLIPSE',
+ 'OBJ_EVENT',
+ 'OBJ_EXPANSION',
+ 'OBJ_FIBOARC',
+ 'OBJ_FIBOCHANNEL',
+ 'OBJ_FIBOFAN',
+ 'OBJ_FIBOTIMES',
+ 'OBJ_FIBO',
+ 'OBJ_GANNFAN',
+ 'OBJ_GANNGRID',
+ 'OBJ_GANNLINE',
+ 'OBJ_HLINE',
+ 'OBJ_LABEL',
+ 'OBJ_NO_PERIODS',
+ 'OBJ_PERIOD_D1',
+ 'OBJ_PERIOD_H1',
+ 'OBJ_PERIOD_H4',
+ 'OBJ_PERIOD_M1',
+ 'OBJ_PERIOD_M5',
+ 'OBJ_PERIOD_M15',
+ 'OBJ_PERIOD_M30',
+ 'OBJ_PERIOD_MN1',
+ 'OBJ_PERIOD_W1',
+ 'OBJ_PITCHFORK',
+ 'OBJ_RECTANGLE',
+ 'OBJ_RECTANGLE_LABEL',
+ 'OBJ_REGRESSION',
+ 'OBJ_STDDEVCHANNEL',
+ 'OBJ_TEXT',
+ 'OBJ_TRENDBYANGLE',
+ 'OBJ_TREND',
+ 'OBJ_TRIANGLE',
+ 'OBJ_VLINE',
+ 'OP_BUYLIMIT',
+ 'OP_BUYSTOP',
+ 'OP_BUY',
+ 'OP_SELLLIMIT',
+ 'OP_SELLSTOP',
+ 'OP_SELL',
+ 'PERIOD_CURRENT',
+ 'PERIOD_D1',
+ 'PERIOD_H1',
+ 'PERIOD_H2',
+ 'PERIOD_H3',
+ 'PERIOD_H4',
+ 'PERIOD_H6',
+ 'PERIOD_H8',
+ 'PERIOD_H12',
+ 'PERIOD_M1',
+ 'PERIOD_M2',
+ 'PERIOD_M3',
+ 'PERIOD_M4',
+ 'PERIOD_M5',
+ 'PERIOD_M6',
+ 'PERIOD_M10',
+ 'PERIOD_M12',
+ 'PERIOD_M15',
+ 'PERIOD_M20',
+ 'PERIOD_M30',
+ 'PERIOD_MN1',
+ 'PERIOD_W1',
+ 'POINTER_AUTOMATIC',
+ 'POINTER_DYNAMIC',
+ 'POINTER_INVALID'
+ 'PRICE_CLOSE',
+ 'PRICE_HIGH',
+ 'PRICE_LOW',
+ 'PRICE_MEDIAN',
+ 'PRICE_OPEN',
+ 'PRICE_TYPICAL',
+ 'PRICE_WEIGHTED',
+ 'PROGRAM_EXPERT',
+ 'PROGRAM_INDICATOR',
+ 'PROGRAM_SCRIPT',
+ 'REASON_ACCOUNT',
+ 'REASON_CHARTCHANGE',
+ 'REASON_CHARTCLOSE',
+ 'REASON_CLOSE',
+ 'REASON_INITFAILED',
+ 'REASON_PARAMETERS',
+ 'REASON_PROGRAM'
+ 'REASON_RECOMPILE',
+ 'REASON_REMOVE',
+ 'REASON_TEMPLATE',
+ 'SATURDAY',
+ 'SEEK_CUR',
+ 'SEEK_END',
+ 'SEEK_SET',
+ 'SERIES_BARS_COUNT',
+ 'SERIES_FIRSTDATE',
+ 'SERIES_LASTBAR_DATE',
+ 'SERIES_SERVER_FIRSTDATE',
+ 'SERIES_SYNCHRONIZED',
+ 'SERIES_TERMINAL_FIRSTDATE',
+ 'SHORT_MAX',
+ 'SHORT_MIN',
+ 'STAT_BALANCEDD_PERCENT',
+ 'STAT_BALANCEMIN',
+ 'STAT_BALANCE_DDREL_PERCENT',
+ 'STAT_BALANCE_DD',
+ 'STAT_BALANCE_DD_RELATIVE',
+ 'STAT_CONLOSSMAX',
+ 'STAT_CONLOSSMAX_TRADES',
+ 'STAT_CONPROFITMAX',
+ 'STAT_CONPROFITMAX_TRADES',
+ 'STAT_CUSTOM_ONTESTER',
+ 'STAT_DEALS',
+ 'STAT_EQUITYDD_PERCENT',
+ 'STAT_EQUITYMIN',
+ 'STAT_EQUITY_DDREL_PERCENT',
+ 'STAT_EQUITY_DD',
+ 'STAT_EQUITY_DD_RELATIVE',
+ 'STAT_EXPECTED_PAYOFF',
+ 'STAT_GROSS_LOSS',
+ 'STAT_GROSS_PROFIT',
+ 'STAT_INITIAL_DEPOSIT',
+ 'STAT_LONG_TRADES',
+ 'STAT_LOSSTRADES_AVGCON',
+ 'STAT_LOSS_TRADES',
+ 'STAT_MAX_CONLOSSES',
+ 'STAT_MAX_CONLOSS_TRADES',
+ 'STAT_MAX_CONPROFIT_TRADES',
+ 'STAT_MAX_CONWINS',
+ 'STAT_MAX_LOSSTRADE',
+ 'STAT_MAX_PROFITTRADE',
+ 'STAT_MIN_MARGINLEVEL',
+ 'STAT_PROFITTRADES_AVGCON',
+ 'STAT_PROFIT',
+ 'STAT_PROFIT_FACTOR',
+ 'STAT_PROFIT_LONGTRADES',
+ 'STAT_PROFIT_SHORTTRADES',
+ 'STAT_PROFIT_TRADES',
+ 'STAT_RECOVERY_FACTOR',
+ 'STAT_SHARPE_RATIO',
+ 'STAT_SHORT_TRADES',
+ 'STAT_TRADES',
+ 'STAT_WITHDRAWAL',
+ 'STO_CLOSECLOSE',
+ 'STO_LOWHIGH',
+ 'STYLE_DASHDOTDOT',
+ 'STYLE_DASHDOT',
+ 'STYLE_DASH',
+ 'STYLE_DOT',
+ 'STYLE_SOLID',
+ 'SUNDAY',
+ 'SYMBOL_ARROWDOWN',
+ 'SYMBOL_ARROWUP',
+ 'SYMBOL_CHECKSIGN',
+ 'SYMBOL_LEFTPRICE',
+ 'SYMBOL_RIGHTPRICE',
+ 'SYMBOL_STOPSIGN',
+ 'SYMBOL_THUMBSDOWN',
+ 'SYMBOL_THUMBSUP',
+ 'TERMINAL_BUILD',
+ 'TERMINAL_CODEPAGE',
+ 'TERMINAL_COMMONDATA_PATH',
+ 'TERMINAL_COMPANY',
+ 'TERMINAL_CONNECTED',
+ 'TERMINAL_CPU_CORES',
+ 'TERMINAL_DATA_PATH',
+ 'TERMINAL_DISK_SPACE',
+ 'TERMINAL_DLLS_ALLOWED',
+ 'TERMINAL_EMAIL_ENABLED',
+ 'TERMINAL_FTP_ENABLED',
+ 'TERMINAL_LANGUAGE',
+ 'TERMINAL_MAXBARS',
+ 'TERMINAL_MEMORY_AVAILABLE',
+ 'TERMINAL_MEMORY_PHYSICAL',
+ 'TERMINAL_MEMORY_TOTAL',
+ 'TERMINAL_MEMORY_USED',
+ 'TERMINAL_NAME',
+ 'TERMINAL_OPENCL_SUPPORT',
+ 'TERMINAL_PATH',
+ 'TERMINAL_TRADE_ALLOWED',
+ 'TERMINAL_X64',
+ 'THURSDAY',
+ 'TRADE_ACTION_DEAL',
+ 'TRADE_ACTION_MODIFY',
+ 'TRADE_ACTION_PENDING',
+ 'TRADE_ACTION_REMOVE',
+ 'TRADE_ACTION_SLTP',
+ 'TUESDAY',
+ 'UCHAR_MAX',
+ 'UINT_MAX',
+ 'ULONG_MAX',
+ 'USHORT_MAX',
+ 'VOLUME_REAL',
+ 'VOLUME_TICK',
+ 'WEDNESDAY',
+ 'WHOLE_ARRAY',
+ 'WRONG_VALUE',
+ 'clrNONE',
+ '__DATETIME__',
+ '__DATE__',
+ '__FILE__',
+ '__FUNCSIG__',
+ '__FUNCTION__',
+ '__LINE__',
+ '__MQL4BUILD__',
+ '__MQLBUILD__',
+ '__PATH__',
+)
+
+colors = (
+ 'AliceBlue',
+ 'AntiqueWhite',
+ 'Aquamarine',
+ 'Aqua',
+ 'Beige',
+ 'Bisque',
+ 'Black',
+ 'BlanchedAlmond',
+ 'BlueViolet',
+ 'Blue',
+ 'Brown',
+ 'BurlyWood',
+ 'CadetBlue',
+ 'Chartreuse',
+ 'Chocolate',
+ 'Coral',
+ 'CornflowerBlue',
+ 'Cornsilk',
+ 'Crimson',
+ 'DarkBlue',
+ 'DarkGoldenrod',
+ 'DarkGray',
+ 'DarkGreen',
+ 'DarkKhaki',
+ 'DarkOliveGreen',
+ 'DarkOrange',
+ 'DarkOrchid',
+ 'DarkSalmon',
+ 'DarkSeaGreen',
+ 'DarkSlateBlue',
+ 'DarkSlateGray',
+ 'DarkTurquoise',
+ 'DarkViolet',
+ 'DeepPink',
+ 'DeepSkyBlue',
+ 'DimGray',
+ 'DodgerBlue',
+ 'FireBrick',
+ 'ForestGreen',
+ 'Gainsboro',
+ 'Goldenrod',
+ 'Gold',
+ 'Gray',
+ 'GreenYellow',
+ 'Green',
+ 'Honeydew',
+ 'HotPink',
+ 'IndianRed',
+ 'Indigo',
+ 'Ivory',
+ 'Khaki',
+ 'LavenderBlush',
+ 'Lavender',
+ 'LawnGreen',
+ 'LemonChiffon',
+ 'LightBlue',
+ 'LightCoral',
+ 'LightCyan',
+ 'LightGoldenrod',
+ 'LightGray',
+ 'LightGreen',
+ 'LightPink',
+ 'LightSalmon',
+ 'LightSeaGreen',
+ 'LightSkyBlue',
+ 'LightSlateGray',
+ 'LightSteelBlue',
+ 'LightYellow',
+ 'LimeGreen',
+ 'Lime',
+ 'Linen',
+ 'Magenta',
+ 'Maroon',
+ 'MediumAquamarine',
+ 'MediumBlue',
+ 'MediumOrchid',
+ 'MediumPurple',
+ 'MediumSeaGreen',
+ 'MediumSlateBlue',
+ 'MediumSpringGreen',
+ 'MediumTurquoise',
+ 'MediumVioletRed',
+ 'MidnightBlue',
+ 'MintCream',
+ 'MistyRose',
+ 'Moccasin',
+ 'NavajoWhite',
+ 'Navy',
+ 'OldLace',
+ 'OliveDrab',
+ 'Olive',
+ 'OrangeRed',
+ 'Orange',
+ 'Orchid',
+ 'PaleGoldenrod',
+ 'PaleGreen',
+ 'PaleTurquoise',
+ 'PaleVioletRed',
+ 'PapayaWhip',
+ 'PeachPuff',
+ 'Peru',
+ 'Pink',
+ 'Plum',
+ 'PowderBlue',
+ 'Purple',
+ 'Red',
+ 'RosyBrown',
+ 'RoyalBlue',
+ 'SaddleBrown',
+ 'Salmon',
+ 'SandyBrown',
+ 'SeaGreen',
+ 'Seashell',
+ 'Sienna',
+ 'Silver',
+ 'SkyBlue',
+ 'SlateBlue',
+ 'SlateGray',
+ 'Snow',
+ 'SpringGreen',
+ 'SteelBlue',
+ 'Tan',
+ 'Teal',
+ 'Thistle',
+ 'Tomato',
+ 'Turquoise',
+ 'Violet',
+ 'Wheat',
+ 'WhiteSmoke',
+ 'White',
+ 'YellowGreen',
+ 'Yellow',
+)
+
+keywords = (
+ 'input', '_Digits', '_Point', '_LastError', '_Period', '_RandomSeed',
+ '_StopFlag', '_Symbol', '_UninitReason', 'Ask', 'Bars', 'Bid',
+ 'Close', 'Digits', 'High', 'Low', 'Open', 'Point', 'Time',
+ 'Volume',
+)
+c_types = (
+ 'void', 'char', 'uchar', 'bool', 'short', 'ushort', 'int', 'uint',
+ 'color', 'long', 'ulong', 'datetime', 'float', 'double',
+ 'string',
+)
diff --git a/wandb/vendor/pygments/lexers/_openedge_builtins.py b/wandb/vendor/pygments/lexers/_openedge_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..0fa7d1b244ca9e2d5c7c724011bb467796cb8541
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_openedge_builtins.py
@@ -0,0 +1,2547 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._openedge_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Builtin list for the OpenEdgeLexer.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+OPENEDGEKEYWORDS = (
+ 'ABSOLUTE',
+ 'ABS',
+ 'ABSO',
+ 'ABSOL',
+ 'ABSOLU',
+ 'ABSOLUT',
+ 'ACCELERATOR',
+ 'ACCUMULATE',
+ 'ACCUM',
+ 'ACCUMU',
+ 'ACCUMUL',
+ 'ACCUMULA',
+ 'ACCUMULAT',
+ 'ACTIVE-FORM',
+ 'ACTIVE-WINDOW',
+ 'ADD',
+ 'ADD-BUFFER',
+ 'ADD-CALC-COLUMN',
+ 'ADD-COLUMNS-FROM',
+ 'ADD-EVENTS-PROCEDURE',
+ 'ADD-FIELDS-FROM',
+ 'ADD-FIRST',
+ 'ADD-INDEX-FIELD',
+ 'ADD-LAST',
+ 'ADD-LIKE-COLUMN',
+ 'ADD-LIKE-FIELD',
+ 'ADD-LIKE-INDEX',
+ 'ADD-NEW-FIELD',
+ 'ADD-NEW-INDEX',
+ 'ADD-SCHEMA-LOCATION',
+ 'ADD-SUPER-PROCEDURE',
+ 'ADM-DATA',
+ 'ADVISE',
+ 'ALERT-BOX',
+ 'ALIAS',
+ 'ALL',
+ 'ALLOW-COLUMN-SEARCHING',
+ 'ALLOW-REPLICATION',
+ 'ALTER',
+ 'ALWAYS-ON-TOP',
+ 'AMBIGUOUS',
+ 'AMBIG',
+ 'AMBIGU',
+ 'AMBIGUO',
+ 'AMBIGUOU',
+ 'ANALYZE',
+ 'ANALYZ',
+ 'AND',
+ 'ANSI-ONLY',
+ 'ANY',
+ 'ANYWHERE',
+ 'APPEND',
+ 'APPL-ALERT-BOXES',
+ 'APPL-ALERT',
+ 'APPL-ALERT-',
+ 'APPL-ALERT-B',
+ 'APPL-ALERT-BO',
+ 'APPL-ALERT-BOX',
+ 'APPL-ALERT-BOXE',
+ 'APPL-CONTEXT-ID',
+ 'APPLICATION',
+ 'APPLY',
+ 'APPSERVER-INFO',
+ 'APPSERVER-PASSWORD',
+ 'APPSERVER-USERID',
+ 'ARRAY-MESSAGE',
+ 'AS',
+ 'ASC',
+ 'ASCENDING',
+ 'ASCE',
+ 'ASCEN',
+ 'ASCEND',
+ 'ASCENDI',
+ 'ASCENDIN',
+ 'ASK-OVERWRITE',
+ 'ASSEMBLY',
+ 'ASSIGN',
+ 'ASYNCHRONOUS',
+ 'ASYNC-REQUEST-COUNT',
+ 'ASYNC-REQUEST-HANDLE',
+ 'AT',
+ 'ATTACHED-PAIRLIST',
+ 'ATTR-SPACE',
+ 'ATTR',
+ 'ATTRI',
+ 'ATTRIB',
+ 'ATTRIBU',
+ 'ATTRIBUT',
+ 'AUDIT-CONTROL',
+ 'AUDIT-ENABLED',
+ 'AUDIT-EVENT-CONTEXT',
+ 'AUDIT-POLICY',
+ 'AUTHENTICATION-FAILED',
+ 'AUTHORIZATION',
+ 'AUTO-COMPLETION',
+ 'AUTO-COMP',
+ 'AUTO-COMPL',
+ 'AUTO-COMPLE',
+ 'AUTO-COMPLET',
+ 'AUTO-COMPLETI',
+ 'AUTO-COMPLETIO',
+ 'AUTO-ENDKEY',
+ 'AUTO-END-KEY',
+ 'AUTO-GO',
+ 'AUTO-INDENT',
+ 'AUTO-IND',
+ 'AUTO-INDE',
+ 'AUTO-INDEN',
+ 'AUTOMATIC',
+ 'AUTO-RESIZE',
+ 'AUTO-RETURN',
+ 'AUTO-RET',
+ 'AUTO-RETU',
+ 'AUTO-RETUR',
+ 'AUTO-SYNCHRONIZE',
+ 'AUTO-ZAP',
+ 'AUTO-Z',
+ 'AUTO-ZA',
+ 'AVAILABLE',
+ 'AVAIL',
+ 'AVAILA',
+ 'AVAILAB',
+ 'AVAILABL',
+ 'AVAILABLE-FORMATS',
+ 'AVERAGE',
+ 'AVE',
+ 'AVER',
+ 'AVERA',
+ 'AVERAG',
+ 'AVG',
+ 'BACKGROUND',
+ 'BACK',
+ 'BACKG',
+ 'BACKGR',
+ 'BACKGRO',
+ 'BACKGROU',
+ 'BACKGROUN',
+ 'BACKWARDS',
+ 'BACKWARD',
+ 'BASE64-DECODE',
+ 'BASE64-ENCODE',
+ 'BASE-ADE',
+ 'BASE-KEY',
+ 'BATCH-MODE',
+ 'BATCH',
+ 'BATCH-',
+ 'BATCH-M',
+ 'BATCH-MO',
+ 'BATCH-MOD',
+ 'BATCH-SIZE',
+ 'BEFORE-HIDE',
+ 'BEFORE-H',
+ 'BEFORE-HI',
+ 'BEFORE-HID',
+ 'BEGIN-EVENT-GROUP',
+ 'BEGINS',
+ 'BELL',
+ 'BETWEEN',
+ 'BGCOLOR',
+ 'BGC',
+ 'BGCO',
+ 'BGCOL',
+ 'BGCOLO',
+ 'BIG-ENDIAN',
+ 'BINARY',
+ 'BIND',
+ 'BIND-WHERE',
+ 'BLANK',
+ 'BLOCK-ITERATION-DISPLAY',
+ 'BORDER-BOTTOM-CHARS',
+ 'BORDER-B',
+ 'BORDER-BO',
+ 'BORDER-BOT',
+ 'BORDER-BOTT',
+ 'BORDER-BOTTO',
+ 'BORDER-BOTTOM-PIXELS',
+ 'BORDER-BOTTOM-P',
+ 'BORDER-BOTTOM-PI',
+ 'BORDER-BOTTOM-PIX',
+ 'BORDER-BOTTOM-PIXE',
+ 'BORDER-BOTTOM-PIXEL',
+ 'BORDER-LEFT-CHARS',
+ 'BORDER-L',
+ 'BORDER-LE',
+ 'BORDER-LEF',
+ 'BORDER-LEFT',
+ 'BORDER-LEFT-',
+ 'BORDER-LEFT-C',
+ 'BORDER-LEFT-CH',
+ 'BORDER-LEFT-CHA',
+ 'BORDER-LEFT-CHAR',
+ 'BORDER-LEFT-PIXELS',
+ 'BORDER-LEFT-P',
+ 'BORDER-LEFT-PI',
+ 'BORDER-LEFT-PIX',
+ 'BORDER-LEFT-PIXE',
+ 'BORDER-LEFT-PIXEL',
+ 'BORDER-RIGHT-CHARS',
+ 'BORDER-R',
+ 'BORDER-RI',
+ 'BORDER-RIG',
+ 'BORDER-RIGH',
+ 'BORDER-RIGHT',
+ 'BORDER-RIGHT-',
+ 'BORDER-RIGHT-C',
+ 'BORDER-RIGHT-CH',
+ 'BORDER-RIGHT-CHA',
+ 'BORDER-RIGHT-CHAR',
+ 'BORDER-RIGHT-PIXELS',
+ 'BORDER-RIGHT-P',
+ 'BORDER-RIGHT-PI',
+ 'BORDER-RIGHT-PIX',
+ 'BORDER-RIGHT-PIXE',
+ 'BORDER-RIGHT-PIXEL',
+ 'BORDER-TOP-CHARS',
+ 'BORDER-T',
+ 'BORDER-TO',
+ 'BORDER-TOP',
+ 'BORDER-TOP-',
+ 'BORDER-TOP-C',
+ 'BORDER-TOP-CH',
+ 'BORDER-TOP-CHA',
+ 'BORDER-TOP-CHAR',
+ 'BORDER-TOP-PIXELS',
+ 'BORDER-TOP-P',
+ 'BORDER-TOP-PI',
+ 'BORDER-TOP-PIX',
+ 'BORDER-TOP-PIXE',
+ 'BORDER-TOP-PIXEL',
+ 'BOX',
+ 'BOX-SELECTABLE',
+ 'BOX-SELECT',
+ 'BOX-SELECTA',
+ 'BOX-SELECTAB',
+ 'BOX-SELECTABL',
+ 'BREAK',
+ 'BROWSE',
+ 'BUFFER',
+ 'BUFFER-CHARS',
+ 'BUFFER-COMPARE',
+ 'BUFFER-COPY',
+ 'BUFFER-CREATE',
+ 'BUFFER-DELETE',
+ 'BUFFER-FIELD',
+ 'BUFFER-HANDLE',
+ 'BUFFER-LINES',
+ 'BUFFER-NAME',
+ 'BUFFER-RELEASE',
+ 'BUFFER-VALUE',
+ 'BUTTON',
+ 'BUTTONS',
+ 'BY',
+ 'BY-POINTER',
+ 'BY-VARIANT-POINTER',
+ 'CACHE',
+ 'CACHE-SIZE',
+ 'CALL',
+ 'CALL-NAME',
+ 'CALL-TYPE',
+ 'CANCEL-BREAK',
+ 'CANCEL-BUTTON',
+ 'CAN-CREATE',
+ 'CAN-DELETE',
+ 'CAN-DO',
+ 'CAN-FIND',
+ 'CAN-QUERY',
+ 'CAN-READ',
+ 'CAN-SET',
+ 'CAN-WRITE',
+ 'CAPS',
+ 'CAREFUL-PAINT',
+ 'CASE',
+ 'CASE-SENSITIVE',
+ 'CASE-SEN',
+ 'CASE-SENS',
+ 'CASE-SENSI',
+ 'CASE-SENSIT',
+ 'CASE-SENSITI',
+ 'CASE-SENSITIV',
+ 'CAST',
+ 'CATCH',
+ 'CDECL',
+ 'CENTERED',
+ 'CENTER',
+ 'CENTERE',
+ 'CHAINED',
+ 'CHARACTER_LENGTH',
+ 'CHARSET',
+ 'CHECK',
+ 'CHECKED',
+ 'CHOOSE',
+ 'CHR',
+ 'CLASS',
+ 'CLASS-TYPE',
+ 'CLEAR',
+ 'CLEAR-APPL-CONTEXT',
+ 'CLEAR-LOG',
+ 'CLEAR-SELECTION',
+ 'CLEAR-SELECT',
+ 'CLEAR-SELECTI',
+ 'CLEAR-SELECTIO',
+ 'CLEAR-SORT-ARROWS',
+ 'CLEAR-SORT-ARROW',
+ 'CLIENT-CONNECTION-ID',
+ 'CLIENT-PRINCIPAL',
+ 'CLIENT-TTY',
+ 'CLIENT-TYPE',
+ 'CLIENT-WORKSTATION',
+ 'CLIPBOARD',
+ 'CLOSE',
+ 'CLOSE-LOG',
+ 'CODE',
+ 'CODEBASE-LOCATOR',
+ 'CODEPAGE',
+ 'CODEPAGE-CONVERT',
+ 'COLLATE',
+ 'COL-OF',
+ 'COLON',
+ 'COLON-ALIGNED',
+ 'COLON-ALIGN',
+ 'COLON-ALIGNE',
+ 'COLOR',
+ 'COLOR-TABLE',
+ 'COLUMN',
+ 'COL',
+ 'COLU',
+ 'COLUM',
+ 'COLUMN-BGCOLOR',
+ 'COLUMN-DCOLOR',
+ 'COLUMN-FGCOLOR',
+ 'COLUMN-FONT',
+ 'COLUMN-LABEL',
+ 'COLUMN-LAB',
+ 'COLUMN-LABE',
+ 'COLUMN-MOVABLE',
+ 'COLUMN-OF',
+ 'COLUMN-PFCOLOR',
+ 'COLUMN-READ-ONLY',
+ 'COLUMN-RESIZABLE',
+ 'COLUMNS',
+ 'COLUMN-SCROLLING',
+ 'COMBO-BOX',
+ 'COMMAND',
+ 'COMPARES',
+ 'COMPILE',
+ 'COMPILER',
+ 'COMPLETE',
+ 'COM-SELF',
+ 'CONFIG-NAME',
+ 'CONNECT',
+ 'CONNECTED',
+ 'CONSTRUCTOR',
+ 'CONTAINS',
+ 'CONTENTS',
+ 'CONTEXT',
+ 'CONTEXT-HELP',
+ 'CONTEXT-HELP-FILE',
+ 'CONTEXT-HELP-ID',
+ 'CONTEXT-POPUP',
+ 'CONTROL',
+ 'CONTROL-BOX',
+ 'CONTROL-FRAME',
+ 'CONVERT',
+ 'CONVERT-3D-COLORS',
+ 'CONVERT-TO-OFFSET',
+ 'CONVERT-TO-OFFS',
+ 'CONVERT-TO-OFFSE',
+ 'COPY-DATASET',
+ 'COPY-LOB',
+ 'COPY-SAX-ATTRIBUTES',
+ 'COPY-TEMP-TABLE',
+ 'COUNT',
+ 'COUNT-OF',
+ 'CPCASE',
+ 'CPCOLL',
+ 'CPINTERNAL',
+ 'CPLOG',
+ 'CPPRINT',
+ 'CPRCODEIN',
+ 'CPRCODEOUT',
+ 'CPSTREAM',
+ 'CPTERM',
+ 'CRC-VALUE',
+ 'CREATE',
+ 'CREATE-LIKE',
+ 'CREATE-LIKE-SEQUENTIAL',
+ 'CREATE-NODE-NAMESPACE',
+ 'CREATE-RESULT-LIST-ENTRY',
+ 'CREATE-TEST-FILE',
+ 'CURRENT',
+ 'CURRENT_DATE',
+ 'CURRENT-CHANGED',
+ 'CURRENT-COLUMN',
+ 'CURRENT-ENVIRONMENT',
+ 'CURRENT-ENV',
+ 'CURRENT-ENVI',
+ 'CURRENT-ENVIR',
+ 'CURRENT-ENVIRO',
+ 'CURRENT-ENVIRON',
+ 'CURRENT-ENVIRONM',
+ 'CURRENT-ENVIRONME',
+ 'CURRENT-ENVIRONMEN',
+ 'CURRENT-ITERATION',
+ 'CURRENT-LANGUAGE',
+ 'CURRENT-LANG',
+ 'CURRENT-LANGU',
+ 'CURRENT-LANGUA',
+ 'CURRENT-LANGUAG',
+ 'CURRENT-QUERY',
+ 'CURRENT-RESULT-ROW',
+ 'CURRENT-ROW-MODIFIED',
+ 'CURRENT-VALUE',
+ 'CURRENT-WINDOW',
+ 'CURSOR',
+ 'CURS',
+ 'CURSO',
+ 'CURSOR-CHAR',
+ 'CURSOR-LINE',
+ 'CURSOR-OFFSET',
+ 'DATABASE',
+ 'DATA-BIND',
+ 'DATA-ENTRY-RETURN',
+ 'DATA-ENTRY-RET',
+ 'DATA-ENTRY-RETU',
+ 'DATA-ENTRY-RETUR',
+ 'DATA-RELATION',
+ 'DATA-REL',
+ 'DATA-RELA',
+ 'DATA-RELAT',
+ 'DATA-RELATI',
+ 'DATA-RELATIO',
+ 'DATASERVERS',
+ 'DATASET',
+ 'DATASET-HANDLE',
+ 'DATA-SOURCE',
+ 'DATA-SOURCE-COMPLETE-MAP',
+ 'DATA-SOURCE-MODIFIED',
+ 'DATA-SOURCE-ROWID',
+ 'DATA-TYPE',
+ 'DATA-T',
+ 'DATA-TY',
+ 'DATA-TYP',
+ 'DATE-FORMAT',
+ 'DATE-F',
+ 'DATE-FO',
+ 'DATE-FOR',
+ 'DATE-FORM',
+ 'DATE-FORMA',
+ 'DAY',
+ 'DBCODEPAGE',
+ 'DBCOLLATION',
+ 'DBNAME',
+ 'DBPARAM',
+ 'DB-REFERENCES',
+ 'DBRESTRICTIONS',
+ 'DBREST',
+ 'DBRESTR',
+ 'DBRESTRI',
+ 'DBRESTRIC',
+ 'DBRESTRICT',
+ 'DBRESTRICTI',
+ 'DBRESTRICTIO',
+ 'DBRESTRICTION',
+ 'DBTASKID',
+ 'DBTYPE',
+ 'DBVERSION',
+ 'DBVERS',
+ 'DBVERSI',
+ 'DBVERSIO',
+ 'DCOLOR',
+ 'DDE',
+ 'DDE-ERROR',
+ 'DDE-ID',
+ 'DDE-I',
+ 'DDE-ITEM',
+ 'DDE-NAME',
+ 'DDE-TOPIC',
+ 'DEBLANK',
+ 'DEBUG',
+ 'DEBU',
+ 'DEBUG-ALERT',
+ 'DEBUGGER',
+ 'DEBUG-LIST',
+ 'DECIMALS',
+ 'DECLARE',
+ 'DECLARE-NAMESPACE',
+ 'DECRYPT',
+ 'DEFAULT',
+ 'DEFAULT-BUFFER-HANDLE',
+ 'DEFAULT-BUTTON',
+ 'DEFAUT-B',
+ 'DEFAUT-BU',
+ 'DEFAUT-BUT',
+ 'DEFAUT-BUTT',
+ 'DEFAUT-BUTTO',
+ 'DEFAULT-COMMIT',
+ 'DEFAULT-EXTENSION',
+ 'DEFAULT-EX',
+ 'DEFAULT-EXT',
+ 'DEFAULT-EXTE',
+ 'DEFAULT-EXTEN',
+ 'DEFAULT-EXTENS',
+ 'DEFAULT-EXTENSI',
+ 'DEFAULT-EXTENSIO',
+ 'DEFAULT-NOXLATE',
+ 'DEFAULT-NOXL',
+ 'DEFAULT-NOXLA',
+ 'DEFAULT-NOXLAT',
+ 'DEFAULT-VALUE',
+ 'DEFAULT-WINDOW',
+ 'DEFINED',
+ 'DEFINE-USER-EVENT-MANAGER',
+ 'DELETE',
+ 'DEL',
+ 'DELE',
+ 'DELET',
+ 'DELETE-CHARACTER',
+ 'DELETE-CHAR',
+ 'DELETE-CHARA',
+ 'DELETE-CHARAC',
+ 'DELETE-CHARACT',
+ 'DELETE-CHARACTE',
+ 'DELETE-CURRENT-ROW',
+ 'DELETE-LINE',
+ 'DELETE-RESULT-LIST-ENTRY',
+ 'DELETE-SELECTED-ROW',
+ 'DELETE-SELECTED-ROWS',
+ 'DELIMITER',
+ 'DESC',
+ 'DESCENDING',
+ 'DESCE',
+ 'DESCEN',
+ 'DESCEND',
+ 'DESCENDI',
+ 'DESCENDIN',
+ 'DESELECT-FOCUSED-ROW',
+ 'DESELECTION',
+ 'DESELECT-ROWS',
+ 'DESELECT-SELECTED-ROW',
+ 'DESTRUCTOR',
+ 'DIALOG-BOX',
+ 'DICTIONARY',
+ 'DICT',
+ 'DICTI',
+ 'DICTIO',
+ 'DICTION',
+ 'DICTIONA',
+ 'DICTIONAR',
+ 'DIR',
+ 'DISABLE',
+ 'DISABLE-AUTO-ZAP',
+ 'DISABLED',
+ 'DISABLE-DUMP-TRIGGERS',
+ 'DISABLE-LOAD-TRIGGERS',
+ 'DISCONNECT',
+ 'DISCON',
+ 'DISCONN',
+ 'DISCONNE',
+ 'DISCONNEC',
+ 'DISP',
+ 'DISPLAY',
+ 'DISPL',
+ 'DISPLA',
+ 'DISPLAY-MESSAGE',
+ 'DISPLAY-TYPE',
+ 'DISPLAY-T',
+ 'DISPLAY-TY',
+ 'DISPLAY-TYP',
+ 'DISTINCT',
+ 'DO',
+ 'DOMAIN-DESCRIPTION',
+ 'DOMAIN-NAME',
+ 'DOMAIN-TYPE',
+ 'DOS',
+ 'DOUBLE',
+ 'DOWN',
+ 'DRAG-ENABLED',
+ 'DROP',
+ 'DROP-DOWN',
+ 'DROP-DOWN-LIST',
+ 'DROP-FILE-NOTIFY',
+ 'DROP-TARGET',
+ 'DUMP',
+ 'DYNAMIC',
+ 'DYNAMIC-FUNCTION',
+ 'EACH',
+ 'ECHO',
+ 'EDGE-CHARS',
+ 'EDGE',
+ 'EDGE-',
+ 'EDGE-C',
+ 'EDGE-CH',
+ 'EDGE-CHA',
+ 'EDGE-CHAR',
+ 'EDGE-PIXELS',
+ 'EDGE-P',
+ 'EDGE-PI',
+ 'EDGE-PIX',
+ 'EDGE-PIXE',
+ 'EDGE-PIXEL',
+ 'EDIT-CAN-PASTE',
+ 'EDIT-CAN-UNDO',
+ 'EDIT-CLEAR',
+ 'EDIT-COPY',
+ 'EDIT-CUT',
+ 'EDITING',
+ 'EDITOR',
+ 'EDIT-PASTE',
+ 'EDIT-UNDO',
+ 'ELSE',
+ 'EMPTY',
+ 'EMPTY-TEMP-TABLE',
+ 'ENABLE',
+ 'ENABLED-FIELDS',
+ 'ENCODE',
+ 'ENCRYPT',
+ 'ENCRYPT-AUDIT-MAC-KEY',
+ 'ENCRYPTION-SALT',
+ 'END',
+ 'END-DOCUMENT',
+ 'END-ELEMENT',
+ 'END-EVENT-GROUP',
+ 'END-FILE-DROP',
+ 'ENDKEY',
+ 'END-KEY',
+ 'END-MOVE',
+ 'END-RESIZE',
+ 'END-ROW-RESIZE',
+ 'END-USER-PROMPT',
+ 'ENTERED',
+ 'ENTRY',
+ 'EQ',
+ 'ERROR',
+ 'ERROR-COLUMN',
+ 'ERROR-COL',
+ 'ERROR-COLU',
+ 'ERROR-COLUM',
+ 'ERROR-ROW',
+ 'ERROR-STACK-TRACE',
+ 'ERROR-STATUS',
+ 'ERROR-STAT',
+ 'ERROR-STATU',
+ 'ESCAPE',
+ 'ETIME',
+ 'EVENT-GROUP-ID',
+ 'EVENT-PROCEDURE',
+ 'EVENT-PROCEDURE-CONTEXT',
+ 'EVENTS',
+ 'EVENT',
+ 'EVENT-TYPE',
+ 'EVENT-T',
+ 'EVENT-TY',
+ 'EVENT-TYP',
+ 'EXCEPT',
+ 'EXCLUSIVE-ID',
+ 'EXCLUSIVE-LOCK',
+ 'EXCLUSIVE',
+ 'EXCLUSIVE-',
+ 'EXCLUSIVE-L',
+ 'EXCLUSIVE-LO',
+ 'EXCLUSIVE-LOC',
+ 'EXCLUSIVE-WEB-USER',
+ 'EXECUTE',
+ 'EXISTS',
+ 'EXP',
+ 'EXPAND',
+ 'EXPANDABLE',
+ 'EXPLICIT',
+ 'EXPORT',
+ 'EXPORT-PRINCIPAL',
+ 'EXTENDED',
+ 'EXTENT',
+ 'EXTERNAL',
+ 'FALSE',
+ 'FETCH',
+ 'FETCH-SELECTED-ROW',
+ 'FGCOLOR',
+ 'FGC',
+ 'FGCO',
+ 'FGCOL',
+ 'FGCOLO',
+ 'FIELD',
+ 'FIELDS',
+ 'FILE',
+ 'FILE-CREATE-DATE',
+ 'FILE-CREATE-TIME',
+ 'FILE-INFORMATION',
+ 'FILE-INFO',
+ 'FILE-INFOR',
+ 'FILE-INFORM',
+ 'FILE-INFORMA',
+ 'FILE-INFORMAT',
+ 'FILE-INFORMATI',
+ 'FILE-INFORMATIO',
+ 'FILE-MOD-DATE',
+ 'FILE-MOD-TIME',
+ 'FILENAME',
+ 'FILE-NAME',
+ 'FILE-OFFSET',
+ 'FILE-OFF',
+ 'FILE-OFFS',
+ 'FILE-OFFSE',
+ 'FILE-SIZE',
+ 'FILE-TYPE',
+ 'FILL',
+ 'FILLED',
+ 'FILL-IN',
+ 'FILTERS',
+ 'FINAL',
+ 'FINALLY',
+ 'FIND',
+ 'FIND-BY-ROWID',
+ 'FIND-CASE-SENSITIVE',
+ 'FIND-CURRENT',
+ 'FINDER',
+ 'FIND-FIRST',
+ 'FIND-GLOBAL',
+ 'FIND-LAST',
+ 'FIND-NEXT-OCCURRENCE',
+ 'FIND-PREV-OCCURRENCE',
+ 'FIND-SELECT',
+ 'FIND-UNIQUE',
+ 'FIND-WRAP-AROUND',
+ 'FIRST',
+ 'FIRST-ASYNCH-REQUEST',
+ 'FIRST-CHILD',
+ 'FIRST-COLUMN',
+ 'FIRST-FORM',
+ 'FIRST-OBJECT',
+ 'FIRST-OF',
+ 'FIRST-PROCEDURE',
+ 'FIRST-PROC',
+ 'FIRST-PROCE',
+ 'FIRST-PROCED',
+ 'FIRST-PROCEDU',
+ 'FIRST-PROCEDUR',
+ 'FIRST-SERVER',
+ 'FIRST-TAB-ITEM',
+ 'FIRST-TAB-I',
+ 'FIRST-TAB-IT',
+ 'FIRST-TAB-ITE',
+ 'FIT-LAST-COLUMN',
+ 'FIXED-ONLY',
+ 'FLAT-BUTTON',
+ 'FLOAT',
+ 'FOCUS',
+ 'FOCUSED-ROW',
+ 'FOCUSED-ROW-SELECTED',
+ 'FONT',
+ 'FONT-TABLE',
+ 'FOR',
+ 'FORCE-FILE',
+ 'FOREGROUND',
+ 'FORE',
+ 'FOREG',
+ 'FOREGR',
+ 'FOREGRO',
+ 'FOREGROU',
+ 'FOREGROUN',
+ 'FORM',
+ 'FORMAT',
+ 'FORMA',
+ 'FORMATTED',
+ 'FORMATTE',
+ 'FORM-LONG-INPUT',
+ 'FORWARD',
+ 'FORWARDS',
+ 'FRAGMENT',
+ 'FRAGMEN',
+ 'FRAME',
+ 'FRAM',
+ 'FRAME-COL',
+ 'FRAME-DB',
+ 'FRAME-DOWN',
+ 'FRAME-FIELD',
+ 'FRAME-FILE',
+ 'FRAME-INDEX',
+ 'FRAME-INDE',
+ 'FRAME-LINE',
+ 'FRAME-NAME',
+ 'FRAME-ROW',
+ 'FRAME-SPACING',
+ 'FRAME-SPA',
+ 'FRAME-SPAC',
+ 'FRAME-SPACI',
+ 'FRAME-SPACIN',
+ 'FRAME-VALUE',
+ 'FRAME-VAL',
+ 'FRAME-VALU',
+ 'FRAME-X',
+ 'FRAME-Y',
+ 'FREQUENCY',
+ 'FROM',
+ 'FROM-CHARS',
+ 'FROM-C',
+ 'FROM-CH',
+ 'FROM-CHA',
+ 'FROM-CHAR',
+ 'FROM-CURRENT',
+ 'FROM-CUR',
+ 'FROM-CURR',
+ 'FROM-CURRE',
+ 'FROM-CURREN',
+ 'FROM-PIXELS',
+ 'FROM-P',
+ 'FROM-PI',
+ 'FROM-PIX',
+ 'FROM-PIXE',
+ 'FROM-PIXEL',
+ 'FULL-HEIGHT-CHARS',
+ 'FULL-HEIGHT',
+ 'FULL-HEIGHT-',
+ 'FULL-HEIGHT-C',
+ 'FULL-HEIGHT-CH',
+ 'FULL-HEIGHT-CHA',
+ 'FULL-HEIGHT-CHAR',
+ 'FULL-HEIGHT-PIXELS',
+ 'FULL-HEIGHT-P',
+ 'FULL-HEIGHT-PI',
+ 'FULL-HEIGHT-PIX',
+ 'FULL-HEIGHT-PIXE',
+ 'FULL-HEIGHT-PIXEL',
+ 'FULL-PATHNAME',
+ 'FULL-PATHN',
+ 'FULL-PATHNA',
+ 'FULL-PATHNAM',
+ 'FULL-WIDTH-CHARS',
+ 'FULL-WIDTH',
+ 'FULL-WIDTH-',
+ 'FULL-WIDTH-C',
+ 'FULL-WIDTH-CH',
+ 'FULL-WIDTH-CHA',
+ 'FULL-WIDTH-CHAR',
+ 'FULL-WIDTH-PIXELS',
+ 'FULL-WIDTH-P',
+ 'FULL-WIDTH-PI',
+ 'FULL-WIDTH-PIX',
+ 'FULL-WIDTH-PIXE',
+ 'FULL-WIDTH-PIXEL',
+ 'FUNCTION',
+ 'FUNCTION-CALL-TYPE',
+ 'GATEWAYS',
+ 'GATEWAY',
+ 'GE',
+ 'GENERATE-MD5',
+ 'GENERATE-PBE-KEY',
+ 'GENERATE-PBE-SALT',
+ 'GENERATE-RANDOM-KEY',
+ 'GENERATE-UUID',
+ 'GET',
+ 'GET-ATTR-CALL-TYPE',
+ 'GET-ATTRIBUTE-NODE',
+ 'GET-BINARY-DATA',
+ 'GET-BLUE-VALUE',
+ 'GET-BLUE',
+ 'GET-BLUE-',
+ 'GET-BLUE-V',
+ 'GET-BLUE-VA',
+ 'GET-BLUE-VAL',
+ 'GET-BLUE-VALU',
+ 'GET-BROWSE-COLUMN',
+ 'GET-BUFFER-HANDLEGETBYTE',
+ 'GET-BYTE',
+ 'GET-CALLBACK-PROC-CONTEXT',
+ 'GET-CALLBACK-PROC-NAME',
+ 'GET-CGI-LIST',
+ 'GET-CGI-LONG-VALUE',
+ 'GET-CGI-VALUE',
+ 'GET-CODEPAGES',
+ 'GET-COLLATIONS',
+ 'GET-CONFIG-VALUE',
+ 'GET-CURRENT',
+ 'GET-DOUBLE',
+ 'GET-DROPPED-FILE',
+ 'GET-DYNAMIC',
+ 'GET-ERROR-COLUMN',
+ 'GET-ERROR-ROW',
+ 'GET-FILE',
+ 'GET-FILE-NAME',
+ 'GET-FILE-OFFSET',
+ 'GET-FILE-OFFSE',
+ 'GET-FIRST',
+ 'GET-FLOAT',
+ 'GET-GREEN-VALUE',
+ 'GET-GREEN',
+ 'GET-GREEN-',
+ 'GET-GREEN-V',
+ 'GET-GREEN-VA',
+ 'GET-GREEN-VAL',
+ 'GET-GREEN-VALU',
+ 'GET-INDEX-BY-NAMESPACE-NAME',
+ 'GET-INDEX-BY-QNAME',
+ 'GET-INT64',
+ 'GET-ITERATION',
+ 'GET-KEY-VALUE',
+ 'GET-KEY-VAL',
+ 'GET-KEY-VALU',
+ 'GET-LAST',
+ 'GET-LOCALNAME-BY-INDEX',
+ 'GET-LONG',
+ 'GET-MESSAGE',
+ 'GET-NEXT',
+ 'GET-NUMBER',
+ 'GET-POINTER-VALUE',
+ 'GET-PREV',
+ 'GET-PRINTERS',
+ 'GET-PROPERTY',
+ 'GET-QNAME-BY-INDEX',
+ 'GET-RED-VALUE',
+ 'GET-RED',
+ 'GET-RED-',
+ 'GET-RED-V',
+ 'GET-RED-VA',
+ 'GET-RED-VAL',
+ 'GET-RED-VALU',
+ 'GET-REPOSITIONED-ROW',
+ 'GET-RGB-VALUE',
+ 'GET-SELECTED-WIDGET',
+ 'GET-SELECTED',
+ 'GET-SELECTED-',
+ 'GET-SELECTED-W',
+ 'GET-SELECTED-WI',
+ 'GET-SELECTED-WID',
+ 'GET-SELECTED-WIDG',
+ 'GET-SELECTED-WIDGE',
+ 'GET-SHORT',
+ 'GET-SIGNATURE',
+ 'GET-SIZE',
+ 'GET-STRING',
+ 'GET-TAB-ITEM',
+ 'GET-TEXT-HEIGHT-CHARS',
+ 'GET-TEXT-HEIGHT',
+ 'GET-TEXT-HEIGHT-',
+ 'GET-TEXT-HEIGHT-C',
+ 'GET-TEXT-HEIGHT-CH',
+ 'GET-TEXT-HEIGHT-CHA',
+ 'GET-TEXT-HEIGHT-CHAR',
+ 'GET-TEXT-HEIGHT-PIXELS',
+ 'GET-TEXT-HEIGHT-P',
+ 'GET-TEXT-HEIGHT-PI',
+ 'GET-TEXT-HEIGHT-PIX',
+ 'GET-TEXT-HEIGHT-PIXE',
+ 'GET-TEXT-HEIGHT-PIXEL',
+ 'GET-TEXT-WIDTH-CHARS',
+ 'GET-TEXT-WIDTH',
+ 'GET-TEXT-WIDTH-',
+ 'GET-TEXT-WIDTH-C',
+ 'GET-TEXT-WIDTH-CH',
+ 'GET-TEXT-WIDTH-CHA',
+ 'GET-TEXT-WIDTH-CHAR',
+ 'GET-TEXT-WIDTH-PIXELS',
+ 'GET-TEXT-WIDTH-P',
+ 'GET-TEXT-WIDTH-PI',
+ 'GET-TEXT-WIDTH-PIX',
+ 'GET-TEXT-WIDTH-PIXE',
+ 'GET-TEXT-WIDTH-PIXEL',
+ 'GET-TYPE-BY-INDEX',
+ 'GET-TYPE-BY-NAMESPACE-NAME',
+ 'GET-TYPE-BY-QNAME',
+ 'GET-UNSIGNED-LONG',
+ 'GET-UNSIGNED-SHORT',
+ 'GET-URI-BY-INDEX',
+ 'GET-VALUE-BY-INDEX',
+ 'GET-VALUE-BY-NAMESPACE-NAME',
+ 'GET-VALUE-BY-QNAME',
+ 'GET-WAIT-STATE',
+ 'GLOBAL',
+ 'GO-ON',
+ 'GO-PENDING',
+ 'GO-PEND',
+ 'GO-PENDI',
+ 'GO-PENDIN',
+ 'GRANT',
+ 'GRAPHIC-EDGE',
+ 'GRAPHIC-E',
+ 'GRAPHIC-ED',
+ 'GRAPHIC-EDG',
+ 'GRID-FACTOR-HORIZONTAL',
+ 'GRID-FACTOR-H',
+ 'GRID-FACTOR-HO',
+ 'GRID-FACTOR-HOR',
+ 'GRID-FACTOR-HORI',
+ 'GRID-FACTOR-HORIZ',
+ 'GRID-FACTOR-HORIZO',
+ 'GRID-FACTOR-HORIZON',
+ 'GRID-FACTOR-HORIZONT',
+ 'GRID-FACTOR-HORIZONTA',
+ 'GRID-FACTOR-VERTICAL',
+ 'GRID-FACTOR-V',
+ 'GRID-FACTOR-VE',
+ 'GRID-FACTOR-VER',
+ 'GRID-FACTOR-VERT',
+ 'GRID-FACTOR-VERTI',
+ 'GRID-FACTOR-VERTIC',
+ 'GRID-FACTOR-VERTICA',
+ 'GRID-SNAP',
+ 'GRID-UNIT-HEIGHT-CHARS',
+ 'GRID-UNIT-HEIGHT',
+ 'GRID-UNIT-HEIGHT-',
+ 'GRID-UNIT-HEIGHT-C',
+ 'GRID-UNIT-HEIGHT-CH',
+ 'GRID-UNIT-HEIGHT-CHA',
+ 'GRID-UNIT-HEIGHT-PIXELS',
+ 'GRID-UNIT-HEIGHT-P',
+ 'GRID-UNIT-HEIGHT-PI',
+ 'GRID-UNIT-HEIGHT-PIX',
+ 'GRID-UNIT-HEIGHT-PIXE',
+ 'GRID-UNIT-HEIGHT-PIXEL',
+ 'GRID-UNIT-WIDTH-CHARS',
+ 'GRID-UNIT-WIDTH',
+ 'GRID-UNIT-WIDTH-',
+ 'GRID-UNIT-WIDTH-C',
+ 'GRID-UNIT-WIDTH-CH',
+ 'GRID-UNIT-WIDTH-CHA',
+ 'GRID-UNIT-WIDTH-CHAR',
+ 'GRID-UNIT-WIDTH-PIXELS',
+ 'GRID-UNIT-WIDTH-P',
+ 'GRID-UNIT-WIDTH-PI',
+ 'GRID-UNIT-WIDTH-PIX',
+ 'GRID-UNIT-WIDTH-PIXE',
+ 'GRID-UNIT-WIDTH-PIXEL',
+ 'GRID-VISIBLE',
+ 'GROUP',
+ 'GT',
+ 'GUID',
+ 'HANDLER',
+ 'HAS-RECORDS',
+ 'HAVING',
+ 'HEADER',
+ 'HEIGHT-CHARS',
+ 'HEIGHT',
+ 'HEIGHT-',
+ 'HEIGHT-C',
+ 'HEIGHT-CH',
+ 'HEIGHT-CHA',
+ 'HEIGHT-CHAR',
+ 'HEIGHT-PIXELS',
+ 'HEIGHT-P',
+ 'HEIGHT-PI',
+ 'HEIGHT-PIX',
+ 'HEIGHT-PIXE',
+ 'HEIGHT-PIXEL',
+ 'HELP',
+ 'HEX-DECODE',
+ 'HEX-ENCODE',
+ 'HIDDEN',
+ 'HIDE',
+ 'HORIZONTAL',
+ 'HORI',
+ 'HORIZ',
+ 'HORIZO',
+ 'HORIZON',
+ 'HORIZONT',
+ 'HORIZONTA',
+ 'HOST-BYTE-ORDER',
+ 'HTML-CHARSET',
+ 'HTML-END-OF-LINE',
+ 'HTML-END-OF-PAGE',
+ 'HTML-FRAME-BEGIN',
+ 'HTML-FRAME-END',
+ 'HTML-HEADER-BEGIN',
+ 'HTML-HEADER-END',
+ 'HTML-TITLE-BEGIN',
+ 'HTML-TITLE-END',
+ 'HWND',
+ 'ICON',
+ 'IF',
+ 'IMAGE',
+ 'IMAGE-DOWN',
+ 'IMAGE-INSENSITIVE',
+ 'IMAGE-SIZE',
+ 'IMAGE-SIZE-CHARS',
+ 'IMAGE-SIZE-C',
+ 'IMAGE-SIZE-CH',
+ 'IMAGE-SIZE-CHA',
+ 'IMAGE-SIZE-CHAR',
+ 'IMAGE-SIZE-PIXELS',
+ 'IMAGE-SIZE-P',
+ 'IMAGE-SIZE-PI',
+ 'IMAGE-SIZE-PIX',
+ 'IMAGE-SIZE-PIXE',
+ 'IMAGE-SIZE-PIXEL',
+ 'IMAGE-UP',
+ 'IMMEDIATE-DISPLAY',
+ 'IMPLEMENTS',
+ 'IMPORT',
+ 'IMPORT-PRINCIPAL',
+ 'IN',
+ 'INCREMENT-EXCLUSIVE-ID',
+ 'INDEX',
+ 'INDEXED-REPOSITION',
+ 'INDEX-HINT',
+ 'INDEX-INFORMATION',
+ 'INDICATOR',
+ 'INFORMATION',
+ 'INFO',
+ 'INFOR',
+ 'INFORM',
+ 'INFORMA',
+ 'INFORMAT',
+ 'INFORMATI',
+ 'INFORMATIO',
+ 'IN-HANDLE',
+ 'INHERIT-BGCOLOR',
+ 'INHERIT-BGC',
+ 'INHERIT-BGCO',
+ 'INHERIT-BGCOL',
+ 'INHERIT-BGCOLO',
+ 'INHERIT-FGCOLOR',
+ 'INHERIT-FGC',
+ 'INHERIT-FGCO',
+ 'INHERIT-FGCOL',
+ 'INHERIT-FGCOLO',
+ 'INHERITS',
+ 'INITIAL',
+ 'INIT',
+ 'INITI',
+ 'INITIA',
+ 'INITIAL-DIR',
+ 'INITIAL-FILTER',
+ 'INITIALIZE-DOCUMENT-TYPE',
+ 'INITIATE',
+ 'INNER-CHARS',
+ 'INNER-LINES',
+ 'INPUT',
+ 'INPUT-OUTPUT',
+ 'INPUT-O',
+ 'INPUT-OU',
+ 'INPUT-OUT',
+ 'INPUT-OUTP',
+ 'INPUT-OUTPU',
+ 'INPUT-VALUE',
+ 'INSERT',
+ 'INSERT-ATTRIBUTE',
+ 'INSERT-BACKTAB',
+ 'INSERT-B',
+ 'INSERT-BA',
+ 'INSERT-BAC',
+ 'INSERT-BACK',
+ 'INSERT-BACKT',
+ 'INSERT-BACKTA',
+ 'INSERT-FILE',
+ 'INSERT-ROW',
+ 'INSERT-STRING',
+ 'INSERT-TAB',
+ 'INSERT-T',
+ 'INSERT-TA',
+ 'INTERFACE',
+ 'INTERNAL-ENTRIES',
+ 'INTO',
+ 'INVOKE',
+ 'IS',
+ 'IS-ATTR-SPACE',
+ 'IS-ATTR',
+ 'IS-ATTR-',
+ 'IS-ATTR-S',
+ 'IS-ATTR-SP',
+ 'IS-ATTR-SPA',
+ 'IS-ATTR-SPAC',
+ 'IS-CLASS',
+ 'IS-CLAS',
+ 'IS-LEAD-BYTE',
+ 'IS-OPEN',
+ 'IS-PARAMETER-SET',
+ 'IS-ROW-SELECTED',
+ 'IS-SELECTED',
+ 'ITEM',
+ 'ITEMS-PER-ROW',
+ 'JOIN',
+ 'JOIN-BY-SQLDB',
+ 'KBLABEL',
+ 'KEEP-CONNECTION-OPEN',
+ 'KEEP-FRAME-Z-ORDER',
+ 'KEEP-FRAME-Z',
+ 'KEEP-FRAME-Z-',
+ 'KEEP-FRAME-Z-O',
+ 'KEEP-FRAME-Z-OR',
+ 'KEEP-FRAME-Z-ORD',
+ 'KEEP-FRAME-Z-ORDE',
+ 'KEEP-MESSAGES',
+ 'KEEP-SECURITY-CACHE',
+ 'KEEP-TAB-ORDER',
+ 'KEY',
+ 'KEYCODE',
+ 'KEY-CODE',
+ 'KEYFUNCTION',
+ 'KEYFUNC',
+ 'KEYFUNCT',
+ 'KEYFUNCTI',
+ 'KEYFUNCTIO',
+ 'KEY-FUNCTION',
+ 'KEY-FUNC',
+ 'KEY-FUNCT',
+ 'KEY-FUNCTI',
+ 'KEY-FUNCTIO',
+ 'KEYLABEL',
+ 'KEY-LABEL',
+ 'KEYS',
+ 'KEYWORD',
+ 'KEYWORD-ALL',
+ 'LABEL',
+ 'LABEL-BGCOLOR',
+ 'LABEL-BGC',
+ 'LABEL-BGCO',
+ 'LABEL-BGCOL',
+ 'LABEL-BGCOLO',
+ 'LABEL-DCOLOR',
+ 'LABEL-DC',
+ 'LABEL-DCO',
+ 'LABEL-DCOL',
+ 'LABEL-DCOLO',
+ 'LABEL-FGCOLOR',
+ 'LABEL-FGC',
+ 'LABEL-FGCO',
+ 'LABEL-FGCOL',
+ 'LABEL-FGCOLO',
+ 'LABEL-FONT',
+ 'LABEL-PFCOLOR',
+ 'LABEL-PFC',
+ 'LABEL-PFCO',
+ 'LABEL-PFCOL',
+ 'LABEL-PFCOLO',
+ 'LABELS',
+ 'LANDSCAPE',
+ 'LANGUAGES',
+ 'LANGUAGE',
+ 'LARGE',
+ 'LARGE-TO-SMALL',
+ 'LAST',
+ 'LAST-ASYNCH-REQUEST',
+ 'LAST-BATCH',
+ 'LAST-CHILD',
+ 'LAST-EVENT',
+ 'LAST-EVEN',
+ 'LAST-FORM',
+ 'LASTKEY',
+ 'LAST-KEY',
+ 'LAST-OBJECT',
+ 'LAST-OF',
+ 'LAST-PROCEDURE',
+ 'LAST-PROCE',
+ 'LAST-PROCED',
+ 'LAST-PROCEDU',
+ 'LAST-PROCEDUR',
+ 'LAST-SERVER',
+ 'LAST-TAB-ITEM',
+ 'LAST-TAB-I',
+ 'LAST-TAB-IT',
+ 'LAST-TAB-ITE',
+ 'LC',
+ 'LDBNAME',
+ 'LE',
+ 'LEAVE',
+ 'LEFT-ALIGNED',
+ 'LEFT-ALIGN',
+ 'LEFT-ALIGNE',
+ 'LEFT-TRIM',
+ 'LENGTH',
+ 'LIBRARY',
+ 'LIKE',
+ 'LIKE-SEQUENTIAL',
+ 'LINE',
+ 'LINE-COUNTER',
+ 'LINE-COUNT',
+ 'LINE-COUNTE',
+ 'LIST-EVENTS',
+ 'LISTING',
+ 'LISTI',
+ 'LISTIN',
+ 'LIST-ITEM-PAIRS',
+ 'LIST-ITEMS',
+ 'LIST-PROPERTY-NAMES',
+ 'LIST-QUERY-ATTRS',
+ 'LIST-SET-ATTRS',
+ 'LIST-WIDGETS',
+ 'LITERAL-QUESTION',
+ 'LITTLE-ENDIAN',
+ 'LOAD',
+ 'LOAD-DOMAINS',
+ 'LOAD-ICON',
+ 'LOAD-IMAGE',
+ 'LOAD-IMAGE-DOWN',
+ 'LOAD-IMAGE-INSENSITIVE',
+ 'LOAD-IMAGE-UP',
+ 'LOAD-MOUSE-POINTER',
+ 'LOAD-MOUSE-P',
+ 'LOAD-MOUSE-PO',
+ 'LOAD-MOUSE-POI',
+ 'LOAD-MOUSE-POIN',
+ 'LOAD-MOUSE-POINT',
+ 'LOAD-MOUSE-POINTE',
+ 'LOAD-PICTURE',
+ 'LOAD-SMALL-ICON',
+ 'LOCAL-NAME',
+ 'LOCATOR-COLUMN-NUMBER',
+ 'LOCATOR-LINE-NUMBER',
+ 'LOCATOR-PUBLIC-ID',
+ 'LOCATOR-SYSTEM-ID',
+ 'LOCATOR-TYPE',
+ 'LOCKED',
+ 'LOCK-REGISTRATION',
+ 'LOG',
+ 'LOG-AUDIT-EVENT',
+ 'LOGIN-EXPIRATION-TIMESTAMP',
+ 'LOGIN-HOST',
+ 'LOGIN-STATE',
+ 'LOG-MANAGER',
+ 'LOGOUT',
+ 'LOOKAHEAD',
+ 'LOOKUP',
+ 'LT',
+ 'MACHINE-CLASS',
+ 'MANDATORY',
+ 'MANUAL-HIGHLIGHT',
+ 'MAP',
+ 'MARGIN-EXTRA',
+ 'MARGIN-HEIGHT-CHARS',
+ 'MARGIN-HEIGHT',
+ 'MARGIN-HEIGHT-',
+ 'MARGIN-HEIGHT-C',
+ 'MARGIN-HEIGHT-CH',
+ 'MARGIN-HEIGHT-CHA',
+ 'MARGIN-HEIGHT-CHAR',
+ 'MARGIN-HEIGHT-PIXELS',
+ 'MARGIN-HEIGHT-P',
+ 'MARGIN-HEIGHT-PI',
+ 'MARGIN-HEIGHT-PIX',
+ 'MARGIN-HEIGHT-PIXE',
+ 'MARGIN-HEIGHT-PIXEL',
+ 'MARGIN-WIDTH-CHARS',
+ 'MARGIN-WIDTH',
+ 'MARGIN-WIDTH-',
+ 'MARGIN-WIDTH-C',
+ 'MARGIN-WIDTH-CH',
+ 'MARGIN-WIDTH-CHA',
+ 'MARGIN-WIDTH-CHAR',
+ 'MARGIN-WIDTH-PIXELS',
+ 'MARGIN-WIDTH-P',
+ 'MARGIN-WIDTH-PI',
+ 'MARGIN-WIDTH-PIX',
+ 'MARGIN-WIDTH-PIXE',
+ 'MARGIN-WIDTH-PIXEL',
+ 'MARK-NEW',
+ 'MARK-ROW-STATE',
+ 'MATCHES',
+ 'MAX-BUTTON',
+ 'MAX-CHARS',
+ 'MAX-DATA-GUESS',
+ 'MAX-HEIGHT',
+ 'MAX-HEIGHT-CHARS',
+ 'MAX-HEIGHT-C',
+ 'MAX-HEIGHT-CH',
+ 'MAX-HEIGHT-CHA',
+ 'MAX-HEIGHT-CHAR',
+ 'MAX-HEIGHT-PIXELS',
+ 'MAX-HEIGHT-P',
+ 'MAX-HEIGHT-PI',
+ 'MAX-HEIGHT-PIX',
+ 'MAX-HEIGHT-PIXE',
+ 'MAX-HEIGHT-PIXEL',
+ 'MAXIMIZE',
+ 'MAXIMUM',
+ 'MAX',
+ 'MAXI',
+ 'MAXIM',
+ 'MAXIMU',
+ 'MAXIMUM-LEVEL',
+ 'MAX-ROWS',
+ 'MAX-SIZE',
+ 'MAX-VALUE',
+ 'MAX-VAL',
+ 'MAX-VALU',
+ 'MAX-WIDTH-CHARS',
+ 'MAX-WIDTH',
+ 'MAX-WIDTH-',
+ 'MAX-WIDTH-C',
+ 'MAX-WIDTH-CH',
+ 'MAX-WIDTH-CHA',
+ 'MAX-WIDTH-CHAR',
+ 'MAX-WIDTH-PIXELS',
+ 'MAX-WIDTH-P',
+ 'MAX-WIDTH-PI',
+ 'MAX-WIDTH-PIX',
+ 'MAX-WIDTH-PIXE',
+ 'MAX-WIDTH-PIXEL',
+ 'MD5-DIGEST',
+ 'MEMBER',
+ 'MEMPTR-TO-NODE-VALUE',
+ 'MENU',
+ 'MENUBAR',
+ 'MENU-BAR',
+ 'MENU-ITEM',
+ 'MENU-KEY',
+ 'MENU-K',
+ 'MENU-KE',
+ 'MENU-MOUSE',
+ 'MENU-M',
+ 'MENU-MO',
+ 'MENU-MOU',
+ 'MENU-MOUS',
+ 'MERGE-BY-FIELD',
+ 'MESSAGE',
+ 'MESSAGE-AREA',
+ 'MESSAGE-AREA-FONT',
+ 'MESSAGE-LINES',
+ 'METHOD',
+ 'MIN-BUTTON',
+ 'MIN-COLUMN-WIDTH-CHARS',
+ 'MIN-COLUMN-WIDTH-C',
+ 'MIN-COLUMN-WIDTH-CH',
+ 'MIN-COLUMN-WIDTH-CHA',
+ 'MIN-COLUMN-WIDTH-CHAR',
+ 'MIN-COLUMN-WIDTH-PIXELS',
+ 'MIN-COLUMN-WIDTH-P',
+ 'MIN-COLUMN-WIDTH-PI',
+ 'MIN-COLUMN-WIDTH-PIX',
+ 'MIN-COLUMN-WIDTH-PIXE',
+ 'MIN-COLUMN-WIDTH-PIXEL',
+ 'MIN-HEIGHT-CHARS',
+ 'MIN-HEIGHT',
+ 'MIN-HEIGHT-',
+ 'MIN-HEIGHT-C',
+ 'MIN-HEIGHT-CH',
+ 'MIN-HEIGHT-CHA',
+ 'MIN-HEIGHT-CHAR',
+ 'MIN-HEIGHT-PIXELS',
+ 'MIN-HEIGHT-P',
+ 'MIN-HEIGHT-PI',
+ 'MIN-HEIGHT-PIX',
+ 'MIN-HEIGHT-PIXE',
+ 'MIN-HEIGHT-PIXEL',
+ 'MINIMUM',
+ 'MIN',
+ 'MINI',
+ 'MINIM',
+ 'MINIMU',
+ 'MIN-SIZE',
+ 'MIN-VALUE',
+ 'MIN-VAL',
+ 'MIN-VALU',
+ 'MIN-WIDTH-CHARS',
+ 'MIN-WIDTH',
+ 'MIN-WIDTH-',
+ 'MIN-WIDTH-C',
+ 'MIN-WIDTH-CH',
+ 'MIN-WIDTH-CHA',
+ 'MIN-WIDTH-CHAR',
+ 'MIN-WIDTH-PIXELS',
+ 'MIN-WIDTH-P',
+ 'MIN-WIDTH-PI',
+ 'MIN-WIDTH-PIX',
+ 'MIN-WIDTH-PIXE',
+ 'MIN-WIDTH-PIXEL',
+ 'MODIFIED',
+ 'MODULO',
+ 'MOD',
+ 'MODU',
+ 'MODUL',
+ 'MONTH',
+ 'MOUSE',
+ 'MOUSE-POINTER',
+ 'MOUSE-P',
+ 'MOUSE-PO',
+ 'MOUSE-POI',
+ 'MOUSE-POIN',
+ 'MOUSE-POINT',
+ 'MOUSE-POINTE',
+ 'MOVABLE',
+ 'MOVE-AFTER-TAB-ITEM',
+ 'MOVE-AFTER',
+ 'MOVE-AFTER-',
+ 'MOVE-AFTER-T',
+ 'MOVE-AFTER-TA',
+ 'MOVE-AFTER-TAB',
+ 'MOVE-AFTER-TAB-',
+ 'MOVE-AFTER-TAB-I',
+ 'MOVE-AFTER-TAB-IT',
+ 'MOVE-AFTER-TAB-ITE',
+ 'MOVE-BEFORE-TAB-ITEM',
+ 'MOVE-BEFOR',
+ 'MOVE-BEFORE',
+ 'MOVE-BEFORE-',
+ 'MOVE-BEFORE-T',
+ 'MOVE-BEFORE-TA',
+ 'MOVE-BEFORE-TAB',
+ 'MOVE-BEFORE-TAB-',
+ 'MOVE-BEFORE-TAB-I',
+ 'MOVE-BEFORE-TAB-IT',
+ 'MOVE-BEFORE-TAB-ITE',
+ 'MOVE-COLUMN',
+ 'MOVE-COL',
+ 'MOVE-COLU',
+ 'MOVE-COLUM',
+ 'MOVE-TO-BOTTOM',
+ 'MOVE-TO-B',
+ 'MOVE-TO-BO',
+ 'MOVE-TO-BOT',
+ 'MOVE-TO-BOTT',
+ 'MOVE-TO-BOTTO',
+ 'MOVE-TO-EOF',
+ 'MOVE-TO-TOP',
+ 'MOVE-TO-T',
+ 'MOVE-TO-TO',
+ 'MPE',
+ 'MULTI-COMPILE',
+ 'MULTIPLE',
+ 'MULTIPLE-KEY',
+ 'MULTITASKING-INTERVAL',
+ 'MUST-EXIST',
+ 'NAME',
+ 'NAMESPACE-PREFIX',
+ 'NAMESPACE-URI',
+ 'NATIVE',
+ 'NE',
+ 'NEEDS-APPSERVER-PROMPT',
+ 'NEEDS-PROMPT',
+ 'NEW',
+ 'NEW-INSTANCE',
+ 'NEW-ROW',
+ 'NEXT',
+ 'NEXT-COLUMN',
+ 'NEXT-PROMPT',
+ 'NEXT-ROWID',
+ 'NEXT-SIBLING',
+ 'NEXT-TAB-ITEM',
+ 'NEXT-TAB-I',
+ 'NEXT-TAB-IT',
+ 'NEXT-TAB-ITE',
+ 'NEXT-VALUE',
+ 'NO',
+ 'NO-APPLY',
+ 'NO-ARRAY-MESSAGE',
+ 'NO-ASSIGN',
+ 'NO-ATTR-LIST',
+ 'NO-ATTR',
+ 'NO-ATTR-',
+ 'NO-ATTR-L',
+ 'NO-ATTR-LI',
+ 'NO-ATTR-LIS',
+ 'NO-ATTR-SPACE',
+ 'NO-ATTR-S',
+ 'NO-ATTR-SP',
+ 'NO-ATTR-SPA',
+ 'NO-ATTR-SPAC',
+ 'NO-AUTO-VALIDATE',
+ 'NO-BIND-WHERE',
+ 'NO-BOX',
+ 'NO-CONSOLE',
+ 'NO-CONVERT',
+ 'NO-CONVERT-3D-COLORS',
+ 'NO-CURRENT-VALUE',
+ 'NO-DEBUG',
+ 'NODE-VALUE-TO-MEMPTR',
+ 'NO-DRAG',
+ 'NO-ECHO',
+ 'NO-EMPTY-SPACE',
+ 'NO-ERROR',
+ 'NO-FILL',
+ 'NO-F',
+ 'NO-FI',
+ 'NO-FIL',
+ 'NO-FOCUS',
+ 'NO-HELP',
+ 'NO-HIDE',
+ 'NO-INDEX-HINT',
+ 'NO-INHERIT-BGCOLOR',
+ 'NO-INHERIT-BGC',
+ 'NO-INHERIT-BGCO',
+ 'NO-INHERIT-FGCOLOR',
+ 'NO-INHERIT-FGC',
+ 'NO-INHERIT-FGCO',
+ 'NO-INHERIT-FGCOL',
+ 'NO-INHERIT-FGCOLO',
+ 'NO-JOIN-BY-SQLDB',
+ 'NO-LABELS',
+ 'NO-LABE',
+ 'NO-LOBS',
+ 'NO-LOCK',
+ 'NO-LOOKAHEAD',
+ 'NO-MAP',
+ 'NO-MESSAGE',
+ 'NO-MES',
+ 'NO-MESS',
+ 'NO-MESSA',
+ 'NO-MESSAG',
+ 'NONAMESPACE-SCHEMA-LOCATION',
+ 'NONE',
+ 'NO-PAUSE',
+ 'NO-PREFETCH',
+ 'NO-PREFE',
+ 'NO-PREFET',
+ 'NO-PREFETC',
+ 'NORMALIZE',
+ 'NO-ROW-MARKERS',
+ 'NO-SCROLLBAR-VERTICAL',
+ 'NO-SEPARATE-CONNECTION',
+ 'NO-SEPARATORS',
+ 'NOT',
+ 'NO-TAB-STOP',
+ 'NOT-ACTIVE',
+ 'NO-UNDERLINE',
+ 'NO-UND',
+ 'NO-UNDE',
+ 'NO-UNDER',
+ 'NO-UNDERL',
+ 'NO-UNDERLI',
+ 'NO-UNDERLIN',
+ 'NO-UNDO',
+ 'NO-VALIDATE',
+ 'NO-VAL',
+ 'NO-VALI',
+ 'NO-VALID',
+ 'NO-VALIDA',
+ 'NO-VALIDAT',
+ 'NOW',
+ 'NO-WAIT',
+ 'NO-WORD-WRAP',
+ 'NULL',
+ 'NUM-ALIASES',
+ 'NUM-ALI',
+ 'NUM-ALIA',
+ 'NUM-ALIAS',
+ 'NUM-ALIASE',
+ 'NUM-BUFFERS',
+ 'NUM-BUTTONS',
+ 'NUM-BUT',
+ 'NUM-BUTT',
+ 'NUM-BUTTO',
+ 'NUM-BUTTON',
+ 'NUM-COLUMNS',
+ 'NUM-COL',
+ 'NUM-COLU',
+ 'NUM-COLUM',
+ 'NUM-COLUMN',
+ 'NUM-COPIES',
+ 'NUM-DBS',
+ 'NUM-DROPPED-FILES',
+ 'NUM-ENTRIES',
+ 'NUMERIC',
+ 'NUMERIC-FORMAT',
+ 'NUMERIC-F',
+ 'NUMERIC-FO',
+ 'NUMERIC-FOR',
+ 'NUMERIC-FORM',
+ 'NUMERIC-FORMA',
+ 'NUM-FIELDS',
+ 'NUM-FORMATS',
+ 'NUM-ITEMS',
+ 'NUM-ITERATIONS',
+ 'NUM-LINES',
+ 'NUM-LOCKED-COLUMNS',
+ 'NUM-LOCKED-COL',
+ 'NUM-LOCKED-COLU',
+ 'NUM-LOCKED-COLUM',
+ 'NUM-LOCKED-COLUMN',
+ 'NUM-MESSAGES',
+ 'NUM-PARAMETERS',
+ 'NUM-REFERENCES',
+ 'NUM-REPLACED',
+ 'NUM-RESULTS',
+ 'NUM-SELECTED-ROWS',
+ 'NUM-SELECTED-WIDGETS',
+ 'NUM-SELECTED',
+ 'NUM-SELECTED-',
+ 'NUM-SELECTED-W',
+ 'NUM-SELECTED-WI',
+ 'NUM-SELECTED-WID',
+ 'NUM-SELECTED-WIDG',
+ 'NUM-SELECTED-WIDGE',
+ 'NUM-SELECTED-WIDGET',
+ 'NUM-TABS',
+ 'NUM-TO-RETAIN',
+ 'NUM-VISIBLE-COLUMNS',
+ 'OCTET-LENGTH',
+ 'OF',
+ 'OFF',
+ 'OK',
+ 'OK-CANCEL',
+ 'OLD',
+ 'ON',
+ 'ON-FRAME-BORDER',
+ 'ON-FRAME',
+ 'ON-FRAME-',
+ 'ON-FRAME-B',
+ 'ON-FRAME-BO',
+ 'ON-FRAME-BOR',
+ 'ON-FRAME-BORD',
+ 'ON-FRAME-BORDE',
+ 'OPEN',
+ 'OPSYS',
+ 'OPTION',
+ 'OR',
+ 'ORDERED-JOIN',
+ 'ORDINAL',
+ 'OS-APPEND',
+ 'OS-COMMAND',
+ 'OS-COPY',
+ 'OS-CREATE-DIR',
+ 'OS-DELETE',
+ 'OS-DIR',
+ 'OS-DRIVES',
+ 'OS-DRIVE',
+ 'OS-ERROR',
+ 'OS-GETENV',
+ 'OS-RENAME',
+ 'OTHERWISE',
+ 'OUTPUT',
+ 'OVERLAY',
+ 'OVERRIDE',
+ 'OWNER',
+ 'PAGE',
+ 'PAGE-BOTTOM',
+ 'PAGE-BOT',
+ 'PAGE-BOTT',
+ 'PAGE-BOTTO',
+ 'PAGED',
+ 'PAGE-NUMBER',
+ 'PAGE-NUM',
+ 'PAGE-NUMB',
+ 'PAGE-NUMBE',
+ 'PAGE-SIZE',
+ 'PAGE-TOP',
+ 'PAGE-WIDTH',
+ 'PAGE-WID',
+ 'PAGE-WIDT',
+ 'PARAMETER',
+ 'PARAM',
+ 'PARAME',
+ 'PARAMET',
+ 'PARAMETE',
+ 'PARENT',
+ 'PARSE-STATUS',
+ 'PARTIAL-KEY',
+ 'PASCAL',
+ 'PASSWORD-FIELD',
+ 'PATHNAME',
+ 'PAUSE',
+ 'PBE-HASH-ALGORITHM',
+ 'PBE-HASH-ALG',
+ 'PBE-HASH-ALGO',
+ 'PBE-HASH-ALGOR',
+ 'PBE-HASH-ALGORI',
+ 'PBE-HASH-ALGORIT',
+ 'PBE-HASH-ALGORITH',
+ 'PBE-KEY-ROUNDS',
+ 'PDBNAME',
+ 'PERSISTENT',
+ 'PERSIST',
+ 'PERSISTE',
+ 'PERSISTEN',
+ 'PERSISTENT-CACHE-DISABLED',
+ 'PFCOLOR',
+ 'PFC',
+ 'PFCO',
+ 'PFCOL',
+ 'PFCOLO',
+ 'PIXELS',
+ 'PIXELS-PER-COLUMN',
+ 'PIXELS-PER-COL',
+ 'PIXELS-PER-COLU',
+ 'PIXELS-PER-COLUM',
+ 'PIXELS-PER-ROW',
+ 'POPUP-MENU',
+ 'POPUP-M',
+ 'POPUP-ME',
+ 'POPUP-MEN',
+ 'POPUP-ONLY',
+ 'POPUP-O',
+ 'POPUP-ON',
+ 'POPUP-ONL',
+ 'PORTRAIT',
+ 'POSITION',
+ 'PRECISION',
+ 'PREFER-DATASET',
+ 'PREPARED',
+ 'PREPARE-STRING',
+ 'PREPROCESS',
+ 'PREPROC',
+ 'PREPROCE',
+ 'PREPROCES',
+ 'PRESELECT',
+ 'PRESEL',
+ 'PRESELE',
+ 'PRESELEC',
+ 'PREV',
+ 'PREV-COLUMN',
+ 'PREV-SIBLING',
+ 'PREV-TAB-ITEM',
+ 'PREV-TAB-I',
+ 'PREV-TAB-IT',
+ 'PREV-TAB-ITE',
+ 'PRIMARY',
+ 'PRINTER',
+ 'PRINTER-CONTROL-HANDLE',
+ 'PRINTER-HDC',
+ 'PRINTER-NAME',
+ 'PRINTER-PORT',
+ 'PRINTER-SETUP',
+ 'PRIVATE',
+ 'PRIVATE-DATA',
+ 'PRIVATE-D',
+ 'PRIVATE-DA',
+ 'PRIVATE-DAT',
+ 'PRIVILEGES',
+ 'PROCEDURE',
+ 'PROCE',
+ 'PROCED',
+ 'PROCEDU',
+ 'PROCEDUR',
+ 'PROCEDURE-CALL-TYPE',
+ 'PROCESS',
+ 'PROC-HANDLE',
+ 'PROC-HA',
+ 'PROC-HAN',
+ 'PROC-HAND',
+ 'PROC-HANDL',
+ 'PROC-STATUS',
+ 'PROC-ST',
+ 'PROC-STA',
+ 'PROC-STAT',
+ 'PROC-STATU',
+ 'proc-text',
+ 'proc-text-buffe',
+ 'PROFILER',
+ 'PROGRAM-NAME',
+ 'PROGRESS',
+ 'PROGRESS-SOURCE',
+ 'PROGRESS-S',
+ 'PROGRESS-SO',
+ 'PROGRESS-SOU',
+ 'PROGRESS-SOUR',
+ 'PROGRESS-SOURC',
+ 'PROMPT',
+ 'PROMPT-FOR',
+ 'PROMPT-F',
+ 'PROMPT-FO',
+ 'PROMSGS',
+ 'PROPATH',
+ 'PROPERTY',
+ 'PROTECTED',
+ 'PROVERSION',
+ 'PROVERS',
+ 'PROVERSI',
+ 'PROVERSIO',
+ 'PROXY',
+ 'PROXY-PASSWORD',
+ 'PROXY-USERID',
+ 'PUBLIC',
+ 'PUBLIC-ID',
+ 'PUBLISH',
+ 'PUBLISHED-EVENTS',
+ 'PUT',
+ 'PUTBYTE',
+ 'PUT-BYTE',
+ 'PUT-DOUBLE',
+ 'PUT-FLOAT',
+ 'PUT-INT64',
+ 'PUT-KEY-VALUE',
+ 'PUT-KEY-VAL',
+ 'PUT-KEY-VALU',
+ 'PUT-LONG',
+ 'PUT-SHORT',
+ 'PUT-STRING',
+ 'PUT-UNSIGNED-LONG',
+ 'QUERY',
+ 'QUERY-CLOSE',
+ 'QUERY-OFF-END',
+ 'QUERY-OPEN',
+ 'QUERY-PREPARE',
+ 'QUERY-TUNING',
+ 'QUESTION',
+ 'QUIT',
+ 'QUOTER',
+ 'RADIO-BUTTONS',
+ 'RADIO-SET',
+ 'RANDOM',
+ 'RAW-TRANSFER',
+ 'RCODE-INFORMATION',
+ 'RCODE-INFO',
+ 'RCODE-INFOR',
+ 'RCODE-INFORM',
+ 'RCODE-INFORMA',
+ 'RCODE-INFORMAT',
+ 'RCODE-INFORMATI',
+ 'RCODE-INFORMATIO',
+ 'READ-AVAILABLE',
+ 'READ-EXACT-NUM',
+ 'READ-FILE',
+ 'READKEY',
+ 'READ-ONLY',
+ 'READ-XML',
+ 'READ-XMLSCHEMA',
+ 'REAL',
+ 'RECORD-LENGTH',
+ 'RECTANGLE',
+ 'RECT',
+ 'RECTA',
+ 'RECTAN',
+ 'RECTANG',
+ 'RECTANGL',
+ 'RECURSIVE',
+ 'REFERENCE-ONLY',
+ 'REFRESH',
+ 'REFRESHABLE',
+ 'REFRESH-AUDIT-POLICY',
+ 'REGISTER-DOMAIN',
+ 'RELEASE',
+ 'REMOTE',
+ 'REMOVE-EVENTS-PROCEDURE',
+ 'REMOVE-SUPER-PROCEDURE',
+ 'REPEAT',
+ 'REPLACE',
+ 'REPLACE-SELECTION-TEXT',
+ 'REPOSITION',
+ 'REPOSITION-BACKWARD',
+ 'REPOSITION-FORWARD',
+ 'REPOSITION-MODE',
+ 'REPOSITION-TO-ROW',
+ 'REPOSITION-TO-ROWID',
+ 'REQUEST',
+ 'RESET',
+ 'RESIZABLE',
+ 'RESIZA',
+ 'RESIZAB',
+ 'RESIZABL',
+ 'RESIZE',
+ 'RESTART-ROW',
+ 'RESTART-ROWID',
+ 'RETAIN',
+ 'RETAIN-SHAPE',
+ 'RETRY',
+ 'RETRY-CANCEL',
+ 'RETURN',
+ 'RETURN-INSERTED',
+ 'RETURN-INS',
+ 'RETURN-INSE',
+ 'RETURN-INSER',
+ 'RETURN-INSERT',
+ 'RETURN-INSERTE',
+ 'RETURNS',
+ 'RETURN-TO-START-DIR',
+ 'RETURN-TO-START-DI',
+ 'RETURN-VALUE',
+ 'RETURN-VAL',
+ 'RETURN-VALU',
+ 'RETURN-VALUE-DATA-TYPE',
+ 'REVERSE-FROM',
+ 'REVERT',
+ 'REVOKE',
+ 'RGB-VALUE',
+ 'RIGHT-ALIGNED',
+ 'RETURN-ALIGN',
+ 'RETURN-ALIGNE',
+ 'RIGHT-TRIM',
+ 'R-INDEX',
+ 'ROLES',
+ 'ROUND',
+ 'ROUTINE-LEVEL',
+ 'ROW',
+ 'ROW-HEIGHT-CHARS',
+ 'ROW-HEIGHT-PIXELS',
+ 'ROW-MARKERS',
+ 'ROW-OF',
+ 'ROW-RESIZABLE',
+ 'RULE',
+ 'RUN',
+ 'RUN-PROCEDURE',
+ 'SAVE',
+ 'SAVE-AS',
+ 'SAVE-FILE',
+ 'SAX-COMPLETE',
+ 'SAX-COMPLE',
+ 'SAX-COMPLET',
+ 'SAX-PARSE',
+ 'SAX-PARSE-FIRST',
+ 'SAX-PARSE-NEXT',
+ 'SAX-PARSER-ERROR',
+ 'SAX-RUNNING',
+ 'SAX-UNINITIALIZED',
+ 'SAX-WRITE-BEGIN',
+ 'SAX-WRITE-COMPLETE',
+ 'SAX-WRITE-CONTENT',
+ 'SAX-WRITE-ELEMENT',
+ 'SAX-WRITE-ERROR',
+ 'SAX-WRITE-IDLE',
+ 'SAX-WRITER',
+ 'SAX-WRITE-TAG',
+ 'SCHEMA',
+ 'SCHEMA-LOCATION',
+ 'SCHEMA-MARSHAL',
+ 'SCHEMA-PATH',
+ 'SCREEN',
+ 'SCREEN-IO',
+ 'SCREEN-LINES',
+ 'SCREEN-VALUE',
+ 'SCREEN-VAL',
+ 'SCREEN-VALU',
+ 'SCROLL',
+ 'SCROLLABLE',
+ 'SCROLLBAR-HORIZONTAL',
+ 'SCROLLBAR-H',
+ 'SCROLLBAR-HO',
+ 'SCROLLBAR-HOR',
+ 'SCROLLBAR-HORI',
+ 'SCROLLBAR-HORIZ',
+ 'SCROLLBAR-HORIZO',
+ 'SCROLLBAR-HORIZON',
+ 'SCROLLBAR-HORIZONT',
+ 'SCROLLBAR-HORIZONTA',
+ 'SCROLL-BARS',
+ 'SCROLLBAR-VERTICAL',
+ 'SCROLLBAR-V',
+ 'SCROLLBAR-VE',
+ 'SCROLLBAR-VER',
+ 'SCROLLBAR-VERT',
+ 'SCROLLBAR-VERTI',
+ 'SCROLLBAR-VERTIC',
+ 'SCROLLBAR-VERTICA',
+ 'SCROLL-DELTA',
+ 'SCROLLED-ROW-POSITION',
+ 'SCROLLED-ROW-POS',
+ 'SCROLLED-ROW-POSI',
+ 'SCROLLED-ROW-POSIT',
+ 'SCROLLED-ROW-POSITI',
+ 'SCROLLED-ROW-POSITIO',
+ 'SCROLLING',
+ 'SCROLL-OFFSET',
+ 'SCROLL-TO-CURRENT-ROW',
+ 'SCROLL-TO-ITEM',
+ 'SCROLL-TO-I',
+ 'SCROLL-TO-IT',
+ 'SCROLL-TO-ITE',
+ 'SCROLL-TO-SELECTED-ROW',
+ 'SDBNAME',
+ 'SEAL',
+ 'SEAL-TIMESTAMP',
+ 'SEARCH',
+ 'SEARCH-SELF',
+ 'SEARCH-TARGET',
+ 'SECTION',
+ 'SECURITY-POLICY',
+ 'SEEK',
+ 'SELECT',
+ 'SELECTABLE',
+ 'SELECT-ALL',
+ 'SELECTED',
+ 'SELECT-FOCUSED-ROW',
+ 'SELECTION',
+ 'SELECTION-END',
+ 'SELECTION-LIST',
+ 'SELECTION-START',
+ 'SELECTION-TEXT',
+ 'SELECT-NEXT-ROW',
+ 'SELECT-PREV-ROW',
+ 'SELECT-ROW',
+ 'SELF',
+ 'SEND',
+ 'send-sql-statement',
+ 'send-sql',
+ 'SENSITIVE',
+ 'SEPARATE-CONNECTION',
+ 'SEPARATOR-FGCOLOR',
+ 'SEPARATORS',
+ 'SERVER',
+ 'SERVER-CONNECTION-BOUND',
+ 'SERVER-CONNECTION-BOUND-REQUEST',
+ 'SERVER-CONNECTION-CONTEXT',
+ 'SERVER-CONNECTION-ID',
+ 'SERVER-OPERATING-MODE',
+ 'SESSION',
+ 'SESSION-ID',
+ 'SET',
+ 'SET-APPL-CONTEXT',
+ 'SET-ATTR-CALL-TYPE',
+ 'SET-ATTRIBUTE-NODE',
+ 'SET-BLUE-VALUE',
+ 'SET-BLUE',
+ 'SET-BLUE-',
+ 'SET-BLUE-V',
+ 'SET-BLUE-VA',
+ 'SET-BLUE-VAL',
+ 'SET-BLUE-VALU',
+ 'SET-BREAK',
+ 'SET-BUFFERS',
+ 'SET-CALLBACK',
+ 'SET-CLIENT',
+ 'SET-COMMIT',
+ 'SET-CONTENTS',
+ 'SET-CURRENT-VALUE',
+ 'SET-DB-CLIENT',
+ 'SET-DYNAMIC',
+ 'SET-EVENT-MANAGER-OPTION',
+ 'SET-GREEN-VALUE',
+ 'SET-GREEN',
+ 'SET-GREEN-',
+ 'SET-GREEN-V',
+ 'SET-GREEN-VA',
+ 'SET-GREEN-VAL',
+ 'SET-GREEN-VALU',
+ 'SET-INPUT-SOURCE',
+ 'SET-OPTION',
+ 'SET-OUTPUT-DESTINATION',
+ 'SET-PARAMETER',
+ 'SET-POINTER-VALUE',
+ 'SET-PROPERTY',
+ 'SET-RED-VALUE',
+ 'SET-RED',
+ 'SET-RED-',
+ 'SET-RED-V',
+ 'SET-RED-VA',
+ 'SET-RED-VAL',
+ 'SET-RED-VALU',
+ 'SET-REPOSITIONED-ROW',
+ 'SET-RGB-VALUE',
+ 'SET-ROLLBACK',
+ 'SET-SELECTION',
+ 'SET-SIZE',
+ 'SET-SORT-ARROW',
+ 'SETUSERID',
+ 'SETUSER',
+ 'SETUSERI',
+ 'SET-WAIT-STATE',
+ 'SHA1-DIGEST',
+ 'SHARED',
+ 'SHARE-LOCK',
+ 'SHARE',
+ 'SHARE-',
+ 'SHARE-L',
+ 'SHARE-LO',
+ 'SHARE-LOC',
+ 'SHOW-IN-TASKBAR',
+ 'SHOW-STATS',
+ 'SHOW-STAT',
+ 'SIDE-LABEL-HANDLE',
+ 'SIDE-LABEL-H',
+ 'SIDE-LABEL-HA',
+ 'SIDE-LABEL-HAN',
+ 'SIDE-LABEL-HAND',
+ 'SIDE-LABEL-HANDL',
+ 'SIDE-LABELS',
+ 'SIDE-LAB',
+ 'SIDE-LABE',
+ 'SIDE-LABEL',
+ 'SILENT',
+ 'SIMPLE',
+ 'SINGLE',
+ 'SIZE',
+ 'SIZE-CHARS',
+ 'SIZE-C',
+ 'SIZE-CH',
+ 'SIZE-CHA',
+ 'SIZE-CHAR',
+ 'SIZE-PIXELS',
+ 'SIZE-P',
+ 'SIZE-PI',
+ 'SIZE-PIX',
+ 'SIZE-PIXE',
+ 'SIZE-PIXEL',
+ 'SKIP',
+ 'SKIP-DELETED-RECORD',
+ 'SLIDER',
+ 'SMALL-ICON',
+ 'SMALLINT',
+ 'SMALL-TITLE',
+ 'SOME',
+ 'SORT',
+ 'SORT-ASCENDING',
+ 'SORT-NUMBER',
+ 'SOURCE',
+ 'SOURCE-PROCEDURE',
+ 'SPACE',
+ 'SQL',
+ 'SQRT',
+ 'SSL-SERVER-NAME',
+ 'STANDALONE',
+ 'START',
+ 'START-DOCUMENT',
+ 'START-ELEMENT',
+ 'START-MOVE',
+ 'START-RESIZE',
+ 'START-ROW-RESIZE',
+ 'STATE-DETAIL',
+ 'STATIC',
+ 'STATUS',
+ 'STATUS-AREA',
+ 'STATUS-AREA-FONT',
+ 'STDCALL',
+ 'STOP',
+ 'STOP-PARSING',
+ 'STOPPED',
+ 'STOPPE',
+ 'STORED-PROCEDURE',
+ 'STORED-PROC',
+ 'STORED-PROCE',
+ 'STORED-PROCED',
+ 'STORED-PROCEDU',
+ 'STORED-PROCEDUR',
+ 'STREAM',
+ 'STREAM-HANDLE',
+ 'STREAM-IO',
+ 'STRETCH-TO-FIT',
+ 'STRICT',
+ 'STRING',
+ 'STRING-VALUE',
+ 'STRING-XREF',
+ 'SUB-AVERAGE',
+ 'SUB-AVE',
+ 'SUB-AVER',
+ 'SUB-AVERA',
+ 'SUB-AVERAG',
+ 'SUB-COUNT',
+ 'SUB-MAXIMUM',
+ 'SUM-MAX',
+ 'SUM-MAXI',
+ 'SUM-MAXIM',
+ 'SUM-MAXIMU',
+ 'SUB-MENU',
+ 'SUBSUB-',
+ 'SUB-MIN',
+ 'SUBSCRIBE',
+ 'SUBSTITUTE',
+ 'SUBST',
+ 'SUBSTI',
+ 'SUBSTIT',
+ 'SUBSTITU',
+ 'SUBSTITUT',
+ 'SUBSTRING',
+ 'SUBSTR',
+ 'SUBSTRI',
+ 'SUBSTRIN',
+ 'SUB-TOTAL',
+ 'SUBTYPE',
+ 'SUM',
+ 'SUPER',
+ 'SUPER-PROCEDURES',
+ 'SUPPRESS-NAMESPACE-PROCESSING',
+ 'SUPPRESS-WARNINGS',
+ 'SUPPRESS-W',
+ 'SUPPRESS-WA',
+ 'SUPPRESS-WAR',
+ 'SUPPRESS-WARN',
+ 'SUPPRESS-WARNI',
+ 'SUPPRESS-WARNIN',
+ 'SUPPRESS-WARNING',
+ 'SYMMETRIC-ENCRYPTION-ALGORITHM',
+ 'SYMMETRIC-ENCRYPTION-IV',
+ 'SYMMETRIC-ENCRYPTION-KEY',
+ 'SYMMETRIC-SUPPORT',
+ 'SYSTEM-ALERT-BOXES',
+ 'SYSTEM-ALERT',
+ 'SYSTEM-ALERT-',
+ 'SYSTEM-ALERT-B',
+ 'SYSTEM-ALERT-BO',
+ 'SYSTEM-ALERT-BOX',
+ 'SYSTEM-ALERT-BOXE',
+ 'SYSTEM-DIALOG',
+ 'SYSTEM-HELP',
+ 'SYSTEM-ID',
+ 'TABLE',
+ 'TABLE-HANDLE',
+ 'TABLE-NUMBER',
+ 'TAB-POSITION',
+ 'TAB-STOP',
+ 'TARGET',
+ 'TARGET-PROCEDURE',
+ 'TEMP-DIRECTORY',
+ 'TEMP-DIR',
+ 'TEMP-DIRE',
+ 'TEMP-DIREC',
+ 'TEMP-DIRECT',
+ 'TEMP-DIRECTO',
+ 'TEMP-DIRECTOR',
+ 'TEMP-TABLE',
+ 'TEMP-TABLE-PREPARE',
+ 'TERM',
+ 'TERMINAL',
+ 'TERMI',
+ 'TERMIN',
+ 'TERMINA',
+ 'TERMINATE',
+ 'TEXT',
+ 'TEXT-CURSOR',
+ 'TEXT-SEG-GROW',
+ 'TEXT-SELECTED',
+ 'THEN',
+ 'THIS-OBJECT',
+ 'THIS-PROCEDURE',
+ 'THREE-D',
+ 'THROW',
+ 'THROUGH',
+ 'THRU',
+ 'TIC-MARKS',
+ 'TIME',
+ 'TIME-SOURCE',
+ 'TITLE',
+ 'TITLE-BGCOLOR',
+ 'TITLE-BGC',
+ 'TITLE-BGCO',
+ 'TITLE-BGCOL',
+ 'TITLE-BGCOLO',
+ 'TITLE-DCOLOR',
+ 'TITLE-DC',
+ 'TITLE-DCO',
+ 'TITLE-DCOL',
+ 'TITLE-DCOLO',
+ 'TITLE-FGCOLOR',
+ 'TITLE-FGC',
+ 'TITLE-FGCO',
+ 'TITLE-FGCOL',
+ 'TITLE-FGCOLO',
+ 'TITLE-FONT',
+ 'TITLE-FO',
+ 'TITLE-FON',
+ 'TO',
+ 'TODAY',
+ 'TOGGLE-BOX',
+ 'TOOLTIP',
+ 'TOOLTIPS',
+ 'TOPIC',
+ 'TOP-NAV-QUERY',
+ 'TOP-ONLY',
+ 'TO-ROWID',
+ 'TOTAL',
+ 'TRAILING',
+ 'TRANS',
+ 'TRANSACTION',
+ 'TRANSACTION-MODE',
+ 'TRANS-INIT-PROCEDURE',
+ 'TRANSPARENT',
+ 'TRIGGER',
+ 'TRIGGERS',
+ 'TRIM',
+ 'TRUE',
+ 'TRUNCATE',
+ 'TRUNC',
+ 'TRUNCA',
+ 'TRUNCAT',
+ 'TYPE',
+ 'TYPE-OF',
+ 'UNBOX',
+ 'UNBUFFERED',
+ 'UNBUFF',
+ 'UNBUFFE',
+ 'UNBUFFER',
+ 'UNBUFFERE',
+ 'UNDERLINE',
+ 'UNDERL',
+ 'UNDERLI',
+ 'UNDERLIN',
+ 'UNDO',
+ 'UNFORMATTED',
+ 'UNFORM',
+ 'UNFORMA',
+ 'UNFORMAT',
+ 'UNFORMATT',
+ 'UNFORMATTE',
+ 'UNION',
+ 'UNIQUE',
+ 'UNIQUE-ID',
+ 'UNIQUE-MATCH',
+ 'UNIX',
+ 'UNLESS-HIDDEN',
+ 'UNLOAD',
+ 'UNSIGNED-LONG',
+ 'UNSUBSCRIBE',
+ 'UP',
+ 'UPDATE',
+ 'UPDATE-ATTRIBUTE',
+ 'URL',
+ 'URL-DECODE',
+ 'URL-ENCODE',
+ 'URL-PASSWORD',
+ 'URL-USERID',
+ 'USE',
+ 'USE-DICT-EXPS',
+ 'USE-FILENAME',
+ 'USE-INDEX',
+ 'USER',
+ 'USE-REVVIDEO',
+ 'USERID',
+ 'USER-ID',
+ 'USE-TEXT',
+ 'USE-UNDERLINE',
+ 'USE-WIDGET-POOL',
+ 'USING',
+ 'V6DISPLAY',
+ 'V6FRAME',
+ 'VALIDATE',
+ 'VALIDATE-EXPRESSION',
+ 'VALIDATE-MESSAGE',
+ 'VALIDATE-SEAL',
+ 'VALIDATION-ENABLED',
+ 'VALID-EVENT',
+ 'VALID-HANDLE',
+ 'VALID-OBJECT',
+ 'VALUE',
+ 'VALUE-CHANGED',
+ 'VALUES',
+ 'VARIABLE',
+ 'VAR',
+ 'VARI',
+ 'VARIA',
+ 'VARIAB',
+ 'VARIABL',
+ 'VERBOSE',
+ 'VERSION',
+ 'VERTICAL',
+ 'VERT',
+ 'VERTI',
+ 'VERTIC',
+ 'VERTICA',
+ 'VIEW',
+ 'VIEW-AS',
+ 'VIEW-FIRST-COLUMN-ON-REOPEN',
+ 'VIRTUAL-HEIGHT-CHARS',
+ 'VIRTUAL-HEIGHT',
+ 'VIRTUAL-HEIGHT-',
+ 'VIRTUAL-HEIGHT-C',
+ 'VIRTUAL-HEIGHT-CH',
+ 'VIRTUAL-HEIGHT-CHA',
+ 'VIRTUAL-HEIGHT-CHAR',
+ 'VIRTUAL-HEIGHT-PIXELS',
+ 'VIRTUAL-HEIGHT-P',
+ 'VIRTUAL-HEIGHT-PI',
+ 'VIRTUAL-HEIGHT-PIX',
+ 'VIRTUAL-HEIGHT-PIXE',
+ 'VIRTUAL-HEIGHT-PIXEL',
+ 'VIRTUAL-WIDTH-CHARS',
+ 'VIRTUAL-WIDTH',
+ 'VIRTUAL-WIDTH-',
+ 'VIRTUAL-WIDTH-C',
+ 'VIRTUAL-WIDTH-CH',
+ 'VIRTUAL-WIDTH-CHA',
+ 'VIRTUAL-WIDTH-CHAR',
+ 'VIRTUAL-WIDTH-PIXELS',
+ 'VIRTUAL-WIDTH-P',
+ 'VIRTUAL-WIDTH-PI',
+ 'VIRTUAL-WIDTH-PIX',
+ 'VIRTUAL-WIDTH-PIXE',
+ 'VIRTUAL-WIDTH-PIXEL',
+ 'VISIBLE',
+ 'VOID',
+ 'WAIT',
+ 'WAIT-FOR',
+ 'WARNING',
+ 'WEB-CONTEXT',
+ 'WEEKDAY',
+ 'WHEN',
+ 'WHERE',
+ 'WHILE',
+ 'WIDGET',
+ 'WIDGET-ENTER',
+ 'WIDGET-E',
+ 'WIDGET-EN',
+ 'WIDGET-ENT',
+ 'WIDGET-ENTE',
+ 'WIDGET-ID',
+ 'WIDGET-LEAVE',
+ 'WIDGET-L',
+ 'WIDGET-LE',
+ 'WIDGET-LEA',
+ 'WIDGET-LEAV',
+ 'WIDGET-POOL',
+ 'WIDTH-CHARS',
+ 'WIDTH',
+ 'WIDTH-',
+ 'WIDTH-C',
+ 'WIDTH-CH',
+ 'WIDTH-CHA',
+ 'WIDTH-CHAR',
+ 'WIDTH-PIXELS',
+ 'WIDTH-P',
+ 'WIDTH-PI',
+ 'WIDTH-PIX',
+ 'WIDTH-PIXE',
+ 'WIDTH-PIXEL',
+ 'WINDOW',
+ 'WINDOW-MAXIMIZED',
+ 'WINDOW-MAXIM',
+ 'WINDOW-MAXIMI',
+ 'WINDOW-MAXIMIZ',
+ 'WINDOW-MAXIMIZE',
+ 'WINDOW-MINIMIZED',
+ 'WINDOW-MINIM',
+ 'WINDOW-MINIMI',
+ 'WINDOW-MINIMIZ',
+ 'WINDOW-MINIMIZE',
+ 'WINDOW-NAME',
+ 'WINDOW-NORMAL',
+ 'WINDOW-STATE',
+ 'WINDOW-STA',
+ 'WINDOW-STAT',
+ 'WINDOW-SYSTEM',
+ 'WITH',
+ 'WORD-INDEX',
+ 'WORD-WRAP',
+ 'WORK-AREA-HEIGHT-PIXELS',
+ 'WORK-AREA-WIDTH-PIXELS',
+ 'WORK-AREA-X',
+ 'WORK-AREA-Y',
+ 'WORKFILE',
+ 'WORK-TABLE',
+ 'WORK-TAB',
+ 'WORK-TABL',
+ 'WRITE',
+ 'WRITE-CDATA',
+ 'WRITE-CHARACTERS',
+ 'WRITE-COMMENT',
+ 'WRITE-DATA-ELEMENT',
+ 'WRITE-EMPTY-ELEMENT',
+ 'WRITE-ENTITY-REF',
+ 'WRITE-EXTERNAL-DTD',
+ 'WRITE-FRAGMENT',
+ 'WRITE-MESSAGE',
+ 'WRITE-PROCESSING-INSTRUCTION',
+ 'WRITE-STATUS',
+ 'WRITE-XML',
+ 'WRITE-XMLSCHEMA',
+ 'X',
+ 'XCODE',
+ 'XML-DATA-TYPE',
+ 'XML-NODE-TYPE',
+ 'XML-SCHEMA-PATH',
+ 'XML-SUPPRESS-NAMESPACE-PROCESSING',
+ 'X-OF',
+ 'XREF',
+ 'XREF-XML',
+ 'Y',
+ 'YEAR',
+ 'YEAR-OFFSET',
+ 'YES',
+ 'YES-NO',
+ 'YES-NO-CANCEL',
+ 'Y-OF'
+)
diff --git a/wandb/vendor/pygments/lexers/_php_builtins.py b/wandb/vendor/pygments/lexers/_php_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..fec3286a8b459040b3d770cec5afad9f0e82ebaf
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_php_builtins.py
@@ -0,0 +1,4756 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._php_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ This file loads the function names and their modules from the
+ php webpage and generates itself.
+
+ Do not alter the MODULES dict by hand!
+
+ WARNING: the generation transfers quite much data over your
+ internet connection. don't run that at home, use
+ a server ;-)
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import print_function
+
+MODULES = {'.NET': ('dotnet_load',),
+ 'APC': ('apc_add',
+ 'apc_bin_dump',
+ 'apc_bin_dumpfile',
+ 'apc_bin_load',
+ 'apc_bin_loadfile',
+ 'apc_cache_info',
+ 'apc_cas',
+ 'apc_clear_cache',
+ 'apc_compile_file',
+ 'apc_dec',
+ 'apc_define_constants',
+ 'apc_delete_file',
+ 'apc_delete',
+ 'apc_exists',
+ 'apc_fetch',
+ 'apc_inc',
+ 'apc_load_constants',
+ 'apc_sma_info',
+ 'apc_store'),
+ 'APD': ('apd_breakpoint',
+ 'apd_callstack',
+ 'apd_clunk',
+ 'apd_continue',
+ 'apd_croak',
+ 'apd_dump_function_table',
+ 'apd_dump_persistent_resources',
+ 'apd_dump_regular_resources',
+ 'apd_echo',
+ 'apd_get_active_symbols',
+ 'apd_set_pprof_trace',
+ 'apd_set_session_trace_socket',
+ 'apd_set_session_trace',
+ 'apd_set_session',
+ 'override_function',
+ 'rename_function'),
+ 'Aliases and deprecated Mysqli': ('mysqli_bind_param',
+ 'mysqli_bind_result',
+ 'mysqli_client_encoding',
+ 'mysqli_connect',
+ 'mysqli_disable_rpl_parse',
+ 'mysqli_enable_reads_from_master',
+ 'mysqli_enable_rpl_parse',
+ 'mysqli_escape_string',
+ 'mysqli_execute',
+ 'mysqli_fetch',
+ 'mysqli_get_cache_stats',
+ 'mysqli_get_metadata',
+ 'mysqli_master_query',
+ 'mysqli_param_count',
+ 'mysqli_report',
+ 'mysqli_rpl_parse_enabled',
+ 'mysqli_rpl_probe',
+ 'mysqli_send_long_data',
+ 'mysqli_slave_query'),
+ 'Apache': ('apache_child_terminate',
+ 'apache_get_modules',
+ 'apache_get_version',
+ 'apache_getenv',
+ 'apache_lookup_uri',
+ 'apache_note',
+ 'apache_request_headers',
+ 'apache_reset_timeout',
+ 'apache_response_headers',
+ 'apache_setenv',
+ 'getallheaders',
+ 'virtual'),
+ 'Array': ('array_change_key_case',
+ 'array_chunk',
+ 'array_column',
+ 'array_combine',
+ 'array_count_values',
+ 'array_diff_assoc',
+ 'array_diff_key',
+ 'array_diff_uassoc',
+ 'array_diff_ukey',
+ 'array_diff',
+ 'array_fill_keys',
+ 'array_fill',
+ 'array_filter',
+ 'array_flip',
+ 'array_intersect_assoc',
+ 'array_intersect_key',
+ 'array_intersect_uassoc',
+ 'array_intersect_ukey',
+ 'array_intersect',
+ 'array_key_exists',
+ 'array_keys',
+ 'array_map',
+ 'array_merge_recursive',
+ 'array_merge',
+ 'array_multisort',
+ 'array_pad',
+ 'array_pop',
+ 'array_product',
+ 'array_push',
+ 'array_rand',
+ 'array_reduce',
+ 'array_replace_recursive',
+ 'array_replace',
+ 'array_reverse',
+ 'array_search',
+ 'array_shift',
+ 'array_slice',
+ 'array_splice',
+ 'array_sum',
+ 'array_udiff_assoc',
+ 'array_udiff_uassoc',
+ 'array_udiff',
+ 'array_uintersect_assoc',
+ 'array_uintersect_uassoc',
+ 'array_uintersect',
+ 'array_unique',
+ 'array_unshift',
+ 'array_values',
+ 'array_walk_recursive',
+ 'array_walk',
+ 'array',
+ 'arsort',
+ 'asort',
+ 'compact',
+ 'count',
+ 'current',
+ 'each',
+ 'end',
+ 'extract',
+ 'in_array',
+ 'key_exists',
+ 'key',
+ 'krsort',
+ 'ksort',
+ 'list',
+ 'natcasesort',
+ 'natsort',
+ 'next',
+ 'pos',
+ 'prev',
+ 'range',
+ 'reset',
+ 'rsort',
+ 'shuffle',
+ 'sizeof',
+ 'sort',
+ 'uasort',
+ 'uksort',
+ 'usort'),
+ 'BBCode': ('bbcode_add_element',
+ 'bbcode_add_smiley',
+ 'bbcode_create',
+ 'bbcode_destroy',
+ 'bbcode_parse',
+ 'bbcode_set_arg_parser',
+ 'bbcode_set_flags'),
+ 'BC Math': ('bcadd',
+ 'bccomp',
+ 'bcdiv',
+ 'bcmod',
+ 'bcmul',
+ 'bcpow',
+ 'bcpowmod',
+ 'bcscale',
+ 'bcsqrt',
+ 'bcsub'),
+ 'Blenc': ('blenc_encrypt',),
+ 'Bzip2': ('bzclose',
+ 'bzcompress',
+ 'bzdecompress',
+ 'bzerrno',
+ 'bzerror',
+ 'bzerrstr',
+ 'bzflush',
+ 'bzopen',
+ 'bzread',
+ 'bzwrite'),
+ 'COM': ('com_addref',
+ 'com_create_guid',
+ 'com_event_sink',
+ 'com_get_active_object',
+ 'com_get',
+ 'com_invoke',
+ 'com_isenum',
+ 'com_load_typelib',
+ 'com_load',
+ 'com_message_pump',
+ 'com_print_typeinfo',
+ 'com_propget',
+ 'com_propput',
+ 'com_propset',
+ 'com_release',
+ 'com_set',
+ 'variant_abs',
+ 'variant_add',
+ 'variant_and',
+ 'variant_cast',
+ 'variant_cat',
+ 'variant_cmp',
+ 'variant_date_from_timestamp',
+ 'variant_date_to_timestamp',
+ 'variant_div',
+ 'variant_eqv',
+ 'variant_fix',
+ 'variant_get_type',
+ 'variant_idiv',
+ 'variant_imp',
+ 'variant_int',
+ 'variant_mod',
+ 'variant_mul',
+ 'variant_neg',
+ 'variant_not',
+ 'variant_or',
+ 'variant_pow',
+ 'variant_round',
+ 'variant_set_type',
+ 'variant_set',
+ 'variant_sub',
+ 'variant_xor'),
+ 'CUBRID': ('cubrid_bind',
+ 'cubrid_close_prepare',
+ 'cubrid_close_request',
+ 'cubrid_col_get',
+ 'cubrid_col_size',
+ 'cubrid_column_names',
+ 'cubrid_column_types',
+ 'cubrid_commit',
+ 'cubrid_connect_with_url',
+ 'cubrid_connect',
+ 'cubrid_current_oid',
+ 'cubrid_disconnect',
+ 'cubrid_drop',
+ 'cubrid_error_code_facility',
+ 'cubrid_error_code',
+ 'cubrid_error_msg',
+ 'cubrid_execute',
+ 'cubrid_fetch',
+ 'cubrid_free_result',
+ 'cubrid_get_autocommit',
+ 'cubrid_get_charset',
+ 'cubrid_get_class_name',
+ 'cubrid_get_client_info',
+ 'cubrid_get_db_parameter',
+ 'cubrid_get_query_timeout',
+ 'cubrid_get_server_info',
+ 'cubrid_get',
+ 'cubrid_insert_id',
+ 'cubrid_is_instance',
+ 'cubrid_lob_close',
+ 'cubrid_lob_export',
+ 'cubrid_lob_get',
+ 'cubrid_lob_send',
+ 'cubrid_lob_size',
+ 'cubrid_lob2_bind',
+ 'cubrid_lob2_close',
+ 'cubrid_lob2_export',
+ 'cubrid_lob2_import',
+ 'cubrid_lob2_new',
+ 'cubrid_lob2_read',
+ 'cubrid_lob2_seek64',
+ 'cubrid_lob2_seek',
+ 'cubrid_lob2_size64',
+ 'cubrid_lob2_size',
+ 'cubrid_lob2_tell64',
+ 'cubrid_lob2_tell',
+ 'cubrid_lob2_write',
+ 'cubrid_lock_read',
+ 'cubrid_lock_write',
+ 'cubrid_move_cursor',
+ 'cubrid_next_result',
+ 'cubrid_num_cols',
+ 'cubrid_num_rows',
+ 'cubrid_pconnect_with_url',
+ 'cubrid_pconnect',
+ 'cubrid_prepare',
+ 'cubrid_put',
+ 'cubrid_rollback',
+ 'cubrid_schema',
+ 'cubrid_seq_drop',
+ 'cubrid_seq_insert',
+ 'cubrid_seq_put',
+ 'cubrid_set_add',
+ 'cubrid_set_autocommit',
+ 'cubrid_set_db_parameter',
+ 'cubrid_set_drop',
+ 'cubrid_set_query_timeout',
+ 'cubrid_version'),
+ 'Cairo': ('cairo_create',
+ 'cairo_font_face_get_type',
+ 'cairo_font_face_status',
+ 'cairo_font_options_create',
+ 'cairo_font_options_equal',
+ 'cairo_font_options_get_antialias',
+ 'cairo_font_options_get_hint_metrics',
+ 'cairo_font_options_get_hint_style',
+ 'cairo_font_options_get_subpixel_order',
+ 'cairo_font_options_hash',
+ 'cairo_font_options_merge',
+ 'cairo_font_options_set_antialias',
+ 'cairo_font_options_set_hint_metrics',
+ 'cairo_font_options_set_hint_style',
+ 'cairo_font_options_set_subpixel_order',
+ 'cairo_font_options_status',
+ 'cairo_format_stride_for_width',
+ 'cairo_image_surface_create_for_data',
+ 'cairo_image_surface_create_from_png',
+ 'cairo_image_surface_create',
+ 'cairo_image_surface_get_data',
+ 'cairo_image_surface_get_format',
+ 'cairo_image_surface_get_height',
+ 'cairo_image_surface_get_stride',
+ 'cairo_image_surface_get_width',
+ 'cairo_matrix_create_scale',
+ 'cairo_matrix_create_translate',
+ 'cairo_matrix_invert',
+ 'cairo_matrix_multiply',
+ 'cairo_matrix_rotate',
+ 'cairo_matrix_transform_distance',
+ 'cairo_matrix_transform_point',
+ 'cairo_matrix_translate',
+ 'cairo_pattern_add_color_stop_rgb',
+ 'cairo_pattern_add_color_stop_rgba',
+ 'cairo_pattern_create_for_surface',
+ 'cairo_pattern_create_linear',
+ 'cairo_pattern_create_radial',
+ 'cairo_pattern_create_rgb',
+ 'cairo_pattern_create_rgba',
+ 'cairo_pattern_get_color_stop_count',
+ 'cairo_pattern_get_color_stop_rgba',
+ 'cairo_pattern_get_extend',
+ 'cairo_pattern_get_filter',
+ 'cairo_pattern_get_linear_points',
+ 'cairo_pattern_get_matrix',
+ 'cairo_pattern_get_radial_circles',
+ 'cairo_pattern_get_rgba',
+ 'cairo_pattern_get_surface',
+ 'cairo_pattern_get_type',
+ 'cairo_pattern_set_extend',
+ 'cairo_pattern_set_filter',
+ 'cairo_pattern_set_matrix',
+ 'cairo_pattern_status',
+ 'cairo_pdf_surface_create',
+ 'cairo_pdf_surface_set_size',
+ 'cairo_ps_get_levels',
+ 'cairo_ps_level_to_string',
+ 'cairo_ps_surface_create',
+ 'cairo_ps_surface_dsc_begin_page_setup',
+ 'cairo_ps_surface_dsc_begin_setup',
+ 'cairo_ps_surface_dsc_comment',
+ 'cairo_ps_surface_get_eps',
+ 'cairo_ps_surface_restrict_to_level',
+ 'cairo_ps_surface_set_eps',
+ 'cairo_ps_surface_set_size',
+ 'cairo_scaled_font_create',
+ 'cairo_scaled_font_extents',
+ 'cairo_scaled_font_get_ctm',
+ 'cairo_scaled_font_get_font_face',
+ 'cairo_scaled_font_get_font_matrix',
+ 'cairo_scaled_font_get_font_options',
+ 'cairo_scaled_font_get_scale_matrix',
+ 'cairo_scaled_font_get_type',
+ 'cairo_scaled_font_glyph_extents',
+ 'cairo_scaled_font_status',
+ 'cairo_scaled_font_text_extents',
+ 'cairo_surface_copy_page',
+ 'cairo_surface_create_similar',
+ 'cairo_surface_finish',
+ 'cairo_surface_flush',
+ 'cairo_surface_get_content',
+ 'cairo_surface_get_device_offset',
+ 'cairo_surface_get_font_options',
+ 'cairo_surface_get_type',
+ 'cairo_surface_mark_dirty_rectangle',
+ 'cairo_surface_mark_dirty',
+ 'cairo_surface_set_device_offset',
+ 'cairo_surface_set_fallback_resolution',
+ 'cairo_surface_show_page',
+ 'cairo_surface_status',
+ 'cairo_surface_write_to_png',
+ 'cairo_svg_surface_create',
+ 'cairo_svg_surface_restrict_to_version',
+ 'cairo_svg_version_to_string'),
+ 'Calendar': ('cal_days_in_month',
+ 'cal_from_jd',
+ 'cal_info',
+ 'cal_to_jd',
+ 'easter_date',
+ 'easter_days',
+ 'FrenchToJD',
+ 'GregorianToJD',
+ 'JDDayOfWeek',
+ 'JDMonthName',
+ 'JDToFrench',
+ 'JDToGregorian',
+ 'jdtojewish',
+ 'JDToJulian',
+ 'jdtounix',
+ 'JewishToJD',
+ 'JulianToJD',
+ 'unixtojd'),
+ 'Classes/Object': ('__autoload',
+ 'call_user_method_array',
+ 'call_user_method',
+ 'class_alias',
+ 'class_exists',
+ 'get_called_class',
+ 'get_class_methods',
+ 'get_class_vars',
+ 'get_class',
+ 'get_declared_classes',
+ 'get_declared_interfaces',
+ 'get_declared_traits',
+ 'get_object_vars',
+ 'get_parent_class',
+ 'interface_exists',
+ 'is_a',
+ 'is_subclass_of',
+ 'method_exists',
+ 'property_exists',
+ 'trait_exists'),
+ 'Classkit': ('classkit_import',
+ 'classkit_method_add',
+ 'classkit_method_copy',
+ 'classkit_method_redefine',
+ 'classkit_method_remove',
+ 'classkit_method_rename'),
+ 'Crack': ('crack_check',
+ 'crack_closedict',
+ 'crack_getlastmessage',
+ 'crack_opendict'),
+ 'Ctype': ('ctype_alnum',
+ 'ctype_alpha',
+ 'ctype_cntrl',
+ 'ctype_digit',
+ 'ctype_graph',
+ 'ctype_lower',
+ 'ctype_print',
+ 'ctype_punct',
+ 'ctype_space',
+ 'ctype_upper',
+ 'ctype_xdigit'),
+ 'Cyrus': ('cyrus_authenticate',
+ 'cyrus_bind',
+ 'cyrus_close',
+ 'cyrus_connect',
+ 'cyrus_query',
+ 'cyrus_unbind'),
+ 'DB++': ('dbplus_add',
+ 'dbplus_aql',
+ 'dbplus_chdir',
+ 'dbplus_close',
+ 'dbplus_curr',
+ 'dbplus_errcode',
+ 'dbplus_errno',
+ 'dbplus_find',
+ 'dbplus_first',
+ 'dbplus_flush',
+ 'dbplus_freealllocks',
+ 'dbplus_freelock',
+ 'dbplus_freerlocks',
+ 'dbplus_getlock',
+ 'dbplus_getunique',
+ 'dbplus_info',
+ 'dbplus_last',
+ 'dbplus_lockrel',
+ 'dbplus_next',
+ 'dbplus_open',
+ 'dbplus_prev',
+ 'dbplus_rchperm',
+ 'dbplus_rcreate',
+ 'dbplus_rcrtexact',
+ 'dbplus_rcrtlike',
+ 'dbplus_resolve',
+ 'dbplus_restorepos',
+ 'dbplus_rkeys',
+ 'dbplus_ropen',
+ 'dbplus_rquery',
+ 'dbplus_rrename',
+ 'dbplus_rsecindex',
+ 'dbplus_runlink',
+ 'dbplus_rzap',
+ 'dbplus_savepos',
+ 'dbplus_setindex',
+ 'dbplus_setindexbynumber',
+ 'dbplus_sql',
+ 'dbplus_tcl',
+ 'dbplus_tremove',
+ 'dbplus_undo',
+ 'dbplus_undoprepare',
+ 'dbplus_unlockrel',
+ 'dbplus_unselect',
+ 'dbplus_update',
+ 'dbplus_xlockrel',
+ 'dbplus_xunlockrel'),
+ 'DBA': ('dba_close',
+ 'dba_delete',
+ 'dba_exists',
+ 'dba_fetch',
+ 'dba_firstkey',
+ 'dba_handlers',
+ 'dba_insert',
+ 'dba_key_split',
+ 'dba_list',
+ 'dba_nextkey',
+ 'dba_open',
+ 'dba_optimize',
+ 'dba_popen',
+ 'dba_replace',
+ 'dba_sync'),
+ 'DOM': ('dom_import_simplexml',),
+ 'Date/Time': ('checkdate',
+ 'date_add',
+ 'date_create_from_format',
+ 'date_create_immutable_from_format',
+ 'date_create_immutable',
+ 'date_create',
+ 'date_date_set',
+ 'date_default_timezone_get',
+ 'date_default_timezone_set',
+ 'date_diff',
+ 'date_format',
+ 'date_get_last_errors',
+ 'date_interval_create_from_date_string',
+ 'date_interval_format',
+ 'date_isodate_set',
+ 'date_modify',
+ 'date_offset_get',
+ 'date_parse_from_format',
+ 'date_parse',
+ 'date_sub',
+ 'date_sun_info',
+ 'date_sunrise',
+ 'date_sunset',
+ 'date_time_set',
+ 'date_timestamp_get',
+ 'date_timestamp_set',
+ 'date_timezone_get',
+ 'date_timezone_set',
+ 'date',
+ 'getdate',
+ 'gettimeofday',
+ 'gmdate',
+ 'gmmktime',
+ 'gmstrftime',
+ 'idate',
+ 'localtime',
+ 'microtime',
+ 'mktime',
+ 'strftime',
+ 'strptime',
+ 'strtotime',
+ 'time',
+ 'timezone_abbreviations_list',
+ 'timezone_identifiers_list',
+ 'timezone_location_get',
+ 'timezone_name_from_abbr',
+ 'timezone_name_get',
+ 'timezone_offset_get',
+ 'timezone_open',
+ 'timezone_transitions_get',
+ 'timezone_version_get'),
+ 'Direct IO': ('dio_close',
+ 'dio_fcntl',
+ 'dio_open',
+ 'dio_read',
+ 'dio_seek',
+ 'dio_stat',
+ 'dio_tcsetattr',
+ 'dio_truncate',
+ 'dio_write'),
+ 'Directory': ('chdir',
+ 'chroot',
+ 'closedir',
+ 'dir',
+ 'getcwd',
+ 'opendir',
+ 'readdir',
+ 'rewinddir',
+ 'scandir'),
+ 'Eio': ('eio_busy',
+ 'eio_cancel',
+ 'eio_chmod',
+ 'eio_chown',
+ 'eio_close',
+ 'eio_custom',
+ 'eio_dup2',
+ 'eio_event_loop',
+ 'eio_fallocate',
+ 'eio_fchmod',
+ 'eio_fchown',
+ 'eio_fdatasync',
+ 'eio_fstat',
+ 'eio_fstatvfs',
+ 'eio_fsync',
+ 'eio_ftruncate',
+ 'eio_futime',
+ 'eio_get_event_stream',
+ 'eio_get_last_error',
+ 'eio_grp_add',
+ 'eio_grp_cancel',
+ 'eio_grp_limit',
+ 'eio_grp',
+ 'eio_init',
+ 'eio_link',
+ 'eio_lstat',
+ 'eio_mkdir',
+ 'eio_mknod',
+ 'eio_nop',
+ 'eio_npending',
+ 'eio_nready',
+ 'eio_nreqs',
+ 'eio_nthreads',
+ 'eio_open',
+ 'eio_poll',
+ 'eio_read',
+ 'eio_readahead',
+ 'eio_readdir',
+ 'eio_readlink',
+ 'eio_realpath',
+ 'eio_rename',
+ 'eio_rmdir',
+ 'eio_seek',
+ 'eio_sendfile',
+ 'eio_set_max_idle',
+ 'eio_set_max_parallel',
+ 'eio_set_max_poll_reqs',
+ 'eio_set_max_poll_time',
+ 'eio_set_min_parallel',
+ 'eio_stat',
+ 'eio_statvfs',
+ 'eio_symlink',
+ 'eio_sync_file_range',
+ 'eio_sync',
+ 'eio_syncfs',
+ 'eio_truncate',
+ 'eio_unlink',
+ 'eio_utime',
+ 'eio_write'),
+ 'Enchant': ('enchant_broker_describe',
+ 'enchant_broker_dict_exists',
+ 'enchant_broker_free_dict',
+ 'enchant_broker_free',
+ 'enchant_broker_get_error',
+ 'enchant_broker_init',
+ 'enchant_broker_list_dicts',
+ 'enchant_broker_request_dict',
+ 'enchant_broker_request_pwl_dict',
+ 'enchant_broker_set_ordering',
+ 'enchant_dict_add_to_personal',
+ 'enchant_dict_add_to_session',
+ 'enchant_dict_check',
+ 'enchant_dict_describe',
+ 'enchant_dict_get_error',
+ 'enchant_dict_is_in_session',
+ 'enchant_dict_quick_check',
+ 'enchant_dict_store_replacement',
+ 'enchant_dict_suggest'),
+ 'Error Handling': ('debug_backtrace',
+ 'debug_print_backtrace',
+ 'error_get_last',
+ 'error_log',
+ 'error_reporting',
+ 'restore_error_handler',
+ 'restore_exception_handler',
+ 'set_error_handler',
+ 'set_exception_handler',
+ 'trigger_error',
+ 'user_error'),
+ 'Exif': ('exif_imagetype',
+ 'exif_read_data',
+ 'exif_tagname',
+ 'exif_thumbnail',
+ 'read_exif_data'),
+ 'Expect': ('expect_expectl', 'expect_popen'),
+ 'FAM': ('fam_cancel_monitor',
+ 'fam_close',
+ 'fam_monitor_collection',
+ 'fam_monitor_directory',
+ 'fam_monitor_file',
+ 'fam_next_event',
+ 'fam_open',
+ 'fam_pending',
+ 'fam_resume_monitor',
+ 'fam_suspend_monitor'),
+ 'FDF': ('fdf_add_doc_javascript',
+ 'fdf_add_template',
+ 'fdf_close',
+ 'fdf_create',
+ 'fdf_enum_values',
+ 'fdf_errno',
+ 'fdf_error',
+ 'fdf_get_ap',
+ 'fdf_get_attachment',
+ 'fdf_get_encoding',
+ 'fdf_get_file',
+ 'fdf_get_flags',
+ 'fdf_get_opt',
+ 'fdf_get_status',
+ 'fdf_get_value',
+ 'fdf_get_version',
+ 'fdf_header',
+ 'fdf_next_field_name',
+ 'fdf_open_string',
+ 'fdf_open',
+ 'fdf_remove_item',
+ 'fdf_save_string',
+ 'fdf_save',
+ 'fdf_set_ap',
+ 'fdf_set_encoding',
+ 'fdf_set_file',
+ 'fdf_set_flags',
+ 'fdf_set_javascript_action',
+ 'fdf_set_on_import_javascript',
+ 'fdf_set_opt',
+ 'fdf_set_status',
+ 'fdf_set_submit_form_action',
+ 'fdf_set_target_frame',
+ 'fdf_set_value',
+ 'fdf_set_version'),
+ 'FPM': ('fastcgi_finish_request',),
+ 'FTP': ('ftp_alloc',
+ 'ftp_cdup',
+ 'ftp_chdir',
+ 'ftp_chmod',
+ 'ftp_close',
+ 'ftp_connect',
+ 'ftp_delete',
+ 'ftp_exec',
+ 'ftp_fget',
+ 'ftp_fput',
+ 'ftp_get_option',
+ 'ftp_get',
+ 'ftp_login',
+ 'ftp_mdtm',
+ 'ftp_mkdir',
+ 'ftp_nb_continue',
+ 'ftp_nb_fget',
+ 'ftp_nb_fput',
+ 'ftp_nb_get',
+ 'ftp_nb_put',
+ 'ftp_nlist',
+ 'ftp_pasv',
+ 'ftp_put',
+ 'ftp_pwd',
+ 'ftp_quit',
+ 'ftp_raw',
+ 'ftp_rawlist',
+ 'ftp_rename',
+ 'ftp_rmdir',
+ 'ftp_set_option',
+ 'ftp_site',
+ 'ftp_size',
+ 'ftp_ssl_connect',
+ 'ftp_systype'),
+ 'Fann': ('fann_cascadetrain_on_data',
+ 'fann_cascadetrain_on_file',
+ 'fann_clear_scaling_params',
+ 'fann_copy',
+ 'fann_create_from_file',
+ 'fann_create_shortcut_array',
+ 'fann_create_shortcut',
+ 'fann_create_sparse_array',
+ 'fann_create_sparse',
+ 'fann_create_standard_array',
+ 'fann_create_standard',
+ 'fann_create_train_from_callback',
+ 'fann_create_train',
+ 'fann_descale_input',
+ 'fann_descale_output',
+ 'fann_descale_train',
+ 'fann_destroy_train',
+ 'fann_destroy',
+ 'fann_duplicate_train_data',
+ 'fann_get_activation_function',
+ 'fann_get_activation_steepness',
+ 'fann_get_bias_array',
+ 'fann_get_bit_fail_limit',
+ 'fann_get_bit_fail',
+ 'fann_get_cascade_activation_functions_count',
+ 'fann_get_cascade_activation_functions',
+ 'fann_get_cascade_activation_steepnesses_count',
+ 'fann_get_cascade_activation_steepnesses',
+ 'fann_get_cascade_candidate_change_fraction',
+ 'fann_get_cascade_candidate_limit',
+ 'fann_get_cascade_candidate_stagnation_epochs',
+ 'fann_get_cascade_max_cand_epochs',
+ 'fann_get_cascade_max_out_epochs',
+ 'fann_get_cascade_min_cand_epochs',
+ 'fann_get_cascade_min_out_epochs',
+ 'fann_get_cascade_num_candidate_groups',
+ 'fann_get_cascade_num_candidates',
+ 'fann_get_cascade_output_change_fraction',
+ 'fann_get_cascade_output_stagnation_epochs',
+ 'fann_get_cascade_weight_multiplier',
+ 'fann_get_connection_array',
+ 'fann_get_connection_rate',
+ 'fann_get_errno',
+ 'fann_get_errstr',
+ 'fann_get_layer_array',
+ 'fann_get_learning_momentum',
+ 'fann_get_learning_rate',
+ 'fann_get_MSE',
+ 'fann_get_network_type',
+ 'fann_get_num_input',
+ 'fann_get_num_layers',
+ 'fann_get_num_output',
+ 'fann_get_quickprop_decay',
+ 'fann_get_quickprop_mu',
+ 'fann_get_rprop_decrease_factor',
+ 'fann_get_rprop_delta_max',
+ 'fann_get_rprop_delta_min',
+ 'fann_get_rprop_delta_zero',
+ 'fann_get_rprop_increase_factor',
+ 'fann_get_sarprop_step_error_shift',
+ 'fann_get_sarprop_step_error_threshold_factor',
+ 'fann_get_sarprop_temperature',
+ 'fann_get_sarprop_weight_decay_shift',
+ 'fann_get_total_connections',
+ 'fann_get_total_neurons',
+ 'fann_get_train_error_function',
+ 'fann_get_train_stop_function',
+ 'fann_get_training_algorithm',
+ 'fann_init_weights',
+ 'fann_length_train_data',
+ 'fann_merge_train_data',
+ 'fann_num_input_train_data',
+ 'fann_num_output_train_data',
+ 'fann_print_error',
+ 'fann_randomize_weights',
+ 'fann_read_train_from_file',
+ 'fann_reset_errno',
+ 'fann_reset_errstr',
+ 'fann_reset_MSE',
+ 'fann_run',
+ 'fann_save_train',
+ 'fann_save',
+ 'fann_scale_input_train_data',
+ 'fann_scale_input',
+ 'fann_scale_output_train_data',
+ 'fann_scale_output',
+ 'fann_scale_train_data',
+ 'fann_scale_train',
+ 'fann_set_activation_function_hidden',
+ 'fann_set_activation_function_layer',
+ 'fann_set_activation_function_output',
+ 'fann_set_activation_function',
+ 'fann_set_activation_steepness_hidden',
+ 'fann_set_activation_steepness_layer',
+ 'fann_set_activation_steepness_output',
+ 'fann_set_activation_steepness',
+ 'fann_set_bit_fail_limit',
+ 'fann_set_callback',
+ 'fann_set_cascade_activation_functions',
+ 'fann_set_cascade_activation_steepnesses',
+ 'fann_set_cascade_candidate_change_fraction',
+ 'fann_set_cascade_candidate_limit',
+ 'fann_set_cascade_candidate_stagnation_epochs',
+ 'fann_set_cascade_max_cand_epochs',
+ 'fann_set_cascade_max_out_epochs',
+ 'fann_set_cascade_min_cand_epochs',
+ 'fann_set_cascade_min_out_epochs',
+ 'fann_set_cascade_num_candidate_groups',
+ 'fann_set_cascade_output_change_fraction',
+ 'fann_set_cascade_output_stagnation_epochs',
+ 'fann_set_cascade_weight_multiplier',
+ 'fann_set_error_log',
+ 'fann_set_input_scaling_params',
+ 'fann_set_learning_momentum',
+ 'fann_set_learning_rate',
+ 'fann_set_output_scaling_params',
+ 'fann_set_quickprop_decay',
+ 'fann_set_quickprop_mu',
+ 'fann_set_rprop_decrease_factor',
+ 'fann_set_rprop_delta_max',
+ 'fann_set_rprop_delta_min',
+ 'fann_set_rprop_delta_zero',
+ 'fann_set_rprop_increase_factor',
+ 'fann_set_sarprop_step_error_shift',
+ 'fann_set_sarprop_step_error_threshold_factor',
+ 'fann_set_sarprop_temperature',
+ 'fann_set_sarprop_weight_decay_shift',
+ 'fann_set_scaling_params',
+ 'fann_set_train_error_function',
+ 'fann_set_train_stop_function',
+ 'fann_set_training_algorithm',
+ 'fann_set_weight_array',
+ 'fann_set_weight',
+ 'fann_shuffle_train_data',
+ 'fann_subset_train_data',
+ 'fann_test_data',
+ 'fann_test',
+ 'fann_train_epoch',
+ 'fann_train_on_data',
+ 'fann_train_on_file',
+ 'fann_train'),
+ 'Fileinfo': ('finfo_buffer',
+ 'finfo_close',
+ 'finfo_file',
+ 'finfo_open',
+ 'finfo_set_flags',
+ 'mime_content_type'),
+ 'Filesystem': ('basename',
+ 'chgrp',
+ 'chmod',
+ 'chown',
+ 'clearstatcache',
+ 'copy',
+ 'dirname',
+ 'disk_free_space',
+ 'disk_total_space',
+ 'diskfreespace',
+ 'fclose',
+ 'feof',
+ 'fflush',
+ 'fgetc',
+ 'fgetcsv',
+ 'fgets',
+ 'fgetss',
+ 'file_exists',
+ 'file_get_contents',
+ 'file_put_contents',
+ 'file',
+ 'fileatime',
+ 'filectime',
+ 'filegroup',
+ 'fileinode',
+ 'filemtime',
+ 'fileowner',
+ 'fileperms',
+ 'filesize',
+ 'filetype',
+ 'flock',
+ 'fnmatch',
+ 'fopen',
+ 'fpassthru',
+ 'fputcsv',
+ 'fputs',
+ 'fread',
+ 'fscanf',
+ 'fseek',
+ 'fstat',
+ 'ftell',
+ 'ftruncate',
+ 'fwrite',
+ 'glob',
+ 'is_dir',
+ 'is_executable',
+ 'is_file',
+ 'is_link',
+ 'is_readable',
+ 'is_uploaded_file',
+ 'is_writable',
+ 'is_writeable',
+ 'lchgrp',
+ 'lchown',
+ 'link',
+ 'linkinfo',
+ 'lstat',
+ 'mkdir',
+ 'move_uploaded_file',
+ 'parse_ini_file',
+ 'parse_ini_string',
+ 'pathinfo',
+ 'pclose',
+ 'popen',
+ 'readfile',
+ 'readlink',
+ 'realpath_cache_get',
+ 'realpath_cache_size',
+ 'realpath',
+ 'rename',
+ 'rewind',
+ 'rmdir',
+ 'set_file_buffer',
+ 'stat',
+ 'symlink',
+ 'tempnam',
+ 'tmpfile',
+ 'touch',
+ 'umask',
+ 'unlink'),
+ 'Filter': ('filter_has_var',
+ 'filter_id',
+ 'filter_input_array',
+ 'filter_input',
+ 'filter_list',
+ 'filter_var_array',
+ 'filter_var'),
+ 'Firebird/InterBase': ('ibase_add_user',
+ 'ibase_affected_rows',
+ 'ibase_backup',
+ 'ibase_blob_add',
+ 'ibase_blob_cancel',
+ 'ibase_blob_close',
+ 'ibase_blob_create',
+ 'ibase_blob_echo',
+ 'ibase_blob_get',
+ 'ibase_blob_import',
+ 'ibase_blob_info',
+ 'ibase_blob_open',
+ 'ibase_close',
+ 'ibase_commit_ret',
+ 'ibase_commit',
+ 'ibase_connect',
+ 'ibase_db_info',
+ 'ibase_delete_user',
+ 'ibase_drop_db',
+ 'ibase_errcode',
+ 'ibase_errmsg',
+ 'ibase_execute',
+ 'ibase_fetch_assoc',
+ 'ibase_fetch_object',
+ 'ibase_fetch_row',
+ 'ibase_field_info',
+ 'ibase_free_event_handler',
+ 'ibase_free_query',
+ 'ibase_free_result',
+ 'ibase_gen_id',
+ 'ibase_maintain_db',
+ 'ibase_modify_user',
+ 'ibase_name_result',
+ 'ibase_num_fields',
+ 'ibase_num_params',
+ 'ibase_param_info',
+ 'ibase_pconnect',
+ 'ibase_prepare',
+ 'ibase_query',
+ 'ibase_restore',
+ 'ibase_rollback_ret',
+ 'ibase_rollback',
+ 'ibase_server_info',
+ 'ibase_service_attach',
+ 'ibase_service_detach',
+ 'ibase_set_event_handler',
+ 'ibase_trans',
+ 'ibase_wait_event'),
+ 'FriBiDi': ('fribidi_log2vis',),
+ 'FrontBase': ('fbsql_affected_rows',
+ 'fbsql_autocommit',
+ 'fbsql_blob_size',
+ 'fbsql_change_user',
+ 'fbsql_clob_size',
+ 'fbsql_close',
+ 'fbsql_commit',
+ 'fbsql_connect',
+ 'fbsql_create_blob',
+ 'fbsql_create_clob',
+ 'fbsql_create_db',
+ 'fbsql_data_seek',
+ 'fbsql_database_password',
+ 'fbsql_database',
+ 'fbsql_db_query',
+ 'fbsql_db_status',
+ 'fbsql_drop_db',
+ 'fbsql_errno',
+ 'fbsql_error',
+ 'fbsql_fetch_array',
+ 'fbsql_fetch_assoc',
+ 'fbsql_fetch_field',
+ 'fbsql_fetch_lengths',
+ 'fbsql_fetch_object',
+ 'fbsql_fetch_row',
+ 'fbsql_field_flags',
+ 'fbsql_field_len',
+ 'fbsql_field_name',
+ 'fbsql_field_seek',
+ 'fbsql_field_table',
+ 'fbsql_field_type',
+ 'fbsql_free_result',
+ 'fbsql_get_autostart_info',
+ 'fbsql_hostname',
+ 'fbsql_insert_id',
+ 'fbsql_list_dbs',
+ 'fbsql_list_fields',
+ 'fbsql_list_tables',
+ 'fbsql_next_result',
+ 'fbsql_num_fields',
+ 'fbsql_num_rows',
+ 'fbsql_password',
+ 'fbsql_pconnect',
+ 'fbsql_query',
+ 'fbsql_read_blob',
+ 'fbsql_read_clob',
+ 'fbsql_result',
+ 'fbsql_rollback',
+ 'fbsql_rows_fetched',
+ 'fbsql_select_db',
+ 'fbsql_set_characterset',
+ 'fbsql_set_lob_mode',
+ 'fbsql_set_password',
+ 'fbsql_set_transaction',
+ 'fbsql_start_db',
+ 'fbsql_stop_db',
+ 'fbsql_table_name',
+ 'fbsql_tablename',
+ 'fbsql_username',
+ 'fbsql_warnings'),
+ 'Function handling': ('call_user_func_array',
+ 'call_user_func',
+ 'create_function',
+ 'forward_static_call_array',
+ 'forward_static_call',
+ 'func_get_arg',
+ 'func_get_args',
+ 'func_num_args',
+ 'function_exists',
+ 'get_defined_functions',
+ 'register_shutdown_function',
+ 'register_tick_function',
+ 'unregister_tick_function'),
+ 'GD and Image': ('gd_info',
+ 'getimagesize',
+ 'getimagesizefromstring',
+ 'image_type_to_extension',
+ 'image_type_to_mime_type',
+ 'image2wbmp',
+ 'imageaffine',
+ 'imageaffinematrixconcat',
+ 'imageaffinematrixget',
+ 'imagealphablending',
+ 'imageantialias',
+ 'imagearc',
+ 'imagechar',
+ 'imagecharup',
+ 'imagecolorallocate',
+ 'imagecolorallocatealpha',
+ 'imagecolorat',
+ 'imagecolorclosest',
+ 'imagecolorclosestalpha',
+ 'imagecolorclosesthwb',
+ 'imagecolordeallocate',
+ 'imagecolorexact',
+ 'imagecolorexactalpha',
+ 'imagecolormatch',
+ 'imagecolorresolve',
+ 'imagecolorresolvealpha',
+ 'imagecolorset',
+ 'imagecolorsforindex',
+ 'imagecolorstotal',
+ 'imagecolortransparent',
+ 'imageconvolution',
+ 'imagecopy',
+ 'imagecopymerge',
+ 'imagecopymergegray',
+ 'imagecopyresampled',
+ 'imagecopyresized',
+ 'imagecreate',
+ 'imagecreatefromgd2',
+ 'imagecreatefromgd2part',
+ 'imagecreatefromgd',
+ 'imagecreatefromgif',
+ 'imagecreatefromjpeg',
+ 'imagecreatefrompng',
+ 'imagecreatefromstring',
+ 'imagecreatefromwbmp',
+ 'imagecreatefromwebp',
+ 'imagecreatefromxbm',
+ 'imagecreatefromxpm',
+ 'imagecreatetruecolor',
+ 'imagecrop',
+ 'imagecropauto',
+ 'imagedashedline',
+ 'imagedestroy',
+ 'imageellipse',
+ 'imagefill',
+ 'imagefilledarc',
+ 'imagefilledellipse',
+ 'imagefilledpolygon',
+ 'imagefilledrectangle',
+ 'imagefilltoborder',
+ 'imagefilter',
+ 'imageflip',
+ 'imagefontheight',
+ 'imagefontwidth',
+ 'imageftbbox',
+ 'imagefttext',
+ 'imagegammacorrect',
+ 'imagegd2',
+ 'imagegd',
+ 'imagegif',
+ 'imagegrabscreen',
+ 'imagegrabwindow',
+ 'imageinterlace',
+ 'imageistruecolor',
+ 'imagejpeg',
+ 'imagelayereffect',
+ 'imageline',
+ 'imageloadfont',
+ 'imagepalettecopy',
+ 'imagepalettetotruecolor',
+ 'imagepng',
+ 'imagepolygon',
+ 'imagepsbbox',
+ 'imagepsencodefont',
+ 'imagepsextendfont',
+ 'imagepsfreefont',
+ 'imagepsloadfont',
+ 'imagepsslantfont',
+ 'imagepstext',
+ 'imagerectangle',
+ 'imagerotate',
+ 'imagesavealpha',
+ 'imagescale',
+ 'imagesetbrush',
+ 'imagesetinterpolation',
+ 'imagesetpixel',
+ 'imagesetstyle',
+ 'imagesetthickness',
+ 'imagesettile',
+ 'imagestring',
+ 'imagestringup',
+ 'imagesx',
+ 'imagesy',
+ 'imagetruecolortopalette',
+ 'imagettfbbox',
+ 'imagettftext',
+ 'imagetypes',
+ 'imagewbmp',
+ 'imagewebp',
+ 'imagexbm',
+ 'iptcembed',
+ 'iptcparse',
+ 'jpeg2wbmp',
+ 'png2wbmp'),
+ 'GMP': ('gmp_abs',
+ 'gmp_add',
+ 'gmp_and',
+ 'gmp_clrbit',
+ 'gmp_cmp',
+ 'gmp_com',
+ 'gmp_div_q',
+ 'gmp_div_qr',
+ 'gmp_div_r',
+ 'gmp_div',
+ 'gmp_divexact',
+ 'gmp_fact',
+ 'gmp_gcd',
+ 'gmp_gcdext',
+ 'gmp_hamdist',
+ 'gmp_init',
+ 'gmp_intval',
+ 'gmp_invert',
+ 'gmp_jacobi',
+ 'gmp_legendre',
+ 'gmp_mod',
+ 'gmp_mul',
+ 'gmp_neg',
+ 'gmp_nextprime',
+ 'gmp_or',
+ 'gmp_perfect_square',
+ 'gmp_popcount',
+ 'gmp_pow',
+ 'gmp_powm',
+ 'gmp_prob_prime',
+ 'gmp_random',
+ 'gmp_scan0',
+ 'gmp_scan1',
+ 'gmp_setbit',
+ 'gmp_sign',
+ 'gmp_sqrt',
+ 'gmp_sqrtrem',
+ 'gmp_strval',
+ 'gmp_sub',
+ 'gmp_testbit',
+ 'gmp_xor'),
+ 'GeoIP': ('geoip_asnum_by_name',
+ 'geoip_continent_code_by_name',
+ 'geoip_country_code_by_name',
+ 'geoip_country_code3_by_name',
+ 'geoip_country_name_by_name',
+ 'geoip_database_info',
+ 'geoip_db_avail',
+ 'geoip_db_filename',
+ 'geoip_db_get_all_info',
+ 'geoip_domain_by_name',
+ 'geoip_id_by_name',
+ 'geoip_isp_by_name',
+ 'geoip_netspeedcell_by_name',
+ 'geoip_org_by_name',
+ 'geoip_record_by_name',
+ 'geoip_region_by_name',
+ 'geoip_region_name_by_code',
+ 'geoip_setup_custom_directory',
+ 'geoip_time_zone_by_country_and_region'),
+ 'Gettext': ('bind_textdomain_codeset',
+ 'bindtextdomain',
+ 'dcgettext',
+ 'dcngettext',
+ 'dgettext',
+ 'dngettext',
+ 'gettext',
+ 'ngettext',
+ 'textdomain'),
+ 'GnuPG': ('gnupg_adddecryptkey',
+ 'gnupg_addencryptkey',
+ 'gnupg_addsignkey',
+ 'gnupg_cleardecryptkeys',
+ 'gnupg_clearencryptkeys',
+ 'gnupg_clearsignkeys',
+ 'gnupg_decrypt',
+ 'gnupg_decryptverify',
+ 'gnupg_encrypt',
+ 'gnupg_encryptsign',
+ 'gnupg_export',
+ 'gnupg_geterror',
+ 'gnupg_getprotocol',
+ 'gnupg_import',
+ 'gnupg_init',
+ 'gnupg_keyinfo',
+ 'gnupg_setarmor',
+ 'gnupg_seterrormode',
+ 'gnupg_setsignmode',
+ 'gnupg_sign',
+ 'gnupg_verify'),
+ 'Gopher': ('gopher_parsedir',),
+ 'Grapheme': ('grapheme_extract',
+ 'grapheme_stripos',
+ 'grapheme_stristr',
+ 'grapheme_strlen',
+ 'grapheme_strpos',
+ 'grapheme_strripos',
+ 'grapheme_strrpos',
+ 'grapheme_strstr',
+ 'grapheme_substr'),
+ 'Gupnp': ('gupnp_context_get_host_ip',
+ 'gupnp_context_get_port',
+ 'gupnp_context_get_subscription_timeout',
+ 'gupnp_context_host_path',
+ 'gupnp_context_new',
+ 'gupnp_context_set_subscription_timeout',
+ 'gupnp_context_timeout_add',
+ 'gupnp_context_unhost_path',
+ 'gupnp_control_point_browse_start',
+ 'gupnp_control_point_browse_stop',
+ 'gupnp_control_point_callback_set',
+ 'gupnp_control_point_new',
+ 'gupnp_device_action_callback_set',
+ 'gupnp_device_info_get_service',
+ 'gupnp_device_info_get',
+ 'gupnp_root_device_get_available',
+ 'gupnp_root_device_get_relative_location',
+ 'gupnp_root_device_new',
+ 'gupnp_root_device_set_available',
+ 'gupnp_root_device_start',
+ 'gupnp_root_device_stop',
+ 'gupnp_service_action_get',
+ 'gupnp_service_action_return_error',
+ 'gupnp_service_action_return',
+ 'gupnp_service_action_set',
+ 'gupnp_service_freeze_notify',
+ 'gupnp_service_info_get_introspection',
+ 'gupnp_service_info_get',
+ 'gupnp_service_introspection_get_state_variable',
+ 'gupnp_service_notify',
+ 'gupnp_service_proxy_action_get',
+ 'gupnp_service_proxy_action_set',
+ 'gupnp_service_proxy_add_notify',
+ 'gupnp_service_proxy_callback_set',
+ 'gupnp_service_proxy_get_subscribed',
+ 'gupnp_service_proxy_remove_notify',
+ 'gupnp_service_proxy_set_subscribed',
+ 'gupnp_service_thaw_notify'),
+ 'HTTP': ('http_cache_etag',
+ 'http_cache_last_modified',
+ 'http_chunked_decode',
+ 'http_deflate',
+ 'http_inflate',
+ 'http_build_cookie',
+ 'http_date',
+ 'http_get_request_body_stream',
+ 'http_get_request_body',
+ 'http_get_request_headers',
+ 'http_match_etag',
+ 'http_match_modified',
+ 'http_match_request_header',
+ 'http_support',
+ 'http_negotiate_charset',
+ 'http_negotiate_content_type',
+ 'http_negotiate_language',
+ 'ob_deflatehandler',
+ 'ob_etaghandler',
+ 'ob_inflatehandler',
+ 'http_parse_cookie',
+ 'http_parse_headers',
+ 'http_parse_message',
+ 'http_parse_params',
+ 'http_persistent_handles_clean',
+ 'http_persistent_handles_count',
+ 'http_persistent_handles_ident',
+ 'http_get',
+ 'http_head',
+ 'http_post_data',
+ 'http_post_fields',
+ 'http_put_data',
+ 'http_put_file',
+ 'http_put_stream',
+ 'http_request_body_encode',
+ 'http_request_method_exists',
+ 'http_request_method_name',
+ 'http_request_method_register',
+ 'http_request_method_unregister',
+ 'http_request',
+ 'http_redirect',
+ 'http_send_content_disposition',
+ 'http_send_content_type',
+ 'http_send_data',
+ 'http_send_file',
+ 'http_send_last_modified',
+ 'http_send_status',
+ 'http_send_stream',
+ 'http_throttle',
+ 'http_build_str',
+ 'http_build_url'),
+ 'Hash': ('hash_algos',
+ 'hash_copy',
+ 'hash_file',
+ 'hash_final',
+ 'hash_hmac_file',
+ 'hash_hmac',
+ 'hash_init',
+ 'hash_pbkdf2',
+ 'hash_update_file',
+ 'hash_update_stream',
+ 'hash_update',
+ 'hash'),
+ 'Hyperwave': ('hw_Array2Objrec',
+ 'hw_changeobject',
+ 'hw_Children',
+ 'hw_ChildrenObj',
+ 'hw_Close',
+ 'hw_Connect',
+ 'hw_connection_info',
+ 'hw_cp',
+ 'hw_Deleteobject',
+ 'hw_DocByAnchor',
+ 'hw_DocByAnchorObj',
+ 'hw_Document_Attributes',
+ 'hw_Document_BodyTag',
+ 'hw_Document_Content',
+ 'hw_Document_SetContent',
+ 'hw_Document_Size',
+ 'hw_dummy',
+ 'hw_EditText',
+ 'hw_Error',
+ 'hw_ErrorMsg',
+ 'hw_Free_Document',
+ 'hw_GetAnchors',
+ 'hw_GetAnchorsObj',
+ 'hw_GetAndLock',
+ 'hw_GetChildColl',
+ 'hw_GetChildCollObj',
+ 'hw_GetChildDocColl',
+ 'hw_GetChildDocCollObj',
+ 'hw_GetObject',
+ 'hw_GetObjectByQuery',
+ 'hw_GetObjectByQueryColl',
+ 'hw_GetObjectByQueryCollObj',
+ 'hw_GetObjectByQueryObj',
+ 'hw_GetParents',
+ 'hw_GetParentsObj',
+ 'hw_getrellink',
+ 'hw_GetRemote',
+ 'hw_getremotechildren',
+ 'hw_GetSrcByDestObj',
+ 'hw_GetText',
+ 'hw_getusername',
+ 'hw_Identify',
+ 'hw_InCollections',
+ 'hw_Info',
+ 'hw_InsColl',
+ 'hw_InsDoc',
+ 'hw_insertanchors',
+ 'hw_InsertDocument',
+ 'hw_InsertObject',
+ 'hw_mapid',
+ 'hw_Modifyobject',
+ 'hw_mv',
+ 'hw_New_Document',
+ 'hw_objrec2array',
+ 'hw_Output_Document',
+ 'hw_pConnect',
+ 'hw_PipeDocument',
+ 'hw_Root',
+ 'hw_setlinkroot',
+ 'hw_stat',
+ 'hw_Unlock',
+ 'hw_Who'),
+ 'Hyperwave API': ('hwapi_attribute_new',
+ 'hwapi_content_new',
+ 'hwapi_hgcsp',
+ 'hwapi_object_new'),
+ 'IBM DB2': ('db2_autocommit',
+ 'db2_bind_param',
+ 'db2_client_info',
+ 'db2_close',
+ 'db2_column_privileges',
+ 'db2_columns',
+ 'db2_commit',
+ 'db2_conn_error',
+ 'db2_conn_errormsg',
+ 'db2_connect',
+ 'db2_cursor_type',
+ 'db2_escape_string',
+ 'db2_exec',
+ 'db2_execute',
+ 'db2_fetch_array',
+ 'db2_fetch_assoc',
+ 'db2_fetch_both',
+ 'db2_fetch_object',
+ 'db2_fetch_row',
+ 'db2_field_display_size',
+ 'db2_field_name',
+ 'db2_field_num',
+ 'db2_field_precision',
+ 'db2_field_scale',
+ 'db2_field_type',
+ 'db2_field_width',
+ 'db2_foreign_keys',
+ 'db2_free_result',
+ 'db2_free_stmt',
+ 'db2_get_option',
+ 'db2_last_insert_id',
+ 'db2_lob_read',
+ 'db2_next_result',
+ 'db2_num_fields',
+ 'db2_num_rows',
+ 'db2_pclose',
+ 'db2_pconnect',
+ 'db2_prepare',
+ 'db2_primary_keys',
+ 'db2_procedure_columns',
+ 'db2_procedures',
+ 'db2_result',
+ 'db2_rollback',
+ 'db2_server_info',
+ 'db2_set_option',
+ 'db2_special_columns',
+ 'db2_statistics',
+ 'db2_stmt_error',
+ 'db2_stmt_errormsg',
+ 'db2_table_privileges',
+ 'db2_tables'),
+ 'ID3': ('id3_get_frame_long_name',
+ 'id3_get_frame_short_name',
+ 'id3_get_genre_id',
+ 'id3_get_genre_list',
+ 'id3_get_genre_name',
+ 'id3_get_tag',
+ 'id3_get_version',
+ 'id3_remove_tag',
+ 'id3_set_tag'),
+ 'IDN': ('grapheme_substr', 'idn_to_ascii', 'idn_to_unicode', 'idn_to_utf8'),
+ 'IIS': ('iis_add_server',
+ 'iis_get_dir_security',
+ 'iis_get_script_map',
+ 'iis_get_server_by_comment',
+ 'iis_get_server_by_path',
+ 'iis_get_server_rights',
+ 'iis_get_service_state',
+ 'iis_remove_server',
+ 'iis_set_app_settings',
+ 'iis_set_dir_security',
+ 'iis_set_script_map',
+ 'iis_set_server_rights',
+ 'iis_start_server',
+ 'iis_start_service',
+ 'iis_stop_server',
+ 'iis_stop_service'),
+ 'IMAP': ('imap_8bit',
+ 'imap_alerts',
+ 'imap_append',
+ 'imap_base64',
+ 'imap_binary',
+ 'imap_body',
+ 'imap_bodystruct',
+ 'imap_check',
+ 'imap_clearflag_full',
+ 'imap_close',
+ 'imap_create',
+ 'imap_createmailbox',
+ 'imap_delete',
+ 'imap_deletemailbox',
+ 'imap_errors',
+ 'imap_expunge',
+ 'imap_fetch_overview',
+ 'imap_fetchbody',
+ 'imap_fetchheader',
+ 'imap_fetchmime',
+ 'imap_fetchstructure',
+ 'imap_fetchtext',
+ 'imap_gc',
+ 'imap_get_quota',
+ 'imap_get_quotaroot',
+ 'imap_getacl',
+ 'imap_getmailboxes',
+ 'imap_getsubscribed',
+ 'imap_header',
+ 'imap_headerinfo',
+ 'imap_headers',
+ 'imap_last_error',
+ 'imap_list',
+ 'imap_listmailbox',
+ 'imap_listscan',
+ 'imap_listsubscribed',
+ 'imap_lsub',
+ 'imap_mail_compose',
+ 'imap_mail_copy',
+ 'imap_mail_move',
+ 'imap_mail',
+ 'imap_mailboxmsginfo',
+ 'imap_mime_header_decode',
+ 'imap_msgno',
+ 'imap_num_msg',
+ 'imap_num_recent',
+ 'imap_open',
+ 'imap_ping',
+ 'imap_qprint',
+ 'imap_rename',
+ 'imap_renamemailbox',
+ 'imap_reopen',
+ 'imap_rfc822_parse_adrlist',
+ 'imap_rfc822_parse_headers',
+ 'imap_rfc822_write_address',
+ 'imap_savebody',
+ 'imap_scan',
+ 'imap_scanmailbox',
+ 'imap_search',
+ 'imap_set_quota',
+ 'imap_setacl',
+ 'imap_setflag_full',
+ 'imap_sort',
+ 'imap_status',
+ 'imap_subscribe',
+ 'imap_thread',
+ 'imap_timeout',
+ 'imap_uid',
+ 'imap_undelete',
+ 'imap_unsubscribe',
+ 'imap_utf7_decode',
+ 'imap_utf7_encode',
+ 'imap_utf8'),
+ 'Informix': ('ifx_affected_rows',
+ 'ifx_blobinfile_mode',
+ 'ifx_byteasvarchar',
+ 'ifx_close',
+ 'ifx_connect',
+ 'ifx_copy_blob',
+ 'ifx_create_blob',
+ 'ifx_create_char',
+ 'ifx_do',
+ 'ifx_error',
+ 'ifx_errormsg',
+ 'ifx_fetch_row',
+ 'ifx_fieldproperties',
+ 'ifx_fieldtypes',
+ 'ifx_free_blob',
+ 'ifx_free_char',
+ 'ifx_free_result',
+ 'ifx_get_blob',
+ 'ifx_get_char',
+ 'ifx_getsqlca',
+ 'ifx_htmltbl_result',
+ 'ifx_nullformat',
+ 'ifx_num_fields',
+ 'ifx_num_rows',
+ 'ifx_pconnect',
+ 'ifx_prepare',
+ 'ifx_query',
+ 'ifx_textasvarchar',
+ 'ifx_update_blob',
+ 'ifx_update_char',
+ 'ifxus_close_slob',
+ 'ifxus_create_slob',
+ 'ifxus_free_slob',
+ 'ifxus_open_slob',
+ 'ifxus_read_slob',
+ 'ifxus_seek_slob',
+ 'ifxus_tell_slob',
+ 'ifxus_write_slob'),
+ 'Ingres': ('ingres_autocommit_state',
+ 'ingres_autocommit',
+ 'ingres_charset',
+ 'ingres_close',
+ 'ingres_commit',
+ 'ingres_connect',
+ 'ingres_cursor',
+ 'ingres_errno',
+ 'ingres_error',
+ 'ingres_errsqlstate',
+ 'ingres_escape_string',
+ 'ingres_execute',
+ 'ingres_fetch_array',
+ 'ingres_fetch_assoc',
+ 'ingres_fetch_object',
+ 'ingres_fetch_proc_return',
+ 'ingres_fetch_row',
+ 'ingres_field_length',
+ 'ingres_field_name',
+ 'ingres_field_nullable',
+ 'ingres_field_precision',
+ 'ingres_field_scale',
+ 'ingres_field_type',
+ 'ingres_free_result',
+ 'ingres_next_error',
+ 'ingres_num_fields',
+ 'ingres_num_rows',
+ 'ingres_pconnect',
+ 'ingres_prepare',
+ 'ingres_query',
+ 'ingres_result_seek',
+ 'ingres_rollback',
+ 'ingres_set_environment',
+ 'ingres_unbuffered_query'),
+ 'Inotify': ('inotify_add_watch',
+ 'inotify_init',
+ 'inotify_queue_len',
+ 'inotify_read',
+ 'inotify_rm_watch'),
+ 'JSON': ('json_decode',
+ 'json_encode',
+ 'json_last_error_msg',
+ 'json_last_error'),
+ 'Java': ('java_last_exception_clear', 'java_last_exception_get'),
+ 'Judy': ('judy_type', 'judy_version'),
+ 'KADM5': ('kadm5_chpass_principal',
+ 'kadm5_create_principal',
+ 'kadm5_delete_principal',
+ 'kadm5_destroy',
+ 'kadm5_flush',
+ 'kadm5_get_policies',
+ 'kadm5_get_principal',
+ 'kadm5_get_principals',
+ 'kadm5_init_with_password',
+ 'kadm5_modify_principal'),
+ 'LDAP': ('ldap_8859_to_t61',
+ 'ldap_add',
+ 'ldap_bind',
+ 'ldap_close',
+ 'ldap_compare',
+ 'ldap_connect',
+ 'ldap_control_paged_result_response',
+ 'ldap_control_paged_result',
+ 'ldap_count_entries',
+ 'ldap_delete',
+ 'ldap_dn2ufn',
+ 'ldap_err2str',
+ 'ldap_errno',
+ 'ldap_error',
+ 'ldap_explode_dn',
+ 'ldap_first_attribute',
+ 'ldap_first_entry',
+ 'ldap_first_reference',
+ 'ldap_free_result',
+ 'ldap_get_attributes',
+ 'ldap_get_dn',
+ 'ldap_get_entries',
+ 'ldap_get_option',
+ 'ldap_get_values_len',
+ 'ldap_get_values',
+ 'ldap_list',
+ 'ldap_mod_add',
+ 'ldap_mod_del',
+ 'ldap_mod_replace',
+ 'ldap_modify',
+ 'ldap_next_attribute',
+ 'ldap_next_entry',
+ 'ldap_next_reference',
+ 'ldap_parse_reference',
+ 'ldap_parse_result',
+ 'ldap_read',
+ 'ldap_rename',
+ 'ldap_sasl_bind',
+ 'ldap_search',
+ 'ldap_set_option',
+ 'ldap_set_rebind_proc',
+ 'ldap_sort',
+ 'ldap_start_tls',
+ 'ldap_t61_to_8859',
+ 'ldap_unbind'),
+ 'LZF': ('lzf_compress', 'lzf_decompress', 'lzf_optimized_for'),
+ 'Libevent': ('event_add',
+ 'event_base_free',
+ 'event_base_loop',
+ 'event_base_loopbreak',
+ 'event_base_loopexit',
+ 'event_base_new',
+ 'event_base_priority_init',
+ 'event_base_set',
+ 'event_buffer_base_set',
+ 'event_buffer_disable',
+ 'event_buffer_enable',
+ 'event_buffer_fd_set',
+ 'event_buffer_free',
+ 'event_buffer_new',
+ 'event_buffer_priority_set',
+ 'event_buffer_read',
+ 'event_buffer_set_callback',
+ 'event_buffer_timeout_set',
+ 'event_buffer_watermark_set',
+ 'event_buffer_write',
+ 'event_del',
+ 'event_free',
+ 'event_new',
+ 'event_set'),
+ 'Lotus Notes': ('notes_body',
+ 'notes_copy_db',
+ 'notes_create_db',
+ 'notes_create_note',
+ 'notes_drop_db',
+ 'notes_find_note',
+ 'notes_header_info',
+ 'notes_list_msgs',
+ 'notes_mark_read',
+ 'notes_mark_unread',
+ 'notes_nav_create',
+ 'notes_search',
+ 'notes_unread',
+ 'notes_version'),
+ 'MCVE': ('m_checkstatus',
+ 'm_completeauthorizations',
+ 'm_connect',
+ 'm_connectionerror',
+ 'm_deletetrans',
+ 'm_destroyconn',
+ 'm_destroyengine',
+ 'm_getcell',
+ 'm_getcellbynum',
+ 'm_getcommadelimited',
+ 'm_getheader',
+ 'm_initconn',
+ 'm_initengine',
+ 'm_iscommadelimited',
+ 'm_maxconntimeout',
+ 'm_monitor',
+ 'm_numcolumns',
+ 'm_numrows',
+ 'm_parsecommadelimited',
+ 'm_responsekeys',
+ 'm_responseparam',
+ 'm_returnstatus',
+ 'm_setblocking',
+ 'm_setdropfile',
+ 'm_setip',
+ 'm_setssl_cafile',
+ 'm_setssl_files',
+ 'm_setssl',
+ 'm_settimeout',
+ 'm_sslcert_gen_hash',
+ 'm_transactionssent',
+ 'm_transinqueue',
+ 'm_transkeyval',
+ 'm_transnew',
+ 'm_transsend',
+ 'm_uwait',
+ 'm_validateidentifier',
+ 'm_verifyconnection',
+ 'm_verifysslcert'),
+ 'Mail': ('ezmlm_hash', 'mail'),
+ 'Mailparse': ('mailparse_determine_best_xfer_encoding',
+ 'mailparse_msg_create',
+ 'mailparse_msg_extract_part_file',
+ 'mailparse_msg_extract_part',
+ 'mailparse_msg_extract_whole_part_file',
+ 'mailparse_msg_free',
+ 'mailparse_msg_get_part_data',
+ 'mailparse_msg_get_part',
+ 'mailparse_msg_get_structure',
+ 'mailparse_msg_parse_file',
+ 'mailparse_msg_parse',
+ 'mailparse_rfc822_parse_addresses',
+ 'mailparse_stream_encode',
+ 'mailparse_uudecode_all'),
+ 'Math': ('abs',
+ 'acos',
+ 'acosh',
+ 'asin',
+ 'asinh',
+ 'atan2',
+ 'atan',
+ 'atanh',
+ 'base_convert',
+ 'bindec',
+ 'ceil',
+ 'cos',
+ 'cosh',
+ 'decbin',
+ 'dechex',
+ 'decoct',
+ 'deg2rad',
+ 'exp',
+ 'expm1',
+ 'floor',
+ 'fmod',
+ 'getrandmax',
+ 'hexdec',
+ 'hypot',
+ 'is_finite',
+ 'is_infinite',
+ 'is_nan',
+ 'lcg_value',
+ 'log10',
+ 'log1p',
+ 'log',
+ 'max',
+ 'min',
+ 'mt_getrandmax',
+ 'mt_rand',
+ 'mt_srand',
+ 'octdec',
+ 'pi',
+ 'pow',
+ 'rad2deg',
+ 'rand',
+ 'round',
+ 'sin',
+ 'sinh',
+ 'sqrt',
+ 'srand',
+ 'tan',
+ 'tanh'),
+ 'MaxDB': ('maxdb_affected_rows',
+ 'maxdb_autocommit',
+ 'maxdb_bind_param',
+ 'maxdb_bind_result',
+ 'maxdb_change_user',
+ 'maxdb_character_set_name',
+ 'maxdb_client_encoding',
+ 'maxdb_close_long_data',
+ 'maxdb_close',
+ 'maxdb_commit',
+ 'maxdb_connect_errno',
+ 'maxdb_connect_error',
+ 'maxdb_connect',
+ 'maxdb_data_seek',
+ 'maxdb_debug',
+ 'maxdb_disable_reads_from_master',
+ 'maxdb_disable_rpl_parse',
+ 'maxdb_dump_debug_info',
+ 'maxdb_embedded_connect',
+ 'maxdb_enable_reads_from_master',
+ 'maxdb_enable_rpl_parse',
+ 'maxdb_errno',
+ 'maxdb_error',
+ 'maxdb_escape_string',
+ 'maxdb_execute',
+ 'maxdb_fetch_array',
+ 'maxdb_fetch_assoc',
+ 'maxdb_fetch_field_direct',
+ 'maxdb_fetch_field',
+ 'maxdb_fetch_fields',
+ 'maxdb_fetch_lengths',
+ 'maxdb_fetch_object',
+ 'maxdb_fetch_row',
+ 'maxdb_fetch',
+ 'maxdb_field_count',
+ 'maxdb_field_seek',
+ 'maxdb_field_tell',
+ 'maxdb_free_result',
+ 'maxdb_get_client_info',
+ 'maxdb_get_client_version',
+ 'maxdb_get_host_info',
+ 'maxdb_get_metadata',
+ 'maxdb_get_proto_info',
+ 'maxdb_get_server_info',
+ 'maxdb_get_server_version',
+ 'maxdb_info',
+ 'maxdb_init',
+ 'maxdb_insert_id',
+ 'maxdb_kill',
+ 'maxdb_master_query',
+ 'maxdb_more_results',
+ 'maxdb_multi_query',
+ 'maxdb_next_result',
+ 'maxdb_num_fields',
+ 'maxdb_num_rows',
+ 'maxdb_options',
+ 'maxdb_param_count',
+ 'maxdb_ping',
+ 'maxdb_prepare',
+ 'maxdb_query',
+ 'maxdb_real_connect',
+ 'maxdb_real_escape_string',
+ 'maxdb_real_query',
+ 'maxdb_report',
+ 'maxdb_rollback',
+ 'maxdb_rpl_parse_enabled',
+ 'maxdb_rpl_probe',
+ 'maxdb_rpl_query_type',
+ 'maxdb_select_db',
+ 'maxdb_send_long_data',
+ 'maxdb_send_query',
+ 'maxdb_server_end',
+ 'maxdb_server_init',
+ 'maxdb_set_opt',
+ 'maxdb_sqlstate',
+ 'maxdb_ssl_set',
+ 'maxdb_stat',
+ 'maxdb_stmt_affected_rows',
+ 'maxdb_stmt_bind_param',
+ 'maxdb_stmt_bind_result',
+ 'maxdb_stmt_close_long_data',
+ 'maxdb_stmt_close',
+ 'maxdb_stmt_data_seek',
+ 'maxdb_stmt_errno',
+ 'maxdb_stmt_error',
+ 'maxdb_stmt_execute',
+ 'maxdb_stmt_fetch',
+ 'maxdb_stmt_free_result',
+ 'maxdb_stmt_init',
+ 'maxdb_stmt_num_rows',
+ 'maxdb_stmt_param_count',
+ 'maxdb_stmt_prepare',
+ 'maxdb_stmt_reset',
+ 'maxdb_stmt_result_metadata',
+ 'maxdb_stmt_send_long_data',
+ 'maxdb_stmt_sqlstate',
+ 'maxdb_stmt_store_result',
+ 'maxdb_store_result',
+ 'maxdb_thread_id',
+ 'maxdb_thread_safe',
+ 'maxdb_use_result',
+ 'maxdb_warning_count'),
+ 'Mcrypt': ('mcrypt_cbc',
+ 'mcrypt_cfb',
+ 'mcrypt_create_iv',
+ 'mcrypt_decrypt',
+ 'mcrypt_ecb',
+ 'mcrypt_enc_get_algorithms_name',
+ 'mcrypt_enc_get_block_size',
+ 'mcrypt_enc_get_iv_size',
+ 'mcrypt_enc_get_key_size',
+ 'mcrypt_enc_get_modes_name',
+ 'mcrypt_enc_get_supported_key_sizes',
+ 'mcrypt_enc_is_block_algorithm_mode',
+ 'mcrypt_enc_is_block_algorithm',
+ 'mcrypt_enc_is_block_mode',
+ 'mcrypt_enc_self_test',
+ 'mcrypt_encrypt',
+ 'mcrypt_generic_deinit',
+ 'mcrypt_generic_end',
+ 'mcrypt_generic_init',
+ 'mcrypt_generic',
+ 'mcrypt_get_block_size',
+ 'mcrypt_get_cipher_name',
+ 'mcrypt_get_iv_size',
+ 'mcrypt_get_key_size',
+ 'mcrypt_list_algorithms',
+ 'mcrypt_list_modes',
+ 'mcrypt_module_close',
+ 'mcrypt_module_get_algo_block_size',
+ 'mcrypt_module_get_algo_key_size',
+ 'mcrypt_module_get_supported_key_sizes',
+ 'mcrypt_module_is_block_algorithm_mode',
+ 'mcrypt_module_is_block_algorithm',
+ 'mcrypt_module_is_block_mode',
+ 'mcrypt_module_open',
+ 'mcrypt_module_self_test',
+ 'mcrypt_ofb',
+ 'mdecrypt_generic'),
+ 'Memcache': ('memcache_debug',),
+ 'Mhash': ('mhash_count',
+ 'mhash_get_block_size',
+ 'mhash_get_hash_name',
+ 'mhash_keygen_s2k',
+ 'mhash'),
+ 'Ming': ('ming_keypress',
+ 'ming_setcubicthreshold',
+ 'ming_setscale',
+ 'ming_setswfcompression',
+ 'ming_useconstants',
+ 'ming_useswfversion'),
+ 'Misc.': ('connection_aborted',
+ 'connection_status',
+ 'connection_timeout',
+ 'constant',
+ 'define',
+ 'defined',
+ 'die',
+ 'eval',
+ 'exit',
+ 'get_browser',
+ '__halt_compiler',
+ 'highlight_file',
+ 'highlight_string',
+ 'ignore_user_abort',
+ 'pack',
+ 'php_check_syntax',
+ 'php_strip_whitespace',
+ 'show_source',
+ 'sleep',
+ 'sys_getloadavg',
+ 'time_nanosleep',
+ 'time_sleep_until',
+ 'uniqid',
+ 'unpack',
+ 'usleep'),
+ 'Mongo': ('bson_decode', 'bson_encode'),
+ 'Msession': ('msession_connect',
+ 'msession_count',
+ 'msession_create',
+ 'msession_destroy',
+ 'msession_disconnect',
+ 'msession_find',
+ 'msession_get_array',
+ 'msession_get_data',
+ 'msession_get',
+ 'msession_inc',
+ 'msession_list',
+ 'msession_listvar',
+ 'msession_lock',
+ 'msession_plugin',
+ 'msession_randstr',
+ 'msession_set_array',
+ 'msession_set_data',
+ 'msession_set',
+ 'msession_timeout',
+ 'msession_uniq',
+ 'msession_unlock'),
+ 'Mssql': ('mssql_bind',
+ 'mssql_close',
+ 'mssql_connect',
+ 'mssql_data_seek',
+ 'mssql_execute',
+ 'mssql_fetch_array',
+ 'mssql_fetch_assoc',
+ 'mssql_fetch_batch',
+ 'mssql_fetch_field',
+ 'mssql_fetch_object',
+ 'mssql_fetch_row',
+ 'mssql_field_length',
+ 'mssql_field_name',
+ 'mssql_field_seek',
+ 'mssql_field_type',
+ 'mssql_free_result',
+ 'mssql_free_statement',
+ 'mssql_get_last_message',
+ 'mssql_guid_string',
+ 'mssql_init',
+ 'mssql_min_error_severity',
+ 'mssql_min_message_severity',
+ 'mssql_next_result',
+ 'mssql_num_fields',
+ 'mssql_num_rows',
+ 'mssql_pconnect',
+ 'mssql_query',
+ 'mssql_result',
+ 'mssql_rows_affected',
+ 'mssql_select_db'),
+ 'Multibyte String': ('mb_check_encoding',
+ 'mb_convert_case',
+ 'mb_convert_encoding',
+ 'mb_convert_kana',
+ 'mb_convert_variables',
+ 'mb_decode_mimeheader',
+ 'mb_decode_numericentity',
+ 'mb_detect_encoding',
+ 'mb_detect_order',
+ 'mb_encode_mimeheader',
+ 'mb_encode_numericentity',
+ 'mb_encoding_aliases',
+ 'mb_ereg_match',
+ 'mb_ereg_replace_callback',
+ 'mb_ereg_replace',
+ 'mb_ereg_search_getpos',
+ 'mb_ereg_search_getregs',
+ 'mb_ereg_search_init',
+ 'mb_ereg_search_pos',
+ 'mb_ereg_search_regs',
+ 'mb_ereg_search_setpos',
+ 'mb_ereg_search',
+ 'mb_ereg',
+ 'mb_eregi_replace',
+ 'mb_eregi',
+ 'mb_get_info',
+ 'mb_http_input',
+ 'mb_http_output',
+ 'mb_internal_encoding',
+ 'mb_language',
+ 'mb_list_encodings',
+ 'mb_output_handler',
+ 'mb_parse_str',
+ 'mb_preferred_mime_name',
+ 'mb_regex_encoding',
+ 'mb_regex_set_options',
+ 'mb_send_mail',
+ 'mb_split',
+ 'mb_strcut',
+ 'mb_strimwidth',
+ 'mb_stripos',
+ 'mb_stristr',
+ 'mb_strlen',
+ 'mb_strpos',
+ 'mb_strrchr',
+ 'mb_strrichr',
+ 'mb_strripos',
+ 'mb_strrpos',
+ 'mb_strstr',
+ 'mb_strtolower',
+ 'mb_strtoupper',
+ 'mb_strwidth',
+ 'mb_substitute_character',
+ 'mb_substr_count',
+ 'mb_substr'),
+ 'MySQL': ('mysql_affected_rows',
+ 'mysql_client_encoding',
+ 'mysql_close',
+ 'mysql_connect',
+ 'mysql_create_db',
+ 'mysql_data_seek',
+ 'mysql_db_name',
+ 'mysql_db_query',
+ 'mysql_drop_db',
+ 'mysql_errno',
+ 'mysql_error',
+ 'mysql_escape_string',
+ 'mysql_fetch_array',
+ 'mysql_fetch_assoc',
+ 'mysql_fetch_field',
+ 'mysql_fetch_lengths',
+ 'mysql_fetch_object',
+ 'mysql_fetch_row',
+ 'mysql_field_flags',
+ 'mysql_field_len',
+ 'mysql_field_name',
+ 'mysql_field_seek',
+ 'mysql_field_table',
+ 'mysql_field_type',
+ 'mysql_free_result',
+ 'mysql_get_client_info',
+ 'mysql_get_host_info',
+ 'mysql_get_proto_info',
+ 'mysql_get_server_info',
+ 'mysql_info',
+ 'mysql_insert_id',
+ 'mysql_list_dbs',
+ 'mysql_list_fields',
+ 'mysql_list_processes',
+ 'mysql_list_tables',
+ 'mysql_num_fields',
+ 'mysql_num_rows',
+ 'mysql_pconnect',
+ 'mysql_ping',
+ 'mysql_query',
+ 'mysql_real_escape_string',
+ 'mysql_result',
+ 'mysql_select_db',
+ 'mysql_set_charset',
+ 'mysql_stat',
+ 'mysql_tablename',
+ 'mysql_thread_id',
+ 'mysql_unbuffered_query'),
+ 'Mysqlnd_memcache': ('mysqlnd_memcache_get_config', 'mysqlnd_memcache_set'),
+ 'Mysqlnd_ms': ('mysqlnd_ms_dump_servers',
+ 'mysqlnd_ms_fabric_select_global',
+ 'mysqlnd_ms_fabric_select_shard',
+ 'mysqlnd_ms_get_last_gtid',
+ 'mysqlnd_ms_get_last_used_connection',
+ 'mysqlnd_ms_get_stats',
+ 'mysqlnd_ms_match_wild',
+ 'mysqlnd_ms_query_is_select',
+ 'mysqlnd_ms_set_qos',
+ 'mysqlnd_ms_set_user_pick_server'),
+ 'Mysqlnd_uh': ('mysqlnd_uh_convert_to_mysqlnd',
+ 'mysqlnd_uh_set_connection_proxy',
+ 'mysqlnd_uh_set_statement_proxy'),
+ 'NSAPI': ('nsapi_request_headers', 'nsapi_response_headers', 'nsapi_virtual'),
+ 'Ncurses': ('ncurses_addch',
+ 'ncurses_addchnstr',
+ 'ncurses_addchstr',
+ 'ncurses_addnstr',
+ 'ncurses_addstr',
+ 'ncurses_assume_default_colors',
+ 'ncurses_attroff',
+ 'ncurses_attron',
+ 'ncurses_attrset',
+ 'ncurses_baudrate',
+ 'ncurses_beep',
+ 'ncurses_bkgd',
+ 'ncurses_bkgdset',
+ 'ncurses_border',
+ 'ncurses_bottom_panel',
+ 'ncurses_can_change_color',
+ 'ncurses_cbreak',
+ 'ncurses_clear',
+ 'ncurses_clrtobot',
+ 'ncurses_clrtoeol',
+ 'ncurses_color_content',
+ 'ncurses_color_set',
+ 'ncurses_curs_set',
+ 'ncurses_def_prog_mode',
+ 'ncurses_def_shell_mode',
+ 'ncurses_define_key',
+ 'ncurses_del_panel',
+ 'ncurses_delay_output',
+ 'ncurses_delch',
+ 'ncurses_deleteln',
+ 'ncurses_delwin',
+ 'ncurses_doupdate',
+ 'ncurses_echo',
+ 'ncurses_echochar',
+ 'ncurses_end',
+ 'ncurses_erase',
+ 'ncurses_erasechar',
+ 'ncurses_filter',
+ 'ncurses_flash',
+ 'ncurses_flushinp',
+ 'ncurses_getch',
+ 'ncurses_getmaxyx',
+ 'ncurses_getmouse',
+ 'ncurses_getyx',
+ 'ncurses_halfdelay',
+ 'ncurses_has_colors',
+ 'ncurses_has_ic',
+ 'ncurses_has_il',
+ 'ncurses_has_key',
+ 'ncurses_hide_panel',
+ 'ncurses_hline',
+ 'ncurses_inch',
+ 'ncurses_init_color',
+ 'ncurses_init_pair',
+ 'ncurses_init',
+ 'ncurses_insch',
+ 'ncurses_insdelln',
+ 'ncurses_insertln',
+ 'ncurses_insstr',
+ 'ncurses_instr',
+ 'ncurses_isendwin',
+ 'ncurses_keyok',
+ 'ncurses_keypad',
+ 'ncurses_killchar',
+ 'ncurses_longname',
+ 'ncurses_meta',
+ 'ncurses_mouse_trafo',
+ 'ncurses_mouseinterval',
+ 'ncurses_mousemask',
+ 'ncurses_move_panel',
+ 'ncurses_move',
+ 'ncurses_mvaddch',
+ 'ncurses_mvaddchnstr',
+ 'ncurses_mvaddchstr',
+ 'ncurses_mvaddnstr',
+ 'ncurses_mvaddstr',
+ 'ncurses_mvcur',
+ 'ncurses_mvdelch',
+ 'ncurses_mvgetch',
+ 'ncurses_mvhline',
+ 'ncurses_mvinch',
+ 'ncurses_mvvline',
+ 'ncurses_mvwaddstr',
+ 'ncurses_napms',
+ 'ncurses_new_panel',
+ 'ncurses_newpad',
+ 'ncurses_newwin',
+ 'ncurses_nl',
+ 'ncurses_nocbreak',
+ 'ncurses_noecho',
+ 'ncurses_nonl',
+ 'ncurses_noqiflush',
+ 'ncurses_noraw',
+ 'ncurses_pair_content',
+ 'ncurses_panel_above',
+ 'ncurses_panel_below',
+ 'ncurses_panel_window',
+ 'ncurses_pnoutrefresh',
+ 'ncurses_prefresh',
+ 'ncurses_putp',
+ 'ncurses_qiflush',
+ 'ncurses_raw',
+ 'ncurses_refresh',
+ 'ncurses_replace_panel',
+ 'ncurses_reset_prog_mode',
+ 'ncurses_reset_shell_mode',
+ 'ncurses_resetty',
+ 'ncurses_savetty',
+ 'ncurses_scr_dump',
+ 'ncurses_scr_init',
+ 'ncurses_scr_restore',
+ 'ncurses_scr_set',
+ 'ncurses_scrl',
+ 'ncurses_show_panel',
+ 'ncurses_slk_attr',
+ 'ncurses_slk_attroff',
+ 'ncurses_slk_attron',
+ 'ncurses_slk_attrset',
+ 'ncurses_slk_clear',
+ 'ncurses_slk_color',
+ 'ncurses_slk_init',
+ 'ncurses_slk_noutrefresh',
+ 'ncurses_slk_refresh',
+ 'ncurses_slk_restore',
+ 'ncurses_slk_set',
+ 'ncurses_slk_touch',
+ 'ncurses_standend',
+ 'ncurses_standout',
+ 'ncurses_start_color',
+ 'ncurses_termattrs',
+ 'ncurses_termname',
+ 'ncurses_timeout',
+ 'ncurses_top_panel',
+ 'ncurses_typeahead',
+ 'ncurses_ungetch',
+ 'ncurses_ungetmouse',
+ 'ncurses_update_panels',
+ 'ncurses_use_default_colors',
+ 'ncurses_use_env',
+ 'ncurses_use_extended_names',
+ 'ncurses_vidattr',
+ 'ncurses_vline',
+ 'ncurses_waddch',
+ 'ncurses_waddstr',
+ 'ncurses_wattroff',
+ 'ncurses_wattron',
+ 'ncurses_wattrset',
+ 'ncurses_wborder',
+ 'ncurses_wclear',
+ 'ncurses_wcolor_set',
+ 'ncurses_werase',
+ 'ncurses_wgetch',
+ 'ncurses_whline',
+ 'ncurses_wmouse_trafo',
+ 'ncurses_wmove',
+ 'ncurses_wnoutrefresh',
+ 'ncurses_wrefresh',
+ 'ncurses_wstandend',
+ 'ncurses_wstandout',
+ 'ncurses_wvline'),
+ 'Network': ('checkdnsrr',
+ 'closelog',
+ 'define_syslog_variables',
+ 'dns_check_record',
+ 'dns_get_mx',
+ 'dns_get_record',
+ 'fsockopen',
+ 'gethostbyaddr',
+ 'gethostbyname',
+ 'gethostbynamel',
+ 'gethostname',
+ 'getmxrr',
+ 'getprotobyname',
+ 'getprotobynumber',
+ 'getservbyname',
+ 'getservbyport',
+ 'header_register_callback',
+ 'header_remove',
+ 'header',
+ 'headers_list',
+ 'headers_sent',
+ 'http_response_code',
+ 'inet_ntop',
+ 'inet_pton',
+ 'ip2long',
+ 'long2ip',
+ 'openlog',
+ 'pfsockopen',
+ 'setcookie',
+ 'setrawcookie',
+ 'socket_get_status',
+ 'socket_set_blocking',
+ 'socket_set_timeout',
+ 'syslog'),
+ 'Newt': ('newt_bell',
+ 'newt_button_bar',
+ 'newt_button',
+ 'newt_centered_window',
+ 'newt_checkbox_get_value',
+ 'newt_checkbox_set_flags',
+ 'newt_checkbox_set_value',
+ 'newt_checkbox_tree_add_item',
+ 'newt_checkbox_tree_find_item',
+ 'newt_checkbox_tree_get_current',
+ 'newt_checkbox_tree_get_entry_value',
+ 'newt_checkbox_tree_get_multi_selection',
+ 'newt_checkbox_tree_get_selection',
+ 'newt_checkbox_tree_multi',
+ 'newt_checkbox_tree_set_current',
+ 'newt_checkbox_tree_set_entry_value',
+ 'newt_checkbox_tree_set_entry',
+ 'newt_checkbox_tree_set_width',
+ 'newt_checkbox_tree',
+ 'newt_checkbox',
+ 'newt_clear_key_buffer',
+ 'newt_cls',
+ 'newt_compact_button',
+ 'newt_component_add_callback',
+ 'newt_component_takes_focus',
+ 'newt_create_grid',
+ 'newt_cursor_off',
+ 'newt_cursor_on',
+ 'newt_delay',
+ 'newt_draw_form',
+ 'newt_draw_root_text',
+ 'newt_entry_get_value',
+ 'newt_entry_set_filter',
+ 'newt_entry_set_flags',
+ 'newt_entry_set',
+ 'newt_entry',
+ 'newt_finished',
+ 'newt_form_add_component',
+ 'newt_form_add_components',
+ 'newt_form_add_hot_key',
+ 'newt_form_destroy',
+ 'newt_form_get_current',
+ 'newt_form_run',
+ 'newt_form_set_background',
+ 'newt_form_set_height',
+ 'newt_form_set_size',
+ 'newt_form_set_timer',
+ 'newt_form_set_width',
+ 'newt_form_watch_fd',
+ 'newt_form',
+ 'newt_get_screen_size',
+ 'newt_grid_add_components_to_form',
+ 'newt_grid_basic_window',
+ 'newt_grid_free',
+ 'newt_grid_get_size',
+ 'newt_grid_h_close_stacked',
+ 'newt_grid_h_stacked',
+ 'newt_grid_place',
+ 'newt_grid_set_field',
+ 'newt_grid_simple_window',
+ 'newt_grid_v_close_stacked',
+ 'newt_grid_v_stacked',
+ 'newt_grid_wrapped_window_at',
+ 'newt_grid_wrapped_window',
+ 'newt_init',
+ 'newt_label_set_text',
+ 'newt_label',
+ 'newt_listbox_append_entry',
+ 'newt_listbox_clear_selection',
+ 'newt_listbox_clear',
+ 'newt_listbox_delete_entry',
+ 'newt_listbox_get_current',
+ 'newt_listbox_get_selection',
+ 'newt_listbox_insert_entry',
+ 'newt_listbox_item_count',
+ 'newt_listbox_select_item',
+ 'newt_listbox_set_current_by_key',
+ 'newt_listbox_set_current',
+ 'newt_listbox_set_data',
+ 'newt_listbox_set_entry',
+ 'newt_listbox_set_width',
+ 'newt_listbox',
+ 'newt_listitem_get_data',
+ 'newt_listitem_set',
+ 'newt_listitem',
+ 'newt_open_window',
+ 'newt_pop_help_line',
+ 'newt_pop_window',
+ 'newt_push_help_line',
+ 'newt_radio_get_current',
+ 'newt_radiobutton',
+ 'newt_redraw_help_line',
+ 'newt_reflow_text',
+ 'newt_refresh',
+ 'newt_resize_screen',
+ 'newt_resume',
+ 'newt_run_form',
+ 'newt_scale_set',
+ 'newt_scale',
+ 'newt_scrollbar_set',
+ 'newt_set_help_callback',
+ 'newt_set_suspend_callback',
+ 'newt_suspend',
+ 'newt_textbox_get_num_lines',
+ 'newt_textbox_reflowed',
+ 'newt_textbox_set_height',
+ 'newt_textbox_set_text',
+ 'newt_textbox',
+ 'newt_vertical_scrollbar',
+ 'newt_wait_for_key',
+ 'newt_win_choice',
+ 'newt_win_entries',
+ 'newt_win_menu',
+ 'newt_win_message',
+ 'newt_win_messagev',
+ 'newt_win_ternary'),
+ 'OAuth': ('oauth_get_sbs', 'oauth_urlencode'),
+ 'OCI8': ('oci_bind_array_by_name',
+ 'oci_bind_by_name',
+ 'oci_cancel',
+ 'oci_client_version',
+ 'oci_close',
+ 'oci_commit',
+ 'oci_connect',
+ 'oci_define_by_name',
+ 'oci_error',
+ 'oci_execute',
+ 'oci_fetch_all',
+ 'oci_fetch_array',
+ 'oci_fetch_assoc',
+ 'oci_fetch_object',
+ 'oci_fetch_row',
+ 'oci_fetch',
+ 'oci_field_is_null',
+ 'oci_field_name',
+ 'oci_field_precision',
+ 'oci_field_scale',
+ 'oci_field_size',
+ 'oci_field_type_raw',
+ 'oci_field_type',
+ 'oci_free_descriptor',
+ 'oci_free_statement',
+ 'oci_get_implicit_resultset',
+ 'oci_internal_debug',
+ 'oci_lob_copy',
+ 'oci_lob_is_equal',
+ 'oci_new_collection',
+ 'oci_new_connect',
+ 'oci_new_cursor',
+ 'oci_new_descriptor',
+ 'oci_num_fields',
+ 'oci_num_rows',
+ 'oci_parse',
+ 'oci_password_change',
+ 'oci_pconnect',
+ 'oci_result',
+ 'oci_rollback',
+ 'oci_server_version',
+ 'oci_set_action',
+ 'oci_set_client_identifier',
+ 'oci_set_client_info',
+ 'oci_set_edition',
+ 'oci_set_module_name',
+ 'oci_set_prefetch',
+ 'oci_statement_type'),
+ 'ODBC': ('odbc_autocommit',
+ 'odbc_binmode',
+ 'odbc_close_all',
+ 'odbc_close',
+ 'odbc_columnprivileges',
+ 'odbc_columns',
+ 'odbc_commit',
+ 'odbc_connect',
+ 'odbc_cursor',
+ 'odbc_data_source',
+ 'odbc_do',
+ 'odbc_error',
+ 'odbc_errormsg',
+ 'odbc_exec',
+ 'odbc_execute',
+ 'odbc_fetch_array',
+ 'odbc_fetch_into',
+ 'odbc_fetch_object',
+ 'odbc_fetch_row',
+ 'odbc_field_len',
+ 'odbc_field_name',
+ 'odbc_field_num',
+ 'odbc_field_precision',
+ 'odbc_field_scale',
+ 'odbc_field_type',
+ 'odbc_foreignkeys',
+ 'odbc_free_result',
+ 'odbc_gettypeinfo',
+ 'odbc_longreadlen',
+ 'odbc_next_result',
+ 'odbc_num_fields',
+ 'odbc_num_rows',
+ 'odbc_pconnect',
+ 'odbc_prepare',
+ 'odbc_primarykeys',
+ 'odbc_procedurecolumns',
+ 'odbc_procedures',
+ 'odbc_result_all',
+ 'odbc_result',
+ 'odbc_rollback',
+ 'odbc_setoption',
+ 'odbc_specialcolumns',
+ 'odbc_statistics',
+ 'odbc_tableprivileges',
+ 'odbc_tables'),
+ 'OPcache': ('opcache_compile_file',
+ 'opcache_get_configuration',
+ 'opcache_get_status',
+ 'opcache_invalidate',
+ 'opcache_reset'),
+ 'Object Aggregation': ('aggregate_info',
+ 'aggregate_methods_by_list',
+ 'aggregate_methods_by_regexp',
+ 'aggregate_methods',
+ 'aggregate_properties_by_list',
+ 'aggregate_properties_by_regexp',
+ 'aggregate_properties',
+ 'aggregate',
+ 'aggregation_info',
+ 'deaggregate'),
+ 'OpenAL': ('openal_buffer_create',
+ 'openal_buffer_data',
+ 'openal_buffer_destroy',
+ 'openal_buffer_get',
+ 'openal_buffer_loadwav',
+ 'openal_context_create',
+ 'openal_context_current',
+ 'openal_context_destroy',
+ 'openal_context_process',
+ 'openal_context_suspend',
+ 'openal_device_close',
+ 'openal_device_open',
+ 'openal_listener_get',
+ 'openal_listener_set',
+ 'openal_source_create',
+ 'openal_source_destroy',
+ 'openal_source_get',
+ 'openal_source_pause',
+ 'openal_source_play',
+ 'openal_source_rewind',
+ 'openal_source_set',
+ 'openal_source_stop',
+ 'openal_stream'),
+ 'OpenSSL': ('openssl_cipher_iv_length',
+ 'openssl_csr_export_to_file',
+ 'openssl_csr_export',
+ 'openssl_csr_get_public_key',
+ 'openssl_csr_get_subject',
+ 'openssl_csr_new',
+ 'openssl_csr_sign',
+ 'openssl_decrypt',
+ 'openssl_dh_compute_key',
+ 'openssl_digest',
+ 'openssl_encrypt',
+ 'openssl_error_string',
+ 'openssl_free_key',
+ 'openssl_get_cipher_methods',
+ 'openssl_get_md_methods',
+ 'openssl_get_privatekey',
+ 'openssl_get_publickey',
+ 'openssl_open',
+ 'openssl_pbkdf2',
+ 'openssl_pkcs12_export_to_file',
+ 'openssl_pkcs12_export',
+ 'openssl_pkcs12_read',
+ 'openssl_pkcs7_decrypt',
+ 'openssl_pkcs7_encrypt',
+ 'openssl_pkcs7_sign',
+ 'openssl_pkcs7_verify',
+ 'openssl_pkey_export_to_file',
+ 'openssl_pkey_export',
+ 'openssl_pkey_free',
+ 'openssl_pkey_get_details',
+ 'openssl_pkey_get_private',
+ 'openssl_pkey_get_public',
+ 'openssl_pkey_new',
+ 'openssl_private_decrypt',
+ 'openssl_private_encrypt',
+ 'openssl_public_decrypt',
+ 'openssl_public_encrypt',
+ 'openssl_random_pseudo_bytes',
+ 'openssl_seal',
+ 'openssl_sign',
+ 'openssl_spki_export_challenge',
+ 'openssl_spki_export',
+ 'openssl_spki_new',
+ 'openssl_spki_verify',
+ 'openssl_verify',
+ 'openssl_x509_check_private_key',
+ 'openssl_x509_checkpurpose',
+ 'openssl_x509_export_to_file',
+ 'openssl_x509_export',
+ 'openssl_x509_free',
+ 'openssl_x509_parse',
+ 'openssl_x509_read'),
+ 'Output Control': ('flush',
+ 'ob_clean',
+ 'ob_end_clean',
+ 'ob_end_flush',
+ 'ob_flush',
+ 'ob_get_clean',
+ 'ob_get_contents',
+ 'ob_get_flush',
+ 'ob_get_length',
+ 'ob_get_level',
+ 'ob_get_status',
+ 'ob_gzhandler',
+ 'ob_implicit_flush',
+ 'ob_list_handlers',
+ 'ob_start',
+ 'output_add_rewrite_var',
+ 'output_reset_rewrite_vars'),
+ 'Ovrimos SQL': ('ovrimos_close',
+ 'ovrimos_commit',
+ 'ovrimos_connect',
+ 'ovrimos_cursor',
+ 'ovrimos_exec',
+ 'ovrimos_execute',
+ 'ovrimos_fetch_into',
+ 'ovrimos_fetch_row',
+ 'ovrimos_field_len',
+ 'ovrimos_field_name',
+ 'ovrimos_field_num',
+ 'ovrimos_field_type',
+ 'ovrimos_free_result',
+ 'ovrimos_longreadlen',
+ 'ovrimos_num_fields',
+ 'ovrimos_num_rows',
+ 'ovrimos_prepare',
+ 'ovrimos_result_all',
+ 'ovrimos_result',
+ 'ovrimos_rollback'),
+ 'PCNTL': ('pcntl_alarm',
+ 'pcntl_errno',
+ 'pcntl_exec',
+ 'pcntl_fork',
+ 'pcntl_get_last_error',
+ 'pcntl_getpriority',
+ 'pcntl_setpriority',
+ 'pcntl_signal_dispatch',
+ 'pcntl_signal',
+ 'pcntl_sigprocmask',
+ 'pcntl_sigtimedwait',
+ 'pcntl_sigwaitinfo',
+ 'pcntl_strerror',
+ 'pcntl_wait',
+ 'pcntl_waitpid',
+ 'pcntl_wexitstatus',
+ 'pcntl_wifexited',
+ 'pcntl_wifsignaled',
+ 'pcntl_wifstopped',
+ 'pcntl_wstopsig',
+ 'pcntl_wtermsig'),
+ 'PCRE': ('preg_filter',
+ 'preg_grep',
+ 'preg_last_error',
+ 'preg_match_all',
+ 'preg_match',
+ 'preg_quote',
+ 'preg_replace_callback',
+ 'preg_replace',
+ 'preg_split'),
+ 'PDF': ('PDF_activate_item',
+ 'PDF_add_annotation',
+ 'PDF_add_bookmark',
+ 'PDF_add_launchlink',
+ 'PDF_add_locallink',
+ 'PDF_add_nameddest',
+ 'PDF_add_note',
+ 'PDF_add_outline',
+ 'PDF_add_pdflink',
+ 'PDF_add_table_cell',
+ 'PDF_add_textflow',
+ 'PDF_add_thumbnail',
+ 'PDF_add_weblink',
+ 'PDF_arc',
+ 'PDF_arcn',
+ 'PDF_attach_file',
+ 'PDF_begin_document',
+ 'PDF_begin_font',
+ 'PDF_begin_glyph',
+ 'PDF_begin_item',
+ 'PDF_begin_layer',
+ 'PDF_begin_page_ext',
+ 'PDF_begin_page',
+ 'PDF_begin_pattern',
+ 'PDF_begin_template_ext',
+ 'PDF_begin_template',
+ 'PDF_circle',
+ 'PDF_clip',
+ 'PDF_close_image',
+ 'PDF_close_pdi_page',
+ 'PDF_close_pdi',
+ 'PDF_close',
+ 'PDF_closepath_fill_stroke',
+ 'PDF_closepath_stroke',
+ 'PDF_closepath',
+ 'PDF_concat',
+ 'PDF_continue_text',
+ 'PDF_create_3dview',
+ 'PDF_create_action',
+ 'PDF_create_annotation',
+ 'PDF_create_bookmark',
+ 'PDF_create_field',
+ 'PDF_create_fieldgroup',
+ 'PDF_create_gstate',
+ 'PDF_create_pvf',
+ 'PDF_create_textflow',
+ 'PDF_curveto',
+ 'PDF_define_layer',
+ 'PDF_delete_pvf',
+ 'PDF_delete_table',
+ 'PDF_delete_textflow',
+ 'PDF_delete',
+ 'PDF_encoding_set_char',
+ 'PDF_end_document',
+ 'PDF_end_font',
+ 'PDF_end_glyph',
+ 'PDF_end_item',
+ 'PDF_end_layer',
+ 'PDF_end_page_ext',
+ 'PDF_end_page',
+ 'PDF_end_pattern',
+ 'PDF_end_template',
+ 'PDF_endpath',
+ 'PDF_fill_imageblock',
+ 'PDF_fill_pdfblock',
+ 'PDF_fill_stroke',
+ 'PDF_fill_textblock',
+ 'PDF_fill',
+ 'PDF_findfont',
+ 'PDF_fit_image',
+ 'PDF_fit_pdi_page',
+ 'PDF_fit_table',
+ 'PDF_fit_textflow',
+ 'PDF_fit_textline',
+ 'PDF_get_apiname',
+ 'PDF_get_buffer',
+ 'PDF_get_errmsg',
+ 'PDF_get_errnum',
+ 'PDF_get_font',
+ 'PDF_get_fontname',
+ 'PDF_get_fontsize',
+ 'PDF_get_image_height',
+ 'PDF_get_image_width',
+ 'PDF_get_majorversion',
+ 'PDF_get_minorversion',
+ 'PDF_get_parameter',
+ 'PDF_get_pdi_parameter',
+ 'PDF_get_pdi_value',
+ 'PDF_get_value',
+ 'PDF_info_font',
+ 'PDF_info_matchbox',
+ 'PDF_info_table',
+ 'PDF_info_textflow',
+ 'PDF_info_textline',
+ 'PDF_initgraphics',
+ 'PDF_lineto',
+ 'PDF_load_3ddata',
+ 'PDF_load_font',
+ 'PDF_load_iccprofile',
+ 'PDF_load_image',
+ 'PDF_makespotcolor',
+ 'PDF_moveto',
+ 'PDF_new',
+ 'PDF_open_ccitt',
+ 'PDF_open_file',
+ 'PDF_open_gif',
+ 'PDF_open_image_file',
+ 'PDF_open_image',
+ 'PDF_open_jpeg',
+ 'PDF_open_memory_image',
+ 'PDF_open_pdi_document',
+ 'PDF_open_pdi_page',
+ 'PDF_open_pdi',
+ 'PDF_open_tiff',
+ 'PDF_pcos_get_number',
+ 'PDF_pcos_get_stream',
+ 'PDF_pcos_get_string',
+ 'PDF_place_image',
+ 'PDF_place_pdi_page',
+ 'PDF_process_pdi',
+ 'PDF_rect',
+ 'PDF_restore',
+ 'PDF_resume_page',
+ 'PDF_rotate',
+ 'PDF_save',
+ 'PDF_scale',
+ 'PDF_set_border_color',
+ 'PDF_set_border_dash',
+ 'PDF_set_border_style',
+ 'PDF_set_char_spacing',
+ 'PDF_set_duration',
+ 'PDF_set_gstate',
+ 'PDF_set_horiz_scaling',
+ 'PDF_set_info_author',
+ 'PDF_set_info_creator',
+ 'PDF_set_info_keywords',
+ 'PDF_set_info_subject',
+ 'PDF_set_info_title',
+ 'PDF_set_info',
+ 'PDF_set_layer_dependency',
+ 'PDF_set_leading',
+ 'PDF_set_parameter',
+ 'PDF_set_text_matrix',
+ 'PDF_set_text_pos',
+ 'PDF_set_text_rendering',
+ 'PDF_set_text_rise',
+ 'PDF_set_value',
+ 'PDF_set_word_spacing',
+ 'PDF_setcolor',
+ 'PDF_setdash',
+ 'PDF_setdashpattern',
+ 'PDF_setflat',
+ 'PDF_setfont',
+ 'PDF_setgray_fill',
+ 'PDF_setgray_stroke',
+ 'PDF_setgray',
+ 'PDF_setlinecap',
+ 'PDF_setlinejoin',
+ 'PDF_setlinewidth',
+ 'PDF_setmatrix',
+ 'PDF_setmiterlimit',
+ 'PDF_setpolydash',
+ 'PDF_setrgbcolor_fill',
+ 'PDF_setrgbcolor_stroke',
+ 'PDF_setrgbcolor',
+ 'PDF_shading_pattern',
+ 'PDF_shading',
+ 'PDF_shfill',
+ 'PDF_show_boxed',
+ 'PDF_show_xy',
+ 'PDF_show',
+ 'PDF_skew',
+ 'PDF_stringwidth',
+ 'PDF_stroke',
+ 'PDF_suspend_page',
+ 'PDF_translate',
+ 'PDF_utf16_to_utf8',
+ 'PDF_utf32_to_utf16',
+ 'PDF_utf8_to_utf16'),
+ 'PHP Options/Info': ('assert_options',
+ 'assert',
+ 'cli_get_process_title',
+ 'cli_set_process_title',
+ 'dl',
+ 'extension_loaded',
+ 'gc_collect_cycles',
+ 'gc_disable',
+ 'gc_enable',
+ 'gc_enabled',
+ 'get_cfg_var',
+ 'get_current_user',
+ 'get_defined_constants',
+ 'get_extension_funcs',
+ 'get_include_path',
+ 'get_included_files',
+ 'get_loaded_extensions',
+ 'get_magic_quotes_gpc',
+ 'get_magic_quotes_runtime',
+ 'get_required_files',
+ 'getenv',
+ 'getlastmod',
+ 'getmygid',
+ 'getmyinode',
+ 'getmypid',
+ 'getmyuid',
+ 'getopt',
+ 'getrusage',
+ 'ini_alter',
+ 'ini_get_all',
+ 'ini_get',
+ 'ini_restore',
+ 'ini_set',
+ 'magic_quotes_runtime',
+ 'memory_get_peak_usage',
+ 'memory_get_usage',
+ 'php_ini_loaded_file',
+ 'php_ini_scanned_files',
+ 'php_logo_guid',
+ 'php_sapi_name',
+ 'php_uname',
+ 'phpcredits',
+ 'phpinfo',
+ 'phpversion',
+ 'putenv',
+ 'restore_include_path',
+ 'set_include_path',
+ 'set_magic_quotes_runtime',
+ 'set_time_limit',
+ 'sys_get_temp_dir',
+ 'version_compare',
+ 'zend_logo_guid',
+ 'zend_thread_id',
+ 'zend_version'),
+ 'POSIX': ('posix_access',
+ 'posix_ctermid',
+ 'posix_errno',
+ 'posix_get_last_error',
+ 'posix_getcwd',
+ 'posix_getegid',
+ 'posix_geteuid',
+ 'posix_getgid',
+ 'posix_getgrgid',
+ 'posix_getgrnam',
+ 'posix_getgroups',
+ 'posix_getlogin',
+ 'posix_getpgid',
+ 'posix_getpgrp',
+ 'posix_getpid',
+ 'posix_getppid',
+ 'posix_getpwnam',
+ 'posix_getpwuid',
+ 'posix_getrlimit',
+ 'posix_getsid',
+ 'posix_getuid',
+ 'posix_initgroups',
+ 'posix_isatty',
+ 'posix_kill',
+ 'posix_mkfifo',
+ 'posix_mknod',
+ 'posix_setegid',
+ 'posix_seteuid',
+ 'posix_setgid',
+ 'posix_setpgid',
+ 'posix_setsid',
+ 'posix_setuid',
+ 'posix_strerror',
+ 'posix_times',
+ 'posix_ttyname',
+ 'posix_uname'),
+ 'POSIX Regex': ('ereg_replace',
+ 'ereg',
+ 'eregi_replace',
+ 'eregi',
+ 'split',
+ 'spliti',
+ 'sql_regcase'),
+ 'PS': ('ps_add_bookmark',
+ 'ps_add_launchlink',
+ 'ps_add_locallink',
+ 'ps_add_note',
+ 'ps_add_pdflink',
+ 'ps_add_weblink',
+ 'ps_arc',
+ 'ps_arcn',
+ 'ps_begin_page',
+ 'ps_begin_pattern',
+ 'ps_begin_template',
+ 'ps_circle',
+ 'ps_clip',
+ 'ps_close_image',
+ 'ps_close',
+ 'ps_closepath_stroke',
+ 'ps_closepath',
+ 'ps_continue_text',
+ 'ps_curveto',
+ 'ps_delete',
+ 'ps_end_page',
+ 'ps_end_pattern',
+ 'ps_end_template',
+ 'ps_fill_stroke',
+ 'ps_fill',
+ 'ps_findfont',
+ 'ps_get_buffer',
+ 'ps_get_parameter',
+ 'ps_get_value',
+ 'ps_hyphenate',
+ 'ps_include_file',
+ 'ps_lineto',
+ 'ps_makespotcolor',
+ 'ps_moveto',
+ 'ps_new',
+ 'ps_open_file',
+ 'ps_open_image_file',
+ 'ps_open_image',
+ 'ps_open_memory_image',
+ 'ps_place_image',
+ 'ps_rect',
+ 'ps_restore',
+ 'ps_rotate',
+ 'ps_save',
+ 'ps_scale',
+ 'ps_set_border_color',
+ 'ps_set_border_dash',
+ 'ps_set_border_style',
+ 'ps_set_info',
+ 'ps_set_parameter',
+ 'ps_set_text_pos',
+ 'ps_set_value',
+ 'ps_setcolor',
+ 'ps_setdash',
+ 'ps_setflat',
+ 'ps_setfont',
+ 'ps_setgray',
+ 'ps_setlinecap',
+ 'ps_setlinejoin',
+ 'ps_setlinewidth',
+ 'ps_setmiterlimit',
+ 'ps_setoverprintmode',
+ 'ps_setpolydash',
+ 'ps_shading_pattern',
+ 'ps_shading',
+ 'ps_shfill',
+ 'ps_show_boxed',
+ 'ps_show_xy2',
+ 'ps_show_xy',
+ 'ps_show2',
+ 'ps_show',
+ 'ps_string_geometry',
+ 'ps_stringwidth',
+ 'ps_stroke',
+ 'ps_symbol_name',
+ 'ps_symbol_width',
+ 'ps_symbol',
+ 'ps_translate'),
+ 'Paradox': ('px_close',
+ 'px_create_fp',
+ 'px_date2string',
+ 'px_delete_record',
+ 'px_delete',
+ 'px_get_field',
+ 'px_get_info',
+ 'px_get_parameter',
+ 'px_get_record',
+ 'px_get_schema',
+ 'px_get_value',
+ 'px_insert_record',
+ 'px_new',
+ 'px_numfields',
+ 'px_numrecords',
+ 'px_open_fp',
+ 'px_put_record',
+ 'px_retrieve_record',
+ 'px_set_blob_file',
+ 'px_set_parameter',
+ 'px_set_tablename',
+ 'px_set_targetencoding',
+ 'px_set_value',
+ 'px_timestamp2string',
+ 'px_update_record'),
+ 'Parsekit': ('parsekit_compile_file',
+ 'parsekit_compile_string',
+ 'parsekit_func_arginfo'),
+ 'Password Hashing': ('password_get_info',
+ 'password_hash',
+ 'password_needs_rehash',
+ 'password_verify'),
+ 'PostgreSQL': ('pg_affected_rows',
+ 'pg_cancel_query',
+ 'pg_client_encoding',
+ 'pg_close',
+ 'pg_connect',
+ 'pg_connection_busy',
+ 'pg_connection_reset',
+ 'pg_connection_status',
+ 'pg_convert',
+ 'pg_copy_from',
+ 'pg_copy_to',
+ 'pg_dbname',
+ 'pg_delete',
+ 'pg_end_copy',
+ 'pg_escape_bytea',
+ 'pg_escape_identifier',
+ 'pg_escape_literal',
+ 'pg_escape_string',
+ 'pg_execute',
+ 'pg_fetch_all_columns',
+ 'pg_fetch_all',
+ 'pg_fetch_array',
+ 'pg_fetch_assoc',
+ 'pg_fetch_object',
+ 'pg_fetch_result',
+ 'pg_fetch_row',
+ 'pg_field_is_null',
+ 'pg_field_name',
+ 'pg_field_num',
+ 'pg_field_prtlen',
+ 'pg_field_size',
+ 'pg_field_table',
+ 'pg_field_type_oid',
+ 'pg_field_type',
+ 'pg_free_result',
+ 'pg_get_notify',
+ 'pg_get_pid',
+ 'pg_get_result',
+ 'pg_host',
+ 'pg_insert',
+ 'pg_last_error',
+ 'pg_last_notice',
+ 'pg_last_oid',
+ 'pg_lo_close',
+ 'pg_lo_create',
+ 'pg_lo_export',
+ 'pg_lo_import',
+ 'pg_lo_open',
+ 'pg_lo_read_all',
+ 'pg_lo_read',
+ 'pg_lo_seek',
+ 'pg_lo_tell',
+ 'pg_lo_truncate',
+ 'pg_lo_unlink',
+ 'pg_lo_write',
+ 'pg_meta_data',
+ 'pg_num_fields',
+ 'pg_num_rows',
+ 'pg_options',
+ 'pg_parameter_status',
+ 'pg_pconnect',
+ 'pg_ping',
+ 'pg_port',
+ 'pg_prepare',
+ 'pg_put_line',
+ 'pg_query_params',
+ 'pg_query',
+ 'pg_result_error_field',
+ 'pg_result_error',
+ 'pg_result_seek',
+ 'pg_result_status',
+ 'pg_select',
+ 'pg_send_execute',
+ 'pg_send_prepare',
+ 'pg_send_query_params',
+ 'pg_send_query',
+ 'pg_set_client_encoding',
+ 'pg_set_error_verbosity',
+ 'pg_trace',
+ 'pg_transaction_status',
+ 'pg_tty',
+ 'pg_unescape_bytea',
+ 'pg_untrace',
+ 'pg_update',
+ 'pg_version'),
+ 'Printer': ('printer_abort',
+ 'printer_close',
+ 'printer_create_brush',
+ 'printer_create_dc',
+ 'printer_create_font',
+ 'printer_create_pen',
+ 'printer_delete_brush',
+ 'printer_delete_dc',
+ 'printer_delete_font',
+ 'printer_delete_pen',
+ 'printer_draw_bmp',
+ 'printer_draw_chord',
+ 'printer_draw_elipse',
+ 'printer_draw_line',
+ 'printer_draw_pie',
+ 'printer_draw_rectangle',
+ 'printer_draw_roundrect',
+ 'printer_draw_text',
+ 'printer_end_doc',
+ 'printer_end_page',
+ 'printer_get_option',
+ 'printer_list',
+ 'printer_logical_fontheight',
+ 'printer_open',
+ 'printer_select_brush',
+ 'printer_select_font',
+ 'printer_select_pen',
+ 'printer_set_option',
+ 'printer_start_doc',
+ 'printer_start_page',
+ 'printer_write'),
+ 'Proctitle': ('setproctitle', 'setthreadtitle'),
+ 'Program execution': ('escapeshellarg',
+ 'escapeshellcmd',
+ 'exec',
+ 'passthru',
+ 'proc_close',
+ 'proc_get_status',
+ 'proc_nice',
+ 'proc_open',
+ 'proc_terminate',
+ 'shell_exec',
+ 'system'),
+ 'Pspell': ('pspell_add_to_personal',
+ 'pspell_add_to_session',
+ 'pspell_check',
+ 'pspell_clear_session',
+ 'pspell_config_create',
+ 'pspell_config_data_dir',
+ 'pspell_config_dict_dir',
+ 'pspell_config_ignore',
+ 'pspell_config_mode',
+ 'pspell_config_personal',
+ 'pspell_config_repl',
+ 'pspell_config_runtogether',
+ 'pspell_config_save_repl',
+ 'pspell_new_config',
+ 'pspell_new_personal',
+ 'pspell_new',
+ 'pspell_save_wordlist',
+ 'pspell_store_replacement',
+ 'pspell_suggest'),
+ 'RPM Reader': ('rpm_close',
+ 'rpm_get_tag',
+ 'rpm_is_valid',
+ 'rpm_open',
+ 'rpm_version'),
+ 'RRD': ('rrd_create',
+ 'rrd_error',
+ 'rrd_fetch',
+ 'rrd_first',
+ 'rrd_graph',
+ 'rrd_info',
+ 'rrd_last',
+ 'rrd_lastupdate',
+ 'rrd_restore',
+ 'rrd_tune',
+ 'rrd_update',
+ 'rrd_version',
+ 'rrd_xport',
+ 'rrdc_disconnect'),
+ 'Radius': ('radius_acct_open',
+ 'radius_add_server',
+ 'radius_auth_open',
+ 'radius_close',
+ 'radius_config',
+ 'radius_create_request',
+ 'radius_cvt_addr',
+ 'radius_cvt_int',
+ 'radius_cvt_string',
+ 'radius_demangle_mppe_key',
+ 'radius_demangle',
+ 'radius_get_attr',
+ 'radius_get_tagged_attr_data',
+ 'radius_get_tagged_attr_tag',
+ 'radius_get_vendor_attr',
+ 'radius_put_addr',
+ 'radius_put_attr',
+ 'radius_put_int',
+ 'radius_put_string',
+ 'radius_put_vendor_addr',
+ 'radius_put_vendor_attr',
+ 'radius_put_vendor_int',
+ 'radius_put_vendor_string',
+ 'radius_request_authenticator',
+ 'radius_salt_encrypt_attr',
+ 'radius_send_request',
+ 'radius_server_secret',
+ 'radius_strerror'),
+ 'Rar': ('rar_wrapper_cache_stats',),
+ 'Readline': ('readline_add_history',
+ 'readline_callback_handler_install',
+ 'readline_callback_handler_remove',
+ 'readline_callback_read_char',
+ 'readline_clear_history',
+ 'readline_completion_function',
+ 'readline_info',
+ 'readline_list_history',
+ 'readline_on_new_line',
+ 'readline_read_history',
+ 'readline_redisplay',
+ 'readline_write_history',
+ 'readline'),
+ 'Recode': ('recode_file', 'recode_string', 'recode'),
+ 'SNMP': ('snmp_get_quick_print',
+ 'snmp_get_valueretrieval',
+ 'snmp_read_mib',
+ 'snmp_set_enum_print',
+ 'snmp_set_oid_numeric_print',
+ 'snmp_set_oid_output_format',
+ 'snmp_set_quick_print',
+ 'snmp_set_valueretrieval',
+ 'snmp2_get',
+ 'snmp2_getnext',
+ 'snmp2_real_walk',
+ 'snmp2_set',
+ 'snmp2_walk',
+ 'snmp3_get',
+ 'snmp3_getnext',
+ 'snmp3_real_walk',
+ 'snmp3_set',
+ 'snmp3_walk',
+ 'snmpget',
+ 'snmpgetnext',
+ 'snmprealwalk',
+ 'snmpset',
+ 'snmpwalk',
+ 'snmpwalkoid'),
+ 'SOAP': ('is_soap_fault', 'use_soap_error_handler'),
+ 'SPL': ('class_implements',
+ 'class_parents',
+ 'class_uses',
+ 'iterator_apply',
+ 'iterator_count',
+ 'iterator_to_array',
+ 'spl_autoload_call',
+ 'spl_autoload_extensions',
+ 'spl_autoload_functions',
+ 'spl_autoload_register',
+ 'spl_autoload_unregister',
+ 'spl_autoload',
+ 'spl_classes',
+ 'spl_object_hash'),
+ 'SPPLUS': ('calcul_hmac', 'calculhmac', 'nthmac', 'signeurlpaiement'),
+ 'SQLSRV': ('sqlsrv_begin_transaction',
+ 'sqlsrv_cancel',
+ 'sqlsrv_client_info',
+ 'sqlsrv_close',
+ 'sqlsrv_commit',
+ 'sqlsrv_configure',
+ 'sqlsrv_connect',
+ 'sqlsrv_errors',
+ 'sqlsrv_execute',
+ 'sqlsrv_fetch_array',
+ 'sqlsrv_fetch_object',
+ 'sqlsrv_fetch',
+ 'sqlsrv_field_metadata',
+ 'sqlsrv_free_stmt',
+ 'sqlsrv_get_config',
+ 'sqlsrv_get_field',
+ 'sqlsrv_has_rows',
+ 'sqlsrv_next_result',
+ 'sqlsrv_num_fields',
+ 'sqlsrv_num_rows',
+ 'sqlsrv_prepare',
+ 'sqlsrv_query',
+ 'sqlsrv_rollback',
+ 'sqlsrv_rows_affected',
+ 'sqlsrv_send_stream_data',
+ 'sqlsrv_server_info'),
+ 'SQLite': ('sqlite_array_query',
+ 'sqlite_busy_timeout',
+ 'sqlite_changes',
+ 'sqlite_close',
+ 'sqlite_column',
+ 'sqlite_create_aggregate',
+ 'sqlite_create_function',
+ 'sqlite_current',
+ 'sqlite_error_string',
+ 'sqlite_escape_string',
+ 'sqlite_exec',
+ 'sqlite_factory',
+ 'sqlite_fetch_all',
+ 'sqlite_fetch_array',
+ 'sqlite_fetch_column_types',
+ 'sqlite_fetch_object',
+ 'sqlite_fetch_single',
+ 'sqlite_fetch_string',
+ 'sqlite_field_name',
+ 'sqlite_has_more',
+ 'sqlite_has_prev',
+ 'sqlite_key',
+ 'sqlite_last_error',
+ 'sqlite_last_insert_rowid',
+ 'sqlite_libencoding',
+ 'sqlite_libversion',
+ 'sqlite_next',
+ 'sqlite_num_fields',
+ 'sqlite_num_rows',
+ 'sqlite_open',
+ 'sqlite_popen',
+ 'sqlite_prev',
+ 'sqlite_query',
+ 'sqlite_rewind',
+ 'sqlite_seek',
+ 'sqlite_single_query',
+ 'sqlite_udf_decode_binary',
+ 'sqlite_udf_encode_binary',
+ 'sqlite_unbuffered_query',
+ 'sqlite_valid'),
+ 'SSH2': ('ssh2_auth_agent',
+ 'ssh2_auth_hostbased_file',
+ 'ssh2_auth_none',
+ 'ssh2_auth_password',
+ 'ssh2_auth_pubkey_file',
+ 'ssh2_connect',
+ 'ssh2_exec',
+ 'ssh2_fetch_stream',
+ 'ssh2_fingerprint',
+ 'ssh2_methods_negotiated',
+ 'ssh2_publickey_add',
+ 'ssh2_publickey_init',
+ 'ssh2_publickey_list',
+ 'ssh2_publickey_remove',
+ 'ssh2_scp_recv',
+ 'ssh2_scp_send',
+ 'ssh2_sftp_chmod',
+ 'ssh2_sftp_lstat',
+ 'ssh2_sftp_mkdir',
+ 'ssh2_sftp_readlink',
+ 'ssh2_sftp_realpath',
+ 'ssh2_sftp_rename',
+ 'ssh2_sftp_rmdir',
+ 'ssh2_sftp_stat',
+ 'ssh2_sftp_symlink',
+ 'ssh2_sftp_unlink',
+ 'ssh2_sftp',
+ 'ssh2_shell',
+ 'ssh2_tunnel'),
+ 'SVN': ('svn_add',
+ 'svn_auth_get_parameter',
+ 'svn_auth_set_parameter',
+ 'svn_blame',
+ 'svn_cat',
+ 'svn_checkout',
+ 'svn_cleanup',
+ 'svn_client_version',
+ 'svn_commit',
+ 'svn_delete',
+ 'svn_diff',
+ 'svn_export',
+ 'svn_fs_abort_txn',
+ 'svn_fs_apply_text',
+ 'svn_fs_begin_txn2',
+ 'svn_fs_change_node_prop',
+ 'svn_fs_check_path',
+ 'svn_fs_contents_changed',
+ 'svn_fs_copy',
+ 'svn_fs_delete',
+ 'svn_fs_dir_entries',
+ 'svn_fs_file_contents',
+ 'svn_fs_file_length',
+ 'svn_fs_is_dir',
+ 'svn_fs_is_file',
+ 'svn_fs_make_dir',
+ 'svn_fs_make_file',
+ 'svn_fs_node_created_rev',
+ 'svn_fs_node_prop',
+ 'svn_fs_props_changed',
+ 'svn_fs_revision_prop',
+ 'svn_fs_revision_root',
+ 'svn_fs_txn_root',
+ 'svn_fs_youngest_rev',
+ 'svn_import',
+ 'svn_log',
+ 'svn_ls',
+ 'svn_mkdir',
+ 'svn_repos_create',
+ 'svn_repos_fs_begin_txn_for_commit',
+ 'svn_repos_fs_commit_txn',
+ 'svn_repos_fs',
+ 'svn_repos_hotcopy',
+ 'svn_repos_open',
+ 'svn_repos_recover',
+ 'svn_revert',
+ 'svn_status',
+ 'svn_update'),
+ 'SWF': ('swf_actiongeturl',
+ 'swf_actiongotoframe',
+ 'swf_actiongotolabel',
+ 'swf_actionnextframe',
+ 'swf_actionplay',
+ 'swf_actionprevframe',
+ 'swf_actionsettarget',
+ 'swf_actionstop',
+ 'swf_actiontogglequality',
+ 'swf_actionwaitforframe',
+ 'swf_addbuttonrecord',
+ 'swf_addcolor',
+ 'swf_closefile',
+ 'swf_definebitmap',
+ 'swf_definefont',
+ 'swf_defineline',
+ 'swf_definepoly',
+ 'swf_definerect',
+ 'swf_definetext',
+ 'swf_endbutton',
+ 'swf_enddoaction',
+ 'swf_endshape',
+ 'swf_endsymbol',
+ 'swf_fontsize',
+ 'swf_fontslant',
+ 'swf_fonttracking',
+ 'swf_getbitmapinfo',
+ 'swf_getfontinfo',
+ 'swf_getframe',
+ 'swf_labelframe',
+ 'swf_lookat',
+ 'swf_modifyobject',
+ 'swf_mulcolor',
+ 'swf_nextid',
+ 'swf_oncondition',
+ 'swf_openfile',
+ 'swf_ortho2',
+ 'swf_ortho',
+ 'swf_perspective',
+ 'swf_placeobject',
+ 'swf_polarview',
+ 'swf_popmatrix',
+ 'swf_posround',
+ 'swf_pushmatrix',
+ 'swf_removeobject',
+ 'swf_rotate',
+ 'swf_scale',
+ 'swf_setfont',
+ 'swf_setframe',
+ 'swf_shapearc',
+ 'swf_shapecurveto3',
+ 'swf_shapecurveto',
+ 'swf_shapefillbitmapclip',
+ 'swf_shapefillbitmaptile',
+ 'swf_shapefilloff',
+ 'swf_shapefillsolid',
+ 'swf_shapelinesolid',
+ 'swf_shapelineto',
+ 'swf_shapemoveto',
+ 'swf_showframe',
+ 'swf_startbutton',
+ 'swf_startdoaction',
+ 'swf_startshape',
+ 'swf_startsymbol',
+ 'swf_textwidth',
+ 'swf_translate',
+ 'swf_viewport'),
+ 'Semaphore': ('ftok',
+ 'msg_get_queue',
+ 'msg_queue_exists',
+ 'msg_receive',
+ 'msg_remove_queue',
+ 'msg_send',
+ 'msg_set_queue',
+ 'msg_stat_queue',
+ 'sem_acquire',
+ 'sem_get',
+ 'sem_release',
+ 'sem_remove',
+ 'shm_attach',
+ 'shm_detach',
+ 'shm_get_var',
+ 'shm_has_var',
+ 'shm_put_var',
+ 'shm_remove_var',
+ 'shm_remove'),
+ 'Session': ('session_cache_expire',
+ 'session_cache_limiter',
+ 'session_commit',
+ 'session_decode',
+ 'session_destroy',
+ 'session_encode',
+ 'session_get_cookie_params',
+ 'session_id',
+ 'session_is_registered',
+ 'session_module_name',
+ 'session_name',
+ 'session_regenerate_id',
+ 'session_register_shutdown',
+ 'session_register',
+ 'session_save_path',
+ 'session_set_cookie_params',
+ 'session_set_save_handler',
+ 'session_start',
+ 'session_status',
+ 'session_unregister',
+ 'session_unset',
+ 'session_write_close'),
+ 'Session PgSQL': ('session_pgsql_add_error',
+ 'session_pgsql_get_error',
+ 'session_pgsql_get_field',
+ 'session_pgsql_reset',
+ 'session_pgsql_set_field',
+ 'session_pgsql_status'),
+ 'Shared Memory': ('shmop_close',
+ 'shmop_delete',
+ 'shmop_open',
+ 'shmop_read',
+ 'shmop_size',
+ 'shmop_write'),
+ 'SimpleXML': ('simplexml_import_dom',
+ 'simplexml_load_file',
+ 'simplexml_load_string'),
+ 'Socket': ('socket_accept',
+ 'socket_bind',
+ 'socket_clear_error',
+ 'socket_close',
+ 'socket_cmsg_space',
+ 'socket_connect',
+ 'socket_create_listen',
+ 'socket_create_pair',
+ 'socket_create',
+ 'socket_get_option',
+ 'socket_getpeername',
+ 'socket_getsockname',
+ 'socket_import_stream',
+ 'socket_last_error',
+ 'socket_listen',
+ 'socket_read',
+ 'socket_recv',
+ 'socket_recvfrom',
+ 'socket_recvmsg',
+ 'socket_select',
+ 'socket_send',
+ 'socket_sendmsg',
+ 'socket_sendto',
+ 'socket_set_block',
+ 'socket_set_nonblock',
+ 'socket_set_option',
+ 'socket_shutdown',
+ 'socket_strerror',
+ 'socket_write'),
+ 'Solr': ('solr_get_version',),
+ 'Statistic': ('stats_absolute_deviation',
+ 'stats_cdf_beta',
+ 'stats_cdf_binomial',
+ 'stats_cdf_cauchy',
+ 'stats_cdf_chisquare',
+ 'stats_cdf_exponential',
+ 'stats_cdf_f',
+ 'stats_cdf_gamma',
+ 'stats_cdf_laplace',
+ 'stats_cdf_logistic',
+ 'stats_cdf_negative_binomial',
+ 'stats_cdf_noncentral_chisquare',
+ 'stats_cdf_noncentral_f',
+ 'stats_cdf_poisson',
+ 'stats_cdf_t',
+ 'stats_cdf_uniform',
+ 'stats_cdf_weibull',
+ 'stats_covariance',
+ 'stats_den_uniform',
+ 'stats_dens_beta',
+ 'stats_dens_cauchy',
+ 'stats_dens_chisquare',
+ 'stats_dens_exponential',
+ 'stats_dens_f',
+ 'stats_dens_gamma',
+ 'stats_dens_laplace',
+ 'stats_dens_logistic',
+ 'stats_dens_negative_binomial',
+ 'stats_dens_normal',
+ 'stats_dens_pmf_binomial',
+ 'stats_dens_pmf_hypergeometric',
+ 'stats_dens_pmf_poisson',
+ 'stats_dens_t',
+ 'stats_dens_weibull',
+ 'stats_harmonic_mean',
+ 'stats_kurtosis',
+ 'stats_rand_gen_beta',
+ 'stats_rand_gen_chisquare',
+ 'stats_rand_gen_exponential',
+ 'stats_rand_gen_f',
+ 'stats_rand_gen_funiform',
+ 'stats_rand_gen_gamma',
+ 'stats_rand_gen_ibinomial_negative',
+ 'stats_rand_gen_ibinomial',
+ 'stats_rand_gen_int',
+ 'stats_rand_gen_ipoisson',
+ 'stats_rand_gen_iuniform',
+ 'stats_rand_gen_noncenral_chisquare',
+ 'stats_rand_gen_noncentral_f',
+ 'stats_rand_gen_noncentral_t',
+ 'stats_rand_gen_normal',
+ 'stats_rand_gen_t',
+ 'stats_rand_get_seeds',
+ 'stats_rand_phrase_to_seeds',
+ 'stats_rand_ranf',
+ 'stats_rand_setall',
+ 'stats_skew',
+ 'stats_standard_deviation',
+ 'stats_stat_binomial_coef',
+ 'stats_stat_correlation',
+ 'stats_stat_gennch',
+ 'stats_stat_independent_t',
+ 'stats_stat_innerproduct',
+ 'stats_stat_noncentral_t',
+ 'stats_stat_paired_t',
+ 'stats_stat_percentile',
+ 'stats_stat_powersum',
+ 'stats_variance'),
+ 'Stomp': ('stomp_connect_error', 'stomp_version'),
+ 'Stream': ('set_socket_blocking',
+ 'stream_bucket_append',
+ 'stream_bucket_make_writeable',
+ 'stream_bucket_new',
+ 'stream_bucket_prepend',
+ 'stream_context_create',
+ 'stream_context_get_default',
+ 'stream_context_get_options',
+ 'stream_context_get_params',
+ 'stream_context_set_default',
+ 'stream_context_set_option',
+ 'stream_context_set_params',
+ 'stream_copy_to_stream',
+ 'stream_encoding',
+ 'stream_filter_append',
+ 'stream_filter_prepend',
+ 'stream_filter_register',
+ 'stream_filter_remove',
+ 'stream_get_contents',
+ 'stream_get_filters',
+ 'stream_get_line',
+ 'stream_get_meta_data',
+ 'stream_get_transports',
+ 'stream_get_wrappers',
+ 'stream_is_local',
+ 'stream_notification_callback',
+ 'stream_register_wrapper',
+ 'stream_resolve_include_path',
+ 'stream_select',
+ 'stream_set_blocking',
+ 'stream_set_chunk_size',
+ 'stream_set_read_buffer',
+ 'stream_set_timeout',
+ 'stream_set_write_buffer',
+ 'stream_socket_accept',
+ 'stream_socket_client',
+ 'stream_socket_enable_crypto',
+ 'stream_socket_get_name',
+ 'stream_socket_pair',
+ 'stream_socket_recvfrom',
+ 'stream_socket_sendto',
+ 'stream_socket_server',
+ 'stream_socket_shutdown',
+ 'stream_supports_lock',
+ 'stream_wrapper_register',
+ 'stream_wrapper_restore',
+ 'stream_wrapper_unregister'),
+ 'String': ('addcslashes',
+ 'addslashes',
+ 'bin2hex',
+ 'chop',
+ 'chr',
+ 'chunk_split',
+ 'convert_cyr_string',
+ 'convert_uudecode',
+ 'convert_uuencode',
+ 'count_chars',
+ 'crc32',
+ 'crypt',
+ 'echo',
+ 'explode',
+ 'fprintf',
+ 'get_html_translation_table',
+ 'hebrev',
+ 'hebrevc',
+ 'hex2bin',
+ 'html_entity_decode',
+ 'htmlentities',
+ 'htmlspecialchars_decode',
+ 'htmlspecialchars',
+ 'implode',
+ 'join',
+ 'lcfirst',
+ 'levenshtein',
+ 'localeconv',
+ 'ltrim',
+ 'md5_file',
+ 'md5',
+ 'metaphone',
+ 'money_format',
+ 'nl_langinfo',
+ 'nl2br',
+ 'number_format',
+ 'ord',
+ 'parse_str',
+ 'print',
+ 'printf',
+ 'quoted_printable_decode',
+ 'quoted_printable_encode',
+ 'quotemeta',
+ 'rtrim',
+ 'setlocale',
+ 'sha1_file',
+ 'sha1',
+ 'similar_text',
+ 'soundex',
+ 'sprintf',
+ 'sscanf',
+ 'str_getcsv',
+ 'str_ireplace',
+ 'str_pad',
+ 'str_repeat',
+ 'str_replace',
+ 'str_rot13',
+ 'str_shuffle',
+ 'str_split',
+ 'str_word_count',
+ 'strcasecmp',
+ 'strchr',
+ 'strcmp',
+ 'strcoll',
+ 'strcspn',
+ 'strip_tags',
+ 'stripcslashes',
+ 'stripos',
+ 'stripslashes',
+ 'stristr',
+ 'strlen',
+ 'strnatcasecmp',
+ 'strnatcmp',
+ 'strncasecmp',
+ 'strncmp',
+ 'strpbrk',
+ 'strpos',
+ 'strrchr',
+ 'strrev',
+ 'strripos',
+ 'strrpos',
+ 'strspn',
+ 'strstr',
+ 'strtok',
+ 'strtolower',
+ 'strtoupper',
+ 'strtr',
+ 'substr_compare',
+ 'substr_count',
+ 'substr_replace',
+ 'substr',
+ 'trim',
+ 'ucfirst',
+ 'ucwords',
+ 'vfprintf',
+ 'vprintf',
+ 'vsprintf',
+ 'wordwrap'),
+ 'Sybase': ('sybase_affected_rows',
+ 'sybase_close',
+ 'sybase_connect',
+ 'sybase_data_seek',
+ 'sybase_deadlock_retry_count',
+ 'sybase_fetch_array',
+ 'sybase_fetch_assoc',
+ 'sybase_fetch_field',
+ 'sybase_fetch_object',
+ 'sybase_fetch_row',
+ 'sybase_field_seek',
+ 'sybase_free_result',
+ 'sybase_get_last_message',
+ 'sybase_min_client_severity',
+ 'sybase_min_error_severity',
+ 'sybase_min_message_severity',
+ 'sybase_min_server_severity',
+ 'sybase_num_fields',
+ 'sybase_num_rows',
+ 'sybase_pconnect',
+ 'sybase_query',
+ 'sybase_result',
+ 'sybase_select_db',
+ 'sybase_set_message_handler',
+ 'sybase_unbuffered_query'),
+ 'TCP': ('tcpwrap_check',),
+ 'Taint': ('is_tainted', 'taint', 'untaint'),
+ 'Tidy': ('ob_tidyhandler',
+ 'tidy_access_count',
+ 'tidy_config_count',
+ 'tidy_error_count',
+ 'tidy_get_output',
+ 'tidy_load_config',
+ 'tidy_reset_config',
+ 'tidy_save_config',
+ 'tidy_set_encoding',
+ 'tidy_setopt',
+ 'tidy_warning_count'),
+ 'Tokenizer': ('token_get_all', 'token_name'),
+ 'Trader': ('trader_acos',
+ 'trader_ad',
+ 'trader_add',
+ 'trader_adosc',
+ 'trader_adx',
+ 'trader_adxr',
+ 'trader_apo',
+ 'trader_aroon',
+ 'trader_aroonosc',
+ 'trader_asin',
+ 'trader_atan',
+ 'trader_atr',
+ 'trader_avgprice',
+ 'trader_bbands',
+ 'trader_beta',
+ 'trader_bop',
+ 'trader_cci',
+ 'trader_cdl2crows',
+ 'trader_cdl3blackcrows',
+ 'trader_cdl3inside',
+ 'trader_cdl3linestrike',
+ 'trader_cdl3outside',
+ 'trader_cdl3starsinsouth',
+ 'trader_cdl3whitesoldiers',
+ 'trader_cdlabandonedbaby',
+ 'trader_cdladvanceblock',
+ 'trader_cdlbelthold',
+ 'trader_cdlbreakaway',
+ 'trader_cdlclosingmarubozu',
+ 'trader_cdlconcealbabyswall',
+ 'trader_cdlcounterattack',
+ 'trader_cdldarkcloudcover',
+ 'trader_cdldoji',
+ 'trader_cdldojistar',
+ 'trader_cdldragonflydoji',
+ 'trader_cdlengulfing',
+ 'trader_cdleveningdojistar',
+ 'trader_cdleveningstar',
+ 'trader_cdlgapsidesidewhite',
+ 'trader_cdlgravestonedoji',
+ 'trader_cdlhammer',
+ 'trader_cdlhangingman',
+ 'trader_cdlharami',
+ 'trader_cdlharamicross',
+ 'trader_cdlhighwave',
+ 'trader_cdlhikkake',
+ 'trader_cdlhikkakemod',
+ 'trader_cdlhomingpigeon',
+ 'trader_cdlidentical3crows',
+ 'trader_cdlinneck',
+ 'trader_cdlinvertedhammer',
+ 'trader_cdlkicking',
+ 'trader_cdlkickingbylength',
+ 'trader_cdlladderbottom',
+ 'trader_cdllongleggeddoji',
+ 'trader_cdllongline',
+ 'trader_cdlmarubozu',
+ 'trader_cdlmatchinglow',
+ 'trader_cdlmathold',
+ 'trader_cdlmorningdojistar',
+ 'trader_cdlmorningstar',
+ 'trader_cdlonneck',
+ 'trader_cdlpiercing',
+ 'trader_cdlrickshawman',
+ 'trader_cdlrisefall3methods',
+ 'trader_cdlseparatinglines',
+ 'trader_cdlshootingstar',
+ 'trader_cdlshortline',
+ 'trader_cdlspinningtop',
+ 'trader_cdlstalledpattern',
+ 'trader_cdlsticksandwich',
+ 'trader_cdltakuri',
+ 'trader_cdltasukigap',
+ 'trader_cdlthrusting',
+ 'trader_cdltristar',
+ 'trader_cdlunique3river',
+ 'trader_cdlupsidegap2crows',
+ 'trader_cdlxsidegap3methods',
+ 'trader_ceil',
+ 'trader_cmo',
+ 'trader_correl',
+ 'trader_cos',
+ 'trader_cosh',
+ 'trader_dema',
+ 'trader_div',
+ 'trader_dx',
+ 'trader_ema',
+ 'trader_errno',
+ 'trader_exp',
+ 'trader_floor',
+ 'trader_get_compat',
+ 'trader_get_unstable_period',
+ 'trader_ht_dcperiod',
+ 'trader_ht_dcphase',
+ 'trader_ht_phasor',
+ 'trader_ht_sine',
+ 'trader_ht_trendline',
+ 'trader_ht_trendmode',
+ 'trader_kama',
+ 'trader_linearreg_angle',
+ 'trader_linearreg_intercept',
+ 'trader_linearreg_slope',
+ 'trader_linearreg',
+ 'trader_ln',
+ 'trader_log10',
+ 'trader_ma',
+ 'trader_macd',
+ 'trader_macdext',
+ 'trader_macdfix',
+ 'trader_mama',
+ 'trader_mavp',
+ 'trader_max',
+ 'trader_maxindex',
+ 'trader_medprice',
+ 'trader_mfi',
+ 'trader_midpoint',
+ 'trader_midprice',
+ 'trader_min',
+ 'trader_minindex',
+ 'trader_minmax',
+ 'trader_minmaxindex',
+ 'trader_minus_di',
+ 'trader_minus_dm',
+ 'trader_mom',
+ 'trader_mult',
+ 'trader_natr',
+ 'trader_obv',
+ 'trader_plus_di',
+ 'trader_plus_dm',
+ 'trader_ppo',
+ 'trader_roc',
+ 'trader_rocp',
+ 'trader_rocr100',
+ 'trader_rocr',
+ 'trader_rsi',
+ 'trader_sar',
+ 'trader_sarext',
+ 'trader_set_compat',
+ 'trader_set_unstable_period',
+ 'trader_sin',
+ 'trader_sinh',
+ 'trader_sma',
+ 'trader_sqrt',
+ 'trader_stddev',
+ 'trader_stoch',
+ 'trader_stochf',
+ 'trader_stochrsi',
+ 'trader_sub',
+ 'trader_sum',
+ 'trader_t3',
+ 'trader_tan',
+ 'trader_tanh',
+ 'trader_tema',
+ 'trader_trange',
+ 'trader_trima',
+ 'trader_trix',
+ 'trader_tsf',
+ 'trader_typprice',
+ 'trader_ultosc',
+ 'trader_var',
+ 'trader_wclprice',
+ 'trader_willr',
+ 'trader_wma'),
+ 'URL': ('base64_decode',
+ 'base64_encode',
+ 'get_headers',
+ 'get_meta_tags',
+ 'http_build_query',
+ 'parse_url',
+ 'rawurldecode',
+ 'rawurlencode',
+ 'urldecode',
+ 'urlencode'),
+ 'Uopz': ('uopz_backup',
+ 'uopz_compose',
+ 'uopz_copy',
+ 'uopz_delete',
+ 'uopz_extend',
+ 'uopz_flags',
+ 'uopz_function',
+ 'uopz_implement',
+ 'uopz_overload',
+ 'uopz_redefine',
+ 'uopz_rename',
+ 'uopz_restore',
+ 'uopz_undefine'),
+ 'Variable handling': ('boolval',
+ 'debug_zval_dump',
+ 'doubleval',
+ 'empty',
+ 'floatval',
+ 'get_defined_vars',
+ 'get_resource_type',
+ 'gettype',
+ 'import_request_variables',
+ 'intval',
+ 'is_array',
+ 'is_bool',
+ 'is_callable',
+ 'is_double',
+ 'is_float',
+ 'is_int',
+ 'is_integer',
+ 'is_long',
+ 'is_null',
+ 'is_numeric',
+ 'is_object',
+ 'is_real',
+ 'is_resource',
+ 'is_scalar',
+ 'is_string',
+ 'isset',
+ 'print_r',
+ 'serialize',
+ 'settype',
+ 'strval',
+ 'unserialize',
+ 'unset',
+ 'var_dump',
+ 'var_export'),
+ 'W32api': ('w32api_deftype',
+ 'w32api_init_dtype',
+ 'w32api_invoke_function',
+ 'w32api_register_function',
+ 'w32api_set_call_method'),
+ 'WDDX': ('wddx_add_vars',
+ 'wddx_deserialize',
+ 'wddx_packet_end',
+ 'wddx_packet_start',
+ 'wddx_serialize_value',
+ 'wddx_serialize_vars'),
+ 'WinCache': ('wincache_fcache_fileinfo',
+ 'wincache_fcache_meminfo',
+ 'wincache_lock',
+ 'wincache_ocache_fileinfo',
+ 'wincache_ocache_meminfo',
+ 'wincache_refresh_if_changed',
+ 'wincache_rplist_fileinfo',
+ 'wincache_rplist_meminfo',
+ 'wincache_scache_info',
+ 'wincache_scache_meminfo',
+ 'wincache_ucache_add',
+ 'wincache_ucache_cas',
+ 'wincache_ucache_clear',
+ 'wincache_ucache_dec',
+ 'wincache_ucache_delete',
+ 'wincache_ucache_exists',
+ 'wincache_ucache_get',
+ 'wincache_ucache_inc',
+ 'wincache_ucache_info',
+ 'wincache_ucache_meminfo',
+ 'wincache_ucache_set',
+ 'wincache_unlock'),
+ 'XML Parser': ('utf8_decode',
+ 'utf8_encode',
+ 'xml_error_string',
+ 'xml_get_current_byte_index',
+ 'xml_get_current_column_number',
+ 'xml_get_current_line_number',
+ 'xml_get_error_code',
+ 'xml_parse_into_struct',
+ 'xml_parse',
+ 'xml_parser_create_ns',
+ 'xml_parser_create',
+ 'xml_parser_free',
+ 'xml_parser_get_option',
+ 'xml_parser_set_option',
+ 'xml_set_character_data_handler',
+ 'xml_set_default_handler',
+ 'xml_set_element_handler',
+ 'xml_set_end_namespace_decl_handler',
+ 'xml_set_external_entity_ref_handler',
+ 'xml_set_notation_decl_handler',
+ 'xml_set_object',
+ 'xml_set_processing_instruction_handler',
+ 'xml_set_start_namespace_decl_handler',
+ 'xml_set_unparsed_entity_decl_handler'),
+ 'XML-RPC': ('xmlrpc_decode_request',
+ 'xmlrpc_decode',
+ 'xmlrpc_encode_request',
+ 'xmlrpc_encode',
+ 'xmlrpc_get_type',
+ 'xmlrpc_is_fault',
+ 'xmlrpc_parse_method_descriptions',
+ 'xmlrpc_server_add_introspection_data',
+ 'xmlrpc_server_call_method',
+ 'xmlrpc_server_create',
+ 'xmlrpc_server_destroy',
+ 'xmlrpc_server_register_introspection_callback',
+ 'xmlrpc_server_register_method',
+ 'xmlrpc_set_type'),
+ 'XSLT (PHP 4)': ('xslt_backend_info',
+ 'xslt_backend_name',
+ 'xslt_backend_version',
+ 'xslt_create',
+ 'xslt_errno',
+ 'xslt_error',
+ 'xslt_free',
+ 'xslt_getopt',
+ 'xslt_process',
+ 'xslt_set_base',
+ 'xslt_set_encoding',
+ 'xslt_set_error_handler',
+ 'xslt_set_log',
+ 'xslt_set_object',
+ 'xslt_set_sax_handler',
+ 'xslt_set_sax_handlers',
+ 'xslt_set_scheme_handler',
+ 'xslt_set_scheme_handlers',
+ 'xslt_setopt'),
+ 'Xhprof': ('xhprof_disable',
+ 'xhprof_enable',
+ 'xhprof_sample_disable',
+ 'xhprof_sample_enable'),
+ 'YAZ': ('yaz_addinfo',
+ 'yaz_ccl_conf',
+ 'yaz_ccl_parse',
+ 'yaz_close',
+ 'yaz_connect',
+ 'yaz_database',
+ 'yaz_element',
+ 'yaz_errno',
+ 'yaz_error',
+ 'yaz_es_result',
+ 'yaz_es',
+ 'yaz_get_option',
+ 'yaz_hits',
+ 'yaz_itemorder',
+ 'yaz_present',
+ 'yaz_range',
+ 'yaz_record',
+ 'yaz_scan_result',
+ 'yaz_scan',
+ 'yaz_schema',
+ 'yaz_search',
+ 'yaz_set_option',
+ 'yaz_sort',
+ 'yaz_syntax',
+ 'yaz_wait'),
+ 'YP/NIS': ('yp_all',
+ 'yp_cat',
+ 'yp_err_string',
+ 'yp_errno',
+ 'yp_first',
+ 'yp_get_default_domain',
+ 'yp_master',
+ 'yp_match',
+ 'yp_next',
+ 'yp_order'),
+ 'Yaml': ('yaml_emit_file',
+ 'yaml_emit',
+ 'yaml_parse_file',
+ 'yaml_parse_url',
+ 'yaml_parse'),
+ 'Zip': ('zip_close',
+ 'zip_entry_close',
+ 'zip_entry_compressedsize',
+ 'zip_entry_compressionmethod',
+ 'zip_entry_filesize',
+ 'zip_entry_name',
+ 'zip_entry_open',
+ 'zip_entry_read',
+ 'zip_open',
+ 'zip_read'),
+ 'Zlib': ('gzclose',
+ 'gzcompress',
+ 'gzdecode',
+ 'gzdeflate',
+ 'gzencode',
+ 'gzeof',
+ 'gzfile',
+ 'gzgetc',
+ 'gzgets',
+ 'gzgetss',
+ 'gzinflate',
+ 'gzopen',
+ 'gzpassthru',
+ 'gzputs',
+ 'gzread',
+ 'gzrewind',
+ 'gzseek',
+ 'gztell',
+ 'gzuncompress',
+ 'gzwrite',
+ 'readgzfile',
+ 'zlib_decode',
+ 'zlib_encode',
+ 'zlib_get_coding_type'),
+ 'bcompiler': ('bcompiler_load_exe',
+ 'bcompiler_load',
+ 'bcompiler_parse_class',
+ 'bcompiler_read',
+ 'bcompiler_write_class',
+ 'bcompiler_write_constant',
+ 'bcompiler_write_exe_footer',
+ 'bcompiler_write_file',
+ 'bcompiler_write_footer',
+ 'bcompiler_write_function',
+ 'bcompiler_write_functions_from_file',
+ 'bcompiler_write_header',
+ 'bcompiler_write_included_filename'),
+ 'cURL': ('curl_close',
+ 'curl_copy_handle',
+ 'curl_errno',
+ 'curl_error',
+ 'curl_escape',
+ 'curl_exec',
+ 'curl_file_create',
+ 'curl_getinfo',
+ 'curl_init',
+ 'curl_multi_add_handle',
+ 'curl_multi_close',
+ 'curl_multi_exec',
+ 'curl_multi_getcontent',
+ 'curl_multi_info_read',
+ 'curl_multi_init',
+ 'curl_multi_remove_handle',
+ 'curl_multi_select',
+ 'curl_multi_setopt',
+ 'curl_multi_strerror',
+ 'curl_pause',
+ 'curl_reset',
+ 'curl_setopt_array',
+ 'curl_setopt',
+ 'curl_share_close',
+ 'curl_share_init',
+ 'curl_share_setopt',
+ 'curl_strerror',
+ 'curl_unescape',
+ 'curl_version'),
+ 'chdb': ('chdb_create',),
+ 'dBase': ('dbase_add_record',
+ 'dbase_close',
+ 'dbase_create',
+ 'dbase_delete_record',
+ 'dbase_get_header_info',
+ 'dbase_get_record_with_names',
+ 'dbase_get_record',
+ 'dbase_numfields',
+ 'dbase_numrecords',
+ 'dbase_open',
+ 'dbase_pack',
+ 'dbase_replace_record'),
+ 'dbx': ('dbx_close',
+ 'dbx_compare',
+ 'dbx_connect',
+ 'dbx_error',
+ 'dbx_escape_string',
+ 'dbx_fetch_row',
+ 'dbx_query',
+ 'dbx_sort'),
+ 'filePro': ('filepro_fieldcount',
+ 'filepro_fieldname',
+ 'filepro_fieldtype',
+ 'filepro_fieldwidth',
+ 'filepro_retrieve',
+ 'filepro_rowcount',
+ 'filepro'),
+ 'iconv': ('iconv_get_encoding',
+ 'iconv_mime_decode_headers',
+ 'iconv_mime_decode',
+ 'iconv_mime_encode',
+ 'iconv_set_encoding',
+ 'iconv_strlen',
+ 'iconv_strpos',
+ 'iconv_strrpos',
+ 'iconv_substr',
+ 'iconv',
+ 'ob_iconv_handler'),
+ 'inclued': ('inclued_get_data',),
+ 'intl': ('intl_error_name',
+ 'intl_get_error_code',
+ 'intl_get_error_message',
+ 'intl_is_failure'),
+ 'libxml': ('libxml_clear_errors',
+ 'libxml_disable_entity_loader',
+ 'libxml_get_errors',
+ 'libxml_get_last_error',
+ 'libxml_set_external_entity_loader',
+ 'libxml_set_streams_context',
+ 'libxml_use_internal_errors'),
+ 'mSQL': ('msql_affected_rows',
+ 'msql_close',
+ 'msql_connect',
+ 'msql_create_db',
+ 'msql_createdb',
+ 'msql_data_seek',
+ 'msql_db_query',
+ 'msql_dbname',
+ 'msql_drop_db',
+ 'msql_error',
+ 'msql_fetch_array',
+ 'msql_fetch_field',
+ 'msql_fetch_object',
+ 'msql_fetch_row',
+ 'msql_field_flags',
+ 'msql_field_len',
+ 'msql_field_name',
+ 'msql_field_seek',
+ 'msql_field_table',
+ 'msql_field_type',
+ 'msql_fieldflags',
+ 'msql_fieldlen',
+ 'msql_fieldname',
+ 'msql_fieldtable',
+ 'msql_fieldtype',
+ 'msql_free_result',
+ 'msql_list_dbs',
+ 'msql_list_fields',
+ 'msql_list_tables',
+ 'msql_num_fields',
+ 'msql_num_rows',
+ 'msql_numfields',
+ 'msql_numrows',
+ 'msql_pconnect',
+ 'msql_query',
+ 'msql_regcase',
+ 'msql_result',
+ 'msql_select_db',
+ 'msql_tablename',
+ 'msql'),
+ 'mnoGoSearch': ('udm_add_search_limit',
+ 'udm_alloc_agent_array',
+ 'udm_alloc_agent',
+ 'udm_api_version',
+ 'udm_cat_list',
+ 'udm_cat_path',
+ 'udm_check_charset',
+ 'udm_check_stored',
+ 'udm_clear_search_limits',
+ 'udm_close_stored',
+ 'udm_crc32',
+ 'udm_errno',
+ 'udm_error',
+ 'udm_find',
+ 'udm_free_agent',
+ 'udm_free_ispell_data',
+ 'udm_free_res',
+ 'udm_get_doc_count',
+ 'udm_get_res_field',
+ 'udm_get_res_param',
+ 'udm_hash32',
+ 'udm_load_ispell_data',
+ 'udm_open_stored',
+ 'udm_set_agent_param'),
+ 'mqseries': ('mqseries_back',
+ 'mqseries_begin',
+ 'mqseries_close',
+ 'mqseries_cmit',
+ 'mqseries_conn',
+ 'mqseries_connx',
+ 'mqseries_disc',
+ 'mqseries_get',
+ 'mqseries_inq',
+ 'mqseries_open',
+ 'mqseries_put1',
+ 'mqseries_put',
+ 'mqseries_set',
+ 'mqseries_strerror'),
+ 'mysqlnd_qc': ('mysqlnd_qc_clear_cache',
+ 'mysqlnd_qc_get_available_handlers',
+ 'mysqlnd_qc_get_cache_info',
+ 'mysqlnd_qc_get_core_stats',
+ 'mysqlnd_qc_get_normalized_query_trace_log',
+ 'mysqlnd_qc_get_query_trace_log',
+ 'mysqlnd_qc_set_cache_condition',
+ 'mysqlnd_qc_set_is_select',
+ 'mysqlnd_qc_set_storage_handler',
+ 'mysqlnd_qc_set_user_handlers'),
+ 'qtdom': ('qdom_error', 'qdom_tree'),
+ 'runkit': ('runkit_class_adopt',
+ 'runkit_class_emancipate',
+ 'runkit_constant_add',
+ 'runkit_constant_redefine',
+ 'runkit_constant_remove',
+ 'runkit_function_add',
+ 'runkit_function_copy',
+ 'runkit_function_redefine',
+ 'runkit_function_remove',
+ 'runkit_function_rename',
+ 'runkit_import',
+ 'runkit_lint_file',
+ 'runkit_lint',
+ 'runkit_method_add',
+ 'runkit_method_copy',
+ 'runkit_method_redefine',
+ 'runkit_method_remove',
+ 'runkit_method_rename',
+ 'runkit_return_value_used',
+ 'runkit_sandbox_output_handler',
+ 'runkit_superglobals'),
+ 'ssdeep': ('ssdeep_fuzzy_compare',
+ 'ssdeep_fuzzy_hash_filename',
+ 'ssdeep_fuzzy_hash'),
+ 'vpopmail': ('vpopmail_add_alias_domain_ex',
+ 'vpopmail_add_alias_domain',
+ 'vpopmail_add_domain_ex',
+ 'vpopmail_add_domain',
+ 'vpopmail_add_user',
+ 'vpopmail_alias_add',
+ 'vpopmail_alias_del_domain',
+ 'vpopmail_alias_del',
+ 'vpopmail_alias_get_all',
+ 'vpopmail_alias_get',
+ 'vpopmail_auth_user',
+ 'vpopmail_del_domain_ex',
+ 'vpopmail_del_domain',
+ 'vpopmail_del_user',
+ 'vpopmail_error',
+ 'vpopmail_passwd',
+ 'vpopmail_set_user_quota'),
+ 'win32ps': ('win32_ps_list_procs', 'win32_ps_stat_mem', 'win32_ps_stat_proc'),
+ 'win32service': ('win32_continue_service',
+ 'win32_create_service',
+ 'win32_delete_service',
+ 'win32_get_last_control_message',
+ 'win32_pause_service',
+ 'win32_query_service_status',
+ 'win32_set_service_status',
+ 'win32_start_service_ctrl_dispatcher',
+ 'win32_start_service',
+ 'win32_stop_service'),
+ 'xattr': ('xattr_get',
+ 'xattr_list',
+ 'xattr_remove',
+ 'xattr_set',
+ 'xattr_supported'),
+ 'xdiff': ('xdiff_file_bdiff_size',
+ 'xdiff_file_bdiff',
+ 'xdiff_file_bpatch',
+ 'xdiff_file_diff_binary',
+ 'xdiff_file_diff',
+ 'xdiff_file_merge3',
+ 'xdiff_file_patch_binary',
+ 'xdiff_file_patch',
+ 'xdiff_file_rabdiff',
+ 'xdiff_string_bdiff_size',
+ 'xdiff_string_bdiff',
+ 'xdiff_string_bpatch',
+ 'xdiff_string_diff_binary',
+ 'xdiff_string_diff',
+ 'xdiff_string_merge3',
+ 'xdiff_string_patch_binary',
+ 'xdiff_string_patch',
+ 'xdiff_string_rabdiff')}
+
+
+if __name__ == '__main__': # pragma: no cover
+ import glob
+ import os
+ import pprint
+ import re
+ import shutil
+ import tarfile
+ try:
+ from urllib import urlretrieve
+ except ImportError:
+ from urllib.request import urlretrieve
+
+ PHP_MANUAL_URL = 'http://us3.php.net/distributions/manual/php_manual_en.tar.gz'
+ PHP_MANUAL_DIR = './php-chunked-xhtml/'
+ PHP_REFERENCE_GLOB = 'ref.*'
+ PHP_FUNCTION_RE = '<a href="function\..*?\.html">(.*?)</a>'
+ PHP_MODULE_RE = '<title>(.*?) Functions</title>'
+
+ def get_php_functions():
+ function_re = re.compile(PHP_FUNCTION_RE)
+ module_re = re.compile(PHP_MODULE_RE)
+ modules = {}
+
+ for file in get_php_references():
+ module = ''
+ for line in open(file):
+ if not module:
+ search = module_re.search(line)
+ if search:
+ module = search.group(1)
+ modules[module] = []
+
+ elif 'href="function.' in line:
+ for match in function_re.finditer(line):
+ fn = match.group(1)
+ if '->' not in fn and '::' not in fn and fn not in modules[module]:
+ modules[module].append(fn)
+
+ if module:
+ # These are dummy manual pages, not actual functions
+ if module == 'PHP Options/Info':
+ modules[module].remove('main')
+
+ if module == 'Filesystem':
+ modules[module].remove('delete')
+
+ if not modules[module]:
+ del modules[module]
+
+ return modules
+
+ def get_php_references():
+ download = urlretrieve(PHP_MANUAL_URL)
+ tar = tarfile.open(download[0])
+ tar.extractall()
+ tar.close()
+ for file in glob.glob("%s%s" % (PHP_MANUAL_DIR, PHP_REFERENCE_GLOB)):
+ yield file
+ os.remove(download[0])
+
+ def regenerate(filename, modules):
+ with open(filename) as fp:
+ content = fp.read()
+
+ header = content[:content.find('MODULES = {')]
+ footer = content[content.find("if __name__ == '__main__':"):]
+
+ with open(filename, 'w') as fp:
+ fp.write(header)
+ fp.write('MODULES = %s\n\n' % pprint.pformat(modules))
+ fp.write(footer)
+
+ def run():
+ print('>> Downloading Function Index')
+ modules = get_php_functions()
+ total = sum(len(v) for v in modules.values())
+ print('%d functions found' % total)
+ regenerate(__file__, modules)
+ shutil.rmtree(PHP_MANUAL_DIR)
+
+ run()
diff --git a/wandb/vendor/pygments/lexers/_postgres_builtins.py b/wandb/vendor/pygments/lexers/_postgres_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..a71360f01a55aeae159849cd08031fe8b008374f
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_postgres_builtins.py
@@ -0,0 +1,621 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._postgres_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Self-updating data files for PostgreSQL lexer.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+
+# Autogenerated: please edit them if you like wasting your time.
+
+KEYWORDS = (
+ 'ABORT',
+ 'ABSOLUTE',
+ 'ACCESS',
+ 'ACTION',
+ 'ADD',
+ 'ADMIN',
+ 'AFTER',
+ 'AGGREGATE',
+ 'ALL',
+ 'ALSO',
+ 'ALTER',
+ 'ALWAYS',
+ 'ANALYSE',
+ 'ANALYZE',
+ 'AND',
+ 'ANY',
+ 'ARRAY',
+ 'AS',
+ 'ASC',
+ 'ASSERTION',
+ 'ASSIGNMENT',
+ 'ASYMMETRIC',
+ 'AT',
+ 'ATTRIBUTE',
+ 'AUTHORIZATION',
+ 'BACKWARD',
+ 'BEFORE',
+ 'BEGIN',
+ 'BETWEEN',
+ 'BIGINT',
+ 'BINARY',
+ 'BIT',
+ 'BOOLEAN',
+ 'BOTH',
+ 'BY',
+ 'CACHE',
+ 'CALLED',
+ 'CASCADE',
+ 'CASCADED',
+ 'CASE',
+ 'CAST',
+ 'CATALOG',
+ 'CHAIN',
+ 'CHAR',
+ 'CHARACTER',
+ 'CHARACTERISTICS',
+ 'CHECK',
+ 'CHECKPOINT',
+ 'CLASS',
+ 'CLOSE',
+ 'CLUSTER',
+ 'COALESCE',
+ 'COLLATE',
+ 'COLLATION',
+ 'COLUMN',
+ 'COMMENT',
+ 'COMMENTS',
+ 'COMMIT',
+ 'COMMITTED',
+ 'CONCURRENTLY',
+ 'CONFIGURATION',
+ 'CONNECTION',
+ 'CONSTRAINT',
+ 'CONSTRAINTS',
+ 'CONTENT',
+ 'CONTINUE',
+ 'CONVERSION',
+ 'COPY',
+ 'COST',
+ 'CREATE',
+ 'CROSS',
+ 'CSV',
+ 'CURRENT',
+ 'CURRENT_CATALOG',
+ 'CURRENT_DATE',
+ 'CURRENT_ROLE',
+ 'CURRENT_SCHEMA',
+ 'CURRENT_TIME',
+ 'CURRENT_TIMESTAMP',
+ 'CURRENT_USER',
+ 'CURSOR',
+ 'CYCLE',
+ 'DATA',
+ 'DATABASE',
+ 'DAY',
+ 'DEALLOCATE',
+ 'DEC',
+ 'DECIMAL',
+ 'DECLARE',
+ 'DEFAULT',
+ 'DEFAULTS',
+ 'DEFERRABLE',
+ 'DEFERRED',
+ 'DEFINER',
+ 'DELETE',
+ 'DELIMITER',
+ 'DELIMITERS',
+ 'DESC',
+ 'DICTIONARY',
+ 'DISABLE',
+ 'DISCARD',
+ 'DISTINCT',
+ 'DO',
+ 'DOCUMENT',
+ 'DOMAIN',
+ 'DOUBLE',
+ 'DROP',
+ 'EACH',
+ 'ELSE',
+ 'ENABLE',
+ 'ENCODING',
+ 'ENCRYPTED',
+ 'END',
+ 'ENUM',
+ 'ESCAPE',
+ 'EVENT',
+ 'EXCEPT',
+ 'EXCLUDE',
+ 'EXCLUDING',
+ 'EXCLUSIVE',
+ 'EXECUTE',
+ 'EXISTS',
+ 'EXPLAIN',
+ 'EXTENSION',
+ 'EXTERNAL',
+ 'EXTRACT',
+ 'FALSE',
+ 'FAMILY',
+ 'FETCH',
+ 'FILTER',
+ 'FIRST',
+ 'FLOAT',
+ 'FOLLOWING',
+ 'FOR',
+ 'FORCE',
+ 'FOREIGN',
+ 'FORWARD',
+ 'FREEZE',
+ 'FROM',
+ 'FULL',
+ 'FUNCTION',
+ 'FUNCTIONS',
+ 'GLOBAL',
+ 'GRANT',
+ 'GRANTED',
+ 'GREATEST',
+ 'GROUP',
+ 'HANDLER',
+ 'HAVING',
+ 'HEADER',
+ 'HOLD',
+ 'HOUR',
+ 'IDENTITY',
+ 'IF',
+ 'ILIKE',
+ 'IMMEDIATE',
+ 'IMMUTABLE',
+ 'IMPLICIT',
+ 'IN',
+ 'INCLUDING',
+ 'INCREMENT',
+ 'INDEX',
+ 'INDEXES',
+ 'INHERIT',
+ 'INHERITS',
+ 'INITIALLY',
+ 'INLINE',
+ 'INNER',
+ 'INOUT',
+ 'INPUT',
+ 'INSENSITIVE',
+ 'INSERT',
+ 'INSTEAD',
+ 'INT',
+ 'INTEGER',
+ 'INTERSECT',
+ 'INTERVAL',
+ 'INTO',
+ 'INVOKER',
+ 'IS',
+ 'ISNULL',
+ 'ISOLATION',
+ 'JOIN',
+ 'KEY',
+ 'LABEL',
+ 'LANGUAGE',
+ 'LARGE',
+ 'LAST',
+ 'LATERAL',
+ 'LC_COLLATE',
+ 'LC_CTYPE',
+ 'LEADING',
+ 'LEAKPROOF',
+ 'LEAST',
+ 'LEFT',
+ 'LEVEL',
+ 'LIKE',
+ 'LIMIT',
+ 'LISTEN',
+ 'LOAD',
+ 'LOCAL',
+ 'LOCALTIME',
+ 'LOCALTIMESTAMP',
+ 'LOCATION',
+ 'LOCK',
+ 'MAPPING',
+ 'MATCH',
+ 'MATERIALIZED',
+ 'MAXVALUE',
+ 'MINUTE',
+ 'MINVALUE',
+ 'MODE',
+ 'MONTH',
+ 'MOVE',
+ 'NAME',
+ 'NAMES',
+ 'NATIONAL',
+ 'NATURAL',
+ 'NCHAR',
+ 'NEXT',
+ 'NO',
+ 'NONE',
+ 'NOT',
+ 'NOTHING',
+ 'NOTIFY',
+ 'NOTNULL',
+ 'NOWAIT',
+ 'NULL',
+ 'NULLIF',
+ 'NULLS',
+ 'NUMERIC',
+ 'OBJECT',
+ 'OF',
+ 'OFF',
+ 'OFFSET',
+ 'OIDS',
+ 'ON',
+ 'ONLY',
+ 'OPERATOR',
+ 'OPTION',
+ 'OPTIONS',
+ 'OR',
+ 'ORDER',
+ 'ORDINALITY',
+ 'OUT',
+ 'OUTER',
+ 'OVER',
+ 'OVERLAPS',
+ 'OVERLAY',
+ 'OWNED',
+ 'OWNER',
+ 'PARSER',
+ 'PARTIAL',
+ 'PARTITION',
+ 'PASSING',
+ 'PASSWORD',
+ 'PLACING',
+ 'PLANS',
+ 'POLICY',
+ 'POSITION',
+ 'PRECEDING',
+ 'PRECISION',
+ 'PREPARE',
+ 'PREPARED',
+ 'PRESERVE',
+ 'PRIMARY',
+ 'PRIOR',
+ 'PRIVILEGES',
+ 'PROCEDURAL',
+ 'PROCEDURE',
+ 'PROGRAM',
+ 'QUOTE',
+ 'RANGE',
+ 'READ',
+ 'REAL',
+ 'REASSIGN',
+ 'RECHECK',
+ 'RECURSIVE',
+ 'REF',
+ 'REFERENCES',
+ 'REFRESH',
+ 'REINDEX',
+ 'RELATIVE',
+ 'RELEASE',
+ 'RENAME',
+ 'REPEATABLE',
+ 'REPLACE',
+ 'REPLICA',
+ 'RESET',
+ 'RESTART',
+ 'RESTRICT',
+ 'RETURNING',
+ 'RETURNS',
+ 'REVOKE',
+ 'RIGHT',
+ 'ROLE',
+ 'ROLLBACK',
+ 'ROW',
+ 'ROWS',
+ 'RULE',
+ 'SAVEPOINT',
+ 'SCHEMA',
+ 'SCROLL',
+ 'SEARCH',
+ 'SECOND',
+ 'SECURITY',
+ 'SELECT',
+ 'SEQUENCE',
+ 'SEQUENCES',
+ 'SERIALIZABLE',
+ 'SERVER',
+ 'SESSION',
+ 'SESSION_USER',
+ 'SET',
+ 'SETOF',
+ 'SHARE',
+ 'SHOW',
+ 'SIMILAR',
+ 'SIMPLE',
+ 'SMALLINT',
+ 'SNAPSHOT',
+ 'SOME',
+ 'STABLE',
+ 'STANDALONE',
+ 'START',
+ 'STATEMENT',
+ 'STATISTICS',
+ 'STDIN',
+ 'STDOUT',
+ 'STORAGE',
+ 'STRICT',
+ 'STRIP',
+ 'SUBSTRING',
+ 'SYMMETRIC',
+ 'SYSID',
+ 'SYSTEM',
+ 'TABLE',
+ 'TABLES',
+ 'TABLESPACE',
+ 'TEMP',
+ 'TEMPLATE',
+ 'TEMPORARY',
+ 'TEXT',
+ 'THEN',
+ 'TIME',
+ 'TIMESTAMP',
+ 'TO',
+ 'TRAILING',
+ 'TRANSACTION',
+ 'TREAT',
+ 'TRIGGER',
+ 'TRIM',
+ 'TRUE',
+ 'TRUNCATE',
+ 'TRUSTED',
+ 'TYPE',
+ 'TYPES',
+ 'UNBOUNDED',
+ 'UNCOMMITTED',
+ 'UNENCRYPTED',
+ 'UNION',
+ 'UNIQUE',
+ 'UNKNOWN',
+ 'UNLISTEN',
+ 'UNLOGGED',
+ 'UNTIL',
+ 'UPDATE',
+ 'USER',
+ 'USING',
+ 'VACUUM',
+ 'VALID',
+ 'VALIDATE',
+ 'VALIDATOR',
+ 'VALUE',
+ 'VALUES',
+ 'VARCHAR',
+ 'VARIADIC',
+ 'VARYING',
+ 'VERBOSE',
+ 'VERSION',
+ 'VIEW',
+ 'VIEWS',
+ 'VOLATILE',
+ 'WHEN',
+ 'WHERE',
+ 'WHITESPACE',
+ 'WINDOW',
+ 'WITH',
+ 'WITHIN',
+ 'WITHOUT',
+ 'WORK',
+ 'WRAPPER',
+ 'WRITE',
+ 'XML',
+ 'XMLATTRIBUTES',
+ 'XMLCONCAT',
+ 'XMLELEMENT',
+ 'XMLEXISTS',
+ 'XMLFOREST',
+ 'XMLPARSE',
+ 'XMLPI',
+ 'XMLROOT',
+ 'XMLSERIALIZE',
+ 'YEAR',
+ 'YES',
+ 'ZONE',
+)
+
+DATATYPES = (
+ 'bigint',
+ 'bigserial',
+ 'bit',
+ 'bit varying',
+ 'bool',
+ 'boolean',
+ 'box',
+ 'bytea',
+ 'char',
+ 'character',
+ 'character varying',
+ 'cidr',
+ 'circle',
+ 'date',
+ 'decimal',
+ 'double precision',
+ 'float4',
+ 'float8',
+ 'inet',
+ 'int',
+ 'int2',
+ 'int4',
+ 'int8',
+ 'integer',
+ 'interval',
+ 'json',
+ 'jsonb',
+ 'line',
+ 'lseg',
+ 'macaddr',
+ 'money',
+ 'numeric',
+ 'path',
+ 'pg_lsn',
+ 'point',
+ 'polygon',
+ 'real',
+ 'serial',
+ 'serial2',
+ 'serial4',
+ 'serial8',
+ 'smallint',
+ 'smallserial',
+ 'text',
+ 'time',
+ 'timestamp',
+ 'timestamptz',
+ 'timetz',
+ 'tsquery',
+ 'tsvector',
+ 'txid_snapshot',
+ 'uuid',
+ 'varbit',
+ 'varchar',
+ 'with time zone',
+ 'without time zone',
+ 'xml',
+)
+
+PSEUDO_TYPES = (
+ 'any',
+ 'anyelement',
+ 'anyarray',
+ 'anynonarray',
+ 'anyenum',
+ 'anyrange',
+ 'cstring',
+ 'internal',
+ 'language_handler',
+ 'fdw_handler',
+ 'record',
+ 'trigger',
+ 'void',
+ 'opaque',
+)
+
+# Remove 'trigger' from types
+PSEUDO_TYPES = tuple(sorted(set(PSEUDO_TYPES) - set(map(str.lower, KEYWORDS))))
+
+PLPGSQL_KEYWORDS = (
+ 'ALIAS', 'CONSTANT', 'DIAGNOSTICS', 'ELSIF', 'EXCEPTION', 'EXIT',
+ 'FOREACH', 'GET', 'LOOP', 'NOTICE', 'OPEN', 'PERFORM', 'QUERY', 'RAISE',
+ 'RETURN', 'REVERSE', 'SQLSTATE', 'WHILE',
+)
+
+
+if __name__ == '__main__': # pragma: no cover
+ import re
+ try:
+ from urllib import urlopen
+ except ImportError:
+ from urllib.request import urlopen
+
+ from pygments.util import format_lines
+
+ # One man's constant is another man's variable.
+ SOURCE_URL = 'https://github.com/postgres/postgres/raw/master'
+ KEYWORDS_URL = SOURCE_URL + '/doc/src/sgml/keywords.sgml'
+ DATATYPES_URL = SOURCE_URL + '/doc/src/sgml/datatype.sgml'
+
+ def update_myself():
+ data_file = list(urlopen(DATATYPES_URL))
+ datatypes = parse_datatypes(data_file)
+ pseudos = parse_pseudos(data_file)
+
+ keywords = parse_keywords(urlopen(KEYWORDS_URL))
+ update_consts(__file__, 'DATATYPES', datatypes)
+ update_consts(__file__, 'PSEUDO_TYPES', pseudos)
+ update_consts(__file__, 'KEYWORDS', keywords)
+
+ def parse_keywords(f):
+ kw = []
+ for m in re.finditer(
+ r'\s*<entry><token>([^<]+)</token></entry>\s*'
+ r'<entry>([^<]+)</entry>', f.read()):
+ kw.append(m.group(1))
+
+ if not kw:
+ raise ValueError('no keyword found')
+
+ kw.sort()
+ return kw
+
+ def parse_datatypes(f):
+ dt = set()
+ for line in f:
+ if '<sect1' in line:
+ break
+ if '<entry><type>' not in line:
+ continue
+
+ # Parse a string such as
+ # time [ (<replaceable>p</replaceable>) ] [ without time zone ]
+ # into types "time" and "without time zone"
+
+ # remove all the tags
+ line = re.sub("<replaceable>[^<]+</replaceable>", "", line)
+ line = re.sub("<[^>]+>", "", line)
+
+ # Drop the parts containing braces
+ for tmp in [t for tmp in line.split('[')
+ for t in tmp.split(']') if "(" not in t]:
+ for t in tmp.split(','):
+ t = t.strip()
+ if not t: continue
+ dt.add(" ".join(t.split()))
+
+ dt = list(dt)
+ dt.sort()
+ return dt
+
+ def parse_pseudos(f):
+ dt = []
+ re_start = re.compile(r'\s*<table id="datatype-pseudotypes-table">')
+ re_entry = re.compile(r'\s*<entry><type>([^<]+)</></entry>')
+ re_end = re.compile(r'\s*</table>')
+
+ f = iter(f)
+ for line in f:
+ if re_start.match(line) is not None:
+ break
+ else:
+ raise ValueError('pseudo datatypes table not found')
+
+ for line in f:
+ m = re_entry.match(line)
+ if m is not None:
+ dt.append(m.group(1))
+
+ if re_end.match(line) is not None:
+ break
+ else:
+ raise ValueError('end of pseudo datatypes table not found')
+
+ if not dt:
+ raise ValueError('pseudo datatypes not found')
+
+ return dt
+
+ def update_consts(filename, constname, content):
+ with open(filename) as f:
+ data = f.read()
+
+ # Line to start/end inserting
+ re_match = re.compile(r'^%s\s*=\s*\($.*?^\s*\)$' % constname, re.M | re.S)
+ m = re_match.search(data)
+ if not m:
+ raise ValueError('Could not find existing definition for %s' %
+ (constname,))
+
+ new_block = format_lines(constname, content)
+ data = data[:m.start()] + new_block + data[m.end():]
+
+ with open(filename, 'w') as f:
+ f.write(data)
+
+ update_myself()
diff --git a/wandb/vendor/pygments/lexers/_scilab_builtins.py b/wandb/vendor/pygments/lexers/_scilab_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..ce0ac67d6f8aa65801fcc4193de9a8605c39e1a8
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_scilab_builtins.py
@@ -0,0 +1,3094 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._scilab_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Builtin list for the ScilabLexer.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+# Autogenerated
+
+commands_kw = (
+ 'abort',
+ 'apropos',
+ 'break',
+ 'case',
+ 'catch',
+ 'continue',
+ 'do',
+ 'else',
+ 'elseif',
+ 'end',
+ 'endfunction',
+ 'for',
+ 'function',
+ 'help',
+ 'if',
+ 'pause',
+ 'quit',
+ 'select',
+ 'then',
+ 'try',
+ 'while',
+)
+
+functions_kw = (
+ '!!_invoke_',
+ '%H5Object_e',
+ '%H5Object_fieldnames',
+ '%H5Object_p',
+ '%XMLAttr_6',
+ '%XMLAttr_e',
+ '%XMLAttr_i_XMLElem',
+ '%XMLAttr_length',
+ '%XMLAttr_p',
+ '%XMLAttr_size',
+ '%XMLDoc_6',
+ '%XMLDoc_e',
+ '%XMLDoc_i_XMLList',
+ '%XMLDoc_p',
+ '%XMLElem_6',
+ '%XMLElem_e',
+ '%XMLElem_i_XMLDoc',
+ '%XMLElem_i_XMLElem',
+ '%XMLElem_i_XMLList',
+ '%XMLElem_p',
+ '%XMLList_6',
+ '%XMLList_e',
+ '%XMLList_i_XMLElem',
+ '%XMLList_i_XMLList',
+ '%XMLList_length',
+ '%XMLList_p',
+ '%XMLList_size',
+ '%XMLNs_6',
+ '%XMLNs_e',
+ '%XMLNs_i_XMLElem',
+ '%XMLNs_p',
+ '%XMLSet_6',
+ '%XMLSet_e',
+ '%XMLSet_length',
+ '%XMLSet_p',
+ '%XMLSet_size',
+ '%XMLValid_p',
+ '%_EClass_6',
+ '%_EClass_e',
+ '%_EClass_p',
+ '%_EObj_0',
+ '%_EObj_1__EObj',
+ '%_EObj_1_b',
+ '%_EObj_1_c',
+ '%_EObj_1_i',
+ '%_EObj_1_s',
+ '%_EObj_2__EObj',
+ '%_EObj_2_b',
+ '%_EObj_2_c',
+ '%_EObj_2_i',
+ '%_EObj_2_s',
+ '%_EObj_3__EObj',
+ '%_EObj_3_b',
+ '%_EObj_3_c',
+ '%_EObj_3_i',
+ '%_EObj_3_s',
+ '%_EObj_4__EObj',
+ '%_EObj_4_b',
+ '%_EObj_4_c',
+ '%_EObj_4_i',
+ '%_EObj_4_s',
+ '%_EObj_5',
+ '%_EObj_6',
+ '%_EObj_a__EObj',
+ '%_EObj_a_b',
+ '%_EObj_a_c',
+ '%_EObj_a_i',
+ '%_EObj_a_s',
+ '%_EObj_d__EObj',
+ '%_EObj_d_b',
+ '%_EObj_d_c',
+ '%_EObj_d_i',
+ '%_EObj_d_s',
+ '%_EObj_disp',
+ '%_EObj_e',
+ '%_EObj_g__EObj',
+ '%_EObj_g_b',
+ '%_EObj_g_c',
+ '%_EObj_g_i',
+ '%_EObj_g_s',
+ '%_EObj_h__EObj',
+ '%_EObj_h_b',
+ '%_EObj_h_c',
+ '%_EObj_h_i',
+ '%_EObj_h_s',
+ '%_EObj_i__EObj',
+ '%_EObj_j__EObj',
+ '%_EObj_j_b',
+ '%_EObj_j_c',
+ '%_EObj_j_i',
+ '%_EObj_j_s',
+ '%_EObj_k__EObj',
+ '%_EObj_k_b',
+ '%_EObj_k_c',
+ '%_EObj_k_i',
+ '%_EObj_k_s',
+ '%_EObj_l__EObj',
+ '%_EObj_l_b',
+ '%_EObj_l_c',
+ '%_EObj_l_i',
+ '%_EObj_l_s',
+ '%_EObj_m__EObj',
+ '%_EObj_m_b',
+ '%_EObj_m_c',
+ '%_EObj_m_i',
+ '%_EObj_m_s',
+ '%_EObj_n__EObj',
+ '%_EObj_n_b',
+ '%_EObj_n_c',
+ '%_EObj_n_i',
+ '%_EObj_n_s',
+ '%_EObj_o__EObj',
+ '%_EObj_o_b',
+ '%_EObj_o_c',
+ '%_EObj_o_i',
+ '%_EObj_o_s',
+ '%_EObj_p',
+ '%_EObj_p__EObj',
+ '%_EObj_p_b',
+ '%_EObj_p_c',
+ '%_EObj_p_i',
+ '%_EObj_p_s',
+ '%_EObj_q__EObj',
+ '%_EObj_q_b',
+ '%_EObj_q_c',
+ '%_EObj_q_i',
+ '%_EObj_q_s',
+ '%_EObj_r__EObj',
+ '%_EObj_r_b',
+ '%_EObj_r_c',
+ '%_EObj_r_i',
+ '%_EObj_r_s',
+ '%_EObj_s__EObj',
+ '%_EObj_s_b',
+ '%_EObj_s_c',
+ '%_EObj_s_i',
+ '%_EObj_s_s',
+ '%_EObj_t',
+ '%_EObj_x__EObj',
+ '%_EObj_x_b',
+ '%_EObj_x_c',
+ '%_EObj_x_i',
+ '%_EObj_x_s',
+ '%_EObj_y__EObj',
+ '%_EObj_y_b',
+ '%_EObj_y_c',
+ '%_EObj_y_i',
+ '%_EObj_y_s',
+ '%_EObj_z__EObj',
+ '%_EObj_z_b',
+ '%_EObj_z_c',
+ '%_EObj_z_i',
+ '%_EObj_z_s',
+ '%_eigs',
+ '%_load',
+ '%b_1__EObj',
+ '%b_2__EObj',
+ '%b_3__EObj',
+ '%b_4__EObj',
+ '%b_a__EObj',
+ '%b_d__EObj',
+ '%b_g__EObj',
+ '%b_h__EObj',
+ '%b_i_XMLList',
+ '%b_i__EObj',
+ '%b_j__EObj',
+ '%b_k__EObj',
+ '%b_l__EObj',
+ '%b_m__EObj',
+ '%b_n__EObj',
+ '%b_o__EObj',
+ '%b_p__EObj',
+ '%b_q__EObj',
+ '%b_r__EObj',
+ '%b_s__EObj',
+ '%b_x__EObj',
+ '%b_y__EObj',
+ '%b_z__EObj',
+ '%c_1__EObj',
+ '%c_2__EObj',
+ '%c_3__EObj',
+ '%c_4__EObj',
+ '%c_a__EObj',
+ '%c_d__EObj',
+ '%c_g__EObj',
+ '%c_h__EObj',
+ '%c_i_XMLAttr',
+ '%c_i_XMLDoc',
+ '%c_i_XMLElem',
+ '%c_i_XMLList',
+ '%c_i__EObj',
+ '%c_j__EObj',
+ '%c_k__EObj',
+ '%c_l__EObj',
+ '%c_m__EObj',
+ '%c_n__EObj',
+ '%c_o__EObj',
+ '%c_p__EObj',
+ '%c_q__EObj',
+ '%c_r__EObj',
+ '%c_s__EObj',
+ '%c_x__EObj',
+ '%c_y__EObj',
+ '%c_z__EObj',
+ '%ce_i_XMLList',
+ '%fptr_i_XMLList',
+ '%h_i_XMLList',
+ '%hm_i_XMLList',
+ '%i_1__EObj',
+ '%i_2__EObj',
+ '%i_3__EObj',
+ '%i_4__EObj',
+ '%i_a__EObj',
+ '%i_abs',
+ '%i_cumprod',
+ '%i_cumsum',
+ '%i_d__EObj',
+ '%i_diag',
+ '%i_g__EObj',
+ '%i_h__EObj',
+ '%i_i_XMLList',
+ '%i_i__EObj',
+ '%i_j__EObj',
+ '%i_k__EObj',
+ '%i_l__EObj',
+ '%i_m__EObj',
+ '%i_matrix',
+ '%i_max',
+ '%i_maxi',
+ '%i_min',
+ '%i_mini',
+ '%i_mput',
+ '%i_n__EObj',
+ '%i_o__EObj',
+ '%i_p',
+ '%i_p__EObj',
+ '%i_prod',
+ '%i_q__EObj',
+ '%i_r__EObj',
+ '%i_s__EObj',
+ '%i_sum',
+ '%i_tril',
+ '%i_triu',
+ '%i_x__EObj',
+ '%i_y__EObj',
+ '%i_z__EObj',
+ '%ip_i_XMLList',
+ '%l_i_XMLList',
+ '%l_i__EObj',
+ '%lss_i_XMLList',
+ '%mc_i_XMLList',
+ '%msp_full',
+ '%msp_i_XMLList',
+ '%msp_spget',
+ '%p_i_XMLList',
+ '%ptr_i_XMLList',
+ '%r_i_XMLList',
+ '%s_1__EObj',
+ '%s_2__EObj',
+ '%s_3__EObj',
+ '%s_4__EObj',
+ '%s_a__EObj',
+ '%s_d__EObj',
+ '%s_g__EObj',
+ '%s_h__EObj',
+ '%s_i_XMLList',
+ '%s_i__EObj',
+ '%s_j__EObj',
+ '%s_k__EObj',
+ '%s_l__EObj',
+ '%s_m__EObj',
+ '%s_n__EObj',
+ '%s_o__EObj',
+ '%s_p__EObj',
+ '%s_q__EObj',
+ '%s_r__EObj',
+ '%s_s__EObj',
+ '%s_x__EObj',
+ '%s_y__EObj',
+ '%s_z__EObj',
+ '%sp_i_XMLList',
+ '%spb_i_XMLList',
+ '%st_i_XMLList',
+ 'Calendar',
+ 'ClipBoard',
+ 'Matplot',
+ 'Matplot1',
+ 'PlaySound',
+ 'TCL_DeleteInterp',
+ 'TCL_DoOneEvent',
+ 'TCL_EvalFile',
+ 'TCL_EvalStr',
+ 'TCL_ExistArray',
+ 'TCL_ExistInterp',
+ 'TCL_ExistVar',
+ 'TCL_GetVar',
+ 'TCL_GetVersion',
+ 'TCL_SetVar',
+ 'TCL_UnsetVar',
+ 'TCL_UpVar',
+ '_',
+ '_code2str',
+ '_d',
+ '_str2code',
+ 'about',
+ 'abs',
+ 'acos',
+ 'addModulePreferences',
+ 'addcolor',
+ 'addf',
+ 'addhistory',
+ 'addinter',
+ 'addlocalizationdomain',
+ 'amell',
+ 'and',
+ 'argn',
+ 'arl2_ius',
+ 'ascii',
+ 'asin',
+ 'atan',
+ 'backslash',
+ 'balanc',
+ 'banner',
+ 'base2dec',
+ 'basename',
+ 'bdiag',
+ 'beep',
+ 'besselh',
+ 'besseli',
+ 'besselj',
+ 'besselk',
+ 'bessely',
+ 'beta',
+ 'bezout',
+ 'bfinit',
+ 'blkfc1i',
+ 'blkslvi',
+ 'bool2s',
+ 'browsehistory',
+ 'browsevar',
+ 'bsplin3val',
+ 'buildDoc',
+ 'buildouttb',
+ 'bvode',
+ 'c_link',
+ 'call',
+ 'callblk',
+ 'captions',
+ 'cd',
+ 'cdfbet',
+ 'cdfbin',
+ 'cdfchi',
+ 'cdfchn',
+ 'cdff',
+ 'cdffnc',
+ 'cdfgam',
+ 'cdfnbn',
+ 'cdfnor',
+ 'cdfpoi',
+ 'cdft',
+ 'ceil',
+ 'champ',
+ 'champ1',
+ 'chdir',
+ 'chol',
+ 'clc',
+ 'clean',
+ 'clear',
+ 'clearfun',
+ 'clearglobal',
+ 'closeEditor',
+ 'closeEditvar',
+ 'closeXcos',
+ 'code2str',
+ 'coeff',
+ 'color',
+ 'comp',
+ 'completion',
+ 'conj',
+ 'contour2di',
+ 'contr',
+ 'conv2',
+ 'convstr',
+ 'copy',
+ 'copyfile',
+ 'corr',
+ 'cos',
+ 'coserror',
+ 'createdir',
+ 'cshep2d',
+ 'csvDefault',
+ 'csvIsnum',
+ 'csvRead',
+ 'csvStringToDouble',
+ 'csvTextScan',
+ 'csvWrite',
+ 'ctree2',
+ 'ctree3',
+ 'ctree4',
+ 'cumprod',
+ 'cumsum',
+ 'curblock',
+ 'curblockc',
+ 'daskr',
+ 'dasrt',
+ 'dassl',
+ 'data2sig',
+ 'datatipCreate',
+ 'datatipManagerMode',
+ 'datatipMove',
+ 'datatipRemove',
+ 'datatipSetDisplay',
+ 'datatipSetInterp',
+ 'datatipSetOrientation',
+ 'datatipSetStyle',
+ 'datatipToggle',
+ 'dawson',
+ 'dct',
+ 'debug',
+ 'dec2base',
+ 'deff',
+ 'definedfields',
+ 'degree',
+ 'delbpt',
+ 'delete',
+ 'deletefile',
+ 'delip',
+ 'delmenu',
+ 'det',
+ 'dgettext',
+ 'dhinf',
+ 'diag',
+ 'diary',
+ 'diffobjs',
+ 'disp',
+ 'dispbpt',
+ 'displayhistory',
+ 'disposefftwlibrary',
+ 'dlgamma',
+ 'dnaupd',
+ 'dneupd',
+ 'double',
+ 'drawaxis',
+ 'drawlater',
+ 'drawnow',
+ 'driver',
+ 'dsaupd',
+ 'dsearch',
+ 'dseupd',
+ 'dst',
+ 'duplicate',
+ 'editvar',
+ 'emptystr',
+ 'end_scicosim',
+ 'ereduc',
+ 'erf',
+ 'erfc',
+ 'erfcx',
+ 'erfi',
+ 'errcatch',
+ 'errclear',
+ 'error',
+ 'eval_cshep2d',
+ 'exec',
+ 'execstr',
+ 'exists',
+ 'exit',
+ 'exp',
+ 'expm',
+ 'exportUI',
+ 'export_to_hdf5',
+ 'eye',
+ 'fadj2sp',
+ 'fec',
+ 'feval',
+ 'fft',
+ 'fftw',
+ 'fftw_flags',
+ 'fftw_forget_wisdom',
+ 'fftwlibraryisloaded',
+ 'figure',
+ 'file',
+ 'filebrowser',
+ 'fileext',
+ 'fileinfo',
+ 'fileparts',
+ 'filesep',
+ 'find',
+ 'findBD',
+ 'findfiles',
+ 'fire_closing_finished',
+ 'floor',
+ 'format',
+ 'fort',
+ 'fprintfMat',
+ 'freq',
+ 'frexp',
+ 'fromc',
+ 'fromjava',
+ 'fscanfMat',
+ 'fsolve',
+ 'fstair',
+ 'full',
+ 'fullpath',
+ 'funcprot',
+ 'funptr',
+ 'gamma',
+ 'gammaln',
+ 'geom3d',
+ 'get',
+ 'getURL',
+ 'get_absolute_file_path',
+ 'get_fftw_wisdom',
+ 'getblocklabel',
+ 'getcallbackobject',
+ 'getdate',
+ 'getdebuginfo',
+ 'getdefaultlanguage',
+ 'getdrives',
+ 'getdynlibext',
+ 'getenv',
+ 'getfield',
+ 'gethistory',
+ 'gethistoryfile',
+ 'getinstalledlookandfeels',
+ 'getio',
+ 'getlanguage',
+ 'getlongpathname',
+ 'getlookandfeel',
+ 'getmd5',
+ 'getmemory',
+ 'getmodules',
+ 'getos',
+ 'getpid',
+ 'getrelativefilename',
+ 'getscicosvars',
+ 'getscilabmode',
+ 'getshortpathname',
+ 'gettext',
+ 'getvariablesonstack',
+ 'getversion',
+ 'glist',
+ 'global',
+ 'glue',
+ 'grand',
+ 'graphicfunction',
+ 'grayplot',
+ 'grep',
+ 'gsort',
+ 'gstacksize',
+ 'h5attr',
+ 'h5close',
+ 'h5cp',
+ 'h5dataset',
+ 'h5dump',
+ 'h5exists',
+ 'h5flush',
+ 'h5get',
+ 'h5group',
+ 'h5isArray',
+ 'h5isAttr',
+ 'h5isCompound',
+ 'h5isFile',
+ 'h5isGroup',
+ 'h5isList',
+ 'h5isRef',
+ 'h5isSet',
+ 'h5isSpace',
+ 'h5isType',
+ 'h5isVlen',
+ 'h5label',
+ 'h5ln',
+ 'h5ls',
+ 'h5mount',
+ 'h5mv',
+ 'h5open',
+ 'h5read',
+ 'h5readattr',
+ 'h5rm',
+ 'h5umount',
+ 'h5write',
+ 'h5writeattr',
+ 'havewindow',
+ 'helpbrowser',
+ 'hess',
+ 'hinf',
+ 'historymanager',
+ 'historysize',
+ 'host',
+ 'htmlDump',
+ 'htmlRead',
+ 'htmlReadStr',
+ 'htmlWrite',
+ 'iconvert',
+ 'ieee',
+ 'ilib_verbose',
+ 'imag',
+ 'impl',
+ 'import_from_hdf5',
+ 'imult',
+ 'inpnvi',
+ 'int',
+ 'int16',
+ 'int2d',
+ 'int32',
+ 'int3d',
+ 'int8',
+ 'interp',
+ 'interp2d',
+ 'interp3d',
+ 'intg',
+ 'intppty',
+ 'inttype',
+ 'inv',
+ 'invoke_lu',
+ 'is_handle_valid',
+ 'is_hdf5_file',
+ 'isalphanum',
+ 'isascii',
+ 'isdef',
+ 'isdigit',
+ 'isdir',
+ 'isequal',
+ 'isequalbitwise',
+ 'iserror',
+ 'isfile',
+ 'isglobal',
+ 'isletter',
+ 'isnum',
+ 'isreal',
+ 'iswaitingforinput',
+ 'jallowClassReloading',
+ 'jarray',
+ 'jautoTranspose',
+ 'jautoUnwrap',
+ 'javaclasspath',
+ 'javalibrarypath',
+ 'jcast',
+ 'jcompile',
+ 'jconvMatrixMethod',
+ 'jcreatejar',
+ 'jdeff',
+ 'jdisableTrace',
+ 'jenableTrace',
+ 'jexists',
+ 'jgetclassname',
+ 'jgetfield',
+ 'jgetfields',
+ 'jgetinfo',
+ 'jgetmethods',
+ 'jimport',
+ 'jinvoke',
+ 'jinvoke_db',
+ 'jnewInstance',
+ 'jremove',
+ 'jsetfield',
+ 'junwrap',
+ 'junwraprem',
+ 'jwrap',
+ 'jwrapinfloat',
+ 'kron',
+ 'lasterror',
+ 'ldiv',
+ 'ldivf',
+ 'legendre',
+ 'length',
+ 'lib',
+ 'librarieslist',
+ 'libraryinfo',
+ 'light',
+ 'linear_interpn',
+ 'lines',
+ 'link',
+ 'linmeq',
+ 'list',
+ 'listvar_in_hdf5',
+ 'load',
+ 'loadGui',
+ 'loadScicos',
+ 'loadXcos',
+ 'loadfftwlibrary',
+ 'loadhistory',
+ 'log',
+ 'log1p',
+ 'lsq',
+ 'lsq_splin',
+ 'lsqrsolve',
+ 'lsslist',
+ 'lstcat',
+ 'lstsize',
+ 'ltitr',
+ 'lu',
+ 'ludel',
+ 'lufact',
+ 'luget',
+ 'lusolve',
+ 'macr2lst',
+ 'macr2tree',
+ 'matfile_close',
+ 'matfile_listvar',
+ 'matfile_open',
+ 'matfile_varreadnext',
+ 'matfile_varwrite',
+ 'matrix',
+ 'max',
+ 'maxfiles',
+ 'mclearerr',
+ 'mclose',
+ 'meof',
+ 'merror',
+ 'messagebox',
+ 'mfprintf',
+ 'mfscanf',
+ 'mget',
+ 'mgeti',
+ 'mgetl',
+ 'mgetstr',
+ 'min',
+ 'mlist',
+ 'mode',
+ 'model2blk',
+ 'mopen',
+ 'move',
+ 'movefile',
+ 'mprintf',
+ 'mput',
+ 'mputl',
+ 'mputstr',
+ 'mscanf',
+ 'mseek',
+ 'msprintf',
+ 'msscanf',
+ 'mtell',
+ 'mtlb_mode',
+ 'mtlb_sparse',
+ 'mucomp',
+ 'mulf',
+ 'name2rgb',
+ 'nearfloat',
+ 'newaxes',
+ 'newest',
+ 'newfun',
+ 'nnz',
+ 'norm',
+ 'notify',
+ 'number_properties',
+ 'ode',
+ 'odedc',
+ 'ones',
+ 'openged',
+ 'opentk',
+ 'optim',
+ 'or',
+ 'ordmmd',
+ 'parallel_concurrency',
+ 'parallel_run',
+ 'param3d',
+ 'param3d1',
+ 'part',
+ 'pathconvert',
+ 'pathsep',
+ 'phase_simulation',
+ 'plot2d',
+ 'plot2d1',
+ 'plot2d2',
+ 'plot2d3',
+ 'plot2d4',
+ 'plot3d',
+ 'plot3d1',
+ 'plotbrowser',
+ 'pointer_xproperty',
+ 'poly',
+ 'ppol',
+ 'pppdiv',
+ 'predef',
+ 'preferences',
+ 'print',
+ 'printf',
+ 'printfigure',
+ 'printsetupbox',
+ 'prod',
+ 'progressionbar',
+ 'prompt',
+ 'pwd',
+ 'qld',
+ 'qp_solve',
+ 'qr',
+ 'raise_window',
+ 'rand',
+ 'rankqr',
+ 'rat',
+ 'rcond',
+ 'rdivf',
+ 'read',
+ 'read4b',
+ 'read_csv',
+ 'readb',
+ 'readgateway',
+ 'readmps',
+ 'real',
+ 'realtime',
+ 'realtimeinit',
+ 'regexp',
+ 'relocate_handle',
+ 'remez',
+ 'removeModulePreferences',
+ 'removedir',
+ 'removelinehistory',
+ 'res_with_prec',
+ 'resethistory',
+ 'residu',
+ 'resume',
+ 'return',
+ 'ricc',
+ 'rlist',
+ 'roots',
+ 'rotate_axes',
+ 'round',
+ 'rpem',
+ 'rtitr',
+ 'rubberbox',
+ 'save',
+ 'saveGui',
+ 'saveafterncommands',
+ 'saveconsecutivecommands',
+ 'savehistory',
+ 'schur',
+ 'sci_haltscicos',
+ 'sci_tree2',
+ 'sci_tree3',
+ 'sci_tree4',
+ 'sciargs',
+ 'scicos_debug',
+ 'scicos_debug_count',
+ 'scicos_time',
+ 'scicosim',
+ 'scinotes',
+ 'sctree',
+ 'semidef',
+ 'set',
+ 'set_blockerror',
+ 'set_fftw_wisdom',
+ 'set_xproperty',
+ 'setbpt',
+ 'setdefaultlanguage',
+ 'setenv',
+ 'setfield',
+ 'sethistoryfile',
+ 'setlanguage',
+ 'setlookandfeel',
+ 'setmenu',
+ 'sfact',
+ 'sfinit',
+ 'show_window',
+ 'sident',
+ 'sig2data',
+ 'sign',
+ 'simp',
+ 'simp_mode',
+ 'sin',
+ 'size',
+ 'slash',
+ 'sleep',
+ 'sorder',
+ 'sparse',
+ 'spchol',
+ 'spcompack',
+ 'spec',
+ 'spget',
+ 'splin',
+ 'splin2d',
+ 'splin3d',
+ 'splitURL',
+ 'spones',
+ 'sprintf',
+ 'sqrt',
+ 'stacksize',
+ 'str2code',
+ 'strcat',
+ 'strchr',
+ 'strcmp',
+ 'strcspn',
+ 'strindex',
+ 'string',
+ 'stringbox',
+ 'stripblanks',
+ 'strncpy',
+ 'strrchr',
+ 'strrev',
+ 'strsplit',
+ 'strspn',
+ 'strstr',
+ 'strsubst',
+ 'strtod',
+ 'strtok',
+ 'subf',
+ 'sum',
+ 'svd',
+ 'swap_handles',
+ 'symfcti',
+ 'syredi',
+ 'system_getproperty',
+ 'system_setproperty',
+ 'ta2lpd',
+ 'tan',
+ 'taucs_chdel',
+ 'taucs_chfact',
+ 'taucs_chget',
+ 'taucs_chinfo',
+ 'taucs_chsolve',
+ 'tempname',
+ 'testmatrix',
+ 'timer',
+ 'tlist',
+ 'tohome',
+ 'tokens',
+ 'toolbar',
+ 'toprint',
+ 'tr_zer',
+ 'tril',
+ 'triu',
+ 'type',
+ 'typename',
+ 'uiDisplayTree',
+ 'uicontextmenu',
+ 'uicontrol',
+ 'uigetcolor',
+ 'uigetdir',
+ 'uigetfile',
+ 'uigetfont',
+ 'uimenu',
+ 'uint16',
+ 'uint32',
+ 'uint8',
+ 'uipopup',
+ 'uiputfile',
+ 'uiwait',
+ 'ulink',
+ 'umf_ludel',
+ 'umf_lufact',
+ 'umf_luget',
+ 'umf_luinfo',
+ 'umf_lusolve',
+ 'umfpack',
+ 'unglue',
+ 'unix',
+ 'unsetmenu',
+ 'unzoom',
+ 'updatebrowsevar',
+ 'usecanvas',
+ 'useeditor',
+ 'user',
+ 'var2vec',
+ 'varn',
+ 'vec2var',
+ 'waitbar',
+ 'warnBlockByUID',
+ 'warning',
+ 'what',
+ 'where',
+ 'whereis',
+ 'who',
+ 'winsid',
+ 'with_module',
+ 'writb',
+ 'write',
+ 'write4b',
+ 'write_csv',
+ 'x_choose',
+ 'x_choose_modeless',
+ 'x_dialog',
+ 'x_mdialog',
+ 'xarc',
+ 'xarcs',
+ 'xarrows',
+ 'xchange',
+ 'xchoicesi',
+ 'xclick',
+ 'xcos',
+ 'xcosAddToolsMenu',
+ 'xcosConfigureXmlFile',
+ 'xcosDiagramToScilab',
+ 'xcosPalCategoryAdd',
+ 'xcosPalDelete',
+ 'xcosPalDisable',
+ 'xcosPalEnable',
+ 'xcosPalGenerateIcon',
+ 'xcosPalGet',
+ 'xcosPalLoad',
+ 'xcosPalMove',
+ 'xcosSimulationStarted',
+ 'xcosUpdateBlock',
+ 'xdel',
+ 'xend',
+ 'xfarc',
+ 'xfarcs',
+ 'xfpoly',
+ 'xfpolys',
+ 'xfrect',
+ 'xget',
+ 'xgetmouse',
+ 'xgraduate',
+ 'xgrid',
+ 'xinit',
+ 'xlfont',
+ 'xls_open',
+ 'xls_read',
+ 'xmlAddNs',
+ 'xmlAppend',
+ 'xmlAsNumber',
+ 'xmlAsText',
+ 'xmlDTD',
+ 'xmlDelete',
+ 'xmlDocument',
+ 'xmlDump',
+ 'xmlElement',
+ 'xmlFormat',
+ 'xmlGetNsByHref',
+ 'xmlGetNsByPrefix',
+ 'xmlGetOpenDocs',
+ 'xmlIsValidObject',
+ 'xmlName',
+ 'xmlNs',
+ 'xmlRead',
+ 'xmlReadStr',
+ 'xmlRelaxNG',
+ 'xmlRemove',
+ 'xmlSchema',
+ 'xmlSetAttributes',
+ 'xmlValidate',
+ 'xmlWrite',
+ 'xmlXPath',
+ 'xname',
+ 'xpause',
+ 'xpoly',
+ 'xpolys',
+ 'xrect',
+ 'xrects',
+ 'xs2bmp',
+ 'xs2emf',
+ 'xs2eps',
+ 'xs2gif',
+ 'xs2jpg',
+ 'xs2pdf',
+ 'xs2png',
+ 'xs2ppm',
+ 'xs2ps',
+ 'xs2svg',
+ 'xsegs',
+ 'xset',
+ 'xstring',
+ 'xstringb',
+ 'xtitle',
+ 'zeros',
+ 'znaupd',
+ 'zneupd',
+ 'zoom_rect',
+)
+
+macros_kw = (
+ '!_deff_wrapper',
+ '%0_i_st',
+ '%3d_i_h',
+ '%Block_xcosUpdateBlock',
+ '%TNELDER_p',
+ '%TNELDER_string',
+ '%TNMPLOT_p',
+ '%TNMPLOT_string',
+ '%TOPTIM_p',
+ '%TOPTIM_string',
+ '%TSIMPLEX_p',
+ '%TSIMPLEX_string',
+ '%_EVoid_p',
+ '%_gsort',
+ '%_listvarinfile',
+ '%_rlist',
+ '%_save',
+ '%_sodload',
+ '%_strsplit',
+ '%_unwrap',
+ '%ar_p',
+ '%asn',
+ '%b_a_b',
+ '%b_a_s',
+ '%b_c_s',
+ '%b_c_spb',
+ '%b_cumprod',
+ '%b_cumsum',
+ '%b_d_s',
+ '%b_diag',
+ '%b_e',
+ '%b_f_s',
+ '%b_f_spb',
+ '%b_g_s',
+ '%b_g_spb',
+ '%b_grand',
+ '%b_h_s',
+ '%b_h_spb',
+ '%b_i_b',
+ '%b_i_ce',
+ '%b_i_h',
+ '%b_i_hm',
+ '%b_i_s',
+ '%b_i_sp',
+ '%b_i_spb',
+ '%b_i_st',
+ '%b_iconvert',
+ '%b_l_b',
+ '%b_l_s',
+ '%b_m_b',
+ '%b_m_s',
+ '%b_matrix',
+ '%b_n_hm',
+ '%b_o_hm',
+ '%b_p_s',
+ '%b_prod',
+ '%b_r_b',
+ '%b_r_s',
+ '%b_s_b',
+ '%b_s_s',
+ '%b_string',
+ '%b_sum',
+ '%b_tril',
+ '%b_triu',
+ '%b_x_b',
+ '%b_x_s',
+ '%bicg',
+ '%bicgstab',
+ '%c_a_c',
+ '%c_b_c',
+ '%c_b_s',
+ '%c_diag',
+ '%c_dsearch',
+ '%c_e',
+ '%c_eye',
+ '%c_f_s',
+ '%c_grand',
+ '%c_i_c',
+ '%c_i_ce',
+ '%c_i_h',
+ '%c_i_hm',
+ '%c_i_lss',
+ '%c_i_r',
+ '%c_i_s',
+ '%c_i_st',
+ '%c_matrix',
+ '%c_n_l',
+ '%c_n_st',
+ '%c_o_l',
+ '%c_o_st',
+ '%c_ones',
+ '%c_rand',
+ '%c_tril',
+ '%c_triu',
+ '%cblock_c_cblock',
+ '%cblock_c_s',
+ '%cblock_e',
+ '%cblock_f_cblock',
+ '%cblock_p',
+ '%cblock_size',
+ '%ce_6',
+ '%ce_c_ce',
+ '%ce_e',
+ '%ce_f_ce',
+ '%ce_i_ce',
+ '%ce_i_s',
+ '%ce_i_st',
+ '%ce_matrix',
+ '%ce_p',
+ '%ce_size',
+ '%ce_string',
+ '%ce_t',
+ '%cgs',
+ '%champdat_i_h',
+ '%choose',
+ '%diagram_xcos',
+ '%dir_p',
+ '%fptr_i_st',
+ '%grand_perm',
+ '%grayplot_i_h',
+ '%h_i_st',
+ '%hmS_k_hmS_generic',
+ '%hm_1_hm',
+ '%hm_1_s',
+ '%hm_2_hm',
+ '%hm_2_s',
+ '%hm_3_hm',
+ '%hm_3_s',
+ '%hm_4_hm',
+ '%hm_4_s',
+ '%hm_5',
+ '%hm_a_hm',
+ '%hm_a_r',
+ '%hm_a_s',
+ '%hm_abs',
+ '%hm_and',
+ '%hm_bool2s',
+ '%hm_c_hm',
+ '%hm_ceil',
+ '%hm_conj',
+ '%hm_cos',
+ '%hm_cumprod',
+ '%hm_cumsum',
+ '%hm_d_hm',
+ '%hm_d_s',
+ '%hm_degree',
+ '%hm_dsearch',
+ '%hm_e',
+ '%hm_exp',
+ '%hm_eye',
+ '%hm_f_hm',
+ '%hm_find',
+ '%hm_floor',
+ '%hm_g_hm',
+ '%hm_grand',
+ '%hm_gsort',
+ '%hm_h_hm',
+ '%hm_i_b',
+ '%hm_i_ce',
+ '%hm_i_h',
+ '%hm_i_hm',
+ '%hm_i_i',
+ '%hm_i_p',
+ '%hm_i_r',
+ '%hm_i_s',
+ '%hm_i_st',
+ '%hm_iconvert',
+ '%hm_imag',
+ '%hm_int',
+ '%hm_isnan',
+ '%hm_isreal',
+ '%hm_j_hm',
+ '%hm_j_s',
+ '%hm_k_hm',
+ '%hm_k_s',
+ '%hm_log',
+ '%hm_m_p',
+ '%hm_m_r',
+ '%hm_m_s',
+ '%hm_matrix',
+ '%hm_max',
+ '%hm_mean',
+ '%hm_median',
+ '%hm_min',
+ '%hm_n_b',
+ '%hm_n_c',
+ '%hm_n_hm',
+ '%hm_n_i',
+ '%hm_n_p',
+ '%hm_n_s',
+ '%hm_o_b',
+ '%hm_o_c',
+ '%hm_o_hm',
+ '%hm_o_i',
+ '%hm_o_p',
+ '%hm_o_s',
+ '%hm_ones',
+ '%hm_or',
+ '%hm_p',
+ '%hm_prod',
+ '%hm_q_hm',
+ '%hm_r_s',
+ '%hm_rand',
+ '%hm_real',
+ '%hm_round',
+ '%hm_s',
+ '%hm_s_hm',
+ '%hm_s_r',
+ '%hm_s_s',
+ '%hm_sign',
+ '%hm_sin',
+ '%hm_size',
+ '%hm_sqrt',
+ '%hm_stdev',
+ '%hm_string',
+ '%hm_sum',
+ '%hm_x_hm',
+ '%hm_x_p',
+ '%hm_x_s',
+ '%hm_zeros',
+ '%i_1_s',
+ '%i_2_s',
+ '%i_3_s',
+ '%i_4_s',
+ '%i_Matplot',
+ '%i_a_i',
+ '%i_a_s',
+ '%i_and',
+ '%i_ascii',
+ '%i_b_s',
+ '%i_bezout',
+ '%i_champ',
+ '%i_champ1',
+ '%i_contour',
+ '%i_contour2d',
+ '%i_d_i',
+ '%i_d_s',
+ '%i_dsearch',
+ '%i_e',
+ '%i_fft',
+ '%i_g_i',
+ '%i_gcd',
+ '%i_grand',
+ '%i_h_i',
+ '%i_i_ce',
+ '%i_i_h',
+ '%i_i_hm',
+ '%i_i_i',
+ '%i_i_s',
+ '%i_i_st',
+ '%i_j_i',
+ '%i_j_s',
+ '%i_l_s',
+ '%i_lcm',
+ '%i_length',
+ '%i_m_i',
+ '%i_m_s',
+ '%i_mfprintf',
+ '%i_mprintf',
+ '%i_msprintf',
+ '%i_n_s',
+ '%i_o_s',
+ '%i_or',
+ '%i_p_i',
+ '%i_p_s',
+ '%i_plot2d',
+ '%i_plot2d1',
+ '%i_plot2d2',
+ '%i_q_s',
+ '%i_r_i',
+ '%i_r_s',
+ '%i_round',
+ '%i_s_i',
+ '%i_s_s',
+ '%i_sign',
+ '%i_string',
+ '%i_x_i',
+ '%i_x_s',
+ '%ip_a_s',
+ '%ip_i_st',
+ '%ip_m_s',
+ '%ip_n_ip',
+ '%ip_o_ip',
+ '%ip_p',
+ '%ip_part',
+ '%ip_s_s',
+ '%ip_string',
+ '%k',
+ '%l_i_h',
+ '%l_i_s',
+ '%l_i_st',
+ '%l_isequal',
+ '%l_n_c',
+ '%l_n_l',
+ '%l_n_m',
+ '%l_n_p',
+ '%l_n_s',
+ '%l_n_st',
+ '%l_o_c',
+ '%l_o_l',
+ '%l_o_m',
+ '%l_o_p',
+ '%l_o_s',
+ '%l_o_st',
+ '%lss_a_lss',
+ '%lss_a_p',
+ '%lss_a_r',
+ '%lss_a_s',
+ '%lss_c_lss',
+ '%lss_c_p',
+ '%lss_c_r',
+ '%lss_c_s',
+ '%lss_e',
+ '%lss_eye',
+ '%lss_f_lss',
+ '%lss_f_p',
+ '%lss_f_r',
+ '%lss_f_s',
+ '%lss_i_ce',
+ '%lss_i_lss',
+ '%lss_i_p',
+ '%lss_i_r',
+ '%lss_i_s',
+ '%lss_i_st',
+ '%lss_inv',
+ '%lss_l_lss',
+ '%lss_l_p',
+ '%lss_l_r',
+ '%lss_l_s',
+ '%lss_m_lss',
+ '%lss_m_p',
+ '%lss_m_r',
+ '%lss_m_s',
+ '%lss_n_lss',
+ '%lss_n_p',
+ '%lss_n_r',
+ '%lss_n_s',
+ '%lss_norm',
+ '%lss_o_lss',
+ '%lss_o_p',
+ '%lss_o_r',
+ '%lss_o_s',
+ '%lss_ones',
+ '%lss_r_lss',
+ '%lss_r_p',
+ '%lss_r_r',
+ '%lss_r_s',
+ '%lss_rand',
+ '%lss_s',
+ '%lss_s_lss',
+ '%lss_s_p',
+ '%lss_s_r',
+ '%lss_s_s',
+ '%lss_size',
+ '%lss_t',
+ '%lss_v_lss',
+ '%lss_v_p',
+ '%lss_v_r',
+ '%lss_v_s',
+ '%lt_i_s',
+ '%m_n_l',
+ '%m_o_l',
+ '%mc_i_h',
+ '%mc_i_s',
+ '%mc_i_st',
+ '%mc_n_st',
+ '%mc_o_st',
+ '%mc_string',
+ '%mps_p',
+ '%mps_string',
+ '%msp_a_s',
+ '%msp_abs',
+ '%msp_e',
+ '%msp_find',
+ '%msp_i_s',
+ '%msp_i_st',
+ '%msp_length',
+ '%msp_m_s',
+ '%msp_maxi',
+ '%msp_n_msp',
+ '%msp_nnz',
+ '%msp_o_msp',
+ '%msp_p',
+ '%msp_sparse',
+ '%msp_spones',
+ '%msp_t',
+ '%p_a_lss',
+ '%p_a_r',
+ '%p_c_lss',
+ '%p_c_r',
+ '%p_cumprod',
+ '%p_cumsum',
+ '%p_d_p',
+ '%p_d_r',
+ '%p_d_s',
+ '%p_det',
+ '%p_e',
+ '%p_f_lss',
+ '%p_f_r',
+ '%p_grand',
+ '%p_i_ce',
+ '%p_i_h',
+ '%p_i_hm',
+ '%p_i_lss',
+ '%p_i_p',
+ '%p_i_r',
+ '%p_i_s',
+ '%p_i_st',
+ '%p_inv',
+ '%p_j_s',
+ '%p_k_p',
+ '%p_k_r',
+ '%p_k_s',
+ '%p_l_lss',
+ '%p_l_p',
+ '%p_l_r',
+ '%p_l_s',
+ '%p_m_hm',
+ '%p_m_lss',
+ '%p_m_r',
+ '%p_matrix',
+ '%p_n_l',
+ '%p_n_lss',
+ '%p_n_r',
+ '%p_o_l',
+ '%p_o_lss',
+ '%p_o_r',
+ '%p_o_sp',
+ '%p_p_s',
+ '%p_part',
+ '%p_prod',
+ '%p_q_p',
+ '%p_q_r',
+ '%p_q_s',
+ '%p_r_lss',
+ '%p_r_p',
+ '%p_r_r',
+ '%p_r_s',
+ '%p_s_lss',
+ '%p_s_r',
+ '%p_simp',
+ '%p_string',
+ '%p_sum',
+ '%p_v_lss',
+ '%p_v_p',
+ '%p_v_r',
+ '%p_v_s',
+ '%p_x_hm',
+ '%p_x_r',
+ '%p_y_p',
+ '%p_y_r',
+ '%p_y_s',
+ '%p_z_p',
+ '%p_z_r',
+ '%p_z_s',
+ '%pcg',
+ '%plist_p',
+ '%plist_string',
+ '%r_0',
+ '%r_a_hm',
+ '%r_a_lss',
+ '%r_a_p',
+ '%r_a_r',
+ '%r_a_s',
+ '%r_c_lss',
+ '%r_c_p',
+ '%r_c_r',
+ '%r_c_s',
+ '%r_clean',
+ '%r_cumprod',
+ '%r_cumsum',
+ '%r_d_p',
+ '%r_d_r',
+ '%r_d_s',
+ '%r_det',
+ '%r_diag',
+ '%r_e',
+ '%r_eye',
+ '%r_f_lss',
+ '%r_f_p',
+ '%r_f_r',
+ '%r_f_s',
+ '%r_i_ce',
+ '%r_i_hm',
+ '%r_i_lss',
+ '%r_i_p',
+ '%r_i_r',
+ '%r_i_s',
+ '%r_i_st',
+ '%r_inv',
+ '%r_j_s',
+ '%r_k_p',
+ '%r_k_r',
+ '%r_k_s',
+ '%r_l_lss',
+ '%r_l_p',
+ '%r_l_r',
+ '%r_l_s',
+ '%r_m_hm',
+ '%r_m_lss',
+ '%r_m_p',
+ '%r_m_r',
+ '%r_m_s',
+ '%r_matrix',
+ '%r_n_lss',
+ '%r_n_p',
+ '%r_n_r',
+ '%r_n_s',
+ '%r_norm',
+ '%r_o_lss',
+ '%r_o_p',
+ '%r_o_r',
+ '%r_o_s',
+ '%r_ones',
+ '%r_p',
+ '%r_p_s',
+ '%r_prod',
+ '%r_q_p',
+ '%r_q_r',
+ '%r_q_s',
+ '%r_r_lss',
+ '%r_r_p',
+ '%r_r_r',
+ '%r_r_s',
+ '%r_rand',
+ '%r_s',
+ '%r_s_hm',
+ '%r_s_lss',
+ '%r_s_p',
+ '%r_s_r',
+ '%r_s_s',
+ '%r_simp',
+ '%r_size',
+ '%r_string',
+ '%r_sum',
+ '%r_t',
+ '%r_tril',
+ '%r_triu',
+ '%r_v_lss',
+ '%r_v_p',
+ '%r_v_r',
+ '%r_v_s',
+ '%r_varn',
+ '%r_x_p',
+ '%r_x_r',
+ '%r_x_s',
+ '%r_y_p',
+ '%r_y_r',
+ '%r_y_s',
+ '%r_z_p',
+ '%r_z_r',
+ '%r_z_s',
+ '%s_1_hm',
+ '%s_1_i',
+ '%s_2_hm',
+ '%s_2_i',
+ '%s_3_hm',
+ '%s_3_i',
+ '%s_4_hm',
+ '%s_4_i',
+ '%s_5',
+ '%s_a_b',
+ '%s_a_hm',
+ '%s_a_i',
+ '%s_a_ip',
+ '%s_a_lss',
+ '%s_a_msp',
+ '%s_a_r',
+ '%s_a_sp',
+ '%s_and',
+ '%s_b_i',
+ '%s_b_s',
+ '%s_bezout',
+ '%s_c_b',
+ '%s_c_cblock',
+ '%s_c_lss',
+ '%s_c_r',
+ '%s_c_sp',
+ '%s_d_b',
+ '%s_d_i',
+ '%s_d_p',
+ '%s_d_r',
+ '%s_d_sp',
+ '%s_e',
+ '%s_f_b',
+ '%s_f_cblock',
+ '%s_f_lss',
+ '%s_f_r',
+ '%s_f_sp',
+ '%s_g_b',
+ '%s_g_s',
+ '%s_gcd',
+ '%s_grand',
+ '%s_h_b',
+ '%s_h_s',
+ '%s_i_b',
+ '%s_i_c',
+ '%s_i_ce',
+ '%s_i_h',
+ '%s_i_hm',
+ '%s_i_i',
+ '%s_i_lss',
+ '%s_i_p',
+ '%s_i_r',
+ '%s_i_s',
+ '%s_i_sp',
+ '%s_i_spb',
+ '%s_i_st',
+ '%s_j_i',
+ '%s_k_hm',
+ '%s_k_p',
+ '%s_k_r',
+ '%s_k_sp',
+ '%s_l_b',
+ '%s_l_hm',
+ '%s_l_i',
+ '%s_l_lss',
+ '%s_l_p',
+ '%s_l_r',
+ '%s_l_s',
+ '%s_l_sp',
+ '%s_lcm',
+ '%s_m_b',
+ '%s_m_hm',
+ '%s_m_i',
+ '%s_m_ip',
+ '%s_m_lss',
+ '%s_m_msp',
+ '%s_m_r',
+ '%s_matrix',
+ '%s_n_hm',
+ '%s_n_i',
+ '%s_n_l',
+ '%s_n_lss',
+ '%s_n_r',
+ '%s_n_st',
+ '%s_o_hm',
+ '%s_o_i',
+ '%s_o_l',
+ '%s_o_lss',
+ '%s_o_r',
+ '%s_o_st',
+ '%s_or',
+ '%s_p_b',
+ '%s_p_i',
+ '%s_pow',
+ '%s_q_hm',
+ '%s_q_i',
+ '%s_q_p',
+ '%s_q_r',
+ '%s_q_sp',
+ '%s_r_b',
+ '%s_r_i',
+ '%s_r_lss',
+ '%s_r_p',
+ '%s_r_r',
+ '%s_r_s',
+ '%s_r_sp',
+ '%s_s_b',
+ '%s_s_hm',
+ '%s_s_i',
+ '%s_s_ip',
+ '%s_s_lss',
+ '%s_s_r',
+ '%s_s_sp',
+ '%s_simp',
+ '%s_v_lss',
+ '%s_v_p',
+ '%s_v_r',
+ '%s_v_s',
+ '%s_x_b',
+ '%s_x_hm',
+ '%s_x_i',
+ '%s_x_r',
+ '%s_y_p',
+ '%s_y_r',
+ '%s_y_sp',
+ '%s_z_p',
+ '%s_z_r',
+ '%s_z_sp',
+ '%sn',
+ '%sp_a_s',
+ '%sp_a_sp',
+ '%sp_and',
+ '%sp_c_s',
+ '%sp_ceil',
+ '%sp_conj',
+ '%sp_cos',
+ '%sp_cumprod',
+ '%sp_cumsum',
+ '%sp_d_s',
+ '%sp_d_sp',
+ '%sp_det',
+ '%sp_diag',
+ '%sp_e',
+ '%sp_exp',
+ '%sp_f_s',
+ '%sp_floor',
+ '%sp_grand',
+ '%sp_gsort',
+ '%sp_i_ce',
+ '%sp_i_h',
+ '%sp_i_s',
+ '%sp_i_sp',
+ '%sp_i_st',
+ '%sp_int',
+ '%sp_inv',
+ '%sp_k_s',
+ '%sp_k_sp',
+ '%sp_l_s',
+ '%sp_l_sp',
+ '%sp_length',
+ '%sp_max',
+ '%sp_min',
+ '%sp_norm',
+ '%sp_or',
+ '%sp_p_s',
+ '%sp_prod',
+ '%sp_q_s',
+ '%sp_q_sp',
+ '%sp_r_s',
+ '%sp_r_sp',
+ '%sp_round',
+ '%sp_s_s',
+ '%sp_s_sp',
+ '%sp_sin',
+ '%sp_sqrt',
+ '%sp_string',
+ '%sp_sum',
+ '%sp_tril',
+ '%sp_triu',
+ '%sp_y_s',
+ '%sp_y_sp',
+ '%sp_z_s',
+ '%sp_z_sp',
+ '%spb_and',
+ '%spb_c_b',
+ '%spb_cumprod',
+ '%spb_cumsum',
+ '%spb_diag',
+ '%spb_e',
+ '%spb_f_b',
+ '%spb_g_b',
+ '%spb_g_spb',
+ '%spb_h_b',
+ '%spb_h_spb',
+ '%spb_i_b',
+ '%spb_i_ce',
+ '%spb_i_h',
+ '%spb_i_st',
+ '%spb_or',
+ '%spb_prod',
+ '%spb_sum',
+ '%spb_tril',
+ '%spb_triu',
+ '%st_6',
+ '%st_c_st',
+ '%st_e',
+ '%st_f_st',
+ '%st_i_b',
+ '%st_i_c',
+ '%st_i_fptr',
+ '%st_i_h',
+ '%st_i_i',
+ '%st_i_ip',
+ '%st_i_lss',
+ '%st_i_msp',
+ '%st_i_p',
+ '%st_i_r',
+ '%st_i_s',
+ '%st_i_sp',
+ '%st_i_spb',
+ '%st_i_st',
+ '%st_matrix',
+ '%st_n_c',
+ '%st_n_l',
+ '%st_n_mc',
+ '%st_n_p',
+ '%st_n_s',
+ '%st_o_c',
+ '%st_o_l',
+ '%st_o_mc',
+ '%st_o_p',
+ '%st_o_s',
+ '%st_o_tl',
+ '%st_p',
+ '%st_size',
+ '%st_string',
+ '%st_t',
+ '%ticks_i_h',
+ '%xls_e',
+ '%xls_p',
+ '%xlssheet_e',
+ '%xlssheet_p',
+ '%xlssheet_size',
+ '%xlssheet_string',
+ 'DominationRank',
+ 'G_make',
+ 'IsAScalar',
+ 'NDcost',
+ 'OS_Version',
+ 'PlotSparse',
+ 'ReadHBSparse',
+ 'TCL_CreateSlave',
+ 'abcd',
+ 'abinv',
+ 'accept_func_default',
+ 'accept_func_vfsa',
+ 'acf',
+ 'acosd',
+ 'acosh',
+ 'acoshm',
+ 'acosm',
+ 'acot',
+ 'acotd',
+ 'acoth',
+ 'acsc',
+ 'acscd',
+ 'acsch',
+ 'add_demo',
+ 'add_help_chapter',
+ 'add_module_help_chapter',
+ 'add_param',
+ 'add_profiling',
+ 'adj2sp',
+ 'aff2ab',
+ 'ana_style',
+ 'analpf',
+ 'analyze',
+ 'aplat',
+ 'arhnk',
+ 'arl2',
+ 'arma2p',
+ 'arma2ss',
+ 'armac',
+ 'armax',
+ 'armax1',
+ 'arobasestring2strings',
+ 'arsimul',
+ 'ascii2string',
+ 'asciimat',
+ 'asec',
+ 'asecd',
+ 'asech',
+ 'asind',
+ 'asinh',
+ 'asinhm',
+ 'asinm',
+ 'assert_checkalmostequal',
+ 'assert_checkequal',
+ 'assert_checkerror',
+ 'assert_checkfalse',
+ 'assert_checkfilesequal',
+ 'assert_checktrue',
+ 'assert_comparecomplex',
+ 'assert_computedigits',
+ 'assert_cond2reltol',
+ 'assert_cond2reqdigits',
+ 'assert_generror',
+ 'atand',
+ 'atanh',
+ 'atanhm',
+ 'atanm',
+ 'atomsAutoload',
+ 'atomsAutoloadAdd',
+ 'atomsAutoloadDel',
+ 'atomsAutoloadList',
+ 'atomsCategoryList',
+ 'atomsCheckModule',
+ 'atomsDepTreeShow',
+ 'atomsGetConfig',
+ 'atomsGetInstalled',
+ 'atomsGetInstalledPath',
+ 'atomsGetLoaded',
+ 'atomsGetLoadedPath',
+ 'atomsInstall',
+ 'atomsIsInstalled',
+ 'atomsIsLoaded',
+ 'atomsList',
+ 'atomsLoad',
+ 'atomsQuit',
+ 'atomsRemove',
+ 'atomsRepositoryAdd',
+ 'atomsRepositoryDel',
+ 'atomsRepositoryList',
+ 'atomsRestoreConfig',
+ 'atomsSaveConfig',
+ 'atomsSearch',
+ 'atomsSetConfig',
+ 'atomsShow',
+ 'atomsSystemInit',
+ 'atomsSystemUpdate',
+ 'atomsTest',
+ 'atomsUpdate',
+ 'atomsVersion',
+ 'augment',
+ 'auread',
+ 'auwrite',
+ 'balreal',
+ 'bench_run',
+ 'bilin',
+ 'bilt',
+ 'bin2dec',
+ 'binomial',
+ 'bitand',
+ 'bitcmp',
+ 'bitget',
+ 'bitor',
+ 'bitset',
+ 'bitxor',
+ 'black',
+ 'blanks',
+ 'bloc2exp',
+ 'bloc2ss',
+ 'block_parameter_error',
+ 'bode',
+ 'bode_asymp',
+ 'bstap',
+ 'buttmag',
+ 'bvodeS',
+ 'bytecode',
+ 'bytecodewalk',
+ 'cainv',
+ 'calendar',
+ 'calerf',
+ 'calfrq',
+ 'canon',
+ 'casc',
+ 'cat',
+ 'cat_code',
+ 'cb_m2sci_gui',
+ 'ccontrg',
+ 'cell',
+ 'cell2mat',
+ 'cellstr',
+ 'center',
+ 'cepstrum',
+ 'cfspec',
+ 'char',
+ 'chart',
+ 'cheb1mag',
+ 'cheb2mag',
+ 'check_gateways',
+ 'check_modules_xml',
+ 'check_versions',
+ 'chepol',
+ 'chfact',
+ 'chsolve',
+ 'classmarkov',
+ 'clean_help',
+ 'clock',
+ 'cls2dls',
+ 'cmb_lin',
+ 'cmndred',
+ 'cmoment',
+ 'coding_ga_binary',
+ 'coding_ga_identity',
+ 'coff',
+ 'coffg',
+ 'colcomp',
+ 'colcompr',
+ 'colinout',
+ 'colregul',
+ 'companion',
+ 'complex',
+ 'compute_initial_temp',
+ 'cond',
+ 'cond2sp',
+ 'condestsp',
+ 'configure_msifort',
+ 'configure_msvc',
+ 'conjgrad',
+ 'cont_frm',
+ 'cont_mat',
+ 'contrss',
+ 'conv',
+ 'convert_to_float',
+ 'convertindex',
+ 'convol',
+ 'convol2d',
+ 'copfac',
+ 'correl',
+ 'cosd',
+ 'cosh',
+ 'coshm',
+ 'cosm',
+ 'cotd',
+ 'cotg',
+ 'coth',
+ 'cothm',
+ 'cov',
+ 'covar',
+ 'createXConfiguration',
+ 'createfun',
+ 'createstruct',
+ 'cross',
+ 'crossover_ga_binary',
+ 'crossover_ga_default',
+ 'csc',
+ 'cscd',
+ 'csch',
+ 'csgn',
+ 'csim',
+ 'cspect',
+ 'ctr_gram',
+ 'czt',
+ 'dae',
+ 'daeoptions',
+ 'damp',
+ 'datafit',
+ 'date',
+ 'datenum',
+ 'datevec',
+ 'dbphi',
+ 'dcf',
+ 'ddp',
+ 'dec2bin',
+ 'dec2hex',
+ 'dec2oct',
+ 'del_help_chapter',
+ 'del_module_help_chapter',
+ 'demo_begin',
+ 'demo_choose',
+ 'demo_compiler',
+ 'demo_end',
+ 'demo_file_choice',
+ 'demo_folder_choice',
+ 'demo_function_choice',
+ 'demo_gui',
+ 'demo_run',
+ 'demo_viewCode',
+ 'denom',
+ 'derivat',
+ 'derivative',
+ 'des2ss',
+ 'des2tf',
+ 'detectmsifort64tools',
+ 'detectmsvc64tools',
+ 'determ',
+ 'detr',
+ 'detrend',
+ 'devtools_run_builder',
+ 'dhnorm',
+ 'diff',
+ 'diophant',
+ 'dir',
+ 'dirname',
+ 'dispfiles',
+ 'dllinfo',
+ 'dscr',
+ 'dsimul',
+ 'dt_ility',
+ 'dtsi',
+ 'edit',
+ 'edit_error',
+ 'editor',
+ 'eigenmarkov',
+ 'eigs',
+ 'ell1mag',
+ 'enlarge_shape',
+ 'entropy',
+ 'eomday',
+ 'epred',
+ 'eqfir',
+ 'eqiir',
+ 'equil',
+ 'equil1',
+ 'erfinv',
+ 'etime',
+ 'eval',
+ 'evans',
+ 'evstr',
+ 'example_run',
+ 'expression2code',
+ 'extract_help_examples',
+ 'factor',
+ 'factorial',
+ 'factors',
+ 'faurre',
+ 'ffilt',
+ 'fft2',
+ 'fftshift',
+ 'fieldnames',
+ 'filt_sinc',
+ 'filter',
+ 'findABCD',
+ 'findAC',
+ 'findBDK',
+ 'findR',
+ 'find_freq',
+ 'find_links',
+ 'find_scicos_version',
+ 'findm',
+ 'findmsifortcompiler',
+ 'findmsvccompiler',
+ 'findx0BD',
+ 'firstnonsingleton',
+ 'fix',
+ 'fixedpointgcd',
+ 'flipdim',
+ 'flts',
+ 'fminsearch',
+ 'formatBlackTip',
+ 'formatBodeMagTip',
+ 'formatBodePhaseTip',
+ 'formatGainplotTip',
+ 'formatHallModuleTip',
+ 'formatHallPhaseTip',
+ 'formatNicholsGainTip',
+ 'formatNicholsPhaseTip',
+ 'formatNyquistTip',
+ 'formatPhaseplotTip',
+ 'formatSgridDampingTip',
+ 'formatSgridFreqTip',
+ 'formatZgridDampingTip',
+ 'formatZgridFreqTip',
+ 'format_txt',
+ 'fourplan',
+ 'frep2tf',
+ 'freson',
+ 'frfit',
+ 'frmag',
+ 'fseek_origin',
+ 'fsfirlin',
+ 'fspec',
+ 'fspecg',
+ 'fstabst',
+ 'ftest',
+ 'ftuneq',
+ 'fullfile',
+ 'fullrf',
+ 'fullrfk',
+ 'fun2string',
+ 'g_margin',
+ 'gainplot',
+ 'gamitg',
+ 'gcare',
+ 'gcd',
+ 'gencompilationflags_unix',
+ 'generateBlockImage',
+ 'generateBlockImages',
+ 'generic_i_ce',
+ 'generic_i_h',
+ 'generic_i_hm',
+ 'generic_i_s',
+ 'generic_i_st',
+ 'genlib',
+ 'genmarkov',
+ 'geomean',
+ 'getDiagramVersion',
+ 'getModelicaPath',
+ 'getPreferencesValue',
+ 'get_file_path',
+ 'get_function_path',
+ 'get_param',
+ 'get_profile',
+ 'get_scicos_version',
+ 'getd',
+ 'getscilabkeywords',
+ 'getshell',
+ 'gettklib',
+ 'gfare',
+ 'gfrancis',
+ 'givens',
+ 'glever',
+ 'gmres',
+ 'group',
+ 'gschur',
+ 'gspec',
+ 'gtild',
+ 'h2norm',
+ 'h_cl',
+ 'h_inf',
+ 'h_inf_st',
+ 'h_norm',
+ 'hallchart',
+ 'halt',
+ 'hank',
+ 'hankelsv',
+ 'harmean',
+ 'haveacompiler',
+ 'head_comments',
+ 'help_from_sci',
+ 'help_skeleton',
+ 'hermit',
+ 'hex2dec',
+ 'hilb',
+ 'hilbert',
+ 'histc',
+ 'horner',
+ 'householder',
+ 'hrmt',
+ 'htrianr',
+ 'hypermat',
+ 'idct',
+ 'idst',
+ 'ifft',
+ 'ifftshift',
+ 'iir',
+ 'iirgroup',
+ 'iirlp',
+ 'iirmod',
+ 'ilib_build',
+ 'ilib_build_jar',
+ 'ilib_compile',
+ 'ilib_for_link',
+ 'ilib_gen_Make',
+ 'ilib_gen_Make_unix',
+ 'ilib_gen_cleaner',
+ 'ilib_gen_gateway',
+ 'ilib_gen_loader',
+ 'ilib_include_flag',
+ 'ilib_mex_build',
+ 'im_inv',
+ 'importScicosDiagram',
+ 'importScicosPal',
+ 'importXcosDiagram',
+ 'imrep2ss',
+ 'ind2sub',
+ 'inistate',
+ 'init_ga_default',
+ 'init_param',
+ 'initial_scicos_tables',
+ 'input',
+ 'instruction2code',
+ 'intc',
+ 'intdec',
+ 'integrate',
+ 'interp1',
+ 'interpln',
+ 'intersect',
+ 'intl',
+ 'intsplin',
+ 'inttrap',
+ 'inv_coeff',
+ 'invr',
+ 'invrs',
+ 'invsyslin',
+ 'iqr',
+ 'isLeapYear',
+ 'is_absolute_path',
+ 'is_param',
+ 'iscell',
+ 'iscellstr',
+ 'iscolumn',
+ 'isempty',
+ 'isfield',
+ 'isinf',
+ 'ismatrix',
+ 'isnan',
+ 'isrow',
+ 'isscalar',
+ 'issparse',
+ 'issquare',
+ 'isstruct',
+ 'isvector',
+ 'jmat',
+ 'justify',
+ 'kalm',
+ 'karmarkar',
+ 'kernel',
+ 'kpure',
+ 'krac2',
+ 'kroneck',
+ 'lattn',
+ 'lattp',
+ 'launchtest',
+ 'lcf',
+ 'lcm',
+ 'lcmdiag',
+ 'leastsq',
+ 'leqe',
+ 'leqr',
+ 'lev',
+ 'levin',
+ 'lex_sort',
+ 'lft',
+ 'lin',
+ 'lin2mu',
+ 'lincos',
+ 'lindquist',
+ 'linf',
+ 'linfn',
+ 'linsolve',
+ 'linspace',
+ 'list2vec',
+ 'list_param',
+ 'listfiles',
+ 'listfunctions',
+ 'listvarinfile',
+ 'lmisolver',
+ 'lmitool',
+ 'loadXcosLibs',
+ 'loadmatfile',
+ 'loadwave',
+ 'log10',
+ 'log2',
+ 'logm',
+ 'logspace',
+ 'lqe',
+ 'lqg',
+ 'lqg2stan',
+ 'lqg_ltr',
+ 'lqr',
+ 'ls',
+ 'lyap',
+ 'm2sci_gui',
+ 'm_circle',
+ 'macglov',
+ 'macrovar',
+ 'mad',
+ 'makecell',
+ 'manedit',
+ 'mapsound',
+ 'markp2ss',
+ 'matfile2sci',
+ 'mdelete',
+ 'mean',
+ 'meanf',
+ 'median',
+ 'members',
+ 'mese',
+ 'meshgrid',
+ 'mfft',
+ 'mfile2sci',
+ 'minreal',
+ 'minss',
+ 'mkdir',
+ 'modulo',
+ 'moment',
+ 'mrfit',
+ 'msd',
+ 'mstr2sci',
+ 'mtlb',
+ 'mtlb_0',
+ 'mtlb_a',
+ 'mtlb_all',
+ 'mtlb_any',
+ 'mtlb_axes',
+ 'mtlb_axis',
+ 'mtlb_beta',
+ 'mtlb_box',
+ 'mtlb_choices',
+ 'mtlb_close',
+ 'mtlb_colordef',
+ 'mtlb_cond',
+ 'mtlb_cov',
+ 'mtlb_cumprod',
+ 'mtlb_cumsum',
+ 'mtlb_dec2hex',
+ 'mtlb_delete',
+ 'mtlb_diag',
+ 'mtlb_diff',
+ 'mtlb_dir',
+ 'mtlb_double',
+ 'mtlb_e',
+ 'mtlb_echo',
+ 'mtlb_error',
+ 'mtlb_eval',
+ 'mtlb_exist',
+ 'mtlb_eye',
+ 'mtlb_false',
+ 'mtlb_fft',
+ 'mtlb_fftshift',
+ 'mtlb_filter',
+ 'mtlb_find',
+ 'mtlb_findstr',
+ 'mtlb_fliplr',
+ 'mtlb_fopen',
+ 'mtlb_format',
+ 'mtlb_fprintf',
+ 'mtlb_fread',
+ 'mtlb_fscanf',
+ 'mtlb_full',
+ 'mtlb_fwrite',
+ 'mtlb_get',
+ 'mtlb_grid',
+ 'mtlb_hold',
+ 'mtlb_i',
+ 'mtlb_ifft',
+ 'mtlb_image',
+ 'mtlb_imp',
+ 'mtlb_int16',
+ 'mtlb_int32',
+ 'mtlb_int8',
+ 'mtlb_is',
+ 'mtlb_isa',
+ 'mtlb_isfield',
+ 'mtlb_isletter',
+ 'mtlb_isspace',
+ 'mtlb_l',
+ 'mtlb_legendre',
+ 'mtlb_linspace',
+ 'mtlb_logic',
+ 'mtlb_logical',
+ 'mtlb_loglog',
+ 'mtlb_lower',
+ 'mtlb_max',
+ 'mtlb_mean',
+ 'mtlb_median',
+ 'mtlb_mesh',
+ 'mtlb_meshdom',
+ 'mtlb_min',
+ 'mtlb_more',
+ 'mtlb_num2str',
+ 'mtlb_ones',
+ 'mtlb_pcolor',
+ 'mtlb_plot',
+ 'mtlb_prod',
+ 'mtlb_qr',
+ 'mtlb_qz',
+ 'mtlb_rand',
+ 'mtlb_randn',
+ 'mtlb_rcond',
+ 'mtlb_realmax',
+ 'mtlb_realmin',
+ 'mtlb_s',
+ 'mtlb_semilogx',
+ 'mtlb_semilogy',
+ 'mtlb_setstr',
+ 'mtlb_size',
+ 'mtlb_sort',
+ 'mtlb_sortrows',
+ 'mtlb_sprintf',
+ 'mtlb_sscanf',
+ 'mtlb_std',
+ 'mtlb_strcmp',
+ 'mtlb_strcmpi',
+ 'mtlb_strfind',
+ 'mtlb_strrep',
+ 'mtlb_subplot',
+ 'mtlb_sum',
+ 'mtlb_t',
+ 'mtlb_toeplitz',
+ 'mtlb_tril',
+ 'mtlb_triu',
+ 'mtlb_true',
+ 'mtlb_type',
+ 'mtlb_uint16',
+ 'mtlb_uint32',
+ 'mtlb_uint8',
+ 'mtlb_upper',
+ 'mtlb_var',
+ 'mtlb_zeros',
+ 'mu2lin',
+ 'mutation_ga_binary',
+ 'mutation_ga_default',
+ 'mvcorrel',
+ 'mvvacov',
+ 'nancumsum',
+ 'nand2mean',
+ 'nanmax',
+ 'nanmean',
+ 'nanmeanf',
+ 'nanmedian',
+ 'nanmin',
+ 'nanreglin',
+ 'nanstdev',
+ 'nansum',
+ 'narsimul',
+ 'ndgrid',
+ 'ndims',
+ 'nehari',
+ 'neigh_func_csa',
+ 'neigh_func_default',
+ 'neigh_func_fsa',
+ 'neigh_func_vfsa',
+ 'neldermead_cget',
+ 'neldermead_configure',
+ 'neldermead_costf',
+ 'neldermead_defaultoutput',
+ 'neldermead_destroy',
+ 'neldermead_function',
+ 'neldermead_get',
+ 'neldermead_log',
+ 'neldermead_new',
+ 'neldermead_restart',
+ 'neldermead_search',
+ 'neldermead_updatesimp',
+ 'nextpow2',
+ 'nfreq',
+ 'nicholschart',
+ 'nlev',
+ 'nmplot_cget',
+ 'nmplot_configure',
+ 'nmplot_contour',
+ 'nmplot_destroy',
+ 'nmplot_function',
+ 'nmplot_get',
+ 'nmplot_historyplot',
+ 'nmplot_log',
+ 'nmplot_new',
+ 'nmplot_outputcmd',
+ 'nmplot_restart',
+ 'nmplot_search',
+ 'nmplot_simplexhistory',
+ 'noisegen',
+ 'nonreg_test_run',
+ 'now',
+ 'nthroot',
+ 'null',
+ 'num2cell',
+ 'numderivative',
+ 'numdiff',
+ 'numer',
+ 'nyquist',
+ 'nyquistfrequencybounds',
+ 'obs_gram',
+ 'obscont',
+ 'observer',
+ 'obsv_mat',
+ 'obsvss',
+ 'oct2dec',
+ 'odeoptions',
+ 'optim_ga',
+ 'optim_moga',
+ 'optim_nsga',
+ 'optim_nsga2',
+ 'optim_sa',
+ 'optimbase_cget',
+ 'optimbase_checkbounds',
+ 'optimbase_checkcostfun',
+ 'optimbase_checkx0',
+ 'optimbase_configure',
+ 'optimbase_destroy',
+ 'optimbase_function',
+ 'optimbase_get',
+ 'optimbase_hasbounds',
+ 'optimbase_hasconstraints',
+ 'optimbase_hasnlcons',
+ 'optimbase_histget',
+ 'optimbase_histset',
+ 'optimbase_incriter',
+ 'optimbase_isfeasible',
+ 'optimbase_isinbounds',
+ 'optimbase_isinnonlincons',
+ 'optimbase_log',
+ 'optimbase_logshutdown',
+ 'optimbase_logstartup',
+ 'optimbase_new',
+ 'optimbase_outputcmd',
+ 'optimbase_outstruct',
+ 'optimbase_proj2bnds',
+ 'optimbase_set',
+ 'optimbase_stoplog',
+ 'optimbase_terminate',
+ 'optimget',
+ 'optimplotfunccount',
+ 'optimplotfval',
+ 'optimplotx',
+ 'optimset',
+ 'optimsimplex_center',
+ 'optimsimplex_check',
+ 'optimsimplex_compsomefv',
+ 'optimsimplex_computefv',
+ 'optimsimplex_deltafv',
+ 'optimsimplex_deltafvmax',
+ 'optimsimplex_destroy',
+ 'optimsimplex_dirmat',
+ 'optimsimplex_fvmean',
+ 'optimsimplex_fvstdev',
+ 'optimsimplex_fvvariance',
+ 'optimsimplex_getall',
+ 'optimsimplex_getallfv',
+ 'optimsimplex_getallx',
+ 'optimsimplex_getfv',
+ 'optimsimplex_getn',
+ 'optimsimplex_getnbve',
+ 'optimsimplex_getve',
+ 'optimsimplex_getx',
+ 'optimsimplex_gradientfv',
+ 'optimsimplex_log',
+ 'optimsimplex_new',
+ 'optimsimplex_reflect',
+ 'optimsimplex_setall',
+ 'optimsimplex_setallfv',
+ 'optimsimplex_setallx',
+ 'optimsimplex_setfv',
+ 'optimsimplex_setn',
+ 'optimsimplex_setnbve',
+ 'optimsimplex_setve',
+ 'optimsimplex_setx',
+ 'optimsimplex_shrink',
+ 'optimsimplex_size',
+ 'optimsimplex_sort',
+ 'optimsimplex_xbar',
+ 'orth',
+ 'output_ga_default',
+ 'output_moga_default',
+ 'output_nsga2_default',
+ 'output_nsga_default',
+ 'p_margin',
+ 'pack',
+ 'pareto_filter',
+ 'parrot',
+ 'pbig',
+ 'pca',
+ 'pcg',
+ 'pdiv',
+ 'pen2ea',
+ 'pencan',
+ 'pencost',
+ 'penlaur',
+ 'perctl',
+ 'perl',
+ 'perms',
+ 'permute',
+ 'pertrans',
+ 'pfactors',
+ 'pfss',
+ 'phasemag',
+ 'phaseplot',
+ 'phc',
+ 'pinv',
+ 'playsnd',
+ 'plotprofile',
+ 'plzr',
+ 'pmodulo',
+ 'pol2des',
+ 'pol2str',
+ 'polar',
+ 'polfact',
+ 'prbs_a',
+ 'prettyprint',
+ 'primes',
+ 'princomp',
+ 'profile',
+ 'proj',
+ 'projsl',
+ 'projspec',
+ 'psmall',
+ 'pspect',
+ 'qmr',
+ 'qpsolve',
+ 'quart',
+ 'quaskro',
+ 'rafiter',
+ 'randpencil',
+ 'range',
+ 'rank',
+ 'readxls',
+ 'recompilefunction',
+ 'recons',
+ 'reglin',
+ 'regress',
+ 'remezb',
+ 'remove_param',
+ 'remove_profiling',
+ 'repfreq',
+ 'replace_Ix_by_Fx',
+ 'repmat',
+ 'reset_profiling',
+ 'resize_matrix',
+ 'returntoscilab',
+ 'rhs2code',
+ 'ric_desc',
+ 'riccati',
+ 'rmdir',
+ 'routh_t',
+ 'rowcomp',
+ 'rowcompr',
+ 'rowinout',
+ 'rowregul',
+ 'rowshuff',
+ 'rref',
+ 'sample',
+ 'samplef',
+ 'samwr',
+ 'savematfile',
+ 'savewave',
+ 'scanf',
+ 'sci2exp',
+ 'sciGUI_init',
+ 'sci_sparse',
+ 'scicos_getvalue',
+ 'scicos_simulate',
+ 'scicos_workspace_init',
+ 'scisptdemo',
+ 'scitest',
+ 'sdiff',
+ 'sec',
+ 'secd',
+ 'sech',
+ 'selection_ga_elitist',
+ 'selection_ga_random',
+ 'sensi',
+ 'setPreferencesValue',
+ 'set_param',
+ 'setdiff',
+ 'sgrid',
+ 'show_margins',
+ 'show_pca',
+ 'showprofile',
+ 'signm',
+ 'sinc',
+ 'sincd',
+ 'sind',
+ 'sinh',
+ 'sinhm',
+ 'sinm',
+ 'sm2des',
+ 'sm2ss',
+ 'smga',
+ 'smooth',
+ 'solve',
+ 'sound',
+ 'soundsec',
+ 'sp2adj',
+ 'spaninter',
+ 'spanplus',
+ 'spantwo',
+ 'specfact',
+ 'speye',
+ 'sprand',
+ 'spzeros',
+ 'sqroot',
+ 'sqrtm',
+ 'squarewave',
+ 'squeeze',
+ 'srfaur',
+ 'srkf',
+ 'ss2des',
+ 'ss2ss',
+ 'ss2tf',
+ 'sskf',
+ 'ssprint',
+ 'ssrand',
+ 'st_deviation',
+ 'st_i_generic',
+ 'st_ility',
+ 'stabil',
+ 'statgain',
+ 'stdev',
+ 'stdevf',
+ 'steadycos',
+ 'strange',
+ 'strcmpi',
+ 'struct',
+ 'sub2ind',
+ 'sva',
+ 'svplot',
+ 'sylm',
+ 'sylv',
+ 'sysconv',
+ 'sysdiag',
+ 'sysfact',
+ 'syslin',
+ 'syssize',
+ 'system',
+ 'systmat',
+ 'tabul',
+ 'tand',
+ 'tanh',
+ 'tanhm',
+ 'tanm',
+ 'tbx_build_blocks',
+ 'tbx_build_cleaner',
+ 'tbx_build_gateway',
+ 'tbx_build_gateway_clean',
+ 'tbx_build_gateway_loader',
+ 'tbx_build_help',
+ 'tbx_build_help_loader',
+ 'tbx_build_loader',
+ 'tbx_build_localization',
+ 'tbx_build_macros',
+ 'tbx_build_pal_loader',
+ 'tbx_build_src',
+ 'tbx_builder',
+ 'tbx_builder_gateway',
+ 'tbx_builder_gateway_lang',
+ 'tbx_builder_help',
+ 'tbx_builder_help_lang',
+ 'tbx_builder_macros',
+ 'tbx_builder_src',
+ 'tbx_builder_src_lang',
+ 'tbx_generate_pofile',
+ 'temp_law_csa',
+ 'temp_law_default',
+ 'temp_law_fsa',
+ 'temp_law_huang',
+ 'temp_law_vfsa',
+ 'test_clean',
+ 'test_on_columns',
+ 'test_run',
+ 'test_run_level',
+ 'testexamples',
+ 'tf2des',
+ 'tf2ss',
+ 'thrownan',
+ 'tic',
+ 'time_id',
+ 'toc',
+ 'toeplitz',
+ 'tokenpos',
+ 'toolboxes',
+ 'trace',
+ 'trans',
+ 'translatepaths',
+ 'tree2code',
+ 'trfmod',
+ 'trianfml',
+ 'trimmean',
+ 'trisolve',
+ 'trzeros',
+ 'typeof',
+ 'ui_observer',
+ 'union',
+ 'unique',
+ 'unit_test_run',
+ 'unix_g',
+ 'unix_s',
+ 'unix_w',
+ 'unix_x',
+ 'unobs',
+ 'unpack',
+ 'unwrap',
+ 'variance',
+ 'variancef',
+ 'vec2list',
+ 'vectorfind',
+ 'ver',
+ 'warnobsolete',
+ 'wavread',
+ 'wavwrite',
+ 'wcenter',
+ 'weekday',
+ 'wfir',
+ 'wfir_gui',
+ 'whereami',
+ 'who_user',
+ 'whos',
+ 'wiener',
+ 'wigner',
+ 'window',
+ 'winlist',
+ 'with_javasci',
+ 'with_macros_source',
+ 'with_modelica_compiler',
+ 'with_tk',
+ 'xcorr',
+ 'xcosBlockEval',
+ 'xcosBlockInterface',
+ 'xcosCodeGeneration',
+ 'xcosConfigureModelica',
+ 'xcosPal',
+ 'xcosPalAdd',
+ 'xcosPalAddBlock',
+ 'xcosPalExport',
+ 'xcosPalGenerateAllIcons',
+ 'xcosShowBlockWarning',
+ 'xcosValidateBlockSet',
+ 'xcosValidateCompareBlock',
+ 'xcos_compile',
+ 'xcos_debug_gui',
+ 'xcos_run',
+ 'xcos_simulate',
+ 'xcov',
+ 'xmltochm',
+ 'xmltoformat',
+ 'xmltohtml',
+ 'xmltojar',
+ 'xmltopdf',
+ 'xmltops',
+ 'xmltoweb',
+ 'yulewalk',
+ 'zeropen',
+ 'zgrid',
+ 'zpbutt',
+ 'zpch1',
+ 'zpch2',
+ 'zpell',
+)
+
+variables_kw = (
+ '$',
+ '%F',
+ '%T',
+ '%e',
+ '%eps',
+ '%f',
+ '%fftw',
+ '%gui',
+ '%i',
+ '%inf',
+ '%io',
+ '%modalWarning',
+ '%nan',
+ '%pi',
+ '%s',
+ '%t',
+ '%tk',
+ '%toolboxes',
+ '%toolboxes_dir',
+ '%z',
+ 'PWD',
+ 'SCI',
+ 'SCIHOME',
+ 'TMPDIR',
+ 'arnoldilib',
+ 'assertlib',
+ 'atomslib',
+ 'cacsdlib',
+ 'compatibility_functilib',
+ 'corelib',
+ 'data_structureslib',
+ 'demo_toolslib',
+ 'development_toolslib',
+ 'differential_equationlib',
+ 'dynamic_linklib',
+ 'elementary_functionslib',
+ 'enull',
+ 'evoid',
+ 'external_objectslib',
+ 'fd',
+ 'fileiolib',
+ 'functionslib',
+ 'genetic_algorithmslib',
+ 'helptoolslib',
+ 'home',
+ 'integerlib',
+ 'interpolationlib',
+ 'iolib',
+ 'jnull',
+ 'jvoid',
+ 'linear_algebralib',
+ 'm2scilib',
+ 'matiolib',
+ 'modules_managerlib',
+ 'neldermeadlib',
+ 'optimbaselib',
+ 'optimizationlib',
+ 'optimsimplexlib',
+ 'output_streamlib',
+ 'overloadinglib',
+ 'parameterslib',
+ 'polynomialslib',
+ 'preferenceslib',
+ 'randliblib',
+ 'scicos_autolib',
+ 'scicos_utilslib',
+ 'scinoteslib',
+ 'signal_processinglib',
+ 'simulated_annealinglib',
+ 'soundlib',
+ 'sparselib',
+ 'special_functionslib',
+ 'spreadsheetlib',
+ 'statisticslib',
+ 'stringlib',
+ 'tclscilib',
+ 'timelib',
+ 'umfpacklib',
+ 'xcoslib',
+)
+
+
+if __name__ == '__main__': # pragma: no cover
+ import subprocess
+ from pygments.util import format_lines, duplicates_removed
+
+ mapping = {'variables': 'builtin'}
+
+ def extract_completion(var_type):
+ s = subprocess.Popen(['scilab', '-nwni'], stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ output = s.communicate('''\
+fd = mopen("/dev/stderr", "wt");
+mputl(strcat(completion("", "%s"), "||"), fd);
+mclose(fd)\n''' % var_type)
+ if '||' not in output[1]:
+ raise Exception(output[0])
+ # Invalid DISPLAY causes this to be output:
+ text = output[1].strip()
+ if text.startswith('Error: unable to open display \n'):
+ text = text[len('Error: unable to open display \n'):]
+ return text.split('||')
+
+ new_data = {}
+ seen = set() # only keep first type for a given word
+ for t in ('functions', 'commands', 'macros', 'variables'):
+ new_data[t] = duplicates_removed(extract_completion(t), seen)
+ seen.update(set(new_data[t]))
+
+
+ with open(__file__) as f:
+ content = f.read()
+
+ header = content[:content.find('# Autogenerated')]
+ footer = content[content.find("if __name__ == '__main__':"):]
+
+ with open(__file__, 'w') as f:
+ f.write(header)
+ f.write('# Autogenerated\n\n')
+ for k, v in sorted(new_data.iteritems()):
+ f.write(format_lines(k + '_kw', v) + '\n\n')
+ f.write(footer)
diff --git a/wandb/vendor/pygments/lexers/_sourcemod_builtins.py b/wandb/vendor/pygments/lexers/_sourcemod_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..f08ea4817c9047dd5fc11c181ffc8cea7ae49522
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_sourcemod_builtins.py
@@ -0,0 +1,1163 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._sourcemod_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ This file contains the names of SourceMod functions.
+ It is able to re-generate itself.
+
+ Do not edit the FUNCTIONS list by hand.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import print_function
+
+FUNCTIONS = (
+ 'OnEntityCreated',
+ 'OnEntityDestroyed',
+ 'OnGetGameDescription',
+ 'OnLevelInit',
+ 'SDKHook',
+ 'SDKHookEx',
+ 'SDKUnhook',
+ 'SDKHooks_TakeDamage',
+ 'SDKHooks_DropWeapon',
+ 'TopMenuHandler',
+ 'CreateTopMenu',
+ 'LoadTopMenuConfig',
+ 'AddToTopMenu',
+ 'GetTopMenuInfoString',
+ 'GetTopMenuObjName',
+ 'RemoveFromTopMenu',
+ 'DisplayTopMenu',
+ 'DisplayTopMenuCategory',
+ 'FindTopMenuCategory',
+ 'SetTopMenuTitleCaching',
+ 'OnAdminMenuCreated',
+ 'OnAdminMenuReady',
+ 'GetAdminTopMenu',
+ 'AddTargetsToMenu',
+ 'AddTargetsToMenu2',
+ 'RedisplayAdminMenu',
+ 'TEHook',
+ 'AddTempEntHook',
+ 'RemoveTempEntHook',
+ 'TE_Start',
+ 'TE_IsValidProp',
+ 'TE_WriteNum',
+ 'TE_ReadNum',
+ 'TE_WriteFloat',
+ 'TE_ReadFloat',
+ 'TE_WriteVector',
+ 'TE_ReadVector',
+ 'TE_WriteAngles',
+ 'TE_WriteFloatArray',
+ 'TE_Send',
+ 'TE_WriteEncodedEnt',
+ 'TE_SendToAll',
+ 'TE_SendToClient',
+ 'CreateKeyValues',
+ 'KvSetString',
+ 'KvSetNum',
+ 'KvSetUInt64',
+ 'KvSetFloat',
+ 'KvSetColor',
+ 'KvSetVector',
+ 'KvGetString',
+ 'KvGetNum',
+ 'KvGetFloat',
+ 'KvGetColor',
+ 'KvGetUInt64',
+ 'KvGetVector',
+ 'KvJumpToKey',
+ 'KvJumpToKeySymbol',
+ 'KvGotoFirstSubKey',
+ 'KvGotoNextKey',
+ 'KvSavePosition',
+ 'KvDeleteKey',
+ 'KvDeleteThis',
+ 'KvGoBack',
+ 'KvRewind',
+ 'KvGetSectionName',
+ 'KvSetSectionName',
+ 'KvGetDataType',
+ 'KeyValuesToFile',
+ 'FileToKeyValues',
+ 'StringToKeyValues',
+ 'KvSetEscapeSequences',
+ 'KvNodesInStack',
+ 'KvCopySubkeys',
+ 'KvFindKeyById',
+ 'KvGetNameSymbol',
+ 'KvGetSectionSymbol',
+ 'TE_SetupSparks',
+ 'TE_SetupSmoke',
+ 'TE_SetupDust',
+ 'TE_SetupMuzzleFlash',
+ 'TE_SetupMetalSparks',
+ 'TE_SetupEnergySplash',
+ 'TE_SetupArmorRicochet',
+ 'TE_SetupGlowSprite',
+ 'TE_SetupExplosion',
+ 'TE_SetupBloodSprite',
+ 'TE_SetupBeamRingPoint',
+ 'TE_SetupBeamPoints',
+ 'TE_SetupBeamLaser',
+ 'TE_SetupBeamRing',
+ 'TE_SetupBeamFollow',
+ 'HookEvent',
+ 'HookEventEx',
+ 'UnhookEvent',
+ 'CreateEvent',
+ 'FireEvent',
+ 'CancelCreatedEvent',
+ 'GetEventBool',
+ 'SetEventBool',
+ 'GetEventInt',
+ 'SetEventInt',
+ 'GetEventFloat',
+ 'SetEventFloat',
+ 'GetEventString',
+ 'SetEventString',
+ 'GetEventName',
+ 'SetEventBroadcast',
+ 'GetUserMessageType',
+ 'GetUserMessageId',
+ 'GetUserMessageName',
+ 'StartMessage',
+ 'StartMessageEx',
+ 'EndMessage',
+ 'MsgHook',
+ 'MsgPostHook',
+ 'HookUserMessage',
+ 'UnhookUserMessage',
+ 'StartMessageAll',
+ 'StartMessageOne',
+ 'InactivateClient',
+ 'ReconnectClient',
+ 'GetMaxEntities',
+ 'GetEntityCount',
+ 'IsValidEntity',
+ 'IsValidEdict',
+ 'IsEntNetworkable',
+ 'CreateEdict',
+ 'RemoveEdict',
+ 'GetEdictFlags',
+ 'SetEdictFlags',
+ 'GetEdictClassname',
+ 'GetEntityNetClass',
+ 'ChangeEdictState',
+ 'GetEntData',
+ 'SetEntData',
+ 'GetEntDataFloat',
+ 'SetEntDataFloat',
+ 'GetEntDataEnt2',
+ 'SetEntDataEnt2',
+ 'GetEntDataVector',
+ 'SetEntDataVector',
+ 'GetEntDataString',
+ 'SetEntDataString',
+ 'FindSendPropOffs',
+ 'FindSendPropInfo',
+ 'FindDataMapOffs',
+ 'FindDataMapInfo',
+ 'GetEntSendPropOffs',
+ 'GetEntProp',
+ 'SetEntProp',
+ 'GetEntPropFloat',
+ 'SetEntPropFloat',
+ 'GetEntPropEnt',
+ 'SetEntPropEnt',
+ 'GetEntPropVector',
+ 'SetEntPropVector',
+ 'GetEntPropString',
+ 'SetEntPropString',
+ 'GetEntPropArraySize',
+ 'GetEntDataArray',
+ 'SetEntDataArray',
+ 'GetEntityAddress',
+ 'GetEntityClassname',
+ 'float',
+ 'FloatMul',
+ 'FloatDiv',
+ 'FloatAdd',
+ 'FloatSub',
+ 'FloatFraction',
+ 'RoundToZero',
+ 'RoundToCeil',
+ 'RoundToFloor',
+ 'RoundToNearest',
+ 'FloatCompare',
+ 'SquareRoot',
+ 'Pow',
+ 'Exponential',
+ 'Logarithm',
+ 'Sine',
+ 'Cosine',
+ 'Tangent',
+ 'FloatAbs',
+ 'ArcTangent',
+ 'ArcCosine',
+ 'ArcSine',
+ 'ArcTangent2',
+ 'RoundFloat',
+ 'operator%',
+ 'DegToRad',
+ 'RadToDeg',
+ 'GetURandomInt',
+ 'GetURandomFloat',
+ 'SetURandomSeed',
+ 'SetURandomSeedSimple',
+ 'RemovePlayerItem',
+ 'GivePlayerItem',
+ 'GetPlayerWeaponSlot',
+ 'IgniteEntity',
+ 'ExtinguishEntity',
+ 'TeleportEntity',
+ 'ForcePlayerSuicide',
+ 'SlapPlayer',
+ 'FindEntityByClassname',
+ 'GetClientEyeAngles',
+ 'CreateEntityByName',
+ 'DispatchSpawn',
+ 'DispatchKeyValue',
+ 'DispatchKeyValueFloat',
+ 'DispatchKeyValueVector',
+ 'GetClientAimTarget',
+ 'GetTeamCount',
+ 'GetTeamName',
+ 'GetTeamScore',
+ 'SetTeamScore',
+ 'GetTeamClientCount',
+ 'SetEntityModel',
+ 'GetPlayerDecalFile',
+ 'GetPlayerJingleFile',
+ 'GetServerNetStats',
+ 'EquipPlayerWeapon',
+ 'ActivateEntity',
+ 'SetClientInfo',
+ 'GivePlayerAmmo',
+ 'SetClientListeningFlags',
+ 'GetClientListeningFlags',
+ 'SetListenOverride',
+ 'GetListenOverride',
+ 'IsClientMuted',
+ 'TR_GetPointContents',
+ 'TR_GetPointContentsEnt',
+ 'TR_TraceRay',
+ 'TR_TraceHull',
+ 'TR_TraceRayFilter',
+ 'TR_TraceHullFilter',
+ 'TR_TraceRayEx',
+ 'TR_TraceHullEx',
+ 'TR_TraceRayFilterEx',
+ 'TR_TraceHullFilterEx',
+ 'TR_GetFraction',
+ 'TR_GetEndPosition',
+ 'TR_GetEntityIndex',
+ 'TR_DidHit',
+ 'TR_GetHitGroup',
+ 'TR_GetPlaneNormal',
+ 'TR_PointOutsideWorld',
+ 'SortIntegers',
+ 'SortFloats',
+ 'SortStrings',
+ 'SortFunc1D',
+ 'SortCustom1D',
+ 'SortCustom2D',
+ 'SortADTArray',
+ 'SortFuncADTArray',
+ 'SortADTArrayCustom',
+ 'CompileRegex',
+ 'MatchRegex',
+ 'GetRegexSubString',
+ 'SimpleRegexMatch',
+ 'TF2_GetPlayerClass',
+ 'TF2_SetPlayerClass',
+ 'TF2_RemoveWeaponSlot',
+ 'TF2_RemoveAllWeapons',
+ 'TF2_IsPlayerInCondition',
+ 'TF2_GetObjectType',
+ 'TF2_GetObjectMode',
+ 'NominateMap',
+ 'RemoveNominationByMap',
+ 'RemoveNominationByOwner',
+ 'GetExcludeMapList',
+ 'GetNominatedMapList',
+ 'CanMapChooserStartVote',
+ 'InitiateMapChooserVote',
+ 'HasEndOfMapVoteFinished',
+ 'EndOfMapVoteEnabled',
+ 'OnNominationRemoved',
+ 'OnMapVoteStarted',
+ 'CreateTimer',
+ 'KillTimer',
+ 'TriggerTimer',
+ 'GetTickedTime',
+ 'GetMapTimeLeft',
+ 'GetMapTimeLimit',
+ 'ExtendMapTimeLimit',
+ 'GetTickInterval',
+ 'OnMapTimeLeftChanged',
+ 'IsServerProcessing',
+ 'CreateDataTimer',
+ 'ByteCountToCells',
+ 'CreateArray',
+ 'ClearArray',
+ 'CloneArray',
+ 'ResizeArray',
+ 'GetArraySize',
+ 'PushArrayCell',
+ 'PushArrayString',
+ 'PushArrayArray',
+ 'GetArrayCell',
+ 'GetArrayString',
+ 'GetArrayArray',
+ 'SetArrayCell',
+ 'SetArrayString',
+ 'SetArrayArray',
+ 'ShiftArrayUp',
+ 'RemoveFromArray',
+ 'SwapArrayItems',
+ 'FindStringInArray',
+ 'FindValueInArray',
+ 'ProcessTargetString',
+ 'ReplyToTargetError',
+ 'MultiTargetFilter',
+ 'AddMultiTargetFilter',
+ 'RemoveMultiTargetFilter',
+ 'OnBanClient',
+ 'OnBanIdentity',
+ 'OnRemoveBan',
+ 'BanClient',
+ 'BanIdentity',
+ 'RemoveBan',
+ 'CreateTrie',
+ 'SetTrieValue',
+ 'SetTrieArray',
+ 'SetTrieString',
+ 'GetTrieValue',
+ 'GetTrieArray',
+ 'GetTrieString',
+ 'RemoveFromTrie',
+ 'ClearTrie',
+ 'GetTrieSize',
+ 'GetFunctionByName',
+ 'CreateGlobalForward',
+ 'CreateForward',
+ 'GetForwardFunctionCount',
+ 'AddToForward',
+ 'RemoveFromForward',
+ 'RemoveAllFromForward',
+ 'Call_StartForward',
+ 'Call_StartFunction',
+ 'Call_PushCell',
+ 'Call_PushCellRef',
+ 'Call_PushFloat',
+ 'Call_PushFloatRef',
+ 'Call_PushArray',
+ 'Call_PushArrayEx',
+ 'Call_PushString',
+ 'Call_PushStringEx',
+ 'Call_Finish',
+ 'Call_Cancel',
+ 'NativeCall',
+ 'CreateNative',
+ 'ThrowNativeError',
+ 'GetNativeStringLength',
+ 'GetNativeString',
+ 'SetNativeString',
+ 'GetNativeCell',
+ 'GetNativeCellRef',
+ 'SetNativeCellRef',
+ 'GetNativeArray',
+ 'SetNativeArray',
+ 'FormatNativeString',
+ 'RequestFrameCallback',
+ 'RequestFrame',
+ 'OnRebuildAdminCache',
+ 'DumpAdminCache',
+ 'AddCommandOverride',
+ 'GetCommandOverride',
+ 'UnsetCommandOverride',
+ 'CreateAdmGroup',
+ 'FindAdmGroup',
+ 'SetAdmGroupAddFlag',
+ 'GetAdmGroupAddFlag',
+ 'GetAdmGroupAddFlags',
+ 'SetAdmGroupImmuneFrom',
+ 'GetAdmGroupImmuneCount',
+ 'GetAdmGroupImmuneFrom',
+ 'AddAdmGroupCmdOverride',
+ 'GetAdmGroupCmdOverride',
+ 'RegisterAuthIdentType',
+ 'CreateAdmin',
+ 'GetAdminUsername',
+ 'BindAdminIdentity',
+ 'SetAdminFlag',
+ 'GetAdminFlag',
+ 'GetAdminFlags',
+ 'AdminInheritGroup',
+ 'GetAdminGroupCount',
+ 'GetAdminGroup',
+ 'SetAdminPassword',
+ 'GetAdminPassword',
+ 'FindAdminByIdentity',
+ 'RemoveAdmin',
+ 'FlagBitsToBitArray',
+ 'FlagBitArrayToBits',
+ 'FlagArrayToBits',
+ 'FlagBitsToArray',
+ 'FindFlagByName',
+ 'FindFlagByChar',
+ 'FindFlagChar',
+ 'ReadFlagString',
+ 'CanAdminTarget',
+ 'CreateAuthMethod',
+ 'SetAdmGroupImmunityLevel',
+ 'GetAdmGroupImmunityLevel',
+ 'SetAdminImmunityLevel',
+ 'GetAdminImmunityLevel',
+ 'FlagToBit',
+ 'BitToFlag',
+ 'ServerCommand',
+ 'ServerCommandEx',
+ 'InsertServerCommand',
+ 'ServerExecute',
+ 'ClientCommand',
+ 'FakeClientCommand',
+ 'FakeClientCommandEx',
+ 'PrintToServer',
+ 'PrintToConsole',
+ 'ReplyToCommand',
+ 'GetCmdReplySource',
+ 'SetCmdReplySource',
+ 'IsChatTrigger',
+ 'ShowActivity2',
+ 'ShowActivity',
+ 'ShowActivityEx',
+ 'FormatActivitySource',
+ 'SrvCmd',
+ 'RegServerCmd',
+ 'ConCmd',
+ 'RegConsoleCmd',
+ 'RegAdminCmd',
+ 'GetCmdArgs',
+ 'GetCmdArg',
+ 'GetCmdArgString',
+ 'CreateConVar',
+ 'FindConVar',
+ 'ConVarChanged',
+ 'HookConVarChange',
+ 'UnhookConVarChange',
+ 'GetConVarBool',
+ 'SetConVarBool',
+ 'GetConVarInt',
+ 'SetConVarInt',
+ 'GetConVarFloat',
+ 'SetConVarFloat',
+ 'GetConVarString',
+ 'SetConVarString',
+ 'ResetConVar',
+ 'GetConVarDefault',
+ 'GetConVarFlags',
+ 'SetConVarFlags',
+ 'GetConVarBounds',
+ 'SetConVarBounds',
+ 'GetConVarName',
+ 'QueryClientConVar',
+ 'GetCommandIterator',
+ 'ReadCommandIterator',
+ 'CheckCommandAccess',
+ 'CheckAccess',
+ 'IsValidConVarChar',
+ 'GetCommandFlags',
+ 'SetCommandFlags',
+ 'FindFirstConCommand',
+ 'FindNextConCommand',
+ 'SendConVarValue',
+ 'AddServerTag',
+ 'RemoveServerTag',
+ 'CommandListener',
+ 'AddCommandListener',
+ 'RemoveCommandListener',
+ 'CommandExists',
+ 'OnClientSayCommand',
+ 'OnClientSayCommand_Post',
+ 'TF2_IgnitePlayer',
+ 'TF2_RespawnPlayer',
+ 'TF2_RegeneratePlayer',
+ 'TF2_AddCondition',
+ 'TF2_RemoveCondition',
+ 'TF2_SetPlayerPowerPlay',
+ 'TF2_DisguisePlayer',
+ 'TF2_RemovePlayerDisguise',
+ 'TF2_StunPlayer',
+ 'TF2_MakeBleed',
+ 'TF2_GetClass',
+ 'TF2_CalcIsAttackCritical',
+ 'TF2_OnIsHolidayActive',
+ 'TF2_IsHolidayActive',
+ 'TF2_IsPlayerInDuel',
+ 'TF2_RemoveWearable',
+ 'TF2_OnConditionAdded',
+ 'TF2_OnConditionRemoved',
+ 'TF2_OnWaitingForPlayersStart',
+ 'TF2_OnWaitingForPlayersEnd',
+ 'TF2_OnPlayerTeleport',
+ 'SQL_Connect',
+ 'SQL_DefConnect',
+ 'SQL_ConnectCustom',
+ 'SQLite_UseDatabase',
+ 'SQL_CheckConfig',
+ 'SQL_GetDriver',
+ 'SQL_ReadDriver',
+ 'SQL_GetDriverIdent',
+ 'SQL_GetDriverProduct',
+ 'SQL_SetCharset',
+ 'SQL_GetAffectedRows',
+ 'SQL_GetInsertId',
+ 'SQL_GetError',
+ 'SQL_EscapeString',
+ 'SQL_QuoteString',
+ 'SQL_FastQuery',
+ 'SQL_Query',
+ 'SQL_PrepareQuery',
+ 'SQL_FetchMoreResults',
+ 'SQL_HasResultSet',
+ 'SQL_GetRowCount',
+ 'SQL_GetFieldCount',
+ 'SQL_FieldNumToName',
+ 'SQL_FieldNameToNum',
+ 'SQL_FetchRow',
+ 'SQL_MoreRows',
+ 'SQL_Rewind',
+ 'SQL_FetchString',
+ 'SQL_FetchFloat',
+ 'SQL_FetchInt',
+ 'SQL_IsFieldNull',
+ 'SQL_FetchSize',
+ 'SQL_BindParamInt',
+ 'SQL_BindParamFloat',
+ 'SQL_BindParamString',
+ 'SQL_Execute',
+ 'SQL_LockDatabase',
+ 'SQL_UnlockDatabase',
+ 'SQLTCallback',
+ 'SQL_IsSameConnection',
+ 'SQL_TConnect',
+ 'SQL_TQuery',
+ 'SQL_CreateTransaction',
+ 'SQL_AddQuery',
+ 'SQLTxnSuccess',
+ 'SQLTxnFailure',
+ 'SQL_ExecuteTransaction',
+ 'CloseHandle',
+ 'CloneHandle',
+ 'MenuHandler',
+ 'CreateMenu',
+ 'DisplayMenu',
+ 'DisplayMenuAtItem',
+ 'AddMenuItem',
+ 'InsertMenuItem',
+ 'RemoveMenuItem',
+ 'RemoveAllMenuItems',
+ 'GetMenuItem',
+ 'GetMenuSelectionPosition',
+ 'GetMenuItemCount',
+ 'SetMenuPagination',
+ 'GetMenuPagination',
+ 'GetMenuStyle',
+ 'SetMenuTitle',
+ 'GetMenuTitle',
+ 'CreatePanelFromMenu',
+ 'GetMenuExitButton',
+ 'SetMenuExitButton',
+ 'GetMenuExitBackButton',
+ 'SetMenuExitBackButton',
+ 'SetMenuNoVoteButton',
+ 'CancelMenu',
+ 'GetMenuOptionFlags',
+ 'SetMenuOptionFlags',
+ 'IsVoteInProgress',
+ 'CancelVote',
+ 'VoteMenu',
+ 'VoteMenuToAll',
+ 'VoteHandler',
+ 'SetVoteResultCallback',
+ 'CheckVoteDelay',
+ 'IsClientInVotePool',
+ 'RedrawClientVoteMenu',
+ 'GetMenuStyleHandle',
+ 'CreatePanel',
+ 'CreateMenuEx',
+ 'GetClientMenu',
+ 'CancelClientMenu',
+ 'GetMaxPageItems',
+ 'GetPanelStyle',
+ 'SetPanelTitle',
+ 'DrawPanelItem',
+ 'DrawPanelText',
+ 'CanPanelDrawFlags',
+ 'SetPanelKeys',
+ 'SendPanelToClient',
+ 'GetPanelTextRemaining',
+ 'GetPanelCurrentKey',
+ 'SetPanelCurrentKey',
+ 'RedrawMenuItem',
+ 'InternalShowMenu',
+ 'GetMenuVoteInfo',
+ 'IsNewVoteAllowed',
+ 'PrefetchSound',
+ 'EmitAmbientSound',
+ 'FadeClientVolume',
+ 'StopSound',
+ 'EmitSound',
+ 'EmitSentence',
+ 'GetDistGainFromSoundLevel',
+ 'AmbientSHook',
+ 'NormalSHook',
+ 'AddAmbientSoundHook',
+ 'AddNormalSoundHook',
+ 'RemoveAmbientSoundHook',
+ 'RemoveNormalSoundHook',
+ 'EmitSoundToClient',
+ 'EmitSoundToAll',
+ 'ATTN_TO_SNDLEVEL',
+ 'GetGameSoundParams',
+ 'EmitGameSound',
+ 'EmitAmbientGameSound',
+ 'EmitGameSoundToClient',
+ 'EmitGameSoundToAll',
+ 'PrecacheScriptSound',
+ 'strlen',
+ 'StrContains',
+ 'strcmp',
+ 'strncmp',
+ 'StrEqual',
+ 'strcopy',
+ 'Format',
+ 'FormatEx',
+ 'VFormat',
+ 'StringToInt',
+ 'StringToIntEx',
+ 'IntToString',
+ 'StringToFloat',
+ 'StringToFloatEx',
+ 'FloatToString',
+ 'BreakString',
+ 'TrimString',
+ 'SplitString',
+ 'ReplaceString',
+ 'ReplaceStringEx',
+ 'GetCharBytes',
+ 'IsCharAlpha',
+ 'IsCharNumeric',
+ 'IsCharSpace',
+ 'IsCharMB',
+ 'IsCharUpper',
+ 'IsCharLower',
+ 'StripQuotes',
+ 'CharToUpper',
+ 'CharToLower',
+ 'FindCharInString',
+ 'StrCat',
+ 'ExplodeString',
+ 'ImplodeStrings',
+ 'GetVectorLength',
+ 'GetVectorDistance',
+ 'GetVectorDotProduct',
+ 'GetVectorCrossProduct',
+ 'NormalizeVector',
+ 'GetAngleVectors',
+ 'GetVectorAngles',
+ 'GetVectorVectors',
+ 'AddVectors',
+ 'SubtractVectors',
+ 'ScaleVector',
+ 'NegateVector',
+ 'MakeVectorFromPoints',
+ 'BaseComm_IsClientGagged',
+ 'BaseComm_IsClientMuted',
+ 'BaseComm_SetClientGag',
+ 'BaseComm_SetClientMute',
+ 'FormatUserLogText',
+ 'FindPluginByFile',
+ 'FindTarget',
+ 'AcceptEntityInput',
+ 'SetVariantBool',
+ 'SetVariantString',
+ 'SetVariantInt',
+ 'SetVariantFloat',
+ 'SetVariantVector3D',
+ 'SetVariantPosVector3D',
+ 'SetVariantColor',
+ 'SetVariantEntity',
+ 'GameRules_GetProp',
+ 'GameRules_SetProp',
+ 'GameRules_GetPropFloat',
+ 'GameRules_SetPropFloat',
+ 'GameRules_GetPropEnt',
+ 'GameRules_SetPropEnt',
+ 'GameRules_GetPropVector',
+ 'GameRules_SetPropVector',
+ 'GameRules_GetPropString',
+ 'GameRules_SetPropString',
+ 'GameRules_GetRoundState',
+ 'OnClientConnect',
+ 'OnClientConnected',
+ 'OnClientPutInServer',
+ 'OnClientDisconnect',
+ 'OnClientDisconnect_Post',
+ 'OnClientCommand',
+ 'OnClientSettingsChanged',
+ 'OnClientAuthorized',
+ 'OnClientPreAdminCheck',
+ 'OnClientPostAdminFilter',
+ 'OnClientPostAdminCheck',
+ 'GetMaxClients',
+ 'GetMaxHumanPlayers',
+ 'GetClientCount',
+ 'GetClientName',
+ 'GetClientIP',
+ 'GetClientAuthString',
+ 'GetClientAuthId',
+ 'GetSteamAccountID',
+ 'GetClientUserId',
+ 'IsClientConnected',
+ 'IsClientInGame',
+ 'IsClientInKickQueue',
+ 'IsClientAuthorized',
+ 'IsFakeClient',
+ 'IsClientSourceTV',
+ 'IsClientReplay',
+ 'IsClientObserver',
+ 'IsPlayerAlive',
+ 'GetClientInfo',
+ 'GetClientTeam',
+ 'SetUserAdmin',
+ 'GetUserAdmin',
+ 'AddUserFlags',
+ 'RemoveUserFlags',
+ 'SetUserFlagBits',
+ 'GetUserFlagBits',
+ 'CanUserTarget',
+ 'RunAdminCacheChecks',
+ 'NotifyPostAdminCheck',
+ 'CreateFakeClient',
+ 'SetFakeClientConVar',
+ 'GetClientHealth',
+ 'GetClientModel',
+ 'GetClientWeapon',
+ 'GetClientMaxs',
+ 'GetClientMins',
+ 'GetClientAbsAngles',
+ 'GetClientAbsOrigin',
+ 'GetClientArmor',
+ 'GetClientDeaths',
+ 'GetClientFrags',
+ 'GetClientDataRate',
+ 'IsClientTimingOut',
+ 'GetClientTime',
+ 'GetClientLatency',
+ 'GetClientAvgLatency',
+ 'GetClientAvgLoss',
+ 'GetClientAvgChoke',
+ 'GetClientAvgData',
+ 'GetClientAvgPackets',
+ 'GetClientOfUserId',
+ 'KickClient',
+ 'KickClientEx',
+ 'ChangeClientTeam',
+ 'GetClientSerial',
+ 'GetClientFromSerial',
+ 'FindStringTable',
+ 'GetNumStringTables',
+ 'GetStringTableNumStrings',
+ 'GetStringTableMaxStrings',
+ 'GetStringTableName',
+ 'FindStringIndex',
+ 'ReadStringTable',
+ 'GetStringTableDataLength',
+ 'GetStringTableData',
+ 'SetStringTableData',
+ 'AddToStringTable',
+ 'LockStringTables',
+ 'AddFileToDownloadsTable',
+ 'GetEntityFlags',
+ 'SetEntityFlags',
+ 'GetEntityMoveType',
+ 'SetEntityMoveType',
+ 'GetEntityRenderMode',
+ 'SetEntityRenderMode',
+ 'GetEntityRenderFx',
+ 'SetEntityRenderFx',
+ 'SetEntityRenderColor',
+ 'GetEntityGravity',
+ 'SetEntityGravity',
+ 'SetEntityHealth',
+ 'GetClientButtons',
+ 'EntityOutput',
+ 'HookEntityOutput',
+ 'UnhookEntityOutput',
+ 'HookSingleEntityOutput',
+ 'UnhookSingleEntityOutput',
+ 'SMC_CreateParser',
+ 'SMC_ParseFile',
+ 'SMC_GetErrorString',
+ 'SMC_ParseStart',
+ 'SMC_SetParseStart',
+ 'SMC_ParseEnd',
+ 'SMC_SetParseEnd',
+ 'SMC_NewSection',
+ 'SMC_KeyValue',
+ 'SMC_EndSection',
+ 'SMC_SetReaders',
+ 'SMC_RawLine',
+ 'SMC_SetRawLine',
+ 'BfWriteBool',
+ 'BfWriteByte',
+ 'BfWriteChar',
+ 'BfWriteShort',
+ 'BfWriteWord',
+ 'BfWriteNum',
+ 'BfWriteFloat',
+ 'BfWriteString',
+ 'BfWriteEntity',
+ 'BfWriteAngle',
+ 'BfWriteCoord',
+ 'BfWriteVecCoord',
+ 'BfWriteVecNormal',
+ 'BfWriteAngles',
+ 'BfReadBool',
+ 'BfReadByte',
+ 'BfReadChar',
+ 'BfReadShort',
+ 'BfReadWord',
+ 'BfReadNum',
+ 'BfReadFloat',
+ 'BfReadString',
+ 'BfReadEntity',
+ 'BfReadAngle',
+ 'BfReadCoord',
+ 'BfReadVecCoord',
+ 'BfReadVecNormal',
+ 'BfReadAngles',
+ 'BfGetNumBytesLeft',
+ 'CreateProfiler',
+ 'StartProfiling',
+ 'StopProfiling',
+ 'GetProfilerTime',
+ 'OnPluginStart',
+ 'AskPluginLoad2',
+ 'OnPluginEnd',
+ 'OnPluginPauseChange',
+ 'OnGameFrame',
+ 'OnMapStart',
+ 'OnMapEnd',
+ 'OnConfigsExecuted',
+ 'OnAutoConfigsBuffered',
+ 'OnAllPluginsLoaded',
+ 'GetMyHandle',
+ 'GetPluginIterator',
+ 'MorePlugins',
+ 'ReadPlugin',
+ 'GetPluginStatus',
+ 'GetPluginFilename',
+ 'IsPluginDebugging',
+ 'GetPluginInfo',
+ 'FindPluginByNumber',
+ 'SetFailState',
+ 'ThrowError',
+ 'GetTime',
+ 'FormatTime',
+ 'LoadGameConfigFile',
+ 'GameConfGetOffset',
+ 'GameConfGetKeyValue',
+ 'GameConfGetAddress',
+ 'GetSysTickCount',
+ 'AutoExecConfig',
+ 'RegPluginLibrary',
+ 'LibraryExists',
+ 'GetExtensionFileStatus',
+ 'OnLibraryAdded',
+ 'OnLibraryRemoved',
+ 'ReadMapList',
+ 'SetMapListCompatBind',
+ 'OnClientFloodCheck',
+ 'OnClientFloodResult',
+ 'CanTestFeatures',
+ 'GetFeatureStatus',
+ 'RequireFeature',
+ 'LoadFromAddress',
+ 'StoreToAddress',
+ 'CreateStack',
+ 'PushStackCell',
+ 'PushStackString',
+ 'PushStackArray',
+ 'PopStackCell',
+ 'PopStackString',
+ 'PopStackArray',
+ 'IsStackEmpty',
+ 'PopStack',
+ 'OnPlayerRunCmd',
+ 'BuildPath',
+ 'OpenDirectory',
+ 'ReadDirEntry',
+ 'OpenFile',
+ 'DeleteFile',
+ 'ReadFileLine',
+ 'ReadFile',
+ 'ReadFileString',
+ 'WriteFile',
+ 'WriteFileString',
+ 'WriteFileLine',
+ 'ReadFileCell',
+ 'WriteFileCell',
+ 'IsEndOfFile',
+ 'FileSeek',
+ 'FilePosition',
+ 'FileExists',
+ 'RenameFile',
+ 'DirExists',
+ 'FileSize',
+ 'FlushFile',
+ 'RemoveDir',
+ 'CreateDirectory',
+ 'GetFileTime',
+ 'LogToOpenFile',
+ 'LogToOpenFileEx',
+ 'PbReadInt',
+ 'PbReadFloat',
+ 'PbReadBool',
+ 'PbReadString',
+ 'PbReadColor',
+ 'PbReadAngle',
+ 'PbReadVector',
+ 'PbReadVector2D',
+ 'PbGetRepeatedFieldCount',
+ 'PbSetInt',
+ 'PbSetFloat',
+ 'PbSetBool',
+ 'PbSetString',
+ 'PbSetColor',
+ 'PbSetAngle',
+ 'PbSetVector',
+ 'PbSetVector2D',
+ 'PbAddInt',
+ 'PbAddFloat',
+ 'PbAddBool',
+ 'PbAddString',
+ 'PbAddColor',
+ 'PbAddAngle',
+ 'PbAddVector',
+ 'PbAddVector2D',
+ 'PbRemoveRepeatedFieldValue',
+ 'PbReadMessage',
+ 'PbReadRepeatedMessage',
+ 'PbAddMessage',
+ 'SetNextMap',
+ 'GetNextMap',
+ 'ForceChangeLevel',
+ 'GetMapHistorySize',
+ 'GetMapHistory',
+ 'GeoipCode2',
+ 'GeoipCode3',
+ 'GeoipCountry',
+ 'MarkNativeAsOptional',
+ 'RegClientCookie',
+ 'FindClientCookie',
+ 'SetClientCookie',
+ 'GetClientCookie',
+ 'SetAuthIdCookie',
+ 'AreClientCookiesCached',
+ 'OnClientCookiesCached',
+ 'CookieMenuHandler',
+ 'SetCookiePrefabMenu',
+ 'SetCookieMenuItem',
+ 'ShowCookieMenu',
+ 'GetCookieIterator',
+ 'ReadCookieIterator',
+ 'GetCookieAccess',
+ 'GetClientCookieTime',
+ 'LoadTranslations',
+ 'SetGlobalTransTarget',
+ 'GetClientLanguage',
+ 'GetServerLanguage',
+ 'GetLanguageCount',
+ 'GetLanguageInfo',
+ 'SetClientLanguage',
+ 'GetLanguageByCode',
+ 'GetLanguageByName',
+ 'CS_OnBuyCommand',
+ 'CS_OnCSWeaponDrop',
+ 'CS_OnGetWeaponPrice',
+ 'CS_OnTerminateRound',
+ 'CS_RespawnPlayer',
+ 'CS_SwitchTeam',
+ 'CS_DropWeapon',
+ 'CS_TerminateRound',
+ 'CS_GetTranslatedWeaponAlias',
+ 'CS_GetWeaponPrice',
+ 'CS_GetClientClanTag',
+ 'CS_SetClientClanTag',
+ 'CS_GetTeamScore',
+ 'CS_SetTeamScore',
+ 'CS_GetMVPCount',
+ 'CS_SetMVPCount',
+ 'CS_GetClientContributionScore',
+ 'CS_SetClientContributionScore',
+ 'CS_GetClientAssists',
+ 'CS_SetClientAssists',
+ 'CS_AliasToWeaponID',
+ 'CS_WeaponIDToAlias',
+ 'CS_IsValidWeaponID',
+ 'CS_UpdateClientModel',
+ 'LogToGame',
+ 'SetRandomSeed',
+ 'GetRandomFloat',
+ 'GetRandomInt',
+ 'IsMapValid',
+ 'IsDedicatedServer',
+ 'GetEngineTime',
+ 'GetGameTime',
+ 'GetGameTickCount',
+ 'GetGameDescription',
+ 'GetGameFolderName',
+ 'GetCurrentMap',
+ 'PrecacheModel',
+ 'PrecacheSentenceFile',
+ 'PrecacheDecal',
+ 'PrecacheGeneric',
+ 'IsModelPrecached',
+ 'IsDecalPrecached',
+ 'IsGenericPrecached',
+ 'PrecacheSound',
+ 'IsSoundPrecached',
+ 'CreateDialog',
+ 'GetEngineVersion',
+ 'PrintToChat',
+ 'PrintToChatAll',
+ 'PrintCenterText',
+ 'PrintCenterTextAll',
+ 'PrintHintText',
+ 'PrintHintTextToAll',
+ 'ShowVGUIPanel',
+ 'CreateHudSynchronizer',
+ 'SetHudTextParams',
+ 'SetHudTextParamsEx',
+ 'ShowSyncHudText',
+ 'ClearSyncHud',
+ 'ShowHudText',
+ 'ShowMOTDPanel',
+ 'DisplayAskConnectBox',
+ 'EntIndexToEntRef',
+ 'EntRefToEntIndex',
+ 'MakeCompatEntRef',
+ 'SetClientViewEntity',
+ 'SetLightStyle',
+ 'GetClientEyePosition',
+ 'CreateDataPack',
+ 'WritePackCell',
+ 'WritePackFloat',
+ 'WritePackString',
+ 'ReadPackCell',
+ 'ReadPackFloat',
+ 'ReadPackString',
+ 'ResetPack',
+ 'GetPackPosition',
+ 'SetPackPosition',
+ 'IsPackReadable',
+ 'LogMessage',
+ 'LogToFile',
+ 'LogToFileEx',
+ 'LogAction',
+ 'LogError',
+ 'OnLogAction',
+ 'GameLogHook',
+ 'AddGameLogHook',
+ 'RemoveGameLogHook',
+ 'FindTeamByName',
+ 'StartPrepSDKCall',
+ 'PrepSDKCall_SetVirtual',
+ 'PrepSDKCall_SetSignature',
+ 'PrepSDKCall_SetAddress',
+ 'PrepSDKCall_SetFromConf',
+ 'PrepSDKCall_SetReturnInfo',
+ 'PrepSDKCall_AddParameter',
+ 'EndPrepSDKCall',
+ 'SDKCall',
+ 'GetPlayerResourceEntity',
+)
+
+
+if __name__ == '__main__': # pragma: no cover
+ import re
+ import sys
+ try:
+ from urllib import FancyURLopener
+ except ImportError:
+ from urllib.request import FancyURLopener
+
+ from pygments.util import format_lines
+
+ # urllib ends up wanting to import a module called 'math' -- if
+ # pygments/lexers is in the path, this ends badly.
+ for i in range(len(sys.path)-1, -1, -1):
+ if sys.path[i].endswith('/lexers'):
+ del sys.path[i]
+
+ class Opener(FancyURLopener):
+ version = 'Mozilla/5.0 (Pygments Sourcemod Builtins Update)'
+
+ opener = Opener()
+
+ def get_version():
+ f = opener.open('http://docs.sourcemod.net/api/index.php')
+ r = re.compile(r'SourceMod v\.<b>([\d\.]+(?:-\w+)?)</td>')
+ for line in f:
+ m = r.search(line)
+ if m is not None:
+ return m.groups()[0]
+ raise ValueError('No version in api docs')
+
+ def get_sm_functions():
+ f = opener.open('http://docs.sourcemod.net/api/SMfuncs.js')
+ r = re.compile(r'SMfunctions\[\d+\] = Array \("(?:public )?([^,]+)",".+"\);')
+ functions = []
+ for line in f:
+ m = r.match(line)
+ if m is not None:
+ functions.append(m.groups()[0])
+ return functions
+
+ def regenerate(filename, natives):
+ with open(filename) as fp:
+ content = fp.read()
+
+ header = content[:content.find('FUNCTIONS = (')]
+ footer = content[content.find("if __name__ == '__main__':")-1:]
+
+
+ with open(filename, 'w') as fp:
+ fp.write(header)
+ fp.write(format_lines('FUNCTIONS', natives))
+ fp.write(footer)
+
+ def run():
+ version = get_version()
+ print('> Downloading function index for SourceMod %s' % version)
+ functions = get_sm_functions()
+ print('> %d functions found:' % len(functions))
+
+ functionlist = []
+ for full_function_name in functions:
+ print('>> %s' % full_function_name)
+ functionlist.append(full_function_name)
+
+ regenerate(__file__, functionlist)
+
+
+ run()
diff --git a/wandb/vendor/pygments/lexers/_stan_builtins.py b/wandb/vendor/pygments/lexers/_stan_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..a189647aab513e423aede97330b2a275de8dd63d
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_stan_builtins.py
@@ -0,0 +1,532 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._stan_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ This file contains the names of functions for Stan used by
+ ``pygments.lexers.math.StanLexer. This is for Stan language version 2.8.0.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+KEYWORDS = (
+ 'else',
+ 'for',
+ 'if',
+ 'in',
+ 'increment_log_prob',
+ 'integrate_ode',
+ 'lp__',
+ 'print',
+ 'reject',
+ 'return',
+ 'while'
+)
+
+TYPES = (
+ 'cholesky_factor_corr',
+ 'cholesky_factor_cov',
+ 'corr_matrix',
+ 'cov_matrix',
+ 'int',
+ 'matrix',
+ 'ordered',
+ 'positive_ordered',
+ 'real',
+ 'row_vector',
+ 'row_vectormatrix',
+ 'simplex',
+ 'unit_vector',
+ 'vector',
+ 'void')
+
+FUNCTIONS = (
+ 'Phi',
+ 'Phi_approx',
+ 'abs',
+ 'acos',
+ 'acosh',
+ 'append_col',
+ 'append_row',
+ 'asin',
+ 'asinh',
+ 'atan',
+ 'atan2',
+ 'atanh',
+ 'bernoulli_ccdf_log',
+ 'bernoulli_cdf',
+ 'bernoulli_cdf_log',
+ 'bernoulli_log',
+ 'bernoulli_logit_log',
+ 'bernoulli_rng',
+ 'bessel_first_kind',
+ 'bessel_second_kind',
+ 'beta_binomial_ccdf_log',
+ 'beta_binomial_cdf',
+ 'beta_binomial_cdf_log',
+ 'beta_binomial_log',
+ 'beta_binomial_rng',
+ 'beta_ccdf_log',
+ 'beta_cdf',
+ 'beta_cdf_log',
+ 'beta_log',
+ 'beta_rng',
+ 'binary_log_loss',
+ 'binomial_ccdf_log',
+ 'binomial_cdf',
+ 'binomial_cdf_log',
+ 'binomial_coefficient_log',
+ 'binomial_log',
+ 'binomial_logit_log',
+ 'binomial_rng',
+ 'block',
+ 'categorical_log',
+ 'categorical_logit_log',
+ 'categorical_rng',
+ 'cauchy_ccdf_log',
+ 'cauchy_cdf',
+ 'cauchy_cdf_log',
+ 'cauchy_log',
+ 'cauchy_rng',
+ 'cbrt',
+ 'ceil',
+ 'chi_square_ccdf_log',
+ 'chi_square_cdf',
+ 'chi_square_cdf_log',
+ 'chi_square_log',
+ 'chi_square_rng',
+ 'cholesky_decompose',
+ 'col',
+ 'cols',
+ 'columns_dot_product',
+ 'columns_dot_self',
+ 'cos',
+ 'cosh',
+ 'crossprod',
+ 'csr_extract_u',
+ 'csr_extract_v',
+ 'csr_extract_w',
+ 'csr_matrix_times_vector',
+ 'csr_to_dense_matrix',
+ 'cumulative_sum',
+ 'determinant',
+ 'diag_matrix',
+ 'diag_post_multiply',
+ 'diag_pre_multiply',
+ 'diagonal',
+ 'digamma',
+ 'dims',
+ 'dirichlet_log',
+ 'dirichlet_rng',
+ 'distance',
+ 'dot_product',
+ 'dot_self',
+ 'double_exponential_ccdf_log',
+ 'double_exponential_cdf',
+ 'double_exponential_cdf_log',
+ 'double_exponential_log',
+ 'double_exponential_rng',
+ 'e',
+ 'eigenvalues_sym',
+ 'eigenvectors_sym',
+ 'erf',
+ 'erfc',
+ 'exp',
+ 'exp2',
+ 'exp_mod_normal_ccdf_log',
+ 'exp_mod_normal_cdf',
+ 'exp_mod_normal_cdf_log',
+ 'exp_mod_normal_log',
+ 'exp_mod_normal_rng',
+ 'expm1',
+ 'exponential_ccdf_log',
+ 'exponential_cdf',
+ 'exponential_cdf_log',
+ 'exponential_log',
+ 'exponential_rng',
+ 'fabs',
+ 'falling_factorial',
+ 'fdim',
+ 'floor',
+ 'fma',
+ 'fmax',
+ 'fmin',
+ 'fmod',
+ 'frechet_ccdf_log',
+ 'frechet_cdf',
+ 'frechet_cdf_log',
+ 'frechet_log',
+ 'frechet_rng',
+ 'gamma_ccdf_log',
+ 'gamma_cdf',
+ 'gamma_cdf_log',
+ 'gamma_log',
+ 'gamma_p',
+ 'gamma_q',
+ 'gamma_rng',
+ 'gaussian_dlm_obs_log',
+ 'get_lp',
+ 'gumbel_ccdf_log',
+ 'gumbel_cdf',
+ 'gumbel_cdf_log',
+ 'gumbel_log',
+ 'gumbel_rng',
+ 'head',
+ 'hypergeometric_log',
+ 'hypergeometric_rng',
+ 'hypot',
+ 'if_else',
+ 'int_step',
+ 'inv',
+ 'inv_chi_square_ccdf_log',
+ 'inv_chi_square_cdf',
+ 'inv_chi_square_cdf_log',
+ 'inv_chi_square_log',
+ 'inv_chi_square_rng',
+ 'inv_cloglog',
+ 'inv_gamma_ccdf_log',
+ 'inv_gamma_cdf',
+ 'inv_gamma_cdf_log',
+ 'inv_gamma_log',
+ 'inv_gamma_rng',
+ 'inv_logit',
+ 'inv_phi',
+ 'inv_sqrt',
+ 'inv_square',
+ 'inv_wishart_log',
+ 'inv_wishart_rng',
+ 'inverse',
+ 'inverse_spd',
+ 'is_inf',
+ 'is_nan',
+ 'lbeta',
+ 'lgamma',
+ 'lkj_corr_cholesky_log',
+ 'lkj_corr_cholesky_rng',
+ 'lkj_corr_log',
+ 'lkj_corr_rng',
+ 'lmgamma',
+ 'log',
+ 'log10',
+ 'log1m',
+ 'log1m_exp',
+ 'log1m_inv_logit',
+ 'log1p',
+ 'log1p_exp',
+ 'log2',
+ 'log_determinant',
+ 'log_diff_exp',
+ 'log_falling_factorial',
+ 'log_inv_logit',
+ 'log_mix',
+ 'log_rising_factorial',
+ 'log_softmax',
+ 'log_sum_exp',
+ 'logistic_ccdf_log',
+ 'logistic_cdf',
+ 'logistic_cdf_log',
+ 'logistic_log',
+ 'logistic_rng',
+ 'logit',
+ 'lognormal_ccdf_log',
+ 'lognormal_cdf',
+ 'lognormal_cdf_log',
+ 'lognormal_log',
+ 'lognormal_rng',
+ 'machine_precision',
+ 'max',
+ 'mdivide_left_tri_low',
+ 'mdivide_right_tri_low',
+ 'mean',
+ 'min',
+ 'modified_bessel_first_kind',
+ 'modified_bessel_second_kind',
+ 'multi_gp_cholesky_log',
+ 'multi_gp_log',
+ 'multi_normal_cholesky_log',
+ 'multi_normal_cholesky_rng',
+ 'multi_normal_log',
+ 'multi_normal_prec_log',
+ 'multi_normal_rng',
+ 'multi_student_t_log',
+ 'multi_student_t_rng',
+ 'multinomial_log',
+ 'multinomial_rng',
+ 'multiply_log',
+ 'multiply_lower_tri_self_transpose',
+ 'neg_binomial_2_ccdf_log',
+ 'neg_binomial_2_cdf',
+ 'neg_binomial_2_cdf_log',
+ 'neg_binomial_2_log',
+ 'neg_binomial_2_log_log',
+ 'neg_binomial_2_log_rng',
+ 'neg_binomial_2_rng',
+ 'neg_binomial_ccdf_log',
+ 'neg_binomial_cdf',
+ 'neg_binomial_cdf_log',
+ 'neg_binomial_log',
+ 'neg_binomial_rng',
+ 'negative_infinity',
+ 'normal_ccdf_log',
+ 'normal_cdf',
+ 'normal_cdf_log',
+ 'normal_log',
+ 'normal_rng',
+ 'not_a_number',
+ 'num_elements',
+ 'ordered_logistic_log',
+ 'ordered_logistic_rng',
+ 'owens_t',
+ 'pareto_ccdf_log',
+ 'pareto_cdf',
+ 'pareto_cdf_log',
+ 'pareto_log',
+ 'pareto_rng',
+ 'pareto_type_2_ccdf_log',
+ 'pareto_type_2_cdf',
+ 'pareto_type_2_cdf_log',
+ 'pareto_type_2_log',
+ 'pareto_type_2_rng',
+ 'pi',
+ 'poisson_ccdf_log',
+ 'poisson_cdf',
+ 'poisson_cdf_log',
+ 'poisson_log',
+ 'poisson_log_log',
+ 'poisson_log_rng',
+ 'poisson_rng',
+ 'positive_infinity',
+ 'pow',
+ 'prod',
+ 'qr_Q',
+ 'qr_R',
+ 'quad_form',
+ 'quad_form_diag',
+ 'quad_form_sym',
+ 'rank',
+ 'rayleigh_ccdf_log',
+ 'rayleigh_cdf',
+ 'rayleigh_cdf_log',
+ 'rayleigh_log',
+ 'rayleigh_rng',
+ 'rep_array',
+ 'rep_matrix',
+ 'rep_row_vector',
+ 'rep_vector',
+ 'rising_factorial',
+ 'round',
+ 'row',
+ 'rows',
+ 'rows_dot_product',
+ 'rows_dot_self',
+ 'scaled_inv_chi_square_ccdf_log',
+ 'scaled_inv_chi_square_cdf',
+ 'scaled_inv_chi_square_cdf_log',
+ 'scaled_inv_chi_square_log',
+ 'scaled_inv_chi_square_rng',
+ 'sd',
+ 'segment',
+ 'sin',
+ 'singular_values',
+ 'sinh',
+ 'size',
+ 'skew_normal_ccdf_log',
+ 'skew_normal_cdf',
+ 'skew_normal_cdf_log',
+ 'skew_normal_log',
+ 'skew_normal_rng',
+ 'softmax',
+ 'sort_asc',
+ 'sort_desc',
+ 'sort_indices_asc',
+ 'sort_indices_desc',
+ 'sqrt',
+ 'sqrt2',
+ 'square',
+ 'squared_distance',
+ 'step',
+ 'student_t_ccdf_log',
+ 'student_t_cdf',
+ 'student_t_cdf_log',
+ 'student_t_log',
+ 'student_t_rng',
+ 'sub_col',
+ 'sub_row',
+ 'sum',
+ 'tail',
+ 'tan',
+ 'tanh',
+ 'tcrossprod',
+ 'tgamma',
+ 'to_array_1d',
+ 'to_array_2d',
+ 'to_matrix',
+ 'to_row_vector',
+ 'to_vector',
+ 'trace',
+ 'trace_gen_quad_form',
+ 'trace_quad_form',
+ 'trigamma',
+ 'trunc',
+ 'uniform_ccdf_log',
+ 'uniform_cdf',
+ 'uniform_cdf_log',
+ 'uniform_log',
+ 'uniform_rng',
+ 'variance',
+ 'von_mises_log',
+ 'von_mises_rng',
+ 'weibull_ccdf_log',
+ 'weibull_cdf',
+ 'weibull_cdf_log',
+ 'weibull_log',
+ 'weibull_rng',
+ 'wiener_log',
+ 'wishart_log',
+ 'wishart_rng'
+)
+
+DISTRIBUTIONS = (
+ 'bernoulli',
+ 'bernoulli_logit',
+ 'beta',
+ 'beta_binomial',
+ 'binomial',
+ 'binomial_logit',
+ 'categorical',
+ 'categorical_logit',
+ 'cauchy',
+ 'chi_square',
+ 'dirichlet',
+ 'double_exponential',
+ 'exp_mod_normal',
+ 'exponential',
+ 'frechet',
+ 'gamma',
+ 'gaussian_dlm_obs',
+ 'gumbel',
+ 'hypergeometric',
+ 'inv_chi_square',
+ 'inv_gamma',
+ 'inv_wishart',
+ 'lkj_corr',
+ 'lkj_corr_cholesky',
+ 'logistic',
+ 'lognormal',
+ 'multi_gp',
+ 'multi_gp_cholesky',
+ 'multi_normal',
+ 'multi_normal_cholesky',
+ 'multi_normal_prec',
+ 'multi_student_t',
+ 'multinomial',
+ 'neg_binomial',
+ 'neg_binomial_2',
+ 'neg_binomial_2_log',
+ 'normal',
+ 'ordered_logistic',
+ 'pareto',
+ 'pareto_type_2',
+ 'poisson',
+ 'poisson_log',
+ 'rayleigh',
+ 'scaled_inv_chi_square',
+ 'skew_normal',
+ 'student_t',
+ 'uniform',
+ 'von_mises',
+ 'weibull',
+ 'wiener',
+ 'wishart'
+)
+
+RESERVED = (
+ 'alignas',
+ 'alignof',
+ 'and',
+ 'and_eq',
+ 'asm',
+ 'auto',
+ 'bitand',
+ 'bitor',
+ 'bool',
+ 'break',
+ 'case',
+ 'catch',
+ 'char',
+ 'char16_t',
+ 'char32_t',
+ 'class',
+ 'compl',
+ 'const',
+ 'const_cast',
+ 'constexpr',
+ 'continue',
+ 'decltype',
+ 'default',
+ 'delete',
+ 'do',
+ 'double',
+ 'dynamic_cast',
+ 'enum',
+ 'explicit',
+ 'export',
+ 'extern',
+ 'false',
+ 'false',
+ 'float',
+ 'friend',
+ 'fvar',
+ 'goto',
+ 'inline',
+ 'int',
+ 'long',
+ 'mutable',
+ 'namespace',
+ 'new',
+ 'noexcept',
+ 'not',
+ 'not_eq',
+ 'nullptr',
+ 'operator',
+ 'or',
+ 'or_eq',
+ 'private',
+ 'protected',
+ 'public',
+ 'register',
+ 'reinterpret_cast',
+ 'repeat',
+ 'short',
+ 'signed',
+ 'sizeof',
+ 'static',
+ 'static_assert',
+ 'static_cast',
+ 'struct',
+ 'switch',
+ 'template',
+ 'then',
+ 'this',
+ 'thread_local',
+ 'throw',
+ 'true',
+ 'true',
+ 'try',
+ 'typedef',
+ 'typeid',
+ 'typename',
+ 'union',
+ 'unsigned',
+ 'until',
+ 'using',
+ 'var',
+ 'virtual',
+ 'void',
+ 'volatile',
+ 'wchar_t',
+ 'xor',
+ 'xor_eq'
+)
+
diff --git a/wandb/vendor/pygments/lexers/_stata_builtins.py b/wandb/vendor/pygments/lexers/_stata_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..5f5f72a9788b3e6f03b88404923548d1b568fe34
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_stata_builtins.py
@@ -0,0 +1,419 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._stata_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Builtins for Stata
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+
+builtins_base = (
+ "if", "else", "in", "foreach", "for", "forv", "forva",
+ "forval", "forvalu", "forvalue", "forvalues", "by", "bys",
+ "bysort", "quietly", "qui", "about", "ac",
+ "ac_7", "acprplot", "acprplot_7", "adjust", "ado", "adopath",
+ "adoupdate", "alpha", "ameans", "an", "ano", "anov", "anova",
+ "anova_estat", "anova_terms", "anovadef", "aorder", "ap", "app",
+ "appe", "appen", "append", "arch", "arch_dr", "arch_estat",
+ "arch_p", "archlm", "areg", "areg_p", "args", "arima",
+ "arima_dr", "arima_estat", "arima_p", "as", "asmprobit",
+ "asmprobit_estat", "asmprobit_lf", "asmprobit_mfx__dlg",
+ "asmprobit_p", "ass", "asse", "asser", "assert", "avplot",
+ "avplot_7", "avplots", "avplots_7", "bcskew0", "bgodfrey",
+ "binreg", "bip0_lf", "biplot", "bipp_lf", "bipr_lf",
+ "bipr_p", "biprobit", "bitest", "bitesti", "bitowt", "blogit",
+ "bmemsize", "boot", "bootsamp", "bootstrap", "bootstrap_8",
+ "boxco_l", "boxco_p", "boxcox", "boxcox_6", "boxcox_p",
+ "bprobit", "br", "break", "brier", "bro", "brow", "brows",
+ "browse", "brr", "brrstat", "bs", "bs_7", "bsampl_w",
+ "bsample", "bsample_7", "bsqreg", "bstat", "bstat_7", "bstat_8",
+ "bstrap", "bstrap_7", "ca", "ca_estat", "ca_p", "cabiplot",
+ "camat", "canon", "canon_8", "canon_8_p", "canon_estat",
+ "canon_p", "cap", "caprojection", "capt", "captu", "captur",
+ "capture", "cat", "cc", "cchart", "cchart_7", "cci",
+ "cd", "censobs_table", "centile", "cf", "char", "chdir",
+ "checkdlgfiles", "checkestimationsample", "checkhlpfiles",
+ "checksum", "chelp", "ci", "cii", "cl", "class", "classutil",
+ "clear", "cli", "clis", "clist", "clo", "clog", "clog_lf",
+ "clog_p", "clogi", "clogi_sw", "clogit", "clogit_lf",
+ "clogit_p", "clogitp", "clogl_sw", "cloglog", "clonevar",
+ "clslistarray", "cluster", "cluster_measures", "cluster_stop",
+ "cluster_tree", "cluster_tree_8", "clustermat", "cmdlog",
+ "cnr", "cnre", "cnreg", "cnreg_p", "cnreg_sw", "cnsreg",
+ "codebook", "collaps4", "collapse", "colormult_nb",
+ "colormult_nw", "compare", "compress", "conf", "confi",
+ "confir", "confirm", "conren", "cons", "const", "constr",
+ "constra", "constrai", "constrain", "constraint", "continue",
+ "contract", "copy", "copyright", "copysource", "cor", "corc",
+ "corr", "corr2data", "corr_anti", "corr_kmo", "corr_smc",
+ "corre", "correl", "correla", "correlat", "correlate",
+ "corrgram", "cou", "coun", "count", "cox", "cox_p", "cox_sw",
+ "coxbase", "coxhaz", "coxvar", "cprplot", "cprplot_7",
+ "crc", "cret", "cretu", "cretur", "creturn", "cross", "cs",
+ "cscript", "cscript_log", "csi", "ct", "ct_is", "ctset",
+ "ctst_5", "ctst_st", "cttost", "cumsp", "cumsp_7", "cumul",
+ "cusum", "cusum_7", "cutil", "d", "datasig", "datasign",
+ "datasigna", "datasignat", "datasignatu", "datasignatur",
+ "datasignature", "datetof", "db", "dbeta", "de", "dec",
+ "deco", "decod", "decode", "deff", "des", "desc", "descr",
+ "descri", "describ", "describe", "destring", "dfbeta",
+ "dfgls", "dfuller", "di", "di_g", "dir", "dirstats", "dis",
+ "discard", "disp", "disp_res", "disp_s", "displ", "displa",
+ "display", "distinct", "do", "doe", "doed", "doedi",
+ "doedit", "dotplot", "dotplot_7", "dprobit", "drawnorm",
+ "drop", "ds", "ds_util", "dstdize", "duplicates", "durbina",
+ "dwstat", "dydx", "e", "ed", "edi", "edit", "egen",
+ "eivreg", "emdef", "en", "enc", "enco", "encod", "encode",
+ "eq", "erase", "ereg", "ereg_lf", "ereg_p", "ereg_sw",
+ "ereghet", "ereghet_glf", "ereghet_glf_sh", "ereghet_gp",
+ "ereghet_ilf", "ereghet_ilf_sh", "ereghet_ip", "eret",
+ "eretu", "eretur", "ereturn", "err", "erro", "error", "est",
+ "est_cfexist", "est_cfname", "est_clickable", "est_expand",
+ "est_hold", "est_table", "est_unhold", "est_unholdok",
+ "estat", "estat_default", "estat_summ", "estat_vce_only",
+ "esti", "estimates", "etodow", "etof", "etomdy", "ex",
+ "exi", "exit", "expand", "expandcl", "fac", "fact", "facto",
+ "factor", "factor_estat", "factor_p", "factor_pca_rotated",
+ "factor_rotate", "factormat", "fcast", "fcast_compute",
+ "fcast_graph", "fdades", "fdadesc", "fdadescr", "fdadescri",
+ "fdadescrib", "fdadescribe", "fdasav", "fdasave", "fdause",
+ "fh_st", "open", "read", "close",
+ "file", "filefilter", "fillin", "find_hlp_file", "findfile",
+ "findit", "findit_7", "fit", "fl", "fli", "flis", "flist",
+ "for5_0", "form", "forma", "format", "fpredict", "frac_154",
+ "frac_adj", "frac_chk", "frac_cox", "frac_ddp", "frac_dis",
+ "frac_dv", "frac_in", "frac_mun", "frac_pp", "frac_pq",
+ "frac_pv", "frac_wgt", "frac_xo", "fracgen", "fracplot",
+ "fracplot_7", "fracpoly", "fracpred", "fron_ex", "fron_hn",
+ "fron_p", "fron_tn", "fron_tn2", "frontier", "ftodate", "ftoe",
+ "ftomdy", "ftowdate", "g", "gamhet_glf", "gamhet_gp",
+ "gamhet_ilf", "gamhet_ip", "gamma", "gamma_d2", "gamma_p",
+ "gamma_sw", "gammahet", "gdi_hexagon", "gdi_spokes", "ge",
+ "gen", "gene", "gener", "genera", "generat", "generate",
+ "genrank", "genstd", "genvmean", "gettoken", "gl", "gladder",
+ "gladder_7", "glim_l01", "glim_l02", "glim_l03", "glim_l04",
+ "glim_l05", "glim_l06", "glim_l07", "glim_l08", "glim_l09",
+ "glim_l10", "glim_l11", "glim_l12", "glim_lf", "glim_mu",
+ "glim_nw1", "glim_nw2", "glim_nw3", "glim_p", "glim_v1",
+ "glim_v2", "glim_v3", "glim_v4", "glim_v5", "glim_v6",
+ "glim_v7", "glm", "glm_6", "glm_p", "glm_sw", "glmpred", "glo",
+ "glob", "globa", "global", "glogit", "glogit_8", "glogit_p",
+ "gmeans", "gnbre_lf", "gnbreg", "gnbreg_5", "gnbreg_p",
+ "gomp_lf", "gompe_sw", "gomper_p", "gompertz", "gompertzhet",
+ "gomphet_glf", "gomphet_glf_sh", "gomphet_gp", "gomphet_ilf",
+ "gomphet_ilf_sh", "gomphet_ip", "gphdot", "gphpen",
+ "gphprint", "gprefs", "gprobi_p", "gprobit", "gprobit_8", "gr",
+ "gr7", "gr_copy", "gr_current", "gr_db", "gr_describe",
+ "gr_dir", "gr_draw", "gr_draw_replay", "gr_drop", "gr_edit",
+ "gr_editviewopts", "gr_example", "gr_example2", "gr_export",
+ "gr_print", "gr_qscheme", "gr_query", "gr_read", "gr_rename",
+ "gr_replay", "gr_save", "gr_set", "gr_setscheme", "gr_table",
+ "gr_undo", "gr_use", "graph", "graph7", "grebar", "greigen",
+ "greigen_7", "greigen_8", "grmeanby", "grmeanby_7",
+ "gs_fileinfo", "gs_filetype", "gs_graphinfo", "gs_stat",
+ "gsort", "gwood", "h", "hadimvo", "hareg", "hausman",
+ "haver", "he", "heck_d2", "heckma_p", "heckman", "heckp_lf",
+ "heckpr_p", "heckprob", "hel", "help", "hereg", "hetpr_lf",
+ "hetpr_p", "hetprob", "hettest", "hexdump", "hilite",
+ "hist", "hist_7", "histogram", "hlogit", "hlu", "hmeans",
+ "hotel", "hotelling", "hprobit", "hreg", "hsearch", "icd9",
+ "icd9_ff", "icd9p", "iis", "impute", "imtest", "inbase",
+ "include", "inf", "infi", "infil", "infile", "infix", "inp",
+ "inpu", "input", "ins", "insheet", "insp", "inspe",
+ "inspec", "inspect", "integ", "inten", "intreg", "intreg_7",
+ "intreg_p", "intrg2_ll", "intrg_ll", "intrg_ll2", "ipolate",
+ "iqreg", "ir", "irf", "irf_create", "irfm", "iri", "is_svy",
+ "is_svysum", "isid", "istdize", "ivprob_1_lf", "ivprob_lf",
+ "ivprobit", "ivprobit_p", "ivreg", "ivreg_footnote",
+ "ivtob_1_lf", "ivtob_lf", "ivtobit", "ivtobit_p", "jackknife",
+ "jacknife", "jknife", "jknife_6", "jknife_8", "jkstat",
+ "joinby", "kalarma1", "kap", "kap_3", "kapmeier", "kappa",
+ "kapwgt", "kdensity", "kdensity_7", "keep", "ksm", "ksmirnov",
+ "ktau", "kwallis", "l", "la", "lab", "labe", "label",
+ "labelbook", "ladder", "levels", "levelsof", "leverage",
+ "lfit", "lfit_p", "li", "lincom", "line", "linktest",
+ "lis", "list", "lloghet_glf", "lloghet_glf_sh", "lloghet_gp",
+ "lloghet_ilf", "lloghet_ilf_sh", "lloghet_ip", "llogi_sw",
+ "llogis_p", "llogist", "llogistic", "llogistichet",
+ "lnorm_lf", "lnorm_sw", "lnorma_p", "lnormal", "lnormalhet",
+ "lnormhet_glf", "lnormhet_glf_sh", "lnormhet_gp",
+ "lnormhet_ilf", "lnormhet_ilf_sh", "lnormhet_ip", "lnskew0",
+ "loadingplot", "loc", "loca", "local", "log", "logi",
+ "logis_lf", "logistic", "logistic_p", "logit", "logit_estat",
+ "logit_p", "loglogs", "logrank", "loneway", "lookfor",
+ "lookup", "lowess", "lowess_7", "lpredict", "lrecomp", "lroc",
+ "lroc_7", "lrtest", "ls", "lsens", "lsens_7", "lsens_x",
+ "lstat", "ltable", "ltable_7", "ltriang", "lv", "lvr2plot",
+ "lvr2plot_7", "m", "ma", "mac", "macr", "macro", "makecns",
+ "man", "manova", "manova_estat", "manova_p", "manovatest",
+ "mantel", "mark", "markin", "markout", "marksample", "mat",
+ "mat_capp", "mat_order", "mat_put_rr", "mat_rapp", "mata",
+ "mata_clear", "mata_describe", "mata_drop", "mata_matdescribe",
+ "mata_matsave", "mata_matuse", "mata_memory", "mata_mlib",
+ "mata_mosave", "mata_rename", "mata_which", "matalabel",
+ "matcproc", "matlist", "matname", "matr", "matri",
+ "matrix", "matrix_input__dlg", "matstrik", "mcc", "mcci",
+ "md0_", "md1_", "md1debug_", "md2_", "md2debug_", "mds",
+ "mds_estat", "mds_p", "mdsconfig", "mdslong", "mdsmat",
+ "mdsshepard", "mdytoe", "mdytof", "me_derd", "mean",
+ "means", "median", "memory", "memsize", "meqparse", "mer",
+ "merg", "merge", "mfp", "mfx", "mhelp", "mhodds", "minbound",
+ "mixed_ll", "mixed_ll_reparm", "mkassert", "mkdir",
+ "mkmat", "mkspline", "ml", "ml_5", "ml_adjs", "ml_bhhhs",
+ "ml_c_d", "ml_check", "ml_clear", "ml_cnt", "ml_debug",
+ "ml_defd", "ml_e0", "ml_e0_bfgs", "ml_e0_cycle", "ml_e0_dfp",
+ "ml_e0i", "ml_e1", "ml_e1_bfgs", "ml_e1_bhhh", "ml_e1_cycle",
+ "ml_e1_dfp", "ml_e2", "ml_e2_cycle", "ml_ebfg0", "ml_ebfr0",
+ "ml_ebfr1", "ml_ebh0q", "ml_ebhh0", "ml_ebhr0", "ml_ebr0i",
+ "ml_ecr0i", "ml_edfp0", "ml_edfr0", "ml_edfr1", "ml_edr0i",
+ "ml_eds", "ml_eer0i", "ml_egr0i", "ml_elf", "ml_elf_bfgs",
+ "ml_elf_bhhh", "ml_elf_cycle", "ml_elf_dfp", "ml_elfi",
+ "ml_elfs", "ml_enr0i", "ml_enrr0", "ml_erdu0", "ml_erdu0_bfgs",
+ "ml_erdu0_bhhh", "ml_erdu0_bhhhq", "ml_erdu0_cycle",
+ "ml_erdu0_dfp", "ml_erdu0_nrbfgs", "ml_exde", "ml_footnote",
+ "ml_geqnr", "ml_grad0", "ml_graph", "ml_hbhhh", "ml_hd0",
+ "ml_hold", "ml_init", "ml_inv", "ml_log", "ml_max",
+ "ml_mlout", "ml_mlout_8", "ml_model", "ml_nb0", "ml_opt",
+ "ml_p", "ml_plot", "ml_query", "ml_rdgrd", "ml_repor",
+ "ml_s_e", "ml_score", "ml_searc", "ml_technique", "ml_unhold",
+ "mleval", "mlf_", "mlmatbysum", "mlmatsum", "mlog", "mlogi",
+ "mlogit", "mlogit_footnote", "mlogit_p", "mlopts", "mlsum",
+ "mlvecsum", "mnl0_", "mor", "more", "mov", "move", "mprobit",
+ "mprobit_lf", "mprobit_p", "mrdu0_", "mrdu1_", "mvdecode",
+ "mvencode", "mvreg", "mvreg_estat", "n", "nbreg",
+ "nbreg_al", "nbreg_lf", "nbreg_p", "nbreg_sw", "nestreg", "net",
+ "newey", "newey_7", "newey_p", "news", "nl", "nl_7", "nl_9",
+ "nl_9_p", "nl_p", "nl_p_7", "nlcom", "nlcom_p", "nlexp2",
+ "nlexp2_7", "nlexp2a", "nlexp2a_7", "nlexp3", "nlexp3_7",
+ "nlgom3", "nlgom3_7", "nlgom4", "nlgom4_7", "nlinit", "nllog3",
+ "nllog3_7", "nllog4", "nllog4_7", "nlog_rd", "nlogit",
+ "nlogit_p", "nlogitgen", "nlogittree", "nlpred", "no",
+ "nobreak", "noi", "nois", "noisi", "noisil", "noisily", "note",
+ "notes", "notes_dlg", "nptrend", "numlabel", "numlist", "odbc",
+ "old_ver", "olo", "olog", "ologi", "ologi_sw", "ologit",
+ "ologit_p", "ologitp", "on", "one", "onew", "onewa", "oneway",
+ "op_colnm", "op_comp", "op_diff", "op_inv", "op_str", "opr",
+ "opro", "oprob", "oprob_sw", "oprobi", "oprobi_p", "oprobit",
+ "oprobitp", "opts_exclusive", "order", "orthog", "orthpoly",
+ "ou", "out", "outf", "outfi", "outfil", "outfile", "outs",
+ "outsh", "outshe", "outshee", "outsheet", "ovtest", "pac",
+ "pac_7", "palette", "parse", "parse_dissim", "pause", "pca",
+ "pca_8", "pca_display", "pca_estat", "pca_p", "pca_rotate",
+ "pcamat", "pchart", "pchart_7", "pchi", "pchi_7", "pcorr",
+ "pctile", "pentium", "pergram", "pergram_7", "permute",
+ "permute_8", "personal", "peto_st", "pkcollapse", "pkcross",
+ "pkequiv", "pkexamine", "pkexamine_7", "pkshape", "pksumm",
+ "pksumm_7", "pl", "plo", "plot", "plugin", "pnorm",
+ "pnorm_7", "poisgof", "poiss_lf", "poiss_sw", "poisso_p",
+ "poisson", "poisson_estat", "post", "postclose", "postfile",
+ "postutil", "pperron", "pr", "prais", "prais_e", "prais_e2",
+ "prais_p", "predict", "predictnl", "preserve", "print",
+ "pro", "prob", "probi", "probit", "probit_estat", "probit_p",
+ "proc_time", "procoverlay", "procrustes", "procrustes_estat",
+ "procrustes_p", "profiler", "prog", "progr", "progra",
+ "program", "prop", "proportion", "prtest", "prtesti", "pwcorr",
+ "pwd", "q", "s", "qby", "qbys", "qchi", "qchi_7", "qladder",
+ "qladder_7", "qnorm", "qnorm_7", "qqplot", "qqplot_7", "qreg",
+ "qreg_c", "qreg_p", "qreg_sw", "qu", "quadchk", "quantile",
+ "quantile_7", "que", "quer", "query", "range", "ranksum",
+ "ratio", "rchart", "rchart_7", "rcof", "recast", "reclink",
+ "recode", "reg", "reg3", "reg3_p", "regdw", "regr", "regre",
+ "regre_p2", "regres", "regres_p", "regress", "regress_estat",
+ "regriv_p", "remap", "ren", "rena", "renam", "rename",
+ "renpfix", "repeat", "replace", "report", "reshape",
+ "restore", "ret", "retu", "retur", "return", "rm", "rmdir",
+ "robvar", "roccomp", "roccomp_7", "roccomp_8", "rocf_lf",
+ "rocfit", "rocfit_8", "rocgold", "rocplot", "rocplot_7",
+ "roctab", "roctab_7", "rolling", "rologit", "rologit_p",
+ "rot", "rota", "rotat", "rotate", "rotatemat", "rreg",
+ "rreg_p", "ru", "run", "runtest", "rvfplot", "rvfplot_7",
+ "rvpplot", "rvpplot_7", "sa", "safesum", "sample",
+ "sampsi", "sav", "save", "savedresults", "saveold", "sc",
+ "sca", "scal", "scala", "scalar", "scatter", "scm_mine",
+ "sco", "scob_lf", "scob_p", "scobi_sw", "scobit", "scor",
+ "score", "scoreplot", "scoreplot_help", "scree", "screeplot",
+ "screeplot_help", "sdtest", "sdtesti", "se", "search",
+ "separate", "seperate", "serrbar", "serrbar_7", "serset", "set",
+ "set_defaults", "sfrancia", "sh", "she", "shel", "shell",
+ "shewhart", "shewhart_7", "signestimationsample", "signrank",
+ "signtest", "simul", "simul_7", "simulate", "simulate_8",
+ "sktest", "sleep", "slogit", "slogit_d2", "slogit_p", "smooth",
+ "snapspan", "so", "sor", "sort", "spearman", "spikeplot",
+ "spikeplot_7", "spikeplt", "spline_x", "split", "sqreg",
+ "sqreg_p", "sret", "sretu", "sretur", "sreturn", "ssc", "st",
+ "st_ct", "st_hc", "st_hcd", "st_hcd_sh", "st_is", "st_issys",
+ "st_note", "st_promo", "st_set", "st_show", "st_smpl",
+ "st_subid", "stack", "statsby", "statsby_8", "stbase", "stci",
+ "stci_7", "stcox", "stcox_estat", "stcox_fr", "stcox_fr_ll",
+ "stcox_p", "stcox_sw", "stcoxkm", "stcoxkm_7", "stcstat",
+ "stcurv", "stcurve", "stcurve_7", "stdes", "stem", "stepwise",
+ "stereg", "stfill", "stgen", "stir", "stjoin", "stmc", "stmh",
+ "stphplot", "stphplot_7", "stphtest", "stphtest_7",
+ "stptime", "strate", "strate_7", "streg", "streg_sw", "streset",
+ "sts", "sts_7", "stset", "stsplit", "stsum", "sttocc",
+ "sttoct", "stvary", "stweib", "su", "suest", "suest_8",
+ "sum", "summ", "summa", "summar", "summari", "summariz",
+ "summarize", "sunflower", "sureg", "survcurv", "survsum",
+ "svar", "svar_p", "svmat", "svy", "svy_disp", "svy_dreg",
+ "svy_est", "svy_est_7", "svy_estat", "svy_get", "svy_gnbreg_p",
+ "svy_head", "svy_header", "svy_heckman_p", "svy_heckprob_p",
+ "svy_intreg_p", "svy_ivreg_p", "svy_logistic_p", "svy_logit_p",
+ "svy_mlogit_p", "svy_nbreg_p", "svy_ologit_p", "svy_oprobit_p",
+ "svy_poisson_p", "svy_probit_p", "svy_regress_p", "svy_sub",
+ "svy_sub_7", "svy_x", "svy_x_7", "svy_x_p", "svydes",
+ "svydes_8", "svygen", "svygnbreg", "svyheckman", "svyheckprob",
+ "svyintreg", "svyintreg_7", "svyintrg", "svyivreg", "svylc",
+ "svylog_p", "svylogit", "svymarkout", "svymarkout_8",
+ "svymean", "svymlog", "svymlogit", "svynbreg", "svyolog",
+ "svyologit", "svyoprob", "svyoprobit", "svyopts",
+ "svypois", "svypois_7", "svypoisson", "svyprobit", "svyprobt",
+ "svyprop", "svyprop_7", "svyratio", "svyreg", "svyreg_p",
+ "svyregress", "svyset", "svyset_7", "svyset_8", "svytab",
+ "svytab_7", "svytest", "svytotal", "sw", "sw_8", "swcnreg",
+ "swcox", "swereg", "swilk", "swlogis", "swlogit",
+ "swologit", "swoprbt", "swpois", "swprobit", "swqreg",
+ "swtobit", "swweib", "symmetry", "symmi", "symplot",
+ "symplot_7", "syntax", "sysdescribe", "sysdir", "sysuse",
+ "szroeter", "ta", "tab", "tab1", "tab2", "tab_or", "tabd",
+ "tabdi", "tabdis", "tabdisp", "tabi", "table", "tabodds",
+ "tabodds_7", "tabstat", "tabu", "tabul", "tabula", "tabulat",
+ "tabulate", "te", "tempfile", "tempname", "tempvar", "tes",
+ "test", "testnl", "testparm", "teststd", "tetrachoric",
+ "time_it", "timer", "tis", "tob", "tobi", "tobit", "tobit_p",
+ "tobit_sw", "token", "tokeni", "tokeniz", "tokenize",
+ "tostring", "total", "translate", "translator", "transmap",
+ "treat_ll", "treatr_p", "treatreg", "trim", "trnb_cons",
+ "trnb_mean", "trpoiss_d2", "trunc_ll", "truncr_p", "truncreg",
+ "tsappend", "tset", "tsfill", "tsline", "tsline_ex",
+ "tsreport", "tsrevar", "tsrline", "tsset", "tssmooth",
+ "tsunab", "ttest", "ttesti", "tut_chk", "tut_wait", "tutorial",
+ "tw", "tware_st", "two", "twoway", "twoway__fpfit_serset",
+ "twoway__function_gen", "twoway__histogram_gen",
+ "twoway__ipoint_serset", "twoway__ipoints_serset",
+ "twoway__kdensity_gen", "twoway__lfit_serset",
+ "twoway__normgen_gen", "twoway__pci_serset",
+ "twoway__qfit_serset", "twoway__scatteri_serset",
+ "twoway__sunflower_gen", "twoway_ksm_serset", "ty", "typ",
+ "type", "typeof", "u", "unab", "unabbrev", "unabcmd",
+ "update", "us", "use", "uselabel", "var", "var_mkcompanion",
+ "var_p", "varbasic", "varfcast", "vargranger", "varirf",
+ "varirf_add", "varirf_cgraph", "varirf_create", "varirf_ctable",
+ "varirf_describe", "varirf_dir", "varirf_drop", "varirf_erase",
+ "varirf_graph", "varirf_ograph", "varirf_rename", "varirf_set",
+ "varirf_table", "varlist", "varlmar", "varnorm", "varsoc",
+ "varstable", "varstable_w", "varstable_w2", "varwle",
+ "vce", "vec", "vec_fevd", "vec_mkphi", "vec_p", "vec_p_w",
+ "vecirf_create", "veclmar", "veclmar_w", "vecnorm",
+ "vecnorm_w", "vecrank", "vecstable", "verinst", "vers",
+ "versi", "versio", "version", "view", "viewsource", "vif",
+ "vwls", "wdatetof", "webdescribe", "webseek", "webuse",
+ "weib1_lf", "weib2_lf", "weib_lf", "weib_lf0", "weibhet_glf",
+ "weibhet_glf_sh", "weibhet_glfa", "weibhet_glfa_sh",
+ "weibhet_gp", "weibhet_ilf", "weibhet_ilf_sh", "weibhet_ilfa",
+ "weibhet_ilfa_sh", "weibhet_ip", "weibu_sw", "weibul_p",
+ "weibull", "weibull_c", "weibull_s", "weibullhet",
+ "wh", "whelp", "whi", "which", "whil", "while", "wilc_st",
+ "wilcoxon", "win", "wind", "windo", "window", "winexec",
+ "wntestb", "wntestb_7", "wntestq", "xchart", "xchart_7",
+ "xcorr", "xcorr_7", "xi", "xi_6", "xmlsav", "xmlsave",
+ "xmluse", "xpose", "xsh", "xshe", "xshel", "xshell",
+ "xt_iis", "xt_tis", "xtab_p", "xtabond", "xtbin_p",
+ "xtclog", "xtcloglog", "xtcloglog_8", "xtcloglog_d2",
+ "xtcloglog_pa_p", "xtcloglog_re_p", "xtcnt_p", "xtcorr",
+ "xtdata", "xtdes", "xtfront_p", "xtfrontier", "xtgee",
+ "xtgee_elink", "xtgee_estat", "xtgee_makeivar", "xtgee_p",
+ "xtgee_plink", "xtgls", "xtgls_p", "xthaus", "xthausman",
+ "xtht_p", "xthtaylor", "xtile", "xtint_p", "xtintreg",
+ "xtintreg_8", "xtintreg_d2", "xtintreg_p", "xtivp_1",
+ "xtivp_2", "xtivreg", "xtline", "xtline_ex", "xtlogit",
+ "xtlogit_8", "xtlogit_d2", "xtlogit_fe_p", "xtlogit_pa_p",
+ "xtlogit_re_p", "xtmixed", "xtmixed_estat", "xtmixed_p",
+ "xtnb_fe", "xtnb_lf", "xtnbreg", "xtnbreg_pa_p",
+ "xtnbreg_refe_p", "xtpcse", "xtpcse_p", "xtpois", "xtpoisson",
+ "xtpoisson_d2", "xtpoisson_pa_p", "xtpoisson_refe_p", "xtpred",
+ "xtprobit", "xtprobit_8", "xtprobit_d2", "xtprobit_re_p",
+ "xtps_fe", "xtps_lf", "xtps_ren", "xtps_ren_8", "xtrar_p",
+ "xtrc", "xtrc_p", "xtrchh", "xtrefe_p", "xtreg", "xtreg_be",
+ "xtreg_fe", "xtreg_ml", "xtreg_pa_p", "xtreg_re",
+ "xtregar", "xtrere_p", "xtset", "xtsf_ll", "xtsf_llti",
+ "xtsum", "xttab", "xttest0", "xttobit", "xttobit_8",
+ "xttobit_p", "xttrans", "yx", "yxview__barlike_draw",
+ "yxview_area_draw", "yxview_bar_draw", "yxview_dot_draw",
+ "yxview_dropline_draw", "yxview_function_draw",
+ "yxview_iarrow_draw", "yxview_ilabels_draw",
+ "yxview_normal_draw", "yxview_pcarrow_draw",
+ "yxview_pcbarrow_draw", "yxview_pccapsym_draw",
+ "yxview_pcscatter_draw", "yxview_pcspike_draw",
+ "yxview_rarea_draw", "yxview_rbar_draw", "yxview_rbarm_draw",
+ "yxview_rcap_draw", "yxview_rcapsym_draw",
+ "yxview_rconnected_draw", "yxview_rline_draw",
+ "yxview_rscatter_draw", "yxview_rspike_draw",
+ "yxview_spike_draw", "yxview_sunflower_draw", "zap_s", "zinb",
+ "zinb_llf", "zinb_plf", "zip", "zip_llf", "zip_p", "zip_plf",
+ "zt_ct_5", "zt_hc_5", "zt_hcd_5", "zt_is_5", "zt_iss_5",
+ "zt_sho_5", "zt_smp_5", "ztbase_5", "ztcox_5", "ztdes_5",
+ "ztereg_5", "ztfill_5", "ztgen_5", "ztir_5", "ztjoin_5", "ztnb",
+ "ztnb_p", "ztp", "ztp_p", "zts_5", "ztset_5", "ztspli_5",
+ "ztsum_5", "zttoct_5", "ztvary_5", "ztweib_5"
+)
+
+builtins_functions = (
+ "Cdhms", "Chms", "Clock", "Cmdyhms", "Cofc", "Cofd", "F",
+ "Fden", "Ftail", "I", "J", "_caller", "abbrev", "abs", "acos",
+ "acosh", "asin", "asinh", "atan", "atan2", "atanh",
+ "autocode", "betaden", "binomial", "binomialp", "binomialtail",
+ "binormal", "bofd", "byteorder", "c", "ceil", "char",
+ "chi2", "chi2den", "chi2tail", "cholesky", "chop", "clip",
+ "clock", "cloglog", "cofC", "cofd", "colnumb", "colsof", "comb",
+ "cond", "corr", "cos", "cosh", "d", "daily", "date", "day",
+ "det", "dgammapda", "dgammapdada", "dgammapdadx", "dgammapdx",
+ "dgammapdxdx", "dhms", "diag", "diag0cnt", "digamma",
+ "dofC", "dofb", "dofc", "dofh", "dofm", "dofq", "dofw",
+ "dofy", "dow", "doy", "dunnettprob", "e", "el", "epsdouble",
+ "epsfloat", "exp", "fileexists", "fileread", "filereaderror",
+ "filewrite", "float", "floor", "fmtwidth", "gammaden",
+ "gammap", "gammaptail", "get", "group", "h", "hadamard",
+ "halfyear", "halfyearly", "has_eprop", "hh", "hhC", "hms",
+ "hofd", "hours", "hypergeometric", "hypergeometricp", "ibeta",
+ "ibetatail", "index", "indexnot", "inlist", "inrange", "int",
+ "inv", "invF", "invFtail", "invbinomial", "invbinomialtail",
+ "invchi2", "invchi2tail", "invcloglog", "invdunnettprob",
+ "invgammap", "invgammaptail", "invibeta", "invibetatail",
+ "invlogit", "invnFtail", "invnbinomial", "invnbinomialtail",
+ "invnchi2", "invnchi2tail", "invnibeta", "invnorm", "invnormal",
+ "invnttail", "invpoisson", "invpoissontail", "invsym", "invt",
+ "invttail", "invtukeyprob", "irecode", "issym", "issymmetric",
+ "itrim", "length", "ln", "lnfact", "lnfactorial", "lngamma",
+ "lnnormal", "lnnormalden", "log", "log10", "logit", "lower",
+ "ltrim", "m", "match", "matmissing", "matrix", "matuniform",
+ "max", "maxbyte", "maxdouble", "maxfloat", "maxint", "maxlong",
+ "mdy", "mdyhms", "mi", "min", "minbyte", "mindouble",
+ "minfloat", "minint", "minlong", "minutes", "missing", "mm",
+ "mmC", "mod", "mofd", "month", "monthly", "mreldif",
+ "msofhours", "msofminutes", "msofseconds", "nF", "nFden",
+ "nFtail", "nbetaden", "nbinomial", "nbinomialp", "nbinomialtail",
+ "nchi2", "nchi2den", "nchi2tail", "nibeta", "norm", "normal",
+ "normalden", "normd", "npnF", "npnchi2", "npnt", "nt", "ntden",
+ "nttail", "nullmat", "plural", "poisson", "poissonp",
+ "poissontail", "proper", "q", "qofd", "quarter", "quarterly",
+ "r", "rbeta", "rbinomial", "rchi2", "real", "recode", "regexm",
+ "regexr", "regexs", "reldif", "replay", "return", "reverse",
+ "rgamma", "rhypergeometric", "rnbinomial", "rnormal", "round",
+ "rownumb", "rowsof", "rpoisson", "rt", "rtrim", "runiform", "s",
+ "scalar", "seconds", "sign", "sin", "sinh", "smallestdouble",
+ "soundex", "soundex_nara", "sqrt", "ss", "ssC", "strcat",
+ "strdup", "string", "strlen", "strlower", "strltrim", "strmatch",
+ "strofreal", "strpos", "strproper", "strreverse", "strrtrim",
+ "strtoname", "strtrim", "strupper", "subinstr", "subinword",
+ "substr", "sum", "sweep", "syminv", "t", "tC", "tan", "tanh",
+ "tc", "td", "tden", "th", "tin", "tm", "tq", "trace",
+ "trigamma", "trim", "trunc", "ttail", "tukeyprob", "tw",
+ "twithin", "uniform", "upper", "vec", "vecdiag", "w", "week",
+ "weekly", "wofd", "word", "wordcount", "year", "yearly",
+ "yh", "ym", "yofd", "yq", "yw"
+)
+
+
diff --git a/wandb/vendor/pygments/lexers/_tsql_builtins.py b/wandb/vendor/pygments/lexers/_tsql_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..e29ed34bacc45ccbf41363abe5bbb248bc6bf434
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_tsql_builtins.py
@@ -0,0 +1,1004 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._tsql_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ These are manually translated lists from https://msdn.microsoft.com.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+# See https://msdn.microsoft.com/en-us/library/ms174986.aspx.
+OPERATORS = (
+ '!<',
+ '!=',
+ '!>',
+ '<',
+ '<=',
+ '<>',
+ '=',
+ '>',
+ '>=',
+ '+',
+ '+=',
+ '-',
+ '-=',
+ '*',
+ '*=',
+ '/',
+ '/=',
+ '%',
+ '%=',
+ '&',
+ '&=',
+ '|',
+ '|=',
+ '^',
+ '^=',
+ '~',
+ '::',
+)
+
+OPERATOR_WORDS = (
+ 'all',
+ 'and',
+ 'any',
+ 'between',
+ 'except',
+ 'exists',
+ 'in',
+ 'intersect',
+ 'like',
+ 'not',
+ 'or',
+ 'some',
+ 'union',
+)
+
+_KEYWORDS_SERVER = (
+ 'add',
+ 'all',
+ 'alter',
+ 'and',
+ 'any',
+ 'as',
+ 'asc',
+ 'authorization',
+ 'backup',
+ 'begin',
+ 'between',
+ 'break',
+ 'browse',
+ 'bulk',
+ 'by',
+ 'cascade',
+ 'case',
+ 'catch',
+ 'check',
+ 'checkpoint',
+ 'close',
+ 'clustered',
+ 'coalesce',
+ 'collate',
+ 'column',
+ 'commit',
+ 'compute',
+ 'constraint',
+ 'contains',
+ 'containstable',
+ 'continue',
+ 'convert',
+ 'create',
+ 'cross',
+ 'current',
+ 'current_date',
+ 'current_time',
+ 'current_timestamp',
+ 'current_user',
+ 'cursor',
+ 'database',
+ 'dbcc',
+ 'deallocate',
+ 'declare',
+ 'default',
+ 'delete',
+ 'deny',
+ 'desc',
+ 'disk',
+ 'distinct',
+ 'distributed',
+ 'double',
+ 'drop',
+ 'dump',
+ 'else',
+ 'end',
+ 'errlvl',
+ 'escape',
+ 'except',
+ 'exec',
+ 'execute',
+ 'exists',
+ 'exit',
+ 'external',
+ 'fetch',
+ 'file',
+ 'fillfactor',
+ 'for',
+ 'foreign',
+ 'freetext',
+ 'freetexttable',
+ 'from',
+ 'full',
+ 'function',
+ 'goto',
+ 'grant',
+ 'group',
+ 'having',
+ 'holdlock',
+ 'identity',
+ 'identity_insert',
+ 'identitycol',
+ 'if',
+ 'in',
+ 'index',
+ 'inner',
+ 'insert',
+ 'intersect',
+ 'into',
+ 'is',
+ 'join',
+ 'key',
+ 'kill',
+ 'left',
+ 'like',
+ 'lineno',
+ 'load',
+ 'merge',
+ 'national',
+ 'nocheck',
+ 'nonclustered',
+ 'not',
+ 'null',
+ 'nullif',
+ 'of',
+ 'off',
+ 'offsets',
+ 'on',
+ 'open',
+ 'opendatasource',
+ 'openquery',
+ 'openrowset',
+ 'openxml',
+ 'option',
+ 'or',
+ 'order',
+ 'outer',
+ 'over',
+ 'percent',
+ 'pivot',
+ 'plan',
+ 'precision',
+ 'primary',
+ 'print',
+ 'proc',
+ 'procedure',
+ 'public',
+ 'raiserror',
+ 'read',
+ 'readtext',
+ 'reconfigure',
+ 'references',
+ 'replication',
+ 'restore',
+ 'restrict',
+ 'return',
+ 'revert',
+ 'revoke',
+ 'right',
+ 'rollback',
+ 'rowcount',
+ 'rowguidcol',
+ 'rule',
+ 'save',
+ 'schema',
+ 'securityaudit',
+ 'select',
+ 'semantickeyphrasetable',
+ 'semanticsimilaritydetailstable',
+ 'semanticsimilaritytable',
+ 'session_user',
+ 'set',
+ 'setuser',
+ 'shutdown',
+ 'some',
+ 'statistics',
+ 'system_user',
+ 'table',
+ 'tablesample',
+ 'textsize',
+ 'then',
+ 'throw',
+ 'to',
+ 'top',
+ 'tran',
+ 'transaction',
+ 'trigger',
+ 'truncate',
+ 'try',
+ 'try_convert',
+ 'tsequal',
+ 'union',
+ 'unique',
+ 'unpivot',
+ 'update',
+ 'updatetext',
+ 'use',
+ 'user',
+ 'values',
+ 'varying',
+ 'view',
+ 'waitfor',
+ 'when',
+ 'where',
+ 'while',
+ 'with',
+ 'within',
+ 'writetext',
+)
+
+_KEYWORDS_FUTURE = (
+ 'absolute',
+ 'action',
+ 'admin',
+ 'after',
+ 'aggregate',
+ 'alias',
+ 'allocate',
+ 'are',
+ 'array',
+ 'asensitive',
+ 'assertion',
+ 'asymmetric',
+ 'at',
+ 'atomic',
+ 'before',
+ 'binary',
+ 'bit',
+ 'blob',
+ 'boolean',
+ 'both',
+ 'breadth',
+ 'call',
+ 'called',
+ 'cardinality',
+ 'cascaded',
+ 'cast',
+ 'catalog',
+ 'char',
+ 'character',
+ 'class',
+ 'clob',
+ 'collation',
+ 'collect',
+ 'completion',
+ 'condition',
+ 'connect',
+ 'connection',
+ 'constraints',
+ 'constructor',
+ 'corr',
+ 'corresponding',
+ 'covar_pop',
+ 'covar_samp',
+ 'cube',
+ 'cume_dist',
+ 'current_catalog',
+ 'current_default_transform_group',
+ 'current_path',
+ 'current_role',
+ 'current_schema',
+ 'current_transform_group_for_type',
+ 'cycle',
+ 'data',
+ 'date',
+ 'day',
+ 'dec',
+ 'decimal',
+ 'deferrable',
+ 'deferred',
+ 'depth',
+ 'deref',
+ 'describe',
+ 'descriptor',
+ 'destroy',
+ 'destructor',
+ 'deterministic',
+ 'diagnostics',
+ 'dictionary',
+ 'disconnect',
+ 'domain',
+ 'dynamic',
+ 'each',
+ 'element',
+ 'end-exec',
+ 'equals',
+ 'every',
+ 'exception',
+ 'false',
+ 'filter',
+ 'first',
+ 'float',
+ 'found',
+ 'free',
+ 'fulltexttable',
+ 'fusion',
+ 'general',
+ 'get',
+ 'global',
+ 'go',
+ 'grouping',
+ 'hold',
+ 'host',
+ 'hour',
+ 'ignore',
+ 'immediate',
+ 'indicator',
+ 'initialize',
+ 'initially',
+ 'inout',
+ 'input',
+ 'int',
+ 'integer',
+ 'intersection',
+ 'interval',
+ 'isolation',
+ 'iterate',
+ 'language',
+ 'large',
+ 'last',
+ 'lateral',
+ 'leading',
+ 'less',
+ 'level',
+ 'like_regex',
+ 'limit',
+ 'ln',
+ 'local',
+ 'localtime',
+ 'localtimestamp',
+ 'locator',
+ 'map',
+ 'match',
+ 'member',
+ 'method',
+ 'minute',
+ 'mod',
+ 'modifies',
+ 'modify',
+ 'module',
+ 'month',
+ 'multiset',
+ 'names',
+ 'natural',
+ 'nchar',
+ 'nclob',
+ 'new',
+ 'next',
+ 'no',
+ 'none',
+ 'normalize',
+ 'numeric',
+ 'object',
+ 'occurrences_regex',
+ 'old',
+ 'only',
+ 'operation',
+ 'ordinality',
+ 'out',
+ 'output',
+ 'overlay',
+ 'pad',
+ 'parameter',
+ 'parameters',
+ 'partial',
+ 'partition',
+ 'path',
+ 'percent_rank',
+ 'percentile_cont',
+ 'percentile_disc',
+ 'position_regex',
+ 'postfix',
+ 'prefix',
+ 'preorder',
+ 'prepare',
+ 'preserve',
+ 'prior',
+ 'privileges',
+ 'range',
+ 'reads',
+ 'real',
+ 'recursive',
+ 'ref',
+ 'referencing',
+ 'regr_avgx',
+ 'regr_avgy',
+ 'regr_count',
+ 'regr_intercept',
+ 'regr_r2',
+ 'regr_slope',
+ 'regr_sxx',
+ 'regr_sxy',
+ 'regr_syy',
+ 'relative',
+ 'release',
+ 'result',
+ 'returns',
+ 'role',
+ 'rollup',
+ 'routine',
+ 'row',
+ 'rows',
+ 'savepoint',
+ 'scope',
+ 'scroll',
+ 'search',
+ 'second',
+ 'section',
+ 'sensitive',
+ 'sequence',
+ 'session',
+ 'sets',
+ 'similar',
+ 'size',
+ 'smallint',
+ 'space',
+ 'specific',
+ 'specifictype',
+ 'sql',
+ 'sqlexception',
+ 'sqlstate',
+ 'sqlwarning',
+ 'start',
+ 'state',
+ 'statement',
+ 'static',
+ 'stddev_pop',
+ 'stddev_samp',
+ 'structure',
+ 'submultiset',
+ 'substring_regex',
+ 'symmetric',
+ 'system',
+ 'temporary',
+ 'terminate',
+ 'than',
+ 'time',
+ 'timestamp',
+ 'timezone_hour',
+ 'timezone_minute',
+ 'trailing',
+ 'translate_regex',
+ 'translation',
+ 'treat',
+ 'true',
+ 'uescape',
+ 'under',
+ 'unknown',
+ 'unnest',
+ 'usage',
+ 'using',
+ 'value',
+ 'var_pop',
+ 'var_samp',
+ 'varchar',
+ 'variable',
+ 'whenever',
+ 'width_bucket',
+ 'window',
+ 'within',
+ 'without',
+ 'work',
+ 'write',
+ 'xmlagg',
+ 'xmlattributes',
+ 'xmlbinary',
+ 'xmlcast',
+ 'xmlcomment',
+ 'xmlconcat',
+ 'xmldocument',
+ 'xmlelement',
+ 'xmlexists',
+ 'xmlforest',
+ 'xmliterate',
+ 'xmlnamespaces',
+ 'xmlparse',
+ 'xmlpi',
+ 'xmlquery',
+ 'xmlserialize',
+ 'xmltable',
+ 'xmltext',
+ 'xmlvalidate',
+ 'year',
+ 'zone',
+)
+
+_KEYWORDS_ODBC = (
+ 'absolute',
+ 'action',
+ 'ada',
+ 'add',
+ 'all',
+ 'allocate',
+ 'alter',
+ 'and',
+ 'any',
+ 'are',
+ 'as',
+ 'asc',
+ 'assertion',
+ 'at',
+ 'authorization',
+ 'avg',
+ 'begin',
+ 'between',
+ 'bit',
+ 'bit_length',
+ 'both',
+ 'by',
+ 'cascade',
+ 'cascaded',
+ 'case',
+ 'cast',
+ 'catalog',
+ 'char',
+ 'char_length',
+ 'character',
+ 'character_length',
+ 'check',
+ 'close',
+ 'coalesce',
+ 'collate',
+ 'collation',
+ 'column',
+ 'commit',
+ 'connect',
+ 'connection',
+ 'constraint',
+ 'constraints',
+ 'continue',
+ 'convert',
+ 'corresponding',
+ 'count',
+ 'create',
+ 'cross',
+ 'current',
+ 'current_date',
+ 'current_time',
+ 'current_timestamp',
+ 'current_user',
+ 'cursor',
+ 'date',
+ 'day',
+ 'deallocate',
+ 'dec',
+ 'decimal',
+ 'declare',
+ 'default',
+ 'deferrable',
+ 'deferred',
+ 'delete',
+ 'desc',
+ 'describe',
+ 'descriptor',
+ 'diagnostics',
+ 'disconnect',
+ 'distinct',
+ 'domain',
+ 'double',
+ 'drop',
+ 'else',
+ 'end',
+ 'end-exec',
+ 'escape',
+ 'except',
+ 'exception',
+ 'exec',
+ 'execute',
+ 'exists',
+ 'external',
+ 'extract',
+ 'false',
+ 'fetch',
+ 'first',
+ 'float',
+ 'for',
+ 'foreign',
+ 'fortran',
+ 'found',
+ 'from',
+ 'full',
+ 'get',
+ 'global',
+ 'go',
+ 'goto',
+ 'grant',
+ 'group',
+ 'having',
+ 'hour',
+ 'identity',
+ 'immediate',
+ 'in',
+ 'include',
+ 'index',
+ 'indicator',
+ 'initially',
+ 'inner',
+ 'input',
+ 'insensitive',
+ 'insert',
+ 'int',
+ 'integer',
+ 'intersect',
+ 'interval',
+ 'into',
+ 'is',
+ 'isolation',
+ 'join',
+ 'key',
+ 'language',
+ 'last',
+ 'leading',
+ 'left',
+ 'level',
+ 'like',
+ 'local',
+ 'lower',
+ 'match',
+ 'max',
+ 'min',
+ 'minute',
+ 'module',
+ 'month',
+ 'names',
+ 'national',
+ 'natural',
+ 'nchar',
+ 'next',
+ 'no',
+ 'none',
+ 'not',
+ 'null',
+ 'nullif',
+ 'numeric',
+ 'octet_length',
+ 'of',
+ 'on',
+ 'only',
+ 'open',
+ 'option',
+ 'or',
+ 'order',
+ 'outer',
+ 'output',
+ 'overlaps',
+ 'pad',
+ 'partial',
+ 'pascal',
+ 'position',
+ 'precision',
+ 'prepare',
+ 'preserve',
+ 'primary',
+ 'prior',
+ 'privileges',
+ 'procedure',
+ 'public',
+ 'read',
+ 'real',
+ 'references',
+ 'relative',
+ 'restrict',
+ 'revoke',
+ 'right',
+ 'rollback',
+ 'rows',
+ 'schema',
+ 'scroll',
+ 'second',
+ 'section',
+ 'select',
+ 'session',
+ 'session_user',
+ 'set',
+ 'size',
+ 'smallint',
+ 'some',
+ 'space',
+ 'sql',
+ 'sqlca',
+ 'sqlcode',
+ 'sqlerror',
+ 'sqlstate',
+ 'sqlwarning',
+ 'substring',
+ 'sum',
+ 'system_user',
+ 'table',
+ 'temporary',
+ 'then',
+ 'time',
+ 'timestamp',
+ 'timezone_hour',
+ 'timezone_minute',
+ 'to',
+ 'trailing',
+ 'transaction',
+ 'translate',
+ 'translation',
+ 'trim',
+ 'true',
+ 'union',
+ 'unique',
+ 'unknown',
+ 'update',
+ 'upper',
+ 'usage',
+ 'user',
+ 'using',
+ 'value',
+ 'values',
+ 'varchar',
+ 'varying',
+ 'view',
+ 'when',
+ 'whenever',
+ 'where',
+ 'with',
+ 'work',
+ 'write',
+ 'year',
+ 'zone',
+)
+
+# See https://msdn.microsoft.com/en-us/library/ms189822.aspx.
+KEYWORDS = sorted(set(_KEYWORDS_FUTURE + _KEYWORDS_ODBC + _KEYWORDS_SERVER))
+
+# See https://msdn.microsoft.com/en-us/library/ms187752.aspx.
+TYPES = (
+ 'bigint',
+ 'binary',
+ 'bit',
+ 'char',
+ 'cursor',
+ 'date',
+ 'datetime',
+ 'datetime2',
+ 'datetimeoffset',
+ 'decimal',
+ 'float',
+ 'hierarchyid',
+ 'image',
+ 'int',
+ 'money',
+ 'nchar',
+ 'ntext',
+ 'numeric',
+ 'nvarchar',
+ 'real',
+ 'smalldatetime',
+ 'smallint',
+ 'smallmoney',
+ 'sql_variant',
+ 'table',
+ 'text',
+ 'time',
+ 'timestamp',
+ 'tinyint',
+ 'uniqueidentifier',
+ 'varbinary',
+ 'varchar',
+ 'xml',
+)
+
+# See https://msdn.microsoft.com/en-us/library/ms174318.aspx.
+FUNCTIONS = (
+ '$partition',
+ 'abs',
+ 'acos',
+ 'app_name',
+ 'applock_mode',
+ 'applock_test',
+ 'ascii',
+ 'asin',
+ 'assemblyproperty',
+ 'atan',
+ 'atn2',
+ 'avg',
+ 'binary_checksum',
+ 'cast',
+ 'ceiling',
+ 'certencoded',
+ 'certprivatekey',
+ 'char',
+ 'charindex',
+ 'checksum',
+ 'checksum_agg',
+ 'choose',
+ 'col_length',
+ 'col_name',
+ 'columnproperty',
+ 'compress',
+ 'concat',
+ 'connectionproperty',
+ 'context_info',
+ 'convert',
+ 'cos',
+ 'cot',
+ 'count',
+ 'count_big',
+ 'current_request_id',
+ 'current_timestamp',
+ 'current_transaction_id',
+ 'current_user',
+ 'cursor_status',
+ 'database_principal_id',
+ 'databasepropertyex',
+ 'dateadd',
+ 'datediff',
+ 'datediff_big',
+ 'datefromparts',
+ 'datename',
+ 'datepart',
+ 'datetime2fromparts',
+ 'datetimefromparts',
+ 'datetimeoffsetfromparts',
+ 'day',
+ 'db_id',
+ 'db_name',
+ 'decompress',
+ 'degrees',
+ 'dense_rank',
+ 'difference',
+ 'eomonth',
+ 'error_line',
+ 'error_message',
+ 'error_number',
+ 'error_procedure',
+ 'error_severity',
+ 'error_state',
+ 'exp',
+ 'file_id',
+ 'file_idex',
+ 'file_name',
+ 'filegroup_id',
+ 'filegroup_name',
+ 'filegroupproperty',
+ 'fileproperty',
+ 'floor',
+ 'format',
+ 'formatmessage',
+ 'fulltextcatalogproperty',
+ 'fulltextserviceproperty',
+ 'get_filestream_transaction_context',
+ 'getansinull',
+ 'getdate',
+ 'getutcdate',
+ 'grouping',
+ 'grouping_id',
+ 'has_perms_by_name',
+ 'host_id',
+ 'host_name',
+ 'iif',
+ 'index_col',
+ 'indexkey_property',
+ 'indexproperty',
+ 'is_member',
+ 'is_rolemember',
+ 'is_srvrolemember',
+ 'isdate',
+ 'isjson',
+ 'isnull',
+ 'isnumeric',
+ 'json_modify',
+ 'json_query',
+ 'json_value',
+ 'left',
+ 'len',
+ 'log',
+ 'log10',
+ 'lower',
+ 'ltrim',
+ 'max',
+ 'min',
+ 'min_active_rowversion',
+ 'month',
+ 'nchar',
+ 'newid',
+ 'newsequentialid',
+ 'ntile',
+ 'object_definition',
+ 'object_id',
+ 'object_name',
+ 'object_schema_name',
+ 'objectproperty',
+ 'objectpropertyex',
+ 'opendatasource',
+ 'openjson',
+ 'openquery',
+ 'openrowset',
+ 'openxml',
+ 'original_db_name',
+ 'original_login',
+ 'parse',
+ 'parsename',
+ 'patindex',
+ 'permissions',
+ 'pi',
+ 'power',
+ 'pwdcompare',
+ 'pwdencrypt',
+ 'quotename',
+ 'radians',
+ 'rand',
+ 'rank',
+ 'replace',
+ 'replicate',
+ 'reverse',
+ 'right',
+ 'round',
+ 'row_number',
+ 'rowcount_big',
+ 'rtrim',
+ 'schema_id',
+ 'schema_name',
+ 'scope_identity',
+ 'serverproperty',
+ 'session_context',
+ 'session_user',
+ 'sign',
+ 'sin',
+ 'smalldatetimefromparts',
+ 'soundex',
+ 'sp_helplanguage',
+ 'space',
+ 'sqrt',
+ 'square',
+ 'stats_date',
+ 'stdev',
+ 'stdevp',
+ 'str',
+ 'string_escape',
+ 'string_split',
+ 'stuff',
+ 'substring',
+ 'sum',
+ 'suser_id',
+ 'suser_name',
+ 'suser_sid',
+ 'suser_sname',
+ 'switchoffset',
+ 'sysdatetime',
+ 'sysdatetimeoffset',
+ 'system_user',
+ 'sysutcdatetime',
+ 'tan',
+ 'textptr',
+ 'textvalid',
+ 'timefromparts',
+ 'todatetimeoffset',
+ 'try_cast',
+ 'try_convert',
+ 'try_parse',
+ 'type_id',
+ 'type_name',
+ 'typeproperty',
+ 'unicode',
+ 'upper',
+ 'user_id',
+ 'user_name',
+ 'var',
+ 'varp',
+ 'xact_state',
+ 'year',
+)
diff --git a/wandb/vendor/pygments/lexers/_vim_builtins.py b/wandb/vendor/pygments/lexers/_vim_builtins.py
new file mode 100644
index 0000000000000000000000000000000000000000..8258628995bb95fca551f879d9126f9afd6a1607
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/_vim_builtins.py
@@ -0,0 +1,1939 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers._vim_builtins
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ This file is autogenerated by scripts/get_vimkw.py
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+# Split up in multiple functions so it's importable by jython, which has a
+# per-method size limit.
+
+def _getauto():
+ var = (
+ ('BufAdd','BufAdd'),
+ ('BufCreate','BufCreate'),
+ ('BufDelete','BufDelete'),
+ ('BufEnter','BufEnter'),
+ ('BufFilePost','BufFilePost'),
+ ('BufFilePre','BufFilePre'),
+ ('BufHidden','BufHidden'),
+ ('BufLeave','BufLeave'),
+ ('BufNew','BufNew'),
+ ('BufNewFile','BufNewFile'),
+ ('BufRead','BufRead'),
+ ('BufReadCmd','BufReadCmd'),
+ ('BufReadPost','BufReadPost'),
+ ('BufReadPre','BufReadPre'),
+ ('BufUnload','BufUnload'),
+ ('BufWinEnter','BufWinEnter'),
+ ('BufWinLeave','BufWinLeave'),
+ ('BufWipeout','BufWipeout'),
+ ('BufWrite','BufWrite'),
+ ('BufWriteCmd','BufWriteCmd'),
+ ('BufWritePost','BufWritePost'),
+ ('BufWritePre','BufWritePre'),
+ ('Cmd','Cmd'),
+ ('CmdwinEnter','CmdwinEnter'),
+ ('CmdwinLeave','CmdwinLeave'),
+ ('ColorScheme','ColorScheme'),
+ ('CompleteDone','CompleteDone'),
+ ('CursorHold','CursorHold'),
+ ('CursorHoldI','CursorHoldI'),
+ ('CursorMoved','CursorMoved'),
+ ('CursorMovedI','CursorMovedI'),
+ ('EncodingChanged','EncodingChanged'),
+ ('FileAppendCmd','FileAppendCmd'),
+ ('FileAppendPost','FileAppendPost'),
+ ('FileAppendPre','FileAppendPre'),
+ ('FileChangedRO','FileChangedRO'),
+ ('FileChangedShell','FileChangedShell'),
+ ('FileChangedShellPost','FileChangedShellPost'),
+ ('FileEncoding','FileEncoding'),
+ ('FileReadCmd','FileReadCmd'),
+ ('FileReadPost','FileReadPost'),
+ ('FileReadPre','FileReadPre'),
+ ('FileType','FileType'),
+ ('FileWriteCmd','FileWriteCmd'),
+ ('FileWritePost','FileWritePost'),
+ ('FileWritePre','FileWritePre'),
+ ('FilterReadPost','FilterReadPost'),
+ ('FilterReadPre','FilterReadPre'),
+ ('FilterWritePost','FilterWritePost'),
+ ('FilterWritePre','FilterWritePre'),
+ ('FocusGained','FocusGained'),
+ ('FocusLost','FocusLost'),
+ ('FuncUndefined','FuncUndefined'),
+ ('GUIEnter','GUIEnter'),
+ ('GUIFailed','GUIFailed'),
+ ('InsertChange','InsertChange'),
+ ('InsertCharPre','InsertCharPre'),
+ ('InsertEnter','InsertEnter'),
+ ('InsertLeave','InsertLeave'),
+ ('MenuPopup','MenuPopup'),
+ ('QuickFixCmdPost','QuickFixCmdPost'),
+ ('QuickFixCmdPre','QuickFixCmdPre'),
+ ('QuitPre','QuitPre'),
+ ('RemoteReply','RemoteReply'),
+ ('SessionLoadPost','SessionLoadPost'),
+ ('ShellCmdPost','ShellCmdPost'),
+ ('ShellFilterPost','ShellFilterPost'),
+ ('SourceCmd','SourceCmd'),
+ ('SourcePre','SourcePre'),
+ ('SpellFileMissing','SpellFileMissing'),
+ ('StdinReadPost','StdinReadPost'),
+ ('StdinReadPre','StdinReadPre'),
+ ('SwapExists','SwapExists'),
+ ('Syntax','Syntax'),
+ ('TabEnter','TabEnter'),
+ ('TabLeave','TabLeave'),
+ ('TermChanged','TermChanged'),
+ ('TermResponse','TermResponse'),
+ ('TextChanged','TextChanged'),
+ ('TextChangedI','TextChangedI'),
+ ('User','User'),
+ ('UserGettingBored','UserGettingBored'),
+ ('VimEnter','VimEnter'),
+ ('VimLeave','VimLeave'),
+ ('VimLeavePre','VimLeavePre'),
+ ('VimResized','VimResized'),
+ ('WinEnter','WinEnter'),
+ ('WinLeave','WinLeave'),
+ ('event','event'),
+ )
+ return var
+auto = _getauto()
+
+def _getcommand():
+ var = (
+ ('a','a'),
+ ('ab','ab'),
+ ('abc','abclear'),
+ ('abo','aboveleft'),
+ ('al','all'),
+ ('ar','ar'),
+ ('ar','args'),
+ ('arga','argadd'),
+ ('argd','argdelete'),
+ ('argdo','argdo'),
+ ('arge','argedit'),
+ ('argg','argglobal'),
+ ('argl','arglocal'),
+ ('argu','argument'),
+ ('as','ascii'),
+ ('au','au'),
+ ('b','buffer'),
+ ('bN','bNext'),
+ ('ba','ball'),
+ ('bad','badd'),
+ ('bd','bdelete'),
+ ('bel','belowright'),
+ ('bf','bfirst'),
+ ('bl','blast'),
+ ('bm','bmodified'),
+ ('bn','bnext'),
+ ('bo','botright'),
+ ('bp','bprevious'),
+ ('br','br'),
+ ('br','brewind'),
+ ('brea','break'),
+ ('breaka','breakadd'),
+ ('breakd','breakdel'),
+ ('breakl','breaklist'),
+ ('bro','browse'),
+ ('bu','bu'),
+ ('buf','buf'),
+ ('bufdo','bufdo'),
+ ('buffers','buffers'),
+ ('bun','bunload'),
+ ('bw','bwipeout'),
+ ('c','c'),
+ ('c','change'),
+ ('cN','cN'),
+ ('cN','cNext'),
+ ('cNf','cNf'),
+ ('cNf','cNfile'),
+ ('cabc','cabclear'),
+ ('cad','cad'),
+ ('cad','caddexpr'),
+ ('caddb','caddbuffer'),
+ ('caddf','caddfile'),
+ ('cal','call'),
+ ('cat','catch'),
+ ('cb','cbuffer'),
+ ('cc','cc'),
+ ('ccl','cclose'),
+ ('cd','cd'),
+ ('ce','center'),
+ ('cex','cexpr'),
+ ('cf','cfile'),
+ ('cfir','cfirst'),
+ ('cg','cgetfile'),
+ ('cgetb','cgetbuffer'),
+ ('cgete','cgetexpr'),
+ ('changes','changes'),
+ ('chd','chdir'),
+ ('che','checkpath'),
+ ('checkt','checktime'),
+ ('cl','cl'),
+ ('cl','clist'),
+ ('cla','clast'),
+ ('clo','close'),
+ ('cmapc','cmapclear'),
+ ('cn','cn'),
+ ('cn','cnext'),
+ ('cnew','cnewer'),
+ ('cnf','cnf'),
+ ('cnf','cnfile'),
+ ('co','copy'),
+ ('col','colder'),
+ ('colo','colorscheme'),
+ ('com','com'),
+ ('comc','comclear'),
+ ('comp','compiler'),
+ ('con','con'),
+ ('con','continue'),
+ ('conf','confirm'),
+ ('cope','copen'),
+ ('cp','cprevious'),
+ ('cpf','cpfile'),
+ ('cq','cquit'),
+ ('cr','crewind'),
+ ('cs','cs'),
+ ('cscope','cscope'),
+ ('cstag','cstag'),
+ ('cuna','cunabbrev'),
+ ('cw','cwindow'),
+ ('d','d'),
+ ('d','delete'),
+ ('de','de'),
+ ('debug','debug'),
+ ('debugg','debuggreedy'),
+ ('del','del'),
+ ('delc','delcommand'),
+ ('delel','delel'),
+ ('delep','delep'),
+ ('deletel','deletel'),
+ ('deletep','deletep'),
+ ('deletl','deletl'),
+ ('deletp','deletp'),
+ ('delf','delf'),
+ ('delf','delfunction'),
+ ('dell','dell'),
+ ('delm','delmarks'),
+ ('delp','delp'),
+ ('dep','dep'),
+ ('di','di'),
+ ('di','display'),
+ ('diffg','diffget'),
+ ('diffo','diffoff'),
+ ('diffp','diffpatch'),
+ ('diffpu','diffput'),
+ ('diffs','diffsplit'),
+ ('difft','diffthis'),
+ ('diffu','diffupdate'),
+ ('dig','dig'),
+ ('dig','digraphs'),
+ ('dir','dir'),
+ ('dj','djump'),
+ ('dl','dl'),
+ ('dli','dlist'),
+ ('do','do'),
+ ('doau','doau'),
+ ('dp','dp'),
+ ('dr','drop'),
+ ('ds','dsearch'),
+ ('dsp','dsplit'),
+ ('e','e'),
+ ('e','edit'),
+ ('ea','ea'),
+ ('earlier','earlier'),
+ ('ec','ec'),
+ ('echoe','echoerr'),
+ ('echom','echomsg'),
+ ('echon','echon'),
+ ('el','else'),
+ ('elsei','elseif'),
+ ('em','emenu'),
+ ('en','en'),
+ ('en','endif'),
+ ('endf','endf'),
+ ('endf','endfunction'),
+ ('endfo','endfor'),
+ ('endfun','endfun'),
+ ('endt','endtry'),
+ ('endw','endwhile'),
+ ('ene','enew'),
+ ('ex','ex'),
+ ('exi','exit'),
+ ('exu','exusage'),
+ ('f','f'),
+ ('f','file'),
+ ('files','files'),
+ ('filet','filet'),
+ ('filetype','filetype'),
+ ('fin','fin'),
+ ('fin','find'),
+ ('fina','finally'),
+ ('fini','finish'),
+ ('fir','first'),
+ ('fix','fixdel'),
+ ('fo','fold'),
+ ('foldc','foldclose'),
+ ('foldd','folddoopen'),
+ ('folddoc','folddoclosed'),
+ ('foldo','foldopen'),
+ ('for','for'),
+ ('fu','fu'),
+ ('fu','function'),
+ ('fun','fun'),
+ ('g','g'),
+ ('go','goto'),
+ ('gr','grep'),
+ ('grepa','grepadd'),
+ ('gui','gui'),
+ ('gvim','gvim'),
+ ('h','h'),
+ ('h','help'),
+ ('ha','hardcopy'),
+ ('helpf','helpfind'),
+ ('helpg','helpgrep'),
+ ('helpt','helptags'),
+ ('hi','hi'),
+ ('hid','hide'),
+ ('his','history'),
+ ('i','i'),
+ ('ia','ia'),
+ ('iabc','iabclear'),
+ ('if','if'),
+ ('ij','ijump'),
+ ('il','ilist'),
+ ('imapc','imapclear'),
+ ('in','in'),
+ ('intro','intro'),
+ ('is','isearch'),
+ ('isp','isplit'),
+ ('iuna','iunabbrev'),
+ ('j','join'),
+ ('ju','jumps'),
+ ('k','k'),
+ ('kee','keepmarks'),
+ ('keepa','keepa'),
+ ('keepalt','keepalt'),
+ ('keepj','keepjumps'),
+ ('keepp','keeppatterns'),
+ ('l','l'),
+ ('l','list'),
+ ('lN','lN'),
+ ('lN','lNext'),
+ ('lNf','lNf'),
+ ('lNf','lNfile'),
+ ('la','la'),
+ ('la','last'),
+ ('lad','lad'),
+ ('lad','laddexpr'),
+ ('laddb','laddbuffer'),
+ ('laddf','laddfile'),
+ ('lan','lan'),
+ ('lan','language'),
+ ('lat','lat'),
+ ('later','later'),
+ ('lb','lbuffer'),
+ ('lc','lcd'),
+ ('lch','lchdir'),
+ ('lcl','lclose'),
+ ('lcs','lcs'),
+ ('lcscope','lcscope'),
+ ('le','left'),
+ ('lefta','leftabove'),
+ ('lex','lexpr'),
+ ('lf','lfile'),
+ ('lfir','lfirst'),
+ ('lg','lgetfile'),
+ ('lgetb','lgetbuffer'),
+ ('lgete','lgetexpr'),
+ ('lgr','lgrep'),
+ ('lgrepa','lgrepadd'),
+ ('lh','lhelpgrep'),
+ ('ll','ll'),
+ ('lla','llast'),
+ ('lli','llist'),
+ ('lmak','lmake'),
+ ('lmapc','lmapclear'),
+ ('lne','lne'),
+ ('lne','lnext'),
+ ('lnew','lnewer'),
+ ('lnf','lnf'),
+ ('lnf','lnfile'),
+ ('lo','lo'),
+ ('lo','loadview'),
+ ('loadk','loadk'),
+ ('loadkeymap','loadkeymap'),
+ ('loc','lockmarks'),
+ ('lockv','lockvar'),
+ ('lol','lolder'),
+ ('lop','lopen'),
+ ('lp','lprevious'),
+ ('lpf','lpfile'),
+ ('lr','lrewind'),
+ ('ls','ls'),
+ ('lt','ltag'),
+ ('lua','lua'),
+ ('luado','luado'),
+ ('luafile','luafile'),
+ ('lv','lvimgrep'),
+ ('lvimgrepa','lvimgrepadd'),
+ ('lw','lwindow'),
+ ('m','move'),
+ ('ma','ma'),
+ ('ma','mark'),
+ ('mak','make'),
+ ('marks','marks'),
+ ('mat','match'),
+ ('menut','menut'),
+ ('menut','menutranslate'),
+ ('mes','mes'),
+ ('messages','messages'),
+ ('mk','mk'),
+ ('mk','mkexrc'),
+ ('mks','mksession'),
+ ('mksp','mkspell'),
+ ('mkv','mkv'),
+ ('mkv','mkvimrc'),
+ ('mkvie','mkview'),
+ ('mo','mo'),
+ ('mod','mode'),
+ ('mz','mz'),
+ ('mz','mzscheme'),
+ ('mzf','mzfile'),
+ ('n','n'),
+ ('n','next'),
+ ('nb','nbkey'),
+ ('nbc','nbclose'),
+ ('nbs','nbstart'),
+ ('ne','ne'),
+ ('new','new'),
+ ('nmapc','nmapclear'),
+ ('noa','noa'),
+ ('noautocmd','noautocmd'),
+ ('noh','nohlsearch'),
+ ('nu','number'),
+ ('o','o'),
+ ('o','open'),
+ ('ol','oldfiles'),
+ ('omapc','omapclear'),
+ ('on','only'),
+ ('opt','options'),
+ ('ownsyntax','ownsyntax'),
+ ('p','p'),
+ ('p','print'),
+ ('pc','pclose'),
+ ('pe','pe'),
+ ('pe','perl'),
+ ('ped','pedit'),
+ ('perld','perldo'),
+ ('po','pop'),
+ ('popu','popu'),
+ ('popu','popup'),
+ ('pp','ppop'),
+ ('pr','pr'),
+ ('pre','preserve'),
+ ('prev','previous'),
+ ('pro','pro'),
+ ('prof','profile'),
+ ('profd','profdel'),
+ ('promptf','promptfind'),
+ ('promptr','promptrepl'),
+ ('ps','psearch'),
+ ('ptN','ptN'),
+ ('ptN','ptNext'),
+ ('pta','ptag'),
+ ('ptf','ptfirst'),
+ ('ptj','ptjump'),
+ ('ptl','ptlast'),
+ ('ptn','ptn'),
+ ('ptn','ptnext'),
+ ('ptp','ptprevious'),
+ ('ptr','ptrewind'),
+ ('pts','ptselect'),
+ ('pu','put'),
+ ('pw','pwd'),
+ ('py','py'),
+ ('py','python'),
+ ('py3','py3'),
+ ('py3','py3'),
+ ('py3do','py3do'),
+ ('pydo','pydo'),
+ ('pyf','pyfile'),
+ ('python3','python3'),
+ ('q','q'),
+ ('q','quit'),
+ ('qa','qall'),
+ ('quita','quitall'),
+ ('r','r'),
+ ('r','read'),
+ ('re','re'),
+ ('rec','recover'),
+ ('red','red'),
+ ('red','redo'),
+ ('redi','redir'),
+ ('redr','redraw'),
+ ('redraws','redrawstatus'),
+ ('reg','registers'),
+ ('res','resize'),
+ ('ret','retab'),
+ ('retu','return'),
+ ('rew','rewind'),
+ ('ri','right'),
+ ('rightb','rightbelow'),
+ ('ru','ru'),
+ ('ru','runtime'),
+ ('rub','ruby'),
+ ('rubyd','rubydo'),
+ ('rubyf','rubyfile'),
+ ('rundo','rundo'),
+ ('rv','rviminfo'),
+ ('sN','sNext'),
+ ('sa','sargument'),
+ ('sal','sall'),
+ ('san','sandbox'),
+ ('sav','saveas'),
+ ('sb','sbuffer'),
+ ('sbN','sbNext'),
+ ('sba','sball'),
+ ('sbf','sbfirst'),
+ ('sbl','sblast'),
+ ('sbm','sbmodified'),
+ ('sbn','sbnext'),
+ ('sbp','sbprevious'),
+ ('sbr','sbrewind'),
+ ('scrip','scrip'),
+ ('scrip','scriptnames'),
+ ('scripte','scriptencoding'),
+ ('scs','scs'),
+ ('scscope','scscope'),
+ ('se','set'),
+ ('setf','setfiletype'),
+ ('setg','setglobal'),
+ ('setl','setlocal'),
+ ('sf','sfind'),
+ ('sfir','sfirst'),
+ ('sh','shell'),
+ ('si','si'),
+ ('sig','sig'),
+ ('sign','sign'),
+ ('sil','silent'),
+ ('sim','simalt'),
+ ('sl','sl'),
+ ('sl','sleep'),
+ ('sla','slast'),
+ ('sm','smagic'),
+ ('sm','smap'),
+ ('sme','sme'),
+ ('smenu','smenu'),
+ ('sn','snext'),
+ ('sni','sniff'),
+ ('sno','snomagic'),
+ ('snoreme','snoreme'),
+ ('snoremenu','snoremenu'),
+ ('so','so'),
+ ('so','source'),
+ ('sor','sort'),
+ ('sp','split'),
+ ('spe','spe'),
+ ('spe','spellgood'),
+ ('spelld','spelldump'),
+ ('spelli','spellinfo'),
+ ('spellr','spellrepall'),
+ ('spellu','spellundo'),
+ ('spellw','spellwrong'),
+ ('spr','sprevious'),
+ ('sre','srewind'),
+ ('st','st'),
+ ('st','stop'),
+ ('sta','stag'),
+ ('star','star'),
+ ('star','startinsert'),
+ ('start','start'),
+ ('startg','startgreplace'),
+ ('startr','startreplace'),
+ ('stj','stjump'),
+ ('stopi','stopinsert'),
+ ('sts','stselect'),
+ ('sun','sunhide'),
+ ('sunme','sunme'),
+ ('sunmenu','sunmenu'),
+ ('sus','suspend'),
+ ('sv','sview'),
+ ('sw','swapname'),
+ ('sy','sy'),
+ ('syn','syn'),
+ ('sync','sync'),
+ ('syncbind','syncbind'),
+ ('syntime','syntime'),
+ ('t','t'),
+ ('tN','tN'),
+ ('tN','tNext'),
+ ('ta','ta'),
+ ('ta','tag'),
+ ('tab','tab'),
+ ('tabN','tabN'),
+ ('tabN','tabNext'),
+ ('tabc','tabclose'),
+ ('tabd','tabdo'),
+ ('tabe','tabedit'),
+ ('tabf','tabfind'),
+ ('tabfir','tabfirst'),
+ ('tabl','tablast'),
+ ('tabm','tabmove'),
+ ('tabn','tabnext'),
+ ('tabnew','tabnew'),
+ ('tabo','tabonly'),
+ ('tabp','tabprevious'),
+ ('tabr','tabrewind'),
+ ('tabs','tabs'),
+ ('tags','tags'),
+ ('tc','tcl'),
+ ('tcld','tcldo'),
+ ('tclf','tclfile'),
+ ('te','tearoff'),
+ ('tf','tfirst'),
+ ('th','throw'),
+ ('tj','tjump'),
+ ('tl','tlast'),
+ ('tm','tm'),
+ ('tm','tmenu'),
+ ('tn','tn'),
+ ('tn','tnext'),
+ ('to','topleft'),
+ ('tp','tprevious'),
+ ('tr','tr'),
+ ('tr','trewind'),
+ ('try','try'),
+ ('ts','tselect'),
+ ('tu','tu'),
+ ('tu','tunmenu'),
+ ('u','u'),
+ ('u','undo'),
+ ('un','un'),
+ ('una','unabbreviate'),
+ ('undoj','undojoin'),
+ ('undol','undolist'),
+ ('unh','unhide'),
+ ('unl','unl'),
+ ('unlo','unlockvar'),
+ ('uns','unsilent'),
+ ('up','update'),
+ ('v','v'),
+ ('ve','ve'),
+ ('ve','version'),
+ ('verb','verbose'),
+ ('vert','vertical'),
+ ('vi','vi'),
+ ('vi','visual'),
+ ('vie','view'),
+ ('vim','vimgrep'),
+ ('vimgrepa','vimgrepadd'),
+ ('viu','viusage'),
+ ('vmapc','vmapclear'),
+ ('vne','vnew'),
+ ('vs','vsplit'),
+ ('w','w'),
+ ('w','write'),
+ ('wN','wNext'),
+ ('wa','wall'),
+ ('wh','while'),
+ ('win','win'),
+ ('win','winsize'),
+ ('winc','wincmd'),
+ ('windo','windo'),
+ ('winp','winpos'),
+ ('wn','wnext'),
+ ('wp','wprevious'),
+ ('wq','wq'),
+ ('wqa','wqall'),
+ ('ws','wsverb'),
+ ('wundo','wundo'),
+ ('wv','wviminfo'),
+ ('x','x'),
+ ('x','xit'),
+ ('xa','xall'),
+ ('xmapc','xmapclear'),
+ ('xme','xme'),
+ ('xmenu','xmenu'),
+ ('xnoreme','xnoreme'),
+ ('xnoremenu','xnoremenu'),
+ ('xunme','xunme'),
+ ('xunmenu','xunmenu'),
+ ('xwininfo','xwininfo'),
+ ('y','yank'),
+ )
+ return var
+command = _getcommand()
+
+def _getoption():
+ var = (
+ ('acd','acd'),
+ ('ai','ai'),
+ ('akm','akm'),
+ ('al','al'),
+ ('aleph','aleph'),
+ ('allowrevins','allowrevins'),
+ ('altkeymap','altkeymap'),
+ ('ambiwidth','ambiwidth'),
+ ('ambw','ambw'),
+ ('anti','anti'),
+ ('antialias','antialias'),
+ ('ar','ar'),
+ ('arab','arab'),
+ ('arabic','arabic'),
+ ('arabicshape','arabicshape'),
+ ('ari','ari'),
+ ('arshape','arshape'),
+ ('autochdir','autochdir'),
+ ('autoindent','autoindent'),
+ ('autoread','autoread'),
+ ('autowrite','autowrite'),
+ ('autowriteall','autowriteall'),
+ ('aw','aw'),
+ ('awa','awa'),
+ ('background','background'),
+ ('backspace','backspace'),
+ ('backup','backup'),
+ ('backupcopy','backupcopy'),
+ ('backupdir','backupdir'),
+ ('backupext','backupext'),
+ ('backupskip','backupskip'),
+ ('balloondelay','balloondelay'),
+ ('ballooneval','ballooneval'),
+ ('balloonexpr','balloonexpr'),
+ ('bdir','bdir'),
+ ('bdlay','bdlay'),
+ ('beval','beval'),
+ ('bex','bex'),
+ ('bexpr','bexpr'),
+ ('bg','bg'),
+ ('bh','bh'),
+ ('bin','bin'),
+ ('binary','binary'),
+ ('biosk','biosk'),
+ ('bioskey','bioskey'),
+ ('bk','bk'),
+ ('bkc','bkc'),
+ ('bl','bl'),
+ ('bomb','bomb'),
+ ('breakat','breakat'),
+ ('brk','brk'),
+ ('browsedir','browsedir'),
+ ('bs','bs'),
+ ('bsdir','bsdir'),
+ ('bsk','bsk'),
+ ('bt','bt'),
+ ('bufhidden','bufhidden'),
+ ('buflisted','buflisted'),
+ ('buftype','buftype'),
+ ('casemap','casemap'),
+ ('cb','cb'),
+ ('cc','cc'),
+ ('ccv','ccv'),
+ ('cd','cd'),
+ ('cdpath','cdpath'),
+ ('cedit','cedit'),
+ ('cf','cf'),
+ ('cfu','cfu'),
+ ('ch','ch'),
+ ('charconvert','charconvert'),
+ ('ci','ci'),
+ ('cin','cin'),
+ ('cindent','cindent'),
+ ('cink','cink'),
+ ('cinkeys','cinkeys'),
+ ('cino','cino'),
+ ('cinoptions','cinoptions'),
+ ('cinw','cinw'),
+ ('cinwords','cinwords'),
+ ('clipboard','clipboard'),
+ ('cmdheight','cmdheight'),
+ ('cmdwinheight','cmdwinheight'),
+ ('cmp','cmp'),
+ ('cms','cms'),
+ ('co','co'),
+ ('cocu','cocu'),
+ ('cole','cole'),
+ ('colorcolumn','colorcolumn'),
+ ('columns','columns'),
+ ('com','com'),
+ ('comments','comments'),
+ ('commentstring','commentstring'),
+ ('compatible','compatible'),
+ ('complete','complete'),
+ ('completefunc','completefunc'),
+ ('completeopt','completeopt'),
+ ('concealcursor','concealcursor'),
+ ('conceallevel','conceallevel'),
+ ('confirm','confirm'),
+ ('consk','consk'),
+ ('conskey','conskey'),
+ ('copyindent','copyindent'),
+ ('cot','cot'),
+ ('cp','cp'),
+ ('cpo','cpo'),
+ ('cpoptions','cpoptions'),
+ ('cpt','cpt'),
+ ('crb','crb'),
+ ('cryptmethod','cryptmethod'),
+ ('cscopepathcomp','cscopepathcomp'),
+ ('cscopeprg','cscopeprg'),
+ ('cscopequickfix','cscopequickfix'),
+ ('cscoperelative','cscoperelative'),
+ ('cscopetag','cscopetag'),
+ ('cscopetagorder','cscopetagorder'),
+ ('cscopeverbose','cscopeverbose'),
+ ('cspc','cspc'),
+ ('csprg','csprg'),
+ ('csqf','csqf'),
+ ('csre','csre'),
+ ('cst','cst'),
+ ('csto','csto'),
+ ('csverb','csverb'),
+ ('cuc','cuc'),
+ ('cul','cul'),
+ ('cursorbind','cursorbind'),
+ ('cursorcolumn','cursorcolumn'),
+ ('cursorline','cursorline'),
+ ('cwh','cwh'),
+ ('debug','debug'),
+ ('deco','deco'),
+ ('def','def'),
+ ('define','define'),
+ ('delcombine','delcombine'),
+ ('dex','dex'),
+ ('dg','dg'),
+ ('dict','dict'),
+ ('dictionary','dictionary'),
+ ('diff','diff'),
+ ('diffexpr','diffexpr'),
+ ('diffopt','diffopt'),
+ ('digraph','digraph'),
+ ('dip','dip'),
+ ('dir','dir'),
+ ('directory','directory'),
+ ('display','display'),
+ ('dy','dy'),
+ ('ea','ea'),
+ ('ead','ead'),
+ ('eadirection','eadirection'),
+ ('eb','eb'),
+ ('ed','ed'),
+ ('edcompatible','edcompatible'),
+ ('ef','ef'),
+ ('efm','efm'),
+ ('ei','ei'),
+ ('ek','ek'),
+ ('enc','enc'),
+ ('encoding','encoding'),
+ ('endofline','endofline'),
+ ('eol','eol'),
+ ('ep','ep'),
+ ('equalalways','equalalways'),
+ ('equalprg','equalprg'),
+ ('errorbells','errorbells'),
+ ('errorfile','errorfile'),
+ ('errorformat','errorformat'),
+ ('esckeys','esckeys'),
+ ('et','et'),
+ ('eventignore','eventignore'),
+ ('ex','ex'),
+ ('expandtab','expandtab'),
+ ('exrc','exrc'),
+ ('fcl','fcl'),
+ ('fcs','fcs'),
+ ('fdc','fdc'),
+ ('fde','fde'),
+ ('fdi','fdi'),
+ ('fdl','fdl'),
+ ('fdls','fdls'),
+ ('fdm','fdm'),
+ ('fdn','fdn'),
+ ('fdo','fdo'),
+ ('fdt','fdt'),
+ ('fen','fen'),
+ ('fenc','fenc'),
+ ('fencs','fencs'),
+ ('fex','fex'),
+ ('ff','ff'),
+ ('ffs','ffs'),
+ ('fic','fic'),
+ ('fileencoding','fileencoding'),
+ ('fileencodings','fileencodings'),
+ ('fileformat','fileformat'),
+ ('fileformats','fileformats'),
+ ('fileignorecase','fileignorecase'),
+ ('filetype','filetype'),
+ ('fillchars','fillchars'),
+ ('fk','fk'),
+ ('fkmap','fkmap'),
+ ('flp','flp'),
+ ('fml','fml'),
+ ('fmr','fmr'),
+ ('fo','fo'),
+ ('foldclose','foldclose'),
+ ('foldcolumn','foldcolumn'),
+ ('foldenable','foldenable'),
+ ('foldexpr','foldexpr'),
+ ('foldignore','foldignore'),
+ ('foldlevel','foldlevel'),
+ ('foldlevelstart','foldlevelstart'),
+ ('foldmarker','foldmarker'),
+ ('foldmethod','foldmethod'),
+ ('foldminlines','foldminlines'),
+ ('foldnestmax','foldnestmax'),
+ ('foldopen','foldopen'),
+ ('foldtext','foldtext'),
+ ('formatexpr','formatexpr'),
+ ('formatlistpat','formatlistpat'),
+ ('formatoptions','formatoptions'),
+ ('formatprg','formatprg'),
+ ('fp','fp'),
+ ('fs','fs'),
+ ('fsync','fsync'),
+ ('ft','ft'),
+ ('gcr','gcr'),
+ ('gd','gd'),
+ ('gdefault','gdefault'),
+ ('gfm','gfm'),
+ ('gfn','gfn'),
+ ('gfs','gfs'),
+ ('gfw','gfw'),
+ ('ghr','ghr'),
+ ('go','go'),
+ ('gp','gp'),
+ ('grepformat','grepformat'),
+ ('grepprg','grepprg'),
+ ('gtl','gtl'),
+ ('gtt','gtt'),
+ ('guicursor','guicursor'),
+ ('guifont','guifont'),
+ ('guifontset','guifontset'),
+ ('guifontwide','guifontwide'),
+ ('guiheadroom','guiheadroom'),
+ ('guioptions','guioptions'),
+ ('guipty','guipty'),
+ ('guitablabel','guitablabel'),
+ ('guitabtooltip','guitabtooltip'),
+ ('helpfile','helpfile'),
+ ('helpheight','helpheight'),
+ ('helplang','helplang'),
+ ('hf','hf'),
+ ('hh','hh'),
+ ('hi','hi'),
+ ('hid','hid'),
+ ('hidden','hidden'),
+ ('highlight','highlight'),
+ ('history','history'),
+ ('hk','hk'),
+ ('hkmap','hkmap'),
+ ('hkmapp','hkmapp'),
+ ('hkp','hkp'),
+ ('hl','hl'),
+ ('hlg','hlg'),
+ ('hls','hls'),
+ ('hlsearch','hlsearch'),
+ ('ic','ic'),
+ ('icon','icon'),
+ ('iconstring','iconstring'),
+ ('ignorecase','ignorecase'),
+ ('im','im'),
+ ('imactivatefunc','imactivatefunc'),
+ ('imactivatekey','imactivatekey'),
+ ('imaf','imaf'),
+ ('imak','imak'),
+ ('imc','imc'),
+ ('imcmdline','imcmdline'),
+ ('imd','imd'),
+ ('imdisable','imdisable'),
+ ('imi','imi'),
+ ('iminsert','iminsert'),
+ ('ims','ims'),
+ ('imsearch','imsearch'),
+ ('imsf','imsf'),
+ ('imstatusfunc','imstatusfunc'),
+ ('inc','inc'),
+ ('include','include'),
+ ('includeexpr','includeexpr'),
+ ('incsearch','incsearch'),
+ ('inde','inde'),
+ ('indentexpr','indentexpr'),
+ ('indentkeys','indentkeys'),
+ ('indk','indk'),
+ ('inex','inex'),
+ ('inf','inf'),
+ ('infercase','infercase'),
+ ('inoremap','inoremap'),
+ ('insertmode','insertmode'),
+ ('invacd','invacd'),
+ ('invai','invai'),
+ ('invakm','invakm'),
+ ('invallowrevins','invallowrevins'),
+ ('invaltkeymap','invaltkeymap'),
+ ('invanti','invanti'),
+ ('invantialias','invantialias'),
+ ('invar','invar'),
+ ('invarab','invarab'),
+ ('invarabic','invarabic'),
+ ('invarabicshape','invarabicshape'),
+ ('invari','invari'),
+ ('invarshape','invarshape'),
+ ('invautochdir','invautochdir'),
+ ('invautoindent','invautoindent'),
+ ('invautoread','invautoread'),
+ ('invautowrite','invautowrite'),
+ ('invautowriteall','invautowriteall'),
+ ('invaw','invaw'),
+ ('invawa','invawa'),
+ ('invbackup','invbackup'),
+ ('invballooneval','invballooneval'),
+ ('invbeval','invbeval'),
+ ('invbin','invbin'),
+ ('invbinary','invbinary'),
+ ('invbiosk','invbiosk'),
+ ('invbioskey','invbioskey'),
+ ('invbk','invbk'),
+ ('invbl','invbl'),
+ ('invbomb','invbomb'),
+ ('invbuflisted','invbuflisted'),
+ ('invcf','invcf'),
+ ('invci','invci'),
+ ('invcin','invcin'),
+ ('invcindent','invcindent'),
+ ('invcompatible','invcompatible'),
+ ('invconfirm','invconfirm'),
+ ('invconsk','invconsk'),
+ ('invconskey','invconskey'),
+ ('invcopyindent','invcopyindent'),
+ ('invcp','invcp'),
+ ('invcrb','invcrb'),
+ ('invcscoperelative','invcscoperelative'),
+ ('invcscopetag','invcscopetag'),
+ ('invcscopeverbose','invcscopeverbose'),
+ ('invcsre','invcsre'),
+ ('invcst','invcst'),
+ ('invcsverb','invcsverb'),
+ ('invcuc','invcuc'),
+ ('invcul','invcul'),
+ ('invcursorbind','invcursorbind'),
+ ('invcursorcolumn','invcursorcolumn'),
+ ('invcursorline','invcursorline'),
+ ('invdeco','invdeco'),
+ ('invdelcombine','invdelcombine'),
+ ('invdg','invdg'),
+ ('invdiff','invdiff'),
+ ('invdigraph','invdigraph'),
+ ('invea','invea'),
+ ('inveb','inveb'),
+ ('inved','inved'),
+ ('invedcompatible','invedcompatible'),
+ ('invek','invek'),
+ ('invendofline','invendofline'),
+ ('inveol','inveol'),
+ ('invequalalways','invequalalways'),
+ ('inverrorbells','inverrorbells'),
+ ('invesckeys','invesckeys'),
+ ('invet','invet'),
+ ('invex','invex'),
+ ('invexpandtab','invexpandtab'),
+ ('invexrc','invexrc'),
+ ('invfen','invfen'),
+ ('invfic','invfic'),
+ ('invfileignorecase','invfileignorecase'),
+ ('invfk','invfk'),
+ ('invfkmap','invfkmap'),
+ ('invfoldenable','invfoldenable'),
+ ('invgd','invgd'),
+ ('invgdefault','invgdefault'),
+ ('invguipty','invguipty'),
+ ('invhid','invhid'),
+ ('invhidden','invhidden'),
+ ('invhk','invhk'),
+ ('invhkmap','invhkmap'),
+ ('invhkmapp','invhkmapp'),
+ ('invhkp','invhkp'),
+ ('invhls','invhls'),
+ ('invhlsearch','invhlsearch'),
+ ('invic','invic'),
+ ('invicon','invicon'),
+ ('invignorecase','invignorecase'),
+ ('invim','invim'),
+ ('invimc','invimc'),
+ ('invimcmdline','invimcmdline'),
+ ('invimd','invimd'),
+ ('invimdisable','invimdisable'),
+ ('invincsearch','invincsearch'),
+ ('invinf','invinf'),
+ ('invinfercase','invinfercase'),
+ ('invinsertmode','invinsertmode'),
+ ('invis','invis'),
+ ('invjoinspaces','invjoinspaces'),
+ ('invjs','invjs'),
+ ('invlazyredraw','invlazyredraw'),
+ ('invlbr','invlbr'),
+ ('invlinebreak','invlinebreak'),
+ ('invlisp','invlisp'),
+ ('invlist','invlist'),
+ ('invloadplugins','invloadplugins'),
+ ('invlpl','invlpl'),
+ ('invlz','invlz'),
+ ('invma','invma'),
+ ('invmacatsui','invmacatsui'),
+ ('invmagic','invmagic'),
+ ('invmh','invmh'),
+ ('invml','invml'),
+ ('invmod','invmod'),
+ ('invmodeline','invmodeline'),
+ ('invmodifiable','invmodifiable'),
+ ('invmodified','invmodified'),
+ ('invmore','invmore'),
+ ('invmousef','invmousef'),
+ ('invmousefocus','invmousefocus'),
+ ('invmousehide','invmousehide'),
+ ('invnu','invnu'),
+ ('invnumber','invnumber'),
+ ('invodev','invodev'),
+ ('invopendevice','invopendevice'),
+ ('invpaste','invpaste'),
+ ('invpi','invpi'),
+ ('invpreserveindent','invpreserveindent'),
+ ('invpreviewwindow','invpreviewwindow'),
+ ('invprompt','invprompt'),
+ ('invpvw','invpvw'),
+ ('invreadonly','invreadonly'),
+ ('invrelativenumber','invrelativenumber'),
+ ('invremap','invremap'),
+ ('invrestorescreen','invrestorescreen'),
+ ('invrevins','invrevins'),
+ ('invri','invri'),
+ ('invrightleft','invrightleft'),
+ ('invrl','invrl'),
+ ('invrnu','invrnu'),
+ ('invro','invro'),
+ ('invrs','invrs'),
+ ('invru','invru'),
+ ('invruler','invruler'),
+ ('invsb','invsb'),
+ ('invsc','invsc'),
+ ('invscb','invscb'),
+ ('invscrollbind','invscrollbind'),
+ ('invscs','invscs'),
+ ('invsecure','invsecure'),
+ ('invsft','invsft'),
+ ('invshellslash','invshellslash'),
+ ('invshelltemp','invshelltemp'),
+ ('invshiftround','invshiftround'),
+ ('invshortname','invshortname'),
+ ('invshowcmd','invshowcmd'),
+ ('invshowfulltag','invshowfulltag'),
+ ('invshowmatch','invshowmatch'),
+ ('invshowmode','invshowmode'),
+ ('invsi','invsi'),
+ ('invsm','invsm'),
+ ('invsmartcase','invsmartcase'),
+ ('invsmartindent','invsmartindent'),
+ ('invsmarttab','invsmarttab'),
+ ('invsmd','invsmd'),
+ ('invsn','invsn'),
+ ('invsol','invsol'),
+ ('invspell','invspell'),
+ ('invsplitbelow','invsplitbelow'),
+ ('invsplitright','invsplitright'),
+ ('invspr','invspr'),
+ ('invsr','invsr'),
+ ('invssl','invssl'),
+ ('invsta','invsta'),
+ ('invstartofline','invstartofline'),
+ ('invstmp','invstmp'),
+ ('invswapfile','invswapfile'),
+ ('invswf','invswf'),
+ ('invta','invta'),
+ ('invtagbsearch','invtagbsearch'),
+ ('invtagrelative','invtagrelative'),
+ ('invtagstack','invtagstack'),
+ ('invtbi','invtbi'),
+ ('invtbidi','invtbidi'),
+ ('invtbs','invtbs'),
+ ('invtermbidi','invtermbidi'),
+ ('invterse','invterse'),
+ ('invtextauto','invtextauto'),
+ ('invtextmode','invtextmode'),
+ ('invtf','invtf'),
+ ('invtgst','invtgst'),
+ ('invtildeop','invtildeop'),
+ ('invtimeout','invtimeout'),
+ ('invtitle','invtitle'),
+ ('invto','invto'),
+ ('invtop','invtop'),
+ ('invtr','invtr'),
+ ('invttimeout','invttimeout'),
+ ('invttybuiltin','invttybuiltin'),
+ ('invttyfast','invttyfast'),
+ ('invtx','invtx'),
+ ('invudf','invudf'),
+ ('invundofile','invundofile'),
+ ('invvb','invvb'),
+ ('invvisualbell','invvisualbell'),
+ ('invwa','invwa'),
+ ('invwarn','invwarn'),
+ ('invwb','invwb'),
+ ('invweirdinvert','invweirdinvert'),
+ ('invwfh','invwfh'),
+ ('invwfw','invwfw'),
+ ('invwic','invwic'),
+ ('invwildignorecase','invwildignorecase'),
+ ('invwildmenu','invwildmenu'),
+ ('invwinfixheight','invwinfixheight'),
+ ('invwinfixwidth','invwinfixwidth'),
+ ('invwiv','invwiv'),
+ ('invwmnu','invwmnu'),
+ ('invwrap','invwrap'),
+ ('invwrapscan','invwrapscan'),
+ ('invwrite','invwrite'),
+ ('invwriteany','invwriteany'),
+ ('invwritebackup','invwritebackup'),
+ ('invws','invws'),
+ ('is','is'),
+ ('isf','isf'),
+ ('isfname','isfname'),
+ ('isi','isi'),
+ ('isident','isident'),
+ ('isk','isk'),
+ ('iskeyword','iskeyword'),
+ ('isp','isp'),
+ ('isprint','isprint'),
+ ('joinspaces','joinspaces'),
+ ('js','js'),
+ ('key','key'),
+ ('keymap','keymap'),
+ ('keymodel','keymodel'),
+ ('keywordprg','keywordprg'),
+ ('km','km'),
+ ('kmp','kmp'),
+ ('kp','kp'),
+ ('langmap','langmap'),
+ ('langmenu','langmenu'),
+ ('laststatus','laststatus'),
+ ('lazyredraw','lazyredraw'),
+ ('lbr','lbr'),
+ ('lcs','lcs'),
+ ('linebreak','linebreak'),
+ ('lines','lines'),
+ ('linespace','linespace'),
+ ('lisp','lisp'),
+ ('lispwords','lispwords'),
+ ('list','list'),
+ ('listchars','listchars'),
+ ('lm','lm'),
+ ('lmap','lmap'),
+ ('loadplugins','loadplugins'),
+ ('lpl','lpl'),
+ ('ls','ls'),
+ ('lsp','lsp'),
+ ('lw','lw'),
+ ('lz','lz'),
+ ('ma','ma'),
+ ('macatsui','macatsui'),
+ ('magic','magic'),
+ ('makeef','makeef'),
+ ('makeprg','makeprg'),
+ ('mat','mat'),
+ ('matchpairs','matchpairs'),
+ ('matchtime','matchtime'),
+ ('maxcombine','maxcombine'),
+ ('maxfuncdepth','maxfuncdepth'),
+ ('maxmapdepth','maxmapdepth'),
+ ('maxmem','maxmem'),
+ ('maxmempattern','maxmempattern'),
+ ('maxmemtot','maxmemtot'),
+ ('mco','mco'),
+ ('mef','mef'),
+ ('menuitems','menuitems'),
+ ('mfd','mfd'),
+ ('mh','mh'),
+ ('mis','mis'),
+ ('mkspellmem','mkspellmem'),
+ ('ml','ml'),
+ ('mls','mls'),
+ ('mm','mm'),
+ ('mmd','mmd'),
+ ('mmp','mmp'),
+ ('mmt','mmt'),
+ ('mod','mod'),
+ ('modeline','modeline'),
+ ('modelines','modelines'),
+ ('modifiable','modifiable'),
+ ('modified','modified'),
+ ('more','more'),
+ ('mouse','mouse'),
+ ('mousef','mousef'),
+ ('mousefocus','mousefocus'),
+ ('mousehide','mousehide'),
+ ('mousem','mousem'),
+ ('mousemodel','mousemodel'),
+ ('mouses','mouses'),
+ ('mouseshape','mouseshape'),
+ ('mouset','mouset'),
+ ('mousetime','mousetime'),
+ ('mp','mp'),
+ ('mps','mps'),
+ ('msm','msm'),
+ ('mzq','mzq'),
+ ('mzquantum','mzquantum'),
+ ('nf','nf'),
+ ('nnoremap','nnoremap'),
+ ('noacd','noacd'),
+ ('noai','noai'),
+ ('noakm','noakm'),
+ ('noallowrevins','noallowrevins'),
+ ('noaltkeymap','noaltkeymap'),
+ ('noanti','noanti'),
+ ('noantialias','noantialias'),
+ ('noar','noar'),
+ ('noarab','noarab'),
+ ('noarabic','noarabic'),
+ ('noarabicshape','noarabicshape'),
+ ('noari','noari'),
+ ('noarshape','noarshape'),
+ ('noautochdir','noautochdir'),
+ ('noautoindent','noautoindent'),
+ ('noautoread','noautoread'),
+ ('noautowrite','noautowrite'),
+ ('noautowriteall','noautowriteall'),
+ ('noaw','noaw'),
+ ('noawa','noawa'),
+ ('nobackup','nobackup'),
+ ('noballooneval','noballooneval'),
+ ('nobeval','nobeval'),
+ ('nobin','nobin'),
+ ('nobinary','nobinary'),
+ ('nobiosk','nobiosk'),
+ ('nobioskey','nobioskey'),
+ ('nobk','nobk'),
+ ('nobl','nobl'),
+ ('nobomb','nobomb'),
+ ('nobuflisted','nobuflisted'),
+ ('nocf','nocf'),
+ ('noci','noci'),
+ ('nocin','nocin'),
+ ('nocindent','nocindent'),
+ ('nocompatible','nocompatible'),
+ ('noconfirm','noconfirm'),
+ ('noconsk','noconsk'),
+ ('noconskey','noconskey'),
+ ('nocopyindent','nocopyindent'),
+ ('nocp','nocp'),
+ ('nocrb','nocrb'),
+ ('nocscoperelative','nocscoperelative'),
+ ('nocscopetag','nocscopetag'),
+ ('nocscopeverbose','nocscopeverbose'),
+ ('nocsre','nocsre'),
+ ('nocst','nocst'),
+ ('nocsverb','nocsverb'),
+ ('nocuc','nocuc'),
+ ('nocul','nocul'),
+ ('nocursorbind','nocursorbind'),
+ ('nocursorcolumn','nocursorcolumn'),
+ ('nocursorline','nocursorline'),
+ ('nodeco','nodeco'),
+ ('nodelcombine','nodelcombine'),
+ ('nodg','nodg'),
+ ('nodiff','nodiff'),
+ ('nodigraph','nodigraph'),
+ ('noea','noea'),
+ ('noeb','noeb'),
+ ('noed','noed'),
+ ('noedcompatible','noedcompatible'),
+ ('noek','noek'),
+ ('noendofline','noendofline'),
+ ('noeol','noeol'),
+ ('noequalalways','noequalalways'),
+ ('noerrorbells','noerrorbells'),
+ ('noesckeys','noesckeys'),
+ ('noet','noet'),
+ ('noex','noex'),
+ ('noexpandtab','noexpandtab'),
+ ('noexrc','noexrc'),
+ ('nofen','nofen'),
+ ('nofic','nofic'),
+ ('nofileignorecase','nofileignorecase'),
+ ('nofk','nofk'),
+ ('nofkmap','nofkmap'),
+ ('nofoldenable','nofoldenable'),
+ ('nogd','nogd'),
+ ('nogdefault','nogdefault'),
+ ('noguipty','noguipty'),
+ ('nohid','nohid'),
+ ('nohidden','nohidden'),
+ ('nohk','nohk'),
+ ('nohkmap','nohkmap'),
+ ('nohkmapp','nohkmapp'),
+ ('nohkp','nohkp'),
+ ('nohls','nohls'),
+ ('nohlsearch','nohlsearch'),
+ ('noic','noic'),
+ ('noicon','noicon'),
+ ('noignorecase','noignorecase'),
+ ('noim','noim'),
+ ('noimc','noimc'),
+ ('noimcmdline','noimcmdline'),
+ ('noimd','noimd'),
+ ('noimdisable','noimdisable'),
+ ('noincsearch','noincsearch'),
+ ('noinf','noinf'),
+ ('noinfercase','noinfercase'),
+ ('noinsertmode','noinsertmode'),
+ ('nois','nois'),
+ ('nojoinspaces','nojoinspaces'),
+ ('nojs','nojs'),
+ ('nolazyredraw','nolazyredraw'),
+ ('nolbr','nolbr'),
+ ('nolinebreak','nolinebreak'),
+ ('nolisp','nolisp'),
+ ('nolist','nolist'),
+ ('noloadplugins','noloadplugins'),
+ ('nolpl','nolpl'),
+ ('nolz','nolz'),
+ ('noma','noma'),
+ ('nomacatsui','nomacatsui'),
+ ('nomagic','nomagic'),
+ ('nomh','nomh'),
+ ('noml','noml'),
+ ('nomod','nomod'),
+ ('nomodeline','nomodeline'),
+ ('nomodifiable','nomodifiable'),
+ ('nomodified','nomodified'),
+ ('nomore','nomore'),
+ ('nomousef','nomousef'),
+ ('nomousefocus','nomousefocus'),
+ ('nomousehide','nomousehide'),
+ ('nonu','nonu'),
+ ('nonumber','nonumber'),
+ ('noodev','noodev'),
+ ('noopendevice','noopendevice'),
+ ('nopaste','nopaste'),
+ ('nopi','nopi'),
+ ('nopreserveindent','nopreserveindent'),
+ ('nopreviewwindow','nopreviewwindow'),
+ ('noprompt','noprompt'),
+ ('nopvw','nopvw'),
+ ('noreadonly','noreadonly'),
+ ('norelativenumber','norelativenumber'),
+ ('noremap','noremap'),
+ ('norestorescreen','norestorescreen'),
+ ('norevins','norevins'),
+ ('nori','nori'),
+ ('norightleft','norightleft'),
+ ('norl','norl'),
+ ('nornu','nornu'),
+ ('noro','noro'),
+ ('nors','nors'),
+ ('noru','noru'),
+ ('noruler','noruler'),
+ ('nosb','nosb'),
+ ('nosc','nosc'),
+ ('noscb','noscb'),
+ ('noscrollbind','noscrollbind'),
+ ('noscs','noscs'),
+ ('nosecure','nosecure'),
+ ('nosft','nosft'),
+ ('noshellslash','noshellslash'),
+ ('noshelltemp','noshelltemp'),
+ ('noshiftround','noshiftround'),
+ ('noshortname','noshortname'),
+ ('noshowcmd','noshowcmd'),
+ ('noshowfulltag','noshowfulltag'),
+ ('noshowmatch','noshowmatch'),
+ ('noshowmode','noshowmode'),
+ ('nosi','nosi'),
+ ('nosm','nosm'),
+ ('nosmartcase','nosmartcase'),
+ ('nosmartindent','nosmartindent'),
+ ('nosmarttab','nosmarttab'),
+ ('nosmd','nosmd'),
+ ('nosn','nosn'),
+ ('nosol','nosol'),
+ ('nospell','nospell'),
+ ('nosplitbelow','nosplitbelow'),
+ ('nosplitright','nosplitright'),
+ ('nospr','nospr'),
+ ('nosr','nosr'),
+ ('nossl','nossl'),
+ ('nosta','nosta'),
+ ('nostartofline','nostartofline'),
+ ('nostmp','nostmp'),
+ ('noswapfile','noswapfile'),
+ ('noswf','noswf'),
+ ('nota','nota'),
+ ('notagbsearch','notagbsearch'),
+ ('notagrelative','notagrelative'),
+ ('notagstack','notagstack'),
+ ('notbi','notbi'),
+ ('notbidi','notbidi'),
+ ('notbs','notbs'),
+ ('notermbidi','notermbidi'),
+ ('noterse','noterse'),
+ ('notextauto','notextauto'),
+ ('notextmode','notextmode'),
+ ('notf','notf'),
+ ('notgst','notgst'),
+ ('notildeop','notildeop'),
+ ('notimeout','notimeout'),
+ ('notitle','notitle'),
+ ('noto','noto'),
+ ('notop','notop'),
+ ('notr','notr'),
+ ('nottimeout','nottimeout'),
+ ('nottybuiltin','nottybuiltin'),
+ ('nottyfast','nottyfast'),
+ ('notx','notx'),
+ ('noudf','noudf'),
+ ('noundofile','noundofile'),
+ ('novb','novb'),
+ ('novisualbell','novisualbell'),
+ ('nowa','nowa'),
+ ('nowarn','nowarn'),
+ ('nowb','nowb'),
+ ('noweirdinvert','noweirdinvert'),
+ ('nowfh','nowfh'),
+ ('nowfw','nowfw'),
+ ('nowic','nowic'),
+ ('nowildignorecase','nowildignorecase'),
+ ('nowildmenu','nowildmenu'),
+ ('nowinfixheight','nowinfixheight'),
+ ('nowinfixwidth','nowinfixwidth'),
+ ('nowiv','nowiv'),
+ ('nowmnu','nowmnu'),
+ ('nowrap','nowrap'),
+ ('nowrapscan','nowrapscan'),
+ ('nowrite','nowrite'),
+ ('nowriteany','nowriteany'),
+ ('nowritebackup','nowritebackup'),
+ ('nows','nows'),
+ ('nrformats','nrformats'),
+ ('nu','nu'),
+ ('number','number'),
+ ('numberwidth','numberwidth'),
+ ('nuw','nuw'),
+ ('odev','odev'),
+ ('oft','oft'),
+ ('ofu','ofu'),
+ ('omnifunc','omnifunc'),
+ ('opendevice','opendevice'),
+ ('operatorfunc','operatorfunc'),
+ ('opfunc','opfunc'),
+ ('osfiletype','osfiletype'),
+ ('pa','pa'),
+ ('para','para'),
+ ('paragraphs','paragraphs'),
+ ('paste','paste'),
+ ('pastetoggle','pastetoggle'),
+ ('patchexpr','patchexpr'),
+ ('patchmode','patchmode'),
+ ('path','path'),
+ ('pdev','pdev'),
+ ('penc','penc'),
+ ('pex','pex'),
+ ('pexpr','pexpr'),
+ ('pfn','pfn'),
+ ('ph','ph'),
+ ('pheader','pheader'),
+ ('pi','pi'),
+ ('pm','pm'),
+ ('pmbcs','pmbcs'),
+ ('pmbfn','pmbfn'),
+ ('popt','popt'),
+ ('preserveindent','preserveindent'),
+ ('previewheight','previewheight'),
+ ('previewwindow','previewwindow'),
+ ('printdevice','printdevice'),
+ ('printencoding','printencoding'),
+ ('printexpr','printexpr'),
+ ('printfont','printfont'),
+ ('printheader','printheader'),
+ ('printmbcharset','printmbcharset'),
+ ('printmbfont','printmbfont'),
+ ('printoptions','printoptions'),
+ ('prompt','prompt'),
+ ('pt','pt'),
+ ('pumheight','pumheight'),
+ ('pvh','pvh'),
+ ('pvw','pvw'),
+ ('qe','qe'),
+ ('quoteescape','quoteescape'),
+ ('rdt','rdt'),
+ ('re','re'),
+ ('readonly','readonly'),
+ ('redrawtime','redrawtime'),
+ ('regexpengine','regexpengine'),
+ ('relativenumber','relativenumber'),
+ ('remap','remap'),
+ ('report','report'),
+ ('restorescreen','restorescreen'),
+ ('revins','revins'),
+ ('ri','ri'),
+ ('rightleft','rightleft'),
+ ('rightleftcmd','rightleftcmd'),
+ ('rl','rl'),
+ ('rlc','rlc'),
+ ('rnu','rnu'),
+ ('ro','ro'),
+ ('rs','rs'),
+ ('rtp','rtp'),
+ ('ru','ru'),
+ ('ruf','ruf'),
+ ('ruler','ruler'),
+ ('rulerformat','rulerformat'),
+ ('runtimepath','runtimepath'),
+ ('sb','sb'),
+ ('sbo','sbo'),
+ ('sbr','sbr'),
+ ('sc','sc'),
+ ('scb','scb'),
+ ('scr','scr'),
+ ('scroll','scroll'),
+ ('scrollbind','scrollbind'),
+ ('scrolljump','scrolljump'),
+ ('scrolloff','scrolloff'),
+ ('scrollopt','scrollopt'),
+ ('scs','scs'),
+ ('sect','sect'),
+ ('sections','sections'),
+ ('secure','secure'),
+ ('sel','sel'),
+ ('selection','selection'),
+ ('selectmode','selectmode'),
+ ('sessionoptions','sessionoptions'),
+ ('sft','sft'),
+ ('sh','sh'),
+ ('shcf','shcf'),
+ ('shell','shell'),
+ ('shellcmdflag','shellcmdflag'),
+ ('shellpipe','shellpipe'),
+ ('shellquote','shellquote'),
+ ('shellredir','shellredir'),
+ ('shellslash','shellslash'),
+ ('shelltemp','shelltemp'),
+ ('shelltype','shelltype'),
+ ('shellxescape','shellxescape'),
+ ('shellxquote','shellxquote'),
+ ('shiftround','shiftround'),
+ ('shiftwidth','shiftwidth'),
+ ('shm','shm'),
+ ('shortmess','shortmess'),
+ ('shortname','shortname'),
+ ('showbreak','showbreak'),
+ ('showcmd','showcmd'),
+ ('showfulltag','showfulltag'),
+ ('showmatch','showmatch'),
+ ('showmode','showmode'),
+ ('showtabline','showtabline'),
+ ('shq','shq'),
+ ('si','si'),
+ ('sidescroll','sidescroll'),
+ ('sidescrolloff','sidescrolloff'),
+ ('siso','siso'),
+ ('sj','sj'),
+ ('slm','slm'),
+ ('sm','sm'),
+ ('smartcase','smartcase'),
+ ('smartindent','smartindent'),
+ ('smarttab','smarttab'),
+ ('smc','smc'),
+ ('smd','smd'),
+ ('sn','sn'),
+ ('so','so'),
+ ('softtabstop','softtabstop'),
+ ('sol','sol'),
+ ('sp','sp'),
+ ('spc','spc'),
+ ('spell','spell'),
+ ('spellcapcheck','spellcapcheck'),
+ ('spellfile','spellfile'),
+ ('spelllang','spelllang'),
+ ('spellsuggest','spellsuggest'),
+ ('spf','spf'),
+ ('spl','spl'),
+ ('splitbelow','splitbelow'),
+ ('splitright','splitright'),
+ ('spr','spr'),
+ ('sps','sps'),
+ ('sr','sr'),
+ ('srr','srr'),
+ ('ss','ss'),
+ ('ssl','ssl'),
+ ('ssop','ssop'),
+ ('st','st'),
+ ('sta','sta'),
+ ('stal','stal'),
+ ('startofline','startofline'),
+ ('statusline','statusline'),
+ ('stl','stl'),
+ ('stmp','stmp'),
+ ('sts','sts'),
+ ('su','su'),
+ ('sua','sua'),
+ ('suffixes','suffixes'),
+ ('suffixesadd','suffixesadd'),
+ ('sw','sw'),
+ ('swapfile','swapfile'),
+ ('swapsync','swapsync'),
+ ('swb','swb'),
+ ('swf','swf'),
+ ('switchbuf','switchbuf'),
+ ('sws','sws'),
+ ('sxe','sxe'),
+ ('sxq','sxq'),
+ ('syn','syn'),
+ ('synmaxcol','synmaxcol'),
+ ('syntax','syntax'),
+ ('t_AB','t_AB'),
+ ('t_AF','t_AF'),
+ ('t_AL','t_AL'),
+ ('t_CS','t_CS'),
+ ('t_CV','t_CV'),
+ ('t_Ce','t_Ce'),
+ ('t_Co','t_Co'),
+ ('t_Cs','t_Cs'),
+ ('t_DL','t_DL'),
+ ('t_EI','t_EI'),
+ ('t_F1','t_F1'),
+ ('t_F2','t_F2'),
+ ('t_F3','t_F3'),
+ ('t_F4','t_F4'),
+ ('t_F5','t_F5'),
+ ('t_F6','t_F6'),
+ ('t_F7','t_F7'),
+ ('t_F8','t_F8'),
+ ('t_F9','t_F9'),
+ ('t_IE','t_IE'),
+ ('t_IS','t_IS'),
+ ('t_K1','t_K1'),
+ ('t_K3','t_K3'),
+ ('t_K4','t_K4'),
+ ('t_K5','t_K5'),
+ ('t_K6','t_K6'),
+ ('t_K7','t_K7'),
+ ('t_K8','t_K8'),
+ ('t_K9','t_K9'),
+ ('t_KA','t_KA'),
+ ('t_KB','t_KB'),
+ ('t_KC','t_KC'),
+ ('t_KD','t_KD'),
+ ('t_KE','t_KE'),
+ ('t_KF','t_KF'),
+ ('t_KG','t_KG'),
+ ('t_KH','t_KH'),
+ ('t_KI','t_KI'),
+ ('t_KJ','t_KJ'),
+ ('t_KK','t_KK'),
+ ('t_KL','t_KL'),
+ ('t_RI','t_RI'),
+ ('t_RV','t_RV'),
+ ('t_SI','t_SI'),
+ ('t_Sb','t_Sb'),
+ ('t_Sf','t_Sf'),
+ ('t_WP','t_WP'),
+ ('t_WS','t_WS'),
+ ('t_ZH','t_ZH'),
+ ('t_ZR','t_ZR'),
+ ('t_al','t_al'),
+ ('t_bc','t_bc'),
+ ('t_cd','t_cd'),
+ ('t_ce','t_ce'),
+ ('t_cl','t_cl'),
+ ('t_cm','t_cm'),
+ ('t_cs','t_cs'),
+ ('t_da','t_da'),
+ ('t_db','t_db'),
+ ('t_dl','t_dl'),
+ ('t_fs','t_fs'),
+ ('t_k1','t_k1'),
+ ('t_k2','t_k2'),
+ ('t_k3','t_k3'),
+ ('t_k4','t_k4'),
+ ('t_k5','t_k5'),
+ ('t_k6','t_k6'),
+ ('t_k7','t_k7'),
+ ('t_k8','t_k8'),
+ ('t_k9','t_k9'),
+ ('t_kB','t_kB'),
+ ('t_kD','t_kD'),
+ ('t_kI','t_kI'),
+ ('t_kN','t_kN'),
+ ('t_kP','t_kP'),
+ ('t_kb','t_kb'),
+ ('t_kd','t_kd'),
+ ('t_ke','t_ke'),
+ ('t_kh','t_kh'),
+ ('t_kl','t_kl'),
+ ('t_kr','t_kr'),
+ ('t_ks','t_ks'),
+ ('t_ku','t_ku'),
+ ('t_le','t_le'),
+ ('t_mb','t_mb'),
+ ('t_md','t_md'),
+ ('t_me','t_me'),
+ ('t_mr','t_mr'),
+ ('t_ms','t_ms'),
+ ('t_nd','t_nd'),
+ ('t_op','t_op'),
+ ('t_se','t_se'),
+ ('t_so','t_so'),
+ ('t_sr','t_sr'),
+ ('t_te','t_te'),
+ ('t_ti','t_ti'),
+ ('t_ts','t_ts'),
+ ('t_u7','t_u7'),
+ ('t_ue','t_ue'),
+ ('t_us','t_us'),
+ ('t_ut','t_ut'),
+ ('t_vb','t_vb'),
+ ('t_ve','t_ve'),
+ ('t_vi','t_vi'),
+ ('t_vs','t_vs'),
+ ('t_xs','t_xs'),
+ ('ta','ta'),
+ ('tabline','tabline'),
+ ('tabpagemax','tabpagemax'),
+ ('tabstop','tabstop'),
+ ('tag','tag'),
+ ('tagbsearch','tagbsearch'),
+ ('taglength','taglength'),
+ ('tagrelative','tagrelative'),
+ ('tags','tags'),
+ ('tagstack','tagstack'),
+ ('tal','tal'),
+ ('tb','tb'),
+ ('tbi','tbi'),
+ ('tbidi','tbidi'),
+ ('tbis','tbis'),
+ ('tbs','tbs'),
+ ('tenc','tenc'),
+ ('term','term'),
+ ('termbidi','termbidi'),
+ ('termencoding','termencoding'),
+ ('terse','terse'),
+ ('textauto','textauto'),
+ ('textmode','textmode'),
+ ('textwidth','textwidth'),
+ ('tf','tf'),
+ ('tgst','tgst'),
+ ('thesaurus','thesaurus'),
+ ('tildeop','tildeop'),
+ ('timeout','timeout'),
+ ('timeoutlen','timeoutlen'),
+ ('title','title'),
+ ('titlelen','titlelen'),
+ ('titleold','titleold'),
+ ('titlestring','titlestring'),
+ ('tl','tl'),
+ ('tm','tm'),
+ ('to','to'),
+ ('toolbar','toolbar'),
+ ('toolbariconsize','toolbariconsize'),
+ ('top','top'),
+ ('tpm','tpm'),
+ ('tr','tr'),
+ ('ts','ts'),
+ ('tsl','tsl'),
+ ('tsr','tsr'),
+ ('ttimeout','ttimeout'),
+ ('ttimeoutlen','ttimeoutlen'),
+ ('ttm','ttm'),
+ ('tty','tty'),
+ ('ttybuiltin','ttybuiltin'),
+ ('ttyfast','ttyfast'),
+ ('ttym','ttym'),
+ ('ttymouse','ttymouse'),
+ ('ttyscroll','ttyscroll'),
+ ('ttytype','ttytype'),
+ ('tw','tw'),
+ ('tx','tx'),
+ ('uc','uc'),
+ ('udf','udf'),
+ ('udir','udir'),
+ ('ul','ul'),
+ ('undodir','undodir'),
+ ('undofile','undofile'),
+ ('undolevels','undolevels'),
+ ('undoreload','undoreload'),
+ ('updatecount','updatecount'),
+ ('updatetime','updatetime'),
+ ('ur','ur'),
+ ('ut','ut'),
+ ('vb','vb'),
+ ('vbs','vbs'),
+ ('vdir','vdir'),
+ ('ve','ve'),
+ ('verbose','verbose'),
+ ('verbosefile','verbosefile'),
+ ('vfile','vfile'),
+ ('vi','vi'),
+ ('viewdir','viewdir'),
+ ('viewoptions','viewoptions'),
+ ('viminfo','viminfo'),
+ ('virtualedit','virtualedit'),
+ ('visualbell','visualbell'),
+ ('vnoremap','vnoremap'),
+ ('vop','vop'),
+ ('wa','wa'),
+ ('wak','wak'),
+ ('warn','warn'),
+ ('wb','wb'),
+ ('wc','wc'),
+ ('wcm','wcm'),
+ ('wd','wd'),
+ ('weirdinvert','weirdinvert'),
+ ('wfh','wfh'),
+ ('wfw','wfw'),
+ ('wh','wh'),
+ ('whichwrap','whichwrap'),
+ ('wi','wi'),
+ ('wic','wic'),
+ ('wig','wig'),
+ ('wildchar','wildchar'),
+ ('wildcharm','wildcharm'),
+ ('wildignore','wildignore'),
+ ('wildignorecase','wildignorecase'),
+ ('wildmenu','wildmenu'),
+ ('wildmode','wildmode'),
+ ('wildoptions','wildoptions'),
+ ('wim','wim'),
+ ('winaltkeys','winaltkeys'),
+ ('window','window'),
+ ('winfixheight','winfixheight'),
+ ('winfixwidth','winfixwidth'),
+ ('winheight','winheight'),
+ ('winminheight','winminheight'),
+ ('winminwidth','winminwidth'),
+ ('winwidth','winwidth'),
+ ('wiv','wiv'),
+ ('wiw','wiw'),
+ ('wm','wm'),
+ ('wmh','wmh'),
+ ('wmnu','wmnu'),
+ ('wmw','wmw'),
+ ('wop','wop'),
+ ('wrap','wrap'),
+ ('wrapmargin','wrapmargin'),
+ ('wrapscan','wrapscan'),
+ ('write','write'),
+ ('writeany','writeany'),
+ ('writebackup','writebackup'),
+ ('writedelay','writedelay'),
+ ('ws','ws'),
+ ('ww','ww'),
+ )
+ return var
+option = _getoption()
+
diff --git a/wandb/vendor/pygments/lexers/actionscript.py b/wandb/vendor/pygments/lexers/actionscript.py
new file mode 100644
index 0000000000000000000000000000000000000000..84607e684bd0d0cef2cbabd9f7296470adc603bc
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/actionscript.py
@@ -0,0 +1,240 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.actionscript
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for ActionScript and MXML.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, using, this, words, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['ActionScriptLexer', 'ActionScript3Lexer', 'MxmlLexer']
+
+
+class ActionScriptLexer(RegexLexer):
+ """
+ For ActionScript source code.
+
+ .. versionadded:: 0.9
+ """
+
+ name = 'ActionScript'
+ aliases = ['as', 'actionscript']
+ filenames = ['*.as']
+ mimetypes = ['application/x-actionscript', 'text/x-actionscript',
+ 'text/actionscript']
+
+ flags = re.DOTALL
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'/(\\\\|\\/|[^/\n])*/[gim]*', String.Regex),
+ (r'[~^*!%&<>|+=:;,/?\\-]+', Operator),
+ (r'[{}\[\]();.]+', Punctuation),
+ (words((
+ 'case', 'default', 'for', 'each', 'in', 'while', 'do', 'break',
+ 'return', 'continue', 'if', 'else', 'throw', 'try', 'catch',
+ 'var', 'with', 'new', 'typeof', 'arguments', 'instanceof', 'this',
+ 'switch'), suffix=r'\b'),
+ Keyword),
+ (words((
+ 'class', 'public', 'final', 'internal', 'native', 'override', 'private',
+ 'protected', 'static', 'import', 'extends', 'implements', 'interface',
+ 'intrinsic', 'return', 'super', 'dynamic', 'function', 'const', 'get',
+ 'namespace', 'package', 'set'), suffix=r'\b'),
+ Keyword.Declaration),
+ (r'(true|false|null|NaN|Infinity|-Infinity|undefined|Void)\b',
+ Keyword.Constant),
+ (words((
+ 'Accessibility', 'AccessibilityProperties', 'ActionScriptVersion',
+ 'ActivityEvent', 'AntiAliasType', 'ApplicationDomain', 'AsBroadcaster', 'Array',
+ 'AsyncErrorEvent', 'AVM1Movie', 'BevelFilter', 'Bitmap', 'BitmapData',
+ 'BitmapDataChannel', 'BitmapFilter', 'BitmapFilterQuality', 'BitmapFilterType',
+ 'BlendMode', 'BlurFilter', 'Boolean', 'ByteArray', 'Camera', 'Capabilities', 'CapsStyle',
+ 'Class', 'Color', 'ColorMatrixFilter', 'ColorTransform', 'ContextMenu',
+ 'ContextMenuBuiltInItems', 'ContextMenuEvent', 'ContextMenuItem',
+ 'ConvultionFilter', 'CSMSettings', 'DataEvent', 'Date', 'DefinitionError',
+ 'DeleteObjectSample', 'Dictionary', 'DisplacmentMapFilter', 'DisplayObject',
+ 'DisplacmentMapFilterMode', 'DisplayObjectContainer', 'DropShadowFilter',
+ 'Endian', 'EOFError', 'Error', 'ErrorEvent', 'EvalError', 'Event', 'EventDispatcher',
+ 'EventPhase', 'ExternalInterface', 'FileFilter', 'FileReference',
+ 'FileReferenceList', 'FocusDirection', 'FocusEvent', 'Font', 'FontStyle', 'FontType',
+ 'FrameLabel', 'FullScreenEvent', 'Function', 'GlowFilter', 'GradientBevelFilter',
+ 'GradientGlowFilter', 'GradientType', 'Graphics', 'GridFitType', 'HTTPStatusEvent',
+ 'IBitmapDrawable', 'ID3Info', 'IDataInput', 'IDataOutput', 'IDynamicPropertyOutput'
+ 'IDynamicPropertyWriter', 'IEventDispatcher', 'IExternalizable',
+ 'IllegalOperationError', 'IME', 'IMEConversionMode', 'IMEEvent', 'int',
+ 'InteractiveObject', 'InterpolationMethod', 'InvalidSWFError', 'InvokeEvent',
+ 'IOError', 'IOErrorEvent', 'JointStyle', 'Key', 'Keyboard', 'KeyboardEvent', 'KeyLocation',
+ 'LineScaleMode', 'Loader', 'LoaderContext', 'LoaderInfo', 'LoadVars', 'LocalConnection',
+ 'Locale', 'Math', 'Matrix', 'MemoryError', 'Microphone', 'MorphShape', 'Mouse', 'MouseEvent',
+ 'MovieClip', 'MovieClipLoader', 'Namespace', 'NetConnection', 'NetStatusEvent',
+ 'NetStream', 'NewObjectSample', 'Number', 'Object', 'ObjectEncoding', 'PixelSnapping',
+ 'Point', 'PrintJob', 'PrintJobOptions', 'PrintJobOrientation', 'ProgressEvent', 'Proxy',
+ 'QName', 'RangeError', 'Rectangle', 'ReferenceError', 'RegExp', 'Responder', 'Sample',
+ 'Scene', 'ScriptTimeoutError', 'Security', 'SecurityDomain', 'SecurityError',
+ 'SecurityErrorEvent', 'SecurityPanel', 'Selection', 'Shape', 'SharedObject',
+ 'SharedObjectFlushStatus', 'SimpleButton', 'Socket', 'Sound', 'SoundChannel',
+ 'SoundLoaderContext', 'SoundMixer', 'SoundTransform', 'SpreadMethod', 'Sprite',
+ 'StackFrame', 'StackOverflowError', 'Stage', 'StageAlign', 'StageDisplayState',
+ 'StageQuality', 'StageScaleMode', 'StaticText', 'StatusEvent', 'String', 'StyleSheet',
+ 'SWFVersion', 'SyncEvent', 'SyntaxError', 'System', 'TextColorType', 'TextField',
+ 'TextFieldAutoSize', 'TextFieldType', 'TextFormat', 'TextFormatAlign',
+ 'TextLineMetrics', 'TextRenderer', 'TextSnapshot', 'Timer', 'TimerEvent', 'Transform',
+ 'TypeError', 'uint', 'URIError', 'URLLoader', 'URLLoaderDataFormat', 'URLRequest',
+ 'URLRequestHeader', 'URLRequestMethod', 'URLStream', 'URLVariabeles', 'VerifyError',
+ 'Video', 'XML', 'XMLDocument', 'XMLList', 'XMLNode', 'XMLNodeType', 'XMLSocket',
+ 'XMLUI'), suffix=r'\b'),
+ Name.Builtin),
+ (words((
+ 'decodeURI', 'decodeURIComponent', 'encodeURI', 'escape', 'eval', 'isFinite', 'isNaN',
+ 'isXMLName', 'clearInterval', 'fscommand', 'getTimer', 'getURL', 'getVersion',
+ 'parseFloat', 'parseInt', 'setInterval', 'trace', 'updateAfterEvent',
+ 'unescape'), suffix=r'\b'),
+ Name.Function),
+ (r'[$a-zA-Z_]\w*', Name.Other),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-f]+', Number.Hex),
+ (r'[0-9]+', Number.Integer),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ ]
+ }
+
+
+class ActionScript3Lexer(RegexLexer):
+ """
+ For ActionScript 3 source code.
+
+ .. versionadded:: 0.11
+ """
+
+ name = 'ActionScript 3'
+ aliases = ['as3', 'actionscript3']
+ filenames = ['*.as']
+ mimetypes = ['application/x-actionscript3', 'text/x-actionscript3',
+ 'text/actionscript3']
+
+ identifier = r'[$a-zA-Z_]\w*'
+ typeidentifier = identifier + '(?:\.<\w+>)?'
+
+ flags = re.DOTALL | re.MULTILINE
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'(function\s+)(' + identifier + r')(\s*)(\()',
+ bygroups(Keyword.Declaration, Name.Function, Text, Operator),
+ 'funcparams'),
+ (r'(var|const)(\s+)(' + identifier + r')(\s*)(:)(\s*)(' +
+ typeidentifier + r')',
+ bygroups(Keyword.Declaration, Text, Name, Text, Punctuation, Text,
+ Keyword.Type)),
+ (r'(import|package)(\s+)((?:' + identifier + r'|\.)+)(\s*)',
+ bygroups(Keyword, Text, Name.Namespace, Text)),
+ (r'(new)(\s+)(' + typeidentifier + r')(\s*)(\()',
+ bygroups(Keyword, Text, Keyword.Type, Text, Operator)),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'/(\\\\|\\/|[^\n])*/[gisx]*', String.Regex),
+ (r'(\.)(' + identifier + r')', bygroups(Operator, Name.Attribute)),
+ (r'(case|default|for|each|in|while|do|break|return|continue|if|else|'
+ r'throw|try|catch|with|new|typeof|arguments|instanceof|this|'
+ r'switch|import|include|as|is)\b',
+ Keyword),
+ (r'(class|public|final|internal|native|override|private|protected|'
+ r'static|import|extends|implements|interface|intrinsic|return|super|'
+ r'dynamic|function|const|get|namespace|package|set)\b',
+ Keyword.Declaration),
+ (r'(true|false|null|NaN|Infinity|-Infinity|undefined|void)\b',
+ Keyword.Constant),
+ (r'(decodeURI|decodeURIComponent|encodeURI|escape|eval|isFinite|isNaN|'
+ r'isXMLName|clearInterval|fscommand|getTimer|getURL|getVersion|'
+ r'isFinite|parseFloat|parseInt|setInterval|trace|updateAfterEvent|'
+ r'unescape)\b', Name.Function),
+ (identifier, Name),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-f]+', Number.Hex),
+ (r'[0-9]+', Number.Integer),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ (r'[~^*!%&<>|+=:;,/?\\{}\[\]().-]+', Operator),
+ ],
+ 'funcparams': [
+ (r'\s+', Text),
+ (r'(\s*)(\.\.\.)?(' + identifier + r')(\s*)(:)(\s*)(' +
+ typeidentifier + r'|\*)(\s*)',
+ bygroups(Text, Punctuation, Name, Text, Operator, Text,
+ Keyword.Type, Text), 'defval'),
+ (r'\)', Operator, 'type')
+ ],
+ 'type': [
+ (r'(\s*)(:)(\s*)(' + typeidentifier + r'|\*)',
+ bygroups(Text, Operator, Text, Keyword.Type), '#pop:2'),
+ (r'\s+', Text, '#pop:2'),
+ default('#pop:2')
+ ],
+ 'defval': [
+ (r'(=)(\s*)([^(),]+)(\s*)(,?)',
+ bygroups(Operator, Text, using(this), Text, Operator), '#pop'),
+ (r',', Operator, '#pop'),
+ default('#pop')
+ ]
+ }
+
+ def analyse_text(text):
+ if re.match(r'\w+\s*:\s*\w', text):
+ return 0.3
+ return 0
+
+
+class MxmlLexer(RegexLexer):
+ """
+ For MXML markup.
+ Nested AS3 in <script> tags is highlighted by the appropriate lexer.
+
+ .. versionadded:: 1.1
+ """
+ flags = re.MULTILINE | re.DOTALL
+ name = 'MXML'
+ aliases = ['mxml']
+ filenames = ['*.mxml']
+ mimetimes = ['text/xml', 'application/xml']
+
+ tokens = {
+ 'root': [
+ ('[^<&]+', Text),
+ (r'&\S*?;', Name.Entity),
+ (r'(\<\!\[CDATA\[)(.*?)(\]\]\>)',
+ bygroups(String, using(ActionScript3Lexer), String)),
+ ('<!--', Comment, 'comment'),
+ (r'<\?.*?\?>', Comment.Preproc),
+ ('<![^>]*>', Comment.Preproc),
+ (r'<\s*[\w:.-]+', Name.Tag, 'tag'),
+ (r'<\s*/\s*[\w:.-]+\s*>', Name.Tag),
+ ],
+ 'comment': [
+ ('[^-]+', Comment),
+ ('-->', Comment, '#pop'),
+ ('-', Comment),
+ ],
+ 'tag': [
+ (r'\s+', Text),
+ (r'[\w.:-]+\s*=', Name.Attribute, 'attr'),
+ (r'/?\s*>', Name.Tag, '#pop'),
+ ],
+ 'attr': [
+ ('\s+', Text),
+ ('".*?"', String, '#pop'),
+ ("'.*?'", String, '#pop'),
+ (r'[^\s>]+', String, '#pop'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/agile.py b/wandb/vendor/pygments/lexers/agile.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb200b9e30bfb3eaba6142fba86995a4cf58808b
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/agile.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.agile
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Just export lexer classes previously contained in this module.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexers.lisp import SchemeLexer
+from pygments.lexers.jvm import IokeLexer, ClojureLexer
+from pygments.lexers.python import PythonLexer, PythonConsoleLexer, \
+ PythonTracebackLexer, Python3Lexer, Python3TracebackLexer, DgLexer
+from pygments.lexers.ruby import RubyLexer, RubyConsoleLexer, FancyLexer
+from pygments.lexers.perl import PerlLexer, Perl6Lexer
+from pygments.lexers.d import CrocLexer, MiniDLexer
+from pygments.lexers.iolang import IoLexer
+from pygments.lexers.tcl import TclLexer
+from pygments.lexers.factor import FactorLexer
+from pygments.lexers.scripting import LuaLexer, MoonScriptLexer
+
+__all__ = []
diff --git a/wandb/vendor/pygments/lexers/algebra.py b/wandb/vendor/pygments/lexers/algebra.py
new file mode 100644
index 0000000000000000000000000000000000000000..15d688422426c91e47effa07de02e005de094fd7
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/algebra.py
@@ -0,0 +1,221 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.algebra
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for computer algebra systems.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['GAPLexer', 'MathematicaLexer', 'MuPADLexer', 'BCLexer']
+
+
+class GAPLexer(RegexLexer):
+ """
+ For `GAP <http://www.gap-system.org>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+ name = 'GAP'
+ aliases = ['gap']
+ filenames = ['*.g', '*.gd', '*.gi', '*.gap']
+
+ tokens = {
+ 'root': [
+ (r'#.*$', Comment.Single),
+ (r'"(?:[^"\\]|\\.)*"', String),
+ (r'\(|\)|\[|\]|\{|\}', Punctuation),
+ (r'''(?x)\b(?:
+ if|then|elif|else|fi|
+ for|while|do|od|
+ repeat|until|
+ break|continue|
+ function|local|return|end|
+ rec|
+ quit|QUIT|
+ IsBound|Unbind|
+ TryNextMethod|
+ Info|Assert
+ )\b''', Keyword),
+ (r'''(?x)\b(?:
+ true|false|fail|infinity
+ )\b''',
+ Name.Constant),
+ (r'''(?x)\b(?:
+ (Declare|Install)([A-Z][A-Za-z]+)|
+ BindGlobal|BIND_GLOBAL
+ )\b''',
+ Name.Builtin),
+ (r'\.|,|:=|;|=|\+|-|\*|/|\^|>|<', Operator),
+ (r'''(?x)\b(?:
+ and|or|not|mod|in
+ )\b''',
+ Operator.Word),
+ (r'''(?x)
+ (?:\w+|`[^`]*`)
+ (?:::\w+|`[^`]*`)*''', Name.Variable),
+ (r'[0-9]+(?:\.[0-9]*)?(?:e[0-9]+)?', Number),
+ (r'\.[0-9]+(?:e[0-9]+)?', Number),
+ (r'.', Text)
+ ],
+ }
+
+
+class MathematicaLexer(RegexLexer):
+ """
+ Lexer for `Mathematica <http://www.wolfram.com/mathematica/>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Mathematica'
+ aliases = ['mathematica', 'mma', 'nb']
+ filenames = ['*.nb', '*.cdf', '*.nbp', '*.ma']
+ mimetypes = ['application/mathematica',
+ 'application/vnd.wolfram.mathematica',
+ 'application/vnd.wolfram.mathematica.package',
+ 'application/vnd.wolfram.cdf']
+
+ # http://reference.wolfram.com/mathematica/guide/Syntax.html
+ operators = (
+ ";;", "=", "=.", "!=" "==", ":=", "->", ":>", "/.", "+", "-", "*", "/",
+ "^", "&&", "||", "!", "<>", "|", "/;", "?", "@", "//", "/@", "@@",
+ "@@@", "~~", "===", "&", "<", ">", "<=", ">=",
+ )
+
+ punctuation = (",", ";", "(", ")", "[", "]", "{", "}")
+
+ def _multi_escape(entries):
+ return '(%s)' % ('|'.join(re.escape(entry) for entry in entries))
+
+ tokens = {
+ 'root': [
+ (r'(?s)\(\*.*?\*\)', Comment),
+
+ (r'([a-zA-Z]+[A-Za-z0-9]*`)', Name.Namespace),
+ (r'([A-Za-z0-9]*_+[A-Za-z0-9]*)', Name.Variable),
+ (r'#\d*', Name.Variable),
+ (r'([a-zA-Z]+[a-zA-Z0-9]*)', Name),
+
+ (r'-?\d+\.\d*', Number.Float),
+ (r'-?\d*\.\d+', Number.Float),
+ (r'-?\d+', Number.Integer),
+
+ (words(operators), Operator),
+ (words(punctuation), Punctuation),
+ (r'".*?"', String),
+ (r'\s+', Text.Whitespace),
+ ],
+ }
+
+
+class MuPADLexer(RegexLexer):
+ """
+ A `MuPAD <http://www.mupad.com>`_ lexer.
+ Contributed by Christopher Creutzig <christopher@creutzig.de>.
+
+ .. versionadded:: 0.8
+ """
+ name = 'MuPAD'
+ aliases = ['mupad']
+ filenames = ['*.mu']
+
+ tokens = {
+ 'root': [
+ (r'//.*?$', Comment.Single),
+ (r'/\*', Comment.Multiline, 'comment'),
+ (r'"(?:[^"\\]|\\.)*"', String),
+ (r'\(|\)|\[|\]|\{|\}', Punctuation),
+ (r'''(?x)\b(?:
+ next|break|end|
+ axiom|end_axiom|category|end_category|domain|end_domain|inherits|
+ if|%if|then|elif|else|end_if|
+ case|of|do|otherwise|end_case|
+ while|end_while|
+ repeat|until|end_repeat|
+ for|from|to|downto|step|end_for|
+ proc|local|option|save|begin|end_proc|
+ delete|frame
+ )\b''', Keyword),
+ (r'''(?x)\b(?:
+ DOM_ARRAY|DOM_BOOL|DOM_COMPLEX|DOM_DOMAIN|DOM_EXEC|DOM_EXPR|
+ DOM_FAIL|DOM_FLOAT|DOM_FRAME|DOM_FUNC_ENV|DOM_HFARRAY|DOM_IDENT|
+ DOM_INT|DOM_INTERVAL|DOM_LIST|DOM_NIL|DOM_NULL|DOM_POLY|DOM_PROC|
+ DOM_PROC_ENV|DOM_RAT|DOM_SET|DOM_STRING|DOM_TABLE|DOM_VAR
+ )\b''', Name.Class),
+ (r'''(?x)\b(?:
+ PI|EULER|E|CATALAN|
+ NIL|FAIL|undefined|infinity|
+ TRUE|FALSE|UNKNOWN
+ )\b''',
+ Name.Constant),
+ (r'\b(?:dom|procname)\b', Name.Builtin.Pseudo),
+ (r'\.|,|:|;|=|\+|-|\*|/|\^|@|>|<|\$|\||!|\'|%|~=', Operator),
+ (r'''(?x)\b(?:
+ and|or|not|xor|
+ assuming|
+ div|mod|
+ union|minus|intersect|in|subset
+ )\b''',
+ Operator.Word),
+ (r'\b(?:I|RDN_INF|RD_NINF|RD_NAN)\b', Number),
+ # (r'\b(?:adt|linalg|newDomain|hold)\b', Name.Builtin),
+ (r'''(?x)
+ ((?:[a-zA-Z_#][\w#]*|`[^`]*`)
+ (?:::[a-zA-Z_#][\w#]*|`[^`]*`)*)(\s*)([(])''',
+ bygroups(Name.Function, Text, Punctuation)),
+ (r'''(?x)
+ (?:[a-zA-Z_#][\w#]*|`[^`]*`)
+ (?:::[a-zA-Z_#][\w#]*|`[^`]*`)*''', Name.Variable),
+ (r'[0-9]+(?:\.[0-9]*)?(?:e[0-9]+)?', Number),
+ (r'\.[0-9]+(?:e[0-9]+)?', Number),
+ (r'.', Text)
+ ],
+ 'comment': [
+ (r'[^*/]', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline)
+ ],
+ }
+
+
+class BCLexer(RegexLexer):
+ """
+ A `BC <https://www.gnu.org/software/bc/>`_ lexer.
+
+ .. versionadded:: 2.1
+ """
+ name = 'BC'
+ aliases = ['bc']
+ filenames = ['*.bc']
+
+ tokens = {
+ 'root': [
+ (r'/\*', Comment.Multiline, 'comment'),
+ (r'"(?:[^"\\]|\\.)*"', String),
+ (r'[{}();,]', Punctuation),
+ (words(('if', 'else', 'while', 'for', 'break', 'continue',
+ 'halt', 'return', 'define', 'auto', 'print', 'read',
+ 'length', 'scale', 'sqrt', 'limits', 'quit',
+ 'warranty'), suffix=r'\b'), Keyword),
+ (r'\+\+|--|\|\||&&|'
+ r'([-<>+*%\^/!=])=?', Operator),
+ # bc doesn't support exponential
+ (r'[0-9]+(\.[0-9]*)?', Number),
+ (r'\.[0-9]+', Number),
+ (r'.', Text)
+ ],
+ 'comment': [
+ (r'[^*/]+', Comment.Multiline),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline)
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/ambient.py b/wandb/vendor/pygments/lexers/ambient.py
new file mode 100644
index 0000000000000000000000000000000000000000..53f3a5e127aa3eb6de27d8da22ae741ad123533b
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/ambient.py
@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.ambient
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for AmbientTalk language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['AmbientTalkLexer']
+
+
+class AmbientTalkLexer(RegexLexer):
+ """
+ Lexer for `AmbientTalk <https://code.google.com/p/ambienttalk>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+ name = 'AmbientTalk'
+ filenames = ['*.at']
+ aliases = ['at', 'ambienttalk', 'ambienttalk/2']
+ mimetypes = ['text/x-ambienttalk']
+
+ flags = re.MULTILINE | re.DOTALL
+
+ builtin = words(('if:', 'then:', 'else:', 'when:', 'whenever:', 'discovered:',
+ 'disconnected:', 'reconnected:', 'takenOffline:', 'becomes:',
+ 'export:', 'as:', 'object:', 'actor:', 'mirror:', 'taggedAs:',
+ 'mirroredBy:', 'is:'))
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'(def|deftype|import|alias|exclude)\b', Keyword),
+ (builtin, Name.Builtin),
+ (r'(true|false|nil)\b', Keyword.Constant),
+ (r'(~|lobby|jlobby|/)\.', Keyword.Constant, 'namespace'),
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r'\|', Punctuation, 'arglist'),
+ (r'<:|[*^!%&<>+=,./?-]|:=', Operator),
+ (r"`[a-zA-Z_]\w*", String.Symbol),
+ (r"[a-zA-Z_]\w*:", Name.Function),
+ (r"[{}()\[\];`]", Punctuation),
+ (r'(self|super)\b', Name.Variable.Instance),
+ (r"[a-zA-Z_]\w*", Name.Variable),
+ (r"@[a-zA-Z_]\w*", Name.Class),
+ (r"@\[", Name.Class, 'annotations'),
+ include('numbers'),
+ ],
+ 'numbers': [
+ (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+ (r'\d+', Number.Integer)
+ ],
+ 'namespace': [
+ (r'[a-zA-Z_]\w*\.', Name.Namespace),
+ (r'[a-zA-Z_]\w*:', Name.Function, '#pop'),
+ (r'[a-zA-Z_]\w*(?!\.)', Name.Function, '#pop')
+ ],
+ 'annotations': [
+ (r"(.*?)\]", Name.Class, '#pop')
+ ],
+ 'arglist': [
+ (r'\|', Punctuation, '#pop'),
+ (r'\s*(,)\s*', Punctuation),
+ (r'[a-zA-Z_]\w*', Name.Variable),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/ampl.py b/wandb/vendor/pygments/lexers/ampl.py
new file mode 100644
index 0000000000000000000000000000000000000000..d439cb19c55a822f6ff976012000f3a630fae472
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/ampl.py
@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.ampl
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the ampl language. <http://ampl.com/>
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, using, this, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['AmplLexer']
+
+
+class AmplLexer(RegexLexer):
+ """
+ For AMPL source code.
+
+ .. versionadded:: 2.2
+ """
+ name = 'Ampl'
+ aliases = ['ampl']
+ filenames = ['*.run']
+
+ tokens = {
+ 'root': [
+ (r'\n', Text),
+ (r'\s+', Text.Whitespace),
+ (r'#.*?\n', Comment.Single),
+ (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+ (words((
+ 'call', 'cd', 'close', 'commands', 'data', 'delete', 'display',
+ 'drop', 'end', 'environ', 'exit', 'expand', 'include', 'load',
+ 'model', 'objective', 'option', 'problem', 'purge', 'quit',
+ 'redeclare', 'reload', 'remove', 'reset', 'restore', 'shell',
+ 'show', 'solexpand', 'solution', 'solve', 'update', 'unload',
+ 'xref', 'coeff', 'coef', 'cover', 'obj', 'interval', 'default',
+ 'from', 'to', 'to_come', 'net_in', 'net_out', 'dimen',
+ 'dimension', 'check', 'complements', 'write', 'function',
+ 'pipe', 'format', 'if', 'then', 'else', 'in', 'while', 'repeat',
+ 'for'), suffix=r'\b'), Keyword.Reserved),
+ (r'(integer|binary|symbolic|ordered|circular|reversed|INOUT|IN|OUT|LOCAL)',
+ Keyword.Type),
+ (r'\".*?\"', String.Double),
+ (r'\'.*?\'', String.Single),
+ (r'[()\[\]{},;:]+', Punctuation),
+ (r'\b(\w+)(\.)(astatus|init0|init|lb0|lb1|lb2|lb|lrc|'
+ r'lslack|rc|relax|slack|sstatus|status|ub0|ub1|ub2|'
+ r'ub|urc|uslack|val)',
+ bygroups(Name.Variable, Punctuation, Keyword.Reserved)),
+ (r'(set|param|var|arc|minimize|maximize|subject to|s\.t\.|subj to|'
+ r'node|table|suffix|read table|write table)(\s+)(\w+)',
+ bygroups(Keyword.Declaration, Text, Name.Variable)),
+ (r'(param)(\s*)(:)(\s*)(\w+)(\s*)(:)(\s*)((\w|\s)+)',
+ bygroups(Keyword.Declaration, Text, Punctuation, Text,
+ Name.Variable, Text, Punctuation, Text, Name.Variable)),
+ (r'(let|fix|unfix)(\s*)((?:\{.*\})?)(\s*)(\w+)',
+ bygroups(Keyword.Declaration, Text, using(this), Text, Name.Variable)),
+ (words((
+ 'abs', 'acos', 'acosh', 'alias', 'asin', 'asinh', 'atan', 'atan2',
+ 'atanh', 'ceil', 'ctime', 'cos', 'exp', 'floor', 'log', 'log10',
+ 'max', 'min', 'precision', 'round', 'sin', 'sinh', 'sqrt', 'tan',
+ 'tanh', 'time', 'trunc', 'Beta', 'Cauchy', 'Exponential', 'Gamma',
+ 'Irand224', 'Normal', 'Normal01', 'Poisson', 'Uniform', 'Uniform01',
+ 'num', 'num0', 'ichar', 'char', 'length', 'substr', 'sprintf',
+ 'match', 'sub', 'gsub', 'print', 'printf', 'next', 'nextw', 'prev',
+ 'prevw', 'first', 'last', 'ord', 'ord0', 'card', 'arity',
+ 'indexarity'), prefix=r'\b', suffix=r'\b'), Name.Builtin),
+ (r'(\+|\-|\*|/|\*\*|=|<=|>=|==|\||\^|<|>|\!|\.\.|:=|\&|\!=|<<|>>)',
+ Operator),
+ (words((
+ 'or', 'exists', 'forall', 'and', 'in', 'not', 'within', 'union',
+ 'diff', 'difference', 'symdiff', 'inter', 'intersect',
+ 'intersection', 'cross', 'setof', 'by', 'less', 'sum', 'prod',
+ 'product', 'div', 'mod'), suffix=r'\b'),
+ Keyword.Reserved), # Operator.Name but not enough emphasized with that
+ (r'(\d+\.(?!\.)\d*|\.(?!.)\d+)([eE][+-]?\d+)?', Number.Float),
+ (r'\d+([eE][+-]?\d+)?', Number.Integer),
+ (r'[+-]?Infinity', Number.Integer),
+ (r'(\w+|(\.(?!\.)))', Text)
+ ]
+
+ }
diff --git a/wandb/vendor/pygments/lexers/apl.py b/wandb/vendor/pygments/lexers/apl.py
new file mode 100644
index 0000000000000000000000000000000000000000..b3414cc05ee6e9328e46e6e494b19ae58619c1ea
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/apl.py
@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.apl
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexers for APL.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['APLLexer']
+
+
+class APLLexer(RegexLexer):
+ """
+ A simple APL lexer.
+
+ .. versionadded:: 2.0
+ """
+ name = 'APL'
+ aliases = ['apl']
+ filenames = ['*.apl']
+
+ tokens = {
+ 'root': [
+ # Whitespace
+ # ==========
+ (r'\s+', Text),
+ #
+ # Comment
+ # =======
+ # 'â' is traditional; '#' is supported by GNU APL and NGN (but not Dyalog)
+ (u'[â#].*$', Comment.Single),
+ #
+ # Strings
+ # =======
+ (r'\'((\'\')|[^\'])*\'', String.Single),
+ (r'"(("")|[^"])*"', String.Double), # supported by NGN APL
+ #
+ # Punctuation
+ # ===========
+ # This token type is used for diamond and parenthesis
+ # but not for bracket and ; (see below)
+ (u'[â‹„â—‡()]', Punctuation),
+ #
+ # Array indexing
+ # ==============
+ # Since this token type is very important in APL, it is not included in
+ # the punctuation token type but rather in the following one
+ (r'[\[\];]', String.Regex),
+ #
+ # Distinguished names
+ # ===================
+ # following IBM APL2 standard
+ (u'⎕[A-Za-zΔ∆â™][A-Za-zΔ∆â™_¯0-9]*', Name.Function),
+ #
+ # Labels
+ # ======
+ # following IBM APL2 standard
+ # (u'[A-Za-zΔ∆â™][A-Za-zΔ∆â™_¯0-9]*:', Name.Label),
+ #
+ # Variables
+ # =========
+ # following IBM APL2 standard
+ (u'[A-Za-zΔ∆â™][A-Za-zΔ∆â™_¯0-9]*', Name.Variable),
+ #
+ # Numbers
+ # =======
+ (u'¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞)'
+ u'([Jj]¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞))?',
+ Number),
+ #
+ # Operators
+ # ==========
+ (u'[\.\\\/⌿â€Â¨â£â¨â â¤âˆ˜]', Name.Attribute), # closest token type
+ (u'[+\-×÷⌈⌊∣|â³?*âŸâ—‹!⌹<≤=>≥≠≡≢∊â·âˆªâˆ©~∨∧â±â²â´,âªâŒ½âŠ–â‰â†‘↓⊂⊃⌷â‹â’⊤⊥â•âŽâŠ£âŠ¢ââ‚≈⌸â¯â†—]',
+ Operator),
+ #
+ # Constant
+ # ========
+ (u'â¬', Name.Constant),
+ #
+ # Quad symbol
+ # ===========
+ (u'[⎕âž]', Name.Variable.Global),
+ #
+ # Arrows left/right
+ # =================
+ (u'[â†â†’]', Keyword.Declaration),
+ #
+ # D-Fn
+ # ====
+ (u'[âºâµâ¶â¹âˆ‡:]', Name.Builtin.Pseudo),
+ (r'[{}]', Keyword.Type),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/archetype.py b/wandb/vendor/pygments/lexers/archetype.py
new file mode 100644
index 0000000000000000000000000000000000000000..5d4eb9aa9d7a978f129bacd5cf04bdb0176490f3
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/archetype.py
@@ -0,0 +1,318 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.archetype
+ ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for Archetype-related syntaxes, including:
+
+ - ODIN syntax <https://github.com/openEHR/odin>
+ - ADL syntax <http://www.openehr.org/releases/trunk/architecture/am/adl2.pdf>
+ - cADL sub-syntax of ADL
+
+ For uses of this syntax, see the openEHR archetypes <http://www.openEHR.org/ckm>
+
+ Contributed by Thomas Beale <https://github.com/wolandscat>,
+ <https://bitbucket.org/thomas_beale>.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, using, default
+from pygments.token import Text, Comment, Name, Literal, Number, String, \
+ Punctuation, Keyword, Operator, Generic
+
+__all__ = ['OdinLexer', 'CadlLexer', 'AdlLexer']
+
+
+class AtomsLexer(RegexLexer):
+ """
+ Lexer for Values used in ADL and ODIN.
+
+ .. versionadded:: 2.1
+ """
+
+ tokens = {
+ # ----- pseudo-states for inclusion -----
+ 'whitespace': [
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'[ \t]*--.*$', Comment),
+ ],
+ 'archetype_id': [
+ (r'[ \t]*([a-zA-Z]\w+(\.[a-zA-Z]\w+)*::)?[a-zA-Z]\w+(-[a-zA-Z]\w+){2}'
+ r'\.\w+[\w-]*\.v\d+(\.\d+){,2}((-[a-z]+)(\.\d+)?)?', Name.Decorator),
+ ],
+ 'date_constraints': [
+ # ISO 8601-based date/time constraints
+ (r'[Xx?YyMmDdHhSs\d]{2,4}([:-][Xx?YyMmDdHhSs\d]{2}){2}', Literal.Date),
+ # ISO 8601-based duration constraints + optional trailing slash
+ (r'(P[YyMmWwDd]+(T[HhMmSs]+)?|PT[HhMmSs]+)/?', Literal.Date),
+ ],
+ 'ordered_values': [
+ # ISO 8601 date with optional 'T' ligature
+ (r'\d{4}-\d{2}-\d{2}T?', Literal.Date),
+ # ISO 8601 time
+ (r'\d{2}:\d{2}:\d{2}(\.\d+)?([+-]\d{4}|Z)?', Literal.Date),
+ # ISO 8601 duration
+ (r'P((\d*(\.\d+)?[YyMmWwDd]){1,3}(T(\d*(\.\d+)?[HhMmSs]){,3})?|'
+ r'T(\d*(\.\d+)?[HhMmSs]){,3})', Literal.Date),
+ (r'[+-]?(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+', Number.Float),
+ (r'[+-]?(\d+)*\.\d+%?', Number.Float),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'[+-]?\d+%?', Number.Integer),
+ ],
+ 'values': [
+ include('ordered_values'),
+ (r'([Tt]rue|[Ff]alse)', Literal),
+ (r'"', String, 'string'),
+ (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+ (r'[a-z][a-z0-9+.-]*:', Literal, 'uri'),
+ # term code
+ (r'(\[)(\w[\w-]*(?:\([^)\n]+\))?)(::)(\w[\w-]*)(\])',
+ bygroups(Punctuation, Name.Decorator, Punctuation, Name.Decorator,
+ Punctuation)),
+ (r'\|', Punctuation, 'interval'),
+ # list continuation
+ (r'\.\.\.', Punctuation),
+ ],
+ 'constraint_values': [
+ (r'(\[)(\w[\w-]*(?:\([^)\n]+\))?)(::)',
+ bygroups(Punctuation, Name.Decorator, Punctuation), 'adl14_code_constraint'),
+ # ADL 1.4 ordinal constraint
+ (r'(\d*)(\|)(\[\w[\w-]*::\w[\w-]*\])((?:[,;])?)',
+ bygroups(Number, Punctuation, Name.Decorator, Punctuation)),
+ include('date_constraints'),
+ include('values'),
+ ],
+
+ # ----- real states -----
+ 'string': [
+ ('"', String, '#pop'),
+ (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|'
+ r'u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})', String.Escape),
+ # all other characters
+ (r'[^\\"]+', String),
+ # stray backslash
+ (r'\\', String),
+ ],
+ 'uri': [
+ # effective URI terminators
+ (r'[,>\s]', Punctuation, '#pop'),
+ (r'[^>\s,]+', Literal),
+ ],
+ 'interval': [
+ (r'\|', Punctuation, '#pop'),
+ include('ordered_values'),
+ (r'\.\.', Punctuation),
+ (r'[<>=] *', Punctuation),
+ # handle +/-
+ (r'\+/-', Punctuation),
+ (r'\s+', Text),
+ ],
+ 'any_code': [
+ include('archetype_id'),
+ # if it is a code
+ (r'[a-z_]\w*[0-9.]+(@[^\]]+)?', Name.Decorator),
+ # if it is tuple with attribute names
+ (r'[a-z_]\w*', Name.Class),
+ # if it is an integer, i.e. Xpath child index
+ (r'[0-9]+', Text),
+ (r'\|', Punctuation, 'code_rubric'),
+ (r'\]', Punctuation, '#pop'),
+ # handle use_archetype statement
+ (r'\s*,\s*', Punctuation),
+ ],
+ 'code_rubric': [
+ (r'\|', Punctuation, '#pop'),
+ (r'[^|]+', String),
+ ],
+ 'adl14_code_constraint': [
+ (r'\]', Punctuation, '#pop'),
+ (r'\|', Punctuation, 'code_rubric'),
+ (r'(\w[\w-]*)([;,]?)', bygroups(Name.Decorator, Punctuation)),
+ include('whitespace'),
+ ],
+ }
+
+
+class OdinLexer(AtomsLexer):
+ """
+ Lexer for ODIN syntax.
+
+ .. versionadded:: 2.1
+ """
+ name = 'ODIN'
+ aliases = ['odin']
+ filenames = ['*.odin']
+ mimetypes = ['text/odin']
+
+ tokens = {
+ 'path': [
+ (r'>', Punctuation, '#pop'),
+ # attribute name
+ (r'[a-z_]\w*', Name.Class),
+ (r'/', Punctuation),
+ (r'\[', Punctuation, 'key'),
+ (r'\s*,\s*', Punctuation, '#pop'),
+ (r'\s+', Text, '#pop'),
+ ],
+ 'key': [
+ include('values'),
+ (r'\]', Punctuation, '#pop'),
+ ],
+ 'type_cast': [
+ (r'\)', Punctuation, '#pop'),
+ (r'[^)]+', Name.Class),
+ ],
+ 'root': [
+ include('whitespace'),
+ (r'([Tt]rue|[Ff]alse)', Literal),
+ include('values'),
+ # x-ref path
+ (r'/', Punctuation, 'path'),
+ # x-ref path starting with key
+ (r'\[', Punctuation, 'key'),
+ # attribute name
+ (r'[a-z_]\w*', Name.Class),
+ (r'=', Operator),
+ (r'\(', Punctuation, 'type_cast'),
+ (r',', Punctuation),
+ (r'<', Punctuation),
+ (r'>', Punctuation),
+ (r';', Punctuation),
+ ],
+ }
+
+
+class CadlLexer(AtomsLexer):
+ """
+ Lexer for cADL syntax.
+
+ .. versionadded:: 2.1
+ """
+ name = 'cADL'
+ aliases = ['cadl']
+ filenames = ['*.cadl']
+
+ tokens = {
+ 'path': [
+ # attribute name
+ (r'[a-z_]\w*', Name.Class),
+ (r'/', Punctuation),
+ (r'\[', Punctuation, 'any_code'),
+ (r'\s+', Punctuation, '#pop'),
+ ],
+ 'root': [
+ include('whitespace'),
+ (r'(cardinality|existence|occurrences|group|include|exclude|'
+ r'allow_archetype|use_archetype|use_node)\W', Keyword.Type),
+ (r'(and|or|not|there_exists|xor|implies|for_all)\W', Keyword.Type),
+ (r'(after|before|closed)\W', Keyword.Type),
+ (r'(not)\W', Operator),
+ (r'(matches|is_in)\W', Operator),
+ # is_in / not is_in char
+ (u'(\u2208|\u2209)', Operator),
+ # there_exists / not there_exists / for_all / and / or
+ (u'(\u2203|\u2204|\u2200|\u2227|\u2228|\u22BB|\223C)',
+ Operator),
+ # regex in slot or as string constraint
+ (r'(\{)(\s*/[^}]+/\s*)(\})',
+ bygroups(Punctuation, String.Regex, Punctuation)),
+ # regex in slot or as string constraint
+ (r'(\{)(\s*\^[^}]+\^\s*)(\})',
+ bygroups(Punctuation, String.Regex, Punctuation)),
+ (r'/', Punctuation, 'path'),
+ # for cardinality etc
+ (r'(\{)((?:\d+\.\.)?(?:\d+|\*))'
+ r'((?:\s*;\s*(?:ordered|unordered|unique)){,2})(\})',
+ bygroups(Punctuation, Number, Number, Punctuation)),
+ # [{ is start of a tuple value
+ (r'\[\{', Punctuation),
+ (r'\}\]', Punctuation),
+ (r'\{', Punctuation),
+ (r'\}', Punctuation),
+ include('constraint_values'),
+ # type name
+ (r'[A-Z]\w+(<[A-Z]\w+([A-Za-z_<>]*)>)?', Name.Class),
+ # attribute name
+ (r'[a-z_]\w*', Name.Class),
+ (r'\[', Punctuation, 'any_code'),
+ (r'(~|//|\\\\|\+|-|/|\*|\^|!=|=|<=|>=|<|>]?)', Operator),
+ (r'\(', Punctuation),
+ (r'\)', Punctuation),
+ # for lists of values
+ (r',', Punctuation),
+ (r'"', String, 'string'),
+ # for assumed value
+ (r';', Punctuation),
+ ],
+ }
+
+
+class AdlLexer(AtomsLexer):
+ """
+ Lexer for ADL syntax.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'ADL'
+ aliases = ['adl']
+ filenames = ['*.adl', '*.adls', '*.adlf', '*.adlx']
+
+ tokens = {
+ 'whitespace': [
+ # blank line ends
+ (r'\s*\n', Text),
+ # comment-only line
+ (r'^[ \t]*--.*$', Comment),
+ ],
+ 'odin_section': [
+ # repeating the following two rules from the root state enable multi-line
+ # strings that start in the first column to be dealt with
+ (r'^(language|description|ontology|terminology|annotations|'
+ r'component_terminologies|revision_history)[ \t]*\n', Generic.Heading),
+ (r'^(definition)[ \t]*\n', Generic.Heading, 'cadl_section'),
+ (r'^([ \t]*|[ \t]+.*)\n', using(OdinLexer)),
+ (r'^([^"]*")(>[ \t]*\n)', bygroups(String, Punctuation)),
+ # template overlay delimiter
+ (r'^----------*\n', Text, '#pop'),
+ (r'^.*\n', String),
+ default('#pop'),
+ ],
+ 'cadl_section': [
+ (r'^([ \t]*|[ \t]+.*)\n', using(CadlLexer)),
+ default('#pop'),
+ ],
+ 'rules_section': [
+ (r'^[ \t]+.*\n', using(CadlLexer)),
+ default('#pop'),
+ ],
+ 'metadata': [
+ (r'\)', Punctuation, '#pop'),
+ (r';', Punctuation),
+ (r'([Tt]rue|[Ff]alse)', Literal),
+ # numbers and version ids
+ (r'\d+(\.\d+)*', Literal),
+ # Guids
+ (r'(\d|[a-fA-F])+(-(\d|[a-fA-F])+){3,}', Literal),
+ (r'\w+', Name.Class),
+ (r'"', String, 'string'),
+ (r'=', Operator),
+ (r'[ \t]+', Text),
+ default('#pop'),
+ ],
+ 'root': [
+ (r'^(archetype|template_overlay|operational_template|template|'
+ r'speciali[sz]e)', Generic.Heading),
+ (r'^(language|description|ontology|terminology|annotations|'
+ r'component_terminologies|revision_history)[ \t]*\n',
+ Generic.Heading, 'odin_section'),
+ (r'^(definition)[ \t]*\n', Generic.Heading, 'cadl_section'),
+ (r'^(rules)[ \t]*\n', Generic.Heading, 'rules_section'),
+ include('archetype_id'),
+ (r'[ \t]*\(', Punctuation, 'metadata'),
+ include('whitespace'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/asm.py b/wandb/vendor/pygments/lexers/asm.py
new file mode 100644
index 0000000000000000000000000000000000000000..9c58478e4b544bae0f1fa70b5dbba6671a4b8f16
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/asm.py
@@ -0,0 +1,641 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.asm
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexers for assembly languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, words, \
+ DelegatingLexer
+from pygments.lexers.c_cpp import CppLexer, CLexer
+from pygments.lexers.d import DLexer
+from pygments.token import Text, Name, Number, String, Comment, Punctuation, \
+ Other, Keyword, Operator
+
+__all__ = ['GasLexer', 'ObjdumpLexer', 'DObjdumpLexer', 'CppObjdumpLexer',
+ 'CObjdumpLexer', 'HsailLexer', 'LlvmLexer', 'NasmLexer',
+ 'NasmObjdumpLexer', 'TasmLexer', 'Ca65Lexer']
+
+
+class GasLexer(RegexLexer):
+ """
+ For Gas (AT&T) assembly code.
+ """
+ name = 'GAS'
+ aliases = ['gas', 'asm']
+ filenames = ['*.s', '*.S']
+ mimetypes = ['text/x-gas']
+
+ #: optional Comment or Whitespace
+ string = r'"(\\"|[^"])*"'
+ char = r'[\w$.@-]'
+ identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+)'
+ number = r'(?:0[xX][a-zA-Z0-9]+|\d+)'
+
+ tokens = {
+ 'root': [
+ include('whitespace'),
+ (identifier + ':', Name.Label),
+ (r'\.' + identifier, Name.Attribute, 'directive-args'),
+ (r'lock|rep(n?z)?|data\d+', Name.Attribute),
+ (identifier, Name.Function, 'instruction-args'),
+ (r'[\r\n]+', Text)
+ ],
+ 'directive-args': [
+ (identifier, Name.Constant),
+ (string, String),
+ ('@' + identifier, Name.Attribute),
+ (number, Number.Integer),
+ (r'[\r\n]+', Text, '#pop'),
+
+ include('punctuation'),
+ include('whitespace')
+ ],
+ 'instruction-args': [
+ # For objdump-disassembled code, shouldn't occur in
+ # actual assembler input
+ ('([a-z0-9]+)( )(<)('+identifier+')(>)',
+ bygroups(Number.Hex, Text, Punctuation, Name.Constant,
+ Punctuation)),
+ ('([a-z0-9]+)( )(<)('+identifier+')([-+])('+number+')(>)',
+ bygroups(Number.Hex, Text, Punctuation, Name.Constant,
+ Punctuation, Number.Integer, Punctuation)),
+
+ # Address constants
+ (identifier, Name.Constant),
+ (number, Number.Integer),
+ # Registers
+ ('%' + identifier, Name.Variable),
+ # Numeric constants
+ ('$'+number, Number.Integer),
+ (r"$'(.|\\')'", String.Char),
+ (r'[\r\n]+', Text, '#pop'),
+
+ include('punctuation'),
+ include('whitespace')
+ ],
+ 'whitespace': [
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'[;#].*?\n', Comment)
+ ],
+ 'punctuation': [
+ (r'[-*,.()\[\]!:]+', Punctuation)
+ ]
+ }
+
+ def analyse_text(text):
+ if re.match(r'^\.(text|data|section)', text, re.M):
+ return True
+ elif re.match(r'^\.\w+', text, re.M):
+ return 0.1
+
+
+def _objdump_lexer_tokens(asm_lexer):
+ """
+ Common objdump lexer tokens to wrap an ASM lexer.
+ """
+ hex_re = r'[0-9A-Za-z]'
+ return {
+ 'root': [
+ # File name & format:
+ ('(.*?)(:)( +file format )(.*?)$',
+ bygroups(Name.Label, Punctuation, Text, String)),
+ # Section header
+ ('(Disassembly of section )(.*?)(:)$',
+ bygroups(Text, Name.Label, Punctuation)),
+ # Function labels
+ # (With offset)
+ ('('+hex_re+'+)( )(<)(.*?)([-+])(0[xX][A-Za-z0-9]+)(>:)$',
+ bygroups(Number.Hex, Text, Punctuation, Name.Function,
+ Punctuation, Number.Hex, Punctuation)),
+ # (Without offset)
+ ('('+hex_re+'+)( )(<)(.*?)(>:)$',
+ bygroups(Number.Hex, Text, Punctuation, Name.Function,
+ Punctuation)),
+ # Code line with disassembled instructions
+ ('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)( *\t)([a-zA-Z].*?)$',
+ bygroups(Text, Name.Label, Text, Number.Hex, Text,
+ using(asm_lexer))),
+ # Code line with ascii
+ ('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)( *)(.*?)$',
+ bygroups(Text, Name.Label, Text, Number.Hex, Text, String)),
+ # Continued code line, only raw opcodes without disassembled
+ # instruction
+ ('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)$',
+ bygroups(Text, Name.Label, Text, Number.Hex)),
+ # Skipped a few bytes
+ (r'\t\.\.\.$', Text),
+ # Relocation line
+ # (With offset)
+ (r'(\t\t\t)('+hex_re+r'+:)( )([^\t]+)(\t)(.*?)([-+])(0x'+hex_re+'+)$',
+ bygroups(Text, Name.Label, Text, Name.Property, Text,
+ Name.Constant, Punctuation, Number.Hex)),
+ # (Without offset)
+ (r'(\t\t\t)('+hex_re+r'+:)( )([^\t]+)(\t)(.*?)$',
+ bygroups(Text, Name.Label, Text, Name.Property, Text,
+ Name.Constant)),
+ (r'[^\n]+\n', Other)
+ ]
+ }
+
+
+class ObjdumpLexer(RegexLexer):
+ """
+ For the output of 'objdump -dr'
+ """
+ name = 'objdump'
+ aliases = ['objdump']
+ filenames = ['*.objdump']
+ mimetypes = ['text/x-objdump']
+
+ tokens = _objdump_lexer_tokens(GasLexer)
+
+
+class DObjdumpLexer(DelegatingLexer):
+ """
+ For the output of 'objdump -Sr on compiled D files'
+ """
+ name = 'd-objdump'
+ aliases = ['d-objdump']
+ filenames = ['*.d-objdump']
+ mimetypes = ['text/x-d-objdump']
+
+ def __init__(self, **options):
+ super(DObjdumpLexer, self).__init__(DLexer, ObjdumpLexer, **options)
+
+
+class CppObjdumpLexer(DelegatingLexer):
+ """
+ For the output of 'objdump -Sr on compiled C++ files'
+ """
+ name = 'cpp-objdump'
+ aliases = ['cpp-objdump', 'c++-objdumb', 'cxx-objdump']
+ filenames = ['*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump']
+ mimetypes = ['text/x-cpp-objdump']
+
+ def __init__(self, **options):
+ super(CppObjdumpLexer, self).__init__(CppLexer, ObjdumpLexer, **options)
+
+
+class CObjdumpLexer(DelegatingLexer):
+ """
+ For the output of 'objdump -Sr on compiled C files'
+ """
+ name = 'c-objdump'
+ aliases = ['c-objdump']
+ filenames = ['*.c-objdump']
+ mimetypes = ['text/x-c-objdump']
+
+ def __init__(self, **options):
+ super(CObjdumpLexer, self).__init__(CLexer, ObjdumpLexer, **options)
+
+
+class HsailLexer(RegexLexer):
+ """
+ For HSAIL assembly code.
+
+ .. versionadded:: 2.2
+ """
+ name = 'HSAIL'
+ aliases = ['hsail', 'hsa']
+ filenames = ['*.hsail']
+ mimetypes = ['text/x-hsail']
+
+ string = r'"[^"]*?"'
+ identifier = r'[a-zA-Z_][\w.]*'
+ # Registers
+ register_number = r'[0-9]+'
+ register = r'(\$(c|s|d|q)' + register_number + ')'
+ # Qualifiers
+ alignQual = r'(align\(\d+\))'
+ widthQual = r'(width\((\d+|all)\))'
+ allocQual = r'(alloc\(agent\))'
+ # Instruction Modifiers
+ roundingMod = (r'((_ftz)?(_up|_down|_zero|_near))')
+ datatypeMod = (r'_('
+ # packedTypes
+ r'u8x4|s8x4|u16x2|s16x2|u8x8|s8x8|u16x4|s16x4|u32x2|s32x2|'
+ r'u8x16|s8x16|u16x8|s16x8|u32x4|s32x4|u64x2|s64x2|'
+ r'f16x2|f16x4|f16x8|f32x2|f32x4|f64x2|'
+ # baseTypes
+ r'u8|s8|u16|s16|u32|s32|u64|s64|'
+ r'b128|b8|b16|b32|b64|b1|'
+ r'f16|f32|f64|'
+ # opaqueType
+ r'roimg|woimg|rwimg|samp|sig32|sig64)')
+
+ # Numeric Constant
+ float = r'((\d+\.)|(\d*\.\d+))[eE][+-]?\d+'
+ hexfloat = r'0[xX](([0-9a-fA-F]+\.[0-9a-fA-F]*)|([0-9a-fA-F]*\.[0-9a-fA-F]+))[pP][+-]?\d+'
+ ieeefloat = r'0((h|H)[0-9a-fA-F]{4}|(f|F)[0-9a-fA-F]{8}|(d|D)[0-9a-fA-F]{16})'
+
+ tokens = {
+ 'root': [
+ include('whitespace'),
+ include('comments'),
+
+ (string, String),
+
+ (r'@' + identifier + ':?', Name.Label),
+
+ (register, Name.Variable.Anonymous),
+
+ include('keyword'),
+
+ (r'&' + identifier, Name.Variable.Global),
+ (r'%' + identifier, Name.Variable),
+
+ (hexfloat, Number.Hex),
+ (r'0[xX][a-fA-F0-9]+', Number.Hex),
+ (ieeefloat, Number.Float),
+ (float, Number.Float),
+ ('\d+', Number.Integer),
+
+ (r'[=<>{}\[\]()*.,:;!]|x\b', Punctuation)
+ ],
+ 'whitespace': [
+ (r'(\n|\s)+', Text),
+ ],
+ 'comments': [
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'//.*?\n', Comment.Singleline),
+ ],
+ 'keyword': [
+ # Types
+ (r'kernarg' + datatypeMod, Keyword.Type),
+
+ # Regular keywords
+ (r'\$(full|base|small|large|default|zero|near)', Keyword),
+ (words((
+ 'module', 'extension', 'pragma', 'prog', 'indirect', 'signature',
+ 'decl', 'kernel', 'function', 'enablebreakexceptions',
+ 'enabledetectexceptions', 'maxdynamicgroupsize', 'maxflatgridsize',
+ 'maxflatworkgroupsize', 'requireddim', 'requiredgridsize',
+ 'requiredworkgroupsize', 'requirenopartialworkgroups'),
+ suffix=r'\b'), Keyword),
+
+ # instructions
+ (roundingMod, Keyword),
+ (datatypeMod, Keyword),
+ (r'_(' + alignQual + '|' + widthQual + ')', Keyword),
+ (r'_kernarg', Keyword),
+ (r'(nop|imagefence)\b', Keyword),
+ (words((
+ 'cleardetectexcept', 'clock', 'cuid', 'debugtrap', 'dim',
+ 'getdetectexcept', 'groupbaseptr', 'kernargbaseptr', 'laneid',
+ 'maxcuid', 'maxwaveid', 'packetid', 'setdetectexcept', 'waveid',
+ 'workitemflatabsid', 'workitemflatid', 'nullptr', 'abs', 'bitrev',
+ 'currentworkgroupsize', 'currentworkitemflatid', 'fract', 'ncos',
+ 'neg', 'nexp2', 'nlog2', 'nrcp', 'nrsqrt', 'nsin', 'nsqrt',
+ 'gridgroups', 'gridsize', 'not', 'sqrt', 'workgroupid',
+ 'workgroupsize', 'workitemabsid', 'workitemid', 'ceil', 'floor',
+ 'rint', 'trunc', 'add', 'bitmask', 'borrow', 'carry', 'copysign',
+ 'div', 'rem', 'sub', 'shl', 'shr', 'and', 'or', 'xor', 'unpackhi',
+ 'unpacklo', 'max', 'min', 'fma', 'mad', 'bitextract', 'bitselect',
+ 'shuffle', 'cmov', 'bitalign', 'bytealign', 'lerp', 'nfma', 'mul',
+ 'mulhi', 'mul24hi', 'mul24', 'mad24', 'mad24hi', 'bitinsert',
+ 'combine', 'expand', 'lda', 'mov', 'pack', 'unpack', 'packcvt',
+ 'unpackcvt', 'sad', 'sementp', 'ftos', 'stof', 'cmp', 'ld', 'st',
+ '_eq', '_ne', '_lt', '_le', '_gt', '_ge', '_equ', '_neu', '_ltu',
+ '_leu', '_gtu', '_geu', '_num', '_nan', '_seq', '_sne', '_slt',
+ '_sle', '_sgt', '_sge', '_snum', '_snan', '_sequ', '_sneu', '_sltu',
+ '_sleu', '_sgtu', '_sgeu', 'atomic', '_ld', '_st', '_cas', '_add',
+ '_and', '_exch', '_max', '_min', '_or', '_sub', '_wrapdec',
+ '_wrapinc', '_xor', 'ret', 'cvt', '_readonly', '_kernarg', '_global',
+ 'br', 'cbr', 'sbr', '_scacq', '_screl', '_scar', '_rlx', '_wave',
+ '_wg', '_agent', '_system', 'ldimage', 'stimage', '_v2', '_v3', '_v4',
+ '_1d', '_2d', '_3d', '_1da', '_2da', '_1db', '_2ddepth', '_2dadepth',
+ '_width', '_height', '_depth', '_array', '_channelorder',
+ '_channeltype', 'querysampler', '_coord', '_filter', '_addressing',
+ 'barrier', 'wavebarrier', 'initfbar', 'joinfbar', 'waitfbar',
+ 'arrivefbar', 'leavefbar', 'releasefbar', 'ldf', 'activelaneid',
+ 'activelanecount', 'activelanemask', 'activelanepermute', 'call',
+ 'scall', 'icall', 'alloca', 'packetcompletionsig',
+ 'addqueuewriteindex', 'casqueuewriteindex', 'ldqueuereadindex',
+ 'stqueuereadindex', 'readonly', 'global', 'private', 'group',
+ 'spill', 'arg', '_upi', '_downi', '_zeroi', '_neari', '_upi_sat',
+ '_downi_sat', '_zeroi_sat', '_neari_sat', '_supi', '_sdowni',
+ '_szeroi', '_sneari', '_supi_sat', '_sdowni_sat', '_szeroi_sat',
+ '_sneari_sat', '_pp', '_ps', '_sp', '_ss', '_s', '_p', '_pp_sat',
+ '_ps_sat', '_sp_sat', '_ss_sat', '_s_sat', '_p_sat')), Keyword),
+
+ # Integer types
+ (r'i[1-9]\d*', Keyword)
+ ]
+ }
+
+
+class LlvmLexer(RegexLexer):
+ """
+ For LLVM assembly code.
+ """
+ name = 'LLVM'
+ aliases = ['llvm']
+ filenames = ['*.ll']
+ mimetypes = ['text/x-llvm']
+
+ #: optional Comment or Whitespace
+ string = r'"[^"]*?"'
+ identifier = r'([-a-zA-Z$._][\w\-$.]*|' + string + ')'
+
+ tokens = {
+ 'root': [
+ include('whitespace'),
+
+ # Before keywords, because keywords are valid label names :(...
+ (identifier + '\s*:', Name.Label),
+
+ include('keyword'),
+
+ (r'%' + identifier, Name.Variable),
+ (r'@' + identifier, Name.Variable.Global),
+ (r'%\d+', Name.Variable.Anonymous),
+ (r'@\d+', Name.Variable.Global),
+ (r'#\d+', Name.Variable.Global),
+ (r'!' + identifier, Name.Variable),
+ (r'!\d+', Name.Variable.Anonymous),
+ (r'c?' + string, String),
+
+ (r'0[xX][a-fA-F0-9]+', Number),
+ (r'-?\d+(?:[.]\d+)?(?:[eE][-+]?\d+(?:[.]\d+)?)?', Number),
+
+ (r'[=<>{}\[\]()*.,!]|x\b', Punctuation)
+ ],
+ 'whitespace': [
+ (r'(\n|\s)+', Text),
+ (r';.*?\n', Comment)
+ ],
+ 'keyword': [
+ # Regular keywords
+ (words((
+ 'begin', 'end', 'true', 'false', 'declare', 'define', 'global',
+ 'constant', 'private', 'linker_private', 'internal',
+ 'available_externally', 'linkonce', 'linkonce_odr', 'weak',
+ 'weak_odr', 'appending', 'dllimport', 'dllexport', 'common',
+ 'default', 'hidden', 'protected', 'extern_weak', 'external',
+ 'thread_local', 'zeroinitializer', 'undef', 'null', 'to', 'tail',
+ 'target', 'triple', 'datalayout', 'volatile', 'nuw', 'nsw', 'nnan',
+ 'ninf', 'nsz', 'arcp', 'fast', 'exact', 'inbounds', 'align',
+ 'addrspace', 'section', 'alias', 'module', 'asm', 'sideeffect',
+ 'gc', 'dbg', 'linker_private_weak', 'attributes', 'blockaddress',
+ 'initialexec', 'localdynamic', 'localexec', 'prefix', 'unnamed_addr',
+ 'ccc', 'fastcc', 'coldcc', 'x86_stdcallcc', 'x86_fastcallcc',
+ 'arm_apcscc', 'arm_aapcscc', 'arm_aapcs_vfpcc', 'ptx_device',
+ 'ptx_kernel', 'intel_ocl_bicc', 'msp430_intrcc', 'spir_func',
+ 'spir_kernel', 'x86_64_sysvcc', 'x86_64_win64cc', 'x86_thiscallcc',
+ 'cc', 'c', 'signext', 'zeroext', 'inreg', 'sret', 'nounwind',
+ 'noreturn', 'noalias', 'nocapture', 'byval', 'nest', 'readnone',
+ 'readonly', 'inlinehint', 'noinline', 'alwaysinline', 'optsize', 'ssp',
+ 'sspreq', 'noredzone', 'noimplicitfloat', 'naked', 'builtin', 'cold',
+ 'nobuiltin', 'noduplicate', 'nonlazybind', 'optnone', 'returns_twice',
+ 'sanitize_address', 'sanitize_memory', 'sanitize_thread', 'sspstrong',
+ 'uwtable', 'returned', 'type', 'opaque', 'eq', 'ne', 'slt', 'sgt',
+ 'sle', 'sge', 'ult', 'ugt', 'ule', 'uge', 'oeq', 'one', 'olt', 'ogt',
+ 'ole', 'oge', 'ord', 'uno', 'ueq', 'une', 'x', 'acq_rel', 'acquire',
+ 'alignstack', 'atomic', 'catch', 'cleanup', 'filter', 'inteldialect',
+ 'max', 'min', 'monotonic', 'nand', 'personality', 'release', 'seq_cst',
+ 'singlethread', 'umax', 'umin', 'unordered', 'xchg', 'add', 'fadd',
+ 'sub', 'fsub', 'mul', 'fmul', 'udiv', 'sdiv', 'fdiv', 'urem', 'srem',
+ 'frem', 'shl', 'lshr', 'ashr', 'and', 'or', 'xor', 'icmp', 'fcmp',
+ 'phi', 'call', 'trunc', 'zext', 'sext', 'fptrunc', 'fpext', 'uitofp',
+ 'sitofp', 'fptoui', 'fptosi', 'inttoptr', 'ptrtoint', 'bitcast',
+ 'addrspacecast', 'select', 'va_arg', 'ret', 'br', 'switch', 'invoke',
+ 'unwind', 'unreachable', 'indirectbr', 'landingpad', 'resume',
+ 'malloc', 'alloca', 'free', 'load', 'store', 'getelementptr',
+ 'extractelement', 'insertelement', 'shufflevector', 'getresult',
+ 'extractvalue', 'insertvalue', 'atomicrmw', 'cmpxchg', 'fence',
+ 'allocsize', 'amdgpu_cs', 'amdgpu_gs', 'amdgpu_kernel', 'amdgpu_ps',
+ 'amdgpu_vs', 'any', 'anyregcc', 'argmemonly', 'avr_intrcc',
+ 'avr_signalcc', 'caller', 'catchpad', 'catchret', 'catchswitch',
+ 'cleanuppad', 'cleanupret', 'comdat', 'convergent', 'cxx_fast_tlscc',
+ 'deplibs', 'dereferenceable', 'dereferenceable_or_null', 'distinct',
+ 'exactmatch', 'externally_initialized', 'from', 'ghccc', 'hhvm_ccc',
+ 'hhvmcc', 'ifunc', 'inaccessiblemem_or_argmemonly', 'inaccessiblememonly',
+ 'inalloca', 'jumptable', 'largest', 'local_unnamed_addr', 'minsize',
+ 'musttail', 'noduplicates', 'none', 'nonnull', 'norecurse', 'notail',
+ 'preserve_allcc', 'preserve_mostcc', 'prologue', 'safestack', 'samesize',
+ 'source_filename', 'swiftcc', 'swifterror', 'swiftself', 'webkit_jscc',
+ 'within', 'writeonly', 'x86_intrcc', 'x86_vectorcallcc'),
+ suffix=r'\b'), Keyword),
+
+ # Types
+ (words(('void', 'half', 'float', 'double', 'x86_fp80', 'fp128',
+ 'ppc_fp128', 'label', 'metadata', 'token')), Keyword.Type),
+
+ # Integer types
+ (r'i[1-9]\d*', Keyword)
+ ]
+ }
+
+
+class NasmLexer(RegexLexer):
+ """
+ For Nasm (Intel) assembly code.
+ """
+ name = 'NASM'
+ aliases = ['nasm']
+ filenames = ['*.asm', '*.ASM']
+ mimetypes = ['text/x-nasm']
+
+ identifier = r'[a-z$._?][\w$.?#@~]*'
+ hexn = r'(?:0x[0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)'
+ octn = r'[0-7]+q'
+ binn = r'[01]+b'
+ decn = r'[0-9]+'
+ floatn = decn + r'\.e?' + decn
+ string = r'"(\\"|[^"\n])*"|' + r"'(\\'|[^'\n])*'|" + r"`(\\`|[^`\n])*`"
+ declkw = r'(?:res|d)[bwdqt]|times'
+ register = (r'r[0-9][0-5]?[bwd]|'
+ r'[a-d][lh]|[er]?[a-d]x|[er]?[sb]p|[er]?[sd]i|[c-gs]s|st[0-7]|'
+ r'mm[0-7]|cr[0-4]|dr[0-367]|tr[3-7]')
+ wordop = r'seg|wrt|strict'
+ type = r'byte|[dq]?word'
+ directives = (r'BITS|USE16|USE32|SECTION|SEGMENT|ABSOLUTE|EXTERN|GLOBAL|'
+ r'ORG|ALIGN|STRUC|ENDSTRUC|COMMON|CPU|GROUP|UPPERCASE|IMPORT|'
+ r'EXPORT|LIBRARY|MODULE')
+
+ flags = re.IGNORECASE | re.MULTILINE
+ tokens = {
+ 'root': [
+ (r'^\s*%', Comment.Preproc, 'preproc'),
+ include('whitespace'),
+ (identifier + ':', Name.Label),
+ (r'(%s)(\s+)(equ)' % identifier,
+ bygroups(Name.Constant, Keyword.Declaration, Keyword.Declaration),
+ 'instruction-args'),
+ (directives, Keyword, 'instruction-args'),
+ (declkw, Keyword.Declaration, 'instruction-args'),
+ (identifier, Name.Function, 'instruction-args'),
+ (r'[\r\n]+', Text)
+ ],
+ 'instruction-args': [
+ (string, String),
+ (hexn, Number.Hex),
+ (octn, Number.Oct),
+ (binn, Number.Bin),
+ (floatn, Number.Float),
+ (decn, Number.Integer),
+ include('punctuation'),
+ (register, Name.Builtin),
+ (identifier, Name.Variable),
+ (r'[\r\n]+', Text, '#pop'),
+ include('whitespace')
+ ],
+ 'preproc': [
+ (r'[^;\n]+', Comment.Preproc),
+ (r';.*?\n', Comment.Single, '#pop'),
+ (r'\n', Comment.Preproc, '#pop'),
+ ],
+ 'whitespace': [
+ (r'\n', Text),
+ (r'[ \t]+', Text),
+ (r';.*', Comment.Single)
+ ],
+ 'punctuation': [
+ (r'[,():\[\]]+', Punctuation),
+ (r'[&|^<>+*/%~-]+', Operator),
+ (r'[$]+', Keyword.Constant),
+ (wordop, Operator.Word),
+ (type, Keyword.Type)
+ ],
+ }
+
+
+class NasmObjdumpLexer(ObjdumpLexer):
+ """
+ For the output of 'objdump -d -M intel'.
+
+ .. versionadded:: 2.0
+ """
+ name = 'objdump-nasm'
+ aliases = ['objdump-nasm']
+ filenames = ['*.objdump-intel']
+ mimetypes = ['text/x-nasm-objdump']
+
+ tokens = _objdump_lexer_tokens(NasmLexer)
+
+
+class TasmLexer(RegexLexer):
+ """
+ For Tasm (Turbo Assembler) assembly code.
+ """
+ name = 'TASM'
+ aliases = ['tasm']
+ filenames = ['*.asm', '*.ASM', '*.tasm']
+ mimetypes = ['text/x-tasm']
+
+ identifier = r'[@a-z$._?][\w$.?#@~]*'
+ hexn = r'(?:0x[0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)'
+ octn = r'[0-7]+q'
+ binn = r'[01]+b'
+ decn = r'[0-9]+'
+ floatn = decn + r'\.e?' + decn
+ string = r'"(\\"|[^"\n])*"|' + r"'(\\'|[^'\n])*'|" + r"`(\\`|[^`\n])*`"
+ declkw = r'(?:res|d)[bwdqt]|times'
+ register = (r'r[0-9][0-5]?[bwd]|'
+ r'[a-d][lh]|[er]?[a-d]x|[er]?[sb]p|[er]?[sd]i|[c-gs]s|st[0-7]|'
+ r'mm[0-7]|cr[0-4]|dr[0-367]|tr[3-7]')
+ wordop = r'seg|wrt|strict'
+ type = r'byte|[dq]?word'
+ directives = (r'BITS|USE16|USE32|SECTION|SEGMENT|ABSOLUTE|EXTERN|GLOBAL|'
+ r'ORG|ALIGN|STRUC|ENDSTRUC|ENDS|COMMON|CPU|GROUP|UPPERCASE|INCLUDE|'
+ r'EXPORT|LIBRARY|MODULE|PROC|ENDP|USES|ARG|DATASEG|UDATASEG|END|IDEAL|'
+ r'P386|MODEL|ASSUME|CODESEG|SIZE')
+ # T[A-Z][a-z] is more of a convention. Lexer should filter out STRUC definitions
+ # and then 'add' them to datatype somehow.
+ datatype = (r'db|dd|dw|T[A-Z][a-z]+')
+
+ flags = re.IGNORECASE | re.MULTILINE
+ tokens = {
+ 'root': [
+ (r'^\s*%', Comment.Preproc, 'preproc'),
+ include('whitespace'),
+ (identifier + ':', Name.Label),
+ (directives, Keyword, 'instruction-args'),
+ (r'(%s)(\s+)(%s)' % (identifier, datatype),
+ bygroups(Name.Constant, Keyword.Declaration, Keyword.Declaration),
+ 'instruction-args'),
+ (declkw, Keyword.Declaration, 'instruction-args'),
+ (identifier, Name.Function, 'instruction-args'),
+ (r'[\r\n]+', Text)
+ ],
+ 'instruction-args': [
+ (string, String),
+ (hexn, Number.Hex),
+ (octn, Number.Oct),
+ (binn, Number.Bin),
+ (floatn, Number.Float),
+ (decn, Number.Integer),
+ include('punctuation'),
+ (register, Name.Builtin),
+ (identifier, Name.Variable),
+ # Do not match newline when it's preceeded by a backslash
+ (r'(\\\s*)(;.*)([\r\n])', bygroups(Text, Comment.Single, Text)),
+ (r'[\r\n]+', Text, '#pop'),
+ include('whitespace')
+ ],
+ 'preproc': [
+ (r'[^;\n]+', Comment.Preproc),
+ (r';.*?\n', Comment.Single, '#pop'),
+ (r'\n', Comment.Preproc, '#pop'),
+ ],
+ 'whitespace': [
+ (r'[\n\r]', Text),
+ (r'\\[\n\r]', Text),
+ (r'[ \t]+', Text),
+ (r';.*', Comment.Single)
+ ],
+ 'punctuation': [
+ (r'[,():\[\]]+', Punctuation),
+ (r'[&|^<>+*=/%~-]+', Operator),
+ (r'[$]+', Keyword.Constant),
+ (wordop, Operator.Word),
+ (type, Keyword.Type)
+ ],
+ }
+
+
+class Ca65Lexer(RegexLexer):
+ """
+ For ca65 assembler sources.
+
+ .. versionadded:: 1.6
+ """
+ name = 'ca65 assembler'
+ aliases = ['ca65']
+ filenames = ['*.s']
+
+ flags = re.IGNORECASE
+
+ tokens = {
+ 'root': [
+ (r';.*', Comment.Single),
+ (r'\s+', Text),
+ (r'[a-z_.@$][\w.@$]*:', Name.Label),
+ (r'((ld|st)[axy]|(in|de)[cxy]|asl|lsr|ro[lr]|adc|sbc|cmp|cp[xy]'
+ r'|cl[cvdi]|se[cdi]|jmp|jsr|bne|beq|bpl|bmi|bvc|bvs|bcc|bcs'
+ r'|p[lh][ap]|rt[is]|brk|nop|ta[xy]|t[xy]a|txs|tsx|and|ora|eor'
+ r'|bit)\b', Keyword),
+ (r'\.\w+', Keyword.Pseudo),
+ (r'[-+~*/^&|!<>=]', Operator),
+ (r'"[^"\n]*.', String),
+ (r"'[^'\n]*.", String.Char),
+ (r'\$[0-9a-f]+|[0-9a-f]+h\b', Number.Hex),
+ (r'\d+', Number.Integer),
+ (r'%[01]+', Number.Bin),
+ (r'[#,.:()=\[\]]', Punctuation),
+ (r'[a-z_.@$][\w.@$]*', Name),
+ ]
+ }
+
+ def analyse_text(self, text):
+ # comments in GAS start with "#"
+ if re.match(r'^\s*;', text, re.MULTILINE):
+ return 0.9
diff --git a/wandb/vendor/pygments/lexers/automation.py b/wandb/vendor/pygments/lexers/automation.py
new file mode 100644
index 0000000000000000000000000000000000000000..be1ec12984d5bc6d346e50046c601e01c2f29884
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/automation.py
@@ -0,0 +1,374 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.automation
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for automation scripting languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, combined
+from pygments.token import Text, Comment, Operator, Name, String, \
+ Number, Punctuation, Generic
+
+__all__ = ['AutohotkeyLexer', 'AutoItLexer']
+
+
+class AutohotkeyLexer(RegexLexer):
+ """
+ For `autohotkey <http://www.autohotkey.com/>`_ source code.
+
+ .. versionadded:: 1.4
+ """
+ name = 'autohotkey'
+ aliases = ['ahk', 'autohotkey']
+ filenames = ['*.ahk', '*.ahkl']
+ mimetypes = ['text/x-autohotkey']
+
+ tokens = {
+ 'root': [
+ (r'^(\s*)(/\*)', bygroups(Text, Comment.Multiline), 'incomment'),
+ (r'^(\s*)(\()', bygroups(Text, Generic), 'incontinuation'),
+ (r'\s+;.*?$', Comment.Singleline),
+ (r'^;.*?$', Comment.Singleline),
+ (r'[]{}(),;[]', Punctuation),
+ (r'(in|is|and|or|not)\b', Operator.Word),
+ (r'\%[a-zA-Z_#@$][\w#@$]*\%', Name.Variable),
+ (r'!=|==|:=|\.=|<<|>>|[-~+/*%=<>&^|?:!.]', Operator),
+ include('commands'),
+ include('labels'),
+ include('builtInFunctions'),
+ include('builtInVariables'),
+ (r'"', String, combined('stringescape', 'dqs')),
+ include('numbers'),
+ (r'[a-zA-Z_#@$][\w#@$]*', Name),
+ (r'\\|\'', Text),
+ (r'\`([,%`abfnrtv\-+;])', String.Escape),
+ include('garbage'),
+ ],
+ 'incomment': [
+ (r'^\s*\*/', Comment.Multiline, '#pop'),
+ (r'[^*/]', Comment.Multiline),
+ (r'[*/]', Comment.Multiline)
+ ],
+ 'incontinuation': [
+ (r'^\s*\)', Generic, '#pop'),
+ (r'[^)]', Generic),
+ (r'[)]', Generic),
+ ],
+ 'commands': [
+ (r'(?i)^(\s*)(global|local|static|'
+ r'#AllowSameLineComments|#ClipboardTimeout|#CommentFlag|'
+ r'#ErrorStdOut|#EscapeChar|#HotkeyInterval|#HotkeyModifierTimeout|'
+ r'#Hotstring|#IfWinActive|#IfWinExist|#IfWinNotActive|'
+ r'#IfWinNotExist|#IncludeAgain|#Include|#InstallKeybdHook|'
+ r'#InstallMouseHook|#KeyHistory|#LTrim|#MaxHotkeysPerInterval|'
+ r'#MaxMem|#MaxThreads|#MaxThreadsBuffer|#MaxThreadsPerHotkey|'
+ r'#NoEnv|#NoTrayIcon|#Persistent|#SingleInstance|#UseHook|'
+ r'#WinActivateForce|AutoTrim|BlockInput|Break|Click|ClipWait|'
+ r'Continue|Control|ControlClick|ControlFocus|ControlGetFocus|'
+ r'ControlGetPos|ControlGetText|ControlGet|ControlMove|ControlSend|'
+ r'ControlSendRaw|ControlSetText|CoordMode|Critical|'
+ r'DetectHiddenText|DetectHiddenWindows|Drive|DriveGet|'
+ r'DriveSpaceFree|Edit|Else|EnvAdd|EnvDiv|EnvGet|EnvMult|EnvSet|'
+ r'EnvSub|EnvUpdate|Exit|ExitApp|FileAppend|'
+ r'FileCopy|FileCopyDir|FileCreateDir|FileCreateShortcut|'
+ r'FileDelete|FileGetAttrib|FileGetShortcut|FileGetSize|'
+ r'FileGetTime|FileGetVersion|FileInstall|FileMove|FileMoveDir|'
+ r'FileRead|FileReadLine|FileRecycle|FileRecycleEmpty|'
+ r'FileRemoveDir|FileSelectFile|FileSelectFolder|FileSetAttrib|'
+ r'FileSetTime|FormatTime|GetKeyState|Gosub|Goto|GroupActivate|'
+ r'GroupAdd|GroupClose|GroupDeactivate|Gui|GuiControl|'
+ r'GuiControlGet|Hotkey|IfEqual|IfExist|IfGreaterOrEqual|IfGreater|'
+ r'IfInString|IfLess|IfLessOrEqual|IfMsgBox|IfNotEqual|IfNotExist|'
+ r'IfNotInString|IfWinActive|IfWinExist|IfWinNotActive|'
+ r'IfWinNotExist|If |ImageSearch|IniDelete|IniRead|IniWrite|'
+ r'InputBox|Input|KeyHistory|KeyWait|ListHotkeys|ListLines|'
+ r'ListVars|Loop|Menu|MouseClickDrag|MouseClick|MouseGetPos|'
+ r'MouseMove|MsgBox|OnExit|OutputDebug|Pause|PixelGetColor|'
+ r'PixelSearch|PostMessage|Process|Progress|Random|RegDelete|'
+ r'RegRead|RegWrite|Reload|Repeat|Return|RunAs|RunWait|Run|'
+ r'SendEvent|SendInput|SendMessage|SendMode|SendPlay|SendRaw|Send|'
+ r'SetBatchLines|SetCapslockState|SetControlDelay|'
+ r'SetDefaultMouseSpeed|SetEnv|SetFormat|SetKeyDelay|'
+ r'SetMouseDelay|SetNumlockState|SetScrollLockState|'
+ r'SetStoreCapslockMode|SetTimer|SetTitleMatchMode|'
+ r'SetWinDelay|SetWorkingDir|Shutdown|Sleep|Sort|SoundBeep|'
+ r'SoundGet|SoundGetWaveVolume|SoundPlay|SoundSet|'
+ r'SoundSetWaveVolume|SplashImage|SplashTextOff|SplashTextOn|'
+ r'SplitPath|StatusBarGetText|StatusBarWait|StringCaseSense|'
+ r'StringGetPos|StringLeft|StringLen|StringLower|StringMid|'
+ r'StringReplace|StringRight|StringSplit|StringTrimLeft|'
+ r'StringTrimRight|StringUpper|Suspend|SysGet|Thread|ToolTip|'
+ r'Transform|TrayTip|URLDownloadToFile|While|WinActivate|'
+ r'WinActivateBottom|WinClose|WinGetActiveStats|WinGetActiveTitle|'
+ r'WinGetClass|WinGetPos|WinGetText|WinGetTitle|WinGet|WinHide|'
+ r'WinKill|WinMaximize|WinMenuSelectItem|WinMinimizeAllUndo|'
+ r'WinMinimizeAll|WinMinimize|WinMove|WinRestore|WinSetTitle|'
+ r'WinSet|WinShow|WinWaitActive|WinWaitClose|WinWaitNotActive|'
+ r'WinWait)\b', bygroups(Text, Name.Builtin)),
+ ],
+ 'builtInFunctions': [
+ (r'(?i)(Abs|ACos|Asc|ASin|ATan|Ceil|Chr|Cos|DllCall|Exp|FileExist|'
+ r'Floor|GetKeyState|IL_Add|IL_Create|IL_Destroy|InStr|IsFunc|'
+ r'IsLabel|Ln|Log|LV_Add|LV_Delete|LV_DeleteCol|LV_GetCount|'
+ r'LV_GetNext|LV_GetText|LV_Insert|LV_InsertCol|LV_Modify|'
+ r'LV_ModifyCol|LV_SetImageList|Mod|NumGet|NumPut|OnMessage|'
+ r'RegExMatch|RegExReplace|RegisterCallback|Round|SB_SetIcon|'
+ r'SB_SetParts|SB_SetText|Sin|Sqrt|StrLen|SubStr|Tan|TV_Add|'
+ r'TV_Delete|TV_GetChild|TV_GetCount|TV_GetNext|TV_Get|'
+ r'TV_GetParent|TV_GetPrev|TV_GetSelection|TV_GetText|TV_Modify|'
+ r'VarSetCapacity|WinActive|WinExist|Object|ComObjActive|'
+ r'ComObjArray|ComObjEnwrap|ComObjUnwrap|ComObjParameter|'
+ r'ComObjType|ComObjConnect|ComObjCreate|ComObjGet|ComObjError|'
+ r'ComObjValue|Insert|MinIndex|MaxIndex|Remove|SetCapacity|'
+ r'GetCapacity|GetAddress|_NewEnum|FileOpen|Read|Write|ReadLine|'
+ r'WriteLine|ReadNumType|WriteNumType|RawRead|RawWrite|Seek|Tell|'
+ r'Close|Next|IsObject|StrPut|StrGet|Trim|LTrim|RTrim)\b',
+ Name.Function),
+ ],
+ 'builtInVariables': [
+ (r'(?i)(A_AhkPath|A_AhkVersion|A_AppData|A_AppDataCommon|'
+ r'A_AutoTrim|A_BatchLines|A_CaretX|A_CaretY|A_ComputerName|'
+ r'A_ControlDelay|A_Cursor|A_DDDD|A_DDD|A_DD|A_DefaultMouseSpeed|'
+ r'A_Desktop|A_DesktopCommon|A_DetectHiddenText|'
+ r'A_DetectHiddenWindows|A_EndChar|A_EventInfo|A_ExitReason|'
+ r'A_FormatFloat|A_FormatInteger|A_Gui|A_GuiEvent|A_GuiControl|'
+ r'A_GuiControlEvent|A_GuiHeight|A_GuiWidth|A_GuiX|A_GuiY|A_Hour|'
+ r'A_IconFile|A_IconHidden|A_IconNumber|A_IconTip|A_Index|'
+ r'A_IPAddress1|A_IPAddress2|A_IPAddress3|A_IPAddress4|A_ISAdmin|'
+ r'A_IsCompiled|A_IsCritical|A_IsPaused|A_IsSuspended|A_KeyDelay|'
+ r'A_Language|A_LastError|A_LineFile|A_LineNumber|A_LoopField|'
+ r'A_LoopFileAttrib|A_LoopFileDir|A_LoopFileExt|A_LoopFileFullPath|'
+ r'A_LoopFileLongPath|A_LoopFileName|A_LoopFileShortName|'
+ r'A_LoopFileShortPath|A_LoopFileSize|A_LoopFileSizeKB|'
+ r'A_LoopFileSizeMB|A_LoopFileTimeAccessed|A_LoopFileTimeCreated|'
+ r'A_LoopFileTimeModified|A_LoopReadLine|A_LoopRegKey|'
+ r'A_LoopRegName|A_LoopRegSubkey|A_LoopRegTimeModified|'
+ r'A_LoopRegType|A_MDAY|A_Min|A_MM|A_MMM|A_MMMM|A_Mon|A_MouseDelay|'
+ r'A_MSec|A_MyDocuments|A_Now|A_NowUTC|A_NumBatchLines|A_OSType|'
+ r'A_OSVersion|A_PriorHotkey|A_ProgramFiles|A_Programs|'
+ r'A_ProgramsCommon|A_ScreenHeight|A_ScreenWidth|A_ScriptDir|'
+ r'A_ScriptFullPath|A_ScriptName|A_Sec|A_Space|A_StartMenu|'
+ r'A_StartMenuCommon|A_Startup|A_StartupCommon|A_StringCaseSense|'
+ r'A_Tab|A_Temp|A_ThisFunc|A_ThisHotkey|A_ThisLabel|A_ThisMenu|'
+ r'A_ThisMenuItem|A_ThisMenuItemPos|A_TickCount|A_TimeIdle|'
+ r'A_TimeIdlePhysical|A_TimeSincePriorHotkey|A_TimeSinceThisHotkey|'
+ r'A_TitleMatchMode|A_TitleMatchModeSpeed|A_UserName|A_WDay|'
+ r'A_WinDelay|A_WinDir|A_WorkingDir|A_YDay|A_YEAR|A_YWeek|A_YYYY|'
+ r'Clipboard|ClipboardAll|ComSpec|ErrorLevel|ProgramFiles|True|'
+ r'False|A_IsUnicode|A_FileEncoding|A_OSVersion|A_PtrSize)\b',
+ Name.Variable),
+ ],
+ 'labels': [
+ # hotkeys and labels
+ # technically, hotkey names are limited to named keys and buttons
+ (r'(^\s*)([^:\s("]+?:{1,2})', bygroups(Text, Name.Label)),
+ (r'(^\s*)(::[^:\s]+?::)', bygroups(Text, Name.Label)),
+ ],
+ 'numbers': [
+ (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+ (r'\d+[eE][+-]?[0-9]+', Number.Float),
+ (r'0\d+', Number.Oct),
+ (r'0[xX][a-fA-F0-9]+', Number.Hex),
+ (r'\d+L', Number.Integer.Long),
+ (r'\d+', Number.Integer)
+ ],
+ 'stringescape': [
+ (r'\"\"|\`([,%`abfnrtv])', String.Escape),
+ ],
+ 'strings': [
+ (r'[^"\n]+', String),
+ ],
+ 'dqs': [
+ (r'"', String, '#pop'),
+ include('strings')
+ ],
+ 'garbage': [
+ (r'[^\S\n]', Text),
+ # (r'.', Text), # no cheating
+ ],
+ }
+
+
+class AutoItLexer(RegexLexer):
+ """
+ For `AutoIt <http://www.autoitscript.com/site/autoit/>`_ files.
+
+ AutoIt is a freeware BASIC-like scripting language
+ designed for automating the Windows GUI and general scripting
+
+ .. versionadded:: 1.6
+ """
+ name = 'AutoIt'
+ aliases = ['autoit']
+ filenames = ['*.au3']
+ mimetypes = ['text/x-autoit']
+
+ # Keywords, functions, macros from au3.keywords.properties
+ # which can be found in AutoIt installed directory, e.g.
+ # c:\Program Files (x86)\AutoIt3\SciTE\au3.keywords.properties
+
+ keywords = """\
+ #include-once #include #endregion #forcedef #forceref #region
+ and byref case continueloop dim do else elseif endfunc endif
+ endselect exit exitloop for func global
+ if local next not or return select step
+ then to until wend while exit""".split()
+
+ functions = """\
+ abs acos adlibregister adlibunregister asc ascw asin assign atan
+ autoitsetoption autoitwingettitle autoitwinsettitle beep binary binarylen
+ binarymid binarytostring bitand bitnot bitor bitrotate bitshift bitxor
+ blockinput break call cdtray ceiling chr chrw clipget clipput consoleread
+ consolewrite consolewriteerror controlclick controlcommand controldisable
+ controlenable controlfocus controlgetfocus controlgethandle controlgetpos
+ controlgettext controlhide controllistview controlmove controlsend
+ controlsettext controlshow controltreeview cos dec dircopy dircreate
+ dirgetsize dirmove dirremove dllcall dllcalladdress dllcallbackfree
+ dllcallbackgetptr dllcallbackregister dllclose dllopen dllstructcreate
+ dllstructgetdata dllstructgetptr dllstructgetsize dllstructsetdata
+ drivegetdrive drivegetfilesystem drivegetlabel drivegetserial drivegettype
+ drivemapadd drivemapdel drivemapget drivesetlabel drivespacefree
+ drivespacetotal drivestatus envget envset envupdate eval execute exp
+ filechangedir fileclose filecopy filecreatentfslink filecreateshortcut
+ filedelete fileexists filefindfirstfile filefindnextfile fileflush
+ filegetattrib filegetencoding filegetlongname filegetpos filegetshortcut
+ filegetshortname filegetsize filegettime filegetversion fileinstall filemove
+ fileopen fileopendialog fileread filereadline filerecycle filerecycleempty
+ filesavedialog fileselectfolder filesetattrib filesetpos filesettime
+ filewrite filewriteline floor ftpsetproxy guicreate guictrlcreateavi
+ guictrlcreatebutton guictrlcreatecheckbox guictrlcreatecombo
+ guictrlcreatecontextmenu guictrlcreatedate guictrlcreatedummy
+ guictrlcreateedit guictrlcreategraphic guictrlcreategroup guictrlcreateicon
+ guictrlcreateinput guictrlcreatelabel guictrlcreatelist
+ guictrlcreatelistview guictrlcreatelistviewitem guictrlcreatemenu
+ guictrlcreatemenuitem guictrlcreatemonthcal guictrlcreateobj
+ guictrlcreatepic guictrlcreateprogress guictrlcreateradio
+ guictrlcreateslider guictrlcreatetab guictrlcreatetabitem
+ guictrlcreatetreeview guictrlcreatetreeviewitem guictrlcreateupdown
+ guictrldelete guictrlgethandle guictrlgetstate guictrlread guictrlrecvmsg
+ guictrlregisterlistviewsort guictrlsendmsg guictrlsendtodummy
+ guictrlsetbkcolor guictrlsetcolor guictrlsetcursor guictrlsetdata
+ guictrlsetdefbkcolor guictrlsetdefcolor guictrlsetfont guictrlsetgraphic
+ guictrlsetimage guictrlsetlimit guictrlsetonevent guictrlsetpos
+ guictrlsetresizing guictrlsetstate guictrlsetstyle guictrlsettip guidelete
+ guigetcursorinfo guigetmsg guigetstyle guiregistermsg guisetaccelerators
+ guisetbkcolor guisetcoord guisetcursor guisetfont guisethelp guiseticon
+ guisetonevent guisetstate guisetstyle guistartgroup guiswitch hex hotkeyset
+ httpsetproxy httpsetuseragent hwnd inetclose inetget inetgetinfo inetgetsize
+ inetread inidelete iniread inireadsection inireadsectionnames
+ inirenamesection iniwrite iniwritesection inputbox int isadmin isarray
+ isbinary isbool isdeclared isdllstruct isfloat ishwnd isint iskeyword
+ isnumber isobj isptr isstring log memgetstats mod mouseclick mouseclickdrag
+ mousedown mousegetcursor mousegetpos mousemove mouseup mousewheel msgbox
+ number objcreate objcreateinterface objevent objevent objget objname
+ onautoitexitregister onautoitexitunregister opt ping pixelchecksum
+ pixelgetcolor pixelsearch pluginclose pluginopen processclose processexists
+ processgetstats processlist processsetpriority processwait processwaitclose
+ progressoff progresson progressset ptr random regdelete regenumkey
+ regenumval regread regwrite round run runas runaswait runwait send
+ sendkeepactive seterror setextended shellexecute shellexecutewait shutdown
+ sin sleep soundplay soundsetwavevolume splashimageon splashoff splashtexton
+ sqrt srandom statusbargettext stderrread stdinwrite stdioclose stdoutread
+ string stringaddcr stringcompare stringformat stringfromasciiarray
+ stringinstr stringisalnum stringisalpha stringisascii stringisdigit
+ stringisfloat stringisint stringislower stringisspace stringisupper
+ stringisxdigit stringleft stringlen stringlower stringmid stringregexp
+ stringregexpreplace stringreplace stringright stringsplit stringstripcr
+ stringstripws stringtoasciiarray stringtobinary stringtrimleft
+ stringtrimright stringupper tan tcpaccept tcpclosesocket tcpconnect
+ tcplisten tcpnametoip tcprecv tcpsend tcpshutdown tcpstartup timerdiff
+ timerinit tooltip traycreateitem traycreatemenu traygetmsg trayitemdelete
+ trayitemgethandle trayitemgetstate trayitemgettext trayitemsetonevent
+ trayitemsetstate trayitemsettext traysetclick trayseticon traysetonevent
+ traysetpauseicon traysetstate traysettooltip traytip ubound udpbind
+ udpclosesocket udpopen udprecv udpsend udpshutdown udpstartup vargettype
+ winactivate winactive winclose winexists winflash wingetcaretpos
+ wingetclasslist wingetclientsize wingethandle wingetpos wingetprocess
+ wingetstate wingettext wingettitle winkill winlist winmenuselectitem
+ winminimizeall winminimizeallundo winmove winsetontop winsetstate
+ winsettitle winsettrans winwait winwaitactive winwaitclose
+ winwaitnotactive""".split()
+
+ macros = """\
+ @appdatacommondir @appdatadir @autoitexe @autoitpid @autoitversion
+ @autoitx64 @com_eventobj @commonfilesdir @compiled @computername @comspec
+ @cpuarch @cr @crlf @desktopcommondir @desktopdepth @desktopdir
+ @desktopheight @desktoprefresh @desktopwidth @documentscommondir @error
+ @exitcode @exitmethod @extended @favoritescommondir @favoritesdir
+ @gui_ctrlhandle @gui_ctrlid @gui_dragfile @gui_dragid @gui_dropid
+ @gui_winhandle @homedrive @homepath @homeshare @hotkeypressed @hour
+ @ipaddress1 @ipaddress2 @ipaddress3 @ipaddress4 @kblayout @lf
+ @logondnsdomain @logondomain @logonserver @mday @min @mon @msec @muilang
+ @mydocumentsdir @numparams @osarch @osbuild @oslang @osservicepack @ostype
+ @osversion @programfilesdir @programscommondir @programsdir @scriptdir
+ @scriptfullpath @scriptlinenumber @scriptname @sec @startmenucommondir
+ @startmenudir @startupcommondir @startupdir @sw_disable @sw_enable @sw_hide
+ @sw_lock @sw_maximize @sw_minimize @sw_restore @sw_show @sw_showdefault
+ @sw_showmaximized @sw_showminimized @sw_showminnoactive @sw_showna
+ @sw_shownoactivate @sw_shownormal @sw_unlock @systemdir @tab @tempdir
+ @tray_id @trayiconflashing @trayiconvisible @username @userprofiledir @wday
+ @windowsdir @workingdir @yday @year""".split()
+
+ tokens = {
+ 'root': [
+ (r';.*\n', Comment.Single),
+ (r'(#comments-start|#cs)(.|\n)*?(#comments-end|#ce)',
+ Comment.Multiline),
+ (r'[\[\]{}(),;]', Punctuation),
+ (r'(and|or|not)\b', Operator.Word),
+ (r'[$|@][a-zA-Z_]\w*', Name.Variable),
+ (r'!=|==|:=|\.=|<<|>>|[-~+/*%=<>&^|?:!.]', Operator),
+ include('commands'),
+ include('labels'),
+ include('builtInFunctions'),
+ include('builtInMarcros'),
+ (r'"', String, combined('stringescape', 'dqs')),
+ include('numbers'),
+ (r'[a-zA-Z_#@$][\w#@$]*', Name),
+ (r'\\|\'', Text),
+ (r'\`([,%`abfnrtv\-+;])', String.Escape),
+ (r'_\n', Text), # Line continuation
+ include('garbage'),
+ ],
+ 'commands': [
+ (r'(?i)(\s*)(%s)\b' % '|'.join(keywords),
+ bygroups(Text, Name.Builtin)),
+ ],
+ 'builtInFunctions': [
+ (r'(?i)(%s)\b' % '|'.join(functions),
+ Name.Function),
+ ],
+ 'builtInMarcros': [
+ (r'(?i)(%s)\b' % '|'.join(macros),
+ Name.Variable.Global),
+ ],
+ 'labels': [
+ # sendkeys
+ (r'(^\s*)(\{\S+?\})', bygroups(Text, Name.Label)),
+ ],
+ 'numbers': [
+ (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+ (r'\d+[eE][+-]?[0-9]+', Number.Float),
+ (r'0\d+', Number.Oct),
+ (r'0[xX][a-fA-F0-9]+', Number.Hex),
+ (r'\d+L', Number.Integer.Long),
+ (r'\d+', Number.Integer)
+ ],
+ 'stringescape': [
+ (r'\"\"|\`([,%`abfnrtv])', String.Escape),
+ ],
+ 'strings': [
+ (r'[^"\n]+', String),
+ ],
+ 'dqs': [
+ (r'"', String, '#pop'),
+ include('strings')
+ ],
+ 'garbage': [
+ (r'[^\S\n]', Text),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/basic.py b/wandb/vendor/pygments/lexers/basic.py
new file mode 100644
index 0000000000000000000000000000000000000000..e6545ee6be73a83382a471b1e83135e631a66cae
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/basic.py
@@ -0,0 +1,500 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.basic
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for BASIC like languages (other than VB.net).
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, default, words, include
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['BlitzBasicLexer', 'BlitzMaxLexer', 'MonkeyLexer', 'CbmBasicV2Lexer',
+ 'QBasicLexer']
+
+
+class BlitzMaxLexer(RegexLexer):
+ """
+ For `BlitzMax <http://blitzbasic.com>`_ source code.
+
+ .. versionadded:: 1.4
+ """
+
+ name = 'BlitzMax'
+ aliases = ['blitzmax', 'bmax']
+ filenames = ['*.bmx']
+ mimetypes = ['text/x-bmx']
+
+ bmax_vopwords = r'\b(Shl|Shr|Sar|Mod)\b'
+ bmax_sktypes = r'@{1,2}|[!#$%]'
+ bmax_lktypes = r'\b(Int|Byte|Short|Float|Double|Long)\b'
+ bmax_name = r'[a-z_]\w*'
+ bmax_var = (r'(%s)(?:(?:([ \t]*)(%s)|([ \t]*:[ \t]*\b(?:Shl|Shr|Sar|Mod)\b)'
+ r'|([ \t]*)(:)([ \t]*)(?:%s|(%s)))(?:([ \t]*)(Ptr))?)') % \
+ (bmax_name, bmax_sktypes, bmax_lktypes, bmax_name)
+ bmax_func = bmax_var + r'?((?:[ \t]|\.\.\n)*)([(])'
+
+ flags = re.MULTILINE | re.IGNORECASE
+ tokens = {
+ 'root': [
+ # Text
+ (r'[ \t]+', Text),
+ (r'\.\.\n', Text), # Line continuation
+ # Comments
+ (r"'.*?\n", Comment.Single),
+ (r'([ \t]*)\bRem\n(\n|.)*?\s*\bEnd([ \t]*)Rem', Comment.Multiline),
+ # Data types
+ ('"', String.Double, 'string'),
+ # Numbers
+ (r'[0-9]+\.[0-9]*(?!\.)', Number.Float),
+ (r'\.[0-9]*(?!\.)', Number.Float),
+ (r'[0-9]+', Number.Integer),
+ (r'\$[0-9a-f]+', Number.Hex),
+ (r'\%[10]+', Number.Bin),
+ # Other
+ (r'(?:(?:(:)?([ \t]*)(:?%s|([+\-*/&|~]))|Or|And|Not|[=<>^]))' %
+ (bmax_vopwords), Operator),
+ (r'[(),.:\[\]]', Punctuation),
+ (r'(?:#[\w \t]*)', Name.Label),
+ (r'(?:\?[\w \t]*)', Comment.Preproc),
+ # Identifiers
+ (r'\b(New)\b([ \t]?)([(]?)(%s)' % (bmax_name),
+ bygroups(Keyword.Reserved, Text, Punctuation, Name.Class)),
+ (r'\b(Import|Framework|Module)([ \t]+)(%s\.%s)' %
+ (bmax_name, bmax_name),
+ bygroups(Keyword.Reserved, Text, Keyword.Namespace)),
+ (bmax_func, bygroups(Name.Function, Text, Keyword.Type,
+ Operator, Text, Punctuation, Text,
+ Keyword.Type, Name.Class, Text,
+ Keyword.Type, Text, Punctuation)),
+ (bmax_var, bygroups(Name.Variable, Text, Keyword.Type, Operator,
+ Text, Punctuation, Text, Keyword.Type,
+ Name.Class, Text, Keyword.Type)),
+ (r'\b(Type|Extends)([ \t]+)(%s)' % (bmax_name),
+ bygroups(Keyword.Reserved, Text, Name.Class)),
+ # Keywords
+ (r'\b(Ptr)\b', Keyword.Type),
+ (r'\b(Pi|True|False|Null|Self|Super)\b', Keyword.Constant),
+ (r'\b(Local|Global|Const|Field)\b', Keyword.Declaration),
+ (words((
+ 'TNullMethodException', 'TNullFunctionException',
+ 'TNullObjectException', 'TArrayBoundsException',
+ 'TRuntimeException'), prefix=r'\b', suffix=r'\b'), Name.Exception),
+ (words((
+ 'Strict', 'SuperStrict', 'Module', 'ModuleInfo',
+ 'End', 'Return', 'Continue', 'Exit', 'Public', 'Private',
+ 'Var', 'VarPtr', 'Chr', 'Len', 'Asc', 'SizeOf', 'Sgn', 'Abs', 'Min', 'Max',
+ 'New', 'Release', 'Delete', 'Incbin', 'IncbinPtr', 'IncbinLen',
+ 'Framework', 'Include', 'Import', 'Extern', 'EndExtern',
+ 'Function', 'EndFunction', 'Type', 'EndType', 'Extends', 'Method', 'EndMethod',
+ 'Abstract', 'Final', 'If', 'Then', 'Else', 'ElseIf', 'EndIf',
+ 'For', 'To', 'Next', 'Step', 'EachIn', 'While', 'Wend', 'EndWhile',
+ 'Repeat', 'Until', 'Forever', 'Select', 'Case', 'Default', 'EndSelect',
+ 'Try', 'Catch', 'EndTry', 'Throw', 'Assert', 'Goto', 'DefData', 'ReadData',
+ 'RestoreData'), prefix=r'\b', suffix=r'\b'),
+ Keyword.Reserved),
+ # Final resolve (for variable names and such)
+ (r'(%s)' % (bmax_name), Name.Variable),
+ ],
+ 'string': [
+ (r'""', String.Double),
+ (r'"C?', String.Double, '#pop'),
+ (r'[^"]+', String.Double),
+ ],
+ }
+
+
+class BlitzBasicLexer(RegexLexer):
+ """
+ For `BlitzBasic <http://blitzbasic.com>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'BlitzBasic'
+ aliases = ['blitzbasic', 'b3d', 'bplus']
+ filenames = ['*.bb', '*.decls']
+ mimetypes = ['text/x-bb']
+
+ bb_sktypes = r'@{1,2}|[#$%]'
+ bb_name = r'[a-z]\w*'
+ bb_var = (r'(%s)(?:([ \t]*)(%s)|([ \t]*)([.])([ \t]*)(?:(%s)))?') % \
+ (bb_name, bb_sktypes, bb_name)
+
+ flags = re.MULTILINE | re.IGNORECASE
+ tokens = {
+ 'root': [
+ # Text
+ (r'[ \t]+', Text),
+ # Comments
+ (r";.*?\n", Comment.Single),
+ # Data types
+ ('"', String.Double, 'string'),
+ # Numbers
+ (r'[0-9]+\.[0-9]*(?!\.)', Number.Float),
+ (r'\.[0-9]+(?!\.)', Number.Float),
+ (r'[0-9]+', Number.Integer),
+ (r'\$[0-9a-f]+', Number.Hex),
+ (r'\%[10]+', Number.Bin),
+ # Other
+ (words(('Shl', 'Shr', 'Sar', 'Mod', 'Or', 'And', 'Not',
+ 'Abs', 'Sgn', 'Handle', 'Int', 'Float', 'Str',
+ 'First', 'Last', 'Before', 'After'),
+ prefix=r'\b', suffix=r'\b'),
+ Operator),
+ (r'([+\-*/~=<>^])', Operator),
+ (r'[(),:\[\]\\]', Punctuation),
+ (r'\.([ \t]*)(%s)' % bb_name, Name.Label),
+ # Identifiers
+ (r'\b(New)\b([ \t]+)(%s)' % (bb_name),
+ bygroups(Keyword.Reserved, Text, Name.Class)),
+ (r'\b(Gosub|Goto)\b([ \t]+)(%s)' % (bb_name),
+ bygroups(Keyword.Reserved, Text, Name.Label)),
+ (r'\b(Object)\b([ \t]*)([.])([ \t]*)(%s)\b' % (bb_name),
+ bygroups(Operator, Text, Punctuation, Text, Name.Class)),
+ (r'\b%s\b([ \t]*)(\()' % bb_var,
+ bygroups(Name.Function, Text, Keyword.Type, Text, Punctuation,
+ Text, Name.Class, Text, Punctuation)),
+ (r'\b(Function)\b([ \t]+)%s' % bb_var,
+ bygroups(Keyword.Reserved, Text, Name.Function, Text, Keyword.Type,
+ Text, Punctuation, Text, Name.Class)),
+ (r'\b(Type)([ \t]+)(%s)' % (bb_name),
+ bygroups(Keyword.Reserved, Text, Name.Class)),
+ # Keywords
+ (r'\b(Pi|True|False|Null)\b', Keyword.Constant),
+ (r'\b(Local|Global|Const|Field|Dim)\b', Keyword.Declaration),
+ (words((
+ 'End', 'Return', 'Exit', 'Chr', 'Len', 'Asc', 'New', 'Delete', 'Insert',
+ 'Include', 'Function', 'Type', 'If', 'Then', 'Else', 'ElseIf', 'EndIf',
+ 'For', 'To', 'Next', 'Step', 'Each', 'While', 'Wend',
+ 'Repeat', 'Until', 'Forever', 'Select', 'Case', 'Default',
+ 'Goto', 'Gosub', 'Data', 'Read', 'Restore'), prefix=r'\b', suffix=r'\b'),
+ Keyword.Reserved),
+ # Final resolve (for variable names and such)
+ # (r'(%s)' % (bb_name), Name.Variable),
+ (bb_var, bygroups(Name.Variable, Text, Keyword.Type,
+ Text, Punctuation, Text, Name.Class)),
+ ],
+ 'string': [
+ (r'""', String.Double),
+ (r'"C?', String.Double, '#pop'),
+ (r'[^"]+', String.Double),
+ ],
+ }
+
+
+class MonkeyLexer(RegexLexer):
+ """
+ For
+ `Monkey <https://en.wikipedia.org/wiki/Monkey_(programming_language)>`_
+ source code.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'Monkey'
+ aliases = ['monkey']
+ filenames = ['*.monkey']
+ mimetypes = ['text/x-monkey']
+
+ name_variable = r'[a-z_]\w*'
+ name_function = r'[A-Z]\w*'
+ name_constant = r'[A-Z_][A-Z0-9_]*'
+ name_class = r'[A-Z]\w*'
+ name_module = r'[a-z0-9_]*'
+
+ keyword_type = r'(?:Int|Float|String|Bool|Object|Array|Void)'
+ # ? == Bool // % == Int // # == Float // $ == String
+ keyword_type_special = r'[?%#$]'
+
+ flags = re.MULTILINE
+
+ tokens = {
+ 'root': [
+ # Text
+ (r'\s+', Text),
+ # Comments
+ (r"'.*", Comment),
+ (r'(?i)^#rem\b', Comment.Multiline, 'comment'),
+ # preprocessor directives
+ (r'(?i)^(?:#If|#ElseIf|#Else|#EndIf|#End|#Print|#Error)\b', Comment.Preproc),
+ # preprocessor variable (any line starting with '#' that is not a directive)
+ (r'^#', Comment.Preproc, 'variables'),
+ # String
+ ('"', String.Double, 'string'),
+ # Numbers
+ (r'[0-9]+\.[0-9]*(?!\.)', Number.Float),
+ (r'\.[0-9]+(?!\.)', Number.Float),
+ (r'[0-9]+', Number.Integer),
+ (r'\$[0-9a-fA-Z]+', Number.Hex),
+ (r'\%[10]+', Number.Bin),
+ # Native data types
+ (r'\b%s\b' % keyword_type, Keyword.Type),
+ # Exception handling
+ (r'(?i)\b(?:Try|Catch|Throw)\b', Keyword.Reserved),
+ (r'Throwable', Name.Exception),
+ # Builtins
+ (r'(?i)\b(?:Null|True|False)\b', Name.Builtin),
+ (r'(?i)\b(?:Self|Super)\b', Name.Builtin.Pseudo),
+ (r'\b(?:HOST|LANG|TARGET|CONFIG)\b', Name.Constant),
+ # Keywords
+ (r'(?i)^(Import)(\s+)(.*)(\n)',
+ bygroups(Keyword.Namespace, Text, Name.Namespace, Text)),
+ (r'(?i)^Strict\b.*\n', Keyword.Reserved),
+ (r'(?i)(Const|Local|Global|Field)(\s+)',
+ bygroups(Keyword.Declaration, Text), 'variables'),
+ (r'(?i)(New|Class|Interface|Extends|Implements)(\s+)',
+ bygroups(Keyword.Reserved, Text), 'classname'),
+ (r'(?i)(Function|Method)(\s+)',
+ bygroups(Keyword.Reserved, Text), 'funcname'),
+ (r'(?i)(?:End|Return|Public|Private|Extern|Property|'
+ r'Final|Abstract)\b', Keyword.Reserved),
+ # Flow Control stuff
+ (r'(?i)(?:If|Then|Else|ElseIf|EndIf|'
+ r'Select|Case|Default|'
+ r'While|Wend|'
+ r'Repeat|Until|Forever|'
+ r'For|To|Until|Step|EachIn|Next|'
+ r'Exit|Continue)\s+', Keyword.Reserved),
+ # not used yet
+ (r'(?i)\b(?:Module|Inline)\b', Keyword.Reserved),
+ # Array
+ (r'[\[\]]', Punctuation),
+ # Other
+ (r'<=|>=|<>|\*=|/=|\+=|-=|&=|~=|\|=|[-&*/^+=<>|~]', Operator),
+ (r'(?i)(?:Not|Mod|Shl|Shr|And|Or)', Operator.Word),
+ (r'[(){}!#,.:]', Punctuation),
+ # catch the rest
+ (r'%s\b' % name_constant, Name.Constant),
+ (r'%s\b' % name_function, Name.Function),
+ (r'%s\b' % name_variable, Name.Variable),
+ ],
+ 'funcname': [
+ (r'(?i)%s\b' % name_function, Name.Function),
+ (r':', Punctuation, 'classname'),
+ (r'\s+', Text),
+ (r'\(', Punctuation, 'variables'),
+ (r'\)', Punctuation, '#pop')
+ ],
+ 'classname': [
+ (r'%s\.' % name_module, Name.Namespace),
+ (r'%s\b' % keyword_type, Keyword.Type),
+ (r'%s\b' % name_class, Name.Class),
+ # array (of given size)
+ (r'(\[)(\s*)(\d*)(\s*)(\])',
+ bygroups(Punctuation, Text, Number.Integer, Text, Punctuation)),
+ # generics
+ (r'\s+(?!<)', Text, '#pop'),
+ (r'<', Punctuation, '#push'),
+ (r'>', Punctuation, '#pop'),
+ (r'\n', Text, '#pop'),
+ default('#pop')
+ ],
+ 'variables': [
+ (r'%s\b' % name_constant, Name.Constant),
+ (r'%s\b' % name_variable, Name.Variable),
+ (r'%s' % keyword_type_special, Keyword.Type),
+ (r'\s+', Text),
+ (r':', Punctuation, 'classname'),
+ (r',', Punctuation, '#push'),
+ default('#pop')
+ ],
+ 'string': [
+ (r'[^"~]+', String.Double),
+ (r'~q|~n|~r|~t|~z|~~', String.Escape),
+ (r'"', String.Double, '#pop'),
+ ],
+ 'comment': [
+ (r'(?i)^#rem.*?', Comment.Multiline, "#push"),
+ (r'(?i)^#end.*?', Comment.Multiline, "#pop"),
+ (r'\n', Comment.Multiline),
+ (r'.+', Comment.Multiline),
+ ],
+ }
+
+
+class CbmBasicV2Lexer(RegexLexer):
+ """
+ For CBM BASIC V2 sources.
+
+ .. versionadded:: 1.6
+ """
+ name = 'CBM BASIC V2'
+ aliases = ['cbmbas']
+ filenames = ['*.bas']
+
+ flags = re.IGNORECASE
+
+ tokens = {
+ 'root': [
+ (r'rem.*\n', Comment.Single),
+ (r'\s+', Text),
+ (r'new|run|end|for|to|next|step|go(to|sub)?|on|return|stop|cont'
+ r'|if|then|input#?|read|wait|load|save|verify|poke|sys|print#?'
+ r'|list|clr|cmd|open|close|get#?', Keyword.Reserved),
+ (r'data|restore|dim|let|def|fn', Keyword.Declaration),
+ (r'tab|spc|sgn|int|abs|usr|fre|pos|sqr|rnd|log|exp|cos|sin|tan|atn'
+ r'|peek|len|val|asc|(str|chr|left|right|mid)\$', Name.Builtin),
+ (r'[-+*/^<>=]', Operator),
+ (r'not|and|or', Operator.Word),
+ (r'"[^"\n]*.', String),
+ (r'\d+|[-+]?\d*\.\d*(e[-+]?\d+)?', Number.Float),
+ (r'[(),:;]', Punctuation),
+ (r'\w+[$%]?', Name),
+ ]
+ }
+
+ def analyse_text(self, text):
+ # if it starts with a line number, it shouldn't be a "modern" Basic
+ # like VB.net
+ if re.match(r'\d+', text):
+ return 0.2
+
+
+class QBasicLexer(RegexLexer):
+ """
+ For
+ `QBasic <http://en.wikipedia.org/wiki/QBasic>`_
+ source code.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'QBasic'
+ aliases = ['qbasic', 'basic']
+ filenames = ['*.BAS', '*.bas']
+ mimetypes = ['text/basic']
+
+ declarations = ('DATA', 'LET')
+
+ functions = (
+ 'ABS', 'ASC', 'ATN', 'CDBL', 'CHR$', 'CINT', 'CLNG',
+ 'COMMAND$', 'COS', 'CSNG', 'CSRLIN', 'CVD', 'CVDMBF', 'CVI',
+ 'CVL', 'CVS', 'CVSMBF', 'DATE$', 'ENVIRON$', 'EOF', 'ERDEV',
+ 'ERDEV$', 'ERL', 'ERR', 'EXP', 'FILEATTR', 'FIX', 'FRE',
+ 'FREEFILE', 'HEX$', 'INKEY$', 'INP', 'INPUT$', 'INSTR', 'INT',
+ 'IOCTL$', 'LBOUND', 'LCASE$', 'LEFT$', 'LEN', 'LOC', 'LOF',
+ 'LOG', 'LPOS', 'LTRIM$', 'MID$', 'MKD$', 'MKDMBF$', 'MKI$',
+ 'MKL$', 'MKS$', 'MKSMBF$', 'OCT$', 'PEEK', 'PEN', 'PLAY',
+ 'PMAP', 'POINT', 'POS', 'RIGHT$', 'RND', 'RTRIM$', 'SADD',
+ 'SCREEN', 'SEEK', 'SETMEM', 'SGN', 'SIN', 'SPACE$', 'SPC',
+ 'SQR', 'STICK', 'STR$', 'STRIG', 'STRING$', 'TAB', 'TAN',
+ 'TIME$', 'TIMER', 'UBOUND', 'UCASE$', 'VAL', 'VARPTR',
+ 'VARPTR$', 'VARSEG'
+ )
+
+ metacommands = ('$DYNAMIC', '$INCLUDE', '$STATIC')
+
+ operators = ('AND', 'EQV', 'IMP', 'NOT', 'OR', 'XOR')
+
+ statements = (
+ 'BEEP', 'BLOAD', 'BSAVE', 'CALL', 'CALL ABSOLUTE',
+ 'CALL INTERRUPT', 'CALLS', 'CHAIN', 'CHDIR', 'CIRCLE', 'CLEAR',
+ 'CLOSE', 'CLS', 'COLOR', 'COM', 'COMMON', 'CONST', 'DATA',
+ 'DATE$', 'DECLARE', 'DEF FN', 'DEF SEG', 'DEFDBL', 'DEFINT',
+ 'DEFLNG', 'DEFSNG', 'DEFSTR', 'DEF', 'DIM', 'DO', 'LOOP',
+ 'DRAW', 'END', 'ENVIRON', 'ERASE', 'ERROR', 'EXIT', 'FIELD',
+ 'FILES', 'FOR', 'NEXT', 'FUNCTION', 'GET', 'GOSUB', 'GOTO',
+ 'IF', 'THEN', 'INPUT', 'INPUT #', 'IOCTL', 'KEY', 'KEY',
+ 'KILL', 'LET', 'LINE', 'LINE INPUT', 'LINE INPUT #', 'LOCATE',
+ 'LOCK', 'UNLOCK', 'LPRINT', 'LSET', 'MID$', 'MKDIR', 'NAME',
+ 'ON COM', 'ON ERROR', 'ON KEY', 'ON PEN', 'ON PLAY',
+ 'ON STRIG', 'ON TIMER', 'ON UEVENT', 'ON', 'OPEN', 'OPEN COM',
+ 'OPTION BASE', 'OUT', 'PAINT', 'PALETTE', 'PCOPY', 'PEN',
+ 'PLAY', 'POKE', 'PRESET', 'PRINT', 'PRINT #', 'PRINT USING',
+ 'PSET', 'PUT', 'PUT', 'RANDOMIZE', 'READ', 'REDIM', 'REM',
+ 'RESET', 'RESTORE', 'RESUME', 'RETURN', 'RMDIR', 'RSET', 'RUN',
+ 'SCREEN', 'SEEK', 'SELECT CASE', 'SHARED', 'SHELL', 'SLEEP',
+ 'SOUND', 'STATIC', 'STOP', 'STRIG', 'SUB', 'SWAP', 'SYSTEM',
+ 'TIME$', 'TIMER', 'TROFF', 'TRON', 'TYPE', 'UEVENT', 'UNLOCK',
+ 'VIEW', 'WAIT', 'WHILE', 'WEND', 'WIDTH', 'WINDOW', 'WRITE'
+ )
+
+ keywords = (
+ 'ACCESS', 'ALIAS', 'ANY', 'APPEND', 'AS', 'BASE', 'BINARY',
+ 'BYVAL', 'CASE', 'CDECL', 'DOUBLE', 'ELSE', 'ELSEIF', 'ENDIF',
+ 'INTEGER', 'IS', 'LIST', 'LOCAL', 'LONG', 'LOOP', 'MOD',
+ 'NEXT', 'OFF', 'ON', 'OUTPUT', 'RANDOM', 'SIGNAL', 'SINGLE',
+ 'STEP', 'STRING', 'THEN', 'TO', 'UNTIL', 'USING', 'WEND'
+ )
+
+ tokens = {
+ 'root': [
+ (r'\n+', Text),
+ (r'\s+', Text.Whitespace),
+ (r'^(\s*)(\d*)(\s*)(REM .*)$',
+ bygroups(Text.Whitespace, Name.Label, Text.Whitespace,
+ Comment.Single)),
+ (r'^(\s*)(\d+)(\s*)',
+ bygroups(Text.Whitespace, Name.Label, Text.Whitespace)),
+ (r'(?=[\s]*)(\w+)(?=[\s]*=)', Name.Variable.Global),
+ (r'(?=[^"]*)\'.*$', Comment.Single),
+ (r'"[^\n"]*"', String.Double),
+ (r'(END)(\s+)(FUNCTION|IF|SELECT|SUB)',
+ bygroups(Keyword.Reserved, Text.Whitespace, Keyword.Reserved)),
+ (r'(DECLARE)(\s+)([A-Z]+)(\s+)(\S+)',
+ bygroups(Keyword.Declaration, Text.Whitespace, Name.Variable,
+ Text.Whitespace, Name)),
+ (r'(DIM)(\s+)(SHARED)(\s+)([^\s(]+)',
+ bygroups(Keyword.Declaration, Text.Whitespace, Name.Variable,
+ Text.Whitespace, Name.Variable.Global)),
+ (r'(DIM)(\s+)([^\s(]+)',
+ bygroups(Keyword.Declaration, Text.Whitespace, Name.Variable.Global)),
+ (r'^(\s*)([a-zA-Z_]+)(\s*)(\=)',
+ bygroups(Text.Whitespace, Name.Variable.Global, Text.Whitespace,
+ Operator)),
+ (r'(GOTO|GOSUB)(\s+)(\w+\:?)',
+ bygroups(Keyword.Reserved, Text.Whitespace, Name.Label)),
+ (r'(SUB)(\s+)(\w+\:?)',
+ bygroups(Keyword.Reserved, Text.Whitespace, Name.Label)),
+ include('declarations'),
+ include('functions'),
+ include('metacommands'),
+ include('operators'),
+ include('statements'),
+ include('keywords'),
+ (r'[a-zA-Z_]\w*[$@#&!]', Name.Variable.Global),
+ (r'[a-zA-Z_]\w*\:', Name.Label),
+ (r'\-?\d*\.\d+[@|#]?', Number.Float),
+ (r'\-?\d+[@|#]', Number.Float),
+ (r'\-?\d+#?', Number.Integer.Long),
+ (r'\-?\d+#?', Number.Integer),
+ (r'!=|==|:=|\.=|<<|>>|[-~+/\\*%=<>&^|?:!.]', Operator),
+ (r'[\[\]{}(),;]', Punctuation),
+ (r'[\w]+', Name.Variable.Global),
+ ],
+ # can't use regular \b because of X$()
+ # XXX: use words() here
+ 'declarations': [
+ (r'\b(%s)(?=\(|\b)' % '|'.join(map(re.escape, declarations)),
+ Keyword.Declaration),
+ ],
+ 'functions': [
+ (r'\b(%s)(?=\(|\b)' % '|'.join(map(re.escape, functions)),
+ Keyword.Reserved),
+ ],
+ 'metacommands': [
+ (r'\b(%s)(?=\(|\b)' % '|'.join(map(re.escape, metacommands)),
+ Keyword.Constant),
+ ],
+ 'operators': [
+ (r'\b(%s)(?=\(|\b)' % '|'.join(map(re.escape, operators)), Operator.Word),
+ ],
+ 'statements': [
+ (r'\b(%s)\b' % '|'.join(map(re.escape, statements)),
+ Keyword.Reserved),
+ ],
+ 'keywords': [
+ (r'\b(%s)\b' % '|'.join(keywords), Keyword),
+ ],
+ }
+
+ def analyse_text(text):
+ if '$DYNAMIC' in text or '$STATIC' in text:
+ return 0.9
diff --git a/wandb/vendor/pygments/lexers/bibtex.py b/wandb/vendor/pygments/lexers/bibtex.py
new file mode 100644
index 0000000000000000000000000000000000000000..a6159f8137f5dfcc2335486ea79ede52ebdd5774
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/bibtex.py
@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.bibtex
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for BibTeX bibliography data and styles
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, ExtendedRegexLexer, include, default, \
+ words
+from pygments.token import Name, Comment, String, Error, Number, Text, \
+ Keyword, Punctuation
+
+__all__ = ['BibTeXLexer', 'BSTLexer']
+
+
+class BibTeXLexer(ExtendedRegexLexer):
+ """
+ A lexer for BibTeX bibliography data format.
+
+ .. versionadded:: 2.2
+ """
+
+ name = 'BibTeX'
+ aliases = ['bib', 'bibtex']
+ filenames = ['*.bib']
+ mimetypes = ["text/x-bibtex"]
+ flags = re.IGNORECASE
+
+ ALLOWED_CHARS = r'@!$&*+\-./:;<>?\[\\\]^`|~'
+ IDENTIFIER = '[{0}][{1}]*'.format('a-z_' + ALLOWED_CHARS, r'\w' + ALLOWED_CHARS)
+
+ def open_brace_callback(self, match, ctx):
+ opening_brace = match.group()
+ ctx.opening_brace = opening_brace
+ yield match.start(), Punctuation, opening_brace
+ ctx.pos = match.end()
+
+ def close_brace_callback(self, match, ctx):
+ closing_brace = match.group()
+ if (
+ ctx.opening_brace == '{' and closing_brace != '}' or
+ ctx.opening_brace == '(' and closing_brace != ')'
+ ):
+ yield match.start(), Error, closing_brace
+ else:
+ yield match.start(), Punctuation, closing_brace
+ del ctx.opening_brace
+ ctx.pos = match.end()
+
+ tokens = {
+ 'root': [
+ include('whitespace'),
+ ('@comment', Comment),
+ ('@preamble', Name.Class, ('closing-brace', 'value', 'opening-brace')),
+ ('@string', Name.Class, ('closing-brace', 'field', 'opening-brace')),
+ ('@' + IDENTIFIER, Name.Class,
+ ('closing-brace', 'command-body', 'opening-brace')),
+ ('.+', Comment),
+ ],
+ 'opening-brace': [
+ include('whitespace'),
+ (r'[{(]', open_brace_callback, '#pop'),
+ ],
+ 'closing-brace': [
+ include('whitespace'),
+ (r'[})]', close_brace_callback, '#pop'),
+ ],
+ 'command-body': [
+ include('whitespace'),
+ (r'[^\s\,\}]+', Name.Label, ('#pop', 'fields')),
+ ],
+ 'fields': [
+ include('whitespace'),
+ (',', Punctuation, 'field'),
+ default('#pop'),
+ ],
+ 'field': [
+ include('whitespace'),
+ (IDENTIFIER, Name.Attribute, ('value', '=')),
+ default('#pop'),
+ ],
+ '=': [
+ include('whitespace'),
+ ('=', Punctuation, '#pop'),
+ ],
+ 'value': [
+ include('whitespace'),
+ (IDENTIFIER, Name.Variable),
+ ('"', String, 'quoted-string'),
+ (r'\{', String, 'braced-string'),
+ (r'[\d]+', Number),
+ ('#', Punctuation),
+ default('#pop'),
+ ],
+ 'quoted-string': [
+ (r'\{', String, 'braced-string'),
+ ('"', String, '#pop'),
+ ('[^\{\"]+', String),
+ ],
+ 'braced-string': [
+ (r'\{', String, '#push'),
+ (r'\}', String, '#pop'),
+ ('[^\{\}]+', String),
+ ],
+ 'whitespace': [
+ (r'\s+', Text),
+ ],
+ }
+
+
+class BSTLexer(RegexLexer):
+ """
+ A lexer for BibTeX bibliography styles.
+
+ .. versionadded:: 2.2
+ """
+
+ name = 'BST'
+ aliases = ['bst', 'bst-pybtex']
+ filenames = ['*.bst']
+ flags = re.IGNORECASE | re.MULTILINE
+
+ tokens = {
+ 'root': [
+ include('whitespace'),
+ (words(['read', 'sort']), Keyword),
+ (words(['execute', 'integers', 'iterate', 'reverse', 'strings']),
+ Keyword, ('group')),
+ (words(['function', 'macro']), Keyword, ('group', 'group')),
+ (words(['entry']), Keyword, ('group', 'group', 'group')),
+ ],
+ 'group': [
+ include('whitespace'),
+ (r'\{', Punctuation, ('#pop', 'group-end', 'body')),
+ ],
+ 'group-end': [
+ include('whitespace'),
+ (r'\}', Punctuation, '#pop'),
+ ],
+ 'body': [
+ include('whitespace'),
+ (r"\'[^#\"\{\}\s]+", Name.Function),
+ (r'[^#\"\{\}\s]+\$', Name.Builtin),
+ (r'[^#\"\{\}\s]+', Name.Variable),
+ (r'"[^\"]*"', String),
+ (r'#-?\d+', Number),
+ (r'\{', Punctuation, ('group-end', 'body')),
+ default('#pop'),
+ ],
+ 'whitespace': [
+ ('\s+', Text),
+ ('%.*?$', Comment.SingleLine),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/business.py b/wandb/vendor/pygments/lexers/business.py
new file mode 100644
index 0000000000000000000000000000000000000000..552f3d9c25033bd31bf756d49c82ac9962467b30
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/business.py
@@ -0,0 +1,612 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.business
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for "business-oriented" languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, words, bygroups
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Error
+
+from pygments.lexers._openedge_builtins import OPENEDGEKEYWORDS
+
+__all__ = ['CobolLexer', 'CobolFreeformatLexer', 'ABAPLexer', 'OpenEdgeLexer',
+ 'GoodDataCLLexer', 'MaqlLexer']
+
+
+class CobolLexer(RegexLexer):
+ """
+ Lexer for OpenCOBOL code.
+
+ .. versionadded:: 1.6
+ """
+ name = 'COBOL'
+ aliases = ['cobol']
+ filenames = ['*.cob', '*.COB', '*.cpy', '*.CPY']
+ mimetypes = ['text/x-cobol']
+ flags = re.IGNORECASE | re.MULTILINE
+
+ # Data Types: by PICTURE and USAGE
+ # Operators: **, *, +, -, /, <, >, <=, >=, =, <>
+ # Logical (?): NOT, AND, OR
+
+ # Reserved words:
+ # http://opencobol.add1tocobol.com/#reserved-words
+ # Intrinsics:
+ # http://opencobol.add1tocobol.com/#does-opencobol-implement-any-intrinsic-functions
+
+ tokens = {
+ 'root': [
+ include('comment'),
+ include('strings'),
+ include('core'),
+ include('nums'),
+ (r'[a-z0-9]([\w\-]*[a-z0-9]+)?', Name.Variable),
+ # (r'[\s]+', Text),
+ (r'[ \t]+', Text),
+ ],
+ 'comment': [
+ (r'(^.{6}[*/].*\n|^.{6}|\*>.*\n)', Comment),
+ ],
+ 'core': [
+ # Figurative constants
+ (r'(^|(?<=[^\w\-]))(ALL\s+)?'
+ r'((ZEROES)|(HIGH-VALUE|LOW-VALUE|QUOTE|SPACE|ZERO)(S)?)'
+ r'\s*($|(?=[^\w\-]))',
+ Name.Constant),
+
+ # Reserved words STATEMENTS and other bolds
+ (words((
+ 'ACCEPT', 'ADD', 'ALLOCATE', 'CALL', 'CANCEL', 'CLOSE', 'COMPUTE',
+ 'CONFIGURATION', 'CONTINUE', 'DATA', 'DELETE', 'DISPLAY', 'DIVIDE',
+ 'DIVISION', 'ELSE', 'END', 'END-ACCEPT',
+ 'END-ADD', 'END-CALL', 'END-COMPUTE', 'END-DELETE', 'END-DISPLAY',
+ 'END-DIVIDE', 'END-EVALUATE', 'END-IF', 'END-MULTIPLY', 'END-OF-PAGE',
+ 'END-PERFORM', 'END-READ', 'END-RETURN', 'END-REWRITE', 'END-SEARCH',
+ 'END-START', 'END-STRING', 'END-SUBTRACT', 'END-UNSTRING', 'END-WRITE',
+ 'ENVIRONMENT', 'EVALUATE', 'EXIT', 'FD', 'FILE', 'FILE-CONTROL', 'FOREVER',
+ 'FREE', 'GENERATE', 'GO', 'GOBACK', 'IDENTIFICATION', 'IF', 'INITIALIZE',
+ 'INITIATE', 'INPUT-OUTPUT', 'INSPECT', 'INVOKE', 'I-O-CONTROL', 'LINKAGE',
+ 'LOCAL-STORAGE', 'MERGE', 'MOVE', 'MULTIPLY', 'OPEN', 'PERFORM',
+ 'PROCEDURE', 'PROGRAM-ID', 'RAISE', 'READ', 'RELEASE', 'RESUME',
+ 'RETURN', 'REWRITE', 'SCREEN', 'SD', 'SEARCH', 'SECTION', 'SET',
+ 'SORT', 'START', 'STOP', 'STRING', 'SUBTRACT', 'SUPPRESS',
+ 'TERMINATE', 'THEN', 'UNLOCK', 'UNSTRING', 'USE', 'VALIDATE',
+ 'WORKING-STORAGE', 'WRITE'), prefix=r'(^|(?<=[^\w\-]))',
+ suffix=r'\s*($|(?=[^\w\-]))'),
+ Keyword.Reserved),
+
+ # Reserved words
+ (words((
+ 'ACCESS', 'ADDRESS', 'ADVANCING', 'AFTER', 'ALL',
+ 'ALPHABET', 'ALPHABETIC', 'ALPHABETIC-LOWER', 'ALPHABETIC-UPPER',
+ 'ALPHANUMERIC', 'ALPHANUMERIC-EDITED', 'ALSO', 'ALTER', 'ALTERNATE'
+ 'ANY', 'ARE', 'AREA', 'AREAS', 'ARGUMENT-NUMBER', 'ARGUMENT-VALUE', 'AS',
+ 'ASCENDING', 'ASSIGN', 'AT', 'AUTO', 'AUTO-SKIP', 'AUTOMATIC',
+ 'AUTOTERMINATE', 'BACKGROUND-COLOR', 'BASED', 'BEEP', 'BEFORE', 'BELL',
+ 'BLANK', 'BLINK', 'BLOCK', 'BOTTOM', 'BY', 'BYTE-LENGTH', 'CHAINING',
+ 'CHARACTER', 'CHARACTERS', 'CLASS', 'CODE', 'CODE-SET', 'COL',
+ 'COLLATING', 'COLS', 'COLUMN', 'COLUMNS', 'COMMA', 'COMMAND-LINE',
+ 'COMMIT', 'COMMON', 'CONSTANT', 'CONTAINS', 'CONTENT', 'CONTROL',
+ 'CONTROLS', 'CONVERTING', 'COPY', 'CORR', 'CORRESPONDING', 'COUNT', 'CRT',
+ 'CURRENCY', 'CURSOR', 'CYCLE', 'DATE', 'DAY', 'DAY-OF-WEEK', 'DE',
+ 'DEBUGGING', 'DECIMAL-POINT', 'DECLARATIVES', 'DEFAULT', 'DELIMITED',
+ 'DELIMITER', 'DEPENDING', 'DESCENDING', 'DETAIL', 'DISK',
+ 'DOWN', 'DUPLICATES', 'DYNAMIC', 'EBCDIC',
+ 'ENTRY', 'ENVIRONMENT-NAME', 'ENVIRONMENT-VALUE', 'EOL', 'EOP',
+ 'EOS', 'ERASE', 'ERROR', 'ESCAPE', 'EXCEPTION',
+ 'EXCLUSIVE', 'EXTEND', 'EXTERNAL', 'FILE-ID', 'FILLER', 'FINAL',
+ 'FIRST', 'FIXED', 'FLOAT-LONG', 'FLOAT-SHORT',
+ 'FOOTING', 'FOR', 'FOREGROUND-COLOR', 'FORMAT', 'FROM', 'FULL',
+ 'FUNCTION', 'FUNCTION-ID', 'GIVING', 'GLOBAL', 'GROUP',
+ 'HEADING', 'HIGHLIGHT', 'I-O', 'ID',
+ 'IGNORE', 'IGNORING', 'IN', 'INDEX', 'INDEXED', 'INDICATE',
+ 'INITIAL', 'INITIALIZED', 'INPUT', 'INTO', 'INTRINSIC', 'INVALID',
+ 'IS', 'JUST', 'JUSTIFIED', 'KEY', 'LABEL',
+ 'LAST', 'LEADING', 'LEFT', 'LENGTH', 'LIMIT', 'LIMITS', 'LINAGE',
+ 'LINAGE-COUNTER', 'LINE', 'LINES', 'LOCALE', 'LOCK',
+ 'LOWLIGHT', 'MANUAL', 'MEMORY', 'MINUS', 'MODE', 'MULTIPLE',
+ 'NATIONAL', 'NATIONAL-EDITED', 'NATIVE', 'NEGATIVE', 'NEXT', 'NO',
+ 'NULL', 'NULLS', 'NUMBER', 'NUMBERS', 'NUMERIC', 'NUMERIC-EDITED',
+ 'OBJECT-COMPUTER', 'OCCURS', 'OF', 'OFF', 'OMITTED', 'ON', 'ONLY',
+ 'OPTIONAL', 'ORDER', 'ORGANIZATION', 'OTHER', 'OUTPUT', 'OVERFLOW',
+ 'OVERLINE', 'PACKED-DECIMAL', 'PADDING', 'PAGE', 'PARAGRAPH',
+ 'PLUS', 'POINTER', 'POSITION', 'POSITIVE', 'PRESENT', 'PREVIOUS',
+ 'PRINTER', 'PRINTING', 'PROCEDURE-POINTER', 'PROCEDURES',
+ 'PROCEED', 'PROGRAM', 'PROGRAM-POINTER', 'PROMPT', 'QUOTE',
+ 'QUOTES', 'RANDOM', 'RD', 'RECORD', 'RECORDING', 'RECORDS', 'RECURSIVE',
+ 'REDEFINES', 'REEL', 'REFERENCE', 'RELATIVE', 'REMAINDER', 'REMOVAL',
+ 'RENAMES', 'REPLACING', 'REPORT', 'REPORTING', 'REPORTS', 'REPOSITORY',
+ 'REQUIRED', 'RESERVE', 'RETURNING', 'REVERSE-VIDEO', 'REWIND',
+ 'RIGHT', 'ROLLBACK', 'ROUNDED', 'RUN', 'SAME', 'SCROLL',
+ 'SECURE', 'SEGMENT-LIMIT', 'SELECT', 'SENTENCE', 'SEPARATE',
+ 'SEQUENCE', 'SEQUENTIAL', 'SHARING', 'SIGN', 'SIGNED', 'SIGNED-INT',
+ 'SIGNED-LONG', 'SIGNED-SHORT', 'SIZE', 'SORT-MERGE', 'SOURCE',
+ 'SOURCE-COMPUTER', 'SPECIAL-NAMES', 'STANDARD',
+ 'STANDARD-1', 'STANDARD-2', 'STATUS', 'SUM',
+ 'SYMBOLIC', 'SYNC', 'SYNCHRONIZED', 'TALLYING', 'TAPE',
+ 'TEST', 'THROUGH', 'THRU', 'TIME', 'TIMES', 'TO', 'TOP', 'TRAILING',
+ 'TRANSFORM', 'TYPE', 'UNDERLINE', 'UNIT', 'UNSIGNED',
+ 'UNSIGNED-INT', 'UNSIGNED-LONG', 'UNSIGNED-SHORT', 'UNTIL', 'UP',
+ 'UPDATE', 'UPON', 'USAGE', 'USING', 'VALUE', 'VALUES', 'VARYING',
+ 'WAIT', 'WHEN', 'WITH', 'WORDS', 'YYYYDDD', 'YYYYMMDD'),
+ prefix=r'(^|(?<=[^\w\-]))', suffix=r'\s*($|(?=[^\w\-]))'),
+ Keyword.Pseudo),
+
+ # inactive reserved words
+ (words((
+ 'ACTIVE-CLASS', 'ALIGNED', 'ANYCASE', 'ARITHMETIC', 'ATTRIBUTE',
+ 'B-AND', 'B-NOT', 'B-OR', 'B-XOR', 'BIT', 'BOOLEAN', 'CD', 'CENTER',
+ 'CF', 'CH', 'CHAIN', 'CLASS-ID', 'CLASSIFICATION', 'COMMUNICATION',
+ 'CONDITION', 'DATA-POINTER', 'DESTINATION', 'DISABLE', 'EC', 'EGI',
+ 'EMI', 'ENABLE', 'END-RECEIVE', 'ENTRY-CONVENTION', 'EO', 'ESI',
+ 'EXCEPTION-OBJECT', 'EXPANDS', 'FACTORY', 'FLOAT-BINARY-16',
+ 'FLOAT-BINARY-34', 'FLOAT-BINARY-7', 'FLOAT-DECIMAL-16',
+ 'FLOAT-DECIMAL-34', 'FLOAT-EXTENDED', 'FORMAT', 'FUNCTION-POINTER',
+ 'GET', 'GROUP-USAGE', 'IMPLEMENTS', 'INFINITY', 'INHERITS',
+ 'INTERFACE', 'INTERFACE-ID', 'INVOKE', 'LC_ALL', 'LC_COLLATE',
+ 'LC_CTYPE', 'LC_MESSAGES', 'LC_MONETARY', 'LC_NUMERIC', 'LC_TIME',
+ 'LINE-COUNTER', 'MESSAGE', 'METHOD', 'METHOD-ID', 'NESTED', 'NONE',
+ 'NORMAL', 'OBJECT', 'OBJECT-REFERENCE', 'OPTIONS', 'OVERRIDE',
+ 'PAGE-COUNTER', 'PF', 'PH', 'PROPERTY', 'PROTOTYPE', 'PURGE',
+ 'QUEUE', 'RAISE', 'RAISING', 'RECEIVE', 'RELATION', 'REPLACE',
+ 'REPRESENTS-NOT-A-NUMBER', 'RESET', 'RESUME', 'RETRY', 'RF', 'RH',
+ 'SECONDS', 'SEGMENT', 'SELF', 'SEND', 'SOURCES', 'STATEMENT',
+ 'STEP', 'STRONG', 'SUB-QUEUE-1', 'SUB-QUEUE-2', 'SUB-QUEUE-3',
+ 'SUPER', 'SYMBOL', 'SYSTEM-DEFAULT', 'TABLE', 'TERMINAL', 'TEXT',
+ 'TYPEDEF', 'UCS-4', 'UNIVERSAL', 'USER-DEFAULT', 'UTF-16', 'UTF-8',
+ 'VAL-STATUS', 'VALID', 'VALIDATE', 'VALIDATE-STATUS'),
+ prefix=r'(^|(?<=[^\w\-]))', suffix=r'\s*($|(?=[^\w\-]))'),
+ Error),
+
+ # Data Types
+ (r'(^|(?<=[^\w\-]))'
+ r'(PIC\s+.+?(?=(\s|\.\s))|PICTURE\s+.+?(?=(\s|\.\s))|'
+ r'(COMPUTATIONAL)(-[1-5X])?|(COMP)(-[1-5X])?|'
+ r'BINARY-C-LONG|'
+ r'BINARY-CHAR|BINARY-DOUBLE|BINARY-LONG|BINARY-SHORT|'
+ r'BINARY)\s*($|(?=[^\w\-]))', Keyword.Type),
+
+ # Operators
+ (r'(\*\*|\*|\+|-|/|<=|>=|<|>|==|/=|=)', Operator),
+
+ # (r'(::)', Keyword.Declaration),
+
+ (r'([(),;:&%.])', Punctuation),
+
+ # Intrinsics
+ (r'(^|(?<=[^\w\-]))(ABS|ACOS|ANNUITY|ASIN|ATAN|BYTE-LENGTH|'
+ r'CHAR|COMBINED-DATETIME|CONCATENATE|COS|CURRENT-DATE|'
+ r'DATE-OF-INTEGER|DATE-TO-YYYYMMDD|DAY-OF-INTEGER|DAY-TO-YYYYDDD|'
+ r'EXCEPTION-(?:FILE|LOCATION|STATEMENT|STATUS)|EXP10|EXP|E|'
+ r'FACTORIAL|FRACTION-PART|INTEGER-OF-(?:DATE|DAY|PART)|INTEGER|'
+ r'LENGTH|LOCALE-(?:DATE|TIME(?:-FROM-SECONDS)?)|LOG(?:10)?|'
+ r'LOWER-CASE|MAX|MEAN|MEDIAN|MIDRANGE|MIN|MOD|NUMVAL(?:-C)?|'
+ r'ORD(?:-MAX|-MIN)?|PI|PRESENT-VALUE|RANDOM|RANGE|REM|REVERSE|'
+ r'SECONDS-FROM-FORMATTED-TIME|SECONDS-PAST-MIDNIGHT|SIGN|SIN|SQRT|'
+ r'STANDARD-DEVIATION|STORED-CHAR-LENGTH|SUBSTITUTE(?:-CASE)?|'
+ r'SUM|TAN|TEST-DATE-YYYYMMDD|TEST-DAY-YYYYDDD|TRIM|'
+ r'UPPER-CASE|VARIANCE|WHEN-COMPILED|YEAR-TO-YYYY)\s*'
+ r'($|(?=[^\w\-]))', Name.Function),
+
+ # Booleans
+ (r'(^|(?<=[^\w\-]))(true|false)\s*($|(?=[^\w\-]))', Name.Builtin),
+ # Comparing Operators
+ (r'(^|(?<=[^\w\-]))(equal|equals|ne|lt|le|gt|ge|'
+ r'greater|less|than|not|and|or)\s*($|(?=[^\w\-]))', Operator.Word),
+ ],
+
+ # \"[^\"\n]*\"|\'[^\'\n]*\'
+ 'strings': [
+ # apparently strings can be delimited by EOL if they are continued
+ # in the next line
+ (r'"[^"\n]*("|\n)', String.Double),
+ (r"'[^'\n]*('|\n)", String.Single),
+ ],
+
+ 'nums': [
+ (r'\d+(\s*|\.$|$)', Number.Integer),
+ (r'[+-]?\d*\.\d+(E[-+]?\d+)?', Number.Float),
+ (r'[+-]?\d+\.\d*(E[-+]?\d+)?', Number.Float),
+ ],
+ }
+
+
+class CobolFreeformatLexer(CobolLexer):
+ """
+ Lexer for Free format OpenCOBOL code.
+
+ .. versionadded:: 1.6
+ """
+ name = 'COBOLFree'
+ aliases = ['cobolfree']
+ filenames = ['*.cbl', '*.CBL']
+ mimetypes = []
+ flags = re.IGNORECASE | re.MULTILINE
+
+ tokens = {
+ 'comment': [
+ (r'(\*>.*\n|^\w*\*.*$)', Comment),
+ ],
+ }
+
+
+class ABAPLexer(RegexLexer):
+ """
+ Lexer for ABAP, SAP's integrated language.
+
+ .. versionadded:: 1.1
+ """
+ name = 'ABAP'
+ aliases = ['abap']
+ filenames = ['*.abap', '*.ABAP']
+ mimetypes = ['text/x-abap']
+
+ flags = re.IGNORECASE | re.MULTILINE
+
+ tokens = {
+ 'common': [
+ (r'\s+', Text),
+ (r'^\*.*$', Comment.Single),
+ (r'\".*?\n', Comment.Single),
+ (r'##\w+', Comment.Special),
+ ],
+ 'variable-names': [
+ (r'<\S+>', Name.Variable),
+ (r'\w[\w~]*(?:(\[\])|->\*)?', Name.Variable),
+ ],
+ 'root': [
+ include('common'),
+ # function calls
+ (r'CALL\s+(?:BADI|CUSTOMER-FUNCTION|FUNCTION)',
+ Keyword),
+ (r'(CALL\s+(?:DIALOG|SCREEN|SUBSCREEN|SELECTION-SCREEN|'
+ r'TRANSACTION|TRANSFORMATION))\b',
+ Keyword),
+ (r'(FORM|PERFORM)(\s+)(\w+)',
+ bygroups(Keyword, Text, Name.Function)),
+ (r'(PERFORM)(\s+)(\()(\w+)(\))',
+ bygroups(Keyword, Text, Punctuation, Name.Variable, Punctuation)),
+ (r'(MODULE)(\s+)(\S+)(\s+)(INPUT|OUTPUT)',
+ bygroups(Keyword, Text, Name.Function, Text, Keyword)),
+
+ # method implementation
+ (r'(METHOD)(\s+)([\w~]+)',
+ bygroups(Keyword, Text, Name.Function)),
+ # method calls
+ (r'(\s+)([\w\-]+)([=\-]>)([\w\-~]+)',
+ bygroups(Text, Name.Variable, Operator, Name.Function)),
+ # call methodnames returning style
+ (r'(?<=(=|-)>)([\w\-~]+)(?=\()', Name.Function),
+
+ # text elements
+ (r'(TEXT)(-)(\d{3})',
+ bygroups(Keyword, Punctuation, Number.Integer)),
+ (r'(TEXT)(-)(\w{3})',
+ bygroups(Keyword, Punctuation, Name.Variable)),
+
+ # keywords with dashes in them.
+ # these need to be first, because for instance the -ID part
+ # of MESSAGE-ID wouldn't get highlighted if MESSAGE was
+ # first in the list of keywords.
+ (r'(ADD-CORRESPONDING|AUTHORITY-CHECK|'
+ r'CLASS-DATA|CLASS-EVENTS|CLASS-METHODS|CLASS-POOL|'
+ r'DELETE-ADJACENT|DIVIDE-CORRESPONDING|'
+ r'EDITOR-CALL|ENHANCEMENT-POINT|ENHANCEMENT-SECTION|EXIT-COMMAND|'
+ r'FIELD-GROUPS|FIELD-SYMBOLS|FUNCTION-POOL|'
+ r'INTERFACE-POOL|INVERTED-DATE|'
+ r'LOAD-OF-PROGRAM|LOG-POINT|'
+ r'MESSAGE-ID|MOVE-CORRESPONDING|MULTIPLY-CORRESPONDING|'
+ r'NEW-LINE|NEW-PAGE|NEW-SECTION|NO-EXTENSION|'
+ r'OUTPUT-LENGTH|PRINT-CONTROL|'
+ r'SELECT-OPTIONS|START-OF-SELECTION|SUBTRACT-CORRESPONDING|'
+ r'SYNTAX-CHECK|SYSTEM-EXCEPTIONS|'
+ r'TYPE-POOL|TYPE-POOLS|NO-DISPLAY'
+ r')\b', Keyword),
+
+ # keyword kombinations
+ (r'(?<![-\>])(CREATE\s+(PUBLIC|PRIVATE|DATA|OBJECT)|'
+ r'(PUBLIC|PRIVATE|PROTECTED)\s+SECTION|'
+ r'(TYPE|LIKE)\s+((LINE\s+OF|REF\s+TO|'
+ r'(SORTED|STANDARD|HASHED)\s+TABLE\s+OF))?|'
+ r'FROM\s+(DATABASE|MEMORY)|CALL\s+METHOD|'
+ r'(GROUP|ORDER) BY|HAVING|SEPARATED BY|'
+ r'GET\s+(BADI|BIT|CURSOR|DATASET|LOCALE|PARAMETER|'
+ r'PF-STATUS|(PROPERTY|REFERENCE)\s+OF|'
+ r'RUN\s+TIME|TIME\s+(STAMP)?)?|'
+ r'SET\s+(BIT|BLANK\s+LINES|COUNTRY|CURSOR|DATASET|EXTENDED\s+CHECK|'
+ r'HANDLER|HOLD\s+DATA|LANGUAGE|LEFT\s+SCROLL-BOUNDARY|'
+ r'LOCALE|MARGIN|PARAMETER|PF-STATUS|PROPERTY\s+OF|'
+ r'RUN\s+TIME\s+(ANALYZER|CLOCK\s+RESOLUTION)|SCREEN|'
+ r'TITLEBAR|UPADTE\s+TASK\s+LOCAL|USER-COMMAND)|'
+ r'CONVERT\s+((INVERTED-)?DATE|TIME|TIME\s+STAMP|TEXT)|'
+ r'(CLOSE|OPEN)\s+(DATASET|CURSOR)|'
+ r'(TO|FROM)\s+(DATA BUFFER|INTERNAL TABLE|MEMORY ID|'
+ r'DATABASE|SHARED\s+(MEMORY|BUFFER))|'
+ r'DESCRIBE\s+(DISTANCE\s+BETWEEN|FIELD|LIST|TABLE)|'
+ r'FREE\s(MEMORY|OBJECT)?|'
+ r'PROCESS\s+(BEFORE\s+OUTPUT|AFTER\s+INPUT|'
+ r'ON\s+(VALUE-REQUEST|HELP-REQUEST))|'
+ r'AT\s+(LINE-SELECTION|USER-COMMAND|END\s+OF|NEW)|'
+ r'AT\s+SELECTION-SCREEN(\s+(ON(\s+(BLOCK|(HELP|VALUE)-REQUEST\s+FOR|'
+ r'END\s+OF|RADIOBUTTON\s+GROUP))?|OUTPUT))?|'
+ r'SELECTION-SCREEN:?\s+((BEGIN|END)\s+OF\s+((TABBED\s+)?BLOCK|LINE|'
+ r'SCREEN)|COMMENT|FUNCTION\s+KEY|'
+ r'INCLUDE\s+BLOCKS|POSITION|PUSHBUTTON|'
+ r'SKIP|ULINE)|'
+ r'LEAVE\s+(LIST-PROCESSING|PROGRAM|SCREEN|'
+ r'TO LIST-PROCESSING|TO TRANSACTION)'
+ r'(ENDING|STARTING)\s+AT|'
+ r'FORMAT\s+(COLOR|INTENSIFIED|INVERSE|HOTSPOT|INPUT|FRAMES|RESET)|'
+ r'AS\s+(CHECKBOX|SUBSCREEN|WINDOW)|'
+ r'WITH\s+(((NON-)?UNIQUE)?\s+KEY|FRAME)|'
+ r'(BEGIN|END)\s+OF|'
+ r'DELETE(\s+ADJACENT\s+DUPLICATES\sFROM)?|'
+ r'COMPARING(\s+ALL\s+FIELDS)?|'
+ r'(INSERT|APPEND)(\s+INITIAL\s+LINE\s+(IN)?TO|\s+LINES\s+OF)?|'
+ r'IN\s+((BYTE|CHARACTER)\s+MODE|PROGRAM)|'
+ r'END-OF-(DEFINITION|PAGE|SELECTION)|'
+ r'WITH\s+FRAME(\s+TITLE)|'
+ r'(REPLACE|FIND)\s+((FIRST|ALL)\s+OCCURRENCES?\s+OF\s+)?(SUBSTRING|REGEX)?|'
+ r'MATCH\s+(LENGTH|COUNT|LINE|OFFSET)|'
+ r'(RESPECTING|IGNORING)\s+CASE|'
+ r'IN\s+UPDATE\s+TASK|'
+ r'(SOURCE|RESULT)\s+(XML)?|'
+ r'REFERENCE\s+INTO|'
+
+ # simple kombinations
+ r'AND\s+(MARK|RETURN)|CLIENT\s+SPECIFIED|CORRESPONDING\s+FIELDS\s+OF|'
+ r'IF\s+FOUND|FOR\s+EVENT|INHERITING\s+FROM|LEAVE\s+TO\s+SCREEN|'
+ r'LOOP\s+AT\s+(SCREEN)?|LOWER\s+CASE|MATCHCODE\s+OBJECT|MODIF\s+ID|'
+ r'MODIFY\s+SCREEN|NESTING\s+LEVEL|NO\s+INTERVALS|OF\s+STRUCTURE|'
+ r'RADIOBUTTON\s+GROUP|RANGE\s+OF|REF\s+TO|SUPPRESS DIALOG|'
+ r'TABLE\s+OF|UPPER\s+CASE|TRANSPORTING\s+NO\s+FIELDS|'
+ r'VALUE\s+CHECK|VISIBLE\s+LENGTH|HEADER\s+LINE|COMMON\s+PART)\b', Keyword),
+
+ # single word keywords.
+ (r'(^|(?<=(\s|\.)))(ABBREVIATED|ABSTRACT|ADD|ALIASES|ALIGN|ALPHA|'
+ r'ASSERT|AS|ASSIGN(ING)?|AT(\s+FIRST)?|'
+ r'BACK|BLOCK|BREAK-POINT|'
+ r'CASE|CATCH|CHANGING|CHECK|CLASS|CLEAR|COLLECT|COLOR|COMMIT|'
+ r'CREATE|COMMUNICATION|COMPONENTS?|COMPUTE|CONCATENATE|CONDENSE|'
+ r'CONSTANTS|CONTEXTS|CONTINUE|CONTROLS|COUNTRY|CURRENCY|'
+ r'DATA|DATE|DECIMALS|DEFAULT|DEFINE|DEFINITION|DEFERRED|DEMAND|'
+ r'DETAIL|DIRECTORY|DIVIDE|DO|DUMMY|'
+ r'ELSE(IF)?|ENDAT|ENDCASE|ENDCATCH|ENDCLASS|ENDDO|ENDFORM|ENDFUNCTION|'
+ r'ENDIF|ENDINTERFACE|ENDLOOP|ENDMETHOD|ENDMODULE|ENDSELECT|ENDTRY|ENDWHILE|'
+ r'ENHANCEMENT|EVENTS|EXACT|EXCEPTIONS?|EXIT|EXPONENT|EXPORT|EXPORTING|EXTRACT|'
+ r'FETCH|FIELDS?|FOR|FORM|FORMAT|FREE|FROM|FUNCTION|'
+ r'HIDE|'
+ r'ID|IF|IMPORT|IMPLEMENTATION|IMPORTING|IN|INCLUDE|INCLUDING|'
+ r'INDEX|INFOTYPES|INITIALIZATION|INTERFACE|INTERFACES|INTO|'
+ r'LANGUAGE|LEAVE|LENGTH|LINES|LOAD|LOCAL|'
+ r'JOIN|'
+ r'KEY|'
+ r'NEXT|'
+ r'MAXIMUM|MESSAGE|METHOD[S]?|MINIMUM|MODULE|MODIFIER|MODIFY|MOVE|MULTIPLY|'
+ r'NODES|NUMBER|'
+ r'OBLIGATORY|OBJECT|OF|OFF|ON|OTHERS|OVERLAY|'
+ r'PACK|PAD|PARAMETERS|PERCENTAGE|POSITION|PROGRAM|PROVIDE|PUBLIC|PUT|PF\d\d|'
+ r'RAISE|RAISING|RANGES?|READ|RECEIVE|REDEFINITION|REFRESH|REJECT|REPORT|RESERVE|'
+ r'RESUME|RETRY|RETURN|RETURNING|RIGHT|ROLLBACK|REPLACE|'
+ r'SCROLL|SEARCH|SELECT|SHIFT|SIGN|SINGLE|SIZE|SKIP|SORT|SPLIT|STATICS|STOP|'
+ r'STYLE|SUBMATCHES|SUBMIT|SUBTRACT|SUM(?!\()|SUMMARY|SUMMING|SUPPLY|'
+ r'TABLE|TABLES|TIMESTAMP|TIMES?|TIMEZONE|TITLE|\??TO|'
+ r'TOP-OF-PAGE|TRANSFER|TRANSLATE|TRY|TYPES|'
+ r'ULINE|UNDER|UNPACK|UPDATE|USING|'
+ r'VALUE|VALUES|VIA|VARYING|VARY|'
+ r'WAIT|WHEN|WHERE|WIDTH|WHILE|WITH|WINDOW|WRITE|XSD|ZERO)\b', Keyword),
+
+ # builtins
+ (r'(abs|acos|asin|atan|'
+ r'boolc|boolx|bit_set|'
+ r'char_off|charlen|ceil|cmax|cmin|condense|contains|'
+ r'contains_any_of|contains_any_not_of|concat_lines_of|cos|cosh|'
+ r'count|count_any_of|count_any_not_of|'
+ r'dbmaxlen|distance|'
+ r'escape|exp|'
+ r'find|find_end|find_any_of|find_any_not_of|floor|frac|from_mixed|'
+ r'insert|'
+ r'lines|log|log10|'
+ r'match|matches|'
+ r'nmax|nmin|numofchar|'
+ r'repeat|replace|rescale|reverse|round|'
+ r'segment|shift_left|shift_right|sign|sin|sinh|sqrt|strlen|'
+ r'substring|substring_after|substring_from|substring_before|substring_to|'
+ r'tan|tanh|to_upper|to_lower|to_mixed|translate|trunc|'
+ r'xstrlen)(\()\b', bygroups(Name.Builtin, Punctuation)),
+
+ (r'&[0-9]', Name),
+ (r'[0-9]+', Number.Integer),
+
+ # operators which look like variable names before
+ # parsing variable names.
+ (r'(?<=(\s|.))(AND|OR|EQ|NE|GT|LT|GE|LE|CO|CN|CA|NA|CS|NOT|NS|CP|NP|'
+ r'BYTE-CO|BYTE-CN|BYTE-CA|BYTE-NA|BYTE-CS|BYTE-NS|'
+ r'IS\s+(NOT\s+)?(INITIAL|ASSIGNED|REQUESTED|BOUND))\b', Operator.Word),
+
+ include('variable-names'),
+
+ # standard operators after variable names,
+ # because < and > are part of field symbols.
+ (r'[?*<>=\-+&]', Operator),
+ (r"'(''|[^'])*'", String.Single),
+ (r"`([^`])*`", String.Single),
+ (r"([|}])([^{}|]*?)([|{])",
+ bygroups(Punctuation, String.Single, Punctuation)),
+ (r'[/;:()\[\],.]', Punctuation),
+ (r'(!)(\w+)', bygroups(Operator, Name)),
+ ],
+ }
+
+
+class OpenEdgeLexer(RegexLexer):
+ """
+ Lexer for `OpenEdge ABL (formerly Progress)
+ <http://web.progress.com/en/openedge/abl.html>`_ source code.
+
+ .. versionadded:: 1.5
+ """
+ name = 'OpenEdge ABL'
+ aliases = ['openedge', 'abl', 'progress']
+ filenames = ['*.p', '*.cls']
+ mimetypes = ['text/x-openedge', 'application/x-openedge']
+
+ types = (r'(?i)(^|(?<=[^\w\-]))(CHARACTER|CHAR|CHARA|CHARAC|CHARACT|CHARACTE|'
+ r'COM-HANDLE|DATE|DATETIME|DATETIME-TZ|'
+ r'DECIMAL|DEC|DECI|DECIM|DECIMA|HANDLE|'
+ r'INT64|INTEGER|INT|INTE|INTEG|INTEGE|'
+ r'LOGICAL|LONGCHAR|MEMPTR|RAW|RECID|ROWID)\s*($|(?=[^\w\-]))')
+
+ keywords = words(OPENEDGEKEYWORDS,
+ prefix=r'(?i)(^|(?<=[^\w\-]))',
+ suffix=r'\s*($|(?=[^\w\-]))')
+
+ tokens = {
+ 'root': [
+ (r'/\*', Comment.Multiline, 'comment'),
+ (r'\{', Comment.Preproc, 'preprocessor'),
+ (r'\s*&.*', Comment.Preproc),
+ (r'0[xX][0-9a-fA-F]+[LlUu]*', Number.Hex),
+ (r'(?i)(DEFINE|DEF|DEFI|DEFIN)\b', Keyword.Declaration),
+ (types, Keyword.Type),
+ (keywords, Name.Builtin),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'[0-9]+', Number.Integer),
+ (r'\s+', Text),
+ (r'[+*/=-]', Operator),
+ (r'[.:()]', Punctuation),
+ (r'.', Name.Variable), # Lazy catch-all
+ ],
+ 'comment': [
+ (r'[^*/]', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline)
+ ],
+ 'preprocessor': [
+ (r'[^{}]', Comment.Preproc),
+ (r'\{', Comment.Preproc, '#push'),
+ (r'\}', Comment.Preproc, '#pop'),
+ ],
+ }
+
+
+class GoodDataCLLexer(RegexLexer):
+ """
+ Lexer for `GoodData-CL
+ <http://github.com/gooddata/GoodData-CL/raw/master/cli/src/main/resources/\
+com/gooddata/processor/COMMANDS.txt>`_
+ script files.
+
+ .. versionadded:: 1.4
+ """
+
+ name = 'GoodData-CL'
+ aliases = ['gooddata-cl']
+ filenames = ['*.gdc']
+ mimetypes = ['text/x-gooddata-cl']
+
+ flags = re.IGNORECASE
+ tokens = {
+ 'root': [
+ # Comments
+ (r'#.*', Comment.Single),
+ # Function call
+ (r'[a-z]\w*', Name.Function),
+ # Argument list
+ (r'\(', Punctuation, 'args-list'),
+ # Punctuation
+ (r';', Punctuation),
+ # Space is not significant
+ (r'\s+', Text)
+ ],
+ 'args-list': [
+ (r'\)', Punctuation, '#pop'),
+ (r',', Punctuation),
+ (r'[a-z]\w*', Name.Variable),
+ (r'=', Operator),
+ (r'"', String, 'string-literal'),
+ (r'[0-9]+(?:\.[0-9]+)?(?:e[+-]?[0-9]{1,3})?', Number),
+ # Space is not significant
+ (r'\s', Text)
+ ],
+ 'string-literal': [
+ (r'\\[tnrfbae"\\]', String.Escape),
+ (r'"', String, '#pop'),
+ (r'[^\\"]+', String)
+ ]
+ }
+
+
+class MaqlLexer(RegexLexer):
+ """
+ Lexer for `GoodData MAQL
+ <https://secure.gooddata.com/docs/html/advanced.metric.tutorial.html>`_
+ scripts.
+
+ .. versionadded:: 1.4
+ """
+
+ name = 'MAQL'
+ aliases = ['maql']
+ filenames = ['*.maql']
+ mimetypes = ['text/x-gooddata-maql', 'application/x-gooddata-maql']
+
+ flags = re.IGNORECASE
+ tokens = {
+ 'root': [
+ # IDENTITY
+ (r'IDENTIFIER\b', Name.Builtin),
+ # IDENTIFIER
+ (r'\{[^}]+\}', Name.Variable),
+ # NUMBER
+ (r'[0-9]+(?:\.[0-9]+)?(?:e[+-]?[0-9]{1,3})?', Number),
+ # STRING
+ (r'"', String, 'string-literal'),
+ # RELATION
+ (r'\<\>|\!\=', Operator),
+ (r'\=|\>\=|\>|\<\=|\<', Operator),
+ # :=
+ (r'\:\=', Operator),
+ # OBJECT
+ (r'\[[^]]+\]', Name.Variable.Class),
+ # keywords
+ (words((
+ 'DIMENSION', 'DIMENSIONS', 'BOTTOM', 'METRIC', 'COUNT', 'OTHER',
+ 'FACT', 'WITH', 'TOP', 'OR', 'ATTRIBUTE', 'CREATE', 'PARENT',
+ 'FALSE', 'ROW', 'ROWS', 'FROM', 'ALL', 'AS', 'PF', 'COLUMN',
+ 'COLUMNS', 'DEFINE', 'REPORT', 'LIMIT', 'TABLE', 'LIKE', 'AND',
+ 'BY', 'BETWEEN', 'EXCEPT', 'SELECT', 'MATCH', 'WHERE', 'TRUE',
+ 'FOR', 'IN', 'WITHOUT', 'FILTER', 'ALIAS', 'WHEN', 'NOT', 'ON',
+ 'KEYS', 'KEY', 'FULLSET', 'PRIMARY', 'LABELS', 'LABEL',
+ 'VISUAL', 'TITLE', 'DESCRIPTION', 'FOLDER', 'ALTER', 'DROP',
+ 'ADD', 'DATASET', 'DATATYPE', 'INT', 'BIGINT', 'DOUBLE', 'DATE',
+ 'VARCHAR', 'DECIMAL', 'SYNCHRONIZE', 'TYPE', 'DEFAULT', 'ORDER',
+ 'ASC', 'DESC', 'HYPERLINK', 'INCLUDE', 'TEMPLATE', 'MODIFY'),
+ suffix=r'\b'),
+ Keyword),
+ # FUNCNAME
+ (r'[a-z]\w*\b', Name.Function),
+ # Comments
+ (r'#.*', Comment.Single),
+ # Punctuation
+ (r'[,;()]', Punctuation),
+ # Space is not significant
+ (r'\s+', Text)
+ ],
+ 'string-literal': [
+ (r'\\[tnrfbae"\\]', String.Escape),
+ (r'"', String, '#pop'),
+ (r'[^\\"]+', String)
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/c_cpp.py b/wandb/vendor/pygments/lexers/c_cpp.py
new file mode 100644
index 0000000000000000000000000000000000000000..691f5ab44a28b4fbd37db87f595dd87ad766234a
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/c_cpp.py
@@ -0,0 +1,252 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.c_cpp
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for C/C++ languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, \
+ this, inherit, default, words
+from pygments.util import get_bool_opt
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Error
+
+__all__ = ['CLexer', 'CppLexer']
+
+
+class CFamilyLexer(RegexLexer):
+ """
+ For C family source code. This is used as a base class to avoid repetitious
+ definitions.
+ """
+
+ #: optional Comment or Whitespace
+ _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+'
+
+ # The trailing ?, rather than *, avoids a geometric performance drop here.
+ #: only one /* */ style comment
+ _ws1 = r'\s*(?:/[*].*?[*]/\s*)?'
+
+ tokens = {
+ 'whitespace': [
+ # preprocessor directives: without whitespace
+ ('^#if\s+0', Comment.Preproc, 'if0'),
+ ('^#', Comment.Preproc, 'macro'),
+ # or with whitespace
+ ('^(' + _ws1 + r')(#if\s+0)',
+ bygroups(using(this), Comment.Preproc), 'if0'),
+ ('^(' + _ws1 + ')(#)',
+ bygroups(using(this), Comment.Preproc), 'macro'),
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'\\\n', Text), # line continuation
+ (r'//(\n|[\w\W]*?[^\\]\n)', Comment.Single),
+ (r'/(\\\n)?[*][\w\W]*?[*](\\\n)?/', Comment.Multiline),
+ # Open until EOF, so no ending delimeter
+ (r'/(\\\n)?[*][\w\W]*', Comment.Multiline),
+ ],
+ 'statements': [
+ (r'(L?)(")', bygroups(String.Affix, String), 'string'),
+ (r"(L?)(')(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])(')",
+ bygroups(String.Affix, String.Char, String.Char, String.Char)),
+ (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float),
+ (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+ (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex),
+ (r'0[0-7]+[LlUu]*', Number.Oct),
+ (r'\d+[LlUu]*', Number.Integer),
+ (r'\*/', Error),
+ (r'[~!%^&*+=|?:<>/-]', Operator),
+ (r'[()\[\],.]', Punctuation),
+ (words(('asm', 'auto', 'break', 'case', 'const', 'continue',
+ 'default', 'do', 'else', 'enum', 'extern', 'for', 'goto',
+ 'if', 'register', 'restricted', 'return', 'sizeof',
+ 'static', 'struct', 'switch', 'typedef', 'union',
+ 'volatile', 'while'),
+ suffix=r'\b'), Keyword),
+ (r'(bool|int|long|float|short|double|char|unsigned|signed|void)\b',
+ Keyword.Type),
+ (words(('inline', '_inline', '__inline', 'naked', 'restrict',
+ 'thread', 'typename'), suffix=r'\b'), Keyword.Reserved),
+ # Vector intrinsics
+ (r'(__m(128i|128d|128|64))\b', Keyword.Reserved),
+ # Microsoft-isms
+ (words((
+ 'asm', 'int8', 'based', 'except', 'int16', 'stdcall', 'cdecl',
+ 'fastcall', 'int32', 'declspec', 'finally', 'int64', 'try',
+ 'leave', 'wchar_t', 'w64', 'unaligned', 'raise', 'noop',
+ 'identifier', 'forceinline', 'assume'),
+ prefix=r'__', suffix=r'\b'), Keyword.Reserved),
+ (r'(true|false|NULL)\b', Name.Builtin),
+ (r'([a-zA-Z_]\w*)(\s*)(:)(?!:)', bygroups(Name.Label, Text, Punctuation)),
+ ('[a-zA-Z_]\w*', Name),
+ ],
+ 'root': [
+ include('whitespace'),
+ # functions
+ (r'((?:[\w*\s])+?(?:\s|[*]))' # return arguments
+ r'([a-zA-Z_]\w*)' # method name
+ r'(\s*\([^;]*?\))' # signature
+ r'([^;{]*)(\{)',
+ bygroups(using(this), Name.Function, using(this), using(this),
+ Punctuation),
+ 'function'),
+ # function declarations
+ (r'((?:[\w*\s])+?(?:\s|[*]))' # return arguments
+ r'([a-zA-Z_]\w*)' # method name
+ r'(\s*\([^;]*?\))' # signature
+ r'([^;]*)(;)',
+ bygroups(using(this), Name.Function, using(this), using(this),
+ Punctuation)),
+ default('statement'),
+ ],
+ 'statement': [
+ include('whitespace'),
+ include('statements'),
+ ('[{}]', Punctuation),
+ (';', Punctuation, '#pop'),
+ ],
+ 'function': [
+ include('whitespace'),
+ include('statements'),
+ (';', Punctuation),
+ (r'\{', Punctuation, '#push'),
+ (r'\}', Punctuation, '#pop'),
+ ],
+ 'string': [
+ (r'"', String, '#pop'),
+ (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|'
+ r'u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})', String.Escape),
+ (r'[^\\"\n]+', String), # all other characters
+ (r'\\\n', String), # line continuation
+ (r'\\', String), # stray backslash
+ ],
+ 'macro': [
+ (r'(include)(' + _ws1 + r')([^\n]+)',
+ bygroups(Comment.Preproc, Text, Comment.PreprocFile)),
+ (r'[^/\n]+', Comment.Preproc),
+ (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+ (r'//.*?\n', Comment.Single, '#pop'),
+ (r'/', Comment.Preproc),
+ (r'(?<=\\)\n', Comment.Preproc),
+ (r'\n', Comment.Preproc, '#pop'),
+ ],
+ 'if0': [
+ (r'^\s*#if.*?(?<!\\)\n', Comment.Preproc, '#push'),
+ (r'^\s*#el(?:se|if).*\n', Comment.Preproc, '#pop'),
+ (r'^\s*#endif.*?(?<!\\)\n', Comment.Preproc, '#pop'),
+ (r'.*?\n', Comment),
+ ]
+ }
+
+ stdlib_types = set((
+ 'size_t', 'ssize_t', 'off_t', 'wchar_t', 'ptrdiff_t', 'sig_atomic_t', 'fpos_t',
+ 'clock_t', 'time_t', 'va_list', 'jmp_buf', 'FILE', 'DIR', 'div_t', 'ldiv_t',
+ 'mbstate_t', 'wctrans_t', 'wint_t', 'wctype_t'))
+ c99_types = set((
+ '_Bool', '_Complex', 'int8_t', 'int16_t', 'int32_t', 'int64_t', 'uint8_t',
+ 'uint16_t', 'uint32_t', 'uint64_t', 'int_least8_t', 'int_least16_t',
+ 'int_least32_t', 'int_least64_t', 'uint_least8_t', 'uint_least16_t',
+ 'uint_least32_t', 'uint_least64_t', 'int_fast8_t', 'int_fast16_t', 'int_fast32_t',
+ 'int_fast64_t', 'uint_fast8_t', 'uint_fast16_t', 'uint_fast32_t', 'uint_fast64_t',
+ 'intptr_t', 'uintptr_t', 'intmax_t', 'uintmax_t'))
+ linux_types = set((
+ 'clockid_t', 'cpu_set_t', 'cpumask_t', 'dev_t', 'gid_t', 'id_t', 'ino_t', 'key_t',
+ 'mode_t', 'nfds_t', 'pid_t', 'rlim_t', 'sig_t', 'sighandler_t', 'siginfo_t',
+ 'sigset_t', 'sigval_t', 'socklen_t', 'timer_t', 'uid_t'))
+
+ def __init__(self, **options):
+ self.stdlibhighlighting = get_bool_opt(options, 'stdlibhighlighting', True)
+ self.c99highlighting = get_bool_opt(options, 'c99highlighting', True)
+ self.platformhighlighting = get_bool_opt(options, 'platformhighlighting', True)
+ RegexLexer.__init__(self, **options)
+
+ def get_tokens_unprocessed(self, text):
+ for index, token, value in \
+ RegexLexer.get_tokens_unprocessed(self, text):
+ if token is Name:
+ if self.stdlibhighlighting and value in self.stdlib_types:
+ token = Keyword.Type
+ elif self.c99highlighting and value in self.c99_types:
+ token = Keyword.Type
+ elif self.platformhighlighting and value in self.linux_types:
+ token = Keyword.Type
+ yield index, token, value
+
+
+class CLexer(CFamilyLexer):
+ """
+ For C source code with preprocessor directives.
+ """
+ name = 'C'
+ aliases = ['c']
+ filenames = ['*.c', '*.h', '*.idc']
+ mimetypes = ['text/x-chdr', 'text/x-csrc']
+ priority = 0.1
+
+ def analyse_text(text):
+ if re.search('^\s*#include [<"]', text, re.MULTILINE):
+ return 0.1
+ if re.search('^\s*#ifn?def ', text, re.MULTILINE):
+ return 0.1
+
+
+class CppLexer(CFamilyLexer):
+ """
+ For C++ source code with preprocessor directives.
+ """
+ name = 'C++'
+ aliases = ['cpp', 'c++']
+ filenames = ['*.cpp', '*.hpp', '*.c++', '*.h++',
+ '*.cc', '*.hh', '*.cxx', '*.hxx',
+ '*.C', '*.H', '*.cp', '*.CPP']
+ mimetypes = ['text/x-c++hdr', 'text/x-c++src']
+ priority = 0.1
+
+ tokens = {
+ 'statements': [
+ (words((
+ 'catch', 'const_cast', 'delete', 'dynamic_cast', 'explicit',
+ 'export', 'friend', 'mutable', 'namespace', 'new', 'operator',
+ 'private', 'protected', 'public', 'reinterpret_cast',
+ 'restrict', 'static_cast', 'template', 'this', 'throw', 'throws',
+ 'try', 'typeid', 'typename', 'using', 'virtual',
+ 'constexpr', 'nullptr', 'decltype', 'thread_local',
+ 'alignas', 'alignof', 'static_assert', 'noexcept', 'override',
+ 'final'), suffix=r'\b'), Keyword),
+ (r'char(16_t|32_t)\b', Keyword.Type),
+ (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
+ # C++11 raw strings
+ (r'(R)(")([^\\()\s]{,16})(\()((?:.|\n)*?)(\)\3)(")',
+ bygroups(String.Affix, String, String.Delimiter, String.Delimiter,
+ String, String.Delimiter, String)),
+ # C++11 UTF-8/16/32 strings
+ (r'(u8|u|U)(")', bygroups(String.Affix, String), 'string'),
+ inherit,
+ ],
+ 'root': [
+ inherit,
+ # C++ Microsoft-isms
+ (words(('virtual_inheritance', 'uuidof', 'super', 'single_inheritance',
+ 'multiple_inheritance', 'interface', 'event'),
+ prefix=r'__', suffix=r'\b'), Keyword.Reserved),
+ # Offload C++ extensions, http://offload.codeplay.com/
+ (r'__(offload|blockingoffload|outer)\b', Keyword.Pseudo),
+ ],
+ 'classname': [
+ (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
+ # template specification
+ (r'\s*(?=>)', Text, '#pop'),
+ ],
+ }
+
+ def analyse_text(text):
+ if re.search('#include <[a-z_]+>', text):
+ return 0.2
+ if re.search('using namespace ', text):
+ return 0.4
diff --git a/wandb/vendor/pygments/lexers/c_like.py b/wandb/vendor/pygments/lexers/c_like.py
new file mode 100644
index 0000000000000000000000000000000000000000..f7ba7e8f841f3f81ed2bcf8ba7c70ec9fab3d798
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/c_like.py
@@ -0,0 +1,541 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.c_like
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for other C-like languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, inherit, words, \
+ default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+from pygments.lexers.c_cpp import CLexer, CppLexer
+from pygments.lexers import _mql_builtins
+
+__all__ = ['PikeLexer', 'NesCLexer', 'ClayLexer', 'ECLexer', 'ValaLexer',
+ 'CudaLexer', 'SwigLexer', 'MqlLexer', 'ArduinoLexer']
+
+
+class PikeLexer(CppLexer):
+ """
+ For `Pike <http://pike.lysator.liu.se/>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Pike'
+ aliases = ['pike']
+ filenames = ['*.pike', '*.pmod']
+ mimetypes = ['text/x-pike']
+
+ tokens = {
+ 'statements': [
+ (words((
+ 'catch', 'new', 'private', 'protected', 'public', 'gauge',
+ 'throw', 'throws', 'class', 'interface', 'implement', 'abstract', 'extends', 'from',
+ 'this', 'super', 'constant', 'final', 'static', 'import', 'use', 'extern',
+ 'inline', 'proto', 'break', 'continue', 'if', 'else', 'for',
+ 'while', 'do', 'switch', 'case', 'as', 'in', 'version', 'return', 'true', 'false', 'null',
+ '__VERSION__', '__MAJOR__', '__MINOR__', '__BUILD__', '__REAL_VERSION__',
+ '__REAL_MAJOR__', '__REAL_MINOR__', '__REAL_BUILD__', '__DATE__', '__TIME__',
+ '__FILE__', '__DIR__', '__LINE__', '__AUTO_BIGNUM__', '__NT__', '__PIKE__',
+ '__amigaos__', '_Pragma', 'static_assert', 'defined', 'sscanf'), suffix=r'\b'),
+ Keyword),
+ (r'(bool|int|long|float|short|double|char|string|object|void|mapping|'
+ r'array|multiset|program|function|lambda|mixed|'
+ r'[a-z_][a-z0-9_]*_t)\b',
+ Keyword.Type),
+ (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
+ (r'[~!%^&*+=|?:<>/@-]', Operator),
+ inherit,
+ ],
+ 'classname': [
+ (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
+ # template specification
+ (r'\s*(?=>)', Text, '#pop'),
+ ],
+ }
+
+
+class NesCLexer(CLexer):
+ """
+ For `nesC <https://github.com/tinyos/nesc>`_ source code with preprocessor
+ directives.
+
+ .. versionadded:: 2.0
+ """
+ name = 'nesC'
+ aliases = ['nesc']
+ filenames = ['*.nc']
+ mimetypes = ['text/x-nescsrc']
+
+ tokens = {
+ 'statements': [
+ (words((
+ 'abstract', 'as', 'async', 'atomic', 'call', 'command', 'component',
+ 'components', 'configuration', 'event', 'extends', 'generic',
+ 'implementation', 'includes', 'interface', 'module', 'new', 'norace',
+ 'post', 'provides', 'signal', 'task', 'uses'), suffix=r'\b'),
+ Keyword),
+ (words(('nx_struct', 'nx_union', 'nx_int8_t', 'nx_int16_t', 'nx_int32_t',
+ 'nx_int64_t', 'nx_uint8_t', 'nx_uint16_t', 'nx_uint32_t',
+ 'nx_uint64_t'), suffix=r'\b'),
+ Keyword.Type),
+ inherit,
+ ],
+ }
+
+
+class ClayLexer(RegexLexer):
+ """
+ For `Clay <http://claylabs.com/clay/>`_ source.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Clay'
+ filenames = ['*.clay']
+ aliases = ['clay']
+ mimetypes = ['text/x-clay']
+ tokens = {
+ 'root': [
+ (r'\s', Text),
+ (r'//.*?$', Comment.Singleline),
+ (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+ (r'\b(public|private|import|as|record|variant|instance'
+ r'|define|overload|default|external|alias'
+ r'|rvalue|ref|forward|inline|noinline|forceinline'
+ r'|enum|var|and|or|not|if|else|goto|return|while'
+ r'|switch|case|break|continue|for|in|true|false|try|catch|throw'
+ r'|finally|onerror|staticassert|eval|when|newtype'
+ r'|__FILE__|__LINE__|__COLUMN__|__ARG__'
+ r')\b', Keyword),
+ (r'[~!%^&*+=|:<>/-]', Operator),
+ (r'[#(){}\[\],;.]', Punctuation),
+ (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex),
+ (r'\d+[LlUu]*', Number.Integer),
+ (r'\b(true|false)\b', Name.Builtin),
+ (r'(?i)[a-z_?][\w?]*', Name),
+ (r'"""', String, 'tdqs'),
+ (r'"', String, 'dqs'),
+ ],
+ 'strings': [
+ (r'(?i)\\(x[0-9a-f]{2}|.)', String.Escape),
+ (r'.', String),
+ ],
+ 'nl': [
+ (r'\n', String),
+ ],
+ 'dqs': [
+ (r'"', String, '#pop'),
+ include('strings'),
+ ],
+ 'tdqs': [
+ (r'"""', String, '#pop'),
+ include('strings'),
+ include('nl'),
+ ],
+ }
+
+
+class ECLexer(CLexer):
+ """
+ For eC source code with preprocessor directives.
+
+ .. versionadded:: 1.5
+ """
+ name = 'eC'
+ aliases = ['ec']
+ filenames = ['*.ec', '*.eh']
+ mimetypes = ['text/x-echdr', 'text/x-ecsrc']
+
+ tokens = {
+ 'statements': [
+ (words((
+ 'virtual', 'class', 'private', 'public', 'property', 'import',
+ 'delete', 'new', 'new0', 'renew', 'renew0', 'define', 'get',
+ 'set', 'remote', 'dllexport', 'dllimport', 'stdcall', 'subclass',
+ '__on_register_module', 'namespace', 'using', 'typed_object',
+ 'any_object', 'incref', 'register', 'watch', 'stopwatching', 'firewatchers',
+ 'watchable', 'class_designer', 'class_fixed', 'class_no_expansion', 'isset',
+ 'class_default_property', 'property_category', 'class_data',
+ 'class_property', 'thisclass', 'dbtable', 'dbindex',
+ 'database_open', 'dbfield'), suffix=r'\b'), Keyword),
+ (words(('uint', 'uint16', 'uint32', 'uint64', 'bool', 'byte',
+ 'unichar', 'int64'), suffix=r'\b'),
+ Keyword.Type),
+ (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
+ (r'(null|value|this)\b', Name.Builtin),
+ inherit,
+ ],
+ 'classname': [
+ (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
+ # template specification
+ (r'\s*(?=>)', Text, '#pop'),
+ ],
+ }
+
+
+class ValaLexer(RegexLexer):
+ """
+ For Vala source code with preprocessor directives.
+
+ .. versionadded:: 1.1
+ """
+ name = 'Vala'
+ aliases = ['vala', 'vapi']
+ filenames = ['*.vala', '*.vapi']
+ mimetypes = ['text/x-vala']
+
+ tokens = {
+ 'whitespace': [
+ (r'^\s*#if\s+0', Comment.Preproc, 'if0'),
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'\\\n', Text), # line continuation
+ (r'//(\n|(.|\n)*?[^\\]\n)', Comment.Single),
+ (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+ ],
+ 'statements': [
+ (r'[L@]?"', String, 'string'),
+ (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'",
+ String.Char),
+ (r'(?s)""".*?"""', String), # verbatim strings
+ (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float),
+ (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+ (r'0x[0-9a-fA-F]+[Ll]?', Number.Hex),
+ (r'0[0-7]+[Ll]?', Number.Oct),
+ (r'\d+[Ll]?', Number.Integer),
+ (r'[~!%^&*+=|?:<>/-]', Operator),
+ (r'(\[)(Compact|Immutable|(?:Boolean|Simple)Type)(\])',
+ bygroups(Punctuation, Name.Decorator, Punctuation)),
+ # TODO: "correctly" parse complex code attributes
+ (r'(\[)(CCode|(?:Integer|Floating)Type)',
+ bygroups(Punctuation, Name.Decorator)),
+ (r'[()\[\],.]', Punctuation),
+ (words((
+ 'as', 'base', 'break', 'case', 'catch', 'construct', 'continue',
+ 'default', 'delete', 'do', 'else', 'enum', 'finally', 'for',
+ 'foreach', 'get', 'if', 'in', 'is', 'lock', 'new', 'out', 'params',
+ 'return', 'set', 'sizeof', 'switch', 'this', 'throw', 'try',
+ 'typeof', 'while', 'yield'), suffix=r'\b'),
+ Keyword),
+ (words((
+ 'abstract', 'const', 'delegate', 'dynamic', 'ensures', 'extern',
+ 'inline', 'internal', 'override', 'owned', 'private', 'protected',
+ 'public', 'ref', 'requires', 'signal', 'static', 'throws', 'unowned',
+ 'var', 'virtual', 'volatile', 'weak', 'yields'), suffix=r'\b'),
+ Keyword.Declaration),
+ (r'(namespace|using)(\s+)', bygroups(Keyword.Namespace, Text),
+ 'namespace'),
+ (r'(class|errordomain|interface|struct)(\s+)',
+ bygroups(Keyword.Declaration, Text), 'class'),
+ (r'(\.)([a-zA-Z_]\w*)',
+ bygroups(Operator, Name.Attribute)),
+ # void is an actual keyword, others are in glib-2.0.vapi
+ (words((
+ 'void', 'bool', 'char', 'double', 'float', 'int', 'int8', 'int16',
+ 'int32', 'int64', 'long', 'short', 'size_t', 'ssize_t', 'string',
+ 'time_t', 'uchar', 'uint', 'uint8', 'uint16', 'uint32', 'uint64',
+ 'ulong', 'unichar', 'ushort'), suffix=r'\b'),
+ Keyword.Type),
+ (r'(true|false|null)\b', Name.Builtin),
+ ('[a-zA-Z_]\w*', Name),
+ ],
+ 'root': [
+ include('whitespace'),
+ default('statement'),
+ ],
+ 'statement': [
+ include('whitespace'),
+ include('statements'),
+ ('[{}]', Punctuation),
+ (';', Punctuation, '#pop'),
+ ],
+ 'string': [
+ (r'"', String, '#pop'),
+ (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+ (r'[^\\"\n]+', String), # all other characters
+ (r'\\\n', String), # line continuation
+ (r'\\', String), # stray backslash
+ ],
+ 'if0': [
+ (r'^\s*#if.*?(?<!\\)\n', Comment.Preproc, '#push'),
+ (r'^\s*#el(?:se|if).*\n', Comment.Preproc, '#pop'),
+ (r'^\s*#endif.*?(?<!\\)\n', Comment.Preproc, '#pop'),
+ (r'.*?\n', Comment),
+ ],
+ 'class': [
+ (r'[a-zA-Z_]\w*', Name.Class, '#pop')
+ ],
+ 'namespace': [
+ (r'[a-zA-Z_][\w.]*', Name.Namespace, '#pop')
+ ],
+ }
+
+
+class CudaLexer(CLexer):
+ """
+ For NVIDIA `CUDAâ„¢ <http://developer.nvidia.com/category/zone/cuda-zone>`_
+ source.
+
+ .. versionadded:: 1.6
+ """
+ name = 'CUDA'
+ filenames = ['*.cu', '*.cuh']
+ aliases = ['cuda', 'cu']
+ mimetypes = ['text/x-cuda']
+
+ function_qualifiers = set(('__device__', '__global__', '__host__',
+ '__noinline__', '__forceinline__'))
+ variable_qualifiers = set(('__device__', '__constant__', '__shared__',
+ '__restrict__'))
+ vector_types = set(('char1', 'uchar1', 'char2', 'uchar2', 'char3', 'uchar3',
+ 'char4', 'uchar4', 'short1', 'ushort1', 'short2', 'ushort2',
+ 'short3', 'ushort3', 'short4', 'ushort4', 'int1', 'uint1',
+ 'int2', 'uint2', 'int3', 'uint3', 'int4', 'uint4', 'long1',
+ 'ulong1', 'long2', 'ulong2', 'long3', 'ulong3', 'long4',
+ 'ulong4', 'longlong1', 'ulonglong1', 'longlong2',
+ 'ulonglong2', 'float1', 'float2', 'float3', 'float4',
+ 'double1', 'double2', 'dim3'))
+ variables = set(('gridDim', 'blockIdx', 'blockDim', 'threadIdx', 'warpSize'))
+ functions = set(('__threadfence_block', '__threadfence', '__threadfence_system',
+ '__syncthreads', '__syncthreads_count', '__syncthreads_and',
+ '__syncthreads_or'))
+ execution_confs = set(('<<<', '>>>'))
+
+ def get_tokens_unprocessed(self, text):
+ for index, token, value in CLexer.get_tokens_unprocessed(self, text):
+ if token is Name:
+ if value in self.variable_qualifiers:
+ token = Keyword.Type
+ elif value in self.vector_types:
+ token = Keyword.Type
+ elif value in self.variables:
+ token = Name.Builtin
+ elif value in self.execution_confs:
+ token = Keyword.Pseudo
+ elif value in self.function_qualifiers:
+ token = Keyword.Reserved
+ elif value in self.functions:
+ token = Name.Function
+ yield index, token, value
+
+
+class SwigLexer(CppLexer):
+ """
+ For `SWIG <http://www.swig.org/>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+ name = 'SWIG'
+ aliases = ['swig']
+ filenames = ['*.swg', '*.i']
+ mimetypes = ['text/swig']
+ priority = 0.04 # Lower than C/C++ and Objective C/C++
+
+ tokens = {
+ 'statements': [
+ # SWIG directives
+ (r'(%[a-z_][a-z0-9_]*)', Name.Function),
+ # Special variables
+ ('\$\**\&?\w+', Name),
+ # Stringification / additional preprocessor directives
+ (r'##*[a-zA-Z_]\w*', Comment.Preproc),
+ inherit,
+ ],
+ }
+
+ # This is a far from complete set of SWIG directives
+ swig_directives = set((
+ # Most common directives
+ '%apply', '%define', '%director', '%enddef', '%exception', '%extend',
+ '%feature', '%fragment', '%ignore', '%immutable', '%import', '%include',
+ '%inline', '%insert', '%module', '%newobject', '%nspace', '%pragma',
+ '%rename', '%shared_ptr', '%template', '%typecheck', '%typemap',
+ # Less common directives
+ '%arg', '%attribute', '%bang', '%begin', '%callback', '%catches', '%clear',
+ '%constant', '%copyctor', '%csconst', '%csconstvalue', '%csenum',
+ '%csmethodmodifiers', '%csnothrowexception', '%default', '%defaultctor',
+ '%defaultdtor', '%defined', '%delete', '%delobject', '%descriptor',
+ '%exceptionclass', '%exceptionvar', '%extend_smart_pointer', '%fragments',
+ '%header', '%ifcplusplus', '%ignorewarn', '%implicit', '%implicitconv',
+ '%init', '%javaconst', '%javaconstvalue', '%javaenum', '%javaexception',
+ '%javamethodmodifiers', '%kwargs', '%luacode', '%mutable', '%naturalvar',
+ '%nestedworkaround', '%perlcode', '%pythonabc', '%pythonappend',
+ '%pythoncallback', '%pythoncode', '%pythondynamic', '%pythonmaybecall',
+ '%pythonnondynamic', '%pythonprepend', '%refobject', '%shadow', '%sizeof',
+ '%trackobjects', '%types', '%unrefobject', '%varargs', '%warn',
+ '%warnfilter'))
+
+ def analyse_text(text):
+ rv = 0
+ # Search for SWIG directives, which are conventionally at the beginning of
+ # a line. The probability of them being within a line is low, so let another
+ # lexer win in this case.
+ matches = re.findall(r'^\s*(%[a-z_][a-z0-9_]*)', text, re.M)
+ for m in matches:
+ if m in SwigLexer.swig_directives:
+ rv = 0.98
+ break
+ else:
+ rv = 0.91 # Fraction higher than MatlabLexer
+ return rv
+
+
+class MqlLexer(CppLexer):
+ """
+ For `MQL4 <http://docs.mql4.com/>`_ and
+ `MQL5 <http://www.mql5.com/en/docs>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+ name = 'MQL'
+ aliases = ['mql', 'mq4', 'mq5', 'mql4', 'mql5']
+ filenames = ['*.mq4', '*.mq5', '*.mqh']
+ mimetypes = ['text/x-mql']
+
+ tokens = {
+ 'statements': [
+ (words(_mql_builtins.keywords, suffix=r'\b'), Keyword),
+ (words(_mql_builtins.c_types, suffix=r'\b'), Keyword.Type),
+ (words(_mql_builtins.types, suffix=r'\b'), Name.Function),
+ (words(_mql_builtins.constants, suffix=r'\b'), Name.Constant),
+ (words(_mql_builtins.colors, prefix='(clr)?', suffix=r'\b'),
+ Name.Constant),
+ inherit,
+ ],
+ }
+
+class ArduinoLexer(CppLexer):
+ """
+ For `Arduino(tm) <https://arduino.cc/>`_ source.
+
+ This is an extension of the CppLexer, as the Arduino® Language is a superset
+ of C++
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'Arduino'
+ aliases = ['arduino']
+ filenames = ['*.ino']
+ mimetypes = ['text/x-arduino']
+
+ # Language sketch main structure functions
+ structure = set(('setup', 'loop'))
+
+ # Language operators
+ operators = set(('not', 'or', 'and', 'xor'))
+
+ # Language 'variables'
+ variables = set((
+ 'DIGITAL_MESSAGE', 'FIRMATA_STRING', 'ANALOG_MESSAGE', 'REPORT_DIGITAL',
+ 'REPORT_ANALOG', 'INPUT_PULLUP', 'SET_PIN_MODE', 'INTERNAL2V56', 'SYSTEM_RESET',
+ 'LED_BUILTIN', 'INTERNAL1V1', 'SYSEX_START', 'INTERNAL', 'EXTERNAL', 'HIGH',
+ 'LOW', 'INPUT', 'OUTPUT', 'INPUT_PULLUP', 'LED_BUILTIN', 'true', 'false',
+ 'void', 'boolean', 'char', 'unsigned char', 'byte', 'int', 'unsigned int',
+ 'word', 'long', 'unsigned long', 'short', 'float', 'double', 'string', 'String',
+ 'array', 'static', 'volatile', 'const', 'boolean', 'byte', 'word', 'string',
+ 'String', 'array', 'int', 'float', 'private', 'char', 'virtual', 'operator',
+ 'sizeof', 'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t', 'int8_t', 'int16_t',
+ 'int32_t', 'int64_t', 'dynamic_cast', 'typedef', 'const_cast', 'const',
+ 'struct', 'static_cast', 'union', 'unsigned', 'long', 'volatile', 'static',
+ 'protected', 'bool', 'public', 'friend', 'auto', 'void', 'enum', 'extern',
+ 'class', 'short', 'reinterpret_cast', 'double', 'register', 'explicit',
+ 'signed', 'inline', 'delete', '_Bool', 'complex', '_Complex', '_Imaginary',
+ 'atomic_bool', 'atomic_char', 'atomic_schar', 'atomic_uchar', 'atomic_short',
+ 'atomic_ushort', 'atomic_int', 'atomic_uint', 'atomic_long', 'atomic_ulong',
+ 'atomic_llong', 'atomic_ullong', 'PROGMEM'))
+
+ # Language shipped functions and class ( )
+ functions = set((
+ 'KeyboardController', 'MouseController', 'SoftwareSerial', 'EthernetServer',
+ 'EthernetClient', 'LiquidCrystal', 'RobotControl', 'GSMVoiceCall',
+ 'EthernetUDP', 'EsploraTFT', 'HttpClient', 'RobotMotor', 'WiFiClient',
+ 'GSMScanner', 'FileSystem', 'Scheduler', 'GSMServer', 'YunClient', 'YunServer',
+ 'IPAddress', 'GSMClient', 'GSMModem', 'Keyboard', 'Ethernet', 'Console',
+ 'GSMBand', 'Esplora', 'Stepper', 'Process', 'WiFiUDP', 'GSM_SMS', 'Mailbox',
+ 'USBHost', 'Firmata', 'PImage', 'Client', 'Server', 'GSMPIN', 'FileIO',
+ 'Bridge', 'Serial', 'EEPROM', 'Stream', 'Mouse', 'Audio', 'Servo', 'File',
+ 'Task', 'GPRS', 'WiFi', 'Wire', 'TFT', 'GSM', 'SPI', 'SD',
+ 'runShellCommandAsynchronously', 'analogWriteResolution',
+ 'retrieveCallingNumber', 'printFirmwareVersion', 'analogReadResolution',
+ 'sendDigitalPortPair', 'noListenOnLocalhost', 'readJoystickButton',
+ 'setFirmwareVersion', 'readJoystickSwitch', 'scrollDisplayRight',
+ 'getVoiceCallStatus', 'scrollDisplayLeft', 'writeMicroseconds',
+ 'delayMicroseconds', 'beginTransmission', 'getSignalStrength',
+ 'runAsynchronously', 'getAsynchronously', 'listenOnLocalhost',
+ 'getCurrentCarrier', 'readAccelerometer', 'messageAvailable',
+ 'sendDigitalPorts', 'lineFollowConfig', 'countryNameWrite', 'runShellCommand',
+ 'readStringUntil', 'rewindDirectory', 'readTemperature', 'setClockDivider',
+ 'readLightSensor', 'endTransmission', 'analogReference', 'detachInterrupt',
+ 'countryNameRead', 'attachInterrupt', 'encryptionType', 'readBytesUntil',
+ 'robotNameWrite', 'readMicrophone', 'robotNameRead', 'cityNameWrite',
+ 'userNameWrite', 'readJoystickY', 'readJoystickX', 'mouseReleased',
+ 'openNextFile', 'scanNetworks', 'noInterrupts', 'digitalWrite', 'beginSpeaker',
+ 'mousePressed', 'isActionDone', 'mouseDragged', 'displayLogos', 'noAutoscroll',
+ 'addParameter', 'remoteNumber', 'getModifiers', 'keyboardRead', 'userNameRead',
+ 'waitContinue', 'processInput', 'parseCommand', 'printVersion', 'readNetworks',
+ 'writeMessage', 'blinkVersion', 'cityNameRead', 'readMessage', 'setDataMode',
+ 'parsePacket', 'isListening', 'setBitOrder', 'beginPacket', 'isDirectory',
+ 'motorsWrite', 'drawCompass', 'digitalRead', 'clearScreen', 'serialEvent',
+ 'rightToLeft', 'setTextSize', 'leftToRight', 'requestFrom', 'keyReleased',
+ 'compassRead', 'analogWrite', 'interrupts', 'WiFiServer', 'disconnect',
+ 'playMelody', 'parseFloat', 'autoscroll', 'getPINUsed', 'setPINUsed',
+ 'setTimeout', 'sendAnalog', 'readSlider', 'analogRead', 'beginWrite',
+ 'createChar', 'motorsStop', 'keyPressed', 'tempoWrite', 'readButton',
+ 'subnetMask', 'debugPrint', 'macAddress', 'writeGreen', 'randomSeed',
+ 'attachGPRS', 'readString', 'sendString', 'remotePort', 'releaseAll',
+ 'mouseMoved', 'background', 'getXChange', 'getYChange', 'answerCall',
+ 'getResult', 'voiceCall', 'endPacket', 'constrain', 'getSocket', 'writeJSON',
+ 'getButton', 'available', 'connected', 'findUntil', 'readBytes', 'exitValue',
+ 'readGreen', 'writeBlue', 'startLoop', 'IPAddress', 'isPressed', 'sendSysex',
+ 'pauseMode', 'gatewayIP', 'setCursor', 'getOemKey', 'tuneWrite', 'noDisplay',
+ 'loadImage', 'switchPIN', 'onRequest', 'onReceive', 'changePIN', 'playFile',
+ 'noBuffer', 'parseInt', 'overflow', 'checkPIN', 'knobRead', 'beginTFT',
+ 'bitClear', 'updateIR', 'bitWrite', 'position', 'writeRGB', 'highByte',
+ 'writeRed', 'setSpeed', 'readBlue', 'noStroke', 'remoteIP', 'transfer',
+ 'shutdown', 'hangCall', 'beginSMS', 'endWrite', 'attached', 'maintain',
+ 'noCursor', 'checkReg', 'checkPUK', 'shiftOut', 'isValid', 'shiftIn', 'pulseIn',
+ 'connect', 'println', 'localIP', 'pinMode', 'getIMEI', 'display', 'noBlink',
+ 'process', 'getBand', 'running', 'beginSD', 'drawBMP', 'lowByte', 'setBand',
+ 'release', 'bitRead', 'prepare', 'pointTo', 'readRed', 'setMode', 'noFill',
+ 'remove', 'listen', 'stroke', 'detach', 'attach', 'noTone', 'exists', 'buffer',
+ 'height', 'bitSet', 'circle', 'config', 'cursor', 'random', 'IRread', 'setDNS',
+ 'endSMS', 'getKey', 'micros', 'millis', 'begin', 'print', 'write', 'ready',
+ 'flush', 'width', 'isPIN', 'blink', 'clear', 'press', 'mkdir', 'rmdir', 'close',
+ 'point', 'yield', 'image', 'BSSID', 'click', 'delay', 'read', 'text', 'move',
+ 'peek', 'beep', 'rect', 'line', 'open', 'seek', 'fill', 'size', 'turn', 'stop',
+ 'home', 'find', 'step', 'tone', 'sqrt', 'RSSI', 'SSID', 'end', 'bit', 'tan',
+ 'cos', 'sin', 'pow', 'map', 'abs', 'max', 'min', 'get', 'run', 'put',
+ 'isAlphaNumeric', 'isAlpha', 'isAscii', 'isWhitespace', 'isControl', 'isDigit',
+ 'isGraph', 'isLowerCase', 'isPrintable', 'isPunct', 'isSpace', 'isUpperCase',
+ 'isHexadecimalDigit'))
+
+ # do not highlight
+ suppress_highlight = set((
+ 'namespace', 'template', 'mutable', 'using', 'asm', 'typeid',
+ 'typename', 'this', 'alignof', 'constexpr', 'decltype', 'noexcept',
+ 'static_assert', 'thread_local', 'restrict'))
+
+
+ def get_tokens_unprocessed(self, text):
+ for index, token, value in CppLexer.get_tokens_unprocessed(self, text):
+ if value in self.structure:
+ yield index, Name.Builtin, value
+ elif value in self.operators:
+ yield index, Operator, value
+ elif value in self.variables:
+ yield index, Keyword.Reserved, value
+ elif value in self.suppress_highlight:
+ yield index, Name, value
+ elif value in self.functions:
+ yield index, Name.Function, value
+ else:
+ yield index, token, value
diff --git a/wandb/vendor/pygments/lexers/capnproto.py b/wandb/vendor/pygments/lexers/capnproto.py
new file mode 100644
index 0000000000000000000000000000000000000000..203523a1adf126b3394768e418ba7a32921a26f7
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/capnproto.py
@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.capnproto
+ ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the Cap'n Proto schema language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, default
+from pygments.token import Text, Comment, Keyword, Name, Literal
+
+__all__ = ['CapnProtoLexer']
+
+
+class CapnProtoLexer(RegexLexer):
+ """
+ For `Cap'n Proto <https://capnproto.org>`_ source.
+
+ .. versionadded:: 2.2
+ """
+ name = 'Cap\'n Proto'
+ filenames = ['*.capnp']
+ aliases = ['capnp']
+
+ flags = re.MULTILINE | re.UNICODE
+
+ tokens = {
+ 'root': [
+ (r'#.*?$', Comment.Single),
+ (r'@[0-9a-zA-Z]*', Name.Decorator),
+ (r'=', Literal, 'expression'),
+ (r':', Name.Class, 'type'),
+ (r'\$', Name.Attribute, 'annotation'),
+ (r'(struct|enum|interface|union|import|using|const|annotation|'
+ r'extends|in|of|on|as|with|from|fixed)\b',
+ Keyword),
+ (r'[\w.]+', Name),
+ (r'[^#@=:$\w]+', Text),
+ ],
+ 'type': [
+ (r'[^][=;,(){}$]+', Name.Class),
+ (r'[[(]', Name.Class, 'parentype'),
+ default('#pop'),
+ ],
+ 'parentype': [
+ (r'[^][;()]+', Name.Class),
+ (r'[[(]', Name.Class, '#push'),
+ (r'[])]', Name.Class, '#pop'),
+ default('#pop'),
+ ],
+ 'expression': [
+ (r'[^][;,(){}$]+', Literal),
+ (r'[[(]', Literal, 'parenexp'),
+ default('#pop'),
+ ],
+ 'parenexp': [
+ (r'[^][;()]+', Literal),
+ (r'[[(]', Literal, '#push'),
+ (r'[])]', Literal, '#pop'),
+ default('#pop'),
+ ],
+ 'annotation': [
+ (r'[^][;,(){}=:]+', Name.Attribute),
+ (r'[[(]', Name.Attribute, 'annexp'),
+ default('#pop'),
+ ],
+ 'annexp': [
+ (r'[^][;()]+', Name.Attribute),
+ (r'[[(]', Name.Attribute, '#push'),
+ (r'[])]', Name.Attribute, '#pop'),
+ default('#pop'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/chapel.py b/wandb/vendor/pygments/lexers/chapel.py
new file mode 100644
index 0000000000000000000000000000000000000000..55bf0e1ed39f25cc863e38656e8b889511deda68
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/chapel.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.chapel
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for the Chapel language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['ChapelLexer']
+
+
+class ChapelLexer(RegexLexer):
+ """
+ For `Chapel <http://chapel.cray.com/>`_ source.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Chapel'
+ filenames = ['*.chpl']
+ aliases = ['chapel', 'chpl']
+ # mimetypes = ['text/x-chapel']
+
+ tokens = {
+ 'root': [
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'\\\n', Text),
+
+ (r'//(.*?)\n', Comment.Single),
+ (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+
+ (r'(config|const|in|inout|out|param|ref|type|var)\b',
+ Keyword.Declaration),
+ (r'(false|nil|true)\b', Keyword.Constant),
+ (r'(bool|complex|imag|int|opaque|range|real|string|uint)\b',
+ Keyword.Type),
+ (words((
+ 'align', 'as', 'atomic', 'begin', 'break', 'by', 'cobegin',
+ 'coforall', 'continue', 'delete', 'dmapped', 'do', 'domain',
+ 'else', 'enum', 'except', 'export', 'extern', 'for', 'forall',
+ 'if', 'index', 'inline', 'iter', 'label', 'lambda', 'let',
+ 'local', 'new', 'noinit', 'on', 'only', 'otherwise', 'pragma',
+ 'private', 'public', 'reduce', 'require', 'return', 'scan',
+ 'select', 'serial', 'single', 'sparse', 'subdomain', 'sync',
+ 'then', 'use', 'when', 'where', 'while', 'with', 'yield',
+ 'zip'), suffix=r'\b'),
+ Keyword),
+ (r'(proc)((?:\s)+)', bygroups(Keyword, Text), 'procname'),
+ (r'(class|module|record|union)(\s+)', bygroups(Keyword, Text),
+ 'classname'),
+
+ # imaginary integers
+ (r'\d+i', Number),
+ (r'\d+\.\d*([Ee][-+]\d+)?i', Number),
+ (r'\.\d+([Ee][-+]\d+)?i', Number),
+ (r'\d+[Ee][-+]\d+i', Number),
+
+ # reals cannot end with a period due to lexical ambiguity with
+ # .. operator. See reference for rationale.
+ (r'(\d*\.\d+)([eE][+-]?[0-9]+)?i?', Number.Float),
+ (r'\d+[eE][+-]?[0-9]+i?', Number.Float),
+
+ # integer literals
+ # -- binary
+ (r'0[bB][01]+', Number.Bin),
+ # -- hex
+ (r'0[xX][0-9a-fA-F]+', Number.Hex),
+ # -- octal
+ (r'0[oO][0-7]+', Number.Oct),
+ # -- decimal
+ (r'[0-9]+', Number.Integer),
+
+ # strings
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r"'(\\\\|\\'|[^'])*'", String),
+
+ # tokens
+ (r'(=|\+=|-=|\*=|/=|\*\*=|%=|&=|\|=|\^=|&&=|\|\|=|<<=|>>=|'
+ r'<=>|<~>|\.\.|by|#|\.\.\.|'
+ r'&&|\|\||!|&|\||\^|~|<<|>>|'
+ r'==|!=|<=|>=|<|>|'
+ r'[+\-*/%]|\*\*)', Operator),
+ (r'[:;,.?()\[\]{}]', Punctuation),
+
+ # identifiers
+ (r'[a-zA-Z_][\w$]*', Name.Other),
+ ],
+ 'classname': [
+ (r'[a-zA-Z_][\w$]*', Name.Class, '#pop'),
+ ],
+ 'procname': [
+ (r'([a-zA-Z_][\w$]+|\~[a-zA-Z_][\w$]+|[+*/!~%<>=&^|\-]{1,2})',
+ Name.Function, '#pop'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/clean.py b/wandb/vendor/pygments/lexers/clean.py
new file mode 100644
index 0000000000000000000000000000000000000000..ba2569f6e173442f64a7e5585a716adef182241d
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/clean.py
@@ -0,0 +1,288 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.clean
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for the Clean language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import ExtendedRegexLexer, LexerContext, \
+ bygroups, words, include, default
+from pygments.token import Comment, Keyword, Literal, Name, Number, Operator, \
+ Punctuation, String, Text, Whitespace
+
+__all__ = ['CleanLexer']
+
+
+class CleanLexer(ExtendedRegexLexer):
+ """
+ Lexer for the general purpose, state-of-the-art, pure and lazy functional
+ programming language Clean (http://clean.cs.ru.nl/Clean).
+
+ .. versionadded: 2.2
+ """
+ name = 'Clean'
+ aliases = ['clean']
+ filenames = ['*.icl', '*.dcl']
+
+ def get_tokens_unprocessed(self, text=None, context=None):
+ ctx = LexerContext(text, 0)
+ ctx.indent = 0
+ return ExtendedRegexLexer.get_tokens_unprocessed(self, text, context=ctx)
+
+ def check_class_not_import(lexer, match, ctx):
+ if match.group(0) == 'import':
+ yield match.start(), Keyword.Namespace, match.group(0)
+ ctx.stack = ctx.stack[:-1] + ['fromimportfunc']
+ else:
+ yield match.start(), Name.Class, match.group(0)
+ ctx.pos = match.end()
+
+ def check_instance_class(lexer, match, ctx):
+ if match.group(0) == 'instance' or match.group(0) == 'class':
+ yield match.start(), Keyword, match.group(0)
+ else:
+ yield match.start(), Name.Function, match.group(0)
+ ctx.stack = ctx.stack + ['fromimportfunctype']
+ ctx.pos = match.end()
+
+ @staticmethod
+ def indent_len(text):
+ # Tabs are four spaces:
+ # https://svn.cs.ru.nl/repos/clean-platform/trunk/doc/STANDARDS.txt
+ text = text.replace('\n', '')
+ return len(text.replace('\t', ' ')), len(text)
+
+ def store_indent(lexer, match, ctx):
+ ctx.indent, _ = CleanLexer.indent_len(match.group(0))
+ ctx.pos = match.end()
+ yield match.start(), Text, match.group(0)
+
+ def check_indent1(lexer, match, ctx):
+ indent, reallen = CleanLexer.indent_len(match.group(0))
+ if indent > ctx.indent:
+ yield match.start(), Whitespace, match.group(0)
+ ctx.pos = match.start() + reallen + 1
+ else:
+ ctx.indent = 0
+ ctx.pos = match.start()
+ ctx.stack = ctx.stack[:-1]
+ yield match.start(), Whitespace, match.group(0)[1:]
+
+ def check_indent2(lexer, match, ctx):
+ indent, reallen = CleanLexer.indent_len(match.group(0))
+ if indent > ctx.indent:
+ yield match.start(), Whitespace, match.group(0)
+ ctx.pos = match.start() + reallen + 1
+ else:
+ ctx.indent = 0
+ ctx.pos = match.start()
+ ctx.stack = ctx.stack[:-2]
+
+ def check_indent3(lexer, match, ctx):
+ indent, reallen = CleanLexer.indent_len(match.group(0))
+ if indent > ctx.indent:
+ yield match.start(), Whitespace, match.group(0)
+ ctx.pos = match.start() + reallen + 1
+ else:
+ ctx.indent = 0
+ ctx.pos = match.start()
+ ctx.stack = ctx.stack[:-3]
+ yield match.start(), Whitespace, match.group(0)[1:]
+ if match.group(0) == '\n\n':
+ ctx.pos = ctx.pos + 1
+
+ def skip(lexer, match, ctx):
+ ctx.stack = ctx.stack[:-1]
+ ctx.pos = match.end()
+ yield match.start(), Comment, match.group(0)
+
+ keywords = ('class', 'instance', 'where', 'with', 'let', 'let!',
+ 'in', 'case', 'of', 'infix', 'infixr', 'infixl', 'generic',
+ 'derive', 'otherwise', 'code', 'inline')
+
+ tokens = {
+ 'common': [
+ (r';', Punctuation, '#pop'),
+ (r'//', Comment, 'singlecomment'),
+ ],
+ 'root': [
+ # Comments
+ (r'//.*\n', Comment.Single),
+ (r'(?s)/\*\*.*?\*/', Comment.Special),
+ (r'(?s)/\*.*?\*/', Comment.Multi),
+
+ # Modules, imports, etc.
+ (r'\b((?:implementation|definition|system)\s+)?(module)(\s+)([\w`.]+)',
+ bygroups(Keyword.Namespace, Keyword.Namespace, Text, Name.Class)),
+ (r'(?<=\n)import(?=\s)', Keyword.Namespace, 'import'),
+ (r'(?<=\n)from(?=\s)', Keyword.Namespace, 'fromimport'),
+
+ # Keywords
+ # We cannot use (?s)^|(?<=\s) as prefix, so need to repeat this
+ (words(keywords, prefix=r'(?<=\s)', suffix=r'(?=\s)'), Keyword),
+ (words(keywords, prefix=r'^', suffix=r'(?=\s)'), Keyword),
+
+ # Function definitions
+ (r'(?=\{\|)', Whitespace, 'genericfunction'),
+ (r'(?<=\n)([ \t]*)([\w`$()=\-<>~*\^|+&%]+)((?:\s+\w)*)(\s*)(::)',
+ bygroups(store_indent, Name.Function, Keyword.Type, Whitespace,
+ Punctuation),
+ 'functiondefargs'),
+
+ # Type definitions
+ (r'(?<=\n)([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'),
+ (r'^([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'),
+
+ # Literals
+ (r'\'\\?.(?<!\\)\'', String.Char),
+ (r'\'\\\d+\'', String.Char),
+ (r'\'\\\\\'', String.Char), # (special case for '\\')
+ (r'[+\-~]?\s*\d+\.\d+(E[+\-~]?\d+)?\b', Number.Float),
+ (r'[+\-~]?\s*0[0-7]\b', Number.Oct),
+ (r'[+\-~]?\s*0x[0-9a-fA-F]\b', Number.Hex),
+ (r'[+\-~]?\s*\d+\b', Number.Integer),
+ (r'"', String.Double, 'doubleqstring'),
+ (words(('True', 'False'), prefix=r'(?<=\s)', suffix=r'(?=\s)'),
+ Literal),
+
+ # Qualified names
+ (r'(\')([\w.]+)(\'\.)',
+ bygroups(Punctuation, Name.Namespace, Punctuation)),
+
+ # Everything else is some name
+ (r'([\w`$%/?@]+\.?)*[\w`$%/?@]+', Name),
+
+ # Punctuation
+ (r'[{}()\[\],:;.#]', Punctuation),
+ (r'[+\-=!<>|&~*\^/]', Operator),
+ (r'\\\\', Operator),
+
+ # Lambda expressions
+ (r'\\.*?(->|\.|=)', Name.Function),
+
+ # Whitespace
+ (r'\s', Whitespace),
+
+ include('common'),
+ ],
+ 'fromimport': [
+ include('common'),
+ (r'([\w`.]+)', check_class_not_import),
+ (r'\n', Whitespace, '#pop'),
+ (r'\s', Whitespace),
+ ],
+ 'fromimportfunc': [
+ include('common'),
+ (r'(::)(\s+)([^,\s]+)', bygroups(Punctuation, Text, Keyword.Type)),
+ (r'([\w`$()=\-<>~*\^|+&%/]+)', check_instance_class),
+ (r',', Punctuation),
+ (r'\n', Whitespace, '#pop'),
+ (r'\s', Whitespace),
+ ],
+ 'fromimportfunctype': [
+ include('common'),
+ (r'[{(\[]', Punctuation, 'combtype'),
+ (r',', Punctuation, '#pop'),
+ (r'[:;.#]', Punctuation),
+ (r'\n', Whitespace, '#pop:2'),
+ (r'[^\S\n]+', Whitespace),
+ (r'\S+', Keyword.Type),
+ ],
+ 'combtype': [
+ include('common'),
+ (r'[})\]]', Punctuation, '#pop'),
+ (r'[{(\[]', Punctuation, '#pop'),
+ (r'[,:;.#]', Punctuation),
+ (r'\s+', Whitespace),
+ (r'\S+', Keyword.Type),
+ ],
+ 'import': [
+ include('common'),
+ (words(('from', 'import', 'as', 'qualified'),
+ prefix='(?<=\s)', suffix='(?=\s)'), Keyword.Namespace),
+ (r'[\w`.]+', Name.Class),
+ (r'\n', Whitespace, '#pop'),
+ (r',', Punctuation),
+ (r'[^\S\n]+', Whitespace),
+ ],
+ 'singlecomment': [
+ (r'(.)(?=\n)', skip),
+ (r'.+(?!\n)', Comment),
+ ],
+ 'doubleqstring': [
+ (r'[^\\"]+', String.Double),
+ (r'"', String.Double, '#pop'),
+ (r'\\.', String.Double),
+ ],
+ 'typedef': [
+ include('common'),
+ (r'[\w`]+', Keyword.Type),
+ (r'[:=|(),\[\]{}!*]', Punctuation),
+ (r'->', Punctuation),
+ (r'\n(?=[^\s|])', Whitespace, '#pop'),
+ (r'\s', Whitespace),
+ (r'.', Keyword.Type),
+ ],
+ 'genericfunction': [
+ include('common'),
+ (r'\{\|', Punctuation),
+ (r'\|\}', Punctuation, '#pop'),
+ (r',', Punctuation),
+ (r'->', Punctuation),
+ (r'(\s+of\s+)(\{)', bygroups(Keyword, Punctuation), 'genericftypes'),
+ (r'\s', Whitespace),
+ (r'[\w`\[\]{}!]+', Keyword.Type),
+ (r'[*()]', Punctuation),
+ ],
+ 'genericftypes': [
+ include('common'),
+ (r'[\w`]+', Keyword.Type),
+ (r',', Punctuation),
+ (r'\s', Whitespace),
+ (r'\}', Punctuation, '#pop'),
+ ],
+ 'functiondefargs': [
+ include('common'),
+ (r'\n(\s*)', check_indent1),
+ (r'[!{}()\[\],:;.#]', Punctuation),
+ (r'->', Punctuation, 'functiondefres'),
+ (r'^(?=\S)', Whitespace, '#pop'),
+ (r'\S', Keyword.Type),
+ (r'\s', Whitespace),
+ ],
+ 'functiondefres': [
+ include('common'),
+ (r'\n(\s*)', check_indent2),
+ (r'^(?=\S)', Whitespace, '#pop:2'),
+ (r'[!{}()\[\],:;.#]', Punctuation),
+ (r'\|', Punctuation, 'functiondefclasses'),
+ (r'\S', Keyword.Type),
+ (r'\s', Whitespace),
+ ],
+ 'functiondefclasses': [
+ include('common'),
+ (r'\n(\s*)', check_indent3),
+ (r'^(?=\S)', Whitespace, '#pop:3'),
+ (r'[,&]', Punctuation),
+ (r'\[', Punctuation, 'functiondefuniquneq'),
+ (r'[\w`$()=\-<>~*\^|+&%/{}\[\]@]', Name.Function, 'functionname'),
+ (r'\s+', Whitespace),
+ ],
+ 'functiondefuniquneq': [
+ include('common'),
+ (r'[a-z]+', Keyword.Type),
+ (r'\s+', Whitespace),
+ (r'<=|,', Punctuation),
+ (r'\]', Punctuation, '#pop')
+ ],
+ 'functionname': [
+ include('common'),
+ (r'[\w`$()=\-<>~*\^|+&%/]+', Name.Function),
+ (r'(?=\{\|)', Punctuation, 'genericfunction'),
+ default('#pop'),
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/compiled.py b/wandb/vendor/pygments/lexers/compiled.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab52a37019c609b3804f979ca075184d08492230
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/compiled.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.compiled
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Just export lexer classes previously contained in this module.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexers.jvm import JavaLexer, ScalaLexer
+from pygments.lexers.c_cpp import CLexer, CppLexer
+from pygments.lexers.d import DLexer
+from pygments.lexers.objective import ObjectiveCLexer, \
+ ObjectiveCppLexer, LogosLexer
+from pygments.lexers.go import GoLexer
+from pygments.lexers.rust import RustLexer
+from pygments.lexers.c_like import ECLexer, ValaLexer, CudaLexer
+from pygments.lexers.pascal import DelphiLexer, Modula2Lexer, AdaLexer
+from pygments.lexers.business import CobolLexer, CobolFreeformatLexer
+from pygments.lexers.fortran import FortranLexer
+from pygments.lexers.prolog import PrologLexer
+from pygments.lexers.python import CythonLexer
+from pygments.lexers.graphics import GLShaderLexer
+from pygments.lexers.ml import OcamlLexer
+from pygments.lexers.basic import BlitzBasicLexer, BlitzMaxLexer, MonkeyLexer
+from pygments.lexers.dylan import DylanLexer, DylanLidLexer, DylanConsoleLexer
+from pygments.lexers.ooc import OocLexer
+from pygments.lexers.felix import FelixLexer
+from pygments.lexers.nimrod import NimrodLexer
+from pygments.lexers.crystal import CrystalLexer
+
+__all__ = []
diff --git a/wandb/vendor/pygments/lexers/configs.py b/wandb/vendor/pygments/lexers/configs.py
new file mode 100644
index 0000000000000000000000000000000000000000..1717a563abec828d39326ba4f12fb448f43fa9ae
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/configs.py
@@ -0,0 +1,833 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.configs
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for configuration file formats.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, default, words, bygroups, include, using
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Whitespace, Literal
+from pygments.lexers.shell import BashLexer
+
+__all__ = ['IniLexer', 'RegeditLexer', 'PropertiesLexer', 'KconfigLexer',
+ 'Cfengine3Lexer', 'ApacheConfLexer', 'SquidConfLexer',
+ 'NginxConfLexer', 'LighttpdConfLexer', 'DockerLexer',
+ 'TerraformLexer', 'TermcapLexer', 'TerminfoLexer',
+ 'PkgConfigLexer', 'PacmanConfLexer']
+
+
+class IniLexer(RegexLexer):
+ """
+ Lexer for configuration files in INI style.
+ """
+
+ name = 'INI'
+ aliases = ['ini', 'cfg', 'dosini']
+ filenames = ['*.ini', '*.cfg', '*.inf']
+ mimetypes = ['text/x-ini', 'text/inf']
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'[;#].*', Comment.Single),
+ (r'\[.*?\]$', Keyword),
+ (r'(.*?)([ \t]*)(=)([ \t]*)(.*(?:\n[ \t].+)*)',
+ bygroups(Name.Attribute, Text, Operator, Text, String)),
+ # standalone option, supported by some INI parsers
+ (r'(.+?)$', Name.Attribute),
+ ],
+ }
+
+ def analyse_text(text):
+ npos = text.find('\n')
+ if npos < 3:
+ return False
+ return text[0] == '[' and text[npos-1] == ']'
+
+
+class RegeditLexer(RegexLexer):
+ """
+ Lexer for `Windows Registry
+ <http://en.wikipedia.org/wiki/Windows_Registry#.REG_files>`_ files produced
+ by regedit.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'reg'
+ aliases = ['registry']
+ filenames = ['*.reg']
+ mimetypes = ['text/x-windows-registry']
+
+ tokens = {
+ 'root': [
+ (r'Windows Registry Editor.*', Text),
+ (r'\s+', Text),
+ (r'[;#].*', Comment.Single),
+ (r'(\[)(-?)(HKEY_[A-Z_]+)(.*?\])$',
+ bygroups(Keyword, Operator, Name.Builtin, Keyword)),
+ # String keys, which obey somewhat normal escaping
+ (r'("(?:\\"|\\\\|[^"])+")([ \t]*)(=)([ \t]*)',
+ bygroups(Name.Attribute, Text, Operator, Text),
+ 'value'),
+ # Bare keys (includes @)
+ (r'(.*?)([ \t]*)(=)([ \t]*)',
+ bygroups(Name.Attribute, Text, Operator, Text),
+ 'value'),
+ ],
+ 'value': [
+ (r'-', Operator, '#pop'), # delete value
+ (r'(dword|hex(?:\([0-9a-fA-F]\))?)(:)([0-9a-fA-F,]+)',
+ bygroups(Name.Variable, Punctuation, Number), '#pop'),
+ # As far as I know, .reg files do not support line continuation.
+ (r'.+', String, '#pop'),
+ default('#pop'),
+ ]
+ }
+
+ def analyse_text(text):
+ return text.startswith('Windows Registry Editor')
+
+
+class PropertiesLexer(RegexLexer):
+ """
+ Lexer for configuration files in Java's properties format.
+
+ Note: trailing whitespace counts as part of the value as per spec
+
+ .. versionadded:: 1.4
+ """
+
+ name = 'Properties'
+ aliases = ['properties', 'jproperties']
+ filenames = ['*.properties']
+ mimetypes = ['text/x-java-properties']
+
+ tokens = {
+ 'root': [
+ (r'^(\w+)([ \t])(\w+\s*)$', bygroups(Name.Attribute, Text, String)),
+ (r'^\w+(\\[ \t]\w*)*$', Name.Attribute),
+ (r'(^ *)([#!].*)', bygroups(Text, Comment)),
+ # More controversial comments
+ (r'(^ *)((?:;|//).*)', bygroups(Text, Comment)),
+ (r'(.*?)([ \t]*)([=:])([ \t]*)(.*(?:(?<=\\)\n.*)*)',
+ bygroups(Name.Attribute, Text, Operator, Text, String)),
+ (r'\s', Text),
+ ],
+ }
+
+
+def _rx_indent(level):
+ # Kconfig *always* interprets a tab as 8 spaces, so this is the default.
+ # Edit this if you are in an environment where KconfigLexer gets expanded
+ # input (tabs expanded to spaces) and the expansion tab width is != 8,
+ # e.g. in connection with Trac (trac.ini, [mimeviewer], tab_width).
+ # Value range here is 2 <= {tab_width} <= 8.
+ tab_width = 8
+ # Regex matching a given indentation {level}, assuming that indentation is
+ # a multiple of {tab_width}. In other cases there might be problems.
+ if tab_width == 2:
+ space_repeat = '+'
+ else:
+ space_repeat = '{1,%d}' % (tab_width - 1)
+ if level == 1:
+ level_repeat = ''
+ else:
+ level_repeat = '{%s}' % level
+ return r'(?:\t| %s\t| {%s})%s.*\n' % (space_repeat, tab_width, level_repeat)
+
+
+class KconfigLexer(RegexLexer):
+ """
+ For Linux-style Kconfig files.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'Kconfig'
+ aliases = ['kconfig', 'menuconfig', 'linux-config', 'kernel-config']
+ # Adjust this if new kconfig file names appear in your environment
+ filenames = ['Kconfig', '*Config.in*', 'external.in*',
+ 'standard-modules.in']
+ mimetypes = ['text/x-kconfig']
+ # No re.MULTILINE, indentation-aware help text needs line-by-line handling
+ flags = 0
+
+ def call_indent(level):
+ # If indentation >= {level} is detected, enter state 'indent{level}'
+ return (_rx_indent(level), String.Doc, 'indent%s' % level)
+
+ def do_indent(level):
+ # Print paragraphs of indentation level >= {level} as String.Doc,
+ # ignoring blank lines. Then return to 'root' state.
+ return [
+ (_rx_indent(level), String.Doc),
+ (r'\s*\n', Text),
+ default('#pop:2')
+ ]
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'#.*?\n', Comment.Single),
+ (words((
+ 'mainmenu', 'config', 'menuconfig', 'choice', 'endchoice',
+ 'comment', 'menu', 'endmenu', 'visible if', 'if', 'endif',
+ 'source', 'prompt', 'select', 'depends on', 'default',
+ 'range', 'option'), suffix=r'\b'),
+ Keyword),
+ (r'(---help---|help)[\t ]*\n', Keyword, 'help'),
+ (r'(bool|tristate|string|hex|int|defconfig_list|modules|env)\b',
+ Name.Builtin),
+ (r'[!=&|]', Operator),
+ (r'[()]', Punctuation),
+ (r'[0-9]+', Number.Integer),
+ (r"'(''|[^'])*'", String.Single),
+ (r'"(""|[^"])*"', String.Double),
+ (r'\S+', Text),
+ ],
+ # Help text is indented, multi-line and ends when a lower indentation
+ # level is detected.
+ 'help': [
+ # Skip blank lines after help token, if any
+ (r'\s*\n', Text),
+ # Determine the first help line's indentation level heuristically(!).
+ # Attention: this is not perfect, but works for 99% of "normal"
+ # indentation schemes up to a max. indentation level of 7.
+ call_indent(7),
+ call_indent(6),
+ call_indent(5),
+ call_indent(4),
+ call_indent(3),
+ call_indent(2),
+ call_indent(1),
+ default('#pop'), # for incomplete help sections without text
+ ],
+ # Handle text for indentation levels 7 to 1
+ 'indent7': do_indent(7),
+ 'indent6': do_indent(6),
+ 'indent5': do_indent(5),
+ 'indent4': do_indent(4),
+ 'indent3': do_indent(3),
+ 'indent2': do_indent(2),
+ 'indent1': do_indent(1),
+ }
+
+
+class Cfengine3Lexer(RegexLexer):
+ """
+ Lexer for `CFEngine3 <http://cfengine.org>`_ policy files.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'CFEngine3'
+ aliases = ['cfengine3', 'cf3']
+ filenames = ['*.cf']
+ mimetypes = []
+
+ tokens = {
+ 'root': [
+ (r'#.*?\n', Comment),
+ (r'(body)(\s+)(\S+)(\s+)(control)',
+ bygroups(Keyword, Text, Keyword, Text, Keyword)),
+ (r'(body|bundle)(\s+)(\S+)(\s+)(\w+)(\()',
+ bygroups(Keyword, Text, Keyword, Text, Name.Function, Punctuation),
+ 'arglist'),
+ (r'(body|bundle)(\s+)(\S+)(\s+)(\w+)',
+ bygroups(Keyword, Text, Keyword, Text, Name.Function)),
+ (r'(")([^"]+)(")(\s+)(string|slist|int|real)(\s*)(=>)(\s*)',
+ bygroups(Punctuation, Name.Variable, Punctuation,
+ Text, Keyword.Type, Text, Operator, Text)),
+ (r'(\S+)(\s*)(=>)(\s*)',
+ bygroups(Keyword.Reserved, Text, Operator, Text)),
+ (r'"', String, 'string'),
+ (r'(\w+)(\()', bygroups(Name.Function, Punctuation)),
+ (r'([\w.!&|()]+)(::)', bygroups(Name.Class, Punctuation)),
+ (r'(\w+)(:)', bygroups(Keyword.Declaration, Punctuation)),
+ (r'@[{(][^)}]+[})]', Name.Variable),
+ (r'[(){},;]', Punctuation),
+ (r'=>', Operator),
+ (r'->', Operator),
+ (r'\d+\.\d+', Number.Float),
+ (r'\d+', Number.Integer),
+ (r'\w+', Name.Function),
+ (r'\s+', Text),
+ ],
+ 'string': [
+ (r'\$[{(]', String.Interpol, 'interpol'),
+ (r'\\.', String.Escape),
+ (r'"', String, '#pop'),
+ (r'\n', String),
+ (r'.', String),
+ ],
+ 'interpol': [
+ (r'\$[{(]', String.Interpol, '#push'),
+ (r'[})]', String.Interpol, '#pop'),
+ (r'[^${()}]+', String.Interpol),
+ ],
+ 'arglist': [
+ (r'\)', Punctuation, '#pop'),
+ (r',', Punctuation),
+ (r'\w+', Name.Variable),
+ (r'\s+', Text),
+ ],
+ }
+
+
+class ApacheConfLexer(RegexLexer):
+ """
+ Lexer for configuration files following the Apache config file
+ format.
+
+ .. versionadded:: 0.6
+ """
+
+ name = 'ApacheConf'
+ aliases = ['apacheconf', 'aconf', 'apache']
+ filenames = ['.htaccess', 'apache.conf', 'apache2.conf']
+ mimetypes = ['text/x-apacheconf']
+ flags = re.MULTILINE | re.IGNORECASE
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'(#.*?)$', Comment),
+ (r'(<[^\s>]+)(?:(\s+)(.*?))?(>)',
+ bygroups(Name.Tag, Text, String, Name.Tag)),
+ (r'([a-z]\w*)(\s+)',
+ bygroups(Name.Builtin, Text), 'value'),
+ (r'\.+', Text),
+ ],
+ 'value': [
+ (r'\\\n', Text),
+ (r'$', Text, '#pop'),
+ (r'\\', Text),
+ (r'[^\S\n]+', Text),
+ (r'\d+\.\d+\.\d+\.\d+(?:/\d+)?', Number),
+ (r'\d+', Number),
+ (r'/([a-z0-9][\w./-]+)', String.Other),
+ (r'(on|off|none|any|all|double|email|dns|min|minimal|'
+ r'os|productonly|full|emerg|alert|crit|error|warn|'
+ r'notice|info|debug|registry|script|inetd|standalone|'
+ r'user|group)\b', Keyword),
+ (r'"([^"\\]*(?:\\.[^"\\]*)*)"', String.Double),
+ (r'[^\s"\\]+', Text)
+ ],
+ }
+
+
+class SquidConfLexer(RegexLexer):
+ """
+ Lexer for `squid <http://www.squid-cache.org/>`_ configuration files.
+
+ .. versionadded:: 0.9
+ """
+
+ name = 'SquidConf'
+ aliases = ['squidconf', 'squid.conf', 'squid']
+ filenames = ['squid.conf']
+ mimetypes = ['text/x-squidconf']
+ flags = re.IGNORECASE
+
+ keywords = (
+ "access_log", "acl", "always_direct", "announce_host",
+ "announce_period", "announce_port", "announce_to", "anonymize_headers",
+ "append_domain", "as_whois_server", "auth_param_basic",
+ "authenticate_children", "authenticate_program", "authenticate_ttl",
+ "broken_posts", "buffered_logs", "cache_access_log", "cache_announce",
+ "cache_dir", "cache_dns_program", "cache_effective_group",
+ "cache_effective_user", "cache_host", "cache_host_acl",
+ "cache_host_domain", "cache_log", "cache_mem", "cache_mem_high",
+ "cache_mem_low", "cache_mgr", "cachemgr_passwd", "cache_peer",
+ "cache_peer_access", "cahce_replacement_policy", "cache_stoplist",
+ "cache_stoplist_pattern", "cache_store_log", "cache_swap",
+ "cache_swap_high", "cache_swap_log", "cache_swap_low", "client_db",
+ "client_lifetime", "client_netmask", "connect_timeout", "coredump_dir",
+ "dead_peer_timeout", "debug_options", "delay_access", "delay_class",
+ "delay_initial_bucket_level", "delay_parameters", "delay_pools",
+ "deny_info", "dns_children", "dns_defnames", "dns_nameservers",
+ "dns_testnames", "emulate_httpd_log", "err_html_text",
+ "fake_user_agent", "firewall_ip", "forwarded_for", "forward_snmpd_port",
+ "fqdncache_size", "ftpget_options", "ftpget_program", "ftp_list_width",
+ "ftp_passive", "ftp_user", "half_closed_clients", "header_access",
+ "header_replace", "hierarchy_stoplist", "high_response_time_warning",
+ "high_page_fault_warning", "hosts_file", "htcp_port", "http_access",
+ "http_anonymizer", "httpd_accel", "httpd_accel_host",
+ "httpd_accel_port", "httpd_accel_uses_host_header",
+ "httpd_accel_with_proxy", "http_port", "http_reply_access",
+ "icp_access", "icp_hit_stale", "icp_port", "icp_query_timeout",
+ "ident_lookup", "ident_lookup_access", "ident_timeout",
+ "incoming_http_average", "incoming_icp_average", "inside_firewall",
+ "ipcache_high", "ipcache_low", "ipcache_size", "local_domain",
+ "local_ip", "logfile_rotate", "log_fqdn", "log_icp_queries",
+ "log_mime_hdrs", "maximum_object_size", "maximum_single_addr_tries",
+ "mcast_groups", "mcast_icp_query_timeout", "mcast_miss_addr",
+ "mcast_miss_encode_key", "mcast_miss_port", "memory_pools",
+ "memory_pools_limit", "memory_replacement_policy", "mime_table",
+ "min_http_poll_cnt", "min_icp_poll_cnt", "minimum_direct_hops",
+ "minimum_object_size", "minimum_retry_timeout", "miss_access",
+ "negative_dns_ttl", "negative_ttl", "neighbor_timeout",
+ "neighbor_type_domain", "netdb_high", "netdb_low", "netdb_ping_period",
+ "netdb_ping_rate", "never_direct", "no_cache", "passthrough_proxy",
+ "pconn_timeout", "pid_filename", "pinger_program", "positive_dns_ttl",
+ "prefer_direct", "proxy_auth", "proxy_auth_realm", "query_icmp",
+ "quick_abort", "quick_abort_max", "quick_abort_min",
+ "quick_abort_pct", "range_offset_limit", "read_timeout",
+ "redirect_children", "redirect_program",
+ "redirect_rewrites_host_header", "reference_age",
+ "refresh_pattern", "reload_into_ims", "request_body_max_size",
+ "request_size", "request_timeout", "shutdown_lifetime",
+ "single_parent_bypass", "siteselect_timeout", "snmp_access",
+ "snmp_incoming_address", "snmp_port", "source_ping", "ssl_proxy",
+ "store_avg_object_size", "store_objects_per_bucket",
+ "strip_query_terms", "swap_level1_dirs", "swap_level2_dirs",
+ "tcp_incoming_address", "tcp_outgoing_address", "tcp_recv_bufsize",
+ "test_reachability", "udp_hit_obj", "udp_hit_obj_size",
+ "udp_incoming_address", "udp_outgoing_address", "unique_hostname",
+ "unlinkd_program", "uri_whitespace", "useragent_log",
+ "visible_hostname", "wais_relay", "wais_relay_host", "wais_relay_port",
+ )
+
+ opts = (
+ "proxy-only", "weight", "ttl", "no-query", "default", "round-robin",
+ "multicast-responder", "on", "off", "all", "deny", "allow", "via",
+ "parent", "no-digest", "heap", "lru", "realm", "children", "q1", "q2",
+ "credentialsttl", "none", "disable", "offline_toggle", "diskd",
+ )
+
+ actions = (
+ "shutdown", "info", "parameter", "server_list", "client_list",
+ r'squid.conf',
+ )
+
+ actions_stats = (
+ "objects", "vm_objects", "utilization", "ipcache", "fqdncache", "dns",
+ "redirector", "io", "reply_headers", "filedescriptors", "netdb",
+ )
+
+ actions_log = ("status", "enable", "disable", "clear")
+
+ acls = (
+ "url_regex", "urlpath_regex", "referer_regex", "port", "proto",
+ "req_mime_type", "rep_mime_type", "method", "browser", "user", "src",
+ "dst", "time", "dstdomain", "ident", "snmp_community",
+ )
+
+ ip_re = (
+ r'(?:(?:(?:[3-9]\d?|2(?:5[0-5]|[0-4]?\d)?|1\d{0,2}|0x0*[0-9a-f]{1,2}|'
+ r'0+[1-3]?[0-7]{0,2})(?:\.(?:[3-9]\d?|2(?:5[0-5]|[0-4]?\d)?|1\d{0,2}|'
+ r'0x0*[0-9a-f]{1,2}|0+[1-3]?[0-7]{0,2})){3})|(?!.*::.*::)(?:(?!:)|'
+ r':(?=:))(?:[0-9a-f]{0,4}(?:(?<=::)|(?<!::):)){6}(?:[0-9a-f]{0,4}'
+ r'(?:(?<=::)|(?<!::):)[0-9a-f]{0,4}(?:(?<=::)|(?<!:)|(?<=:)(?<!::):)|'
+ r'(?:25[0-4]|2[0-4]\d|1\d\d|[1-9]?\d)(?:\.(?:25[0-4]|2[0-4]\d|1\d\d|'
+ r'[1-9]?\d)){3}))'
+ )
+
+ tokens = {
+ 'root': [
+ (r'\s+', Whitespace),
+ (r'#', Comment, 'comment'),
+ (words(keywords, prefix=r'\b', suffix=r'\b'), Keyword),
+ (words(opts, prefix=r'\b', suffix=r'\b'), Name.Constant),
+ # Actions
+ (words(actions, prefix=r'\b', suffix=r'\b'), String),
+ (words(actions_stats, prefix=r'stats/', suffix=r'\b'), String),
+ (words(actions_log, prefix=r'log/', suffix=r'='), String),
+ (words(acls, prefix=r'\b', suffix=r'\b'), Keyword),
+ (ip_re + r'(?:/(?:' + ip_re + r'|\b\d+\b))?', Number.Float),
+ (r'(?:\b\d+\b(?:-\b\d+|%)?)', Number),
+ (r'\S+', Text),
+ ],
+ 'comment': [
+ (r'\s*TAG:.*', String.Escape, '#pop'),
+ (r'.+', Comment, '#pop'),
+ default('#pop'),
+ ],
+ }
+
+
+class NginxConfLexer(RegexLexer):
+ """
+ Lexer for `Nginx <http://nginx.net/>`_ configuration files.
+
+ .. versionadded:: 0.11
+ """
+ name = 'Nginx configuration file'
+ aliases = ['nginx']
+ filenames = ['nginx.conf']
+ mimetypes = ['text/x-nginx-conf']
+
+ tokens = {
+ 'root': [
+ (r'(include)(\s+)([^\s;]+)', bygroups(Keyword, Text, Name)),
+ (r'[^\s;#]+', Keyword, 'stmt'),
+ include('base'),
+ ],
+ 'block': [
+ (r'\}', Punctuation, '#pop:2'),
+ (r'[^\s;#]+', Keyword.Namespace, 'stmt'),
+ include('base'),
+ ],
+ 'stmt': [
+ (r'\{', Punctuation, 'block'),
+ (r';', Punctuation, '#pop'),
+ include('base'),
+ ],
+ 'base': [
+ (r'#.*\n', Comment.Single),
+ (r'on|off', Name.Constant),
+ (r'\$[^\s;#()]+', Name.Variable),
+ (r'([a-z0-9.-]+)(:)([0-9]+)',
+ bygroups(Name, Punctuation, Number.Integer)),
+ (r'[a-z-]+/[a-z-+]+', String), # mimetype
+ # (r'[a-zA-Z._-]+', Keyword),
+ (r'[0-9]+[km]?\b', Number.Integer),
+ (r'(~)(\s*)([^\s{]+)', bygroups(Punctuation, Text, String.Regex)),
+ (r'[:=~]', Punctuation),
+ (r'[^\s;#{}$]+', String), # catch all
+ (r'/[^\s;#]*', Name), # pathname
+ (r'\s+', Text),
+ (r'[$;]', Text), # leftover characters
+ ],
+ }
+
+
+class LighttpdConfLexer(RegexLexer):
+ """
+ Lexer for `Lighttpd <http://lighttpd.net/>`_ configuration files.
+
+ .. versionadded:: 0.11
+ """
+ name = 'Lighttpd configuration file'
+ aliases = ['lighty', 'lighttpd']
+ filenames = []
+ mimetypes = ['text/x-lighttpd-conf']
+
+ tokens = {
+ 'root': [
+ (r'#.*\n', Comment.Single),
+ (r'/\S*', Name), # pathname
+ (r'[a-zA-Z._-]+', Keyword),
+ (r'\d+\.\d+\.\d+\.\d+(?:/\d+)?', Number),
+ (r'[0-9]+', Number),
+ (r'=>|=~|\+=|==|=|\+', Operator),
+ (r'\$[A-Z]+', Name.Builtin),
+ (r'[(){}\[\],]', Punctuation),
+ (r'"([^"\\]*(?:\\.[^"\\]*)*)"', String.Double),
+ (r'\s+', Text),
+ ],
+
+ }
+
+
+class DockerLexer(RegexLexer):
+ """
+ Lexer for `Docker <http://docker.io>`_ configuration files.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Docker'
+ aliases = ['docker', 'dockerfile']
+ filenames = ['Dockerfile', '*.docker']
+ mimetypes = ['text/x-dockerfile-config']
+
+ _keywords = (r'(?:FROM|MAINTAINER|CMD|EXPOSE|ENV|ADD|ENTRYPOINT|'
+ r'VOLUME|WORKDIR)')
+
+ flags = re.IGNORECASE | re.MULTILINE
+
+ tokens = {
+ 'root': [
+ (r'^(ONBUILD)(\s+)(%s)\b' % (_keywords,),
+ bygroups(Name.Keyword, Whitespace, Keyword)),
+ (r'^(%s)\b(.*)' % (_keywords,), bygroups(Keyword, String)),
+ (r'#.*', Comment),
+ (r'RUN', Keyword), # Rest of line falls through
+ (r'(.*\\\n)*.+', using(BashLexer)),
+ ],
+ }
+
+
+class TerraformLexer(RegexLexer):
+ """
+ Lexer for `terraformi .tf files <https://www.terraform.io/>`_.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'Terraform'
+ aliases = ['terraform', 'tf']
+ filenames = ['*.tf']
+ mimetypes = ['application/x-tf', 'application/x-terraform']
+
+ tokens = {
+ 'root': [
+ include('string'),
+ include('punctuation'),
+ include('curly'),
+ include('basic'),
+ include('whitespace'),
+ (r'[0-9]+', Number),
+ ],
+ 'basic': [
+ (words(('true', 'false'), prefix=r'\b', suffix=r'\b'), Keyword.Type),
+ (r'\s*/\*', Comment.Multiline, 'comment'),
+ (r'\s*#.*\n', Comment.Single),
+ (r'(.*?)(\s*)(=)', bygroups(Name.Attribute, Text, Operator)),
+ (words(('variable', 'resource', 'provider', 'provisioner', 'module'),
+ prefix=r'\b', suffix=r'\b'), Keyword.Reserved, 'function'),
+ (words(('ingress', 'egress', 'listener', 'default', 'connection'),
+ prefix=r'\b', suffix=r'\b'), Keyword.Declaration),
+ ('\$\{', String.Interpol, 'var_builtin'),
+ ],
+ 'function': [
+ (r'(\s+)(".*")(\s+)', bygroups(Text, String, Text)),
+ include('punctuation'),
+ include('curly'),
+ ],
+ 'var_builtin': [
+ (r'\$\{', String.Interpol, '#push'),
+ (words(('concat', 'file', 'join', 'lookup', 'element'),
+ prefix=r'\b', suffix=r'\b'), Name.Builtin),
+ include('string'),
+ include('punctuation'),
+ (r'\s+', Text),
+ (r'\}', String.Interpol, '#pop'),
+ ],
+ 'string': [
+ (r'(".*")', bygroups(String.Double)),
+ ],
+ 'punctuation': [
+ (r'[\[\](),.]', Punctuation),
+ ],
+ # Keep this seperate from punctuation - we sometimes want to use different
+ # Tokens for { }
+ 'curly': [
+ (r'\{', Text.Punctuation),
+ (r'\}', Text.Punctuation),
+ ],
+ 'comment': [
+ (r'[^*/]', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline)
+ ],
+ 'whitespace': [
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'\\\n', Text),
+ ],
+ }
+
+
+class TermcapLexer(RegexLexer):
+ """
+ Lexer for termcap database source.
+
+ This is very simple and minimal.
+
+ .. versionadded:: 2.1
+ """
+ name = 'Termcap'
+ aliases = ['termcap']
+ filenames = ['termcap', 'termcap.src']
+ mimetypes = []
+
+ # NOTE:
+ # * multiline with trailing backslash
+ # * separator is ':'
+ # * to embed colon as data, we must use \072
+ # * space after separator is not allowed (mayve)
+ tokens = {
+ 'root': [
+ (r'^#.*$', Comment),
+ (r'^[^\s#:|]+', Name.Tag, 'names'),
+ ],
+ 'names': [
+ (r'\n', Text, '#pop'),
+ (r':', Punctuation, 'defs'),
+ (r'\|', Punctuation),
+ (r'[^:|]+', Name.Attribute),
+ ],
+ 'defs': [
+ (r'\\\n[ \t]*', Text),
+ (r'\n[ \t]*', Text, '#pop:2'),
+ (r'(#)([0-9]+)', bygroups(Operator, Number)),
+ (r'=', Operator, 'data'),
+ (r':', Punctuation),
+ (r'[^\s:=#]+', Name.Class),
+ ],
+ 'data': [
+ (r'\\072', Literal),
+ (r':', Punctuation, '#pop'),
+ (r'[^:\\]+', Literal), # for performance
+ (r'.', Literal),
+ ],
+ }
+
+
+class TerminfoLexer(RegexLexer):
+ """
+ Lexer for terminfo database source.
+
+ This is very simple and minimal.
+
+ .. versionadded:: 2.1
+ """
+ name = 'Terminfo'
+ aliases = ['terminfo']
+ filenames = ['terminfo', 'terminfo.src']
+ mimetypes = []
+
+ # NOTE:
+ # * multiline with leading whitespace
+ # * separator is ','
+ # * to embed comma as data, we can use \,
+ # * space after separator is allowed
+ tokens = {
+ 'root': [
+ (r'^#.*$', Comment),
+ (r'^[^\s#,|]+', Name.Tag, 'names'),
+ ],
+ 'names': [
+ (r'\n', Text, '#pop'),
+ (r'(,)([ \t]*)', bygroups(Punctuation, Text), 'defs'),
+ (r'\|', Punctuation),
+ (r'[^,|]+', Name.Attribute),
+ ],
+ 'defs': [
+ (r'\n[ \t]+', Text),
+ (r'\n', Text, '#pop:2'),
+ (r'(#)([0-9]+)', bygroups(Operator, Number)),
+ (r'=', Operator, 'data'),
+ (r'(,)([ \t]*)', bygroups(Punctuation, Text)),
+ (r'[^\s,=#]+', Name.Class),
+ ],
+ 'data': [
+ (r'\\[,\\]', Literal),
+ (r'(,)([ \t]*)', bygroups(Punctuation, Text), '#pop'),
+ (r'[^\\,]+', Literal), # for performance
+ (r'.', Literal),
+ ],
+ }
+
+
+class PkgConfigLexer(RegexLexer):
+ """
+ Lexer for `pkg-config
+ <http://www.freedesktop.org/wiki/Software/pkg-config/>`_
+ (see also `manual page <http://linux.die.net/man/1/pkg-config>`_).
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'PkgConfig'
+ aliases = ['pkgconfig']
+ filenames = ['*.pc']
+ mimetypes = []
+
+ tokens = {
+ 'root': [
+ (r'#.*$', Comment.Single),
+
+ # variable definitions
+ (r'^(\w+)(=)', bygroups(Name.Attribute, Operator)),
+
+ # keyword lines
+ (r'^([\w.]+)(:)',
+ bygroups(Name.Tag, Punctuation), 'spvalue'),
+
+ # variable references
+ include('interp'),
+
+ # fallback
+ (r'[^${}#=:\n.]+', Text),
+ (r'.', Text),
+ ],
+ 'interp': [
+ # you can escape literal "$" as "$$"
+ (r'\$\$', Text),
+
+ # variable references
+ (r'\$\{', String.Interpol, 'curly'),
+ ],
+ 'curly': [
+ (r'\}', String.Interpol, '#pop'),
+ (r'\w+', Name.Attribute),
+ ],
+ 'spvalue': [
+ include('interp'),
+
+ (r'#.*$', Comment.Single, '#pop'),
+ (r'\n', Text, '#pop'),
+
+ # fallback
+ (r'[^${}#\n]+', Text),
+ (r'.', Text),
+ ],
+ }
+
+
+class PacmanConfLexer(RegexLexer):
+ """
+ Lexer for `pacman.conf
+ <https://www.archlinux.org/pacman/pacman.conf.5.html>`_.
+
+ Actually, IniLexer works almost fine for this format,
+ but it yield error token. It is because pacman.conf has
+ a form without assignment like:
+
+ UseSyslog
+ Color
+ TotalDownload
+ CheckSpace
+ VerbosePkgLists
+
+ These are flags to switch on.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'PacmanConf'
+ aliases = ['pacmanconf']
+ filenames = ['pacman.conf']
+ mimetypes = []
+
+ tokens = {
+ 'root': [
+ # comment
+ (r'#.*$', Comment.Single),
+
+ # section header
+ (r'^\s*\[.*?\]\s*$', Keyword),
+
+ # variable definitions
+ # (Leading space is allowed...)
+ (r'(\w+)(\s*)(=)',
+ bygroups(Name.Attribute, Text, Operator)),
+
+ # flags to on
+ (r'^(\s*)(\w+)(\s*)$',
+ bygroups(Text, Name.Attribute, Text)),
+
+ # built-in special values
+ (words((
+ '$repo', # repository
+ '$arch', # architecture
+ '%o', # outfile
+ '%u', # url
+ ), suffix=r'\b'),
+ Name.Variable),
+
+ # fallback
+ (r'.', Text),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/console.py b/wandb/vendor/pygments/lexers/console.py
new file mode 100644
index 0000000000000000000000000000000000000000..77bb72e5a3a33f352db047dcf35a8503a7571786
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/console.py
@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.console
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for misc console output.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups
+from pygments.token import Generic, Comment, String, Text, Keyword, Name, \
+ Punctuation, Number
+
+__all__ = ['VCTreeStatusLexer', 'PyPyLogLexer']
+
+
+class VCTreeStatusLexer(RegexLexer):
+ """
+ For colorizing output of version control status commands, like "hg
+ status" or "svn status".
+
+ .. versionadded:: 2.0
+ """
+ name = 'VCTreeStatus'
+ aliases = ['vctreestatus']
+ filenames = []
+ mimetypes = []
+
+ tokens = {
+ 'root': [
+ (r'^A \+ C\s+', Generic.Error),
+ (r'^A\s+\+?\s+', String),
+ (r'^M\s+', Generic.Inserted),
+ (r'^C\s+', Generic.Error),
+ (r'^D\s+', Generic.Deleted),
+ (r'^[?!]\s+', Comment.Preproc),
+ (r' >\s+.*\n', Comment.Preproc),
+ (r'.*\n', Text)
+ ]
+ }
+
+
+class PyPyLogLexer(RegexLexer):
+ """
+ Lexer for PyPy log files.
+
+ .. versionadded:: 1.5
+ """
+ name = "PyPy Log"
+ aliases = ["pypylog", "pypy"]
+ filenames = ["*.pypylog"]
+ mimetypes = ['application/x-pypylog']
+
+ tokens = {
+ "root": [
+ (r"\[\w+\] \{jit-log-.*?$", Keyword, "jit-log"),
+ (r"\[\w+\] \{jit-backend-counts$", Keyword, "jit-backend-counts"),
+ include("extra-stuff"),
+ ],
+ "jit-log": [
+ (r"\[\w+\] jit-log-.*?}$", Keyword, "#pop"),
+ (r"^\+\d+: ", Comment),
+ (r"--end of the loop--", Comment),
+ (r"[ifp]\d+", Name),
+ (r"ptr\d+", Name),
+ (r"(\()(\w+(?:\.\w+)?)(\))",
+ bygroups(Punctuation, Name.Builtin, Punctuation)),
+ (r"[\[\]=,()]", Punctuation),
+ (r"(\d+\.\d+|inf|-inf)", Number.Float),
+ (r"-?\d+", Number.Integer),
+ (r"'.*'", String),
+ (r"(None|descr|ConstClass|ConstPtr|TargetToken)", Name),
+ (r"<.*?>+", Name.Builtin),
+ (r"(label|debug_merge_point|jump|finish)", Name.Class),
+ (r"(int_add_ovf|int_add|int_sub_ovf|int_sub|int_mul_ovf|int_mul|"
+ r"int_floordiv|int_mod|int_lshift|int_rshift|int_and|int_or|"
+ r"int_xor|int_eq|int_ne|int_ge|int_gt|int_le|int_lt|int_is_zero|"
+ r"int_is_true|"
+ r"uint_floordiv|uint_ge|uint_lt|"
+ r"float_add|float_sub|float_mul|float_truediv|float_neg|"
+ r"float_eq|float_ne|float_ge|float_gt|float_le|float_lt|float_abs|"
+ r"ptr_eq|ptr_ne|instance_ptr_eq|instance_ptr_ne|"
+ r"cast_int_to_float|cast_float_to_int|"
+ r"force_token|quasiimmut_field|same_as|virtual_ref_finish|"
+ r"virtual_ref|mark_opaque_ptr|"
+ r"call_may_force|call_assembler|call_loopinvariant|"
+ r"call_release_gil|call_pure|call|"
+ r"new_with_vtable|new_array|newstr|newunicode|new|"
+ r"arraylen_gc|"
+ r"getarrayitem_gc_pure|getarrayitem_gc|setarrayitem_gc|"
+ r"getarrayitem_raw|setarrayitem_raw|getfield_gc_pure|"
+ r"getfield_gc|getinteriorfield_gc|setinteriorfield_gc|"
+ r"getfield_raw|setfield_gc|setfield_raw|"
+ r"strgetitem|strsetitem|strlen|copystrcontent|"
+ r"unicodegetitem|unicodesetitem|unicodelen|"
+ r"guard_true|guard_false|guard_value|guard_isnull|"
+ r"guard_nonnull_class|guard_nonnull|guard_class|guard_no_overflow|"
+ r"guard_not_forced|guard_no_exception|guard_not_invalidated)",
+ Name.Builtin),
+ include("extra-stuff"),
+ ],
+ "jit-backend-counts": [
+ (r"\[\w+\] jit-backend-counts}$", Keyword, "#pop"),
+ (r":", Punctuation),
+ (r"\d+", Number),
+ include("extra-stuff"),
+ ],
+ "extra-stuff": [
+ (r"\s+", Text),
+ (r"#.*?$", Comment),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/crystal.py b/wandb/vendor/pygments/lexers/crystal.py
new file mode 100644
index 0000000000000000000000000000000000000000..bea4833f2ea011bc1953222092ec8b2c0f52b50c
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/crystal.py
@@ -0,0 +1,393 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.crystal
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for Crystal.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import ExtendedRegexLexer, include, \
+ bygroups, default, LexerContext, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Error
+
+__all__ = ['CrystalLexer']
+
+line_re = re.compile('.*?\n')
+
+
+CRYSTAL_OPERATORS = [
+ '!=', '!~', '!', '%', '&&', '&', '**', '*', '+', '-', '/', '<=>', '<<', '<=', '<',
+ '===', '==', '=~', '=', '>=', '>>', '>', '[]=', '[]?', '[]', '^', '||', '|', '~'
+]
+
+
+class CrystalLexer(ExtendedRegexLexer):
+ """
+ For `Crystal <http://crystal-lang.org>`_ source code.
+
+ .. versionadded:: 2.2
+ """
+
+ name = 'Crystal'
+ aliases = ['cr', 'crystal']
+ filenames = ['*.cr']
+ mimetypes = ['text/x-crystal']
+
+ flags = re.DOTALL | re.MULTILINE
+
+ def heredoc_callback(self, match, ctx):
+ # okay, this is the hardest part of parsing Crystal...
+ # match: 1 = <<-?, 2 = quote? 3 = name 4 = quote? 5 = rest of line
+
+ start = match.start(1)
+ yield start, Operator, match.group(1) # <<-?
+ yield match.start(2), String.Heredoc, match.group(2) # quote ", ', `
+ yield match.start(3), String.Delimiter, match.group(3) # heredoc name
+ yield match.start(4), String.Heredoc, match.group(4) # quote again
+
+ heredocstack = ctx.__dict__.setdefault('heredocstack', [])
+ outermost = not bool(heredocstack)
+ heredocstack.append((match.group(1) == '<<-', match.group(3)))
+
+ ctx.pos = match.start(5)
+ ctx.end = match.end(5)
+ # this may find other heredocs
+ for i, t, v in self.get_tokens_unprocessed(context=ctx):
+ yield i, t, v
+ ctx.pos = match.end()
+
+ if outermost:
+ # this is the outer heredoc again, now we can process them all
+ for tolerant, hdname in heredocstack:
+ lines = []
+ for match in line_re.finditer(ctx.text, ctx.pos):
+ if tolerant:
+ check = match.group().strip()
+ else:
+ check = match.group().rstrip()
+ if check == hdname:
+ for amatch in lines:
+ yield amatch.start(), String.Heredoc, amatch.group()
+ yield match.start(), String.Delimiter, match.group()
+ ctx.pos = match.end()
+ break
+ else:
+ lines.append(match)
+ else:
+ # end of heredoc not found -- error!
+ for amatch in lines:
+ yield amatch.start(), Error, amatch.group()
+ ctx.end = len(ctx.text)
+ del heredocstack[:]
+
+ def gen_crystalstrings_rules():
+ def intp_regex_callback(self, match, ctx):
+ yield match.start(1), String.Regex, match.group(1) # begin
+ nctx = LexerContext(match.group(3), 0, ['interpolated-regex'])
+ for i, t, v in self.get_tokens_unprocessed(context=nctx):
+ yield match.start(3)+i, t, v
+ yield match.start(4), String.Regex, match.group(4) # end[imsx]*
+ ctx.pos = match.end()
+
+ def intp_string_callback(self, match, ctx):
+ yield match.start(1), String.Other, match.group(1)
+ nctx = LexerContext(match.group(3), 0, ['interpolated-string'])
+ for i, t, v in self.get_tokens_unprocessed(context=nctx):
+ yield match.start(3)+i, t, v
+ yield match.start(4), String.Other, match.group(4) # end
+ ctx.pos = match.end()
+
+ states = {}
+ states['strings'] = [
+ (r'\:@{0,2}[a-zA-Z_]\w*[!?]?', String.Symbol),
+ (words(CRYSTAL_OPERATORS, prefix=r'\:@{0,2}'), String.Symbol),
+ (r":'(\\\\|\\'|[^'])*'", String.Symbol),
+ # This allows arbitrary text after '\ for simplicity
+ (r"'(\\\\|\\'|[^']|\\[^'\\]+)'", String.Char),
+ (r':"', String.Symbol, 'simple-sym'),
+ # Crystal doesn't have "symbol:"s but this simplifies function args
+ (r'([a-zA-Z_]\w*)(:)(?!:)', bygroups(String.Symbol, Punctuation)),
+ (r'"', String.Double, 'simple-string'),
+ (r'(?<!\.)`', String.Backtick, 'simple-backtick'),
+ ]
+
+ # double-quoted string and symbol
+ for name, ttype, end in ('string', String.Double, '"'), \
+ ('sym', String.Symbol, '"'), \
+ ('backtick', String.Backtick, '`'):
+ states['simple-'+name] = [
+ include('string-escaped' if name == 'sym' else 'string-intp-escaped'),
+ (r'[^\\%s#]+' % end, ttype),
+ (r'[\\#]', ttype),
+ (end, ttype, '#pop'),
+ ]
+
+ # braced quoted strings
+ for lbrace, rbrace, bracecc, name in \
+ ('\\{', '\\}', '{}', 'cb'), \
+ ('\\[', '\\]', '\\[\\]', 'sb'), \
+ ('\\(', '\\)', '()', 'pa'), \
+ ('<', '>', '<>', 'ab'):
+ states[name+'-intp-string'] = [
+ (r'\\[' + lbrace + ']', String.Other),
+ (lbrace, String.Other, '#push'),
+ (rbrace, String.Other, '#pop'),
+ include('string-intp-escaped'),
+ (r'[\\#' + bracecc + ']', String.Other),
+ (r'[^\\#' + bracecc + ']+', String.Other),
+ ]
+ states['strings'].append((r'%' + lbrace, String.Other,
+ name+'-intp-string'))
+ states[name+'-string'] = [
+ (r'\\[\\' + bracecc + ']', String.Other),
+ (lbrace, String.Other, '#push'),
+ (rbrace, String.Other, '#pop'),
+ (r'[\\#' + bracecc + ']', String.Other),
+ (r'[^\\#' + bracecc + ']+', String.Other),
+ ]
+ # http://crystal-lang.org/docs/syntax_and_semantics/literals/array.html
+ states['strings'].append((r'%[wi]' + lbrace, String.Other,
+ name+'-string'))
+ states[name+'-regex'] = [
+ (r'\\[\\' + bracecc + ']', String.Regex),
+ (lbrace, String.Regex, '#push'),
+ (rbrace + '[imsx]*', String.Regex, '#pop'),
+ include('string-intp'),
+ (r'[\\#' + bracecc + ']', String.Regex),
+ (r'[^\\#' + bracecc + ']+', String.Regex),
+ ]
+ states['strings'].append((r'%r' + lbrace, String.Regex,
+ name+'-regex'))
+
+ # these must come after %<brace>!
+ states['strings'] += [
+ # %r regex
+ (r'(%r([\W_]))((?:\\\2|(?!\2).)*)(\2[imsx]*)',
+ intp_regex_callback),
+ # regular fancy strings with qsw
+ (r'(%[wi]([\W_]))((?:\\\2|(?!\2).)*)(\2)',
+ intp_string_callback),
+ # special forms of fancy strings after operators or
+ # in method calls with braces
+ (r'(?<=[-+/*%=<>&!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
+ bygroups(Text, String.Other, None)),
+ # and because of fixed width lookbehinds the whole thing a
+ # second time for line startings...
+ (r'^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
+ bygroups(Text, String.Other, None)),
+ # all regular fancy strings without qsw
+ (r'(%([\[{(<]))((?:\\\2|(?!\2).)*)(\2)',
+ intp_string_callback),
+ ]
+
+ return states
+
+ tokens = {
+ 'root': [
+ (r'#.*?$', Comment.Single),
+ # keywords
+ (words('''
+ abstract asm as begin break case do else elsif end ensure extend ifdef if
+ include instance_sizeof next of pointerof private protected rescue return
+ require sizeof super then typeof unless until when while with yield
+ '''.split(), suffix=r'\b'), Keyword),
+ (words(['true', 'false', 'nil'], suffix=r'\b'), Keyword.Constant),
+ # start of function, class and module names
+ (r'(module|lib)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)',
+ bygroups(Keyword, Text, Name.Namespace)),
+ (r'(def|fun|macro)(\s+)((?:[a-zA-Z_]\w*::)*)',
+ bygroups(Keyword, Text, Name.Namespace), 'funcname'),
+ (r'def(?=[*%&^`~+-/\[<>=])', Keyword, 'funcname'),
+ (r'(class|struct|union|type|alias|enum)(\s+)((?:[a-zA-Z_]\w*::)*)',
+ bygroups(Keyword, Text, Name.Namespace), 'classname'),
+ (r'(self|out|uninitialized)\b|(is_a|responds_to)\?', Keyword.Pseudo),
+ # macros
+ (words('''
+ debugger record pp assert_responds_to spawn parallel
+ getter setter property delegate def_hash def_equals def_equals_and_hash
+ forward_missing_to
+ '''.split(), suffix=r'\b'), Name.Builtin.Pseudo),
+ (r'getter[!?]|property[!?]|__(DIR|FILE|LINE)__\b', Name.Builtin.Pseudo),
+ # builtins
+ # http://crystal-lang.org/api/toplevel.html
+ (words('''
+ Object Value Struct Reference Proc Class Nil Symbol Enum Void
+ Bool Number Int Int8 Int16 Int32 Int64 UInt8 UInt16 UInt32 UInt64
+ Float Float32 Float64 Char String
+ Pointer Slice Range Exception Regex
+ Mutex StaticArray Array Hash Set Tuple Deque Box Process File
+ Dir Time Channel Concurrent Scheduler
+ abort at_exit caller delay exit fork future get_stack_top gets
+ lazy loop main p print printf puts
+ raise rand read_line sleep sprintf system with_color
+ '''.split(), prefix=r'(?<!\.)', suffix=r'\b'), Name.Builtin),
+ # normal heredocs
+ (r'(?<!\w)(<<-?)(["`\']?)([a-zA-Z_]\w*)(\2)(.*?\n)',
+ heredoc_callback),
+ # empty string heredocs
+ (r'(<<-?)("|\')()(\2)(.*?\n)', heredoc_callback),
+ (r'__END__', Comment.Preproc, 'end-part'),
+ # multiline regex (after keywords or assignments)
+ (r'(?:^|(?<=[=<>~!:])|'
+ r'(?<=(?:\s|;)when\s)|'
+ r'(?<=(?:\s|;)or\s)|'
+ r'(?<=(?:\s|;)and\s)|'
+ r'(?<=\.index\s)|'
+ r'(?<=\.scan\s)|'
+ r'(?<=\.sub\s)|'
+ r'(?<=\.sub!\s)|'
+ r'(?<=\.gsub\s)|'
+ r'(?<=\.gsub!\s)|'
+ r'(?<=\.match\s)|'
+ r'(?<=(?:\s|;)if\s)|'
+ r'(?<=(?:\s|;)elsif\s)|'
+ r'(?<=^when\s)|'
+ r'(?<=^index\s)|'
+ r'(?<=^scan\s)|'
+ r'(?<=^sub\s)|'
+ r'(?<=^gsub\s)|'
+ r'(?<=^sub!\s)|'
+ r'(?<=^gsub!\s)|'
+ r'(?<=^match\s)|'
+ r'(?<=^if\s)|'
+ r'(?<=^elsif\s)'
+ r')(\s*)(/)', bygroups(Text, String.Regex), 'multiline-regex'),
+ # multiline regex (in method calls or subscripts)
+ (r'(?<=\(|,|\[)/', String.Regex, 'multiline-regex'),
+ # multiline regex (this time the funny no whitespace rule)
+ (r'(\s+)(/)(?![\s=])', bygroups(Text, String.Regex),
+ 'multiline-regex'),
+ # lex numbers and ignore following regular expressions which
+ # are division operators in fact (grrrr. i hate that. any
+ # better ideas?)
+ # since pygments 0.7 we also eat a "?" operator after numbers
+ # so that the char operator does not work. Chars are not allowed
+ # there so that you can use the ternary operator.
+ # stupid example:
+ # x>=0?n[x]:""
+ (r'(0o[0-7]+(?:_[0-7]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
+ bygroups(Number.Oct, Text, Operator)),
+ (r'(0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
+ bygroups(Number.Hex, Text, Operator)),
+ (r'(0b[01]+(?:_[01]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
+ bygroups(Number.Bin, Text, Operator)),
+ # 3 separate expressions for floats because any of the 3 optional
+ # parts makes it a float
+ (r'((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)(?:e[+-]?[0-9]+)?'
+ r'(?:_?f[0-9]+)?)(\s*)([/?])?',
+ bygroups(Number.Float, Text, Operator)),
+ (r'((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)?(?:e[+-]?[0-9]+)'
+ r'(?:_?f[0-9]+)?)(\s*)([/?])?',
+ bygroups(Number.Float, Text, Operator)),
+ (r'((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)?(?:e[+-]?[0-9]+)?'
+ r'(?:_?f[0-9]+))(\s*)([/?])?',
+ bygroups(Number.Float, Text, Operator)),
+ (r'(0\b|[1-9][\d]*(?:_\d+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
+ bygroups(Number.Integer, Text, Operator)),
+ # Names
+ (r'@@[a-zA-Z_]\w*', Name.Variable.Class),
+ (r'@[a-zA-Z_]\w*', Name.Variable.Instance),
+ (r'\$\w+', Name.Variable.Global),
+ (r'\$[!@&`\'+~=/\\,;.<>_*$?:"^-]', Name.Variable.Global),
+ (r'\$-[0adFiIlpvw]', Name.Variable.Global),
+ (r'::', Operator),
+ include('strings'),
+ # chars
+ (r'\?(\\[MC]-)*' # modifiers
+ r'(\\([\\befnrtv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)'
+ r'(?!\w)',
+ String.Char),
+ (r'[A-Z][A-Z_]+\b', Name.Constant),
+ # macro expansion
+ (r'\{%', String.Interpol, 'in-macro-control'),
+ (r'\{\{', String.Interpol, 'in-macro-expr'),
+ # attributes
+ (r'(@\[)(\s*)([A-Z]\w*)',
+ bygroups(Operator, Text, Name.Decorator), 'in-attr'),
+ # this is needed because Crystal attributes can look
+ # like keywords (class) or like this: ` ?!?
+ (words(CRYSTAL_OPERATORS, prefix=r'(\.|::)'),
+ bygroups(Operator, Name.Operator)),
+ (r'(\.|::)([a-zA-Z_]\w*[!?]?|[*%&^`~+\-/\[<>=])',
+ bygroups(Operator, Name)),
+ # Names can end with [!?] unless it's "!="
+ (r'[a-zA-Z_]\w*(?:[!?](?!=))?', Name),
+ (r'(\[|\]\??|\*\*|<=>?|>=|<<?|>>?|=~|===|'
+ r'!~|&&?|\|\||\.{1,3})', Operator),
+ (r'[-+/*%=<>&!^|~]=?', Operator),
+ (r'[(){};,/?:\\]', Punctuation),
+ (r'\s+', Text)
+ ],
+ 'funcname': [
+ (r'(?:([a-zA-Z_]\w*)(\.))?'
+ r'([a-zA-Z_]\w*[!?]?|\*\*?|[-+]@?|'
+ r'[/%&|^`~]|\[\]=?|<<|>>|<=?>|>=?|===?)',
+ bygroups(Name.Class, Operator, Name.Function), '#pop'),
+ default('#pop')
+ ],
+ 'classname': [
+ (r'[A-Z_]\w*', Name.Class),
+ (r'(\()(\s*)([A-Z_]\w*)(\s*)(\))',
+ bygroups(Punctuation, Text, Name.Class, Text, Punctuation)),
+ default('#pop')
+ ],
+ 'in-intp': [
+ (r'\{', String.Interpol, '#push'),
+ (r'\}', String.Interpol, '#pop'),
+ include('root'),
+ ],
+ 'string-intp': [
+ (r'#\{', String.Interpol, 'in-intp'),
+ ],
+ 'string-escaped': [
+ (r'\\([\\befnstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})', String.Escape)
+ ],
+ 'string-intp-escaped': [
+ include('string-intp'),
+ include('string-escaped'),
+ ],
+ 'interpolated-regex': [
+ include('string-intp'),
+ (r'[\\#]', String.Regex),
+ (r'[^\\#]+', String.Regex),
+ ],
+ 'interpolated-string': [
+ include('string-intp'),
+ (r'[\\#]', String.Other),
+ (r'[^\\#]+', String.Other),
+ ],
+ 'multiline-regex': [
+ include('string-intp'),
+ (r'\\\\', String.Regex),
+ (r'\\/', String.Regex),
+ (r'[\\#]', String.Regex),
+ (r'[^\\/#]+', String.Regex),
+ (r'/[imsx]*', String.Regex, '#pop'),
+ ],
+ 'end-part': [
+ (r'.+', Comment.Preproc, '#pop')
+ ],
+ 'in-macro-control': [
+ (r'\{%', String.Interpol, '#push'),
+ (r'%\}', String.Interpol, '#pop'),
+ (r'for\b|in\b', Keyword),
+ include('root'),
+ ],
+ 'in-macro-expr': [
+ (r'\{\{', String.Interpol, '#push'),
+ (r'\}\}', String.Interpol, '#pop'),
+ include('root'),
+ ],
+ 'in-attr': [
+ (r'\[', Operator, '#push'),
+ (r'\]', Operator, '#pop'),
+ include('root'),
+ ],
+ }
+ tokens.update(gen_crystalstrings_rules())
diff --git a/wandb/vendor/pygments/lexers/csound.py b/wandb/vendor/pygments/lexers/csound.py
new file mode 100644
index 0000000000000000000000000000000000000000..858aa3482e34491b7153ab5718f606cec4b1d2eb
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/csound.py
@@ -0,0 +1,366 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.csound
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for CSound languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, default, include, using, words
+from pygments.token import Comment, Keyword, Name, Number, Operator, Punctuation, \
+ String, Text
+from pygments.lexers._csound_builtins import OPCODES
+from pygments.lexers.html import HtmlLexer
+from pygments.lexers.python import PythonLexer
+from pygments.lexers.scripting import LuaLexer
+
+__all__ = ['CsoundScoreLexer', 'CsoundOrchestraLexer', 'CsoundDocumentLexer']
+
+newline = (r'((?:(?:;|//).*)*)(\n)', bygroups(Comment.Single, Text))
+
+
+class CsoundLexer(RegexLexer):
+ # Subclasses must define a 'single-line string' state.
+ tokens = {
+ 'whitespace': [
+ (r'[ \t]+', Text),
+ (r'\\\n', Text),
+ (r'/[*](.|\n)*?[*]/', Comment.Multiline)
+ ],
+
+ 'macro call': [
+ (r'(\$\w+\.?)(\()', bygroups(Comment.Preproc, Punctuation),
+ 'function macro call'),
+ (r'\$\w+(\.|\b)', Comment.Preproc)
+ ],
+ 'function macro call': [
+ (r"((?:\\['\)]|[^'\)])+)(')", bygroups(Comment.Preproc, Punctuation)),
+ (r"([^'\)]+)(\))", bygroups(Comment.Preproc, Punctuation), '#pop')
+ ],
+
+ 'whitespace or macro call': [
+ include('whitespace'),
+ include('macro call')
+ ],
+
+ 'preprocessor directives': [
+ (r'#(e(nd(if)?|lse)|ifn?def|undef)\b|##', Comment.Preproc),
+ (r'#include\b', Comment.Preproc, 'include'),
+ (r'#[ \t]*define\b', Comment.Preproc, 'macro name'),
+ (r'@+[ \t]*\d*', Comment.Preproc)
+ ],
+
+ 'include': [
+ include('whitespace'),
+ (r'"', String, 'single-line string')
+ ],
+
+ 'macro name': [
+ include('whitespace'),
+ (r'(\w+)(\()', bygroups(Comment.Preproc, Text),
+ 'function macro argument list'),
+ (r'\w+', Comment.Preproc, 'object macro definition after name')
+ ],
+ 'object macro definition after name': [
+ include('whitespace'),
+ (r'#', Punctuation, 'object macro replacement text')
+ ],
+ 'object macro replacement text': [
+ (r'(\\#|[^#])+', Comment.Preproc),
+ (r'#', Punctuation, '#pop:3')
+ ],
+ 'function macro argument list': [
+ (r"(\w+)(['#])", bygroups(Comment.Preproc, Punctuation)),
+ (r'(\w+)(\))', bygroups(Comment.Preproc, Punctuation),
+ 'function macro definition after name')
+ ],
+ 'function macro definition after name': [
+ (r'[ \t]+', Text),
+ (r'#', Punctuation, 'function macro replacement text')
+ ],
+ 'function macro replacement text': [
+ (r'(\\#|[^#])+', Comment.Preproc),
+ (r'#', Punctuation, '#pop:4')
+ ]
+ }
+
+
+class CsoundScoreLexer(CsoundLexer):
+ """
+ For `Csound <http://csound.github.io>`_ scores.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'Csound Score'
+ aliases = ['csound-score', 'csound-sco']
+ filenames = ['*.sco']
+
+ tokens = {
+ 'partial statement': [
+ include('preprocessor directives'),
+ (r'\d+e[+-]?\d+|(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?', Number.Float),
+ (r'0[xX][a-fA-F0-9]+', Number.Hex),
+ (r'\d+', Number.Integer),
+ (r'"', String, 'single-line string'),
+ (r'[+\-*/%^!=<>|&#~.]', Operator),
+ (r'[]()[]', Punctuation),
+ (r'\w+', Comment.Preproc)
+ ],
+
+ 'statement': [
+ include('whitespace or macro call'),
+ newline + ('#pop',),
+ include('partial statement')
+ ],
+
+ 'root': [
+ newline,
+ include('whitespace or macro call'),
+ (r'[{}]', Punctuation, 'statement'),
+ (r'[abefimq-tv-z]|[nN][pP]?', Keyword, 'statement')
+ ],
+
+ 'single-line string': [
+ (r'"', String, '#pop'),
+ (r'[^\\"]+', String)
+ ]
+ }
+
+
+class CsoundOrchestraLexer(CsoundLexer):
+ """
+ For `Csound <http://csound.github.io>`_ orchestras.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'Csound Orchestra'
+ aliases = ['csound', 'csound-orc']
+ filenames = ['*.orc']
+
+ user_defined_opcodes = set()
+
+ def opcode_name_callback(lexer, match):
+ opcode = match.group(0)
+ lexer.user_defined_opcodes.add(opcode)
+ yield match.start(), Name.Function, opcode
+
+ def name_callback(lexer, match):
+ name = match.group(0)
+ if re.match('p\d+$', name) or name in OPCODES:
+ yield match.start(), Name.Builtin, name
+ elif name in lexer.user_defined_opcodes:
+ yield match.start(), Name.Function, name
+ else:
+ nameMatch = re.search(r'^(g?[aikSw])(\w+)', name)
+ if nameMatch:
+ yield nameMatch.start(1), Keyword.Type, nameMatch.group(1)
+ yield nameMatch.start(2), Name, nameMatch.group(2)
+ else:
+ yield match.start(), Name, name
+
+ tokens = {
+ 'label': [
+ (r'\b(\w+)(:)', bygroups(Name.Label, Punctuation))
+ ],
+
+ 'partial expression': [
+ include('preprocessor directives'),
+ (r'\b(0dbfs|k(r|smps)|nchnls(_i)?|sr)\b', Name.Variable.Global),
+ (r'\d+e[+-]?\d+|(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?', Number.Float),
+ (r'0[xX][a-fA-F0-9]+', Number.Hex),
+ (r'\d+', Number.Integer),
+ (r'"', String, 'single-line string'),
+ (r'\{\{', String, 'multi-line string'),
+ (r'[+\-*/%^!=&|<>#~¬]', Operator),
+ (r'[](),?:[]', Punctuation),
+ (words((
+ # Keywords
+ 'do', 'else', 'elseif', 'endif', 'enduntil', 'fi', 'if', 'ithen', 'kthen',
+ 'od', 'then', 'until', 'while',
+ # Opcodes that act as control structures
+ 'return', 'timout'
+ ), prefix=r'\b', suffix=r'\b'), Keyword),
+ (words(('goto', 'igoto', 'kgoto', 'rigoto', 'tigoto'),
+ prefix=r'\b', suffix=r'\b'), Keyword, 'goto label'),
+ (words(('cggoto', 'cigoto', 'cingoto', 'ckgoto', 'cngoto'),
+ prefix=r'\b', suffix=r'\b'), Keyword,
+ ('goto label', 'goto expression')),
+ (words(('loop_ge', 'loop_gt', 'loop_le', 'loop_lt'),
+ prefix=r'\b', suffix=r'\b'), Keyword,
+ ('goto label', 'goto expression', 'goto expression', 'goto expression')),
+ (r'\bscoreline(_i)?\b', Name.Builtin, 'scoreline opcode'),
+ (r'\bpyl?run[it]?\b', Name.Builtin, 'python opcode'),
+ (r'\blua_(exec|opdef)\b', Name.Builtin, 'lua opcode'),
+ (r'\b[a-zA-Z_]\w*\b', name_callback)
+ ],
+
+ 'expression': [
+ include('whitespace or macro call'),
+ newline + ('#pop',),
+ include('partial expression')
+ ],
+
+ 'root': [
+ newline,
+ include('whitespace or macro call'),
+ (r'\binstr\b', Keyword, ('instrument block', 'instrument name list')),
+ (r'\bopcode\b', Keyword, ('opcode block', 'opcode parameter list',
+ 'opcode types', 'opcode types', 'opcode name')),
+ include('label'),
+ default('expression')
+ ],
+
+ 'instrument name list': [
+ include('whitespace or macro call'),
+ (r'\d+|\+?[a-zA-Z_]\w*', Name.Function),
+ (r',', Punctuation),
+ newline + ('#pop',)
+ ],
+ 'instrument block': [
+ newline,
+ include('whitespace or macro call'),
+ (r'\bendin\b', Keyword, '#pop'),
+ include('label'),
+ default('expression')
+ ],
+
+ 'opcode name': [
+ include('whitespace or macro call'),
+ (r'[a-zA-Z_]\w*', opcode_name_callback, '#pop')
+ ],
+ 'opcode types': [
+ include('whitespace or macro call'),
+ (r'0|[]afijkKoOpPStV[]+', Keyword.Type, '#pop'),
+ (r',', Punctuation)
+ ],
+ 'opcode parameter list': [
+ include('whitespace or macro call'),
+ newline + ('#pop',)
+ ],
+ 'opcode block': [
+ newline,
+ include('whitespace or macro call'),
+ (r'\bendop\b', Keyword, '#pop'),
+ include('label'),
+ default('expression')
+ ],
+
+ 'goto label': [
+ include('whitespace or macro call'),
+ (r'\w+', Name.Label, '#pop'),
+ default('#pop')
+ ],
+ 'goto expression': [
+ include('whitespace or macro call'),
+ (r',', Punctuation, '#pop'),
+ include('partial expression')
+ ],
+
+ 'single-line string': [
+ include('macro call'),
+ (r'"', String, '#pop'),
+ # From https://github.com/csound/csound/blob/develop/Opcodes/fout.c#L1405
+ (r'%\d*(\.\d+)?[cdhilouxX]', String.Interpol),
+ (r'%[!%nNrRtT]|[~^]|\\([\\aAbBnNrRtT"]|[0-7]{1,3})', String.Escape),
+ (r'[^\\"~$%\^\n]+', String),
+ (r'[\\"~$%\^\n]', String)
+ ],
+ 'multi-line string': [
+ (r'\}\}', String, '#pop'),
+ (r'[^}]+|\}(?!\})', String)
+ ],
+
+ 'scoreline opcode': [
+ include('whitespace or macro call'),
+ (r'\{\{', String, 'scoreline'),
+ default('#pop')
+ ],
+ 'scoreline': [
+ (r'\}\}', String, '#pop'),
+ (r'([^}]+)|\}(?!\})', using(CsoundScoreLexer))
+ ],
+
+ 'python opcode': [
+ include('whitespace or macro call'),
+ (r'\{\{', String, 'python'),
+ default('#pop')
+ ],
+ 'python': [
+ (r'\}\}', String, '#pop'),
+ (r'([^}]+)|\}(?!\})', using(PythonLexer))
+ ],
+
+ 'lua opcode': [
+ include('whitespace or macro call'),
+ (r'"', String, 'single-line string'),
+ (r'\{\{', String, 'lua'),
+ (r',', Punctuation),
+ default('#pop')
+ ],
+ 'lua': [
+ (r'\}\}', String, '#pop'),
+ (r'([^}]+)|\}(?!\})', using(LuaLexer))
+ ]
+ }
+
+
+class CsoundDocumentLexer(RegexLexer):
+ """
+ For `Csound <http://csound.github.io>`_ documents.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'Csound Document'
+ aliases = ['csound-document', 'csound-csd']
+ filenames = ['*.csd']
+
+ # These tokens are based on those in XmlLexer in pygments/lexers/html.py. Making
+ # CsoundDocumentLexer a subclass of XmlLexer rather than RegexLexer may seem like a
+ # better idea, since Csound Document files look like XML files. However, Csound
+ # Documents can contain Csound comments (preceded by //, for example) before and
+ # after the root element, unescaped bitwise AND & and less than < operators, etc. In
+ # other words, while Csound Document files look like XML files, they may not actually
+ # be XML files.
+ tokens = {
+ 'root': [
+ newline,
+ (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+ (r'[^<&;/]+', Text),
+ (r'<\s*CsInstruments', Name.Tag, ('orchestra', 'tag')),
+ (r'<\s*CsScore', Name.Tag, ('score', 'tag')),
+ (r'<\s*[hH][tT][mM][lL]', Name.Tag, ('HTML', 'tag')),
+ (r'<\s*[\w:.-]+', Name.Tag, 'tag'),
+ (r'<\s*/\s*[\w:.-]+\s*>', Name.Tag)
+ ],
+ 'orchestra': [
+ (r'<\s*/\s*CsInstruments\s*>', Name.Tag, '#pop'),
+ (r'(.|\n)+?(?=<\s*/\s*CsInstruments\s*>)', using(CsoundOrchestraLexer))
+ ],
+ 'score': [
+ (r'<\s*/\s*CsScore\s*>', Name.Tag, '#pop'),
+ (r'(.|\n)+?(?=<\s*/\s*CsScore\s*>)', using(CsoundScoreLexer))
+ ],
+ 'HTML': [
+ (r'<\s*/\s*[hH][tT][mM][lL]\s*>', Name.Tag, '#pop'),
+ (r'(.|\n)+?(?=<\s*/\s*[hH][tT][mM][lL]\s*>)', using(HtmlLexer))
+ ],
+ 'tag': [
+ (r'\s+', Text),
+ (r'[\w.:-]+\s*=', Name.Attribute, 'attr'),
+ (r'/?\s*>', Name.Tag, '#pop')
+ ],
+ 'attr': [
+ (r'\s+', Text),
+ (r'".*?"', String, '#pop'),
+ (r"'.*?'", String, '#pop'),
+ (r'[^\s>]+', String, '#pop')
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/css.py b/wandb/vendor/pygments/lexers/css.py
new file mode 100644
index 0000000000000000000000000000000000000000..29d83707e15bf89d5f864693f84960a6c70aec27
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/css.py
@@ -0,0 +1,689 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.css
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexers for CSS and related stylesheet formats.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+import copy
+
+from pygments.lexer import ExtendedRegexLexer, RegexLexer, include, bygroups, \
+ default, words, inherit
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+from pygments.util import iteritems
+
+__all__ = ['CssLexer', 'SassLexer', 'ScssLexer', 'LessCssLexer']
+
+
+# List of vendor prefixes obtained from:
+# https://www.w3.org/TR/CSS21/syndata.html#vendor-keyword-history
+_vendor_prefixes = (
+ '-ms-', 'mso-', '-moz-', '-o-', '-xv-', '-atsc-', '-wap-', '-khtml-',
+ '-webkit-', 'prince-', '-ah-', '-hp-', '-ro-', '-rim-', '-tc-',
+)
+
+# List of CSS properties obtained from:
+# https://www.w3.org/Style/CSS/all-properties.en.html
+# Note: handle --* separately
+_css_properties = (
+ 'align-content', 'align-items', 'align-self', 'alignment-baseline', 'all',
+ 'animation', 'animation-delay', 'animation-direction',
+ 'animation-duration', 'animation-fill-mode', 'animation-iteration-count',
+ 'animation-name', 'animation-play-state', 'animation-timing-function',
+ 'appearance', 'azimuth', 'backface-visibility', 'background',
+ 'background-attachment', 'background-blend-mode', 'background-clip',
+ 'background-color', 'background-image', 'background-origin',
+ 'background-position', 'background-repeat', 'background-size',
+ 'baseline-shift', 'bookmark-label', 'bookmark-level', 'bookmark-state',
+ 'border', 'border-bottom', 'border-bottom-color',
+ 'border-bottom-left-radius', 'border-bottom-right-radius',
+ 'border-bottom-style', 'border-bottom-width', 'border-boundary',
+ 'border-collapse', 'border-color', 'border-image', 'border-image-outset',
+ 'border-image-repeat', 'border-image-slice', 'border-image-source',
+ 'border-image-width', 'border-left', 'border-left-color',
+ 'border-left-style', 'border-left-width', 'border-radius', 'border-right',
+ 'border-right-color', 'border-right-style', 'border-right-width',
+ 'border-spacing', 'border-style', 'border-top', 'border-top-color',
+ 'border-top-left-radius', 'border-top-right-radius', 'border-top-style',
+ 'border-top-width', 'border-width', 'bottom', 'box-decoration-break',
+ 'box-shadow', 'box-sizing', 'box-snap', 'box-suppress', 'break-after',
+ 'break-before', 'break-inside', 'caption-side', 'caret', 'caret-animation',
+ 'caret-color', 'caret-shape', 'chains', 'clear', 'clip', 'clip-path',
+ 'clip-rule', 'color', 'color-interpolation-filters', 'column-count',
+ 'column-fill', 'column-gap', 'column-rule', 'column-rule-color',
+ 'column-rule-style', 'column-rule-width', 'column-span', 'column-width',
+ 'columns', 'content', 'counter-increment', 'counter-reset', 'counter-set',
+ 'crop', 'cue', 'cue-after', 'cue-before', 'cursor', 'direction', 'display',
+ 'dominant-baseline', 'elevation', 'empty-cells', 'filter', 'flex',
+ 'flex-basis', 'flex-direction', 'flex-flow', 'flex-grow', 'flex-shrink',
+ 'flex-wrap', 'float', 'float-defer', 'float-offset', 'float-reference',
+ 'flood-color', 'flood-opacity', 'flow', 'flow-from', 'flow-into', 'font',
+ 'font-family', 'font-feature-settings', 'font-kerning',
+ 'font-language-override', 'font-size', 'font-size-adjust', 'font-stretch',
+ 'font-style', 'font-synthesis', 'font-variant', 'font-variant-alternates',
+ 'font-variant-caps', 'font-variant-east-asian', 'font-variant-ligatures',
+ 'font-variant-numeric', 'font-variant-position', 'font-weight',
+ 'footnote-display', 'footnote-policy', 'glyph-orientation-vertical',
+ 'grid', 'grid-area', 'grid-auto-columns', 'grid-auto-flow',
+ 'grid-auto-rows', 'grid-column', 'grid-column-end', 'grid-column-gap',
+ 'grid-column-start', 'grid-gap', 'grid-row', 'grid-row-end',
+ 'grid-row-gap', 'grid-row-start', 'grid-template', 'grid-template-areas',
+ 'grid-template-columns', 'grid-template-rows', 'hanging-punctuation',
+ 'height', 'hyphenate-character', 'hyphenate-limit-chars',
+ 'hyphenate-limit-last', 'hyphenate-limit-lines', 'hyphenate-limit-zone',
+ 'hyphens', 'image-orientation', 'image-resolution', 'initial-letter',
+ 'initial-letter-align', 'initial-letter-wrap', 'isolation',
+ 'justify-content', 'justify-items', 'justify-self', 'left',
+ 'letter-spacing', 'lighting-color', 'line-break', 'line-grid',
+ 'line-height', 'line-snap', 'list-style', 'list-style-image',
+ 'list-style-position', 'list-style-type', 'margin', 'margin-bottom',
+ 'margin-left', 'margin-right', 'margin-top', 'marker-side',
+ 'marquee-direction', 'marquee-loop', 'marquee-speed', 'marquee-style',
+ 'mask', 'mask-border', 'mask-border-mode', 'mask-border-outset',
+ 'mask-border-repeat', 'mask-border-slice', 'mask-border-source',
+ 'mask-border-width', 'mask-clip', 'mask-composite', 'mask-image',
+ 'mask-mode', 'mask-origin', 'mask-position', 'mask-repeat', 'mask-size',
+ 'mask-type', 'max-height', 'max-lines', 'max-width', 'min-height',
+ 'min-width', 'mix-blend-mode', 'motion', 'motion-offset', 'motion-path',
+ 'motion-rotation', 'move-to', 'nav-down', 'nav-left', 'nav-right',
+ 'nav-up', 'object-fit', 'object-position', 'offset-after', 'offset-before',
+ 'offset-end', 'offset-start', 'opacity', 'order', 'orphans', 'outline',
+ 'outline-color', 'outline-offset', 'outline-style', 'outline-width',
+ 'overflow', 'overflow-style', 'overflow-wrap', 'overflow-x', 'overflow-y',
+ 'padding', 'padding-bottom', 'padding-left', 'padding-right', 'padding-top',
+ 'page', 'page-break-after', 'page-break-before', 'page-break-inside',
+ 'page-policy', 'pause', 'pause-after', 'pause-before', 'perspective',
+ 'perspective-origin', 'pitch', 'pitch-range', 'play-during', 'polar-angle',
+ 'polar-distance', 'position', 'presentation-level', 'quotes',
+ 'region-fragment', 'resize', 'rest', 'rest-after', 'rest-before',
+ 'richness', 'right', 'rotation', 'rotation-point', 'ruby-align',
+ 'ruby-merge', 'ruby-position', 'running', 'scroll-snap-coordinate',
+ 'scroll-snap-destination', 'scroll-snap-points-x', 'scroll-snap-points-y',
+ 'scroll-snap-type', 'shape-image-threshold', 'shape-inside', 'shape-margin',
+ 'shape-outside', 'size', 'speak', 'speak-as', 'speak-header',
+ 'speak-numeral', 'speak-punctuation', 'speech-rate', 'stress', 'string-set',
+ 'tab-size', 'table-layout', 'text-align', 'text-align-last',
+ 'text-combine-upright', 'text-decoration', 'text-decoration-color',
+ 'text-decoration-line', 'text-decoration-skip', 'text-decoration-style',
+ 'text-emphasis', 'text-emphasis-color', 'text-emphasis-position',
+ 'text-emphasis-style', 'text-indent', 'text-justify', 'text-orientation',
+ 'text-overflow', 'text-shadow', 'text-space-collapse', 'text-space-trim',
+ 'text-spacing', 'text-transform', 'text-underline-position', 'text-wrap',
+ 'top', 'transform', 'transform-origin', 'transform-style', 'transition',
+ 'transition-delay', 'transition-duration', 'transition-property',
+ 'transition-timing-function', 'unicode-bidi', 'user-select',
+ 'vertical-align', 'visibility', 'voice-balance', 'voice-duration',
+ 'voice-family', 'voice-pitch', 'voice-range', 'voice-rate', 'voice-stress',
+ 'voice-volume', 'volume', 'white-space', 'widows', 'width', 'will-change',
+ 'word-break', 'word-spacing', 'word-wrap', 'wrap-after', 'wrap-before',
+ 'wrap-flow', 'wrap-inside', 'wrap-through', 'writing-mode', 'z-index',
+)
+
+# List of keyword values obtained from:
+# http://cssvalues.com/
+_keyword_values = (
+ 'absolute', 'alias', 'all', 'all-petite-caps', 'all-scroll',
+ 'all-small-caps', 'allow-end', 'alpha', 'alternate', 'alternate-reverse',
+ 'always', 'armenian', 'auto', 'avoid', 'avoid-column', 'avoid-page',
+ 'backwards', 'balance', 'baseline', 'below', 'blink', 'block', 'bold',
+ 'bolder', 'border-box', 'both', 'bottom', 'box-decoration', 'break-word',
+ 'capitalize', 'cell', 'center', 'circle', 'clip', 'clone', 'close-quote',
+ 'col-resize', 'collapse', 'color', 'color-burn', 'color-dodge', 'column',
+ 'column-reverse', 'compact', 'condensed', 'contain', 'container',
+ 'content-box', 'context-menu', 'copy', 'cover', 'crisp-edges', 'crosshair',
+ 'currentColor', 'cursive', 'darken', 'dashed', 'decimal',
+ 'decimal-leading-zero', 'default', 'descendants', 'difference', 'digits',
+ 'disc', 'distribute', 'dot', 'dotted', 'double', 'double-circle', 'e-resize',
+ 'each-line', 'ease', 'ease-in', 'ease-in-out', 'ease-out', 'edges',
+ 'ellipsis', 'end', 'ew-resize', 'exclusion', 'expanded', 'extra-condensed',
+ 'extra-expanded', 'fantasy', 'fill', 'fill-box', 'filled', 'first', 'fixed',
+ 'flat', 'flex', 'flex-end', 'flex-start', 'flip', 'force-end', 'forwards',
+ 'from-image', 'full-width', 'geometricPrecision', 'georgian', 'groove',
+ 'hanging', 'hard-light', 'help', 'hidden', 'hide', 'horizontal', 'hue',
+ 'icon', 'infinite', 'inherit', 'initial', 'ink', 'inline', 'inline-block',
+ 'inline-flex', 'inline-table', 'inset', 'inside', 'inter-word', 'invert',
+ 'isolate', 'italic', 'justify', 'large', 'larger', 'last', 'left',
+ 'lighten', 'lighter', 'line-through', 'linear', 'list-item', 'local',
+ 'loose', 'lower-alpha', 'lower-greek', 'lower-latin', 'lower-roman',
+ 'lowercase', 'ltr', 'luminance', 'luminosity', 'mandatory', 'manipulation',
+ 'manual', 'margin-box', 'match-parent', 'medium', 'mixed', 'monospace',
+ 'move', 'multiply', 'n-resize', 'ne-resize', 'nesw-resize',
+ 'no-close-quote', 'no-drop', 'no-open-quote', 'no-repeat', 'none', 'normal',
+ 'not-allowed', 'nowrap', 'ns-resize', 'nw-resize', 'nwse-resize', 'objects',
+ 'oblique', 'off', 'on', 'open', 'open-quote', 'optimizeLegibility',
+ 'optimizeSpeed', 'outset', 'outside', 'over', 'overlay', 'overline',
+ 'padding-box', 'page', 'pan-down', 'pan-left', 'pan-right', 'pan-up',
+ 'pan-x', 'pan-y', 'paused', 'petite-caps', 'pixelated', 'pointer',
+ 'preserve-3d', 'progress', 'proximity', 'relative', 'repeat',
+ 'repeat no-repeat', 'repeat-x', 'repeat-y', 'reverse', 'ridge', 'right',
+ 'round', 'row', 'row-resize', 'row-reverse', 'rtl', 'ruby', 'ruby-base',
+ 'ruby-base-container', 'ruby-text', 'ruby-text-container', 'run-in',
+ 'running', 's-resize', 'sans-serif', 'saturation', 'scale-down', 'screen',
+ 'scroll', 'se-resize', 'semi-condensed', 'semi-expanded', 'separate',
+ 'serif', 'sesame', 'show', 'sideways', 'sideways-left', 'sideways-right',
+ 'slice', 'small', 'small-caps', 'smaller', 'smooth', 'snap', 'soft-light',
+ 'solid', 'space', 'space-around', 'space-between', 'spaces', 'square',
+ 'start', 'static', 'step-end', 'step-start', 'sticky', 'stretch', 'strict',
+ 'stroke-box', 'style', 'sw-resize', 'table', 'table-caption', 'table-cell',
+ 'table-column', 'table-column-group', 'table-footer-group',
+ 'table-header-group', 'table-row', 'table-row-group', 'text', 'thick',
+ 'thin', 'titling-caps', 'to', 'top', 'triangle', 'ultra-condensed',
+ 'ultra-expanded', 'under', 'underline', 'unicase', 'unset', 'upper-alpha',
+ 'upper-latin', 'upper-roman', 'uppercase', 'upright', 'use-glyph-orientation',
+ 'vertical', 'vertical-text', 'view-box', 'visible', 'w-resize', 'wait',
+ 'wavy', 'weight', 'weight style', 'wrap', 'wrap-reverse', 'x-large',
+ 'x-small', 'xx-large', 'xx-small', 'zoom-in', 'zoom-out',
+)
+
+# List of extended color keywords obtained from:
+# https://drafts.csswg.org/css-color/#named-colors
+_color_keywords = (
+ 'aliceblue', 'antiquewhite', 'aqua', 'aquamarine', 'azure', 'beige',
+ 'bisque', 'black', 'blanchedalmond', 'blue', 'blueviolet', 'brown',
+ 'burlywood', 'cadetblue', 'chartreuse', 'chocolate', 'coral',
+ 'cornflowerblue', 'cornsilk', 'crimson', 'cyan', 'darkblue', 'darkcyan',
+ 'darkgoldenrod', 'darkgray', 'darkgreen', 'darkgrey', 'darkkhaki',
+ 'darkmagenta', 'darkolivegreen', 'darkorange', 'darkorchid', 'darkred',
+ 'darksalmon', 'darkseagreen', 'darkslateblue', 'darkslategray',
+ 'darkslategrey', 'darkturquoise', 'darkviolet', 'deeppink', 'deepskyblue',
+ 'dimgray', 'dimgrey', 'dodgerblue', 'firebrick', 'floralwhite',
+ 'forestgreen', 'fuchsia', 'gainsboro', 'ghostwhite', 'gold', 'goldenrod',
+ 'gray', 'green', 'greenyellow', 'grey', 'honeydew', 'hotpink', 'indianred',
+ 'indigo', 'ivory', 'khaki', 'lavender', 'lavenderblush', 'lawngreen',
+ 'lemonchiffon', 'lightblue', 'lightcoral', 'lightcyan',
+ 'lightgoldenrodyellow', 'lightgray', 'lightgreen', 'lightgrey',
+ 'lightpink', 'lightsalmon', 'lightseagreen', 'lightskyblue',
+ 'lightslategray', 'lightslategrey', 'lightsteelblue', 'lightyellow',
+ 'lime', 'limegreen', 'linen', 'magenta', 'maroon', 'mediumaquamarine',
+ 'mediumblue', 'mediumorchid', 'mediumpurple', 'mediumseagreen',
+ 'mediumslateblue', 'mediumspringgreen', 'mediumturquoise',
+ 'mediumvioletred', 'midnightblue', 'mintcream', 'mistyrose', 'moccasin',
+ 'navajowhite', 'navy', 'oldlace', 'olive', 'olivedrab', 'orange',
+ 'orangered', 'orchid', 'palegoldenrod', 'palegreen', 'paleturquoise',
+ 'palevioletred', 'papayawhip', 'peachpuff', 'peru', 'pink', 'plum',
+ 'powderblue', 'purple', 'rebeccapurple', 'red', 'rosybrown', 'royalblue',
+ 'saddlebrown', 'salmon', 'sandybrown', 'seagreen', 'seashell', 'sienna',
+ 'silver', 'skyblue', 'slateblue', 'slategray', 'slategrey', 'snow',
+ 'springgreen', 'steelblue', 'tan', 'teal', 'thistle', 'tomato', 'turquoise',
+ 'violet', 'wheat', 'white', 'whitesmoke', 'yellow', 'yellowgreen',
+) + ('transparent',)
+
+# List of other keyword values from other sources:
+_other_keyword_values = (
+ 'above', 'aural', 'behind', 'bidi-override', 'center-left', 'center-right',
+ 'cjk-ideographic', 'continuous', 'crop', 'cross', 'embed', 'far-left',
+ 'far-right', 'fast', 'faster', 'hebrew', 'high', 'higher', 'hiragana',
+ 'hiragana-iroha', 'katakana', 'katakana-iroha', 'landscape', 'left-side',
+ 'leftwards', 'level', 'loud', 'low', 'lower', 'message-box', 'middle',
+ 'mix', 'narrower', 'once', 'portrait', 'right-side', 'rightwards', 'silent',
+ 'slow', 'slower', 'small-caption', 'soft', 'spell-out', 'status-bar',
+ 'super', 'text-bottom', 'text-top', 'wider', 'x-fast', 'x-high', 'x-loud',
+ 'x-low', 'x-soft', 'yes', 'pre', 'pre-wrap', 'pre-line',
+)
+
+# List of functional notation and function keyword values:
+_functional_notation_keyword_values = (
+ 'attr', 'blackness', 'blend', 'blenda', 'blur', 'brightness', 'calc',
+ 'circle', 'color-mod', 'contrast', 'counter', 'cubic-bezier', 'device-cmyk',
+ 'drop-shadow', 'ellipse', 'gray', 'grayscale', 'hsl', 'hsla', 'hue',
+ 'hue-rotate', 'hwb', 'image', 'inset', 'invert', 'lightness',
+ 'linear-gradient', 'matrix', 'matrix3d', 'opacity', 'perspective',
+ 'polygon', 'radial-gradient', 'rect', 'repeating-linear-gradient',
+ 'repeating-radial-gradient', 'rgb', 'rgba', 'rotate', 'rotate3d', 'rotateX',
+ 'rotateY', 'rotateZ', 'saturate', 'saturation', 'scale', 'scale3d',
+ 'scaleX', 'scaleY', 'scaleZ', 'sepia', 'shade', 'skewX', 'skewY', 'steps',
+ 'tint', 'toggle', 'translate', 'translate3d', 'translateX', 'translateY',
+ 'translateZ', 'whiteness',
+)
+# Note! Handle url(...) separately.
+
+# List of units obtained from:
+# https://www.w3.org/TR/css3-values/
+_angle_units = (
+ 'deg', 'grad', 'rad', 'turn',
+)
+_frequency_units = (
+ 'Hz', 'kHz',
+)
+_length_units = (
+ 'em', 'ex', 'ch', 'rem',
+ 'vh', 'vw', 'vmin', 'vmax',
+ 'px', 'mm', 'cm', 'in', 'pt', 'pc', 'q',
+)
+_resolution_units = (
+ 'dpi', 'dpcm', 'dppx',
+)
+_time_units = (
+ 's', 'ms',
+)
+_all_units = _angle_units + _frequency_units + _length_units + \
+ _resolution_units + _time_units
+
+
+class CssLexer(RegexLexer):
+ """
+ For CSS (Cascading Style Sheets).
+ """
+
+ name = 'CSS'
+ aliases = ['css']
+ filenames = ['*.css']
+ mimetypes = ['text/css']
+
+ tokens = {
+ 'root': [
+ include('basics'),
+ ],
+ 'basics': [
+ (r'\s+', Text),
+ (r'/\*(?:.|\n)*?\*/', Comment),
+ (r'\{', Punctuation, 'content'),
+ (r'(\:{1,2})([\w-]+)', bygroups(Punctuation, Name.Decorator)),
+ (r'(\.)([\w-]+)', bygroups(Punctuation, Name.Class)),
+ (r'(\#)([\w-]+)', bygroups(Punctuation, Name.Namespace)),
+ (r'(@)([\w-]+)', bygroups(Punctuation, Keyword), 'atrule'),
+ (r'[\w-]+', Name.Tag),
+ (r'[~^*!%&$\[\]()<>|+=@:;,./?-]', Operator),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single)
+ ],
+ 'atrule': [
+ (r'\{', Punctuation, 'atcontent'),
+ (r';', Punctuation, '#pop'),
+ include('basics'),
+ ],
+ 'atcontent': [
+ include('basics'),
+ (r'\}', Punctuation, '#pop:2'),
+ ],
+ 'content': [
+ (r'\s+', Text),
+ (r'\}', Punctuation, '#pop'),
+ (r';', Punctuation),
+ (r'^@.*?$', Comment.Preproc),
+
+ (words(_vendor_prefixes,), Keyword.Pseudo),
+ (r'('+r'|'.join(_css_properties)+r')(\s*)(\:)',
+ bygroups(Keyword, Text, Punctuation), 'value-start'),
+ (r'([a-zA-Z_][\w-]*)(\s*)(\:)', bygroups(Name, Text, Punctuation),
+ 'value-start'),
+
+ (r'/\*(?:.|\n)*?\*/', Comment),
+ ],
+ 'value-start': [
+ (r'\s+', Text),
+ (words(_vendor_prefixes,), Name.Builtin.Pseudo),
+ include('urls'),
+ (r'('+r'|'.join(_functional_notation_keyword_values)+r')(\()',
+ bygroups(Name.Builtin, Punctuation), 'function-start'),
+ (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'),
+ (words(_keyword_values, suffix=r'\b'), Keyword.Constant),
+ (words(_other_keyword_values, suffix=r'\b'), Keyword.Constant),
+ (words(_color_keywords, suffix=r'\b'), Keyword.Constant),
+ (words(_css_properties, suffix=r'\b'), Keyword), # for transition-property etc.
+ (r'\!important', Comment.Preproc),
+ (r'/\*(?:.|\n)*?\*/', Comment),
+
+ include('numeric-values'),
+
+ (r'[~^*!%&<>|+=@:./?-]+', Operator),
+ (r'[\[\](),]+', Punctuation),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ (r'[a-zA-Z_][\w-]*', Name),
+ (r';', Punctuation, '#pop'),
+ (r'\}', Punctuation, '#pop:2'),
+ ],
+ 'function-start': [
+ (r'\s+', Text),
+ include('urls'),
+ (words(_vendor_prefixes,), Keyword.Pseudo),
+ (words(_keyword_values, suffix=r'\b'), Keyword.Constant),
+ (words(_other_keyword_values, suffix=r'\b'), Keyword.Constant),
+ (words(_color_keywords, suffix=r'\b'), Keyword.Constant),
+
+ # function-start may be entered recursively
+ (r'(' + r'|'.join(_functional_notation_keyword_values) + r')(\()',
+ bygroups(Name.Builtin, Punctuation), 'function-start'),
+ (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'),
+
+ (r'/\*(?:.|\n)*?\*/', Comment),
+ include('numeric-values'),
+ (r'[*+/-]', Operator),
+ (r'[,]', Punctuation),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ (r'[a-zA-Z_-]\w*', Name),
+ (r'\)', Punctuation, '#pop'),
+ ],
+ 'urls': [
+ (r'(url)(\()(".*?")(\))', bygroups(Name.Builtin, Punctuation,
+ String.Double, Punctuation)),
+ (r"(url)(\()('.*?')(\))", bygroups(Name.Builtin, Punctuation,
+ String.Single, Punctuation)),
+ (r'(url)(\()(.*?)(\))', bygroups(Name.Builtin, Punctuation,
+ String.Other, Punctuation)),
+ ],
+ 'numeric-values': [
+ (r'\#[a-zA-Z0-9]{1,6}', Number.Hex),
+ (r'[+\-]?[0-9]*[.][0-9]+', Number.Float, 'numeric-end'),
+ (r'[+\-]?[0-9]+', Number.Integer, 'numeric-end'),
+ ],
+ 'numeric-end': [
+ (words(_all_units, suffix=r'\b'), Keyword.Type),
+ (r'%', Keyword.Type),
+ default('#pop'),
+ ],
+ }
+
+
+common_sass_tokens = {
+ 'value': [
+ (r'[ \t]+', Text),
+ (r'[!$][\w-]+', Name.Variable),
+ (r'url\(', String.Other, 'string-url'),
+ (r'[a-z_-][\w-]*(?=\()', Name.Function),
+ (words(_css_properties + (
+ 'above', 'absolute', 'always', 'armenian', 'aural', 'auto', 'avoid', 'baseline',
+ 'behind', 'below', 'bidi-override', 'blink', 'block', 'bold', 'bolder', 'both',
+ 'capitalize', 'center-left', 'center-right', 'center', 'circle',
+ 'cjk-ideographic', 'close-quote', 'collapse', 'condensed', 'continuous',
+ 'crop', 'crosshair', 'cross', 'cursive', 'dashed', 'decimal-leading-zero',
+ 'decimal', 'default', 'digits', 'disc', 'dotted', 'double', 'e-resize', 'embed',
+ 'extra-condensed', 'extra-expanded', 'expanded', 'fantasy', 'far-left',
+ 'far-right', 'faster', 'fast', 'fixed', 'georgian', 'groove', 'hebrew', 'help',
+ 'hidden', 'hide', 'higher', 'high', 'hiragana-iroha', 'hiragana', 'icon',
+ 'inherit', 'inline-table', 'inline', 'inset', 'inside', 'invert', 'italic',
+ 'justify', 'katakana-iroha', 'katakana', 'landscape', 'larger', 'large',
+ 'left-side', 'leftwards', 'level', 'lighter', 'line-through', 'list-item',
+ 'loud', 'lower-alpha', 'lower-greek', 'lower-roman', 'lowercase', 'ltr',
+ 'lower', 'low', 'medium', 'message-box', 'middle', 'mix', 'monospace',
+ 'n-resize', 'narrower', 'ne-resize', 'no-close-quote', 'no-open-quote',
+ 'no-repeat', 'none', 'normal', 'nowrap', 'nw-resize', 'oblique', 'once',
+ 'open-quote', 'outset', 'outside', 'overline', 'pointer', 'portrait', 'px',
+ 'relative', 'repeat-x', 'repeat-y', 'repeat', 'rgb', 'ridge', 'right-side',
+ 'rightwards', 's-resize', 'sans-serif', 'scroll', 'se-resize',
+ 'semi-condensed', 'semi-expanded', 'separate', 'serif', 'show', 'silent',
+ 'slow', 'slower', 'small-caps', 'small-caption', 'smaller', 'soft', 'solid',
+ 'spell-out', 'square', 'static', 'status-bar', 'super', 'sw-resize',
+ 'table-caption', 'table-cell', 'table-column', 'table-column-group',
+ 'table-footer-group', 'table-header-group', 'table-row',
+ 'table-row-group', 'text', 'text-bottom', 'text-top', 'thick', 'thin',
+ 'transparent', 'ultra-condensed', 'ultra-expanded', 'underline',
+ 'upper-alpha', 'upper-latin', 'upper-roman', 'uppercase', 'url',
+ 'visible', 'w-resize', 'wait', 'wider', 'x-fast', 'x-high', 'x-large', 'x-loud',
+ 'x-low', 'x-small', 'x-soft', 'xx-large', 'xx-small', 'yes'), suffix=r'\b'),
+ Name.Constant),
+ (words(_color_keywords, suffix=r'\b'), Name.Entity),
+ (words((
+ 'black', 'silver', 'gray', 'white', 'maroon', 'red', 'purple', 'fuchsia', 'green',
+ 'lime', 'olive', 'yellow', 'navy', 'blue', 'teal', 'aqua'), suffix=r'\b'),
+ Name.Builtin),
+ (r'\!(important|default)', Name.Exception),
+ (r'(true|false)', Name.Pseudo),
+ (r'(and|or|not)', Operator.Word),
+ (r'/\*', Comment.Multiline, 'inline-comment'),
+ (r'//[^\n]*', Comment.Single),
+ (r'\#[a-z0-9]{1,6}', Number.Hex),
+ (r'(-?\d+)(\%|[a-z]+)?', bygroups(Number.Integer, Keyword.Type)),
+ (r'(-?\d*\.\d+)(\%|[a-z]+)?', bygroups(Number.Float, Keyword.Type)),
+ (r'#\{', String.Interpol, 'interpolation'),
+ (r'[~^*!&%<>|+=@:,./?-]+', Operator),
+ (r'[\[\]()]+', Punctuation),
+ (r'"', String.Double, 'string-double'),
+ (r"'", String.Single, 'string-single'),
+ (r'[a-z_-][\w-]*', Name),
+ ],
+
+ 'interpolation': [
+ (r'\}', String.Interpol, '#pop'),
+ include('value'),
+ ],
+
+ 'selector': [
+ (r'[ \t]+', Text),
+ (r'\:', Name.Decorator, 'pseudo-class'),
+ (r'\.', Name.Class, 'class'),
+ (r'\#', Name.Namespace, 'id'),
+ (r'[\w-]+', Name.Tag),
+ (r'#\{', String.Interpol, 'interpolation'),
+ (r'&', Keyword),
+ (r'[~^*!&\[\]()<>|+=@:;,./?-]', Operator),
+ (r'"', String.Double, 'string-double'),
+ (r"'", String.Single, 'string-single'),
+ ],
+
+ 'string-double': [
+ (r'(\\.|#(?=[^\n{])|[^\n"#])+', String.Double),
+ (r'#\{', String.Interpol, 'interpolation'),
+ (r'"', String.Double, '#pop'),
+ ],
+
+ 'string-single': [
+ (r"(\\.|#(?=[^\n{])|[^\n'#])+", String.Double),
+ (r'#\{', String.Interpol, 'interpolation'),
+ (r"'", String.Double, '#pop'),
+ ],
+
+ 'string-url': [
+ (r'(\\#|#(?=[^\n{])|[^\n#)])+', String.Other),
+ (r'#\{', String.Interpol, 'interpolation'),
+ (r'\)', String.Other, '#pop'),
+ ],
+
+ 'pseudo-class': [
+ (r'[\w-]+', Name.Decorator),
+ (r'#\{', String.Interpol, 'interpolation'),
+ default('#pop'),
+ ],
+
+ 'class': [
+ (r'[\w-]+', Name.Class),
+ (r'#\{', String.Interpol, 'interpolation'),
+ default('#pop'),
+ ],
+
+ 'id': [
+ (r'[\w-]+', Name.Namespace),
+ (r'#\{', String.Interpol, 'interpolation'),
+ default('#pop'),
+ ],
+
+ 'for': [
+ (r'(from|to|through)', Operator.Word),
+ include('value'),
+ ],
+}
+
+
+def _indentation(lexer, match, ctx):
+ indentation = match.group(0)
+ yield match.start(), Text, indentation
+ ctx.last_indentation = indentation
+ ctx.pos = match.end()
+
+ if hasattr(ctx, 'block_state') and ctx.block_state and \
+ indentation.startswith(ctx.block_indentation) and \
+ indentation != ctx.block_indentation:
+ ctx.stack.append(ctx.block_state)
+ else:
+ ctx.block_state = None
+ ctx.block_indentation = None
+ ctx.stack.append('content')
+
+
+def _starts_block(token, state):
+ def callback(lexer, match, ctx):
+ yield match.start(), token, match.group(0)
+
+ if hasattr(ctx, 'last_indentation'):
+ ctx.block_indentation = ctx.last_indentation
+ else:
+ ctx.block_indentation = ''
+
+ ctx.block_state = state
+ ctx.pos = match.end()
+
+ return callback
+
+
+class SassLexer(ExtendedRegexLexer):
+ """
+ For Sass stylesheets.
+
+ .. versionadded:: 1.3
+ """
+
+ name = 'Sass'
+ aliases = ['sass']
+ filenames = ['*.sass']
+ mimetypes = ['text/x-sass']
+
+ flags = re.IGNORECASE | re.MULTILINE
+
+ tokens = {
+ 'root': [
+ (r'[ \t]*\n', Text),
+ (r'[ \t]*', _indentation),
+ ],
+
+ 'content': [
+ (r'//[^\n]*', _starts_block(Comment.Single, 'single-comment'),
+ 'root'),
+ (r'/\*[^\n]*', _starts_block(Comment.Multiline, 'multi-comment'),
+ 'root'),
+ (r'@import', Keyword, 'import'),
+ (r'@for', Keyword, 'for'),
+ (r'@(debug|warn|if|while)', Keyword, 'value'),
+ (r'(@mixin)( [\w-]+)', bygroups(Keyword, Name.Function), 'value'),
+ (r'(@include)( [\w-]+)', bygroups(Keyword, Name.Decorator), 'value'),
+ (r'@extend', Keyword, 'selector'),
+ (r'@[\w-]+', Keyword, 'selector'),
+ (r'=[\w-]+', Name.Function, 'value'),
+ (r'\+[\w-]+', Name.Decorator, 'value'),
+ (r'([!$][\w-]\w*)([ \t]*(?:(?:\|\|)?=|:))',
+ bygroups(Name.Variable, Operator), 'value'),
+ (r':', Name.Attribute, 'old-style-attr'),
+ (r'(?=.+?[=:]([^a-z]|$))', Name.Attribute, 'new-style-attr'),
+ default('selector'),
+ ],
+
+ 'single-comment': [
+ (r'.+', Comment.Single),
+ (r'\n', Text, 'root'),
+ ],
+
+ 'multi-comment': [
+ (r'.+', Comment.Multiline),
+ (r'\n', Text, 'root'),
+ ],
+
+ 'import': [
+ (r'[ \t]+', Text),
+ (r'\S+', String),
+ (r'\n', Text, 'root'),
+ ],
+
+ 'old-style-attr': [
+ (r'[^\s:="\[]+', Name.Attribute),
+ (r'#\{', String.Interpol, 'interpolation'),
+ (r'[ \t]*=', Operator, 'value'),
+ default('value'),
+ ],
+
+ 'new-style-attr': [
+ (r'[^\s:="\[]+', Name.Attribute),
+ (r'#\{', String.Interpol, 'interpolation'),
+ (r'[ \t]*[=:]', Operator, 'value'),
+ ],
+
+ 'inline-comment': [
+ (r"(\\#|#(?=[^\n{])|\*(?=[^\n/])|[^\n#*])+", Comment.Multiline),
+ (r'#\{', String.Interpol, 'interpolation'),
+ (r"\*/", Comment, '#pop'),
+ ],
+ }
+ for group, common in iteritems(common_sass_tokens):
+ tokens[group] = copy.copy(common)
+ tokens['value'].append((r'\n', Text, 'root'))
+ tokens['selector'].append((r'\n', Text, 'root'))
+
+
+class ScssLexer(RegexLexer):
+ """
+ For SCSS stylesheets.
+ """
+
+ name = 'SCSS'
+ aliases = ['scss']
+ filenames = ['*.scss']
+ mimetypes = ['text/x-scss']
+
+ flags = re.IGNORECASE | re.DOTALL
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'@import', Keyword, 'value'),
+ (r'@for', Keyword, 'for'),
+ (r'@(debug|warn|if|while)', Keyword, 'value'),
+ (r'(@mixin)( [\w-]+)', bygroups(Keyword, Name.Function), 'value'),
+ (r'(@include)( [\w-]+)', bygroups(Keyword, Name.Decorator), 'value'),
+ (r'@extend', Keyword, 'selector'),
+ (r'(@media)(\s+)', bygroups(Keyword, Text), 'value'),
+ (r'@[\w-]+', Keyword, 'selector'),
+ (r'(\$[\w-]*\w)([ \t]*:)', bygroups(Name.Variable, Operator), 'value'),
+ # TODO: broken, and prone to infinite loops.
+ # (r'(?=[^;{}][;}])', Name.Attribute, 'attr'),
+ # (r'(?=[^;{}:]+:[^a-z])', Name.Attribute, 'attr'),
+ default('selector'),
+ ],
+
+ 'attr': [
+ (r'[^\s:="\[]+', Name.Attribute),
+ (r'#\{', String.Interpol, 'interpolation'),
+ (r'[ \t]*:', Operator, 'value'),
+ default('#pop'),
+ ],
+
+ 'inline-comment': [
+ (r"(\\#|#(?=[^{])|\*(?=[^/])|[^#*])+", Comment.Multiline),
+ (r'#\{', String.Interpol, 'interpolation'),
+ (r"\*/", Comment, '#pop'),
+ ],
+ }
+ for group, common in iteritems(common_sass_tokens):
+ tokens[group] = copy.copy(common)
+ tokens['value'].extend([(r'\n', Text), (r'[;{}]', Punctuation, '#pop')])
+ tokens['selector'].extend([(r'\n', Text), (r'[;{}]', Punctuation, '#pop')])
+
+
+class LessCssLexer(CssLexer):
+ """
+ For `LESS <http://lesscss.org/>`_ styleshets.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'LessCss'
+ aliases = ['less']
+ filenames = ['*.less']
+ mimetypes = ['text/x-less-css']
+
+ tokens = {
+ 'root': [
+ (r'@\w+', Name.Variable),
+ inherit,
+ ],
+ 'content': [
+ (r'\{', Punctuation, '#push'),
+ inherit,
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/d.py b/wandb/vendor/pygments/lexers/d.py
new file mode 100644
index 0000000000000000000000000000000000000000..09e6fe87e96c5980dd162e8fa59a882c0007d9b7
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/d.py
@@ -0,0 +1,251 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.d
+ ~~~~~~~~~~~~~~~~~
+
+ Lexers for D languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Text, Comment, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['DLexer', 'CrocLexer', 'MiniDLexer']
+
+
+class DLexer(RegexLexer):
+ """
+ For D source.
+
+ .. versionadded:: 1.2
+ """
+ name = 'D'
+ filenames = ['*.d', '*.di']
+ aliases = ['d']
+ mimetypes = ['text/x-dsrc']
+
+ tokens = {
+ 'root': [
+ (r'\n', Text),
+ (r'\s+', Text),
+ # (r'\\\n', Text), # line continuations
+ # Comments
+ (r'//(.*?)\n', Comment.Single),
+ (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+ (r'/\+', Comment.Multiline, 'nested_comment'),
+ # Keywords
+ (words((
+ 'abstract', 'alias', 'align', 'asm', 'assert', 'auto', 'body',
+ 'break', 'case', 'cast', 'catch', 'class', 'const', 'continue',
+ 'debug', 'default', 'delegate', 'delete', 'deprecated', 'do', 'else',
+ 'enum', 'export', 'extern', 'finally', 'final', 'foreach_reverse',
+ 'foreach', 'for', 'function', 'goto', 'if', 'immutable', 'import',
+ 'interface', 'invariant', 'inout', 'in', 'is', 'lazy', 'mixin',
+ 'module', 'new', 'nothrow', 'out', 'override', 'package', 'pragma',
+ 'private', 'protected', 'public', 'pure', 'ref', 'return', 'scope',
+ 'shared', 'static', 'struct', 'super', 'switch', 'synchronized',
+ 'template', 'this', 'throw', 'try', 'typedef', 'typeid', 'typeof',
+ 'union', 'unittest', 'version', 'volatile', 'while', 'with',
+ '__gshared', '__traits', '__vector', '__parameters'),
+ suffix=r'\b'),
+ Keyword),
+ (words((
+ 'bool', 'byte', 'cdouble', 'cent', 'cfloat', 'char', 'creal',
+ 'dchar', 'double', 'float', 'idouble', 'ifloat', 'int', 'ireal',
+ 'long', 'real', 'short', 'ubyte', 'ucent', 'uint', 'ulong',
+ 'ushort', 'void', 'wchar'), suffix=r'\b'),
+ Keyword.Type),
+ (r'(false|true|null)\b', Keyword.Constant),
+ (words((
+ '__FILE__', '__MODULE__', '__LINE__', '__FUNCTION__', '__PRETTY_FUNCTION__'
+ '', '__DATE__', '__EOF__', '__TIME__', '__TIMESTAMP__', '__VENDOR__',
+ '__VERSION__'), suffix=r'\b'),
+ Keyword.Pseudo),
+ (r'macro\b', Keyword.Reserved),
+ (r'(string|wstring|dstring|size_t|ptrdiff_t)\b', Name.Builtin),
+ # FloatLiteral
+ # -- HexFloat
+ (r'0[xX]([0-9a-fA-F_]*\.[0-9a-fA-F_]+|[0-9a-fA-F_]+)'
+ r'[pP][+\-]?[0-9_]+[fFL]?[i]?', Number.Float),
+ # -- DecimalFloat
+ (r'[0-9_]+(\.[0-9_]+[eE][+\-]?[0-9_]+|'
+ r'\.[0-9_]*|[eE][+\-]?[0-9_]+)[fFL]?[i]?', Number.Float),
+ (r'\.(0|[1-9][0-9_]*)([eE][+\-]?[0-9_]+)?[fFL]?[i]?', Number.Float),
+ # IntegerLiteral
+ # -- Binary
+ (r'0[Bb][01_]+', Number.Bin),
+ # -- Octal
+ (r'0[0-7_]+', Number.Oct),
+ # -- Hexadecimal
+ (r'0[xX][0-9a-fA-F_]+', Number.Hex),
+ # -- Decimal
+ (r'(0|[1-9][0-9_]*)([LUu]|Lu|LU|uL|UL)?', Number.Integer),
+ # CharacterLiteral
+ (r"""'(\\['"?\\abfnrtv]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}"""
+ r"""|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|\\&\w+;|.)'""",
+ String.Char),
+ # StringLiteral
+ # -- WysiwygString
+ (r'r"[^"]*"[cwd]?', String),
+ # -- AlternateWysiwygString
+ (r'`[^`]*`[cwd]?', String),
+ # -- DoubleQuotedString
+ (r'"(\\\\|\\"|[^"])*"[cwd]?', String),
+ # -- EscapeSequence
+ (r"\\(['\"?\\abfnrtv]|x[0-9a-fA-F]{2}|[0-7]{1,3}"
+ r"|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8}|&\w+;)",
+ String),
+ # -- HexString
+ (r'x"[0-9a-fA-F_\s]*"[cwd]?', String),
+ # -- DelimitedString
+ (r'q"\[', String, 'delimited_bracket'),
+ (r'q"\(', String, 'delimited_parenthesis'),
+ (r'q"<', String, 'delimited_angle'),
+ (r'q"\{', String, 'delimited_curly'),
+ (r'q"([a-zA-Z_]\w*)\n.*?\n\1"', String),
+ (r'q"(.).*?\1"', String),
+ # -- TokenString
+ (r'q\{', String, 'token_string'),
+ # Attributes
+ (r'@([a-zA-Z_]\w*)?', Name.Decorator),
+ # Tokens
+ (r'(~=|\^=|%=|\*=|==|!>=|!<=|!<>=|!<>|!<|!>|!=|>>>=|>>>|>>=|>>|>='
+ r'|<>=|<>|<<=|<<|<=|\+\+|\+=|--|-=|\|\||\|=|&&|&=|\.\.\.|\.\.|/=)'
+ r'|[/.&|\-+<>!()\[\]{}?,;:$=*%^~]', Punctuation),
+ # Identifier
+ (r'[a-zA-Z_]\w*', Name),
+ # Line
+ (r'#line\s.*\n', Comment.Special),
+ ],
+ 'nested_comment': [
+ (r'[^+/]+', Comment.Multiline),
+ (r'/\+', Comment.Multiline, '#push'),
+ (r'\+/', Comment.Multiline, '#pop'),
+ (r'[+/]', Comment.Multiline),
+ ],
+ 'token_string': [
+ (r'\{', Punctuation, 'token_string_nest'),
+ (r'\}', String, '#pop'),
+ include('root'),
+ ],
+ 'token_string_nest': [
+ (r'\{', Punctuation, '#push'),
+ (r'\}', Punctuation, '#pop'),
+ include('root'),
+ ],
+ 'delimited_bracket': [
+ (r'[^\[\]]+', String),
+ (r'\[', String, 'delimited_inside_bracket'),
+ (r'\]"', String, '#pop'),
+ ],
+ 'delimited_inside_bracket': [
+ (r'[^\[\]]+', String),
+ (r'\[', String, '#push'),
+ (r'\]', String, '#pop'),
+ ],
+ 'delimited_parenthesis': [
+ (r'[^()]+', String),
+ (r'\(', String, 'delimited_inside_parenthesis'),
+ (r'\)"', String, '#pop'),
+ ],
+ 'delimited_inside_parenthesis': [
+ (r'[^()]+', String),
+ (r'\(', String, '#push'),
+ (r'\)', String, '#pop'),
+ ],
+ 'delimited_angle': [
+ (r'[^<>]+', String),
+ (r'<', String, 'delimited_inside_angle'),
+ (r'>"', String, '#pop'),
+ ],
+ 'delimited_inside_angle': [
+ (r'[^<>]+', String),
+ (r'<', String, '#push'),
+ (r'>', String, '#pop'),
+ ],
+ 'delimited_curly': [
+ (r'[^{}]+', String),
+ (r'\{', String, 'delimited_inside_curly'),
+ (r'\}"', String, '#pop'),
+ ],
+ 'delimited_inside_curly': [
+ (r'[^{}]+', String),
+ (r'\{', String, '#push'),
+ (r'\}', String, '#pop'),
+ ],
+ }
+
+
+class CrocLexer(RegexLexer):
+ """
+ For `Croc <http://jfbillingsley.com/croc>`_ source.
+ """
+ name = 'Croc'
+ filenames = ['*.croc']
+ aliases = ['croc']
+ mimetypes = ['text/x-crocsrc']
+
+ tokens = {
+ 'root': [
+ (r'\n', Text),
+ (r'\s+', Text),
+ # Comments
+ (r'//(.*?)\n', Comment.Single),
+ (r'/\*', Comment.Multiline, 'nestedcomment'),
+ # Keywords
+ (words((
+ 'as', 'assert', 'break', 'case', 'catch', 'class', 'continue',
+ 'default', 'do', 'else', 'finally', 'for', 'foreach', 'function',
+ 'global', 'namespace', 'if', 'import', 'in', 'is', 'local',
+ 'module', 'return', 'scope', 'super', 'switch', 'this', 'throw',
+ 'try', 'vararg', 'while', 'with', 'yield'), suffix=r'\b'),
+ Keyword),
+ (r'(false|true|null)\b', Keyword.Constant),
+ # FloatLiteral
+ (r'([0-9][0-9_]*)(?=[.eE])(\.[0-9][0-9_]*)?([eE][+\-]?[0-9_]+)?',
+ Number.Float),
+ # IntegerLiteral
+ # -- Binary
+ (r'0[bB][01][01_]*', Number.Bin),
+ # -- Hexadecimal
+ (r'0[xX][0-9a-fA-F][0-9a-fA-F_]*', Number.Hex),
+ # -- Decimal
+ (r'([0-9][0-9_]*)(?![.eE])', Number.Integer),
+ # CharacterLiteral
+ (r"""'(\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\[0-9]{1,3}"""
+ r"""|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|.)'""",
+ String.Char),
+ # StringLiteral
+ # -- WysiwygString
+ (r'@"(""|[^"])*"', String),
+ (r'@`(``|[^`])*`', String),
+ (r"@'(''|[^'])*'", String),
+ # -- DoubleQuotedString
+ (r'"(\\\\|\\"|[^"])*"', String),
+ # Tokens
+ (r'(~=|\^=|%=|\*=|==|!=|>>>=|>>>|>>=|>>|>=|<=>|\?=|-\>'
+ r'|<<=|<<|<=|\+\+|\+=|--|-=|\|\||\|=|&&|&=|\.\.|/=)'
+ r'|[-/.&$@|\+<>!()\[\]{}?,;:=*%^~#\\]', Punctuation),
+ # Identifier
+ (r'[a-zA-Z_]\w*', Name),
+ ],
+ 'nestedcomment': [
+ (r'[^*/]+', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline),
+ ],
+ }
+
+
+class MiniDLexer(CrocLexer):
+ """
+ For MiniD source. MiniD is now known as Croc.
+ """
+ name = 'MiniD'
+ filenames = [] # don't lex .md as MiniD, reserve for Markdown
+ aliases = ['minid']
+ mimetypes = ['text/x-minidsrc']
diff --git a/wandb/vendor/pygments/lexers/dalvik.py b/wandb/vendor/pygments/lexers/dalvik.py
new file mode 100644
index 0000000000000000000000000000000000000000..c211f13ef2a6bc152cb126504d8f674ebb94cb34
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/dalvik.py
@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.dalvik
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Pygments lexers for Dalvik VM-related languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups
+from pygments.token import Keyword, Text, Comment, Name, String, Number, \
+ Punctuation
+
+__all__ = ['SmaliLexer']
+
+
+class SmaliLexer(RegexLexer):
+ """
+ For `Smali <http://code.google.com/p/smali/>`_ (Android/Dalvik) assembly
+ code.
+
+ .. versionadded:: 1.6
+ """
+ name = 'Smali'
+ aliases = ['smali']
+ filenames = ['*.smali']
+ mimetypes = ['text/smali']
+
+ tokens = {
+ 'root': [
+ include('comment'),
+ include('label'),
+ include('field'),
+ include('method'),
+ include('class'),
+ include('directive'),
+ include('access-modifier'),
+ include('instruction'),
+ include('literal'),
+ include('punctuation'),
+ include('type'),
+ include('whitespace')
+ ],
+ 'directive': [
+ (r'^[ \t]*\.(class|super|implements|field|subannotation|annotation|'
+ r'enum|method|registers|locals|array-data|packed-switch|'
+ r'sparse-switch|catchall|catch|line|parameter|local|prologue|'
+ r'epilogue|source)', Keyword),
+ (r'^[ \t]*\.end (field|subannotation|annotation|method|array-data|'
+ 'packed-switch|sparse-switch|parameter|local)', Keyword),
+ (r'^[ \t]*\.restart local', Keyword),
+ ],
+ 'access-modifier': [
+ (r'(public|private|protected|static|final|synchronized|bridge|'
+ r'varargs|native|abstract|strictfp|synthetic|constructor|'
+ r'declared-synchronized|interface|enum|annotation|volatile|'
+ r'transient)', Keyword),
+ ],
+ 'whitespace': [
+ (r'\n', Text),
+ (r'\s+', Text),
+ ],
+ 'instruction': [
+ (r'\b[vp]\d+\b', Name.Builtin), # registers
+ (r'\b[a-z][A-Za-z0-9/-]+\s+', Text), # instructions
+ ],
+ 'literal': [
+ (r'".*"', String),
+ (r'0x[0-9A-Fa-f]+t?', Number.Hex),
+ (r'[0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'[0-9]+L?', Number.Integer),
+ ],
+ 'field': [
+ (r'(\$?\b)([\w$]*)(:)',
+ bygroups(Punctuation, Name.Variable, Punctuation)),
+ ],
+ 'method': [
+ (r'<(?:cl)?init>', Name.Function), # constructor
+ (r'(\$?\b)([\w$]*)(\()',
+ bygroups(Punctuation, Name.Function, Punctuation)),
+ ],
+ 'label': [
+ (r':\w+', Name.Label),
+ ],
+ 'class': [
+ # class names in the form Lcom/namespace/ClassName;
+ # I only want to color the ClassName part, so the namespace part is
+ # treated as 'Text'
+ (r'(L)((?:[\w$]+/)*)([\w$]+)(;)',
+ bygroups(Keyword.Type, Text, Name.Class, Text)),
+ ],
+ 'punctuation': [
+ (r'->', Punctuation),
+ (r'[{},():=.-]', Punctuation),
+ ],
+ 'type': [
+ (r'[ZBSCIJFDV\[]+', Keyword.Type),
+ ],
+ 'comment': [
+ (r'#.*?\n', Comment),
+ ],
+ }
+
+ def analyse_text(text):
+ score = 0
+ if re.search(r'^\s*\.class\s', text, re.MULTILINE):
+ score += 0.5
+ if re.search(r'\b((check-cast|instance-of|throw-verification-error'
+ r')\b|(-to|add|[ais]get|[ais]put|and|cmpl|const|div|'
+ r'if|invoke|move|mul|neg|not|or|rem|return|rsub|shl|'
+ r'shr|sub|ushr)[-/])|{|}', text, re.MULTILINE):
+ score += 0.3
+ if re.search(r'(\.(catchall|epilogue|restart local|prologue)|'
+ r'\b(array-data|class-change-error|declared-synchronized|'
+ r'(field|inline|vtable)@0x[0-9a-fA-F]|generic-error|'
+ r'illegal-class-access|illegal-field-access|'
+ r'illegal-method-access|instantiation-error|no-error|'
+ r'no-such-class|no-such-field|no-such-method|'
+ r'packed-switch|sparse-switch))\b', text, re.MULTILINE):
+ score += 0.6
+ return score
diff --git a/wandb/vendor/pygments/lexers/data.py b/wandb/vendor/pygments/lexers/data.py
new file mode 100644
index 0000000000000000000000000000000000000000..296366c225604574dcaf362203839d0cf05373c0
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/data.py
@@ -0,0 +1,555 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.data
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for data file format.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, ExtendedRegexLexer, LexerContext, \
+ include, bygroups, inherit
+from pygments.token import Text, Comment, Keyword, Name, String, Number, \
+ Punctuation, Literal, Error
+
+__all__ = ['YamlLexer', 'JsonLexer', 'JsonBareObjectLexer', 'JsonLdLexer']
+
+
+class YamlLexerContext(LexerContext):
+ """Indentation context for the YAML lexer."""
+
+ def __init__(self, *args, **kwds):
+ super(YamlLexerContext, self).__init__(*args, **kwds)
+ self.indent_stack = []
+ self.indent = -1
+ self.next_indent = 0
+ self.block_scalar_indent = None
+
+
+class YamlLexer(ExtendedRegexLexer):
+ """
+ Lexer for `YAML <http://yaml.org/>`_, a human-friendly data serialization
+ language.
+
+ .. versionadded:: 0.11
+ """
+
+ name = 'YAML'
+ aliases = ['yaml']
+ filenames = ['*.yaml', '*.yml']
+ mimetypes = ['text/x-yaml']
+
+ def something(token_class):
+ """Do not produce empty tokens."""
+ def callback(lexer, match, context):
+ text = match.group()
+ if not text:
+ return
+ yield match.start(), token_class, text
+ context.pos = match.end()
+ return callback
+
+ def reset_indent(token_class):
+ """Reset the indentation levels."""
+ def callback(lexer, match, context):
+ text = match.group()
+ context.indent_stack = []
+ context.indent = -1
+ context.next_indent = 0
+ context.block_scalar_indent = None
+ yield match.start(), token_class, text
+ context.pos = match.end()
+ return callback
+
+ def save_indent(token_class, start=False):
+ """Save a possible indentation level."""
+ def callback(lexer, match, context):
+ text = match.group()
+ extra = ''
+ if start:
+ context.next_indent = len(text)
+ if context.next_indent < context.indent:
+ while context.next_indent < context.indent:
+ context.indent = context.indent_stack.pop()
+ if context.next_indent > context.indent:
+ extra = text[context.indent:]
+ text = text[:context.indent]
+ else:
+ context.next_indent += len(text)
+ if text:
+ yield match.start(), token_class, text
+ if extra:
+ yield match.start()+len(text), token_class.Error, extra
+ context.pos = match.end()
+ return callback
+
+ def set_indent(token_class, implicit=False):
+ """Set the previously saved indentation level."""
+ def callback(lexer, match, context):
+ text = match.group()
+ if context.indent < context.next_indent:
+ context.indent_stack.append(context.indent)
+ context.indent = context.next_indent
+ if not implicit:
+ context.next_indent += len(text)
+ yield match.start(), token_class, text
+ context.pos = match.end()
+ return callback
+
+ def set_block_scalar_indent(token_class):
+ """Set an explicit indentation level for a block scalar."""
+ def callback(lexer, match, context):
+ text = match.group()
+ context.block_scalar_indent = None
+ if not text:
+ return
+ increment = match.group(1)
+ if increment:
+ current_indent = max(context.indent, 0)
+ increment = int(increment)
+ context.block_scalar_indent = current_indent + increment
+ if text:
+ yield match.start(), token_class, text
+ context.pos = match.end()
+ return callback
+
+ def parse_block_scalar_empty_line(indent_token_class, content_token_class):
+ """Process an empty line in a block scalar."""
+ def callback(lexer, match, context):
+ text = match.group()
+ if (context.block_scalar_indent is None or
+ len(text) <= context.block_scalar_indent):
+ if text:
+ yield match.start(), indent_token_class, text
+ else:
+ indentation = text[:context.block_scalar_indent]
+ content = text[context.block_scalar_indent:]
+ yield match.start(), indent_token_class, indentation
+ yield (match.start()+context.block_scalar_indent,
+ content_token_class, content)
+ context.pos = match.end()
+ return callback
+
+ def parse_block_scalar_indent(token_class):
+ """Process indentation spaces in a block scalar."""
+ def callback(lexer, match, context):
+ text = match.group()
+ if context.block_scalar_indent is None:
+ if len(text) <= max(context.indent, 0):
+ context.stack.pop()
+ context.stack.pop()
+ return
+ context.block_scalar_indent = len(text)
+ else:
+ if len(text) < context.block_scalar_indent:
+ context.stack.pop()
+ context.stack.pop()
+ return
+ if text:
+ yield match.start(), token_class, text
+ context.pos = match.end()
+ return callback
+
+ def parse_plain_scalar_indent(token_class):
+ """Process indentation spaces in a plain scalar."""
+ def callback(lexer, match, context):
+ text = match.group()
+ if len(text) <= context.indent:
+ context.stack.pop()
+ context.stack.pop()
+ return
+ if text:
+ yield match.start(), token_class, text
+ context.pos = match.end()
+ return callback
+
+ tokens = {
+ # the root rules
+ 'root': [
+ # ignored whitespaces
+ (r'[ ]+(?=#|$)', Text),
+ # line breaks
+ (r'\n+', Text),
+ # a comment
+ (r'#[^\n]*', Comment.Single),
+ # the '%YAML' directive
+ (r'^%YAML(?=[ ]|$)', reset_indent(Name.Tag), 'yaml-directive'),
+ # the %TAG directive
+ (r'^%TAG(?=[ ]|$)', reset_indent(Name.Tag), 'tag-directive'),
+ # document start and document end indicators
+ (r'^(?:---|\.\.\.)(?=[ ]|$)', reset_indent(Name.Namespace),
+ 'block-line'),
+ # indentation spaces
+ (r'[ ]*(?!\s|$)', save_indent(Text, start=True),
+ ('block-line', 'indentation')),
+ ],
+
+ # trailing whitespaces after directives or a block scalar indicator
+ 'ignored-line': [
+ # ignored whitespaces
+ (r'[ ]+(?=#|$)', Text),
+ # a comment
+ (r'#[^\n]*', Comment.Single),
+ # line break
+ (r'\n', Text, '#pop:2'),
+ ],
+
+ # the %YAML directive
+ 'yaml-directive': [
+ # the version number
+ (r'([ ]+)([0-9]+\.[0-9]+)',
+ bygroups(Text, Number), 'ignored-line'),
+ ],
+
+ # the %YAG directive
+ 'tag-directive': [
+ # a tag handle and the corresponding prefix
+ (r'([ ]+)(!|![\w-]*!)'
+ r'([ ]+)(!|!?[\w;/?:@&=+$,.!~*\'()\[\]%-]+)',
+ bygroups(Text, Keyword.Type, Text, Keyword.Type),
+ 'ignored-line'),
+ ],
+
+ # block scalar indicators and indentation spaces
+ 'indentation': [
+ # trailing whitespaces are ignored
+ (r'[ ]*$', something(Text), '#pop:2'),
+ # whitespaces preceeding block collection indicators
+ (r'[ ]+(?=[?:-](?:[ ]|$))', save_indent(Text)),
+ # block collection indicators
+ (r'[?:-](?=[ ]|$)', set_indent(Punctuation.Indicator)),
+ # the beginning a block line
+ (r'[ ]*', save_indent(Text), '#pop'),
+ ],
+
+ # an indented line in the block context
+ 'block-line': [
+ # the line end
+ (r'[ ]*(?=#|$)', something(Text), '#pop'),
+ # whitespaces separating tokens
+ (r'[ ]+', Text),
+ # tags, anchors and aliases,
+ include('descriptors'),
+ # block collections and scalars
+ include('block-nodes'),
+ # flow collections and quoted scalars
+ include('flow-nodes'),
+ # a plain scalar
+ (r'(?=[^\s?:,\[\]{}#&*!|>\'"%@`-]|[?:-]\S)',
+ something(Name.Variable),
+ 'plain-scalar-in-block-context'),
+ ],
+
+ # tags, anchors, aliases
+ 'descriptors': [
+ # a full-form tag
+ (r'!<[\w#;/?:@&=+$,.!~*\'()\[\]%-]+>', Keyword.Type),
+ # a tag in the form '!', '!suffix' or '!handle!suffix'
+ (r'!(?:[\w-]+!)?'
+ r'[\w#;/?:@&=+$,.!~*\'()\[\]%-]+', Keyword.Type),
+ # an anchor
+ (r'&[\w-]+', Name.Label),
+ # an alias
+ (r'\*[\w-]+', Name.Variable),
+ ],
+
+ # block collections and scalars
+ 'block-nodes': [
+ # implicit key
+ (r':(?=[ ]|$)', set_indent(Punctuation.Indicator, implicit=True)),
+ # literal and folded scalars
+ (r'[|>]', Punctuation.Indicator,
+ ('block-scalar-content', 'block-scalar-header')),
+ ],
+
+ # flow collections and quoted scalars
+ 'flow-nodes': [
+ # a flow sequence
+ (r'\[', Punctuation.Indicator, 'flow-sequence'),
+ # a flow mapping
+ (r'\{', Punctuation.Indicator, 'flow-mapping'),
+ # a single-quoted scalar
+ (r'\'', String, 'single-quoted-scalar'),
+ # a double-quoted scalar
+ (r'\"', String, 'double-quoted-scalar'),
+ ],
+
+ # the content of a flow collection
+ 'flow-collection': [
+ # whitespaces
+ (r'[ ]+', Text),
+ # line breaks
+ (r'\n+', Text),
+ # a comment
+ (r'#[^\n]*', Comment.Single),
+ # simple indicators
+ (r'[?:,]', Punctuation.Indicator),
+ # tags, anchors and aliases
+ include('descriptors'),
+ # nested collections and quoted scalars
+ include('flow-nodes'),
+ # a plain scalar
+ (r'(?=[^\s?:,\[\]{}#&*!|>\'"%@`])',
+ something(Name.Variable),
+ 'plain-scalar-in-flow-context'),
+ ],
+
+ # a flow sequence indicated by '[' and ']'
+ 'flow-sequence': [
+ # include flow collection rules
+ include('flow-collection'),
+ # the closing indicator
+ (r'\]', Punctuation.Indicator, '#pop'),
+ ],
+
+ # a flow mapping indicated by '{' and '}'
+ 'flow-mapping': [
+ # include flow collection rules
+ include('flow-collection'),
+ # the closing indicator
+ (r'\}', Punctuation.Indicator, '#pop'),
+ ],
+
+ # block scalar lines
+ 'block-scalar-content': [
+ # line break
+ (r'\n', Text),
+ # empty line
+ (r'^[ ]+$',
+ parse_block_scalar_empty_line(Text, Name.Constant)),
+ # indentation spaces (we may leave the state here)
+ (r'^[ ]*', parse_block_scalar_indent(Text)),
+ # line content
+ (r'[\S\t ]+', Name.Constant),
+ ],
+
+ # the content of a literal or folded scalar
+ 'block-scalar-header': [
+ # indentation indicator followed by chomping flag
+ (r'([1-9])?[+-]?(?=[ ]|$)',
+ set_block_scalar_indent(Punctuation.Indicator),
+ 'ignored-line'),
+ # chomping flag followed by indentation indicator
+ (r'[+-]?([1-9])?(?=[ ]|$)',
+ set_block_scalar_indent(Punctuation.Indicator),
+ 'ignored-line'),
+ ],
+
+ # ignored and regular whitespaces in quoted scalars
+ 'quoted-scalar-whitespaces': [
+ # leading and trailing whitespaces are ignored
+ (r'^[ ]+', Text),
+ (r'[ ]+$', Text),
+ # line breaks are ignored
+ (r'\n+', Text),
+ # other whitespaces are a part of the value
+ (r'[ ]+', Name.Variable),
+ ],
+
+ # single-quoted scalars
+ 'single-quoted-scalar': [
+ # include whitespace and line break rules
+ include('quoted-scalar-whitespaces'),
+ # escaping of the quote character
+ (r'\'\'', String.Escape),
+ # regular non-whitespace characters
+ (r'[^\s\']+', String),
+ # the closing quote
+ (r'\'', String, '#pop'),
+ ],
+
+ # double-quoted scalars
+ 'double-quoted-scalar': [
+ # include whitespace and line break rules
+ include('quoted-scalar-whitespaces'),
+ # escaping of special characters
+ (r'\\[0abt\tn\nvfre "\\N_LP]', String),
+ # escape codes
+ (r'\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})',
+ String.Escape),
+ # regular non-whitespace characters
+ (r'[^\s"\\]+', String),
+ # the closing quote
+ (r'"', String, '#pop'),
+ ],
+
+ # the beginning of a new line while scanning a plain scalar
+ 'plain-scalar-in-block-context-new-line': [
+ # empty lines
+ (r'^[ ]+$', Text),
+ # line breaks
+ (r'\n+', Text),
+ # document start and document end indicators
+ (r'^(?=---|\.\.\.)', something(Name.Namespace), '#pop:3'),
+ # indentation spaces (we may leave the block line state here)
+ (r'^[ ]*', parse_plain_scalar_indent(Text), '#pop'),
+ ],
+
+ # a plain scalar in the block context
+ 'plain-scalar-in-block-context': [
+ # the scalar ends with the ':' indicator
+ (r'[ ]*(?=:[ ]|:$)', something(Text), '#pop'),
+ # the scalar ends with whitespaces followed by a comment
+ (r'[ ]+(?=#)', Text, '#pop'),
+ # trailing whitespaces are ignored
+ (r'[ ]+$', Text),
+ # line breaks are ignored
+ (r'\n+', Text, 'plain-scalar-in-block-context-new-line'),
+ # other whitespaces are a part of the value
+ (r'[ ]+', Literal.Scalar.Plain),
+ # regular non-whitespace characters
+ (r'(?::(?!\s)|[^\s:])+', Literal.Scalar.Plain),
+ ],
+
+ # a plain scalar is the flow context
+ 'plain-scalar-in-flow-context': [
+ # the scalar ends with an indicator character
+ (r'[ ]*(?=[,:?\[\]{}])', something(Text), '#pop'),
+ # the scalar ends with a comment
+ (r'[ ]+(?=#)', Text, '#pop'),
+ # leading and trailing whitespaces are ignored
+ (r'^[ ]+', Text),
+ (r'[ ]+$', Text),
+ # line breaks are ignored
+ (r'\n+', Text),
+ # other whitespaces are a part of the value
+ (r'[ ]+', Name.Variable),
+ # regular non-whitespace characters
+ (r'[^\s,:?\[\]{}]+', Name.Variable),
+ ],
+
+ }
+
+ def get_tokens_unprocessed(self, text=None, context=None):
+ if context is None:
+ context = YamlLexerContext(text, 0)
+ return super(YamlLexer, self).get_tokens_unprocessed(text, context)
+
+
+class JsonLexer(RegexLexer):
+ """
+ For JSON data structures.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'JSON'
+ aliases = ['json']
+ filenames = ['*.json']
+ mimetypes = ['application/json']
+
+ flags = re.DOTALL
+
+ # integer part of a number
+ int_part = r'-?(0|[1-9]\d*)'
+
+ # fractional part of a number
+ frac_part = r'\.\d+'
+
+ # exponential part of a number
+ exp_part = r'[eE](\+|-)?\d+'
+
+ tokens = {
+ 'whitespace': [
+ (r'\s+', Text),
+ ],
+
+ # represents a simple terminal value
+ 'simplevalue': [
+ (r'(true|false|null)\b', Keyword.Constant),
+ (('%(int_part)s(%(frac_part)s%(exp_part)s|'
+ '%(exp_part)s|%(frac_part)s)') % vars(),
+ Number.Float),
+ (int_part, Number.Integer),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ ],
+
+
+ # the right hand side of an object, after the attribute name
+ 'objectattribute': [
+ include('value'),
+ (r':', Punctuation),
+ # comma terminates the attribute but expects more
+ (r',', Punctuation, '#pop'),
+ # a closing bracket terminates the entire object, so pop twice
+ (r'\}', Punctuation, '#pop:2'),
+ ],
+
+ # a json object - { attr, attr, ... }
+ 'objectvalue': [
+ include('whitespace'),
+ (r'"(\\\\|\\"|[^"])*"', Name.Tag, 'objectattribute'),
+ (r'\}', Punctuation, '#pop'),
+ ],
+
+ # json array - [ value, value, ... }
+ 'arrayvalue': [
+ include('whitespace'),
+ include('value'),
+ (r',', Punctuation),
+ (r'\]', Punctuation, '#pop'),
+ ],
+
+ # a json value - either a simple value or a complex value (object or array)
+ 'value': [
+ include('whitespace'),
+ include('simplevalue'),
+ (r'\{', Punctuation, 'objectvalue'),
+ (r'\[', Punctuation, 'arrayvalue'),
+ ],
+
+ # the root of a json document whould be a value
+ 'root': [
+ include('value'),
+ ],
+ }
+
+
+class JsonBareObjectLexer(JsonLexer):
+ """
+ For JSON data structures (with missing object curly braces).
+
+ .. versionadded:: 2.2
+ """
+
+ name = 'JSONBareObject'
+ aliases = ['json-object']
+ filenames = []
+ mimetypes = ['application/json-object']
+
+ tokens = {
+ 'root': [
+ (r'\}', Error),
+ include('objectvalue'),
+ ],
+ 'objectattribute': [
+ (r'\}', Error),
+ inherit,
+ ],
+ }
+
+
+class JsonLdLexer(JsonLexer):
+ """
+ For `JSON-LD <http://json-ld.org/>`_ linked data.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'JSON-LD'
+ aliases = ['jsonld', 'json-ld']
+ filenames = ['*.jsonld']
+ mimetypes = ['application/ld+json']
+
+ tokens = {
+ 'objectvalue': [
+ (r'"@(context|id|value|language|type|container|list|set|'
+ r'reverse|index|base|vocab|graph)"', Name.Decorator,
+ 'objectattribute'),
+ inherit,
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/diff.py b/wandb/vendor/pygments/lexers/diff.py
new file mode 100644
index 0000000000000000000000000000000000000000..f7019440290b694e69787f2fa909cb9fd6add0b0
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/diff.py
@@ -0,0 +1,165 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.diff
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for diff/patch formats.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups
+from pygments.token import Text, Comment, Operator, Keyword, Name, Generic, \
+ Literal
+
+__all__ = ['DiffLexer', 'DarcsPatchLexer', 'WDiffLexer']
+
+
+class DiffLexer(RegexLexer):
+ """
+ Lexer for unified or context-style diffs or patches.
+ """
+
+ name = 'Diff'
+ aliases = ['diff', 'udiff']
+ filenames = ['*.diff', '*.patch']
+ mimetypes = ['text/x-diff', 'text/x-patch']
+
+ tokens = {
+ 'root': [
+ (r' .*\n', Text),
+ (r'\+.*\n', Generic.Inserted),
+ (r'-.*\n', Generic.Deleted),
+ (r'!.*\n', Generic.Strong),
+ (r'@.*\n', Generic.Subheading),
+ (r'([Ii]ndex|diff).*\n', Generic.Heading),
+ (r'=.*\n', Generic.Heading),
+ (r'.*\n', Text),
+ ]
+ }
+
+ def analyse_text(text):
+ if text[:7] == 'Index: ':
+ return True
+ if text[:5] == 'diff ':
+ return True
+ if text[:4] == '--- ':
+ return 0.9
+
+
+class DarcsPatchLexer(RegexLexer):
+ """
+ DarcsPatchLexer is a lexer for the various versions of the darcs patch
+ format. Examples of this format are derived by commands such as
+ ``darcs annotate --patch`` and ``darcs send``.
+
+ .. versionadded:: 0.10
+ """
+
+ name = 'Darcs Patch'
+ aliases = ['dpatch']
+ filenames = ['*.dpatch', '*.darcspatch']
+
+ DPATCH_KEYWORDS = ('hunk', 'addfile', 'adddir', 'rmfile', 'rmdir', 'move',
+ 'replace')
+
+ tokens = {
+ 'root': [
+ (r'<', Operator),
+ (r'>', Operator),
+ (r'\{', Operator),
+ (r'\}', Operator),
+ (r'(\[)((?:TAG )?)(.*)(\n)(.*)(\*\*)(\d+)(\s?)(\])',
+ bygroups(Operator, Keyword, Name, Text, Name, Operator,
+ Literal.Date, Text, Operator)),
+ (r'(\[)((?:TAG )?)(.*)(\n)(.*)(\*\*)(\d+)(\s?)',
+ bygroups(Operator, Keyword, Name, Text, Name, Operator,
+ Literal.Date, Text), 'comment'),
+ (r'New patches:', Generic.Heading),
+ (r'Context:', Generic.Heading),
+ (r'Patch bundle hash:', Generic.Heading),
+ (r'(\s*)(%s)(.*\n)' % '|'.join(DPATCH_KEYWORDS),
+ bygroups(Text, Keyword, Text)),
+ (r'\+', Generic.Inserted, "insert"),
+ (r'-', Generic.Deleted, "delete"),
+ (r'.*\n', Text),
+ ],
+ 'comment': [
+ (r'[^\]].*\n', Comment),
+ (r'\]', Operator, "#pop"),
+ ],
+ 'specialText': [ # darcs add [_CODE_] special operators for clarity
+ (r'\n', Text, "#pop"), # line-based
+ (r'\[_[^_]*_]', Operator),
+ ],
+ 'insert': [
+ include('specialText'),
+ (r'\[', Generic.Inserted),
+ (r'[^\n\[]+', Generic.Inserted),
+ ],
+ 'delete': [
+ include('specialText'),
+ (r'\[', Generic.Deleted),
+ (r'[^\n\[]+', Generic.Deleted),
+ ],
+ }
+
+
+class WDiffLexer(RegexLexer):
+ """
+ A `wdiff <https://www.gnu.org/software/wdiff/>`_ lexer.
+
+ Note that:
+
+ * only to normal output (without option like -l).
+ * if target files of wdiff contain "[-", "-]", "{+", "+}",
+ especially they are unbalanced, this lexer will get confusing.
+
+ .. versionadded:: 2.2
+ """
+
+ name = 'WDiff'
+ aliases = ['wdiff']
+ filenames = ['*.wdiff']
+ mimetypes = []
+
+ flags = re.MULTILINE | re.DOTALL
+
+ # We can only assume "[-" after "[-" before "-]" is `nested`,
+ # for instance wdiff to wdiff outputs. We have no way to
+ # distinct these marker is of wdiff output from original text.
+
+ ins_op = r"\{\+"
+ ins_cl = r"\+\}"
+ del_op = r"\[\-"
+ del_cl = r"\-\]"
+ normal = r'[^{}[\]+-]+' # for performance
+ tokens = {
+ 'root': [
+ (ins_op, Generic.Inserted, 'inserted'),
+ (del_op, Generic.Deleted, 'deleted'),
+ (normal, Text),
+ (r'.', Text),
+ ],
+ 'inserted': [
+ (ins_op, Generic.Inserted, '#push'),
+ (del_op, Generic.Inserted, '#push'),
+ (del_cl, Generic.Inserted, '#pop'),
+
+ (ins_cl, Generic.Inserted, '#pop'),
+ (normal, Generic.Inserted),
+ (r'.', Generic.Inserted),
+ ],
+ 'deleted': [
+ (del_op, Generic.Deleted, '#push'),
+ (ins_op, Generic.Deleted, '#push'),
+ (ins_cl, Generic.Deleted, '#pop'),
+
+ (del_cl, Generic.Deleted, '#pop'),
+ (normal, Generic.Deleted),
+ (r'.', Generic.Deleted),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/dotnet.py b/wandb/vendor/pygments/lexers/dotnet.py
new file mode 100644
index 0000000000000000000000000000000000000000..4e2bc8ab5c91b5444a90a31948e4dd35b10b79f8
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/dotnet.py
@@ -0,0 +1,691 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.dotnet
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for .net languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+import re
+
+from pygments.lexer import RegexLexer, DelegatingLexer, bygroups, include, \
+ using, this, default, words
+from pygments.token import Punctuation, \
+ Text, Comment, Operator, Keyword, Name, String, Number, Literal, Other
+from pygments.util import get_choice_opt, iteritems
+from pygments import unistring as uni
+
+from pygments.lexers.html import XmlLexer
+
+__all__ = ['CSharpLexer', 'NemerleLexer', 'BooLexer', 'VbNetLexer',
+ 'CSharpAspxLexer', 'VbNetAspxLexer', 'FSharpLexer']
+
+
+class CSharpLexer(RegexLexer):
+ """
+ For `C# <http://msdn2.microsoft.com/en-us/vcsharp/default.aspx>`_
+ source code.
+
+ Additional options accepted:
+
+ `unicodelevel`
+ Determines which Unicode characters this lexer allows for identifiers.
+ The possible values are:
+
+ * ``none`` -- only the ASCII letters and numbers are allowed. This
+ is the fastest selection.
+ * ``basic`` -- all Unicode characters from the specification except
+ category ``Lo`` are allowed.
+ * ``full`` -- all Unicode characters as specified in the C# specs
+ are allowed. Note that this means a considerable slowdown since the
+ ``Lo`` category has more than 40,000 characters in it!
+
+ The default value is ``basic``.
+
+ .. versionadded:: 0.8
+ """
+
+ name = 'C#'
+ aliases = ['csharp', 'c#']
+ filenames = ['*.cs']
+ mimetypes = ['text/x-csharp'] # inferred
+
+ flags = re.MULTILINE | re.DOTALL | re.UNICODE
+
+ # for the range of allowed unicode characters in identifiers, see
+ # http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf
+
+ levels = {
+ 'none': '@?[_a-zA-Z]\w*',
+ 'basic': ('@?[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' +
+ '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc',
+ 'Cf', 'Mn', 'Mc') + ']*'),
+ 'full': ('@?(?:_|[^' +
+ uni.allexcept('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl') + '])'
+ + '[^' + uni.allexcept('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl',
+ 'Nd', 'Pc', 'Cf', 'Mn', 'Mc') + ']*'),
+ }
+
+ tokens = {}
+ token_variants = True
+
+ for levelname, cs_ident in iteritems(levels):
+ tokens[levelname] = {
+ 'root': [
+ # method names
+ (r'^([ \t]*(?:' + cs_ident + r'(?:\[\])?\s+)+?)' # return type
+ r'(' + cs_ident + ')' # method name
+ r'(\s*)(\()', # signature start
+ bygroups(using(this), Name.Function, Text, Punctuation)),
+ (r'^\s*\[.*?\]', Name.Attribute),
+ (r'[^\S\n]+', Text),
+ (r'\\\n', Text), # line continuation
+ (r'//.*?\n', Comment.Single),
+ (r'/[*].*?[*]/', Comment.Multiline),
+ (r'\n', Text),
+ (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation),
+ (r'[{}]', Punctuation),
+ (r'@"(""|[^"])*"', String),
+ (r'"(\\\\|\\"|[^"\n])*["\n]', String),
+ (r"'\\.'|'[^\\]'", String.Char),
+ (r"[0-9](\.[0-9]*)?([eE][+-][0-9]+)?"
+ r"[flFLdD]?|0[xX][0-9a-fA-F]+[Ll]?", Number),
+ (r'#[ \t]*(if|endif|else|elif|define|undef|'
+ r'line|error|warning|region|endregion|pragma)\b.*?\n',
+ Comment.Preproc),
+ (r'\b(extern)(\s+)(alias)\b', bygroups(Keyword, Text,
+ Keyword)),
+ (r'(abstract|as|async|await|base|break|by|case|catch|'
+ r'checked|const|continue|default|delegate|'
+ r'do|else|enum|event|explicit|extern|false|finally|'
+ r'fixed|for|foreach|goto|if|implicit|in|interface|'
+ r'internal|is|let|lock|new|null|on|operator|'
+ r'out|override|params|private|protected|public|readonly|'
+ r'ref|return|sealed|sizeof|stackalloc|static|'
+ r'switch|this|throw|true|try|typeof|'
+ r'unchecked|unsafe|virtual|void|while|'
+ r'get|set|new|partial|yield|add|remove|value|alias|ascending|'
+ r'descending|from|group|into|orderby|select|thenby|where|'
+ r'join|equals)\b', Keyword),
+ (r'(global)(::)', bygroups(Keyword, Punctuation)),
+ (r'(bool|byte|char|decimal|double|dynamic|float|int|long|object|'
+ r'sbyte|short|string|uint|ulong|ushort|var)\b\??', Keyword.Type),
+ (r'(class|struct)(\s+)', bygroups(Keyword, Text), 'class'),
+ (r'(namespace|using)(\s+)', bygroups(Keyword, Text), 'namespace'),
+ (cs_ident, Name),
+ ],
+ 'class': [
+ (cs_ident, Name.Class, '#pop'),
+ default('#pop'),
+ ],
+ 'namespace': [
+ (r'(?=\()', Text, '#pop'), # using (resource)
+ ('(' + cs_ident + r'|\.)+', Name.Namespace, '#pop'),
+ ]
+ }
+
+ def __init__(self, **options):
+ level = get_choice_opt(options, 'unicodelevel', list(self.tokens), 'basic')
+ if level not in self._all_tokens:
+ # compile the regexes now
+ self._tokens = self.__class__.process_tokendef(level)
+ else:
+ self._tokens = self._all_tokens[level]
+
+ RegexLexer.__init__(self, **options)
+
+
+class NemerleLexer(RegexLexer):
+ """
+ For `Nemerle <http://nemerle.org>`_ source code.
+
+ Additional options accepted:
+
+ `unicodelevel`
+ Determines which Unicode characters this lexer allows for identifiers.
+ The possible values are:
+
+ * ``none`` -- only the ASCII letters and numbers are allowed. This
+ is the fastest selection.
+ * ``basic`` -- all Unicode characters from the specification except
+ category ``Lo`` are allowed.
+ * ``full`` -- all Unicode characters as specified in the C# specs
+ are allowed. Note that this means a considerable slowdown since the
+ ``Lo`` category has more than 40,000 characters in it!
+
+ The default value is ``basic``.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'Nemerle'
+ aliases = ['nemerle']
+ filenames = ['*.n']
+ mimetypes = ['text/x-nemerle'] # inferred
+
+ flags = re.MULTILINE | re.DOTALL | re.UNICODE
+
+ # for the range of allowed unicode characters in identifiers, see
+ # http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf
+
+ levels = {
+ 'none': '@?[_a-zA-Z]\w*',
+ 'basic': ('@?[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' +
+ '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc',
+ 'Cf', 'Mn', 'Mc') + ']*'),
+ 'full': ('@?(?:_|[^' +
+ uni.allexcept('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl') + '])'
+ + '[^' + uni.allexcept('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl',
+ 'Nd', 'Pc', 'Cf', 'Mn', 'Mc') + ']*'),
+ }
+
+ tokens = {}
+ token_variants = True
+
+ for levelname, cs_ident in iteritems(levels):
+ tokens[levelname] = {
+ 'root': [
+ # method names
+ (r'^([ \t]*(?:' + cs_ident + r'(?:\[\])?\s+)+?)' # return type
+ r'(' + cs_ident + ')' # method name
+ r'(\s*)(\()', # signature start
+ bygroups(using(this), Name.Function, Text, Punctuation)),
+ (r'^\s*\[.*?\]', Name.Attribute),
+ (r'[^\S\n]+', Text),
+ (r'\\\n', Text), # line continuation
+ (r'//.*?\n', Comment.Single),
+ (r'/[*].*?[*]/', Comment.Multiline),
+ (r'\n', Text),
+ (r'\$\s*"', String, 'splice-string'),
+ (r'\$\s*<#', String, 'splice-string2'),
+ (r'<#', String, 'recursive-string'),
+
+ (r'(<\[)\s*(' + cs_ident + ':)?', Keyword),
+ (r'\]\>', Keyword),
+
+ # quasiquotation only
+ (r'\$' + cs_ident, Name),
+ (r'(\$)(\()', bygroups(Name, Punctuation),
+ 'splice-string-content'),
+
+ (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation),
+ (r'[{}]', Punctuation),
+ (r'@"(""|[^"])*"', String),
+ (r'"(\\\\|\\"|[^"\n])*["\n]', String),
+ (r"'\\.'|'[^\\]'", String.Char),
+ (r"0[xX][0-9a-fA-F]+[Ll]?", Number),
+ (r"[0-9](\.[0-9]*)?([eE][+-][0-9]+)?[flFLdD]?", Number),
+ (r'#[ \t]*(if|endif|else|elif|define|undef|'
+ r'line|error|warning|region|endregion|pragma)\b.*?\n',
+ Comment.Preproc),
+ (r'\b(extern)(\s+)(alias)\b', bygroups(Keyword, Text,
+ Keyword)),
+ (r'(abstract|and|as|base|catch|def|delegate|'
+ r'enum|event|extern|false|finally|'
+ r'fun|implements|interface|internal|'
+ r'is|macro|match|matches|module|mutable|new|'
+ r'null|out|override|params|partial|private|'
+ r'protected|public|ref|sealed|static|'
+ r'syntax|this|throw|true|try|type|typeof|'
+ r'virtual|volatile|when|where|with|'
+ r'assert|assert2|async|break|checked|continue|do|else|'
+ r'ensures|for|foreach|if|late|lock|new|nolate|'
+ r'otherwise|regexp|repeat|requires|return|surroundwith|'
+ r'unchecked|unless|using|while|yield)\b', Keyword),
+ (r'(global)(::)', bygroups(Keyword, Punctuation)),
+ (r'(bool|byte|char|decimal|double|float|int|long|object|sbyte|'
+ r'short|string|uint|ulong|ushort|void|array|list)\b\??',
+ Keyword.Type),
+ (r'(:>?)\s*(' + cs_ident + r'\??)',
+ bygroups(Punctuation, Keyword.Type)),
+ (r'(class|struct|variant|module)(\s+)',
+ bygroups(Keyword, Text), 'class'),
+ (r'(namespace|using)(\s+)', bygroups(Keyword, Text),
+ 'namespace'),
+ (cs_ident, Name),
+ ],
+ 'class': [
+ (cs_ident, Name.Class, '#pop')
+ ],
+ 'namespace': [
+ (r'(?=\()', Text, '#pop'), # using (resource)
+ ('(' + cs_ident + r'|\.)+', Name.Namespace, '#pop')
+ ],
+ 'splice-string': [
+ (r'[^"$]', String),
+ (r'\$' + cs_ident, Name),
+ (r'(\$)(\()', bygroups(Name, Punctuation),
+ 'splice-string-content'),
+ (r'\\"', String),
+ (r'"', String, '#pop')
+ ],
+ 'splice-string2': [
+ (r'[^#<>$]', String),
+ (r'\$' + cs_ident, Name),
+ (r'(\$)(\()', bygroups(Name, Punctuation),
+ 'splice-string-content'),
+ (r'<#', String, '#push'),
+ (r'#>', String, '#pop')
+ ],
+ 'recursive-string': [
+ (r'[^#<>]', String),
+ (r'<#', String, '#push'),
+ (r'#>', String, '#pop')
+ ],
+ 'splice-string-content': [
+ (r'if|match', Keyword),
+ (r'[~!%^&*+=|\[\]:;,.<>/?-\\"$ ]', Punctuation),
+ (cs_ident, Name),
+ (r'\d+', Number),
+ (r'\(', Punctuation, '#push'),
+ (r'\)', Punctuation, '#pop')
+ ]
+ }
+
+ def __init__(self, **options):
+ level = get_choice_opt(options, 'unicodelevel', list(self.tokens),
+ 'basic')
+ if level not in self._all_tokens:
+ # compile the regexes now
+ self._tokens = self.__class__.process_tokendef(level)
+ else:
+ self._tokens = self._all_tokens[level]
+
+ RegexLexer.__init__(self, **options)
+
+
+class BooLexer(RegexLexer):
+ """
+ For `Boo <http://boo.codehaus.org/>`_ source code.
+ """
+
+ name = 'Boo'
+ aliases = ['boo']
+ filenames = ['*.boo']
+ mimetypes = ['text/x-boo']
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'(#|//).*$', Comment.Single),
+ (r'/[*]', Comment.Multiline, 'comment'),
+ (r'[]{}:(),.;[]', Punctuation),
+ (r'\\\n', Text),
+ (r'\\', Text),
+ (r'(in|is|and|or|not)\b', Operator.Word),
+ (r'/(\\\\|\\/|[^/\s])/', String.Regex),
+ (r'@/(\\\\|\\/|[^/])*/', String.Regex),
+ (r'=~|!=|==|<<|>>|[-+/*%=<>&^|]', Operator),
+ (r'(as|abstract|callable|constructor|destructor|do|import|'
+ r'enum|event|final|get|interface|internal|of|override|'
+ r'partial|private|protected|public|return|set|static|'
+ r'struct|transient|virtual|yield|super|and|break|cast|'
+ r'continue|elif|else|ensure|except|for|given|goto|if|in|'
+ r'is|isa|not|or|otherwise|pass|raise|ref|try|unless|when|'
+ r'while|from|as)\b', Keyword),
+ (r'def(?=\s+\(.*?\))', Keyword),
+ (r'(def)(\s+)', bygroups(Keyword, Text), 'funcname'),
+ (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
+ (r'(namespace)(\s+)', bygroups(Keyword, Text), 'namespace'),
+ (r'(?<!\.)(true|false|null|self|__eval__|__switch__|array|'
+ r'assert|checked|enumerate|filter|getter|len|lock|map|'
+ r'matrix|max|min|normalArrayIndexing|print|property|range|'
+ r'rawArrayIndexing|required|typeof|unchecked|using|'
+ r'yieldAll|zip)\b', Name.Builtin),
+ (r'"""(\\\\|\\"|.*?)"""', String.Double),
+ (r'"(\\\\|\\"|[^"]*?)"', String.Double),
+ (r"'(\\\\|\\'|[^']*?)'", String.Single),
+ (r'[a-zA-Z_]\w*', Name),
+ (r'(\d+\.\d*|\d*\.\d+)([fF][+-]?[0-9]+)?', Number.Float),
+ (r'[0-9][0-9.]*(ms?|d|h|s)', Number),
+ (r'0\d+', Number.Oct),
+ (r'0x[a-fA-F0-9]+', Number.Hex),
+ (r'\d+L', Number.Integer.Long),
+ (r'\d+', Number.Integer),
+ ],
+ 'comment': [
+ ('/[*]', Comment.Multiline, '#push'),
+ ('[*]/', Comment.Multiline, '#pop'),
+ ('[^/*]', Comment.Multiline),
+ ('[*/]', Comment.Multiline)
+ ],
+ 'funcname': [
+ ('[a-zA-Z_]\w*', Name.Function, '#pop')
+ ],
+ 'classname': [
+ ('[a-zA-Z_]\w*', Name.Class, '#pop')
+ ],
+ 'namespace': [
+ ('[a-zA-Z_][\w.]*', Name.Namespace, '#pop')
+ ]
+ }
+
+
+class VbNetLexer(RegexLexer):
+ """
+ For
+ `Visual Basic.NET <http://msdn2.microsoft.com/en-us/vbasic/default.aspx>`_
+ source code.
+ """
+
+ name = 'VB.net'
+ aliases = ['vb.net', 'vbnet']
+ filenames = ['*.vb', '*.bas']
+ mimetypes = ['text/x-vbnet', 'text/x-vba'] # (?)
+
+ uni_name = '[_' + uni.combine('Ll', 'Lt', 'Lm', 'Nl') + ']' + \
+ '[' + uni.combine('Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc',
+ 'Cf', 'Mn', 'Mc') + ']*'
+
+ flags = re.MULTILINE | re.IGNORECASE
+ tokens = {
+ 'root': [
+ (r'^\s*<.*?>', Name.Attribute),
+ (r'\s+', Text),
+ (r'\n', Text),
+ (r'rem\b.*?\n', Comment),
+ (r"'.*?\n", Comment),
+ (r'#If\s.*?\sThen|#ElseIf\s.*?\sThen|#Else|#End\s+If|#Const|'
+ r'#ExternalSource.*?\n|#End\s+ExternalSource|'
+ r'#Region.*?\n|#End\s+Region|#ExternalChecksum',
+ Comment.Preproc),
+ (r'[(){}!#,.:]', Punctuation),
+ (r'Option\s+(Strict|Explicit|Compare)\s+'
+ r'(On|Off|Binary|Text)', Keyword.Declaration),
+ (words((
+ 'AddHandler', 'Alias', 'ByRef', 'ByVal', 'Call', 'Case',
+ 'Catch', 'CBool', 'CByte', 'CChar', 'CDate', 'CDec', 'CDbl',
+ 'CInt', 'CLng', 'CObj', 'Continue', 'CSByte', 'CShort', 'CSng',
+ 'CStr', 'CType', 'CUInt', 'CULng', 'CUShort', 'Declare',
+ 'Default', 'Delegate', 'DirectCast', 'Do', 'Each', 'Else',
+ 'ElseIf', 'EndIf', 'Erase', 'Error', 'Event', 'Exit', 'False',
+ 'Finally', 'For', 'Friend', 'Get', 'Global', 'GoSub', 'GoTo',
+ 'Handles', 'If', 'Implements', 'Inherits', 'Interface', 'Let',
+ 'Lib', 'Loop', 'Me', 'MustInherit', 'MustOverride', 'MyBase',
+ 'MyClass', 'Narrowing', 'New', 'Next', 'Not', 'Nothing',
+ 'NotInheritable', 'NotOverridable', 'Of', 'On', 'Operator',
+ 'Option', 'Optional', 'Overloads', 'Overridable', 'Overrides',
+ 'ParamArray', 'Partial', 'Private', 'Protected', 'Public',
+ 'RaiseEvent', 'ReadOnly', 'ReDim', 'RemoveHandler', 'Resume',
+ 'Return', 'Select', 'Set', 'Shadows', 'Shared', 'Single',
+ 'Static', 'Step', 'Stop', 'SyncLock', 'Then', 'Throw', 'To',
+ 'True', 'Try', 'TryCast', 'Wend', 'Using', 'When', 'While',
+ 'Widening', 'With', 'WithEvents', 'WriteOnly'),
+ prefix='(?<!\.)', suffix=r'\b'), Keyword),
+ (r'(?<!\.)End\b', Keyword, 'end'),
+ (r'(?<!\.)(Dim|Const)\b', Keyword, 'dim'),
+ (r'(?<!\.)(Function|Sub|Property)(\s+)',
+ bygroups(Keyword, Text), 'funcname'),
+ (r'(?<!\.)(Class|Structure|Enum)(\s+)',
+ bygroups(Keyword, Text), 'classname'),
+ (r'(?<!\.)(Module|Namespace|Imports)(\s+)',
+ bygroups(Keyword, Text), 'namespace'),
+ (r'(?<!\.)(Boolean|Byte|Char|Date|Decimal|Double|Integer|Long|'
+ r'Object|SByte|Short|Single|String|Variant|UInteger|ULong|'
+ r'UShort)\b', Keyword.Type),
+ (r'(?<!\.)(AddressOf|And|AndAlso|As|GetType|In|Is|IsNot|Like|Mod|'
+ r'Or|OrElse|TypeOf|Xor)\b', Operator.Word),
+ (r'&=|[*]=|/=|\\=|\^=|\+=|-=|<<=|>>=|<<|>>|:=|'
+ r'<=|>=|<>|[-&*/\\^+=<>\[\]]',
+ Operator),
+ ('"', String, 'string'),
+ (r'_\n', Text), # Line continuation (must be before Name)
+ (uni_name + '[%&@!#$]?', Name),
+ ('#.*?#', Literal.Date),
+ (r'(\d+\.\d*|\d*\.\d+)(F[+-]?[0-9]+)?', Number.Float),
+ (r'\d+([SILDFR]|US|UI|UL)?', Number.Integer),
+ (r'&H[0-9a-f]+([SILDFR]|US|UI|UL)?', Number.Integer),
+ (r'&O[0-7]+([SILDFR]|US|UI|UL)?', Number.Integer),
+ ],
+ 'string': [
+ (r'""', String),
+ (r'"C?', String, '#pop'),
+ (r'[^"]+', String),
+ ],
+ 'dim': [
+ (uni_name, Name.Variable, '#pop'),
+ default('#pop'), # any other syntax
+ ],
+ 'funcname': [
+ (uni_name, Name.Function, '#pop'),
+ ],
+ 'classname': [
+ (uni_name, Name.Class, '#pop'),
+ ],
+ 'namespace': [
+ (uni_name, Name.Namespace),
+ (r'\.', Name.Namespace),
+ default('#pop'),
+ ],
+ 'end': [
+ (r'\s+', Text),
+ (r'(Function|Sub|Property|Class|Structure|Enum|Module|Namespace)\b',
+ Keyword, '#pop'),
+ default('#pop'),
+ ]
+ }
+
+ def analyse_text(text):
+ if re.search(r'^\s*(#If|Module|Namespace)', text, re.MULTILINE):
+ return 0.5
+
+
+class GenericAspxLexer(RegexLexer):
+ """
+ Lexer for ASP.NET pages.
+ """
+
+ name = 'aspx-gen'
+ filenames = []
+ mimetypes = []
+
+ flags = re.DOTALL
+
+ tokens = {
+ 'root': [
+ (r'(<%[@=#]?)(.*?)(%>)', bygroups(Name.Tag, Other, Name.Tag)),
+ (r'(<script.*?>)(.*?)(</script>)', bygroups(using(XmlLexer),
+ Other,
+ using(XmlLexer))),
+ (r'(.+?)(?=<)', using(XmlLexer)),
+ (r'.+', using(XmlLexer)),
+ ],
+ }
+
+
+# TODO support multiple languages within the same source file
+class CSharpAspxLexer(DelegatingLexer):
+ """
+ Lexer for highlighting C# within ASP.NET pages.
+ """
+
+ name = 'aspx-cs'
+ aliases = ['aspx-cs']
+ filenames = ['*.aspx', '*.asax', '*.ascx', '*.ashx', '*.asmx', '*.axd']
+ mimetypes = []
+
+ def __init__(self, **options):
+ super(CSharpAspxLexer, self).__init__(CSharpLexer, GenericAspxLexer,
+ **options)
+
+ def analyse_text(text):
+ if re.search(r'Page\s*Language="C#"', text, re.I) is not None:
+ return 0.2
+ elif re.search(r'script[^>]+language=["\']C#', text, re.I) is not None:
+ return 0.15
+
+
+class VbNetAspxLexer(DelegatingLexer):
+ """
+ Lexer for highlighting Visual Basic.net within ASP.NET pages.
+ """
+
+ name = 'aspx-vb'
+ aliases = ['aspx-vb']
+ filenames = ['*.aspx', '*.asax', '*.ascx', '*.ashx', '*.asmx', '*.axd']
+ mimetypes = []
+
+ def __init__(self, **options):
+ super(VbNetAspxLexer, self).__init__(VbNetLexer, GenericAspxLexer,
+ **options)
+
+ def analyse_text(text):
+ if re.search(r'Page\s*Language="Vb"', text, re.I) is not None:
+ return 0.2
+ elif re.search(r'script[^>]+language=["\']vb', text, re.I) is not None:
+ return 0.15
+
+
+# Very close to functional.OcamlLexer
+class FSharpLexer(RegexLexer):
+ """
+ For the F# language (version 3.0).
+
+ AAAAACK Strings
+ http://research.microsoft.com/en-us/um/cambridge/projects/fsharp/manual/spec.html#_Toc335818775
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'FSharp'
+ aliases = ['fsharp']
+ filenames = ['*.fs', '*.fsi']
+ mimetypes = ['text/x-fsharp']
+
+ keywords = [
+ 'abstract', 'as', 'assert', 'base', 'begin', 'class', 'default',
+ 'delegate', 'do!', 'do', 'done', 'downcast', 'downto', 'elif', 'else',
+ 'end', 'exception', 'extern', 'false', 'finally', 'for', 'function',
+ 'fun', 'global', 'if', 'inherit', 'inline', 'interface', 'internal',
+ 'in', 'lazy', 'let!', 'let', 'match', 'member', 'module', 'mutable',
+ 'namespace', 'new', 'null', 'of', 'open', 'override', 'private', 'public',
+ 'rec', 'return!', 'return', 'select', 'static', 'struct', 'then', 'to',
+ 'true', 'try', 'type', 'upcast', 'use!', 'use', 'val', 'void', 'when',
+ 'while', 'with', 'yield!', 'yield',
+ ]
+ # Reserved words; cannot hurt to color them as keywords too.
+ keywords += [
+ 'atomic', 'break', 'checked', 'component', 'const', 'constraint',
+ 'constructor', 'continue', 'eager', 'event', 'external', 'fixed',
+ 'functor', 'include', 'method', 'mixin', 'object', 'parallel',
+ 'process', 'protected', 'pure', 'sealed', 'tailcall', 'trait',
+ 'virtual', 'volatile',
+ ]
+ keyopts = [
+ '!=', '#', '&&', '&', '\(', '\)', '\*', '\+', ',', '-\.',
+ '->', '-', '\.\.', '\.', '::', ':=', ':>', ':', ';;', ';', '<-',
+ '<\]', '<', '>\]', '>', '\?\?', '\?', '\[<', '\[\|', '\[', '\]',
+ '_', '`', '\{', '\|\]', '\|', '\}', '~', '<@@', '<@', '=', '@>', '@@>',
+ ]
+
+ operators = r'[!$%&*+\./:<=>?@^|~-]'
+ word_operators = ['and', 'or', 'not']
+ prefix_syms = r'[!?~]'
+ infix_syms = r'[=<>@^|&+\*/$%-]'
+ primitives = [
+ 'sbyte', 'byte', 'char', 'nativeint', 'unativeint', 'float32', 'single',
+ 'float', 'double', 'int8', 'uint8', 'int16', 'uint16', 'int32',
+ 'uint32', 'int64', 'uint64', 'decimal', 'unit', 'bool', 'string',
+ 'list', 'exn', 'obj', 'enum',
+ ]
+
+ # See http://msdn.microsoft.com/en-us/library/dd233181.aspx and/or
+ # http://fsharp.org/about/files/spec.pdf for reference. Good luck.
+
+ tokens = {
+ 'escape-sequence': [
+ (r'\\[\\"\'ntbrafv]', String.Escape),
+ (r'\\[0-9]{3}', String.Escape),
+ (r'\\u[0-9a-fA-F]{4}', String.Escape),
+ (r'\\U[0-9a-fA-F]{8}', String.Escape),
+ ],
+ 'root': [
+ (r'\s+', Text),
+ (r'\(\)|\[\]', Name.Builtin.Pseudo),
+ (r'\b(?<!\.)([A-Z][\w\']*)(?=\s*\.)',
+ Name.Namespace, 'dotted'),
+ (r'\b([A-Z][\w\']*)', Name),
+ (r'///.*?\n', String.Doc),
+ (r'//.*?\n', Comment.Single),
+ (r'\(\*(?!\))', Comment, 'comment'),
+
+ (r'@"', String, 'lstring'),
+ (r'"""', String, 'tqs'),
+ (r'"', String, 'string'),
+
+ (r'\b(open|module)(\s+)([\w.]+)',
+ bygroups(Keyword, Text, Name.Namespace)),
+ (r'\b(let!?)(\s+)(\w+)',
+ bygroups(Keyword, Text, Name.Variable)),
+ (r'\b(type)(\s+)(\w+)',
+ bygroups(Keyword, Text, Name.Class)),
+ (r'\b(member|override)(\s+)(\w+)(\.)(\w+)',
+ bygroups(Keyword, Text, Name, Punctuation, Name.Function)),
+ (r'\b(%s)\b' % '|'.join(keywords), Keyword),
+ (r'``([^`\n\r\t]|`[^`\n\r\t])+``', Name),
+ (r'(%s)' % '|'.join(keyopts), Operator),
+ (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator),
+ (r'\b(%s)\b' % '|'.join(word_operators), Operator.Word),
+ (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type),
+ (r'#[ \t]*(if|endif|else|line|nowarn|light|\d+)\b.*?\n',
+ Comment.Preproc),
+
+ (r"[^\W\d][\w']*", Name),
+
+ (r'\d[\d_]*[uU]?[yslLnQRZINGmM]?', Number.Integer),
+ (r'0[xX][\da-fA-F][\da-fA-F_]*[uU]?[yslLn]?[fF]?', Number.Hex),
+ (r'0[oO][0-7][0-7_]*[uU]?[yslLn]?', Number.Oct),
+ (r'0[bB][01][01_]*[uU]?[yslLn]?', Number.Bin),
+ (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)[fFmM]?',
+ Number.Float),
+
+ (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'B?",
+ String.Char),
+ (r"'.'", String.Char),
+ (r"'", Keyword), # a stray quote is another syntax element
+
+ (r'@?"', String.Double, 'string'),
+
+ (r'[~?][a-z][\w\']*:', Name.Variable),
+ ],
+ 'dotted': [
+ (r'\s+', Text),
+ (r'\.', Punctuation),
+ (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace),
+ (r'[A-Z][\w\']*', Name, '#pop'),
+ (r'[a-z_][\w\']*', Name, '#pop'),
+ # e.g. dictionary index access
+ default('#pop'),
+ ],
+ 'comment': [
+ (r'[^(*)@"]+', Comment),
+ (r'\(\*', Comment, '#push'),
+ (r'\*\)', Comment, '#pop'),
+ # comments cannot be closed within strings in comments
+ (r'@"', String, 'lstring'),
+ (r'"""', String, 'tqs'),
+ (r'"', String, 'string'),
+ (r'[(*)@]', Comment),
+ ],
+ 'string': [
+ (r'[^\\"]+', String),
+ include('escape-sequence'),
+ (r'\\\n', String),
+ (r'\n', String), # newlines are allowed in any string
+ (r'"B?', String, '#pop'),
+ ],
+ 'lstring': [
+ (r'[^"]+', String),
+ (r'\n', String),
+ (r'""', String),
+ (r'"B?', String, '#pop'),
+ ],
+ 'tqs': [
+ (r'[^"]+', String),
+ (r'\n', String),
+ (r'"""B?', String, '#pop'),
+ (r'"', String),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/dsls.py b/wandb/vendor/pygments/lexers/dsls.py
new file mode 100644
index 0000000000000000000000000000000000000000..a1426bd6020890373f76b368a54c049968e8d573
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/dsls.py
@@ -0,0 +1,878 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.dsls
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for various domain-specific languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import ExtendedRegexLexer, RegexLexer, bygroups, words, \
+ include, default, this, using, combined
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Literal, Whitespace
+
+__all__ = ['ProtoBufLexer', 'BroLexer', 'PuppetLexer', 'RslLexer',
+ 'MscgenLexer', 'VGLLexer', 'AlloyLexer', 'PanLexer',
+ 'CrmshLexer', 'ThriftLexer', 'FlatlineLexer', 'SnowballLexer']
+
+
+class ProtoBufLexer(RegexLexer):
+ """
+ Lexer for `Protocol Buffer <http://code.google.com/p/protobuf/>`_
+ definition files.
+
+ .. versionadded:: 1.4
+ """
+
+ name = 'Protocol Buffer'
+ aliases = ['protobuf', 'proto']
+ filenames = ['*.proto']
+
+ tokens = {
+ 'root': [
+ (r'[ \t]+', Text),
+ (r'[,;{}\[\]()<>]', Punctuation),
+ (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single),
+ (r'/(\\\n)?\*(.|\n)*?\*(\\\n)?/', Comment.Multiline),
+ (words((
+ 'import', 'option', 'optional', 'required', 'repeated', 'default',
+ 'packed', 'ctype', 'extensions', 'to', 'max', 'rpc', 'returns',
+ 'oneof'), prefix=r'\b', suffix=r'\b'),
+ Keyword),
+ (words((
+ 'int32', 'int64', 'uint32', 'uint64', 'sint32', 'sint64',
+ 'fixed32', 'fixed64', 'sfixed32', 'sfixed64',
+ 'float', 'double', 'bool', 'string', 'bytes'), suffix=r'\b'),
+ Keyword.Type),
+ (r'(true|false)\b', Keyword.Constant),
+ (r'(package)(\s+)', bygroups(Keyword.Namespace, Text), 'package'),
+ (r'(message|extend)(\s+)',
+ bygroups(Keyword.Declaration, Text), 'message'),
+ (r'(enum|group|service)(\s+)',
+ bygroups(Keyword.Declaration, Text), 'type'),
+ (r'\".*?\"', String),
+ (r'\'.*?\'', String),
+ (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float),
+ (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+ (r'(\-?(inf|nan))\b', Number.Float),
+ (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex),
+ (r'0[0-7]+[LlUu]*', Number.Oct),
+ (r'\d+[LlUu]*', Number.Integer),
+ (r'[+-=]', Operator),
+ (r'([a-zA-Z_][\w.]*)([ \t]*)(=)',
+ bygroups(Name.Attribute, Text, Operator)),
+ ('[a-zA-Z_][\w.]*', Name),
+ ],
+ 'package': [
+ (r'[a-zA-Z_]\w*', Name.Namespace, '#pop'),
+ default('#pop'),
+ ],
+ 'message': [
+ (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
+ default('#pop'),
+ ],
+ 'type': [
+ (r'[a-zA-Z_]\w*', Name, '#pop'),
+ default('#pop'),
+ ],
+ }
+
+
+class ThriftLexer(RegexLexer):
+ """
+ For `Thrift <https://thrift.apache.org/>`__ interface definitions.
+
+ .. versionadded:: 2.1
+ """
+ name = 'Thrift'
+ aliases = ['thrift']
+ filenames = ['*.thrift']
+ mimetypes = ['application/x-thrift']
+
+ tokens = {
+ 'root': [
+ include('whitespace'),
+ include('comments'),
+ (r'"', String.Double, combined('stringescape', 'dqs')),
+ (r'\'', String.Single, combined('stringescape', 'sqs')),
+ (r'(namespace)(\s+)',
+ bygroups(Keyword.Namespace, Text.Whitespace), 'namespace'),
+ (r'(enum|union|struct|service|exception)(\s+)',
+ bygroups(Keyword.Declaration, Text.Whitespace), 'class'),
+ (r'((?:(?:[^\W\d]|\$)[\w.\[\]$<>]*\s+)+?)' # return arguments
+ r'((?:[^\W\d]|\$)[\w$]*)' # method name
+ r'(\s*)(\()', # signature start
+ bygroups(using(this), Name.Function, Text, Operator)),
+ include('keywords'),
+ include('numbers'),
+ (r'[&=]', Operator),
+ (r'[:;,{}()<>\[\]]', Punctuation),
+ (r'[a-zA-Z_](\.\w|\w)*', Name),
+ ],
+ 'whitespace': [
+ (r'\n', Text.Whitespace),
+ (r'\s+', Text.Whitespace),
+ ],
+ 'comments': [
+ (r'#.*$', Comment),
+ (r'//.*?\n', Comment),
+ (r'/\*[\w\W]*?\*/', Comment.Multiline),
+ ],
+ 'stringescape': [
+ (r'\\([\\nrt"\'])', String.Escape),
+ ],
+ 'dqs': [
+ (r'"', String.Double, '#pop'),
+ (r'[^\\"\n]+', String.Double),
+ ],
+ 'sqs': [
+ (r"'", String.Single, '#pop'),
+ (r'[^\\\'\n]+', String.Single),
+ ],
+ 'namespace': [
+ (r'[a-z*](\.\w|\w)*', Name.Namespace, '#pop'),
+ default('#pop'),
+ ],
+ 'class': [
+ (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
+ default('#pop'),
+ ],
+ 'keywords': [
+ (r'(async|oneway|extends|throws|required|optional)\b', Keyword),
+ (r'(true|false)\b', Keyword.Constant),
+ (r'(const|typedef)\b', Keyword.Declaration),
+ (words((
+ 'cpp_namespace', 'cpp_include', 'cpp_type', 'java_package',
+ 'cocoa_prefix', 'csharp_namespace', 'delphi_namespace',
+ 'php_namespace', 'py_module', 'perl_package',
+ 'ruby_namespace', 'smalltalk_category', 'smalltalk_prefix',
+ 'xsd_all', 'xsd_optional', 'xsd_nillable', 'xsd_namespace',
+ 'xsd_attrs', 'include'), suffix=r'\b'),
+ Keyword.Namespace),
+ (words((
+ 'void', 'bool', 'byte', 'i16', 'i32', 'i64', 'double',
+ 'string', 'binary', 'map', 'list', 'set', 'slist',
+ 'senum'), suffix=r'\b'),
+ Keyword.Type),
+ (words((
+ 'BEGIN', 'END', '__CLASS__', '__DIR__', '__FILE__',
+ '__FUNCTION__', '__LINE__', '__METHOD__', '__NAMESPACE__',
+ 'abstract', 'alias', 'and', 'args', 'as', 'assert', 'begin',
+ 'break', 'case', 'catch', 'class', 'clone', 'continue',
+ 'declare', 'def', 'default', 'del', 'delete', 'do', 'dynamic',
+ 'elif', 'else', 'elseif', 'elsif', 'end', 'enddeclare',
+ 'endfor', 'endforeach', 'endif', 'endswitch', 'endwhile',
+ 'ensure', 'except', 'exec', 'finally', 'float', 'for',
+ 'foreach', 'function', 'global', 'goto', 'if', 'implements',
+ 'import', 'in', 'inline', 'instanceof', 'interface', 'is',
+ 'lambda', 'module', 'native', 'new', 'next', 'nil', 'not',
+ 'or', 'pass', 'public', 'print', 'private', 'protected',
+ 'raise', 'redo', 'rescue', 'retry', 'register', 'return',
+ 'self', 'sizeof', 'static', 'super', 'switch', 'synchronized',
+ 'then', 'this', 'throw', 'transient', 'try', 'undef',
+ 'unless', 'unsigned', 'until', 'use', 'var', 'virtual',
+ 'volatile', 'when', 'while', 'with', 'xor', 'yield'),
+ prefix=r'\b', suffix=r'\b'),
+ Keyword.Reserved),
+ ],
+ 'numbers': [
+ (r'[+-]?(\d+\.\d+([eE][+-]?\d+)?|\.?\d+[eE][+-]?\d+)', Number.Float),
+ (r'[+-]?0x[0-9A-Fa-f]+', Number.Hex),
+ (r'[+-]?[0-9]+', Number.Integer),
+ ],
+ }
+
+
+class BroLexer(RegexLexer):
+ """
+ For `Bro <http://bro-ids.org/>`_ scripts.
+
+ .. versionadded:: 1.5
+ """
+ name = 'Bro'
+ aliases = ['bro']
+ filenames = ['*.bro']
+
+ _hex = r'[0-9a-fA-F_]'
+ _float = r'((\d*\.?\d+)|(\d+\.?\d*))([eE][-+]?\d+)?'
+ _h = r'[A-Za-z0-9][-A-Za-z0-9]*'
+
+ tokens = {
+ 'root': [
+ # Whitespace
+ (r'^@.*?\n', Comment.Preproc),
+ (r'#.*?\n', Comment.Single),
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'\\\n', Text),
+ # Keywords
+ (r'(add|alarm|break|case|const|continue|delete|do|else|enum|event'
+ r'|export|for|function|if|global|hook|local|module|next'
+ r'|of|print|redef|return|schedule|switch|type|when|while)\b', Keyword),
+ (r'(addr|any|bool|count|counter|double|file|int|interval|net'
+ r'|pattern|port|record|set|string|subnet|table|time|timer'
+ r'|vector)\b', Keyword.Type),
+ (r'(T|F)\b', Keyword.Constant),
+ (r'(&)((?:add|delete|expire)_func|attr|(?:create|read|write)_expire'
+ r'|default|disable_print_hook|raw_output|encrypt|group|log'
+ r'|mergeable|optional|persistent|priority|redef'
+ r'|rotate_(?:interval|size)|synchronized)\b',
+ bygroups(Punctuation, Keyword)),
+ (r'\s+module\b', Keyword.Namespace),
+ # Addresses, ports and networks
+ (r'\d+/(tcp|udp|icmp|unknown)\b', Number),
+ (r'(\d+\.){3}\d+', Number),
+ (r'(' + _hex + r'){7}' + _hex, Number),
+ (r'0x' + _hex + r'(' + _hex + r'|:)*::(' + _hex + r'|:)*', Number),
+ (r'((\d+|:)(' + _hex + r'|:)*)?::(' + _hex + r'|:)*', Number),
+ (r'(\d+\.\d+\.|(\d+\.){2}\d+)', Number),
+ # Hostnames
+ (_h + r'(\.' + _h + r')+', String),
+ # Numeric
+ (_float + r'\s+(day|hr|min|sec|msec|usec)s?\b', Literal.Date),
+ (r'0[xX]' + _hex, Number.Hex),
+ (_float, Number.Float),
+ (r'\d+', Number.Integer),
+ (r'/', String.Regex, 'regex'),
+ (r'"', String, 'string'),
+ # Operators
+ (r'[!%*/+:<=>?~|-]', Operator),
+ (r'([-+=&|]{2}|[+=!><-]=)', Operator),
+ (r'(in|match)\b', Operator.Word),
+ (r'[{}()\[\]$.,;]', Punctuation),
+ # Identfier
+ (r'([_a-zA-Z]\w*)(::)', bygroups(Name, Name.Namespace)),
+ (r'[a-zA-Z_]\w*', Name)
+ ],
+ 'string': [
+ (r'"', String, '#pop'),
+ (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+ (r'[^\\"\n]+', String),
+ (r'\\\n', String),
+ (r'\\', String)
+ ],
+ 'regex': [
+ (r'/', String.Regex, '#pop'),
+ (r'\\[\\nt/]', String.Regex), # String.Escape is too intense here.
+ (r'[^\\/\n]+', String.Regex),
+ (r'\\\n', String.Regex),
+ (r'\\', String.Regex)
+ ]
+ }
+
+
+class PuppetLexer(RegexLexer):
+ """
+ For `Puppet <http://puppetlabs.com/>`__ configuration DSL.
+
+ .. versionadded:: 1.6
+ """
+ name = 'Puppet'
+ aliases = ['puppet']
+ filenames = ['*.pp']
+
+ tokens = {
+ 'root': [
+ include('comments'),
+ include('keywords'),
+ include('names'),
+ include('numbers'),
+ include('operators'),
+ include('strings'),
+
+ (r'[]{}:(),;[]', Punctuation),
+ (r'[^\S\n]+', Text),
+ ],
+
+ 'comments': [
+ (r'\s*#.*$', Comment),
+ (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+ ],
+
+ 'operators': [
+ (r'(=>|\?|<|>|=|\+|-|/|\*|~|!|\|)', Operator),
+ (r'(in|and|or|not)\b', Operator.Word),
+ ],
+
+ 'names': [
+ ('[a-zA-Z_]\w*', Name.Attribute),
+ (r'(\$\S+)(\[)(\S+)(\])', bygroups(Name.Variable, Punctuation,
+ String, Punctuation)),
+ (r'\$\S+', Name.Variable),
+ ],
+
+ 'numbers': [
+ # Copypasta from the Python lexer
+ (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
+ (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
+ (r'0[0-7]+j?', Number.Oct),
+ (r'0[xX][a-fA-F0-9]+', Number.Hex),
+ (r'\d+L', Number.Integer.Long),
+ (r'\d+j?', Number.Integer)
+ ],
+
+ 'keywords': [
+ # Left out 'group' and 'require'
+ # Since they're often used as attributes
+ (words((
+ 'absent', 'alert', 'alias', 'audit', 'augeas', 'before', 'case',
+ 'check', 'class', 'computer', 'configured', 'contained',
+ 'create_resources', 'crit', 'cron', 'debug', 'default',
+ 'define', 'defined', 'directory', 'else', 'elsif', 'emerg',
+ 'err', 'exec', 'extlookup', 'fail', 'false', 'file',
+ 'filebucket', 'fqdn_rand', 'generate', 'host', 'if', 'import',
+ 'include', 'info', 'inherits', 'inline_template', 'installed',
+ 'interface', 'k5login', 'latest', 'link', 'loglevel',
+ 'macauthorization', 'mailalias', 'maillist', 'mcx', 'md5',
+ 'mount', 'mounted', 'nagios_command', 'nagios_contact',
+ 'nagios_contactgroup', 'nagios_host', 'nagios_hostdependency',
+ 'nagios_hostescalation', 'nagios_hostextinfo', 'nagios_hostgroup',
+ 'nagios_service', 'nagios_servicedependency', 'nagios_serviceescalation',
+ 'nagios_serviceextinfo', 'nagios_servicegroup', 'nagios_timeperiod',
+ 'node', 'noop', 'notice', 'notify', 'package', 'present', 'purged',
+ 'realize', 'regsubst', 'resources', 'role', 'router', 'running',
+ 'schedule', 'scheduled_task', 'search', 'selboolean', 'selmodule',
+ 'service', 'sha1', 'shellquote', 'split', 'sprintf',
+ 'ssh_authorized_key', 'sshkey', 'stage', 'stopped', 'subscribe',
+ 'tag', 'tagged', 'template', 'tidy', 'true', 'undef', 'unmounted',
+ 'user', 'versioncmp', 'vlan', 'warning', 'yumrepo', 'zfs', 'zone',
+ 'zpool'), prefix='(?i)', suffix=r'\b'),
+ Keyword),
+ ],
+
+ 'strings': [
+ (r'"([^"])*"', String),
+ (r"'(\\'|[^'])*'", String),
+ ],
+
+ }
+
+
+class RslLexer(RegexLexer):
+ """
+ `RSL <http://en.wikipedia.org/wiki/RAISE>`_ is the formal specification
+ language used in RAISE (Rigorous Approach to Industrial Software Engineering)
+ method.
+
+ .. versionadded:: 2.0
+ """
+ name = 'RSL'
+ aliases = ['rsl']
+ filenames = ['*.rsl']
+ mimetypes = ['text/rsl']
+
+ flags = re.MULTILINE | re.DOTALL
+
+ tokens = {
+ 'root': [
+ (words((
+ 'Bool', 'Char', 'Int', 'Nat', 'Real', 'Text', 'Unit', 'abs',
+ 'all', 'always', 'any', 'as', 'axiom', 'card', 'case', 'channel',
+ 'chaos', 'class', 'devt_relation', 'dom', 'elems', 'else', 'elif',
+ 'end', 'exists', 'extend', 'false', 'for', 'hd', 'hide', 'if',
+ 'in', 'is', 'inds', 'initialise', 'int', 'inter', 'isin', 'len',
+ 'let', 'local', 'ltl_assertion', 'object', 'of', 'out', 'post',
+ 'pre', 'read', 'real', 'rng', 'scheme', 'skip', 'stop', 'swap',
+ 'then', 'theory', 'test_case', 'tl', 'transition_system', 'true',
+ 'type', 'union', 'until', 'use', 'value', 'variable', 'while',
+ 'with', 'write', '~isin', '-inflist', '-infset', '-list',
+ '-set'), prefix=r'\b', suffix=r'\b'),
+ Keyword),
+ (r'(variable|value)\b', Keyword.Declaration),
+ (r'--.*?\n', Comment),
+ (r'<:.*?:>', Comment),
+ (r'\{!.*?!\}', Comment),
+ (r'/\*.*?\*/', Comment),
+ (r'^[ \t]*([\w]+)[ \t]*:[^:]', Name.Function),
+ (r'(^[ \t]*)([\w]+)([ \t]*\([\w\s,]*\)[ \t]*)(is|as)',
+ bygroups(Text, Name.Function, Text, Keyword)),
+ (r'\b[A-Z]\w*\b', Keyword.Type),
+ (r'(true|false)\b', Keyword.Constant),
+ (r'".*"', String),
+ (r'\'.\'', String.Char),
+ (r'(><|->|-m->|/\\|<=|<<=|<\.|\|\||\|\^\||-~->|-~m->|\\/|>=|>>|'
+ r'\.>|\+\+|-\\|<->|=>|:-|~=|\*\*|<<|>>=|\+>|!!|\|=\||#)',
+ Operator),
+ (r'[0-9]+\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-f]+', Number.Hex),
+ (r'[0-9]+', Number.Integer),
+ (r'.', Text),
+ ],
+ }
+
+ def analyse_text(text):
+ """
+ Check for the most common text in the beginning of a RSL file.
+ """
+ if re.search(r'scheme\s*.*?=\s*class\s*type', text, re.I) is not None:
+ return 1.0
+
+
+class MscgenLexer(RegexLexer):
+ """
+ For `Mscgen <http://www.mcternan.me.uk/mscgen/>`_ files.
+
+ .. versionadded:: 1.6
+ """
+ name = 'Mscgen'
+ aliases = ['mscgen', 'msc']
+ filenames = ['*.msc']
+
+ _var = r'(\w+|"(?:\\"|[^"])*")'
+
+ tokens = {
+ 'root': [
+ (r'msc\b', Keyword.Type),
+ # Options
+ (r'(hscale|HSCALE|width|WIDTH|wordwraparcs|WORDWRAPARCS'
+ r'|arcgradient|ARCGRADIENT)\b', Name.Property),
+ # Operators
+ (r'(abox|ABOX|rbox|RBOX|box|BOX|note|NOTE)\b', Operator.Word),
+ (r'(\.|-|\|){3}', Keyword),
+ (r'(?:-|=|\.|:){2}'
+ r'|<<=>>|<->|<=>|<<>>|<:>'
+ r'|->|=>>|>>|=>|:>|-x|-X'
+ r'|<-|<<=|<<|<=|<:|x-|X-|=', Operator),
+ # Names
+ (r'\*', Name.Builtin),
+ (_var, Name.Variable),
+ # Other
+ (r'\[', Punctuation, 'attrs'),
+ (r'\{|\}|,|;', Punctuation),
+ include('comments')
+ ],
+ 'attrs': [
+ (r'\]', Punctuation, '#pop'),
+ (_var + r'(\s*)(=)(\s*)' + _var,
+ bygroups(Name.Attribute, Text.Whitespace, Operator, Text.Whitespace,
+ String)),
+ (r',', Punctuation),
+ include('comments')
+ ],
+ 'comments': [
+ (r'(?://|#).*?\n', Comment.Single),
+ (r'/\*(?:.|\n)*?\*/', Comment.Multiline),
+ (r'[ \t\r\n]+', Text.Whitespace)
+ ]
+ }
+
+
+class VGLLexer(RegexLexer):
+ """
+ For `SampleManager VGL <http://www.thermoscientific.com/samplemanager>`_
+ source code.
+
+ .. versionadded:: 1.6
+ """
+ name = 'VGL'
+ aliases = ['vgl']
+ filenames = ['*.rpf']
+
+ flags = re.MULTILINE | re.DOTALL | re.IGNORECASE
+
+ tokens = {
+ 'root': [
+ (r'\{[^}]*\}', Comment.Multiline),
+ (r'declare', Keyword.Constant),
+ (r'(if|then|else|endif|while|do|endwhile|and|or|prompt|object'
+ r'|create|on|line|with|global|routine|value|endroutine|constant'
+ r'|global|set|join|library|compile_option|file|exists|create|copy'
+ r'|delete|enable|windows|name|notprotected)(?! *[=<>.,()])',
+ Keyword),
+ (r'(true|false|null|empty|error|locked)', Keyword.Constant),
+ (r'[~^*#!%&\[\]()<>|+=:;,./?-]', Operator),
+ (r'"[^"]*"', String),
+ (r'(\.)([a-z_$][\w$]*)', bygroups(Operator, Name.Attribute)),
+ (r'[0-9][0-9]*(\.[0-9]+(e[+\-]?[0-9]+)?)?', Number),
+ (r'[a-z_$][\w$]*', Name),
+ (r'[\r\n]+', Text),
+ (r'\s+', Text)
+ ]
+ }
+
+
+class AlloyLexer(RegexLexer):
+ """
+ For `Alloy <http://alloy.mit.edu>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Alloy'
+ aliases = ['alloy']
+ filenames = ['*.als']
+ mimetypes = ['text/x-alloy']
+
+ flags = re.MULTILINE | re.DOTALL
+
+ iden_rex = r'[a-zA-Z_][\w\']*'
+ text_tuple = (r'[^\S\n]+', Text)
+
+ tokens = {
+ 'sig': [
+ (r'(extends)\b', Keyword, '#pop'),
+ (iden_rex, Name),
+ text_tuple,
+ (r',', Punctuation),
+ (r'\{', Operator, '#pop'),
+ ],
+ 'module': [
+ text_tuple,
+ (iden_rex, Name, '#pop'),
+ ],
+ 'fun': [
+ text_tuple,
+ (r'\{', Operator, '#pop'),
+ (iden_rex, Name, '#pop'),
+ ],
+ 'root': [
+ (r'--.*?$', Comment.Single),
+ (r'//.*?$', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ text_tuple,
+ (r'(module|open)(\s+)', bygroups(Keyword.Namespace, Text),
+ 'module'),
+ (r'(sig|enum)(\s+)', bygroups(Keyword.Declaration, Text), 'sig'),
+ (r'(iden|univ|none)\b', Keyword.Constant),
+ (r'(int|Int)\b', Keyword.Type),
+ (r'(this|abstract|extends|set|seq|one|lone|let)\b', Keyword),
+ (r'(all|some|no|sum|disj|when|else)\b', Keyword),
+ (r'(run|check|for|but|exactly|expect|as)\b', Keyword),
+ (r'(and|or|implies|iff|in)\b', Operator.Word),
+ (r'(fun|pred|fact|assert)(\s+)', bygroups(Keyword, Text), 'fun'),
+ (r'!|#|&&|\+\+|<<|>>|>=|<=>|<=|\.|->', Operator),
+ (r'[-+/*%=<>&!^|~{}\[\]().]', Operator),
+ (iden_rex, Name),
+ (r'[:,]', Punctuation),
+ (r'[0-9]+', Number.Integer),
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r'\n', Text),
+ ]
+ }
+
+
+class PanLexer(RegexLexer):
+ """
+ Lexer for `pan <http://github.com/quattor/pan/>`_ source files.
+
+ Based on tcsh lexer.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Pan'
+ aliases = ['pan']
+ filenames = ['*.pan']
+
+ tokens = {
+ 'root': [
+ include('basic'),
+ (r'\(', Keyword, 'paren'),
+ (r'\{', Keyword, 'curly'),
+ include('data'),
+ ],
+ 'basic': [
+ (words((
+ 'if', 'for', 'with', 'else', 'type', 'bind', 'while', 'valid', 'final',
+ 'prefix', 'unique', 'object', 'foreach', 'include', 'template',
+ 'function', 'variable', 'structure', 'extensible', 'declaration'),
+ prefix=r'\b', suffix=r'\s*\b'),
+ Keyword),
+ (words((
+ 'file_contents', 'format', 'index', 'length', 'match', 'matches',
+ 'replace', 'splice', 'split', 'substr', 'to_lowercase', 'to_uppercase',
+ 'debug', 'error', 'traceback', 'deprecated', 'base64_decode',
+ 'base64_encode', 'digest', 'escape', 'unescape', 'append', 'create',
+ 'first', 'nlist', 'key', 'list', 'merge', 'next', 'prepend', 'is_boolean',
+ 'is_defined', 'is_double', 'is_list', 'is_long', 'is_nlist', 'is_null',
+ 'is_number', 'is_property', 'is_resource', 'is_string', 'to_boolean',
+ 'to_double', 'to_long', 'to_string', 'clone', 'delete', 'exists',
+ 'path_exists', 'if_exists', 'return', 'value'),
+ prefix=r'\b', suffix=r'\s*\b'),
+ Name.Builtin),
+ (r'#.*', Comment),
+ (r'\\[\w\W]', String.Escape),
+ (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Text, Operator)),
+ (r'[\[\]{}()=]+', Operator),
+ (r'<<\s*(\'?)\\?(\w+)[\w\W]+?\2', String),
+ (r';', Punctuation),
+ ],
+ 'data': [
+ (r'(?s)"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double),
+ (r"(?s)'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single),
+ (r'\s+', Text),
+ (r'[^=\s\[\]{}()$"\'`\\;#]+', Text),
+ (r'\d+(?= |\Z)', Number),
+ ],
+ 'curly': [
+ (r'\}', Keyword, '#pop'),
+ (r':-', Keyword),
+ (r'\w+', Name.Variable),
+ (r'[^}:"\'`$]+', Punctuation),
+ (r':', Punctuation),
+ include('root'),
+ ],
+ 'paren': [
+ (r'\)', Keyword, '#pop'),
+ include('root'),
+ ],
+ }
+
+
+class CrmshLexer(RegexLexer):
+ """
+ Lexer for `crmsh <http://crmsh.github.io/>`_ configuration files
+ for Pacemaker clusters.
+
+ .. versionadded:: 2.1
+ """
+ name = 'Crmsh'
+ aliases = ['crmsh', 'pcmk']
+ filenames = ['*.crmsh', '*.pcmk']
+ mimetypes = []
+
+ elem = words((
+ 'node', 'primitive', 'group', 'clone', 'ms', 'location',
+ 'colocation', 'order', 'fencing_topology', 'rsc_ticket',
+ 'rsc_template', 'property', 'rsc_defaults',
+ 'op_defaults', 'acl_target', 'acl_group', 'user', 'role',
+ 'tag'), suffix=r'(?![\w#$-])')
+ sub = words((
+ 'params', 'meta', 'operations', 'op', 'rule',
+ 'attributes', 'utilization'), suffix=r'(?![\w#$-])')
+ acl = words(('read', 'write', 'deny'), suffix=r'(?![\w#$-])')
+ bin_rel = words(('and', 'or'), suffix=r'(?![\w#$-])')
+ un_ops = words(('defined', 'not_defined'), suffix=r'(?![\w#$-])')
+ date_exp = words(('in_range', 'date', 'spec', 'in'), suffix=r'(?![\w#$-])')
+ acl_mod = (r'(?:tag|ref|reference|attribute|type|xpath)')
+ bin_ops = (r'(?:lt|gt|lte|gte|eq|ne)')
+ val_qual = (r'(?:string|version|number)')
+ rsc_role_action = (r'(?:Master|Started|Slave|Stopped|'
+ r'start|promote|demote|stop)')
+
+ tokens = {
+ 'root': [
+ (r'^#.*\n?', Comment),
+ # attr=value (nvpair)
+ (r'([\w#$-]+)(=)("(?:""|[^"])*"|\S+)',
+ bygroups(Name.Attribute, Punctuation, String)),
+ # need this construct, otherwise numeric node ids
+ # are matched as scores
+ # elem id:
+ (r'(node)(\s+)([\w#$-]+)(:)',
+ bygroups(Keyword, Whitespace, Name, Punctuation)),
+ # scores
+ (r'([+-]?([0-9]+|inf)):', Number),
+ # keywords (elements and other)
+ (elem, Keyword),
+ (sub, Keyword),
+ (acl, Keyword),
+ # binary operators
+ (r'(?:%s:)?(%s)(?![\w#$-])' % (val_qual, bin_ops), Operator.Word),
+ # other operators
+ (bin_rel, Operator.Word),
+ (un_ops, Operator.Word),
+ (date_exp, Operator.Word),
+ # builtin attributes (e.g. #uname)
+ (r'#[a-z]+(?![\w#$-])', Name.Builtin),
+ # acl_mod:blah
+ (r'(%s)(:)("(?:""|[^"])*"|\S+)' % acl_mod,
+ bygroups(Keyword, Punctuation, Name)),
+ # rsc_id[:(role|action)]
+ # NB: this matches all other identifiers
+ (r'([\w#$-]+)(?:(:)(%s))?(?![\w#$-])' % rsc_role_action,
+ bygroups(Name, Punctuation, Operator.Word)),
+ # punctuation
+ (r'(\\(?=\n)|[[\](){}/:@])', Punctuation),
+ (r'\s+|\n', Whitespace),
+ ],
+ }
+
+
+class FlatlineLexer(RegexLexer):
+ """
+ Lexer for `Flatline <https://github.com/bigmlcom/flatline>`_ expressions.
+
+ .. versionadded:: 2.2
+ """
+ name = 'Flatline'
+ aliases = ['flatline']
+ filenames = []
+ mimetypes = ['text/x-flatline']
+
+ special_forms = ('let',)
+
+ builtins = (
+ "!=", "*", "+", "-", "<", "<=", "=", ">", ">=", "abs", "acos", "all",
+ "all-but", "all-with-defaults", "all-with-numeric-default", "and",
+ "asin", "atan", "avg", "avg-window", "bin-center", "bin-count", "call",
+ "category-count", "ceil", "cond", "cond-window", "cons", "cos", "cosh",
+ "count", "diff-window", "div", "ensure-value", "ensure-weighted-value",
+ "epoch", "epoch-day", "epoch-fields", "epoch-hour", "epoch-millisecond",
+ "epoch-minute", "epoch-month", "epoch-second", "epoch-weekday",
+ "epoch-year", "exp", "f", "field", "field-prop", "fields", "filter",
+ "first", "floor", "head", "if", "in", "integer", "language", "length",
+ "levenshtein", "linear-regression", "list", "ln", "log", "log10", "map",
+ "matches", "matches?", "max", "maximum", "md5", "mean", "median", "min",
+ "minimum", "missing", "missing-count", "missing?", "missing_count",
+ "mod", "mode", "normalize", "not", "nth", "occurrences", "or",
+ "percentile", "percentile-label", "population", "population-fraction",
+ "pow", "preferred", "preferred?", "quantile-label", "rand", "rand-int",
+ "random-value", "re-quote", "real", "replace", "replace-first", "rest",
+ "round", "row-number", "segment-label", "sha1", "sha256", "sin", "sinh",
+ "sqrt", "square", "standard-deviation", "standard_deviation", "str",
+ "subs", "sum", "sum-squares", "sum-window", "sum_squares", "summary",
+ "summary-no", "summary-str", "tail", "tan", "tanh", "to-degrees",
+ "to-radians", "variance", "vectorize", "weighted-random-value", "window",
+ "winnow", "within-percentiles?", "z-score",
+ )
+
+ valid_name = r'(?!#)[\w!$%*+<=>?/.#-]+'
+
+ tokens = {
+ 'root': [
+ # whitespaces - usually not relevant
+ (r'[,\s]+', Text),
+
+ # numbers
+ (r'-?\d+\.\d+', Number.Float),
+ (r'-?\d+', Number.Integer),
+ (r'0x-?[a-f\d]+', Number.Hex),
+
+ # strings, symbols and characters
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r"\\(.|[a-z]+)", String.Char),
+
+ # expression template placeholder
+ (r'_', String.Symbol),
+
+ # highlight the special forms
+ (words(special_forms, suffix=' '), Keyword),
+
+ # highlight the builtins
+ (words(builtins, suffix=' '), Name.Builtin),
+
+ # the remaining functions
+ (r'(?<=\()' + valid_name, Name.Function),
+
+ # find the remaining variables
+ (valid_name, Name.Variable),
+
+ # parentheses
+ (r'(\(|\))', Punctuation),
+ ],
+ }
+
+
+class SnowballLexer(ExtendedRegexLexer):
+ """
+ Lexer for `Snowball <http://snowballstem.org/>`_ source code.
+
+ .. versionadded:: 2.2
+ """
+
+ name = 'Snowball'
+ aliases = ['snowball']
+ filenames = ['*.sbl']
+
+ _ws = r'\n\r\t '
+
+ def __init__(self, **options):
+ self._reset_stringescapes()
+ ExtendedRegexLexer.__init__(self, **options)
+
+ def _reset_stringescapes(self):
+ self._start = "'"
+ self._end = "'"
+
+ def _string(do_string_first):
+ def callback(lexer, match, ctx):
+ s = match.start()
+ text = match.group()
+ string = re.compile(r'([^%s]*)(.)' % re.escape(lexer._start)).match
+ escape = re.compile(r'([^%s]*)(.)' % re.escape(lexer._end)).match
+ pos = 0
+ do_string = do_string_first
+ while pos < len(text):
+ if do_string:
+ match = string(text, pos)
+ yield s + match.start(1), String.Single, match.group(1)
+ if match.group(2) == "'":
+ yield s + match.start(2), String.Single, match.group(2)
+ ctx.stack.pop()
+ break
+ yield s + match.start(2), String.Escape, match.group(2)
+ pos = match.end()
+ match = escape(text, pos)
+ yield s + match.start(), String.Escape, match.group()
+ if match.group(2) != lexer._end:
+ ctx.stack[-1] = 'escape'
+ break
+ pos = match.end()
+ do_string = True
+ ctx.pos = s + match.end()
+ return callback
+
+ def _stringescapes(lexer, match, ctx):
+ lexer._start = match.group(3)
+ lexer._end = match.group(5)
+ return bygroups(Keyword.Reserved, Text, String.Escape, Text,
+ String.Escape)(lexer, match, ctx)
+
+ tokens = {
+ 'root': [
+ (words(('len', 'lenof'), suffix=r'\b'), Operator.Word),
+ include('root1'),
+ ],
+ 'root1': [
+ (r'[%s]+' % _ws, Text),
+ (r'\d+', Number.Integer),
+ (r"'", String.Single, 'string'),
+ (r'[()]', Punctuation),
+ (r'/\*[\w\W]*?\*/', Comment.Multiline),
+ (r'//.*', Comment.Single),
+ (r'[!*+\-/<=>]=|[-=]>|<[+-]|[$*+\-/<=>?\[\]]', Operator),
+ (words(('as', 'get', 'hex', 'among', 'define', 'decimal',
+ 'backwardmode'), suffix=r'\b'),
+ Keyword.Reserved),
+ (words(('strings', 'booleans', 'integers', 'routines', 'externals',
+ 'groupings'), suffix=r'\b'),
+ Keyword.Reserved, 'declaration'),
+ (words(('do', 'or', 'and', 'for', 'hop', 'non', 'not', 'set', 'try',
+ 'fail', 'goto', 'loop', 'next', 'test', 'true',
+ 'false', 'unset', 'atmark', 'attach', 'delete', 'gopast',
+ 'insert', 'repeat', 'sizeof', 'tomark', 'atleast',
+ 'atlimit', 'reverse', 'setmark', 'tolimit', 'setlimit',
+ 'backwards', 'substring'), suffix=r'\b'),
+ Operator.Word),
+ (words(('size', 'limit', 'cursor', 'maxint', 'minint'),
+ suffix=r'\b'),
+ Name.Builtin),
+ (r'(stringdef\b)([%s]*)([^%s]+)' % (_ws, _ws),
+ bygroups(Keyword.Reserved, Text, String.Escape)),
+ (r'(stringescapes\b)([%s]*)(.)([%s]*)(.)' % (_ws, _ws),
+ _stringescapes),
+ (r'[A-Za-z]\w*', Name),
+ ],
+ 'declaration': [
+ (r'\)', Punctuation, '#pop'),
+ (words(('len', 'lenof'), suffix=r'\b'), Name,
+ ('root1', 'declaration')),
+ include('root1'),
+ ],
+ 'string': [
+ (r"[^']*'", _string(True)),
+ ],
+ 'escape': [
+ (r"[^']*'", _string(False)),
+ ],
+ }
+
+ def get_tokens_unprocessed(self, text=None, context=None):
+ self._reset_stringescapes()
+ return ExtendedRegexLexer.get_tokens_unprocessed(self, text, context)
diff --git a/wandb/vendor/pygments/lexers/dylan.py b/wandb/vendor/pygments/lexers/dylan.py
new file mode 100644
index 0000000000000000000000000000000000000000..f61bb60d162282c73af505ea649f13064ab557c5
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/dylan.py
@@ -0,0 +1,289 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.dylan
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the Dylan language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, bygroups, do_insertions, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Generic, Literal
+
+__all__ = ['DylanLexer', 'DylanConsoleLexer', 'DylanLidLexer']
+
+
+class DylanLexer(RegexLexer):
+ """
+ For the `Dylan <http://www.opendylan.org/>`_ language.
+
+ .. versionadded:: 0.7
+ """
+
+ name = 'Dylan'
+ aliases = ['dylan']
+ filenames = ['*.dylan', '*.dyl', '*.intr']
+ mimetypes = ['text/x-dylan']
+
+ flags = re.IGNORECASE
+
+ builtins = set((
+ 'subclass', 'abstract', 'block', 'concrete', 'constant', 'class',
+ 'compiler-open', 'compiler-sideways', 'domain', 'dynamic',
+ 'each-subclass', 'exception', 'exclude', 'function', 'generic',
+ 'handler', 'inherited', 'inline', 'inline-only', 'instance',
+ 'interface', 'import', 'keyword', 'library', 'macro', 'method',
+ 'module', 'open', 'primary', 'required', 'sealed', 'sideways',
+ 'singleton', 'slot', 'thread', 'variable', 'virtual'))
+
+ keywords = set((
+ 'above', 'afterwards', 'begin', 'below', 'by', 'case', 'cleanup',
+ 'create', 'define', 'else', 'elseif', 'end', 'export', 'finally',
+ 'for', 'from', 'if', 'in', 'let', 'local', 'otherwise', 'rename',
+ 'select', 'signal', 'then', 'to', 'unless', 'until', 'use', 'when',
+ 'while'))
+
+ operators = set((
+ '~', '+', '-', '*', '|', '^', '=', '==', '~=', '~==', '<', '<=',
+ '>', '>=', '&', '|'))
+
+ functions = set((
+ 'abort', 'abs', 'add', 'add!', 'add-method', 'add-new', 'add-new!',
+ 'all-superclasses', 'always', 'any?', 'applicable-method?', 'apply',
+ 'aref', 'aref-setter', 'as', 'as-lowercase', 'as-lowercase!',
+ 'as-uppercase', 'as-uppercase!', 'ash', 'backward-iteration-protocol',
+ 'break', 'ceiling', 'ceiling/', 'cerror', 'check-type', 'choose',
+ 'choose-by', 'complement', 'compose', 'concatenate', 'concatenate-as',
+ 'condition-format-arguments', 'condition-format-string', 'conjoin',
+ 'copy-sequence', 'curry', 'default-handler', 'dimension', 'dimensions',
+ 'direct-subclasses', 'direct-superclasses', 'disjoin', 'do',
+ 'do-handlers', 'element', 'element-setter', 'empty?', 'error', 'even?',
+ 'every?', 'false-or', 'fill!', 'find-key', 'find-method', 'first',
+ 'first-setter', 'floor', 'floor/', 'forward-iteration-protocol',
+ 'function-arguments', 'function-return-values',
+ 'function-specializers', 'gcd', 'generic-function-mandatory-keywords',
+ 'generic-function-methods', 'head', 'head-setter', 'identity',
+ 'initialize', 'instance?', 'integral?', 'intersection',
+ 'key-sequence', 'key-test', 'last', 'last-setter', 'lcm', 'limited',
+ 'list', 'logand', 'logbit?', 'logior', 'lognot', 'logxor', 'make',
+ 'map', 'map-as', 'map-into', 'max', 'member?', 'merge-hash-codes',
+ 'min', 'modulo', 'negative', 'negative?', 'next-method',
+ 'object-class', 'object-hash', 'odd?', 'one-of', 'pair', 'pop',
+ 'pop-last', 'positive?', 'push', 'push-last', 'range', 'rank',
+ 'rcurry', 'reduce', 'reduce1', 'remainder', 'remove', 'remove!',
+ 'remove-duplicates', 'remove-duplicates!', 'remove-key!',
+ 'remove-method', 'replace-elements!', 'replace-subsequence!',
+ 'restart-query', 'return-allowed?', 'return-description',
+ 'return-query', 'reverse', 'reverse!', 'round', 'round/',
+ 'row-major-index', 'second', 'second-setter', 'shallow-copy',
+ 'signal', 'singleton', 'size', 'size-setter', 'slot-initialized?',
+ 'sort', 'sort!', 'sorted-applicable-methods', 'subsequence-position',
+ 'subtype?', 'table-protocol', 'tail', 'tail-setter', 'third',
+ 'third-setter', 'truncate', 'truncate/', 'type-error-expected-type',
+ 'type-error-value', 'type-for-copy', 'type-union', 'union', 'values',
+ 'vector', 'zero?'))
+
+ valid_name = '\\\\?[\\w!&*<>|^$%@\\-+~?/=]+'
+
+ def get_tokens_unprocessed(self, text):
+ for index, token, value in RegexLexer.get_tokens_unprocessed(self, text):
+ if token is Name:
+ lowercase_value = value.lower()
+ if lowercase_value in self.builtins:
+ yield index, Name.Builtin, value
+ continue
+ if lowercase_value in self.keywords:
+ yield index, Keyword, value
+ continue
+ if lowercase_value in self.functions:
+ yield index, Name.Builtin, value
+ continue
+ if lowercase_value in self.operators:
+ yield index, Operator, value
+ continue
+ yield index, token, value
+
+ tokens = {
+ 'root': [
+ # Whitespace
+ (r'\s+', Text),
+
+ # single line comment
+ (r'//.*?\n', Comment.Single),
+
+ # lid header
+ (r'([a-z0-9-]+)(:)([ \t]*)(.*(?:\n[ \t].+)*)',
+ bygroups(Name.Attribute, Operator, Text, String)),
+
+ default('code') # no header match, switch to code
+ ],
+ 'code': [
+ # Whitespace
+ (r'\s+', Text),
+
+ # single line comment
+ (r'//.*?\n', Comment.Single),
+
+ # multi-line comment
+ (r'/\*', Comment.Multiline, 'comment'),
+
+ # strings and characters
+ (r'"', String, 'string'),
+ (r"'(\\.|\\[0-7]{1,3}|\\x[a-f0-9]{1,2}|[^\\\'\n])'", String.Char),
+
+ # binary integer
+ (r'#b[01]+', Number.Bin),
+
+ # octal integer
+ (r'#o[0-7]+', Number.Oct),
+
+ # floating point
+ (r'[-+]?(\d*\.\d+(e[-+]?\d+)?|\d+(\.\d*)?e[-+]?\d+)', Number.Float),
+
+ # decimal integer
+ (r'[-+]?\d+', Number.Integer),
+
+ # hex integer
+ (r'#x[0-9a-f]+', Number.Hex),
+
+ # Macro parameters
+ (r'(\?' + valid_name + ')(:)'
+ r'(token|name|variable|expression|body|case-body|\*)',
+ bygroups(Name.Tag, Operator, Name.Builtin)),
+ (r'(\?)(:)(token|name|variable|expression|body|case-body|\*)',
+ bygroups(Name.Tag, Operator, Name.Builtin)),
+ (r'\?' + valid_name, Name.Tag),
+
+ # Punctuation
+ (r'(=>|::|#\(|#\[|##|\?\?|\?=|\?|[(){}\[\],.;])', Punctuation),
+
+ # Most operators are picked up as names and then re-flagged.
+ # This one isn't valid in a name though, so we pick it up now.
+ (r':=', Operator),
+
+ # Pick up #t / #f before we match other stuff with #.
+ (r'#[tf]', Literal),
+
+ # #"foo" style keywords
+ (r'#"', String.Symbol, 'keyword'),
+
+ # #rest, #key, #all-keys, etc.
+ (r'#[a-z0-9-]+', Keyword),
+
+ # required-init-keyword: style keywords.
+ (valid_name + ':', Keyword),
+
+ # class names
+ (r'<' + valid_name + '>', Name.Class),
+
+ # define variable forms.
+ (r'\*' + valid_name + '\*', Name.Variable.Global),
+
+ # define constant forms.
+ (r'\$' + valid_name, Name.Constant),
+
+ # everything else. We re-flag some of these in the method above.
+ (valid_name, Name),
+ ],
+ 'comment': [
+ (r'[^*/]', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline)
+ ],
+ 'keyword': [
+ (r'"', String.Symbol, '#pop'),
+ (r'[^\\"]+', String.Symbol), # all other characters
+ ],
+ 'string': [
+ (r'"', String, '#pop'),
+ (r'\\([\\abfnrtv"\']|x[a-f0-9]{2,4}|[0-7]{1,3})', String.Escape),
+ (r'[^\\"\n]+', String), # all other characters
+ (r'\\\n', String), # line continuation
+ (r'\\', String), # stray backslash
+ ]
+ }
+
+
+class DylanLidLexer(RegexLexer):
+ """
+ For Dylan LID (Library Interchange Definition) files.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'DylanLID'
+ aliases = ['dylan-lid', 'lid']
+ filenames = ['*.lid', '*.hdp']
+ mimetypes = ['text/x-dylan-lid']
+
+ flags = re.IGNORECASE
+
+ tokens = {
+ 'root': [
+ # Whitespace
+ (r'\s+', Text),
+
+ # single line comment
+ (r'//.*?\n', Comment.Single),
+
+ # lid header
+ (r'(.*?)(:)([ \t]*)(.*(?:\n[ \t].+)*)',
+ bygroups(Name.Attribute, Operator, Text, String)),
+ ]
+ }
+
+
+class DylanConsoleLexer(Lexer):
+ """
+ For Dylan interactive console output like:
+
+ .. sourcecode:: dylan-console
+
+ ? let a = 1;
+ => 1
+ ? a
+ => 1
+
+ This is based on a copy of the RubyConsoleLexer.
+
+ .. versionadded:: 1.6
+ """
+ name = 'Dylan session'
+ aliases = ['dylan-console', 'dylan-repl']
+ filenames = ['*.dylan-console']
+ mimetypes = ['text/x-dylan-console']
+
+ _line_re = re.compile('.*?\n')
+ _prompt_re = re.compile('\?| ')
+
+ def get_tokens_unprocessed(self, text):
+ dylexer = DylanLexer(**self.options)
+
+ curcode = ''
+ insertions = []
+ for match in self._line_re.finditer(text):
+ line = match.group()
+ m = self._prompt_re.match(line)
+ if m is not None:
+ end = m.end()
+ insertions.append((len(curcode),
+ [(0, Generic.Prompt, line[:end])]))
+ curcode += line[end:]
+ else:
+ if curcode:
+ for item in do_insertions(insertions,
+ dylexer.get_tokens_unprocessed(curcode)):
+ yield item
+ curcode = ''
+ insertions = []
+ yield match.start(), Generic.Output, line
+ if curcode:
+ for item in do_insertions(insertions,
+ dylexer.get_tokens_unprocessed(curcode)):
+ yield item
diff --git a/wandb/vendor/pygments/lexers/ecl.py b/wandb/vendor/pygments/lexers/ecl.py
new file mode 100644
index 0000000000000000000000000000000000000000..bd80ad19df4b7872e817a7d37447a077cf8eca00
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/ecl.py
@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.ecl
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the ECL language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Error
+
+__all__ = ['ECLLexer']
+
+
+class ECLLexer(RegexLexer):
+ """
+ Lexer for the declarative big-data `ECL
+ <http://hpccsystems.com/community/docs/ecl-language-reference/html>`_
+ language.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'ECL'
+ aliases = ['ecl']
+ filenames = ['*.ecl']
+ mimetypes = ['application/x-ecl']
+
+ flags = re.IGNORECASE | re.MULTILINE
+
+ tokens = {
+ 'root': [
+ include('whitespace'),
+ include('statements'),
+ ],
+ 'whitespace': [
+ (r'\s+', Text),
+ (r'\/\/.*', Comment.Single),
+ (r'/(\\\n)?\*(.|\n)*?\*(\\\n)?/', Comment.Multiline),
+ ],
+ 'statements': [
+ include('types'),
+ include('keywords'),
+ include('functions'),
+ include('hash'),
+ (r'"', String, 'string'),
+ (r'\'', String, 'string'),
+ (r'(\d+\.\d*|\.\d+|\d+)e[+-]?\d+[lu]*', Number.Float),
+ (r'(\d+\.\d*|\.\d+|\d+f)f?', Number.Float),
+ (r'0x[0-9a-f]+[lu]*', Number.Hex),
+ (r'0[0-7]+[lu]*', Number.Oct),
+ (r'\d+[lu]*', Number.Integer),
+ (r'\*/', Error),
+ (r'[~!%^&*+=|?:<>/-]+', Operator),
+ (r'[{}()\[\],.;]', Punctuation),
+ (r'[a-z_]\w*', Name),
+ ],
+ 'hash': [
+ (r'^#.*$', Comment.Preproc),
+ ],
+ 'types': [
+ (r'(RECORD|END)\D', Keyword.Declaration),
+ (r'((?:ASCII|BIG_ENDIAN|BOOLEAN|DATA|DECIMAL|EBCDIC|INTEGER|PATTERN|'
+ r'QSTRING|REAL|RECORD|RULE|SET OF|STRING|TOKEN|UDECIMAL|UNICODE|'
+ r'UNSIGNED|VARSTRING|VARUNICODE)\d*)(\s+)',
+ bygroups(Keyword.Type, Text)),
+ ],
+ 'keywords': [
+ (words((
+ 'APPLY', 'ASSERT', 'BUILD', 'BUILDINDEX', 'EVALUATE', 'FAIL',
+ 'KEYDIFF', 'KEYPATCH', 'LOADXML', 'NOTHOR', 'NOTIFY', 'OUTPUT',
+ 'PARALLEL', 'SEQUENTIAL', 'SOAPCALL', 'CHECKPOINT', 'DEPRECATED',
+ 'FAILCODE', 'FAILMESSAGE', 'FAILURE', 'GLOBAL', 'INDEPENDENT',
+ 'ONWARNING', 'PERSIST', 'PRIORITY', 'RECOVERY', 'STORED', 'SUCCESS',
+ 'WAIT', 'WHEN'), suffix=r'\b'),
+ Keyword.Reserved),
+ # These are classed differently, check later
+ (words((
+ 'ALL', 'AND', 'ANY', 'AS', 'ATMOST', 'BEFORE', 'BEGINC++', 'BEST', 'BETWEEN', 'CASE',
+ 'CONST', 'COUNTER', 'CSV', 'DESCEND', 'ENCRYPT', 'ENDC++', 'ENDMACRO', 'EXCEPT',
+ 'EXCLUSIVE', 'EXPIRE', 'EXPORT', 'EXTEND', 'FALSE', 'FEW', 'FIRST', 'FLAT', 'FULL',
+ 'FUNCTION', 'GROUP', 'HEADER', 'HEADING', 'HOLE', 'IFBLOCK', 'IMPORT', 'IN', 'JOINED',
+ 'KEEP', 'KEYED', 'LAST', 'LEFT', 'LIMIT', 'LOAD', 'LOCAL', 'LOCALE', 'LOOKUP', 'MACRO',
+ 'MANY', 'MAXCOUNT', 'MAXLENGTH', 'MIN SKEW', 'MODULE', 'INTERFACE', 'NAMED', 'NOCASE',
+ 'NOROOT', 'NOSCAN', 'NOSORT', 'NOT', 'OF', 'ONLY', 'OPT', 'OR', 'OUTER', 'OVERWRITE',
+ 'PACKED', 'PARTITION', 'PENALTY', 'PHYSICALLENGTH', 'PIPE', 'QUOTE', 'RELATIONSHIP',
+ 'REPEAT', 'RETURN', 'RIGHT', 'SCAN', 'SELF', 'SEPARATOR', 'SERVICE', 'SHARED', 'SKEW',
+ 'SKIP', 'SQL', 'STORE', 'TERMINATOR', 'THOR', 'THRESHOLD', 'TOKEN', 'TRANSFORM', 'TRIM',
+ 'TRUE', 'TYPE', 'UNICODEORDER', 'UNSORTED', 'VALIDATE', 'VIRTUAL', 'WHOLE', 'WILD',
+ 'WITHIN', 'XML', 'XPATH', '__COMPRESSED__'), suffix=r'\b'),
+ Keyword.Reserved),
+ ],
+ 'functions': [
+ (words((
+ 'ABS', 'ACOS', 'ALLNODES', 'ASCII', 'ASIN', 'ASSTRING', 'ATAN', 'ATAN2', 'AVE', 'CASE',
+ 'CHOOSE', 'CHOOSEN', 'CHOOSESETS', 'CLUSTERSIZE', 'COMBINE', 'CORRELATION', 'COS',
+ 'COSH', 'COUNT', 'COVARIANCE', 'CRON', 'DATASET', 'DEDUP', 'DEFINE', 'DENORMALIZE',
+ 'DISTRIBUTE', 'DISTRIBUTED', 'DISTRIBUTION', 'EBCDIC', 'ENTH', 'ERROR', 'EVALUATE',
+ 'EVENT', 'EVENTEXTRA', 'EVENTNAME', 'EXISTS', 'EXP', 'FAILCODE', 'FAILMESSAGE',
+ 'FETCH', 'FROMUNICODE', 'GETISVALID', 'GLOBAL', 'GRAPH', 'GROUP', 'HASH', 'HASH32',
+ 'HASH64', 'HASHCRC', 'HASHMD5', 'HAVING', 'IF', 'INDEX', 'INTFORMAT', 'ISVALID',
+ 'ITERATE', 'JOIN', 'KEYUNICODE', 'LENGTH', 'LIBRARY', 'LIMIT', 'LN', 'LOCAL', 'LOG', 'LOOP',
+ 'MAP', 'MATCHED', 'MATCHLENGTH', 'MATCHPOSITION', 'MATCHTEXT', 'MATCHUNICODE',
+ 'MAX', 'MERGE', 'MERGEJOIN', 'MIN', 'NOLOCAL', 'NONEMPTY', 'NORMALIZE', 'PARSE', 'PIPE',
+ 'POWER', 'PRELOAD', 'PROCESS', 'PROJECT', 'PULL', 'RANDOM', 'RANGE', 'RANK', 'RANKED',
+ 'REALFORMAT', 'RECORDOF', 'REGEXFIND', 'REGEXREPLACE', 'REGROUP', 'REJECTED',
+ 'ROLLUP', 'ROUND', 'ROUNDUP', 'ROW', 'ROWDIFF', 'SAMPLE', 'SET', 'SIN', 'SINH', 'SIZEOF',
+ 'SOAPCALL', 'SORT', 'SORTED', 'SQRT', 'STEPPED', 'STORED', 'SUM', 'TABLE', 'TAN', 'TANH',
+ 'THISNODE', 'TOPN', 'TOUNICODE', 'TRANSFER', 'TRIM', 'TRUNCATE', 'TYPEOF', 'UNGROUP',
+ 'UNICODEORDER', 'VARIANCE', 'WHICH', 'WORKUNIT', 'XMLDECODE', 'XMLENCODE',
+ 'XMLTEXT', 'XMLUNICODE'), suffix=r'\b'),
+ Name.Function),
+ ],
+ 'string': [
+ (r'"', String, '#pop'),
+ (r'\'', String, '#pop'),
+ (r'[^"\']+', String),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/eiffel.py b/wandb/vendor/pygments/lexers/eiffel.py
new file mode 100644
index 0000000000000000000000000000000000000000..a90ab0a5535599649c4e63043bd03a6dcc6c3f13
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/eiffel.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.eiffel
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for the Eiffel language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['EiffelLexer']
+
+
+class EiffelLexer(RegexLexer):
+ """
+ For `Eiffel <http://www.eiffel.com>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Eiffel'
+ aliases = ['eiffel']
+ filenames = ['*.e']
+ mimetypes = ['text/x-eiffel']
+
+ tokens = {
+ 'root': [
+ (r'[^\S\n]+', Text),
+ (r'--.*?\n', Comment.Single),
+ (r'[^\S\n]+', Text),
+ # Please note that keyword and operator are case insensitive.
+ (r'(?i)(true|false|void|current|result|precursor)\b', Keyword.Constant),
+ (r'(?i)(and(\s+then)?|not|xor|implies|or(\s+else)?)\b', Operator.Word),
+ (words((
+ 'across', 'agent', 'alias', 'all', 'as', 'assign', 'attached',
+ 'attribute', 'check', 'class', 'convert', 'create', 'debug',
+ 'deferred', 'detachable', 'do', 'else', 'elseif', 'end', 'ensure',
+ 'expanded', 'export', 'external', 'feature', 'from', 'frozen', 'if',
+ 'inherit', 'inspect', 'invariant', 'like', 'local', 'loop', 'none',
+ 'note', 'obsolete', 'old', 'once', 'only', 'redefine', 'rename',
+ 'require', 'rescue', 'retry', 'select', 'separate', 'then',
+ 'undefine', 'until', 'variant', 'when'), prefix=r'(?i)\b', suffix=r'\b'),
+ Keyword.Reserved),
+ (r'"\[(([^\]%]|\n)|%(.|\n)|\][^"])*?\]"', String),
+ (r'"([^"%\n]|%.)*?"', String),
+ include('numbers'),
+ (r"'([^'%]|%'|%%)'", String.Char),
+ (r"(//|\\\\|>=|<=|:=|/=|~|/~|[\\?!#%&@|+/\-=>*$<^\[\]])", Operator),
+ (r"([{}():;,.])", Punctuation),
+ (r'([a-z]\w*)|([A-Z][A-Z0-9_]*[a-z]\w*)', Name),
+ (r'([A-Z][A-Z0-9_]*)', Name.Class),
+ (r'\n+', Text),
+ ],
+ 'numbers': [
+ (r'0[xX][a-fA-F0-9]+', Number.Hex),
+ (r'0[bB][01]+', Number.Bin),
+ (r'0[cC][0-7]+', Number.Oct),
+ (r'([0-9]+\.[0-9]*)|([0-9]*\.[0-9]+)', Number.Float),
+ (r'[0-9]+', Number.Integer),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/elm.py b/wandb/vendor/pygments/lexers/elm.py
new file mode 100644
index 0000000000000000000000000000000000000000..0fa36367880ee572e60dff018278dd26881fc3a1
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/elm.py
@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.elm
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexer for the Elm programming language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words, include
+from pygments.token import Comment, Keyword, Name, Number, Punctuation, String, Text
+
+__all__ = ['ElmLexer']
+
+
+class ElmLexer(RegexLexer):
+ """
+ For `Elm <http://elm-lang.org/>`_ source code.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'Elm'
+ aliases = ['elm']
+ filenames = ['*.elm']
+ mimetypes = ['text/x-elm']
+
+ validName = r'[a-z_][a-zA-Z_\']*'
+
+ specialName = r'^main '
+
+ builtinOps = (
+ '~', '||', '|>', '|', '`', '^', '\\', '\'', '>>', '>=', '>', '==',
+ '=', '<~', '<|', '<=', '<<', '<-', '<', '::', ':', '/=', '//', '/',
+ '..', '.', '->', '-', '++', '+', '*', '&&', '%',
+ )
+
+ reservedWords = words((
+ 'alias', 'as', 'case', 'else', 'if', 'import', 'in',
+ 'let', 'module', 'of', 'port', 'then', 'type', 'where',
+ ), suffix=r'\b')
+
+ tokens = {
+ 'root': [
+
+ # Comments
+ (r'\{-', Comment.Multiline, 'comment'),
+ (r'--.*', Comment.Single),
+
+ # Whitespace
+ (r'\s+', Text),
+
+ # Strings
+ (r'"', String, 'doublequote'),
+
+ # Modules
+ (r'^\s*module\s*', Keyword.Namespace, 'imports'),
+
+ # Imports
+ (r'^\s*import\s*', Keyword.Namespace, 'imports'),
+
+ # Shaders
+ (r'\[glsl\|.*', Name.Entity, 'shader'),
+
+ # Keywords
+ (reservedWords, Keyword.Reserved),
+
+ # Types
+ (r'[A-Z]\w*', Keyword.Type),
+
+ # Main
+ (specialName, Keyword.Reserved),
+
+ # Prefix Operators
+ (words((builtinOps), prefix=r'\(', suffix=r'\)'), Name.Function),
+
+ # Infix Operators
+ (words((builtinOps)), Name.Function),
+
+ # Numbers
+ include('numbers'),
+
+ # Variable Names
+ (validName, Name.Variable),
+
+ # Parens
+ (r'[,()\[\]{}]', Punctuation),
+
+ ],
+
+ 'comment': [
+ (r'-(?!\})', Comment.Multiline),
+ (r'\{-', Comment.Multiline, 'comment'),
+ (r'[^-}]', Comment.Multiline),
+ (r'-\}', Comment.Multiline, '#pop'),
+ ],
+
+ 'doublequote': [
+ (r'\\u[0-9a-fA-F]{4}', String.Escape),
+ (r'\\[nrfvb\\"]', String.Escape),
+ (r'[^"]', String),
+ (r'"', String, '#pop'),
+ ],
+
+ 'imports': [
+ (r'\w+(\.\w+)*', Name.Class, '#pop'),
+ ],
+
+ 'numbers': [
+ (r'_?\d+\.(?=\d+)', Number.Float),
+ (r'_?\d+', Number.Integer),
+ ],
+
+ 'shader': [
+ (r'\|(?!\])', Name.Entity),
+ (r'\|\]', Name.Entity, '#pop'),
+ (r'.*\n', Name.Entity),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/erlang.py b/wandb/vendor/pygments/lexers/erlang.py
new file mode 100644
index 0000000000000000000000000000000000000000..9e7f85c1adaec89a7dee56a511388858a17a2926
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/erlang.py
@@ -0,0 +1,533 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.erlang
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Erlang.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, bygroups, words, do_insertions, \
+ include, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Generic
+
+__all__ = ['ErlangLexer', 'ErlangShellLexer', 'ElixirConsoleLexer',
+ 'ElixirLexer']
+
+
+line_re = re.compile('.*?\n')
+
+
+class ErlangLexer(RegexLexer):
+ """
+ For the Erlang functional programming language.
+
+ Blame Jeremy Thurgood (http://jerith.za.net/).
+
+ .. versionadded:: 0.9
+ """
+
+ name = 'Erlang'
+ aliases = ['erlang']
+ filenames = ['*.erl', '*.hrl', '*.es', '*.escript']
+ mimetypes = ['text/x-erlang']
+
+ keywords = (
+ 'after', 'begin', 'case', 'catch', 'cond', 'end', 'fun', 'if',
+ 'let', 'of', 'query', 'receive', 'try', 'when',
+ )
+
+ builtins = ( # See erlang(3) man page
+ 'abs', 'append_element', 'apply', 'atom_to_list', 'binary_to_list',
+ 'bitstring_to_list', 'binary_to_term', 'bit_size', 'bump_reductions',
+ 'byte_size', 'cancel_timer', 'check_process_code', 'delete_module',
+ 'demonitor', 'disconnect_node', 'display', 'element', 'erase', 'exit',
+ 'float', 'float_to_list', 'fun_info', 'fun_to_list',
+ 'function_exported', 'garbage_collect', 'get', 'get_keys',
+ 'group_leader', 'hash', 'hd', 'integer_to_list', 'iolist_to_binary',
+ 'iolist_size', 'is_atom', 'is_binary', 'is_bitstring', 'is_boolean',
+ 'is_builtin', 'is_float', 'is_function', 'is_integer', 'is_list',
+ 'is_number', 'is_pid', 'is_port', 'is_process_alive', 'is_record',
+ 'is_reference', 'is_tuple', 'length', 'link', 'list_to_atom',
+ 'list_to_binary', 'list_to_bitstring', 'list_to_existing_atom',
+ 'list_to_float', 'list_to_integer', 'list_to_pid', 'list_to_tuple',
+ 'load_module', 'localtime_to_universaltime', 'make_tuple', 'md5',
+ 'md5_final', 'md5_update', 'memory', 'module_loaded', 'monitor',
+ 'monitor_node', 'node', 'nodes', 'open_port', 'phash', 'phash2',
+ 'pid_to_list', 'port_close', 'port_command', 'port_connect',
+ 'port_control', 'port_call', 'port_info', 'port_to_list',
+ 'process_display', 'process_flag', 'process_info', 'purge_module',
+ 'put', 'read_timer', 'ref_to_list', 'register', 'resume_process',
+ 'round', 'send', 'send_after', 'send_nosuspend', 'set_cookie',
+ 'setelement', 'size', 'spawn', 'spawn_link', 'spawn_monitor',
+ 'spawn_opt', 'split_binary', 'start_timer', 'statistics',
+ 'suspend_process', 'system_flag', 'system_info', 'system_monitor',
+ 'system_profile', 'term_to_binary', 'tl', 'trace', 'trace_delivered',
+ 'trace_info', 'trace_pattern', 'trunc', 'tuple_size', 'tuple_to_list',
+ 'universaltime_to_localtime', 'unlink', 'unregister', 'whereis'
+ )
+
+ operators = r'(\+\+?|--?|\*|/|<|>|/=|=:=|=/=|=<|>=|==?|<-|!|\?)'
+ word_operators = (
+ 'and', 'andalso', 'band', 'bnot', 'bor', 'bsl', 'bsr', 'bxor',
+ 'div', 'not', 'or', 'orelse', 'rem', 'xor'
+ )
+
+ atom_re = r"(?:[a-z]\w*|'[^\n']*[^\\]')"
+
+ variable_re = r'(?:[A-Z_]\w*)'
+
+ esc_char_re = r'[bdefnrstv\'"\\]'
+ esc_octal_re = r'[0-7][0-7]?[0-7]?'
+ esc_hex_re = r'(?:x[0-9a-fA-F]{2}|x\{[0-9a-fA-F]+\})'
+ esc_ctrl_re = r'\^[a-zA-Z]'
+ escape_re = r'(?:\\(?:'+esc_char_re+r'|'+esc_octal_re+r'|'+esc_hex_re+r'|'+esc_ctrl_re+r'))'
+
+ macro_re = r'(?:'+variable_re+r'|'+atom_re+r')'
+
+ base_re = r'(?:[2-9]|[12][0-9]|3[0-6])'
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'%.*\n', Comment),
+ (words(keywords, suffix=r'\b'), Keyword),
+ (words(builtins, suffix=r'\b'), Name.Builtin),
+ (words(word_operators, suffix=r'\b'), Operator.Word),
+ (r'^-', Punctuation, 'directive'),
+ (operators, Operator),
+ (r'"', String, 'string'),
+ (r'<<', Name.Label),
+ (r'>>', Name.Label),
+ ('(' + atom_re + ')(:)', bygroups(Name.Namespace, Punctuation)),
+ ('(?:^|(?<=:))(' + atom_re + r')(\s*)(\()',
+ bygroups(Name.Function, Text, Punctuation)),
+ (r'[+-]?' + base_re + r'#[0-9a-zA-Z]+', Number.Integer),
+ (r'[+-]?\d+', Number.Integer),
+ (r'[+-]?\d+.\d+', Number.Float),
+ (r'[]\[:_@\".{}()|;,]', Punctuation),
+ (variable_re, Name.Variable),
+ (atom_re, Name),
+ (r'\?'+macro_re, Name.Constant),
+ (r'\$(?:'+escape_re+r'|\\[ %]|[^\\])', String.Char),
+ (r'#'+atom_re+r'(:?\.'+atom_re+r')?', Name.Label),
+
+ # Erlang script shebang
+ (r'\A#!.+\n', Comment.Hashbang),
+
+ # EEP 43: Maps
+ # http://www.erlang.org/eeps/eep-0043.html
+ (r'#\{', Punctuation, 'map_key'),
+ ],
+ 'string': [
+ (escape_re, String.Escape),
+ (r'"', String, '#pop'),
+ (r'~[0-9.*]*[~#+BPWXb-ginpswx]', String.Interpol),
+ (r'[^"\\~]+', String),
+ (r'~', String),
+ ],
+ 'directive': [
+ (r'(define)(\s*)(\()('+macro_re+r')',
+ bygroups(Name.Entity, Text, Punctuation, Name.Constant), '#pop'),
+ (r'(record)(\s*)(\()('+macro_re+r')',
+ bygroups(Name.Entity, Text, Punctuation, Name.Label), '#pop'),
+ (atom_re, Name.Entity, '#pop'),
+ ],
+ 'map_key': [
+ include('root'),
+ (r'=>', Punctuation, 'map_val'),
+ (r':=', Punctuation, 'map_val'),
+ (r'\}', Punctuation, '#pop'),
+ ],
+ 'map_val': [
+ include('root'),
+ (r',', Punctuation, '#pop'),
+ (r'(?=\})', Punctuation, '#pop'),
+ ],
+ }
+
+
+class ErlangShellLexer(Lexer):
+ """
+ Shell sessions in erl (for Erlang code).
+
+ .. versionadded:: 1.1
+ """
+ name = 'Erlang erl session'
+ aliases = ['erl']
+ filenames = ['*.erl-sh']
+ mimetypes = ['text/x-erl-shellsession']
+
+ _prompt_re = re.compile(r'\d+>(?=\s|\Z)')
+
+ def get_tokens_unprocessed(self, text):
+ erlexer = ErlangLexer(**self.options)
+
+ curcode = ''
+ insertions = []
+ for match in line_re.finditer(text):
+ line = match.group()
+ m = self._prompt_re.match(line)
+ if m is not None:
+ end = m.end()
+ insertions.append((len(curcode),
+ [(0, Generic.Prompt, line[:end])]))
+ curcode += line[end:]
+ else:
+ if curcode:
+ for item in do_insertions(insertions,
+ erlexer.get_tokens_unprocessed(curcode)):
+ yield item
+ curcode = ''
+ insertions = []
+ if line.startswith('*'):
+ yield match.start(), Generic.Traceback, line
+ else:
+ yield match.start(), Generic.Output, line
+ if curcode:
+ for item in do_insertions(insertions,
+ erlexer.get_tokens_unprocessed(curcode)):
+ yield item
+
+
+def gen_elixir_string_rules(name, symbol, token):
+ states = {}
+ states['string_' + name] = [
+ (r'[^#%s\\]+' % (symbol,), token),
+ include('escapes'),
+ (r'\\.', token),
+ (r'(%s)' % (symbol,), bygroups(token), "#pop"),
+ include('interpol')
+ ]
+ return states
+
+
+def gen_elixir_sigstr_rules(term, token, interpol=True):
+ if interpol:
+ return [
+ (r'[^#%s\\]+' % (term,), token),
+ include('escapes'),
+ (r'\\.', token),
+ (r'%s[a-zA-Z]*' % (term,), token, '#pop'),
+ include('interpol')
+ ]
+ else:
+ return [
+ (r'[^%s\\]+' % (term,), token),
+ (r'\\.', token),
+ (r'%s[a-zA-Z]*' % (term,), token, '#pop'),
+ ]
+
+
+class ElixirLexer(RegexLexer):
+ """
+ For the `Elixir language <http://elixir-lang.org>`_.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'Elixir'
+ aliases = ['elixir', 'ex', 'exs']
+ filenames = ['*.ex', '*.exs']
+ mimetypes = ['text/x-elixir']
+
+ KEYWORD = ('fn', 'do', 'end', 'after', 'else', 'rescue', 'catch')
+ KEYWORD_OPERATOR = ('not', 'and', 'or', 'when', 'in')
+ BUILTIN = (
+ 'case', 'cond', 'for', 'if', 'unless', 'try', 'receive', 'raise',
+ 'quote', 'unquote', 'unquote_splicing', 'throw', 'super',
+ )
+ BUILTIN_DECLARATION = (
+ 'def', 'defp', 'defmodule', 'defprotocol', 'defmacro', 'defmacrop',
+ 'defdelegate', 'defexception', 'defstruct', 'defimpl', 'defcallback',
+ )
+
+ BUILTIN_NAMESPACE = ('import', 'require', 'use', 'alias')
+ CONSTANT = ('nil', 'true', 'false')
+
+ PSEUDO_VAR = ('_', '__MODULE__', '__DIR__', '__ENV__', '__CALLER__')
+
+ OPERATORS3 = (
+ '<<<', '>>>', '|||', '&&&', '^^^', '~~~', '===', '!==',
+ '~>>', '<~>', '|~>', '<|>',
+ )
+ OPERATORS2 = (
+ '==', '!=', '<=', '>=', '&&', '||', '<>', '++', '--', '|>', '=~',
+ '->', '<-', '|', '.', '=', '~>', '<~',
+ )
+ OPERATORS1 = ('<', '>', '+', '-', '*', '/', '!', '^', '&')
+
+ PUNCTUATION = (
+ '\\\\', '<<', '>>', '=>', '(', ')', ':', ';', ',', '[', ']',
+ )
+
+ def get_tokens_unprocessed(self, text):
+ for index, token, value in RegexLexer.get_tokens_unprocessed(self, text):
+ if token is Name:
+ if value in self.KEYWORD:
+ yield index, Keyword, value
+ elif value in self.KEYWORD_OPERATOR:
+ yield index, Operator.Word, value
+ elif value in self.BUILTIN:
+ yield index, Keyword, value
+ elif value in self.BUILTIN_DECLARATION:
+ yield index, Keyword.Declaration, value
+ elif value in self.BUILTIN_NAMESPACE:
+ yield index, Keyword.Namespace, value
+ elif value in self.CONSTANT:
+ yield index, Name.Constant, value
+ elif value in self.PSEUDO_VAR:
+ yield index, Name.Builtin.Pseudo, value
+ else:
+ yield index, token, value
+ else:
+ yield index, token, value
+
+ def gen_elixir_sigil_rules():
+ # all valid sigil terminators (excluding heredocs)
+ terminators = [
+ (r'\{', r'\}', 'cb'),
+ (r'\[', r'\]', 'sb'),
+ (r'\(', r'\)', 'pa'),
+ (r'<', r'>', 'ab'),
+ (r'/', r'/', 'slas'),
+ (r'\|', r'\|', 'pipe'),
+ ('"', '"', 'quot'),
+ ("'", "'", 'apos'),
+ ]
+
+ # heredocs have slightly different rules
+ triquotes = [(r'"""', 'triquot'), (r"'''", 'triapos')]
+
+ token = String.Other
+ states = {'sigils': []}
+
+ for term, name in triquotes:
+ states['sigils'] += [
+ (r'(~[a-z])(%s)' % (term,), bygroups(token, String.Heredoc),
+ (name + '-end', name + '-intp')),
+ (r'(~[A-Z])(%s)' % (term,), bygroups(token, String.Heredoc),
+ (name + '-end', name + '-no-intp')),
+ ]
+
+ states[name + '-end'] = [
+ (r'[a-zA-Z]+', token, '#pop'),
+ default('#pop'),
+ ]
+ states[name + '-intp'] = [
+ (r'^\s*' + term, String.Heredoc, '#pop'),
+ include('heredoc_interpol'),
+ ]
+ states[name + '-no-intp'] = [
+ (r'^\s*' + term, String.Heredoc, '#pop'),
+ include('heredoc_no_interpol'),
+ ]
+
+ for lterm, rterm, name in terminators:
+ states['sigils'] += [
+ (r'~[a-z]' + lterm, token, name + '-intp'),
+ (r'~[A-Z]' + lterm, token, name + '-no-intp'),
+ ]
+ states[name + '-intp'] = gen_elixir_sigstr_rules(rterm, token)
+ states[name + '-no-intp'] = \
+ gen_elixir_sigstr_rules(rterm, token, interpol=False)
+
+ return states
+
+ op3_re = "|".join(re.escape(s) for s in OPERATORS3)
+ op2_re = "|".join(re.escape(s) for s in OPERATORS2)
+ op1_re = "|".join(re.escape(s) for s in OPERATORS1)
+ ops_re = r'(?:%s|%s|%s)' % (op3_re, op2_re, op1_re)
+ punctuation_re = "|".join(re.escape(s) for s in PUNCTUATION)
+ alnum = '\w'
+ name_re = r'(?:\.\.\.|[a-z_]%s*[!?]?)' % alnum
+ modname_re = r'[A-Z]%(alnum)s*(?:\.[A-Z]%(alnum)s*)*' % {'alnum': alnum}
+ complex_name_re = r'(?:%s|%s|%s)' % (name_re, modname_re, ops_re)
+ special_atom_re = r'(?:\.\.\.|<<>>|%\{\}|%|\{\})'
+
+ long_hex_char_re = r'(\\x\{)([\da-fA-F]+)(\})'
+ hex_char_re = r'(\\x[\da-fA-F]{1,2})'
+ escape_char_re = r'(\\[abdefnrstv])'
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'#.*$', Comment.Single),
+
+ # Various kinds of characters
+ (r'(\?)' + long_hex_char_re,
+ bygroups(String.Char,
+ String.Escape, Number.Hex, String.Escape)),
+ (r'(\?)' + hex_char_re,
+ bygroups(String.Char, String.Escape)),
+ (r'(\?)' + escape_char_re,
+ bygroups(String.Char, String.Escape)),
+ (r'\?\\?.', String.Char),
+
+ # '::' has to go before atoms
+ (r':::', String.Symbol),
+ (r'::', Operator),
+
+ # atoms
+ (r':' + special_atom_re, String.Symbol),
+ (r':' + complex_name_re, String.Symbol),
+ (r':"', String.Symbol, 'string_double_atom'),
+ (r":'", String.Symbol, 'string_single_atom'),
+
+ # [keywords: ...]
+ (r'(%s|%s)(:)(?=\s|\n)' % (special_atom_re, complex_name_re),
+ bygroups(String.Symbol, Punctuation)),
+
+ # @attributes
+ (r'@' + name_re, Name.Attribute),
+
+ # identifiers
+ (name_re, Name),
+ (r'(%%?)(%s)' % (modname_re,), bygroups(Punctuation, Name.Class)),
+
+ # operators and punctuation
+ (op3_re, Operator),
+ (op2_re, Operator),
+ (punctuation_re, Punctuation),
+ (r'&\d', Name.Entity), # anon func arguments
+ (op1_re, Operator),
+
+ # numbers
+ (r'0b[01]+', Number.Bin),
+ (r'0o[0-7]+', Number.Oct),
+ (r'0x[\da-fA-F]+', Number.Hex),
+ (r'\d(_?\d)*\.\d(_?\d)*([eE][-+]?\d(_?\d)*)?', Number.Float),
+ (r'\d(_?\d)*', Number.Integer),
+
+ # strings and heredocs
+ (r'"""\s*', String.Heredoc, 'heredoc_double'),
+ (r"'''\s*$", String.Heredoc, 'heredoc_single'),
+ (r'"', String.Double, 'string_double'),
+ (r"'", String.Single, 'string_single'),
+
+ include('sigils'),
+
+ (r'%\{', Punctuation, 'map_key'),
+ (r'\{', Punctuation, 'tuple'),
+ ],
+ 'heredoc_double': [
+ (r'^\s*"""', String.Heredoc, '#pop'),
+ include('heredoc_interpol'),
+ ],
+ 'heredoc_single': [
+ (r"^\s*'''", String.Heredoc, '#pop'),
+ include('heredoc_interpol'),
+ ],
+ 'heredoc_interpol': [
+ (r'[^#\\\n]+', String.Heredoc),
+ include('escapes'),
+ (r'\\.', String.Heredoc),
+ (r'\n+', String.Heredoc),
+ include('interpol'),
+ ],
+ 'heredoc_no_interpol': [
+ (r'[^\\\n]+', String.Heredoc),
+ (r'\\.', String.Heredoc),
+ (r'\n+', String.Heredoc),
+ ],
+ 'escapes': [
+ (long_hex_char_re,
+ bygroups(String.Escape, Number.Hex, String.Escape)),
+ (hex_char_re, String.Escape),
+ (escape_char_re, String.Escape),
+ ],
+ 'interpol': [
+ (r'#\{', String.Interpol, 'interpol_string'),
+ ],
+ 'interpol_string': [
+ (r'\}', String.Interpol, "#pop"),
+ include('root')
+ ],
+ 'map_key': [
+ include('root'),
+ (r':', Punctuation, 'map_val'),
+ (r'=>', Punctuation, 'map_val'),
+ (r'\}', Punctuation, '#pop'),
+ ],
+ 'map_val': [
+ include('root'),
+ (r',', Punctuation, '#pop'),
+ (r'(?=\})', Punctuation, '#pop'),
+ ],
+ 'tuple': [
+ include('root'),
+ (r'\}', Punctuation, '#pop'),
+ ],
+ }
+ tokens.update(gen_elixir_string_rules('double', '"', String.Double))
+ tokens.update(gen_elixir_string_rules('single', "'", String.Single))
+ tokens.update(gen_elixir_string_rules('double_atom', '"', String.Symbol))
+ tokens.update(gen_elixir_string_rules('single_atom', "'", String.Symbol))
+ tokens.update(gen_elixir_sigil_rules())
+
+
+class ElixirConsoleLexer(Lexer):
+ """
+ For Elixir interactive console (iex) output like:
+
+ .. sourcecode:: iex
+
+ iex> [head | tail] = [1,2,3]
+ [1,2,3]
+ iex> head
+ 1
+ iex> tail
+ [2,3]
+ iex> [head | tail]
+ [1,2,3]
+ iex> length [head | tail]
+ 3
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'Elixir iex session'
+ aliases = ['iex']
+ mimetypes = ['text/x-elixir-shellsession']
+
+ _prompt_re = re.compile('(iex|\.{3})(\(\d+\))?> ')
+
+ def get_tokens_unprocessed(self, text):
+ exlexer = ElixirLexer(**self.options)
+
+ curcode = ''
+ in_error = False
+ insertions = []
+ for match in line_re.finditer(text):
+ line = match.group()
+ if line.startswith(u'** '):
+ in_error = True
+ insertions.append((len(curcode),
+ [(0, Generic.Error, line[:-1])]))
+ curcode += line[-1:]
+ else:
+ m = self._prompt_re.match(line)
+ if m is not None:
+ in_error = False
+ end = m.end()
+ insertions.append((len(curcode),
+ [(0, Generic.Prompt, line[:end])]))
+ curcode += line[end:]
+ else:
+ if curcode:
+ for item in do_insertions(
+ insertions, exlexer.get_tokens_unprocessed(curcode)):
+ yield item
+ curcode = ''
+ insertions = []
+ token = Generic.Error if in_error else Generic.Output
+ yield match.start(), token, line
+ if curcode:
+ for item in do_insertions(
+ insertions, exlexer.get_tokens_unprocessed(curcode)):
+ yield item
diff --git a/wandb/vendor/pygments/lexers/esoteric.py b/wandb/vendor/pygments/lexers/esoteric.py
new file mode 100644
index 0000000000000000000000000000000000000000..793c28be4b038adbf0502ce0faa209e2a970d791
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/esoteric.py
@@ -0,0 +1,277 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.esoteric
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for esoteric languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Error
+
+__all__ = ['BrainfuckLexer', 'BefungeLexer', 'RedcodeLexer', 'CAmkESLexer',
+ 'CapDLLexer', 'AheuiLexer']
+
+
+class BrainfuckLexer(RegexLexer):
+ """
+ Lexer for the esoteric `BrainFuck <http://www.muppetlabs.com/~breadbox/bf/>`_
+ language.
+ """
+
+ name = 'Brainfuck'
+ aliases = ['brainfuck', 'bf']
+ filenames = ['*.bf', '*.b']
+ mimetypes = ['application/x-brainfuck']
+
+ tokens = {
+ 'common': [
+ # use different colors for different instruction types
+ (r'[.,]+', Name.Tag),
+ (r'[+-]+', Name.Builtin),
+ (r'[<>]+', Name.Variable),
+ (r'[^.,+\-<>\[\]]+', Comment),
+ ],
+ 'root': [
+ (r'\[', Keyword, 'loop'),
+ (r'\]', Error),
+ include('common'),
+ ],
+ 'loop': [
+ (r'\[', Keyword, '#push'),
+ (r'\]', Keyword, '#pop'),
+ include('common'),
+ ]
+ }
+
+
+class BefungeLexer(RegexLexer):
+ """
+ Lexer for the esoteric `Befunge <http://en.wikipedia.org/wiki/Befunge>`_
+ language.
+
+ .. versionadded:: 0.7
+ """
+ name = 'Befunge'
+ aliases = ['befunge']
+ filenames = ['*.befunge']
+ mimetypes = ['application/x-befunge']
+
+ tokens = {
+ 'root': [
+ (r'[0-9a-f]', Number),
+ (r'[+*/%!`-]', Operator), # Traditional math
+ (r'[<>^v?\[\]rxjk]', Name.Variable), # Move, imperatives
+ (r'[:\\$.,n]', Name.Builtin), # Stack ops, imperatives
+ (r'[|_mw]', Keyword),
+ (r'[{}]', Name.Tag), # Befunge-98 stack ops
+ (r'".*?"', String.Double), # Strings don't appear to allow escapes
+ (r'\'.', String.Single), # Single character
+ (r'[#;]', Comment), # Trampoline... depends on direction hit
+ (r'[pg&~=@iotsy]', Keyword), # Misc
+ (r'[()A-Z]', Comment), # Fingerprints
+ (r'\s+', Text), # Whitespace doesn't matter
+ ],
+ }
+
+
+class CAmkESLexer(RegexLexer):
+ """
+ Basic lexer for the input language for the
+ `CAmkES <https://sel4.systems/CAmkES/>`_ component platform.
+
+ .. versionadded:: 2.1
+ """
+ name = 'CAmkES'
+ aliases = ['camkes', 'idl4']
+ filenames = ['*.camkes', '*.idl4']
+
+ tokens = {
+ 'root': [
+ # C pre-processor directive
+ (r'^\s*#.*\n', Comment.Preproc),
+
+ # Whitespace, comments
+ (r'\s+', Text),
+ (r'/\*(.|\n)*?\*/', Comment),
+ (r'//.*\n', Comment),
+
+ (r'[\[(){},.;\]]', Punctuation),
+ (r'[~!%^&*+=|?:<>/-]', Operator),
+
+ (words(('assembly', 'attribute', 'component', 'composition',
+ 'configuration', 'connection', 'connector', 'consumes',
+ 'control', 'dataport', 'Dataport', 'Dataports', 'emits',
+ 'event', 'Event', 'Events', 'export', 'from', 'group',
+ 'hardware', 'has', 'interface', 'Interface', 'maybe',
+ 'procedure', 'Procedure', 'Procedures', 'provides',
+ 'template', 'thread', 'threads', 'to', 'uses', 'with'),
+ suffix=r'\b'), Keyword),
+
+ (words(('bool', 'boolean', 'Buf', 'char', 'character', 'double',
+ 'float', 'in', 'inout', 'int', 'int16_6', 'int32_t',
+ 'int64_t', 'int8_t', 'integer', 'mutex', 'out', 'real',
+ 'refin', 'semaphore', 'signed', 'string', 'struct',
+ 'uint16_t', 'uint32_t', 'uint64_t', 'uint8_t', 'uintptr_t',
+ 'unsigned', 'void'),
+ suffix=r'\b'), Keyword.Type),
+
+ # Recognised attributes
+ (r'[a-zA-Z_]\w*_(priority|domain|buffer)', Keyword.Reserved),
+ (words(('dma_pool', 'from_access', 'to_access'), suffix=r'\b'),
+ Keyword.Reserved),
+
+ # CAmkES-level include
+ (r'import\s+(<[^>]*>|"[^"]*");', Comment.Preproc),
+
+ # C-level include
+ (r'include\s+(<[^>]*>|"[^"]*");', Comment.Preproc),
+
+ # Literals
+ (r'0[xX][\da-fA-F]+', Number.Hex),
+ (r'-?[\d]+', Number),
+ (r'-?[\d]+\.[\d]+', Number.Float),
+ (r'"[^"]*"', String),
+ (r'[Tt]rue|[Ff]alse', Name.Builtin),
+
+ # Identifiers
+ (r'[a-zA-Z_]\w*', Name),
+ ],
+ }
+
+
+class CapDLLexer(RegexLexer):
+ """
+ Basic lexer for
+ `CapDL <https://ssrg.nicta.com.au/publications/nictaabstracts/Kuz_KLW_10.abstract.pml>`_.
+
+ The source of the primary tool that reads such specifications is available
+ at https://github.com/seL4/capdl/tree/master/capDL-tool. Note that this
+ lexer only supports a subset of the grammar. For example, identifiers can
+ shadow type names, but these instances are currently incorrectly
+ highlighted as types. Supporting this would need a stateful lexer that is
+ considered unnecessarily complex for now.
+
+ .. versionadded:: 2.2
+ """
+ name = 'CapDL'
+ aliases = ['capdl']
+ filenames = ['*.cdl']
+
+ tokens = {
+ 'root': [
+ # C pre-processor directive
+ (r'^\s*#.*\n', Comment.Preproc),
+
+ # Whitespace, comments
+ (r'\s+', Text),
+ (r'/\*(.|\n)*?\*/', Comment),
+ (r'(//|--).*\n', Comment),
+
+ (r'[<>\[(){},:;=\]]', Punctuation),
+ (r'\.\.', Punctuation),
+
+ (words(('arch', 'arm11', 'caps', 'child_of', 'ia32', 'irq', 'maps',
+ 'objects'), suffix=r'\b'), Keyword),
+
+ (words(('aep', 'asid_pool', 'cnode', 'ep', 'frame', 'io_device',
+ 'io_ports', 'io_pt', 'notification', 'pd', 'pt', 'tcb',
+ 'ut', 'vcpu'), suffix=r'\b'), Keyword.Type),
+
+ # Properties
+ (words(('asid', 'addr', 'badge', 'cached', 'dom', 'domainID', 'elf',
+ 'fault_ep', 'G', 'guard', 'guard_size', 'init', 'ip',
+ 'prio', 'sp', 'R', 'RG', 'RX', 'RW', 'RWG', 'RWX', 'W',
+ 'WG', 'WX', 'level', 'masked', 'master_reply', 'paddr',
+ 'ports', 'reply', 'uncached'), suffix=r'\b'),
+ Keyword.Reserved),
+
+ # Literals
+ (r'0[xX][\da-fA-F]+', Number.Hex),
+ (r'\d+(\.\d+)?(k|M)?', Number),
+ (words(('bits',), suffix=r'\b'), Number),
+ (words(('cspace', 'vspace', 'reply_slot', 'caller_slot',
+ 'ipc_buffer_slot'), suffix=r'\b'), Number),
+
+ # Identifiers
+ (r'[a-zA-Z_][-@\.\w]*', Name),
+ ],
+ }
+
+
+class RedcodeLexer(RegexLexer):
+ """
+ A simple Redcode lexer based on ICWS'94.
+ Contributed by Adam Blinkinsop <blinks@acm.org>.
+
+ .. versionadded:: 0.8
+ """
+ name = 'Redcode'
+ aliases = ['redcode']
+ filenames = ['*.cw']
+
+ opcodes = ('DAT', 'MOV', 'ADD', 'SUB', 'MUL', 'DIV', 'MOD',
+ 'JMP', 'JMZ', 'JMN', 'DJN', 'CMP', 'SLT', 'SPL',
+ 'ORG', 'EQU', 'END')
+ modifiers = ('A', 'B', 'AB', 'BA', 'F', 'X', 'I')
+
+ tokens = {
+ 'root': [
+ # Whitespace:
+ (r'\s+', Text),
+ (r';.*$', Comment.Single),
+ # Lexemes:
+ # Identifiers
+ (r'\b(%s)\b' % '|'.join(opcodes), Name.Function),
+ (r'\b(%s)\b' % '|'.join(modifiers), Name.Decorator),
+ (r'[A-Za-z_]\w+', Name),
+ # Operators
+ (r'[-+*/%]', Operator),
+ (r'[#$@<>]', Operator), # mode
+ (r'[.,]', Punctuation), # mode
+ # Numbers
+ (r'[-+]?\d+', Number.Integer),
+ ],
+ }
+
+
+class AheuiLexer(RegexLexer):
+ """
+ Aheui_ Lexer.
+
+ Aheui_ is esoteric language based on Korean alphabets.
+
+ .. _Aheui:: http://aheui.github.io/
+
+ """
+
+ name = 'Aheui'
+ aliases = ['aheui']
+ filenames = ['*.aheui']
+
+ tokens = {
+ 'root': [
+ (u'['
+ u'나-낳ëƒ-냫너-넣녀-녛노-놓뇨-눟뉴-닇'
+ u'다-닿댜-댷ë”-ë¯ëŽŒ-뎧ë„-ëŸë´-ë‘«ë“€-딓'
+ u'ë”°-땋땨-ë–ƒë– -떻뗘-ë—³ë˜-똫뚀-뚷뜌-ëŸ'
+ u'ë¼-ëž—ëž´-ëŸëŸ¬-ë ‡ë ¤-ë ¿ë¡œ-롷료-뤃류-릫'
+ u'마-맣먀-먛머-ë©“ë©°-몋모-뫃묘-ë뮤-믷'
+ u'ë°”-밯뱌-뱧버-벟벼-ë³—ë³´-ë´ëµ¤-붛뷰-빃'
+ u'ë¹ -빻뺘-뺳ë»-뻫뼈-뼣뽀-뽛뾰-뿧쀼-ì‚'
+ u'사-샇샤-샿서-섷셔-셯소-솧쇼-숳슈-싛'
+ u'싸-쌓쌰-ì‹ì¨-ìŽƒìŽ -쎻ì˜-ì³ì‘ˆ-ì‘¿ì“”-씧'
+ u'ìž-잫쟈-ìŸ£ì €-ì ›ì ¸-ì¡“ì¡°-ì¢‹ì£ -줗쥬-즿'
+ u'ì°¨-ì±ƒì± -챻처-ì²³ì³-쳫초-촣쵸-춯츄-ì¹—'
+ u'ì¹´-ìºìº¬-컇커-컿켜-ì¼·ì½”-콯쿄-ì¿»í-í‚£'
+ u'타-탛탸-í„“í„°-í…‹í…¨-í†ƒí† -톻íˆ-퉇튜-틯'
+ u'파-팧í„-íŸí¼-펗펴-íí¬-í‡í‘œ-풓퓨-í”»'
+ u'하-핳í–-í–«í—ˆ-헣혀-혛호-홓효-훟휴-힇'
+ u']', Operator),
+ ('.', Comment),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/ezhil.py b/wandb/vendor/pygments/lexers/ezhil.py
new file mode 100644
index 0000000000000000000000000000000000000000..ce1cdb2df1ab5a385d1c88076845738195f3e54e
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/ezhil.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.ezhil
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Pygments lexers for Ezhil language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Keyword, Text, Comment, Name
+from pygments.token import String, Number, Punctuation, Operator
+
+__all__ = ['EzhilLexer']
+
+
+class EzhilLexer(RegexLexer):
+ """
+ Lexer for `Ezhil, a Tamil script-based programming language <http://ezhillang.org>`_
+
+ .. versionadded:: 2.1
+ """
+ name = 'Ezhil'
+ aliases = ['ezhil']
+ filenames = ['*.n']
+ mimetypes = ['text/x-ezhil']
+ flags = re.MULTILINE | re.UNICODE
+ # Refer to tamil.utf8.tamil_letters from open-tamil for a stricter version of this.
+ # This much simpler version is close enough, and includes combining marks.
+ _TALETTERS = u'[a-zA-Z_]|[\u0b80-\u0bff]'
+ tokens = {
+ 'root': [
+ include('keywords'),
+ (r'#.*\n', Comment.Single),
+ (r'[@+/*,^\-%]|[!<>=]=?|&&?|\|\|?', Operator),
+ (u'இலà¯', Operator.Word),
+ (words((u'assert', u'max', u'min',
+ u'நீளமà¯', u'சரமà¯_இடமாறà¯à®±à¯', u'சரமà¯_கணà¯à®Ÿà¯à®ªà®¿à®Ÿà®¿',
+ u'படà¯à®Ÿà®¿à®¯à®²à¯', u'பினà¯à®‡à®£à¯ˆ', u'வரிசைபà¯à®ªà®Ÿà¯à®¤à¯à®¤à¯',
+ u'எடà¯', u'தலைகீழà¯', u'நீடà¯à®Ÿà®¿à®•à¯à®•', u'நà¯à®´à¯ˆà®•à¯à®•', u'வை',
+ u'கோபà¯à®ªà¯ˆ_திற', u'கோபà¯à®ªà¯ˆ_எழà¯à®¤à¯', u'கோபà¯à®ªà¯ˆ_மூடà¯',
+ u'pi', u'sin', u'cos', u'tan', u'sqrt', u'hypot', u'pow',
+ u'exp', u'log', u'log10', u'exit',
+ ), suffix=r'\b'), Name.Builtin),
+ (r'(True|False)\b', Keyword.Constant),
+ (r'[^\S\n]+', Text),
+ include('identifier'),
+ include('literal'),
+ (r'[(){}\[\]:;.]', Punctuation),
+ ],
+ 'keywords': [
+ (u'பதிபà¯à®ªà®¿|தேரà¯à®¨à¯à®¤à¯†à®Ÿà¯|தேரà¯à®µà¯|à®à®¤à¯‡à®©à®¿à®²à¯|ஆனாலà¯|இலà¯à®²à¯ˆà®†à®©à®¾à®²à¯|இலà¯à®²à¯ˆ|ஆக|ஒவà¯à®µà¯Šà®©à¯à®±à®¾à®•|இலà¯|வரை|செயà¯|à®®à¯à®Ÿà®¿à®¯à¯‡à®©à®¿à®²à¯|பினà¯à®•à¯Šà®Ÿà¯|à®®à¯à®Ÿà®¿|நிரலà¯à®ªà®¾à®•à®®à¯|தொடரà¯|நிறà¯à®¤à¯à®¤à¯|நிரலà¯à®ªà®¾à®•à®®à¯', Keyword),
+ ],
+ 'identifier': [
+ (u'(?:'+_TALETTERS+u')(?:[0-9]|'+_TALETTERS+u')*', Name),
+ ],
+ 'literal': [
+ (r'".*?"', String),
+ (r'(?u)\d+((\.\d*)?[eE][+-]?\d+|\.\d*)', Number.Float),
+ (r'(?u)\d+', Number.Integer),
+ ]
+ }
+
+ def __init__(self, **options):
+ super(EzhilLexer, self).__init__(**options)
+ self.encoding = options.get('encoding', 'utf-8')
diff --git a/wandb/vendor/pygments/lexers/factor.py b/wandb/vendor/pygments/lexers/factor.py
new file mode 100644
index 0000000000000000000000000000000000000000..09d85c276a2f6ef102f8c8614daf785ba56604a6
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/factor.py
@@ -0,0 +1,344 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.factor
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the Factor language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, default, words
+from pygments.token import Text, Comment, Keyword, Name, String, Number
+
+__all__ = ['FactorLexer']
+
+
+class FactorLexer(RegexLexer):
+ """
+ Lexer for the `Factor <http://factorcode.org>`_ language.
+
+ .. versionadded:: 1.4
+ """
+ name = 'Factor'
+ aliases = ['factor']
+ filenames = ['*.factor']
+ mimetypes = ['text/x-factor']
+
+ flags = re.MULTILINE | re.UNICODE
+
+ builtin_kernel = words((
+ '-rot', '2bi', '2bi@', '2bi*', '2curry', '2dip', '2drop', '2dup', '2keep', '2nip',
+ '2over', '2tri', '2tri@', '2tri*', '3bi', '3curry', '3dip', '3drop', '3dup', '3keep',
+ '3tri', '4dip', '4drop', '4dup', '4keep', '<wrapper>', '=', '>boolean', 'clone',
+ '?', '?execute', '?if', 'and', 'assert', 'assert=', 'assert?', 'bi', 'bi-curry',
+ 'bi-curry@', 'bi-curry*', 'bi@', 'bi*', 'boa', 'boolean', 'boolean?', 'both?',
+ 'build', 'call', 'callstack', 'callstack>array', 'callstack?', 'clear', '(clone)',
+ 'compose', 'compose?', 'curry', 'curry?', 'datastack', 'die', 'dip', 'do', 'drop',
+ 'dup', 'dupd', 'either?', 'eq?', 'equal?', 'execute', 'hashcode', 'hashcode*',
+ 'identity-hashcode', 'identity-tuple', 'identity-tuple?', 'if', 'if*',
+ 'keep', 'loop', 'most', 'new', 'nip', 'not', 'null', 'object', 'or', 'over',
+ 'pick', 'prepose', 'retainstack', 'rot', 'same?', 'swap', 'swapd', 'throw',
+ 'tri', 'tri-curry', 'tri-curry@', 'tri-curry*', 'tri@', 'tri*', 'tuple',
+ 'tuple?', 'unless', 'unless*', 'until', 'when', 'when*', 'while', 'with',
+ 'wrapper', 'wrapper?', 'xor'), suffix=r'\s')
+
+ builtin_assocs = words((
+ '2cache', '<enum>', '>alist', '?at', '?of', 'assoc', 'assoc-all?',
+ 'assoc-any?', 'assoc-clone-like', 'assoc-combine', 'assoc-diff',
+ 'assoc-diff!', 'assoc-differ', 'assoc-each', 'assoc-empty?',
+ 'assoc-filter', 'assoc-filter!', 'assoc-filter-as', 'assoc-find',
+ 'assoc-hashcode', 'assoc-intersect', 'assoc-like', 'assoc-map',
+ 'assoc-map-as', 'assoc-partition', 'assoc-refine', 'assoc-size',
+ 'assoc-stack', 'assoc-subset?', 'assoc-union', 'assoc-union!',
+ 'assoc=', 'assoc>map', 'assoc?', 'at', 'at+', 'at*', 'cache', 'change-at',
+ 'clear-assoc', 'delete-at', 'delete-at*', 'enum', 'enum?', 'extract-keys',
+ 'inc-at', 'key?', 'keys', 'map>assoc', 'maybe-set-at', 'new-assoc', 'of',
+ 'push-at', 'rename-at', 'set-at', 'sift-keys', 'sift-values', 'substitute',
+ 'unzip', 'value-at', 'value-at*', 'value?', 'values', 'zip'), suffix=r'\s')
+
+ builtin_combinators = words((
+ '2cleave', '2cleave>quot', '3cleave', '3cleave>quot', '4cleave',
+ '4cleave>quot', 'alist>quot', 'call-effect', 'case', 'case-find',
+ 'case>quot', 'cleave', 'cleave>quot', 'cond', 'cond>quot', 'deep-spread>quot',
+ 'execute-effect', 'linear-case-quot', 'no-case', 'no-case?', 'no-cond',
+ 'no-cond?', 'recursive-hashcode', 'shallow-spread>quot', 'spread',
+ 'to-fixed-point', 'wrong-values', 'wrong-values?'), suffix=r'\s')
+
+ builtin_math = words((
+ '-', '/', '/f', '/i', '/mod', '2/', '2^', '<', '<=', '<fp-nan>', '>',
+ '>=', '>bignum', '>fixnum', '>float', '>integer', '(all-integers?)',
+ '(each-integer)', '(find-integer)', '*', '+', '?1+',
+ 'abs', 'align', 'all-integers?', 'bignum', 'bignum?', 'bit?', 'bitand',
+ 'bitnot', 'bitor', 'bits>double', 'bits>float', 'bitxor', 'complex',
+ 'complex?', 'denominator', 'double>bits', 'each-integer', 'even?',
+ 'find-integer', 'find-last-integer', 'fixnum', 'fixnum?', 'float',
+ 'float>bits', 'float?', 'fp-bitwise=', 'fp-infinity?', 'fp-nan-payload',
+ 'fp-nan?', 'fp-qnan?', 'fp-sign', 'fp-snan?', 'fp-special?',
+ 'if-zero', 'imaginary-part', 'integer', 'integer>fixnum',
+ 'integer>fixnum-strict', 'integer?', 'log2', 'log2-expects-positive',
+ 'log2-expects-positive?', 'mod', 'neg', 'neg?', 'next-float',
+ 'next-power-of-2', 'number', 'number=', 'number?', 'numerator', 'odd?',
+ 'out-of-fixnum-range', 'out-of-fixnum-range?', 'power-of-2?',
+ 'prev-float', 'ratio', 'ratio?', 'rational', 'rational?', 'real',
+ 'real-part', 'real?', 'recip', 'rem', 'sgn', 'shift', 'sq', 'times',
+ 'u<', 'u<=', 'u>', 'u>=', 'unless-zero', 'unordered?', 'when-zero',
+ 'zero?'), suffix=r'\s')
+
+ builtin_sequences = words((
+ '1sequence', '2all?', '2each', '2map', '2map-as', '2map-reduce', '2reduce',
+ '2selector', '2sequence', '3append', '3append-as', '3each', '3map', '3map-as',
+ '3sequence', '4sequence', '<repetition>', '<reversed>', '<slice>', '?first',
+ '?last', '?nth', '?second', '?set-nth', 'accumulate', 'accumulate!',
+ 'accumulate-as', 'all?', 'any?', 'append', 'append!', 'append-as',
+ 'assert-sequence', 'assert-sequence=', 'assert-sequence?',
+ 'binary-reduce', 'bounds-check', 'bounds-check?', 'bounds-error',
+ 'bounds-error?', 'but-last', 'but-last-slice', 'cartesian-each',
+ 'cartesian-map', 'cartesian-product', 'change-nth', 'check-slice',
+ 'check-slice-error', 'clone-like', 'collapse-slice', 'collector',
+ 'collector-for', 'concat', 'concat-as', 'copy', 'count', 'cut', 'cut-slice',
+ 'cut*', 'delete-all', 'delete-slice', 'drop-prefix', 'each', 'each-from',
+ 'each-index', 'empty?', 'exchange', 'filter', 'filter!', 'filter-as', 'find',
+ 'find-from', 'find-index', 'find-index-from', 'find-last', 'find-last-from',
+ 'first', 'first2', 'first3', 'first4', 'flip', 'follow', 'fourth', 'glue', 'halves',
+ 'harvest', 'head', 'head-slice', 'head-slice*', 'head*', 'head?',
+ 'if-empty', 'immutable', 'immutable-sequence', 'immutable-sequence?',
+ 'immutable?', 'index', 'index-from', 'indices', 'infimum', 'infimum-by',
+ 'insert-nth', 'interleave', 'iota', 'iota-tuple', 'iota-tuple?', 'join',
+ 'join-as', 'last', 'last-index', 'last-index-from', 'length', 'lengthen',
+ 'like', 'longer', 'longer?', 'longest', 'map', 'map!', 'map-as', 'map-find',
+ 'map-find-last', 'map-index', 'map-integers', 'map-reduce', 'map-sum',
+ 'max-length', 'member-eq?', 'member?', 'midpoint@', 'min-length',
+ 'mismatch', 'move', 'new-like', 'new-resizable', 'new-sequence',
+ 'non-negative-integer-expected', 'non-negative-integer-expected?',
+ 'nth', 'nths', 'pad-head', 'pad-tail', 'padding', 'partition', 'pop', 'pop*',
+ 'prefix', 'prepend', 'prepend-as', 'produce', 'produce-as', 'product', 'push',
+ 'push-all', 'push-either', 'push-if', 'reduce', 'reduce-index', 'remove',
+ 'remove!', 'remove-eq', 'remove-eq!', 'remove-nth', 'remove-nth!', 'repetition',
+ 'repetition?', 'replace-slice', 'replicate', 'replicate-as', 'rest',
+ 'rest-slice', 'reverse', 'reverse!', 'reversed', 'reversed?', 'second',
+ 'selector', 'selector-for', 'sequence', 'sequence-hashcode', 'sequence=',
+ 'sequence?', 'set-first', 'set-fourth', 'set-last', 'set-length', 'set-nth',
+ 'set-second', 'set-third', 'short', 'shorten', 'shorter', 'shorter?',
+ 'shortest', 'sift', 'slice', 'slice-error', 'slice-error?', 'slice?',
+ 'snip', 'snip-slice', 'start', 'start*', 'subseq', 'subseq?', 'suffix',
+ 'suffix!', 'sum', 'sum-lengths', 'supremum', 'supremum-by', 'surround', 'tail',
+ 'tail-slice', 'tail-slice*', 'tail*', 'tail?', 'third', 'trim',
+ 'trim-head', 'trim-head-slice', 'trim-slice', 'trim-tail', 'trim-tail-slice',
+ 'unclip', 'unclip-last', 'unclip-last-slice', 'unclip-slice', 'unless-empty',
+ 'virtual-exemplar', 'virtual-sequence', 'virtual-sequence?', 'virtual@',
+ 'when-empty'), suffix=r'\s')
+
+ builtin_namespaces = words((
+ '+@', 'change', 'change-global', 'counter', 'dec', 'get', 'get-global',
+ 'global', 'inc', 'init-namespaces', 'initialize', 'is-global', 'make-assoc',
+ 'namespace', 'namestack', 'off', 'on', 'set', 'set-global', 'set-namestack',
+ 'toggle', 'with-global', 'with-scope', 'with-variable', 'with-variables'),
+ suffix=r'\s')
+
+ builtin_arrays = words((
+ '1array', '2array', '3array', '4array', '<array>', '>array', 'array',
+ 'array?', 'pair', 'pair?', 'resize-array'), suffix=r'\s')
+
+ builtin_io = words((
+ '(each-stream-block-slice)', '(each-stream-block)',
+ '(stream-contents-by-block)', '(stream-contents-by-element)',
+ '(stream-contents-by-length-or-block)',
+ '(stream-contents-by-length)', '+byte+', '+character+',
+ 'bad-seek-type', 'bad-seek-type?', 'bl', 'contents', 'each-block',
+ 'each-block-size', 'each-block-slice', 'each-line', 'each-morsel',
+ 'each-stream-block', 'each-stream-block-slice', 'each-stream-line',
+ 'error-stream', 'flush', 'input-stream', 'input-stream?',
+ 'invalid-read-buffer', 'invalid-read-buffer?', 'lines', 'nl',
+ 'output-stream', 'output-stream?', 'print', 'read', 'read-into',
+ 'read-partial', 'read-partial-into', 'read-until', 'read1', 'readln',
+ 'seek-absolute', 'seek-absolute?', 'seek-end', 'seek-end?',
+ 'seek-input', 'seek-output', 'seek-relative', 'seek-relative?',
+ 'stream-bl', 'stream-contents', 'stream-contents*', 'stream-copy',
+ 'stream-copy*', 'stream-element-type', 'stream-flush',
+ 'stream-length', 'stream-lines', 'stream-nl', 'stream-print',
+ 'stream-read', 'stream-read-into', 'stream-read-partial',
+ 'stream-read-partial-into', 'stream-read-partial-unsafe',
+ 'stream-read-unsafe', 'stream-read-until', 'stream-read1',
+ 'stream-readln', 'stream-seek', 'stream-seekable?', 'stream-tell',
+ 'stream-write', 'stream-write1', 'tell-input', 'tell-output',
+ 'with-error-stream', 'with-error-stream*', 'with-error>output',
+ 'with-input-output+error-streams',
+ 'with-input-output+error-streams*', 'with-input-stream',
+ 'with-input-stream*', 'with-output-stream', 'with-output-stream*',
+ 'with-output>error', 'with-output+error-stream',
+ 'with-output+error-stream*', 'with-streams', 'with-streams*',
+ 'write', 'write1'), suffix=r'\s')
+
+ builtin_strings = words((
+ '1string', '<string>', '>string', 'resize-string', 'string',
+ 'string?'), suffix=r'\s')
+
+ builtin_vectors = words((
+ '1vector', '<vector>', '>vector', '?push', 'vector', 'vector?'),
+ suffix=r'\s')
+
+ builtin_continuations = words((
+ '<condition>', '<continuation>', '<restart>', 'attempt-all',
+ 'attempt-all-error', 'attempt-all-error?', 'callback-error-hook',
+ 'callcc0', 'callcc1', 'cleanup', 'compute-restarts', 'condition',
+ 'condition?', 'continuation', 'continuation?', 'continue',
+ 'continue-restart', 'continue-with', 'current-continuation',
+ 'error', 'error-continuation', 'error-in-thread', 'error-thread',
+ 'ifcc', 'ignore-errors', 'in-callback?', 'original-error', 'recover',
+ 'restart', 'restart?', 'restarts', 'rethrow', 'rethrow-restarts',
+ 'return', 'return-continuation', 'thread-error-hook', 'throw-continue',
+ 'throw-restarts', 'with-datastack', 'with-return'), suffix=r'\s')
+
+ tokens = {
+ 'root': [
+ # factor allows a file to start with a shebang
+ (r'#!.*$', Comment.Preproc),
+ default('base'),
+ ],
+ 'base': [
+ (r'\s+', Text),
+
+ # defining words
+ (r'((?:MACRO|MEMO|TYPED)?:[:]?)(\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Function)),
+ (r'(M:[:]?)(\s+)(\S+)(\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Class, Text, Name.Function)),
+ (r'(C:)(\s+)(\S+)(\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Function, Text, Name.Class)),
+ (r'(GENERIC:)(\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Function)),
+ (r'(HOOK:|GENERIC#)(\s+)(\S+)(\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Function, Text, Name.Function)),
+ (r'\(\s', Name.Function, 'stackeffect'),
+ (r';\s', Keyword),
+
+ # imports and namespaces
+ (r'(USING:)(\s+)',
+ bygroups(Keyword.Namespace, Text), 'vocabs'),
+ (r'(USE:|UNUSE:|IN:|QUALIFIED:)(\s+)(\S+)',
+ bygroups(Keyword.Namespace, Text, Name.Namespace)),
+ (r'(QUALIFIED-WITH:)(\s+)(\S+)(\s+)(\S+)',
+ bygroups(Keyword.Namespace, Text, Name.Namespace, Text, Name.Namespace)),
+ (r'(FROM:|EXCLUDE:)(\s+)(\S+)(\s+=>\s)',
+ bygroups(Keyword.Namespace, Text, Name.Namespace, Text), 'words'),
+ (r'(RENAME:)(\s+)(\S+)(\s+)(\S+)(\s+=>\s+)(\S+)',
+ bygroups(Keyword.Namespace, Text, Name.Function, Text, Name.Namespace, Text, Name.Function)),
+ (r'(ALIAS:|TYPEDEF:)(\s+)(\S+)(\s+)(\S+)',
+ bygroups(Keyword.Namespace, Text, Name.Function, Text, Name.Function)),
+ (r'(DEFER:|FORGET:|POSTPONE:)(\s+)(\S+)',
+ bygroups(Keyword.Namespace, Text, Name.Function)),
+
+ # tuples and classes
+ (r'(TUPLE:|ERROR:)(\s+)(\S+)(\s+<\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Class, Text, Name.Class), 'slots'),
+ (r'(TUPLE:|ERROR:|BUILTIN:)(\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Class), 'slots'),
+ (r'(MIXIN:|UNION:|INTERSECTION:)(\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Class)),
+ (r'(PREDICATE:)(\s+)(\S+)(\s+<\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Class, Text, Name.Class)),
+ (r'(C:)(\s+)(\S+)(\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Function, Text, Name.Class)),
+ (r'(INSTANCE:)(\s+)(\S+)(\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Class, Text, Name.Class)),
+ (r'(SLOT:)(\s+)(\S+)', bygroups(Keyword, Text, Name.Function)),
+ (r'(SINGLETON:)(\s+)(\S+)', bygroups(Keyword, Text, Name.Class)),
+ (r'SINGLETONS:', Keyword, 'classes'),
+
+ # other syntax
+ (r'(CONSTANT:|SYMBOL:|MAIN:|HELP:)(\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Function)),
+ (r'SYMBOLS:\s', Keyword, 'words'),
+ (r'SYNTAX:\s', Keyword),
+ (r'ALIEN:\s', Keyword),
+ (r'(STRUCT:)(\s+)(\S+)', bygroups(Keyword, Text, Name.Class)),
+ (r'(FUNCTION:)(\s+\S+\s+)(\S+)(\s+\(\s+[^)]+\)\s)',
+ bygroups(Keyword.Namespace, Text, Name.Function, Text)),
+ (r'(FUNCTION-ALIAS:)(\s+)(\S+)(\s+\S+\s+)(\S+)(\s+\(\s+[^)]+\)\s)',
+ bygroups(Keyword.Namespace, Text, Name.Function, Text, Name.Function, Text)),
+
+ # vocab.private
+ (r'(?:<PRIVATE|PRIVATE>)\s', Keyword.Namespace),
+
+ # strings
+ (r'"""\s+(?:.|\n)*?\s+"""', String),
+ (r'"(?:\\\\|\\"|[^"])*"', String),
+ (r'\S+"\s+(?:\\\\|\\"|[^"])*"', String),
+ (r'CHAR:\s+(?:\\[\\abfnrstv]|[^\\]\S*)\s', String.Char),
+
+ # comments
+ (r'!\s+.*$', Comment),
+ (r'#!\s+.*$', Comment),
+ (r'/\*\s+(?:.|\n)*?\s\*/\s', Comment),
+
+ # boolean constants
+ (r'[tf]\s', Name.Constant),
+
+ # symbols and literals
+ (r'[\\$]\s+\S+', Name.Constant),
+ (r'M\\\s+\S+\s+\S+', Name.Constant),
+
+ # numbers
+ (r'[+-]?(?:[\d,]*\d)?\.(?:\d([\d,]*\d)?)?(?:[eE][+-]?\d+)?\s', Number),
+ (r'[+-]?\d(?:[\d,]*\d)?(?:[eE][+-]?\d+)?\s', Number),
+ (r'0x[a-fA-F\d](?:[a-fA-F\d,]*[a-fA-F\d])?(?:p\d([\d,]*\d)?)?\s', Number),
+ (r'NAN:\s+[a-fA-F\d](?:[a-fA-F\d,]*[a-fA-F\d])?(?:p\d([\d,]*\d)?)?\s', Number),
+ (r'0b[01]+\s', Number.Bin),
+ (r'0o[0-7]+\s', Number.Oct),
+ (r'(?:\d([\d,]*\d)?)?\+\d(?:[\d,]*\d)?/\d(?:[\d,]*\d)?\s', Number),
+ (r'(?:\-\d([\d,]*\d)?)?\-\d(?:[\d,]*\d)?/\d(?:[\d,]*\d)?\s', Number),
+
+ # keywords
+ (r'(?:deprecated|final|foldable|flushable|inline|recursive)\s',
+ Keyword),
+
+ # builtins
+ (builtin_kernel, Name.Builtin),
+ (builtin_assocs, Name.Builtin),
+ (builtin_combinators, Name.Builtin),
+ (builtin_math, Name.Builtin),
+ (builtin_sequences, Name.Builtin),
+ (builtin_namespaces, Name.Builtin),
+ (builtin_arrays, Name.Builtin),
+ (builtin_io, Name.Builtin),
+ (builtin_strings, Name.Builtin),
+ (builtin_vectors, Name.Builtin),
+ (builtin_continuations, Name.Builtin),
+
+ # everything else is text
+ (r'\S+', Text),
+ ],
+ 'stackeffect': [
+ (r'\s+', Text),
+ (r'\(\s+', Name.Function, 'stackeffect'),
+ (r'\)\s', Name.Function, '#pop'),
+ (r'--\s', Name.Function),
+ (r'\S+', Name.Variable),
+ ],
+ 'slots': [
+ (r'\s+', Text),
+ (r';\s', Keyword, '#pop'),
+ (r'(\{\s+)(\S+)(\s+[^}]+\s+\}\s)',
+ bygroups(Text, Name.Variable, Text)),
+ (r'\S+', Name.Variable),
+ ],
+ 'vocabs': [
+ (r'\s+', Text),
+ (r';\s', Keyword, '#pop'),
+ (r'\S+', Name.Namespace),
+ ],
+ 'classes': [
+ (r'\s+', Text),
+ (r';\s', Keyword, '#pop'),
+ (r'\S+', Name.Class),
+ ],
+ 'words': [
+ (r'\s+', Text),
+ (r';\s', Keyword, '#pop'),
+ (r'\S+', Name.Function),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/fantom.py b/wandb/vendor/pygments/lexers/fantom.py
new file mode 100644
index 0000000000000000000000000000000000000000..3ea2177c7bebe7f2743a5ebca49336a3ebbbb292
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/fantom.py
@@ -0,0 +1,250 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.fantom
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for the Fantom language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from string import Template
+
+from pygments.lexer import RegexLexer, include, bygroups, using, \
+ this, default, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Literal
+
+__all__ = ['FantomLexer']
+
+
+class FantomLexer(RegexLexer):
+ """
+ For Fantom source code.
+
+ .. versionadded:: 1.5
+ """
+ name = 'Fantom'
+ aliases = ['fan']
+ filenames = ['*.fan']
+ mimetypes = ['application/x-fantom']
+
+ # often used regexes
+ def s(str):
+ return Template(str).substitute(
+ dict(
+ pod=r'[\"\w\.]+',
+ eos=r'\n|;',
+ id=r'[a-zA-Z_]\w*',
+ # all chars which can be part of type definition. Starts with
+ # either letter, or [ (maps), or | (funcs)
+ type=r'(?:\[|[a-zA-Z_]|\|)[:\w\[\]|\->?]*?',
+ )
+ )
+
+ tokens = {
+ 'comments': [
+ (r'(?s)/\*.*?\*/', Comment.Multiline), # Multiline
+ (r'//.*?\n', Comment.Single), # Single line
+ # TODO: highlight references in fandocs
+ (r'\*\*.*?\n', Comment.Special), # Fandoc
+ (r'#.*\n', Comment.Single) # Shell-style
+ ],
+ 'literals': [
+ (r'\b-?[\d_]+(ns|ms|sec|min|hr|day)', Number), # Duration
+ (r'\b-?[\d_]*\.[\d_]+(ns|ms|sec|min|hr|day)', Number), # Duration with dot
+ (r'\b-?(\d+)?\.\d+(f|F|d|D)?', Number.Float), # Float/Decimal
+ (r'\b-?0x[0-9a-fA-F_]+', Number.Hex), # Hex
+ (r'\b-?[\d_]+', Number.Integer), # Int
+ (r"'\\.'|'[^\\]'|'\\u[0-9a-f]{4}'", String.Char), # Char
+ (r'"', Punctuation, 'insideStr'), # Opening quote
+ (r'`', Punctuation, 'insideUri'), # Opening accent
+ (r'\b(true|false|null)\b', Keyword.Constant), # Bool & null
+ (r'(?:(\w+)(::))?(\w+)(<\|)(.*?)(\|>)', # DSL
+ bygroups(Name.Namespace, Punctuation, Name.Class,
+ Punctuation, String, Punctuation)),
+ (r'(?:(\w+)(::))?(\w+)?(#)(\w+)?', # Type/slot literal
+ bygroups(Name.Namespace, Punctuation, Name.Class,
+ Punctuation, Name.Function)),
+ (r'\[,\]', Literal), # Empty list
+ (s(r'($type)(\[,\])'), # Typed empty list
+ bygroups(using(this, state='inType'), Literal)),
+ (r'\[:\]', Literal), # Empty Map
+ (s(r'($type)(\[:\])'),
+ bygroups(using(this, state='inType'), Literal)),
+ ],
+ 'insideStr': [
+ (r'\\\\', String.Escape), # Escaped backslash
+ (r'\\"', String.Escape), # Escaped "
+ (r'\\`', String.Escape), # Escaped `
+ (r'\$\w+', String.Interpol), # Subst var
+ (r'\$\{.*?\}', String.Interpol), # Subst expr
+ (r'"', Punctuation, '#pop'), # Closing quot
+ (r'.', String) # String content
+ ],
+ 'insideUri': [ # TODO: remove copy/paste str/uri
+ (r'\\\\', String.Escape), # Escaped backslash
+ (r'\\"', String.Escape), # Escaped "
+ (r'\\`', String.Escape), # Escaped `
+ (r'\$\w+', String.Interpol), # Subst var
+ (r'\$\{.*?\}', String.Interpol), # Subst expr
+ (r'`', Punctuation, '#pop'), # Closing tick
+ (r'.', String.Backtick) # URI content
+ ],
+ 'protectionKeywords': [
+ (r'\b(public|protected|private|internal)\b', Keyword),
+ ],
+ 'typeKeywords': [
+ (r'\b(abstract|final|const|native|facet|enum)\b', Keyword),
+ ],
+ 'methodKeywords': [
+ (r'\b(abstract|native|once|override|static|virtual|final)\b',
+ Keyword),
+ ],
+ 'fieldKeywords': [
+ (r'\b(abstract|const|final|native|override|static|virtual|'
+ r'readonly)\b', Keyword)
+ ],
+ 'otherKeywords': [
+ (words((
+ 'try', 'catch', 'throw', 'finally', 'for', 'if', 'else', 'while',
+ 'as', 'is', 'isnot', 'switch', 'case', 'default', 'continue',
+ 'break', 'do', 'return', 'get', 'set'), prefix=r'\b', suffix=r'\b'),
+ Keyword),
+ (r'\b(it|this|super)\b', Name.Builtin.Pseudo),
+ ],
+ 'operators': [
+ (r'\+\+|\-\-|\+|\-|\*|/|\|\||&&|<=>|<=|<|>=|>|=|!|\[|\]', Operator)
+ ],
+ 'inType': [
+ (r'[\[\]|\->:?]', Punctuation),
+ (s(r'$id'), Name.Class),
+ default('#pop'),
+
+ ],
+ 'root': [
+ include('comments'),
+ include('protectionKeywords'),
+ include('typeKeywords'),
+ include('methodKeywords'),
+ include('fieldKeywords'),
+ include('literals'),
+ include('otherKeywords'),
+ include('operators'),
+ (r'using\b', Keyword.Namespace, 'using'), # Using stmt
+ (r'@\w+', Name.Decorator, 'facet'), # Symbol
+ (r'(class|mixin)(\s+)(\w+)', bygroups(Keyword, Text, Name.Class),
+ 'inheritance'), # Inheritance list
+
+ # Type var := val
+ (s(r'($type)([ \t]+)($id)(\s*)(:=)'),
+ bygroups(using(this, state='inType'), Text,
+ Name.Variable, Text, Operator)),
+
+ # var := val
+ (s(r'($id)(\s*)(:=)'),
+ bygroups(Name.Variable, Text, Operator)),
+
+ # .someId( or ->someId( ###
+ (s(r'(\.|(?:\->))($id)(\s*)(\()'),
+ bygroups(Operator, Name.Function, Text, Punctuation),
+ 'insideParen'),
+
+ # .someId or ->someId
+ (s(r'(\.|(?:\->))($id)'),
+ bygroups(Operator, Name.Function)),
+
+ # new makeXXX (
+ (r'(new)(\s+)(make\w*)(\s*)(\()',
+ bygroups(Keyword, Text, Name.Function, Text, Punctuation),
+ 'insideMethodDeclArgs'),
+
+ # Type name (
+ (s(r'($type)([ \t]+)' # Return type and whitespace
+ r'($id)(\s*)(\()'), # method name + open brace
+ bygroups(using(this, state='inType'), Text,
+ Name.Function, Text, Punctuation),
+ 'insideMethodDeclArgs'),
+
+ # ArgType argName,
+ (s(r'($type)(\s+)($id)(\s*)(,)'),
+ bygroups(using(this, state='inType'), Text, Name.Variable,
+ Text, Punctuation)),
+
+ # ArgType argName)
+ # Covered in 'insideParen' state
+
+ # ArgType argName -> ArgType|
+ (s(r'($type)(\s+)($id)(\s*)(\->)(\s*)($type)(\|)'),
+ bygroups(using(this, state='inType'), Text, Name.Variable,
+ Text, Punctuation, Text, using(this, state='inType'),
+ Punctuation)),
+
+ # ArgType argName|
+ (s(r'($type)(\s+)($id)(\s*)(\|)'),
+ bygroups(using(this, state='inType'), Text, Name.Variable,
+ Text, Punctuation)),
+
+ # Type var
+ (s(r'($type)([ \t]+)($id)'),
+ bygroups(using(this, state='inType'), Text,
+ Name.Variable)),
+
+ (r'\(', Punctuation, 'insideParen'),
+ (r'\{', Punctuation, 'insideBrace'),
+ (r'.', Text)
+ ],
+ 'insideParen': [
+ (r'\)', Punctuation, '#pop'),
+ include('root'),
+ ],
+ 'insideMethodDeclArgs': [
+ (r'\)', Punctuation, '#pop'),
+ (s(r'($type)(\s+)($id)(\s*)(\))'),
+ bygroups(using(this, state='inType'), Text, Name.Variable,
+ Text, Punctuation), '#pop'),
+ include('root'),
+ ],
+ 'insideBrace': [
+ (r'\}', Punctuation, '#pop'),
+ include('root'),
+ ],
+ 'inheritance': [
+ (r'\s+', Text), # Whitespace
+ (r':|,', Punctuation),
+ (r'(?:(\w+)(::))?(\w+)',
+ bygroups(Name.Namespace, Punctuation, Name.Class)),
+ (r'\{', Punctuation, '#pop')
+ ],
+ 'using': [
+ (r'[ \t]+', Text), # consume whitespaces
+ (r'(\[)(\w+)(\])',
+ bygroups(Punctuation, Comment.Special, Punctuation)), # ffi
+ (r'(\")?([\w.]+)(\")?',
+ bygroups(Punctuation, Name.Namespace, Punctuation)), # podname
+ (r'::', Punctuation, 'usingClass'),
+ default('#pop')
+ ],
+ 'usingClass': [
+ (r'[ \t]+', Text), # consume whitespaces
+ (r'(as)(\s+)(\w+)',
+ bygroups(Keyword.Declaration, Text, Name.Class), '#pop:2'),
+ (r'[\w$]+', Name.Class),
+ default('#pop:2') # jump out to root state
+ ],
+ 'facet': [
+ (r'\s+', Text),
+ (r'\{', Punctuation, 'facetFields'),
+ default('#pop')
+ ],
+ 'facetFields': [
+ include('comments'),
+ include('literals'),
+ include('operators'),
+ (r'\s+', Text),
+ (r'(\s*)(\w+)(\s*)(=)', bygroups(Text, Name, Text, Operator)),
+ (r'\}', Punctuation, '#pop'),
+ (r'.', Text)
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/felix.py b/wandb/vendor/pygments/lexers/felix.py
new file mode 100644
index 0000000000000000000000000000000000000000..8f0695b5cc6be7706cd68e19d4da7bb18b32019e
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/felix.py
@@ -0,0 +1,273 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.felix
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for the Felix language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, default, words, \
+ combined
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['FelixLexer']
+
+
+class FelixLexer(RegexLexer):
+ """
+ For `Felix <http://www.felix-lang.org>`_ source code.
+
+ .. versionadded:: 1.2
+ """
+
+ name = 'Felix'
+ aliases = ['felix', 'flx']
+ filenames = ['*.flx', '*.flxh']
+ mimetypes = ['text/x-felix']
+
+ preproc = (
+ 'elif', 'else', 'endif', 'if', 'ifdef', 'ifndef',
+ )
+
+ keywords = (
+ '_', '_deref', 'all', 'as',
+ 'assert', 'attempt', 'call', 'callback', 'case', 'caseno', 'cclass',
+ 'code', 'compound', 'ctypes', 'do', 'done', 'downto', 'elif', 'else',
+ 'endattempt', 'endcase', 'endif', 'endmatch', 'enum', 'except',
+ 'exceptions', 'expect', 'finally', 'for', 'forall', 'forget', 'fork',
+ 'functor', 'goto', 'ident', 'if', 'incomplete', 'inherit', 'instance',
+ 'interface', 'jump', 'lambda', 'loop', 'match', 'module', 'namespace',
+ 'new', 'noexpand', 'nonterm', 'obj', 'of', 'open', 'parse', 'raise',
+ 'regexp', 'reglex', 'regmatch', 'rename', 'return', 'the', 'then',
+ 'to', 'type', 'typecase', 'typedef', 'typematch', 'typeof', 'upto',
+ 'when', 'whilst', 'with', 'yield',
+ )
+
+ keyword_directives = (
+ '_gc_pointer', '_gc_type', 'body', 'comment', 'const', 'export',
+ 'header', 'inline', 'lval', 'macro', 'noinline', 'noreturn',
+ 'package', 'private', 'pod', 'property', 'public', 'publish',
+ 'requires', 'todo', 'virtual', 'use',
+ )
+
+ keyword_declarations = (
+ 'def', 'let', 'ref', 'val', 'var',
+ )
+
+ keyword_types = (
+ 'unit', 'void', 'any', 'bool',
+ 'byte', 'offset',
+ 'address', 'caddress', 'cvaddress', 'vaddress',
+ 'tiny', 'short', 'int', 'long', 'vlong',
+ 'utiny', 'ushort', 'vshort', 'uint', 'ulong', 'uvlong',
+ 'int8', 'int16', 'int32', 'int64',
+ 'uint8', 'uint16', 'uint32', 'uint64',
+ 'float', 'double', 'ldouble',
+ 'complex', 'dcomplex', 'lcomplex',
+ 'imaginary', 'dimaginary', 'limaginary',
+ 'char', 'wchar', 'uchar',
+ 'charp', 'charcp', 'ucharp', 'ucharcp',
+ 'string', 'wstring', 'ustring',
+ 'cont',
+ 'array', 'varray', 'list',
+ 'lvalue', 'opt', 'slice',
+ )
+
+ keyword_constants = (
+ 'false', 'true',
+ )
+
+ operator_words = (
+ 'and', 'not', 'in', 'is', 'isin', 'or', 'xor',
+ )
+
+ name_builtins = (
+ '_svc', 'while',
+ )
+
+ name_pseudo = (
+ 'root', 'self', 'this',
+ )
+
+ decimal_suffixes = '([tTsSiIlLvV]|ll|LL|([iIuU])(8|16|32|64))?'
+
+ tokens = {
+ 'root': [
+ include('whitespace'),
+
+ # Keywords
+ (words(('axiom', 'ctor', 'fun', 'gen', 'proc', 'reduce',
+ 'union'), suffix=r'\b'),
+ Keyword, 'funcname'),
+ (words(('class', 'cclass', 'cstruct', 'obj', 'struct'), suffix=r'\b'),
+ Keyword, 'classname'),
+ (r'(instance|module|typeclass)\b', Keyword, 'modulename'),
+
+ (words(keywords, suffix=r'\b'), Keyword),
+ (words(keyword_directives, suffix=r'\b'), Name.Decorator),
+ (words(keyword_declarations, suffix=r'\b'), Keyword.Declaration),
+ (words(keyword_types, suffix=r'\b'), Keyword.Type),
+ (words(keyword_constants, suffix=r'\b'), Keyword.Constant),
+
+ # Operators
+ include('operators'),
+
+ # Float Literal
+ # -- Hex Float
+ (r'0[xX]([0-9a-fA-F_]*\.[0-9a-fA-F_]+|[0-9a-fA-F_]+)'
+ r'[pP][+\-]?[0-9_]+[lLfFdD]?', Number.Float),
+ # -- DecimalFloat
+ (r'[0-9_]+(\.[0-9_]+[eE][+\-]?[0-9_]+|'
+ r'\.[0-9_]*|[eE][+\-]?[0-9_]+)[lLfFdD]?', Number.Float),
+ (r'\.(0|[1-9][0-9_]*)([eE][+\-]?[0-9_]+)?[lLfFdD]?',
+ Number.Float),
+
+ # IntegerLiteral
+ # -- Binary
+ (r'0[Bb][01_]+%s' % decimal_suffixes, Number.Bin),
+ # -- Octal
+ (r'0[0-7_]+%s' % decimal_suffixes, Number.Oct),
+ # -- Hexadecimal
+ (r'0[xX][0-9a-fA-F_]+%s' % decimal_suffixes, Number.Hex),
+ # -- Decimal
+ (r'(0|[1-9][0-9_]*)%s' % decimal_suffixes, Number.Integer),
+
+ # Strings
+ ('([rR][cC]?|[cC][rR])"""', String, 'tdqs'),
+ ("([rR][cC]?|[cC][rR])'''", String, 'tsqs'),
+ ('([rR][cC]?|[cC][rR])"', String, 'dqs'),
+ ("([rR][cC]?|[cC][rR])'", String, 'sqs'),
+ ('[cCfFqQwWuU]?"""', String, combined('stringescape', 'tdqs')),
+ ("[cCfFqQwWuU]?'''", String, combined('stringescape', 'tsqs')),
+ ('[cCfFqQwWuU]?"', String, combined('stringescape', 'dqs')),
+ ("[cCfFqQwWuU]?'", String, combined('stringescape', 'sqs')),
+
+ # Punctuation
+ (r'[\[\]{}:(),;?]', Punctuation),
+
+ # Labels
+ (r'[a-zA-Z_]\w*:>', Name.Label),
+
+ # Identifiers
+ (r'(%s)\b' % '|'.join(name_builtins), Name.Builtin),
+ (r'(%s)\b' % '|'.join(name_pseudo), Name.Builtin.Pseudo),
+ (r'[a-zA-Z_]\w*', Name),
+ ],
+ 'whitespace': [
+ (r'\n', Text),
+ (r'\s+', Text),
+
+ include('comment'),
+
+ # Preprocessor
+ (r'#\s*if\s+0', Comment.Preproc, 'if0'),
+ (r'#', Comment.Preproc, 'macro'),
+ ],
+ 'operators': [
+ (r'(%s)\b' % '|'.join(operator_words), Operator.Word),
+ (r'!=|==|<<|>>|\|\||&&|[-~+/*%=<>&^|.$]', Operator),
+ ],
+ 'comment': [
+ (r'//(.*?)\n', Comment.Single),
+ (r'/[*]', Comment.Multiline, 'comment2'),
+ ],
+ 'comment2': [
+ (r'[^/*]', Comment.Multiline),
+ (r'/[*]', Comment.Multiline, '#push'),
+ (r'[*]/', Comment.Multiline, '#pop'),
+ (r'[/*]', Comment.Multiline),
+ ],
+ 'if0': [
+ (r'^\s*#if.*?(?<!\\)\n', Comment, '#push'),
+ (r'^\s*#endif.*?(?<!\\)\n', Comment, '#pop'),
+ (r'.*?\n', Comment),
+ ],
+ 'macro': [
+ include('comment'),
+ (r'(import|include)(\s+)(<[^>]*?>)',
+ bygroups(Comment.Preproc, Text, String), '#pop'),
+ (r'(import|include)(\s+)("[^"]*?")',
+ bygroups(Comment.Preproc, Text, String), '#pop'),
+ (r"(import|include)(\s+)('[^']*?')",
+ bygroups(Comment.Preproc, Text, String), '#pop'),
+ (r'[^/\n]+', Comment.Preproc),
+ # (r'/[*](.|\n)*?[*]/', Comment),
+ # (r'//.*?\n', Comment, '#pop'),
+ (r'/', Comment.Preproc),
+ (r'(?<=\\)\n', Comment.Preproc),
+ (r'\n', Comment.Preproc, '#pop'),
+ ],
+ 'funcname': [
+ include('whitespace'),
+ (r'[a-zA-Z_]\w*', Name.Function, '#pop'),
+ # anonymous functions
+ (r'(?=\()', Text, '#pop'),
+ ],
+ 'classname': [
+ include('whitespace'),
+ (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
+ # anonymous classes
+ (r'(?=\{)', Text, '#pop'),
+ ],
+ 'modulename': [
+ include('whitespace'),
+ (r'\[', Punctuation, ('modulename2', 'tvarlist')),
+ default('modulename2'),
+ ],
+ 'modulename2': [
+ include('whitespace'),
+ (r'([a-zA-Z_]\w*)', Name.Namespace, '#pop:2'),
+ ],
+ 'tvarlist': [
+ include('whitespace'),
+ include('operators'),
+ (r'\[', Punctuation, '#push'),
+ (r'\]', Punctuation, '#pop'),
+ (r',', Punctuation),
+ (r'(with|where)\b', Keyword),
+ (r'[a-zA-Z_]\w*', Name),
+ ],
+ 'stringescape': [
+ (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
+ r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
+ ],
+ 'strings': [
+ (r'%(\([a-zA-Z0-9]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
+ '[hlL]?[E-GXc-giorsux%]', String.Interpol),
+ (r'[^\\\'"%\n]+', String),
+ # quotes, percents and backslashes must be parsed one at a time
+ (r'[\'"\\]', String),
+ # unhandled string formatting sign
+ (r'%', String)
+ # newlines are an error (use "nl" state)
+ ],
+ 'nl': [
+ (r'\n', String)
+ ],
+ 'dqs': [
+ (r'"', String, '#pop'),
+ # included here again for raw strings
+ (r'\\\\|\\"|\\\n', String.Escape),
+ include('strings')
+ ],
+ 'sqs': [
+ (r"'", String, '#pop'),
+ # included here again for raw strings
+ (r"\\\\|\\'|\\\n", String.Escape),
+ include('strings')
+ ],
+ 'tdqs': [
+ (r'"""', String, '#pop'),
+ include('strings'),
+ include('nl')
+ ],
+ 'tsqs': [
+ (r"'''", String, '#pop'),
+ include('strings'),
+ include('nl')
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/forth.py b/wandb/vendor/pygments/lexers/forth.py
new file mode 100644
index 0000000000000000000000000000000000000000..a51f1b57e5f012a0e1da8bd28d2dba0260e15bb1
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/forth.py
@@ -0,0 +1,177 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.forth
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups
+from pygments.token import Error, Punctuation, Literal, Token, \
+ Text, Comment, Operator, Keyword, Name, String, Number, Generic
+
+
+__all__ = ['ForthLexer']
+
+
+class ForthLexer(RegexLexer):
+ """
+ Lexer for Forth files.
+
+ .. versionadded:: 2.2
+ """
+ name = 'Forth'
+ aliases = ['forth']
+ filenames = ['*.frt', '*.fs']
+ mimetypes = ['application/x-forth']
+
+ delimiter = r'\s'
+ delimiter_end = r'(?=[%s])' % delimiter
+
+ valid_name_chars = r'[^%s]' % delimiter
+ valid_name = r"%s+%s" % (valid_name_chars, delimiter_end)
+
+ flags = re.IGNORECASE | re.MULTILINE
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ # All comment types
+ (r'\\.*?\n', Comment.Single),
+ (r'\([\s].*?\)', Comment.Single),
+ # defining words. The next word is a new command name
+ (r'(:|variable|constant|value|buffer:)(\s+)',
+ bygroups(Keyword.Namespace, Text), 'worddef'),
+ # strings are rather simple
+ (r'([.sc]")(\s+?)', bygroups(String, Text), 'stringdef'),
+ # keywords from the various wordsets
+ # *** Wordset BLOCK
+ (r'(blk|block|buffer|evaluate|flush|load|save-buffers|update|'
+ # *** Wordset BLOCK-EXT
+ r'empty-buffers|list|refill|scr|thru|'
+ # *** Wordset CORE
+ r'\#s|\*\/mod|\+loop|\/mod|0<|0=|1\+|1-|2!|'
+ r'2\*|2\/|2@|2drop|2dup|2over|2swap|>body|'
+ r'>in|>number|>r|\?dup|abort|abort\"|abs|'
+ r'accept|align|aligned|allot|and|base|begin|'
+ r'bl|c!|c,|c@|cell\+|cells|char|char\+|'
+ r'chars|constant|count|cr|create|decimal|'
+ r'depth|do|does>|drop|dup|else|emit|environment\?|'
+ r'evaluate|execute|exit|fill|find|fm\/mod|'
+ r'here|hold|i|if|immediate|invert|j|key|'
+ r'leave|literal|loop|lshift|m\*|max|min|'
+ r'mod|move|negate|or|over|postpone|quit|'
+ r'r>|r@|recurse|repeat|rot|rshift|s\"|s>d|'
+ r'sign|sm\/rem|source|space|spaces|state|swap|'
+ r'then|type|u\.|u\<|um\*|um\/mod|unloop|until|'
+ r'variable|while|word|xor|\[char\]|\[\'\]|'
+ r'@|!|\#|<\#|\#>|:|;|\+|-|\*|\/|,|<|>|\|1\+|1-|\.|'
+ # *** Wordset CORE-EXT
+ r'\.r|0<>|'
+ r'0>|2>r|2r>|2r@|:noname|\?do|again|c\"|'
+ r'case|compile,|endcase|endof|erase|false|'
+ r'hex|marker|nip|of|pad|parse|pick|refill|'
+ r'restore-input|roll|save-input|source-id|to|'
+ r'true|tuck|u\.r|u>|unused|value|within|'
+ r'\[compile\]|'
+ # *** Wordset CORE-EXT-obsolescent
+ r'\#tib|convert|expect|query|span|'
+ r'tib|'
+ # *** Wordset DOUBLE
+ r'2constant|2literal|2variable|d\+|d-|'
+ r'd\.|d\.r|d0<|d0=|d2\*|d2\/|d<|d=|d>s|'
+ r'dabs|dmax|dmin|dnegate|m\*\/|m\+|'
+ # *** Wordset DOUBLE-EXT
+ r'2rot|du<|'
+ # *** Wordset EXCEPTION
+ r'catch|throw|'
+ # *** Wordset EXCEPTION-EXT
+ r'abort|abort\"|'
+ # *** Wordset FACILITY
+ r'at-xy|key\?|page|'
+ # *** Wordset FACILITY-EXT
+ r'ekey|ekey>char|ekey\?|emit\?|ms|time&date|'
+ # *** Wordset FILE
+ r'BIN|CLOSE-FILE|CREATE-FILE|DELETE-FILE|FILE-POSITION|'
+ r'FILE-SIZE|INCLUDE-FILE|INCLUDED|OPEN-FILE|R\/O|'
+ r'R\/W|READ-FILE|READ-LINE|REPOSITION-FILE|RESIZE-FILE|'
+ r'S\"|SOURCE-ID|W/O|WRITE-FILE|WRITE-LINE|'
+ # *** Wordset FILE-EXT
+ r'FILE-STATUS|FLUSH-FILE|REFILL|RENAME-FILE|'
+ # *** Wordset FLOAT
+ r'>float|d>f|'
+ r'f!|f\*|f\+|f-|f\/|f0<|f0=|f<|f>d|f@|'
+ r'falign|faligned|fconstant|fdepth|fdrop|fdup|'
+ r'fliteral|float\+|floats|floor|fmax|fmin|'
+ r'fnegate|fover|frot|fround|fswap|fvariable|'
+ r'represent|'
+ # *** Wordset FLOAT-EXT
+ r'df!|df@|dfalign|dfaligned|dfloat\+|'
+ r'dfloats|f\*\*|f\.|fabs|facos|facosh|falog|'
+ r'fasin|fasinh|fatan|fatan2|fatanh|fcos|fcosh|'
+ r'fe\.|fexp|fexpm1|fln|flnp1|flog|fs\.|fsin|'
+ r'fsincos|fsinh|fsqrt|ftan|ftanh|f~|precision|'
+ r'set-precision|sf!|sf@|sfalign|sfaligned|sfloat\+|'
+ r'sfloats|'
+ # *** Wordset LOCAL
+ r'\(local\)|to|'
+ # *** Wordset LOCAL-EXT
+ r'locals\||'
+ # *** Wordset MEMORY
+ r'allocate|free|resize|'
+ # *** Wordset SEARCH
+ r'definitions|find|forth-wordlist|get-current|'
+ r'get-order|search-wordlist|set-current|set-order|'
+ r'wordlist|'
+ # *** Wordset SEARCH-EXT
+ r'also|forth|only|order|previous|'
+ # *** Wordset STRING
+ r'-trailing|\/string|blank|cmove|cmove>|compare|'
+ r'search|sliteral|'
+ # *** Wordset TOOLS
+ r'.s|dump|see|words|'
+ # *** Wordset TOOLS-EXT
+ r';code|'
+ r'ahead|assembler|bye|code|cs-pick|cs-roll|'
+ r'editor|state|\[else\]|\[if\]|\[then\]|'
+ # *** Wordset TOOLS-EXT-obsolescent
+ r'forget|'
+ # Forth 2012
+ r'defer|defer@|defer!|action-of|begin-structure|field:|buffer:|'
+ r'parse-name|buffer:|traverse-wordlist|n>r|nr>|2value|fvalue|'
+ r'name>interpret|name>compile|name>string|'
+ r'cfield:|end-structure)'+delimiter, Keyword),
+
+ # Numbers
+ (r'(\$[0-9A-F]+)', Number.Hex),
+ (r'(\#|%|&|\-|\+)?[0-9]+', Number.Integer),
+ (r'(\#|%|&|\-|\+)?[0-9.]+', Keyword.Type),
+ # amforth specific
+ (r'(@i|!i|@e|!e|pause|noop|turnkey|sleep|'
+ r'itype|icompare|sp@|sp!|rp@|rp!|up@|up!|'
+ r'>a|a>|a@|a!|a@+|a@-|>b|b>|b@|b!|b@+|b@-|'
+ r'find-name|1ms|'
+ r'sp0|rp0|\(evaluate\)|int-trap|int!)' + delimiter,
+ Name.Constant),
+ # a proposal
+ (r'(do-recognizer|r:fail|recognizer:|get-recognizers|'
+ r'set-recognizers|r:float|r>comp|r>int|r>post|'
+ r'r:name|r:word|r:dnum|r:num|recognizer|forth-recognizer|'
+ r'rec:num|rec:float|rec:word)' + delimiter, Name.Decorator),
+ # defining words. The next word is a new command name
+ (r'(Evalue|Rvalue|Uvalue|Edefer|Rdefer|Udefer)(\s+)',
+ bygroups(Keyword.Namespace, Text), 'worddef'),
+
+ (valid_name, Name.Function), # Anything else is executed
+
+ ],
+ 'worddef': [
+ (r'\S+', Name.Class, '#pop'),
+ ],
+ 'stringdef': [
+ (r'[^"]+', String, '#pop'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/fortran.py b/wandb/vendor/pygments/lexers/fortran.py
new file mode 100644
index 0000000000000000000000000000000000000000..1a611c9dc21104b0043c6fcd9d10f89a6dba35fb
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/fortran.py
@@ -0,0 +1,205 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.fortran
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Fortran languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, include, words, using, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Generic
+
+__all__ = ['FortranLexer', 'FortranFixedLexer']
+
+
+class FortranLexer(RegexLexer):
+ """
+ Lexer for FORTRAN 90 code.
+
+ .. versionadded:: 0.10
+ """
+ name = 'Fortran'
+ aliases = ['fortran']
+ filenames = ['*.f03', '*.f90', '*.F03', '*.F90']
+ mimetypes = ['text/x-fortran']
+ flags = re.IGNORECASE | re.MULTILINE
+
+ # Data Types: INTEGER, REAL, COMPLEX, LOGICAL, CHARACTER and DOUBLE PRECISION
+ # Operators: **, *, +, -, /, <, >, <=, >=, ==, /=
+ # Logical (?): NOT, AND, OR, EQV, NEQV
+
+ # Builtins:
+ # http://gcc.gnu.org/onlinedocs/gcc-3.4.6/g77/Table-of-Intrinsic-Functions.html
+
+ tokens = {
+ 'root': [
+ (r'^#.*\n', Comment.Preproc),
+ (r'!.*\n', Comment),
+ include('strings'),
+ include('core'),
+ (r'[a-z][\w$]*', Name),
+ include('nums'),
+ (r'[\s]+', Text),
+ ],
+ 'core': [
+ # Statements
+ (words((
+ 'ABSTRACT', 'ACCEPT', 'ALL', 'ALLSTOP', 'ALLOCATABLE', 'ALLOCATE',
+ 'ARRAY', 'ASSIGN', 'ASSOCIATE', 'ASYNCHRONOUS', 'BACKSPACE', 'BIND',
+ 'BLOCK', 'BLOCKDATA', 'BYTE', 'CALL', 'CASE', 'CLASS', 'CLOSE',
+ 'CODIMENSION', 'COMMON', 'CONCURRRENT', 'CONTIGUOUS', 'CONTAINS',
+ 'CONTINUE', 'CRITICAL', 'CYCLE', 'DATA', 'DEALLOCATE', 'DECODE',
+ 'DEFERRED', 'DIMENSION', 'DO', 'ELEMENTAL', 'ELSE', 'ENCODE', 'END',
+ 'ENTRY', 'ENUM', 'ENUMERATOR', 'EQUIVALENCE', 'EXIT', 'EXTENDS',
+ 'EXTERNAL', 'EXTRINSIC', 'FILE', 'FINAL', 'FORALL', 'FORMAT',
+ 'FUNCTION', 'GENERIC', 'GOTO', 'IF', 'IMAGES', 'IMPLICIT',
+ 'IMPORT', 'IMPURE', 'INCLUDE', 'INQUIRE', 'INTENT', 'INTERFACE',
+ 'INTRINSIC', 'IS', 'LOCK', 'MEMORY', 'MODULE', 'NAMELIST', 'NULLIFY',
+ 'NONE', 'NON_INTRINSIC', 'NON_OVERRIDABLE', 'NOPASS', 'OPEN', 'OPTIONAL',
+ 'OPTIONS', 'PARAMETER', 'PASS', 'PAUSE', 'POINTER', 'PRINT', 'PRIVATE',
+ 'PROGRAM', 'PROCEDURE', 'PROTECTED', 'PUBLIC', 'PURE', 'READ',
+ 'RECURSIVE', 'RESULT', 'RETURN', 'REWIND', 'SAVE', 'SELECT', 'SEQUENCE',
+ 'STOP', 'SUBMODULE', 'SUBROUTINE', 'SYNC', 'SYNCALL', 'SYNCIMAGES',
+ 'SYNCMEMORY', 'TARGET', 'THEN', 'TYPE', 'UNLOCK', 'USE', 'VALUE',
+ 'VOLATILE', 'WHERE', 'WRITE', 'WHILE'), prefix=r'\b', suffix=r'\s*\b'),
+ Keyword),
+
+ # Data Types
+ (words((
+ 'CHARACTER', 'COMPLEX', 'DOUBLE PRECISION', 'DOUBLE COMPLEX', 'INTEGER',
+ 'LOGICAL', 'REAL', 'C_INT', 'C_SHORT', 'C_LONG', 'C_LONG_LONG',
+ 'C_SIGNED_CHAR', 'C_SIZE_T', 'C_INT8_T', 'C_INT16_T', 'C_INT32_T',
+ 'C_INT64_T', 'C_INT_LEAST8_T', 'C_INT_LEAST16_T', 'C_INT_LEAST32_T',
+ 'C_INT_LEAST64_T', 'C_INT_FAST8_T', 'C_INT_FAST16_T', 'C_INT_FAST32_T',
+ 'C_INT_FAST64_T', 'C_INTMAX_T', 'C_INTPTR_T', 'C_FLOAT', 'C_DOUBLE',
+ 'C_LONG_DOUBLE', 'C_FLOAT_COMPLEX', 'C_DOUBLE_COMPLEX',
+ 'C_LONG_DOUBLE_COMPLEX', 'C_BOOL', 'C_CHAR', 'C_PTR', 'C_FUNPTR'),
+ prefix=r'\b', suffix=r'\s*\b'),
+ Keyword.Type),
+
+ # Operators
+ (r'(\*\*|\*|\+|-|\/|<|>|<=|>=|==|\/=|=)', Operator),
+
+ (r'(::)', Keyword.Declaration),
+
+ (r'[()\[\],:&%;.]', Punctuation),
+ # Intrinsics
+ (words((
+ 'Abort', 'Abs', 'Access', 'AChar', 'ACos', 'ACosH', 'AdjustL',
+ 'AdjustR', 'AImag', 'AInt', 'Alarm', 'All', 'Allocated', 'ALog',
+ 'AMax', 'AMin', 'AMod', 'And', 'ANInt', 'Any', 'ASin', 'ASinH',
+ 'Associated', 'ATan', 'ATanH', 'Atomic_Define', 'Atomic_Ref',
+ 'BesJ', 'BesJN', 'Bessel_J0', 'Bessel_J1', 'Bessel_JN', 'Bessel_Y0',
+ 'Bessel_Y1', 'Bessel_YN', 'BesY', 'BesYN', 'BGE', 'BGT', 'BLE',
+ 'BLT', 'Bit_Size', 'BTest', 'CAbs', 'CCos', 'Ceiling', 'CExp',
+ 'Char', 'ChDir', 'ChMod', 'CLog', 'Cmplx', 'Command_Argument_Count',
+ 'Complex', 'Conjg', 'Cos', 'CosH', 'Count', 'CPU_Time', 'CShift',
+ 'CSin', 'CSqRt', 'CTime', 'C_Loc', 'C_Associated',
+ 'C_Null_Ptr', 'C_Null_Funptr', 'C_F_Pointer', 'C_F_ProcPointer',
+ 'C_Null_Char', 'C_Alert', 'C_Backspace', 'C_Form_Feed', 'C_FunLoc',
+ 'C_Sizeof', 'C_New_Line', 'C_Carriage_Return',
+ 'C_Horizontal_Tab', 'C_Vertical_Tab', 'DAbs', 'DACos', 'DASin',
+ 'DATan', 'Date_and_Time', 'DbesJ', 'DbesJN', 'DbesY',
+ 'DbesYN', 'Dble', 'DCos', 'DCosH', 'DDiM', 'DErF',
+ 'DErFC', 'DExp', 'Digits', 'DiM', 'DInt', 'DLog', 'DMax',
+ 'DMin', 'DMod', 'DNInt', 'Dot_Product', 'DProd', 'DSign', 'DSinH',
+ 'DShiftL', 'DShiftR', 'DSin', 'DSqRt', 'DTanH', 'DTan', 'DTime',
+ 'EOShift', 'Epsilon', 'ErF', 'ErFC', 'ErFC_Scaled', 'ETime',
+ 'Execute_Command_Line', 'Exit', 'Exp', 'Exponent', 'Extends_Type_Of',
+ 'FDate', 'FGet', 'FGetC', 'FindLoc', 'Float', 'Floor', 'Flush',
+ 'FNum', 'FPutC', 'FPut', 'Fraction', 'FSeek', 'FStat', 'FTell',
+ 'Gamma', 'GError', 'GetArg', 'Get_Command', 'Get_Command_Argument',
+ 'Get_Environment_Variable', 'GetCWD', 'GetEnv', 'GetGId', 'GetLog',
+ 'GetPId', 'GetUId', 'GMTime', 'HostNm', 'Huge', 'Hypot', 'IAbs',
+ 'IAChar', 'IAll', 'IAnd', 'IAny', 'IArgC', 'IBClr', 'IBits',
+ 'IBSet', 'IChar', 'IDate', 'IDiM', 'IDInt', 'IDNInt', 'IEOr',
+ 'IErrNo', 'IFix', 'Imag', 'ImagPart', 'Image_Index', 'Index',
+ 'Int', 'IOr', 'IParity', 'IRand', 'IsaTty', 'IShft', 'IShftC',
+ 'ISign', 'Iso_C_Binding', 'Is_Contiguous', 'Is_Iostat_End',
+ 'Is_Iostat_Eor', 'ITime', 'Kill', 'Kind', 'LBound', 'LCoBound',
+ 'Len', 'Len_Trim', 'LGe', 'LGt', 'Link', 'LLe', 'LLt', 'LnBlnk',
+ 'Loc', 'Log', 'Log_Gamma', 'Logical', 'Long', 'LShift', 'LStat',
+ 'LTime', 'MaskL', 'MaskR', 'MatMul', 'Max', 'MaxExponent',
+ 'MaxLoc', 'MaxVal', 'MClock', 'Merge', 'Merge_Bits', 'Move_Alloc',
+ 'Min', 'MinExponent', 'MinLoc', 'MinVal', 'Mod', 'Modulo', 'MvBits',
+ 'Nearest', 'New_Line', 'NInt', 'Norm2', 'Not', 'Null', 'Num_Images',
+ 'Or', 'Pack', 'Parity', 'PError', 'Precision', 'Present', 'Product',
+ 'Radix', 'Rand', 'Random_Number', 'Random_Seed', 'Range', 'Real',
+ 'RealPart', 'Rename', 'Repeat', 'Reshape', 'RRSpacing', 'RShift',
+ 'Same_Type_As', 'Scale', 'Scan', 'Second', 'Selected_Char_Kind',
+ 'Selected_Int_Kind', 'Selected_Real_Kind', 'Set_Exponent', 'Shape',
+ 'ShiftA', 'ShiftL', 'ShiftR', 'Short', 'Sign', 'Signal', 'SinH',
+ 'Sin', 'Sleep', 'Sngl', 'Spacing', 'Spread', 'SqRt', 'SRand',
+ 'Stat', 'Storage_Size', 'Sum', 'SymLnk', 'System', 'System_Clock',
+ 'Tan', 'TanH', 'Time', 'This_Image', 'Tiny', 'TrailZ', 'Transfer',
+ 'Transpose', 'Trim', 'TtyNam', 'UBound', 'UCoBound', 'UMask',
+ 'Unlink', 'Unpack', 'Verify', 'XOr', 'ZAbs', 'ZCos', 'ZExp',
+ 'ZLog', 'ZSin', 'ZSqRt'), prefix=r'\b', suffix=r'\s*\b'),
+ Name.Builtin),
+
+ # Booleans
+ (r'\.(true|false)\.', Name.Builtin),
+ # Comparing Operators
+ (r'\.(eq|ne|lt|le|gt|ge|not|and|or|eqv|neqv)\.', Operator.Word),
+ ],
+
+ 'strings': [
+ (r'(?s)"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double),
+ (r"(?s)'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single),
+ ],
+
+ 'nums': [
+ (r'\d+(?![.e])(_[a-z]\w+)?', Number.Integer),
+ (r'[+-]?\d*\.\d+([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float),
+ (r'[+-]?\d+\.\d*([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float),
+ ],
+ }
+
+
+class FortranFixedLexer(RegexLexer):
+ """
+ Lexer for fixed format Fortran.
+
+ .. versionadded:: 2.1
+ """
+ name = 'FortranFixed'
+ aliases = ['fortranfixed']
+ filenames = ['*.f', '*.F']
+
+ flags = re.IGNORECASE
+
+ def _lex_fortran(self, match, ctx=None):
+ """Lex a line just as free form fortran without line break."""
+ lexer = FortranLexer()
+ text = match.group(0) + "\n"
+ for index, token, value in lexer.get_tokens_unprocessed(text):
+ value = value.replace('\n', '')
+ if value != '':
+ yield index, token, value
+
+ tokens = {
+ 'root': [
+ (r'[C*].*\n', Comment),
+ (r'#.*\n', Comment.Preproc),
+ (r' {0,4}!.*\n', Comment),
+ (r'(.{5})', Name.Label, 'cont-char'),
+ (r'.*\n', using(FortranLexer)),
+ ],
+ 'cont-char': [
+ (' ', Text, 'code'),
+ ('0', Comment, 'code'),
+ ('.', Generic.Strong, 'code'),
+ ],
+ 'code': [
+ (r'(.{66})(.*)(\n)',
+ bygroups(_lex_fortran, Comment, Text), 'root'),
+ (r'(.*)(\n)', bygroups(_lex_fortran, Text), 'root'),
+ default('root'),
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/foxpro.py b/wandb/vendor/pygments/lexers/foxpro.py
new file mode 100644
index 0000000000000000000000000000000000000000..7c0d2621919abde9cb0cd70de2c108b88bb7ab3c
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/foxpro.py
@@ -0,0 +1,428 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.foxpro
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Simple lexer for Microsoft Visual FoxPro source code.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer
+from pygments.token import Punctuation, Text, Comment, Operator, Keyword, \
+ Name, String
+
+__all__ = ['FoxProLexer']
+
+
+class FoxProLexer(RegexLexer):
+ """Lexer for Microsoft Visual FoxPro language.
+
+ FoxPro syntax allows to shorten all keywords and function names
+ to 4 characters. Shortened forms are not recognized by this lexer.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'FoxPro'
+ aliases = ['foxpro', 'vfp', 'clipper', 'xbase']
+ filenames = ['*.PRG', '*.prg']
+ mimetype = []
+
+ flags = re.IGNORECASE | re.MULTILINE
+
+ tokens = {
+ 'root': [
+ (r';\s*\n', Punctuation), # consume newline
+ (r'(^|\n)\s*', Text, 'newline'),
+
+ # Square brackets may be used for array indices
+ # and for string literal. Look for arrays
+ # before matching string literals.
+ (r'(?<=\w)\[[0-9, ]+\]', Text),
+ (r'\'[^\'\n]*\'|"[^"\n]*"|\[[^]*]\]', String),
+ (r'(^\s*\*|&&|&&).*?\n', Comment.Single),
+
+ (r'(ABS|ACLASS|ACOPY|ACOS|ADATABASES|ADBOBJECTS|ADDBS|'
+ r'ADDPROPERTY|ADEL|ADIR|ADLLS|ADOCKSTATE|AELEMENT|AERROR|'
+ r'AEVENTS|AFIELDS|AFONT|AGETCLASS|AGETFILEVERSION|AINS|'
+ r'AINSTANCE|ALANGUAGE|ALEN|ALIAS|ALINES|ALLTRIM|'
+ r'AMEMBERS|AMOUSEOBJ|ANETRESOURCES|APRINTERS|APROCINFO|'
+ r'ASC|ASCAN|ASELOBJ|ASESSIONS|ASIN|ASORT|ASQLHANDLES|'
+ r'ASTACKINFO|ASUBSCRIPT|AT|AT_C|ATAGINFO|ATAN|ATC|ATCC|'
+ r'ATCLINE|ATLINE|ATN2|AUSED|AVCXCLASSES|BAR|BARCOUNT|'
+ r'BARPROMPT|BETWEEN|BINDEVENT|BINTOC|BITAND|BITCLEAR|'
+ r'BITLSHIFT|BITNOT|BITOR|BITRSHIFT|BITSET|BITTEST|BITXOR|'
+ r'BOF|CANDIDATE|CAPSLOCK|CAST|CDOW|CDX|CEILING|CHR|CHRSAW|'
+ r'CHRTRAN|CHRTRANC|CLEARRESULTSET|CMONTH|CNTBAR|CNTPAD|COL|'
+ r'COM|Functions|COMARRAY|COMCLASSINFO|COMPOBJ|COMPROP|'
+ r'COMRETURNERROR|COS|CPCONVERT|CPCURRENT|CPDBF|CREATEBINARY|'
+ r'CREATEOBJECT|CREATEOBJECTEX|CREATEOFFLINE|CTOBIN|CTOD|'
+ r'CTOT|CURDIR|CURSORGETPROP|CURSORSETPROP|CURSORTOXML|'
+ r'CURVAL|DATE|DATETIME|DAY|DBC|DBF|DBGETPROP|DBSETPROP|'
+ r'DBUSED|DDEAbortTrans|DDEAdvise|DDEEnabled|DDEExecute|'
+ r'DDEInitiate|DDELastError|DDEPoke|DDERequest|DDESetOption|'
+ r'DDESetService|DDESetTopic|DDETerminate|DEFAULTEXT|'
+ r'DELETED|DESCENDING|DIFFERENCE|DIRECTORY|DISKSPACE|'
+ r'DisplayPath|DMY|DODEFAULT|DOW|DRIVETYPE|DROPOFFLINE|'
+ r'DTOC|DTOR|DTOS|DTOT|EDITSOURCE|EMPTY|EOF|ERROR|EVAL(UATE)?|'
+ r'EVENTHANDLER|EVL|EXECSCRIPT|EXP|FCHSIZE|FCLOSE|FCOUNT|'
+ r'FCREATE|FDATE|FEOF|FERROR|FFLUSH|FGETS|FIELD|FILE|'
+ r'FILETOSTR|FILTER|FKLABEL|FKMAX|FLDLIST|FLOCK|FLOOR|'
+ r'FONTMETRIC|FOPEN|FOR|FORCEEXT|FORCEPATH|FOUND|FPUTS|'
+ r'FREAD|FSEEK|FSIZE|FTIME|FULLPATH|FV|FWRITE|'
+ r'GETAUTOINCVALUE|GETBAR|GETCOLOR|GETCP|GETDIR|GETENV|'
+ r'GETFILE|GETFLDSTATE|GETFONT|GETINTERFACE|'
+ r'GETNEXTMODIFIED|GETOBJECT|GETPAD|GETPEM|GETPICT|'
+ r'GETPRINTER|GETRESULTSET|GETWORDCOUNT|GETWORDNUM|'
+ r'GETCURSORADAPTER|GOMONTH|HEADER|HOME|HOUR|ICASE|'
+ r'IDXCOLLATE|IIF|IMESTATUS|INDBC|INDEXSEEK|INKEY|INLIST|'
+ r'INPUTBOX|INSMODE|INT|ISALPHA|ISBLANK|ISCOLOR|ISDIGIT|'
+ r'ISEXCLUSIVE|ISFLOCKED|ISLEADBYTE|ISLOWER|ISMEMOFETCHED|'
+ r'ISMOUSE|ISNULL|ISPEN|ISREADONLY|ISRLOCKED|'
+ r'ISTRANSACTABLE|ISUPPER|JUSTDRIVE|JUSTEXT|JUSTFNAME|'
+ r'JUSTPATH|JUSTSTEM|KEY|KEYMATCH|LASTKEY|LEFT|LEFTC|LEN|'
+ r'LENC|LIKE|LIKEC|LINENO|LOADPICTURE|LOCFILE|LOCK|LOG|'
+ r'LOG10|LOOKUP|LOWER|LTRIM|LUPDATE|MAKETRANSACTABLE|MAX|'
+ r'MCOL|MDOWN|MDX|MDY|MEMLINES|MEMORY|MENU|MESSAGE|'
+ r'MESSAGEBOX|MIN|MINUTE|MLINE|MOD|MONTH|MRKBAR|MRKPAD|'
+ r'MROW|MTON|MWINDOW|NDX|NEWOBJECT|NORMALIZE|NTOM|NUMLOCK|'
+ r'NVL|OBJNUM|OBJTOCLIENT|OBJVAR|OCCURS|OEMTOANSI|OLDVAL|'
+ r'ON|ORDER|OS|PAD|PADL|PARAMETERS|PAYMENT|PCOL|PCOUNT|'
+ r'PEMSTATUS|PI|POPUP|PRIMARY|PRINTSTATUS|PRMBAR|PRMPAD|'
+ r'PROGRAM|PROMPT|PROPER|PROW|PRTINFO|PUTFILE|PV|QUARTER|'
+ r'RAISEEVENT|RAND|RAT|RATC|RATLINE|RDLEVEL|READKEY|RECCOUNT|'
+ r'RECNO|RECSIZE|REFRESH|RELATION|REPLICATE|REQUERY|RGB|'
+ r'RGBSCHEME|RIGHT|RIGHTC|RLOCK|ROUND|ROW|RTOD|RTRIM|'
+ r'SAVEPICTURE|SCHEME|SCOLS|SEC|SECONDS|SEEK|SELECT|SET|'
+ r'SETFLDSTATE|SETRESULTSET|SIGN|SIN|SKPBAR|SKPPAD|SOUNDEX|'
+ r'SPACE|SQLCANCEL|SQLCOLUMNS|SQLCOMMIT|SQLCONNECT|'
+ r'SQLDISCONNECT|SQLEXEC|SQLGETPROP|SQLIDLEDISCONNECT|'
+ r'SQLMORERESULTS|SQLPREPARE|SQLROLLBACK|SQLSETPROP|'
+ r'SQLSTRINGCONNECT|SQLTABLES|SQRT|SROWS|STR|STRCONV|'
+ r'STREXTRACT|STRTOFILE|STRTRAN|STUFF|STUFFC|SUBSTR|'
+ r'SUBSTRC|SYS|SYSMETRIC|TABLEREVERT|TABLEUPDATE|TAG|'
+ r'TAGCOUNT|TAGNO|TAN|TARGET|TEXTMERGE|TIME|TRANSFORM|'
+ r'TRIM|TTOC|TTOD|TXNLEVEL|TXTWIDTH|TYPE|UNBINDEVENTS|'
+ r'UNIQUE|UPDATED|UPPER|USED|VAL|VARREAD|VARTYPE|VERSION|'
+ r'WBORDER|WCHILD|WCOLS|WDOCKABLE|WEEK|WEXIST|WFONT|WLAST|'
+ r'WLCOL|WLROW|WMAXIMUM|WMINIMUM|WONTOP|WOUTPUT|WPARENT|'
+ r'WREAD|WROWS|WTITLE|WVISIBLE|XMLTOCURSOR|XMLUPDATEGRAM|'
+ r'YEAR)(?=\s*\()', Name.Function),
+
+ (r'_ALIGNMENT|_ASCIICOLS|_ASCIIROWS|_ASSIST|_BEAUTIFY|_BOX|'
+ r'_BROWSER|_BUILDER|_CALCMEM|_CALCVALUE|_CLIPTEXT|_CONVERTER|'
+ r'_COVERAGE|_CUROBJ|_DBLCLICK|_DIARYDATE|_DOS|_FOXDOC|_FOXREF|'
+ r'_GALLERY|_GENGRAPH|_GENHTML|_GENMENU|_GENPD|_GENSCRN|'
+ r'_GENXTAB|_GETEXPR|_INCLUDE|_INCSEEK|_INDENT|_LMARGIN|_MAC|'
+ r'_MENUDESIGNER|_MLINE|_PADVANCE|_PAGENO|_PAGETOTAL|_PBPAGE|'
+ r'_PCOLNO|_PCOPIES|_PDRIVER|_PDSETUP|_PECODE|_PEJECT|_PEPAGE|'
+ r'_PLENGTH|_PLINENO|_PLOFFSET|_PPITCH|_PQUALITY|_PRETEXT|'
+ r'_PSCODE|_PSPACING|_PWAIT|_RMARGIN|_REPORTBUILDER|'
+ r'_REPORTOUTPUT|_REPORTPREVIEW|_SAMPLES|_SCCTEXT|_SCREEN|'
+ r'_SHELL|_SPELLCHK|_STARTUP|_TABS|_TALLY|_TASKPANE|_TEXT|'
+ r'_THROTTLE|_TOOLBOX|_TOOLTIPTIMEOUT|_TRANSPORT|_TRIGGERLEVEL|'
+ r'_UNIX|_VFP|_WINDOWS|_WIZARD|_WRAP', Keyword.Pseudo),
+
+ (r'THISFORMSET|THISFORM|THIS', Name.Builtin),
+
+ (r'Application|CheckBox|Collection|Column|ComboBox|'
+ r'CommandButton|CommandGroup|Container|Control|CursorAdapter|'
+ r'Cursor|Custom|DataEnvironment|DataObject|EditBox|'
+ r'Empty|Exception|Fields|Files|File|FormSet|Form|FoxCode|'
+ r'Grid|Header|Hyperlink|Image|Label|Line|ListBox|Objects|'
+ r'OptionButton|OptionGroup|PageFrame|Page|ProjectHook|Projects|'
+ r'Project|Relation|ReportListener|Separator|Servers|Server|'
+ r'Session|Shape|Spinner|Tables|TextBox|Timer|ToolBar|'
+ r'XMLAdapter|XMLField|XMLTable', Name.Class),
+
+ (r'm\.[a-z_]\w*', Name.Variable),
+ (r'\.(F|T|AND|OR|NOT|NULL)\.|\b(AND|OR|NOT|NULL)\b', Operator.Word),
+
+ (r'\.(ActiveColumn|ActiveControl|ActiveForm|ActivePage|'
+ r'ActiveProject|ActiveRow|AddLineFeeds|ADOCodePage|Alias|'
+ r'Alignment|Align|AllowAddNew|AllowAutoColumnFit|'
+ r'AllowCellSelection|AllowDelete|AllowHeaderSizing|'
+ r'AllowInsert|AllowModalMessages|AllowOutput|AllowRowSizing|'
+ r'AllowSimultaneousFetch|AllowTabs|AllowUpdate|'
+ r'AlwaysOnBottom|AlwaysOnTop|Anchor|Application|'
+ r'AutoActivate|AutoCenter|AutoCloseTables|AutoComplete|'
+ r'AutoCompSource|AutoCompTable|AutoHideScrollBar|'
+ r'AutoIncrement|AutoOpenTables|AutoRelease|AutoSize|'
+ r'AutoVerbMenu|AutoYield|BackColor|ForeColor|BackStyle|'
+ r'BaseClass|BatchUpdateCount|BindControls|BorderColor|'
+ r'BorderStyle|BorderWidth|BoundColumn|BoundTo|Bound|'
+ r'BreakOnError|BufferModeOverride|BufferMode|'
+ r'BuildDateTime|ButtonCount|Buttons|Cancel|Caption|'
+ r'Centered|Century|ChildAlias|ChildOrder|ChildTable|'
+ r'ClassLibrary|Class|ClipControls|Closable|CLSID|CodePage|'
+ r'ColorScheme|ColorSource|ColumnCount|ColumnLines|'
+ r'ColumnOrder|Columns|ColumnWidths|CommandClauses|'
+ r'Comment|CompareMemo|ConflictCheckCmd|ConflictCheckType|'
+ r'ContinuousScroll|ControlBox|ControlCount|Controls|'
+ r'ControlSource|ConversionFunc|Count|CurrentControl|'
+ r'CurrentDataSession|CurrentPass|CurrentX|CurrentY|'
+ r'CursorSchema|CursorSource|CursorStatus|Curvature|'
+ r'Database|DataSessionID|DataSession|DataSourceType|'
+ r'DataSource|DataType|DateFormat|DateMark|Debug|'
+ r'DeclareXMLPrefix|DEClassLibrary|DEClass|DefaultFilePath|'
+ r'Default|DefOLELCID|DeleteCmdDataSourceType|DeleteCmdDataSource|'
+ r'DeleteCmd|DeleteMark|Description|Desktop|'
+ r'Details|DisabledBackColor|DisabledForeColor|'
+ r'DisabledItemBackColor|DisabledItemForeColor|'
+ r'DisabledPicture|DisableEncode|DisplayCount|'
+ r'DisplayValue|Dockable|Docked|DockPosition|'
+ r'DocumentFile|DownPicture|DragIcon|DragMode|DrawMode|'
+ r'DrawStyle|DrawWidth|DynamicAlignment|DynamicBackColor|'
+ r'DynamicForeColor|DynamicCurrentControl|DynamicFontBold|'
+ r'DynamicFontItalic|DynamicFontStrikethru|'
+ r'DynamicFontUnderline|DynamicFontName|DynamicFontOutline|'
+ r'DynamicFontShadow|DynamicFontSize|DynamicInputMask|'
+ r'DynamicLineHeight|EditorOptions|Enabled|'
+ r'EnableHyperlinks|Encrypted|ErrorNo|Exclude|Exclusive|'
+ r'FetchAsNeeded|FetchMemoCmdList|FetchMemoDataSourceType|'
+ r'FetchMemoDataSource|FetchMemo|FetchSize|'
+ r'FileClassLibrary|FileClass|FillColor|FillStyle|Filter|'
+ r'FirstElement|FirstNestedTable|Flags|FontBold|FontItalic|'
+ r'FontStrikethru|FontUnderline|FontCharSet|FontCondense|'
+ r'FontExtend|FontName|FontOutline|FontShadow|FontSize|'
+ r'ForceCloseTag|Format|FormCount|FormattedOutput|Forms|'
+ r'FractionDigits|FRXDataSession|FullName|GDIPlusGraphics|'
+ r'GridLineColor|GridLines|GridLineWidth|HalfHeightCaption|'
+ r'HeaderClassLibrary|HeaderClass|HeaderHeight|Height|'
+ r'HelpContextID|HideSelection|HighlightBackColor|'
+ r'HighlightForeColor|HighlightStyle|HighlightRowLineWidth|'
+ r'HighlightRow|Highlight|HomeDir|Hours|HostName|'
+ r'HScrollSmallChange|hWnd|Icon|IncrementalSearch|Increment|'
+ r'InitialSelectedAlias|InputMask|InsertCmdDataSourceType|'
+ r'InsertCmdDataSource|InsertCmdRefreshCmd|'
+ r'InsertCmdRefreshFieldList|InsertCmdRefreshKeyFieldList|'
+ r'InsertCmd|Instancing|IntegralHeight|'
+ r'Interval|IMEMode|IsAttribute|IsBase64|IsBinary|IsNull|'
+ r'IsDiffGram|IsLoaded|ItemBackColor,|ItemData|ItemIDData|'
+ r'ItemTips|IXMLDOMElement|KeyboardHighValue|KeyboardLowValue|'
+ r'Keyfield|KeyFieldList|KeyPreview|KeySort|LanguageOptions|'
+ r'LeftColumn|Left|LineContents|LineNo|LineSlant|LinkMaster|'
+ r'ListCount|ListenerType|ListIndex|ListItemID|ListItem|'
+ r'List|LockColumnsLeft|LockColumns|LockScreen|MacDesktop|'
+ r'MainFile|MapN19_4ToCurrency|MapBinary|MapVarchar|Margin|'
+ r'MaxButton|MaxHeight|MaxLeft|MaxLength|MaxRecords|MaxTop|'
+ r'MaxWidth|MDIForm|MemberClassLibrary|MemberClass|'
+ r'MemoWindow|Message|MinButton|MinHeight|MinWidth|'
+ r'MouseIcon|MousePointer|Movable|MoverBars|MultiSelect|'
+ r'Name|NestedInto|NewIndex|NewItemID|NextSiblingTable|'
+ r'NoCpTrans|NoDataOnLoad|NoData|NullDisplay|'
+ r'NumberOfElements|Object|OLEClass|OLEDragMode|'
+ r'OLEDragPicture|OLEDropEffects|OLEDropHasData|'
+ r'OLEDropMode|OLEDropTextInsertion|OLELCID|'
+ r'OLERequestPendingTimeout|OLEServerBusyRaiseError|'
+ r'OLEServerBusyTimeout|OLETypeAllowed|OneToMany|'
+ r'OpenViews|OpenWindow|Optimize|OrderDirection|Order|'
+ r'OutputPageCount|OutputType|PageCount|PageHeight|'
+ r'PageNo|PageOrder|Pages|PageTotal|PageWidth|'
+ r'PanelLink|Panel|ParentAlias|ParentClass|ParentTable|'
+ r'Parent|Partition|PasswordChar|PictureMargin|'
+ r'PicturePosition|PictureSpacing|PictureSelectionDisplay|'
+ r'PictureVal|Picture|Prepared|'
+ r'PolyPoints|PreserveWhiteSpace|PreviewContainer|'
+ r'PrintJobName|Procedure|PROCESSID|ProgID|ProjectHookClass|'
+ r'ProjectHookLibrary|ProjectHook|QuietMode|'
+ r'ReadCycle|ReadLock|ReadMouse|ReadObject|ReadOnly|'
+ r'ReadSave|ReadTimeout|RecordMark|RecordSourceType|'
+ r'RecordSource|RefreshAlias|'
+ r'RefreshCmdDataSourceType|RefreshCmdDataSource|RefreshCmd|'
+ r'RefreshIgnoreFieldList|RefreshTimeStamp|RelationalExpr|'
+ r'RelativeColumn|RelativeRow|ReleaseType|Resizable|'
+ r'RespectCursorCP|RespectNesting|RightToLeft|RotateFlip|'
+ r'Rotation|RowColChange|RowHeight|RowSourceType|'
+ r'RowSource|ScaleMode|SCCProvider|SCCStatus|ScrollBars|'
+ r'Seconds|SelectCmd|SelectedID|'
+ r'SelectedItemBackColor|SelectedItemForeColor|Selected|'
+ r'SelectionNamespaces|SelectOnEntry|SelLength|SelStart|'
+ r'SelText|SendGDIPlusImage|SendUpdates|ServerClassLibrary|'
+ r'ServerClass|ServerHelpFile|ServerName|'
+ r'ServerProject|ShowTips|ShowInTaskbar|ShowWindow|'
+ r'Sizable|SizeBox|SOM|Sorted|Sparse|SpecialEffect|'
+ r'SpinnerHighValue|SpinnerLowValue|SplitBar|StackLevel|'
+ r'StartMode|StatusBarText|StatusBar|Stretch|StrictDateEntry|'
+ r'Style|TabIndex|Tables|TabOrientation|Tabs|TabStop|'
+ r'TabStretch|TabStyle|Tag|TerminateRead|Text|Themes|'
+ r'ThreadID|TimestampFieldList|TitleBar|ToolTipText|'
+ r'TopIndex|TopItemID|Top|TwoPassProcess|TypeLibCLSID|'
+ r'TypeLibDesc|TypeLibName|Type|Unicode|UpdatableFieldList|'
+ r'UpdateCmdDataSourceType|UpdateCmdDataSource|'
+ r'UpdateCmdRefreshCmd|UpdateCmdRefreshFieldList|'
+ r'UpdateCmdRefreshKeyFieldList|UpdateCmd|'
+ r'UpdateGramSchemaLocation|UpdateGram|UpdateNameList|UpdateType|'
+ r'UseCodePage|UseCursorSchema|UseDeDataSource|UseMemoSize|'
+ r'UserValue|UseTransactions|UTF8Encoded|Value|VersionComments|'
+ r'VersionCompany|VersionCopyright|VersionDescription|'
+ r'VersionNumber|VersionProduct|VersionTrademarks|Version|'
+ r'VFPXMLProgID|ViewPortHeight|ViewPortLeft|'
+ r'ViewPortTop|ViewPortWidth|VScrollSmallChange|View|Visible|'
+ r'VisualEffect|WhatsThisButton|WhatsThisHelpID|WhatsThisHelp|'
+ r'WhereType|Width|WindowList|WindowState|WindowType|WordWrap|'
+ r'WrapCharInCDATA|WrapInCDATA|WrapMemoInCDATA|XMLAdapter|'
+ r'XMLConstraints|XMLNameIsXPath|XMLNamespace|XMLName|'
+ r'XMLPrefix|XMLSchemaLocation|XMLTable|XMLType|'
+ r'XSDfractionDigits|XSDmaxLength|XSDtotalDigits|'
+ r'XSDtype|ZoomBox)', Name.Attribute),
+
+ (r'\.(ActivateCell|AddColumn|AddItem|AddListItem|AddObject|'
+ r'AddProperty|AddTableSchema|AddToSCC|Add|'
+ r'ApplyDiffgram|Attach|AutoFit|AutoOpen|Box|Build|'
+ r'CancelReport|ChangesToCursor|CheckIn|CheckOut|Circle|'
+ r'CleanUp|ClearData|ClearStatus|Clear|CloneObject|CloseTables|'
+ r'Close|Cls|CursorAttach|CursorDetach|CursorFill|'
+ r'CursorRefresh|DataToClip|DelayedMemoFetch|DeleteColumn|'
+ r'Dock|DoMessage|DoScroll|DoStatus|DoVerb|Drag|Draw|Eval|'
+ r'GetData|GetDockState|GetFormat|GetKey|GetLatestVersion|'
+ r'GetPageHeight|GetPageWidth|Help|Hide|IncludePageInOutput|'
+ r'IndexToItemID|ItemIDToIndex|Item|LoadXML|Line|Modify|'
+ r'MoveItem|Move|Nest|OLEDrag|OnPreviewClose|OutputPage|'
+ r'Point|Print|PSet|Quit|ReadExpression|ReadMethod|'
+ r'RecordRefresh|Refresh|ReleaseXML|Release|RemoveFromSCC|'
+ r'RemoveItem|RemoveListItem|RemoveObject|Remove|'
+ r'Render|Requery|RequestData|ResetToDefault|Reset|Run|'
+ r'SaveAsClass|SaveAs|SetAll|SetData|SetFocus|SetFormat|'
+ r'SetMain|SetVar|SetViewPort|ShowWhatsThis|Show|'
+ r'SupportsListenerType|TextHeight|TextWidth|ToCursor|'
+ r'ToXML|UndoCheckOut|Unnest|UpdateStatus|WhatsThisMode|'
+ r'WriteExpression|WriteMethod|ZOrder)', Name.Function),
+
+ (r'\.(Activate|AdjustObjectSize|AfterBand|AfterBuild|'
+ r'AfterCloseTables|AfterCursorAttach|AfterCursorClose|'
+ r'AfterCursorDetach|AfterCursorFill|AfterCursorRefresh|'
+ r'AfterCursorUpdate|AfterDelete|AfterInsert|'
+ r'AfterRecordRefresh|AfterUpdate|AfterDock|AfterReport|'
+ r'AfterRowColChange|BeforeBand|BeforeCursorAttach|'
+ r'BeforeCursorClose|BeforeCursorDetach|BeforeCursorFill|'
+ r'BeforeCursorRefresh|BeforeCursorUpdate|BeforeDelete|'
+ r'BeforeInsert|BeforeDock|BeforeOpenTables|'
+ r'BeforeRecordRefresh|BeforeReport|BeforeRowColChange|'
+ r'BeforeUpdate|Click|dbc_Activate|dbc_AfterAddTable|'
+ r'dbc_AfterAppendProc|dbc_AfterCloseTable|dbc_AfterCopyProc|'
+ r'dbc_AfterCreateConnection|dbc_AfterCreateOffline|'
+ r'dbc_AfterCreateTable|dbc_AfterCreateView|dbc_AfterDBGetProp|'
+ r'dbc_AfterDBSetProp|dbc_AfterDeleteConnection|'
+ r'dbc_AfterDropOffline|dbc_AfterDropTable|'
+ r'dbc_AfterModifyConnection|dbc_AfterModifyProc|'
+ r'dbc_AfterModifyTable|dbc_AfterModifyView|dbc_AfterOpenTable|'
+ r'dbc_AfterRemoveTable|dbc_AfterRenameConnection|'
+ r'dbc_AfterRenameTable|dbc_AfterRenameView|'
+ r'dbc_AfterValidateData|dbc_BeforeAddTable|'
+ r'dbc_BeforeAppendProc|dbc_BeforeCloseTable|'
+ r'dbc_BeforeCopyProc|dbc_BeforeCreateConnection|'
+ r'dbc_BeforeCreateOffline|dbc_BeforeCreateTable|'
+ r'dbc_BeforeCreateView|dbc_BeforeDBGetProp|'
+ r'dbc_BeforeDBSetProp|dbc_BeforeDeleteConnection|'
+ r'dbc_BeforeDropOffline|dbc_BeforeDropTable|'
+ r'dbc_BeforeModifyConnection|dbc_BeforeModifyProc|'
+ r'dbc_BeforeModifyTable|dbc_BeforeModifyView|'
+ r'dbc_BeforeOpenTable|dbc_BeforeRemoveTable|'
+ r'dbc_BeforeRenameConnection|dbc_BeforeRenameTable|'
+ r'dbc_BeforeRenameView|dbc_BeforeValidateData|'
+ r'dbc_CloseData|dbc_Deactivate|dbc_ModifyData|dbc_OpenData|'
+ r'dbc_PackData|DblClick|Deactivate|Deleted|Destroy|DoCmd|'
+ r'DownClick|DragDrop|DragOver|DropDown|ErrorMessage|Error|'
+ r'EvaluateContents|GotFocus|Init|InteractiveChange|KeyPress|'
+ r'LoadReport|Load|LostFocus|Message|MiddleClick|MouseDown|'
+ r'MouseEnter|MouseLeave|MouseMove|MouseUp|MouseWheel|Moved|'
+ r'OLECompleteDrag|OLEDragOver|OLEGiveFeedback|OLESetData|'
+ r'OLEStartDrag|OnMoveItem|Paint|ProgrammaticChange|'
+ r'QueryAddFile|QueryModifyFile|QueryNewFile|QueryRemoveFile|'
+ r'QueryRunFile|QueryUnload|RangeHigh|RangeLow|ReadActivate|'
+ r'ReadDeactivate|ReadShow|ReadValid|ReadWhen|Resize|'
+ r'RightClick|SCCInit|SCCDestroy|Scrolled|Timer|UIEnable|'
+ r'UnDock|UnloadReport|Unload|UpClick|Valid|When)', Name.Function),
+
+ (r'\s+', Text),
+ # everything else is not colored
+ (r'.', Text),
+ ],
+ 'newline': [
+ (r'\*.*?$', Comment.Single, '#pop'),
+ (r'(ACCEPT|ACTIVATE\s*MENU|ACTIVATE\s*POPUP|ACTIVATE\s*SCREEN|'
+ r'ACTIVATE\s*WINDOW|APPEND|APPEND\s*FROM|APPEND\s*FROM\s*ARRAY|'
+ r'APPEND\s*GENERAL|APPEND\s*MEMO|ASSIST|AVERAGE|BLANK|BROWSE|'
+ r'BUILD\s*APP|BUILD\s*EXE|BUILD\s*PROJECT|CALCULATE|CALL|'
+ r'CANCEL|CHANGE|CLEAR|CLOSE|CLOSE\s*MEMO|COMPILE|CONTINUE|'
+ r'COPY\s*FILE|COPY\s*INDEXES|COPY\s*MEMO|COPY\s*STRUCTURE|'
+ r'COPY\s*STRUCTURE\s*EXTENDED|COPY\s*TAG|COPY\s*TO|'
+ r'COPY\s*TO\s*ARRAY|COUNT|CREATE|CREATE\s*COLOR\s*SET|'
+ r'CREATE\s*CURSOR|CREATE\s*FROM|CREATE\s*LABEL|CREATE\s*MENU|'
+ r'CREATE\s*PROJECT|CREATE\s*QUERY|CREATE\s*REPORT|'
+ r'CREATE\s*SCREEN|CREATE\s*TABLE|CREATE\s*VIEW|DDE|'
+ r'DEACTIVATE\s*MENU|DEACTIVATE\s*POPUP|DEACTIVATE\s*WINDOW|'
+ r'DECLARE|DEFINE\s*BAR|DEFINE\s*BOX|DEFINE\s*MENU|'
+ r'DEFINE\s*PAD|DEFINE\s*POPUP|DEFINE\s*WINDOW|DELETE|'
+ r'DELETE\s*FILE|DELETE\s*TAG|DIMENSION|DIRECTORY|DISPLAY|'
+ r'DISPLAY\s*FILES|DISPLAY\s*MEMORY|DISPLAY\s*STATUS|'
+ r'DISPLAY\s*STRUCTURE|DO|EDIT|EJECT|EJECT\s*PAGE|ERASE|'
+ r'EXIT|EXPORT|EXTERNAL|FILER|FIND|FLUSH|FUNCTION|GATHER|'
+ r'GETEXPR|GO|GOTO|HELP|HIDE\s*MENU|HIDE\s*POPUP|'
+ r'HIDE\s*WINDOW|IMPORT|INDEX|INPUT|INSERT|JOIN|KEYBOARD|'
+ r'LABEL|LIST|LOAD|LOCATE|LOOP|MENU|MENU\s*TO|MODIFY\s*COMMAND|'
+ r'MODIFY\s*FILE|MODIFY\s*GENERAL|MODIFY\s*LABEL|MODIFY\s*MEMO|'
+ r'MODIFY\s*MENU|MODIFY\s*PROJECT|MODIFY\s*QUERY|'
+ r'MODIFY\s*REPORT|MODIFY\s*SCREEN|MODIFY\s*STRUCTURE|'
+ r'MODIFY\s*WINDOW|MOVE\s*POPUP|MOVE\s*WINDOW|NOTE|'
+ r'ON\s*APLABOUT|ON\s*BAR|ON\s*ERROR|ON\s*ESCAPE|'
+ r'ON\s*EXIT\s*BAR|ON\s*EXIT\s*MENU|ON\s*EXIT\s*PAD|'
+ r'ON\s*EXIT\s*POPUP|ON\s*KEY|ON\s*KEY\s*=|ON\s*KEY\s*LABEL|'
+ r'ON\s*MACHELP|ON\s*PAD|ON\s*PAGE|ON\s*READERROR|'
+ r'ON\s*SELECTION\s*BAR|ON\s*SELECTION\s*MENU|'
+ r'ON\s*SELECTION\s*PAD|ON\s*SELECTION\s*POPUP|ON\s*SHUTDOWN|'
+ r'PACK|PARAMETERS|PLAY\s*MACRO|POP\s*KEY|POP\s*MENU|'
+ r'POP\s*POPUP|PRIVATE|PROCEDURE|PUBLIC|PUSH\s*KEY|'
+ r'PUSH\s*MENU|PUSH\s*POPUP|QUIT|READ|READ\s*MENU|RECALL|'
+ r'REINDEX|RELEASE|RELEASE\s*MODULE|RENAME|REPLACE|'
+ r'REPLACE\s*FROM\s*ARRAY|REPORT|RESTORE\s*FROM|'
+ r'RESTORE\s*MACROS|RESTORE\s*SCREEN|RESTORE\s*WINDOW|'
+ r'RESUME|RETRY|RETURN|RUN|RUN\s*\/N"|RUNSCRIPT|'
+ r'SAVE\s*MACROS|SAVE\s*SCREEN|SAVE\s*TO|SAVE\s*WINDOWS|'
+ r'SCATTER|SCROLL|SEEK|SELECT|SET|SET\s*ALTERNATE|'
+ r'SET\s*ANSI|SET\s*APLABOUT|SET\s*AUTOSAVE|SET\s*BELL|'
+ r'SET\s*BLINK|SET\s*BLOCKSIZE|SET\s*BORDER|SET\s*BRSTATUS|'
+ r'SET\s*CARRY|SET\s*CENTURY|SET\s*CLEAR|SET\s*CLOCK|'
+ r'SET\s*COLLATE|SET\s*COLOR\s*OF|SET\s*COLOR\s*OF\s*SCHEME|'
+ r'SET\s*COLOR\s*SET|SET\s*COLOR\s*TO|SET\s*COMPATIBLE|'
+ r'SET\s*CONFIRM|SET\s*CONSOLE|SET\s*CURRENCY|SET\s*CURSOR|'
+ r'SET\s*DATE|SET\s*DEBUG|SET\s*DECIMALS|SET\s*DEFAULT|'
+ r'SET\s*DELETED|SET\s*DELIMITERS|SET\s*DEVELOPMENT|'
+ r'SET\s*DEVICE|SET\s*DISPLAY|SET\s*DOHISTORY|SET\s*ECHO|'
+ r'SET\s*ESCAPE|SET\s*EXACT|SET\s*EXCLUSIVE|SET\s*FIELDS|'
+ r'SET\s*FILTER|SET\s*FIXED|SET\s*FORMAT|SET\s*FULLPATH|'
+ r'SET\s*FUNCTION|SET\s*HEADINGS|SET\s*HELP|SET\s*HELPFILTER|'
+ r'SET\s*HOURS|SET\s*INDEX|SET\s*INTENSITY|SET\s*KEY|'
+ r'SET\s*KEYCOMP|SET\s*LIBRARY|SET\s*LOCK|SET\s*LOGERRORS|'
+ r'SET\s*MACDESKTOP|SET\s*MACHELP|SET\s*MACKEY|SET\s*MARGIN|'
+ r'SET\s*MARK\s*OF|SET\s*MARK\s*TO|SET\s*MEMOWIDTH|'
+ r'SET\s*MESSAGE|SET\s*MOUSE|SET\s*MULTILOCKS|SET\s*NEAR|'
+ r'SET\s*NOCPTRANS|SET\s*NOTIFY|SET\s*ODOMETER|SET\s*OPTIMIZE|'
+ r'SET\s*ORDER|SET\s*PALETTE|SET\s*PATH|SET\s*PDSETUP|'
+ r'SET\s*POINT|SET\s*PRINTER|SET\s*PROCEDURE|SET\s*READBORDER|'
+ r'SET\s*REFRESH|SET\s*RELATION|SET\s*RELATION\s*OFF|'
+ r'SET\s*REPROCESS|SET\s*RESOURCE|SET\s*SAFETY|SET\s*SCOREBOARD|'
+ r'SET\s*SEPARATOR|SET\s*SHADOWS|SET\s*SKIP|SET\s*SKIP\s*OF|'
+ r'SET\s*SPACE|SET\s*STATUS|SET\s*STATUS\s*BAR|SET\s*STEP|'
+ r'SET\s*STICKY|SET\s*SYSMENU|SET\s*TALK|SET\s*TEXTMERGE|'
+ r'SET\s*TEXTMERGE\s*DELIMITERS|SET\s*TOPIC|SET\s*TRBETWEEN|'
+ r'SET\s*TYPEAHEAD|SET\s*UDFPARMS|SET\s*UNIQUE|SET\s*VIEW|'
+ r'SET\s*VOLUME|SET\s*WINDOW\s*OF\s*MEMO|SET\s*XCMDFILE|'
+ r'SHOW\s*GET|SHOW\s*GETS|SHOW\s*MENU|SHOW\s*OBJECT|'
+ r'SHOW\s*POPUP|SHOW\s*WINDOW|SIZE\s*POPUP|SKIP|SORT|'
+ r'STORE|SUM|SUSPEND|TOTAL|TYPE|UNLOCK|UPDATE|USE|WAIT|'
+ r'ZAP|ZOOM\s*WINDOW|DO\s*CASE|CASE|OTHERWISE|ENDCASE|'
+ r'DO\s*WHILE|ENDDO|FOR|ENDFOR|NEXT|IF|ELSE|ENDIF|PRINTJOB|'
+ r'ENDPRINTJOB|SCAN|ENDSCAN|TEXT|ENDTEXT|=)',
+ Keyword.Reserved, '#pop'),
+ (r'#\s*(IF|ELIF|ELSE|ENDIF|DEFINE|IFDEF|IFNDEF|INCLUDE)',
+ Comment.Preproc, '#pop'),
+ (r'(m\.)?[a-z_]\w*', Name.Variable, '#pop'),
+ (r'.', Text, '#pop'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/functional.py b/wandb/vendor/pygments/lexers/functional.py
new file mode 100644
index 0000000000000000000000000000000000000000..254df795f6b54090a0d00bf96e8c235240b17258
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/functional.py
@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.functional
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Just export lexer classes previously contained in this module.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexers.lisp import SchemeLexer, CommonLispLexer, RacketLexer, \
+ NewLispLexer, ShenLexer
+from pygments.lexers.haskell import HaskellLexer, LiterateHaskellLexer, \
+ KokaLexer
+from pygments.lexers.theorem import CoqLexer
+from pygments.lexers.erlang import ErlangLexer, ErlangShellLexer, \
+ ElixirConsoleLexer, ElixirLexer
+from pygments.lexers.ml import SMLLexer, OcamlLexer, OpaLexer
+
+__all__ = []
diff --git a/wandb/vendor/pygments/lexers/go.py b/wandb/vendor/pygments/lexers/go.py
new file mode 100644
index 0000000000000000000000000000000000000000..cc2a6d631225b683622f12b87d8600d055b1512c
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/go.py
@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.go
+ ~~~~~~~~~~~~~~~~~~
+
+ Lexers for the Google Go language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['GoLexer']
+
+
+class GoLexer(RegexLexer):
+ """
+ For `Go <http://golang.org>`_ source.
+
+ .. versionadded:: 1.2
+ """
+ name = 'Go'
+ filenames = ['*.go']
+ aliases = ['go']
+ mimetypes = ['text/x-gosrc']
+
+ flags = re.MULTILINE | re.UNICODE
+
+ tokens = {
+ 'root': [
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'\\\n', Text), # line continuations
+ (r'//(.*?)\n', Comment.Single),
+ (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+ (r'(import|package)\b', Keyword.Namespace),
+ (r'(var|func|struct|map|chan|type|interface|const)\b',
+ Keyword.Declaration),
+ (words((
+ 'break', 'default', 'select', 'case', 'defer', 'go',
+ 'else', 'goto', 'switch', 'fallthrough', 'if', 'range',
+ 'continue', 'for', 'return'), suffix=r'\b'),
+ Keyword),
+ (r'(true|false|iota|nil)\b', Keyword.Constant),
+ # It seems the builtin types aren't actually keywords, but
+ # can be used as functions. So we need two declarations.
+ (words((
+ 'uint', 'uint8', 'uint16', 'uint32', 'uint64',
+ 'int', 'int8', 'int16', 'int32', 'int64',
+ 'float', 'float32', 'float64',
+ 'complex64', 'complex128', 'byte', 'rune',
+ 'string', 'bool', 'error', 'uintptr',
+ 'print', 'println', 'panic', 'recover', 'close', 'complex',
+ 'real', 'imag', 'len', 'cap', 'append', 'copy', 'delete',
+ 'new', 'make'), suffix=r'\b(\()'),
+ bygroups(Name.Builtin, Punctuation)),
+ (words((
+ 'uint', 'uint8', 'uint16', 'uint32', 'uint64',
+ 'int', 'int8', 'int16', 'int32', 'int64',
+ 'float', 'float32', 'float64',
+ 'complex64', 'complex128', 'byte', 'rune',
+ 'string', 'bool', 'error', 'uintptr'), suffix=r'\b'),
+ Keyword.Type),
+ # imaginary_lit
+ (r'\d+i', Number),
+ (r'\d+\.\d*([Ee][-+]\d+)?i', Number),
+ (r'\.\d+([Ee][-+]\d+)?i', Number),
+ (r'\d+[Ee][-+]\d+i', Number),
+ # float_lit
+ (r'\d+(\.\d+[eE][+\-]?\d+|'
+ r'\.\d*|[eE][+\-]?\d+)', Number.Float),
+ (r'\.\d+([eE][+\-]?\d+)?', Number.Float),
+ # int_lit
+ # -- octal_lit
+ (r'0[0-7]+', Number.Oct),
+ # -- hex_lit
+ (r'0[xX][0-9a-fA-F]+', Number.Hex),
+ # -- decimal_lit
+ (r'(0|[1-9][0-9]*)', Number.Integer),
+ # char_lit
+ (r"""'(\\['"\\abfnrtv]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}"""
+ r"""|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|[^\\])'""",
+ String.Char),
+ # StringLiteral
+ # -- raw_string_lit
+ (r'`[^`]*`', String),
+ # -- interpreted_string_lit
+ (r'"(\\\\|\\"|[^"])*"', String),
+ # Tokens
+ (r'(<<=|>>=|<<|>>|<=|>=|&\^=|&\^|\+=|-=|\*=|/=|%=|&=|\|=|&&|\|\|'
+ r'|<-|\+\+|--|==|!=|:=|\.\.\.|[+\-*/%&])', Operator),
+ (r'[|^<>=!()\[\]{}.,;:]', Punctuation),
+ # identifier
+ (r'[^\W\d]\w*', Name.Other),
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/grammar_notation.py b/wandb/vendor/pygments/lexers/grammar_notation.py
new file mode 100644
index 0000000000000000000000000000000000000000..076249d3df6c4f81a38e8c76172d1589d5e35840
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/grammar_notation.py
@@ -0,0 +1,213 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.grammar_notation
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for grammer notations like BNF.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, include, this, using, words
+from pygments.token import Comment, Keyword, Literal, Name, Number, \
+ Operator, Punctuation, String, Text
+
+__all__ = ['BnfLexer', 'AbnfLexer', 'JsgfLexer']
+
+
+class BnfLexer(RegexLexer):
+ """
+ This lexer is for grammer notations which are similar to
+ original BNF.
+
+ In order to maximize a number of targets of this lexer,
+ let's decide some designs:
+
+ * We don't distinguish `Terminal Symbol`.
+
+ * We do assume that `NonTerminal Symbol` are always enclosed
+ with arrow brackets.
+
+ * We do assume that `NonTerminal Symbol` may include
+ any printable characters except arrow brackets and ASCII 0x20.
+ This assumption is for `RBNF <http://www.rfc-base.org/txt/rfc-5511.txt>`_.
+
+ * We do assume that target notation doesn't support comment.
+
+ * We don't distinguish any operators and punctuation except
+ `::=`.
+
+ Though these desision making might cause too minimal highlighting
+ and you might be disappointed, but it is reasonable for us.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'BNF'
+ aliases = ['bnf']
+ filenames = ['*.bnf']
+ mimetypes = ['text/x-bnf']
+
+ tokens = {
+ 'root': [
+ (r'(<)([ -;=?-~]+)(>)',
+ bygroups(Punctuation, Name.Class, Punctuation)),
+
+ # an only operator
+ (r'::=', Operator),
+
+ # fallback
+ (r'[^<>:]+', Text), # for performance
+ (r'.', Text),
+ ],
+ }
+
+
+class AbnfLexer(RegexLexer):
+ """
+ Lexer for `IETF 7405 ABNF
+ <http://www.ietf.org/rfc/rfc7405.txt>`_
+ (Updates `5234 <http://www.ietf.org/rfc/rfc5234.txt>`_)
+ grammars.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'ABNF'
+ aliases = ['abnf']
+ filenames = ['*.abnf']
+ mimetypes = ['text/x-abnf']
+
+ _core_rules = (
+ 'ALPHA', 'BIT', 'CHAR', 'CR', 'CRLF', 'CTL', 'DIGIT',
+ 'DQUOTE', 'HEXDIG', 'HTAB', 'LF', 'LWSP', 'OCTET',
+ 'SP', 'VCHAR', 'WSP')
+
+ tokens = {
+ 'root': [
+ # comment
+ (r';.*$', Comment.Single),
+
+ # quoted
+ # double quote itself in this state, it is as '%x22'.
+ (r'(%[si])?"[^"]*"', Literal),
+
+ # binary (but i have never seen...)
+ (r'%b[01]+\-[01]+\b', Literal), # range
+ (r'%b[01]+(\.[01]+)*\b', Literal), # concat
+
+ # decimal
+ (r'%d[0-9]+\-[0-9]+\b', Literal), # range
+ (r'%d[0-9]+(\.[0-9]+)*\b', Literal), # concat
+
+ # hexadecimal
+ (r'%x[0-9a-fA-F]+\-[0-9a-fA-F]+\b', Literal), # range
+ (r'%x[0-9a-fA-F]+(\.[0-9a-fA-F]+)*\b', Literal), # concat
+
+ # repetition (<a>*<b>element) including nRule
+ (r'\b[0-9]+\*[0-9]+', Operator),
+ (r'\b[0-9]+\*', Operator),
+ (r'\b[0-9]+', Operator),
+ (r'\*', Operator),
+
+ # Strictly speaking, these are not keyword but
+ # are called `Core Rule'.
+ (words(_core_rules, suffix=r'\b'), Keyword),
+
+ # nonterminals (ALPHA *(ALPHA / DIGIT / "-"))
+ (r'[a-zA-Z][a-zA-Z0-9-]+\b', Name.Class),
+
+ # operators
+ (r'(=/|=|/)', Operator),
+
+ # punctuation
+ (r'[\[\]()]', Punctuation),
+
+ # fallback
+ (r'\s+', Text),
+ (r'.', Text),
+ ],
+ }
+
+
+class JsgfLexer(RegexLexer):
+ """
+ For `JSpeech Grammar Format <https://www.w3.org/TR/jsgf/>`_
+ grammars.
+
+ .. versionadded:: 2.2
+ """
+ name = 'JSGF'
+ aliases = ['jsgf']
+ filenames = ['*.jsgf']
+ mimetypes = ['application/jsgf', 'application/x-jsgf', 'text/jsgf']
+
+ flags = re.MULTILINE | re.UNICODE
+
+ tokens = {
+ 'root': [
+ include('comments'),
+ include('non-comments'),
+ ],
+ 'comments': [
+ (r'/\*\*(?!/)', Comment.Multiline, 'documentation comment'),
+ (r'/\*[\w\W]*?\*/', Comment.Multiline),
+ (r'//.*', Comment.Single),
+ ],
+ 'non-comments': [
+ ('\A#JSGF[^;]*', Comment.Preproc),
+ (r'\s+', Text),
+ (r';', Punctuation),
+ (r'[=|()\[\]*+]', Operator),
+ (r'/[^/]+/', Number.Float),
+ (r'"', String.Double, 'string'),
+ (r'\{', String.Other, 'tag'),
+ (words(('import', 'public'), suffix=r'\b'), Keyword.Reserved),
+ (r'grammar\b', Keyword.Reserved, 'grammar name'),
+ (r'(<)(NULL|VOID)(>)',
+ bygroups(Punctuation, Name.Builtin, Punctuation)),
+ (r'<', Punctuation, 'rulename'),
+ (r'\w+|[^\s;=|()\[\]*+/"{<\w]+', Text),
+ ],
+ 'string': [
+ (r'"', String.Double, '#pop'),
+ (r'\\.', String.Escape),
+ (r'[^\\"]+', String.Double),
+ ],
+ 'tag': [
+ (r'\}', String.Other, '#pop'),
+ (r'\\.', String.Escape),
+ (r'[^\\}]+', String.Other),
+ ],
+ 'grammar name': [
+ (r';', Punctuation, '#pop'),
+ (r'\s+', Text),
+ (r'\.', Punctuation),
+ (r'[^;\s.]+', Name.Namespace),
+ ],
+ 'rulename': [
+ (r'>', Punctuation, '#pop'),
+ (r'\*', Punctuation),
+ (r'\s+', Text),
+ (r'([^.>]+)(\s*)(\.)', bygroups(Name.Namespace, Text, Punctuation)),
+ (r'[^.>]+', Name.Constant),
+ ],
+ 'documentation comment': [
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'(^\s*\*?\s*)(@(?:example|see)\s+)'
+ r'([\w\W]*?(?=(?:^\s*\*?\s*@|\*/)))',
+ bygroups(Comment.Multiline, Comment.Special,
+ using(this, state='example'))),
+ (r'(^\s*\*?\s*)(@\S*)',
+ bygroups(Comment.Multiline, Comment.Special)),
+ (r'[^*\n@]+|\w|\W', Comment.Multiline),
+ ],
+ 'example': [
+ (r'\n\s*\*', Comment.Multiline),
+ include('non-comments'),
+ (r'.', Comment.Multiline),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/graph.py b/wandb/vendor/pygments/lexers/graph.py
new file mode 100644
index 0000000000000000000000000000000000000000..1a33824603a62ff243a7fec94c576dd447a75625
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/graph.py
@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.graph
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for graph query languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, this
+from pygments.token import Keyword, Punctuation, Comment, Operator, Name,\
+ String, Number, Whitespace
+
+
+__all__ = ['CypherLexer']
+
+
+class CypherLexer(RegexLexer):
+ """
+ For `Cypher Query Language
+ <http://docs.neo4j.org/chunked/milestone/cypher-query-lang.html>`_
+
+ For the Cypher version in Neo4J 2.0
+
+ .. versionadded:: 2.0
+ """
+ name = 'Cypher'
+ aliases = ['cypher']
+ filenames = ['*.cyp', '*.cypher']
+
+ flags = re.MULTILINE | re.IGNORECASE
+
+ tokens = {
+ 'root': [
+ include('comment'),
+ include('keywords'),
+ include('clauses'),
+ include('relations'),
+ include('strings'),
+ include('whitespace'),
+ include('barewords'),
+ ],
+ 'comment': [
+ (r'^.*//.*\n', Comment.Single),
+ ],
+ 'keywords': [
+ (r'(create|order|match|limit|set|skip|start|return|with|where|'
+ r'delete|foreach|not|by)\b', Keyword),
+ ],
+ 'clauses': [
+ # TODO: many missing ones, see http://docs.neo4j.org/refcard/2.0/
+ (r'(all|any|as|asc|create|create\s+unique|delete|'
+ r'desc|distinct|foreach|in|is\s+null|limit|match|none|'
+ r'order\s+by|return|set|skip|single|start|union|where|with)\b',
+ Keyword),
+ ],
+ 'relations': [
+ (r'(-\[)(.*?)(\]->)', bygroups(Operator, using(this), Operator)),
+ (r'(<-\[)(.*?)(\]-)', bygroups(Operator, using(this), Operator)),
+ (r'(-\[)(.*?)(\]-)', bygroups(Operator, using(this), Operator)),
+ (r'-->|<--|\[|\]', Operator),
+ (r'<|>|<>|=|<=|=>|\(|\)|\||:|,|;', Punctuation),
+ (r'[.*{}]', Punctuation),
+ ],
+ 'strings': [
+ (r'"(?:\\[tbnrf\'"\\]|[^\\"])*"', String),
+ (r'`(?:``|[^`])+`', Name.Variable),
+ ],
+ 'whitespace': [
+ (r'\s+', Whitespace),
+ ],
+ 'barewords': [
+ (r'[a-z]\w*', Name),
+ (r'\d+', Number),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/graphics.py b/wandb/vendor/pygments/lexers/graphics.py
new file mode 100644
index 0000000000000000000000000000000000000000..c8af9f9916eb453e44d6ff1e950bf38c6ee85778
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/graphics.py
@@ -0,0 +1,553 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.graphics
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for computer graphics and plotting related languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words, include, bygroups, using, \
+ this, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, \
+ Number, Punctuation, String
+
+__all__ = ['GLShaderLexer', 'PostScriptLexer', 'AsymptoteLexer', 'GnuplotLexer',
+ 'PovrayLexer']
+
+
+class GLShaderLexer(RegexLexer):
+ """
+ GLSL (OpenGL Shader) lexer.
+
+ .. versionadded:: 1.1
+ """
+ name = 'GLSL'
+ aliases = ['glsl']
+ filenames = ['*.vert', '*.frag', '*.geo']
+ mimetypes = ['text/x-glslsrc']
+
+ tokens = {
+ 'root': [
+ (r'^#.*', Comment.Preproc),
+ (r'//.*', Comment.Single),
+ (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+ (r'\+|-|~|!=?|\*|/|%|<<|>>|<=?|>=?|==?|&&?|\^|\|\|?',
+ Operator),
+ (r'[?:]', Operator), # quick hack for ternary
+ (r'\bdefined\b', Operator),
+ (r'[;{}(),\[\]]', Punctuation),
+ # FIXME when e is present, no decimal point needed
+ (r'[+-]?\d*\.\d+([eE][-+]?\d+)?', Number.Float),
+ (r'[+-]?\d+\.\d*([eE][-+]?\d+)?', Number.Float),
+ (r'0[xX][0-9a-fA-F]*', Number.Hex),
+ (r'0[0-7]*', Number.Oct),
+ (r'[1-9][0-9]*', Number.Integer),
+ (words((
+ 'attribute', 'const', 'uniform', 'varying', 'centroid', 'break',
+ 'continue', 'do', 'for', 'while', 'if', 'else', 'in', 'out',
+ 'inout', 'float', 'int', 'void', 'bool', 'true', 'false',
+ 'invariant', 'discard', 'return', 'mat2', 'mat3' 'mat4',
+ 'mat2x2', 'mat3x2', 'mat4x2', 'mat2x3', 'mat3x3', 'mat4x3',
+ 'mat2x4', 'mat3x4', 'mat4x4', 'vec2', 'vec3', 'vec4',
+ 'ivec2', 'ivec3', 'ivec4', 'bvec2', 'bvec3', 'bvec4',
+ 'sampler1D', 'sampler2D', 'sampler3D' 'samplerCube',
+ 'sampler1DShadow', 'sampler2DShadow', 'struct'),
+ prefix=r'\b', suffix=r'\b'),
+ Keyword),
+ (words((
+ 'asm', 'class', 'union', 'enum', 'typedef', 'template', 'this',
+ 'packed', 'goto', 'switch', 'default', 'inline', 'noinline',
+ 'volatile', 'public', 'static', 'extern', 'external', 'interface',
+ 'long', 'short', 'double', 'half', 'fixed', 'unsigned', 'lowp',
+ 'mediump', 'highp', 'precision', 'input', 'output',
+ 'hvec2', 'hvec3', 'hvec4', 'dvec2', 'dvec3', 'dvec4',
+ 'fvec2', 'fvec3', 'fvec4', 'sampler2DRect', 'sampler3DRect',
+ 'sampler2DRectShadow', 'sizeof', 'cast', 'namespace', 'using'),
+ prefix=r'\b', suffix=r'\b'),
+ Keyword), # future use
+ (r'[a-zA-Z_]\w*', Name),
+ (r'\.', Punctuation),
+ (r'\s+', Text),
+ ],
+ }
+
+
+class PostScriptLexer(RegexLexer):
+ """
+ Lexer for PostScript files.
+
+ The PostScript Language Reference published by Adobe at
+ <http://partners.adobe.com/public/developer/en/ps/PLRM.pdf>
+ is the authority for this.
+
+ .. versionadded:: 1.4
+ """
+ name = 'PostScript'
+ aliases = ['postscript', 'postscr']
+ filenames = ['*.ps', '*.eps']
+ mimetypes = ['application/postscript']
+
+ delimiter = r'()<>\[\]{}/%\s'
+ delimiter_end = r'(?=[%s])' % delimiter
+
+ valid_name_chars = r'[^%s]' % delimiter
+ valid_name = r"%s+%s" % (valid_name_chars, delimiter_end)
+
+ tokens = {
+ 'root': [
+ # All comment types
+ (r'^%!.+\n', Comment.Preproc),
+ (r'%%.*\n', Comment.Special),
+ (r'(^%.*\n){2,}', Comment.Multiline),
+ (r'%.*\n', Comment.Single),
+
+ # String literals are awkward; enter separate state.
+ (r'\(', String, 'stringliteral'),
+
+ (r'[{}<>\[\]]', Punctuation),
+
+ # Numbers
+ (r'<[0-9A-Fa-f]+>' + delimiter_end, Number.Hex),
+ # Slight abuse: use Oct to signify any explicit base system
+ (r'[0-9]+\#(\-|\+)?([0-9]+\.?|[0-9]*\.[0-9]+|[0-9]+\.[0-9]*)'
+ r'((e|E)[0-9]+)?' + delimiter_end, Number.Oct),
+ (r'(\-|\+)?([0-9]+\.?|[0-9]*\.[0-9]+|[0-9]+\.[0-9]*)((e|E)[0-9]+)?'
+ + delimiter_end, Number.Float),
+ (r'(\-|\+)?[0-9]+' + delimiter_end, Number.Integer),
+
+ # References
+ (r'\/%s' % valid_name, Name.Variable),
+
+ # Names
+ (valid_name, Name.Function), # Anything else is executed
+
+ # These keywords taken from
+ # <http://www.math.ubc.ca/~cass/graphics/manual/pdf/a1.pdf>
+ # Is there an authoritative list anywhere that doesn't involve
+ # trawling documentation?
+
+ (r'(false|true)' + delimiter_end, Keyword.Constant),
+
+ # Conditionals / flow control
+ (r'(eq|ne|g[et]|l[et]|and|or|not|if(?:else)?|for(?:all)?)'
+ + delimiter_end, Keyword.Reserved),
+
+ (words((
+ 'abs', 'add', 'aload', 'arc', 'arcn', 'array', 'atan', 'begin',
+ 'bind', 'ceiling', 'charpath', 'clip', 'closepath', 'concat',
+ 'concatmatrix', 'copy', 'cos', 'currentlinewidth', 'currentmatrix',
+ 'currentpoint', 'curveto', 'cvi', 'cvs', 'def', 'defaultmatrix',
+ 'dict', 'dictstackoverflow', 'div', 'dtransform', 'dup', 'end',
+ 'exch', 'exec', 'exit', 'exp', 'fill', 'findfont', 'floor', 'get',
+ 'getinterval', 'grestore', 'gsave', 'gt', 'identmatrix', 'idiv',
+ 'idtransform', 'index', 'invertmatrix', 'itransform', 'length',
+ 'lineto', 'ln', 'load', 'log', 'loop', 'matrix', 'mod', 'moveto',
+ 'mul', 'neg', 'newpath', 'pathforall', 'pathbbox', 'pop', 'print',
+ 'pstack', 'put', 'quit', 'rand', 'rangecheck', 'rcurveto', 'repeat',
+ 'restore', 'rlineto', 'rmoveto', 'roll', 'rotate', 'round', 'run',
+ 'save', 'scale', 'scalefont', 'setdash', 'setfont', 'setgray',
+ 'setlinecap', 'setlinejoin', 'setlinewidth', 'setmatrix',
+ 'setrgbcolor', 'shfill', 'show', 'showpage', 'sin', 'sqrt',
+ 'stack', 'stringwidth', 'stroke', 'strokepath', 'sub', 'syntaxerror',
+ 'transform', 'translate', 'truncate', 'typecheck', 'undefined',
+ 'undefinedfilename', 'undefinedresult'), suffix=delimiter_end),
+ Name.Builtin),
+
+ (r'\s+', Text),
+ ],
+
+ 'stringliteral': [
+ (r'[^()\\]+', String),
+ (r'\\', String.Escape, 'escape'),
+ (r'\(', String, '#push'),
+ (r'\)', String, '#pop'),
+ ],
+
+ 'escape': [
+ (r'[0-8]{3}|n|r|t|b|f|\\|\(|\)', String.Escape, '#pop'),
+ default('#pop'),
+ ],
+ }
+
+
+class AsymptoteLexer(RegexLexer):
+ """
+ For `Asymptote <http://asymptote.sf.net/>`_ source code.
+
+ .. versionadded:: 1.2
+ """
+ name = 'Asymptote'
+ aliases = ['asy', 'asymptote']
+ filenames = ['*.asy']
+ mimetypes = ['text/x-asymptote']
+
+ #: optional Comment or Whitespace
+ _ws = r'(?:\s|//.*?\n|/\*.*?\*/)+'
+
+ tokens = {
+ 'whitespace': [
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'\\\n', Text), # line continuation
+ (r'//(\n|(.|\n)*?[^\\]\n)', Comment),
+ (r'/(\\\n)?\*(.|\n)*?\*(\\\n)?/', Comment),
+ ],
+ 'statements': [
+ # simple string (TeX friendly)
+ (r'"(\\\\|\\"|[^"])*"', String),
+ # C style string (with character escapes)
+ (r"'", String, 'string'),
+ (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float),
+ (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+ (r'0x[0-9a-fA-F]+[Ll]?', Number.Hex),
+ (r'0[0-7]+[Ll]?', Number.Oct),
+ (r'\d+[Ll]?', Number.Integer),
+ (r'[~!%^&*+=|?:<>/-]', Operator),
+ (r'[()\[\],.]', Punctuation),
+ (r'\b(case)(.+?)(:)', bygroups(Keyword, using(this), Text)),
+ (r'(and|controls|tension|atleast|curl|if|else|while|for|do|'
+ r'return|break|continue|struct|typedef|new|access|import|'
+ r'unravel|from|include|quote|static|public|private|restricted|'
+ r'this|explicit|true|false|null|cycle|newframe|operator)\b', Keyword),
+ # Since an asy-type-name can be also an asy-function-name,
+ # in the following we test if the string " [a-zA-Z]" follows
+ # the Keyword.Type.
+ # Of course it is not perfect !
+ (r'(Braid|FitResult|Label|Legend|TreeNode|abscissa|arc|arrowhead|'
+ r'binarytree|binarytreeNode|block|bool|bool3|bounds|bqe|circle|'
+ r'conic|coord|coordsys|cputime|ellipse|file|filltype|frame|grid3|'
+ r'guide|horner|hsv|hyperbola|indexedTransform|int|inversion|key|'
+ r'light|line|linefit|marginT|marker|mass|object|pair|parabola|path|'
+ r'path3|pen|picture|point|position|projection|real|revolution|'
+ r'scaleT|scientific|segment|side|slice|splitface|string|surface|'
+ r'tensionSpecifier|ticklocate|ticksgridT|tickvalues|transform|'
+ r'transformation|tree|triangle|trilinear|triple|vector|'
+ r'vertex|void)(?=\s+[a-zA-Z])', Keyword.Type),
+ # Now the asy-type-name which are not asy-function-name
+ # except yours !
+ # Perhaps useless
+ (r'(Braid|FitResult|TreeNode|abscissa|arrowhead|block|bool|bool3|'
+ r'bounds|coord|frame|guide|horner|int|linefit|marginT|pair|pen|'
+ r'picture|position|real|revolution|slice|splitface|ticksgridT|'
+ r'tickvalues|tree|triple|vertex|void)\b', Keyword.Type),
+ ('[a-zA-Z_]\w*:(?!:)', Name.Label),
+ ('[a-zA-Z_]\w*', Name),
+ ],
+ 'root': [
+ include('whitespace'),
+ # functions
+ (r'((?:[\w*\s])+?(?:\s|\*))' # return arguments
+ r'([a-zA-Z_]\w*)' # method name
+ r'(\s*\([^;]*?\))' # signature
+ r'(' + _ws + r')(\{)',
+ bygroups(using(this), Name.Function, using(this), using(this),
+ Punctuation),
+ 'function'),
+ # function declarations
+ (r'((?:[\w*\s])+?(?:\s|\*))' # return arguments
+ r'([a-zA-Z_]\w*)' # method name
+ r'(\s*\([^;]*?\))' # signature
+ r'(' + _ws + r')(;)',
+ bygroups(using(this), Name.Function, using(this), using(this),
+ Punctuation)),
+ default('statement'),
+ ],
+ 'statement': [
+ include('whitespace'),
+ include('statements'),
+ ('[{}]', Punctuation),
+ (';', Punctuation, '#pop'),
+ ],
+ 'function': [
+ include('whitespace'),
+ include('statements'),
+ (';', Punctuation),
+ (r'\{', Punctuation, '#push'),
+ (r'\}', Punctuation, '#pop'),
+ ],
+ 'string': [
+ (r"'", String, '#pop'),
+ (r'\\([\\abfnrtv"\'?]|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+ (r'\n', String),
+ (r"[^\\'\n]+", String), # all other characters
+ (r'\\\n', String),
+ (r'\\n', String), # line continuation
+ (r'\\', String), # stray backslash
+ ],
+ }
+
+ def get_tokens_unprocessed(self, text):
+ from pygments.lexers._asy_builtins import ASYFUNCNAME, ASYVARNAME
+ for index, token, value in \
+ RegexLexer.get_tokens_unprocessed(self, text):
+ if token is Name and value in ASYFUNCNAME:
+ token = Name.Function
+ elif token is Name and value in ASYVARNAME:
+ token = Name.Variable
+ yield index, token, value
+
+
+def _shortened(word):
+ dpos = word.find('$')
+ return '|'.join(word[:dpos] + word[dpos+1:i] + r'\b'
+ for i in range(len(word), dpos, -1))
+
+
+def _shortened_many(*words):
+ return '|'.join(map(_shortened, words))
+
+
+class GnuplotLexer(RegexLexer):
+ """
+ For `Gnuplot <http://gnuplot.info/>`_ plotting scripts.
+
+ .. versionadded:: 0.11
+ """
+
+ name = 'Gnuplot'
+ aliases = ['gnuplot']
+ filenames = ['*.plot', '*.plt']
+ mimetypes = ['text/x-gnuplot']
+
+ tokens = {
+ 'root': [
+ include('whitespace'),
+ (_shortened('bi$nd'), Keyword, 'bind'),
+ (_shortened_many('ex$it', 'q$uit'), Keyword, 'quit'),
+ (_shortened('f$it'), Keyword, 'fit'),
+ (r'(if)(\s*)(\()', bygroups(Keyword, Text, Punctuation), 'if'),
+ (r'else\b', Keyword),
+ (_shortened('pa$use'), Keyword, 'pause'),
+ (_shortened_many('p$lot', 'rep$lot', 'sp$lot'), Keyword, 'plot'),
+ (_shortened('sa$ve'), Keyword, 'save'),
+ (_shortened('se$t'), Keyword, ('genericargs', 'optionarg')),
+ (_shortened_many('sh$ow', 'uns$et'),
+ Keyword, ('noargs', 'optionarg')),
+ (_shortened_many('low$er', 'ra$ise', 'ca$ll', 'cd$', 'cl$ear',
+ 'h$elp', '\\?$', 'hi$story', 'l$oad', 'pr$int',
+ 'pwd$', 're$read', 'res$et', 'scr$eendump',
+ 'she$ll', 'sy$stem', 'up$date'),
+ Keyword, 'genericargs'),
+ (_shortened_many('pwd$', 're$read', 'res$et', 'scr$eendump',
+ 'she$ll', 'test$'),
+ Keyword, 'noargs'),
+ ('([a-zA-Z_]\w*)(\s*)(=)',
+ bygroups(Name.Variable, Text, Operator), 'genericargs'),
+ ('([a-zA-Z_]\w*)(\s*\(.*?\)\s*)(=)',
+ bygroups(Name.Function, Text, Operator), 'genericargs'),
+ (r'@[a-zA-Z_]\w*', Name.Constant), # macros
+ (r';', Keyword),
+ ],
+ 'comment': [
+ (r'[^\\\n]', Comment),
+ (r'\\\n', Comment),
+ (r'\\', Comment),
+ # don't add the newline to the Comment token
+ default('#pop'),
+ ],
+ 'whitespace': [
+ ('#', Comment, 'comment'),
+ (r'[ \t\v\f]+', Text),
+ ],
+ 'noargs': [
+ include('whitespace'),
+ # semicolon and newline end the argument list
+ (r';', Punctuation, '#pop'),
+ (r'\n', Text, '#pop'),
+ ],
+ 'dqstring': [
+ (r'"', String, '#pop'),
+ (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+ (r'[^\\"\n]+', String), # all other characters
+ (r'\\\n', String), # line continuation
+ (r'\\', String), # stray backslash
+ (r'\n', String, '#pop'), # newline ends the string too
+ ],
+ 'sqstring': [
+ (r"''", String), # escaped single quote
+ (r"'", String, '#pop'),
+ (r"[^\\'\n]+", String), # all other characters
+ (r'\\\n', String), # line continuation
+ (r'\\', String), # normal backslash
+ (r'\n', String, '#pop'), # newline ends the string too
+ ],
+ 'genericargs': [
+ include('noargs'),
+ (r'"', String, 'dqstring'),
+ (r"'", String, 'sqstring'),
+ (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+', Number.Float),
+ (r'(\d+\.\d*|\.\d+)', Number.Float),
+ (r'-?\d+', Number.Integer),
+ ('[,.~!%^&*+=|?:<>/-]', Operator),
+ ('[{}()\[\]]', Punctuation),
+ (r'(eq|ne)\b', Operator.Word),
+ (r'([a-zA-Z_]\w*)(\s*)(\()',
+ bygroups(Name.Function, Text, Punctuation)),
+ (r'[a-zA-Z_]\w*', Name),
+ (r'@[a-zA-Z_]\w*', Name.Constant), # macros
+ (r'\\\n', Text),
+ ],
+ 'optionarg': [
+ include('whitespace'),
+ (_shortened_many(
+ "a$ll", "an$gles", "ar$row", "au$toscale", "b$ars", "bor$der",
+ "box$width", "cl$abel", "c$lip", "cn$trparam", "co$ntour", "da$ta",
+ "data$file", "dg$rid3d", "du$mmy", "enc$oding", "dec$imalsign",
+ "fit$", "font$path", "fo$rmat", "fu$nction", "fu$nctions", "g$rid",
+ "hid$den3d", "his$torysize", "is$osamples", "k$ey", "keyt$itle",
+ "la$bel", "li$nestyle", "ls$", "loa$dpath", "loc$ale", "log$scale",
+ "mac$ros", "map$ping", "map$ping3d", "mar$gin", "lmar$gin",
+ "rmar$gin", "tmar$gin", "bmar$gin", "mo$use", "multi$plot",
+ "mxt$ics", "nomxt$ics", "mx2t$ics", "nomx2t$ics", "myt$ics",
+ "nomyt$ics", "my2t$ics", "nomy2t$ics", "mzt$ics", "nomzt$ics",
+ "mcbt$ics", "nomcbt$ics", "of$fsets", "or$igin", "o$utput",
+ "pa$rametric", "pm$3d", "pal$ette", "colorb$ox", "p$lot",
+ "poi$ntsize", "pol$ar", "pr$int", "obj$ect", "sa$mples", "si$ze",
+ "st$yle", "su$rface", "table$", "t$erminal", "termo$ptions", "ti$cs",
+ "ticsc$ale", "ticsl$evel", "timef$mt", "tim$estamp", "tit$le",
+ "v$ariables", "ve$rsion", "vi$ew", "xyp$lane", "xda$ta", "x2da$ta",
+ "yda$ta", "y2da$ta", "zda$ta", "cbda$ta", "xl$abel", "x2l$abel",
+ "yl$abel", "y2l$abel", "zl$abel", "cbl$abel", "xti$cs", "noxti$cs",
+ "x2ti$cs", "nox2ti$cs", "yti$cs", "noyti$cs", "y2ti$cs", "noy2ti$cs",
+ "zti$cs", "nozti$cs", "cbti$cs", "nocbti$cs", "xdti$cs", "noxdti$cs",
+ "x2dti$cs", "nox2dti$cs", "ydti$cs", "noydti$cs", "y2dti$cs",
+ "noy2dti$cs", "zdti$cs", "nozdti$cs", "cbdti$cs", "nocbdti$cs",
+ "xmti$cs", "noxmti$cs", "x2mti$cs", "nox2mti$cs", "ymti$cs",
+ "noymti$cs", "y2mti$cs", "noy2mti$cs", "zmti$cs", "nozmti$cs",
+ "cbmti$cs", "nocbmti$cs", "xr$ange", "x2r$ange", "yr$ange",
+ "y2r$ange", "zr$ange", "cbr$ange", "rr$ange", "tr$ange", "ur$ange",
+ "vr$ange", "xzeroa$xis", "x2zeroa$xis", "yzeroa$xis", "y2zeroa$xis",
+ "zzeroa$xis", "zeroa$xis", "z$ero"), Name.Builtin, '#pop'),
+ ],
+ 'bind': [
+ ('!', Keyword, '#pop'),
+ (_shortened('all$windows'), Name.Builtin),
+ include('genericargs'),
+ ],
+ 'quit': [
+ (r'gnuplot\b', Keyword),
+ include('noargs'),
+ ],
+ 'fit': [
+ (r'via\b', Name.Builtin),
+ include('plot'),
+ ],
+ 'if': [
+ (r'\)', Punctuation, '#pop'),
+ include('genericargs'),
+ ],
+ 'pause': [
+ (r'(mouse|any|button1|button2|button3)\b', Name.Builtin),
+ (_shortened('key$press'), Name.Builtin),
+ include('genericargs'),
+ ],
+ 'plot': [
+ (_shortened_many('ax$es', 'axi$s', 'bin$ary', 'ev$ery', 'i$ndex',
+ 'mat$rix', 's$mooth', 'thru$', 't$itle',
+ 'not$itle', 'u$sing', 'w$ith'),
+ Name.Builtin),
+ include('genericargs'),
+ ],
+ 'save': [
+ (_shortened_many('f$unctions', 's$et', 't$erminal', 'v$ariables'),
+ Name.Builtin),
+ include('genericargs'),
+ ],
+ }
+
+
+class PovrayLexer(RegexLexer):
+ """
+ For `Persistence of Vision Raytracer <http://www.povray.org/>`_ files.
+
+ .. versionadded:: 0.11
+ """
+ name = 'POVRay'
+ aliases = ['pov']
+ filenames = ['*.pov', '*.inc']
+ mimetypes = ['text/x-povray']
+
+ tokens = {
+ 'root': [
+ (r'/\*[\w\W]*?\*/', Comment.Multiline),
+ (r'//.*\n', Comment.Single),
+ (r'(?s)"(?:\\.|[^"\\])+"', String.Double),
+ (words((
+ 'break', 'case', 'debug', 'declare', 'default', 'define', 'else',
+ 'elseif', 'end', 'error', 'fclose', 'fopen', 'for', 'if', 'ifdef',
+ 'ifndef', 'include', 'local', 'macro', 'range', 'read', 'render',
+ 'statistics', 'switch', 'undef', 'version', 'warning', 'while',
+ 'write'), prefix=r'#', suffix=r'\b'),
+ Comment.Preproc),
+ (words((
+ 'aa_level', 'aa_threshold', 'abs', 'acos', 'acosh', 'adaptive', 'adc_bailout',
+ 'agate', 'agate_turb', 'all', 'alpha', 'ambient', 'ambient_light', 'angle',
+ 'aperture', 'arc_angle', 'area_light', 'asc', 'asin', 'asinh', 'assumed_gamma',
+ 'atan', 'atan2', 'atanh', 'atmosphere', 'atmospheric_attenuation',
+ 'attenuating', 'average', 'background', 'black_hole', 'blue', 'blur_samples',
+ 'bounded_by', 'box_mapping', 'bozo', 'break', 'brick', 'brick_size',
+ 'brightness', 'brilliance', 'bumps', 'bumpy1', 'bumpy2', 'bumpy3', 'bump_map',
+ 'bump_size', 'case', 'caustics', 'ceil', 'checker', 'chr', 'clipped_by', 'clock',
+ 'color', 'color_map', 'colour', 'colour_map', 'component', 'composite', 'concat',
+ 'confidence', 'conic_sweep', 'constant', 'control0', 'control1', 'cos', 'cosh',
+ 'count', 'crackle', 'crand', 'cube', 'cubic_spline', 'cylindrical_mapping',
+ 'debug', 'declare', 'default', 'degrees', 'dents', 'diffuse', 'direction',
+ 'distance', 'distance_maximum', 'div', 'dust', 'dust_type', 'eccentricity',
+ 'else', 'emitting', 'end', 'error', 'error_bound', 'exp', 'exponent',
+ 'fade_distance', 'fade_power', 'falloff', 'falloff_angle', 'false',
+ 'file_exists', 'filter', 'finish', 'fisheye', 'flatness', 'flip', 'floor',
+ 'focal_point', 'fog', 'fog_alt', 'fog_offset', 'fog_type', 'frequency', 'gif',
+ 'global_settings', 'glowing', 'gradient', 'granite', 'gray_threshold',
+ 'green', 'halo', 'hexagon', 'hf_gray_16', 'hierarchy', 'hollow', 'hypercomplex',
+ 'if', 'ifdef', 'iff', 'image_map', 'incidence', 'include', 'int', 'interpolate',
+ 'inverse', 'ior', 'irid', 'irid_wavelength', 'jitter', 'lambda', 'leopard',
+ 'linear', 'linear_spline', 'linear_sweep', 'location', 'log', 'looks_like',
+ 'look_at', 'low_error_factor', 'mandel', 'map_type', 'marble', 'material_map',
+ 'matrix', 'max', 'max_intersections', 'max_iteration', 'max_trace_level',
+ 'max_value', 'metallic', 'min', 'minimum_reuse', 'mod', 'mortar',
+ 'nearest_count', 'no', 'normal', 'normal_map', 'no_shadow', 'number_of_waves',
+ 'octaves', 'off', 'offset', 'omega', 'omnimax', 'on', 'once', 'onion', 'open',
+ 'orthographic', 'panoramic', 'pattern1', 'pattern2', 'pattern3',
+ 'perspective', 'pgm', 'phase', 'phong', 'phong_size', 'pi', 'pigment',
+ 'pigment_map', 'planar_mapping', 'png', 'point_at', 'pot', 'pow', 'ppm',
+ 'precision', 'pwr', 'quadratic_spline', 'quaternion', 'quick_color',
+ 'quick_colour', 'quilted', 'radial', 'radians', 'radiosity', 'radius', 'rainbow',
+ 'ramp_wave', 'rand', 'range', 'reciprocal', 'recursion_limit', 'red',
+ 'reflection', 'refraction', 'render', 'repeat', 'rgb', 'rgbf', 'rgbft', 'rgbt',
+ 'right', 'ripples', 'rotate', 'roughness', 'samples', 'scale', 'scallop_wave',
+ 'scattering', 'seed', 'shadowless', 'sin', 'sine_wave', 'sinh', 'sky', 'sky_sphere',
+ 'slice', 'slope_map', 'smooth', 'specular', 'spherical_mapping', 'spiral',
+ 'spiral1', 'spiral2', 'spotlight', 'spotted', 'sqr', 'sqrt', 'statistics', 'str',
+ 'strcmp', 'strength', 'strlen', 'strlwr', 'strupr', 'sturm', 'substr', 'switch', 'sys',
+ 't', 'tan', 'tanh', 'test_camera_1', 'test_camera_2', 'test_camera_3',
+ 'test_camera_4', 'texture', 'texture_map', 'tga', 'thickness', 'threshold',
+ 'tightness', 'tile2', 'tiles', 'track', 'transform', 'translate', 'transmit',
+ 'triangle_wave', 'true', 'ttf', 'turbulence', 'turb_depth', 'type',
+ 'ultra_wide_angle', 'up', 'use_color', 'use_colour', 'use_index', 'u_steps',
+ 'val', 'variance', 'vaxis_rotate', 'vcross', 'vdot', 'version', 'vlength',
+ 'vnormalize', 'volume_object', 'volume_rendered', 'vol_with_light',
+ 'vrotate', 'v_steps', 'warning', 'warp', 'water_level', 'waves', 'while', 'width',
+ 'wood', 'wrinkles', 'yes'), prefix=r'\b', suffix=r'\b'),
+ Keyword),
+ (words((
+ 'bicubic_patch', 'blob', 'box', 'camera', 'cone', 'cubic', 'cylinder', 'difference',
+ 'disc', 'height_field', 'intersection', 'julia_fractal', 'lathe',
+ 'light_source', 'merge', 'mesh', 'object', 'plane', 'poly', 'polygon', 'prism',
+ 'quadric', 'quartic', 'smooth_triangle', 'sor', 'sphere', 'superellipsoid',
+ 'text', 'torus', 'triangle', 'union'), suffix=r'\b'),
+ Name.Builtin),
+ # TODO: <=, etc
+ (r'[\[\](){}<>;,]', Punctuation),
+ (r'[-+*/=]', Operator),
+ (r'\b(x|y|z|u|v)\b', Name.Builtin.Pseudo),
+ (r'[a-zA-Z_]\w*', Name),
+ (r'[0-9]+\.[0-9]*', Number.Float),
+ (r'\.[0-9]+', Number.Float),
+ (r'[0-9]+', Number.Integer),
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r'\s+', Text),
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/haskell.py b/wandb/vendor/pygments/lexers/haskell.py
new file mode 100644
index 0000000000000000000000000000000000000000..1a2f221771ff1b30867835352a207e9f01e1e3bd
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/haskell.py
@@ -0,0 +1,843 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.haskell
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Haskell and related languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, bygroups, do_insertions, \
+ default, include
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Generic
+from pygments import unistring as uni
+
+__all__ = ['HaskellLexer', 'IdrisLexer', 'AgdaLexer', 'CryptolLexer',
+ 'LiterateHaskellLexer', 'LiterateIdrisLexer', 'LiterateAgdaLexer',
+ 'LiterateCryptolLexer', 'KokaLexer']
+
+
+line_re = re.compile('.*?\n')
+
+
+class HaskellLexer(RegexLexer):
+ """
+ A Haskell lexer based on the lexemes defined in the Haskell 98 Report.
+
+ .. versionadded:: 0.8
+ """
+ name = 'Haskell'
+ aliases = ['haskell', 'hs']
+ filenames = ['*.hs']
+ mimetypes = ['text/x-haskell']
+
+ flags = re.MULTILINE | re.UNICODE
+
+ reserved = ('case', 'class', 'data', 'default', 'deriving', 'do', 'else',
+ 'family', 'if', 'in', 'infix[lr]?', 'instance',
+ 'let', 'newtype', 'of', 'then', 'type', 'where', '_')
+ ascii = ('NUL', 'SOH', '[SE]TX', 'EOT', 'ENQ', 'ACK',
+ 'BEL', 'BS', 'HT', 'LF', 'VT', 'FF', 'CR', 'S[OI]', 'DLE',
+ 'DC[1-4]', 'NAK', 'SYN', 'ETB', 'CAN',
+ 'EM', 'SUB', 'ESC', '[FGRU]S', 'SP', 'DEL')
+
+ tokens = {
+ 'root': [
+ # Whitespace:
+ (r'\s+', Text),
+ # (r'--\s*|.*$', Comment.Doc),
+ (r'--(?![!#$%&*+./<=>?@^|_~:\\]).*?$', Comment.Single),
+ (r'\{-', Comment.Multiline, 'comment'),
+ # Lexemes:
+ # Identifiers
+ (r'\bimport\b', Keyword.Reserved, 'import'),
+ (r'\bmodule\b', Keyword.Reserved, 'module'),
+ (r'\berror\b', Name.Exception),
+ (r'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved),
+ (r"'[^\\]'", String.Char), # this has to come before the TH quote
+ (r'^[_' + uni.Ll + r'][\w\']*', Name.Function),
+ (r"'?[_" + uni.Ll + r"][\w']*", Name),
+ (r"('')?[" + uni.Lu + r"][\w\']*", Keyword.Type),
+ (r"(')[" + uni.Lu + r"][\w\']*", Keyword.Type),
+ (r"(')\[[^\]]*\]", Keyword.Type), # tuples and lists get special treatment in GHC
+ (r"(')\([^)]*\)", Keyword.Type), # ..
+ # Operators
+ (r'\\(?![:!#$%&*+.\\/<=>?@^|~-]+)', Name.Function), # lambda operator
+ (r'(<-|::|->|=>|=)(?![:!#$%&*+.\\/<=>?@^|~-]+)', Operator.Word), # specials
+ (r':[:!#$%&*+.\\/<=>?@^|~-]*', Keyword.Type), # Constructor operators
+ (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator), # Other operators
+ # Numbers
+ (r'\d+[eE][+-]?\d+', Number.Float),
+ (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float),
+ (r'0[oO][0-7]+', Number.Oct),
+ (r'0[xX][\da-fA-F]+', Number.Hex),
+ (r'\d+', Number.Integer),
+ # Character/String Literals
+ (r"'", String.Char, 'character'),
+ (r'"', String, 'string'),
+ # Special
+ (r'\[\]', Keyword.Type),
+ (r'\(\)', Name.Builtin),
+ (r'[][(),;`{}]', Punctuation),
+ ],
+ 'import': [
+ # Import statements
+ (r'\s+', Text),
+ (r'"', String, 'string'),
+ # after "funclist" state
+ (r'\)', Punctuation, '#pop'),
+ (r'qualified\b', Keyword),
+ # import X as Y
+ (r'([' + uni.Lu + r'][\w.]*)(\s+)(as)(\s+)([' + uni.Lu + r'][\w.]*)',
+ bygroups(Name.Namespace, Text, Keyword, Text, Name), '#pop'),
+ # import X hiding (functions)
+ (r'([' + uni.Lu + r'][\w.]*)(\s+)(hiding)(\s+)(\()',
+ bygroups(Name.Namespace, Text, Keyword, Text, Punctuation), 'funclist'),
+ # import X (functions)
+ (r'([' + uni.Lu + r'][\w.]*)(\s+)(\()',
+ bygroups(Name.Namespace, Text, Punctuation), 'funclist'),
+ # import X
+ (r'[\w.]+', Name.Namespace, '#pop'),
+ ],
+ 'module': [
+ (r'\s+', Text),
+ (r'([' + uni.Lu + r'][\w.]*)(\s+)(\()',
+ bygroups(Name.Namespace, Text, Punctuation), 'funclist'),
+ (r'[' + uni.Lu + r'][\w.]*', Name.Namespace, '#pop'),
+ ],
+ 'funclist': [
+ (r'\s+', Text),
+ (r'[' + uni.Lu + r']\w*', Keyword.Type),
+ (r'(_[\w\']+|[' + uni.Ll + r'][\w\']*)', Name.Function),
+ (r'--(?![!#$%&*+./<=>?@^|_~:\\]).*?$', Comment.Single),
+ (r'\{-', Comment.Multiline, 'comment'),
+ (r',', Punctuation),
+ (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator),
+ # (HACK, but it makes sense to push two instances, believe me)
+ (r'\(', Punctuation, ('funclist', 'funclist')),
+ (r'\)', Punctuation, '#pop:2'),
+ ],
+ # NOTE: the next four states are shared in the AgdaLexer; make sure
+ # any change is compatible with Agda as well or copy over and change
+ 'comment': [
+ # Multiline Comments
+ (r'[^-{}]+', Comment.Multiline),
+ (r'\{-', Comment.Multiline, '#push'),
+ (r'-\}', Comment.Multiline, '#pop'),
+ (r'[-{}]', Comment.Multiline),
+ ],
+ 'character': [
+ # Allows multi-chars, incorrectly.
+ (r"[^\\']'", String.Char, '#pop'),
+ (r"\\", String.Escape, 'escape'),
+ ("'", String.Char, '#pop'),
+ ],
+ 'string': [
+ (r'[^\\"]+', String),
+ (r"\\", String.Escape, 'escape'),
+ ('"', String, '#pop'),
+ ],
+ 'escape': [
+ (r'[abfnrtv"\'&\\]', String.Escape, '#pop'),
+ (r'\^[][' + uni.Lu + r'@^_]', String.Escape, '#pop'),
+ ('|'.join(ascii), String.Escape, '#pop'),
+ (r'o[0-7]+', String.Escape, '#pop'),
+ (r'x[\da-fA-F]+', String.Escape, '#pop'),
+ (r'\d+', String.Escape, '#pop'),
+ (r'\s+\\', String.Escape, '#pop'),
+ ],
+ }
+
+
+class IdrisLexer(RegexLexer):
+ """
+ A lexer for the dependently typed programming language Idris.
+
+ Based on the Haskell and Agda Lexer.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Idris'
+ aliases = ['idris', 'idr']
+ filenames = ['*.idr']
+ mimetypes = ['text/x-idris']
+
+ reserved = ('case', 'class', 'data', 'default', 'using', 'do', 'else',
+ 'if', 'in', 'infix[lr]?', 'instance', 'rewrite', 'auto',
+ 'namespace', 'codata', 'mutual', 'private', 'public', 'abstract',
+ 'total', 'partial',
+ 'let', 'proof', 'of', 'then', 'static', 'where', '_', 'with',
+ 'pattern', 'term', 'syntax', 'prefix',
+ 'postulate', 'parameters', 'record', 'dsl', 'impossible', 'implicit',
+ 'tactics', 'intros', 'intro', 'compute', 'refine', 'exact', 'trivial')
+
+ ascii = ('NUL', 'SOH', '[SE]TX', 'EOT', 'ENQ', 'ACK',
+ 'BEL', 'BS', 'HT', 'LF', 'VT', 'FF', 'CR', 'S[OI]', 'DLE',
+ 'DC[1-4]', 'NAK', 'SYN', 'ETB', 'CAN',
+ 'EM', 'SUB', 'ESC', '[FGRU]S', 'SP', 'DEL')
+
+ directives = ('lib', 'link', 'flag', 'include', 'hide', 'freeze', 'access',
+ 'default', 'logging', 'dynamic', 'name', 'error_handlers', 'language')
+
+ tokens = {
+ 'root': [
+ # Comments
+ (r'^(\s*)(%%%s)' % '|'.join(directives),
+ bygroups(Text, Keyword.Reserved)),
+ (r'(\s*)(--(?![!#$%&*+./<=>?@^|_~:\\]).*?)$', bygroups(Text, Comment.Single)),
+ (r'(\s*)(\|{3}.*?)$', bygroups(Text, Comment.Single)),
+ (r'(\s*)(\{-)', bygroups(Text, Comment.Multiline), 'comment'),
+ # Declaration
+ (r'^(\s*)([^\s(){}]+)(\s*)(:)(\s*)',
+ bygroups(Text, Name.Function, Text, Operator.Word, Text)),
+ # Identifiers
+ (r'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved),
+ (r'(import|module)(\s+)', bygroups(Keyword.Reserved, Text), 'module'),
+ (r"('')?[A-Z][\w\']*", Keyword.Type),
+ (r'[a-z][\w\']*', Text),
+ # Special Symbols
+ (r'(<-|::|->|=>|=)', Operator.Word), # specials
+ (r'([(){}\[\]:!#$%&*+.\\/<=>?@^|~-]+)', Operator.Word), # specials
+ # Numbers
+ (r'\d+[eE][+-]?\d+', Number.Float),
+ (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float),
+ (r'0[xX][\da-fA-F]+', Number.Hex),
+ (r'\d+', Number.Integer),
+ # Strings
+ (r"'", String.Char, 'character'),
+ (r'"', String, 'string'),
+ (r'[^\s(){}]+', Text),
+ (r'\s+?', Text), # Whitespace
+ ],
+ 'module': [
+ (r'\s+', Text),
+ (r'([A-Z][\w.]*)(\s+)(\()',
+ bygroups(Name.Namespace, Text, Punctuation), 'funclist'),
+ (r'[A-Z][\w.]*', Name.Namespace, '#pop'),
+ ],
+ 'funclist': [
+ (r'\s+', Text),
+ (r'[A-Z]\w*', Keyword.Type),
+ (r'(_[\w\']+|[a-z][\w\']*)', Name.Function),
+ (r'--.*$', Comment.Single),
+ (r'\{-', Comment.Multiline, 'comment'),
+ (r',', Punctuation),
+ (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator),
+ # (HACK, but it makes sense to push two instances, believe me)
+ (r'\(', Punctuation, ('funclist', 'funclist')),
+ (r'\)', Punctuation, '#pop:2'),
+ ],
+ # NOTE: the next four states are shared in the AgdaLexer; make sure
+ # any change is compatible with Agda as well or copy over and change
+ 'comment': [
+ # Multiline Comments
+ (r'[^-{}]+', Comment.Multiline),
+ (r'\{-', Comment.Multiline, '#push'),
+ (r'-\}', Comment.Multiline, '#pop'),
+ (r'[-{}]', Comment.Multiline),
+ ],
+ 'character': [
+ # Allows multi-chars, incorrectly.
+ (r"[^\\']", String.Char),
+ (r"\\", String.Escape, 'escape'),
+ ("'", String.Char, '#pop'),
+ ],
+ 'string': [
+ (r'[^\\"]+', String),
+ (r"\\", String.Escape, 'escape'),
+ ('"', String, '#pop'),
+ ],
+ 'escape': [
+ (r'[abfnrtv"\'&\\]', String.Escape, '#pop'),
+ (r'\^[][A-Z@^_]', String.Escape, '#pop'),
+ ('|'.join(ascii), String.Escape, '#pop'),
+ (r'o[0-7]+', String.Escape, '#pop'),
+ (r'x[\da-fA-F]+', String.Escape, '#pop'),
+ (r'\d+', String.Escape, '#pop'),
+ (r'\s+\\', String.Escape, '#pop')
+ ],
+ }
+
+
+class AgdaLexer(RegexLexer):
+ """
+ For the `Agda <http://wiki.portal.chalmers.se/agda/pmwiki.php>`_
+ dependently typed functional programming language and proof assistant.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Agda'
+ aliases = ['agda']
+ filenames = ['*.agda']
+ mimetypes = ['text/x-agda']
+
+ reserved = ['abstract', 'codata', 'coinductive', 'constructor', 'data',
+ 'field', 'forall', 'hiding', 'in', 'inductive', 'infix',
+ 'infixl', 'infixr', 'instance', 'let', 'mutual', 'open',
+ 'pattern', 'postulate', 'primitive', 'private',
+ 'quote', 'quoteGoal', 'quoteTerm',
+ 'record', 'renaming', 'rewrite', 'syntax', 'tactic',
+ 'unquote', 'unquoteDecl', 'using', 'where', 'with']
+
+ tokens = {
+ 'root': [
+ # Declaration
+ (r'^(\s*)([^\s(){}]+)(\s*)(:)(\s*)',
+ bygroups(Text, Name.Function, Text, Operator.Word, Text)),
+ # Comments
+ (r'--(?![!#$%&*+./<=>?@^|_~:\\]).*?$', Comment.Single),
+ (r'\{-', Comment.Multiline, 'comment'),
+ # Holes
+ (r'\{!', Comment.Directive, 'hole'),
+ # Lexemes:
+ # Identifiers
+ (r'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved),
+ (r'(import|module)(\s+)', bygroups(Keyword.Reserved, Text), 'module'),
+ (r'\b(Set|Prop)\b', Keyword.Type),
+ # Special Symbols
+ (r'(\(|\)|\{|\})', Operator),
+ (u'(\\.{1,3}|\\||\u039B|\u2200|\u2192|:|=|->)', Operator.Word),
+ # Numbers
+ (r'\d+[eE][+-]?\d+', Number.Float),
+ (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float),
+ (r'0[xX][\da-fA-F]+', Number.Hex),
+ (r'\d+', Number.Integer),
+ # Strings
+ (r"'", String.Char, 'character'),
+ (r'"', String, 'string'),
+ (r'[^\s(){}]+', Text),
+ (r'\s+?', Text), # Whitespace
+ ],
+ 'hole': [
+ # Holes
+ (r'[^!{}]+', Comment.Directive),
+ (r'\{!', Comment.Directive, '#push'),
+ (r'!\}', Comment.Directive, '#pop'),
+ (r'[!{}]', Comment.Directive),
+ ],
+ 'module': [
+ (r'\{-', Comment.Multiline, 'comment'),
+ (r'[a-zA-Z][\w.]*', Name, '#pop'),
+ (r'[\W0-9_]+', Text)
+ ],
+ 'comment': HaskellLexer.tokens['comment'],
+ 'character': HaskellLexer.tokens['character'],
+ 'string': HaskellLexer.tokens['string'],
+ 'escape': HaskellLexer.tokens['escape']
+ }
+
+
+class CryptolLexer(RegexLexer):
+ """
+ FIXME: A Cryptol2 lexer based on the lexemes defined in the Haskell 98 Report.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Cryptol'
+ aliases = ['cryptol', 'cry']
+ filenames = ['*.cry']
+ mimetypes = ['text/x-cryptol']
+
+ reserved = ('Arith', 'Bit', 'Cmp', 'False', 'Inf', 'True', 'else',
+ 'export', 'extern', 'fin', 'if', 'import', 'inf', 'lg2',
+ 'max', 'min', 'module', 'newtype', 'pragma', 'property',
+ 'then', 'type', 'where', 'width')
+ ascii = ('NUL', 'SOH', '[SE]TX', 'EOT', 'ENQ', 'ACK',
+ 'BEL', 'BS', 'HT', 'LF', 'VT', 'FF', 'CR', 'S[OI]', 'DLE',
+ 'DC[1-4]', 'NAK', 'SYN', 'ETB', 'CAN',
+ 'EM', 'SUB', 'ESC', '[FGRU]S', 'SP', 'DEL')
+
+ tokens = {
+ 'root': [
+ # Whitespace:
+ (r'\s+', Text),
+ # (r'--\s*|.*$', Comment.Doc),
+ (r'//.*$', Comment.Single),
+ (r'/\*', Comment.Multiline, 'comment'),
+ # Lexemes:
+ # Identifiers
+ (r'\bimport\b', Keyword.Reserved, 'import'),
+ (r'\bmodule\b', Keyword.Reserved, 'module'),
+ (r'\berror\b', Name.Exception),
+ (r'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved),
+ (r'^[_a-z][\w\']*', Name.Function),
+ (r"'?[_a-z][\w']*", Name),
+ (r"('')?[A-Z][\w\']*", Keyword.Type),
+ # Operators
+ (r'\\(?![:!#$%&*+.\\/<=>?@^|~-]+)', Name.Function), # lambda operator
+ (r'(<-|::|->|=>|=)(?![:!#$%&*+.\\/<=>?@^|~-]+)', Operator.Word), # specials
+ (r':[:!#$%&*+.\\/<=>?@^|~-]*', Keyword.Type), # Constructor operators
+ (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator), # Other operators
+ # Numbers
+ (r'\d+[eE][+-]?\d+', Number.Float),
+ (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float),
+ (r'0[oO][0-7]+', Number.Oct),
+ (r'0[xX][\da-fA-F]+', Number.Hex),
+ (r'\d+', Number.Integer),
+ # Character/String Literals
+ (r"'", String.Char, 'character'),
+ (r'"', String, 'string'),
+ # Special
+ (r'\[\]', Keyword.Type),
+ (r'\(\)', Name.Builtin),
+ (r'[][(),;`{}]', Punctuation),
+ ],
+ 'import': [
+ # Import statements
+ (r'\s+', Text),
+ (r'"', String, 'string'),
+ # after "funclist" state
+ (r'\)', Punctuation, '#pop'),
+ (r'qualified\b', Keyword),
+ # import X as Y
+ (r'([A-Z][\w.]*)(\s+)(as)(\s+)([A-Z][\w.]*)',
+ bygroups(Name.Namespace, Text, Keyword, Text, Name), '#pop'),
+ # import X hiding (functions)
+ (r'([A-Z][\w.]*)(\s+)(hiding)(\s+)(\()',
+ bygroups(Name.Namespace, Text, Keyword, Text, Punctuation), 'funclist'),
+ # import X (functions)
+ (r'([A-Z][\w.]*)(\s+)(\()',
+ bygroups(Name.Namespace, Text, Punctuation), 'funclist'),
+ # import X
+ (r'[\w.]+', Name.Namespace, '#pop'),
+ ],
+ 'module': [
+ (r'\s+', Text),
+ (r'([A-Z][\w.]*)(\s+)(\()',
+ bygroups(Name.Namespace, Text, Punctuation), 'funclist'),
+ (r'[A-Z][\w.]*', Name.Namespace, '#pop'),
+ ],
+ 'funclist': [
+ (r'\s+', Text),
+ (r'[A-Z]\w*', Keyword.Type),
+ (r'(_[\w\']+|[a-z][\w\']*)', Name.Function),
+ # TODO: these don't match the comments in docs, remove.
+ #(r'--(?![!#$%&*+./<=>?@^|_~:\\]).*?$', Comment.Single),
+ #(r'{-', Comment.Multiline, 'comment'),
+ (r',', Punctuation),
+ (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator),
+ # (HACK, but it makes sense to push two instances, believe me)
+ (r'\(', Punctuation, ('funclist', 'funclist')),
+ (r'\)', Punctuation, '#pop:2'),
+ ],
+ 'comment': [
+ # Multiline Comments
+ (r'[^/*]+', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline),
+ ],
+ 'character': [
+ # Allows multi-chars, incorrectly.
+ (r"[^\\']'", String.Char, '#pop'),
+ (r"\\", String.Escape, 'escape'),
+ ("'", String.Char, '#pop'),
+ ],
+ 'string': [
+ (r'[^\\"]+', String),
+ (r"\\", String.Escape, 'escape'),
+ ('"', String, '#pop'),
+ ],
+ 'escape': [
+ (r'[abfnrtv"\'&\\]', String.Escape, '#pop'),
+ (r'\^[][A-Z@^_]', String.Escape, '#pop'),
+ ('|'.join(ascii), String.Escape, '#pop'),
+ (r'o[0-7]+', String.Escape, '#pop'),
+ (r'x[\da-fA-F]+', String.Escape, '#pop'),
+ (r'\d+', String.Escape, '#pop'),
+ (r'\s+\\', String.Escape, '#pop'),
+ ],
+ }
+
+ EXTRA_KEYWORDS = set(('join', 'split', 'reverse', 'transpose', 'width',
+ 'length', 'tail', '<<', '>>', '<<<', '>>>', 'const',
+ 'reg', 'par', 'seq', 'ASSERT', 'undefined', 'error',
+ 'trace'))
+
+ def get_tokens_unprocessed(self, text):
+ stack = ['root']
+ for index, token, value in \
+ RegexLexer.get_tokens_unprocessed(self, text, stack):
+ if token is Name and value in self.EXTRA_KEYWORDS:
+ yield index, Name.Builtin, value
+ else:
+ yield index, token, value
+
+
+class LiterateLexer(Lexer):
+ """
+ Base class for lexers of literate file formats based on LaTeX or Bird-style
+ (prefixing each code line with ">").
+
+ Additional options accepted:
+
+ `litstyle`
+ If given, must be ``"bird"`` or ``"latex"``. If not given, the style
+ is autodetected: if the first non-whitespace character in the source
+ is a backslash or percent character, LaTeX is assumed, else Bird.
+ """
+
+ bird_re = re.compile(r'(>[ \t]*)(.*\n)')
+
+ def __init__(self, baselexer, **options):
+ self.baselexer = baselexer
+ Lexer.__init__(self, **options)
+
+ def get_tokens_unprocessed(self, text):
+ style = self.options.get('litstyle')
+ if style is None:
+ style = (text.lstrip()[0:1] in '%\\') and 'latex' or 'bird'
+
+ code = ''
+ insertions = []
+ if style == 'bird':
+ # bird-style
+ for match in line_re.finditer(text):
+ line = match.group()
+ m = self.bird_re.match(line)
+ if m:
+ insertions.append((len(code),
+ [(0, Comment.Special, m.group(1))]))
+ code += m.group(2)
+ else:
+ insertions.append((len(code), [(0, Text, line)]))
+ else:
+ # latex-style
+ from pygments.lexers.markup import TexLexer
+ lxlexer = TexLexer(**self.options)
+ codelines = 0
+ latex = ''
+ for match in line_re.finditer(text):
+ line = match.group()
+ if codelines:
+ if line.lstrip().startswith('\\end{code}'):
+ codelines = 0
+ latex += line
+ else:
+ code += line
+ elif line.lstrip().startswith('\\begin{code}'):
+ codelines = 1
+ latex += line
+ insertions.append((len(code),
+ list(lxlexer.get_tokens_unprocessed(latex))))
+ latex = ''
+ else:
+ latex += line
+ insertions.append((len(code),
+ list(lxlexer.get_tokens_unprocessed(latex))))
+ for item in do_insertions(insertions, self.baselexer.get_tokens_unprocessed(code)):
+ yield item
+
+
+class LiterateHaskellLexer(LiterateLexer):
+ """
+ For Literate Haskell (Bird-style or LaTeX) source.
+
+ Additional options accepted:
+
+ `litstyle`
+ If given, must be ``"bird"`` or ``"latex"``. If not given, the style
+ is autodetected: if the first non-whitespace character in the source
+ is a backslash or percent character, LaTeX is assumed, else Bird.
+
+ .. versionadded:: 0.9
+ """
+ name = 'Literate Haskell'
+ aliases = ['lhs', 'literate-haskell', 'lhaskell']
+ filenames = ['*.lhs']
+ mimetypes = ['text/x-literate-haskell']
+
+ def __init__(self, **options):
+ hslexer = HaskellLexer(**options)
+ LiterateLexer.__init__(self, hslexer, **options)
+
+
+class LiterateIdrisLexer(LiterateLexer):
+ """
+ For Literate Idris (Bird-style or LaTeX) source.
+
+ Additional options accepted:
+
+ `litstyle`
+ If given, must be ``"bird"`` or ``"latex"``. If not given, the style
+ is autodetected: if the first non-whitespace character in the source
+ is a backslash or percent character, LaTeX is assumed, else Bird.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Literate Idris'
+ aliases = ['lidr', 'literate-idris', 'lidris']
+ filenames = ['*.lidr']
+ mimetypes = ['text/x-literate-idris']
+
+ def __init__(self, **options):
+ hslexer = IdrisLexer(**options)
+ LiterateLexer.__init__(self, hslexer, **options)
+
+
+class LiterateAgdaLexer(LiterateLexer):
+ """
+ For Literate Agda source.
+
+ Additional options accepted:
+
+ `litstyle`
+ If given, must be ``"bird"`` or ``"latex"``. If not given, the style
+ is autodetected: if the first non-whitespace character in the source
+ is a backslash or percent character, LaTeX is assumed, else Bird.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Literate Agda'
+ aliases = ['lagda', 'literate-agda']
+ filenames = ['*.lagda']
+ mimetypes = ['text/x-literate-agda']
+
+ def __init__(self, **options):
+ agdalexer = AgdaLexer(**options)
+ LiterateLexer.__init__(self, agdalexer, litstyle='latex', **options)
+
+
+class LiterateCryptolLexer(LiterateLexer):
+ """
+ For Literate Cryptol (Bird-style or LaTeX) source.
+
+ Additional options accepted:
+
+ `litstyle`
+ If given, must be ``"bird"`` or ``"latex"``. If not given, the style
+ is autodetected: if the first non-whitespace character in the source
+ is a backslash or percent character, LaTeX is assumed, else Bird.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Literate Cryptol'
+ aliases = ['lcry', 'literate-cryptol', 'lcryptol']
+ filenames = ['*.lcry']
+ mimetypes = ['text/x-literate-cryptol']
+
+ def __init__(self, **options):
+ crylexer = CryptolLexer(**options)
+ LiterateLexer.__init__(self, crylexer, **options)
+
+
+class KokaLexer(RegexLexer):
+ """
+ Lexer for the `Koka <http://koka.codeplex.com>`_
+ language.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'Koka'
+ aliases = ['koka']
+ filenames = ['*.kk', '*.kki']
+ mimetypes = ['text/x-koka']
+
+ keywords = [
+ 'infix', 'infixr', 'infixl',
+ 'type', 'cotype', 'rectype', 'alias',
+ 'struct', 'con',
+ 'fun', 'function', 'val', 'var',
+ 'external',
+ 'if', 'then', 'else', 'elif', 'return', 'match',
+ 'private', 'public', 'private',
+ 'module', 'import', 'as',
+ 'include', 'inline',
+ 'rec',
+ 'try', 'yield', 'enum',
+ 'interface', 'instance',
+ ]
+
+ # keywords that are followed by a type
+ typeStartKeywords = [
+ 'type', 'cotype', 'rectype', 'alias', 'struct', 'enum',
+ ]
+
+ # keywords valid in a type
+ typekeywords = [
+ 'forall', 'exists', 'some', 'with',
+ ]
+
+ # builtin names and special names
+ builtin = [
+ 'for', 'while', 'repeat',
+ 'foreach', 'foreach-indexed',
+ 'error', 'catch', 'finally',
+ 'cs', 'js', 'file', 'ref', 'assigned',
+ ]
+
+ # symbols that can be in an operator
+ symbols = r'[$%&*+@!/\\^~=.:\-?|<>]+'
+
+ # symbol boundary: an operator keyword should not be followed by any of these
+ sboundary = '(?!'+symbols+')'
+
+ # name boundary: a keyword should not be followed by any of these
+ boundary = '(?![\w/])'
+
+ # koka token abstractions
+ tokenType = Name.Attribute
+ tokenTypeDef = Name.Class
+ tokenConstructor = Generic.Emph
+
+ # main lexer
+ tokens = {
+ 'root': [
+ include('whitespace'),
+
+ # go into type mode
+ (r'::?' + sboundary, tokenType, 'type'),
+ (r'(alias)(\s+)([a-z]\w*)?', bygroups(Keyword, Text, tokenTypeDef),
+ 'alias-type'),
+ (r'(struct)(\s+)([a-z]\w*)?', bygroups(Keyword, Text, tokenTypeDef),
+ 'struct-type'),
+ ((r'(%s)' % '|'.join(typeStartKeywords)) +
+ r'(\s+)([a-z]\w*)?', bygroups(Keyword, Text, tokenTypeDef),
+ 'type'),
+
+ # special sequences of tokens (we use ?: for non-capturing group as
+ # required by 'bygroups')
+ (r'(module)(\s+)(interface\s+)?((?:[a-z]\w*/)*[a-z]\w*)',
+ bygroups(Keyword, Text, Keyword, Name.Namespace)),
+ (r'(import)(\s+)((?:[a-z]\w*/)*[a-z]\w*)'
+ r'(?:(\s*)(=)(\s*)((?:qualified\s*)?)'
+ r'((?:[a-z]\w*/)*[a-z]\w*))?',
+ bygroups(Keyword, Text, Name.Namespace, Text, Keyword, Text,
+ Keyword, Name.Namespace)),
+
+ (r'(^(?:(?:public|private)\s*)?(?:function|fun|val))'
+ r'(\s+)([a-z]\w*|\((?:' + symbols + r'|/)\))',
+ bygroups(Keyword, Text, Name.Function)),
+ (r'(^(?:(?:public|private)\s*)?external)(\s+)(inline\s+)?'
+ r'([a-z]\w*|\((?:' + symbols + r'|/)\))',
+ bygroups(Keyword, Text, Keyword, Name.Function)),
+
+ # keywords
+ (r'(%s)' % '|'.join(typekeywords) + boundary, Keyword.Type),
+ (r'(%s)' % '|'.join(keywords) + boundary, Keyword),
+ (r'(%s)' % '|'.join(builtin) + boundary, Keyword.Pseudo),
+ (r'::?|:=|\->|[=.]' + sboundary, Keyword),
+
+ # names
+ (r'((?:[a-z]\w*/)*)([A-Z]\w*)',
+ bygroups(Name.Namespace, tokenConstructor)),
+ (r'((?:[a-z]\w*/)*)([a-z]\w*)', bygroups(Name.Namespace, Name)),
+ (r'((?:[a-z]\w*/)*)(\((?:' + symbols + r'|/)\))',
+ bygroups(Name.Namespace, Name)),
+ (r'_\w*', Name.Variable),
+
+ # literal string
+ (r'@"', String.Double, 'litstring'),
+
+ # operators
+ (symbols + "|/(?![*/])", Operator),
+ (r'`', Operator),
+ (r'[{}()\[\];,]', Punctuation),
+
+ # literals. No check for literal characters with len > 1
+ (r'[0-9]+\.[0-9]+([eE][\-+]?[0-9]+)?', Number.Float),
+ (r'0[xX][0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+', Number.Integer),
+
+ (r"'", String.Char, 'char'),
+ (r'"', String.Double, 'string'),
+ ],
+
+ # type started by alias
+ 'alias-type': [
+ (r'=', Keyword),
+ include('type')
+ ],
+
+ # type started by struct
+ 'struct-type': [
+ (r'(?=\((?!,*\)))', Punctuation, '#pop'),
+ include('type')
+ ],
+
+ # type started by colon
+ 'type': [
+ (r'[(\[<]', tokenType, 'type-nested'),
+ include('type-content')
+ ],
+
+ # type nested in brackets: can contain parameters, comma etc.
+ 'type-nested': [
+ (r'[)\]>]', tokenType, '#pop'),
+ (r'[(\[<]', tokenType, 'type-nested'),
+ (r',', tokenType),
+ (r'([a-z]\w*)(\s*)(:)(?!:)',
+ bygroups(Name, Text, tokenType)), # parameter name
+ include('type-content')
+ ],
+
+ # shared contents of a type
+ 'type-content': [
+ include('whitespace'),
+
+ # keywords
+ (r'(%s)' % '|'.join(typekeywords) + boundary, Keyword),
+ (r'(?=((%s)' % '|'.join(keywords) + boundary + '))',
+ Keyword, '#pop'), # need to match because names overlap...
+
+ # kinds
+ (r'[EPHVX]' + boundary, tokenType),
+
+ # type names
+ (r'[a-z][0-9]*(?![\w/])', tokenType),
+ (r'_\w*', tokenType.Variable), # Generic.Emph
+ (r'((?:[a-z]\w*/)*)([A-Z]\w*)',
+ bygroups(Name.Namespace, tokenType)),
+ (r'((?:[a-z]\w*/)*)([a-z]\w+)',
+ bygroups(Name.Namespace, tokenType)),
+
+ # type keyword operators
+ (r'::|->|[.:|]', tokenType),
+
+ # catchall
+ default('#pop')
+ ],
+
+ # comments and literals
+ 'whitespace': [
+ (r'\n\s*#.*$', Comment.Preproc),
+ (r'\s+', Text),
+ (r'/\*', Comment.Multiline, 'comment'),
+ (r'//.*$', Comment.Single)
+ ],
+ 'comment': [
+ (r'[^/*]+', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline),
+ ],
+ 'litstring': [
+ (r'[^"]+', String.Double),
+ (r'""', String.Escape),
+ (r'"', String.Double, '#pop'),
+ ],
+ 'string': [
+ (r'[^\\"\n]+', String.Double),
+ include('escape-sequence'),
+ (r'["\n]', String.Double, '#pop'),
+ ],
+ 'char': [
+ (r'[^\\\'\n]+', String.Char),
+ include('escape-sequence'),
+ (r'[\'\n]', String.Char, '#pop'),
+ ],
+ 'escape-sequence': [
+ (r'\\[nrt\\"\']', String.Escape),
+ (r'\\x[0-9a-fA-F]{2}', String.Escape),
+ (r'\\u[0-9a-fA-F]{4}', String.Escape),
+ # Yes, \U literals are 6 hex digits.
+ (r'\\U[0-9a-fA-F]{6}', String.Escape)
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/haxe.py b/wandb/vendor/pygments/lexers/haxe.py
new file mode 100644
index 0000000000000000000000000000000000000000..6f5c3599c607c55c6b43a058179be99be4ebfb0a
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/haxe.py
@@ -0,0 +1,936 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.haxe
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Haxe and related stuff.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import ExtendedRegexLexer, RegexLexer, include, bygroups, \
+ default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Generic, Whitespace
+
+__all__ = ['HaxeLexer', 'HxmlLexer']
+
+
+class HaxeLexer(ExtendedRegexLexer):
+ """
+ For Haxe source code (http://haxe.org/).
+
+ .. versionadded:: 1.3
+ """
+
+ name = 'Haxe'
+ aliases = ['hx', 'haxe', 'hxsl']
+ filenames = ['*.hx', '*.hxsl']
+ mimetypes = ['text/haxe', 'text/x-haxe', 'text/x-hx']
+
+ # keywords extracted from lexer.mll in the haxe compiler source
+ keyword = (r'(?:function|class|static|var|if|else|while|do|for|'
+ r'break|return|continue|extends|implements|import|'
+ r'switch|case|default|public|private|try|untyped|'
+ r'catch|new|this|throw|extern|enum|in|interface|'
+ r'cast|override|dynamic|typedef|package|'
+ r'inline|using|null|true|false|abstract)\b')
+
+ # idtype in lexer.mll
+ typeid = r'_*[A-Z]\w*'
+
+ # combined ident and dollar and idtype
+ ident = r'(?:_*[a-z]\w*|_+[0-9]\w*|' + typeid + '|_+|\$\w+)'
+
+ binop = (r'(?:%=|&=|\|=|\^=|\+=|\-=|\*=|/=|<<=|>\s*>\s*=|>\s*>\s*>\s*=|==|'
+ r'!=|<=|>\s*=|&&|\|\||<<|>>>|>\s*>|\.\.\.|<|>|%|&|\||\^|\+|\*|'
+ r'/|\-|=>|=)')
+
+ # ident except keywords
+ ident_no_keyword = r'(?!' + keyword + ')' + ident
+
+ flags = re.DOTALL | re.MULTILINE
+
+ preproc_stack = []
+
+ def preproc_callback(self, match, ctx):
+ proc = match.group(2)
+
+ if proc == 'if':
+ # store the current stack
+ self.preproc_stack.append(ctx.stack[:])
+ elif proc in ['else', 'elseif']:
+ # restore the stack back to right before #if
+ if self.preproc_stack:
+ ctx.stack = self.preproc_stack[-1][:]
+ elif proc == 'end':
+ # remove the saved stack of previous #if
+ if self.preproc_stack:
+ self.preproc_stack.pop()
+
+ # #if and #elseif should follow by an expr
+ if proc in ['if', 'elseif']:
+ ctx.stack.append('preproc-expr')
+
+ # #error can be optionally follow by the error msg
+ if proc in ['error']:
+ ctx.stack.append('preproc-error')
+
+ yield match.start(), Comment.Preproc, '#' + proc
+ ctx.pos = match.end()
+
+ tokens = {
+ 'root': [
+ include('spaces'),
+ include('meta'),
+ (r'(?:package)\b', Keyword.Namespace, ('semicolon', 'package')),
+ (r'(?:import)\b', Keyword.Namespace, ('semicolon', 'import')),
+ (r'(?:using)\b', Keyword.Namespace, ('semicolon', 'using')),
+ (r'(?:extern|private)\b', Keyword.Declaration),
+ (r'(?:abstract)\b', Keyword.Declaration, 'abstract'),
+ (r'(?:class|interface)\b', Keyword.Declaration, 'class'),
+ (r'(?:enum)\b', Keyword.Declaration, 'enum'),
+ (r'(?:typedef)\b', Keyword.Declaration, 'typedef'),
+
+ # top-level expression
+ # although it is not supported in haxe, but it is common to write
+ # expression in web pages the positive lookahead here is to prevent
+ # an infinite loop at the EOF
+ (r'(?=.)', Text, 'expr-statement'),
+ ],
+
+ # space/tab/comment/preproc
+ 'spaces': [
+ (r'\s+', Text),
+ (r'//[^\n\r]*', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'(#)(if|elseif|else|end|error)\b', preproc_callback),
+ ],
+
+ 'string-single-interpol': [
+ (r'\$\{', String.Interpol, ('string-interpol-close', 'expr')),
+ (r'\$\$', String.Escape),
+ (r'\$(?=' + ident + ')', String.Interpol, 'ident'),
+ include('string-single'),
+ ],
+
+ 'string-single': [
+ (r"'", String.Single, '#pop'),
+ (r'\\.', String.Escape),
+ (r'.', String.Single),
+ ],
+
+ 'string-double': [
+ (r'"', String.Double, '#pop'),
+ (r'\\.', String.Escape),
+ (r'.', String.Double),
+ ],
+
+ 'string-interpol-close': [
+ (r'\$'+ident, String.Interpol),
+ (r'\}', String.Interpol, '#pop'),
+ ],
+
+ 'package': [
+ include('spaces'),
+ (ident, Name.Namespace),
+ (r'\.', Punctuation, 'import-ident'),
+ default('#pop'),
+ ],
+
+ 'import': [
+ include('spaces'),
+ (ident, Name.Namespace),
+ (r'\*', Keyword), # wildcard import
+ (r'\.', Punctuation, 'import-ident'),
+ (r'in', Keyword.Namespace, 'ident'),
+ default('#pop'),
+ ],
+
+ 'import-ident': [
+ include('spaces'),
+ (r'\*', Keyword, '#pop'), # wildcard import
+ (ident, Name.Namespace, '#pop'),
+ ],
+
+ 'using': [
+ include('spaces'),
+ (ident, Name.Namespace),
+ (r'\.', Punctuation, 'import-ident'),
+ default('#pop'),
+ ],
+
+ 'preproc-error': [
+ (r'\s+', Comment.Preproc),
+ (r"'", String.Single, ('#pop', 'string-single')),
+ (r'"', String.Double, ('#pop', 'string-double')),
+ default('#pop'),
+ ],
+
+ 'preproc-expr': [
+ (r'\s+', Comment.Preproc),
+ (r'\!', Comment.Preproc),
+ (r'\(', Comment.Preproc, ('#pop', 'preproc-parenthesis')),
+
+ (ident, Comment.Preproc, '#pop'),
+
+ # Float
+ (r'\.[0-9]+', Number.Float),
+ (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float),
+ (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float),
+ (r'[0-9]+\.[0-9]+', Number.Float),
+ (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float),
+
+ # Int
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+', Number.Integer),
+
+ # String
+ (r"'", String.Single, ('#pop', 'string-single')),
+ (r'"', String.Double, ('#pop', 'string-double')),
+ ],
+
+ 'preproc-parenthesis': [
+ (r'\s+', Comment.Preproc),
+ (r'\)', Comment.Preproc, '#pop'),
+ default('preproc-expr-in-parenthesis'),
+ ],
+
+ 'preproc-expr-chain': [
+ (r'\s+', Comment.Preproc),
+ (binop, Comment.Preproc, ('#pop', 'preproc-expr-in-parenthesis')),
+ default('#pop'),
+ ],
+
+ # same as 'preproc-expr' but able to chain 'preproc-expr-chain'
+ 'preproc-expr-in-parenthesis': [
+ (r'\s+', Comment.Preproc),
+ (r'\!', Comment.Preproc),
+ (r'\(', Comment.Preproc,
+ ('#pop', 'preproc-expr-chain', 'preproc-parenthesis')),
+
+ (ident, Comment.Preproc, ('#pop', 'preproc-expr-chain')),
+
+ # Float
+ (r'\.[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')),
+ (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')),
+ (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')),
+ (r'[0-9]+\.[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')),
+ (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, ('#pop', 'preproc-expr-chain')),
+
+ # Int
+ (r'0x[0-9a-fA-F]+', Number.Hex, ('#pop', 'preproc-expr-chain')),
+ (r'[0-9]+', Number.Integer, ('#pop', 'preproc-expr-chain')),
+
+ # String
+ (r"'", String.Single,
+ ('#pop', 'preproc-expr-chain', 'string-single')),
+ (r'"', String.Double,
+ ('#pop', 'preproc-expr-chain', 'string-double')),
+ ],
+
+ 'abstract': [
+ include('spaces'),
+ default(('#pop', 'abstract-body', 'abstract-relation',
+ 'abstract-opaque', 'type-param-constraint', 'type-name')),
+ ],
+
+ 'abstract-body': [
+ include('spaces'),
+ (r'\{', Punctuation, ('#pop', 'class-body')),
+ ],
+
+ 'abstract-opaque': [
+ include('spaces'),
+ (r'\(', Punctuation, ('#pop', 'parenthesis-close', 'type')),
+ default('#pop'),
+ ],
+
+ 'abstract-relation': [
+ include('spaces'),
+ (r'(?:to|from)', Keyword.Declaration, 'type'),
+ (r',', Punctuation),
+ default('#pop'),
+ ],
+
+ 'meta': [
+ include('spaces'),
+ (r'@', Name.Decorator, ('meta-body', 'meta-ident', 'meta-colon')),
+ ],
+
+ # optional colon
+ 'meta-colon': [
+ include('spaces'),
+ (r':', Name.Decorator, '#pop'),
+ default('#pop'),
+ ],
+
+ # same as 'ident' but set token as Name.Decorator instead of Name
+ 'meta-ident': [
+ include('spaces'),
+ (ident, Name.Decorator, '#pop'),
+ ],
+
+ 'meta-body': [
+ include('spaces'),
+ (r'\(', Name.Decorator, ('#pop', 'meta-call')),
+ default('#pop'),
+ ],
+
+ 'meta-call': [
+ include('spaces'),
+ (r'\)', Name.Decorator, '#pop'),
+ default(('#pop', 'meta-call-sep', 'expr')),
+ ],
+
+ 'meta-call-sep': [
+ include('spaces'),
+ (r'\)', Name.Decorator, '#pop'),
+ (r',', Punctuation, ('#pop', 'meta-call')),
+ ],
+
+ 'typedef': [
+ include('spaces'),
+ default(('#pop', 'typedef-body', 'type-param-constraint',
+ 'type-name')),
+ ],
+
+ 'typedef-body': [
+ include('spaces'),
+ (r'=', Operator, ('#pop', 'optional-semicolon', 'type')),
+ ],
+
+ 'enum': [
+ include('spaces'),
+ default(('#pop', 'enum-body', 'bracket-open',
+ 'type-param-constraint', 'type-name')),
+ ],
+
+ 'enum-body': [
+ include('spaces'),
+ include('meta'),
+ (r'\}', Punctuation, '#pop'),
+ (ident_no_keyword, Name, ('enum-member', 'type-param-constraint')),
+ ],
+
+ 'enum-member': [
+ include('spaces'),
+ (r'\(', Punctuation,
+ ('#pop', 'semicolon', 'flag', 'function-param')),
+ default(('#pop', 'semicolon', 'flag')),
+ ],
+
+ 'class': [
+ include('spaces'),
+ default(('#pop', 'class-body', 'bracket-open', 'extends',
+ 'type-param-constraint', 'type-name')),
+ ],
+
+ 'extends': [
+ include('spaces'),
+ (r'(?:extends|implements)\b', Keyword.Declaration, 'type'),
+ (r',', Punctuation), # the comma is made optional here, since haxe2
+ # requires the comma but haxe3 does not allow it
+ default('#pop'),
+ ],
+
+ 'bracket-open': [
+ include('spaces'),
+ (r'\{', Punctuation, '#pop'),
+ ],
+
+ 'bracket-close': [
+ include('spaces'),
+ (r'\}', Punctuation, '#pop'),
+ ],
+
+ 'class-body': [
+ include('spaces'),
+ include('meta'),
+ (r'\}', Punctuation, '#pop'),
+ (r'(?:static|public|private|override|dynamic|inline|macro)\b',
+ Keyword.Declaration),
+ default('class-member'),
+ ],
+
+ 'class-member': [
+ include('spaces'),
+ (r'(var)\b', Keyword.Declaration,
+ ('#pop', 'optional-semicolon', 'var')),
+ (r'(function)\b', Keyword.Declaration,
+ ('#pop', 'optional-semicolon', 'class-method')),
+ ],
+
+ # local function, anonymous or not
+ 'function-local': [
+ include('spaces'),
+ (ident_no_keyword, Name.Function,
+ ('#pop', 'optional-expr', 'flag', 'function-param',
+ 'parenthesis-open', 'type-param-constraint')),
+ default(('#pop', 'optional-expr', 'flag', 'function-param',
+ 'parenthesis-open', 'type-param-constraint')),
+ ],
+
+ 'optional-expr': [
+ include('spaces'),
+ include('expr'),
+ default('#pop'),
+ ],
+
+ 'class-method': [
+ include('spaces'),
+ (ident, Name.Function, ('#pop', 'optional-expr', 'flag',
+ 'function-param', 'parenthesis-open',
+ 'type-param-constraint')),
+ ],
+
+ # function arguments
+ 'function-param': [
+ include('spaces'),
+ (r'\)', Punctuation, '#pop'),
+ (r'\?', Punctuation),
+ (ident_no_keyword, Name,
+ ('#pop', 'function-param-sep', 'assign', 'flag')),
+ ],
+
+ 'function-param-sep': [
+ include('spaces'),
+ (r'\)', Punctuation, '#pop'),
+ (r',', Punctuation, ('#pop', 'function-param')),
+ ],
+
+ 'prop-get-set': [
+ include('spaces'),
+ (r'\(', Punctuation, ('#pop', 'parenthesis-close',
+ 'prop-get-set-opt', 'comma', 'prop-get-set-opt')),
+ default('#pop'),
+ ],
+
+ 'prop-get-set-opt': [
+ include('spaces'),
+ (r'(?:default|null|never|dynamic|get|set)\b', Keyword, '#pop'),
+ (ident_no_keyword, Text, '#pop'), # custom getter/setter
+ ],
+
+ 'expr-statement': [
+ include('spaces'),
+ # makes semicolon optional here, just to avoid checking the last
+ # one is bracket or not.
+ default(('#pop', 'optional-semicolon', 'expr')),
+ ],
+
+ 'expr': [
+ include('spaces'),
+ (r'@', Name.Decorator, ('#pop', 'optional-expr', 'meta-body',
+ 'meta-ident', 'meta-colon')),
+ (r'(?:\+\+|\-\-|~(?!/)|!|\-)', Operator),
+ (r'\(', Punctuation, ('#pop', 'expr-chain', 'parenthesis')),
+ (r'(?:static|public|private|override|dynamic|inline)\b',
+ Keyword.Declaration),
+ (r'(?:function)\b', Keyword.Declaration, ('#pop', 'expr-chain',
+ 'function-local')),
+ (r'\{', Punctuation, ('#pop', 'expr-chain', 'bracket')),
+ (r'(?:true|false|null)\b', Keyword.Constant, ('#pop', 'expr-chain')),
+ (r'(?:this)\b', Keyword, ('#pop', 'expr-chain')),
+ (r'(?:cast)\b', Keyword, ('#pop', 'expr-chain', 'cast')),
+ (r'(?:try)\b', Keyword, ('#pop', 'catch', 'expr')),
+ (r'(?:var)\b', Keyword.Declaration, ('#pop', 'var')),
+ (r'(?:new)\b', Keyword, ('#pop', 'expr-chain', 'new')),
+ (r'(?:switch)\b', Keyword, ('#pop', 'switch')),
+ (r'(?:if)\b', Keyword, ('#pop', 'if')),
+ (r'(?:do)\b', Keyword, ('#pop', 'do')),
+ (r'(?:while)\b', Keyword, ('#pop', 'while')),
+ (r'(?:for)\b', Keyword, ('#pop', 'for')),
+ (r'(?:untyped|throw)\b', Keyword),
+ (r'(?:return)\b', Keyword, ('#pop', 'optional-expr')),
+ (r'(?:macro)\b', Keyword, ('#pop', 'macro')),
+ (r'(?:continue|break)\b', Keyword, '#pop'),
+ (r'(?:\$\s*[a-z]\b|\$(?!'+ident+'))', Name, ('#pop', 'dollar')),
+ (ident_no_keyword, Name, ('#pop', 'expr-chain')),
+
+ # Float
+ (r'\.[0-9]+', Number.Float, ('#pop', 'expr-chain')),
+ (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'expr-chain')),
+ (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'expr-chain')),
+ (r'[0-9]+\.[0-9]+', Number.Float, ('#pop', 'expr-chain')),
+ (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, ('#pop', 'expr-chain')),
+
+ # Int
+ (r'0x[0-9a-fA-F]+', Number.Hex, ('#pop', 'expr-chain')),
+ (r'[0-9]+', Number.Integer, ('#pop', 'expr-chain')),
+
+ # String
+ (r"'", String.Single, ('#pop', 'expr-chain', 'string-single-interpol')),
+ (r'"', String.Double, ('#pop', 'expr-chain', 'string-double')),
+
+ # EReg
+ (r'~/(\\\\|\\/|[^/\n])*/[gimsu]*', String.Regex, ('#pop', 'expr-chain')),
+
+ # Array
+ (r'\[', Punctuation, ('#pop', 'expr-chain', 'array-decl')),
+ ],
+
+ 'expr-chain': [
+ include('spaces'),
+ (r'(?:\+\+|\-\-)', Operator),
+ (binop, Operator, ('#pop', 'expr')),
+ (r'(?:in)\b', Keyword, ('#pop', 'expr')),
+ (r'\?', Operator, ('#pop', 'expr', 'ternary', 'expr')),
+ (r'(\.)(' + ident_no_keyword + ')', bygroups(Punctuation, Name)),
+ (r'\[', Punctuation, 'array-access'),
+ (r'\(', Punctuation, 'call'),
+ default('#pop'),
+ ],
+
+ # macro reification
+ 'macro': [
+ include('spaces'),
+ include('meta'),
+ (r':', Punctuation, ('#pop', 'type')),
+
+ (r'(?:extern|private)\b', Keyword.Declaration),
+ (r'(?:abstract)\b', Keyword.Declaration, ('#pop', 'optional-semicolon', 'abstract')),
+ (r'(?:class|interface)\b', Keyword.Declaration, ('#pop', 'optional-semicolon', 'macro-class')),
+ (r'(?:enum)\b', Keyword.Declaration, ('#pop', 'optional-semicolon', 'enum')),
+ (r'(?:typedef)\b', Keyword.Declaration, ('#pop', 'optional-semicolon', 'typedef')),
+
+ default(('#pop', 'expr')),
+ ],
+
+ 'macro-class': [
+ (r'\{', Punctuation, ('#pop', 'class-body')),
+ include('class')
+ ],
+
+ # cast can be written as "cast expr" or "cast(expr, type)"
+ 'cast': [
+ include('spaces'),
+ (r'\(', Punctuation, ('#pop', 'parenthesis-close',
+ 'cast-type', 'expr')),
+ default(('#pop', 'expr')),
+ ],
+
+ # optionally give a type as the 2nd argument of cast()
+ 'cast-type': [
+ include('spaces'),
+ (r',', Punctuation, ('#pop', 'type')),
+ default('#pop'),
+ ],
+
+ 'catch': [
+ include('spaces'),
+ (r'(?:catch)\b', Keyword, ('expr', 'function-param',
+ 'parenthesis-open')),
+ default('#pop'),
+ ],
+
+ # do-while loop
+ 'do': [
+ include('spaces'),
+ default(('#pop', 'do-while', 'expr')),
+ ],
+
+ # the while after do
+ 'do-while': [
+ include('spaces'),
+ (r'(?:while)\b', Keyword, ('#pop', 'parenthesis',
+ 'parenthesis-open')),
+ ],
+
+ 'while': [
+ include('spaces'),
+ (r'\(', Punctuation, ('#pop', 'expr', 'parenthesis')),
+ ],
+
+ 'for': [
+ include('spaces'),
+ (r'\(', Punctuation, ('#pop', 'expr', 'parenthesis')),
+ ],
+
+ 'if': [
+ include('spaces'),
+ (r'\(', Punctuation, ('#pop', 'else', 'optional-semicolon', 'expr',
+ 'parenthesis')),
+ ],
+
+ 'else': [
+ include('spaces'),
+ (r'(?:else)\b', Keyword, ('#pop', 'expr')),
+ default('#pop'),
+ ],
+
+ 'switch': [
+ include('spaces'),
+ default(('#pop', 'switch-body', 'bracket-open', 'expr')),
+ ],
+
+ 'switch-body': [
+ include('spaces'),
+ (r'(?:case|default)\b', Keyword, ('case-block', 'case')),
+ (r'\}', Punctuation, '#pop'),
+ ],
+
+ 'case': [
+ include('spaces'),
+ (r':', Punctuation, '#pop'),
+ default(('#pop', 'case-sep', 'case-guard', 'expr')),
+ ],
+
+ 'case-sep': [
+ include('spaces'),
+ (r':', Punctuation, '#pop'),
+ (r',', Punctuation, ('#pop', 'case')),
+ ],
+
+ 'case-guard': [
+ include('spaces'),
+ (r'(?:if)\b', Keyword, ('#pop', 'parenthesis', 'parenthesis-open')),
+ default('#pop'),
+ ],
+
+ # optional multiple expr under a case
+ 'case-block': [
+ include('spaces'),
+ (r'(?!(?:case|default)\b|\})', Keyword, 'expr-statement'),
+ default('#pop'),
+ ],
+
+ 'new': [
+ include('spaces'),
+ default(('#pop', 'call', 'parenthesis-open', 'type')),
+ ],
+
+ 'array-decl': [
+ include('spaces'),
+ (r'\]', Punctuation, '#pop'),
+ default(('#pop', 'array-decl-sep', 'expr')),
+ ],
+
+ 'array-decl-sep': [
+ include('spaces'),
+ (r'\]', Punctuation, '#pop'),
+ (r',', Punctuation, ('#pop', 'array-decl')),
+ ],
+
+ 'array-access': [
+ include('spaces'),
+ default(('#pop', 'array-access-close', 'expr')),
+ ],
+
+ 'array-access-close': [
+ include('spaces'),
+ (r'\]', Punctuation, '#pop'),
+ ],
+
+ 'comma': [
+ include('spaces'),
+ (r',', Punctuation, '#pop'),
+ ],
+
+ 'colon': [
+ include('spaces'),
+ (r':', Punctuation, '#pop'),
+ ],
+
+ 'semicolon': [
+ include('spaces'),
+ (r';', Punctuation, '#pop'),
+ ],
+
+ 'optional-semicolon': [
+ include('spaces'),
+ (r';', Punctuation, '#pop'),
+ default('#pop'),
+ ],
+
+ # identity that CAN be a Haxe keyword
+ 'ident': [
+ include('spaces'),
+ (ident, Name, '#pop'),
+ ],
+
+ 'dollar': [
+ include('spaces'),
+ (r'\{', Punctuation, ('#pop', 'expr-chain', 'bracket-close', 'expr')),
+ default(('#pop', 'expr-chain')),
+ ],
+
+ 'type-name': [
+ include('spaces'),
+ (typeid, Name, '#pop'),
+ ],
+
+ 'type-full-name': [
+ include('spaces'),
+ (r'\.', Punctuation, 'ident'),
+ default('#pop'),
+ ],
+
+ 'type': [
+ include('spaces'),
+ (r'\?', Punctuation),
+ (ident, Name, ('#pop', 'type-check', 'type-full-name')),
+ (r'\{', Punctuation, ('#pop', 'type-check', 'type-struct')),
+ (r'\(', Punctuation, ('#pop', 'type-check', 'type-parenthesis')),
+ ],
+
+ 'type-parenthesis': [
+ include('spaces'),
+ default(('#pop', 'parenthesis-close', 'type')),
+ ],
+
+ 'type-check': [
+ include('spaces'),
+ (r'->', Punctuation, ('#pop', 'type')),
+ (r'<(?!=)', Punctuation, 'type-param'),
+ default('#pop'),
+ ],
+
+ 'type-struct': [
+ include('spaces'),
+ (r'\}', Punctuation, '#pop'),
+ (r'\?', Punctuation),
+ (r'>', Punctuation, ('comma', 'type')),
+ (ident_no_keyword, Name, ('#pop', 'type-struct-sep', 'type', 'colon')),
+ include('class-body'),
+ ],
+
+ 'type-struct-sep': [
+ include('spaces'),
+ (r'\}', Punctuation, '#pop'),
+ (r',', Punctuation, ('#pop', 'type-struct')),
+ ],
+
+ # type-param can be a normal type or a constant literal...
+ 'type-param-type': [
+ # Float
+ (r'\.[0-9]+', Number.Float, '#pop'),
+ (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float, '#pop'),
+ (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float, '#pop'),
+ (r'[0-9]+\.[0-9]+', Number.Float, '#pop'),
+ (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, '#pop'),
+
+ # Int
+ (r'0x[0-9a-fA-F]+', Number.Hex, '#pop'),
+ (r'[0-9]+', Number.Integer, '#pop'),
+
+ # String
+ (r"'", String.Single, ('#pop', 'string-single')),
+ (r'"', String.Double, ('#pop', 'string-double')),
+
+ # EReg
+ (r'~/(\\\\|\\/|[^/\n])*/[gim]*', String.Regex, '#pop'),
+
+ # Array
+ (r'\[', Operator, ('#pop', 'array-decl')),
+
+ include('type'),
+ ],
+
+ # type-param part of a type
+ # ie. the <A,B> path in Map<A,B>
+ 'type-param': [
+ include('spaces'),
+ default(('#pop', 'type-param-sep', 'type-param-type')),
+ ],
+
+ 'type-param-sep': [
+ include('spaces'),
+ (r'>', Punctuation, '#pop'),
+ (r',', Punctuation, ('#pop', 'type-param')),
+ ],
+
+ # optional type-param that may include constraint
+ # ie. <T:Constraint, T2:(ConstraintA,ConstraintB)>
+ 'type-param-constraint': [
+ include('spaces'),
+ (r'<(?!=)', Punctuation, ('#pop', 'type-param-constraint-sep',
+ 'type-param-constraint-flag', 'type-name')),
+ default('#pop'),
+ ],
+
+ 'type-param-constraint-sep': [
+ include('spaces'),
+ (r'>', Punctuation, '#pop'),
+ (r',', Punctuation, ('#pop', 'type-param-constraint-sep',
+ 'type-param-constraint-flag', 'type-name')),
+ ],
+
+ # the optional constraint inside type-param
+ 'type-param-constraint-flag': [
+ include('spaces'),
+ (r':', Punctuation, ('#pop', 'type-param-constraint-flag-type')),
+ default('#pop'),
+ ],
+
+ 'type-param-constraint-flag-type': [
+ include('spaces'),
+ (r'\(', Punctuation, ('#pop', 'type-param-constraint-flag-type-sep',
+ 'type')),
+ default(('#pop', 'type')),
+ ],
+
+ 'type-param-constraint-flag-type-sep': [
+ include('spaces'),
+ (r'\)', Punctuation, '#pop'),
+ (r',', Punctuation, 'type'),
+ ],
+
+ # a parenthesis expr that contain exactly one expr
+ 'parenthesis': [
+ include('spaces'),
+ default(('#pop', 'parenthesis-close', 'flag', 'expr')),
+ ],
+
+ 'parenthesis-open': [
+ include('spaces'),
+ (r'\(', Punctuation, '#pop'),
+ ],
+
+ 'parenthesis-close': [
+ include('spaces'),
+ (r'\)', Punctuation, '#pop'),
+ ],
+
+ 'var': [
+ include('spaces'),
+ (ident_no_keyword, Text, ('#pop', 'var-sep', 'assign', 'flag', 'prop-get-set')),
+ ],
+
+ # optional more var decl.
+ 'var-sep': [
+ include('spaces'),
+ (r',', Punctuation, ('#pop', 'var')),
+ default('#pop'),
+ ],
+
+ # optional assignment
+ 'assign': [
+ include('spaces'),
+ (r'=', Operator, ('#pop', 'expr')),
+ default('#pop'),
+ ],
+
+ # optional type flag
+ 'flag': [
+ include('spaces'),
+ (r':', Punctuation, ('#pop', 'type')),
+ default('#pop'),
+ ],
+
+ # colon as part of a ternary operator (?:)
+ 'ternary': [
+ include('spaces'),
+ (r':', Operator, '#pop'),
+ ],
+
+ # function call
+ 'call': [
+ include('spaces'),
+ (r'\)', Punctuation, '#pop'),
+ default(('#pop', 'call-sep', 'expr')),
+ ],
+
+ # after a call param
+ 'call-sep': [
+ include('spaces'),
+ (r'\)', Punctuation, '#pop'),
+ (r',', Punctuation, ('#pop', 'call')),
+ ],
+
+ # bracket can be block or object
+ 'bracket': [
+ include('spaces'),
+ (r'(?!(?:\$\s*[a-z]\b|\$(?!'+ident+')))' + ident_no_keyword, Name,
+ ('#pop', 'bracket-check')),
+ (r"'", String.Single, ('#pop', 'bracket-check', 'string-single')),
+ (r'"', String.Double, ('#pop', 'bracket-check', 'string-double')),
+ default(('#pop', 'block')),
+ ],
+
+ 'bracket-check': [
+ include('spaces'),
+ (r':', Punctuation, ('#pop', 'object-sep', 'expr')), # is object
+ default(('#pop', 'block', 'optional-semicolon', 'expr-chain')), # is block
+ ],
+
+ # code block
+ 'block': [
+ include('spaces'),
+ (r'\}', Punctuation, '#pop'),
+ default('expr-statement'),
+ ],
+
+ # object in key-value pairs
+ 'object': [
+ include('spaces'),
+ (r'\}', Punctuation, '#pop'),
+ default(('#pop', 'object-sep', 'expr', 'colon', 'ident-or-string'))
+ ],
+
+ # a key of an object
+ 'ident-or-string': [
+ include('spaces'),
+ (ident_no_keyword, Name, '#pop'),
+ (r"'", String.Single, ('#pop', 'string-single')),
+ (r'"', String.Double, ('#pop', 'string-double')),
+ ],
+
+ # after a key-value pair in object
+ 'object-sep': [
+ include('spaces'),
+ (r'\}', Punctuation, '#pop'),
+ (r',', Punctuation, ('#pop', 'object')),
+ ],
+
+
+
+ }
+
+ def analyse_text(text):
+ if re.match(r'\w+\s*:\s*\w', text):
+ return 0.3
+
+
+class HxmlLexer(RegexLexer):
+ """
+ Lexer for `haXe build <http://haxe.org/doc/compiler>`_ files.
+
+ .. versionadded:: 1.6
+ """
+ name = 'Hxml'
+ aliases = ['haxeml', 'hxml']
+ filenames = ['*.hxml']
+
+ tokens = {
+ 'root': [
+ # Seperator
+ (r'(--)(next)', bygroups(Punctuation, Generic.Heading)),
+ # Compiler switches with one dash
+ (r'(-)(prompt|debug|v)', bygroups(Punctuation, Keyword.Keyword)),
+ # Compilerswitches with two dashes
+ (r'(--)(neko-source|flash-strict|flash-use-stage|no-opt|no-traces|'
+ r'no-inline|times|no-output)', bygroups(Punctuation, Keyword)),
+ # Targets and other options that take an argument
+ (r'(-)(cpp|js|neko|x|as3|swf9?|swf-lib|php|xml|main|lib|D|resource|'
+ r'cp|cmd)( +)(.+)',
+ bygroups(Punctuation, Keyword, Whitespace, String)),
+ # Options that take only numerical arguments
+ (r'(-)(swf-version)( +)(\d+)',
+ bygroups(Punctuation, Keyword, Number.Integer)),
+ # An Option that defines the size, the fps and the background
+ # color of an flash movie
+ (r'(-)(swf-header)( +)(\d+)(:)(\d+)(:)(\d+)(:)([A-Fa-f0-9]{6})',
+ bygroups(Punctuation, Keyword, Whitespace, Number.Integer,
+ Punctuation, Number.Integer, Punctuation, Number.Integer,
+ Punctuation, Number.Hex)),
+ # options with two dashes that takes arguments
+ (r'(--)(js-namespace|php-front|php-lib|remap|gen-hx-classes)( +)'
+ r'(.+)', bygroups(Punctuation, Keyword, Whitespace, String)),
+ # Single line comment, multiline ones are not allowed.
+ (r'#.*', Comment.Single)
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/hdl.py b/wandb/vendor/pygments/lexers/hdl.py
new file mode 100644
index 0000000000000000000000000000000000000000..57fb7ac96242ce7e0084bed66d22e058b5c02cef
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/hdl.py
@@ -0,0 +1,382 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.hdl
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexers for hardware descriptor languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, include, using, this, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Error
+
+__all__ = ['VerilogLexer', 'SystemVerilogLexer', 'VhdlLexer']
+
+
+class VerilogLexer(RegexLexer):
+ """
+ For verilog source code with preprocessor directives.
+
+ .. versionadded:: 1.4
+ """
+ name = 'verilog'
+ aliases = ['verilog', 'v']
+ filenames = ['*.v']
+ mimetypes = ['text/x-verilog']
+
+ #: optional Comment or Whitespace
+ _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+'
+
+ tokens = {
+ 'root': [
+ (r'^\s*`define', Comment.Preproc, 'macro'),
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'\\\n', Text), # line continuation
+ (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single),
+ (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+ (r'[{}#@]', Punctuation),
+ (r'L?"', String, 'string'),
+ (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+ (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float),
+ (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+ (r'([0-9]+)|(\'h)[0-9a-fA-F]+', Number.Hex),
+ (r'([0-9]+)|(\'b)[01]+', Number.Bin),
+ (r'([0-9]+)|(\'d)[0-9]+', Number.Integer),
+ (r'([0-9]+)|(\'o)[0-7]+', Number.Oct),
+ (r'\'[01xz]', Number),
+ (r'\d+[Ll]?', Number.Integer),
+ (r'\*/', Error),
+ (r'[~!%^&*+=|?:<>/-]', Operator),
+ (r'[()\[\],.;\']', Punctuation),
+ (r'`[a-zA-Z_]\w*', Name.Constant),
+
+ (r'^(\s*)(package)(\s+)', bygroups(Text, Keyword.Namespace, Text)),
+ (r'^(\s*)(import)(\s+)', bygroups(Text, Keyword.Namespace, Text),
+ 'import'),
+
+ (words((
+ 'always', 'always_comb', 'always_ff', 'always_latch', 'and',
+ 'assign', 'automatic', 'begin', 'break', 'buf', 'bufif0', 'bufif1',
+ 'case', 'casex', 'casez', 'cmos', 'const', 'continue', 'deassign',
+ 'default', 'defparam', 'disable', 'do', 'edge', 'else', 'end', 'endcase',
+ 'endfunction', 'endgenerate', 'endmodule', 'endpackage', 'endprimitive',
+ 'endspecify', 'endtable', 'endtask', 'enum', 'event', 'final', 'for',
+ 'force', 'forever', 'fork', 'function', 'generate', 'genvar', 'highz0',
+ 'highz1', 'if', 'initial', 'inout', 'input', 'integer', 'join', 'large',
+ 'localparam', 'macromodule', 'medium', 'module', 'nand', 'negedge',
+ 'nmos', 'nor', 'not', 'notif0', 'notif1', 'or', 'output', 'packed',
+ 'parameter', 'pmos', 'posedge', 'primitive', 'pull0', 'pull1',
+ 'pulldown', 'pullup', 'rcmos', 'ref', 'release', 'repeat', 'return',
+ 'rnmos', 'rpmos', 'rtran', 'rtranif0', 'rtranif1', 'scalared', 'signed',
+ 'small', 'specify', 'specparam', 'strength', 'string', 'strong0',
+ 'strong1', 'struct', 'table', 'task', 'tran', 'tranif0', 'tranif1',
+ 'type', 'typedef', 'unsigned', 'var', 'vectored', 'void', 'wait',
+ 'weak0', 'weak1', 'while', 'xnor', 'xor'), suffix=r'\b'),
+ Keyword),
+
+ (words((
+ 'accelerate', 'autoexpand_vectornets', 'celldefine', 'default_nettype',
+ 'else', 'elsif', 'endcelldefine', 'endif', 'endprotect', 'endprotected',
+ 'expand_vectornets', 'ifdef', 'ifndef', 'include', 'noaccelerate',
+ 'noexpand_vectornets', 'noremove_gatenames', 'noremove_netnames',
+ 'nounconnected_drive', 'protect', 'protected', 'remove_gatenames',
+ 'remove_netnames', 'resetall', 'timescale', 'unconnected_drive',
+ 'undef'), prefix=r'`', suffix=r'\b'),
+ Comment.Preproc),
+
+ (words((
+ 'bits', 'bitstoreal', 'bitstoshortreal', 'countdrivers', 'display', 'fclose',
+ 'fdisplay', 'finish', 'floor', 'fmonitor', 'fopen', 'fstrobe', 'fwrite',
+ 'getpattern', 'history', 'incsave', 'input', 'itor', 'key', 'list', 'log',
+ 'monitor', 'monitoroff', 'monitoron', 'nokey', 'nolog', 'printtimescale',
+ 'random', 'readmemb', 'readmemh', 'realtime', 'realtobits', 'reset',
+ 'reset_count', 'reset_value', 'restart', 'rtoi', 'save', 'scale', 'scope',
+ 'shortrealtobits', 'showscopes', 'showvariables', 'showvars', 'sreadmemb',
+ 'sreadmemh', 'stime', 'stop', 'strobe', 'time', 'timeformat', 'write'),
+ prefix=r'\$', suffix=r'\b'),
+ Name.Builtin),
+
+ (words((
+ 'byte', 'shortint', 'int', 'longint', 'integer', 'time',
+ 'bit', 'logic', 'reg', 'supply0', 'supply1', 'tri', 'triand',
+ 'trior', 'tri0', 'tri1', 'trireg', 'uwire', 'wire', 'wand', 'wo'
+ 'shortreal', 'real', 'realtime'), suffix=r'\b'),
+ Keyword.Type),
+ (r'[a-zA-Z_]\w*:(?!:)', Name.Label),
+ (r'\$?[a-zA-Z_]\w*', Name),
+ ],
+ 'string': [
+ (r'"', String, '#pop'),
+ (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+ (r'[^\\"\n]+', String), # all other characters
+ (r'\\\n', String), # line continuation
+ (r'\\', String), # stray backslash
+ ],
+ 'macro': [
+ (r'[^/\n]+', Comment.Preproc),
+ (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+ (r'//.*?\n', Comment.Single, '#pop'),
+ (r'/', Comment.Preproc),
+ (r'(?<=\\)\n', Comment.Preproc),
+ (r'\n', Comment.Preproc, '#pop'),
+ ],
+ 'import': [
+ (r'[\w:]+\*?', Name.Namespace, '#pop')
+ ]
+ }
+
+ def get_tokens_unprocessed(self, text):
+ for index, token, value in \
+ RegexLexer.get_tokens_unprocessed(self, text):
+ # Convention: mark all upper case names as constants
+ if token is Name:
+ if value.isupper():
+ token = Name.Constant
+ yield index, token, value
+
+
+class SystemVerilogLexer(RegexLexer):
+ """
+ Extends verilog lexer to recognise all SystemVerilog keywords from IEEE
+ 1800-2009 standard.
+
+ .. versionadded:: 1.5
+ """
+ name = 'systemverilog'
+ aliases = ['systemverilog', 'sv']
+ filenames = ['*.sv', '*.svh']
+ mimetypes = ['text/x-systemverilog']
+
+ #: optional Comment or Whitespace
+ _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+'
+
+ tokens = {
+ 'root': [
+ (r'^\s*`define', Comment.Preproc, 'macro'),
+ (r'^(\s*)(package)(\s+)', bygroups(Text, Keyword.Namespace, Text)),
+ (r'^(\s*)(import)(\s+)', bygroups(Text, Keyword.Namespace, Text), 'import'),
+
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'\\\n', Text), # line continuation
+ (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single),
+ (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+ (r'[{}#@]', Punctuation),
+ (r'L?"', String, 'string'),
+ (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+ (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float),
+ (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+ (r'([0-9]+)|(\'h)[0-9a-fA-F]+', Number.Hex),
+ (r'([0-9]+)|(\'b)[01]+', Number.Bin),
+ (r'([0-9]+)|(\'d)[0-9]+', Number.Integer),
+ (r'([0-9]+)|(\'o)[0-7]+', Number.Oct),
+ (r'\'[01xz]', Number),
+ (r'\d+[Ll]?', Number.Integer),
+ (r'\*/', Error),
+ (r'[~!%^&*+=|?:<>/-]', Operator),
+ (r'[()\[\],.;\']', Punctuation),
+ (r'`[a-zA-Z_]\w*', Name.Constant),
+
+ (words((
+ 'accept_on', 'alias', 'always', 'always_comb', 'always_ff', 'always_latch',
+ 'and', 'assert', 'assign', 'assume', 'automatic', 'before', 'begin', 'bind', 'bins',
+ 'binsof', 'bit', 'break', 'buf', 'bufif0', 'bufif1', 'byte', 'case', 'casex', 'casez',
+ 'cell', 'chandle', 'checker', 'class', 'clocking', 'cmos', 'config', 'const', 'constraint',
+ 'context', 'continue', 'cover', 'covergroup', 'coverpoint', 'cross', 'deassign',
+ 'default', 'defparam', 'design', 'disable', 'dist', 'do', 'edge', 'else', 'end', 'endcase',
+ 'endchecker', 'endclass', 'endclocking', 'endconfig', 'endfunction', 'endgenerate',
+ 'endgroup', 'endinterface', 'endmodule', 'endpackage', 'endprimitive',
+ 'endprogram', 'endproperty', 'endsequence', 'endspecify', 'endtable',
+ 'endtask', 'enum', 'event', 'eventually', 'expect', 'export', 'extends', 'extern',
+ 'final', 'first_match', 'for', 'force', 'foreach', 'forever', 'fork', 'forkjoin',
+ 'function', 'generate', 'genvar', 'global', 'highz0', 'highz1', 'if', 'iff', 'ifnone',
+ 'ignore_bins', 'illegal_bins', 'implies', 'import', 'incdir', 'include',
+ 'initial', 'inout', 'input', 'inside', 'instance', 'int', 'integer', 'interface',
+ 'intersect', 'join', 'join_any', 'join_none', 'large', 'let', 'liblist', 'library',
+ 'local', 'localparam', 'logic', 'longint', 'macromodule', 'matches', 'medium',
+ 'modport', 'module', 'nand', 'negedge', 'new', 'nexttime', 'nmos', 'nor', 'noshowcancelled',
+ 'not', 'notif0', 'notif1', 'null', 'or', 'output', 'package', 'packed', 'parameter',
+ 'pmos', 'posedge', 'primitive', 'priority', 'program', 'property', 'protected',
+ 'pull0', 'pull1', 'pulldown', 'pullup', 'pulsestyle_ondetect', 'pulsestyle_onevent',
+ 'pure', 'rand', 'randc', 'randcase', 'randsequence', 'rcmos', 'real', 'realtime',
+ 'ref', 'reg', 'reject_on', 'release', 'repeat', 'restrict', 'return', 'rnmos',
+ 'rpmos', 'rtran', 'rtranif0', 'rtranif1', 's_always', 's_eventually', 's_nexttime',
+ 's_until', 's_until_with', 'scalared', 'sequence', 'shortint', 'shortreal',
+ 'showcancelled', 'signed', 'small', 'solve', 'specify', 'specparam', 'static',
+ 'string', 'strong', 'strong0', 'strong1', 'struct', 'super', 'supply0', 'supply1',
+ 'sync_accept_on', 'sync_reject_on', 'table', 'tagged', 'task', 'this', 'throughout',
+ 'time', 'timeprecision', 'timeunit', 'tran', 'tranif0', 'tranif1', 'tri', 'tri0',
+ 'tri1', 'triand', 'trior', 'trireg', 'type', 'typedef', 'union', 'unique', 'unique0',
+ 'unsigned', 'until', 'until_with', 'untyped', 'use', 'uwire', 'var', 'vectored',
+ 'virtual', 'void', 'wait', 'wait_order', 'wand', 'weak', 'weak0', 'weak1', 'while',
+ 'wildcard', 'wire', 'with', 'within', 'wor', 'xnor', 'xor'), suffix=r'\b'),
+ Keyword),
+
+ (words((
+ '`__FILE__', '`__LINE__', '`begin_keywords', '`celldefine', '`default_nettype',
+ '`define', '`else', '`elsif', '`end_keywords', '`endcelldefine', '`endif',
+ '`ifdef', '`ifndef', '`include', '`line', '`nounconnected_drive', '`pragma',
+ '`resetall', '`timescale', '`unconnected_drive', '`undef', '`undefineall'),
+ suffix=r'\b'),
+ Comment.Preproc),
+
+ (words((
+ '$display', '$displayb', '$displayh', '$displayo', '$dumpall', '$dumpfile',
+ '$dumpflush', '$dumplimit', '$dumpoff', '$dumpon', '$dumpports',
+ '$dumpportsall', '$dumpportsflush', '$dumpportslimit', '$dumpportsoff',
+ '$dumpportson', '$dumpvars', '$fclose', '$fdisplay', '$fdisplayb',
+ '$fdisplayh', '$fdisplayo', '$feof', '$ferror', '$fflush', '$fgetc',
+ '$fgets', '$finish', '$fmonitor', '$fmonitorb', '$fmonitorh', '$fmonitoro',
+ '$fopen', '$fread', '$fscanf', '$fseek', '$fstrobe', '$fstrobeb', '$fstrobeh',
+ '$fstrobeo', '$ftell', '$fwrite', '$fwriteb', '$fwriteh', '$fwriteo',
+ '$monitor', '$monitorb', '$monitorh', '$monitoro', '$monitoroff',
+ '$monitoron', '$plusargs', '$random', '$readmemb', '$readmemh', '$rewind',
+ '$sformat', '$sformatf', '$sscanf', '$strobe', '$strobeb', '$strobeh', '$strobeo',
+ '$swrite', '$swriteb', '$swriteh', '$swriteo', '$test', '$ungetc',
+ '$value$plusargs', '$write', '$writeb', '$writeh', '$writememb',
+ '$writememh', '$writeo'), suffix=r'\b'),
+ Name.Builtin),
+
+ (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
+ (words((
+ 'byte', 'shortint', 'int', 'longint', 'integer', 'time',
+ 'bit', 'logic', 'reg', 'supply0', 'supply1', 'tri', 'triand',
+ 'trior', 'tri0', 'tri1', 'trireg', 'uwire', 'wire', 'wand', 'wo'
+ 'shortreal', 'real', 'realtime'), suffix=r'\b'),
+ Keyword.Type),
+ (r'[a-zA-Z_]\w*:(?!:)', Name.Label),
+ (r'\$?[a-zA-Z_]\w*', Name),
+ ],
+ 'classname': [
+ (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
+ ],
+ 'string': [
+ (r'"', String, '#pop'),
+ (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+ (r'[^\\"\n]+', String), # all other characters
+ (r'\\\n', String), # line continuation
+ (r'\\', String), # stray backslash
+ ],
+ 'macro': [
+ (r'[^/\n]+', Comment.Preproc),
+ (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+ (r'//.*?\n', Comment.Single, '#pop'),
+ (r'/', Comment.Preproc),
+ (r'(?<=\\)\n', Comment.Preproc),
+ (r'\n', Comment.Preproc, '#pop'),
+ ],
+ 'import': [
+ (r'[\w:]+\*?', Name.Namespace, '#pop')
+ ]
+ }
+
+ def get_tokens_unprocessed(self, text):
+ for index, token, value in \
+ RegexLexer.get_tokens_unprocessed(self, text):
+ # Convention: mark all upper case names as constants
+ if token is Name:
+ if value.isupper():
+ token = Name.Constant
+ yield index, token, value
+
+
+class VhdlLexer(RegexLexer):
+ """
+ For VHDL source code.
+
+ .. versionadded:: 1.5
+ """
+ name = 'vhdl'
+ aliases = ['vhdl']
+ filenames = ['*.vhdl', '*.vhd']
+ mimetypes = ['text/x-vhdl']
+ flags = re.MULTILINE | re.IGNORECASE
+
+ tokens = {
+ 'root': [
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'\\\n', Text), # line continuation
+ (r'--.*?$', Comment.Single),
+ (r"'(U|X|0|1|Z|W|L|H|-)'", String.Char),
+ (r'[~!%^&*+=|?:<>/-]', Operator),
+ (r"'[a-z_]\w*", Name.Attribute),
+ (r'[()\[\],.;\']', Punctuation),
+ (r'"[^\n\\"]*"', String),
+
+ (r'(library)(\s+)([a-z_]\w*)',
+ bygroups(Keyword, Text, Name.Namespace)),
+ (r'(use)(\s+)(entity)', bygroups(Keyword, Text, Keyword)),
+ (r'(use)(\s+)([a-z_][\w.]*\.)(all)',
+ bygroups(Keyword, Text, Name.Namespace, Keyword)),
+ (r'(use)(\s+)([a-z_][\w.]*)',
+ bygroups(Keyword, Text, Name.Namespace)),
+ (r'(std|ieee)(\.[a-z_]\w*)',
+ bygroups(Name.Namespace, Name.Namespace)),
+ (words(('std', 'ieee', 'work'), suffix=r'\b'),
+ Name.Namespace),
+ (r'(entity|component)(\s+)([a-z_]\w*)',
+ bygroups(Keyword, Text, Name.Class)),
+ (r'(architecture|configuration)(\s+)([a-z_]\w*)(\s+)'
+ r'(of)(\s+)([a-z_]\w*)(\s+)(is)',
+ bygroups(Keyword, Text, Name.Class, Text, Keyword, Text,
+ Name.Class, Text, Keyword)),
+ (r'([a-z_]\w*)(:)(\s+)(process|for)',
+ bygroups(Name.Class, Operator, Text, Keyword)),
+ (r'(end)(\s+)', bygroups(using(this), Text), 'endblock'),
+
+ include('types'),
+ include('keywords'),
+ include('numbers'),
+
+ (r'[a-z_]\w*', Name),
+ ],
+ 'endblock': [
+ include('keywords'),
+ (r'[a-z_]\w*', Name.Class),
+ (r'(\s+)', Text),
+ (r';', Punctuation, '#pop'),
+ ],
+ 'types': [
+ (words((
+ 'boolean', 'bit', 'character', 'severity_level', 'integer', 'time',
+ 'delay_length', 'natural', 'positive', 'string', 'bit_vector',
+ 'file_open_kind', 'file_open_status', 'std_ulogic', 'std_ulogic_vector',
+ 'std_logic', 'std_logic_vector', 'signed', 'unsigned'), suffix=r'\b'),
+ Keyword.Type),
+ ],
+ 'keywords': [
+ (words((
+ 'abs', 'access', 'after', 'alias', 'all', 'and',
+ 'architecture', 'array', 'assert', 'attribute', 'begin', 'block',
+ 'body', 'buffer', 'bus', 'case', 'component', 'configuration',
+ 'constant', 'disconnect', 'downto', 'else', 'elsif', 'end',
+ 'entity', 'exit', 'file', 'for', 'function', 'generate',
+ 'generic', 'group', 'guarded', 'if', 'impure', 'in',
+ 'inertial', 'inout', 'is', 'label', 'library', 'linkage',
+ 'literal', 'loop', 'map', 'mod', 'nand', 'new',
+ 'next', 'nor', 'not', 'null', 'of', 'on',
+ 'open', 'or', 'others', 'out', 'package', 'port',
+ 'postponed', 'procedure', 'process', 'pure', 'range', 'record',
+ 'register', 'reject', 'rem', 'return', 'rol', 'ror', 'select',
+ 'severity', 'signal', 'shared', 'sla', 'sll', 'sra',
+ 'srl', 'subtype', 'then', 'to', 'transport', 'type',
+ 'units', 'until', 'use', 'variable', 'wait', 'when',
+ 'while', 'with', 'xnor', 'xor'), suffix=r'\b'),
+ Keyword),
+ ],
+ 'numbers': [
+ (r'\d{1,2}#[0-9a-f_]+#?', Number.Integer),
+ (r'\d+', Number.Integer),
+ (r'(\d+\.\d*|\.\d+|\d+)E[+-]?\d+', Number.Float),
+ (r'X"[0-9a-f_]+"', Number.Hex),
+ (r'O"[0-7_]+"', Number.Oct),
+ (r'B"[01_]+"', Number.Bin),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/hexdump.py b/wandb/vendor/pygments/lexers/hexdump.py
new file mode 100644
index 0000000000000000000000000000000000000000..cba49be720eca200e6753b1fe023a5c8752942af
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/hexdump.py
@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.hexdump
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for hexadecimal dumps.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, include
+from pygments.token import Text, Name, Number, String, Punctuation
+
+__all__ = ['HexdumpLexer']
+
+
+class HexdumpLexer(RegexLexer):
+ """
+ For typical hex dump output formats by the UNIX and GNU/Linux tools ``hexdump``,
+ ``hd``, ``hexcat``, ``od`` and ``xxd``, and the DOS tool ``DEBUG``. For example:
+
+ .. sourcecode:: hexdump
+
+ 00000000 7f 45 4c 46 02 01 01 00 00 00 00 00 00 00 00 00 |.ELF............|
+ 00000010 02 00 3e 00 01 00 00 00 c5 48 40 00 00 00 00 00 |..>......H@.....|
+
+ The specific supported formats are the outputs of:
+
+ * ``hexdump FILE``
+ * ``hexdump -C FILE`` -- the `canonical` format used in the example.
+ * ``hd FILE`` -- same as ``hexdump -C FILE``.
+ * ``hexcat FILE``
+ * ``od -t x1z FILE``
+ * ``xxd FILE``
+ * ``DEBUG.EXE FILE.COM`` and entering ``d`` to the prompt.
+
+ .. versionadded:: 2.1
+ """
+ name = 'Hexdump'
+ aliases = ['hexdump']
+
+ hd = r'[0-9A-Ha-h]'
+
+ tokens = {
+ 'root': [
+ (r'\n', Text),
+ include('offset'),
+ (r'('+hd+r'{2})(\-)('+hd+r'{2})',
+ bygroups(Number.Hex, Punctuation, Number.Hex)),
+ (hd+r'{2}', Number.Hex),
+ (r'(\s{2,3})(\>)(.{16})(\<)$',
+ bygroups(Text, Punctuation, String, Punctuation), 'bracket-strings'),
+ (r'(\s{2,3})(\|)(.{16})(\|)$',
+ bygroups(Text, Punctuation, String, Punctuation), 'piped-strings'),
+ (r'(\s{2,3})(\>)(.{1,15})(\<)$',
+ bygroups(Text, Punctuation, String, Punctuation)),
+ (r'(\s{2,3})(\|)(.{1,15})(\|)$',
+ bygroups(Text, Punctuation, String, Punctuation)),
+ (r'(\s{2,3})(.{1,15})$', bygroups(Text, String)),
+ (r'(\s{2,3})(.{16}|.{20})$', bygroups(Text, String), 'nonpiped-strings'),
+ (r'\s', Text),
+ (r'^\*', Punctuation),
+ ],
+ 'offset': [
+ (r'^('+hd+'+)(:)', bygroups(Name.Label, Punctuation), 'offset-mode'),
+ (r'^'+hd+'+', Name.Label),
+ ],
+ 'offset-mode': [
+ (r'\s', Text, '#pop'),
+ (hd+'+', Name.Label),
+ (r':', Punctuation)
+ ],
+ 'piped-strings': [
+ (r'\n', Text),
+ include('offset'),
+ (hd+r'{2}', Number.Hex),
+ (r'(\s{2,3})(\|)(.{1,16})(\|)$',
+ bygroups(Text, Punctuation, String, Punctuation)),
+ (r'\s', Text),
+ (r'^\*', Punctuation),
+ ],
+ 'bracket-strings': [
+ (r'\n', Text),
+ include('offset'),
+ (hd+r'{2}', Number.Hex),
+ (r'(\s{2,3})(\>)(.{1,16})(\<)$',
+ bygroups(Text, Punctuation, String, Punctuation)),
+ (r'\s', Text),
+ (r'^\*', Punctuation),
+ ],
+ 'nonpiped-strings': [
+ (r'\n', Text),
+ include('offset'),
+ (r'('+hd+r'{2})(\-)('+hd+r'{2})',
+ bygroups(Number.Hex, Punctuation, Number.Hex)),
+ (hd+r'{2}', Number.Hex),
+ (r'(\s{19,})(.{1,20}?)$', bygroups(Text, String)),
+ (r'(\s{2,3})(.{1,20})$', bygroups(Text, String)),
+ (r'\s', Text),
+ (r'^\*', Punctuation),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/html.py b/wandb/vendor/pygments/lexers/html.py
new file mode 100644
index 0000000000000000000000000000000000000000..73f020fa825d2c80441c3958ea3936873007a1e1
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/html.py
@@ -0,0 +1,602 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.html
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for HTML, XML and related markup.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, ExtendedRegexLexer, include, bygroups, \
+ default, using
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Punctuation
+from pygments.util import looks_like_xml, html_doctype_matches
+
+from pygments.lexers.javascript import JavascriptLexer
+from pygments.lexers.jvm import ScalaLexer
+from pygments.lexers.css import CssLexer, _indentation, _starts_block
+from pygments.lexers.ruby import RubyLexer
+
+__all__ = ['HtmlLexer', 'DtdLexer', 'XmlLexer', 'XsltLexer', 'HamlLexer',
+ 'ScamlLexer', 'PugLexer']
+
+
+class HtmlLexer(RegexLexer):
+ """
+ For HTML 4 and XHTML 1 markup. Nested JavaScript and CSS is highlighted
+ by the appropriate lexer.
+ """
+
+ name = 'HTML'
+ aliases = ['html']
+ filenames = ['*.html', '*.htm', '*.xhtml', '*.xslt']
+ mimetypes = ['text/html', 'application/xhtml+xml']
+
+ flags = re.IGNORECASE | re.DOTALL
+ tokens = {
+ 'root': [
+ ('[^<&]+', Text),
+ (r'&\S*?;', Name.Entity),
+ (r'\<\!\[CDATA\[.*?\]\]\>', Comment.Preproc),
+ ('<!--', Comment, 'comment'),
+ (r'<\?.*?\?>', Comment.Preproc),
+ ('<![^>]*>', Comment.Preproc),
+ (r'(<)(\s*)(script)(\s*)',
+ bygroups(Punctuation, Text, Name.Tag, Text),
+ ('script-content', 'tag')),
+ (r'(<)(\s*)(style)(\s*)',
+ bygroups(Punctuation, Text, Name.Tag, Text),
+ ('style-content', 'tag')),
+ # note: this allows tag names not used in HTML like <x:with-dash>,
+ # this is to support yet-unknown template engines and the like
+ (r'(<)(\s*)([\w:.-]+)',
+ bygroups(Punctuation, Text, Name.Tag), 'tag'),
+ (r'(<)(\s*)(/)(\s*)([\w:.-]+)(\s*)(>)',
+ bygroups(Punctuation, Text, Punctuation, Text, Name.Tag, Text,
+ Punctuation)),
+ ],
+ 'comment': [
+ ('[^-]+', Comment),
+ ('-->', Comment, '#pop'),
+ ('-', Comment),
+ ],
+ 'tag': [
+ (r'\s+', Text),
+ (r'([\w:-]+\s*)(=)(\s*)', bygroups(Name.Attribute, Operator, Text),
+ 'attr'),
+ (r'[\w:-]+', Name.Attribute),
+ (r'(/?)(\s*)(>)', bygroups(Punctuation, Text, Punctuation), '#pop'),
+ ],
+ 'script-content': [
+ (r'(<)(\s*)(/)(\s*)(script)(\s*)(>)',
+ bygroups(Punctuation, Text, Punctuation, Text, Name.Tag, Text,
+ Punctuation), '#pop'),
+ (r'.+?(?=<\s*/\s*script\s*>)', using(JavascriptLexer)),
+ ],
+ 'style-content': [
+ (r'(<)(\s*)(/)(\s*)(style)(\s*)(>)',
+ bygroups(Punctuation, Text, Punctuation, Text, Name.Tag, Text,
+ Punctuation),'#pop'),
+ (r'.+?(?=<\s*/\s*style\s*>)', using(CssLexer)),
+ ],
+ 'attr': [
+ ('".*?"', String, '#pop'),
+ ("'.*?'", String, '#pop'),
+ (r'[^\s>]+', String, '#pop'),
+ ],
+ }
+
+ def analyse_text(text):
+ if html_doctype_matches(text):
+ return 0.5
+
+
+class DtdLexer(RegexLexer):
+ """
+ A lexer for DTDs (Document Type Definitions).
+
+ .. versionadded:: 1.5
+ """
+
+ flags = re.MULTILINE | re.DOTALL
+
+ name = 'DTD'
+ aliases = ['dtd']
+ filenames = ['*.dtd']
+ mimetypes = ['application/xml-dtd']
+
+ tokens = {
+ 'root': [
+ include('common'),
+
+ (r'(<!ELEMENT)(\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Tag), 'element'),
+ (r'(<!ATTLIST)(\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Tag), 'attlist'),
+ (r'(<!ENTITY)(\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Entity), 'entity'),
+ (r'(<!NOTATION)(\s+)(\S+)',
+ bygroups(Keyword, Text, Name.Tag), 'notation'),
+ (r'(<!\[)([^\[\s]+)(\s*)(\[)', # conditional sections
+ bygroups(Keyword, Name.Entity, Text, Keyword)),
+
+ (r'(<!DOCTYPE)(\s+)([^>\s]+)',
+ bygroups(Keyword, Text, Name.Tag)),
+ (r'PUBLIC|SYSTEM', Keyword.Constant),
+ (r'[\[\]>]', Keyword),
+ ],
+
+ 'common': [
+ (r'\s+', Text),
+ (r'(%|&)[^;]*;', Name.Entity),
+ ('<!--', Comment, 'comment'),
+ (r'[(|)*,?+]', Operator),
+ (r'"[^"]*"', String.Double),
+ (r'\'[^\']*\'', String.Single),
+ ],
+
+ 'comment': [
+ ('[^-]+', Comment),
+ ('-->', Comment, '#pop'),
+ ('-', Comment),
+ ],
+
+ 'element': [
+ include('common'),
+ (r'EMPTY|ANY|#PCDATA', Keyword.Constant),
+ (r'[^>\s|()?+*,]+', Name.Tag),
+ (r'>', Keyword, '#pop'),
+ ],
+
+ 'attlist': [
+ include('common'),
+ (r'CDATA|IDREFS|IDREF|ID|NMTOKENS|NMTOKEN|ENTITIES|ENTITY|NOTATION',
+ Keyword.Constant),
+ (r'#REQUIRED|#IMPLIED|#FIXED', Keyword.Constant),
+ (r'xml:space|xml:lang', Keyword.Reserved),
+ (r'[^>\s|()?+*,]+', Name.Attribute),
+ (r'>', Keyword, '#pop'),
+ ],
+
+ 'entity': [
+ include('common'),
+ (r'SYSTEM|PUBLIC|NDATA', Keyword.Constant),
+ (r'[^>\s|()?+*,]+', Name.Entity),
+ (r'>', Keyword, '#pop'),
+ ],
+
+ 'notation': [
+ include('common'),
+ (r'SYSTEM|PUBLIC', Keyword.Constant),
+ (r'[^>\s|()?+*,]+', Name.Attribute),
+ (r'>', Keyword, '#pop'),
+ ],
+ }
+
+ def analyse_text(text):
+ if not looks_like_xml(text) and \
+ ('<!ELEMENT' in text or '<!ATTLIST' in text or '<!ENTITY' in text):
+ return 0.8
+
+
+class XmlLexer(RegexLexer):
+ """
+ Generic lexer for XML (eXtensible Markup Language).
+ """
+
+ flags = re.MULTILINE | re.DOTALL | re.UNICODE
+
+ name = 'XML'
+ aliases = ['xml']
+ filenames = ['*.xml', '*.xsl', '*.rss', '*.xslt', '*.xsd',
+ '*.wsdl', '*.wsf']
+ mimetypes = ['text/xml', 'application/xml', 'image/svg+xml',
+ 'application/rss+xml', 'application/atom+xml']
+
+ tokens = {
+ 'root': [
+ ('[^<&]+', Text),
+ (r'&\S*?;', Name.Entity),
+ (r'\<\!\[CDATA\[.*?\]\]\>', Comment.Preproc),
+ ('<!--', Comment, 'comment'),
+ (r'<\?.*?\?>', Comment.Preproc),
+ ('<![^>]*>', Comment.Preproc),
+ (r'<\s*[\w:.-]+', Name.Tag, 'tag'),
+ (r'<\s*/\s*[\w:.-]+\s*>', Name.Tag),
+ ],
+ 'comment': [
+ ('[^-]+', Comment),
+ ('-->', Comment, '#pop'),
+ ('-', Comment),
+ ],
+ 'tag': [
+ (r'\s+', Text),
+ (r'[\w.:-]+\s*=', Name.Attribute, 'attr'),
+ (r'/?\s*>', Name.Tag, '#pop'),
+ ],
+ 'attr': [
+ ('\s+', Text),
+ ('".*?"', String, '#pop'),
+ ("'.*?'", String, '#pop'),
+ (r'[^\s>]+', String, '#pop'),
+ ],
+ }
+
+ def analyse_text(text):
+ if looks_like_xml(text):
+ return 0.45 # less than HTML
+
+
+class XsltLexer(XmlLexer):
+ """
+ A lexer for XSLT.
+
+ .. versionadded:: 0.10
+ """
+
+ name = 'XSLT'
+ aliases = ['xslt']
+ filenames = ['*.xsl', '*.xslt', '*.xpl'] # xpl is XProc
+ mimetypes = ['application/xsl+xml', 'application/xslt+xml']
+
+ EXTRA_KEYWORDS = set((
+ 'apply-imports', 'apply-templates', 'attribute',
+ 'attribute-set', 'call-template', 'choose', 'comment',
+ 'copy', 'copy-of', 'decimal-format', 'element', 'fallback',
+ 'for-each', 'if', 'import', 'include', 'key', 'message',
+ 'namespace-alias', 'number', 'otherwise', 'output', 'param',
+ 'preserve-space', 'processing-instruction', 'sort',
+ 'strip-space', 'stylesheet', 'template', 'text', 'transform',
+ 'value-of', 'variable', 'when', 'with-param'
+ ))
+
+ def get_tokens_unprocessed(self, text):
+ for index, token, value in XmlLexer.get_tokens_unprocessed(self, text):
+ m = re.match('</?xsl:([^>]*)/?>?', value)
+
+ if token is Name.Tag and m and m.group(1) in self.EXTRA_KEYWORDS:
+ yield index, Keyword, value
+ else:
+ yield index, token, value
+
+ def analyse_text(text):
+ if looks_like_xml(text) and '<xsl' in text:
+ return 0.8
+
+
+class HamlLexer(ExtendedRegexLexer):
+ """
+ For Haml markup.
+
+ .. versionadded:: 1.3
+ """
+
+ name = 'Haml'
+ aliases = ['haml']
+ filenames = ['*.haml']
+ mimetypes = ['text/x-haml']
+
+ flags = re.IGNORECASE
+ # Haml can include " |\n" anywhere,
+ # which is ignored and used to wrap long lines.
+ # To accomodate this, use this custom faux dot instead.
+ _dot = r'(?: \|\n(?=.* \|)|.)'
+
+ # In certain places, a comma at the end of the line
+ # allows line wrapping as well.
+ _comma_dot = r'(?:,\s*\n|' + _dot + ')'
+ tokens = {
+ 'root': [
+ (r'[ \t]*\n', Text),
+ (r'[ \t]*', _indentation),
+ ],
+
+ 'css': [
+ (r'\.[\w:-]+', Name.Class, 'tag'),
+ (r'\#[\w:-]+', Name.Function, 'tag'),
+ ],
+
+ 'eval-or-plain': [
+ (r'[&!]?==', Punctuation, 'plain'),
+ (r'([&!]?[=~])(' + _comma_dot + r'*\n)',
+ bygroups(Punctuation, using(RubyLexer)),
+ 'root'),
+ default('plain'),
+ ],
+
+ 'content': [
+ include('css'),
+ (r'%[\w:-]+', Name.Tag, 'tag'),
+ (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'),
+ (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)',
+ bygroups(Comment, Comment.Special, Comment),
+ '#pop'),
+ (r'/' + _dot + r'*\n', _starts_block(Comment, 'html-comment-block'),
+ '#pop'),
+ (r'-#' + _dot + r'*\n', _starts_block(Comment.Preproc,
+ 'haml-comment-block'), '#pop'),
+ (r'(-)(' + _comma_dot + r'*\n)',
+ bygroups(Punctuation, using(RubyLexer)),
+ '#pop'),
+ (r':' + _dot + r'*\n', _starts_block(Name.Decorator, 'filter-block'),
+ '#pop'),
+ include('eval-or-plain'),
+ ],
+
+ 'tag': [
+ include('css'),
+ (r'\{(,\n|' + _dot + ')*?\}', using(RubyLexer)),
+ (r'\[' + _dot + '*?\]', using(RubyLexer)),
+ (r'\(', Text, 'html-attributes'),
+ (r'/[ \t]*\n', Punctuation, '#pop:2'),
+ (r'[<>]{1,2}(?=[ \t=])', Punctuation),
+ include('eval-or-plain'),
+ ],
+
+ 'plain': [
+ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Text),
+ (r'(#\{)(' + _dot + '*?)(\})',
+ bygroups(String.Interpol, using(RubyLexer), String.Interpol)),
+ (r'\n', Text, 'root'),
+ ],
+
+ 'html-attributes': [
+ (r'\s+', Text),
+ (r'[\w:-]+[ \t]*=', Name.Attribute, 'html-attribute-value'),
+ (r'[\w:-]+', Name.Attribute),
+ (r'\)', Text, '#pop'),
+ ],
+
+ 'html-attribute-value': [
+ (r'[ \t]+', Text),
+ (r'\w+', Name.Variable, '#pop'),
+ (r'@\w+', Name.Variable.Instance, '#pop'),
+ (r'\$\w+', Name.Variable.Global, '#pop'),
+ (r"'(\\\\|\\'|[^'\n])*'", String, '#pop'),
+ (r'"(\\\\|\\"|[^"\n])*"', String, '#pop'),
+ ],
+
+ 'html-comment-block': [
+ (_dot + '+', Comment),
+ (r'\n', Text, 'root'),
+ ],
+
+ 'haml-comment-block': [
+ (_dot + '+', Comment.Preproc),
+ (r'\n', Text, 'root'),
+ ],
+
+ 'filter-block': [
+ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Name.Decorator),
+ (r'(#\{)(' + _dot + '*?)(\})',
+ bygroups(String.Interpol, using(RubyLexer), String.Interpol)),
+ (r'\n', Text, 'root'),
+ ],
+ }
+
+
+class ScamlLexer(ExtendedRegexLexer):
+ """
+ For `Scaml markup <http://scalate.fusesource.org/>`_. Scaml is Haml for Scala.
+
+ .. versionadded:: 1.4
+ """
+
+ name = 'Scaml'
+ aliases = ['scaml']
+ filenames = ['*.scaml']
+ mimetypes = ['text/x-scaml']
+
+ flags = re.IGNORECASE
+ # Scaml does not yet support the " |\n" notation to
+ # wrap long lines. Once it does, use the custom faux
+ # dot instead.
+ # _dot = r'(?: \|\n(?=.* \|)|.)'
+ _dot = r'.'
+
+ tokens = {
+ 'root': [
+ (r'[ \t]*\n', Text),
+ (r'[ \t]*', _indentation),
+ ],
+
+ 'css': [
+ (r'\.[\w:-]+', Name.Class, 'tag'),
+ (r'\#[\w:-]+', Name.Function, 'tag'),
+ ],
+
+ 'eval-or-plain': [
+ (r'[&!]?==', Punctuation, 'plain'),
+ (r'([&!]?[=~])(' + _dot + r'*\n)',
+ bygroups(Punctuation, using(ScalaLexer)),
+ 'root'),
+ default('plain'),
+ ],
+
+ 'content': [
+ include('css'),
+ (r'%[\w:-]+', Name.Tag, 'tag'),
+ (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'),
+ (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)',
+ bygroups(Comment, Comment.Special, Comment),
+ '#pop'),
+ (r'/' + _dot + r'*\n', _starts_block(Comment, 'html-comment-block'),
+ '#pop'),
+ (r'-#' + _dot + r'*\n', _starts_block(Comment.Preproc,
+ 'scaml-comment-block'), '#pop'),
+ (r'(-@\s*)(import)?(' + _dot + r'*\n)',
+ bygroups(Punctuation, Keyword, using(ScalaLexer)),
+ '#pop'),
+ (r'(-)(' + _dot + r'*\n)',
+ bygroups(Punctuation, using(ScalaLexer)),
+ '#pop'),
+ (r':' + _dot + r'*\n', _starts_block(Name.Decorator, 'filter-block'),
+ '#pop'),
+ include('eval-or-plain'),
+ ],
+
+ 'tag': [
+ include('css'),
+ (r'\{(,\n|' + _dot + ')*?\}', using(ScalaLexer)),
+ (r'\[' + _dot + '*?\]', using(ScalaLexer)),
+ (r'\(', Text, 'html-attributes'),
+ (r'/[ \t]*\n', Punctuation, '#pop:2'),
+ (r'[<>]{1,2}(?=[ \t=])', Punctuation),
+ include('eval-or-plain'),
+ ],
+
+ 'plain': [
+ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Text),
+ (r'(#\{)(' + _dot + '*?)(\})',
+ bygroups(String.Interpol, using(ScalaLexer), String.Interpol)),
+ (r'\n', Text, 'root'),
+ ],
+
+ 'html-attributes': [
+ (r'\s+', Text),
+ (r'[\w:-]+[ \t]*=', Name.Attribute, 'html-attribute-value'),
+ (r'[\w:-]+', Name.Attribute),
+ (r'\)', Text, '#pop'),
+ ],
+
+ 'html-attribute-value': [
+ (r'[ \t]+', Text),
+ (r'\w+', Name.Variable, '#pop'),
+ (r'@\w+', Name.Variable.Instance, '#pop'),
+ (r'\$\w+', Name.Variable.Global, '#pop'),
+ (r"'(\\\\|\\'|[^'\n])*'", String, '#pop'),
+ (r'"(\\\\|\\"|[^"\n])*"', String, '#pop'),
+ ],
+
+ 'html-comment-block': [
+ (_dot + '+', Comment),
+ (r'\n', Text, 'root'),
+ ],
+
+ 'scaml-comment-block': [
+ (_dot + '+', Comment.Preproc),
+ (r'\n', Text, 'root'),
+ ],
+
+ 'filter-block': [
+ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Name.Decorator),
+ (r'(#\{)(' + _dot + '*?)(\})',
+ bygroups(String.Interpol, using(ScalaLexer), String.Interpol)),
+ (r'\n', Text, 'root'),
+ ],
+ }
+
+
+class PugLexer(ExtendedRegexLexer):
+ """
+ For Pug markup.
+ Pug is a variant of Scaml, see:
+ http://scalate.fusesource.org/documentation/scaml-reference.html
+
+ .. versionadded:: 1.4
+ """
+
+ name = 'Pug'
+ aliases = ['pug', 'jade']
+ filenames = ['*.pug', '*.jade']
+ mimetypes = ['text/x-pug', 'text/x-jade']
+
+ flags = re.IGNORECASE
+ _dot = r'.'
+
+ tokens = {
+ 'root': [
+ (r'[ \t]*\n', Text),
+ (r'[ \t]*', _indentation),
+ ],
+
+ 'css': [
+ (r'\.[\w:-]+', Name.Class, 'tag'),
+ (r'\#[\w:-]+', Name.Function, 'tag'),
+ ],
+
+ 'eval-or-plain': [
+ (r'[&!]?==', Punctuation, 'plain'),
+ (r'([&!]?[=~])(' + _dot + r'*\n)',
+ bygroups(Punctuation, using(ScalaLexer)), 'root'),
+ default('plain'),
+ ],
+
+ 'content': [
+ include('css'),
+ (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'),
+ (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)',
+ bygroups(Comment, Comment.Special, Comment),
+ '#pop'),
+ (r'/' + _dot + r'*\n', _starts_block(Comment, 'html-comment-block'),
+ '#pop'),
+ (r'-#' + _dot + r'*\n', _starts_block(Comment.Preproc,
+ 'scaml-comment-block'), '#pop'),
+ (r'(-@\s*)(import)?(' + _dot + r'*\n)',
+ bygroups(Punctuation, Keyword, using(ScalaLexer)),
+ '#pop'),
+ (r'(-)(' + _dot + r'*\n)',
+ bygroups(Punctuation, using(ScalaLexer)),
+ '#pop'),
+ (r':' + _dot + r'*\n', _starts_block(Name.Decorator, 'filter-block'),
+ '#pop'),
+ (r'[\w:-]+', Name.Tag, 'tag'),
+ (r'\|', Text, 'eval-or-plain'),
+ ],
+
+ 'tag': [
+ include('css'),
+ (r'\{(,\n|' + _dot + ')*?\}', using(ScalaLexer)),
+ (r'\[' + _dot + '*?\]', using(ScalaLexer)),
+ (r'\(', Text, 'html-attributes'),
+ (r'/[ \t]*\n', Punctuation, '#pop:2'),
+ (r'[<>]{1,2}(?=[ \t=])', Punctuation),
+ include('eval-or-plain'),
+ ],
+
+ 'plain': [
+ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Text),
+ (r'(#\{)(' + _dot + '*?)(\})',
+ bygroups(String.Interpol, using(ScalaLexer), String.Interpol)),
+ (r'\n', Text, 'root'),
+ ],
+
+ 'html-attributes': [
+ (r'\s+', Text),
+ (r'[\w:-]+[ \t]*=', Name.Attribute, 'html-attribute-value'),
+ (r'[\w:-]+', Name.Attribute),
+ (r'\)', Text, '#pop'),
+ ],
+
+ 'html-attribute-value': [
+ (r'[ \t]+', Text),
+ (r'\w+', Name.Variable, '#pop'),
+ (r'@\w+', Name.Variable.Instance, '#pop'),
+ (r'\$\w+', Name.Variable.Global, '#pop'),
+ (r"'(\\\\|\\'|[^'\n])*'", String, '#pop'),
+ (r'"(\\\\|\\"|[^"\n])*"', String, '#pop'),
+ ],
+
+ 'html-comment-block': [
+ (_dot + '+', Comment),
+ (r'\n', Text, 'root'),
+ ],
+
+ 'scaml-comment-block': [
+ (_dot + '+', Comment.Preproc),
+ (r'\n', Text, 'root'),
+ ],
+
+ 'filter-block': [
+ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Name.Decorator),
+ (r'(#\{)(' + _dot + '*?)(\})',
+ bygroups(String.Interpol, using(ScalaLexer), String.Interpol)),
+ (r'\n', Text, 'root'),
+ ],
+ }
+JadeLexer = PugLexer # compat
diff --git a/wandb/vendor/pygments/lexers/idl.py b/wandb/vendor/pygments/lexers/idl.py
new file mode 100644
index 0000000000000000000000000000000000000000..990789702799df1a66dc7f854df55b0fbc4b78b2
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/idl.py
@@ -0,0 +1,270 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.idl
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexers for IDL.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, Number, String
+
+__all__ = ['IDLLexer']
+
+
+class IDLLexer(RegexLexer):
+ """
+ Pygments Lexer for IDL (Interactive Data Language).
+
+ .. versionadded:: 1.6
+ """
+ name = 'IDL'
+ aliases = ['idl']
+ filenames = ['*.pro']
+ mimetypes = ['text/idl']
+
+ flags = re.IGNORECASE | re.MULTILINE
+
+ _RESERVED = (
+ 'and', 'begin', 'break', 'case', 'common', 'compile_opt',
+ 'continue', 'do', 'else', 'end', 'endcase', 'elseelse',
+ 'endfor', 'endforeach', 'endif', 'endrep', 'endswitch',
+ 'endwhile', 'eq', 'for', 'foreach', 'forward_function',
+ 'function', 'ge', 'goto', 'gt', 'if', 'inherits', 'le',
+ 'lt', 'mod', 'ne', 'not', 'of', 'on_ioerror', 'or', 'pro',
+ 'repeat', 'switch', 'then', 'until', 'while', 'xor')
+ """Reserved words from: http://www.exelisvis.com/docs/reswords.html"""
+
+ _BUILTIN_LIB = (
+ 'abs', 'acos', 'adapt_hist_equal', 'alog', 'alog10',
+ 'amoeba', 'annotate', 'app_user_dir', 'app_user_dir_query',
+ 'arg_present', 'array_equal', 'array_indices', 'arrow',
+ 'ascii_template', 'asin', 'assoc', 'atan', 'axis',
+ 'a_correlate', 'bandpass_filter', 'bandreject_filter',
+ 'barplot', 'bar_plot', 'beseli', 'beselj', 'beselk',
+ 'besely', 'beta', 'bilinear', 'binary_template', 'bindgen',
+ 'binomial', 'bin_date', 'bit_ffs', 'bit_population',
+ 'blas_axpy', 'blk_con', 'box_cursor', 'breakpoint',
+ 'broyden', 'butterworth', 'bytarr', 'byte', 'byteorder',
+ 'bytscl', 'caldat', 'calendar', 'call_external',
+ 'call_function', 'call_method', 'call_procedure', 'canny',
+ 'catch', 'cd', 'cdf_\w*', 'ceil', 'chebyshev',
+ 'check_math',
+ 'chisqr_cvf', 'chisqr_pdf', 'choldc', 'cholsol', 'cindgen',
+ 'cir_3pnt', 'close', 'cluster', 'cluster_tree', 'clust_wts',
+ 'cmyk_convert', 'colorbar', 'colorize_sample',
+ 'colormap_applicable', 'colormap_gradient',
+ 'colormap_rotation', 'colortable', 'color_convert',
+ 'color_exchange', 'color_quan', 'color_range_map', 'comfit',
+ 'command_line_args', 'complex', 'complexarr', 'complexround',
+ 'compute_mesh_normals', 'cond', 'congrid', 'conj',
+ 'constrained_min', 'contour', 'convert_coord', 'convol',
+ 'convol_fft', 'coord2to3', 'copy_lun', 'correlate', 'cos',
+ 'cosh', 'cpu', 'cramer', 'create_cursor', 'create_struct',
+ 'create_view', 'crossp', 'crvlength', 'cti_test',
+ 'ct_luminance', 'cursor', 'curvefit', 'cvttobm', 'cv_coord',
+ 'cw_animate', 'cw_animate_getp', 'cw_animate_load',
+ 'cw_animate_run', 'cw_arcball', 'cw_bgroup', 'cw_clr_index',
+ 'cw_colorsel', 'cw_defroi', 'cw_field', 'cw_filesel',
+ 'cw_form', 'cw_fslider', 'cw_light_editor',
+ 'cw_light_editor_get', 'cw_light_editor_set', 'cw_orient',
+ 'cw_palette_editor', 'cw_palette_editor_get',
+ 'cw_palette_editor_set', 'cw_pdmenu', 'cw_rgbslider',
+ 'cw_tmpl', 'cw_zoom', 'c_correlate', 'dblarr', 'db_exists',
+ 'dcindgen', 'dcomplex', 'dcomplexarr', 'define_key',
+ 'define_msgblk', 'define_msgblk_from_file', 'defroi',
+ 'defsysv', 'delvar', 'dendrogram', 'dendro_plot', 'deriv',
+ 'derivsig', 'determ', 'device', 'dfpmin', 'diag_matrix',
+ 'dialog_dbconnect', 'dialog_message', 'dialog_pickfile',
+ 'dialog_printersetup', 'dialog_printjob',
+ 'dialog_read_image', 'dialog_write_image', 'digital_filter',
+ 'dilate', 'dindgen', 'dissolve', 'dist', 'distance_measure',
+ 'dlm_load', 'dlm_register', 'doc_library', 'double',
+ 'draw_roi', 'edge_dog', 'efont', 'eigenql', 'eigenvec',
+ 'ellipse', 'elmhes', 'emboss', 'empty', 'enable_sysrtn',
+ 'eof', 'eos_\w*', 'erase', 'erf', 'erfc', 'erfcx',
+ 'erode', 'errorplot', 'errplot', 'estimator_filter',
+ 'execute', 'exit', 'exp', 'expand', 'expand_path', 'expint',
+ 'extrac', 'extract_slice', 'factorial', 'fft', 'filepath',
+ 'file_basename', 'file_chmod', 'file_copy', 'file_delete',
+ 'file_dirname', 'file_expand_path', 'file_info',
+ 'file_lines', 'file_link', 'file_mkdir', 'file_move',
+ 'file_poll_input', 'file_readlink', 'file_same',
+ 'file_search', 'file_test', 'file_which', 'findgen',
+ 'finite', 'fix', 'flick', 'float', 'floor', 'flow3',
+ 'fltarr', 'flush', 'format_axis_values', 'free_lun',
+ 'fstat', 'fulstr', 'funct', 'fv_test', 'fx_root',
+ 'fz_roots', 'f_cvf', 'f_pdf', 'gamma', 'gamma_ct',
+ 'gauss2dfit', 'gaussfit', 'gaussian_function', 'gaussint',
+ 'gauss_cvf', 'gauss_pdf', 'gauss_smooth', 'getenv',
+ 'getwindows', 'get_drive_list', 'get_dxf_objects',
+ 'get_kbrd', 'get_login_info', 'get_lun', 'get_screen_size',
+ 'greg2jul', 'grib_\w*', 'grid3', 'griddata',
+ 'grid_input', 'grid_tps', 'gs_iter',
+ 'h5[adfgirst]_\w*', 'h5_browser', 'h5_close',
+ 'h5_create', 'h5_get_libversion', 'h5_open', 'h5_parse',
+ 'hanning', 'hash', 'hdf_\w*', 'heap_free',
+ 'heap_gc', 'heap_nosave', 'heap_refcount', 'heap_save',
+ 'help', 'hilbert', 'histogram', 'hist_2d', 'hist_equal',
+ 'hls', 'hough', 'hqr', 'hsv', 'h_eq_ct', 'h_eq_int',
+ 'i18n_multibytetoutf8', 'i18n_multibytetowidechar',
+ 'i18n_utf8tomultibyte', 'i18n_widechartomultibyte',
+ 'ibeta', 'icontour', 'iconvertcoord', 'idelete', 'identity',
+ 'idlexbr_assistant', 'idlitsys_createtool', 'idl_base64',
+ 'idl_validname', 'iellipse', 'igamma', 'igetcurrent',
+ 'igetdata', 'igetid', 'igetproperty', 'iimage', 'image',
+ 'image_cont', 'image_statistics', 'imaginary', 'imap',
+ 'indgen', 'intarr', 'interpol', 'interpolate',
+ 'interval_volume', 'int_2d', 'int_3d', 'int_tabulated',
+ 'invert', 'ioctl', 'iopen', 'iplot', 'ipolygon',
+ 'ipolyline', 'iputdata', 'iregister', 'ireset', 'iresolve',
+ 'irotate', 'ir_filter', 'isa', 'isave', 'iscale',
+ 'isetcurrent', 'isetproperty', 'ishft', 'isocontour',
+ 'isosurface', 'isurface', 'itext', 'itranslate', 'ivector',
+ 'ivolume', 'izoom', 'i_beta', 'journal', 'json_parse',
+ 'json_serialize', 'jul2greg', 'julday', 'keyword_set',
+ 'krig2d', 'kurtosis', 'kw_test', 'l64indgen', 'label_date',
+ 'label_region', 'ladfit', 'laguerre', 'laplacian',
+ 'la_choldc', 'la_cholmprove', 'la_cholsol', 'la_determ',
+ 'la_eigenproblem', 'la_eigenql', 'la_eigenvec', 'la_elmhes',
+ 'la_gm_linear_model', 'la_hqr', 'la_invert',
+ 'la_least_squares', 'la_least_square_equality',
+ 'la_linear_equation', 'la_ludc', 'la_lumprove', 'la_lusol',
+ 'la_svd', 'la_tridc', 'la_trimprove', 'la_triql',
+ 'la_trired', 'la_trisol', 'least_squares_filter', 'leefilt',
+ 'legend', 'legendre', 'linbcg', 'lindgen', 'linfit',
+ 'linkimage', 'list', 'll_arc_distance', 'lmfit', 'lmgr',
+ 'lngamma', 'lnp_test', 'loadct', 'locale_get',
+ 'logical_and', 'logical_or', 'logical_true', 'lon64arr',
+ 'lonarr', 'long', 'long64', 'lsode', 'ludc', 'lumprove',
+ 'lusol', 'lu_complex', 'machar', 'make_array', 'make_dll',
+ 'make_rt', 'map', 'mapcontinents', 'mapgrid', 'map_2points',
+ 'map_continents', 'map_grid', 'map_image', 'map_patch',
+ 'map_proj_forward', 'map_proj_image', 'map_proj_info',
+ 'map_proj_init', 'map_proj_inverse', 'map_set',
+ 'matrix_multiply', 'matrix_power', 'max', 'md_test',
+ 'mean', 'meanabsdev', 'mean_filter', 'median', 'memory',
+ 'mesh_clip', 'mesh_decimate', 'mesh_issolid', 'mesh_merge',
+ 'mesh_numtriangles', 'mesh_obj', 'mesh_smooth',
+ 'mesh_surfacearea', 'mesh_validate', 'mesh_volume',
+ 'message', 'min', 'min_curve_surf', 'mk_html_help',
+ 'modifyct', 'moment', 'morph_close', 'morph_distance',
+ 'morph_gradient', 'morph_hitormiss', 'morph_open',
+ 'morph_thin', 'morph_tophat', 'multi', 'm_correlate',
+ 'ncdf_\w*', 'newton', 'noise_hurl', 'noise_pick',
+ 'noise_scatter', 'noise_slur', 'norm', 'n_elements',
+ 'n_params', 'n_tags', 'objarr', 'obj_class', 'obj_destroy',
+ 'obj_hasmethod', 'obj_isa', 'obj_new', 'obj_valid',
+ 'online_help', 'on_error', 'open', 'oplot', 'oploterr',
+ 'parse_url', 'particle_trace', 'path_cache', 'path_sep',
+ 'pcomp', 'plot', 'plot3d', 'ploterr', 'plots', 'plot_3dbox',
+ 'plot_field', 'pnt_line', 'point_lun', 'polarplot',
+ 'polar_contour', 'polar_surface', 'poly', 'polyfill',
+ 'polyfillv', 'polygon', 'polyline', 'polyshade', 'polywarp',
+ 'poly_2d', 'poly_area', 'poly_fit', 'popd', 'powell',
+ 'pref_commit', 'pref_get', 'pref_set', 'prewitt', 'primes',
+ 'print', 'printd', 'product', 'profile', 'profiler',
+ 'profiles', 'project_vol', 'psafm', 'pseudo',
+ 'ps_show_fonts', 'ptrarr', 'ptr_free', 'ptr_new',
+ 'ptr_valid', 'pushd', 'p_correlate', 'qgrid3', 'qhull',
+ 'qromb', 'qromo', 'qsimp', 'query_ascii', 'query_bmp',
+ 'query_csv', 'query_dicom', 'query_gif', 'query_image',
+ 'query_jpeg', 'query_jpeg2000', 'query_mrsid', 'query_pict',
+ 'query_png', 'query_ppm', 'query_srf', 'query_tiff',
+ 'query_wav', 'radon', 'randomn', 'randomu', 'ranks',
+ 'rdpix', 'read', 'reads', 'readu', 'read_ascii',
+ 'read_binary', 'read_bmp', 'read_csv', 'read_dicom',
+ 'read_gif', 'read_image', 'read_interfile', 'read_jpeg',
+ 'read_jpeg2000', 'read_mrsid', 'read_pict', 'read_png',
+ 'read_ppm', 'read_spr', 'read_srf', 'read_sylk',
+ 'read_tiff', 'read_wav', 'read_wave', 'read_x11_bitmap',
+ 'read_xwd', 'real_part', 'rebin', 'recall_commands',
+ 'recon3', 'reduce_colors', 'reform', 'region_grow',
+ 'register_cursor', 'regress', 'replicate',
+ 'replicate_inplace', 'resolve_all', 'resolve_routine',
+ 'restore', 'retall', 'return', 'reverse', 'rk4', 'roberts',
+ 'rot', 'rotate', 'round', 'routine_filepath',
+ 'routine_info', 'rs_test', 'r_correlate', 'r_test',
+ 'save', 'savgol', 'scale3', 'scale3d', 'scope_level',
+ 'scope_traceback', 'scope_varfetch', 'scope_varname',
+ 'search2d', 'search3d', 'sem_create', 'sem_delete',
+ 'sem_lock', 'sem_release', 'setenv', 'set_plot',
+ 'set_shading', 'sfit', 'shade_surf', 'shade_surf_irr',
+ 'shade_volume', 'shift', 'shift_diff', 'shmdebug', 'shmmap',
+ 'shmunmap', 'shmvar', 'show3', 'showfont', 'simplex', 'sin',
+ 'sindgen', 'sinh', 'size', 'skewness', 'skip_lun',
+ 'slicer3', 'slide_image', 'smooth', 'sobel', 'socket',
+ 'sort', 'spawn', 'spher_harm', 'sph_4pnt', 'sph_scat',
+ 'spline', 'spline_p', 'spl_init', 'spl_interp', 'sprsab',
+ 'sprsax', 'sprsin', 'sprstp', 'sqrt', 'standardize',
+ 'stddev', 'stop', 'strarr', 'strcmp', 'strcompress',
+ 'streamline', 'stregex', 'stretch', 'string', 'strjoin',
+ 'strlen', 'strlowcase', 'strmatch', 'strmessage', 'strmid',
+ 'strpos', 'strput', 'strsplit', 'strtrim', 'struct_assign',
+ 'struct_hide', 'strupcase', 'surface', 'surfr', 'svdc',
+ 'svdfit', 'svsol', 'swap_endian', 'swap_endian_inplace',
+ 'symbol', 'systime', 's_test', 't3d', 'tag_names', 'tan',
+ 'tanh', 'tek_color', 'temporary', 'tetra_clip',
+ 'tetra_surface', 'tetra_volume', 'text', 'thin', 'threed',
+ 'timegen', 'time_test2', 'tm_test', 'total', 'trace',
+ 'transpose', 'triangulate', 'trigrid', 'triql', 'trired',
+ 'trisol', 'tri_surf', 'truncate_lun', 'ts_coef', 'ts_diff',
+ 'ts_fcast', 'ts_smooth', 'tv', 'tvcrs', 'tvlct', 'tvrd',
+ 'tvscl', 'typename', 't_cvt', 't_pdf', 'uindgen', 'uint',
+ 'uintarr', 'ul64indgen', 'ulindgen', 'ulon64arr', 'ulonarr',
+ 'ulong', 'ulong64', 'uniq', 'unsharp_mask', 'usersym',
+ 'value_locate', 'variance', 'vector', 'vector_field', 'vel',
+ 'velovect', 'vert_t3d', 'voigt', 'voronoi', 'voxel_proj',
+ 'wait', 'warp_tri', 'watershed', 'wdelete', 'wf_draw',
+ 'where', 'widget_base', 'widget_button', 'widget_combobox',
+ 'widget_control', 'widget_displaycontextmen', 'widget_draw',
+ 'widget_droplist', 'widget_event', 'widget_info',
+ 'widget_label', 'widget_list', 'widget_propertysheet',
+ 'widget_slider', 'widget_tab', 'widget_table',
+ 'widget_text', 'widget_tree', 'widget_tree_move',
+ 'widget_window', 'wiener_filter', 'window', 'writeu',
+ 'write_bmp', 'write_csv', 'write_gif', 'write_image',
+ 'write_jpeg', 'write_jpeg2000', 'write_nrif', 'write_pict',
+ 'write_png', 'write_ppm', 'write_spr', 'write_srf',
+ 'write_sylk', 'write_tiff', 'write_wav', 'write_wave',
+ 'wset', 'wshow', 'wtn', 'wv_applet', 'wv_cwt',
+ 'wv_cw_wavelet', 'wv_denoise', 'wv_dwt', 'wv_fn_coiflet',
+ 'wv_fn_daubechies', 'wv_fn_gaussian', 'wv_fn_haar',
+ 'wv_fn_morlet', 'wv_fn_paul', 'wv_fn_symlet',
+ 'wv_import_data', 'wv_import_wavelet', 'wv_plot3d_wps',
+ 'wv_plot_multires', 'wv_pwt', 'wv_tool_denoise',
+ 'xbm_edit', 'xdisplayfile', 'xdxf', 'xfont',
+ 'xinteranimate', 'xloadct', 'xmanager', 'xmng_tmpl',
+ 'xmtool', 'xobjview', 'xobjview_rotate',
+ 'xobjview_write_image', 'xpalette', 'xpcolor', 'xplot3d',
+ 'xregistered', 'xroi', 'xsq_test', 'xsurface', 'xvaredit',
+ 'xvolume', 'xvolume_rotate', 'xvolume_write_image',
+ 'xyouts', 'zoom', 'zoom_24')
+ """Functions from: http://www.exelisvis.com/docs/routines-1.html"""
+
+ tokens = {
+ 'root': [
+ (r'^\s*;.*?\n', Comment.Singleline),
+ (words(_RESERVED, prefix=r'\b', suffix=r'\b'), Keyword),
+ (words(_BUILTIN_LIB, prefix=r'\b', suffix=r'\b'), Name.Builtin),
+ (r'\+=|-=|\^=|\*=|/=|#=|##=|<=|>=|=', Operator),
+ (r'\+\+|--|->|\+|-|##|#|\*|/|<|>|&&|\^|~|\|\|\?|:', Operator),
+ (r'\b(mod=|lt=|le=|eq=|ne=|ge=|gt=|not=|and=|or=|xor=)', Operator),
+ (r'\b(mod|lt|le|eq|ne|ge|gt|not|and|or|xor)\b', Operator),
+ (r'"[^\"]*"', String.Double),
+ (r"'[^\']*'", String.Single),
+ (r'\b[+\-]?([0-9]*\.[0-9]+|[0-9]+\.[0-9]*)(D|E)?([+\-]?[0-9]+)?\b',
+ Number.Float),
+ (r'\b\'[+\-]?[0-9A-F]+\'X(U?(S?|L{1,2})|B)\b', Number.Hex),
+ (r'\b\'[+\-]?[0-7]+\'O(U?(S?|L{1,2})|B)\b', Number.Oct),
+ (r'\b[+\-]?[0-9]+U?L{1,2}\b', Number.Integer.Long),
+ (r'\b[+\-]?[0-9]+U?S?\b', Number.Integer),
+ (r'\b[+\-]?[0-9]+B\b', Number),
+ (r'.', Text),
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/igor.py b/wandb/vendor/pygments/lexers/igor.py
new file mode 100644
index 0000000000000000000000000000000000000000..1a21fe878967fa58131061d5b0ad2d5c62503ffb
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/igor.py
@@ -0,0 +1,288 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.igor
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Igor Pro.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Text, Comment, Keyword, Name, String
+
+__all__ = ['IgorLexer']
+
+
+class IgorLexer(RegexLexer):
+ """
+ Pygments Lexer for Igor Pro procedure files (.ipf).
+ See http://www.wavemetrics.com/ and http://www.igorexchange.com/.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Igor'
+ aliases = ['igor', 'igorpro']
+ filenames = ['*.ipf']
+ mimetypes = ['text/ipf']
+
+ flags = re.IGNORECASE | re.MULTILINE
+
+ flowControl = (
+ 'if', 'else', 'elseif', 'endif', 'for', 'endfor', 'strswitch', 'switch',
+ 'case', 'default', 'endswitch', 'do', 'while', 'try', 'catch', 'endtry',
+ 'break', 'continue', 'return', 'AbortOnRTE', 'AbortOnValue'
+ )
+ types = (
+ 'variable', 'string', 'constant', 'strconstant', 'NVAR', 'SVAR', 'WAVE',
+ 'STRUCT', 'dfref', 'funcref', 'char', 'uchar', 'int16', 'uint16', 'int32',
+ 'uint32', 'int64', 'uint64', 'float', 'double'
+ )
+ keywords = (
+ 'override', 'ThreadSafe', 'MultiThread', 'static', 'Proc',
+ 'Picture', 'Prompt', 'DoPrompt', 'macro', 'window', 'function', 'end',
+ 'Structure', 'EndStructure', 'EndMacro', 'Menu', 'SubMenu'
+ )
+ operations = (
+ 'Abort', 'AddFIFOData', 'AddFIFOVectData', 'AddMovieAudio', 'AddMovieFrame',
+ 'AdoptFiles', 'APMath', 'Append', 'AppendImage', 'AppendLayoutObject',
+ 'AppendMatrixContour', 'AppendText', 'AppendToGizmo', 'AppendToGraph',
+ 'AppendToLayout', 'AppendToTable', 'AppendXYZContour', 'AutoPositionWindow',
+ 'BackgroundInfo', 'Beep', 'BoundingBall', 'BoxSmooth', 'BrowseURL', 'BuildMenu',
+ 'Button', 'cd', 'Chart', 'CheckBox', 'CheckDisplayed', 'ChooseColor', 'Close',
+ 'CloseHelp', 'CloseMovie', 'CloseProc', 'ColorScale', 'ColorTab2Wave',
+ 'Concatenate', 'ControlBar', 'ControlInfo', 'ControlUpdate',
+ 'ConvertGlobalStringTextEncoding', 'ConvexHull', 'Convolve', 'CopyFile',
+ 'CopyFolder', 'CopyScales', 'Correlate', 'CreateAliasShortcut', 'CreateBrowser',
+ 'Cross', 'CtrlBackground', 'CtrlFIFO', 'CtrlNamedBackground', 'Cursor',
+ 'CurveFit', 'CustomControl', 'CWT', 'Debugger', 'DebuggerOptions', 'DefaultFont',
+ 'DefaultGuiControls', 'DefaultGuiFont', 'DefaultTextEncoding', 'DefineGuide',
+ 'DelayUpdate', 'DeleteAnnotations', 'DeleteFile', 'DeleteFolder', 'DeletePoints',
+ 'Differentiate', 'dir', 'Display', 'DisplayHelpTopic', 'DisplayProcedure',
+ 'DoAlert', 'DoIgorMenu', 'DoUpdate', 'DoWindow', 'DoXOPIdle', 'DPSS',
+ 'DrawAction', 'DrawArc', 'DrawBezier', 'DrawLine', 'DrawOval', 'DrawPICT',
+ 'DrawPoly', 'DrawRect', 'DrawRRect', 'DrawText', 'DrawUserShape', 'DSPDetrend',
+ 'DSPPeriodogram', 'Duplicate', 'DuplicateDataFolder', 'DWT', 'EdgeStats', 'Edit',
+ 'ErrorBars', 'EstimatePeakSizes', 'Execute', 'ExecuteScriptText',
+ 'ExperimentModified', 'ExportGizmo', 'Extract', 'FastGaussTransform', 'FastOp',
+ 'FBinRead', 'FBinWrite', 'FFT', 'FIFOStatus', 'FIFO2Wave', 'FilterFIR',
+ 'FilterIIR', 'FindAPeak', 'FindContour', 'FindDuplicates', 'FindLevel',
+ 'FindLevels', 'FindPeak', 'FindPointsInPoly', 'FindRoots', 'FindSequence',
+ 'FindValue', 'FPClustering', 'fprintf', 'FReadLine', 'FSetPos', 'FStatus',
+ 'FTPCreateDirectory', 'FTPDelete', 'FTPDownload', 'FTPUpload', 'FuncFit',
+ 'FuncFitMD', 'GBLoadWave', 'GetAxis', 'GetCamera', 'GetFileFolderInfo',
+ 'GetGizmo', 'GetLastUserMenuInfo', 'GetMarquee', 'GetMouse', 'GetSelection',
+ 'GetWindow', 'GPIBReadBinaryWave2', 'GPIBReadBinary2', 'GPIBReadWave2',
+ 'GPIBRead2', 'GPIBWriteBinaryWave2', 'GPIBWriteBinary2', 'GPIBWriteWave2',
+ 'GPIBWrite2', 'GPIB2', 'GraphNormal', 'GraphWaveDraw', 'GraphWaveEdit', 'Grep',
+ 'GroupBox', 'Hanning', 'HDF5CloseFile', 'HDF5CloseGroup', 'HDF5ConvertColors',
+ 'HDF5CreateFile', 'HDF5CreateGroup', 'HDF5CreateLink', 'HDF5Dump',
+ 'HDF5DumpErrors', 'HDF5DumpState', 'HDF5ListAttributes', 'HDF5ListGroup',
+ 'HDF5LoadData', 'HDF5LoadGroup', 'HDF5LoadImage', 'HDF5OpenFile', 'HDF5OpenGroup',
+ 'HDF5SaveData', 'HDF5SaveGroup', 'HDF5SaveImage', 'HDF5TestOperation',
+ 'HDF5UnlinkObject', 'HideIgorMenus', 'HideInfo', 'HideProcedures', 'HideTools',
+ 'HilbertTransform', 'Histogram', 'ICA', 'IFFT', 'ImageAnalyzeParticles',
+ 'ImageBlend', 'ImageBoundaryToMask', 'ImageEdgeDetection', 'ImageFileInfo',
+ 'ImageFilter', 'ImageFocus', 'ImageFromXYZ', 'ImageGenerateROIMask', 'ImageGLCM',
+ 'ImageHistModification', 'ImageHistogram', 'ImageInterpolate', 'ImageLineProfile',
+ 'ImageLoad', 'ImageMorphology', 'ImageRegistration', 'ImageRemoveBackground',
+ 'ImageRestore', 'ImageRotate', 'ImageSave', 'ImageSeedFill', 'ImageSkeleton3d',
+ 'ImageSnake', 'ImageStats', 'ImageThreshold', 'ImageTransform',
+ 'ImageUnwrapPhase', 'ImageWindow', 'IndexSort', 'InsertPoints', 'Integrate',
+ 'IntegrateODE', 'Integrate2D', 'Interpolate2', 'Interpolate3D', 'Interp3DPath',
+ 'JCAMPLoadWave', 'JointHistogram', 'KillBackground', 'KillControl',
+ 'KillDataFolder', 'KillFIFO', 'KillFreeAxis', 'KillPath', 'KillPICTs',
+ 'KillStrings', 'KillVariables', 'KillWaves', 'KillWindow', 'KMeans', 'Label',
+ 'Layout', 'LayoutPageAction', 'LayoutSlideShow', 'Legend',
+ 'LinearFeedbackShiftRegister', 'ListBox', 'LoadData', 'LoadPackagePreferences',
+ 'LoadPICT', 'LoadWave', 'Loess', 'LombPeriodogram', 'Make', 'MakeIndex',
+ 'MarkPerfTestTime', 'MatrixConvolve', 'MatrixCorr', 'MatrixEigenV',
+ 'MatrixFilter', 'MatrixGaussJ', 'MatrixGLM', 'MatrixInverse', 'MatrixLinearSolve',
+ 'MatrixLinearSolveTD', 'MatrixLLS', 'MatrixLUBkSub', 'MatrixLUD', 'MatrixLUDTD',
+ 'MatrixMultiply', 'MatrixOP', 'MatrixSchur', 'MatrixSolve', 'MatrixSVBkSub',
+ 'MatrixSVD', 'MatrixTranspose', 'MeasureStyledText', 'MLLoadWave', 'Modify',
+ 'ModifyBrowser', 'ModifyCamera', 'ModifyContour', 'ModifyControl',
+ 'ModifyControlList', 'ModifyFreeAxis', 'ModifyGizmo', 'ModifyGraph',
+ 'ModifyImage', 'ModifyLayout', 'ModifyPanel', 'ModifyTable', 'ModifyWaterfall',
+ 'MoveDataFolder', 'MoveFile', 'MoveFolder', 'MoveString', 'MoveSubwindow',
+ 'MoveVariable', 'MoveWave', 'MoveWindow', 'MultiTaperPSD',
+ 'MultiThreadingControl', 'NeuralNetworkRun', 'NeuralNetworkTrain', 'NewCamera',
+ 'NewDataFolder', 'NewFIFO', 'NewFIFOChan', 'NewFreeAxis', 'NewGizmo', 'NewImage',
+ 'NewLayout', 'NewMovie', 'NewNotebook', 'NewPanel', 'NewPath', 'NewWaterfall',
+ 'NI4882', 'Note', 'Notebook', 'NotebookAction', 'Open', 'OpenHelp',
+ 'OpenNotebook', 'Optimize', 'ParseOperationTemplate', 'PathInfo', 'PauseForUser',
+ 'PauseUpdate', 'PCA', 'PlayMovie', 'PlayMovieAction', 'PlaySound',
+ 'PopupContextualMenu', 'PopupMenu', 'Preferences', 'PrimeFactors', 'Print',
+ 'printf', 'PrintGraphs', 'PrintLayout', 'PrintNotebook', 'PrintSettings',
+ 'PrintTable', 'Project', 'PulseStats', 'PutScrapText', 'pwd', 'Quit',
+ 'RatioFromNumber', 'Redimension', 'Remove', 'RemoveContour', 'RemoveFromGizmo',
+ 'RemoveFromGraph', 'RemoveFromLayout', 'RemoveFromTable', 'RemoveImage',
+ 'RemoveLayoutObjects', 'RemovePath', 'Rename', 'RenameDataFolder', 'RenamePath',
+ 'RenamePICT', 'RenameWindow', 'ReorderImages', 'ReorderTraces', 'ReplaceText',
+ 'ReplaceWave', 'Resample', 'ResumeUpdate', 'Reverse', 'Rotate', 'Save',
+ 'SaveData', 'SaveExperiment', 'SaveGraphCopy', 'SaveNotebook',
+ 'SavePackagePreferences', 'SavePICT', 'SaveTableCopy', 'SetActiveSubwindow',
+ 'SetAxis', 'SetBackground', 'SetDashPattern', 'SetDataFolder', 'SetDimLabel',
+ 'SetDrawEnv', 'SetDrawLayer', 'SetFileFolderInfo', 'SetFormula', 'SetIgorHook',
+ 'SetIgorMenuMode', 'SetIgorOption', 'SetMarquee', 'SetProcessSleep',
+ 'SetRandomSeed', 'SetScale', 'SetVariable', 'SetWaveLock', 'SetWaveTextEncoding',
+ 'SetWindow', 'ShowIgorMenus', 'ShowInfo', 'ShowTools', 'Silent', 'Sleep',
+ 'Slider', 'Smooth', 'SmoothCustom', 'Sort', 'SortColumns', 'SoundInRecord',
+ 'SoundInSet', 'SoundInStartChart', 'SoundInStatus', 'SoundInStopChart',
+ 'SoundLoadWave', 'SoundSaveWave', 'SphericalInterpolate', 'SphericalTriangulate',
+ 'SplitString', 'SplitWave', 'sprintf', 'sscanf', 'Stack', 'StackWindows',
+ 'StatsAngularDistanceTest', 'StatsANOVA1Test', 'StatsANOVA2NRTest',
+ 'StatsANOVA2RMTest', 'StatsANOVA2Test', 'StatsChiTest',
+ 'StatsCircularCorrelationTest', 'StatsCircularMeans', 'StatsCircularMoments',
+ 'StatsCircularTwoSampleTest', 'StatsCochranTest', 'StatsContingencyTable',
+ 'StatsDIPTest', 'StatsDunnettTest', 'StatsFriedmanTest', 'StatsFTest',
+ 'StatsHodgesAjneTest', 'StatsJBTest', 'StatsKDE', 'StatsKendallTauTest',
+ 'StatsKSTest', 'StatsKWTest', 'StatsLinearCorrelationTest',
+ 'StatsLinearRegression', 'StatsMultiCorrelationTest', 'StatsNPMCTest',
+ 'StatsNPNominalSRTest', 'StatsQuantiles', 'StatsRankCorrelationTest',
+ 'StatsResample', 'StatsSample', 'StatsScheffeTest', 'StatsShapiroWilkTest',
+ 'StatsSignTest', 'StatsSRTest', 'StatsTTest', 'StatsTukeyTest',
+ 'StatsVariancesTest', 'StatsWatsonUSquaredTest', 'StatsWatsonWilliamsTest',
+ 'StatsWheelerWatsonTest', 'StatsWilcoxonRankTest', 'StatsWRCorrelationTest',
+ 'String', 'StructGet', 'StructPut', 'SumDimension', 'SumSeries', 'TabControl',
+ 'Tag', 'TextBox', 'ThreadGroupPutDF', 'ThreadStart', 'Tile', 'TileWindows',
+ 'TitleBox', 'ToCommandLine', 'ToolsGrid', 'Triangulate3d', 'Unwrap', 'URLRequest',
+ 'ValDisplay', 'Variable', 'VDTClosePort2', 'VDTGetPortList2', 'VDTGetStatus2',
+ 'VDTOpenPort2', 'VDTOperationsPort2', 'VDTReadBinaryWave2', 'VDTReadBinary2',
+ 'VDTReadHexWave2', 'VDTReadHex2', 'VDTReadWave2', 'VDTRead2', 'VDTTerminalPort2',
+ 'VDTWriteBinaryWave2', 'VDTWriteBinary2', 'VDTWriteHexWave2', 'VDTWriteHex2',
+ 'VDTWriteWave2', 'VDTWrite2', 'VDT2', 'WaveMeanStdv', 'WaveStats',
+ 'WaveTransform', 'wfprintf', 'WignerTransform', 'WindowFunction', 'XLLoadWave'
+ )
+ functions = (
+ 'abs', 'acos', 'acosh', 'AddListItem', 'AiryA', 'AiryAD', 'AiryB', 'AiryBD',
+ 'alog', 'AnnotationInfo', 'AnnotationList', 'area', 'areaXY', 'asin', 'asinh',
+ 'atan', 'atanh', 'atan2', 'AxisInfo', 'AxisList', 'AxisValFromPixel', 'Besseli',
+ 'Besselj', 'Besselk', 'Bessely', 'beta', 'betai', 'BinarySearch',
+ 'BinarySearchInterp', 'binomial', 'binomialln', 'binomialNoise', 'cabs',
+ 'CaptureHistory', 'CaptureHistoryStart', 'ceil', 'cequal', 'char2num',
+ 'chebyshev', 'chebyshevU', 'CheckName', 'ChildWindowList', 'CleanupName', 'cmplx',
+ 'cmpstr', 'conj', 'ContourInfo', 'ContourNameList', 'ContourNameToWaveRef',
+ 'ContourZ', 'ControlNameList', 'ConvertTextEncoding', 'cos', 'cosh',
+ 'cosIntegral', 'cot', 'coth', 'CountObjects', 'CountObjectsDFR', 'cpowi',
+ 'CreationDate', 'csc', 'csch', 'CsrInfo', 'CsrWave', 'CsrWaveRef', 'CsrXWave',
+ 'CsrXWaveRef', 'CTabList', 'DataFolderDir', 'DataFolderExists',
+ 'DataFolderRefsEqual', 'DataFolderRefStatus', 'date', 'datetime', 'DateToJulian',
+ 'date2secs', 'Dawson', 'DDERequestString', 'defined', 'deltax', 'digamma',
+ 'dilogarithm', 'DimDelta', 'DimOffset', 'DimSize', 'ei', 'enoise', 'equalWaves',
+ 'erf', 'erfc', 'erfcw', 'exists', 'exp', 'ExpConvExp', 'ExpConvExpFit',
+ 'ExpConvExpFitBL', 'ExpConvExpFit1Shape', 'ExpConvExpFit1ShapeBL', 'ExpGauss',
+ 'ExpGaussFit', 'ExpGaussFitBL', 'ExpGaussFit1Shape', 'ExpGaussFit1ShapeBL',
+ 'expInt', 'expIntegralE1', 'expNoise', 'factorial', 'fakedata', 'faverage',
+ 'faverageXY', 'FetchURL', 'FindDimLabel', 'FindListItem', 'floor', 'FontList',
+ 'FontSizeHeight', 'FontSizeStringWidth', 'FresnelCos', 'FresnelSin',
+ 'FuncRefInfo', 'FunctionInfo', 'FunctionList', 'FunctionPath', 'gamma',
+ 'gammaEuler', 'gammaInc', 'gammaNoise', 'gammln', 'gammp', 'gammq', 'Gauss',
+ 'GaussFit', 'GaussFitBL', 'GaussFit1Width', 'GaussFit1WidthBL', 'Gauss1D',
+ 'Gauss2D', 'gcd', 'GetBrowserLine', 'GetBrowserSelection', 'GetDataFolder',
+ 'GetDataFolderDFR', 'GetDefaultFont', 'GetDefaultFontSize', 'GetDefaultFontStyle',
+ 'GetDimLabel', 'GetEnvironmentVariable', 'GetErrMessage', 'GetFormula',
+ 'GetIndependentModuleName', 'GetIndexedObjName', 'GetIndexedObjNameDFR',
+ 'GetKeyState', 'GetRTErrMessage', 'GetRTError', 'GetRTLocation', 'GetRTLocInfo',
+ 'GetRTStackInfo', 'GetScrapText', 'GetUserData', 'GetWavesDataFolder',
+ 'GetWavesDataFolderDFR', 'GizmoInfo', 'GizmoScale', 'gnoise', 'GrepList',
+ 'GrepString', 'GuideInfo', 'GuideNameList', 'Hash', 'hcsr', 'HDF5AttributeInfo',
+ 'HDF5DatasetInfo', 'HDF5LibraryInfo', 'HDF5TypeInfo', 'hermite', 'hermiteGauss',
+ 'HyperGNoise', 'HyperGPFQ', 'HyperG0F1', 'HyperG1F1', 'HyperG2F1', 'IgorInfo',
+ 'IgorVersion', 'imag', 'ImageInfo', 'ImageNameList', 'ImageNameToWaveRef',
+ 'IndependentModuleList', 'IndexedDir', 'IndexedFile', 'Inf', 'Integrate1D',
+ 'interp', 'Interp2D', 'Interp3D', 'inverseERF', 'inverseERFC', 'ItemsInList',
+ 'JacobiCn', 'JacobiSn', 'JulianToDate', 'Laguerre', 'LaguerreA', 'LaguerreGauss',
+ 'LambertW', 'LayoutInfo', 'leftx', 'LegendreA', 'limit', 'ListMatch',
+ 'ListToTextWave', 'ListToWaveRefWave', 'ln', 'log', 'logNormalNoise',
+ 'LorentzianFit', 'LorentzianFitBL', 'LorentzianFit1Width',
+ 'LorentzianFit1WidthBL', 'lorentzianNoise', 'LowerStr', 'MacroList', 'magsqr',
+ 'MandelbrotPoint', 'MarcumQ', 'MatrixCondition', 'MatrixDet', 'MatrixDot',
+ 'MatrixRank', 'MatrixTrace', 'max', 'mean', 'median', 'min', 'mod', 'ModDate',
+ 'MPFXEMGPeak', 'MPFXExpConvExpPeak', 'MPFXGaussPeak', 'MPFXLorenzianPeak',
+ 'MPFXVoigtPeak', 'NameOfWave', 'NaN', 'NewFreeDataFolder', 'NewFreeWave', 'norm',
+ 'NormalizeUnicode', 'note', 'NumberByKey', 'numpnts', 'numtype',
+ 'NumVarOrDefault', 'num2char', 'num2istr', 'num2str', 'NVAR_Exists',
+ 'OperationList', 'PadString', 'PanelResolution', 'ParamIsDefault',
+ 'ParseFilePath', 'PathList', 'pcsr', 'Pi', 'PICTInfo', 'PICTList',
+ 'PixelFromAxisVal', 'pnt2x', 'poissonNoise', 'poly', 'PolygonArea', 'poly2D',
+ 'PossiblyQuoteName', 'ProcedureText', 'p2rect', 'qcsr', 'real', 'RemoveByKey',
+ 'RemoveEnding', 'RemoveFromList', 'RemoveListItem', 'ReplaceNumberByKey',
+ 'ReplaceString', 'ReplaceStringByKey', 'rightx', 'round', 'r2polar', 'sawtooth',
+ 'scaleToIndex', 'ScreenResolution', 'sec', 'sech', 'Secs2Date', 'Secs2Time',
+ 'SelectNumber', 'SelectString', 'SetEnvironmentVariable', 'sign', 'sin', 'sinc',
+ 'sinh', 'sinIntegral', 'SortList', 'SpecialCharacterInfo', 'SpecialCharacterList',
+ 'SpecialDirPath', 'SphericalBessJ', 'SphericalBessJD', 'SphericalBessY',
+ 'SphericalBessYD', 'SphericalHarmonics', 'sqrt', 'StartMSTimer', 'StatsBetaCDF',
+ 'StatsBetaPDF', 'StatsBinomialCDF', 'StatsBinomialPDF', 'StatsCauchyCDF',
+ 'StatsCauchyPDF', 'StatsChiCDF', 'StatsChiPDF', 'StatsCMSSDCDF',
+ 'StatsCorrelation', 'StatsDExpCDF', 'StatsDExpPDF', 'StatsErlangCDF',
+ 'StatsErlangPDF', 'StatsErrorPDF', 'StatsEValueCDF', 'StatsEValuePDF',
+ 'StatsExpCDF', 'StatsExpPDF', 'StatsFCDF', 'StatsFPDF', 'StatsFriedmanCDF',
+ 'StatsGammaCDF', 'StatsGammaPDF', 'StatsGeometricCDF', 'StatsGeometricPDF',
+ 'StatsGEVCDF', 'StatsGEVPDF', 'StatsHyperGCDF', 'StatsHyperGPDF',
+ 'StatsInvBetaCDF', 'StatsInvBinomialCDF', 'StatsInvCauchyCDF', 'StatsInvChiCDF',
+ 'StatsInvCMSSDCDF', 'StatsInvDExpCDF', 'StatsInvEValueCDF', 'StatsInvExpCDF',
+ 'StatsInvFCDF', 'StatsInvFriedmanCDF', 'StatsInvGammaCDF', 'StatsInvGeometricCDF',
+ 'StatsInvKuiperCDF', 'StatsInvLogisticCDF', 'StatsInvLogNormalCDF',
+ 'StatsInvMaxwellCDF', 'StatsInvMooreCDF', 'StatsInvNBinomialCDF',
+ 'StatsInvNCChiCDF', 'StatsInvNCFCDF', 'StatsInvNormalCDF', 'StatsInvParetoCDF',
+ 'StatsInvPoissonCDF', 'StatsInvPowerCDF', 'StatsInvQCDF', 'StatsInvQpCDF',
+ 'StatsInvRayleighCDF', 'StatsInvRectangularCDF', 'StatsInvSpearmanCDF',
+ 'StatsInvStudentCDF', 'StatsInvTopDownCDF', 'StatsInvTriangularCDF',
+ 'StatsInvUsquaredCDF', 'StatsInvVonMisesCDF', 'StatsInvWeibullCDF',
+ 'StatsKuiperCDF', 'StatsLogisticCDF', 'StatsLogisticPDF', 'StatsLogNormalCDF',
+ 'StatsLogNormalPDF', 'StatsMaxwellCDF', 'StatsMaxwellPDF', 'StatsMedian',
+ 'StatsMooreCDF', 'StatsNBinomialCDF', 'StatsNBinomialPDF', 'StatsNCChiCDF',
+ 'StatsNCChiPDF', 'StatsNCFCDF', 'StatsNCFPDF', 'StatsNCTCDF', 'StatsNCTPDF',
+ 'StatsNormalCDF', 'StatsNormalPDF', 'StatsParetoCDF', 'StatsParetoPDF',
+ 'StatsPermute', 'StatsPoissonCDF', 'StatsPoissonPDF', 'StatsPowerCDF',
+ 'StatsPowerNoise', 'StatsPowerPDF', 'StatsQCDF', 'StatsQpCDF', 'StatsRayleighCDF',
+ 'StatsRayleighPDF', 'StatsRectangularCDF', 'StatsRectangularPDF', 'StatsRunsCDF',
+ 'StatsSpearmanRhoCDF', 'StatsStudentCDF', 'StatsStudentPDF', 'StatsTopDownCDF',
+ 'StatsTriangularCDF', 'StatsTriangularPDF', 'StatsTrimmedMean',
+ 'StatsUSquaredCDF', 'StatsVonMisesCDF', 'StatsVonMisesNoise', 'StatsVonMisesPDF',
+ 'StatsWaldCDF', 'StatsWaldPDF', 'StatsWeibullCDF', 'StatsWeibullPDF',
+ 'StopMSTimer', 'StringByKey', 'stringCRC', 'StringFromList', 'StringList',
+ 'stringmatch', 'strlen', 'strsearch', 'StrVarOrDefault', 'str2num', 'StudentA',
+ 'StudentT', 'sum', 'SVAR_Exists', 'TableInfo', 'TagVal', 'TagWaveRef', 'tan',
+ 'tanh', 'TextEncodingCode', 'TextEncodingName', 'TextFile', 'ThreadGroupCreate',
+ 'ThreadGroupGetDF', 'ThreadGroupGetDFR', 'ThreadGroupRelease', 'ThreadGroupWait',
+ 'ThreadProcessorCount', 'ThreadReturnValue', 'ticks', 'time', 'TraceFromPixel',
+ 'TraceInfo', 'TraceNameList', 'TraceNameToWaveRef', 'trunc', 'UniqueName',
+ 'UnPadString', 'UnsetEnvironmentVariable', 'UpperStr', 'URLDecode', 'URLEncode',
+ 'VariableList', 'Variance', 'vcsr', 'Voigt', 'VoigtFit', 'VoigtFitBL',
+ 'VoigtFit1Shape', 'VoigtFit1ShapeBL', 'VoigtFit1Shape1Width',
+ 'VoigtFit1Shape1WidthBL', 'VoigtFunc', 'WaveCRC', 'WaveDims', 'WaveExists',
+ 'WaveInfo', 'WaveList', 'WaveMax', 'WaveMin', 'WaveName', 'WaveRefIndexed',
+ 'WaveRefIndexedDFR', 'WaveRefsEqual', 'WaveRefWaveToList', 'WaveTextEncoding',
+ 'WaveType', 'WaveUnits', 'WhichListItem', 'WinList', 'WinName', 'WinRecreation',
+ 'WinType', 'WMFindWholeWord', 'WNoise', 'xcsr', 'XWaveName', 'XWaveRefFromTrace',
+ 'x2pnt', 'zcsr', 'ZernikeR', 'zeta'
+ )
+
+ tokens = {
+ 'root': [
+ (r'//.*$', Comment.Single),
+ (r'"([^"\\]|\\.)*"', String),
+ # Flow Control.
+ (words(flowControl, prefix=r'\b', suffix=r'\b'), Keyword),
+ # Types.
+ (words(types, prefix=r'\b', suffix=r'\b'), Keyword.Type),
+ # Keywords.
+ (words(keywords, prefix=r'\b', suffix=r'\b'), Keyword.Reserved),
+ # Built-in operations.
+ (words(operations, prefix=r'\b', suffix=r'\b'), Name.Class),
+ # Built-in functions.
+ (words(functions, prefix=r'\b', suffix=r'\b'), Name.Function),
+ # Compiler directives.
+ (r'^#(include|pragma|define|undef|ifdef|ifndef|if|elif|else|endif)',
+ Name.Decorator),
+ (r'[^a-z"/]+$', Text),
+ (r'.', Text),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/inferno.py b/wandb/vendor/pygments/lexers/inferno.py
new file mode 100644
index 0000000000000000000000000000000000000000..5fc5a0bad34b927a5955c7d5f8e7d0c03e5df60f
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/inferno.py
@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.inferno
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Inferno os and all the related stuff.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, default
+from pygments.token import Punctuation, Text, Comment, Operator, Keyword, \
+ Name, String, Number
+
+__all__ = ['LimboLexer']
+
+
+class LimboLexer(RegexLexer):
+ """
+ Lexer for `Limbo programming language <http://www.vitanuova.com/inferno/limbo.html>`_
+
+ TODO:
+ - maybe implement better var declaration highlighting
+ - some simple syntax error highlighting
+
+ .. versionadded:: 2.0
+ """
+ name = 'Limbo'
+ aliases = ['limbo']
+ filenames = ['*.b']
+ mimetypes = ['text/limbo']
+
+ tokens = {
+ 'whitespace': [
+ (r'^(\s*)([a-zA-Z_]\w*:(\s*)\n)',
+ bygroups(Text, Name.Label)),
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'#(\n|(.|\n)*?[^\\]\n)', Comment.Single),
+ ],
+ 'string': [
+ (r'"', String, '#pop'),
+ (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|'
+ r'u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})', String.Escape),
+ (r'[^\\"\n]+', String), # all other characters
+ (r'\\', String), # stray backslash
+ ],
+ 'statements': [
+ (r'"', String, 'string'),
+ (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+ (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+', Number.Float),
+ (r'(\d+\.\d*|\.\d+|\d+[fF])', Number.Float),
+ (r'16r[0-9a-fA-F]+', Number.Hex),
+ (r'8r[0-7]+', Number.Oct),
+ (r'((([1-3]\d)|([2-9]))r)?(\d+)', Number.Integer),
+ (r'[()\[\],.]', Punctuation),
+ (r'[~!%^&*+=|?:<>/-]|(->)|(<-)|(=>)|(::)', Operator),
+ (r'(alt|break|case|continue|cyclic|do|else|exit'
+ r'for|hd|if|implement|import|include|len|load|or'
+ r'pick|return|spawn|tagof|tl|to|while)\b', Keyword),
+ (r'(byte|int|big|real|string|array|chan|list|adt'
+ r'|fn|ref|of|module|self|type)\b', Keyword.Type),
+ (r'(con|iota|nil)\b', Keyword.Constant),
+ ('[a-zA-Z_]\w*', Name),
+ ],
+ 'statement' : [
+ include('whitespace'),
+ include('statements'),
+ ('[{}]', Punctuation),
+ (';', Punctuation, '#pop'),
+ ],
+ 'root': [
+ include('whitespace'),
+ default('statement'),
+ ],
+ }
+
+ def analyse_text(text):
+ # Any limbo module implements something
+ if re.search(r'^implement \w+;', text, re.MULTILINE):
+ return 0.7
+
+# TODO:
+# - Make lexers for:
+# - asm sources
+# - man pages
+# - mkfiles
+# - module definitions
+# - namespace definitions
+# - shell scripts
+# - maybe keyfiles and fonts
+# they all seem to be quite similar to their equivalents
+# from unix world, so there should not be a lot of problems
diff --git a/wandb/vendor/pygments/lexers/installers.py b/wandb/vendor/pygments/lexers/installers.py
new file mode 100644
index 0000000000000000000000000000000000000000..0323d140925d97b02da198d58769e2ef504ac015
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/installers.py
@@ -0,0 +1,322 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.installers
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for installer/packager DSLs and formats.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, this, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Punctuation, Generic, Number, Whitespace
+
+__all__ = ['NSISLexer', 'RPMSpecLexer', 'SourcesListLexer',
+ 'DebianControlLexer']
+
+
+class NSISLexer(RegexLexer):
+ """
+ For `NSIS <http://nsis.sourceforge.net/>`_ scripts.
+
+ .. versionadded:: 1.6
+ """
+ name = 'NSIS'
+ aliases = ['nsis', 'nsi', 'nsh']
+ filenames = ['*.nsi', '*.nsh']
+ mimetypes = ['text/x-nsis']
+
+ flags = re.IGNORECASE
+
+ tokens = {
+ 'root': [
+ (r'[;#].*\n', Comment),
+ (r"'.*?'", String.Single),
+ (r'"', String.Double, 'str_double'),
+ (r'`', String.Backtick, 'str_backtick'),
+ include('macro'),
+ include('interpol'),
+ include('basic'),
+ (r'\$\{[a-z_|][\w|]*\}', Keyword.Pseudo),
+ (r'/[a-z_]\w*', Name.Attribute),
+ ('.', Text),
+ ],
+ 'basic': [
+ (r'(\n)(Function)(\s+)([._a-z][.\w]*)\b',
+ bygroups(Text, Keyword, Text, Name.Function)),
+ (r'\b([_a-z]\w*)(::)([a-z][a-z0-9]*)\b',
+ bygroups(Keyword.Namespace, Punctuation, Name.Function)),
+ (r'\b([_a-z]\w*)(:)', bygroups(Name.Label, Punctuation)),
+ (r'(\b[ULS]|\B)([!<>=]?=|\<\>?|\>)\B', Operator),
+ (r'[|+-]', Operator),
+ (r'\\', Punctuation),
+ (r'\b(Abort|Add(?:BrandingImage|Size)|'
+ r'Allow(?:RootDirInstall|SkipFiles)|AutoCloseWindow|'
+ r'BG(?:Font|Gradient)|BrandingText|BringToFront|Call(?:InstDLL)?|'
+ r'(?:Sub)?Caption|ChangeUI|CheckBitmap|ClearErrors|CompletedText|'
+ r'ComponentText|CopyFiles|CRCCheck|'
+ r'Create(?:Directory|Font|Shortcut)|Delete(?:INI(?:Sec|Str)|'
+ r'Reg(?:Key|Value))?|DetailPrint|DetailsButtonText|'
+ r'Dir(?:Show|Text|Var|Verify)|(?:Disabled|Enabled)Bitmap|'
+ r'EnableWindow|EnumReg(?:Key|Value)|Exch|Exec(?:Shell|Wait)?|'
+ r'ExpandEnvStrings|File(?:BufSize|Close|ErrorText|Open|'
+ r'Read(?:Byte)?|Seek|Write(?:Byte)?)?|'
+ r'Find(?:Close|First|Next|Window)|FlushINI|Function(?:End)?|'
+ r'Get(?:CurInstType|CurrentAddress|DlgItem|DLLVersion(?:Local)?|'
+ r'ErrorLevel|FileTime(?:Local)?|FullPathName|FunctionAddress|'
+ r'InstDirError|LabelAddress|TempFileName)|'
+ r'Goto|HideWindow|Icon|'
+ r'If(?:Abort|Errors|FileExists|RebootFlag|Silent)|'
+ r'InitPluginsDir|Install(?:ButtonText|Colors|Dir(?:RegKey)?)|'
+ r'Inst(?:ProgressFlags|Type(?:[GS]etText)?)|Int(?:CmpU?|Fmt|Op)|'
+ r'IsWindow|LangString(?:UP)?|'
+ r'License(?:BkColor|Data|ForceSelection|LangString|Text)|'
+ r'LoadLanguageFile|LockWindow|Log(?:Set|Text)|MessageBox|'
+ r'MiscButtonText|Name|Nop|OutFile|(?:Uninst)?Page(?:Ex(?:End)?)?|'
+ r'PluginDir|Pop|Push|Quit|Read(?:(?:Env|INI|Reg)Str|RegDWORD)|'
+ r'Reboot|(?:Un)?RegDLL|Rename|RequestExecutionLevel|ReserveFile|'
+ r'Return|RMDir|SearchPath|Section(?:Divider|End|'
+ r'(?:(?:Get|Set)(?:Flags|InstTypes|Size|Text))|Group(?:End)?|In)?|'
+ r'SendMessage|Set(?:AutoClose|BrandingImage|Compress(?:ionLevel|'
+ r'or(?:DictSize)?)?|CtlColors|CurInstType|DatablockOptimize|'
+ r'DateSave|Details(?:Print|View)|Error(?:s|Level)|FileAttributes|'
+ r'Font|OutPath|Overwrite|PluginUnload|RebootFlag|ShellVarContext|'
+ r'Silent|StaticBkColor)|'
+ r'Show(?:(?:I|Uni)nstDetails|Window)|Silent(?:Un)?Install|Sleep|'
+ r'SpaceTexts|Str(?:CmpS?|Cpy|Len)|SubSection(?:End)?|'
+ r'Uninstall(?:ButtonText|(?:Sub)?Caption|EXEName|Icon|Text)|'
+ r'UninstPage|Var|VI(?:AddVersionKey|ProductVersion)|WindowIcon|'
+ r'Write(?:INIStr|Reg(:?Bin|DWORD|(?:Expand)?Str)|Uninstaller)|'
+ r'XPStyle)\b', Keyword),
+ (r'\b(CUR|END|(?:FILE_ATTRIBUTE_)?'
+ r'(?:ARCHIVE|HIDDEN|NORMAL|OFFLINE|READONLY|SYSTEM|TEMPORARY)|'
+ r'HK(CC|CR|CU|DD|LM|PD|U)|'
+ r'HKEY_(?:CLASSES_ROOT|CURRENT_(?:CONFIG|USER)|DYN_DATA|'
+ r'LOCAL_MACHINE|PERFORMANCE_DATA|USERS)|'
+ r'ID(?:ABORT|CANCEL|IGNORE|NO|OK|RETRY|YES)|'
+ r'MB_(?:ABORTRETRYIGNORE|DEFBUTTON[1-4]|'
+ r'ICON(?:EXCLAMATION|INFORMATION|QUESTION|STOP)|'
+ r'OK(?:CANCEL)?|RETRYCANCEL|RIGHT|SETFOREGROUND|TOPMOST|USERICON|'
+ r'YESNO(?:CANCEL)?)|SET|SHCTX|'
+ r'SW_(?:HIDE|SHOW(?:MAXIMIZED|MINIMIZED|NORMAL))|'
+ r'admin|all|auto|both|bottom|bzip2|checkbox|colored|current|false|'
+ r'force|hide|highest|if(?:diff|newer)|lastused|leave|left|'
+ r'listonly|lzma|nevershow|none|normal|off|on|pop|push|'
+ r'radiobuttons|right|show|silent|silentlog|smooth|textonly|top|'
+ r'true|try|user|zlib)\b', Name.Constant),
+ ],
+ 'macro': [
+ (r'\!(addincludedir(?:dir)?|addplugindir|appendfile|cd|define|'
+ r'delfilefile|echo(?:message)?|else|endif|error|execute|'
+ r'if(?:macro)?n?(?:def)?|include|insertmacro|macro(?:end)?|packhdr|'
+ r'search(?:parse|replace)|system|tempfilesymbol|undef|verbose|'
+ r'warning)\b', Comment.Preproc),
+ ],
+ 'interpol': [
+ (r'\$(R?[0-9])', Name.Builtin.Pseudo), # registers
+ (r'\$(ADMINTOOLS|APPDATA|CDBURN_AREA|COOKIES|COMMONFILES(?:32|64)|'
+ r'DESKTOP|DOCUMENTS|EXE(?:DIR|FILE|PATH)|FAVORITES|FONTS|HISTORY|'
+ r'HWNDPARENT|INTERNET_CACHE|LOCALAPPDATA|MUSIC|NETHOOD|PICTURES|'
+ r'PLUGINSDIR|PRINTHOOD|PROFILE|PROGRAMFILES(?:32|64)|QUICKLAUNCH|'
+ r'RECENT|RESOURCES(?:_LOCALIZED)?|SENDTO|SM(?:PROGRAMS|STARTUP)|'
+ r'STARTMENU|SYSDIR|TEMP(?:LATES)?|VIDEOS|WINDIR|\{NSISDIR\})',
+ Name.Builtin),
+ (r'\$(CMDLINE|INSTDIR|OUTDIR|LANGUAGE)', Name.Variable.Global),
+ (r'\$[a-z_]\w*', Name.Variable),
+ ],
+ 'str_double': [
+ (r'"', String, '#pop'),
+ (r'\$(\\[nrt"]|\$)', String.Escape),
+ include('interpol'),
+ (r'.', String.Double),
+ ],
+ 'str_backtick': [
+ (r'`', String, '#pop'),
+ (r'\$(\\[nrt"]|\$)', String.Escape),
+ include('interpol'),
+ (r'.', String.Double),
+ ],
+ }
+
+
+class RPMSpecLexer(RegexLexer):
+ """
+ For RPM ``.spec`` files.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'RPMSpec'
+ aliases = ['spec']
+ filenames = ['*.spec']
+ mimetypes = ['text/x-rpm-spec']
+
+ _directives = ('(?:package|prep|build|install|clean|check|pre[a-z]*|'
+ 'post[a-z]*|trigger[a-z]*|files)')
+
+ tokens = {
+ 'root': [
+ (r'#.*\n', Comment),
+ include('basic'),
+ ],
+ 'description': [
+ (r'^(%' + _directives + ')(.*)$',
+ bygroups(Name.Decorator, Text), '#pop'),
+ (r'\n', Text),
+ (r'.', Text),
+ ],
+ 'changelog': [
+ (r'\*.*\n', Generic.Subheading),
+ (r'^(%' + _directives + ')(.*)$',
+ bygroups(Name.Decorator, Text), '#pop'),
+ (r'\n', Text),
+ (r'.', Text),
+ ],
+ 'string': [
+ (r'"', String.Double, '#pop'),
+ (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+ include('interpol'),
+ (r'.', String.Double),
+ ],
+ 'basic': [
+ include('macro'),
+ (r'(?i)^(Name|Version|Release|Epoch|Summary|Group|License|Packager|'
+ r'Vendor|Icon|URL|Distribution|Prefix|Patch[0-9]*|Source[0-9]*|'
+ r'Requires\(?[a-z]*\)?|[a-z]+Req|Obsoletes|Suggests|Provides|Conflicts|'
+ r'Build[a-z]+|[a-z]+Arch|Auto[a-z]+)(:)(.*)$',
+ bygroups(Generic.Heading, Punctuation, using(this))),
+ (r'^%description', Name.Decorator, 'description'),
+ (r'^%changelog', Name.Decorator, 'changelog'),
+ (r'^(%' + _directives + ')(.*)$', bygroups(Name.Decorator, Text)),
+ (r'%(attr|defattr|dir|doc(?:dir)?|setup|config(?:ure)?|'
+ r'make(?:install)|ghost|patch[0-9]+|find_lang|exclude|verify)',
+ Keyword),
+ include('interpol'),
+ (r"'.*?'", String.Single),
+ (r'"', String.Double, 'string'),
+ (r'.', Text),
+ ],
+ 'macro': [
+ (r'%define.*\n', Comment.Preproc),
+ (r'%\{\!\?.*%define.*\}', Comment.Preproc),
+ (r'(%(?:if(?:n?arch)?|else(?:if)?|endif))(.*)$',
+ bygroups(Comment.Preproc, Text)),
+ ],
+ 'interpol': [
+ (r'%\{?__[a-z_]+\}?', Name.Function),
+ (r'%\{?_([a-z_]+dir|[a-z_]+path|prefix)\}?', Keyword.Pseudo),
+ (r'%\{\?\w+\}', Name.Variable),
+ (r'\$\{?RPM_[A-Z0-9_]+\}?', Name.Variable.Global),
+ (r'%\{[a-zA-Z]\w+\}', Keyword.Constant),
+ ]
+ }
+
+
+class SourcesListLexer(RegexLexer):
+ """
+ Lexer that highlights debian sources.list files.
+
+ .. versionadded:: 0.7
+ """
+
+ name = 'Debian Sourcelist'
+ aliases = ['sourceslist', 'sources.list', 'debsources']
+ filenames = ['sources.list']
+ mimetype = ['application/x-debian-sourceslist']
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'#.*?$', Comment),
+ (r'^(deb(?:-src)?)(\s+)',
+ bygroups(Keyword, Text), 'distribution')
+ ],
+ 'distribution': [
+ (r'#.*?$', Comment, '#pop'),
+ (r'\$\(ARCH\)', Name.Variable),
+ (r'[^\s$[]+', String),
+ (r'\[', String.Other, 'escaped-distribution'),
+ (r'\$', String),
+ (r'\s+', Text, 'components')
+ ],
+ 'escaped-distribution': [
+ (r'\]', String.Other, '#pop'),
+ (r'\$\(ARCH\)', Name.Variable),
+ (r'[^\]$]+', String.Other),
+ (r'\$', String.Other)
+ ],
+ 'components': [
+ (r'#.*?$', Comment, '#pop:2'),
+ (r'$', Text, '#pop:2'),
+ (r'\s+', Text),
+ (r'\S+', Keyword.Pseudo),
+ ]
+ }
+
+ def analyse_text(text):
+ for line in text.splitlines():
+ line = line.strip()
+ if line.startswith('deb ') or line.startswith('deb-src '):
+ return True
+
+
+class DebianControlLexer(RegexLexer):
+ """
+ Lexer for Debian ``control`` files and ``apt-cache show <pkg>`` outputs.
+
+ .. versionadded:: 0.9
+ """
+ name = 'Debian Control file'
+ aliases = ['control', 'debcontrol']
+ filenames = ['control']
+
+ tokens = {
+ 'root': [
+ (r'^(Description)', Keyword, 'description'),
+ (r'^(Maintainer)(:\s*)', bygroups(Keyword, Text), 'maintainer'),
+ (r'^((Build-)?Depends)', Keyword, 'depends'),
+ (r'^((?:Python-)?Version)(:\s*)(\S+)$',
+ bygroups(Keyword, Text, Number)),
+ (r'^((?:Installed-)?Size)(:\s*)(\S+)$',
+ bygroups(Keyword, Text, Number)),
+ (r'^(MD5Sum|SHA1|SHA256)(:\s*)(\S+)$',
+ bygroups(Keyword, Text, Number)),
+ (r'^([a-zA-Z\-0-9\.]*?)(:\s*)(.*?)$',
+ bygroups(Keyword, Whitespace, String)),
+ ],
+ 'maintainer': [
+ (r'<[^>]+>', Generic.Strong),
+ (r'<[^>]+>$', Generic.Strong, '#pop'),
+ (r',\n?', Text),
+ (r'.', Text),
+ ],
+ 'description': [
+ (r'(.*)(Homepage)(: )(\S+)',
+ bygroups(Text, String, Name, Name.Class)),
+ (r':.*\n', Generic.Strong),
+ (r' .*\n', Text),
+ default('#pop'),
+ ],
+ 'depends': [
+ (r':\s*', Text),
+ (r'(\$)(\{)(\w+\s*:\s*\w+)', bygroups(Operator, Text, Name.Entity)),
+ (r'\(', Text, 'depend_vers'),
+ (r',', Text),
+ (r'\|', Operator),
+ (r'[\s]+', Text),
+ (r'[})]\s*$', Text, '#pop'),
+ (r'\}', Text),
+ (r'[^,]$', Name.Function, '#pop'),
+ (r'([+.a-zA-Z0-9-])(\s*)', bygroups(Name.Function, Text)),
+ (r'\[.*?\]', Name.Entity),
+ ],
+ 'depend_vers': [
+ (r'\),', Text, '#pop'),
+ (r'\)[^,]', Text, '#pop:2'),
+ (r'([><=]+)(\s*)([^)]+)', bygroups(Operator, Text, Number))
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/int_fiction.py b/wandb/vendor/pygments/lexers/int_fiction.py
new file mode 100644
index 0000000000000000000000000000000000000000..f280a56d85fe98204428e689f7332843e2490b0d
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/int_fiction.py
@@ -0,0 +1,1343 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.int_fiction
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for interactive fiction languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, \
+ this, default, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Error, Generic
+
+__all__ = ['Inform6Lexer', 'Inform6TemplateLexer', 'Inform7Lexer',
+ 'Tads3Lexer']
+
+
+class Inform6Lexer(RegexLexer):
+ """
+ For `Inform 6 <http://inform-fiction.org/>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Inform 6'
+ aliases = ['inform6', 'i6']
+ filenames = ['*.inf']
+
+ flags = re.MULTILINE | re.DOTALL | re.UNICODE
+
+ _name = r'[a-zA-Z_]\w*'
+
+ # Inform 7 maps these four character classes to their ASCII
+ # equivalents. To support Inform 6 inclusions within Inform 7,
+ # Inform6Lexer maps them too.
+ _dash = u'\\-\u2010-\u2014'
+ _dquote = u'"\u201c\u201d'
+ _squote = u"'\u2018\u2019"
+ _newline = u'\\n\u0085\u2028\u2029'
+
+ tokens = {
+ 'root': [
+ (r'\A(!%%[^%s]*[%s])+' % (_newline, _newline), Comment.Preproc,
+ 'directive'),
+ default('directive')
+ ],
+ '_whitespace': [
+ (r'\s+', Text),
+ (r'![^%s]*' % _newline, Comment.Single)
+ ],
+ 'default': [
+ include('_whitespace'),
+ (r'\[', Punctuation, 'many-values'), # Array initialization
+ (r':|(?=;)', Punctuation, '#pop'),
+ (r'<', Punctuation), # Second angle bracket in an action statement
+ default(('expression', '_expression'))
+ ],
+
+ # Expressions
+ '_expression': [
+ include('_whitespace'),
+ (r'(?=sp\b)', Text, '#pop'),
+ (r'(?=[%s%s$0-9#a-zA-Z_])' % (_dquote, _squote), Text,
+ ('#pop', 'value')),
+ (r'\+\+|[%s]{1,2}(?!>)|~~?' % _dash, Operator),
+ (r'(?=[()\[%s,?@{:;])' % _dash, Text, '#pop')
+ ],
+ 'expression': [
+ include('_whitespace'),
+ (r'\(', Punctuation, ('expression', '_expression')),
+ (r'\)', Punctuation, '#pop'),
+ (r'\[', Punctuation, ('#pop', 'statements', 'locals')),
+ (r'>(?=(\s+|(![^%s]*))*[>;])' % _newline, Punctuation),
+ (r'\+\+|[%s]{2}(?!>)' % _dash, Operator),
+ (r',', Punctuation, '_expression'),
+ (r'&&?|\|\|?|[=~><]?=|[%s]{1,2}>?|\.\.?[&#]?|::|[<>+*/%%]' % _dash,
+ Operator, '_expression'),
+ (r'(has|hasnt|in|notin|ofclass|or|provides)\b', Operator.Word,
+ '_expression'),
+ (r'sp\b', Name),
+ (r'\?~?', Name.Label, 'label?'),
+ (r'[@{]', Error),
+ default('#pop')
+ ],
+ '_assembly-expression': [
+ (r'\(', Punctuation, ('#push', '_expression')),
+ (r'[\[\]]', Punctuation),
+ (r'[%s]>' % _dash, Punctuation, '_expression'),
+ (r'sp\b', Keyword.Pseudo),
+ (r';', Punctuation, '#pop:3'),
+ include('expression')
+ ],
+ '_for-expression': [
+ (r'\)', Punctuation, '#pop:2'),
+ (r':', Punctuation, '#pop'),
+ include('expression')
+ ],
+ '_keyword-expression': [
+ (r'(from|near|to)\b', Keyword, '_expression'),
+ include('expression')
+ ],
+ '_list-expression': [
+ (r',', Punctuation, '#pop'),
+ include('expression')
+ ],
+ '_object-expression': [
+ (r'has\b', Keyword.Declaration, '#pop'),
+ include('_list-expression')
+ ],
+
+ # Values
+ 'value': [
+ include('_whitespace'),
+ # Strings
+ (r'[%s][^@][%s]' % (_squote, _squote), String.Char, '#pop'),
+ (r'([%s])(@\{[0-9a-fA-F]{1,4}\})([%s])' % (_squote, _squote),
+ bygroups(String.Char, String.Escape, String.Char), '#pop'),
+ (r'([%s])(@.{2})([%s])' % (_squote, _squote),
+ bygroups(String.Char, String.Escape, String.Char), '#pop'),
+ (r'[%s]' % _squote, String.Single, ('#pop', 'dictionary-word')),
+ (r'[%s]' % _dquote, String.Double, ('#pop', 'string')),
+ # Numbers
+ (r'\$[+%s][0-9]*\.?[0-9]*([eE][+%s]?[0-9]+)?' % (_dash, _dash),
+ Number.Float, '#pop'),
+ (r'\$[0-9a-fA-F]+', Number.Hex, '#pop'),
+ (r'\$\$[01]+', Number.Bin, '#pop'),
+ (r'[0-9]+', Number.Integer, '#pop'),
+ # Values prefixed by hashes
+ (r'(##|#a\$)(%s)' % _name, bygroups(Operator, Name), '#pop'),
+ (r'(#g\$)(%s)' % _name,
+ bygroups(Operator, Name.Variable.Global), '#pop'),
+ (r'#[nw]\$', Operator, ('#pop', 'obsolete-dictionary-word')),
+ (r'(#r\$)(%s)' % _name, bygroups(Operator, Name.Function), '#pop'),
+ (r'#', Name.Builtin, ('#pop', 'system-constant')),
+ # System functions
+ (words((
+ 'child', 'children', 'elder', 'eldest', 'glk', 'indirect', 'metaclass',
+ 'parent', 'random', 'sibling', 'younger', 'youngest'), suffix=r'\b'),
+ Name.Builtin, '#pop'),
+ # Metaclasses
+ (r'(?i)(Class|Object|Routine|String)\b', Name.Builtin, '#pop'),
+ # Veneer routines
+ (words((
+ 'Box__Routine', 'CA__Pr', 'CDefArt', 'CInDefArt', 'Cl__Ms',
+ 'Copy__Primitive', 'CP__Tab', 'DA__Pr', 'DB__Pr', 'DefArt', 'Dynam__String',
+ 'EnglishNumber', 'Glk__Wrap', 'IA__Pr', 'IB__Pr', 'InDefArt', 'Main__',
+ 'Meta__class', 'OB__Move', 'OB__Remove', 'OC__Cl', 'OP__Pr', 'Print__Addr',
+ 'Print__PName', 'PrintShortName', 'RA__Pr', 'RA__Sc', 'RL__Pr', 'R_Process',
+ 'RT__ChG', 'RT__ChGt', 'RT__ChLDB', 'RT__ChLDW', 'RT__ChPR', 'RT__ChPrintA',
+ 'RT__ChPrintC', 'RT__ChPrintO', 'RT__ChPrintS', 'RT__ChPS', 'RT__ChR',
+ 'RT__ChSTB', 'RT__ChSTW', 'RT__ChT', 'RT__Err', 'RT__TrPS', 'RV__Pr',
+ 'Symb__Tab', 'Unsigned__Compare', 'WV__Pr', 'Z__Region'),
+ prefix='(?i)', suffix=r'\b'),
+ Name.Builtin, '#pop'),
+ # Other built-in symbols
+ (words((
+ 'call', 'copy', 'create', 'DEBUG', 'destroy', 'DICT_CHAR_SIZE',
+ 'DICT_ENTRY_BYTES', 'DICT_IS_UNICODE', 'DICT_WORD_SIZE', 'false',
+ 'FLOAT_INFINITY', 'FLOAT_NAN', 'FLOAT_NINFINITY', 'GOBJFIELD_CHAIN',
+ 'GOBJFIELD_CHILD', 'GOBJFIELD_NAME', 'GOBJFIELD_PARENT',
+ 'GOBJFIELD_PROPTAB', 'GOBJFIELD_SIBLING', 'GOBJ_EXT_START',
+ 'GOBJ_TOTAL_LENGTH', 'Grammar__Version', 'INDIV_PROP_START', 'INFIX',
+ 'infix__watching', 'MODULE_MODE', 'name', 'nothing', 'NUM_ATTR_BYTES', 'print',
+ 'print_to_array', 'recreate', 'remaining', 'self', 'sender', 'STRICT_MODE',
+ 'sw__var', 'sys__glob0', 'sys__glob1', 'sys__glob2', 'sys_statusline_flag',
+ 'TARGET_GLULX', 'TARGET_ZCODE', 'temp__global2', 'temp__global3',
+ 'temp__global4', 'temp_global', 'true', 'USE_MODULES', 'WORDSIZE'),
+ prefix='(?i)', suffix=r'\b'),
+ Name.Builtin, '#pop'),
+ # Other values
+ (_name, Name, '#pop')
+ ],
+ # Strings
+ 'dictionary-word': [
+ (r'[~^]+', String.Escape),
+ (r'[^~^\\@({%s]+' % _squote, String.Single),
+ (r'[({]', String.Single),
+ (r'@\{[0-9a-fA-F]{,4}\}', String.Escape),
+ (r'@.{2}', String.Escape),
+ (r'[%s]' % _squote, String.Single, '#pop')
+ ],
+ 'string': [
+ (r'[~^]+', String.Escape),
+ (r'[^~^\\@({%s]+' % _dquote, String.Double),
+ (r'[({]', String.Double),
+ (r'\\', String.Escape),
+ (r'@(\\\s*[%s]\s*)*@((\\\s*[%s]\s*)*[0-9])*' %
+ (_newline, _newline), String.Escape),
+ (r'@(\\\s*[%s]\s*)*\{((\\\s*[%s]\s*)*[0-9a-fA-F]){,4}'
+ r'(\\\s*[%s]\s*)*\}' % (_newline, _newline, _newline),
+ String.Escape),
+ (r'@(\\\s*[%s]\s*)*.(\\\s*[%s]\s*)*.' % (_newline, _newline),
+ String.Escape),
+ (r'[%s]' % _dquote, String.Double, '#pop')
+ ],
+ 'plain-string': [
+ (r'[^~^\\({\[\]%s]+' % _dquote, String.Double),
+ (r'[~^({\[\]]', String.Double),
+ (r'\\', String.Escape),
+ (r'[%s]' % _dquote, String.Double, '#pop')
+ ],
+ # Names
+ '_constant': [
+ include('_whitespace'),
+ (_name, Name.Constant, '#pop'),
+ include('value')
+ ],
+ '_global': [
+ include('_whitespace'),
+ (_name, Name.Variable.Global, '#pop'),
+ include('value')
+ ],
+ 'label?': [
+ include('_whitespace'),
+ (_name, Name.Label, '#pop'),
+ default('#pop')
+ ],
+ 'variable?': [
+ include('_whitespace'),
+ (_name, Name.Variable, '#pop'),
+ default('#pop')
+ ],
+ # Values after hashes
+ 'obsolete-dictionary-word': [
+ (r'\S\w*', String.Other, '#pop')
+ ],
+ 'system-constant': [
+ include('_whitespace'),
+ (_name, Name.Builtin, '#pop')
+ ],
+
+ # Directives
+ 'directive': [
+ include('_whitespace'),
+ (r'#', Punctuation),
+ (r';', Punctuation, '#pop'),
+ (r'\[', Punctuation,
+ ('default', 'statements', 'locals', 'routine-name?')),
+ (words((
+ 'abbreviate', 'endif', 'dictionary', 'ifdef', 'iffalse', 'ifndef', 'ifnot',
+ 'iftrue', 'ifv3', 'ifv5', 'release', 'serial', 'switches', 'system_file',
+ 'version'), prefix='(?i)', suffix=r'\b'),
+ Keyword, 'default'),
+ (r'(?i)(array|global)\b', Keyword,
+ ('default', 'directive-keyword?', '_global')),
+ (r'(?i)attribute\b', Keyword, ('default', 'alias?', '_constant')),
+ (r'(?i)class\b', Keyword,
+ ('object-body', 'duplicates', 'class-name')),
+ (r'(?i)(constant|default)\b', Keyword,
+ ('default', 'expression', '_constant')),
+ (r'(?i)(end\b)(.*)', bygroups(Keyword, Text)),
+ (r'(?i)(extend|verb)\b', Keyword, 'grammar'),
+ (r'(?i)fake_action\b', Keyword, ('default', '_constant')),
+ (r'(?i)import\b', Keyword, 'manifest'),
+ (r'(?i)(include|link)\b', Keyword,
+ ('default', 'before-plain-string')),
+ (r'(?i)(lowstring|undef)\b', Keyword, ('default', '_constant')),
+ (r'(?i)message\b', Keyword, ('default', 'diagnostic')),
+ (r'(?i)(nearby|object)\b', Keyword,
+ ('object-body', '_object-head')),
+ (r'(?i)property\b', Keyword,
+ ('default', 'alias?', '_constant', 'property-keyword*')),
+ (r'(?i)replace\b', Keyword,
+ ('default', 'routine-name?', 'routine-name?')),
+ (r'(?i)statusline\b', Keyword, ('default', 'directive-keyword?')),
+ (r'(?i)stub\b', Keyword, ('default', 'routine-name?')),
+ (r'(?i)trace\b', Keyword,
+ ('default', 'trace-keyword?', 'trace-keyword?')),
+ (r'(?i)zcharacter\b', Keyword,
+ ('default', 'directive-keyword?', 'directive-keyword?')),
+ (_name, Name.Class, ('object-body', '_object-head'))
+ ],
+ # [, Replace, Stub
+ 'routine-name?': [
+ include('_whitespace'),
+ (_name, Name.Function, '#pop'),
+ default('#pop')
+ ],
+ 'locals': [
+ include('_whitespace'),
+ (r';', Punctuation, '#pop'),
+ (r'\*', Punctuation),
+ (r'"', String.Double, 'plain-string'),
+ (_name, Name.Variable)
+ ],
+ # Array
+ 'many-values': [
+ include('_whitespace'),
+ (r';', Punctuation),
+ (r'\]', Punctuation, '#pop'),
+ (r':', Error),
+ default(('expression', '_expression'))
+ ],
+ # Attribute, Property
+ 'alias?': [
+ include('_whitespace'),
+ (r'alias\b', Keyword, ('#pop', '_constant')),
+ default('#pop')
+ ],
+ # Class, Object, Nearby
+ 'class-name': [
+ include('_whitespace'),
+ (r'(?=[,;]|(class|has|private|with)\b)', Text, '#pop'),
+ (_name, Name.Class, '#pop')
+ ],
+ 'duplicates': [
+ include('_whitespace'),
+ (r'\(', Punctuation, ('#pop', 'expression', '_expression')),
+ default('#pop')
+ ],
+ '_object-head': [
+ (r'[%s]>' % _dash, Punctuation),
+ (r'(class|has|private|with)\b', Keyword.Declaration, '#pop'),
+ include('_global')
+ ],
+ 'object-body': [
+ include('_whitespace'),
+ (r';', Punctuation, '#pop:2'),
+ (r',', Punctuation),
+ (r'class\b', Keyword.Declaration, 'class-segment'),
+ (r'(has|private|with)\b', Keyword.Declaration),
+ (r':', Error),
+ default(('_object-expression', '_expression'))
+ ],
+ 'class-segment': [
+ include('_whitespace'),
+ (r'(?=[,;]|(class|has|private|with)\b)', Text, '#pop'),
+ (_name, Name.Class),
+ default('value')
+ ],
+ # Extend, Verb
+ 'grammar': [
+ include('_whitespace'),
+ (r'=', Punctuation, ('#pop', 'default')),
+ (r'\*', Punctuation, ('#pop', 'grammar-line')),
+ default('_directive-keyword')
+ ],
+ 'grammar-line': [
+ include('_whitespace'),
+ (r';', Punctuation, '#pop'),
+ (r'[/*]', Punctuation),
+ (r'[%s]>' % _dash, Punctuation, 'value'),
+ (r'(noun|scope)\b', Keyword, '=routine'),
+ default('_directive-keyword')
+ ],
+ '=routine': [
+ include('_whitespace'),
+ (r'=', Punctuation, 'routine-name?'),
+ default('#pop')
+ ],
+ # Import
+ 'manifest': [
+ include('_whitespace'),
+ (r';', Punctuation, '#pop'),
+ (r',', Punctuation),
+ (r'(?i)global\b', Keyword, '_global'),
+ default('_global')
+ ],
+ # Include, Link, Message
+ 'diagnostic': [
+ include('_whitespace'),
+ (r'[%s]' % _dquote, String.Double, ('#pop', 'message-string')),
+ default(('#pop', 'before-plain-string', 'directive-keyword?'))
+ ],
+ 'before-plain-string': [
+ include('_whitespace'),
+ (r'[%s]' % _dquote, String.Double, ('#pop', 'plain-string'))
+ ],
+ 'message-string': [
+ (r'[~^]+', String.Escape),
+ include('plain-string')
+ ],
+
+ # Keywords used in directives
+ '_directive-keyword!': [
+ include('_whitespace'),
+ (words((
+ 'additive', 'alias', 'buffer', 'class', 'creature', 'data', 'error', 'fatalerror',
+ 'first', 'has', 'held', 'initial', 'initstr', 'last', 'long', 'meta', 'multi',
+ 'multiexcept', 'multiheld', 'multiinside', 'noun', 'number', 'only', 'private',
+ 'replace', 'reverse', 'scope', 'score', 'special', 'string', 'table', 'terminating',
+ 'time', 'topic', 'warning', 'with'), suffix=r'\b'),
+ Keyword, '#pop'),
+ (r'[%s]{1,2}>|[+=]' % _dash, Punctuation, '#pop')
+ ],
+ '_directive-keyword': [
+ include('_directive-keyword!'),
+ include('value')
+ ],
+ 'directive-keyword?': [
+ include('_directive-keyword!'),
+ default('#pop')
+ ],
+ 'property-keyword*': [
+ include('_whitespace'),
+ (r'(additive|long)\b', Keyword),
+ default('#pop')
+ ],
+ 'trace-keyword?': [
+ include('_whitespace'),
+ (words((
+ 'assembly', 'dictionary', 'expressions', 'lines', 'linker',
+ 'objects', 'off', 'on', 'symbols', 'tokens', 'verbs'), suffix=r'\b'),
+ Keyword, '#pop'),
+ default('#pop')
+ ],
+
+ # Statements
+ 'statements': [
+ include('_whitespace'),
+ (r'\]', Punctuation, '#pop'),
+ (r'[;{}]', Punctuation),
+ (words((
+ 'box', 'break', 'continue', 'default', 'give', 'inversion',
+ 'new_line', 'quit', 'read', 'remove', 'return', 'rfalse', 'rtrue',
+ 'spaces', 'string', 'until'), suffix=r'\b'),
+ Keyword, 'default'),
+ (r'(do|else)\b', Keyword),
+ (r'(font|style)\b', Keyword,
+ ('default', 'miscellaneous-keyword?')),
+ (r'for\b', Keyword, ('for', '(?')),
+ (r'(if|switch|while)', Keyword,
+ ('expression', '_expression', '(?')),
+ (r'(jump|save|restore)\b', Keyword, ('default', 'label?')),
+ (r'objectloop\b', Keyword,
+ ('_keyword-expression', 'variable?', '(?')),
+ (r'print(_ret)?\b|(?=[%s])' % _dquote, Keyword, 'print-list'),
+ (r'\.', Name.Label, 'label?'),
+ (r'@', Keyword, 'opcode'),
+ (r'#(?![agrnw]\$|#)', Punctuation, 'directive'),
+ (r'<', Punctuation, 'default'),
+ (r'move\b', Keyword,
+ ('default', '_keyword-expression', '_expression')),
+ default(('default', '_keyword-expression', '_expression'))
+ ],
+ 'miscellaneous-keyword?': [
+ include('_whitespace'),
+ (r'(bold|fixed|from|near|off|on|reverse|roman|to|underline)\b',
+ Keyword, '#pop'),
+ (r'(a|A|an|address|char|name|number|object|property|string|the|'
+ r'The)\b(?=(\s+|(![^%s]*))*\))' % _newline, Keyword.Pseudo,
+ '#pop'),
+ (r'%s(?=(\s+|(![^%s]*))*\))' % (_name, _newline), Name.Function,
+ '#pop'),
+ default('#pop')
+ ],
+ '(?': [
+ include('_whitespace'),
+ (r'\(', Punctuation, '#pop'),
+ default('#pop')
+ ],
+ 'for': [
+ include('_whitespace'),
+ (r';', Punctuation, ('_for-expression', '_expression')),
+ default(('_for-expression', '_expression'))
+ ],
+ 'print-list': [
+ include('_whitespace'),
+ (r';', Punctuation, '#pop'),
+ (r':', Error),
+ default(('_list-expression', '_expression', '_list-expression', 'form'))
+ ],
+ 'form': [
+ include('_whitespace'),
+ (r'\(', Punctuation, ('#pop', 'miscellaneous-keyword?')),
+ default('#pop')
+ ],
+
+ # Assembly
+ 'opcode': [
+ include('_whitespace'),
+ (r'[%s]' % _dquote, String.Double, ('operands', 'plain-string')),
+ (_name, Keyword, 'operands')
+ ],
+ 'operands': [
+ (r':', Error),
+ default(('_assembly-expression', '_expression'))
+ ]
+ }
+
+ def get_tokens_unprocessed(self, text):
+ # 'in' is either a keyword or an operator.
+ # If the token two tokens after 'in' is ')', 'in' is a keyword:
+ # objectloop(a in b)
+ # Otherwise, it is an operator:
+ # objectloop(a in b && true)
+ objectloop_queue = []
+ objectloop_token_count = -1
+ previous_token = None
+ for index, token, value in RegexLexer.get_tokens_unprocessed(self,
+ text):
+ if previous_token is Name.Variable and value == 'in':
+ objectloop_queue = [[index, token, value]]
+ objectloop_token_count = 2
+ elif objectloop_token_count > 0:
+ if token not in Comment and token not in Text:
+ objectloop_token_count -= 1
+ objectloop_queue.append((index, token, value))
+ else:
+ if objectloop_token_count == 0:
+ if objectloop_queue[-1][2] == ')':
+ objectloop_queue[0][1] = Keyword
+ while objectloop_queue:
+ yield objectloop_queue.pop(0)
+ objectloop_token_count = -1
+ yield index, token, value
+ if token not in Comment and token not in Text:
+ previous_token = token
+ while objectloop_queue:
+ yield objectloop_queue.pop(0)
+
+
+class Inform7Lexer(RegexLexer):
+ """
+ For `Inform 7 <http://inform7.com/>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Inform 7'
+ aliases = ['inform7', 'i7']
+ filenames = ['*.ni', '*.i7x']
+
+ flags = re.MULTILINE | re.DOTALL | re.UNICODE
+
+ _dash = Inform6Lexer._dash
+ _dquote = Inform6Lexer._dquote
+ _newline = Inform6Lexer._newline
+ _start = r'\A|(?<=[%s])' % _newline
+
+ # There are three variants of Inform 7, differing in how to
+ # interpret at signs and braces in I6T. In top-level inclusions, at
+ # signs in the first column are inweb syntax. In phrase definitions
+ # and use options, tokens in braces are treated as I7. Use options
+ # also interpret "{N}".
+ tokens = {}
+ token_variants = ['+i6t-not-inline', '+i6t-inline', '+i6t-use-option']
+
+ for level in token_variants:
+ tokens[level] = {
+ '+i6-root': list(Inform6Lexer.tokens['root']),
+ '+i6t-root': [ # For Inform6TemplateLexer
+ (r'[^%s]*' % Inform6Lexer._newline, Comment.Preproc,
+ ('directive', '+p'))
+ ],
+ 'root': [
+ (r'(\|?\s)+', Text),
+ (r'\[', Comment.Multiline, '+comment'),
+ (r'[%s]' % _dquote, Generic.Heading,
+ ('+main', '+titling', '+titling-string')),
+ default(('+main', '+heading?'))
+ ],
+ '+titling-string': [
+ (r'[^%s]+' % _dquote, Generic.Heading),
+ (r'[%s]' % _dquote, Generic.Heading, '#pop')
+ ],
+ '+titling': [
+ (r'\[', Comment.Multiline, '+comment'),
+ (r'[^%s.;:|%s]+' % (_dquote, _newline), Generic.Heading),
+ (r'[%s]' % _dquote, Generic.Heading, '+titling-string'),
+ (r'[%s]{2}|(?<=[\s%s])\|[\s%s]' % (_newline, _dquote, _dquote),
+ Text, ('#pop', '+heading?')),
+ (r'[.;:]|(?<=[\s%s])\|' % _dquote, Text, '#pop'),
+ (r'[|%s]' % _newline, Generic.Heading)
+ ],
+ '+main': [
+ (r'(?i)[^%s:a\[(|%s]+' % (_dquote, _newline), Text),
+ (r'[%s]' % _dquote, String.Double, '+text'),
+ (r':', Text, '+phrase-definition'),
+ (r'(?i)\bas\b', Text, '+use-option'),
+ (r'\[', Comment.Multiline, '+comment'),
+ (r'(\([%s])(.*?)([%s]\))' % (_dash, _dash),
+ bygroups(Punctuation,
+ using(this, state=('+i6-root', 'directive'),
+ i6t='+i6t-not-inline'), Punctuation)),
+ (r'(%s|(?<=[\s;:.%s]))\|\s|[%s]{2,}' %
+ (_start, _dquote, _newline), Text, '+heading?'),
+ (r'(?i)[a(|%s]' % _newline, Text)
+ ],
+ '+phrase-definition': [
+ (r'\s+', Text),
+ (r'\[', Comment.Multiline, '+comment'),
+ (r'(\([%s])(.*?)([%s]\))' % (_dash, _dash),
+ bygroups(Punctuation,
+ using(this, state=('+i6-root', 'directive',
+ 'default', 'statements'),
+ i6t='+i6t-inline'), Punctuation), '#pop'),
+ default('#pop')
+ ],
+ '+use-option': [
+ (r'\s+', Text),
+ (r'\[', Comment.Multiline, '+comment'),
+ (r'(\([%s])(.*?)([%s]\))' % (_dash, _dash),
+ bygroups(Punctuation,
+ using(this, state=('+i6-root', 'directive'),
+ i6t='+i6t-use-option'), Punctuation), '#pop'),
+ default('#pop')
+ ],
+ '+comment': [
+ (r'[^\[\]]+', Comment.Multiline),
+ (r'\[', Comment.Multiline, '#push'),
+ (r'\]', Comment.Multiline, '#pop')
+ ],
+ '+text': [
+ (r'[^\[%s]+' % _dquote, String.Double),
+ (r'\[.*?\]', String.Interpol),
+ (r'[%s]' % _dquote, String.Double, '#pop')
+ ],
+ '+heading?': [
+ (r'(\|?\s)+', Text),
+ (r'\[', Comment.Multiline, '+comment'),
+ (r'[%s]{4}\s+' % _dash, Text, '+documentation-heading'),
+ (r'[%s]{1,3}' % _dash, Text),
+ (r'(?i)(volume|book|part|chapter|section)\b[^%s]*' % _newline,
+ Generic.Heading, '#pop'),
+ default('#pop')
+ ],
+ '+documentation-heading': [
+ (r'\s+', Text),
+ (r'\[', Comment.Multiline, '+comment'),
+ (r'(?i)documentation\s+', Text, '+documentation-heading2'),
+ default('#pop')
+ ],
+ '+documentation-heading2': [
+ (r'\s+', Text),
+ (r'\[', Comment.Multiline, '+comment'),
+ (r'[%s]{4}\s' % _dash, Text, '+documentation'),
+ default('#pop:2')
+ ],
+ '+documentation': [
+ (r'(?i)(%s)\s*(chapter|example)\s*:[^%s]*' %
+ (_start, _newline), Generic.Heading),
+ (r'(?i)(%s)\s*section\s*:[^%s]*' % (_start, _newline),
+ Generic.Subheading),
+ (r'((%s)\t.*?[%s])+' % (_start, _newline),
+ using(this, state='+main')),
+ (r'[^%s\[]+|[%s\[]' % (_newline, _newline), Text),
+ (r'\[', Comment.Multiline, '+comment'),
+ ],
+ '+i6t-not-inline': [
+ (r'(%s)@c( .*?)?([%s]|\Z)' % (_start, _newline),
+ Comment.Preproc),
+ (r'(%s)@([%s]+|Purpose:)[^%s]*' % (_start, _dash, _newline),
+ Comment.Preproc),
+ (r'(%s)@p( .*?)?([%s]|\Z)' % (_start, _newline),
+ Generic.Heading, '+p')
+ ],
+ '+i6t-use-option': [
+ include('+i6t-not-inline'),
+ (r'(\{)(N)(\})', bygroups(Punctuation, Text, Punctuation))
+ ],
+ '+i6t-inline': [
+ (r'(\{)(\S[^}]*)?(\})',
+ bygroups(Punctuation, using(this, state='+main'),
+ Punctuation))
+ ],
+ '+i6t': [
+ (r'(\{[%s])(![^}]*)(\}?)' % _dash,
+ bygroups(Punctuation, Comment.Single, Punctuation)),
+ (r'(\{[%s])(lines)(:)([^}]*)(\}?)' % _dash,
+ bygroups(Punctuation, Keyword, Punctuation, Text,
+ Punctuation), '+lines'),
+ (r'(\{[%s])([^:}]*)(:?)([^}]*)(\}?)' % _dash,
+ bygroups(Punctuation, Keyword, Punctuation, Text,
+ Punctuation)),
+ (r'(\(\+)(.*?)(\+\)|\Z)',
+ bygroups(Punctuation, using(this, state='+main'),
+ Punctuation))
+ ],
+ '+p': [
+ (r'[^@]+', Comment.Preproc),
+ (r'(%s)@c( .*?)?([%s]|\Z)' % (_start, _newline),
+ Comment.Preproc, '#pop'),
+ (r'(%s)@([%s]|Purpose:)' % (_start, _dash), Comment.Preproc),
+ (r'(%s)@p( .*?)?([%s]|\Z)' % (_start, _newline),
+ Generic.Heading),
+ (r'@', Comment.Preproc)
+ ],
+ '+lines': [
+ (r'(%s)@c( .*?)?([%s]|\Z)' % (_start, _newline),
+ Comment.Preproc),
+ (r'(%s)@([%s]|Purpose:)[^%s]*' % (_start, _dash, _newline),
+ Comment.Preproc),
+ (r'(%s)@p( .*?)?([%s]|\Z)' % (_start, _newline),
+ Generic.Heading, '+p'),
+ (r'(%s)@\w*[ %s]' % (_start, _newline), Keyword),
+ (r'![^%s]*' % _newline, Comment.Single),
+ (r'(\{)([%s]endlines)(\})' % _dash,
+ bygroups(Punctuation, Keyword, Punctuation), '#pop'),
+ (r'[^@!{]+?([%s]|\Z)|.' % _newline, Text)
+ ]
+ }
+ # Inform 7 can include snippets of Inform 6 template language,
+ # so all of Inform6Lexer's states are copied here, with
+ # modifications to account for template syntax. Inform7Lexer's
+ # own states begin with '+' to avoid name conflicts. Some of
+ # Inform6Lexer's states begin with '_': these are not modified.
+ # They deal with template syntax either by including modified
+ # states, or by matching r'' then pushing to modified states.
+ for token in Inform6Lexer.tokens:
+ if token == 'root':
+ continue
+ tokens[level][token] = list(Inform6Lexer.tokens[token])
+ if not token.startswith('_'):
+ tokens[level][token][:0] = [include('+i6t'), include(level)]
+
+ def __init__(self, **options):
+ level = options.get('i6t', '+i6t-not-inline')
+ if level not in self._all_tokens:
+ self._tokens = self.__class__.process_tokendef(level)
+ else:
+ self._tokens = self._all_tokens[level]
+ RegexLexer.__init__(self, **options)
+
+
+class Inform6TemplateLexer(Inform7Lexer):
+ """
+ For `Inform 6 template
+ <http://inform7.com/sources/src/i6template/Woven/index.html>`_ code.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Inform 6 template'
+ aliases = ['i6t']
+ filenames = ['*.i6t']
+
+ def get_tokens_unprocessed(self, text, stack=('+i6t-root',)):
+ return Inform7Lexer.get_tokens_unprocessed(self, text, stack)
+
+
+class Tads3Lexer(RegexLexer):
+ """
+ For `TADS 3 <http://www.tads.org/>`_ source code.
+ """
+
+ name = 'TADS 3'
+ aliases = ['tads3']
+ filenames = ['*.t']
+
+ flags = re.DOTALL | re.MULTILINE
+
+ _comment_single = r'(?://(?:[^\\\n]|\\+[\w\W])*$)'
+ _comment_multiline = r'(?:/\*(?:[^*]|\*(?!/))*\*/)'
+ _escape = (r'(?:\\(?:[\n\\<>"\'^v bnrt]|u[\da-fA-F]{,4}|x[\da-fA-F]{,2}|'
+ r'[0-3]?[0-7]{1,2}))')
+ _name = r'(?:[_a-zA-Z]\w*)'
+ _no_quote = r'(?=\s|\\?>)'
+ _operator = (r'(?:&&|\|\||\+\+|--|\?\?|::|[.,@\[\]~]|'
+ r'(?:[=+\-*/%!&|^]|<<?|>>?>?)=?)')
+ _ws = r'(?:\\|\s|%s|%s)' % (_comment_single, _comment_multiline)
+ _ws_pp = r'(?:\\\n|[^\S\n]|%s|%s)' % (_comment_single, _comment_multiline)
+
+ def _make_string_state(triple, double, verbatim=None, _escape=_escape):
+ if verbatim:
+ verbatim = ''.join(['(?:%s|%s)' % (re.escape(c.lower()),
+ re.escape(c.upper()))
+ for c in verbatim])
+ char = r'"' if double else r"'"
+ token = String.Double if double else String.Single
+ escaped_quotes = r'+|%s(?!%s{2})' % (char, char) if triple else r''
+ prefix = '%s%s' % ('t' if triple else '', 'd' if double else 's')
+ tag_state_name = '%sqt' % prefix
+ state = []
+ if triple:
+ state += [
+ (r'%s{3,}' % char, token, '#pop'),
+ (r'\\%s+' % char, String.Escape),
+ (char, token)
+ ]
+ else:
+ state.append((char, token, '#pop'))
+ state += [
+ include('s/verbatim'),
+ (r'[^\\<&{}%s]+' % char, token)
+ ]
+ if verbatim:
+ # This regex can't use `(?i)` because escape sequences are
+ # case-sensitive. `<\XMP>` works; `<\xmp>` doesn't.
+ state.append((r'\\?<(/|\\\\|(?!%s)\\)%s(?=[\s=>])' %
+ (_escape, verbatim),
+ Name.Tag, ('#pop', '%sqs' % prefix, tag_state_name)))
+ else:
+ state += [
+ (r'\\?<!([^><\\%s]|<(?!<)|\\%s%s|%s|\\.)*>?' %
+ (char, char, escaped_quotes, _escape), Comment.Multiline),
+ (r'(?i)\\?<listing(?=[\s=>]|\\>)', Name.Tag,
+ ('#pop', '%sqs/listing' % prefix, tag_state_name)),
+ (r'(?i)\\?<xmp(?=[\s=>]|\\>)', Name.Tag,
+ ('#pop', '%sqs/xmp' % prefix, tag_state_name)),
+ (r'\\?<([^\s=><\\%s]|<(?!<)|\\%s%s|%s|\\.)*' %
+ (char, char, escaped_quotes, _escape), Name.Tag,
+ tag_state_name),
+ include('s/entity')
+ ]
+ state += [
+ include('s/escape'),
+ (r'\{([^}<\\%s]|<(?!<)|\\%s%s|%s|\\.)*\}' %
+ (char, char, escaped_quotes, _escape), String.Interpol),
+ (r'[\\&{}<]', token)
+ ]
+ return state
+
+ def _make_tag_state(triple, double, _escape=_escape):
+ char = r'"' if double else r"'"
+ quantifier = r'{3,}' if triple else r''
+ state_name = '%s%sqt' % ('t' if triple else '', 'd' if double else 's')
+ token = String.Double if double else String.Single
+ escaped_quotes = r'+|%s(?!%s{2})' % (char, char) if triple else r''
+ return [
+ (r'%s%s' % (char, quantifier), token, '#pop:2'),
+ (r'(\s|\\\n)+', Text),
+ (r'(=)(\\?")', bygroups(Punctuation, String.Double),
+ 'dqs/%s' % state_name),
+ (r"(=)(\\?')", bygroups(Punctuation, String.Single),
+ 'sqs/%s' % state_name),
+ (r'=', Punctuation, 'uqs/%s' % state_name),
+ (r'\\?>', Name.Tag, '#pop'),
+ (r'\{([^}<\\%s]|<(?!<)|\\%s%s|%s|\\.)*\}' %
+ (char, char, escaped_quotes, _escape), String.Interpol),
+ (r'([^\s=><\\%s]|<(?!<)|\\%s%s|%s|\\.)+' %
+ (char, char, escaped_quotes, _escape), Name.Attribute),
+ include('s/escape'),
+ include('s/verbatim'),
+ include('s/entity'),
+ (r'[\\{}&]', Name.Attribute)
+ ]
+
+ def _make_attribute_value_state(terminator, host_triple, host_double,
+ _escape=_escape):
+ token = (String.Double if terminator == r'"' else
+ String.Single if terminator == r"'" else String.Other)
+ host_char = r'"' if host_double else r"'"
+ host_quantifier = r'{3,}' if host_triple else r''
+ host_token = String.Double if host_double else String.Single
+ escaped_quotes = (r'+|%s(?!%s{2})' % (host_char, host_char)
+ if host_triple else r'')
+ return [
+ (r'%s%s' % (host_char, host_quantifier), host_token, '#pop:3'),
+ (r'%s%s' % (r'' if token is String.Other else r'\\?', terminator),
+ token, '#pop'),
+ include('s/verbatim'),
+ include('s/entity'),
+ (r'\{([^}<\\%s]|<(?!<)|\\%s%s|%s|\\.)*\}' %
+ (host_char, host_char, escaped_quotes, _escape), String.Interpol),
+ (r'([^\s"\'<%s{}\\&])+' % (r'>' if token is String.Other else r''),
+ token),
+ include('s/escape'),
+ (r'["\'\s&{<}\\]', token)
+ ]
+
+ tokens = {
+ 'root': [
+ (u'\ufeff', Text),
+ (r'\{', Punctuation, 'object-body'),
+ (r';+', Punctuation),
+ (r'(?=(argcount|break|case|catch|continue|default|definingobj|'
+ r'delegated|do|else|for|foreach|finally|goto|if|inherited|'
+ r'invokee|local|nil|new|operator|replaced|return|self|switch|'
+ r'targetobj|targetprop|throw|true|try|while)\b)', Text, 'block'),
+ (r'(%s)(%s*)(\()' % (_name, _ws),
+ bygroups(Name.Function, using(this, state='whitespace'),
+ Punctuation),
+ ('block?/root', 'more/parameters', 'main/parameters')),
+ include('whitespace'),
+ (r'\++', Punctuation),
+ (r'[^\s!"%-(*->@-_a-z{-~]+', Error), # Averts an infinite loop
+ (r'(?!\Z)', Text, 'main/root')
+ ],
+ 'main/root': [
+ include('main/basic'),
+ default(('#pop', 'object-body/no-braces', 'classes', 'class'))
+ ],
+ 'object-body/no-braces': [
+ (r';', Punctuation, '#pop'),
+ (r'\{', Punctuation, ('#pop', 'object-body')),
+ include('object-body')
+ ],
+ 'object-body': [
+ (r';', Punctuation),
+ (r'\{', Punctuation, '#push'),
+ (r'\}', Punctuation, '#pop'),
+ (r':', Punctuation, ('classes', 'class')),
+ (r'(%s?)(%s*)(\()' % (_name, _ws),
+ bygroups(Name.Function, using(this, state='whitespace'),
+ Punctuation),
+ ('block?', 'more/parameters', 'main/parameters')),
+ (r'(%s)(%s*)(\{)' % (_name, _ws),
+ bygroups(Name.Function, using(this, state='whitespace'),
+ Punctuation), 'block'),
+ (r'(%s)(%s*)(:)' % (_name, _ws),
+ bygroups(Name.Variable, using(this, state='whitespace'),
+ Punctuation),
+ ('object-body/no-braces', 'classes', 'class')),
+ include('whitespace'),
+ (r'->|%s' % _operator, Punctuation, 'main'),
+ default('main/object-body')
+ ],
+ 'main/object-body': [
+ include('main/basic'),
+ (r'(%s)(%s*)(=?)' % (_name, _ws),
+ bygroups(Name.Variable, using(this, state='whitespace'),
+ Punctuation), ('#pop', 'more', 'main')),
+ default('#pop:2')
+ ],
+ 'block?/root': [
+ (r'\{', Punctuation, ('#pop', 'block')),
+ include('whitespace'),
+ (r'(?=[[\'"<(:])', Text, # It might be a VerbRule macro.
+ ('#pop', 'object-body/no-braces', 'grammar', 'grammar-rules')),
+ # It might be a macro like DefineAction.
+ default(('#pop', 'object-body/no-braces'))
+ ],
+ 'block?': [
+ (r'\{', Punctuation, ('#pop', 'block')),
+ include('whitespace'),
+ default('#pop')
+ ],
+ 'block/basic': [
+ (r'[;:]+', Punctuation),
+ (r'\{', Punctuation, '#push'),
+ (r'\}', Punctuation, '#pop'),
+ (r'default\b', Keyword.Reserved),
+ (r'(%s)(%s*)(:)' % (_name, _ws),
+ bygroups(Name.Label, using(this, state='whitespace'),
+ Punctuation)),
+ include('whitespace')
+ ],
+ 'block': [
+ include('block/basic'),
+ (r'(?!\Z)', Text, ('more', 'main'))
+ ],
+ 'block/embed': [
+ (r'>>', String.Interpol, '#pop'),
+ include('block/basic'),
+ (r'(?!\Z)', Text, ('more/embed', 'main'))
+ ],
+ 'main/basic': [
+ include('whitespace'),
+ (r'\(', Punctuation, ('#pop', 'more', 'main')),
+ (r'\[', Punctuation, ('#pop', 'more/list', 'main')),
+ (r'\{', Punctuation, ('#pop', 'more/inner', 'main/inner',
+ 'more/parameters', 'main/parameters')),
+ (r'\*|\.{3}', Punctuation, '#pop'),
+ (r'(?i)0x[\da-f]+', Number.Hex, '#pop'),
+ (r'(\d+\.(?!\.)\d*|\.\d+)([eE][-+]?\d+)?|\d+[eE][-+]?\d+',
+ Number.Float, '#pop'),
+ (r'0[0-7]+', Number.Oct, '#pop'),
+ (r'\d+', Number.Integer, '#pop'),
+ (r'"""', String.Double, ('#pop', 'tdqs')),
+ (r"'''", String.Single, ('#pop', 'tsqs')),
+ (r'"', String.Double, ('#pop', 'dqs')),
+ (r"'", String.Single, ('#pop', 'sqs')),
+ (r'R"""', String.Regex, ('#pop', 'tdqr')),
+ (r"R'''", String.Regex, ('#pop', 'tsqr')),
+ (r'R"', String.Regex, ('#pop', 'dqr')),
+ (r"R'", String.Regex, ('#pop', 'sqr')),
+ # Two-token keywords
+ (r'(extern)(%s+)(object\b)' % _ws,
+ bygroups(Keyword.Reserved, using(this, state='whitespace'),
+ Keyword.Reserved)),
+ (r'(function|method)(%s*)(\()' % _ws,
+ bygroups(Keyword.Reserved, using(this, state='whitespace'),
+ Punctuation),
+ ('#pop', 'block?', 'more/parameters', 'main/parameters')),
+ (r'(modify)(%s+)(grammar\b)' % _ws,
+ bygroups(Keyword.Reserved, using(this, state='whitespace'),
+ Keyword.Reserved),
+ ('#pop', 'object-body/no-braces', ':', 'grammar')),
+ (r'(new)(%s+(?=(?:function|method)\b))' % _ws,
+ bygroups(Keyword.Reserved, using(this, state='whitespace'))),
+ (r'(object)(%s+)(template\b)' % _ws,
+ bygroups(Keyword.Reserved, using(this, state='whitespace'),
+ Keyword.Reserved), ('#pop', 'template')),
+ (r'(string)(%s+)(template\b)' % _ws,
+ bygroups(Keyword, using(this, state='whitespace'),
+ Keyword.Reserved), ('#pop', 'function-name')),
+ # Keywords
+ (r'(argcount|definingobj|invokee|replaced|targetobj|targetprop)\b',
+ Name.Builtin, '#pop'),
+ (r'(break|continue|goto)\b', Keyword.Reserved, ('#pop', 'label')),
+ (r'(case|extern|if|intrinsic|return|static|while)\b',
+ Keyword.Reserved),
+ (r'catch\b', Keyword.Reserved, ('#pop', 'catch')),
+ (r'class\b', Keyword.Reserved,
+ ('#pop', 'object-body/no-braces', 'class')),
+ (r'(default|do|else|finally|try)\b', Keyword.Reserved, '#pop'),
+ (r'(dictionary|property)\b', Keyword.Reserved,
+ ('#pop', 'constants')),
+ (r'enum\b', Keyword.Reserved, ('#pop', 'enum')),
+ (r'export\b', Keyword.Reserved, ('#pop', 'main')),
+ (r'(for|foreach)\b', Keyword.Reserved,
+ ('#pop', 'more/inner', 'main/inner')),
+ (r'(function|method)\b', Keyword.Reserved,
+ ('#pop', 'block?', 'function-name')),
+ (r'grammar\b', Keyword.Reserved,
+ ('#pop', 'object-body/no-braces', 'grammar')),
+ (r'inherited\b', Keyword.Reserved, ('#pop', 'inherited')),
+ (r'local\b', Keyword.Reserved,
+ ('#pop', 'more/local', 'main/local')),
+ (r'(modify|replace|switch|throw|transient)\b', Keyword.Reserved,
+ '#pop'),
+ (r'new\b', Keyword.Reserved, ('#pop', 'class')),
+ (r'(nil|true)\b', Keyword.Constant, '#pop'),
+ (r'object\b', Keyword.Reserved, ('#pop', 'object-body/no-braces')),
+ (r'operator\b', Keyword.Reserved, ('#pop', 'operator')),
+ (r'propertyset\b', Keyword.Reserved,
+ ('#pop', 'propertyset', 'main')),
+ (r'self\b', Name.Builtin.Pseudo, '#pop'),
+ (r'template\b', Keyword.Reserved, ('#pop', 'template')),
+ # Operators
+ (r'(__objref|defined)(%s*)(\()' % _ws,
+ bygroups(Operator.Word, using(this, state='whitespace'),
+ Operator), ('#pop', 'more/__objref', 'main')),
+ (r'delegated\b', Operator.Word),
+ # Compiler-defined macros and built-in properties
+ (r'(__DATE__|__DEBUG|__LINE__|__FILE__|'
+ r'__TADS_MACRO_FORMAT_VERSION|__TADS_SYS_\w*|__TADS_SYSTEM_NAME|'
+ r'__TADS_VERSION_MAJOR|__TADS_VERSION_MINOR|__TADS3|__TIME__|'
+ r'construct|finalize|grammarInfo|grammarTag|lexicalParent|'
+ r'miscVocab|sourceTextGroup|sourceTextGroupName|'
+ r'sourceTextGroupOrder|sourceTextOrder)\b', Name.Builtin, '#pop')
+ ],
+ 'main': [
+ include('main/basic'),
+ (_name, Name, '#pop'),
+ default('#pop')
+ ],
+ 'more/basic': [
+ (r'\(', Punctuation, ('more/list', 'main')),
+ (r'\[', Punctuation, ('more', 'main')),
+ (r'\.{3}', Punctuation),
+ (r'->|\.\.', Punctuation, 'main'),
+ (r'(?=;)|[:)\]]', Punctuation, '#pop'),
+ include('whitespace'),
+ (_operator, Operator, 'main'),
+ (r'\?', Operator, ('main', 'more/conditional', 'main')),
+ (r'(is|not)(%s+)(in\b)' % _ws,
+ bygroups(Operator.Word, using(this, state='whitespace'),
+ Operator.Word)),
+ (r'[^\s!"%-_a-z{-~]+', Error) # Averts an infinite loop
+ ],
+ 'more': [
+ include('more/basic'),
+ default('#pop')
+ ],
+ # Then expression (conditional operator)
+ 'more/conditional': [
+ (r':(?!:)', Operator, '#pop'),
+ include('more')
+ ],
+ # Embedded expressions
+ 'more/embed': [
+ (r'>>', String.Interpol, '#pop:2'),
+ include('more')
+ ],
+ # For/foreach loop initializer or short-form anonymous function
+ 'main/inner': [
+ (r'\(', Punctuation, ('#pop', 'more/inner', 'main/inner')),
+ (r'local\b', Keyword.Reserved, ('#pop', 'main/local')),
+ include('main')
+ ],
+ 'more/inner': [
+ (r'\}', Punctuation, '#pop'),
+ (r',', Punctuation, 'main/inner'),
+ (r'(in|step)\b', Keyword, 'main/inner'),
+ include('more')
+ ],
+ # Local
+ 'main/local': [
+ (_name, Name.Variable, '#pop'),
+ include('whitespace')
+ ],
+ 'more/local': [
+ (r',', Punctuation, 'main/local'),
+ include('more')
+ ],
+ # List
+ 'more/list': [
+ (r'[,:]', Punctuation, 'main'),
+ include('more')
+ ],
+ # Parameter list
+ 'main/parameters': [
+ (r'(%s)(%s*)(?=:)' % (_name, _ws),
+ bygroups(Name.Variable, using(this, state='whitespace')), '#pop'),
+ (r'(%s)(%s+)(%s)' % (_name, _ws, _name),
+ bygroups(Name.Class, using(this, state='whitespace'),
+ Name.Variable), '#pop'),
+ (r'\[+', Punctuation),
+ include('main/basic'),
+ (_name, Name.Variable, '#pop'),
+ default('#pop')
+ ],
+ 'more/parameters': [
+ (r'(:)(%s*(?=[?=,:)]))' % _ws,
+ bygroups(Punctuation, using(this, state='whitespace'))),
+ (r'[?\]]+', Punctuation),
+ (r'[:)]', Punctuation, ('#pop', 'multimethod?')),
+ (r',', Punctuation, 'main/parameters'),
+ (r'=', Punctuation, ('more/parameter', 'main')),
+ include('more')
+ ],
+ 'more/parameter': [
+ (r'(?=[,)])', Text, '#pop'),
+ include('more')
+ ],
+ 'multimethod?': [
+ (r'multimethod\b', Keyword, '#pop'),
+ include('whitespace'),
+ default('#pop')
+ ],
+
+ # Statements and expressions
+ 'more/__objref': [
+ (r',', Punctuation, 'mode'),
+ (r'\)', Operator, '#pop'),
+ include('more')
+ ],
+ 'mode': [
+ (r'(error|warn)\b', Keyword, '#pop'),
+ include('whitespace')
+ ],
+ 'catch': [
+ (r'\(+', Punctuation),
+ (_name, Name.Exception, ('#pop', 'variables')),
+ include('whitespace')
+ ],
+ 'enum': [
+ include('whitespace'),
+ (r'token\b', Keyword, ('#pop', 'constants')),
+ default(('#pop', 'constants'))
+ ],
+ 'grammar': [
+ (r'\)+', Punctuation),
+ (r'\(', Punctuation, 'grammar-tag'),
+ (r':', Punctuation, 'grammar-rules'),
+ (_name, Name.Class),
+ include('whitespace')
+ ],
+ 'grammar-tag': [
+ include('whitespace'),
+ (r'"""([^\\"<]|""?(?!")|\\"+|\\.|<(?!<))+("{3,}|<<)|'
+ r'R"""([^\\"]|""?(?!")|\\"+|\\.)+"{3,}|'
+ r"'''([^\\'<]|''?(?!')|\\'+|\\.|<(?!<))+('{3,}|<<)|"
+ r"R'''([^\\']|''?(?!')|\\'+|\\.)+'{3,}|"
+ r'"([^\\"<]|\\.|<(?!<))+("|<<)|R"([^\\"]|\\.)+"|'
+ r"'([^\\'<]|\\.|<(?!<))+('|<<)|R'([^\\']|\\.)+'|"
+ r"([^)\s\\/]|/(?![/*]))+|\)", String.Other, '#pop')
+ ],
+ 'grammar-rules': [
+ include('string'),
+ include('whitespace'),
+ (r'(\[)(%s*)(badness)' % _ws,
+ bygroups(Punctuation, using(this, state='whitespace'), Keyword),
+ 'main'),
+ (r'->|%s|[()]' % _operator, Punctuation),
+ (_name, Name.Constant),
+ default('#pop:2')
+ ],
+ ':': [
+ (r':', Punctuation, '#pop')
+ ],
+ 'function-name': [
+ (r'(<<([^>]|>>>|>(?!>))*>>)+', String.Interpol),
+ (r'(?=%s?%s*[({])' % (_name, _ws), Text, '#pop'),
+ (_name, Name.Function, '#pop'),
+ include('whitespace')
+ ],
+ 'inherited': [
+ (r'<', Punctuation, ('#pop', 'classes', 'class')),
+ include('whitespace'),
+ (_name, Name.Class, '#pop'),
+ default('#pop')
+ ],
+ 'operator': [
+ (r'negate\b', Operator.Word, '#pop'),
+ include('whitespace'),
+ (_operator, Operator),
+ default('#pop')
+ ],
+ 'propertyset': [
+ (r'\(', Punctuation, ('more/parameters', 'main/parameters')),
+ (r'\{', Punctuation, ('#pop', 'object-body')),
+ include('whitespace')
+ ],
+ 'template': [
+ (r'(?=;)', Text, '#pop'),
+ include('string'),
+ (r'inherited\b', Keyword.Reserved),
+ include('whitespace'),
+ (r'->|\?|%s' % _operator, Punctuation),
+ (_name, Name.Variable)
+ ],
+
+ # Identifiers
+ 'class': [
+ (r'\*|\.{3}', Punctuation, '#pop'),
+ (r'object\b', Keyword.Reserved, '#pop'),
+ (r'transient\b', Keyword.Reserved),
+ (_name, Name.Class, '#pop'),
+ include('whitespace'),
+ default('#pop')
+ ],
+ 'classes': [
+ (r'[:,]', Punctuation, 'class'),
+ include('whitespace'),
+ (r'>', Punctuation, '#pop'),
+ default('#pop')
+ ],
+ 'constants': [
+ (r',+', Punctuation),
+ (r';', Punctuation, '#pop'),
+ (r'property\b', Keyword.Reserved),
+ (_name, Name.Constant),
+ include('whitespace')
+ ],
+ 'label': [
+ (_name, Name.Label, '#pop'),
+ include('whitespace'),
+ default('#pop')
+ ],
+ 'variables': [
+ (r',+', Punctuation),
+ (r'\)', Punctuation, '#pop'),
+ include('whitespace'),
+ (_name, Name.Variable)
+ ],
+
+ # Whitespace and comments
+ 'whitespace': [
+ (r'^%s*#(%s|[^\n]|(?<=\\)\n)*\n?' % (_ws_pp, _comment_multiline),
+ Comment.Preproc),
+ (_comment_single, Comment.Single),
+ (_comment_multiline, Comment.Multiline),
+ (r'\\+\n+%s*#?|\n+|([^\S\n]|\\)+' % _ws_pp, Text)
+ ],
+
+ # Strings
+ 'string': [
+ (r'"""', String.Double, 'tdqs'),
+ (r"'''", String.Single, 'tsqs'),
+ (r'"', String.Double, 'dqs'),
+ (r"'", String.Single, 'sqs')
+ ],
+ 's/escape': [
+ (r'\{\{|\}\}|%s' % _escape, String.Escape)
+ ],
+ 's/verbatim': [
+ (r'<<\s*(as\s+decreasingly\s+likely\s+outcomes|cycling|else|end|'
+ r'first\s+time|one\s+of|only|or|otherwise|'
+ r'(sticky|(then\s+)?(purely\s+)?at)\s+random|stopping|'
+ r'(then\s+)?(half\s+)?shuffled|\|\|)\s*>>', String.Interpol),
+ (r'<<(%%(_(%s|\\?.)|[\-+ ,#]|\[\d*\]?)*\d*\.?\d*(%s|\\?.)|'
+ r'\s*((else|otherwise)\s+)?(if|unless)\b)?' % (_escape, _escape),
+ String.Interpol, ('block/embed', 'more/embed', 'main'))
+ ],
+ 's/entity': [
+ (r'(?i)&(#(x[\da-f]+|\d+)|[a-z][\da-z]*);?', Name.Entity)
+ ],
+ 'tdqs': _make_string_state(True, True),
+ 'tsqs': _make_string_state(True, False),
+ 'dqs': _make_string_state(False, True),
+ 'sqs': _make_string_state(False, False),
+ 'tdqs/listing': _make_string_state(True, True, 'listing'),
+ 'tsqs/listing': _make_string_state(True, False, 'listing'),
+ 'dqs/listing': _make_string_state(False, True, 'listing'),
+ 'sqs/listing': _make_string_state(False, False, 'listing'),
+ 'tdqs/xmp': _make_string_state(True, True, 'xmp'),
+ 'tsqs/xmp': _make_string_state(True, False, 'xmp'),
+ 'dqs/xmp': _make_string_state(False, True, 'xmp'),
+ 'sqs/xmp': _make_string_state(False, False, 'xmp'),
+
+ # Tags
+ 'tdqt': _make_tag_state(True, True),
+ 'tsqt': _make_tag_state(True, False),
+ 'dqt': _make_tag_state(False, True),
+ 'sqt': _make_tag_state(False, False),
+ 'dqs/tdqt': _make_attribute_value_state(r'"', True, True),
+ 'dqs/tsqt': _make_attribute_value_state(r'"', True, False),
+ 'dqs/dqt': _make_attribute_value_state(r'"', False, True),
+ 'dqs/sqt': _make_attribute_value_state(r'"', False, False),
+ 'sqs/tdqt': _make_attribute_value_state(r"'", True, True),
+ 'sqs/tsqt': _make_attribute_value_state(r"'", True, False),
+ 'sqs/dqt': _make_attribute_value_state(r"'", False, True),
+ 'sqs/sqt': _make_attribute_value_state(r"'", False, False),
+ 'uqs/tdqt': _make_attribute_value_state(_no_quote, True, True),
+ 'uqs/tsqt': _make_attribute_value_state(_no_quote, True, False),
+ 'uqs/dqt': _make_attribute_value_state(_no_quote, False, True),
+ 'uqs/sqt': _make_attribute_value_state(_no_quote, False, False),
+
+ # Regular expressions
+ 'tdqr': [
+ (r'[^\\"]+', String.Regex),
+ (r'\\"*', String.Regex),
+ (r'"{3,}', String.Regex, '#pop'),
+ (r'"', String.Regex)
+ ],
+ 'tsqr': [
+ (r"[^\\']+", String.Regex),
+ (r"\\'*", String.Regex),
+ (r"'{3,}", String.Regex, '#pop'),
+ (r"'", String.Regex)
+ ],
+ 'dqr': [
+ (r'[^\\"]+', String.Regex),
+ (r'\\"?', String.Regex),
+ (r'"', String.Regex, '#pop')
+ ],
+ 'sqr': [
+ (r"[^\\']+", String.Regex),
+ (r"\\'?", String.Regex),
+ (r"'", String.Regex, '#pop')
+ ]
+ }
+
+ def get_tokens_unprocessed(self, text, **kwargs):
+ pp = r'^%s*#%s*' % (self._ws_pp, self._ws_pp)
+ if_false_level = 0
+ for index, token, value in (
+ RegexLexer.get_tokens_unprocessed(self, text, **kwargs)):
+ if if_false_level == 0: # Not in a false #if
+ if (token is Comment.Preproc and
+ re.match(r'%sif%s+(0|nil)%s*$\n?' %
+ (pp, self._ws_pp, self._ws_pp), value)):
+ if_false_level = 1
+ else: # In a false #if
+ if token is Comment.Preproc:
+ if (if_false_level == 1 and
+ re.match(r'%sel(if|se)\b' % pp, value)):
+ if_false_level = 0
+ elif re.match(r'%sif' % pp, value):
+ if_false_level += 1
+ elif re.match(r'%sendif\b' % pp, value):
+ if_false_level -= 1
+ else:
+ token = Comment
+ yield index, token, value
diff --git a/wandb/vendor/pygments/lexers/iolang.py b/wandb/vendor/pygments/lexers/iolang.py
new file mode 100644
index 0000000000000000000000000000000000000000..bbc17faff05686e6bc1c20123cbb52980a289b9b
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/iolang.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.iolang
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the Io language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number
+
+__all__ = ['IoLexer']
+
+
+class IoLexer(RegexLexer):
+ """
+ For `Io <http://iolanguage.com/>`_ (a small, prototype-based
+ programming language) source.
+
+ .. versionadded:: 0.10
+ """
+ name = 'Io'
+ filenames = ['*.io']
+ aliases = ['io']
+ mimetypes = ['text/x-iosrc']
+ tokens = {
+ 'root': [
+ (r'\n', Text),
+ (r'\s+', Text),
+ # Comments
+ (r'//(.*?)\n', Comment.Single),
+ (r'#(.*?)\n', Comment.Single),
+ (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+ (r'/\+', Comment.Multiline, 'nestedcomment'),
+ # DoubleQuotedString
+ (r'"(\\\\|\\"|[^"])*"', String),
+ # Operators
+ (r'::=|:=|=|\(|\)|;|,|\*|-|\+|>|<|@|!|/|\||\^|\.|%|&|\[|\]|\{|\}',
+ Operator),
+ # keywords
+ (r'(clone|do|doFile|doString|method|for|if|else|elseif|then)\b',
+ Keyword),
+ # constants
+ (r'(nil|false|true)\b', Name.Constant),
+ # names
+ (r'(Object|list|List|Map|args|Sequence|Coroutine|File)\b',
+ Name.Builtin),
+ ('[a-zA-Z_]\w*', Name),
+ # numbers
+ (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+ (r'\d+', Number.Integer)
+ ],
+ 'nestedcomment': [
+ (r'[^+/]+', Comment.Multiline),
+ (r'/\+', Comment.Multiline, '#push'),
+ (r'\+/', Comment.Multiline, '#pop'),
+ (r'[+/]', Comment.Multiline),
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/j.py b/wandb/vendor/pygments/lexers/j.py
new file mode 100644
index 0000000000000000000000000000000000000000..434964fec125d243532c30a66803fb5716bf3de0
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/j.py
@@ -0,0 +1,146 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.j
+ ~~~~~~~~~~~~~~~~~
+
+ Lexer for the J programming language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words, include
+from pygments.token import Comment, Keyword, Name, Number, Operator, Punctuation, \
+ String, Text
+
+__all__ = ['JLexer']
+
+
+class JLexer(RegexLexer):
+ """
+ For `J <http://jsoftware.com/>`_ source code.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'J'
+ aliases = ['j']
+ filenames = ['*.ijs']
+ mimetypes = ['text/x-j']
+
+ validName = r'\b[a-zA-Z]\w*'
+
+ tokens = {
+ 'root': [
+ # Shebang script
+ (r'#!.*$', Comment.Preproc),
+
+ # Comments
+ (r'NB\..*', Comment.Single),
+ (r'\n+\s*Note', Comment.Multiline, 'comment'),
+ (r'\s*Note.*', Comment.Single),
+
+ # Whitespace
+ (r'\s+', Text),
+
+ # Strings
+ (r"'", String, 'singlequote'),
+
+ # Definitions
+ (r'0\s+:\s*0|noun\s+define\s*$', Name.Entity, 'nounDefinition'),
+ (r'(([1-4]|13)\s+:\s*0|(adverb|conjunction|dyad|monad|verb)\s+define)\b',
+ Name.Function, 'explicitDefinition'),
+
+ # Flow Control
+ (words(('for_', 'goto_', 'label_'), suffix=validName+'\.'), Name.Label),
+ (words((
+ 'assert', 'break', 'case', 'catch', 'catchd',
+ 'catcht', 'continue', 'do', 'else', 'elseif',
+ 'end', 'fcase', 'for', 'if', 'return',
+ 'select', 'throw', 'try', 'while', 'whilst',
+ ), suffix='\.'), Name.Label),
+
+ # Variable Names
+ (validName, Name.Variable),
+
+ # Standard Library
+ (words((
+ 'ARGV', 'CR', 'CRLF', 'DEL', 'Debug',
+ 'EAV', 'EMPTY', 'FF', 'JVERSION', 'LF',
+ 'LF2', 'Note', 'TAB', 'alpha17', 'alpha27',
+ 'apply', 'bind', 'boxopen', 'boxxopen', 'bx',
+ 'clear', 'cutLF', 'cutopen', 'datatype', 'def',
+ 'dfh', 'drop', 'each', 'echo', 'empty',
+ 'erase', 'every', 'evtloop', 'exit', 'expand',
+ 'fetch', 'file2url', 'fixdotdot', 'fliprgb', 'getargs',
+ 'getenv', 'hfd', 'inv', 'inverse', 'iospath',
+ 'isatty', 'isutf8', 'items', 'leaf', 'list',
+ 'nameclass', 'namelist', 'names', 'nc',
+ 'nl', 'on', 'pick', 'rows',
+ 'script', 'scriptd', 'sign', 'sminfo', 'smoutput',
+ 'sort', 'split', 'stderr', 'stdin', 'stdout',
+ 'table', 'take', 'timespacex', 'timex', 'tmoutput',
+ 'toCRLF', 'toHOST', 'toJ', 'tolower', 'toupper',
+ 'type', 'ucp', 'ucpcount', 'usleep', 'utf8',
+ 'uucp',
+ )), Name.Function),
+
+ # Copula
+ (r'=[.:]', Operator),
+
+ # Builtins
+ (r'[-=+*#$%@!~`^&";:.,<>{}\[\]\\|/]', Operator),
+
+ # Short Keywords
+ (r'[abCdDeEfHiIjLMoprtT]\.', Keyword.Reserved),
+ (r'[aDiLpqsStux]\:', Keyword.Reserved),
+ (r'(_[0-9])\:', Keyword.Constant),
+
+ # Parens
+ (r'\(', Punctuation, 'parentheses'),
+
+ # Numbers
+ include('numbers'),
+ ],
+
+ 'comment': [
+ (r'[^)]', Comment.Multiline),
+ (r'^\)', Comment.Multiline, '#pop'),
+ (r'[)]', Comment.Multiline),
+ ],
+
+ 'explicitDefinition': [
+ (r'\b[nmuvxy]\b', Name.Decorator),
+ include('root'),
+ (r'[^)]', Name),
+ (r'^\)', Name.Label, '#pop'),
+ (r'[)]', Name),
+ ],
+
+ 'numbers': [
+ (r'\b_{1,2}\b', Number),
+ (r'_?\d+(\.\d+)?(\s*[ejr]\s*)_?\d+(\.?=\d+)?', Number),
+ (r'_?\d+\.(?=\d+)', Number.Float),
+ (r'_?\d+x', Number.Integer.Long),
+ (r'_?\d+', Number.Integer),
+ ],
+
+ 'nounDefinition': [
+ (r'[^)]', String),
+ (r'^\)', Name.Label, '#pop'),
+ (r'[)]', String),
+ ],
+
+ 'parentheses': [
+ (r'\)', Punctuation, '#pop'),
+ # include('nounDefinition'),
+ include('explicitDefinition'),
+ include('root'),
+ ],
+
+ 'singlequote': [
+ (r"[^']", String),
+ (r"''", String),
+ (r"'", String, '#pop'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/javascript.py b/wandb/vendor/pygments/lexers/javascript.py
new file mode 100644
index 0000000000000000000000000000000000000000..862535c9867e8d39b8d672dc398f75e457c1e944
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/javascript.py
@@ -0,0 +1,1525 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.javascript
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for JavaScript and related languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, default, using, \
+ this, words, combined
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Other
+from pygments.util import get_bool_opt, iteritems
+import pygments.unistring as uni
+
+__all__ = ['JavascriptLexer', 'KalLexer', 'LiveScriptLexer', 'DartLexer',
+ 'TypeScriptLexer', 'LassoLexer', 'ObjectiveJLexer',
+ 'CoffeeScriptLexer', 'MaskLexer', 'EarlGreyLexer', 'JuttleLexer']
+
+JS_IDENT_START = ('(?:[$_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl') +
+ ']|\\\\u[a-fA-F0-9]{4})')
+JS_IDENT_PART = ('(?:[$' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl',
+ 'Mn', 'Mc', 'Nd', 'Pc') +
+ u'\u200c\u200d]|\\\\u[a-fA-F0-9]{4})')
+JS_IDENT = JS_IDENT_START + '(?:' + JS_IDENT_PART + ')*'
+
+
+class JavascriptLexer(RegexLexer):
+ """
+ For JavaScript source code.
+ """
+
+ name = 'JavaScript'
+ aliases = ['js', 'javascript']
+ filenames = ['*.js', '*.jsm']
+ mimetypes = ['application/javascript', 'application/x-javascript',
+ 'text/x-javascript', 'text/javascript']
+
+ flags = re.DOTALL | re.UNICODE | re.MULTILINE
+
+ tokens = {
+ 'commentsandwhitespace': [
+ (r'\s+', Text),
+ (r'<!--', Comment),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline)
+ ],
+ 'slashstartsregex': [
+ include('commentsandwhitespace'),
+ (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+ r'([gimuy]+\b|\B)', String.Regex, '#pop'),
+ (r'(?=/)', Text, ('#pop', 'badregex')),
+ default('#pop')
+ ],
+ 'badregex': [
+ (r'\n', Text, '#pop')
+ ],
+ 'root': [
+ (r'\A#! ?/.*?\n', Comment.Hashbang), # recognized by node.js
+ (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'),
+ include('commentsandwhitespace'),
+ (r'(\.\d+|[0-9]+\.[0-9]*)([eE][-+]?[0-9]+)?', Number.Float),
+ (r'0[bB][01]+', Number.Bin),
+ (r'0[oO][0-7]+', Number.Oct),
+ (r'0[xX][0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+', Number.Integer),
+ (r'\.\.\.|=>', Punctuation),
+ (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|'
+ r'(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'),
+ (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
+ (r'[})\].]', Punctuation),
+ (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|'
+ r'throw|try|catch|finally|new|delete|typeof|instanceof|void|yield|'
+ r'this|of)\b', Keyword, 'slashstartsregex'),
+ (r'(var|let|with|function)\b', Keyword.Declaration, 'slashstartsregex'),
+ (r'(abstract|boolean|byte|char|class|const|debugger|double|enum|export|'
+ r'extends|final|float|goto|implements|import|int|interface|long|native|'
+ r'package|private|protected|public|short|static|super|synchronized|throws|'
+ r'transient|volatile)\b', Keyword.Reserved),
+ (r'(true|false|null|NaN|Infinity|undefined)\b', Keyword.Constant),
+ (r'(Array|Boolean|Date|Error|Function|Math|netscape|'
+ r'Number|Object|Packages|RegExp|String|Promise|Proxy|sun|decodeURI|'
+ r'decodeURIComponent|encodeURI|encodeURIComponent|'
+ r'Error|eval|isFinite|isNaN|isSafeInteger|parseFloat|parseInt|'
+ r'document|this|window)\b', Name.Builtin),
+ (JS_IDENT, Name.Other),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ (r'`', String.Backtick, 'interp'),
+ ],
+ 'interp': [
+ (r'`', String.Backtick, '#pop'),
+ (r'\\\\', String.Backtick),
+ (r'\\`', String.Backtick),
+ (r'\$\{', String.Interpol, 'interp-inside'),
+ (r'\$', String.Backtick),
+ (r'[^`\\$]+', String.Backtick),
+ ],
+ 'interp-inside': [
+ # TODO: should this include single-line comments and allow nesting strings?
+ (r'\}', String.Interpol, '#pop'),
+ include('root'),
+ ],
+ # (\\\\|\\`|[^`])*`', String.Backtick),
+ }
+
+
+class KalLexer(RegexLexer):
+ """
+ For `Kal`_ source code.
+
+ .. _Kal: http://rzimmerman.github.io/kal
+
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Kal'
+ aliases = ['kal']
+ filenames = ['*.kal']
+ mimetypes = ['text/kal', 'application/kal']
+
+ flags = re.DOTALL
+ tokens = {
+ 'commentsandwhitespace': [
+ (r'\s+', Text),
+ (r'###[^#].*?###', Comment.Multiline),
+ (r'#(?!##[^#]).*?\n', Comment.Single),
+ ],
+ 'functiondef': [
+ (r'[$a-zA-Z_][\w$]*\s*', Name.Function, '#pop'),
+ include('commentsandwhitespace'),
+ ],
+ 'classdef': [
+ (r'\binherits\s+from\b', Keyword),
+ (r'[$a-zA-Z_][\w$]*\s*\n', Name.Class, '#pop'),
+ (r'[$a-zA-Z_][\w$]*\s*', Name.Class),
+ include('commentsandwhitespace'),
+ ],
+ 'listcomprehension': [
+ (r'\]', Punctuation, '#pop'),
+ (r'\b(property|value)\b', Keyword),
+ include('root'),
+ ],
+ 'waitfor': [
+ (r'\n', Punctuation, '#pop'),
+ (r'\bfrom\b', Keyword),
+ include('root'),
+ ],
+ 'root': [
+ include('commentsandwhitespace'),
+ (r'/(?! )(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+ r'([gim]+\b|\B)', String.Regex),
+ (r'\?|:|_(?=\n)|==?|!=|-(?!>)|[<>+*/-]=?',
+ Operator),
+ (r'\b(and|or|isnt|is|not|but|bitwise|mod|\^|xor|exists|'
+ r'doesnt\s+exist)\b', Operator.Word),
+ (r'(?:\([^()]+\))?\s*>', Name.Function),
+ (r'[{(]', Punctuation),
+ (r'\[', Punctuation, 'listcomprehension'),
+ (r'[})\].,]', Punctuation),
+ (r'\b(function|method|task)\b', Keyword.Declaration, 'functiondef'),
+ (r'\bclass\b', Keyword.Declaration, 'classdef'),
+ (r'\b(safe\s+)?wait\s+for\b', Keyword, 'waitfor'),
+ (r'\b(me|this)(\.[$a-zA-Z_][\w.$]*)?\b', Name.Variable.Instance),
+ (r'(?<![.$])(for(\s+(parallel|series))?|in|of|while|until|'
+ r'break|return|continue|'
+ r'when|if|unless|else|otherwise|except\s+when|'
+ r'throw|raise|fail\s+with|try|catch|finally|new|delete|'
+ r'typeof|instanceof|super|run\s+in\s+parallel|'
+ r'inherits\s+from)\b', Keyword),
+ (r'(?<![.$])(true|false|yes|no|on|off|null|nothing|none|'
+ r'NaN|Infinity|undefined)\b',
+ Keyword.Constant),
+ (r'(Array|Boolean|Date|Error|Function|Math|netscape|'
+ r'Number|Object|Packages|RegExp|String|sun|decodeURI|'
+ r'decodeURIComponent|encodeURI|encodeURIComponent|'
+ r'eval|isFinite|isNaN|isSafeInteger|parseFloat|parseInt|document|'
+ r'window|'
+ r'print)\b',
+ Name.Builtin),
+ (r'[$a-zA-Z_][\w.$]*\s*(:|[+\-*/]?\=)?\b', Name.Variable),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+', Number.Integer),
+ ('"""', String, 'tdqs'),
+ ("'''", String, 'tsqs'),
+ ('"', String, 'dqs'),
+ ("'", String, 'sqs'),
+ ],
+ 'strings': [
+ (r'[^#\\\'"]+', String),
+ # note that all kal strings are multi-line.
+ # hashmarks, quotes and backslashes must be parsed one at a time
+ ],
+ 'interpoling_string': [
+ (r'\}', String.Interpol, "#pop"),
+ include('root')
+ ],
+ 'dqs': [
+ (r'"', String, '#pop'),
+ (r'\\.|\'', String), # double-quoted string don't need ' escapes
+ (r'#\{', String.Interpol, "interpoling_string"),
+ include('strings')
+ ],
+ 'sqs': [
+ (r"'", String, '#pop'),
+ (r'#|\\.|"', String), # single quoted strings don't need " escapses
+ include('strings')
+ ],
+ 'tdqs': [
+ (r'"""', String, '#pop'),
+ (r'\\.|\'|"', String), # no need to escape quotes in triple-string
+ (r'#\{', String.Interpol, "interpoling_string"),
+ include('strings'),
+ ],
+ 'tsqs': [
+ (r"'''", String, '#pop'),
+ (r'#|\\.|\'|"', String), # no need to escape quotes in triple-strings
+ include('strings')
+ ],
+ }
+
+
+class LiveScriptLexer(RegexLexer):
+ """
+ For `LiveScript`_ source code.
+
+ .. _LiveScript: http://gkz.github.com/LiveScript/
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'LiveScript'
+ aliases = ['live-script', 'livescript']
+ filenames = ['*.ls']
+ mimetypes = ['text/livescript']
+
+ flags = re.DOTALL
+ tokens = {
+ 'commentsandwhitespace': [
+ (r'\s+', Text),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'#.*?\n', Comment.Single),
+ ],
+ 'multilineregex': [
+ include('commentsandwhitespace'),
+ (r'//([gim]+\b|\B)', String.Regex, '#pop'),
+ (r'/', String.Regex),
+ (r'[^/#]+', String.Regex)
+ ],
+ 'slashstartsregex': [
+ include('commentsandwhitespace'),
+ (r'//', String.Regex, ('#pop', 'multilineregex')),
+ (r'/(?! )(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+ r'([gim]+\b|\B)', String.Regex, '#pop'),
+ default('#pop'),
+ ],
+ 'root': [
+ # this next expr leads to infinite loops root -> slashstartsregex
+ # (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'),
+ include('commentsandwhitespace'),
+ (r'(?:\([^()]+\))?[ ]*[~-]{1,2}>|'
+ r'(?:\(?[^()\n]+\)?)?[ ]*<[~-]{1,2}', Name.Function),
+ (r'\+\+|&&|(?<![.$])\b(?:and|x?or|is|isnt|not)\b|\?|:|=|'
+ r'\|\||\\(?=\n)|(<<|>>>?|==?|!=?|'
+ r'~(?!\~?>)|-(?!\-?>)|<(?!\[)|(?<!\])>|'
+ r'[+*`%&|^/])=?',
+ Operator, 'slashstartsregex'),
+ (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
+ (r'[})\].]', Punctuation),
+ (r'(?<![.$])(for|own|in|of|while|until|loop|break|'
+ r'return|continue|switch|when|then|if|unless|else|'
+ r'throw|try|catch|finally|new|delete|typeof|instanceof|super|'
+ r'extends|this|class|by|const|var|to|til)\b', Keyword,
+ 'slashstartsregex'),
+ (r'(?<![.$])(true|false|yes|no|on|off|'
+ r'null|NaN|Infinity|undefined|void)\b',
+ Keyword.Constant),
+ (r'(Array|Boolean|Date|Error|Function|Math|netscape|'
+ r'Number|Object|Packages|RegExp|String|sun|decodeURI|'
+ r'decodeURIComponent|encodeURI|encodeURIComponent|'
+ r'eval|isFinite|isNaN|parseFloat|parseInt|document|window)\b',
+ Name.Builtin),
+ (r'[$a-zA-Z_][\w.\-:$]*\s*[:=]\s', Name.Variable,
+ 'slashstartsregex'),
+ (r'@[$a-zA-Z_][\w.\-:$]*\s*[:=]\s', Name.Variable.Instance,
+ 'slashstartsregex'),
+ (r'@', Name.Other, 'slashstartsregex'),
+ (r'@?[$a-zA-Z_][\w-]*', Name.Other, 'slashstartsregex'),
+ (r'[0-9]+\.[0-9]+([eE][0-9]+)?[fd]?(?:[a-zA-Z_]+)?', Number.Float),
+ (r'[0-9]+(~[0-9a-z]+)?(?:[a-zA-Z_]+)?', Number.Integer),
+ ('"""', String, 'tdqs'),
+ ("'''", String, 'tsqs'),
+ ('"', String, 'dqs'),
+ ("'", String, 'sqs'),
+ (r'\\\S+', String),
+ (r'<\[.*?\]>', String),
+ ],
+ 'strings': [
+ (r'[^#\\\'"]+', String),
+ # note that all coffee script strings are multi-line.
+ # hashmarks, quotes and backslashes must be parsed one at a time
+ ],
+ 'interpoling_string': [
+ (r'\}', String.Interpol, "#pop"),
+ include('root')
+ ],
+ 'dqs': [
+ (r'"', String, '#pop'),
+ (r'\\.|\'', String), # double-quoted string don't need ' escapes
+ (r'#\{', String.Interpol, "interpoling_string"),
+ (r'#', String),
+ include('strings')
+ ],
+ 'sqs': [
+ (r"'", String, '#pop'),
+ (r'#|\\.|"', String), # single quoted strings don't need " escapses
+ include('strings')
+ ],
+ 'tdqs': [
+ (r'"""', String, '#pop'),
+ (r'\\.|\'|"', String), # no need to escape quotes in triple-string
+ (r'#\{', String.Interpol, "interpoling_string"),
+ (r'#', String),
+ include('strings'),
+ ],
+ 'tsqs': [
+ (r"'''", String, '#pop'),
+ (r'#|\\.|\'|"', String), # no need to escape quotes in triple-strings
+ include('strings')
+ ],
+ }
+
+
+class DartLexer(RegexLexer):
+ """
+ For `Dart <http://dartlang.org/>`_ source code.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'Dart'
+ aliases = ['dart']
+ filenames = ['*.dart']
+ mimetypes = ['text/x-dart']
+
+ flags = re.MULTILINE | re.DOTALL
+
+ tokens = {
+ 'root': [
+ include('string_literal'),
+ (r'#!(.*?)$', Comment.Preproc),
+ (r'\b(import|export)\b', Keyword, 'import_decl'),
+ (r'\b(library|source|part of|part)\b', Keyword),
+ (r'[^\S\n]+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'\b(class)\b(\s+)',
+ bygroups(Keyword.Declaration, Text), 'class'),
+ (r'\b(assert|break|case|catch|continue|default|do|else|finally|for|'
+ r'if|in|is|new|return|super|switch|this|throw|try|while)\b',
+ Keyword),
+ (r'\b(abstract|async|await|const|extends|factory|final|get|'
+ r'implements|native|operator|set|static|sync|typedef|var|with|'
+ r'yield)\b', Keyword.Declaration),
+ (r'\b(bool|double|dynamic|int|num|Object|String|void)\b', Keyword.Type),
+ (r'\b(false|null|true)\b', Keyword.Constant),
+ (r'[~!%^&*+=|?:<>/-]|as\b', Operator),
+ (r'[a-zA-Z_$]\w*:', Name.Label),
+ (r'[a-zA-Z_$]\w*', Name),
+ (r'[(){}\[\],.;]', Punctuation),
+ (r'0[xX][0-9a-fA-F]+', Number.Hex),
+ # DIGIT+ (‘.’ DIGIT*)? EXPONENT?
+ (r'\d+(\.\d*)?([eE][+-]?\d+)?', Number),
+ (r'\.\d+([eE][+-]?\d+)?', Number), # ‘.’ DIGIT+ EXPONENT?
+ (r'\n', Text)
+ # pseudo-keyword negate intentionally left out
+ ],
+ 'class': [
+ (r'[a-zA-Z_$]\w*', Name.Class, '#pop')
+ ],
+ 'import_decl': [
+ include('string_literal'),
+ (r'\s+', Text),
+ (r'\b(as|show|hide)\b', Keyword),
+ (r'[a-zA-Z_$]\w*', Name),
+ (r'\,', Punctuation),
+ (r'\;', Punctuation, '#pop')
+ ],
+ 'string_literal': [
+ # Raw strings.
+ (r'r"""([\w\W]*?)"""', String.Double),
+ (r"r'''([\w\W]*?)'''", String.Single),
+ (r'r"(.*?)"', String.Double),
+ (r"r'(.*?)'", String.Single),
+ # Normal Strings.
+ (r'"""', String.Double, 'string_double_multiline'),
+ (r"'''", String.Single, 'string_single_multiline'),
+ (r'"', String.Double, 'string_double'),
+ (r"'", String.Single, 'string_single')
+ ],
+ 'string_common': [
+ (r"\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|u\{[0-9A-Fa-f]*\}|[a-z'\"$\\])",
+ String.Escape),
+ (r'(\$)([a-zA-Z_]\w*)', bygroups(String.Interpol, Name)),
+ (r'(\$\{)(.*?)(\})',
+ bygroups(String.Interpol, using(this), String.Interpol))
+ ],
+ 'string_double': [
+ (r'"', String.Double, '#pop'),
+ (r'[^"$\\\n]+', String.Double),
+ include('string_common'),
+ (r'\$+', String.Double)
+ ],
+ 'string_double_multiline': [
+ (r'"""', String.Double, '#pop'),
+ (r'[^"$\\]+', String.Double),
+ include('string_common'),
+ (r'(\$|\")+', String.Double)
+ ],
+ 'string_single': [
+ (r"'", String.Single, '#pop'),
+ (r"[^'$\\\n]+", String.Single),
+ include('string_common'),
+ (r'\$+', String.Single)
+ ],
+ 'string_single_multiline': [
+ (r"'''", String.Single, '#pop'),
+ (r'[^\'$\\]+', String.Single),
+ include('string_common'),
+ (r'(\$|\')+', String.Single)
+ ]
+ }
+
+
+class TypeScriptLexer(RegexLexer):
+ """
+ For `TypeScript <http://typescriptlang.org/>`_ source code.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'TypeScript'
+ aliases = ['ts', 'typescript']
+ filenames = ['*.ts', '*.tsx']
+ mimetypes = ['text/x-typescript']
+
+ flags = re.DOTALL | re.MULTILINE
+
+ tokens = {
+ 'commentsandwhitespace': [
+ (r'\s+', Text),
+ (r'<!--', Comment),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline)
+ ],
+ 'slashstartsregex': [
+ include('commentsandwhitespace'),
+ (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+ r'([gim]+\b|\B)', String.Regex, '#pop'),
+ (r'(?=/)', Text, ('#pop', 'badregex')),
+ default('#pop')
+ ],
+ 'badregex': [
+ (r'\n', Text, '#pop')
+ ],
+ 'root': [
+ (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'),
+ include('commentsandwhitespace'),
+ (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|'
+ r'(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'),
+ (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
+ (r'[})\].]', Punctuation),
+ (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|'
+ r'throw|try|catch|finally|new|delete|typeof|instanceof|void|'
+ r'this)\b', Keyword, 'slashstartsregex'),
+ (r'(var|let|with|function)\b', Keyword.Declaration, 'slashstartsregex'),
+ (r'(abstract|boolean|byte|char|class|const|debugger|double|enum|export|'
+ r'extends|final|float|goto|implements|import|int|interface|long|native|'
+ r'package|private|protected|public|short|static|super|synchronized|throws|'
+ r'transient|volatile)\b', Keyword.Reserved),
+ (r'(true|false|null|NaN|Infinity|undefined)\b', Keyword.Constant),
+ (r'(Array|Boolean|Date|Error|Function|Math|netscape|'
+ r'Number|Object|Packages|RegExp|String|sun|decodeURI|'
+ r'decodeURIComponent|encodeURI|encodeURIComponent|'
+ r'Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|'
+ r'window)\b', Name.Builtin),
+ # Match stuff like: module name {...}
+ (r'\b(module)(\s*)(\s*[\w?.$][\w?.$]*)(\s*)',
+ bygroups(Keyword.Reserved, Text, Name.Other, Text), 'slashstartsregex'),
+ # Match variable type keywords
+ (r'\b(string|bool|number)\b', Keyword.Type),
+ # Match stuff like: constructor
+ (r'\b(constructor|declare|interface|as|AS)\b', Keyword.Reserved),
+ # Match stuff like: super(argument, list)
+ (r'(super)(\s*)(\([\w,?.$\s]+\s*\))',
+ bygroups(Keyword.Reserved, Text), 'slashstartsregex'),
+ # Match stuff like: function() {...}
+ (r'([a-zA-Z_?.$][\w?.$]*)\(\) \{', Name.Other, 'slashstartsregex'),
+ # Match stuff like: (function: return type)
+ (r'([\w?.$][\w?.$]*)(\s*:\s*)([\w?.$][\w?.$]*)',
+ bygroups(Name.Other, Text, Keyword.Type)),
+ (r'[$a-zA-Z_]\w*', Name.Other),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+', Number.Integer),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ (r'`', String.Backtick, 'interp'),
+ # Match stuff like: Decorators
+ (r'@\w+', Keyword.Declaration),
+ ],
+
+ # The 'interp*' rules match those in JavascriptLexer. Changes made
+ # there should be reflected here as well.
+ 'interp': [
+ (r'`', String.Backtick, '#pop'),
+ (r'\\\\', String.Backtick),
+ (r'\\`', String.Backtick),
+ (r'\$\{', String.Interpol, 'interp-inside'),
+ (r'\$', String.Backtick),
+ (r'[^`\\$]+', String.Backtick),
+ ],
+ 'interp-inside': [
+ # TODO: should this include single-line comments and allow nesting strings?
+ (r'\}', String.Interpol, '#pop'),
+ include('root'),
+ ],
+ }
+
+ def analyse_text(text):
+ if re.search('^(import.+(from\s+)?["\']|'
+ '(export\s*)?(interface|class|function)\s+)',
+ text, re.MULTILINE):
+ return 1.0
+
+
+class LassoLexer(RegexLexer):
+ """
+ For `Lasso <http://www.lassosoft.com/>`_ source code, covering both Lasso 9
+ syntax and LassoScript for Lasso 8.6 and earlier. For Lasso embedded in
+ HTML, use the `LassoHtmlLexer`.
+
+ Additional options accepted:
+
+ `builtinshighlighting`
+ If given and ``True``, highlight builtin types, traits, methods, and
+ members (default: ``True``).
+ `requiredelimiters`
+ If given and ``True``, only highlight code between delimiters as Lasso
+ (default: ``False``).
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'Lasso'
+ aliases = ['lasso', 'lassoscript']
+ filenames = ['*.lasso', '*.lasso[89]']
+ alias_filenames = ['*.incl', '*.inc', '*.las']
+ mimetypes = ['text/x-lasso']
+ flags = re.IGNORECASE | re.DOTALL | re.MULTILINE
+
+ tokens = {
+ 'root': [
+ (r'^#![ \S]+lasso9\b', Comment.Preproc, 'lasso'),
+ (r'(?=\[|<)', Other, 'delimiters'),
+ (r'\s+', Other),
+ default(('delimiters', 'lassofile')),
+ ],
+ 'delimiters': [
+ (r'\[no_square_brackets\]', Comment.Preproc, 'nosquarebrackets'),
+ (r'\[noprocess\]', Comment.Preproc, 'noprocess'),
+ (r'\[', Comment.Preproc, 'squarebrackets'),
+ (r'<\?(lasso(script)?|=)', Comment.Preproc, 'anglebrackets'),
+ (r'<(!--.*?-->)?', Other),
+ (r'[^[<]+', Other),
+ ],
+ 'nosquarebrackets': [
+ (r'\[noprocess\]', Comment.Preproc, 'noprocess'),
+ (r'\[', Other),
+ (r'<\?(lasso(script)?|=)', Comment.Preproc, 'anglebrackets'),
+ (r'<(!--.*?-->)?', Other),
+ (r'[^[<]+', Other),
+ ],
+ 'noprocess': [
+ (r'\[/noprocess\]', Comment.Preproc, '#pop'),
+ (r'\[', Other),
+ (r'[^[]', Other),
+ ],
+ 'squarebrackets': [
+ (r'\]', Comment.Preproc, '#pop'),
+ include('lasso'),
+ ],
+ 'anglebrackets': [
+ (r'\?>', Comment.Preproc, '#pop'),
+ include('lasso'),
+ ],
+ 'lassofile': [
+ (r'\]|\?>', Comment.Preproc, '#pop'),
+ include('lasso'),
+ ],
+ 'whitespacecomments': [
+ (r'\s+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*\*!.*?\*/', String.Doc),
+ (r'/\*.*?\*/', Comment.Multiline),
+ ],
+ 'lasso': [
+ # whitespace/comments
+ include('whitespacecomments'),
+
+ # literals
+ (r'\d*\.\d+(e[+-]?\d+)?', Number.Float),
+ (r'0x[\da-f]+', Number.Hex),
+ (r'\d+', Number.Integer),
+ (r'(infinity|NaN)\b', Number),
+ (r"'", String.Single, 'singlestring'),
+ (r'"', String.Double, 'doublestring'),
+ (r'`[^`]*`', String.Backtick),
+
+ # names
+ (r'\$[a-z_][\w.]*', Name.Variable),
+ (r'#([a-z_][\w.]*|\d+\b)', Name.Variable.Instance),
+ (r"(\.\s*)('[a-z_][\w.]*')",
+ bygroups(Name.Builtin.Pseudo, Name.Variable.Class)),
+ (r"(self)(\s*->\s*)('[a-z_][\w.]*')",
+ bygroups(Name.Builtin.Pseudo, Operator, Name.Variable.Class)),
+ (r'(\.\.?\s*)([a-z_][\w.]*(=(?!=))?)',
+ bygroups(Name.Builtin.Pseudo, Name.Other.Member)),
+ (r'(->\\?\s*|&\s*)([a-z_][\w.]*(=(?!=))?)',
+ bygroups(Operator, Name.Other.Member)),
+ (r'(?<!->)(self|inherited|currentcapture|givenblock)\b',
+ Name.Builtin.Pseudo),
+ (r'-(?!infinity)[a-z_][\w.]*', Name.Attribute),
+ (r'::\s*[a-z_][\w.]*', Name.Label),
+ (r'(error_(code|msg)_\w+|Error_AddError|Error_ColumnRestriction|'
+ r'Error_DatabaseConnectionUnavailable|Error_DatabaseTimeout|'
+ r'Error_DeleteError|Error_FieldRestriction|Error_FileNotFound|'
+ r'Error_InvalidDatabase|Error_InvalidPassword|'
+ r'Error_InvalidUsername|Error_ModuleNotFound|'
+ r'Error_NoError|Error_NoPermission|Error_OutOfMemory|'
+ r'Error_ReqColumnMissing|Error_ReqFieldMissing|'
+ r'Error_RequiredColumnMissing|Error_RequiredFieldMissing|'
+ r'Error_UpdateError)\b', Name.Exception),
+
+ # definitions
+ (r'(define)(\s+)([a-z_][\w.]*)(\s*=>\s*)(type|trait|thread)\b',
+ bygroups(Keyword.Declaration, Text, Name.Class, Operator, Keyword)),
+ (r'(define)(\s+)([a-z_][\w.]*)(\s*->\s*)([a-z_][\w.]*=?|[-+*/%])',
+ bygroups(Keyword.Declaration, Text, Name.Class, Operator,
+ Name.Function), 'signature'),
+ (r'(define)(\s+)([a-z_][\w.]*)',
+ bygroups(Keyword.Declaration, Text, Name.Function), 'signature'),
+ (r'(public|protected|private|provide)(\s+)(([a-z_][\w.]*=?|[-+*/%])'
+ r'(?=\s*\())', bygroups(Keyword, Text, Name.Function),
+ 'signature'),
+ (r'(public|protected|private|provide)(\s+)([a-z_][\w.]*)',
+ bygroups(Keyword, Text, Name.Function)),
+
+ # keywords
+ (r'(true|false|none|minimal|full|all|void)\b', Keyword.Constant),
+ (r'(local|var|variable|global|data(?=\s))\b', Keyword.Declaration),
+ (r'(array|date|decimal|duration|integer|map|pair|string|tag|xml|'
+ r'null|boolean|bytes|keyword|list|locale|queue|set|stack|'
+ r'staticarray)\b', Keyword.Type),
+ (r'([a-z_][\w.]*)(\s+)(in)\b', bygroups(Name, Text, Keyword)),
+ (r'(let|into)(\s+)([a-z_][\w.]*)', bygroups(Keyword, Text, Name)),
+ (r'require\b', Keyword, 'requiresection'),
+ (r'(/?)(Namespace_Using)\b', bygroups(Punctuation, Keyword.Namespace)),
+ (r'(/?)(Cache|Database_Names|Database_SchemaNames|'
+ r'Database_TableNames|Define_Tag|Define_Type|Email_Batch|'
+ r'Encode_Set|HTML_Comment|Handle|Handle_Error|Header|If|Inline|'
+ r'Iterate|LJAX_Target|Link|Link_CurrentAction|Link_CurrentGroup|'
+ r'Link_CurrentRecord|Link_Detail|Link_FirstGroup|Link_FirstRecord|'
+ r'Link_LastGroup|Link_LastRecord|Link_NextGroup|Link_NextRecord|'
+ r'Link_PrevGroup|Link_PrevRecord|Log|Loop|Output_None|Portal|'
+ r'Private|Protect|Records|Referer|Referrer|Repeating|ResultSet|'
+ r'Rows|Search_Args|Search_Arguments|Select|Sort_Args|'
+ r'Sort_Arguments|Thread_Atomic|Value_List|While|Abort|Case|Else|'
+ r'Fail_If|Fail_IfNot|Fail|If_Empty|If_False|If_Null|If_True|'
+ r'Loop_Abort|Loop_Continue|Loop_Count|Params|Params_Up|Return|'
+ r'Return_Value|Run_Children|SOAP_DefineTag|SOAP_LastRequest|'
+ r'SOAP_LastResponse|Tag_Name|ascending|average|by|define|'
+ r'descending|do|equals|frozen|group|handle_failure|import|in|into|'
+ r'join|let|match|max|min|on|order|parent|protected|provide|public|'
+ r'require|returnhome|skip|split_thread|sum|take|thread|to|trait|'
+ r'type|where|with|yield|yieldhome)\b',
+ bygroups(Punctuation, Keyword)),
+
+ # other
+ (r',', Punctuation, 'commamember'),
+ (r'(and|or|not)\b', Operator.Word),
+ (r'([a-z_][\w.]*)(\s*::\s*[a-z_][\w.]*)?(\s*=(?!=))',
+ bygroups(Name, Name.Label, Operator)),
+ (r'(/?)([\w.]+)', bygroups(Punctuation, Name.Other)),
+ (r'(=)(n?bw|n?ew|n?cn|lte?|gte?|n?eq|n?rx|ft)\b',
+ bygroups(Operator, Operator.Word)),
+ (r':=|[-+*/%=<>&|!?\\]+', Operator),
+ (r'[{}():;,@^]', Punctuation),
+ ],
+ 'singlestring': [
+ (r"'", String.Single, '#pop'),
+ (r"[^'\\]+", String.Single),
+ include('escape'),
+ (r"\\", String.Single),
+ ],
+ 'doublestring': [
+ (r'"', String.Double, '#pop'),
+ (r'[^"\\]+', String.Double),
+ include('escape'),
+ (r'\\', String.Double),
+ ],
+ 'escape': [
+ (r'\\(U[\da-f]{8}|u[\da-f]{4}|x[\da-f]{1,2}|[0-7]{1,3}|:[^:\n\r]+:|'
+ r'[abefnrtv?"\'\\]|$)', String.Escape),
+ ],
+ 'signature': [
+ (r'=>', Operator, '#pop'),
+ (r'\)', Punctuation, '#pop'),
+ (r'[(,]', Punctuation, 'parameter'),
+ include('lasso'),
+ ],
+ 'parameter': [
+ (r'\)', Punctuation, '#pop'),
+ (r'-?[a-z_][\w.]*', Name.Attribute, '#pop'),
+ (r'\.\.\.', Name.Builtin.Pseudo),
+ include('lasso'),
+ ],
+ 'requiresection': [
+ (r'(([a-z_][\w.]*=?|[-+*/%])(?=\s*\())', Name, 'requiresignature'),
+ (r'(([a-z_][\w.]*=?|[-+*/%])(?=(\s*::\s*[\w.]+)?\s*,))', Name),
+ (r'[a-z_][\w.]*=?|[-+*/%]', Name, '#pop'),
+ (r'::\s*[a-z_][\w.]*', Name.Label),
+ (r',', Punctuation),
+ include('whitespacecomments'),
+ ],
+ 'requiresignature': [
+ (r'(\)(?=(\s*::\s*[\w.]+)?\s*,))', Punctuation, '#pop'),
+ (r'\)', Punctuation, '#pop:2'),
+ (r'-?[a-z_][\w.]*', Name.Attribute),
+ (r'::\s*[a-z_][\w.]*', Name.Label),
+ (r'\.\.\.', Name.Builtin.Pseudo),
+ (r'[(,]', Punctuation),
+ include('whitespacecomments'),
+ ],
+ 'commamember': [
+ (r'(([a-z_][\w.]*=?|[-+*/%])'
+ r'(?=\s*(\(([^()]*\([^()]*\))*[^)]*\)\s*)?(::[\w.\s]+)?=>))',
+ Name.Function, 'signature'),
+ include('whitespacecomments'),
+ default('#pop'),
+ ],
+ }
+
+ def __init__(self, **options):
+ self.builtinshighlighting = get_bool_opt(
+ options, 'builtinshighlighting', True)
+ self.requiredelimiters = get_bool_opt(
+ options, 'requiredelimiters', False)
+
+ self._builtins = set()
+ self._members = set()
+ if self.builtinshighlighting:
+ from pygments.lexers._lasso_builtins import BUILTINS, MEMBERS
+ for key, value in iteritems(BUILTINS):
+ self._builtins.update(value)
+ for key, value in iteritems(MEMBERS):
+ self._members.update(value)
+ RegexLexer.__init__(self, **options)
+
+ def get_tokens_unprocessed(self, text):
+ stack = ['root']
+ if self.requiredelimiters:
+ stack.append('delimiters')
+ for index, token, value in \
+ RegexLexer.get_tokens_unprocessed(self, text, stack):
+ if (token is Name.Other and value.lower() in self._builtins or
+ token is Name.Other.Member and
+ value.lower().rstrip('=') in self._members):
+ yield index, Name.Builtin, value
+ continue
+ yield index, token, value
+
+ def analyse_text(text):
+ rv = 0.0
+ if 'bin/lasso9' in text:
+ rv += 0.8
+ if re.search(r'<\?lasso', text, re.I):
+ rv += 0.4
+ if re.search(r'local\(', text, re.I):
+ rv += 0.4
+ return rv
+
+
+class ObjectiveJLexer(RegexLexer):
+ """
+ For Objective-J source code with preprocessor directives.
+
+ .. versionadded:: 1.3
+ """
+
+ name = 'Objective-J'
+ aliases = ['objective-j', 'objectivej', 'obj-j', 'objj']
+ filenames = ['*.j']
+ mimetypes = ['text/x-objective-j']
+
+ #: optional Comment or Whitespace
+ _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)*'
+
+ flags = re.DOTALL | re.MULTILINE
+
+ tokens = {
+ 'root': [
+ include('whitespace'),
+
+ # function definition
+ (r'^(' + _ws + r'[+-]' + _ws + r')([(a-zA-Z_].*?[^(])(' + _ws + r'\{)',
+ bygroups(using(this), using(this, state='function_signature'),
+ using(this))),
+
+ # class definition
+ (r'(@interface|@implementation)(\s+)', bygroups(Keyword, Text),
+ 'classname'),
+ (r'(@class|@protocol)(\s*)', bygroups(Keyword, Text),
+ 'forward_classname'),
+ (r'(\s*)(@end)(\s*)', bygroups(Text, Keyword, Text)),
+
+ include('statements'),
+ ('[{()}]', Punctuation),
+ (';', Punctuation),
+ ],
+ 'whitespace': [
+ (r'(@import)(\s+)("(?:\\\\|\\"|[^"])*")',
+ bygroups(Comment.Preproc, Text, String.Double)),
+ (r'(@import)(\s+)(<(?:\\\\|\\>|[^>])*>)',
+ bygroups(Comment.Preproc, Text, String.Double)),
+ (r'(#(?:include|import))(\s+)("(?:\\\\|\\"|[^"])*")',
+ bygroups(Comment.Preproc, Text, String.Double)),
+ (r'(#(?:include|import))(\s+)(<(?:\\\\|\\>|[^>])*>)',
+ bygroups(Comment.Preproc, Text, String.Double)),
+
+ (r'#if\s+0', Comment.Preproc, 'if0'),
+ (r'#', Comment.Preproc, 'macro'),
+
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'\\\n', Text), # line continuation
+ (r'//(\n|(.|\n)*?[^\\]\n)', Comment.Single),
+ (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+ (r'<!--', Comment),
+ ],
+ 'slashstartsregex': [
+ include('whitespace'),
+ (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+ r'([gim]+\b|\B)', String.Regex, '#pop'),
+ (r'(?=/)', Text, ('#pop', 'badregex')),
+ default('#pop'),
+ ],
+ 'badregex': [
+ (r'\n', Text, '#pop'),
+ ],
+ 'statements': [
+ (r'(L|@)?"', String, 'string'),
+ (r"(L|@)?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'",
+ String.Char),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float),
+ (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+ (r'0x[0-9a-fA-F]+[Ll]?', Number.Hex),
+ (r'0[0-7]+[Ll]?', Number.Oct),
+ (r'\d+[Ll]?', Number.Integer),
+
+ (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'),
+
+ (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|'
+ r'(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?',
+ Operator, 'slashstartsregex'),
+ (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
+ (r'[})\].]', Punctuation),
+
+ (r'(for|in|while|do|break|return|continue|switch|case|default|if|'
+ r'else|throw|try|catch|finally|new|delete|typeof|instanceof|void|'
+ r'prototype|__proto__)\b', Keyword, 'slashstartsregex'),
+
+ (r'(var|with|function)\b', Keyword.Declaration, 'slashstartsregex'),
+
+ (r'(@selector|@private|@protected|@public|@encode|'
+ r'@synchronized|@try|@throw|@catch|@finally|@end|@property|'
+ r'@synthesize|@dynamic|@for|@accessors|new)\b', Keyword),
+
+ (r'(int|long|float|short|double|char|unsigned|signed|void|'
+ r'id|BOOL|bool|boolean|IBOutlet|IBAction|SEL|@outlet|@action)\b',
+ Keyword.Type),
+
+ (r'(self|super)\b', Name.Builtin),
+
+ (r'(TRUE|YES|FALSE|NO|Nil|nil|NULL)\b', Keyword.Constant),
+ (r'(true|false|null|NaN|Infinity|undefined)\b', Keyword.Constant),
+ (r'(ABS|ASIN|ACOS|ATAN|ATAN2|SIN|COS|TAN|EXP|POW|CEIL|FLOOR|ROUND|'
+ r'MIN|MAX|RAND|SQRT|E|LN2|LN10|LOG2E|LOG10E|PI|PI2|PI_2|SQRT1_2|'
+ r'SQRT2)\b', Keyword.Constant),
+
+ (r'(Array|Boolean|Date|Error|Function|Math|netscape|'
+ r'Number|Object|Packages|RegExp|String|sun|decodeURI|'
+ r'decodeURIComponent|encodeURI|encodeURIComponent|'
+ r'Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|'
+ r'window)\b', Name.Builtin),
+
+ (r'([$a-zA-Z_]\w*)(' + _ws + r')(?=\()',
+ bygroups(Name.Function, using(this))),
+
+ (r'[$a-zA-Z_]\w*', Name),
+ ],
+ 'classname': [
+ # interface definition that inherits
+ (r'([a-zA-Z_]\w*)(' + _ws + r':' + _ws +
+ r')([a-zA-Z_]\w*)?',
+ bygroups(Name.Class, using(this), Name.Class), '#pop'),
+ # interface definition for a category
+ (r'([a-zA-Z_]\w*)(' + _ws + r'\()([a-zA-Z_]\w*)(\))',
+ bygroups(Name.Class, using(this), Name.Label, Text), '#pop'),
+ # simple interface / implementation
+ (r'([a-zA-Z_]\w*)', Name.Class, '#pop'),
+ ],
+ 'forward_classname': [
+ (r'([a-zA-Z_]\w*)(\s*,\s*)',
+ bygroups(Name.Class, Text), '#push'),
+ (r'([a-zA-Z_]\w*)(\s*;?)',
+ bygroups(Name.Class, Text), '#pop'),
+ ],
+ 'function_signature': [
+ include('whitespace'),
+
+ # start of a selector w/ parameters
+ (r'(\(' + _ws + r')' # open paren
+ r'([a-zA-Z_]\w+)' # return type
+ r'(' + _ws + r'\)' + _ws + r')' # close paren
+ r'([$a-zA-Z_]\w+' + _ws + r':)', # function name
+ bygroups(using(this), Keyword.Type, using(this),
+ Name.Function), 'function_parameters'),
+
+ # no-param function
+ (r'(\(' + _ws + r')' # open paren
+ r'([a-zA-Z_]\w+)' # return type
+ r'(' + _ws + r'\)' + _ws + r')' # close paren
+ r'([$a-zA-Z_]\w+)', # function name
+ bygroups(using(this), Keyword.Type, using(this),
+ Name.Function), "#pop"),
+
+ # no return type given, start of a selector w/ parameters
+ (r'([$a-zA-Z_]\w+' + _ws + r':)', # function name
+ bygroups(Name.Function), 'function_parameters'),
+
+ # no return type given, no-param function
+ (r'([$a-zA-Z_]\w+)', # function name
+ bygroups(Name.Function), "#pop"),
+
+ default('#pop'),
+ ],
+ 'function_parameters': [
+ include('whitespace'),
+
+ # parameters
+ (r'(\(' + _ws + ')' # open paren
+ r'([^)]+)' # type
+ r'(' + _ws + r'\)' + _ws + r')' # close paren
+ r'([$a-zA-Z_]\w+)', # param name
+ bygroups(using(this), Keyword.Type, using(this), Text)),
+
+ # one piece of a selector name
+ (r'([$a-zA-Z_]\w+' + _ws + r':)', # function name
+ Name.Function),
+
+ # smallest possible selector piece
+ (r'(:)', Name.Function),
+
+ # var args
+ (r'(,' + _ws + r'\.\.\.)', using(this)),
+
+ # param name
+ (r'([$a-zA-Z_]\w+)', Text),
+ ],
+ 'expression': [
+ (r'([$a-zA-Z_]\w*)(\()', bygroups(Name.Function,
+ Punctuation)),
+ (r'(\))', Punctuation, "#pop"),
+ ],
+ 'string': [
+ (r'"', String, '#pop'),
+ (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+ (r'[^\\"\n]+', String), # all other characters
+ (r'\\\n', String), # line continuation
+ (r'\\', String), # stray backslash
+ ],
+ 'macro': [
+ (r'[^/\n]+', Comment.Preproc),
+ (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+ (r'//.*?\n', Comment.Single, '#pop'),
+ (r'/', Comment.Preproc),
+ (r'(?<=\\)\n', Comment.Preproc),
+ (r'\n', Comment.Preproc, '#pop'),
+ ],
+ 'if0': [
+ (r'^\s*#if.*?(?<!\\)\n', Comment.Preproc, '#push'),
+ (r'^\s*#endif.*?(?<!\\)\n', Comment.Preproc, '#pop'),
+ (r'.*?\n', Comment),
+ ]
+ }
+
+ def analyse_text(text):
+ if re.search('^\s*@import\s+[<"]', text, re.MULTILINE):
+ # special directive found in most Objective-J files
+ return True
+ return False
+
+
+class CoffeeScriptLexer(RegexLexer):
+ """
+ For `CoffeeScript`_ source code.
+
+ .. _CoffeeScript: http://coffeescript.org
+
+ .. versionadded:: 1.3
+ """
+
+ name = 'CoffeeScript'
+ aliases = ['coffee-script', 'coffeescript', 'coffee']
+ filenames = ['*.coffee']
+ mimetypes = ['text/coffeescript']
+
+
+ _operator_re = (
+ r'\+\+|~|&&|\band\b|\bor\b|\bis\b|\bisnt\b|\bnot\b|\?|:|'
+ r'\|\||\\(?=\n)|'
+ r'(<<|>>>?|==?(?!>)|!=?|=(?!>)|-(?!>)|[<>+*`%&\|\^/])=?')
+
+ flags = re.DOTALL
+ tokens = {
+ 'commentsandwhitespace': [
+ (r'\s+', Text),
+ (r'###[^#].*?###', Comment.Multiline),
+ (r'#(?!##[^#]).*?\n', Comment.Single),
+ ],
+ 'multilineregex': [
+ (r'[^/#]+', String.Regex),
+ (r'///([gim]+\b|\B)', String.Regex, '#pop'),
+ (r'#\{', String.Interpol, 'interpoling_string'),
+ (r'[/#]', String.Regex),
+ ],
+ 'slashstartsregex': [
+ include('commentsandwhitespace'),
+ (r'///', String.Regex, ('#pop', 'multilineregex')),
+ (r'/(?! )(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+ r'([gim]+\b|\B)', String.Regex, '#pop'),
+ # This isn't really guarding against mishighlighting well-formed
+ # code, just the ability to infinite-loop between root and
+ # slashstartsregex.
+ (r'/', Operator),
+ default('#pop'),
+ ],
+ 'root': [
+ include('commentsandwhitespace'),
+ (r'^(?=\s|/)', Text, 'slashstartsregex'),
+ (_operator_re, Operator, 'slashstartsregex'),
+ (r'(?:\([^()]*\))?\s*[=-]>', Name.Function, 'slashstartsregex'),
+ (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
+ (r'[})\].]', Punctuation),
+ (r'(?<![.$])(for|own|in|of|while|until|'
+ r'loop|break|return|continue|'
+ r'switch|when|then|if|unless|else|'
+ r'throw|try|catch|finally|new|delete|typeof|instanceof|super|'
+ r'extends|this|class|by)\b', Keyword, 'slashstartsregex'),
+ (r'(?<![.$])(true|false|yes|no|on|off|null|'
+ r'NaN|Infinity|undefined)\b',
+ Keyword.Constant),
+ (r'(Array|Boolean|Date|Error|Function|Math|netscape|'
+ r'Number|Object|Packages|RegExp|String|sun|decodeURI|'
+ r'decodeURIComponent|encodeURI|encodeURIComponent|'
+ r'eval|isFinite|isNaN|parseFloat|parseInt|document|window)\b',
+ Name.Builtin),
+ (r'[$a-zA-Z_][\w.:$]*\s*[:=]\s', Name.Variable,
+ 'slashstartsregex'),
+ (r'@[$a-zA-Z_][\w.:$]*\s*[:=]\s', Name.Variable.Instance,
+ 'slashstartsregex'),
+ (r'@', Name.Other, 'slashstartsregex'),
+ (r'@?[$a-zA-Z_][\w$]*', Name.Other),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+', Number.Integer),
+ ('"""', String, 'tdqs'),
+ ("'''", String, 'tsqs'),
+ ('"', String, 'dqs'),
+ ("'", String, 'sqs'),
+ ],
+ 'strings': [
+ (r'[^#\\\'"]+', String),
+ # note that all coffee script strings are multi-line.
+ # hashmarks, quotes and backslashes must be parsed one at a time
+ ],
+ 'interpoling_string': [
+ (r'\}', String.Interpol, "#pop"),
+ include('root')
+ ],
+ 'dqs': [
+ (r'"', String, '#pop'),
+ (r'\\.|\'', String), # double-quoted string don't need ' escapes
+ (r'#\{', String.Interpol, "interpoling_string"),
+ (r'#', String),
+ include('strings')
+ ],
+ 'sqs': [
+ (r"'", String, '#pop'),
+ (r'#|\\.|"', String), # single quoted strings don't need " escapses
+ include('strings')
+ ],
+ 'tdqs': [
+ (r'"""', String, '#pop'),
+ (r'\\.|\'|"', String), # no need to escape quotes in triple-string
+ (r'#\{', String.Interpol, "interpoling_string"),
+ (r'#', String),
+ include('strings'),
+ ],
+ 'tsqs': [
+ (r"'''", String, '#pop'),
+ (r'#|\\.|\'|"', String), # no need to escape quotes in triple-strings
+ include('strings')
+ ],
+ }
+
+
+class MaskLexer(RegexLexer):
+ """
+ For `Mask <http://github.com/atmajs/MaskJS>`__ markup.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Mask'
+ aliases = ['mask']
+ filenames = ['*.mask']
+ mimetypes = ['text/x-mask']
+
+ flags = re.MULTILINE | re.IGNORECASE | re.DOTALL
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'[{};>]', Punctuation),
+ (r"'''", String, 'string-trpl-single'),
+ (r'"""', String, 'string-trpl-double'),
+ (r"'", String, 'string-single'),
+ (r'"', String, 'string-double'),
+ (r'([\w-]+)', Name.Tag, 'node'),
+ (r'([^.#;{>\s]+)', Name.Class, 'node'),
+ (r'(#[\w-]+)', Name.Function, 'node'),
+ (r'(\.[\w-]+)', Name.Variable.Class, 'node')
+ ],
+ 'string-base': [
+ (r'\\.', String.Escape),
+ (r'~\[', String.Interpol, 'interpolation'),
+ (r'.', String.Single),
+ ],
+ 'string-single': [
+ (r"'", String.Single, '#pop'),
+ include('string-base')
+ ],
+ 'string-double': [
+ (r'"', String.Single, '#pop'),
+ include('string-base')
+ ],
+ 'string-trpl-single': [
+ (r"'''", String.Single, '#pop'),
+ include('string-base')
+ ],
+ 'string-trpl-double': [
+ (r'"""', String.Single, '#pop'),
+ include('string-base')
+ ],
+ 'interpolation': [
+ (r'\]', String.Interpol, '#pop'),
+ (r'\s*:', String.Interpol, 'expression'),
+ (r'\s*\w+:', Name.Other),
+ (r'[^\]]+', String.Interpol)
+ ],
+ 'expression': [
+ (r'[^\]]+', using(JavascriptLexer), '#pop')
+ ],
+ 'node': [
+ (r'\s+', Text),
+ (r'\.', Name.Variable.Class, 'node-class'),
+ (r'\#', Name.Function, 'node-id'),
+ (r'style[ \t]*=', Name.Attribute, 'node-attr-style-value'),
+ (r'[\w:-]+[ \t]*=', Name.Attribute, 'node-attr-value'),
+ (r'[\w:-]+', Name.Attribute),
+ (r'[>{;]', Punctuation, '#pop')
+ ],
+ 'node-class': [
+ (r'[\w-]+', Name.Variable.Class),
+ (r'~\[', String.Interpol, 'interpolation'),
+ default('#pop')
+ ],
+ 'node-id': [
+ (r'[\w-]+', Name.Function),
+ (r'~\[', String.Interpol, 'interpolation'),
+ default('#pop')
+ ],
+ 'node-attr-value': [
+ (r'\s+', Text),
+ (r'\w+', Name.Variable, '#pop'),
+ (r"'", String, 'string-single-pop2'),
+ (r'"', String, 'string-double-pop2'),
+ default('#pop')
+ ],
+ 'node-attr-style-value': [
+ (r'\s+', Text),
+ (r"'", String.Single, 'css-single-end'),
+ (r'"', String.Single, 'css-double-end'),
+ include('node-attr-value')
+ ],
+ 'css-base': [
+ (r'\s+', Text),
+ (r";", Punctuation),
+ (r"[\w\-]+\s*:", Name.Builtin)
+ ],
+ 'css-single-end': [
+ include('css-base'),
+ (r"'", String.Single, '#pop:2'),
+ (r"[^;']+", Name.Entity)
+ ],
+ 'css-double-end': [
+ include('css-base'),
+ (r'"', String.Single, '#pop:2'),
+ (r'[^;"]+', Name.Entity)
+ ],
+ 'string-single-pop2': [
+ (r"'", String.Single, '#pop:2'),
+ include('string-base')
+ ],
+ 'string-double-pop2': [
+ (r'"', String.Single, '#pop:2'),
+ include('string-base')
+ ],
+ }
+
+
+class EarlGreyLexer(RegexLexer):
+ """
+ For `Earl-Grey`_ source code.
+
+ .. _Earl-Grey: https://breuleux.github.io/earl-grey/
+
+ .. versionadded: 2.1
+ """
+
+ name = 'Earl Grey'
+ aliases = ['earl-grey', 'earlgrey', 'eg']
+ filenames = ['*.eg']
+ mimetypes = ['text/x-earl-grey']
+
+ tokens = {
+ 'root': [
+ (r'\n', Text),
+ include('control'),
+ (r'[^\S\n]+', Text),
+ (r';;.*\n', Comment),
+ (r'[\[\]{}:(),;]', Punctuation),
+ (r'\\\n', Text),
+ (r'\\', Text),
+ include('errors'),
+ (words((
+ 'with', 'where', 'when', 'and', 'not', 'or', 'in',
+ 'as', 'of', 'is'),
+ prefix=r'(?<=\s|\[)', suffix=r'(?![\w$\-])'),
+ Operator.Word),
+ (r'[*@]?->', Name.Function),
+ (r'[+\-*/~^<>%&|?!@#.]*=', Operator.Word),
+ (r'\.{2,3}', Operator.Word), # Range Operator
+ (r'([+*/~^<>&|?!]+)|([#\-](?=\s))|@@+(?=\s)|=+', Operator),
+ (r'(?<![\w$\-])(var|let)(?:[^\w$])', Keyword.Declaration),
+ include('keywords'),
+ include('builtins'),
+ include('assignment'),
+ (r'''(?x)
+ (?:()([a-zA-Z$_](?:[\w$\-]*[\w$])?)|
+ (?<=[\s{\[(])(\.)([a-zA-Z$_](?:[\w$\-]*[\w$])?))
+ (?=.*%)''',
+ bygroups(Punctuation, Name.Tag, Punctuation, Name.Class.Start), 'dbs'),
+ (r'[rR]?`', String.Backtick, 'bt'),
+ (r'[rR]?```', String.Backtick, 'tbt'),
+ (r'(?<=[\s\[{(,;])\.([a-zA-Z$_](?:[\w$\-]*[\w$])?)'
+ r'(?=[\s\]}),;])', String.Symbol),
+ include('nested'),
+ (r'(?:[rR]|[rR]\.[gmi]{1,3})?"', String, combined('stringescape', 'dqs')),
+ (r'(?:[rR]|[rR]\.[gmi]{1,3})?\'', String, combined('stringescape', 'sqs')),
+ (r'"""', String, combined('stringescape', 'tdqs')),
+ include('tuple'),
+ include('import_paths'),
+ include('name'),
+ include('numbers'),
+ ],
+ 'dbs': [
+ (r'(\.)([a-zA-Z$_](?:[\w$\-]*[\w$])?)(?=[.\[\s])',
+ bygroups(Punctuation, Name.Class.DBS)),
+ (r'(\[)([\^#][a-zA-Z$_](?:[\w$\-]*[\w$])?)(\])',
+ bygroups(Punctuation, Name.Entity.DBS, Punctuation)),
+ (r'\s+', Text),
+ (r'%', Operator.DBS, '#pop'),
+ ],
+ 'import_paths': [
+ (r'(?<=[\s:;,])(\.{1,3}(?:[\w\-]*/)*)(\w(?:[\w\-]*\w)*)(?=[\s;,])',
+ bygroups(Text.Whitespace, Text)),
+ ],
+ 'assignment': [
+ (r'(\.)?([a-zA-Z$_](?:[\w$\-]*[\w$])?)'
+ r'(?=\s+[+\-*/~^<>%&|?!@#.]*\=\s)',
+ bygroups(Punctuation, Name.Variable))
+ ],
+ 'errors': [
+ (words(('Error', 'TypeError', 'ReferenceError'),
+ prefix=r'(?<![\w\-$.])', suffix=r'(?![\w\-$.])'),
+ Name.Exception),
+ (r'''(?x)
+ (?<![\w$])
+ E\.[\w$](?:[\w$\-]*[\w$])?
+ (?:\.[\w$](?:[\w$\-]*[\w$])?)*
+ (?=[({\[?!\s])''',
+ Name.Exception),
+ ],
+ 'control': [
+ (r'''(?x)
+ ([a-zA-Z$_](?:[\w$-]*[\w$])?)
+ (?!\n)\s+
+ (?!and|as|each\*|each|in|is|mod|of|or|when|where|with)
+ (?=(?:[+\-*/~^<>%&|?!@#.])?[a-zA-Z$_](?:[\w$-]*[\w$])?)''',
+ Keyword.Control),
+ (r'([a-zA-Z$_](?:[\w$-]*[\w$])?)(?!\n)\s+(?=[\'"\d{\[(])',
+ Keyword.Control),
+ (r'''(?x)
+ (?:
+ (?<=[%=])|
+ (?<=[=\-]>)|
+ (?<=with|each|with)|
+ (?<=each\*|where)
+ )(\s+)
+ ([a-zA-Z$_](?:[\w$-]*[\w$])?)(:)''',
+ bygroups(Text, Keyword.Control, Punctuation)),
+ (r'''(?x)
+ (?<![+\-*/~^<>%&|?!@#.])(\s+)
+ ([a-zA-Z$_](?:[\w$-]*[\w$])?)(:)''',
+ bygroups(Text, Keyword.Control, Punctuation)),
+ ],
+ 'nested': [
+ (r'''(?x)
+ (?<=[\w$\]})])(\.)
+ ([a-zA-Z$_](?:[\w$-]*[\w$])?)
+ (?=\s+with(?:\s|\n))''',
+ bygroups(Punctuation, Name.Function)),
+ (r'''(?x)
+ (?<!\s)(\.)
+ ([a-zA-Z$_](?:[\w$-]*[\w$])?)
+ (?=[}\]).,;:\s])''',
+ bygroups(Punctuation, Name.Field)),
+ (r'''(?x)
+ (?<=[\w$\]})])(\.)
+ ([a-zA-Z$_](?:[\w$-]*[\w$])?)
+ (?=[\[{(:])''',
+ bygroups(Punctuation, Name.Function)),
+ ],
+ 'keywords': [
+ (words((
+ 'each', 'each*', 'mod', 'await', 'break', 'chain',
+ 'continue', 'elif', 'expr-value', 'if', 'match',
+ 'return', 'yield', 'pass', 'else', 'require', 'var',
+ 'let', 'async', 'method', 'gen'),
+ prefix=r'(?<![\w\-$.])', suffix=r'(?![\w\-$.])'),
+ Keyword.Pseudo),
+ (words(('this', 'self', '@'),
+ prefix=r'(?<![\w\-$.])', suffix=r'(?![\w\-$])'),
+ Keyword.Constant),
+ (words((
+ 'Function', 'Object', 'Array', 'String', 'Number',
+ 'Boolean', 'ErrorFactory', 'ENode', 'Promise'),
+ prefix=r'(?<![\w\-$.])', suffix=r'(?![\w\-$])'),
+ Keyword.Type),
+ ],
+ 'builtins': [
+ (words((
+ 'send', 'object', 'keys', 'items', 'enumerate', 'zip',
+ 'product', 'neighbours', 'predicate', 'equal',
+ 'nequal', 'contains', 'repr', 'clone', 'range',
+ 'getChecker', 'get-checker', 'getProperty', 'get-property',
+ 'getProjector', 'get-projector', 'consume', 'take',
+ 'promisify', 'spawn', 'constructor'),
+ prefix=r'(?<![\w\-#.])', suffix=r'(?![\w\-.])'),
+ Name.Builtin),
+ (words((
+ 'true', 'false', 'null', 'undefined'),
+ prefix=r'(?<![\w\-$.])', suffix=r'(?![\w\-$.])'),
+ Name.Constant),
+ ],
+ 'name': [
+ (r'@([a-zA-Z$_](?:[\w$-]*[\w$])?)', Name.Variable.Instance),
+ (r'([a-zA-Z$_](?:[\w$-]*[\w$])?)(\+\+|\-\-)?',
+ bygroups(Name.Symbol, Operator.Word))
+ ],
+ 'tuple': [
+ (r'#[a-zA-Z_][\w\-]*(?=[\s{(,;])', Name.Namespace)
+ ],
+ 'interpoling_string': [
+ (r'\}', String.Interpol, '#pop'),
+ include('root')
+ ],
+ 'stringescape': [
+ (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
+ r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
+ ],
+ 'strings': [
+ (r'[^\\\'"]', String),
+ (r'[\'"\\]', String),
+ (r'\n', String) # All strings are multiline in EG
+ ],
+ 'dqs': [
+ (r'"', String, '#pop'),
+ (r'\\\\|\\"|\\\n', String.Escape),
+ include('strings')
+ ],
+ 'sqs': [
+ (r"'", String, '#pop'),
+ (r"\\\\|\\'|\\\n", String.Escape),
+ (r'\{', String.Interpol, 'interpoling_string'),
+ include('strings')
+ ],
+ 'tdqs': [
+ (r'"""', String, '#pop'),
+ include('strings'),
+ ],
+ 'bt': [
+ (r'`', String.Backtick, '#pop'),
+ (r'(?<!`)\n', String.Backtick),
+ (r'\^=?', String.Escape),
+ (r'.+', String.Backtick),
+ ],
+ 'tbt': [
+ (r'```', String.Backtick, '#pop'),
+ (r'\n', String.Backtick),
+ (r'\^=?', String.Escape),
+ (r'[^`]+', String.Backtick),
+ ],
+ 'numbers': [
+ (r'\d+\.(?!\.)\d*([eE][+-]?[0-9]+)?', Number.Float),
+ (r'\d+[eE][+-]?[0-9]+', Number.Float),
+ (r'8r[0-7]+', Number.Oct),
+ (r'2r[01]+', Number.Bin),
+ (r'16r[a-fA-F0-9]+', Number.Hex),
+ (r'([3-79]|[12][0-9]|3[0-6])r[a-zA-Z\d]+(\.[a-zA-Z\d]+)?', Number.Radix),
+ (r'\d+', Number.Integer)
+ ],
+ }
+
+class JuttleLexer(RegexLexer):
+ """
+ For `Juttle`_ source code.
+
+ .. _Juttle: https://github.com/juttle/juttle
+
+ """
+
+ name = 'Juttle'
+ aliases = ['juttle', 'juttle']
+ filenames = ['*.juttle']
+ mimetypes = ['application/juttle', 'application/x-juttle',
+ 'text/x-juttle', 'text/juttle']
+
+ flags = re.DOTALL | re.UNICODE | re.MULTILINE
+
+ tokens = {
+ 'commentsandwhitespace': [
+ (r'\s+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline)
+ ],
+ 'slashstartsregex': [
+ include('commentsandwhitespace'),
+ (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+ r'([gim]+\b|\B)', String.Regex, '#pop'),
+ (r'(?=/)', Text, ('#pop', 'badregex')),
+ default('#pop')
+ ],
+ 'badregex': [
+ (r'\n', Text, '#pop')
+ ],
+ 'root': [
+ (r'^(?=\s|/)', Text, 'slashstartsregex'),
+ include('commentsandwhitespace'),
+ (r':\d{2}:\d{2}:\d{2}(\.\d*)?:', String.Moment),
+ (r':(now|beginning|end|forever|yesterday|today|tomorrow|(\d+(\.\d*)?|\.\d+)(ms|[smhdwMy])?):', String.Moment),
+ (r':\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d*)?)?(Z|[+-]\d{2}:\d{2}|[+-]\d{4})?:', String.Moment),
+ (r':((\d+(\.\d*)?|\.\d+)[ ]+)?(millisecond|second|minute|hour|day|week|month|year)[s]?'
+ r'(([ ]+and[ ]+(\d+[ ]+)?(millisecond|second|minute|hour|day|week|month|year)[s]?)'
+ r'|[ ]+(ago|from[ ]+now))*:', String.Moment),
+ (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|'
+ r'(==?|!=?|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'),
+ (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
+ (r'[})\].]', Punctuation),
+ (r'(import|return|continue|if|else)\b', Keyword, 'slashstartsregex'),
+ (r'(var|const|function|reducer|sub|input)\b', Keyword.Declaration, 'slashstartsregex'),
+ (r'(batch|emit|filter|head|join|keep|pace|pass|put|read|reduce|remove|'
+ r'sequence|skip|sort|split|tail|unbatch|uniq|view|write)\b', Keyword.Reserved),
+ (r'(true|false|null|Infinity)\b', Keyword.Constant),
+ (r'(Array|Date|Juttle|Math|Number|Object|RegExp|String)\b', Name.Builtin),
+ (JS_IDENT, Name.Other),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'[0-9]+', Number.Integer),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single)
+ ]
+
+ }
diff --git a/wandb/vendor/pygments/lexers/julia.py b/wandb/vendor/pygments/lexers/julia.py
new file mode 100644
index 0000000000000000000000000000000000000000..67453aba5082e24ac6b33f11229f49864f067bba
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/julia.py
@@ -0,0 +1,333 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.julia
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the Julia language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, bygroups, do_insertions, \
+ words, include
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Generic
+from pygments.util import shebang_matches, unirange
+
+__all__ = ['JuliaLexer', 'JuliaConsoleLexer']
+
+allowed_variable = (
+ u'(?:[a-zA-Z_\u00A1-\uffff]|%s)(?:[a-zA-Z_0-9\u00A1-\uffff]|%s)*!*' %
+ ((unirange(0x10000, 0x10ffff),) * 2))
+
+
+class JuliaLexer(RegexLexer):
+ """
+ For `Julia <http://julialang.org/>`_ source code.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'Julia'
+ aliases = ['julia', 'jl']
+ filenames = ['*.jl']
+ mimetypes = ['text/x-julia', 'application/x-julia']
+
+ flags = re.MULTILINE | re.UNICODE
+
+ tokens = {
+ 'root': [
+ (r'\n', Text),
+ (r'[^\S\n]+', Text),
+ (r'#=', Comment.Multiline, "blockcomment"),
+ (r'#.*$', Comment),
+ (r'[\[\]{}(),;]', Punctuation),
+
+ # keywords
+ (r'in\b', Keyword.Pseudo),
+ (r'(true|false)\b', Keyword.Constant),
+ (r'(local|global|const)\b', Keyword.Declaration),
+ (words([
+ 'function', 'type', 'typealias', 'abstract', 'immutable',
+ 'baremodule', 'begin', 'bitstype', 'break', 'catch', 'ccall',
+ 'continue', 'do', 'else', 'elseif', 'end', 'export', 'finally',
+ 'for', 'if', 'import', 'importall', 'let', 'macro', 'module',
+ 'quote', 'return', 'try', 'using', 'while'],
+ suffix=r'\b'), Keyword),
+
+ # NOTE
+ # Patterns below work only for definition sites and thus hardly reliable.
+ #
+ # functions
+ # (r'(function)(\s+)(' + allowed_variable + ')',
+ # bygroups(Keyword, Text, Name.Function)),
+ #
+ # types
+ # (r'(type|typealias|abstract|immutable)(\s+)(' + allowed_variable + ')',
+ # bygroups(Keyword, Text, Name.Class)),
+
+ # type names
+ (words([
+ 'ANY', 'ASCIIString', 'AbstractArray', 'AbstractChannel',
+ 'AbstractFloat', 'AbstractMatrix', 'AbstractRNG',
+ 'AbstractSparseArray', 'AbstractSparseMatrix',
+ 'AbstractSparseVector', 'AbstractString', 'AbstractVecOrMat',
+ 'AbstractVector', 'Any', 'ArgumentError', 'Array',
+ 'AssertionError', 'Associative', 'Base64DecodePipe',
+ 'Base64EncodePipe', 'Bidiagonal', 'BigFloat', 'BigInt',
+ 'BitArray', 'BitMatrix', 'BitVector', 'Bool', 'BoundsError',
+ 'Box', 'BufferStream', 'CapturedException', 'CartesianIndex',
+ 'CartesianRange', 'Cchar', 'Cdouble', 'Cfloat', 'Channel',
+ 'Char', 'Cint', 'Cintmax_t', 'Clong', 'Clonglong',
+ 'ClusterManager', 'Cmd', 'Coff_t', 'Colon', 'Complex',
+ 'Complex128', 'Complex32', 'Complex64', 'CompositeException',
+ 'Condition', 'Cptrdiff_t', 'Cshort', 'Csize_t', 'Cssize_t',
+ 'Cstring', 'Cuchar', 'Cuint', 'Cuintmax_t', 'Culong',
+ 'Culonglong', 'Cushort', 'Cwchar_t', 'Cwstring', 'DataType',
+ 'Date', 'DateTime', 'DenseArray', 'DenseMatrix',
+ 'DenseVecOrMat', 'DenseVector', 'Diagonal', 'Dict',
+ 'DimensionMismatch', 'Dims', 'DirectIndexString', 'Display',
+ 'DivideError', 'DomainError', 'EOFError', 'EachLine', 'Enum',
+ 'Enumerate', 'ErrorException', 'Exception', 'Expr',
+ 'Factorization', 'FileMonitor', 'FileOffset', 'Filter',
+ 'Float16', 'Float32', 'Float64', 'FloatRange', 'Function',
+ 'GenSym', 'GlobalRef', 'GotoNode', 'HTML', 'Hermitian', 'IO',
+ 'IOBuffer', 'IOStream', 'IPv4', 'IPv6', 'InexactError',
+ 'InitError', 'Int', 'Int128', 'Int16', 'Int32', 'Int64', 'Int8',
+ 'IntSet', 'Integer', 'InterruptException', 'IntrinsicFunction',
+ 'InvalidStateException', 'Irrational', 'KeyError', 'LabelNode',
+ 'LambdaStaticData', 'LinSpace', 'LineNumberNode', 'LoadError',
+ 'LocalProcess', 'LowerTriangular', 'MIME', 'Matrix',
+ 'MersenneTwister', 'Method', 'MethodError', 'MethodTable',
+ 'Module', 'NTuple', 'NewvarNode', 'NullException', 'Nullable',
+ 'Number', 'ObjectIdDict', 'OrdinalRange', 'OutOfMemoryError',
+ 'OverflowError', 'Pair', 'ParseError', 'PartialQuickSort',
+ 'Pipe', 'PollingFileWatcher', 'ProcessExitedException',
+ 'ProcessGroup', 'Ptr', 'QuoteNode', 'RandomDevice', 'Range',
+ 'Rational', 'RawFD', 'ReadOnlyMemoryError', 'Real',
+ 'ReentrantLock', 'Ref', 'Regex', 'RegexMatch',
+ 'RemoteException', 'RemoteRef', 'RepString', 'RevString',
+ 'RopeString', 'RoundingMode', 'SegmentationFault',
+ 'SerializationState', 'Set', 'SharedArray', 'SharedMatrix',
+ 'SharedVector', 'Signed', 'SimpleVector', 'SparseMatrixCSC',
+ 'StackOverflowError', 'StatStruct', 'StepRange', 'StridedArray',
+ 'StridedMatrix', 'StridedVecOrMat', 'StridedVector', 'SubArray',
+ 'SubString', 'SymTridiagonal', 'Symbol', 'SymbolNode',
+ 'Symmetric', 'SystemError', 'TCPSocket', 'Task', 'Text',
+ 'TextDisplay', 'Timer', 'TopNode', 'Tridiagonal', 'Tuple',
+ 'Type', 'TypeConstructor', 'TypeError', 'TypeName', 'TypeVar',
+ 'UDPSocket', 'UInt', 'UInt128', 'UInt16', 'UInt32', 'UInt64',
+ 'UInt8', 'UTF16String', 'UTF32String', 'UTF8String',
+ 'UndefRefError', 'UndefVarError', 'UnicodeError', 'UniformScaling',
+ 'Union', 'UnitRange', 'Unsigned', 'UpperTriangular', 'Val',
+ 'Vararg', 'VecOrMat', 'Vector', 'VersionNumber', 'Void', 'WString',
+ 'WeakKeyDict', 'WeakRef', 'WorkerConfig', 'Zip'], suffix=r'\b'),
+ Keyword.Type),
+
+ # builtins
+ (words([
+ u'ARGS', u'CPU_CORES', u'C_NULL', u'DevNull', u'ENDIAN_BOM',
+ u'ENV', u'I', u'Inf', u'Inf16', u'Inf32', u'Inf64',
+ u'InsertionSort', u'JULIA_HOME', u'LOAD_PATH', u'MergeSort',
+ u'NaN', u'NaN16', u'NaN32', u'NaN64', u'OS_NAME',
+ u'QuickSort', u'RoundDown', u'RoundFromZero', u'RoundNearest',
+ u'RoundNearestTiesAway', u'RoundNearestTiesUp',
+ u'RoundToZero', u'RoundUp', u'STDERR', u'STDIN', u'STDOUT',
+ u'VERSION', u'WORD_SIZE', u'catalan', u'e', u'eu',
+ u'eulergamma', u'golden', u'im', u'nothing', u'pi', u'γ',
+ u'π', u'φ'],
+ suffix=r'\b'), Name.Builtin),
+
+ # operators
+ # see: https://github.com/JuliaLang/julia/blob/master/src/julia-parser.scm
+ (words([
+ # prec-assignment
+ u'=', u':=', u'+=', u'-=', u'*=', u'/=', u'//=', u'.//=', u'.*=', u'./=',
+ u'\=', u'.\=', u'^=', u'.^=', u'÷=', u'.÷=', u'%=', u'.%=', u'|=', u'&=',
+ u'$=', u'=>', u'<<=', u'>>=', u'>>>=', u'~', u'.+=', u'.-=',
+ # prec-conditional
+ u'?',
+ # prec-arrow
+ u'--', u'-->',
+ # prec-lazy-or
+ u'||',
+ # prec-lazy-and
+ u'&&',
+ # prec-comparison
+ u'>', u'<', u'>=', u'≥', u'<=', u'≤', u'==', u'===', u'≡', u'!=', u'≠',
+ u'!==', u'≢', u'.>', u'.<', u'.>=', u'.≥', u'.<=', u'.≤', u'.==', u'.!=',
+ u'.≠', u'.=', u'.!', u'<:', u'>:', u'∈', u'∉', u'∋', u'∌', u'⊆',
+ u'⊈', u'⊂',
+ u'⊄', u'⊊',
+ # prec-pipe
+ u'|>', u'<|',
+ # prec-colon
+ u':',
+ # prec-plus
+ u'+', u'-', u'.+', u'.-', u'|', u'∪', u'$',
+ # prec-bitshift
+ u'<<', u'>>', u'>>>', u'.<<', u'.>>', u'.>>>',
+ # prec-times
+ u'*', u'/', u'./', u'÷', u'.÷', u'%', u'⋅', u'.%', u'.*', u'\\', u'.\\', u'&', u'∩',
+ # prec-rational
+ u'//', u'.//',
+ # prec-power
+ u'^', u'.^',
+ # prec-decl
+ u'::',
+ # prec-dot
+ u'.',
+ # unary op
+ u'+', u'-', u'!', u'~', u'√', u'∛', u'∜'
+ ]), Operator),
+
+ # chars
+ (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,3}|\\u[a-fA-F0-9]{1,4}|"
+ r"\\U[a-fA-F0-9]{1,6}|[^\\\'\n])'", String.Char),
+
+ # try to match trailing transpose
+ (r'(?<=[.\w)\]])\'+', Operator),
+
+ # strings
+ (r'"""', String, 'tqstring'),
+ (r'"', String, 'string'),
+
+ # regular expressions
+ (r'r"""', String.Regex, 'tqregex'),
+ (r'r"', String.Regex, 'regex'),
+
+ # backticks
+ (r'`', String.Backtick, 'command'),
+
+ # names
+ (allowed_variable, Name),
+ (r'@' + allowed_variable, Name.Decorator),
+
+ # numbers
+ (r'(\d+(_\d+)+\.\d*|\d*\.\d+(_\d+)+)([eEf][+-]?[0-9]+)?', Number.Float),
+ (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float),
+ (r'\d+(_\d+)+[eEf][+-]?[0-9]+', Number.Float),
+ (r'\d+[eEf][+-]?[0-9]+', Number.Float),
+ (r'0b[01]+(_[01]+)+', Number.Bin),
+ (r'0b[01]+', Number.Bin),
+ (r'0o[0-7]+(_[0-7]+)+', Number.Oct),
+ (r'0o[0-7]+', Number.Oct),
+ (r'0x[a-fA-F0-9]+(_[a-fA-F0-9]+)+', Number.Hex),
+ (r'0x[a-fA-F0-9]+', Number.Hex),
+ (r'\d+(_\d+)+', Number.Integer),
+ (r'\d+', Number.Integer)
+ ],
+
+ "blockcomment": [
+ (r'[^=#]', Comment.Multiline),
+ (r'#=', Comment.Multiline, '#push'),
+ (r'=#', Comment.Multiline, '#pop'),
+ (r'[=#]', Comment.Multiline),
+ ],
+
+ 'string': [
+ (r'"', String, '#pop'),
+ # FIXME: This escape pattern is not perfect.
+ (r'\\([\\"\'$nrbtfav]|(x|u|U)[a-fA-F0-9]+|\d+)', String.Escape),
+ # Interpolation is defined as "$" followed by the shortest full
+ # expression, which is something we can't parse.
+ # Include the most common cases here: $word, and $(paren'd expr).
+ (r'\$' + allowed_variable, String.Interpol),
+ # (r'\$[a-zA-Z_]+', String.Interpol),
+ (r'(\$)(\()', bygroups(String.Interpol, Punctuation), 'in-intp'),
+ # @printf and @sprintf formats
+ (r'%[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?[hlL]?[E-GXc-giorsux%]',
+ String.Interpol),
+ (r'.|\s', String),
+ ],
+
+ 'tqstring': [
+ (r'"""', String, '#pop'),
+ (r'\\([\\"\'$nrbtfav]|(x|u|U)[a-fA-F0-9]+|\d+)', String.Escape),
+ (r'\$' + allowed_variable, String.Interpol),
+ (r'(\$)(\()', bygroups(String.Interpol, Punctuation), 'in-intp'),
+ (r'.|\s', String),
+ ],
+
+ 'regex': [
+ (r'"', String.Regex, '#pop'),
+ (r'\\"', String.Regex),
+ (r'.|\s', String.Regex),
+ ],
+
+ 'tqregex': [
+ (r'"""', String.Regex, '#pop'),
+ (r'.|\s', String.Regex),
+ ],
+
+ 'command': [
+ (r'`', String.Backtick, '#pop'),
+ (r'\$' + allowed_variable, String.Interpol),
+ (r'(\$)(\()', bygroups(String.Interpol, Punctuation), 'in-intp'),
+ (r'.|\s', String.Backtick)
+ ],
+
+ 'in-intp': [
+ (r'\(', Punctuation, '#push'),
+ (r'\)', Punctuation, '#pop'),
+ include('root'),
+ ]
+ }
+
+ def analyse_text(text):
+ return shebang_matches(text, r'julia')
+
+
+class JuliaConsoleLexer(Lexer):
+ """
+ For Julia console sessions. Modeled after MatlabSessionLexer.
+
+ .. versionadded:: 1.6
+ """
+ name = 'Julia console'
+ aliases = ['jlcon']
+
+ def get_tokens_unprocessed(self, text):
+ jllexer = JuliaLexer(**self.options)
+ start = 0
+ curcode = ''
+ insertions = []
+ output = False
+ error = False
+
+ for line in text.splitlines(True):
+ if line.startswith('julia>'):
+ insertions.append((len(curcode), [(0, Generic.Prompt, line[:6])]))
+ curcode += line[6:]
+ output = False
+ error = False
+ elif line.startswith('help?>') or line.startswith('shell>'):
+ yield start, Generic.Prompt, line[:6]
+ yield start + 6, Text, line[6:]
+ output = False
+ error = False
+ elif line.startswith(' ') and not output:
+ insertions.append((len(curcode), [(0, Text, line[:6])]))
+ curcode += line[6:]
+ else:
+ if curcode:
+ for item in do_insertions(
+ insertions, jllexer.get_tokens_unprocessed(curcode)):
+ yield item
+ curcode = ''
+ insertions = []
+ if line.startswith('ERROR: ') or error:
+ yield start, Generic.Error, line
+ error = True
+ else:
+ yield start, Generic.Output, line
+ output = True
+ start += len(line)
+
+ if curcode:
+ for item in do_insertions(
+ insertions, jllexer.get_tokens_unprocessed(curcode)):
+ yield item
diff --git a/wandb/vendor/pygments/lexers/jvm.py b/wandb/vendor/pygments/lexers/jvm.py
new file mode 100644
index 0000000000000000000000000000000000000000..f4392839ea7f4251e0851b31553d6106daeae40e
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/jvm.py
@@ -0,0 +1,1573 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.jvm
+ ~~~~~~~~~~~~~~~~~~~
+
+ Pygments lexers for JVM languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \
+ this, combined, default, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+from pygments.util import shebang_matches
+from pygments import unistring as uni
+
+__all__ = ['JavaLexer', 'ScalaLexer', 'GosuLexer', 'GosuTemplateLexer',
+ 'GroovyLexer', 'IokeLexer', 'ClojureLexer', 'ClojureScriptLexer',
+ 'KotlinLexer', 'XtendLexer', 'AspectJLexer', 'CeylonLexer',
+ 'PigLexer', 'GoloLexer', 'JasminLexer']
+
+
+class JavaLexer(RegexLexer):
+ """
+ For `Java <http://www.sun.com/java/>`_ source code.
+ """
+
+ name = 'Java'
+ aliases = ['java']
+ filenames = ['*.java']
+ mimetypes = ['text/x-java']
+
+ flags = re.MULTILINE | re.DOTALL | re.UNICODE
+
+ tokens = {
+ 'root': [
+ (r'[^\S\n]+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ # keywords: go before method names to avoid lexing "throw new XYZ"
+ # as a method signature
+ (r'(assert|break|case|catch|continue|default|do|else|finally|for|'
+ r'if|goto|instanceof|new|return|switch|this|throw|try|while)\b',
+ Keyword),
+ # method names
+ (r'((?:(?:[^\W\d]|\$)[\w.\[\]$<>]*\s+)+?)' # return arguments
+ r'((?:[^\W\d]|\$)[\w$]*)' # method name
+ r'(\s*)(\()', # signature start
+ bygroups(using(this), Name.Function, Text, Operator)),
+ (r'@[^\W\d][\w.]*', Name.Decorator),
+ (r'(abstract|const|enum|extends|final|implements|native|private|'
+ r'protected|public|static|strictfp|super|synchronized|throws|'
+ r'transient|volatile)\b', Keyword.Declaration),
+ (r'(boolean|byte|char|double|float|int|long|short|void)\b',
+ Keyword.Type),
+ (r'(package)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
+ (r'(true|false|null)\b', Keyword.Constant),
+ (r'(class|interface)(\s+)', bygroups(Keyword.Declaration, Text),
+ 'class'),
+ (r'(import(?:\s+static)?)(\s+)', bygroups(Keyword.Namespace, Text),
+ 'import'),
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Char),
+ (r'(\.)((?:[^\W\d]|\$)[\w$]*)', bygroups(Operator, Name.Attribute)),
+ (r'^\s*([^\W\d]|\$)[\w$]*:', Name.Label),
+ (r'([^\W\d]|\$)[\w$]*', Name),
+ (r'([0-9][0-9_]*\.([0-9][0-9_]*)?|'
+ r'\.[0-9][0-9_]*)'
+ r'([eE][+\-]?[0-9][0-9_]*)?[fFdD]?|'
+ r'[0-9][eE][+\-]?[0-9][0-9_]*[fFdD]?|'
+ r'[0-9]([eE][+\-]?[0-9][0-9_]*)?[fFdD]|'
+ r'0[xX]([0-9a-fA-F][0-9a-fA-F_]*\.?|'
+ r'([0-9a-fA-F][0-9a-fA-F_]*)?\.[0-9a-fA-F][0-9a-fA-F_]*)'
+ r'[pP][+\-]?[0-9][0-9_]*[fFdD]?', Number.Float),
+ (r'0[xX][0-9a-fA-F][0-9a-fA-F_]*[lL]?', Number.Hex),
+ (r'0[bB][01][01_]*[lL]?', Number.Bin),
+ (r'0[0-7_]+[lL]?', Number.Oct),
+ (r'0|[1-9][0-9_]*[lL]?', Number.Integer),
+ (r'[~^*!%&\[\](){}<>|+=:;,./?-]', Operator),
+ (r'\n', Text)
+ ],
+ 'class': [
+ (r'([^\W\d]|\$)[\w$]*', Name.Class, '#pop')
+ ],
+ 'import': [
+ (r'[\w.]+\*?', Name.Namespace, '#pop')
+ ],
+ }
+
+
+class AspectJLexer(JavaLexer):
+ """
+ For `AspectJ <http://www.eclipse.org/aspectj/>`_ source code.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'AspectJ'
+ aliases = ['aspectj']
+ filenames = ['*.aj']
+ mimetypes = ['text/x-aspectj']
+
+ aj_keywords = set((
+ 'aspect', 'pointcut', 'privileged', 'call', 'execution',
+ 'initialization', 'preinitialization', 'handler', 'get', 'set',
+ 'staticinitialization', 'target', 'args', 'within', 'withincode',
+ 'cflow', 'cflowbelow', 'annotation', 'before', 'after', 'around',
+ 'proceed', 'throwing', 'returning', 'adviceexecution', 'declare',
+ 'parents', 'warning', 'error', 'soft', 'precedence', 'thisJoinPoint',
+ 'thisJoinPointStaticPart', 'thisEnclosingJoinPointStaticPart',
+ 'issingleton', 'perthis', 'pertarget', 'percflow', 'percflowbelow',
+ 'pertypewithin', 'lock', 'unlock', 'thisAspectInstance'
+ ))
+ aj_inter_type = set(('parents:', 'warning:', 'error:', 'soft:', 'precedence:'))
+ aj_inter_type_annotation = set(('@type', '@method', '@constructor', '@field'))
+
+ def get_tokens_unprocessed(self, text):
+ for index, token, value in JavaLexer.get_tokens_unprocessed(self, text):
+ if token is Name and value in self.aj_keywords:
+ yield index, Keyword, value
+ elif token is Name.Label and value in self.aj_inter_type:
+ yield index, Keyword, value[:-1]
+ yield index, Operator, value[-1]
+ elif token is Name.Decorator and value in self.aj_inter_type_annotation:
+ yield index, Keyword, value
+ else:
+ yield index, token, value
+
+
+class ScalaLexer(RegexLexer):
+ """
+ For `Scala <http://www.scala-lang.org>`_ source code.
+ """
+
+ name = 'Scala'
+ aliases = ['scala']
+ filenames = ['*.scala']
+ mimetypes = ['text/x-scala']
+
+ flags = re.MULTILINE | re.DOTALL
+
+ # don't use raw unicode strings!
+ op = (u'[-~\\^\\*!%&\\\\<>\\|+=:/?@\u00a6-\u00a7\u00a9\u00ac\u00ae\u00b0-\u00b1'
+ u'\u00b6\u00d7\u00f7\u03f6\u0482\u0606-\u0608\u060e-\u060f\u06e9'
+ u'\u06fd-\u06fe\u07f6\u09fa\u0b70\u0bf3-\u0bf8\u0bfa\u0c7f\u0cf1-\u0cf2'
+ u'\u0d79\u0f01-\u0f03\u0f13-\u0f17\u0f1a-\u0f1f\u0f34\u0f36\u0f38'
+ u'\u0fbe-\u0fc5\u0fc7-\u0fcf\u109e-\u109f\u1360\u1390-\u1399\u1940'
+ u'\u19e0-\u19ff\u1b61-\u1b6a\u1b74-\u1b7c\u2044\u2052\u207a-\u207c'
+ u'\u208a-\u208c\u2100-\u2101\u2103-\u2106\u2108-\u2109\u2114\u2116-\u2118'
+ u'\u211e-\u2123\u2125\u2127\u2129\u212e\u213a-\u213b\u2140-\u2144'
+ u'\u214a-\u214d\u214f\u2190-\u2328\u232b-\u244a\u249c-\u24e9\u2500-\u2767'
+ u'\u2794-\u27c4\u27c7-\u27e5\u27f0-\u2982\u2999-\u29d7\u29dc-\u29fb'
+ u'\u29fe-\u2b54\u2ce5-\u2cea\u2e80-\u2ffb\u3004\u3012-\u3013\u3020'
+ u'\u3036-\u3037\u303e-\u303f\u3190-\u3191\u3196-\u319f\u31c0-\u31e3'
+ u'\u3200-\u321e\u322a-\u3250\u3260-\u327f\u328a-\u32b0\u32c0-\u33ff'
+ u'\u4dc0-\u4dff\ua490-\ua4c6\ua828-\ua82b\ufb29\ufdfd\ufe62\ufe64-\ufe66'
+ u'\uff0b\uff1c-\uff1e\uff5c\uff5e\uffe2\uffe4\uffe8-\uffee\ufffc-\ufffd]+')
+
+ letter = (u'[a-zA-Z\\$_\u00aa\u00b5\u00ba\u00c0-\u00d6\u00d8-\u00f6'
+ u'\u00f8-\u02af\u0370-\u0373\u0376-\u0377\u037b-\u037d\u0386'
+ u'\u0388-\u03f5\u03f7-\u0481\u048a-\u0556\u0561-\u0587\u05d0-\u05f2'
+ u'\u0621-\u063f\u0641-\u064a\u066e-\u066f\u0671-\u06d3\u06d5'
+ u'\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5'
+ u'\u07b1\u07ca-\u07ea\u0904-\u0939\u093d\u0950\u0958-\u0961'
+ u'\u0972-\u097f\u0985-\u09b9\u09bd\u09ce\u09dc-\u09e1\u09f0-\u09f1'
+ u'\u0a05-\u0a39\u0a59-\u0a5e\u0a72-\u0a74\u0a85-\u0ab9\u0abd'
+ u'\u0ad0-\u0ae1\u0b05-\u0b39\u0b3d\u0b5c-\u0b61\u0b71\u0b83-\u0bb9'
+ u'\u0bd0\u0c05-\u0c3d\u0c58-\u0c61\u0c85-\u0cb9\u0cbd\u0cde-\u0ce1'
+ u'\u0d05-\u0d3d\u0d60-\u0d61\u0d7a-\u0d7f\u0d85-\u0dc6\u0e01-\u0e30'
+ u'\u0e32-\u0e33\u0e40-\u0e45\u0e81-\u0eb0\u0eb2-\u0eb3\u0ebd-\u0ec4'
+ u'\u0edc-\u0f00\u0f40-\u0f6c\u0f88-\u0f8b\u1000-\u102a\u103f'
+ u'\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070'
+ u'\u1075-\u1081\u108e\u10a0-\u10fa\u1100-\u135a\u1380-\u138f'
+ u'\u13a0-\u166c\u166f-\u1676\u1681-\u169a\u16a0-\u16ea\u16ee-\u1711'
+ u'\u1720-\u1731\u1740-\u1751\u1760-\u1770\u1780-\u17b3\u17dc'
+ u'\u1820-\u1842\u1844-\u18a8\u18aa-\u191c\u1950-\u19a9\u19c1-\u19c7'
+ u'\u1a00-\u1a16\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf'
+ u'\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c77\u1d00-\u1d2b\u1d62-\u1d77'
+ u'\u1d79-\u1d9a\u1e00-\u1fbc\u1fbe\u1fc2-\u1fcc\u1fd0-\u1fdb'
+ u'\u1fe0-\u1fec\u1ff2-\u1ffc\u2071\u207f\u2102\u2107\u210a-\u2113'
+ u'\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u212f-\u2139'
+ u'\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c7c'
+ u'\u2c80-\u2ce4\u2d00-\u2d65\u2d80-\u2dde\u3006-\u3007\u3021-\u3029'
+ u'\u3038-\u303a\u303c\u3041-\u3096\u309f\u30a1-\u30fa\u30ff-\u318e'
+ u'\u31a0-\u31b7\u31f0-\u31ff\u3400-\u4db5\u4e00-\ua014\ua016-\ua48c'
+ u'\ua500-\ua60b\ua610-\ua61f\ua62a-\ua66e\ua680-\ua697\ua722-\ua76f'
+ u'\ua771-\ua787\ua78b-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822'
+ u'\ua840-\ua873\ua882-\ua8b3\ua90a-\ua925\ua930-\ua946\uaa00-\uaa28'
+ u'\uaa40-\uaa42\uaa44-\uaa4b\uac00-\ud7a3\uf900-\ufb1d\ufb1f-\ufb28'
+ u'\ufb2a-\ufd3d\ufd50-\ufdfb\ufe70-\ufefc\uff21-\uff3a\uff41-\uff5a'
+ u'\uff66-\uff6f\uff71-\uff9d\uffa0-\uffdc]')
+
+ upper = (u'[A-Z\\$_\u00c0-\u00d6\u00d8-\u00de\u0100\u0102\u0104\u0106\u0108'
+ u'\u010a\u010c\u010e\u0110\u0112\u0114\u0116\u0118\u011a\u011c'
+ u'\u011e\u0120\u0122\u0124\u0126\u0128\u012a\u012c\u012e\u0130'
+ u'\u0132\u0134\u0136\u0139\u013b\u013d\u013f\u0141\u0143\u0145'
+ u'\u0147\u014a\u014c\u014e\u0150\u0152\u0154\u0156\u0158\u015a'
+ u'\u015c\u015e\u0160\u0162\u0164\u0166\u0168\u016a\u016c\u016e'
+ u'\u0170\u0172\u0174\u0176\u0178-\u0179\u017b\u017d\u0181-\u0182'
+ u'\u0184\u0186-\u0187\u0189-\u018b\u018e-\u0191\u0193-\u0194'
+ u'\u0196-\u0198\u019c-\u019d\u019f-\u01a0\u01a2\u01a4\u01a6-\u01a7'
+ u'\u01a9\u01ac\u01ae-\u01af\u01b1-\u01b3\u01b5\u01b7-\u01b8\u01bc'
+ u'\u01c4\u01c7\u01ca\u01cd\u01cf\u01d1\u01d3\u01d5\u01d7\u01d9'
+ u'\u01db\u01de\u01e0\u01e2\u01e4\u01e6\u01e8\u01ea\u01ec\u01ee'
+ u'\u01f1\u01f4\u01f6-\u01f8\u01fa\u01fc\u01fe\u0200\u0202\u0204'
+ u'\u0206\u0208\u020a\u020c\u020e\u0210\u0212\u0214\u0216\u0218'
+ u'\u021a\u021c\u021e\u0220\u0222\u0224\u0226\u0228\u022a\u022c'
+ u'\u022e\u0230\u0232\u023a-\u023b\u023d-\u023e\u0241\u0243-\u0246'
+ u'\u0248\u024a\u024c\u024e\u0370\u0372\u0376\u0386\u0388-\u038f'
+ u'\u0391-\u03ab\u03cf\u03d2-\u03d4\u03d8\u03da\u03dc\u03de\u03e0'
+ u'\u03e2\u03e4\u03e6\u03e8\u03ea\u03ec\u03ee\u03f4\u03f7'
+ u'\u03f9-\u03fa\u03fd-\u042f\u0460\u0462\u0464\u0466\u0468\u046a'
+ u'\u046c\u046e\u0470\u0472\u0474\u0476\u0478\u047a\u047c\u047e'
+ u'\u0480\u048a\u048c\u048e\u0490\u0492\u0494\u0496\u0498\u049a'
+ u'\u049c\u049e\u04a0\u04a2\u04a4\u04a6\u04a8\u04aa\u04ac\u04ae'
+ u'\u04b0\u04b2\u04b4\u04b6\u04b8\u04ba\u04bc\u04be\u04c0-\u04c1'
+ u'\u04c3\u04c5\u04c7\u04c9\u04cb\u04cd\u04d0\u04d2\u04d4\u04d6'
+ u'\u04d8\u04da\u04dc\u04de\u04e0\u04e2\u04e4\u04e6\u04e8\u04ea'
+ u'\u04ec\u04ee\u04f0\u04f2\u04f4\u04f6\u04f8\u04fa\u04fc\u04fe'
+ u'\u0500\u0502\u0504\u0506\u0508\u050a\u050c\u050e\u0510\u0512'
+ u'\u0514\u0516\u0518\u051a\u051c\u051e\u0520\u0522\u0531-\u0556'
+ u'\u10a0-\u10c5\u1e00\u1e02\u1e04\u1e06\u1e08\u1e0a\u1e0c\u1e0e'
+ u'\u1e10\u1e12\u1e14\u1e16\u1e18\u1e1a\u1e1c\u1e1e\u1e20\u1e22'
+ u'\u1e24\u1e26\u1e28\u1e2a\u1e2c\u1e2e\u1e30\u1e32\u1e34\u1e36'
+ u'\u1e38\u1e3a\u1e3c\u1e3e\u1e40\u1e42\u1e44\u1e46\u1e48\u1e4a'
+ u'\u1e4c\u1e4e\u1e50\u1e52\u1e54\u1e56\u1e58\u1e5a\u1e5c\u1e5e'
+ u'\u1e60\u1e62\u1e64\u1e66\u1e68\u1e6a\u1e6c\u1e6e\u1e70\u1e72'
+ u'\u1e74\u1e76\u1e78\u1e7a\u1e7c\u1e7e\u1e80\u1e82\u1e84\u1e86'
+ u'\u1e88\u1e8a\u1e8c\u1e8e\u1e90\u1e92\u1e94\u1e9e\u1ea0\u1ea2'
+ u'\u1ea4\u1ea6\u1ea8\u1eaa\u1eac\u1eae\u1eb0\u1eb2\u1eb4\u1eb6'
+ u'\u1eb8\u1eba\u1ebc\u1ebe\u1ec0\u1ec2\u1ec4\u1ec6\u1ec8\u1eca'
+ u'\u1ecc\u1ece\u1ed0\u1ed2\u1ed4\u1ed6\u1ed8\u1eda\u1edc\u1ede'
+ u'\u1ee0\u1ee2\u1ee4\u1ee6\u1ee8\u1eea\u1eec\u1eee\u1ef0\u1ef2'
+ u'\u1ef4\u1ef6\u1ef8\u1efa\u1efc\u1efe\u1f08-\u1f0f\u1f18-\u1f1d'
+ u'\u1f28-\u1f2f\u1f38-\u1f3f\u1f48-\u1f4d\u1f59-\u1f5f'
+ u'\u1f68-\u1f6f\u1fb8-\u1fbb\u1fc8-\u1fcb\u1fd8-\u1fdb'
+ u'\u1fe8-\u1fec\u1ff8-\u1ffb\u2102\u2107\u210b-\u210d\u2110-\u2112'
+ u'\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u2130-\u2133'
+ u'\u213e-\u213f\u2145\u2183\u2c00-\u2c2e\u2c60\u2c62-\u2c64\u2c67'
+ u'\u2c69\u2c6b\u2c6d-\u2c6f\u2c72\u2c75\u2c80\u2c82\u2c84\u2c86'
+ u'\u2c88\u2c8a\u2c8c\u2c8e\u2c90\u2c92\u2c94\u2c96\u2c98\u2c9a'
+ u'\u2c9c\u2c9e\u2ca0\u2ca2\u2ca4\u2ca6\u2ca8\u2caa\u2cac\u2cae'
+ u'\u2cb0\u2cb2\u2cb4\u2cb6\u2cb8\u2cba\u2cbc\u2cbe\u2cc0\u2cc2'
+ u'\u2cc4\u2cc6\u2cc8\u2cca\u2ccc\u2cce\u2cd0\u2cd2\u2cd4\u2cd6'
+ u'\u2cd8\u2cda\u2cdc\u2cde\u2ce0\u2ce2\ua640\ua642\ua644\ua646'
+ u'\ua648\ua64a\ua64c\ua64e\ua650\ua652\ua654\ua656\ua658\ua65a'
+ u'\ua65c\ua65e\ua662\ua664\ua666\ua668\ua66a\ua66c\ua680\ua682'
+ u'\ua684\ua686\ua688\ua68a\ua68c\ua68e\ua690\ua692\ua694\ua696'
+ u'\ua722\ua724\ua726\ua728\ua72a\ua72c\ua72e\ua732\ua734\ua736'
+ u'\ua738\ua73a\ua73c\ua73e\ua740\ua742\ua744\ua746\ua748\ua74a'
+ u'\ua74c\ua74e\ua750\ua752\ua754\ua756\ua758\ua75a\ua75c\ua75e'
+ u'\ua760\ua762\ua764\ua766\ua768\ua76a\ua76c\ua76e\ua779\ua77b'
+ u'\ua77d-\ua77e\ua780\ua782\ua784\ua786\ua78b\uff21-\uff3a]')
+
+ idrest = u'%s(?:%s|[0-9])*(?:(?<=_)%s)?' % (letter, letter, op)
+ letter_letter_digit = u'%s(?:%s|\d)*' % (letter, letter)
+
+ tokens = {
+ 'root': [
+ # method names
+ (r'(class|trait|object)(\s+)', bygroups(Keyword, Text), 'class'),
+ (r'[^\S\n]+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*', Comment.Multiline, 'comment'),
+ (u'@%s' % idrest, Name.Decorator),
+ (u'(abstract|ca(?:se|tch)|d(?:ef|o)|e(?:lse|xtends)|'
+ u'f(?:inal(?:ly)?|or(?:Some)?)|i(?:f|mplicit)|'
+ u'lazy|match|new|override|pr(?:ivate|otected)'
+ u'|re(?:quires|turn)|s(?:ealed|uper)|'
+ u't(?:h(?:is|row)|ry)|va[lr]|w(?:hile|ith)|yield)\\b|'
+ u'(<[%:-]|=>|>:|[#=@_\u21D2\u2190])(\\b|(?=\\s)|$)', Keyword),
+ (u':(?!%s)' % op, Keyword, 'type'),
+ (u'%s%s\\b' % (upper, idrest), Name.Class),
+ (r'(true|false|null)\b', Keyword.Constant),
+ (r'(import|package)(\s+)', bygroups(Keyword, Text), 'import'),
+ (r'(type)(\s+)', bygroups(Keyword, Text), 'type'),
+ (r'""".*?"""(?!")', String),
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Char),
+ (u"'%s" % idrest, Text.Symbol),
+ (r'[fs]"""', String, 'interptriplestring'), # interpolated strings
+ (r'[fs]"', String, 'interpstring'), # interpolated strings
+ (r'raw"(\\\\|\\"|[^"])*"', String), # raw strings
+ # (ur'(\.)(%s|%s|`[^`]+`)' % (idrest, op), bygroups(Operator,
+ # Name.Attribute)),
+ (idrest, Name),
+ (r'`[^`]+`', Name),
+ (r'\[', Operator, 'typeparam'),
+ (r'[(){};,.#]', Operator),
+ (op, Operator),
+ (r'([0-9][0-9]*\.[0-9]*|\.[0-9]+)([eE][+-]?[0-9]+)?[fFdD]?',
+ Number.Float),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+L?', Number.Integer),
+ (r'\n', Text)
+ ],
+ 'class': [
+ (u'(%s|%s|`[^`]+`)(\\s*)(\\[)' % (idrest, op),
+ bygroups(Name.Class, Text, Operator), 'typeparam'),
+ (r'\s+', Text),
+ (r'\{', Operator, '#pop'),
+ (r'\(', Operator, '#pop'),
+ (r'//.*?\n', Comment.Single, '#pop'),
+ (u'%s|%s|`[^`]+`' % (idrest, op), Name.Class, '#pop'),
+ ],
+ 'type': [
+ (r'\s+', Text),
+ (r'<[%:]|>:|[#_]|forSome|type', Keyword),
+ (u'([,);}]|=>|=|\u21d2)(\\s*)', bygroups(Operator, Text), '#pop'),
+ (r'[({]', Operator, '#push'),
+ (u'((?:%s|%s|`[^`]+`)(?:\\.(?:%s|%s|`[^`]+`))*)(\\s*)(\\[)' %
+ (idrest, op, idrest, op),
+ bygroups(Keyword.Type, Text, Operator), ('#pop', 'typeparam')),
+ (u'((?:%s|%s|`[^`]+`)(?:\\.(?:%s|%s|`[^`]+`))*)(\\s*)$' %
+ (idrest, op, idrest, op),
+ bygroups(Keyword.Type, Text), '#pop'),
+ (r'//.*?\n', Comment.Single, '#pop'),
+ (u'\\.|%s|%s|`[^`]+`' % (idrest, op), Keyword.Type)
+ ],
+ 'typeparam': [
+ (r'[\s,]+', Text),
+ (u'<[%:]|=>|>:|[#_\u21D2]|forSome|type', Keyword),
+ (r'([\])}])', Operator, '#pop'),
+ (r'[(\[{]', Operator, '#push'),
+ (u'\\.|%s|%s|`[^`]+`' % (idrest, op), Keyword.Type)
+ ],
+ 'comment': [
+ (r'[^/*]+', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline)
+ ],
+ 'import': [
+ (u'(%s|\\.)+' % idrest, Name.Namespace, '#pop')
+ ],
+ 'interpstringcommon': [
+ (r'[^"$\\]+', String),
+ (r'\$\$', String),
+ (r'\$' + letter_letter_digit, String.Interpol),
+ (r'\$\{', String.Interpol, 'interpbrace'),
+ (r'\\.', String),
+ ],
+ 'interptriplestring': [
+ (r'"""(?!")', String, '#pop'),
+ (r'"', String),
+ include('interpstringcommon'),
+ ],
+ 'interpstring': [
+ (r'"', String, '#pop'),
+ include('interpstringcommon'),
+ ],
+ 'interpbrace': [
+ (r'\}', String.Interpol, '#pop'),
+ (r'\{', String.Interpol, '#push'),
+ include('root'),
+ ],
+ }
+
+
+class GosuLexer(RegexLexer):
+ """
+ For Gosu source code.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'Gosu'
+ aliases = ['gosu']
+ filenames = ['*.gs', '*.gsx', '*.gsp', '*.vark']
+ mimetypes = ['text/x-gosu']
+
+ flags = re.MULTILINE | re.DOTALL
+
+ tokens = {
+ 'root': [
+ # method names
+ (r'^(\s*(?:[a-zA-Z_][\w.\[\]]*\s+)+?)' # modifiers etc.
+ r'([a-zA-Z_]\w*)' # method name
+ r'(\s*)(\()', # signature start
+ bygroups(using(this), Name.Function, Text, Operator)),
+ (r'[^\S\n]+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'@[a-zA-Z_][\w.]*', Name.Decorator),
+ (r'(in|as|typeof|statictypeof|typeis|typeas|if|else|foreach|for|'
+ r'index|while|do|continue|break|return|try|catch|finally|this|'
+ r'throw|new|switch|case|default|eval|super|outer|classpath|'
+ r'using)\b', Keyword),
+ (r'(var|delegate|construct|function|private|internal|protected|'
+ r'public|abstract|override|final|static|extends|transient|'
+ r'implements|represents|readonly)\b', Keyword.Declaration),
+ (r'(property\s+)(get|set)?', Keyword.Declaration),
+ (r'(boolean|byte|char|double|float|int|long|short|void|block)\b',
+ Keyword.Type),
+ (r'(package)(\s+)', bygroups(Keyword.Namespace, Text)),
+ (r'(true|false|null|NaN|Infinity)\b', Keyword.Constant),
+ (r'(class|interface|enhancement|enum)(\s+)([a-zA-Z_]\w*)',
+ bygroups(Keyword.Declaration, Text, Name.Class)),
+ (r'(uses)(\s+)([\w.]+\*?)',
+ bygroups(Keyword.Namespace, Text, Name.Namespace)),
+ (r'"', String, 'string'),
+ (r'(\??[.#])([a-zA-Z_]\w*)',
+ bygroups(Operator, Name.Attribute)),
+ (r'(:)([a-zA-Z_]\w*)',
+ bygroups(Operator, Name.Attribute)),
+ (r'[a-zA-Z_$]\w*', Name),
+ (r'and|or|not|[\\~^*!%&\[\](){}<>|+=:;,./?-]', Operator),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'[0-9]+', Number.Integer),
+ (r'\n', Text)
+ ],
+ 'templateText': [
+ (r'(\\<)|(\\\$)', String),
+ (r'(<%@\s+)(extends|params)',
+ bygroups(Operator, Name.Decorator), 'stringTemplate'),
+ (r'<%!--.*?--%>', Comment.Multiline),
+ (r'(<%)|(<%=)', Operator, 'stringTemplate'),
+ (r'\$\{', Operator, 'stringTemplateShorthand'),
+ (r'.', String)
+ ],
+ 'string': [
+ (r'"', String, '#pop'),
+ include('templateText')
+ ],
+ 'stringTemplate': [
+ (r'"', String, 'string'),
+ (r'%>', Operator, '#pop'),
+ include('root')
+ ],
+ 'stringTemplateShorthand': [
+ (r'"', String, 'string'),
+ (r'\{', Operator, 'stringTemplateShorthand'),
+ (r'\}', Operator, '#pop'),
+ include('root')
+ ],
+ }
+
+
+class GosuTemplateLexer(Lexer):
+ """
+ For Gosu templates.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'Gosu Template'
+ aliases = ['gst']
+ filenames = ['*.gst']
+ mimetypes = ['text/x-gosu-template']
+
+ def get_tokens_unprocessed(self, text):
+ lexer = GosuLexer()
+ stack = ['templateText']
+ for item in lexer.get_tokens_unprocessed(text, stack):
+ yield item
+
+
+class GroovyLexer(RegexLexer):
+ """
+ For `Groovy <http://groovy.codehaus.org/>`_ source code.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'Groovy'
+ aliases = ['groovy']
+ filenames = ['*.groovy','*.gradle']
+ mimetypes = ['text/x-groovy']
+
+ flags = re.MULTILINE | re.DOTALL
+
+ tokens = {
+ 'root': [
+ # Groovy allows a file to start with a shebang
+ (r'#!(.*?)$', Comment.Preproc, 'base'),
+ default('base'),
+ ],
+ 'base': [
+ # method names
+ (r'^(\s*(?:[a-zA-Z_][\w.\[\]]*\s+)+?)' # return arguments
+ r'([a-zA-Z_]\w*)' # method name
+ r'(\s*)(\()', # signature start
+ bygroups(using(this), Name.Function, Text, Operator)),
+ (r'[^\S\n]+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'@[a-zA-Z_][\w.]*', Name.Decorator),
+ (r'(assert|break|case|catch|continue|default|do|else|finally|for|'
+ r'if|goto|instanceof|new|return|switch|this|throw|try|while|in|as)\b',
+ Keyword),
+ (r'(abstract|const|enum|extends|final|implements|native|private|'
+ r'protected|public|static|strictfp|super|synchronized|throws|'
+ r'transient|volatile)\b', Keyword.Declaration),
+ (r'(def|boolean|byte|char|double|float|int|long|short|void)\b',
+ Keyword.Type),
+ (r'(package)(\s+)', bygroups(Keyword.Namespace, Text)),
+ (r'(true|false|null)\b', Keyword.Constant),
+ (r'(class|interface)(\s+)', bygroups(Keyword.Declaration, Text),
+ 'class'),
+ (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
+ (r'""".*?"""', String.Double),
+ (r"'''.*?'''", String.Single),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ (r'\$/((?!/\$).)*/\$', String),
+ (r'/(\\\\|\\"|[^/])*/', String),
+ (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Char),
+ (r'(\.)([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)),
+ (r'[a-zA-Z_]\w*:', Name.Label),
+ (r'[a-zA-Z_$]\w*', Name),
+ (r'[~^*!%&\[\](){}<>|+=:;,./?-]', Operator),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+L?', Number.Integer),
+ (r'\n', Text)
+ ],
+ 'class': [
+ (r'[a-zA-Z_]\w*', Name.Class, '#pop')
+ ],
+ 'import': [
+ (r'[\w.]+\*?', Name.Namespace, '#pop')
+ ],
+ }
+
+ def analyse_text(text):
+ return shebang_matches(text, r'groovy')
+
+
+class IokeLexer(RegexLexer):
+ """
+ For `Ioke <http://ioke.org/>`_ (a strongly typed, dynamic,
+ prototype based programming language) source.
+
+ .. versionadded:: 1.4
+ """
+ name = 'Ioke'
+ filenames = ['*.ik']
+ aliases = ['ioke', 'ik']
+ mimetypes = ['text/x-iokesrc']
+ tokens = {
+ 'interpolatableText': [
+ (r'(\\b|\\e|\\t|\\n|\\f|\\r|\\"|\\\\|\\#|\\\Z|\\u[0-9a-fA-F]{1,4}'
+ r'|\\[0-3]?[0-7]?[0-7])', String.Escape),
+ (r'#\{', Punctuation, 'textInterpolationRoot')
+ ],
+
+ 'text': [
+ (r'(?<!\\)"', String, '#pop'),
+ include('interpolatableText'),
+ (r'[^"]', String)
+ ],
+
+ 'documentation': [
+ (r'(?<!\\)"', String.Doc, '#pop'),
+ include('interpolatableText'),
+ (r'[^"]', String.Doc)
+ ],
+
+ 'textInterpolationRoot': [
+ (r'\}', Punctuation, '#pop'),
+ include('root')
+ ],
+
+ 'slashRegexp': [
+ (r'(?<!\\)/[im-psux]*', String.Regex, '#pop'),
+ include('interpolatableText'),
+ (r'\\/', String.Regex),
+ (r'[^/]', String.Regex)
+ ],
+
+ 'squareRegexp': [
+ (r'(?<!\\)][im-psux]*', String.Regex, '#pop'),
+ include('interpolatableText'),
+ (r'\\]', String.Regex),
+ (r'[^\]]', String.Regex)
+ ],
+
+ 'squareText': [
+ (r'(?<!\\)]', String, '#pop'),
+ include('interpolatableText'),
+ (r'[^\]]', String)
+ ],
+
+ 'root': [
+ (r'\n', Text),
+ (r'\s+', Text),
+
+ # Comments
+ (r';(.*?)\n', Comment),
+ (r'\A#!(.*?)\n', Comment),
+
+ # Regexps
+ (r'#/', String.Regex, 'slashRegexp'),
+ (r'#r\[', String.Regex, 'squareRegexp'),
+
+ # Symbols
+ (r':[\w!:?]+', String.Symbol),
+ (r'[\w!:?]+:(?![\w!?])', String.Other),
+ (r':"(\\\\|\\"|[^"])*"', String.Symbol),
+
+ # Documentation
+ (r'((?<=fn\()|(?<=fnx\()|(?<=method\()|(?<=macro\()|(?<=lecro\()'
+ r'|(?<=syntax\()|(?<=dmacro\()|(?<=dlecro\()|(?<=dlecrox\()'
+ r'|(?<=dsyntax\())\s*"', String.Doc, 'documentation'),
+
+ # Text
+ (r'"', String, 'text'),
+ (r'#\[', String, 'squareText'),
+
+ # Mimic
+ (r'\w[\w!:?]+(?=\s*=.*mimic\s)', Name.Entity),
+
+ # Assignment
+ (r'[a-zA-Z_][\w!:?]*(?=[\s]*[+*/-]?=[^=].*($|\.))',
+ Name.Variable),
+
+ # keywords
+ (r'(break|cond|continue|do|ensure|for|for:dict|for:set|if|let|'
+ r'loop|p:for|p:for:dict|p:for:set|return|unless|until|while|'
+ r'with)(?![\w!:?])', Keyword.Reserved),
+
+ # Origin
+ (r'(eval|mimic|print|println)(?![\w!:?])', Keyword),
+
+ # Base
+ (r'(cell\?|cellNames|cellOwner\?|cellOwner|cells|cell|'
+ r'documentation|hash|identity|mimic|removeCell\!|undefineCell\!)'
+ r'(?![\w!:?])', Keyword),
+
+ # Ground
+ (r'(stackTraceAsText)(?![\w!:?])', Keyword),
+
+ # DefaultBehaviour Literals
+ (r'(dict|list|message|set)(?![\w!:?])', Keyword.Reserved),
+
+ # DefaultBehaviour Case
+ (r'(case|case:and|case:else|case:nand|case:nor|case:not|case:or|'
+ r'case:otherwise|case:xor)(?![\w!:?])', Keyword.Reserved),
+
+ # DefaultBehaviour Reflection
+ (r'(asText|become\!|derive|freeze\!|frozen\?|in\?|is\?|kind\?|'
+ r'mimic\!|mimics|mimics\?|prependMimic\!|removeAllMimics\!|'
+ r'removeMimic\!|same\?|send|thaw\!|uniqueHexId)'
+ r'(?![\w!:?])', Keyword),
+
+ # DefaultBehaviour Aspects
+ (r'(after|around|before)(?![\w!:?])', Keyword.Reserved),
+
+ # DefaultBehaviour
+ (r'(kind|cellDescriptionDict|cellSummary|genSym|inspect|notice)'
+ r'(?![\w!:?])', Keyword),
+ (r'(use|destructuring)', Keyword.Reserved),
+
+ # DefaultBehavior BaseBehavior
+ (r'(cell\?|cellOwner\?|cellOwner|cellNames|cells|cell|'
+ r'documentation|identity|removeCell!|undefineCell)'
+ r'(?![\w!:?])', Keyword),
+
+ # DefaultBehavior Internal
+ (r'(internal:compositeRegexp|internal:concatenateText|'
+ r'internal:createDecimal|internal:createNumber|'
+ r'internal:createRegexp|internal:createText)'
+ r'(?![\w!:?])', Keyword.Reserved),
+
+ # DefaultBehaviour Conditions
+ (r'(availableRestarts|bind|error\!|findRestart|handle|'
+ r'invokeRestart|rescue|restart|signal\!|warn\!)'
+ r'(?![\w!:?])', Keyword.Reserved),
+
+ # constants
+ (r'(nil|false|true)(?![\w!:?])', Name.Constant),
+
+ # names
+ (r'(Arity|Base|Call|Condition|DateTime|Aspects|Pointcut|'
+ r'Assignment|BaseBehavior|Boolean|Case|AndCombiner|Else|'
+ r'NAndCombiner|NOrCombiner|NotCombiner|OrCombiner|XOrCombiner|'
+ r'Conditions|Definitions|FlowControl|Internal|Literals|'
+ r'Reflection|DefaultMacro|DefaultMethod|DefaultSyntax|Dict|'
+ r'FileSystem|Ground|Handler|Hook|IO|IokeGround|Struct|'
+ r'LexicalBlock|LexicalMacro|List|Message|Method|Mixins|'
+ r'NativeMethod|Number|Origin|Pair|Range|Reflector|Regexp Match|'
+ r'Regexp|Rescue|Restart|Runtime|Sequence|Set|Symbol|'
+ r'System|Text|Tuple)(?![\w!:?])', Name.Builtin),
+
+ # functions
+ (u'(generateMatchMethod|aliasMethod|\u03bb|\u028E|fnx|fn|method|'
+ u'dmacro|dlecro|syntax|macro|dlecrox|lecrox|lecro|syntax)'
+ u'(?![\w!:?])', Name.Function),
+
+ # Numbers
+ (r'-?0[xX][0-9a-fA-F]+', Number.Hex),
+ (r'-?(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+ (r'-?\d+', Number.Integer),
+
+ (r'#\(', Punctuation),
+
+ # Operators
+ (r'(&&>>|\|\|>>|\*\*>>|:::|::|\.\.\.|===|\*\*>|\*\*=|&&>|&&=|'
+ r'\|\|>|\|\|=|\->>|\+>>|!>>|<>>>|<>>|&>>|%>>|#>>|@>>|/>>|\*>>|'
+ r'\?>>|\|>>|\^>>|~>>|\$>>|=>>|<<=|>>=|<=>|<\->|=~|!~|=>|\+\+|'
+ r'\-\-|<=|>=|==|!=|&&|\.\.|\+=|\-=|\*=|\/=|%=|&=|\^=|\|=|<\-|'
+ r'\+>|!>|<>|&>|%>|#>|\@>|\/>|\*>|\?>|\|>|\^>|~>|\$>|<\->|\->|'
+ r'<<|>>|\*\*|\?\||\?&|\|\||>|<|\*|\/|%|\+|\-|&|\^|\||=|\$|!|~|'
+ u'\\?|#|\u2260|\u2218|\u2208|\u2209)', Operator),
+ (r'(and|nand|or|xor|nor|return|import)(?![\w!?])',
+ Operator),
+
+ # Punctuation
+ (r'(\`\`|\`|\'\'|\'|\.|\,|@@|@|\[|\]|\(|\)|\{|\})', Punctuation),
+
+ # kinds
+ (r'[A-Z][\w!:?]*', Name.Class),
+
+ # default cellnames
+ (r'[a-z_][\w!:?]*', Name)
+ ]
+ }
+
+
+class ClojureLexer(RegexLexer):
+ """
+ Lexer for `Clojure <http://clojure.org/>`_ source code.
+
+ .. versionadded:: 0.11
+ """
+ name = 'Clojure'
+ aliases = ['clojure', 'clj']
+ filenames = ['*.clj']
+ mimetypes = ['text/x-clojure', 'application/x-clojure']
+
+ special_forms = (
+ '.', 'def', 'do', 'fn', 'if', 'let', 'new', 'quote', 'var', 'loop'
+ )
+
+ # It's safe to consider 'ns' a declaration thing because it defines a new
+ # namespace.
+ declarations = (
+ 'def-', 'defn', 'defn-', 'defmacro', 'defmulti', 'defmethod',
+ 'defstruct', 'defonce', 'declare', 'definline', 'definterface',
+ 'defprotocol', 'defrecord', 'deftype', 'defproject', 'ns'
+ )
+
+ builtins = (
+ '*', '+', '-', '->', '/', '<', '<=', '=', '==', '>', '>=', '..',
+ 'accessor', 'agent', 'agent-errors', 'aget', 'alength', 'all-ns',
+ 'alter', 'and', 'append-child', 'apply', 'array-map', 'aset',
+ 'aset-boolean', 'aset-byte', 'aset-char', 'aset-double', 'aset-float',
+ 'aset-int', 'aset-long', 'aset-short', 'assert', 'assoc', 'await',
+ 'await-for', 'bean', 'binding', 'bit-and', 'bit-not', 'bit-or',
+ 'bit-shift-left', 'bit-shift-right', 'bit-xor', 'boolean', 'branch?',
+ 'butlast', 'byte', 'cast', 'char', 'children', 'class',
+ 'clear-agent-errors', 'comment', 'commute', 'comp', 'comparator',
+ 'complement', 'concat', 'conj', 'cons', 'constantly', 'cond', 'if-not',
+ 'construct-proxy', 'contains?', 'count', 'create-ns', 'create-struct',
+ 'cycle', 'dec', 'deref', 'difference', 'disj', 'dissoc', 'distinct',
+ 'doall', 'doc', 'dorun', 'doseq', 'dosync', 'dotimes', 'doto',
+ 'double', 'down', 'drop', 'drop-while', 'edit', 'end?', 'ensure',
+ 'eval', 'every?', 'false?', 'ffirst', 'file-seq', 'filter', 'find',
+ 'find-doc', 'find-ns', 'find-var', 'first', 'float', 'flush', 'for',
+ 'fnseq', 'frest', 'gensym', 'get-proxy-class', 'get',
+ 'hash-map', 'hash-set', 'identical?', 'identity', 'if-let', 'import',
+ 'in-ns', 'inc', 'index', 'insert-child', 'insert-left', 'insert-right',
+ 'inspect-table', 'inspect-tree', 'instance?', 'int', 'interleave',
+ 'intersection', 'into', 'into-array', 'iterate', 'join', 'key', 'keys',
+ 'keyword', 'keyword?', 'last', 'lazy-cat', 'lazy-cons', 'left',
+ 'lefts', 'line-seq', 'list*', 'list', 'load', 'load-file',
+ 'locking', 'long', 'loop', 'macroexpand', 'macroexpand-1',
+ 'make-array', 'make-node', 'map', 'map-invert', 'map?', 'mapcat',
+ 'max', 'max-key', 'memfn', 'merge', 'merge-with', 'meta', 'min',
+ 'min-key', 'name', 'namespace', 'neg?', 'new', 'newline', 'next',
+ 'nil?', 'node', 'not', 'not-any?', 'not-every?', 'not=', 'ns-imports',
+ 'ns-interns', 'ns-map', 'ns-name', 'ns-publics', 'ns-refers',
+ 'ns-resolve', 'ns-unmap', 'nth', 'nthrest', 'or', 'parse', 'partial',
+ 'path', 'peek', 'pop', 'pos?', 'pr', 'pr-str', 'print', 'print-str',
+ 'println', 'println-str', 'prn', 'prn-str', 'project', 'proxy',
+ 'proxy-mappings', 'quot', 'rand', 'rand-int', 'range', 're-find',
+ 're-groups', 're-matcher', 're-matches', 're-pattern', 're-seq',
+ 'read', 'read-line', 'reduce', 'ref', 'ref-set', 'refer', 'rem',
+ 'remove', 'remove-method', 'remove-ns', 'rename', 'rename-keys',
+ 'repeat', 'replace', 'replicate', 'resolve', 'rest', 'resultset-seq',
+ 'reverse', 'rfirst', 'right', 'rights', 'root', 'rrest', 'rseq',
+ 'second', 'select', 'select-keys', 'send', 'send-off', 'seq',
+ 'seq-zip', 'seq?', 'set', 'short', 'slurp', 'some', 'sort',
+ 'sort-by', 'sorted-map', 'sorted-map-by', 'sorted-set',
+ 'special-symbol?', 'split-at', 'split-with', 'str', 'string?',
+ 'struct', 'struct-map', 'subs', 'subvec', 'symbol', 'symbol?',
+ 'sync', 'take', 'take-nth', 'take-while', 'test', 'time', 'to-array',
+ 'to-array-2d', 'tree-seq', 'true?', 'union', 'up', 'update-proxy',
+ 'val', 'vals', 'var-get', 'var-set', 'var?', 'vector', 'vector-zip',
+ 'vector?', 'when', 'when-first', 'when-let', 'when-not',
+ 'with-local-vars', 'with-meta', 'with-open', 'with-out-str',
+ 'xml-seq', 'xml-zip', 'zero?', 'zipmap', 'zipper')
+
+ # valid names for identifiers
+ # well, names can only not consist fully of numbers
+ # but this should be good enough for now
+
+ # TODO / should divide keywords/symbols into namespace/rest
+ # but that's hard, so just pretend / is part of the name
+ valid_name = r'(?!#)[\w!$%*+<=>?/.#-]+'
+
+ tokens = {
+ 'root': [
+ # the comments - always starting with semicolon
+ # and going to the end of the line
+ (r';.*$', Comment.Single),
+
+ # whitespaces - usually not relevant
+ (r'[,\s]+', Text),
+
+ # numbers
+ (r'-?\d+\.\d+', Number.Float),
+ (r'-?\d+', Number.Integer),
+ (r'0x-?[abcdef\d]+', Number.Hex),
+
+ # strings, symbols and characters
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r"'" + valid_name, String.Symbol),
+ (r"\\(.|[a-z]+)", String.Char),
+
+ # keywords
+ (r'::?#?' + valid_name, String.Symbol),
+
+ # special operators
+ (r'~@|[`\'#^~&@]', Operator),
+
+ # highlight the special forms
+ (words(special_forms, suffix=' '), Keyword),
+
+ # Technically, only the special forms are 'keywords'. The problem
+ # is that only treating them as keywords means that things like
+ # 'defn' and 'ns' need to be highlighted as builtins. This is ugly
+ # and weird for most styles. So, as a compromise we're going to
+ # highlight them as Keyword.Declarations.
+ (words(declarations, suffix=' '), Keyword.Declaration),
+
+ # highlight the builtins
+ (words(builtins, suffix=' '), Name.Builtin),
+
+ # the remaining functions
+ (r'(?<=\()' + valid_name, Name.Function),
+
+ # find the remaining variables
+ (valid_name, Name.Variable),
+
+ # Clojure accepts vector notation
+ (r'(\[|\])', Punctuation),
+
+ # Clojure accepts map notation
+ (r'(\{|\})', Punctuation),
+
+ # the famous parentheses!
+ (r'(\(|\))', Punctuation),
+ ],
+ }
+
+
+class ClojureScriptLexer(ClojureLexer):
+ """
+ Lexer for `ClojureScript <http://clojure.org/clojurescript>`_
+ source code.
+
+ .. versionadded:: 2.0
+ """
+ name = 'ClojureScript'
+ aliases = ['clojurescript', 'cljs']
+ filenames = ['*.cljs']
+ mimetypes = ['text/x-clojurescript', 'application/x-clojurescript']
+
+
+class TeaLangLexer(RegexLexer):
+ """
+ For `Tea <http://teatrove.org/>`_ source code. Only used within a
+ TeaTemplateLexer.
+
+ .. versionadded:: 1.5
+ """
+
+ flags = re.MULTILINE | re.DOTALL
+
+ tokens = {
+ 'root': [
+ # method names
+ (r'^(\s*(?:[a-zA-Z_][\w\.\[\]]*\s+)+?)' # return arguments
+ r'([a-zA-Z_]\w*)' # method name
+ r'(\s*)(\()', # signature start
+ bygroups(using(this), Name.Function, Text, Operator)),
+ (r'[^\S\n]+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'@[a-zA-Z_][\w\.]*', Name.Decorator),
+ (r'(and|break|else|foreach|if|in|not|or|reverse)\b',
+ Keyword),
+ (r'(as|call|define)\b', Keyword.Declaration),
+ (r'(true|false|null)\b', Keyword.Constant),
+ (r'(template)(\s+)', bygroups(Keyword.Declaration, Text), 'template'),
+ (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r'\'(\\\\|\\\'|[^\'])*\'', String),
+ (r'(\.)([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)),
+ (r'[a-zA-Z_]\w*:', Name.Label),
+ (r'[a-zA-Z_\$]\w*', Name),
+ (r'(isa|[.]{3}|[.]{2}|[=#!<>+-/%&;,.\*\\\(\)\[\]\{\}])', Operator),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+L?', Number.Integer),
+ (r'\n', Text)
+ ],
+ 'template': [
+ (r'[a-zA-Z_]\w*', Name.Class, '#pop')
+ ],
+ 'import': [
+ (r'[\w.]+\*?', Name.Namespace, '#pop')
+ ],
+ }
+
+
+class CeylonLexer(RegexLexer):
+ """
+ For `Ceylon <http://ceylon-lang.org/>`_ source code.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'Ceylon'
+ aliases = ['ceylon']
+ filenames = ['*.ceylon']
+ mimetypes = ['text/x-ceylon']
+
+ flags = re.MULTILINE | re.DOTALL
+
+ #: optional Comment or Whitespace
+ _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+'
+
+ tokens = {
+ 'root': [
+ # method names
+ (r'^(\s*(?:[a-zA-Z_][\w.\[\]]*\s+)+?)' # return arguments
+ r'([a-zA-Z_]\w*)' # method name
+ r'(\s*)(\()', # signature start
+ bygroups(using(this), Name.Function, Text, Operator)),
+ (r'[^\S\n]+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*', Comment.Multiline, 'comment'),
+ (r'(shared|abstract|formal|default|actual|variable|deprecated|small|'
+ r'late|literal|doc|by|see|throws|optional|license|tagged|final|native|'
+ r'annotation|sealed)\b', Name.Decorator),
+ (r'(break|case|catch|continue|else|finally|for|in|'
+ r'if|return|switch|this|throw|try|while|is|exists|dynamic|'
+ r'nonempty|then|outer|assert|let)\b', Keyword),
+ (r'(abstracts|extends|satisfies|'
+ r'super|given|of|out|assign)\b', Keyword.Declaration),
+ (r'(function|value|void|new)\b',
+ Keyword.Type),
+ (r'(assembly|module|package)(\s+)', bygroups(Keyword.Namespace, Text)),
+ (r'(true|false|null)\b', Keyword.Constant),
+ (r'(class|interface|object|alias)(\s+)',
+ bygroups(Keyword.Declaration, Text), 'class'),
+ (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r"'\\.'|'[^\\]'|'\\\{#[0-9a-fA-F]{4}\}'", String.Char),
+ (r'".*``.*``.*"', String.Interpol),
+ (r'(\.)([a-z_]\w*)',
+ bygroups(Operator, Name.Attribute)),
+ (r'[a-zA-Z_]\w*:', Name.Label),
+ (r'[a-zA-Z_]\w*', Name),
+ (r'[~^*!%&\[\](){}<>|+=:;,./?-]', Operator),
+ (r'\d{1,3}(_\d{3})+\.\d{1,3}(_\d{3})+[kMGTPmunpf]?', Number.Float),
+ (r'\d{1,3}(_\d{3})+\.[0-9]+([eE][+-]?[0-9]+)?[kMGTPmunpf]?',
+ Number.Float),
+ (r'[0-9][0-9]*\.\d{1,3}(_\d{3})+[kMGTPmunpf]?', Number.Float),
+ (r'[0-9][0-9]*\.[0-9]+([eE][+-]?[0-9]+)?[kMGTPmunpf]?',
+ Number.Float),
+ (r'#([0-9a-fA-F]{4})(_[0-9a-fA-F]{4})+', Number.Hex),
+ (r'#[0-9a-fA-F]+', Number.Hex),
+ (r'\$([01]{4})(_[01]{4})+', Number.Bin),
+ (r'\$[01]+', Number.Bin),
+ (r'\d{1,3}(_\d{3})+[kMGTP]?', Number.Integer),
+ (r'[0-9]+[kMGTP]?', Number.Integer),
+ (r'\n', Text)
+ ],
+ 'class': [
+ (r'[A-Za-z_]\w*', Name.Class, '#pop')
+ ],
+ 'import': [
+ (r'[a-z][\w.]*',
+ Name.Namespace, '#pop')
+ ],
+ 'comment': [
+ (r'[^*/]', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline)
+ ],
+ }
+
+
+class KotlinLexer(RegexLexer):
+ """
+ For `Kotlin <http://kotlinlang.org/>`_
+ source code.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'Kotlin'
+ aliases = ['kotlin']
+ filenames = ['*.kt']
+ mimetypes = ['text/x-kotlin']
+
+ flags = re.MULTILINE | re.DOTALL | re.UNICODE
+
+ kt_name = ('@?[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' +
+ '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc', 'Cf',
+ 'Mn', 'Mc') + ']*')
+ kt_id = '(' + kt_name + '|`' + kt_name + '`)'
+
+ tokens = {
+ 'root': [
+ (r'^\s*\[.*?\]', Name.Attribute),
+ (r'[^\S\n]+', Text),
+ (r'\\\n', Text), # line continuation
+ (r'//.*?\n', Comment.Single),
+ (r'/[*].*?[*]/', Comment.Multiline),
+ (r'\n', Text),
+ (r'::|!!|\?[:.]', Operator),
+ (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation),
+ (r'[{}]', Punctuation),
+ (r'@"(""|[^"])*"', String),
+ (r'"(\\\\|\\"|[^"\n])*["\n]', String),
+ (r"'\\.'|'[^\\]'", String.Char),
+ (r"[0-9](\.[0-9]*)?([eE][+-][0-9]+)?[flFL]?|"
+ r"0[xX][0-9a-fA-F]+[Ll]?", Number),
+ (r'(class)(\s+)(object)', bygroups(Keyword, Text, Keyword)),
+ (r'(class|interface|object)(\s+)', bygroups(Keyword, Text), 'class'),
+ (r'(package|import)(\s+)', bygroups(Keyword, Text), 'package'),
+ (r'(val|var)(\s+)', bygroups(Keyword, Text), 'property'),
+ (r'(fun)(\s+)', bygroups(Keyword, Text), 'function'),
+ (r'(abstract|annotation|as|break|by|catch|class|companion|const|'
+ r'constructor|continue|crossinline|data|do|dynamic|else|enum|'
+ r'external|false|final|finally|for|fun|get|if|import|in|infix|'
+ r'inline|inner|interface|internal|is|lateinit|noinline|null|'
+ r'object|open|operator|out|override|package|private|protected|'
+ r'public|reified|return|sealed|set|super|tailrec|this|throw|'
+ r'true|try|val|var|vararg|when|where|while)\b', Keyword),
+ (kt_id, Name),
+ ],
+ 'package': [
+ (r'\S+', Name.Namespace, '#pop')
+ ],
+ 'class': [
+ (kt_id, Name.Class, '#pop')
+ ],
+ 'property': [
+ (kt_id, Name.Property, '#pop')
+ ],
+ 'function': [
+ (kt_id, Name.Function, '#pop')
+ ],
+ }
+
+
+class XtendLexer(RegexLexer):
+ """
+ For `Xtend <http://xtend-lang.org/>`_ source code.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'Xtend'
+ aliases = ['xtend']
+ filenames = ['*.xtend']
+ mimetypes = ['text/x-xtend']
+
+ flags = re.MULTILINE | re.DOTALL
+
+ tokens = {
+ 'root': [
+ # method names
+ (r'^(\s*(?:[a-zA-Z_][\w.\[\]]*\s+)+?)' # return arguments
+ r'([a-zA-Z_$][\w$]*)' # method name
+ r'(\s*)(\()', # signature start
+ bygroups(using(this), Name.Function, Text, Operator)),
+ (r'[^\S\n]+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'@[a-zA-Z_][\w.]*', Name.Decorator),
+ (r'(assert|break|case|catch|continue|default|do|else|finally|for|'
+ r'if|goto|instanceof|new|return|switch|this|throw|try|while|IF|'
+ r'ELSE|ELSEIF|ENDIF|FOR|ENDFOR|SEPARATOR|BEFORE|AFTER)\b',
+ Keyword),
+ (r'(def|abstract|const|enum|extends|final|implements|native|private|'
+ r'protected|public|static|strictfp|super|synchronized|throws|'
+ r'transient|volatile)\b', Keyword.Declaration),
+ (r'(boolean|byte|char|double|float|int|long|short|void)\b',
+ Keyword.Type),
+ (r'(package)(\s+)', bygroups(Keyword.Namespace, Text)),
+ (r'(true|false|null)\b', Keyword.Constant),
+ (r'(class|interface)(\s+)', bygroups(Keyword.Declaration, Text),
+ 'class'),
+ (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
+ (r"(''')", String, 'template'),
+ (u'(\u00BB)', String, 'template'),
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r"'(\\\\|\\'|[^'])*'", String),
+ (r'[a-zA-Z_]\w*:', Name.Label),
+ (r'[a-zA-Z_$]\w*', Name),
+ (r'[~^*!%&\[\](){}<>\|+=:;,./?-]', Operator),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+L?', Number.Integer),
+ (r'\n', Text)
+ ],
+ 'class': [
+ (r'[a-zA-Z_]\w*', Name.Class, '#pop')
+ ],
+ 'import': [
+ (r'[\w.]+\*?', Name.Namespace, '#pop')
+ ],
+ 'template': [
+ (r"'''", String, '#pop'),
+ (u'\u00AB', String, '#pop'),
+ (r'.', String)
+ ],
+ }
+
+
+class PigLexer(RegexLexer):
+ """
+ For `Pig Latin <https://pig.apache.org/>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Pig'
+ aliases = ['pig']
+ filenames = ['*.pig']
+ mimetypes = ['text/x-pig']
+
+ flags = re.MULTILINE | re.IGNORECASE
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'--.*', Comment),
+ (r'/\*[\w\W]*?\*/', Comment.Multiline),
+ (r'\\\n', Text),
+ (r'\\', Text),
+ (r'\'(?:\\[ntbrf\\\']|\\u[0-9a-f]{4}|[^\'\\\n\r])*\'', String),
+ include('keywords'),
+ include('types'),
+ include('builtins'),
+ include('punct'),
+ include('operators'),
+ (r'[0-9]*\.[0-9]+(e[0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-f]+', Number.Hex),
+ (r'[0-9]+L?', Number.Integer),
+ (r'\n', Text),
+ (r'([a-z_]\w*)(\s*)(\()',
+ bygroups(Name.Function, Text, Punctuation)),
+ (r'[()#:]', Text),
+ (r'[^(:#\'")\s]+', Text),
+ (r'\S+\s+', Text) # TODO: make tests pass without \s+
+ ],
+ 'keywords': [
+ (r'(assert|and|any|all|arrange|as|asc|bag|by|cache|CASE|cat|cd|cp|'
+ r'%declare|%default|define|dense|desc|describe|distinct|du|dump|'
+ r'eval|exex|explain|filter|flatten|foreach|full|generate|group|'
+ r'help|if|illustrate|import|inner|input|into|is|join|kill|left|'
+ r'limit|load|ls|map|matches|mkdir|mv|not|null|onschema|or|order|'
+ r'outer|output|parallel|pig|pwd|quit|register|returns|right|rm|'
+ r'rmf|rollup|run|sample|set|ship|split|stderr|stdin|stdout|store|'
+ r'stream|through|union|using|void)\b', Keyword)
+ ],
+ 'builtins': [
+ (r'(AVG|BinStorage|cogroup|CONCAT|copyFromLocal|copyToLocal|COUNT|'
+ r'cross|DIFF|MAX|MIN|PigDump|PigStorage|SIZE|SUM|TextLoader|'
+ r'TOKENIZE)\b', Name.Builtin)
+ ],
+ 'types': [
+ (r'(bytearray|BIGINTEGER|BIGDECIMAL|chararray|datetime|double|float|'
+ r'int|long|tuple)\b', Keyword.Type)
+ ],
+ 'punct': [
+ (r'[;(){}\[\]]', Punctuation),
+ ],
+ 'operators': [
+ (r'[#=,./%+\-?]', Operator),
+ (r'(eq|gt|lt|gte|lte|neq|matches)\b', Operator),
+ (r'(==|<=|<|>=|>|!=)', Operator),
+ ],
+ }
+
+
+class GoloLexer(RegexLexer):
+ """
+ For `Golo <http://golo-lang.org/>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Golo'
+ filenames = ['*.golo']
+ aliases = ['golo']
+
+ tokens = {
+ 'root': [
+ (r'[^\S\n]+', Text),
+
+ (r'#.*$', Comment),
+
+ (r'(\^|\.\.\.|:|\?:|->|==|!=|=|\+|\*|%|/|<=|<|>=|>|=|\.)',
+ Operator),
+ (r'(?<=[^-])(-)(?=[^-])', Operator),
+
+ (r'(?<=[^`])(is|isnt|and|or|not|oftype|in|orIfNull)\b', Operator.Word),
+ (r'[]{}|(),[]', Punctuation),
+
+ (r'(module|import)(\s+)',
+ bygroups(Keyword.Namespace, Text),
+ 'modname'),
+ (r'\b([a-zA-Z_][\w$.]*)(::)', bygroups(Name.Namespace, Punctuation)),
+ (r'\b([a-zA-Z_][\w$]*(?:\.[a-zA-Z_][\w$]*)+)\b', Name.Namespace),
+
+ (r'(let|var)(\s+)',
+ bygroups(Keyword.Declaration, Text),
+ 'varname'),
+ (r'(struct)(\s+)',
+ bygroups(Keyword.Declaration, Text),
+ 'structname'),
+ (r'(function)(\s+)',
+ bygroups(Keyword.Declaration, Text),
+ 'funcname'),
+
+ (r'(null|true|false)\b', Keyword.Constant),
+ (r'(augment|pimp'
+ r'|if|else|case|match|return'
+ r'|case|when|then|otherwise'
+ r'|while|for|foreach'
+ r'|try|catch|finally|throw'
+ r'|local'
+ r'|continue|break)\b', Keyword),
+
+ (r'(map|array|list|set|vector|tuple)(\[)',
+ bygroups(Name.Builtin, Punctuation)),
+ (r'(print|println|readln|raise|fun'
+ r'|asInterfaceInstance)\b', Name.Builtin),
+ (r'(`?[a-zA-Z_][\w$]*)(\()',
+ bygroups(Name.Function, Punctuation)),
+
+ (r'-?[\d_]*\.[\d_]*([eE][+-]?\d[\d_]*)?F?', Number.Float),
+ (r'0[0-7]+j?', Number.Oct),
+ (r'0[xX][a-fA-F0-9]+', Number.Hex),
+ (r'-?\d[\d_]*L', Number.Integer.Long),
+ (r'-?\d[\d_]*', Number.Integer),
+
+ ('`?[a-zA-Z_][\w$]*', Name),
+ (r'@[a-zA-Z_][\w$.]*', Name.Decorator),
+
+ (r'"""', String, combined('stringescape', 'triplestring')),
+ (r'"', String, combined('stringescape', 'doublestring')),
+ (r"'", String, combined('stringescape', 'singlestring')),
+ (r'----((.|\n)*?)----', String.Doc)
+
+ ],
+
+ 'funcname': [
+ (r'`?[a-zA-Z_][\w$]*', Name.Function, '#pop'),
+ ],
+ 'modname': [
+ (r'[a-zA-Z_][\w$.]*\*?', Name.Namespace, '#pop')
+ ],
+ 'structname': [
+ (r'`?[\w.]+\*?', Name.Class, '#pop')
+ ],
+ 'varname': [
+ (r'`?[a-zA-Z_][\w$]*', Name.Variable, '#pop'),
+ ],
+ 'string': [
+ (r'[^\\\'"\n]+', String),
+ (r'[\'"\\]', String)
+ ],
+ 'stringescape': [
+ (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
+ r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
+ ],
+ 'triplestring': [
+ (r'"""', String, '#pop'),
+ include('string'),
+ (r'\n', String),
+ ],
+ 'doublestring': [
+ (r'"', String.Double, '#pop'),
+ include('string'),
+ ],
+ 'singlestring': [
+ (r"'", String, '#pop'),
+ include('string'),
+ ],
+ 'operators': [
+ (r'[#=,./%+\-?]', Operator),
+ (r'(eq|gt|lt|gte|lte|neq|matches)\b', Operator),
+ (r'(==|<=|<|>=|>|!=)', Operator),
+ ],
+ }
+
+
+class JasminLexer(RegexLexer):
+ """
+ For `Jasmin <http://jasmin.sourceforge.net/>`_ assembly code.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Jasmin'
+ aliases = ['jasmin', 'jasminxt']
+ filenames = ['*.j']
+
+ _whitespace = r' \n\t\r'
+ _ws = r'(?:[%s]+)' % _whitespace
+ _separator = r'%s:=' % _whitespace
+ _break = r'(?=[%s]|$)' % _separator
+ _name = r'[^%s]+' % _separator
+ _unqualified_name = r'(?:[^%s.;\[/]+)' % _separator
+
+ tokens = {
+ 'default': [
+ (r'\n', Text, '#pop'),
+ (r"'", String.Single, ('#pop', 'quote')),
+ (r'"', String.Double, 'string'),
+ (r'=', Punctuation),
+ (r':', Punctuation, 'label'),
+ (_ws, Text),
+ (r';.*', Comment.Single),
+ (r'(\$[-+])?0x-?[\da-fA-F]+%s' % _break, Number.Hex),
+ (r'(\$[-+]|\+)?-?\d+%s' % _break, Number.Integer),
+ (r'-?(\d+\.\d*|\.\d+)([eE][-+]?\d+)?[fFdD]?'
+ r'[\x00-\x08\x0b\x0c\x0e-\x1f]*%s' % _break, Number.Float),
+ (r'\$%s' % _name, Name.Variable),
+
+ # Directives
+ (r'\.annotation%s' % _break, Keyword.Reserved, 'annotation'),
+ (r'(\.attribute|\.bytecode|\.debug|\.deprecated|\.enclosing|'
+ r'\.interface|\.line|\.signature|\.source|\.stack|\.var|abstract|'
+ r'annotation|bridge|class|default|enum|field|final|fpstrict|'
+ r'interface|native|private|protected|public|signature|static|'
+ r'synchronized|synthetic|transient|varargs|volatile)%s' % _break,
+ Keyword.Reserved),
+ (r'\.catch%s' % _break, Keyword.Reserved, 'caught-exception'),
+ (r'(\.class|\.implements|\.inner|\.super|inner|invisible|'
+ r'invisibleparam|outer|visible|visibleparam)%s' % _break,
+ Keyword.Reserved, 'class/convert-dots'),
+ (r'\.field%s' % _break, Keyword.Reserved,
+ ('descriptor/convert-dots', 'field')),
+ (r'(\.end|\.limit|use)%s' % _break, Keyword.Reserved,
+ 'no-verification'),
+ (r'\.method%s' % _break, Keyword.Reserved, 'method'),
+ (r'\.set%s' % _break, Keyword.Reserved, 'var'),
+ (r'\.throws%s' % _break, Keyword.Reserved, 'exception'),
+ (r'(from|offset|to|using)%s' % _break, Keyword.Reserved, 'label'),
+ (r'is%s' % _break, Keyword.Reserved,
+ ('descriptor/convert-dots', 'var')),
+ (r'(locals|stack)%s' % _break, Keyword.Reserved, 'verification'),
+ (r'method%s' % _break, Keyword.Reserved, 'enclosing-method'),
+
+ # Instructions
+ (words((
+ 'aaload', 'aastore', 'aconst_null', 'aload', 'aload_0', 'aload_1', 'aload_2',
+ 'aload_3', 'aload_w', 'areturn', 'arraylength', 'astore', 'astore_0', 'astore_1',
+ 'astore_2', 'astore_3', 'astore_w', 'athrow', 'baload', 'bastore', 'bipush',
+ 'breakpoint', 'caload', 'castore', 'd2f', 'd2i', 'd2l', 'dadd', 'daload', 'dastore',
+ 'dcmpg', 'dcmpl', 'dconst_0', 'dconst_1', 'ddiv', 'dload', 'dload_0', 'dload_1',
+ 'dload_2', 'dload_3', 'dload_w', 'dmul', 'dneg', 'drem', 'dreturn', 'dstore', 'dstore_0',
+ 'dstore_1', 'dstore_2', 'dstore_3', 'dstore_w', 'dsub', 'dup', 'dup2', 'dup2_x1',
+ 'dup2_x2', 'dup_x1', 'dup_x2', 'f2d', 'f2i', 'f2l', 'fadd', 'faload', 'fastore', 'fcmpg',
+ 'fcmpl', 'fconst_0', 'fconst_1', 'fconst_2', 'fdiv', 'fload', 'fload_0', 'fload_1',
+ 'fload_2', 'fload_3', 'fload_w', 'fmul', 'fneg', 'frem', 'freturn', 'fstore', 'fstore_0',
+ 'fstore_1', 'fstore_2', 'fstore_3', 'fstore_w', 'fsub', 'i2b', 'i2c', 'i2d', 'i2f', 'i2l',
+ 'i2s', 'iadd', 'iaload', 'iand', 'iastore', 'iconst_0', 'iconst_1', 'iconst_2',
+ 'iconst_3', 'iconst_4', 'iconst_5', 'iconst_m1', 'idiv', 'iinc', 'iinc_w', 'iload',
+ 'iload_0', 'iload_1', 'iload_2', 'iload_3', 'iload_w', 'imul', 'ineg', 'int2byte',
+ 'int2char', 'int2short', 'ior', 'irem', 'ireturn', 'ishl', 'ishr', 'istore', 'istore_0',
+ 'istore_1', 'istore_2', 'istore_3', 'istore_w', 'isub', 'iushr', 'ixor', 'l2d', 'l2f',
+ 'l2i', 'ladd', 'laload', 'land', 'lastore', 'lcmp', 'lconst_0', 'lconst_1', 'ldc2_w',
+ 'ldiv', 'lload', 'lload_0', 'lload_1', 'lload_2', 'lload_3', 'lload_w', 'lmul', 'lneg',
+ 'lookupswitch', 'lor', 'lrem', 'lreturn', 'lshl', 'lshr', 'lstore', 'lstore_0',
+ 'lstore_1', 'lstore_2', 'lstore_3', 'lstore_w', 'lsub', 'lushr', 'lxor',
+ 'monitorenter', 'monitorexit', 'nop', 'pop', 'pop2', 'ret', 'ret_w', 'return', 'saload',
+ 'sastore', 'sipush', 'swap'), suffix=_break), Keyword.Reserved),
+ (r'(anewarray|checkcast|instanceof|ldc|ldc_w|new)%s' % _break,
+ Keyword.Reserved, 'class/no-dots'),
+ (r'invoke(dynamic|interface|nonvirtual|special|'
+ r'static|virtual)%s' % _break, Keyword.Reserved,
+ 'invocation'),
+ (r'(getfield|putfield)%s' % _break, Keyword.Reserved,
+ ('descriptor/no-dots', 'field')),
+ (r'(getstatic|putstatic)%s' % _break, Keyword.Reserved,
+ ('descriptor/no-dots', 'static')),
+ (words((
+ 'goto', 'goto_w', 'if_acmpeq', 'if_acmpne', 'if_icmpeq',
+ 'if_icmpge', 'if_icmpgt', 'if_icmple', 'if_icmplt', 'if_icmpne',
+ 'ifeq', 'ifge', 'ifgt', 'ifle', 'iflt', 'ifne', 'ifnonnull',
+ 'ifnull', 'jsr', 'jsr_w'), suffix=_break),
+ Keyword.Reserved, 'label'),
+ (r'(multianewarray|newarray)%s' % _break, Keyword.Reserved,
+ 'descriptor/convert-dots'),
+ (r'tableswitch%s' % _break, Keyword.Reserved, 'table')
+ ],
+ 'quote': [
+ (r"'", String.Single, '#pop'),
+ (r'\\u[\da-fA-F]{4}', String.Escape),
+ (r"[^'\\]+", String.Single)
+ ],
+ 'string': [
+ (r'"', String.Double, '#pop'),
+ (r'\\([nrtfb"\'\\]|u[\da-fA-F]{4}|[0-3]?[0-7]{1,2})',
+ String.Escape),
+ (r'[^"\\]+', String.Double)
+ ],
+ 'root': [
+ (r'\n+', Text),
+ (r"'", String.Single, 'quote'),
+ include('default'),
+ (r'(%s)([ \t\r]*)(:)' % _name,
+ bygroups(Name.Label, Text, Punctuation)),
+ (_name, String.Other)
+ ],
+ 'annotation': [
+ (r'\n', Text, ('#pop', 'annotation-body')),
+ (r'default%s' % _break, Keyword.Reserved,
+ ('#pop', 'annotation-default')),
+ include('default')
+ ],
+ 'annotation-body': [
+ (r'\n+', Text),
+ (r'\.end%s' % _break, Keyword.Reserved, '#pop'),
+ include('default'),
+ (_name, String.Other, ('annotation-items', 'descriptor/no-dots'))
+ ],
+ 'annotation-default': [
+ (r'\n+', Text),
+ (r'\.end%s' % _break, Keyword.Reserved, '#pop'),
+ include('default'),
+ default(('annotation-items', 'descriptor/no-dots'))
+ ],
+ 'annotation-items': [
+ (r"'", String.Single, 'quote'),
+ include('default'),
+ (_name, String.Other)
+ ],
+ 'caught-exception': [
+ (r'all%s' % _break, Keyword, '#pop'),
+ include('exception')
+ ],
+ 'class/convert-dots': [
+ include('default'),
+ (r'(L)((?:%s[/.])*)(%s)(;)' % (_unqualified_name, _name),
+ bygroups(Keyword.Type, Name.Namespace, Name.Class, Punctuation),
+ '#pop'),
+ (r'((?:%s[/.])*)(%s)' % (_unqualified_name, _name),
+ bygroups(Name.Namespace, Name.Class), '#pop')
+ ],
+ 'class/no-dots': [
+ include('default'),
+ (r'\[+', Punctuation, ('#pop', 'descriptor/no-dots')),
+ (r'(L)((?:%s/)*)(%s)(;)' % (_unqualified_name, _name),
+ bygroups(Keyword.Type, Name.Namespace, Name.Class, Punctuation),
+ '#pop'),
+ (r'((?:%s/)*)(%s)' % (_unqualified_name, _name),
+ bygroups(Name.Namespace, Name.Class), '#pop')
+ ],
+ 'descriptor/convert-dots': [
+ include('default'),
+ (r'\[+', Punctuation),
+ (r'(L)((?:%s[/.])*)(%s?)(;)' % (_unqualified_name, _name),
+ bygroups(Keyword.Type, Name.Namespace, Name.Class, Punctuation),
+ '#pop'),
+ (r'[^%s\[)L]+' % _separator, Keyword.Type, '#pop'),
+ default('#pop')
+ ],
+ 'descriptor/no-dots': [
+ include('default'),
+ (r'\[+', Punctuation),
+ (r'(L)((?:%s/)*)(%s)(;)' % (_unqualified_name, _name),
+ bygroups(Keyword.Type, Name.Namespace, Name.Class, Punctuation),
+ '#pop'),
+ (r'[^%s\[)L]+' % _separator, Keyword.Type, '#pop'),
+ default('#pop')
+ ],
+ 'descriptors/convert-dots': [
+ (r'\)', Punctuation, '#pop'),
+ default('descriptor/convert-dots')
+ ],
+ 'enclosing-method': [
+ (_ws, Text),
+ (r'(?=[^%s]*\()' % _separator, Text, ('#pop', 'invocation')),
+ default(('#pop', 'class/convert-dots'))
+ ],
+ 'exception': [
+ include('default'),
+ (r'((?:%s[/.])*)(%s)' % (_unqualified_name, _name),
+ bygroups(Name.Namespace, Name.Exception), '#pop')
+ ],
+ 'field': [
+ (r'static%s' % _break, Keyword.Reserved, ('#pop', 'static')),
+ include('default'),
+ (r'((?:%s[/.](?=[^%s]*[/.]))*)(%s[/.])?(%s)' %
+ (_unqualified_name, _separator, _unqualified_name, _name),
+ bygroups(Name.Namespace, Name.Class, Name.Variable.Instance),
+ '#pop')
+ ],
+ 'invocation': [
+ include('default'),
+ (r'((?:%s[/.](?=[^%s(]*[/.]))*)(%s[/.])?(%s)(\()' %
+ (_unqualified_name, _separator, _unqualified_name, _name),
+ bygroups(Name.Namespace, Name.Class, Name.Function, Punctuation),
+ ('#pop', 'descriptor/convert-dots', 'descriptors/convert-dots',
+ 'descriptor/convert-dots'))
+ ],
+ 'label': [
+ include('default'),
+ (_name, Name.Label, '#pop')
+ ],
+ 'method': [
+ include('default'),
+ (r'(%s)(\()' % _name, bygroups(Name.Function, Punctuation),
+ ('#pop', 'descriptor/convert-dots', 'descriptors/convert-dots',
+ 'descriptor/convert-dots'))
+ ],
+ 'no-verification': [
+ (r'(locals|method|stack)%s' % _break, Keyword.Reserved, '#pop'),
+ include('default')
+ ],
+ 'static': [
+ include('default'),
+ (r'((?:%s[/.](?=[^%s]*[/.]))*)(%s[/.])?(%s)' %
+ (_unqualified_name, _separator, _unqualified_name, _name),
+ bygroups(Name.Namespace, Name.Class, Name.Variable.Class), '#pop')
+ ],
+ 'table': [
+ (r'\n+', Text),
+ (r'default%s' % _break, Keyword.Reserved, '#pop'),
+ include('default'),
+ (_name, Name.Label)
+ ],
+ 'var': [
+ include('default'),
+ (_name, Name.Variable, '#pop')
+ ],
+ 'verification': [
+ include('default'),
+ (r'(Double|Float|Integer|Long|Null|Top|UninitializedThis)%s' %
+ _break, Keyword, '#pop'),
+ (r'Object%s' % _break, Keyword, ('#pop', 'class/no-dots')),
+ (r'Uninitialized%s' % _break, Keyword, ('#pop', 'label'))
+ ]
+ }
+
+ def analyse_text(text):
+ score = 0
+ if re.search(r'^\s*\.class\s', text, re.MULTILINE):
+ score += 0.5
+ if re.search(r'^\s*[a-z]+_[a-z]+\b', text, re.MULTILINE):
+ score += 0.3
+ if re.search(r'^\s*\.(attribute|bytecode|debug|deprecated|enclosing|'
+ r'inner|interface|limit|set|signature|stack)\b', text,
+ re.MULTILINE):
+ score += 0.6
+ return score
diff --git a/wandb/vendor/pygments/lexers/lisp.py b/wandb/vendor/pygments/lexers/lisp.py
new file mode 100644
index 0000000000000000000000000000000000000000..e258c347bbfc71d79009bd8d79fcba4141296b50
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/lisp.py
@@ -0,0 +1,2621 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.lisp
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Lispy languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, words, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Literal, Error
+
+from pygments.lexers.python import PythonLexer
+
+__all__ = ['SchemeLexer', 'CommonLispLexer', 'HyLexer', 'RacketLexer',
+ 'NewLispLexer', 'EmacsLispLexer', 'ShenLexer', 'CPSALexer',
+ 'XtlangLexer']
+
+
+class SchemeLexer(RegexLexer):
+ """
+ A Scheme lexer, parsing a stream and outputting the tokens
+ needed to highlight scheme code.
+ This lexer could be most probably easily subclassed to parse
+ other LISP-Dialects like Common Lisp, Emacs Lisp or AutoLisp.
+
+ This parser is checked with pastes from the LISP pastebin
+ at http://paste.lisp.org/ to cover as much syntax as possible.
+
+ It supports the full Scheme syntax as defined in R5RS.
+
+ .. versionadded:: 0.6
+ """
+ name = 'Scheme'
+ aliases = ['scheme', 'scm']
+ filenames = ['*.scm', '*.ss']
+ mimetypes = ['text/x-scheme', 'application/x-scheme']
+
+ # list of known keywords and builtins taken form vim 6.4 scheme.vim
+ # syntax file.
+ keywords = (
+ 'lambda', 'define', 'if', 'else', 'cond', 'and', 'or', 'case', 'let',
+ 'let*', 'letrec', 'begin', 'do', 'delay', 'set!', '=>', 'quote',
+ 'quasiquote', 'unquote', 'unquote-splicing', 'define-syntax',
+ 'let-syntax', 'letrec-syntax', 'syntax-rules'
+ )
+ builtins = (
+ '*', '+', '-', '/', '<', '<=', '=', '>', '>=', 'abs', 'acos', 'angle',
+ 'append', 'apply', 'asin', 'assoc', 'assq', 'assv', 'atan',
+ 'boolean?', 'caaaar', 'caaadr', 'caaar', 'caadar', 'caaddr', 'caadr',
+ 'caar', 'cadaar', 'cadadr', 'cadar', 'caddar', 'cadddr', 'caddr',
+ 'cadr', 'call-with-current-continuation', 'call-with-input-file',
+ 'call-with-output-file', 'call-with-values', 'call/cc', 'car',
+ 'cdaaar', 'cdaadr', 'cdaar', 'cdadar', 'cdaddr', 'cdadr', 'cdar',
+ 'cddaar', 'cddadr', 'cddar', 'cdddar', 'cddddr', 'cdddr', 'cddr',
+ 'cdr', 'ceiling', 'char->integer', 'char-alphabetic?', 'char-ci<=?',
+ 'char-ci<?', 'char-ci=?', 'char-ci>=?', 'char-ci>?', 'char-downcase',
+ 'char-lower-case?', 'char-numeric?', 'char-ready?', 'char-upcase',
+ 'char-upper-case?', 'char-whitespace?', 'char<=?', 'char<?', 'char=?',
+ 'char>=?', 'char>?', 'char?', 'close-input-port', 'close-output-port',
+ 'complex?', 'cons', 'cos', 'current-input-port', 'current-output-port',
+ 'denominator', 'display', 'dynamic-wind', 'eof-object?', 'eq?',
+ 'equal?', 'eqv?', 'eval', 'even?', 'exact->inexact', 'exact?', 'exp',
+ 'expt', 'floor', 'for-each', 'force', 'gcd', 'imag-part',
+ 'inexact->exact', 'inexact?', 'input-port?', 'integer->char',
+ 'integer?', 'interaction-environment', 'lcm', 'length', 'list',
+ 'list->string', 'list->vector', 'list-ref', 'list-tail', 'list?',
+ 'load', 'log', 'magnitude', 'make-polar', 'make-rectangular',
+ 'make-string', 'make-vector', 'map', 'max', 'member', 'memq', 'memv',
+ 'min', 'modulo', 'negative?', 'newline', 'not', 'null-environment',
+ 'null?', 'number->string', 'number?', 'numerator', 'odd?',
+ 'open-input-file', 'open-output-file', 'output-port?', 'pair?',
+ 'peek-char', 'port?', 'positive?', 'procedure?', 'quotient',
+ 'rational?', 'rationalize', 'read', 'read-char', 'real-part', 'real?',
+ 'remainder', 'reverse', 'round', 'scheme-report-environment',
+ 'set-car!', 'set-cdr!', 'sin', 'sqrt', 'string', 'string->list',
+ 'string->number', 'string->symbol', 'string-append', 'string-ci<=?',
+ 'string-ci<?', 'string-ci=?', 'string-ci>=?', 'string-ci>?',
+ 'string-copy', 'string-fill!', 'string-length', 'string-ref',
+ 'string-set!', 'string<=?', 'string<?', 'string=?', 'string>=?',
+ 'string>?', 'string?', 'substring', 'symbol->string', 'symbol?',
+ 'tan', 'transcript-off', 'transcript-on', 'truncate', 'values',
+ 'vector', 'vector->list', 'vector-fill!', 'vector-length',
+ 'vector-ref', 'vector-set!', 'vector?', 'with-input-from-file',
+ 'with-output-to-file', 'write', 'write-char', 'zero?'
+ )
+
+ # valid names for identifiers
+ # well, names can only not consist fully of numbers
+ # but this should be good enough for now
+ valid_name = r'[\w!$%&*+,/:<=>?@^~|-]+'
+
+ tokens = {
+ 'root': [
+ # the comments
+ # and going to the end of the line
+ (r';.*$', Comment.Single),
+ # multi-line comment
+ (r'#\|', Comment.Multiline, 'multiline-comment'),
+ # commented form (entire sexpr folliwng)
+ (r'#;\s*\(', Comment, 'commented-form'),
+ # signifies that the program text that follows is written with the
+ # lexical and datum syntax described in r6rs
+ (r'#!r6rs', Comment),
+
+ # whitespaces - usually not relevant
+ (r'\s+', Text),
+
+ # numbers
+ (r'-?\d+\.\d+', Number.Float),
+ (r'-?\d+', Number.Integer),
+ # support for uncommon kinds of numbers -
+ # have to figure out what the characters mean
+ # (r'(#e|#i|#b|#o|#d|#x)[\d.]+', Number),
+
+ # strings, symbols and characters
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r"'" + valid_name, String.Symbol),
+ (r"#\\([()/'\"._!§$%& ?=+-]|[a-zA-Z0-9]+)", String.Char),
+
+ # constants
+ (r'(#t|#f)', Name.Constant),
+
+ # special operators
+ (r"('|#|`|,@|,|\.)", Operator),
+
+ # highlight the keywords
+ ('(%s)' % '|'.join(re.escape(entry) + ' ' for entry in keywords),
+ Keyword),
+
+ # first variable in a quoted string like
+ # '(this is syntactic sugar)
+ (r"(?<='\()" + valid_name, Name.Variable),
+ (r"(?<=#\()" + valid_name, Name.Variable),
+
+ # highlight the builtins
+ ("(?<=\()(%s)" % '|'.join(re.escape(entry) + ' ' for entry in builtins),
+ Name.Builtin),
+
+ # the remaining functions
+ (r'(?<=\()' + valid_name, Name.Function),
+ # find the remaining variables
+ (valid_name, Name.Variable),
+
+ # the famous parentheses!
+ (r'(\(|\))', Punctuation),
+ (r'(\[|\])', Punctuation),
+ ],
+ 'multiline-comment': [
+ (r'#\|', Comment.Multiline, '#push'),
+ (r'\|#', Comment.Multiline, '#pop'),
+ (r'[^|#]+', Comment.Multiline),
+ (r'[|#]', Comment.Multiline),
+ ],
+ 'commented-form': [
+ (r'\(', Comment, '#push'),
+ (r'\)', Comment, '#pop'),
+ (r'[^()]+', Comment),
+ ],
+ }
+
+
+class CommonLispLexer(RegexLexer):
+ """
+ A Common Lisp lexer.
+
+ .. versionadded:: 0.9
+ """
+ name = 'Common Lisp'
+ aliases = ['common-lisp', 'cl', 'lisp']
+ filenames = ['*.cl', '*.lisp']
+ mimetypes = ['text/x-common-lisp']
+
+ flags = re.IGNORECASE | re.MULTILINE
+
+ # couple of useful regexes
+
+ # characters that are not macro-characters and can be used to begin a symbol
+ nonmacro = r'\\.|[\w!$%&*+-/<=>?@\[\]^{}~]'
+ constituent = nonmacro + '|[#.:]'
+ terminated = r'(?=[ "()\'\n,;`])' # whitespace or terminating macro characters
+
+ # symbol token, reverse-engineered from hyperspec
+ # Take a deep breath...
+ symbol = r'(\|[^|]+\||(?:%s)(?:%s)*)' % (nonmacro, constituent)
+
+ def __init__(self, **options):
+ from pygments.lexers._cl_builtins import BUILTIN_FUNCTIONS, \
+ SPECIAL_FORMS, MACROS, LAMBDA_LIST_KEYWORDS, DECLARATIONS, \
+ BUILTIN_TYPES, BUILTIN_CLASSES
+ self.builtin_function = BUILTIN_FUNCTIONS
+ self.special_forms = SPECIAL_FORMS
+ self.macros = MACROS
+ self.lambda_list_keywords = LAMBDA_LIST_KEYWORDS
+ self.declarations = DECLARATIONS
+ self.builtin_types = BUILTIN_TYPES
+ self.builtin_classes = BUILTIN_CLASSES
+ RegexLexer.__init__(self, **options)
+
+ def get_tokens_unprocessed(self, text):
+ stack = ['root']
+ for index, token, value in RegexLexer.get_tokens_unprocessed(self, text, stack):
+ if token is Name.Variable:
+ if value in self.builtin_function:
+ yield index, Name.Builtin, value
+ continue
+ if value in self.special_forms:
+ yield index, Keyword, value
+ continue
+ if value in self.macros:
+ yield index, Name.Builtin, value
+ continue
+ if value in self.lambda_list_keywords:
+ yield index, Keyword, value
+ continue
+ if value in self.declarations:
+ yield index, Keyword, value
+ continue
+ if value in self.builtin_types:
+ yield index, Keyword.Type, value
+ continue
+ if value in self.builtin_classes:
+ yield index, Name.Class, value
+ continue
+ yield index, token, value
+
+ tokens = {
+ 'root': [
+ default('body'),
+ ],
+ 'multiline-comment': [
+ (r'#\|', Comment.Multiline, '#push'), # (cf. Hyperspec 2.4.8.19)
+ (r'\|#', Comment.Multiline, '#pop'),
+ (r'[^|#]+', Comment.Multiline),
+ (r'[|#]', Comment.Multiline),
+ ],
+ 'commented-form': [
+ (r'\(', Comment.Preproc, '#push'),
+ (r'\)', Comment.Preproc, '#pop'),
+ (r'[^()]+', Comment.Preproc),
+ ],
+ 'body': [
+ # whitespace
+ (r'\s+', Text),
+
+ # single-line comment
+ (r';.*$', Comment.Single),
+
+ # multi-line comment
+ (r'#\|', Comment.Multiline, 'multiline-comment'),
+
+ # encoding comment (?)
+ (r'#\d*Y.*$', Comment.Special),
+
+ # strings and characters
+ (r'"(\\.|\\\n|[^"\\])*"', String),
+ # quoting
+ (r":" + symbol, String.Symbol),
+ (r"::" + symbol, String.Symbol),
+ (r":#" + symbol, String.Symbol),
+ (r"'" + symbol, String.Symbol),
+ (r"'", Operator),
+ (r"`", Operator),
+
+ # decimal numbers
+ (r'[-+]?\d+\.?' + terminated, Number.Integer),
+ (r'[-+]?\d+/\d+' + terminated, Number),
+ (r'[-+]?(\d*\.\d+([defls][-+]?\d+)?|\d+(\.\d*)?[defls][-+]?\d+)' +
+ terminated, Number.Float),
+
+ # sharpsign strings and characters
+ (r"#\\." + terminated, String.Char),
+ (r"#\\" + symbol, String.Char),
+
+ # vector
+ (r'#\(', Operator, 'body'),
+
+ # bitstring
+ (r'#\d*\*[01]*', Literal.Other),
+
+ # uninterned symbol
+ (r'#:' + symbol, String.Symbol),
+
+ # read-time and load-time evaluation
+ (r'#[.,]', Operator),
+
+ # function shorthand
+ (r'#\'', Name.Function),
+
+ # binary rational
+ (r'#b[+-]?[01]+(/[01]+)?', Number.Bin),
+
+ # octal rational
+ (r'#o[+-]?[0-7]+(/[0-7]+)?', Number.Oct),
+
+ # hex rational
+ (r'#x[+-]?[0-9a-f]+(/[0-9a-f]+)?', Number.Hex),
+
+ # radix rational
+ (r'#\d+r[+-]?[0-9a-z]+(/[0-9a-z]+)?', Number),
+
+ # complex
+ (r'(#c)(\()', bygroups(Number, Punctuation), 'body'),
+
+ # array
+ (r'(#\d+a)(\()', bygroups(Literal.Other, Punctuation), 'body'),
+
+ # structure
+ (r'(#s)(\()', bygroups(Literal.Other, Punctuation), 'body'),
+
+ # path
+ (r'#p?"(\\.|[^"])*"', Literal.Other),
+
+ # reference
+ (r'#\d+=', Operator),
+ (r'#\d+#', Operator),
+
+ # read-time comment
+ (r'#+nil' + terminated + '\s*\(', Comment.Preproc, 'commented-form'),
+
+ # read-time conditional
+ (r'#[+-]', Operator),
+
+ # special operators that should have been parsed already
+ (r'(,@|,|\.)', Operator),
+
+ # special constants
+ (r'(t|nil)' + terminated, Name.Constant),
+
+ # functions and variables
+ (r'\*' + symbol + '\*', Name.Variable.Global),
+ (symbol, Name.Variable),
+
+ # parentheses
+ (r'\(', Punctuation, 'body'),
+ (r'\)', Punctuation, '#pop'),
+ ],
+ }
+
+
+class HyLexer(RegexLexer):
+ """
+ Lexer for `Hy <http://hylang.org/>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Hy'
+ aliases = ['hylang']
+ filenames = ['*.hy']
+ mimetypes = ['text/x-hy', 'application/x-hy']
+
+ special_forms = (
+ 'cond', 'for', '->', '->>', 'car',
+ 'cdr', 'first', 'rest', 'let', 'when', 'unless',
+ 'import', 'do', 'progn', 'get', 'slice', 'assoc', 'with-decorator',
+ ',', 'list_comp', 'kwapply', '~', 'is', 'in', 'is-not', 'not-in',
+ 'quasiquote', 'unquote', 'unquote-splice', 'quote', '|', '<<=', '>>=',
+ 'foreach', 'while',
+ 'eval-and-compile', 'eval-when-compile'
+ )
+
+ declarations = (
+ 'def', 'defn', 'defun', 'defmacro', 'defclass', 'lambda', 'fn', 'setv'
+ )
+
+ hy_builtins = ()
+
+ hy_core = (
+ 'cycle', 'dec', 'distinct', 'drop', 'even?', 'filter', 'inc',
+ 'instance?', 'iterable?', 'iterate', 'iterator?', 'neg?',
+ 'none?', 'nth', 'numeric?', 'odd?', 'pos?', 'remove', 'repeat',
+ 'repeatedly', 'take', 'take_nth', 'take_while', 'zero?'
+ )
+
+ builtins = hy_builtins + hy_core
+
+ # valid names for identifiers
+ # well, names can only not consist fully of numbers
+ # but this should be good enough for now
+ valid_name = r'(?!#)[\w!$%*+<=>?/.#-]+'
+
+ def _multi_escape(entries):
+ return words(entries, suffix=' ')
+
+ tokens = {
+ 'root': [
+ # the comments - always starting with semicolon
+ # and going to the end of the line
+ (r';.*$', Comment.Single),
+
+ # whitespaces - usually not relevant
+ (r'[,\s]+', Text),
+
+ # numbers
+ (r'-?\d+\.\d+', Number.Float),
+ (r'-?\d+', Number.Integer),
+ (r'0[0-7]+j?', Number.Oct),
+ (r'0[xX][a-fA-F0-9]+', Number.Hex),
+
+ # strings, symbols and characters
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r"'" + valid_name, String.Symbol),
+ (r"\\(.|[a-z]+)", String.Char),
+ (r'^(\s*)([rRuU]{,2}"""(?:.|\n)*?""")', bygroups(Text, String.Doc)),
+ (r"^(\s*)([rRuU]{,2}'''(?:.|\n)*?''')", bygroups(Text, String.Doc)),
+
+ # keywords
+ (r'::?' + valid_name, String.Symbol),
+
+ # special operators
+ (r'~@|[`\'#^~&@]', Operator),
+
+ include('py-keywords'),
+ include('py-builtins'),
+
+ # highlight the special forms
+ (_multi_escape(special_forms), Keyword),
+
+ # Technically, only the special forms are 'keywords'. The problem
+ # is that only treating them as keywords means that things like
+ # 'defn' and 'ns' need to be highlighted as builtins. This is ugly
+ # and weird for most styles. So, as a compromise we're going to
+ # highlight them as Keyword.Declarations.
+ (_multi_escape(declarations), Keyword.Declaration),
+
+ # highlight the builtins
+ (_multi_escape(builtins), Name.Builtin),
+
+ # the remaining functions
+ (r'(?<=\()' + valid_name, Name.Function),
+
+ # find the remaining variables
+ (valid_name, Name.Variable),
+
+ # Hy accepts vector notation
+ (r'(\[|\])', Punctuation),
+
+ # Hy accepts map notation
+ (r'(\{|\})', Punctuation),
+
+ # the famous parentheses!
+ (r'(\(|\))', Punctuation),
+
+ ],
+ 'py-keywords': PythonLexer.tokens['keywords'],
+ 'py-builtins': PythonLexer.tokens['builtins'],
+ }
+
+ def analyse_text(text):
+ if '(import ' in text or '(defn ' in text:
+ return 0.9
+
+
+class RacketLexer(RegexLexer):
+ """
+ Lexer for `Racket <http://racket-lang.org/>`_ source code (formerly
+ known as PLT Scheme).
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'Racket'
+ aliases = ['racket', 'rkt']
+ filenames = ['*.rkt', '*.rktd', '*.rktl']
+ mimetypes = ['text/x-racket', 'application/x-racket']
+
+ # Generated by example.rkt
+ _keywords = (
+ u'#%app', u'#%datum', u'#%declare', u'#%expression', u'#%module-begin',
+ u'#%plain-app', u'#%plain-lambda', u'#%plain-module-begin',
+ u'#%printing-module-begin', u'#%provide', u'#%require',
+ u'#%stratified-body', u'#%top', u'#%top-interaction',
+ u'#%variable-reference', u'->', u'->*', u'->*m', u'->d', u'->dm', u'->i',
+ u'->m', u'...', u':do-in', u'==', u'=>', u'_', u'absent', u'abstract',
+ u'all-defined-out', u'all-from-out', u'and', u'any', u'augment', u'augment*',
+ u'augment-final', u'augment-final*', u'augride', u'augride*', u'begin',
+ u'begin-for-syntax', u'begin0', u'case', u'case->', u'case->m',
+ u'case-lambda', u'class', u'class*', u'class-field-accessor',
+ u'class-field-mutator', u'class/c', u'class/derived', u'combine-in',
+ u'combine-out', u'command-line', u'compound-unit', u'compound-unit/infer',
+ u'cond', u'cons/dc', u'contract', u'contract-out', u'contract-struct',
+ u'contracted', u'define', u'define-compound-unit',
+ u'define-compound-unit/infer', u'define-contract-struct',
+ u'define-custom-hash-types', u'define-custom-set-types',
+ u'define-for-syntax', u'define-local-member-name', u'define-logger',
+ u'define-match-expander', u'define-member-name',
+ u'define-module-boundary-contract', u'define-namespace-anchor',
+ u'define-opt/c', u'define-sequence-syntax', u'define-serializable-class',
+ u'define-serializable-class*', u'define-signature',
+ u'define-signature-form', u'define-struct', u'define-struct/contract',
+ u'define-struct/derived', u'define-syntax', u'define-syntax-rule',
+ u'define-syntaxes', u'define-unit', u'define-unit-binding',
+ u'define-unit-from-context', u'define-unit/contract',
+ u'define-unit/new-import-export', u'define-unit/s', u'define-values',
+ u'define-values-for-export', u'define-values-for-syntax',
+ u'define-values/invoke-unit', u'define-values/invoke-unit/infer',
+ u'define/augment', u'define/augment-final', u'define/augride',
+ u'define/contract', u'define/final-prop', u'define/match',
+ u'define/overment', u'define/override', u'define/override-final',
+ u'define/private', u'define/public', u'define/public-final',
+ u'define/pubment', u'define/subexpression-pos-prop',
+ u'define/subexpression-pos-prop/name', u'delay', u'delay/idle',
+ u'delay/name', u'delay/strict', u'delay/sync', u'delay/thread', u'do',
+ u'else', u'except', u'except-in', u'except-out', u'export', u'extends',
+ u'failure-cont', u'false', u'false/c', u'field', u'field-bound?', u'file',
+ u'flat-murec-contract', u'flat-rec-contract', u'for', u'for*', u'for*/and',
+ u'for*/async', u'for*/first', u'for*/fold', u'for*/fold/derived',
+ u'for*/hash', u'for*/hasheq', u'for*/hasheqv', u'for*/last', u'for*/list',
+ u'for*/lists', u'for*/mutable-set', u'for*/mutable-seteq',
+ u'for*/mutable-seteqv', u'for*/or', u'for*/product', u'for*/set',
+ u'for*/seteq', u'for*/seteqv', u'for*/stream', u'for*/sum', u'for*/vector',
+ u'for*/weak-set', u'for*/weak-seteq', u'for*/weak-seteqv', u'for-label',
+ u'for-meta', u'for-syntax', u'for-template', u'for/and', u'for/async',
+ u'for/first', u'for/fold', u'for/fold/derived', u'for/hash', u'for/hasheq',
+ u'for/hasheqv', u'for/last', u'for/list', u'for/lists', u'for/mutable-set',
+ u'for/mutable-seteq', u'for/mutable-seteqv', u'for/or', u'for/product',
+ u'for/set', u'for/seteq', u'for/seteqv', u'for/stream', u'for/sum',
+ u'for/vector', u'for/weak-set', u'for/weak-seteq', u'for/weak-seteqv',
+ u'gen:custom-write', u'gen:dict', u'gen:equal+hash', u'gen:set',
+ u'gen:stream', u'generic', u'get-field', u'hash/dc', u'if', u'implies',
+ u'import', u'include', u'include-at/relative-to',
+ u'include-at/relative-to/reader', u'include/reader', u'inherit',
+ u'inherit-field', u'inherit/inner', u'inherit/super', u'init',
+ u'init-depend', u'init-field', u'init-rest', u'inner', u'inspect',
+ u'instantiate', u'interface', u'interface*', u'invariant-assertion',
+ u'invoke-unit', u'invoke-unit/infer', u'lambda', u'lazy', u'let', u'let*',
+ u'let*-values', u'let-syntax', u'let-syntaxes', u'let-values', u'let/cc',
+ u'let/ec', u'letrec', u'letrec-syntax', u'letrec-syntaxes',
+ u'letrec-syntaxes+values', u'letrec-values', u'lib', u'link', u'local',
+ u'local-require', u'log-debug', u'log-error', u'log-fatal', u'log-info',
+ u'log-warning', u'match', u'match*', u'match*/derived', u'match-define',
+ u'match-define-values', u'match-lambda', u'match-lambda*',
+ u'match-lambda**', u'match-let', u'match-let*', u'match-let*-values',
+ u'match-let-values', u'match-letrec', u'match-letrec-values',
+ u'match/derived', u'match/values', u'member-name-key', u'mixin', u'module',
+ u'module*', u'module+', u'nand', u'new', u'nor', u'object-contract',
+ u'object/c', u'only', u'only-in', u'only-meta-in', u'open', u'opt/c', u'or',
+ u'overment', u'overment*', u'override', u'override*', u'override-final',
+ u'override-final*', u'parameterize', u'parameterize*',
+ u'parameterize-break', u'parametric->/c', u'place', u'place*',
+ u'place/context', u'planet', u'prefix', u'prefix-in', u'prefix-out',
+ u'private', u'private*', u'prompt-tag/c', u'protect-out', u'provide',
+ u'provide-signature-elements', u'provide/contract', u'public', u'public*',
+ u'public-final', u'public-final*', u'pubment', u'pubment*', u'quasiquote',
+ u'quasisyntax', u'quasisyntax/loc', u'quote', u'quote-syntax',
+ u'quote-syntax/prune', u'recontract-out', u'recursive-contract',
+ u'relative-in', u'rename', u'rename-in', u'rename-inner', u'rename-out',
+ u'rename-super', u'require', u'send', u'send*', u'send+', u'send-generic',
+ u'send/apply', u'send/keyword-apply', u'set!', u'set!-values',
+ u'set-field!', u'shared', u'stream', u'stream*', u'stream-cons', u'struct',
+ u'struct*', u'struct-copy', u'struct-field-index', u'struct-out',
+ u'struct/c', u'struct/ctc', u'struct/dc', u'submod', u'super',
+ u'super-instantiate', u'super-make-object', u'super-new', u'syntax',
+ u'syntax-case', u'syntax-case*', u'syntax-id-rules', u'syntax-rules',
+ u'syntax/loc', u'tag', u'this', u'this%', u'thunk', u'thunk*', u'time',
+ u'unconstrained-domain->', u'unit', u'unit-from-context', u'unit/c',
+ u'unit/new-import-export', u'unit/s', u'unless', u'unquote',
+ u'unquote-splicing', u'unsyntax', u'unsyntax-splicing', u'values/drop',
+ u'when', u'with-continuation-mark', u'with-contract',
+ u'with-contract-continuation-mark', u'with-handlers', u'with-handlers*',
+ u'with-method', u'with-syntax', u'λ'
+ )
+
+ # Generated by example.rkt
+ _builtins = (
+ u'*', u'*list/c', u'+', u'-', u'/', u'<', u'</c', u'<=', u'<=/c', u'=', u'=/c',
+ u'>', u'>/c', u'>=', u'>=/c', u'abort-current-continuation', u'abs',
+ u'absolute-path?', u'acos', u'add-between', u'add1', u'alarm-evt',
+ u'always-evt', u'and/c', u'andmap', u'angle', u'any/c', u'append', u'append*',
+ u'append-map', u'apply', u'argmax', u'argmin', u'arithmetic-shift',
+ u'arity-at-least', u'arity-at-least-value', u'arity-at-least?',
+ u'arity-checking-wrapper', u'arity-includes?', u'arity=?',
+ u'arrow-contract-info', u'arrow-contract-info-accepts-arglist',
+ u'arrow-contract-info-chaperone-procedure',
+ u'arrow-contract-info-check-first-order', u'arrow-contract-info?',
+ u'asin', u'assf', u'assoc', u'assq', u'assv', u'atan',
+ u'bad-number-of-results', u'banner', u'base->-doms/c', u'base->-rngs/c',
+ u'base->?', u'between/c', u'bitwise-and', u'bitwise-bit-field',
+ u'bitwise-bit-set?', u'bitwise-ior', u'bitwise-not', u'bitwise-xor',
+ u'blame-add-car-context', u'blame-add-cdr-context', u'blame-add-context',
+ u'blame-add-missing-party', u'blame-add-nth-arg-context',
+ u'blame-add-range-context', u'blame-add-unknown-context',
+ u'blame-context', u'blame-contract', u'blame-fmt->-string',
+ u'blame-missing-party?', u'blame-negative', u'blame-original?',
+ u'blame-positive', u'blame-replace-negative', u'blame-source',
+ u'blame-swap', u'blame-swapped?', u'blame-update', u'blame-value',
+ u'blame?', u'boolean=?', u'boolean?', u'bound-identifier=?', u'box',
+ u'box-cas!', u'box-immutable', u'box-immutable/c', u'box/c', u'box?',
+ u'break-enabled', u'break-parameterization?', u'break-thread',
+ u'build-chaperone-contract-property', u'build-compound-type-name',
+ u'build-contract-property', u'build-flat-contract-property',
+ u'build-list', u'build-path', u'build-path/convention-type',
+ u'build-string', u'build-vector', u'byte-pregexp', u'byte-pregexp?',
+ u'byte-ready?', u'byte-regexp', u'byte-regexp?', u'byte?', u'bytes',
+ u'bytes->immutable-bytes', u'bytes->list', u'bytes->path',
+ u'bytes->path-element', u'bytes->string/latin-1', u'bytes->string/locale',
+ u'bytes->string/utf-8', u'bytes-append', u'bytes-append*',
+ u'bytes-close-converter', u'bytes-convert', u'bytes-convert-end',
+ u'bytes-converter?', u'bytes-copy', u'bytes-copy!',
+ u'bytes-environment-variable-name?', u'bytes-fill!', u'bytes-join',
+ u'bytes-length', u'bytes-no-nuls?', u'bytes-open-converter', u'bytes-ref',
+ u'bytes-set!', u'bytes-utf-8-index', u'bytes-utf-8-length',
+ u'bytes-utf-8-ref', u'bytes<?', u'bytes=?', u'bytes>?', u'bytes?', u'caaaar',
+ u'caaadr', u'caaar', u'caadar', u'caaddr', u'caadr', u'caar', u'cadaar',
+ u'cadadr', u'cadar', u'caddar', u'cadddr', u'caddr', u'cadr',
+ u'call-in-nested-thread', u'call-with-atomic-output-file',
+ u'call-with-break-parameterization',
+ u'call-with-composable-continuation', u'call-with-continuation-barrier',
+ u'call-with-continuation-prompt', u'call-with-current-continuation',
+ u'call-with-default-reading-parameterization',
+ u'call-with-escape-continuation', u'call-with-exception-handler',
+ u'call-with-file-lock/timeout', u'call-with-immediate-continuation-mark',
+ u'call-with-input-bytes', u'call-with-input-file',
+ u'call-with-input-file*', u'call-with-input-string',
+ u'call-with-output-bytes', u'call-with-output-file',
+ u'call-with-output-file*', u'call-with-output-string',
+ u'call-with-parameterization', u'call-with-semaphore',
+ u'call-with-semaphore/enable-break', u'call-with-values', u'call/cc',
+ u'call/ec', u'car', u'cartesian-product', u'cdaaar', u'cdaadr', u'cdaar',
+ u'cdadar', u'cdaddr', u'cdadr', u'cdar', u'cddaar', u'cddadr', u'cddar',
+ u'cdddar', u'cddddr', u'cdddr', u'cddr', u'cdr', u'ceiling', u'channel-get',
+ u'channel-put', u'channel-put-evt', u'channel-put-evt?',
+ u'channel-try-get', u'channel/c', u'channel?', u'chaperone-box',
+ u'chaperone-channel', u'chaperone-continuation-mark-key',
+ u'chaperone-contract-property?', u'chaperone-contract?', u'chaperone-evt',
+ u'chaperone-hash', u'chaperone-hash-set', u'chaperone-of?',
+ u'chaperone-procedure', u'chaperone-procedure*', u'chaperone-prompt-tag',
+ u'chaperone-struct', u'chaperone-struct-type', u'chaperone-vector',
+ u'chaperone?', u'char->integer', u'char-alphabetic?', u'char-blank?',
+ u'char-ci<=?', u'char-ci<?', u'char-ci=?', u'char-ci>=?', u'char-ci>?',
+ u'char-downcase', u'char-foldcase', u'char-general-category',
+ u'char-graphic?', u'char-in', u'char-in/c', u'char-iso-control?',
+ u'char-lower-case?', u'char-numeric?', u'char-punctuation?',
+ u'char-ready?', u'char-symbolic?', u'char-title-case?', u'char-titlecase',
+ u'char-upcase', u'char-upper-case?', u'char-utf-8-length',
+ u'char-whitespace?', u'char<=?', u'char<?', u'char=?', u'char>=?', u'char>?',
+ u'char?', u'check-duplicate-identifier', u'check-duplicates',
+ u'checked-procedure-check-and-extract', u'choice-evt',
+ u'class->interface', u'class-info', u'class-seal', u'class-unseal',
+ u'class?', u'cleanse-path', u'close-input-port', u'close-output-port',
+ u'coerce-chaperone-contract', u'coerce-chaperone-contracts',
+ u'coerce-contract', u'coerce-contract/f', u'coerce-contracts',
+ u'coerce-flat-contract', u'coerce-flat-contracts', u'collect-garbage',
+ u'collection-file-path', u'collection-path', u'combinations', u'compile',
+ u'compile-allow-set!-undefined', u'compile-context-preservation-enabled',
+ u'compile-enforce-module-constants', u'compile-syntax',
+ u'compiled-expression-recompile', u'compiled-expression?',
+ u'compiled-module-expression?', u'complete-path?', u'complex?', u'compose',
+ u'compose1', u'conjoin', u'conjugate', u'cons', u'cons/c', u'cons?', u'const',
+ u'continuation-mark-key/c', u'continuation-mark-key?',
+ u'continuation-mark-set->context', u'continuation-mark-set->list',
+ u'continuation-mark-set->list*', u'continuation-mark-set-first',
+ u'continuation-mark-set?', u'continuation-marks',
+ u'continuation-prompt-available?', u'continuation-prompt-tag?',
+ u'continuation?', u'contract-continuation-mark-key',
+ u'contract-custom-write-property-proc', u'contract-exercise',
+ u'contract-first-order', u'contract-first-order-passes?',
+ u'contract-late-neg-projection', u'contract-name', u'contract-proc',
+ u'contract-projection', u'contract-property?',
+ u'contract-random-generate', u'contract-random-generate-fail',
+ u'contract-random-generate-fail?',
+ u'contract-random-generate-get-current-environment',
+ u'contract-random-generate-stash', u'contract-random-generate/choose',
+ u'contract-stronger?', u'contract-struct-exercise',
+ u'contract-struct-generate', u'contract-struct-late-neg-projection',
+ u'contract-struct-list-contract?', u'contract-val-first-projection',
+ u'contract?', u'convert-stream', u'copy-directory/files', u'copy-file',
+ u'copy-port', u'cos', u'cosh', u'count', u'current-blame-format',
+ u'current-break-parameterization', u'current-code-inspector',
+ u'current-command-line-arguments', u'current-compile',
+ u'current-compiled-file-roots', u'current-continuation-marks',
+ u'current-contract-region', u'current-custodian', u'current-directory',
+ u'current-directory-for-user', u'current-drive',
+ u'current-environment-variables', u'current-error-port', u'current-eval',
+ u'current-evt-pseudo-random-generator',
+ u'current-force-delete-permissions', u'current-future',
+ u'current-gc-milliseconds', u'current-get-interaction-input-port',
+ u'current-inexact-milliseconds', u'current-input-port',
+ u'current-inspector', u'current-library-collection-links',
+ u'current-library-collection-paths', u'current-load',
+ u'current-load-extension', u'current-load-relative-directory',
+ u'current-load/use-compiled', u'current-locale', u'current-logger',
+ u'current-memory-use', u'current-milliseconds',
+ u'current-module-declare-name', u'current-module-declare-source',
+ u'current-module-name-resolver', u'current-module-path-for-load',
+ u'current-namespace', u'current-output-port', u'current-parameterization',
+ u'current-plumber', u'current-preserved-thread-cell-values',
+ u'current-print', u'current-process-milliseconds', u'current-prompt-read',
+ u'current-pseudo-random-generator', u'current-read-interaction',
+ u'current-reader-guard', u'current-readtable', u'current-seconds',
+ u'current-security-guard', u'current-subprocess-custodian-mode',
+ u'current-thread', u'current-thread-group',
+ u'current-thread-initial-stack-size',
+ u'current-write-relative-directory', u'curry', u'curryr',
+ u'custodian-box-value', u'custodian-box?', u'custodian-limit-memory',
+ u'custodian-managed-list', u'custodian-memory-accounting-available?',
+ u'custodian-require-memory', u'custodian-shutdown-all', u'custodian?',
+ u'custom-print-quotable-accessor', u'custom-print-quotable?',
+ u'custom-write-accessor', u'custom-write-property-proc', u'custom-write?',
+ u'date', u'date*', u'date*-nanosecond', u'date*-time-zone-name', u'date*?',
+ u'date-day', u'date-dst?', u'date-hour', u'date-minute', u'date-month',
+ u'date-second', u'date-time-zone-offset', u'date-week-day', u'date-year',
+ u'date-year-day', u'date?', u'datum->syntax', u'datum-intern-literal',
+ u'default-continuation-prompt-tag', u'degrees->radians',
+ u'delete-directory', u'delete-directory/files', u'delete-file',
+ u'denominator', u'dict->list', u'dict-can-functional-set?',
+ u'dict-can-remove-keys?', u'dict-clear', u'dict-clear!', u'dict-copy',
+ u'dict-count', u'dict-empty?', u'dict-for-each', u'dict-has-key?',
+ u'dict-implements/c', u'dict-implements?', u'dict-iter-contract',
+ u'dict-iterate-first', u'dict-iterate-key', u'dict-iterate-next',
+ u'dict-iterate-value', u'dict-key-contract', u'dict-keys', u'dict-map',
+ u'dict-mutable?', u'dict-ref', u'dict-ref!', u'dict-remove',
+ u'dict-remove!', u'dict-set', u'dict-set!', u'dict-set*', u'dict-set*!',
+ u'dict-update', u'dict-update!', u'dict-value-contract', u'dict-values',
+ u'dict?', u'directory-exists?', u'directory-list', u'disjoin', u'display',
+ u'display-lines', u'display-lines-to-file', u'display-to-file',
+ u'displayln', u'double-flonum?', u'drop', u'drop-common-prefix',
+ u'drop-right', u'dropf', u'dropf-right', u'dump-memory-stats',
+ u'dup-input-port', u'dup-output-port', u'dynamic->*', u'dynamic-get-field',
+ u'dynamic-object/c', u'dynamic-place', u'dynamic-place*',
+ u'dynamic-require', u'dynamic-require-for-syntax', u'dynamic-send',
+ u'dynamic-set-field!', u'dynamic-wind', u'eighth', u'empty',
+ u'empty-sequence', u'empty-stream', u'empty?',
+ u'environment-variables-copy', u'environment-variables-names',
+ u'environment-variables-ref', u'environment-variables-set!',
+ u'environment-variables?', u'eof', u'eof-evt', u'eof-object?',
+ u'ephemeron-value', u'ephemeron?', u'eprintf', u'eq-contract-val',
+ u'eq-contract?', u'eq-hash-code', u'eq?', u'equal-contract-val',
+ u'equal-contract?', u'equal-hash-code', u'equal-secondary-hash-code',
+ u'equal<%>', u'equal?', u'equal?/recur', u'eqv-hash-code', u'eqv?', u'error',
+ u'error-display-handler', u'error-escape-handler',
+ u'error-print-context-length', u'error-print-source-location',
+ u'error-print-width', u'error-value->string-handler', u'eval',
+ u'eval-jit-enabled', u'eval-syntax', u'even?', u'evt/c', u'evt?',
+ u'exact->inexact', u'exact-ceiling', u'exact-floor', u'exact-integer?',
+ u'exact-nonnegative-integer?', u'exact-positive-integer?', u'exact-round',
+ u'exact-truncate', u'exact?', u'executable-yield-handler', u'exit',
+ u'exit-handler', u'exn', u'exn-continuation-marks', u'exn-message',
+ u'exn:break', u'exn:break-continuation', u'exn:break:hang-up',
+ u'exn:break:hang-up?', u'exn:break:terminate', u'exn:break:terminate?',
+ u'exn:break?', u'exn:fail', u'exn:fail:contract',
+ u'exn:fail:contract:arity', u'exn:fail:contract:arity?',
+ u'exn:fail:contract:blame', u'exn:fail:contract:blame-object',
+ u'exn:fail:contract:blame?', u'exn:fail:contract:continuation',
+ u'exn:fail:contract:continuation?', u'exn:fail:contract:divide-by-zero',
+ u'exn:fail:contract:divide-by-zero?',
+ u'exn:fail:contract:non-fixnum-result',
+ u'exn:fail:contract:non-fixnum-result?', u'exn:fail:contract:variable',
+ u'exn:fail:contract:variable-id', u'exn:fail:contract:variable?',
+ u'exn:fail:contract?', u'exn:fail:filesystem',
+ u'exn:fail:filesystem:errno', u'exn:fail:filesystem:errno-errno',
+ u'exn:fail:filesystem:errno?', u'exn:fail:filesystem:exists',
+ u'exn:fail:filesystem:exists?', u'exn:fail:filesystem:missing-module',
+ u'exn:fail:filesystem:missing-module-path',
+ u'exn:fail:filesystem:missing-module?', u'exn:fail:filesystem:version',
+ u'exn:fail:filesystem:version?', u'exn:fail:filesystem?',
+ u'exn:fail:network', u'exn:fail:network:errno',
+ u'exn:fail:network:errno-errno', u'exn:fail:network:errno?',
+ u'exn:fail:network?', u'exn:fail:object', u'exn:fail:object?',
+ u'exn:fail:out-of-memory', u'exn:fail:out-of-memory?', u'exn:fail:read',
+ u'exn:fail:read-srclocs', u'exn:fail:read:eof', u'exn:fail:read:eof?',
+ u'exn:fail:read:non-char', u'exn:fail:read:non-char?', u'exn:fail:read?',
+ u'exn:fail:syntax', u'exn:fail:syntax-exprs',
+ u'exn:fail:syntax:missing-module',
+ u'exn:fail:syntax:missing-module-path',
+ u'exn:fail:syntax:missing-module?', u'exn:fail:syntax:unbound',
+ u'exn:fail:syntax:unbound?', u'exn:fail:syntax?', u'exn:fail:unsupported',
+ u'exn:fail:unsupported?', u'exn:fail:user', u'exn:fail:user?',
+ u'exn:fail?', u'exn:misc:match?', u'exn:missing-module-accessor',
+ u'exn:missing-module?', u'exn:srclocs-accessor', u'exn:srclocs?', u'exn?',
+ u'exp', u'expand', u'expand-once', u'expand-syntax', u'expand-syntax-once',
+ u'expand-syntax-to-top-form', u'expand-to-top-form', u'expand-user-path',
+ u'explode-path', u'expt', u'externalizable<%>', u'failure-result/c',
+ u'false?', u'field-names', u'fifth', u'file->bytes', u'file->bytes-lines',
+ u'file->lines', u'file->list', u'file->string', u'file->value',
+ u'file-exists?', u'file-name-from-path', u'file-or-directory-identity',
+ u'file-or-directory-modify-seconds', u'file-or-directory-permissions',
+ u'file-position', u'file-position*', u'file-size',
+ u'file-stream-buffer-mode', u'file-stream-port?', u'file-truncate',
+ u'filename-extension', u'filesystem-change-evt',
+ u'filesystem-change-evt-cancel', u'filesystem-change-evt?',
+ u'filesystem-root-list', u'filter', u'filter-map', u'filter-not',
+ u'filter-read-input-port', u'find-executable-path', u'find-files',
+ u'find-library-collection-links', u'find-library-collection-paths',
+ u'find-relative-path', u'find-system-path', u'findf', u'first',
+ u'first-or/c', u'fixnum?', u'flat-contract', u'flat-contract-predicate',
+ u'flat-contract-property?', u'flat-contract?', u'flat-named-contract',
+ u'flatten', u'floating-point-bytes->real', u'flonum?', u'floor',
+ u'flush-output', u'fold-files', u'foldl', u'foldr', u'for-each', u'force',
+ u'format', u'fourth', u'fprintf', u'free-identifier=?',
+ u'free-label-identifier=?', u'free-template-identifier=?',
+ u'free-transformer-identifier=?', u'fsemaphore-count', u'fsemaphore-post',
+ u'fsemaphore-try-wait?', u'fsemaphore-wait', u'fsemaphore?', u'future',
+ u'future?', u'futures-enabled?', u'gcd', u'generate-member-key',
+ u'generate-temporaries', u'generic-set?', u'generic?', u'gensym',
+ u'get-output-bytes', u'get-output-string', u'get-preference',
+ u'get/build-late-neg-projection', u'get/build-val-first-projection',
+ u'getenv', u'global-port-print-handler', u'group-by', u'group-execute-bit',
+ u'group-read-bit', u'group-write-bit', u'guard-evt', u'handle-evt',
+ u'handle-evt?', u'has-blame?', u'has-contract?', u'hash', u'hash->list',
+ u'hash-clear', u'hash-clear!', u'hash-copy', u'hash-copy-clear',
+ u'hash-count', u'hash-empty?', u'hash-eq?', u'hash-equal?', u'hash-eqv?',
+ u'hash-for-each', u'hash-has-key?', u'hash-iterate-first',
+ u'hash-iterate-key', u'hash-iterate-key+value', u'hash-iterate-next',
+ u'hash-iterate-pair', u'hash-iterate-value', u'hash-keys', u'hash-map',
+ u'hash-placeholder?', u'hash-ref', u'hash-ref!', u'hash-remove',
+ u'hash-remove!', u'hash-set', u'hash-set!', u'hash-set*', u'hash-set*!',
+ u'hash-update', u'hash-update!', u'hash-values', u'hash-weak?', u'hash/c',
+ u'hash?', u'hasheq', u'hasheqv', u'identifier-binding',
+ u'identifier-binding-symbol', u'identifier-label-binding',
+ u'identifier-prune-lexical-context',
+ u'identifier-prune-to-source-module',
+ u'identifier-remove-from-definition-context',
+ u'identifier-template-binding', u'identifier-transformer-binding',
+ u'identifier?', u'identity', u'if/c', u'imag-part', u'immutable?',
+ u'impersonate-box', u'impersonate-channel',
+ u'impersonate-continuation-mark-key', u'impersonate-hash',
+ u'impersonate-hash-set', u'impersonate-procedure',
+ u'impersonate-procedure*', u'impersonate-prompt-tag',
+ u'impersonate-struct', u'impersonate-vector', u'impersonator-contract?',
+ u'impersonator-ephemeron', u'impersonator-of?',
+ u'impersonator-prop:application-mark', u'impersonator-prop:blame',
+ u'impersonator-prop:contracted',
+ u'impersonator-property-accessor-procedure?', u'impersonator-property?',
+ u'impersonator?', u'implementation?', u'implementation?/c', u'in-bytes',
+ u'in-bytes-lines', u'in-combinations', u'in-cycle', u'in-dict',
+ u'in-dict-keys', u'in-dict-pairs', u'in-dict-values', u'in-directory',
+ u'in-hash', u'in-hash-keys', u'in-hash-pairs', u'in-hash-values',
+ u'in-immutable-hash', u'in-immutable-hash-keys',
+ u'in-immutable-hash-pairs', u'in-immutable-hash-values',
+ u'in-immutable-set', u'in-indexed', u'in-input-port-bytes',
+ u'in-input-port-chars', u'in-lines', u'in-list', u'in-mlist',
+ u'in-mutable-hash', u'in-mutable-hash-keys', u'in-mutable-hash-pairs',
+ u'in-mutable-hash-values', u'in-mutable-set', u'in-naturals',
+ u'in-parallel', u'in-permutations', u'in-port', u'in-producer', u'in-range',
+ u'in-sequences', u'in-set', u'in-slice', u'in-stream', u'in-string',
+ u'in-syntax', u'in-value', u'in-values*-sequence', u'in-values-sequence',
+ u'in-vector', u'in-weak-hash', u'in-weak-hash-keys', u'in-weak-hash-pairs',
+ u'in-weak-hash-values', u'in-weak-set', u'inexact->exact',
+ u'inexact-real?', u'inexact?', u'infinite?', u'input-port-append',
+ u'input-port?', u'inspector?', u'instanceof/c', u'integer->char',
+ u'integer->integer-bytes', u'integer-bytes->integer', u'integer-in',
+ u'integer-length', u'integer-sqrt', u'integer-sqrt/remainder', u'integer?',
+ u'interface->method-names', u'interface-extension?', u'interface?',
+ u'internal-definition-context-binding-identifiers',
+ u'internal-definition-context-introduce',
+ u'internal-definition-context-seal', u'internal-definition-context?',
+ u'is-a?', u'is-a?/c', u'keyword->string', u'keyword-apply', u'keyword<?',
+ u'keyword?', u'keywords-match', u'kill-thread', u'last', u'last-pair',
+ u'lcm', u'length', u'liberal-define-context?', u'link-exists?', u'list',
+ u'list*', u'list*of', u'list->bytes', u'list->mutable-set',
+ u'list->mutable-seteq', u'list->mutable-seteqv', u'list->set',
+ u'list->seteq', u'list->seteqv', u'list->string', u'list->vector',
+ u'list->weak-set', u'list->weak-seteq', u'list->weak-seteqv',
+ u'list-contract?', u'list-prefix?', u'list-ref', u'list-set', u'list-tail',
+ u'list-update', u'list/c', u'list?', u'listen-port-number?', u'listof',
+ u'load', u'load-extension', u'load-on-demand-enabled', u'load-relative',
+ u'load-relative-extension', u'load/cd', u'load/use-compiled',
+ u'local-expand', u'local-expand/capture-lifts',
+ u'local-transformer-expand', u'local-transformer-expand/capture-lifts',
+ u'locale-string-encoding', u'log', u'log-all-levels', u'log-level-evt',
+ u'log-level?', u'log-max-level', u'log-message', u'log-receiver?',
+ u'logger-name', u'logger?', u'magnitude', u'make-arity-at-least',
+ u'make-base-empty-namespace', u'make-base-namespace', u'make-bytes',
+ u'make-channel', u'make-chaperone-contract',
+ u'make-continuation-mark-key', u'make-continuation-prompt-tag',
+ u'make-contract', u'make-custodian', u'make-custodian-box',
+ u'make-custom-hash', u'make-custom-hash-types', u'make-custom-set',
+ u'make-custom-set-types', u'make-date', u'make-date*',
+ u'make-derived-parameter', u'make-directory', u'make-directory*',
+ u'make-do-sequence', u'make-empty-namespace',
+ u'make-environment-variables', u'make-ephemeron', u'make-exn',
+ u'make-exn:break', u'make-exn:break:hang-up', u'make-exn:break:terminate',
+ u'make-exn:fail', u'make-exn:fail:contract',
+ u'make-exn:fail:contract:arity', u'make-exn:fail:contract:blame',
+ u'make-exn:fail:contract:continuation',
+ u'make-exn:fail:contract:divide-by-zero',
+ u'make-exn:fail:contract:non-fixnum-result',
+ u'make-exn:fail:contract:variable', u'make-exn:fail:filesystem',
+ u'make-exn:fail:filesystem:errno', u'make-exn:fail:filesystem:exists',
+ u'make-exn:fail:filesystem:missing-module',
+ u'make-exn:fail:filesystem:version', u'make-exn:fail:network',
+ u'make-exn:fail:network:errno', u'make-exn:fail:object',
+ u'make-exn:fail:out-of-memory', u'make-exn:fail:read',
+ u'make-exn:fail:read:eof', u'make-exn:fail:read:non-char',
+ u'make-exn:fail:syntax', u'make-exn:fail:syntax:missing-module',
+ u'make-exn:fail:syntax:unbound', u'make-exn:fail:unsupported',
+ u'make-exn:fail:user', u'make-file-or-directory-link',
+ u'make-flat-contract', u'make-fsemaphore', u'make-generic',
+ u'make-handle-get-preference-locked', u'make-hash',
+ u'make-hash-placeholder', u'make-hasheq', u'make-hasheq-placeholder',
+ u'make-hasheqv', u'make-hasheqv-placeholder',
+ u'make-immutable-custom-hash', u'make-immutable-hash',
+ u'make-immutable-hasheq', u'make-immutable-hasheqv',
+ u'make-impersonator-property', u'make-input-port',
+ u'make-input-port/read-to-peek', u'make-inspector',
+ u'make-keyword-procedure', u'make-known-char-range-list',
+ u'make-limited-input-port', u'make-list', u'make-lock-file-name',
+ u'make-log-receiver', u'make-logger', u'make-mixin-contract',
+ u'make-mutable-custom-set', u'make-none/c', u'make-object',
+ u'make-output-port', u'make-parameter', u'make-parent-directory*',
+ u'make-phantom-bytes', u'make-pipe', u'make-pipe-with-specials',
+ u'make-placeholder', u'make-plumber', u'make-polar', u'make-prefab-struct',
+ u'make-primitive-class', u'make-proj-contract',
+ u'make-pseudo-random-generator', u'make-reader-graph', u'make-readtable',
+ u'make-rectangular', u'make-rename-transformer',
+ u'make-resolved-module-path', u'make-security-guard', u'make-semaphore',
+ u'make-set!-transformer', u'make-shared-bytes', u'make-sibling-inspector',
+ u'make-special-comment', u'make-srcloc', u'make-string',
+ u'make-struct-field-accessor', u'make-struct-field-mutator',
+ u'make-struct-type', u'make-struct-type-property',
+ u'make-syntax-delta-introducer', u'make-syntax-introducer',
+ u'make-temporary-file', u'make-tentative-pretty-print-output-port',
+ u'make-thread-cell', u'make-thread-group', u'make-vector',
+ u'make-weak-box', u'make-weak-custom-hash', u'make-weak-custom-set',
+ u'make-weak-hash', u'make-weak-hasheq', u'make-weak-hasheqv',
+ u'make-will-executor', u'map', u'match-equality-test',
+ u'matches-arity-exactly?', u'max', u'mcar', u'mcdr', u'mcons', u'member',
+ u'member-name-key-hash-code', u'member-name-key=?', u'member-name-key?',
+ u'memf', u'memq', u'memv', u'merge-input', u'method-in-interface?', u'min',
+ u'mixin-contract', u'module->exports', u'module->imports',
+ u'module->language-info', u'module->namespace',
+ u'module-compiled-cross-phase-persistent?', u'module-compiled-exports',
+ u'module-compiled-imports', u'module-compiled-language-info',
+ u'module-compiled-name', u'module-compiled-submodules',
+ u'module-declared?', u'module-path-index-join',
+ u'module-path-index-resolve', u'module-path-index-split',
+ u'module-path-index-submodule', u'module-path-index?', u'module-path?',
+ u'module-predefined?', u'module-provide-protected?', u'modulo', u'mpair?',
+ u'mutable-set', u'mutable-seteq', u'mutable-seteqv', u'n->th',
+ u'nack-guard-evt', u'namespace-anchor->empty-namespace',
+ u'namespace-anchor->namespace', u'namespace-anchor?',
+ u'namespace-attach-module', u'namespace-attach-module-declaration',
+ u'namespace-base-phase', u'namespace-mapped-symbols',
+ u'namespace-module-identifier', u'namespace-module-registry',
+ u'namespace-require', u'namespace-require/constant',
+ u'namespace-require/copy', u'namespace-require/expansion-time',
+ u'namespace-set-variable-value!', u'namespace-symbol->identifier',
+ u'namespace-syntax-introduce', u'namespace-undefine-variable!',
+ u'namespace-unprotect-module', u'namespace-variable-value', u'namespace?',
+ u'nan?', u'natural-number/c', u'negate', u'negative?', u'never-evt',
+ u'new-∀/c', u'new-∃/c', u'newline', u'ninth', u'non-empty-listof',
+ u'non-empty-string?', u'none/c', u'normal-case-path', u'normalize-arity',
+ u'normalize-path', u'normalized-arity?', u'not', u'not/c', u'null', u'null?',
+ u'number->string', u'number?', u'numerator', u'object%', u'object->vector',
+ u'object-info', u'object-interface', u'object-method-arity-includes?',
+ u'object-name', u'object-or-false=?', u'object=?', u'object?', u'odd?',
+ u'one-of/c', u'open-input-bytes', u'open-input-file',
+ u'open-input-output-file', u'open-input-string', u'open-output-bytes',
+ u'open-output-file', u'open-output-nowhere', u'open-output-string',
+ u'or/c', u'order-of-magnitude', u'ormap', u'other-execute-bit',
+ u'other-read-bit', u'other-write-bit', u'output-port?', u'pair?',
+ u'parameter-procedure=?', u'parameter/c', u'parameter?',
+ u'parameterization?', u'parse-command-line', u'partition', u'path->bytes',
+ u'path->complete-path', u'path->directory-path', u'path->string',
+ u'path-add-suffix', u'path-convention-type', u'path-element->bytes',
+ u'path-element->string', u'path-element?', u'path-for-some-system?',
+ u'path-list-string->path-list', u'path-only', u'path-replace-suffix',
+ u'path-string?', u'path<?', u'path?', u'pathlist-closure', u'peek-byte',
+ u'peek-byte-or-special', u'peek-bytes', u'peek-bytes!', u'peek-bytes!-evt',
+ u'peek-bytes-avail!', u'peek-bytes-avail!*', u'peek-bytes-avail!-evt',
+ u'peek-bytes-avail!/enable-break', u'peek-bytes-evt', u'peek-char',
+ u'peek-char-or-special', u'peek-string', u'peek-string!',
+ u'peek-string!-evt', u'peek-string-evt', u'peeking-input-port',
+ u'permutations', u'phantom-bytes?', u'pi', u'pi.f', u'pipe-content-length',
+ u'place-break', u'place-channel', u'place-channel-get',
+ u'place-channel-put', u'place-channel-put/get', u'place-channel?',
+ u'place-dead-evt', u'place-enabled?', u'place-kill', u'place-location?',
+ u'place-message-allowed?', u'place-sleep', u'place-wait', u'place?',
+ u'placeholder-get', u'placeholder-set!', u'placeholder?',
+ u'plumber-add-flush!', u'plumber-flush-all',
+ u'plumber-flush-handle-remove!', u'plumber-flush-handle?', u'plumber?',
+ u'poll-guard-evt', u'port->bytes', u'port->bytes-lines', u'port->lines',
+ u'port->list', u'port->string', u'port-closed-evt', u'port-closed?',
+ u'port-commit-peeked', u'port-count-lines!', u'port-count-lines-enabled',
+ u'port-counts-lines?', u'port-display-handler', u'port-file-identity',
+ u'port-file-unlock', u'port-next-location', u'port-number?',
+ u'port-print-handler', u'port-progress-evt',
+ u'port-provides-progress-evts?', u'port-read-handler',
+ u'port-try-file-lock?', u'port-write-handler', u'port-writes-atomic?',
+ u'port-writes-special?', u'port?', u'positive?', u'predicate/c',
+ u'prefab-key->struct-type', u'prefab-key?', u'prefab-struct-key',
+ u'preferences-lock-file-mode', u'pregexp', u'pregexp?', u'pretty-display',
+ u'pretty-format', u'pretty-print', u'pretty-print-.-symbol-without-bars',
+ u'pretty-print-abbreviate-read-macros', u'pretty-print-columns',
+ u'pretty-print-current-style-table', u'pretty-print-depth',
+ u'pretty-print-exact-as-decimal', u'pretty-print-extend-style-table',
+ u'pretty-print-handler', u'pretty-print-newline',
+ u'pretty-print-post-print-hook', u'pretty-print-pre-print-hook',
+ u'pretty-print-print-hook', u'pretty-print-print-line',
+ u'pretty-print-remap-stylable', u'pretty-print-show-inexactness',
+ u'pretty-print-size-hook', u'pretty-print-style-table?',
+ u'pretty-printing', u'pretty-write', u'primitive-closure?',
+ u'primitive-result-arity', u'primitive?', u'print', u'print-as-expression',
+ u'print-boolean-long-form', u'print-box', u'print-graph',
+ u'print-hash-table', u'print-mpair-curly-braces',
+ u'print-pair-curly-braces', u'print-reader-abbreviations',
+ u'print-struct', u'print-syntax-width', u'print-unreadable',
+ u'print-vector-length', u'printable/c', u'printable<%>', u'printf',
+ u'println', u'procedure->method', u'procedure-arity',
+ u'procedure-arity-includes/c', u'procedure-arity-includes?',
+ u'procedure-arity?', u'procedure-closure-contents-eq?',
+ u'procedure-extract-target', u'procedure-keywords',
+ u'procedure-reduce-arity', u'procedure-reduce-keyword-arity',
+ u'procedure-rename', u'procedure-result-arity', u'procedure-specialize',
+ u'procedure-struct-type?', u'procedure?', u'process', u'process*',
+ u'process*/ports', u'process/ports', u'processor-count', u'progress-evt?',
+ u'promise-forced?', u'promise-running?', u'promise/c', u'promise/name?',
+ u'promise?', u'prop:arity-string', u'prop:arrow-contract',
+ u'prop:arrow-contract-get-info', u'prop:arrow-contract?', u'prop:blame',
+ u'prop:chaperone-contract', u'prop:checked-procedure', u'prop:contract',
+ u'prop:contracted', u'prop:custom-print-quotable', u'prop:custom-write',
+ u'prop:dict', u'prop:dict/contract', u'prop:equal+hash', u'prop:evt',
+ u'prop:exn:missing-module', u'prop:exn:srclocs',
+ u'prop:expansion-contexts', u'prop:flat-contract',
+ u'prop:impersonator-of', u'prop:input-port',
+ u'prop:liberal-define-context', u'prop:object-name',
+ u'prop:opt-chaperone-contract', u'prop:opt-chaperone-contract-get-test',
+ u'prop:opt-chaperone-contract?', u'prop:orc-contract',
+ u'prop:orc-contract-get-subcontracts', u'prop:orc-contract?',
+ u'prop:output-port', u'prop:place-location', u'prop:procedure',
+ u'prop:recursive-contract', u'prop:recursive-contract-unroll',
+ u'prop:recursive-contract?', u'prop:rename-transformer', u'prop:sequence',
+ u'prop:set!-transformer', u'prop:stream', u'proper-subset?',
+ u'pseudo-random-generator->vector', u'pseudo-random-generator-vector?',
+ u'pseudo-random-generator?', u'put-preferences', u'putenv', u'quotient',
+ u'quotient/remainder', u'radians->degrees', u'raise',
+ u'raise-argument-error', u'raise-arguments-error', u'raise-arity-error',
+ u'raise-blame-error', u'raise-contract-error', u'raise-mismatch-error',
+ u'raise-not-cons-blame-error', u'raise-range-error',
+ u'raise-result-error', u'raise-syntax-error', u'raise-type-error',
+ u'raise-user-error', u'random', u'random-seed', u'range', u'rational?',
+ u'rationalize', u'read', u'read-accept-bar-quote', u'read-accept-box',
+ u'read-accept-compiled', u'read-accept-dot', u'read-accept-graph',
+ u'read-accept-infix-dot', u'read-accept-lang', u'read-accept-quasiquote',
+ u'read-accept-reader', u'read-byte', u'read-byte-or-special',
+ u'read-bytes', u'read-bytes!', u'read-bytes!-evt', u'read-bytes-avail!',
+ u'read-bytes-avail!*', u'read-bytes-avail!-evt',
+ u'read-bytes-avail!/enable-break', u'read-bytes-evt', u'read-bytes-line',
+ u'read-bytes-line-evt', u'read-case-sensitive', u'read-cdot', u'read-char',
+ u'read-char-or-special', u'read-curly-brace-as-paren',
+ u'read-curly-brace-with-tag', u'read-decimal-as-inexact',
+ u'read-eval-print-loop', u'read-language', u'read-line', u'read-line-evt',
+ u'read-on-demand-source', u'read-square-bracket-as-paren',
+ u'read-square-bracket-with-tag', u'read-string', u'read-string!',
+ u'read-string!-evt', u'read-string-evt', u'read-syntax',
+ u'read-syntax/recursive', u'read/recursive', u'readtable-mapping',
+ u'readtable?', u'real->decimal-string', u'real->double-flonum',
+ u'real->floating-point-bytes', u'real->single-flonum', u'real-in',
+ u'real-part', u'real?', u'reencode-input-port', u'reencode-output-port',
+ u'regexp', u'regexp-match', u'regexp-match*', u'regexp-match-evt',
+ u'regexp-match-exact?', u'regexp-match-peek',
+ u'regexp-match-peek-immediate', u'regexp-match-peek-positions',
+ u'regexp-match-peek-positions*',
+ u'regexp-match-peek-positions-immediate',
+ u'regexp-match-peek-positions-immediate/end',
+ u'regexp-match-peek-positions/end', u'regexp-match-positions',
+ u'regexp-match-positions*', u'regexp-match-positions/end',
+ u'regexp-match/end', u'regexp-match?', u'regexp-max-lookbehind',
+ u'regexp-quote', u'regexp-replace', u'regexp-replace*',
+ u'regexp-replace-quote', u'regexp-replaces', u'regexp-split',
+ u'regexp-try-match', u'regexp?', u'relative-path?', u'relocate-input-port',
+ u'relocate-output-port', u'remainder', u'remf', u'remf*', u'remove',
+ u'remove*', u'remove-duplicates', u'remq', u'remq*', u'remv', u'remv*',
+ u'rename-contract', u'rename-file-or-directory',
+ u'rename-transformer-target', u'rename-transformer?', u'replace-evt',
+ u'reroot-path', u'resolve-path', u'resolved-module-path-name',
+ u'resolved-module-path?', u'rest', u'reverse', u'round', u'second',
+ u'seconds->date', u'security-guard?', u'semaphore-peek-evt',
+ u'semaphore-peek-evt?', u'semaphore-post', u'semaphore-try-wait?',
+ u'semaphore-wait', u'semaphore-wait/enable-break', u'semaphore?',
+ u'sequence->list', u'sequence->stream', u'sequence-add-between',
+ u'sequence-andmap', u'sequence-append', u'sequence-count',
+ u'sequence-filter', u'sequence-fold', u'sequence-for-each',
+ u'sequence-generate', u'sequence-generate*', u'sequence-length',
+ u'sequence-map', u'sequence-ormap', u'sequence-ref', u'sequence-tail',
+ u'sequence/c', u'sequence?', u'set', u'set!-transformer-procedure',
+ u'set!-transformer?', u'set->list', u'set->stream', u'set-add', u'set-add!',
+ u'set-box!', u'set-clear', u'set-clear!', u'set-copy', u'set-copy-clear',
+ u'set-count', u'set-empty?', u'set-eq?', u'set-equal?', u'set-eqv?',
+ u'set-first', u'set-for-each', u'set-implements/c', u'set-implements?',
+ u'set-intersect', u'set-intersect!', u'set-map', u'set-mcar!', u'set-mcdr!',
+ u'set-member?', u'set-mutable?', u'set-phantom-bytes!',
+ u'set-port-next-location!', u'set-remove', u'set-remove!', u'set-rest',
+ u'set-some-basic-contracts!', u'set-subtract', u'set-subtract!',
+ u'set-symmetric-difference', u'set-symmetric-difference!', u'set-union',
+ u'set-union!', u'set-weak?', u'set/c', u'set=?', u'set?', u'seteq', u'seteqv',
+ u'seventh', u'sgn', u'shared-bytes', u'shell-execute', u'shrink-path-wrt',
+ u'shuffle', u'simple-form-path', u'simplify-path', u'sin',
+ u'single-flonum?', u'sinh', u'sixth', u'skip-projection-wrapper?', u'sleep',
+ u'some-system-path->string', u'sort', u'special-comment-value',
+ u'special-comment?', u'special-filter-input-port', u'split-at',
+ u'split-at-right', u'split-common-prefix', u'split-path', u'splitf-at',
+ u'splitf-at-right', u'sqr', u'sqrt', u'srcloc', u'srcloc->string',
+ u'srcloc-column', u'srcloc-line', u'srcloc-position', u'srcloc-source',
+ u'srcloc-span', u'srcloc?', u'stop-after', u'stop-before', u'stream->list',
+ u'stream-add-between', u'stream-andmap', u'stream-append', u'stream-count',
+ u'stream-empty?', u'stream-filter', u'stream-first', u'stream-fold',
+ u'stream-for-each', u'stream-length', u'stream-map', u'stream-ormap',
+ u'stream-ref', u'stream-rest', u'stream-tail', u'stream/c', u'stream?',
+ u'string', u'string->bytes/latin-1', u'string->bytes/locale',
+ u'string->bytes/utf-8', u'string->immutable-string', u'string->keyword',
+ u'string->list', u'string->number', u'string->path',
+ u'string->path-element', u'string->some-system-path', u'string->symbol',
+ u'string->uninterned-symbol', u'string->unreadable-symbol',
+ u'string-append', u'string-append*', u'string-ci<=?', u'string-ci<?',
+ u'string-ci=?', u'string-ci>=?', u'string-ci>?', u'string-contains?',
+ u'string-copy', u'string-copy!', u'string-downcase',
+ u'string-environment-variable-name?', u'string-fill!', u'string-foldcase',
+ u'string-join', u'string-len/c', u'string-length', u'string-locale-ci<?',
+ u'string-locale-ci=?', u'string-locale-ci>?', u'string-locale-downcase',
+ u'string-locale-upcase', u'string-locale<?', u'string-locale=?',
+ u'string-locale>?', u'string-no-nuls?', u'string-normalize-nfc',
+ u'string-normalize-nfd', u'string-normalize-nfkc',
+ u'string-normalize-nfkd', u'string-normalize-spaces', u'string-port?',
+ u'string-prefix?', u'string-ref', u'string-replace', u'string-set!',
+ u'string-split', u'string-suffix?', u'string-titlecase', u'string-trim',
+ u'string-upcase', u'string-utf-8-length', u'string<=?', u'string<?',
+ u'string=?', u'string>=?', u'string>?', u'string?', u'struct->vector',
+ u'struct-accessor-procedure?', u'struct-constructor-procedure?',
+ u'struct-info', u'struct-mutator-procedure?',
+ u'struct-predicate-procedure?', u'struct-type-info',
+ u'struct-type-make-constructor', u'struct-type-make-predicate',
+ u'struct-type-property-accessor-procedure?', u'struct-type-property/c',
+ u'struct-type-property?', u'struct-type?', u'struct:arity-at-least',
+ u'struct:arrow-contract-info', u'struct:date', u'struct:date*',
+ u'struct:exn', u'struct:exn:break', u'struct:exn:break:hang-up',
+ u'struct:exn:break:terminate', u'struct:exn:fail',
+ u'struct:exn:fail:contract', u'struct:exn:fail:contract:arity',
+ u'struct:exn:fail:contract:blame',
+ u'struct:exn:fail:contract:continuation',
+ u'struct:exn:fail:contract:divide-by-zero',
+ u'struct:exn:fail:contract:non-fixnum-result',
+ u'struct:exn:fail:contract:variable', u'struct:exn:fail:filesystem',
+ u'struct:exn:fail:filesystem:errno',
+ u'struct:exn:fail:filesystem:exists',
+ u'struct:exn:fail:filesystem:missing-module',
+ u'struct:exn:fail:filesystem:version', u'struct:exn:fail:network',
+ u'struct:exn:fail:network:errno', u'struct:exn:fail:object',
+ u'struct:exn:fail:out-of-memory', u'struct:exn:fail:read',
+ u'struct:exn:fail:read:eof', u'struct:exn:fail:read:non-char',
+ u'struct:exn:fail:syntax', u'struct:exn:fail:syntax:missing-module',
+ u'struct:exn:fail:syntax:unbound', u'struct:exn:fail:unsupported',
+ u'struct:exn:fail:user', u'struct:srcloc',
+ u'struct:wrapped-extra-arg-arrow', u'struct?', u'sub1', u'subbytes',
+ u'subclass?', u'subclass?/c', u'subprocess', u'subprocess-group-enabled',
+ u'subprocess-kill', u'subprocess-pid', u'subprocess-status',
+ u'subprocess-wait', u'subprocess?', u'subset?', u'substring', u'suggest/c',
+ u'symbol->string', u'symbol-interned?', u'symbol-unreadable?', u'symbol<?',
+ u'symbol=?', u'symbol?', u'symbols', u'sync', u'sync/enable-break',
+ u'sync/timeout', u'sync/timeout/enable-break', u'syntax->datum',
+ u'syntax->list', u'syntax-arm', u'syntax-column', u'syntax-debug-info',
+ u'syntax-disarm', u'syntax-e', u'syntax-line',
+ u'syntax-local-bind-syntaxes', u'syntax-local-certifier',
+ u'syntax-local-context', u'syntax-local-expand-expression',
+ u'syntax-local-get-shadower', u'syntax-local-identifier-as-binding',
+ u'syntax-local-introduce', u'syntax-local-lift-context',
+ u'syntax-local-lift-expression', u'syntax-local-lift-module',
+ u'syntax-local-lift-module-end-declaration',
+ u'syntax-local-lift-provide', u'syntax-local-lift-require',
+ u'syntax-local-lift-values-expression',
+ u'syntax-local-make-definition-context',
+ u'syntax-local-make-delta-introducer',
+ u'syntax-local-module-defined-identifiers',
+ u'syntax-local-module-exports',
+ u'syntax-local-module-required-identifiers', u'syntax-local-name',
+ u'syntax-local-phase-level', u'syntax-local-submodules',
+ u'syntax-local-transforming-module-provides?', u'syntax-local-value',
+ u'syntax-local-value/immediate', u'syntax-original?', u'syntax-position',
+ u'syntax-property', u'syntax-property-preserved?',
+ u'syntax-property-symbol-keys', u'syntax-protect', u'syntax-rearm',
+ u'syntax-recertify', u'syntax-shift-phase-level', u'syntax-source',
+ u'syntax-source-module', u'syntax-span', u'syntax-taint',
+ u'syntax-tainted?', u'syntax-track-origin',
+ u'syntax-transforming-module-expression?',
+ u'syntax-transforming-with-lifts?', u'syntax-transforming?', u'syntax/c',
+ u'syntax?', u'system', u'system*', u'system*/exit-code',
+ u'system-big-endian?', u'system-idle-evt', u'system-language+country',
+ u'system-library-subpath', u'system-path-convention-type', u'system-type',
+ u'system/exit-code', u'tail-marks-match?', u'take', u'take-common-prefix',
+ u'take-right', u'takef', u'takef-right', u'tan', u'tanh',
+ u'tcp-abandon-port', u'tcp-accept', u'tcp-accept-evt',
+ u'tcp-accept-ready?', u'tcp-accept/enable-break', u'tcp-addresses',
+ u'tcp-close', u'tcp-connect', u'tcp-connect/enable-break', u'tcp-listen',
+ u'tcp-listener?', u'tcp-port?', u'tentative-pretty-print-port-cancel',
+ u'tentative-pretty-print-port-transfer', u'tenth', u'terminal-port?',
+ u'the-unsupplied-arg', u'third', u'thread', u'thread-cell-ref',
+ u'thread-cell-set!', u'thread-cell-values?', u'thread-cell?',
+ u'thread-dead-evt', u'thread-dead?', u'thread-group?', u'thread-receive',
+ u'thread-receive-evt', u'thread-resume', u'thread-resume-evt',
+ u'thread-rewind-receive', u'thread-running?', u'thread-send',
+ u'thread-suspend', u'thread-suspend-evt', u'thread-try-receive',
+ u'thread-wait', u'thread/suspend-to-kill', u'thread?', u'time-apply',
+ u'touch', u'transplant-input-port', u'transplant-output-port', u'true',
+ u'truncate', u'udp-addresses', u'udp-bind!', u'udp-bound?', u'udp-close',
+ u'udp-connect!', u'udp-connected?', u'udp-multicast-interface',
+ u'udp-multicast-join-group!', u'udp-multicast-leave-group!',
+ u'udp-multicast-loopback?', u'udp-multicast-set-interface!',
+ u'udp-multicast-set-loopback!', u'udp-multicast-set-ttl!',
+ u'udp-multicast-ttl', u'udp-open-socket', u'udp-receive!',
+ u'udp-receive!*', u'udp-receive!-evt', u'udp-receive!/enable-break',
+ u'udp-receive-ready-evt', u'udp-send', u'udp-send*', u'udp-send-evt',
+ u'udp-send-ready-evt', u'udp-send-to', u'udp-send-to*', u'udp-send-to-evt',
+ u'udp-send-to/enable-break', u'udp-send/enable-break', u'udp?', u'unbox',
+ u'uncaught-exception-handler', u'unit?', u'unspecified-dom',
+ u'unsupplied-arg?', u'use-collection-link-paths',
+ u'use-compiled-file-paths', u'use-user-specific-search-paths',
+ u'user-execute-bit', u'user-read-bit', u'user-write-bit', u'value-blame',
+ u'value-contract', u'values', u'variable-reference->empty-namespace',
+ u'variable-reference->module-base-phase',
+ u'variable-reference->module-declaration-inspector',
+ u'variable-reference->module-path-index',
+ u'variable-reference->module-source', u'variable-reference->namespace',
+ u'variable-reference->phase',
+ u'variable-reference->resolved-module-path',
+ u'variable-reference-constant?', u'variable-reference?', u'vector',
+ u'vector->immutable-vector', u'vector->list',
+ u'vector->pseudo-random-generator', u'vector->pseudo-random-generator!',
+ u'vector->values', u'vector-append', u'vector-argmax', u'vector-argmin',
+ u'vector-copy', u'vector-copy!', u'vector-count', u'vector-drop',
+ u'vector-drop-right', u'vector-fill!', u'vector-filter',
+ u'vector-filter-not', u'vector-immutable', u'vector-immutable/c',
+ u'vector-immutableof', u'vector-length', u'vector-map', u'vector-map!',
+ u'vector-member', u'vector-memq', u'vector-memv', u'vector-ref',
+ u'vector-set!', u'vector-set*!', u'vector-set-performance-stats!',
+ u'vector-split-at', u'vector-split-at-right', u'vector-take',
+ u'vector-take-right', u'vector/c', u'vector?', u'vectorof', u'version',
+ u'void', u'void?', u'weak-box-value', u'weak-box?', u'weak-set',
+ u'weak-seteq', u'weak-seteqv', u'will-execute', u'will-executor?',
+ u'will-register', u'will-try-execute', u'with-input-from-bytes',
+ u'with-input-from-file', u'with-input-from-string',
+ u'with-output-to-bytes', u'with-output-to-file', u'with-output-to-string',
+ u'would-be-future', u'wrap-evt', u'wrapped-extra-arg-arrow',
+ u'wrapped-extra-arg-arrow-extra-neg-party-argument',
+ u'wrapped-extra-arg-arrow-real-func', u'wrapped-extra-arg-arrow?',
+ u'writable<%>', u'write', u'write-byte', u'write-bytes',
+ u'write-bytes-avail', u'write-bytes-avail*', u'write-bytes-avail-evt',
+ u'write-bytes-avail/enable-break', u'write-char', u'write-special',
+ u'write-special-avail*', u'write-special-evt', u'write-string',
+ u'write-to-file', u'writeln', u'xor', u'zero?', u'~.a', u'~.s', u'~.v', u'~a',
+ u'~e', u'~r', u'~s', u'~v'
+ )
+
+ _opening_parenthesis = r'[([{]'
+ _closing_parenthesis = r'[)\]}]'
+ _delimiters = r'()[\]{}",\'`;\s'
+ _symbol = r'(?u)(?:\|[^|]*\||\\[\w\W]|[^|\\%s]+)+' % _delimiters
+ _exact_decimal_prefix = r'(?:#e)?(?:#d)?(?:#e)?'
+ _exponent = r'(?:[defls][-+]?\d+)'
+ _inexact_simple_no_hashes = r'(?:\d+(?:/\d+|\.\d*)?|\.\d+)'
+ _inexact_simple = (r'(?:%s|(?:\d+#+(?:\.#*|/\d+#*)?|\.\d+#+|'
+ r'\d+(?:\.\d*#+|/\d+#+)))' % _inexact_simple_no_hashes)
+ _inexact_normal_no_hashes = r'(?:%s%s?)' % (_inexact_simple_no_hashes,
+ _exponent)
+ _inexact_normal = r'(?:%s%s?)' % (_inexact_simple, _exponent)
+ _inexact_special = r'(?:(?:inf|nan)\.[0f])'
+ _inexact_real = r'(?:[-+]?%s|[-+]%s)' % (_inexact_normal,
+ _inexact_special)
+ _inexact_unsigned = r'(?:%s|%s)' % (_inexact_normal, _inexact_special)
+
+ tokens = {
+ 'root': [
+ (_closing_parenthesis, Error),
+ (r'(?!\Z)', Text, 'unquoted-datum')
+ ],
+ 'datum': [
+ (r'(?s)#;|#*', Comment),
+ (u';[^\\n\\r\x85\u2028\u2029]*', Comment.Single),
+ (r'#\|', Comment.Multiline, 'block-comment'),
+
+ # Whitespaces
+ (r'(?u)\s+', Text),
+
+ # Numbers: Keep in mind Racket reader hash prefixes, which
+ # can denote the base or the type. These don't map neatly
+ # onto Pygments token types; some judgment calls here.
+
+ # #d or no prefix
+ (r'(?i)%s[-+]?\d+(?=[%s])' % (_exact_decimal_prefix, _delimiters),
+ Number.Integer, '#pop'),
+ (r'(?i)%s[-+]?(\d+(\.\d*)?|\.\d+)([deflst][-+]?\d+)?(?=[%s])' %
+ (_exact_decimal_prefix, _delimiters), Number.Float, '#pop'),
+ (r'(?i)%s[-+]?(%s([-+]%s?i)?|[-+]%s?i)(?=[%s])' %
+ (_exact_decimal_prefix, _inexact_normal_no_hashes,
+ _inexact_normal_no_hashes, _inexact_normal_no_hashes,
+ _delimiters), Number, '#pop'),
+
+ # Inexact without explicit #i
+ (r'(?i)(#d)?(%s([-+]%s?i)?|[-+]%s?i|%s@%s)(?=[%s])' %
+ (_inexact_real, _inexact_unsigned, _inexact_unsigned,
+ _inexact_real, _inexact_real, _delimiters), Number.Float,
+ '#pop'),
+
+ # The remaining extflonums
+ (r'(?i)(([-+]?%st[-+]?\d+)|[-+](inf|nan)\.t)(?=[%s])' %
+ (_inexact_simple, _delimiters), Number.Float, '#pop'),
+
+ # #b
+ (r'(?i)(#[ei])?#b%s' % _symbol, Number.Bin, '#pop'),
+
+ # #o
+ (r'(?i)(#[ei])?#o%s' % _symbol, Number.Oct, '#pop'),
+
+ # #x
+ (r'(?i)(#[ei])?#x%s' % _symbol, Number.Hex, '#pop'),
+
+ # #i is always inexact, i.e. float
+ (r'(?i)(#d)?#i%s' % _symbol, Number.Float, '#pop'),
+
+ # Strings and characters
+ (r'#?"', String.Double, ('#pop', 'string')),
+ (r'#<<(.+)\n(^(?!\1$).*$\n)*^\1$', String.Heredoc, '#pop'),
+ (r'#\\(u[\da-fA-F]{1,4}|U[\da-fA-F]{1,8})', String.Char, '#pop'),
+ (r'(?is)#\\([0-7]{3}|[a-z]+|.)', String.Char, '#pop'),
+ (r'(?s)#[pr]x#?"(\\?.)*?"', String.Regex, '#pop'),
+
+ # Constants
+ (r'#(true|false|[tTfF])', Name.Constant, '#pop'),
+
+ # Keyword argument names (e.g. #:keyword)
+ (r'#:%s' % _symbol, Keyword.Declaration, '#pop'),
+
+ # Reader extensions
+ (r'(#lang |#!)(\S+)',
+ bygroups(Keyword.Namespace, Name.Namespace)),
+ (r'#reader', Keyword.Namespace, 'quoted-datum'),
+
+ # Other syntax
+ (r"(?i)\.(?=[%s])|#c[is]|#['`]|#,@?" % _delimiters, Operator),
+ (r"'|#[s&]|#hash(eqv?)?|#\d*(?=%s)" % _opening_parenthesis,
+ Operator, ('#pop', 'quoted-datum'))
+ ],
+ 'datum*': [
+ (r'`|,@?', Operator),
+ (_symbol, String.Symbol, '#pop'),
+ (r'[|\\]', Error),
+ default('#pop')
+ ],
+ 'list': [
+ (_closing_parenthesis, Punctuation, '#pop')
+ ],
+ 'unquoted-datum': [
+ include('datum'),
+ (r'quote(?=[%s])' % _delimiters, Keyword,
+ ('#pop', 'quoted-datum')),
+ (r'`', Operator, ('#pop', 'quasiquoted-datum')),
+ (r'quasiquote(?=[%s])' % _delimiters, Keyword,
+ ('#pop', 'quasiquoted-datum')),
+ (_opening_parenthesis, Punctuation, ('#pop', 'unquoted-list')),
+ (words(_keywords, prefix='(?u)', suffix='(?=[%s])' % _delimiters),
+ Keyword, '#pop'),
+ (words(_builtins, prefix='(?u)', suffix='(?=[%s])' % _delimiters),
+ Name.Builtin, '#pop'),
+ (_symbol, Name, '#pop'),
+ include('datum*')
+ ],
+ 'unquoted-list': [
+ include('list'),
+ (r'(?!\Z)', Text, 'unquoted-datum')
+ ],
+ 'quasiquoted-datum': [
+ include('datum'),
+ (r',@?', Operator, ('#pop', 'unquoted-datum')),
+ (r'unquote(-splicing)?(?=[%s])' % _delimiters, Keyword,
+ ('#pop', 'unquoted-datum')),
+ (_opening_parenthesis, Punctuation, ('#pop', 'quasiquoted-list')),
+ include('datum*')
+ ],
+ 'quasiquoted-list': [
+ include('list'),
+ (r'(?!\Z)', Text, 'quasiquoted-datum')
+ ],
+ 'quoted-datum': [
+ include('datum'),
+ (_opening_parenthesis, Punctuation, ('#pop', 'quoted-list')),
+ include('datum*')
+ ],
+ 'quoted-list': [
+ include('list'),
+ (r'(?!\Z)', Text, 'quoted-datum')
+ ],
+ 'block-comment': [
+ (r'#\|', Comment.Multiline, '#push'),
+ (r'\|#', Comment.Multiline, '#pop'),
+ (r'[^#|]+|.', Comment.Multiline)
+ ],
+ 'string': [
+ (r'"', String.Double, '#pop'),
+ (r'(?s)\\([0-7]{1,3}|x[\da-fA-F]{1,2}|u[\da-fA-F]{1,4}|'
+ r'U[\da-fA-F]{1,8}|.)', String.Escape),
+ (r'[^\\"]+', String.Double)
+ ]
+ }
+
+
+class NewLispLexer(RegexLexer):
+ """
+ For `newLISP. <www.newlisp.org>`_ source code (version 10.3.0).
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'NewLisp'
+ aliases = ['newlisp']
+ filenames = ['*.lsp', '*.nl', '*.kif']
+ mimetypes = ['text/x-newlisp', 'application/x-newlisp']
+
+ flags = re.IGNORECASE | re.MULTILINE | re.UNICODE
+
+ # list of built-in functions for newLISP version 10.3
+ builtins = (
+ '^', '--', '-', ':', '!', '!=', '?', '@', '*', '/', '&', '%', '+', '++',
+ '<', '<<', '<=', '=', '>', '>=', '>>', '|', '~', '$', '$0', '$1', '$10',
+ '$11', '$12', '$13', '$14', '$15', '$2', '$3', '$4', '$5', '$6', '$7',
+ '$8', '$9', '$args', '$idx', '$it', '$main-args', 'abort', 'abs',
+ 'acos', 'acosh', 'add', 'address', 'amb', 'and', 'append-file',
+ 'append', 'apply', 'args', 'array-list', 'array?', 'array', 'asin',
+ 'asinh', 'assoc', 'atan', 'atan2', 'atanh', 'atom?', 'base64-dec',
+ 'base64-enc', 'bayes-query', 'bayes-train', 'begin',
+ 'beta', 'betai', 'bind', 'binomial', 'bits', 'callback',
+ 'case', 'catch', 'ceil', 'change-dir', 'char', 'chop', 'Class', 'clean',
+ 'close', 'command-event', 'cond', 'cons', 'constant',
+ 'context?', 'context', 'copy-file', 'copy', 'cos', 'cosh', 'count',
+ 'cpymem', 'crc32', 'crit-chi2', 'crit-z', 'current-line', 'curry',
+ 'date-list', 'date-parse', 'date-value', 'date', 'debug', 'dec',
+ 'def-new', 'default', 'define-macro', 'define',
+ 'delete-file', 'delete-url', 'delete', 'destroy', 'det', 'device',
+ 'difference', 'directory?', 'directory', 'div', 'do-until', 'do-while',
+ 'doargs', 'dolist', 'dostring', 'dotimes', 'dotree', 'dump', 'dup',
+ 'empty?', 'encrypt', 'ends-with', 'env', 'erf', 'error-event',
+ 'eval-string', 'eval', 'exec', 'exists', 'exit', 'exp', 'expand',
+ 'explode', 'extend', 'factor', 'fft', 'file-info', 'file?', 'filter',
+ 'find-all', 'find', 'first', 'flat', 'float?', 'float', 'floor', 'flt',
+ 'fn', 'for-all', 'for', 'fork', 'format', 'fv', 'gammai', 'gammaln',
+ 'gcd', 'get-char', 'get-float', 'get-int', 'get-long', 'get-string',
+ 'get-url', 'global?', 'global', 'if-not', 'if', 'ifft', 'import', 'inc',
+ 'index', 'inf?', 'int', 'integer?', 'integer', 'intersect', 'invert',
+ 'irr', 'join', 'lambda-macro', 'lambda?', 'lambda', 'last-error',
+ 'last', 'legal?', 'length', 'let', 'letex', 'letn',
+ 'list?', 'list', 'load', 'local', 'log', 'lookup',
+ 'lower-case', 'macro?', 'main-args', 'MAIN', 'make-dir', 'map', 'mat',
+ 'match', 'max', 'member', 'min', 'mod', 'module', 'mul', 'multiply',
+ 'NaN?', 'net-accept', 'net-close', 'net-connect', 'net-error',
+ 'net-eval', 'net-interface', 'net-ipv', 'net-listen', 'net-local',
+ 'net-lookup', 'net-packet', 'net-peek', 'net-peer', 'net-ping',
+ 'net-receive-from', 'net-receive-udp', 'net-receive', 'net-select',
+ 'net-send-to', 'net-send-udp', 'net-send', 'net-service',
+ 'net-sessions', 'new', 'nil?', 'nil', 'normal', 'not', 'now', 'nper',
+ 'npv', 'nth', 'null?', 'number?', 'open', 'or', 'ostype', 'pack',
+ 'parse-date', 'parse', 'peek', 'pipe', 'pmt', 'pop-assoc', 'pop',
+ 'post-url', 'pow', 'prefix', 'pretty-print', 'primitive?', 'print',
+ 'println', 'prob-chi2', 'prob-z', 'process', 'prompt-event',
+ 'protected?', 'push', 'put-url', 'pv', 'quote?', 'quote', 'rand',
+ 'random', 'randomize', 'read', 'read-char', 'read-expr', 'read-file',
+ 'read-key', 'read-line', 'read-utf8', 'reader-event',
+ 'real-path', 'receive', 'ref-all', 'ref', 'regex-comp', 'regex',
+ 'remove-dir', 'rename-file', 'replace', 'reset', 'rest', 'reverse',
+ 'rotate', 'round', 'save', 'search', 'seed', 'seek', 'select', 'self',
+ 'semaphore', 'send', 'sequence', 'series', 'set-locale', 'set-ref-all',
+ 'set-ref', 'set', 'setf', 'setq', 'sgn', 'share', 'signal', 'silent',
+ 'sin', 'sinh', 'sleep', 'slice', 'sort', 'source', 'spawn', 'sqrt',
+ 'starts-with', 'string?', 'string', 'sub', 'swap', 'sym', 'symbol?',
+ 'symbols', 'sync', 'sys-error', 'sys-info', 'tan', 'tanh', 'term',
+ 'throw-error', 'throw', 'time-of-day', 'time', 'timer', 'title-case',
+ 'trace-highlight', 'trace', 'transpose', 'Tree', 'trim', 'true?',
+ 'true', 'unicode', 'unify', 'unique', 'unless', 'unpack', 'until',
+ 'upper-case', 'utf8', 'utf8len', 'uuid', 'wait-pid', 'when', 'while',
+ 'write', 'write-char', 'write-file', 'write-line',
+ 'xfer-event', 'xml-error', 'xml-parse', 'xml-type-tags', 'zero?',
+ )
+
+ # valid names
+ valid_name = r'([\w!$%&*+.,/<=>?@^~|-])+|(\[.*?\])+'
+
+ tokens = {
+ 'root': [
+ # shebang
+ (r'#!(.*?)$', Comment.Preproc),
+ # comments starting with semicolon
+ (r';.*$', Comment.Single),
+ # comments starting with #
+ (r'#.*$', Comment.Single),
+
+ # whitespace
+ (r'\s+', Text),
+
+ # strings, symbols and characters
+ (r'"(\\\\|\\"|[^"])*"', String),
+
+ # braces
+ (r'\{', String, "bracestring"),
+
+ # [text] ... [/text] delimited strings
+ (r'\[text\]*', String, "tagstring"),
+
+ # 'special' operators...
+ (r"('|:)", Operator),
+
+ # highlight the builtins
+ (words(builtins, suffix=r'\b'),
+ Keyword),
+
+ # the remaining functions
+ (r'(?<=\()' + valid_name, Name.Variable),
+
+ # the remaining variables
+ (valid_name, String.Symbol),
+
+ # parentheses
+ (r'(\(|\))', Punctuation),
+ ],
+
+ # braced strings...
+ 'bracestring': [
+ (r'\{', String, "#push"),
+ (r'\}', String, "#pop"),
+ ('[^{}]+', String),
+ ],
+
+ # tagged [text]...[/text] delimited strings...
+ 'tagstring': [
+ (r'(?s)(.*?)(\[/text\])', String, '#pop'),
+ ],
+ }
+
+
+class EmacsLispLexer(RegexLexer):
+ """
+ An ELisp lexer, parsing a stream and outputting the tokens
+ needed to highlight elisp code.
+
+ .. versionadded:: 2.1
+ """
+ name = 'EmacsLisp'
+ aliases = ['emacs', 'elisp', 'emacs-lisp']
+ filenames = ['*.el']
+ mimetypes = ['text/x-elisp', 'application/x-elisp']
+
+ flags = re.MULTILINE
+
+ # couple of useful regexes
+
+ # characters that are not macro-characters and can be used to begin a symbol
+ nonmacro = r'\\.|[\w!$%&*+-/<=>?@^{}~|]'
+ constituent = nonmacro + '|[#.:]'
+ terminated = r'(?=[ "()\]\'\n,;`])' # whitespace or terminating macro characters
+
+ # symbol token, reverse-engineered from hyperspec
+ # Take a deep breath...
+ symbol = r'((?:%s)(?:%s)*)' % (nonmacro, constituent)
+
+ macros = set((
+ 'atomic-change-group', 'case', 'block', 'cl-block', 'cl-callf', 'cl-callf2',
+ 'cl-case', 'cl-decf', 'cl-declaim', 'cl-declare',
+ 'cl-define-compiler-macro', 'cl-defmacro', 'cl-defstruct',
+ 'cl-defsubst', 'cl-deftype', 'cl-defun', 'cl-destructuring-bind',
+ 'cl-do', 'cl-do*', 'cl-do-all-symbols', 'cl-do-symbols', 'cl-dolist',
+ 'cl-dotimes', 'cl-ecase', 'cl-etypecase', 'eval-when', 'cl-eval-when', 'cl-flet',
+ 'cl-flet*', 'cl-function', 'cl-incf', 'cl-labels', 'cl-letf',
+ 'cl-letf*', 'cl-load-time-value', 'cl-locally', 'cl-loop',
+ 'cl-macrolet', 'cl-multiple-value-bind', 'cl-multiple-value-setq',
+ 'cl-progv', 'cl-psetf', 'cl-psetq', 'cl-pushnew', 'cl-remf',
+ 'cl-return', 'cl-return-from', 'cl-rotatef', 'cl-shiftf',
+ 'cl-symbol-macrolet', 'cl-tagbody', 'cl-the', 'cl-typecase',
+ 'combine-after-change-calls', 'condition-case-unless-debug', 'decf',
+ 'declaim', 'declare', 'declare-function', 'def-edebug-spec',
+ 'defadvice', 'defclass', 'defcustom', 'defface', 'defgeneric',
+ 'defgroup', 'define-advice', 'define-alternatives',
+ 'define-compiler-macro', 'define-derived-mode', 'define-generic-mode',
+ 'define-global-minor-mode', 'define-globalized-minor-mode',
+ 'define-minor-mode', 'define-modify-macro',
+ 'define-obsolete-face-alias', 'define-obsolete-function-alias',
+ 'define-obsolete-variable-alias', 'define-setf-expander',
+ 'define-skeleton', 'defmacro', 'defmethod', 'defsetf', 'defstruct',
+ 'defsubst', 'deftheme', 'deftype', 'defun', 'defvar-local',
+ 'delay-mode-hooks', 'destructuring-bind', 'do', 'do*',
+ 'do-all-symbols', 'do-symbols', 'dolist', 'dont-compile', 'dotimes',
+ 'dotimes-with-progress-reporter', 'ecase', 'ert-deftest', 'etypecase',
+ 'eval-and-compile', 'eval-when-compile', 'flet', 'ignore-errors',
+ 'incf', 'labels', 'lambda', 'letrec', 'lexical-let', 'lexical-let*',
+ 'loop', 'multiple-value-bind', 'multiple-value-setq', 'noreturn',
+ 'oref', 'oref-default', 'oset', 'oset-default', 'pcase',
+ 'pcase-defmacro', 'pcase-dolist', 'pcase-exhaustive', 'pcase-let',
+ 'pcase-let*', 'pop', 'psetf', 'psetq', 'push', 'pushnew', 'remf',
+ 'return', 'rotatef', 'rx', 'save-match-data', 'save-selected-window',
+ 'save-window-excursion', 'setf', 'setq-local', 'shiftf',
+ 'track-mouse', 'typecase', 'unless', 'use-package', 'when',
+ 'while-no-input', 'with-case-table', 'with-category-table',
+ 'with-coding-priority', 'with-current-buffer', 'with-demoted-errors',
+ 'with-eval-after-load', 'with-file-modes', 'with-local-quit',
+ 'with-output-to-string', 'with-output-to-temp-buffer',
+ 'with-parsed-tramp-file-name', 'with-selected-frame',
+ 'with-selected-window', 'with-silent-modifications', 'with-slots',
+ 'with-syntax-table', 'with-temp-buffer', 'with-temp-file',
+ 'with-temp-message', 'with-timeout', 'with-tramp-connection-property',
+ 'with-tramp-file-property', 'with-tramp-progress-reporter',
+ 'with-wrapper-hook', 'load-time-value', 'locally', 'macrolet', 'progv',
+ 'return-from',
+ ))
+
+ special_forms = set((
+ 'and', 'catch', 'cond', 'condition-case', 'defconst', 'defvar',
+ 'function', 'if', 'interactive', 'let', 'let*', 'or', 'prog1',
+ 'prog2', 'progn', 'quote', 'save-current-buffer', 'save-excursion',
+ 'save-restriction', 'setq', 'setq-default', 'subr-arity',
+ 'unwind-protect', 'while',
+ ))
+
+ builtin_function = set((
+ '%', '*', '+', '-', '/', '/=', '1+', '1-', '<', '<=', '=', '>', '>=',
+ 'Snarf-documentation', 'abort-recursive-edit', 'abs',
+ 'accept-process-output', 'access-file', 'accessible-keymaps', 'acos',
+ 'active-minibuffer-window', 'add-face-text-property',
+ 'add-name-to-file', 'add-text-properties', 'all-completions',
+ 'append', 'apply', 'apropos-internal', 'aref', 'arrayp', 'aset',
+ 'ash', 'asin', 'assoc', 'assoc-string', 'assq', 'atan', 'atom',
+ 'autoload', 'autoload-do-load', 'backtrace', 'backtrace--locals',
+ 'backtrace-debug', 'backtrace-eval', 'backtrace-frame',
+ 'backward-char', 'backward-prefix-chars', 'barf-if-buffer-read-only',
+ 'base64-decode-region', 'base64-decode-string',
+ 'base64-encode-region', 'base64-encode-string', 'beginning-of-line',
+ 'bidi-find-overridden-directionality', 'bidi-resolved-levels',
+ 'bitmap-spec-p', 'bobp', 'bolp', 'bool-vector',
+ 'bool-vector-count-consecutive', 'bool-vector-count-population',
+ 'bool-vector-exclusive-or', 'bool-vector-intersection',
+ 'bool-vector-not', 'bool-vector-p', 'bool-vector-set-difference',
+ 'bool-vector-subsetp', 'bool-vector-union', 'boundp',
+ 'buffer-base-buffer', 'buffer-chars-modified-tick',
+ 'buffer-enable-undo', 'buffer-file-name', 'buffer-has-markers-at',
+ 'buffer-list', 'buffer-live-p', 'buffer-local-value',
+ 'buffer-local-variables', 'buffer-modified-p', 'buffer-modified-tick',
+ 'buffer-name', 'buffer-size', 'buffer-string', 'buffer-substring',
+ 'buffer-substring-no-properties', 'buffer-swap-text', 'bufferp',
+ 'bury-buffer-internal', 'byte-code', 'byte-code-function-p',
+ 'byte-to-position', 'byte-to-string', 'byteorder',
+ 'call-interactively', 'call-last-kbd-macro', 'call-process',
+ 'call-process-region', 'cancel-kbd-macro-events', 'capitalize',
+ 'capitalize-region', 'capitalize-word', 'car', 'car-less-than-car',
+ 'car-safe', 'case-table-p', 'category-docstring',
+ 'category-set-mnemonics', 'category-table', 'category-table-p',
+ 'ccl-execute', 'ccl-execute-on-string', 'ccl-program-p', 'cdr',
+ 'cdr-safe', 'ceiling', 'char-after', 'char-before',
+ 'char-category-set', 'char-charset', 'char-equal', 'char-or-string-p',
+ 'char-resolve-modifiers', 'char-syntax', 'char-table-extra-slot',
+ 'char-table-p', 'char-table-parent', 'char-table-range',
+ 'char-table-subtype', 'char-to-string', 'char-width', 'characterp',
+ 'charset-after', 'charset-id-internal', 'charset-plist',
+ 'charset-priority-list', 'charsetp', 'check-coding-system',
+ 'check-coding-systems-region', 'clear-buffer-auto-save-failure',
+ 'clear-charset-maps', 'clear-face-cache', 'clear-font-cache',
+ 'clear-image-cache', 'clear-string', 'clear-this-command-keys',
+ 'close-font', 'clrhash', 'coding-system-aliases',
+ 'coding-system-base', 'coding-system-eol-type', 'coding-system-p',
+ 'coding-system-plist', 'coding-system-priority-list',
+ 'coding-system-put', 'color-distance', 'color-gray-p',
+ 'color-supported-p', 'combine-after-change-execute',
+ 'command-error-default-function', 'command-remapping', 'commandp',
+ 'compare-buffer-substrings', 'compare-strings',
+ 'compare-window-configurations', 'completing-read',
+ 'compose-region-internal', 'compose-string-internal',
+ 'composition-get-gstring', 'compute-motion', 'concat', 'cons',
+ 'consp', 'constrain-to-field', 'continue-process',
+ 'controlling-tty-p', 'coordinates-in-window-p', 'copy-alist',
+ 'copy-category-table', 'copy-file', 'copy-hash-table', 'copy-keymap',
+ 'copy-marker', 'copy-sequence', 'copy-syntax-table', 'copysign',
+ 'cos', 'current-active-maps', 'current-bidi-paragraph-direction',
+ 'current-buffer', 'current-case-table', 'current-column',
+ 'current-global-map', 'current-idle-time', 'current-indentation',
+ 'current-input-mode', 'current-local-map', 'current-message',
+ 'current-minor-mode-maps', 'current-time', 'current-time-string',
+ 'current-time-zone', 'current-window-configuration',
+ 'cygwin-convert-file-name-from-windows',
+ 'cygwin-convert-file-name-to-windows', 'daemon-initialized',
+ 'daemonp', 'dbus--init-bus', 'dbus-get-unique-name',
+ 'dbus-message-internal', 'debug-timer-check', 'declare-equiv-charset',
+ 'decode-big5-char', 'decode-char', 'decode-coding-region',
+ 'decode-coding-string', 'decode-sjis-char', 'decode-time',
+ 'default-boundp', 'default-file-modes', 'default-printer-name',
+ 'default-toplevel-value', 'default-value', 'define-category',
+ 'define-charset-alias', 'define-charset-internal',
+ 'define-coding-system-alias', 'define-coding-system-internal',
+ 'define-fringe-bitmap', 'define-hash-table-test', 'define-key',
+ 'define-prefix-command', 'delete',
+ 'delete-all-overlays', 'delete-and-extract-region', 'delete-char',
+ 'delete-directory-internal', 'delete-field', 'delete-file',
+ 'delete-frame', 'delete-other-windows-internal', 'delete-overlay',
+ 'delete-process', 'delete-region', 'delete-terminal',
+ 'delete-window-internal', 'delq', 'describe-buffer-bindings',
+ 'describe-vector', 'destroy-fringe-bitmap', 'detect-coding-region',
+ 'detect-coding-string', 'ding', 'directory-file-name',
+ 'directory-files', 'directory-files-and-attributes', 'discard-input',
+ 'display-supports-face-attributes-p', 'do-auto-save', 'documentation',
+ 'documentation-property', 'downcase', 'downcase-region',
+ 'downcase-word', 'draw-string', 'dump-colors', 'dump-emacs',
+ 'dump-face', 'dump-frame-glyph-matrix', 'dump-glyph-matrix',
+ 'dump-glyph-row', 'dump-redisplay-history', 'dump-tool-bar-row',
+ 'elt', 'emacs-pid', 'encode-big5-char', 'encode-char',
+ 'encode-coding-region', 'encode-coding-string', 'encode-sjis-char',
+ 'encode-time', 'end-kbd-macro', 'end-of-line', 'eobp', 'eolp', 'eq',
+ 'eql', 'equal', 'equal-including-properties', 'erase-buffer',
+ 'error-message-string', 'eval', 'eval-buffer', 'eval-region',
+ 'event-convert-list', 'execute-kbd-macro', 'exit-recursive-edit',
+ 'exp', 'expand-file-name', 'expt', 'external-debugging-output',
+ 'face-attribute-relative-p', 'face-attributes-as-vector', 'face-font',
+ 'fboundp', 'fceiling', 'fetch-bytecode', 'ffloor',
+ 'field-beginning', 'field-end', 'field-string',
+ 'field-string-no-properties', 'file-accessible-directory-p',
+ 'file-acl', 'file-attributes', 'file-attributes-lessp',
+ 'file-directory-p', 'file-executable-p', 'file-exists-p',
+ 'file-locked-p', 'file-modes', 'file-name-absolute-p',
+ 'file-name-all-completions', 'file-name-as-directory',
+ 'file-name-completion', 'file-name-directory',
+ 'file-name-nondirectory', 'file-newer-than-file-p', 'file-readable-p',
+ 'file-regular-p', 'file-selinux-context', 'file-symlink-p',
+ 'file-system-info', 'file-system-info', 'file-writable-p',
+ 'fillarray', 'find-charset-region', 'find-charset-string',
+ 'find-coding-systems-region-internal', 'find-composition-internal',
+ 'find-file-name-handler', 'find-font', 'find-operation-coding-system',
+ 'float', 'float-time', 'floatp', 'floor', 'fmakunbound',
+ 'following-char', 'font-at', 'font-drive-otf', 'font-face-attributes',
+ 'font-family-list', 'font-get', 'font-get-glyphs',
+ 'font-get-system-font', 'font-get-system-normal-font', 'font-info',
+ 'font-match-p', 'font-otf-alternates', 'font-put',
+ 'font-shape-gstring', 'font-spec', 'font-variation-glyphs',
+ 'font-xlfd-name', 'fontp', 'fontset-font', 'fontset-info',
+ 'fontset-list', 'fontset-list-all', 'force-mode-line-update',
+ 'force-window-update', 'format', 'format-mode-line',
+ 'format-network-address', 'format-time-string', 'forward-char',
+ 'forward-comment', 'forward-line', 'forward-word',
+ 'frame-border-width', 'frame-bottom-divider-width',
+ 'frame-can-run-window-configuration-change-hook', 'frame-char-height',
+ 'frame-char-width', 'frame-face-alist', 'frame-first-window',
+ 'frame-focus', 'frame-font-cache', 'frame-fringe-width', 'frame-list',
+ 'frame-live-p', 'frame-or-buffer-changed-p', 'frame-parameter',
+ 'frame-parameters', 'frame-pixel-height', 'frame-pixel-width',
+ 'frame-pointer-visible-p', 'frame-right-divider-width',
+ 'frame-root-window', 'frame-scroll-bar-height',
+ 'frame-scroll-bar-width', 'frame-selected-window', 'frame-terminal',
+ 'frame-text-cols', 'frame-text-height', 'frame-text-lines',
+ 'frame-text-width', 'frame-total-cols', 'frame-total-lines',
+ 'frame-visible-p', 'framep', 'frexp', 'fringe-bitmaps-at-pos',
+ 'fround', 'fset', 'ftruncate', 'funcall', 'funcall-interactively',
+ 'function-equal', 'functionp', 'gap-position', 'gap-size',
+ 'garbage-collect', 'gc-status', 'generate-new-buffer-name', 'get',
+ 'get-buffer', 'get-buffer-create', 'get-buffer-process',
+ 'get-buffer-window', 'get-byte', 'get-char-property',
+ 'get-char-property-and-overlay', 'get-file-buffer', 'get-file-char',
+ 'get-internal-run-time', 'get-load-suffixes', 'get-pos-property',
+ 'get-process', 'get-screen-color', 'get-text-property',
+ 'get-unicode-property-internal', 'get-unused-category',
+ 'get-unused-iso-final-char', 'getenv-internal', 'gethash',
+ 'gfile-add-watch', 'gfile-rm-watch', 'global-key-binding',
+ 'gnutls-available-p', 'gnutls-boot', 'gnutls-bye', 'gnutls-deinit',
+ 'gnutls-error-fatalp', 'gnutls-error-string', 'gnutls-errorp',
+ 'gnutls-get-initstage', 'gnutls-peer-status',
+ 'gnutls-peer-status-warning-describe', 'goto-char', 'gpm-mouse-start',
+ 'gpm-mouse-stop', 'group-gid', 'group-real-gid',
+ 'handle-save-session', 'handle-switch-frame', 'hash-table-count',
+ 'hash-table-p', 'hash-table-rehash-size',
+ 'hash-table-rehash-threshold', 'hash-table-size', 'hash-table-test',
+ 'hash-table-weakness', 'iconify-frame', 'identity', 'image-flush',
+ 'image-mask-p', 'image-metadata', 'image-size', 'imagemagick-types',
+ 'imagep', 'indent-to', 'indirect-function', 'indirect-variable',
+ 'init-image-library', 'inotify-add-watch', 'inotify-rm-watch',
+ 'input-pending-p', 'insert', 'insert-and-inherit',
+ 'insert-before-markers', 'insert-before-markers-and-inherit',
+ 'insert-buffer-substring', 'insert-byte', 'insert-char',
+ 'insert-file-contents', 'insert-startup-screen', 'int86',
+ 'integer-or-marker-p', 'integerp', 'interactive-form', 'intern',
+ 'intern-soft', 'internal--track-mouse', 'internal-char-font',
+ 'internal-complete-buffer', 'internal-copy-lisp-face',
+ 'internal-default-process-filter',
+ 'internal-default-process-sentinel', 'internal-describe-syntax-value',
+ 'internal-event-symbol-parse-modifiers',
+ 'internal-face-x-get-resource', 'internal-get-lisp-face-attribute',
+ 'internal-lisp-face-attribute-values', 'internal-lisp-face-empty-p',
+ 'internal-lisp-face-equal-p', 'internal-lisp-face-p',
+ 'internal-make-lisp-face', 'internal-make-var-non-special',
+ 'internal-merge-in-global-face',
+ 'internal-set-alternative-font-family-alist',
+ 'internal-set-alternative-font-registry-alist',
+ 'internal-set-font-selection-order',
+ 'internal-set-lisp-face-attribute',
+ 'internal-set-lisp-face-attribute-from-resource',
+ 'internal-show-cursor', 'internal-show-cursor-p', 'interrupt-process',
+ 'invisible-p', 'invocation-directory', 'invocation-name', 'isnan',
+ 'iso-charset', 'key-binding', 'key-description',
+ 'keyboard-coding-system', 'keymap-parent', 'keymap-prompt', 'keymapp',
+ 'keywordp', 'kill-all-local-variables', 'kill-buffer', 'kill-emacs',
+ 'kill-local-variable', 'kill-process', 'last-nonminibuffer-frame',
+ 'lax-plist-get', 'lax-plist-put', 'ldexp', 'length',
+ 'libxml-parse-html-region', 'libxml-parse-xml-region',
+ 'line-beginning-position', 'line-end-position', 'line-pixel-height',
+ 'list', 'list-fonts', 'list-system-processes', 'listp', 'load',
+ 'load-average', 'local-key-binding', 'local-variable-if-set-p',
+ 'local-variable-p', 'locale-info', 'locate-file-internal',
+ 'lock-buffer', 'log', 'logand', 'logb', 'logior', 'lognot', 'logxor',
+ 'looking-at', 'lookup-image', 'lookup-image-map', 'lookup-key',
+ 'lower-frame', 'lsh', 'macroexpand', 'make-bool-vector',
+ 'make-byte-code', 'make-category-set', 'make-category-table',
+ 'make-char', 'make-char-table', 'make-directory-internal',
+ 'make-frame-invisible', 'make-frame-visible', 'make-hash-table',
+ 'make-indirect-buffer', 'make-keymap', 'make-list',
+ 'make-local-variable', 'make-marker', 'make-network-process',
+ 'make-overlay', 'make-serial-process', 'make-sparse-keymap',
+ 'make-string', 'make-symbol', 'make-symbolic-link', 'make-temp-name',
+ 'make-terminal-frame', 'make-variable-buffer-local',
+ 'make-variable-frame-local', 'make-vector', 'makunbound',
+ 'map-char-table', 'map-charset-chars', 'map-keymap',
+ 'map-keymap-internal', 'mapatoms', 'mapc', 'mapcar', 'mapconcat',
+ 'maphash', 'mark-marker', 'marker-buffer', 'marker-insertion-type',
+ 'marker-position', 'markerp', 'match-beginning', 'match-data',
+ 'match-end', 'matching-paren', 'max', 'max-char', 'md5', 'member',
+ 'memory-info', 'memory-limit', 'memory-use-counts', 'memq', 'memql',
+ 'menu-bar-menu-at-x-y', 'menu-or-popup-active-p',
+ 'menu-or-popup-active-p', 'merge-face-attribute', 'message',
+ 'message-box', 'message-or-box', 'min',
+ 'minibuffer-completion-contents', 'minibuffer-contents',
+ 'minibuffer-contents-no-properties', 'minibuffer-depth',
+ 'minibuffer-prompt', 'minibuffer-prompt-end',
+ 'minibuffer-selected-window', 'minibuffer-window', 'minibufferp',
+ 'minor-mode-key-binding', 'mod', 'modify-category-entry',
+ 'modify-frame-parameters', 'modify-syntax-entry',
+ 'mouse-pixel-position', 'mouse-position', 'move-overlay',
+ 'move-point-visually', 'move-to-column', 'move-to-window-line',
+ 'msdos-downcase-filename', 'msdos-long-file-names', 'msdos-memget',
+ 'msdos-memput', 'msdos-mouse-disable', 'msdos-mouse-enable',
+ 'msdos-mouse-init', 'msdos-mouse-p', 'msdos-remember-default-colors',
+ 'msdos-set-keyboard', 'msdos-set-mouse-buttons',
+ 'multibyte-char-to-unibyte', 'multibyte-string-p', 'narrow-to-region',
+ 'natnump', 'nconc', 'network-interface-info',
+ 'network-interface-list', 'new-fontset', 'newline-cache-check',
+ 'next-char-property-change', 'next-frame', 'next-overlay-change',
+ 'next-property-change', 'next-read-file-uses-dialog-p',
+ 'next-single-char-property-change', 'next-single-property-change',
+ 'next-window', 'nlistp', 'nreverse', 'nth', 'nthcdr', 'null',
+ 'number-or-marker-p', 'number-to-string', 'numberp',
+ 'open-dribble-file', 'open-font', 'open-termscript',
+ 'optimize-char-table', 'other-buffer', 'other-window-for-scrolling',
+ 'overlay-buffer', 'overlay-end', 'overlay-get', 'overlay-lists',
+ 'overlay-properties', 'overlay-put', 'overlay-recenter',
+ 'overlay-start', 'overlayp', 'overlays-at', 'overlays-in',
+ 'parse-partial-sexp', 'play-sound-internal', 'plist-get',
+ 'plist-member', 'plist-put', 'point', 'point-marker', 'point-max',
+ 'point-max-marker', 'point-min', 'point-min-marker',
+ 'pos-visible-in-window-p', 'position-bytes', 'posix-looking-at',
+ 'posix-search-backward', 'posix-search-forward', 'posix-string-match',
+ 'posn-at-point', 'posn-at-x-y', 'preceding-char',
+ 'prefix-numeric-value', 'previous-char-property-change',
+ 'previous-frame', 'previous-overlay-change',
+ 'previous-property-change', 'previous-single-char-property-change',
+ 'previous-single-property-change', 'previous-window', 'prin1',
+ 'prin1-to-string', 'princ', 'print', 'process-attributes',
+ 'process-buffer', 'process-coding-system', 'process-command',
+ 'process-connection', 'process-contact', 'process-datagram-address',
+ 'process-exit-status', 'process-filter', 'process-filter-multibyte-p',
+ 'process-id', 'process-inherit-coding-system-flag', 'process-list',
+ 'process-mark', 'process-name', 'process-plist',
+ 'process-query-on-exit-flag', 'process-running-child-p',
+ 'process-send-eof', 'process-send-region', 'process-send-string',
+ 'process-sentinel', 'process-status', 'process-tty-name',
+ 'process-type', 'processp', 'profiler-cpu-log',
+ 'profiler-cpu-running-p', 'profiler-cpu-start', 'profiler-cpu-stop',
+ 'profiler-memory-log', 'profiler-memory-running-p',
+ 'profiler-memory-start', 'profiler-memory-stop', 'propertize',
+ 'purecopy', 'put', 'put-text-property',
+ 'put-unicode-property-internal', 'puthash', 'query-font',
+ 'query-fontset', 'quit-process', 'raise-frame', 'random', 'rassoc',
+ 'rassq', 're-search-backward', 're-search-forward', 'read',
+ 'read-buffer', 'read-char', 'read-char-exclusive',
+ 'read-coding-system', 'read-command', 'read-event',
+ 'read-from-minibuffer', 'read-from-string', 'read-function',
+ 'read-key-sequence', 'read-key-sequence-vector',
+ 'read-no-blanks-input', 'read-non-nil-coding-system', 'read-string',
+ 'read-variable', 'recent-auto-save-p', 'recent-doskeys',
+ 'recent-keys', 'recenter', 'recursion-depth', 'recursive-edit',
+ 'redirect-debugging-output', 'redirect-frame-focus', 'redisplay',
+ 'redraw-display', 'redraw-frame', 'regexp-quote', 'region-beginning',
+ 'region-end', 'register-ccl-program', 'register-code-conversion-map',
+ 'remhash', 'remove-list-of-text-properties', 'remove-text-properties',
+ 'rename-buffer', 'rename-file', 'replace-match',
+ 'reset-this-command-lengths', 'resize-mini-window-internal',
+ 'restore-buffer-modified-p', 'resume-tty', 'reverse', 'round',
+ 'run-hook-with-args', 'run-hook-with-args-until-failure',
+ 'run-hook-with-args-until-success', 'run-hook-wrapped', 'run-hooks',
+ 'run-window-configuration-change-hook', 'run-window-scroll-functions',
+ 'safe-length', 'scan-lists', 'scan-sexps', 'scroll-down',
+ 'scroll-left', 'scroll-other-window', 'scroll-right', 'scroll-up',
+ 'search-backward', 'search-forward', 'secure-hash', 'select-frame',
+ 'select-window', 'selected-frame', 'selected-window',
+ 'self-insert-command', 'send-string-to-terminal', 'sequencep',
+ 'serial-process-configure', 'set', 'set-buffer',
+ 'set-buffer-auto-saved', 'set-buffer-major-mode',
+ 'set-buffer-modified-p', 'set-buffer-multibyte', 'set-case-table',
+ 'set-category-table', 'set-char-table-extra-slot',
+ 'set-char-table-parent', 'set-char-table-range', 'set-charset-plist',
+ 'set-charset-priority', 'set-coding-system-priority',
+ 'set-cursor-size', 'set-default', 'set-default-file-modes',
+ 'set-default-toplevel-value', 'set-file-acl', 'set-file-modes',
+ 'set-file-selinux-context', 'set-file-times', 'set-fontset-font',
+ 'set-frame-height', 'set-frame-position', 'set-frame-selected-window',
+ 'set-frame-size', 'set-frame-width', 'set-fringe-bitmap-face',
+ 'set-input-interrupt-mode', 'set-input-meta-mode', 'set-input-mode',
+ 'set-keyboard-coding-system-internal', 'set-keymap-parent',
+ 'set-marker', 'set-marker-insertion-type', 'set-match-data',
+ 'set-message-beep', 'set-minibuffer-window',
+ 'set-mouse-pixel-position', 'set-mouse-position',
+ 'set-network-process-option', 'set-output-flow-control',
+ 'set-process-buffer', 'set-process-coding-system',
+ 'set-process-datagram-address', 'set-process-filter',
+ 'set-process-filter-multibyte',
+ 'set-process-inherit-coding-system-flag', 'set-process-plist',
+ 'set-process-query-on-exit-flag', 'set-process-sentinel',
+ 'set-process-window-size', 'set-quit-char',
+ 'set-safe-terminal-coding-system-internal', 'set-screen-color',
+ 'set-standard-case-table', 'set-syntax-table',
+ 'set-terminal-coding-system-internal', 'set-terminal-local-value',
+ 'set-terminal-parameter', 'set-text-properties', 'set-time-zone-rule',
+ 'set-visited-file-modtime', 'set-window-buffer',
+ 'set-window-combination-limit', 'set-window-configuration',
+ 'set-window-dedicated-p', 'set-window-display-table',
+ 'set-window-fringes', 'set-window-hscroll', 'set-window-margins',
+ 'set-window-new-normal', 'set-window-new-pixel',
+ 'set-window-new-total', 'set-window-next-buffers',
+ 'set-window-parameter', 'set-window-point', 'set-window-prev-buffers',
+ 'set-window-redisplay-end-trigger', 'set-window-scroll-bars',
+ 'set-window-start', 'set-window-vscroll', 'setcar', 'setcdr',
+ 'setplist', 'show-face-resources', 'signal', 'signal-process', 'sin',
+ 'single-key-description', 'skip-chars-backward', 'skip-chars-forward',
+ 'skip-syntax-backward', 'skip-syntax-forward', 'sleep-for', 'sort',
+ 'sort-charsets', 'special-variable-p', 'split-char',
+ 'split-window-internal', 'sqrt', 'standard-case-table',
+ 'standard-category-table', 'standard-syntax-table', 'start-kbd-macro',
+ 'start-process', 'stop-process', 'store-kbd-macro-event', 'string',
+ 'string-as-multibyte', 'string-as-unibyte', 'string-bytes',
+ 'string-collate-equalp', 'string-collate-lessp', 'string-equal',
+ 'string-lessp', 'string-make-multibyte', 'string-make-unibyte',
+ 'string-match', 'string-to-char', 'string-to-multibyte',
+ 'string-to-number', 'string-to-syntax', 'string-to-unibyte',
+ 'string-width', 'stringp', 'subr-name', 'subrp',
+ 'subst-char-in-region', 'substitute-command-keys',
+ 'substitute-in-file-name', 'substring', 'substring-no-properties',
+ 'suspend-emacs', 'suspend-tty', 'suspicious-object', 'sxhash',
+ 'symbol-function', 'symbol-name', 'symbol-plist', 'symbol-value',
+ 'symbolp', 'syntax-table', 'syntax-table-p', 'system-groups',
+ 'system-move-file-to-trash', 'system-name', 'system-users', 'tan',
+ 'terminal-coding-system', 'terminal-list', 'terminal-live-p',
+ 'terminal-local-value', 'terminal-name', 'terminal-parameter',
+ 'terminal-parameters', 'terpri', 'test-completion',
+ 'text-char-description', 'text-properties-at', 'text-property-any',
+ 'text-property-not-all', 'this-command-keys',
+ 'this-command-keys-vector', 'this-single-command-keys',
+ 'this-single-command-raw-keys', 'time-add', 'time-less-p',
+ 'time-subtract', 'tool-bar-get-system-style', 'tool-bar-height',
+ 'tool-bar-pixel-width', 'top-level', 'trace-redisplay',
+ 'trace-to-stderr', 'translate-region-internal', 'transpose-regions',
+ 'truncate', 'try-completion', 'tty-display-color-cells',
+ 'tty-display-color-p', 'tty-no-underline',
+ 'tty-suppress-bold-inverse-default-colors', 'tty-top-frame',
+ 'tty-type', 'type-of', 'undo-boundary', 'unencodable-char-position',
+ 'unhandled-file-name-directory', 'unibyte-char-to-multibyte',
+ 'unibyte-string', 'unicode-property-table-internal', 'unify-charset',
+ 'unintern', 'unix-sync', 'unlock-buffer', 'upcase', 'upcase-initials',
+ 'upcase-initials-region', 'upcase-region', 'upcase-word',
+ 'use-global-map', 'use-local-map', 'user-full-name',
+ 'user-login-name', 'user-real-login-name', 'user-real-uid',
+ 'user-uid', 'variable-binding-locus', 'vconcat', 'vector',
+ 'vector-or-char-table-p', 'vectorp', 'verify-visited-file-modtime',
+ 'vertical-motion', 'visible-frame-list', 'visited-file-modtime',
+ 'w16-get-clipboard-data', 'w16-selection-exists-p',
+ 'w16-set-clipboard-data', 'w32-battery-status',
+ 'w32-default-color-map', 'w32-define-rgb-color',
+ 'w32-display-monitor-attributes-list', 'w32-frame-menu-bar-size',
+ 'w32-frame-rect', 'w32-get-clipboard-data',
+ 'w32-get-codepage-charset', 'w32-get-console-codepage',
+ 'w32-get-console-output-codepage', 'w32-get-current-locale-id',
+ 'w32-get-default-locale-id', 'w32-get-keyboard-layout',
+ 'w32-get-locale-info', 'w32-get-valid-codepages',
+ 'w32-get-valid-keyboard-layouts', 'w32-get-valid-locale-ids',
+ 'w32-has-winsock', 'w32-long-file-name', 'w32-reconstruct-hot-key',
+ 'w32-register-hot-key', 'w32-registered-hot-keys',
+ 'w32-selection-exists-p', 'w32-send-sys-command',
+ 'w32-set-clipboard-data', 'w32-set-console-codepage',
+ 'w32-set-console-output-codepage', 'w32-set-current-locale',
+ 'w32-set-keyboard-layout', 'w32-set-process-priority',
+ 'w32-shell-execute', 'w32-short-file-name', 'w32-toggle-lock-key',
+ 'w32-unload-winsock', 'w32-unregister-hot-key', 'w32-window-exists-p',
+ 'w32notify-add-watch', 'w32notify-rm-watch',
+ 'waiting-for-user-input-p', 'where-is-internal', 'widen',
+ 'widget-apply', 'widget-get', 'widget-put',
+ 'window-absolute-pixel-edges', 'window-at', 'window-body-height',
+ 'window-body-width', 'window-bottom-divider-width', 'window-buffer',
+ 'window-combination-limit', 'window-configuration-frame',
+ 'window-configuration-p', 'window-dedicated-p',
+ 'window-display-table', 'window-edges', 'window-end', 'window-frame',
+ 'window-fringes', 'window-header-line-height', 'window-hscroll',
+ 'window-inside-absolute-pixel-edges', 'window-inside-edges',
+ 'window-inside-pixel-edges', 'window-left-child',
+ 'window-left-column', 'window-line-height', 'window-list',
+ 'window-list-1', 'window-live-p', 'window-margins',
+ 'window-minibuffer-p', 'window-mode-line-height', 'window-new-normal',
+ 'window-new-pixel', 'window-new-total', 'window-next-buffers',
+ 'window-next-sibling', 'window-normal-size', 'window-old-point',
+ 'window-parameter', 'window-parameters', 'window-parent',
+ 'window-pixel-edges', 'window-pixel-height', 'window-pixel-left',
+ 'window-pixel-top', 'window-pixel-width', 'window-point',
+ 'window-prev-buffers', 'window-prev-sibling',
+ 'window-redisplay-end-trigger', 'window-resize-apply',
+ 'window-resize-apply-total', 'window-right-divider-width',
+ 'window-scroll-bar-height', 'window-scroll-bar-width',
+ 'window-scroll-bars', 'window-start', 'window-system',
+ 'window-text-height', 'window-text-pixel-size', 'window-text-width',
+ 'window-top-child', 'window-top-line', 'window-total-height',
+ 'window-total-width', 'window-use-time', 'window-valid-p',
+ 'window-vscroll', 'windowp', 'write-char', 'write-region',
+ 'x-backspace-delete-keys-p', 'x-change-window-property',
+ 'x-change-window-property', 'x-close-connection',
+ 'x-close-connection', 'x-create-frame', 'x-create-frame',
+ 'x-delete-window-property', 'x-delete-window-property',
+ 'x-disown-selection-internal', 'x-display-backing-store',
+ 'x-display-backing-store', 'x-display-color-cells',
+ 'x-display-color-cells', 'x-display-grayscale-p',
+ 'x-display-grayscale-p', 'x-display-list', 'x-display-list',
+ 'x-display-mm-height', 'x-display-mm-height', 'x-display-mm-width',
+ 'x-display-mm-width', 'x-display-monitor-attributes-list',
+ 'x-display-pixel-height', 'x-display-pixel-height',
+ 'x-display-pixel-width', 'x-display-pixel-width', 'x-display-planes',
+ 'x-display-planes', 'x-display-save-under', 'x-display-save-under',
+ 'x-display-screens', 'x-display-screens', 'x-display-visual-class',
+ 'x-display-visual-class', 'x-family-fonts', 'x-file-dialog',
+ 'x-file-dialog', 'x-file-dialog', 'x-focus-frame', 'x-frame-geometry',
+ 'x-frame-geometry', 'x-get-atom-name', 'x-get-resource',
+ 'x-get-selection-internal', 'x-hide-tip', 'x-hide-tip',
+ 'x-list-fonts', 'x-load-color-file', 'x-menu-bar-open-internal',
+ 'x-menu-bar-open-internal', 'x-open-connection', 'x-open-connection',
+ 'x-own-selection-internal', 'x-parse-geometry', 'x-popup-dialog',
+ 'x-popup-menu', 'x-register-dnd-atom', 'x-select-font',
+ 'x-select-font', 'x-selection-exists-p', 'x-selection-owner-p',
+ 'x-send-client-message', 'x-server-max-request-size',
+ 'x-server-max-request-size', 'x-server-vendor', 'x-server-vendor',
+ 'x-server-version', 'x-server-version', 'x-show-tip', 'x-show-tip',
+ 'x-synchronize', 'x-synchronize', 'x-uses-old-gtk-dialog',
+ 'x-window-property', 'x-window-property', 'x-wm-set-size-hint',
+ 'xw-color-defined-p', 'xw-color-defined-p', 'xw-color-values',
+ 'xw-color-values', 'xw-display-color-p', 'xw-display-color-p',
+ 'yes-or-no-p', 'zlib-available-p', 'zlib-decompress-region',
+ 'forward-point',
+ ))
+
+ builtin_function_highlighted = set((
+ 'defvaralias', 'provide', 'require',
+ 'with-no-warnings', 'define-widget', 'with-electric-help',
+ 'throw', 'defalias', 'featurep'
+ ))
+
+ lambda_list_keywords = set((
+ '&allow-other-keys', '&aux', '&body', '&environment', '&key', '&optional',
+ '&rest', '&whole',
+ ))
+
+ error_keywords = set((
+ 'cl-assert', 'cl-check-type', 'error', 'signal',
+ 'user-error', 'warn',
+ ))
+
+ def get_tokens_unprocessed(self, text):
+ stack = ['root']
+ for index, token, value in RegexLexer.get_tokens_unprocessed(self, text, stack):
+ if token is Name.Variable:
+ if value in EmacsLispLexer.builtin_function:
+ yield index, Name.Function, value
+ continue
+ if value in EmacsLispLexer.special_forms:
+ yield index, Keyword, value
+ continue
+ if value in EmacsLispLexer.error_keywords:
+ yield index, Name.Exception, value
+ continue
+ if value in EmacsLispLexer.builtin_function_highlighted:
+ yield index, Name.Builtin, value
+ continue
+ if value in EmacsLispLexer.macros:
+ yield index, Name.Builtin, value
+ continue
+ if value in EmacsLispLexer.lambda_list_keywords:
+ yield index, Keyword.Pseudo, value
+ continue
+ yield index, token, value
+
+ tokens = {
+ 'root': [
+ default('body'),
+ ],
+ 'body': [
+ # whitespace
+ (r'\s+', Text),
+
+ # single-line comment
+ (r';.*$', Comment.Single),
+
+ # strings and characters
+ (r'"', String, 'string'),
+ (r'\?([^\\]|\\.)', String.Char),
+ # quoting
+ (r":" + symbol, Name.Builtin),
+ (r"::" + symbol, String.Symbol),
+ (r"'" + symbol, String.Symbol),
+ (r"'", Operator),
+ (r"`", Operator),
+
+ # decimal numbers
+ (r'[-+]?\d+\.?' + terminated, Number.Integer),
+ (r'[-+]?\d+/\d+' + terminated, Number),
+ (r'[-+]?(\d*\.\d+([defls][-+]?\d+)?|\d+(\.\d*)?[defls][-+]?\d+)' +
+ terminated, Number.Float),
+
+ # vectors
+ (r'\[|\]', Punctuation),
+
+ # uninterned symbol
+ (r'#:' + symbol, String.Symbol),
+
+ # read syntax for char tables
+ (r'#\^\^?', Operator),
+
+ # function shorthand
+ (r'#\'', Name.Function),
+
+ # binary rational
+ (r'#[bB][+-]?[01]+(/[01]+)?', Number.Bin),
+
+ # octal rational
+ (r'#[oO][+-]?[0-7]+(/[0-7]+)?', Number.Oct),
+
+ # hex rational
+ (r'#[xX][+-]?[0-9a-fA-F]+(/[0-9a-fA-F]+)?', Number.Hex),
+
+ # radix rational
+ (r'#\d+r[+-]?[0-9a-zA-Z]+(/[0-9a-zA-Z]+)?', Number),
+
+ # reference
+ (r'#\d+=', Operator),
+ (r'#\d+#', Operator),
+
+ # special operators that should have been parsed already
+ (r'(,@|,|\.|:)', Operator),
+
+ # special constants
+ (r'(t|nil)' + terminated, Name.Constant),
+
+ # functions and variables
+ (r'\*' + symbol + '\*', Name.Variable.Global),
+ (symbol, Name.Variable),
+
+ # parentheses
+ (r'#\(', Operator, 'body'),
+ (r'\(', Punctuation, 'body'),
+ (r'\)', Punctuation, '#pop'),
+ ],
+ 'string': [
+ (r'[^"\\`]+', String),
+ (r'`%s\'' % symbol, String.Symbol),
+ (r'`', String),
+ (r'\\.', String),
+ (r'\\\n', String),
+ (r'"', String, '#pop'),
+ ],
+ }
+
+
+class ShenLexer(RegexLexer):
+ """
+ Lexer for `Shen <http://shenlanguage.org/>`_ source code.
+
+ .. versionadded:: 2.1
+ """
+ name = 'Shen'
+ aliases = ['shen']
+ filenames = ['*.shen']
+ mimetypes = ['text/x-shen', 'application/x-shen']
+
+ DECLARATIONS = (
+ 'datatype', 'define', 'defmacro', 'defprolog', 'defcc',
+ 'synonyms', 'declare', 'package', 'type', 'function',
+ )
+
+ SPECIAL_FORMS = (
+ 'lambda', 'get', 'let', 'if', 'cases', 'cond', 'put', 'time', 'freeze',
+ 'value', 'load', '$', 'protect', 'or', 'and', 'not', 'do', 'output',
+ 'prolog?', 'trap-error', 'error', 'make-string', '/.', 'set', '@p',
+ '@s', '@v',
+ )
+
+ BUILTINS = (
+ '==', '=', '*', '+', '-', '/', '<', '>', '>=', '<=', '<-address',
+ '<-vector', 'abort', 'absvector', 'absvector?', 'address->', 'adjoin',
+ 'append', 'arity', 'assoc', 'bind', 'boolean?', 'bound?', 'call', 'cd',
+ 'close', 'cn', 'compile', 'concat', 'cons', 'cons?', 'cut', 'destroy',
+ 'difference', 'element?', 'empty?', 'enable-type-theory',
+ 'error-to-string', 'eval', 'eval-kl', 'exception', 'explode', 'external',
+ 'fail', 'fail-if', 'file', 'findall', 'fix', 'fst', 'fwhen', 'gensym',
+ 'get-time', 'hash', 'hd', 'hdstr', 'hdv', 'head', 'identical',
+ 'implementation', 'in', 'include', 'include-all-but', 'inferences',
+ 'input', 'input+', 'integer?', 'intern', 'intersection', 'is', 'kill',
+ 'language', 'length', 'limit', 'lineread', 'loaded', 'macro', 'macroexpand',
+ 'map', 'mapcan', 'maxinferences', 'mode', 'n->string', 'nl', 'nth', 'null',
+ 'number?', 'occurrences', 'occurs-check', 'open', 'os', 'out', 'port',
+ 'porters', 'pos', 'pr', 'preclude', 'preclude-all-but', 'print', 'profile',
+ 'profile-results', 'ps', 'quit', 'read', 'read+', 'read-byte', 'read-file',
+ 'read-file-as-bytelist', 'read-file-as-string', 'read-from-string',
+ 'release', 'remove', 'return', 'reverse', 'run', 'save', 'set',
+ 'simple-error', 'snd', 'specialise', 'spy', 'step', 'stinput', 'stoutput',
+ 'str', 'string->n', 'string->symbol', 'string?', 'subst', 'symbol?',
+ 'systemf', 'tail', 'tc', 'tc?', 'thaw', 'tl', 'tlstr', 'tlv', 'track',
+ 'tuple?', 'undefmacro', 'unify', 'unify!', 'union', 'unprofile',
+ 'unspecialise', 'untrack', 'variable?', 'vector', 'vector->', 'vector?',
+ 'verified', 'version', 'warn', 'when', 'write-byte', 'write-to-file',
+ 'y-or-n?',
+ )
+
+ BUILTINS_ANYWHERE = ('where', 'skip', '>>', '_', '!', '<e>', '<!>')
+
+ MAPPINGS = dict((s, Keyword) for s in DECLARATIONS)
+ MAPPINGS.update((s, Name.Builtin) for s in BUILTINS)
+ MAPPINGS.update((s, Keyword) for s in SPECIAL_FORMS)
+
+ valid_symbol_chars = r'[\w!$%*+,<=>?/.\'@&#:-]'
+ valid_name = '%s+' % valid_symbol_chars
+ symbol_name = r'[a-z!$%%*+,<=>?/.\'@&#_-]%s*' % valid_symbol_chars
+ variable = r'[A-Z]%s*' % valid_symbol_chars
+
+ tokens = {
+ 'string': [
+ (r'"', String, '#pop'),
+ (r'c#\d{1,3};', String.Escape),
+ (r'~[ARS%]', String.Interpol),
+ (r'(?s).', String),
+ ],
+
+ 'root': [
+ (r'(?s)\\\*.*?\*\\', Comment.Multiline), # \* ... *\
+ (r'\\\\.*', Comment.Single), # \\ ...
+ (r'\s+', Text),
+ (r'_{5,}', Punctuation),
+ (r'={5,}', Punctuation),
+ (r'(;|:=|\||--?>|<--?)', Punctuation),
+ (r'(:-|:|\{|\})', Literal),
+ (r'[+-]*\d*\.\d+(e[+-]?\d+)?', Number.Float),
+ (r'[+-]*\d+', Number.Integer),
+ (r'"', String, 'string'),
+ (variable, Name.Variable),
+ (r'(true|false|<>|\[\])', Keyword.Pseudo),
+ (symbol_name, Literal),
+ (r'(\[|\]|\(|\))', Punctuation),
+ ],
+ }
+
+ def get_tokens_unprocessed(self, text):
+ tokens = RegexLexer.get_tokens_unprocessed(self, text)
+ tokens = self._process_symbols(tokens)
+ tokens = self._process_declarations(tokens)
+ return tokens
+
+ def _relevant(self, token):
+ return token not in (Text, Comment.Single, Comment.Multiline)
+
+ def _process_declarations(self, tokens):
+ opening_paren = False
+ for index, token, value in tokens:
+ yield index, token, value
+ if self._relevant(token):
+ if opening_paren and token == Keyword and value in self.DECLARATIONS:
+ declaration = value
+ for index, token, value in \
+ self._process_declaration(declaration, tokens):
+ yield index, token, value
+ opening_paren = value == '(' and token == Punctuation
+
+ def _process_symbols(self, tokens):
+ opening_paren = False
+ for index, token, value in tokens:
+ if opening_paren and token in (Literal, Name.Variable):
+ token = self.MAPPINGS.get(value, Name.Function)
+ elif token == Literal and value in self.BUILTINS_ANYWHERE:
+ token = Name.Builtin
+ opening_paren = value == '(' and token == Punctuation
+ yield index, token, value
+
+ def _process_declaration(self, declaration, tokens):
+ for index, token, value in tokens:
+ if self._relevant(token):
+ break
+ yield index, token, value
+
+ if declaration == 'datatype':
+ prev_was_colon = False
+ token = Keyword.Type if token == Literal else token
+ yield index, token, value
+ for index, token, value in tokens:
+ if prev_was_colon and token == Literal:
+ token = Keyword.Type
+ yield index, token, value
+ if self._relevant(token):
+ prev_was_colon = token == Literal and value == ':'
+ elif declaration == 'package':
+ token = Name.Namespace if token == Literal else token
+ yield index, token, value
+ elif declaration == 'define':
+ token = Name.Function if token == Literal else token
+ yield index, token, value
+ for index, token, value in tokens:
+ if self._relevant(token):
+ break
+ yield index, token, value
+ if value == '{' and token == Literal:
+ yield index, Punctuation, value
+ for index, token, value in self._process_signature(tokens):
+ yield index, token, value
+ else:
+ yield index, token, value
+ else:
+ token = Name.Function if token == Literal else token
+ yield index, token, value
+
+ raise StopIteration
+
+ def _process_signature(self, tokens):
+ for index, token, value in tokens:
+ if token == Literal and value == '}':
+ yield index, Punctuation, value
+ raise StopIteration
+ elif token in (Literal, Name.Function):
+ token = Name.Variable if value.istitle() else Keyword.Type
+ yield index, token, value
+
+
+class CPSALexer(SchemeLexer):
+ """
+ A CPSA lexer based on the CPSA language as of version 2.2.12
+
+ .. versionadded:: 2.1
+ """
+ name = 'CPSA'
+ aliases = ['cpsa']
+ filenames = ['*.cpsa']
+ mimetypes = []
+
+ # list of known keywords and builtins taken form vim 6.4 scheme.vim
+ # syntax file.
+ _keywords = (
+ 'herald', 'vars', 'defmacro', 'include', 'defprotocol', 'defrole',
+ 'defskeleton', 'defstrand', 'deflistener', 'non-orig', 'uniq-orig',
+ 'pen-non-orig', 'precedes', 'trace', 'send', 'recv', 'name', 'text',
+ 'skey', 'akey', 'data', 'mesg',
+ )
+ _builtins = (
+ 'cat', 'enc', 'hash', 'privk', 'pubk', 'invk', 'ltk', 'gen', 'exp',
+ )
+
+ # valid names for identifiers
+ # well, names can only not consist fully of numbers
+ # but this should be good enough for now
+ valid_name = r'[\w!$%&*+,/:<=>?@^~|-]+'
+
+ tokens = {
+ 'root': [
+ # the comments - always starting with semicolon
+ # and going to the end of the line
+ (r';.*$', Comment.Single),
+
+ # whitespaces - usually not relevant
+ (r'\s+', Text),
+
+ # numbers
+ (r'-?\d+\.\d+', Number.Float),
+ (r'-?\d+', Number.Integer),
+ # support for uncommon kinds of numbers -
+ # have to figure out what the characters mean
+ # (r'(#e|#i|#b|#o|#d|#x)[\d.]+', Number),
+
+ # strings, symbols and characters
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r"'" + valid_name, String.Symbol),
+ (r"#\\([()/'\"._!§$%& ?=+-]|[a-zA-Z0-9]+)", String.Char),
+
+ # constants
+ (r'(#t|#f)', Name.Constant),
+
+ # special operators
+ (r"('|#|`|,@|,|\.)", Operator),
+
+ # highlight the keywords
+ (words(_keywords, suffix=r'\b'), Keyword),
+
+ # first variable in a quoted string like
+ # '(this is syntactic sugar)
+ (r"(?<='\()" + valid_name, Name.Variable),
+ (r"(?<=#\()" + valid_name, Name.Variable),
+
+ # highlight the builtins
+ (words(_builtins, prefix=r'(?<=\()', suffix=r'\b'), Name.Builtin),
+
+ # the remaining functions
+ (r'(?<=\()' + valid_name, Name.Function),
+ # find the remaining variables
+ (valid_name, Name.Variable),
+
+ # the famous parentheses!
+ (r'(\(|\))', Punctuation),
+ (r'(\[|\])', Punctuation),
+ ],
+ }
+
+
+class XtlangLexer(RegexLexer):
+ """An xtlang lexer for the `Extempore programming environment
+ <http://extempore.moso.com.au>`_.
+
+ This is a mixture of Scheme and xtlang, really. Keyword lists are
+ taken from the Extempore Emacs mode
+ (https://github.com/extemporelang/extempore-emacs-mode)
+
+ .. versionadded:: 2.2
+ """
+ name = 'xtlang'
+ aliases = ['extempore']
+ filenames = ['*.xtm']
+ mimetypes = []
+
+ common_keywords = (
+ 'lambda', 'define', 'if', 'else', 'cond', 'and',
+ 'or', 'let', 'begin', 'set!', 'map', 'for-each',
+ )
+ scheme_keywords = (
+ 'do', 'delay', 'quasiquote', 'unquote', 'unquote-splicing', 'eval',
+ 'case', 'let*', 'letrec', 'quote',
+ )
+ xtlang_bind_keywords = (
+ 'bind-func', 'bind-val', 'bind-lib', 'bind-type', 'bind-alias',
+ 'bind-poly', 'bind-dylib', 'bind-lib-func', 'bind-lib-val',
+ )
+ xtlang_keywords = (
+ 'letz', 'memzone', 'cast', 'convert', 'dotimes', 'doloop',
+ )
+ common_functions = (
+ '*', '+', '-', '/', '<', '<=', '=', '>', '>=', '%', 'abs', 'acos',
+ 'angle', 'append', 'apply', 'asin', 'assoc', 'assq', 'assv',
+ 'atan', 'boolean?', 'caaaar', 'caaadr', 'caaar', 'caadar',
+ 'caaddr', 'caadr', 'caar', 'cadaar', 'cadadr', 'cadar',
+ 'caddar', 'cadddr', 'caddr', 'cadr', 'car', 'cdaaar',
+ 'cdaadr', 'cdaar', 'cdadar', 'cdaddr', 'cdadr', 'cdar',
+ 'cddaar', 'cddadr', 'cddar', 'cdddar', 'cddddr', 'cdddr',
+ 'cddr', 'cdr', 'ceiling', 'cons', 'cos', 'floor', 'length',
+ 'list', 'log', 'max', 'member', 'min', 'modulo', 'not',
+ 'reverse', 'round', 'sin', 'sqrt', 'substring', 'tan',
+ 'println', 'random', 'null?', 'callback', 'now',
+ )
+ scheme_functions = (
+ 'call-with-current-continuation', 'call-with-input-file',
+ 'call-with-output-file', 'call-with-values', 'call/cc',
+ 'char->integer', 'char-alphabetic?', 'char-ci<=?', 'char-ci<?',
+ 'char-ci=?', 'char-ci>=?', 'char-ci>?', 'char-downcase',
+ 'char-lower-case?', 'char-numeric?', 'char-ready?',
+ 'char-upcase', 'char-upper-case?', 'char-whitespace?',
+ 'char<=?', 'char<?', 'char=?', 'char>=?', 'char>?', 'char?',
+ 'close-input-port', 'close-output-port', 'complex?',
+ 'current-input-port', 'current-output-port', 'denominator',
+ 'display', 'dynamic-wind', 'eof-object?', 'eq?', 'equal?',
+ 'eqv?', 'even?', 'exact->inexact', 'exact?', 'exp', 'expt',
+ 'force', 'gcd', 'imag-part', 'inexact->exact', 'inexact?',
+ 'input-port?', 'integer->char', 'integer?',
+ 'interaction-environment', 'lcm', 'list->string',
+ 'list->vector', 'list-ref', 'list-tail', 'list?', 'load',
+ 'magnitude', 'make-polar', 'make-rectangular', 'make-string',
+ 'make-vector', 'memq', 'memv', 'negative?', 'newline',
+ 'null-environment', 'number->string', 'number?',
+ 'numerator', 'odd?', 'open-input-file', 'open-output-file',
+ 'output-port?', 'pair?', 'peek-char', 'port?', 'positive?',
+ 'procedure?', 'quotient', 'rational?', 'rationalize', 'read',
+ 'read-char', 'real-part', 'real?',
+ 'remainder', 'scheme-report-environment', 'set-car!', 'set-cdr!',
+ 'string', 'string->list', 'string->number', 'string->symbol',
+ 'string-append', 'string-ci<=?', 'string-ci<?', 'string-ci=?',
+ 'string-ci>=?', 'string-ci>?', 'string-copy', 'string-fill!',
+ 'string-length', 'string-ref', 'string-set!', 'string<=?',
+ 'string<?', 'string=?', 'string>=?', 'string>?', 'string?',
+ 'symbol->string', 'symbol?', 'transcript-off', 'transcript-on',
+ 'truncate', 'values', 'vector', 'vector->list', 'vector-fill!',
+ 'vector-length', 'vector?',
+ 'with-input-from-file', 'with-output-to-file', 'write',
+ 'write-char', 'zero?',
+ )
+ xtlang_functions = (
+ 'toString', 'afill!', 'pfill!', 'tfill!', 'tbind', 'vfill!',
+ 'array-fill!', 'pointer-fill!', 'tuple-fill!', 'vector-fill!', 'free',
+ 'array', 'tuple', 'list', '~', 'cset!', 'cref', '&', 'bor',
+ 'ang-names', '<<', '>>', 'nil', 'printf', 'sprintf', 'null', 'now',
+ 'pset!', 'pref-ptr', 'vset!', 'vref', 'aset!', 'aref', 'aref-ptr',
+ 'tset!', 'tref', 'tref-ptr', 'salloc', 'halloc', 'zalloc', 'alloc',
+ 'schedule', 'exp', 'log', 'sin', 'cos', 'tan', 'asin', 'acos', 'atan',
+ 'sqrt', 'expt', 'floor', 'ceiling', 'truncate', 'round',
+ 'llvm_printf', 'push_zone', 'pop_zone', 'memzone', 'callback',
+ 'llvm_sprintf', 'make-array', 'array-set!', 'array-ref',
+ 'array-ref-ptr', 'pointer-set!', 'pointer-ref', 'pointer-ref-ptr',
+ 'stack-alloc', 'heap-alloc', 'zone-alloc', 'make-tuple', 'tuple-set!',
+ 'tuple-ref', 'tuple-ref-ptr', 'closure-set!', 'closure-ref', 'pref',
+ 'pdref', 'impc_null', 'bitcast', 'void', 'ifret', 'ret->', 'clrun->',
+ 'make-env-zone', 'make-env', '<>', 'dtof', 'ftod', 'i1tof',
+ 'i1tod', 'i1toi8', 'i1toi32', 'i1toi64', 'i8tof', 'i8tod',
+ 'i8toi1', 'i8toi32', 'i8toi64', 'i32tof', 'i32tod', 'i32toi1',
+ 'i32toi8', 'i32toi64', 'i64tof', 'i64tod', 'i64toi1',
+ 'i64toi8', 'i64toi32',
+ )
+
+ # valid names for Scheme identifiers (names cannot consist fully
+ # of numbers, but this should be good enough for now)
+ valid_scheme_name = r'[\w!$%&*+,/:<=>?@^~|-]+'
+
+ # valid characters in xtlang names & types
+ valid_xtlang_name = r'[\w.!-]+'
+ valid_xtlang_type = r'[]{}[\w<>,*/|!-]+'
+
+ tokens = {
+ # keep track of when we're exiting the xtlang form
+ 'xtlang': [
+ (r'\(', Punctuation, '#push'),
+ (r'\)', Punctuation, '#pop'),
+
+ (r'(?<=bind-func\s)' + valid_xtlang_name, Name.Function),
+ (r'(?<=bind-val\s)' + valid_xtlang_name, Name.Function),
+ (r'(?<=bind-type\s)' + valid_xtlang_name, Name.Function),
+ (r'(?<=bind-alias\s)' + valid_xtlang_name, Name.Function),
+ (r'(?<=bind-poly\s)' + valid_xtlang_name, Name.Function),
+ (r'(?<=bind-lib\s)' + valid_xtlang_name, Name.Function),
+ (r'(?<=bind-dylib\s)' + valid_xtlang_name, Name.Function),
+ (r'(?<=bind-lib-func\s)' + valid_xtlang_name, Name.Function),
+ (r'(?<=bind-lib-val\s)' + valid_xtlang_name, Name.Function),
+
+ # type annotations
+ (r':' + valid_xtlang_type, Keyword.Type),
+
+ # types
+ (r'(<' + valid_xtlang_type + r'>|\|' + valid_xtlang_type + r'\||/' +
+ valid_xtlang_type + r'/|' + valid_xtlang_type + r'\*)\**',
+ Keyword.Type),
+
+ # keywords
+ (words(xtlang_keywords, prefix=r'(?<=\()'), Keyword),
+
+ # builtins
+ (words(xtlang_functions, prefix=r'(?<=\()'), Name.Function),
+
+ include('common'),
+
+ # variables
+ (valid_xtlang_name, Name.Variable),
+ ],
+ 'scheme': [
+ # quoted symbols
+ (r"'" + valid_scheme_name, String.Symbol),
+
+ # char literals
+ (r"#\\([()/'\"._!§$%& ?=+-]|[a-zA-Z0-9]+)", String.Char),
+
+ # special operators
+ (r"('|#|`|,@|,|\.)", Operator),
+
+ # keywords
+ (words(scheme_keywords, prefix=r'(?<=\()'), Keyword),
+
+ # builtins
+ (words(scheme_functions, prefix=r'(?<=\()'), Name.Function),
+
+ include('common'),
+
+ # variables
+ (valid_scheme_name, Name.Variable),
+ ],
+ # common to both xtlang and Scheme
+ 'common': [
+ # comments
+ (r';.*$', Comment.Single),
+
+ # whitespaces - usually not relevant
+ (r'\s+', Text),
+
+ # numbers
+ (r'-?\d+\.\d+', Number.Float),
+ (r'-?\d+', Number.Integer),
+
+ # binary/oct/hex literals
+ (r'(#b|#o|#x)[\d.]+', Number),
+
+ # strings
+ (r'"(\\\\|\\"|[^"])*"', String),
+
+ # true/false constants
+ (r'(#t|#f)', Name.Constant),
+
+ # keywords
+ (words(common_keywords, prefix=r'(?<=\()'), Keyword),
+
+ # builtins
+ (words(common_functions, prefix=r'(?<=\()'), Name.Function),
+
+ # the famous parentheses!
+ (r'(\(|\))', Punctuation),
+ ],
+ 'root': [
+ # go into xtlang mode
+ (words(xtlang_bind_keywords, prefix=r'(?<=\()', suffix=r'\b'),
+ Keyword, 'xtlang'),
+
+ include('scheme')
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/make.py b/wandb/vendor/pygments/lexers/make.py
new file mode 100644
index 0000000000000000000000000000000000000000..b222b67292777163db321d769f131473a903267e
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/make.py
@@ -0,0 +1,202 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.make
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Makefiles and similar.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, include, bygroups, \
+ do_insertions, using
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Punctuation
+from pygments.lexers.shell import BashLexer
+
+__all__ = ['MakefileLexer', 'BaseMakefileLexer', 'CMakeLexer']
+
+
+class MakefileLexer(Lexer):
+ """
+ Lexer for BSD and GNU make extensions (lenient enough to handle both in
+ the same file even).
+
+ *Rewritten in Pygments 0.10.*
+ """
+
+ name = 'Makefile'
+ aliases = ['make', 'makefile', 'mf', 'bsdmake']
+ filenames = ['*.mak', '*.mk', 'Makefile', 'makefile', 'Makefile.*', 'GNUmakefile']
+ mimetypes = ['text/x-makefile']
+
+ r_special = re.compile(
+ r'^(?:'
+ # BSD Make
+ r'\.\s*(include|undef|error|warning|if|else|elif|endif|for|endfor)|'
+ # GNU Make
+ r'\s*(ifeq|ifneq|ifdef|ifndef|else|endif|-?include|define|endef|:|vpath)|'
+ # GNU Automake
+ r'\s*(if|else|endif))(?=\s)')
+ r_comment = re.compile(r'^\s*@?#')
+
+ def get_tokens_unprocessed(self, text):
+ ins = []
+ lines = text.splitlines(True)
+ done = ''
+ lex = BaseMakefileLexer(**self.options)
+ backslashflag = False
+ for line in lines:
+ if self.r_special.match(line) or backslashflag:
+ ins.append((len(done), [(0, Comment.Preproc, line)]))
+ backslashflag = line.strip().endswith('\\')
+ elif self.r_comment.match(line):
+ ins.append((len(done), [(0, Comment, line)]))
+ else:
+ done += line
+ for item in do_insertions(ins, lex.get_tokens_unprocessed(done)):
+ yield item
+
+ def analyse_text(text):
+ # Many makefiles have $(BIG_CAPS) style variables
+ if re.search(r'\$\([A-Z_]+\)', text):
+ return 0.1
+
+
+class BaseMakefileLexer(RegexLexer):
+ """
+ Lexer for simple Makefiles (no preprocessing).
+
+ .. versionadded:: 0.10
+ """
+
+ name = 'Base Makefile'
+ aliases = ['basemake']
+ filenames = []
+ mimetypes = []
+
+ tokens = {
+ 'root': [
+ # recipes (need to allow spaces because of expandtabs)
+ (r'^(?:[\t ]+.*\n|\n)+', using(BashLexer)),
+ # special variables
+ (r'\$[<@$+%?|*]', Keyword),
+ (r'\s+', Text),
+ (r'#.*?\n', Comment),
+ (r'(export)(\s+)(?=[\w${}\t -]+\n)',
+ bygroups(Keyword, Text), 'export'),
+ (r'export\s+', Keyword),
+ # assignment
+ (r'([\w${}().-]+)(\s*)([!?:+]?=)([ \t]*)((?:.*\\\n)+|.*\n)',
+ bygroups(Name.Variable, Text, Operator, Text, using(BashLexer))),
+ # strings
+ (r'(?s)"(\\\\|\\.|[^"\\])*"', String.Double),
+ (r"(?s)'(\\\\|\\.|[^'\\])*'", String.Single),
+ # targets
+ (r'([^\n:]+)(:+)([ \t]*)', bygroups(Name.Function, Operator, Text),
+ 'block-header'),
+ # expansions
+ (r'\$\(', Keyword, 'expansion'),
+ ],
+ 'expansion': [
+ (r'[^$a-zA-Z_()]+', Text),
+ (r'[a-zA-Z_]+', Name.Variable),
+ (r'\$', Keyword),
+ (r'\(', Keyword, '#push'),
+ (r'\)', Keyword, '#pop'),
+ ],
+ 'export': [
+ (r'[\w${}-]+', Name.Variable),
+ (r'\n', Text, '#pop'),
+ (r'\s+', Text),
+ ],
+ 'block-header': [
+ (r'[,|]', Punctuation),
+ (r'#.*?\n', Comment, '#pop'),
+ (r'\\\n', Text), # line continuation
+ (r'\$\(', Keyword, 'expansion'),
+ (r'[a-zA-Z_]+', Name),
+ (r'\n', Text, '#pop'),
+ (r'.', Text),
+ ],
+ }
+
+
+class CMakeLexer(RegexLexer):
+ """
+ Lexer for `CMake <http://cmake.org/Wiki/CMake>`_ files.
+
+ .. versionadded:: 1.2
+ """
+ name = 'CMake'
+ aliases = ['cmake']
+ filenames = ['*.cmake', 'CMakeLists.txt']
+ mimetypes = ['text/x-cmake']
+
+ tokens = {
+ 'root': [
+ # (r'(ADD_CUSTOM_COMMAND|ADD_CUSTOM_TARGET|ADD_DEFINITIONS|'
+ # r'ADD_DEPENDENCIES|ADD_EXECUTABLE|ADD_LIBRARY|ADD_SUBDIRECTORY|'
+ # r'ADD_TEST|AUX_SOURCE_DIRECTORY|BUILD_COMMAND|BUILD_NAME|'
+ # r'CMAKE_MINIMUM_REQUIRED|CONFIGURE_FILE|CREATE_TEST_SOURCELIST|'
+ # r'ELSE|ELSEIF|ENABLE_LANGUAGE|ENABLE_TESTING|ENDFOREACH|'
+ # r'ENDFUNCTION|ENDIF|ENDMACRO|ENDWHILE|EXEC_PROGRAM|'
+ # r'EXECUTE_PROCESS|EXPORT_LIBRARY_DEPENDENCIES|FILE|FIND_FILE|'
+ # r'FIND_LIBRARY|FIND_PACKAGE|FIND_PATH|FIND_PROGRAM|FLTK_WRAP_UI|'
+ # r'FOREACH|FUNCTION|GET_CMAKE_PROPERTY|GET_DIRECTORY_PROPERTY|'
+ # r'GET_FILENAME_COMPONENT|GET_SOURCE_FILE_PROPERTY|'
+ # r'GET_TARGET_PROPERTY|GET_TEST_PROPERTY|IF|INCLUDE|'
+ # r'INCLUDE_DIRECTORIES|INCLUDE_EXTERNAL_MSPROJECT|'
+ # r'INCLUDE_REGULAR_EXPRESSION|INSTALL|INSTALL_FILES|'
+ # r'INSTALL_PROGRAMS|INSTALL_TARGETS|LINK_DIRECTORIES|'
+ # r'LINK_LIBRARIES|LIST|LOAD_CACHE|LOAD_COMMAND|MACRO|'
+ # r'MAKE_DIRECTORY|MARK_AS_ADVANCED|MATH|MESSAGE|OPTION|'
+ # r'OUTPUT_REQUIRED_FILES|PROJECT|QT_WRAP_CPP|QT_WRAP_UI|REMOVE|'
+ # r'REMOVE_DEFINITIONS|SEPARATE_ARGUMENTS|SET|'
+ # r'SET_DIRECTORY_PROPERTIES|SET_SOURCE_FILES_PROPERTIES|'
+ # r'SET_TARGET_PROPERTIES|SET_TESTS_PROPERTIES|SITE_NAME|'
+ # r'SOURCE_GROUP|STRING|SUBDIR_DEPENDS|SUBDIRS|'
+ # r'TARGET_LINK_LIBRARIES|TRY_COMPILE|TRY_RUN|UNSET|'
+ # r'USE_MANGLED_MESA|UTILITY_SOURCE|VARIABLE_REQUIRES|'
+ # r'VTK_MAKE_INSTANTIATOR|VTK_WRAP_JAVA|VTK_WRAP_PYTHON|'
+ # r'VTK_WRAP_TCL|WHILE|WRITE_FILE|'
+ # r'COUNTARGS)\b', Name.Builtin, 'args'),
+ (r'\b(\w+)([ \t]*)(\()', bygroups(Name.Builtin, Text,
+ Punctuation), 'args'),
+ include('keywords'),
+ include('ws')
+ ],
+ 'args': [
+ (r'\(', Punctuation, '#push'),
+ (r'\)', Punctuation, '#pop'),
+ (r'(\$\{)(.+?)(\})', bygroups(Operator, Name.Variable, Operator)),
+ (r'(\$ENV\{)(.+?)(\})', bygroups(Operator, Name.Variable, Operator)),
+ (r'(\$<)(.+?)(>)', bygroups(Operator, Name.Variable, Operator)),
+ (r'(?s)".*?"', String.Double),
+ (r'\\\S+', String),
+ (r'[^)$"# \t\n]+', String),
+ (r'\n', Text), # explicitly legal
+ include('keywords'),
+ include('ws')
+ ],
+ 'string': [
+
+ ],
+ 'keywords': [
+ (r'\b(WIN32|UNIX|APPLE|CYGWIN|BORLAND|MINGW|MSVC|MSVC_IDE|MSVC60|'
+ r'MSVC70|MSVC71|MSVC80|MSVC90)\b', Keyword),
+ ],
+ 'ws': [
+ (r'[ \t]+', Text),
+ (r'#.*\n', Comment),
+ ]
+ }
+
+ def analyse_text(text):
+ exp = r'^ *CMAKE_MINIMUM_REQUIRED *\( *VERSION *\d(\.\d)* *( FATAL_ERROR)? *\) *$'
+ if re.search(exp, text, flags=re.MULTILINE | re.IGNORECASE):
+ return 0.8
+ return 0.0
diff --git a/wandb/vendor/pygments/lexers/markup.py b/wandb/vendor/pygments/lexers/markup.py
new file mode 100644
index 0000000000000000000000000000000000000000..92dc9e7a6185d7f7cb2f5bcc1e0fcea9242f3a32
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/markup.py
@@ -0,0 +1,595 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.markup
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for non-HTML markup languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexers.html import HtmlLexer, XmlLexer
+from pygments.lexers.javascript import JavascriptLexer
+from pygments.lexers.css import CssLexer
+
+from pygments.lexer import RegexLexer, DelegatingLexer, include, bygroups, \
+ using, this, do_insertions, default, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Generic, Other
+from pygments.util import get_bool_opt, ClassNotFound
+
+__all__ = ['BBCodeLexer', 'MoinWikiLexer', 'RstLexer', 'TexLexer', 'GroffLexer',
+ 'MozPreprocHashLexer', 'MozPreprocPercentLexer',
+ 'MozPreprocXulLexer', 'MozPreprocJavascriptLexer',
+ 'MozPreprocCssLexer', 'MarkdownLexer']
+
+
+class BBCodeLexer(RegexLexer):
+ """
+ A lexer that highlights BBCode(-like) syntax.
+
+ .. versionadded:: 0.6
+ """
+
+ name = 'BBCode'
+ aliases = ['bbcode']
+ mimetypes = ['text/x-bbcode']
+
+ tokens = {
+ 'root': [
+ (r'[^[]+', Text),
+ # tag/end tag begin
+ (r'\[/?\w+', Keyword, 'tag'),
+ # stray bracket
+ (r'\[', Text),
+ ],
+ 'tag': [
+ (r'\s+', Text),
+ # attribute with value
+ (r'(\w+)(=)("?[^\s"\]]+"?)',
+ bygroups(Name.Attribute, Operator, String)),
+ # tag argument (a la [color=green])
+ (r'(=)("?[^\s"\]]+"?)',
+ bygroups(Operator, String)),
+ # tag end
+ (r'\]', Keyword, '#pop'),
+ ],
+ }
+
+
+class MoinWikiLexer(RegexLexer):
+ """
+ For MoinMoin (and Trac) Wiki markup.
+
+ .. versionadded:: 0.7
+ """
+
+ name = 'MoinMoin/Trac Wiki markup'
+ aliases = ['trac-wiki', 'moin']
+ filenames = []
+ mimetypes = ['text/x-trac-wiki']
+ flags = re.MULTILINE | re.IGNORECASE
+
+ tokens = {
+ 'root': [
+ (r'^#.*$', Comment),
+ (r'(!)(\S+)', bygroups(Keyword, Text)), # Ignore-next
+ # Titles
+ (r'^(=+)([^=]+)(=+)(\s*#.+)?$',
+ bygroups(Generic.Heading, using(this), Generic.Heading, String)),
+ # Literal code blocks, with optional shebang
+ (r'(\{\{\{)(\n#!.+)?', bygroups(Name.Builtin, Name.Namespace), 'codeblock'),
+ (r'(\'\'\'?|\|\||`|__|~~|\^|,,|::)', Comment), # Formatting
+ # Lists
+ (r'^( +)([.*-])( )', bygroups(Text, Name.Builtin, Text)),
+ (r'^( +)([a-z]{1,5}\.)( )', bygroups(Text, Name.Builtin, Text)),
+ # Other Formatting
+ (r'\[\[\w+.*?\]\]', Keyword), # Macro
+ (r'(\[[^\s\]]+)(\s+[^\]]+?)?(\])',
+ bygroups(Keyword, String, Keyword)), # Link
+ (r'^----+$', Keyword), # Horizontal rules
+ (r'[^\n\'\[{!_~^,|]+', Text),
+ (r'\n', Text),
+ (r'.', Text),
+ ],
+ 'codeblock': [
+ (r'\}\}\}', Name.Builtin, '#pop'),
+ # these blocks are allowed to be nested in Trac, but not MoinMoin
+ (r'\{\{\{', Text, '#push'),
+ (r'[^{}]+', Comment.Preproc), # slurp boring text
+ (r'.', Comment.Preproc), # allow loose { or }
+ ],
+ }
+
+
+class RstLexer(RegexLexer):
+ """
+ For `reStructuredText <http://docutils.sf.net/rst.html>`_ markup.
+
+ .. versionadded:: 0.7
+
+ Additional options accepted:
+
+ `handlecodeblocks`
+ Highlight the contents of ``.. sourcecode:: language``,
+ ``.. code:: language`` and ``.. code-block:: language``
+ directives with a lexer for the given language (default:
+ ``True``).
+
+ .. versionadded:: 0.8
+ """
+ name = 'reStructuredText'
+ aliases = ['rst', 'rest', 'restructuredtext']
+ filenames = ['*.rst', '*.rest']
+ mimetypes = ["text/x-rst", "text/prs.fallenstein.rst"]
+ flags = re.MULTILINE
+
+ def _handle_sourcecode(self, match):
+ from pygments.lexers import get_lexer_by_name
+
+ # section header
+ yield match.start(1), Punctuation, match.group(1)
+ yield match.start(2), Text, match.group(2)
+ yield match.start(3), Operator.Word, match.group(3)
+ yield match.start(4), Punctuation, match.group(4)
+ yield match.start(5), Text, match.group(5)
+ yield match.start(6), Keyword, match.group(6)
+ yield match.start(7), Text, match.group(7)
+
+ # lookup lexer if wanted and existing
+ lexer = None
+ if self.handlecodeblocks:
+ try:
+ lexer = get_lexer_by_name(match.group(6).strip())
+ except ClassNotFound:
+ pass
+ indention = match.group(8)
+ indention_size = len(indention)
+ code = (indention + match.group(9) + match.group(10) + match.group(11))
+
+ # no lexer for this language. handle it like it was a code block
+ if lexer is None:
+ yield match.start(8), String, code
+ return
+
+ # highlight the lines with the lexer.
+ ins = []
+ codelines = code.splitlines(True)
+ code = ''
+ for line in codelines:
+ if len(line) > indention_size:
+ ins.append((len(code), [(0, Text, line[:indention_size])]))
+ code += line[indention_size:]
+ else:
+ code += line
+ for item in do_insertions(ins, lexer.get_tokens_unprocessed(code)):
+ yield item
+
+ # from docutils.parsers.rst.states
+ closers = u'\'")]}>\u2019\u201d\xbb!?'
+ unicode_delimiters = u'\u2010\u2011\u2012\u2013\u2014\u00a0'
+ end_string_suffix = (r'((?=$)|(?=[-/:.,; \n\x00%s%s]))'
+ % (re.escape(unicode_delimiters),
+ re.escape(closers)))
+
+ tokens = {
+ 'root': [
+ # Heading with overline
+ (r'^(=+|-+|`+|:+|\.+|\'+|"+|~+|\^+|_+|\*+|\++|#+)([ \t]*\n)'
+ r'(.+)(\n)(\1)(\n)',
+ bygroups(Generic.Heading, Text, Generic.Heading,
+ Text, Generic.Heading, Text)),
+ # Plain heading
+ (r'^(\S.*)(\n)(={3,}|-{3,}|`{3,}|:{3,}|\.{3,}|\'{3,}|"{3,}|'
+ r'~{3,}|\^{3,}|_{3,}|\*{3,}|\+{3,}|#{3,})(\n)',
+ bygroups(Generic.Heading, Text, Generic.Heading, Text)),
+ # Bulleted lists
+ (r'^(\s*)([-*+])( .+\n(?:\1 .+\n)*)',
+ bygroups(Text, Number, using(this, state='inline'))),
+ # Numbered lists
+ (r'^(\s*)([0-9#ivxlcmIVXLCM]+\.)( .+\n(?:\1 .+\n)*)',
+ bygroups(Text, Number, using(this, state='inline'))),
+ (r'^(\s*)(\(?[0-9#ivxlcmIVXLCM]+\))( .+\n(?:\1 .+\n)*)',
+ bygroups(Text, Number, using(this, state='inline'))),
+ # Numbered, but keep words at BOL from becoming lists
+ (r'^(\s*)([A-Z]+\.)( .+\n(?:\1 .+\n)+)',
+ bygroups(Text, Number, using(this, state='inline'))),
+ (r'^(\s*)(\(?[A-Za-z]+\))( .+\n(?:\1 .+\n)+)',
+ bygroups(Text, Number, using(this, state='inline'))),
+ # Line blocks
+ (r'^(\s*)(\|)( .+\n(?:\| .+\n)*)',
+ bygroups(Text, Operator, using(this, state='inline'))),
+ # Sourcecode directives
+ (r'^( *\.\.)(\s*)((?:source)?code(?:-block)?)(::)([ \t]*)([^\n]+)'
+ r'(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\8.*|)\n)+)',
+ _handle_sourcecode),
+ # A directive
+ (r'^( *\.\.)(\s*)([\w:-]+?)(::)(?:([ \t]*)(.*))',
+ bygroups(Punctuation, Text, Operator.Word, Punctuation, Text,
+ using(this, state='inline'))),
+ # A reference target
+ (r'^( *\.\.)(\s*)(_(?:[^:\\]|\\.)+:)(.*?)$',
+ bygroups(Punctuation, Text, Name.Tag, using(this, state='inline'))),
+ # A footnote/citation target
+ (r'^( *\.\.)(\s*)(\[.+\])(.*?)$',
+ bygroups(Punctuation, Text, Name.Tag, using(this, state='inline'))),
+ # A substitution def
+ (r'^( *\.\.)(\s*)(\|.+\|)(\s*)([\w:-]+?)(::)(?:([ \t]*)(.*))',
+ bygroups(Punctuation, Text, Name.Tag, Text, Operator.Word,
+ Punctuation, Text, using(this, state='inline'))),
+ # Comments
+ (r'^ *\.\..*(\n( +.*\n|\n)+)?', Comment.Preproc),
+ # Field list
+ (r'^( *)(:[a-zA-Z-]+:)(\s*)$', bygroups(Text, Name.Class, Text)),
+ (r'^( *)(:.*?:)([ \t]+)(.*?)$',
+ bygroups(Text, Name.Class, Text, Name.Function)),
+ # Definition list
+ (r'^(\S.*(?<!::)\n)((?:(?: +.*)\n)+)',
+ bygroups(using(this, state='inline'), using(this, state='inline'))),
+ # Code blocks
+ (r'(::)(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\3.*|)\n)+)',
+ bygroups(String.Escape, Text, String, String, Text, String)),
+ include('inline'),
+ ],
+ 'inline': [
+ (r'\\.', Text), # escape
+ (r'``', String, 'literal'), # code
+ (r'(`.+?)(<.+?>)(`__?)', # reference with inline target
+ bygroups(String, String.Interpol, String)),
+ (r'`.+?`__?', String), # reference
+ (r'(`.+?`)(:[a-zA-Z0-9:-]+?:)?',
+ bygroups(Name.Variable, Name.Attribute)), # role
+ (r'(:[a-zA-Z0-9:-]+?:)(`.+?`)',
+ bygroups(Name.Attribute, Name.Variable)), # role (content first)
+ (r'\*\*.+?\*\*', Generic.Strong), # Strong emphasis
+ (r'\*.+?\*', Generic.Emph), # Emphasis
+ (r'\[.*?\]_', String), # Footnote or citation
+ (r'<.+?>', Name.Tag), # Hyperlink
+ (r'[^\\\n\[*`:]+', Text),
+ (r'.', Text),
+ ],
+ 'literal': [
+ (r'[^`]+', String),
+ (r'``' + end_string_suffix, String, '#pop'),
+ (r'`', String),
+ ]
+ }
+
+ def __init__(self, **options):
+ self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True)
+ RegexLexer.__init__(self, **options)
+
+ def analyse_text(text):
+ if text[:2] == '..' and text[2:3] != '.':
+ return 0.3
+ p1 = text.find("\n")
+ p2 = text.find("\n", p1 + 1)
+ if (p2 > -1 and # has two lines
+ p1 * 2 + 1 == p2 and # they are the same length
+ text[p1+1] in '-=' and # the next line both starts and ends with
+ text[p1+1] == text[p2-1]): # ...a sufficiently high header
+ return 0.5
+
+
+class TexLexer(RegexLexer):
+ """
+ Lexer for the TeX and LaTeX typesetting languages.
+ """
+
+ name = 'TeX'
+ aliases = ['tex', 'latex']
+ filenames = ['*.tex', '*.aux', '*.toc']
+ mimetypes = ['text/x-tex', 'text/x-latex']
+
+ tokens = {
+ 'general': [
+ (r'%.*?\n', Comment),
+ (r'[{}]', Name.Builtin),
+ (r'[&_^]', Name.Builtin),
+ ],
+ 'root': [
+ (r'\\\[', String.Backtick, 'displaymath'),
+ (r'\\\(', String, 'inlinemath'),
+ (r'\$\$', String.Backtick, 'displaymath'),
+ (r'\$', String, 'inlinemath'),
+ (r'\\([a-zA-Z]+|.)', Keyword, 'command'),
+ (r'\\$', Keyword),
+ include('general'),
+ (r'[^\\$%&_^{}]+', Text),
+ ],
+ 'math': [
+ (r'\\([a-zA-Z]+|.)', Name.Variable),
+ include('general'),
+ (r'[0-9]+', Number),
+ (r'[-=!+*/()\[\]]', Operator),
+ (r'[^=!+*/()\[\]\\$%&_^{}0-9-]+', Name.Builtin),
+ ],
+ 'inlinemath': [
+ (r'\\\)', String, '#pop'),
+ (r'\$', String, '#pop'),
+ include('math'),
+ ],
+ 'displaymath': [
+ (r'\\\]', String, '#pop'),
+ (r'\$\$', String, '#pop'),
+ (r'\$', Name.Builtin),
+ include('math'),
+ ],
+ 'command': [
+ (r'\[.*?\]', Name.Attribute),
+ (r'\*', Keyword),
+ default('#pop'),
+ ],
+ }
+
+ def analyse_text(text):
+ for start in ("\\documentclass", "\\input", "\\documentstyle",
+ "\\relax"):
+ if text[:len(start)] == start:
+ return True
+
+
+class GroffLexer(RegexLexer):
+ """
+ Lexer for the (g)roff typesetting language, supporting groff
+ extensions. Mainly useful for highlighting manpage sources.
+
+ .. versionadded:: 0.6
+ """
+
+ name = 'Groff'
+ aliases = ['groff', 'nroff', 'man']
+ filenames = ['*.[1234567]', '*.man']
+ mimetypes = ['application/x-troff', 'text/troff']
+
+ tokens = {
+ 'root': [
+ (r'(\.)(\w+)', bygroups(Text, Keyword), 'request'),
+ (r'\.', Punctuation, 'request'),
+ # Regular characters, slurp till we find a backslash or newline
+ (r'[^\\\n]+', Text, 'textline'),
+ default('textline'),
+ ],
+ 'textline': [
+ include('escapes'),
+ (r'[^\\\n]+', Text),
+ (r'\n', Text, '#pop'),
+ ],
+ 'escapes': [
+ # groff has many ways to write escapes.
+ (r'\\"[^\n]*', Comment),
+ (r'\\[fn]\w', String.Escape),
+ (r'\\\(.{2}', String.Escape),
+ (r'\\.\[.*\]', String.Escape),
+ (r'\\.', String.Escape),
+ (r'\\\n', Text, 'request'),
+ ],
+ 'request': [
+ (r'\n', Text, '#pop'),
+ include('escapes'),
+ (r'"[^\n"]+"', String.Double),
+ (r'\d+', Number),
+ (r'\S+', String),
+ (r'\s+', Text),
+ ],
+ }
+
+ def analyse_text(text):
+ if text[:1] != '.':
+ return False
+ if text[:3] == '.\\"':
+ return True
+ if text[:4] == '.TH ':
+ return True
+ if text[1:3].isalnum() and text[3].isspace():
+ return 0.9
+
+
+class MozPreprocHashLexer(RegexLexer):
+ """
+ Lexer for Mozilla Preprocessor files (with '#' as the marker).
+
+ Other data is left untouched.
+
+ .. versionadded:: 2.0
+ """
+ name = 'mozhashpreproc'
+ aliases = [name]
+ filenames = []
+ mimetypes = []
+
+ tokens = {
+ 'root': [
+ (r'^#', Comment.Preproc, ('expr', 'exprstart')),
+ (r'.+', Other),
+ ],
+ 'exprstart': [
+ (r'(literal)(.*)', bygroups(Comment.Preproc, Text), '#pop:2'),
+ (words((
+ 'define', 'undef', 'if', 'ifdef', 'ifndef', 'else', 'elif',
+ 'elifdef', 'elifndef', 'endif', 'expand', 'filter', 'unfilter',
+ 'include', 'includesubst', 'error')),
+ Comment.Preproc, '#pop'),
+ ],
+ 'expr': [
+ (words(('!', '!=', '==', '&&', '||')), Operator),
+ (r'(defined)(\()', bygroups(Keyword, Punctuation)),
+ (r'\)', Punctuation),
+ (r'[0-9]+', Number.Decimal),
+ (r'__\w+?__', Name.Variable),
+ (r'@\w+?@', Name.Class),
+ (r'\w+', Name),
+ (r'\n', Text, '#pop'),
+ (r'\s+', Text),
+ (r'\S', Punctuation),
+ ],
+ }
+
+
+class MozPreprocPercentLexer(MozPreprocHashLexer):
+ """
+ Lexer for Mozilla Preprocessor files (with '%' as the marker).
+
+ Other data is left untouched.
+
+ .. versionadded:: 2.0
+ """
+ name = 'mozpercentpreproc'
+ aliases = [name]
+ filenames = []
+ mimetypes = []
+
+ tokens = {
+ 'root': [
+ (r'^%', Comment.Preproc, ('expr', 'exprstart')),
+ (r'.+', Other),
+ ],
+ }
+
+
+class MozPreprocXulLexer(DelegatingLexer):
+ """
+ Subclass of the `MozPreprocHashLexer` that highlights unlexed data with the
+ `XmlLexer`.
+
+ .. versionadded:: 2.0
+ """
+ name = "XUL+mozpreproc"
+ aliases = ['xul+mozpreproc']
+ filenames = ['*.xul.in']
+ mimetypes = []
+
+ def __init__(self, **options):
+ super(MozPreprocXulLexer, self).__init__(
+ XmlLexer, MozPreprocHashLexer, **options)
+
+
+class MozPreprocJavascriptLexer(DelegatingLexer):
+ """
+ Subclass of the `MozPreprocHashLexer` that highlights unlexed data with the
+ `JavascriptLexer`.
+
+ .. versionadded:: 2.0
+ """
+ name = "Javascript+mozpreproc"
+ aliases = ['javascript+mozpreproc']
+ filenames = ['*.js.in']
+ mimetypes = []
+
+ def __init__(self, **options):
+ super(MozPreprocJavascriptLexer, self).__init__(
+ JavascriptLexer, MozPreprocHashLexer, **options)
+
+
+class MozPreprocCssLexer(DelegatingLexer):
+ """
+ Subclass of the `MozPreprocHashLexer` that highlights unlexed data with the
+ `CssLexer`.
+
+ .. versionadded:: 2.0
+ """
+ name = "CSS+mozpreproc"
+ aliases = ['css+mozpreproc']
+ filenames = ['*.css.in']
+ mimetypes = []
+
+ def __init__(self, **options):
+ super(MozPreprocCssLexer, self).__init__(
+ CssLexer, MozPreprocPercentLexer, **options)
+
+
+class MarkdownLexer(RegexLexer):
+ """
+ For `Markdown <https://help.github.com/categories/writing-on-github/>`_ markup.
+
+ .. versionadded:: 2.2
+ """
+ name = 'markdown'
+ aliases = ['md']
+ filenames = ['*.md']
+ mimetypes = ["text/x-markdown"]
+ flags = re.MULTILINE
+
+ def _handle_codeblock(self, match):
+ """
+ match args: 1:backticks, 2:lang_name, 3:newline, 4:code, 5:backticks
+ """
+ from pygments.lexers import get_lexer_by_name
+
+ # section header
+ yield match.start(1), String , match.group(1)
+ yield match.start(2), String , match.group(2)
+ yield match.start(3), Text , match.group(3)
+
+ # lookup lexer if wanted and existing
+ lexer = None
+ if self.handlecodeblocks:
+ try:
+ lexer = get_lexer_by_name( match.group(2).strip() )
+ except ClassNotFound:
+ pass
+ code = match.group(4)
+
+ # no lexer for this language. handle it like it was a code block
+ if lexer is None:
+ yield match.start(4), String, code
+ return
+
+ for item in do_insertions([], lexer.get_tokens_unprocessed(code)):
+ yield item
+
+ yield match.start(5), String , match.group(5)
+
+ tokens = {
+ 'root': [
+ # heading with pound prefix
+ (r'^(#)([^#].+\n)', bygroups(Generic.Heading, Text)),
+ (r'^(#{2,6})(.+\n)', bygroups(Generic.Subheading, Text)),
+ # task list
+ (r'^(\s*)([*-] )(\[[ xX]\])( .+\n)',
+ bygroups(Text, Keyword, Keyword, using(this, state='inline'))),
+ # bulleted lists
+ (r'^(\s*)([*-])(\s)(.+\n)',
+ bygroups(Text, Keyword, Text, using(this, state='inline'))),
+ # numbered lists
+ (r'^(\s*)([0-9]+\.)( .+\n)',
+ bygroups(Text, Keyword, using(this, state='inline'))),
+ # quote
+ (r'^(\s*>\s)(.+\n)', bygroups(Keyword, Generic.Emph)),
+ # text block
+ (r'^(```\n)([\w\W]*?)(^```$)', bygroups(String, Text, String)),
+ # code block with language
+ (r'^(```)(\w+)(\n)([\w\W]*?)(^```$)', _handle_codeblock),
+
+ include('inline'),
+ ],
+ 'inline': [
+ # escape
+ (r'\\.', Text),
+ # italics
+ (r'(\s)([*_][^*_]+[*_])(\W|\n)', bygroups(Text, Generic.Emph, Text)),
+ # bold
+ # warning: the following rule eats internal tags. eg. **foo _bar_ baz** bar is not italics
+ (r'(\s)((\*\*|__).*\3)((?=\W|\n))', bygroups(Text, Generic.Strong, None, Text)),
+ # "proper way" (r'(\s)([*_]{2}[^*_]+[*_]{2})((?=\W|\n))', bygroups(Text, Generic.Strong, Text)),
+ # strikethrough
+ (r'(\s)(~~[^~]+~~)((?=\W|\n))', bygroups(Text, Generic.Deleted, Text)),
+ # inline code
+ (r'`[^`]+`', String.Backtick),
+ # mentions and topics (twitter and github stuff)
+ (r'[@#][\w/:]+', Name.Entity),
+ # (image?) links eg: 
+ (r'(!?\[)([^]]+)(\])(\()([^)]+)(\))', bygroups(Text, Name.Tag, Text, Text, Name.Attribute, Text)),
+
+ # general text, must come last!
+ (r'[^\\\s]+', Text),
+ (r'.', Text),
+ ],
+ }
+
+ def __init__(self, **options):
+ self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True)
+ RegexLexer.__init__(self, **options)
diff --git a/wandb/vendor/pygments/lexers/math.py b/wandb/vendor/pygments/lexers/math.py
new file mode 100644
index 0000000000000000000000000000000000000000..ea0ebee2a777da643726fc943f2ba52d3691760a
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/math.py
@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.math
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Just export lexers that were contained in this module.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexers.python import NumPyLexer
+from pygments.lexers.matlab import MatlabLexer, MatlabSessionLexer, \
+ OctaveLexer, ScilabLexer
+from pygments.lexers.julia import JuliaLexer, JuliaConsoleLexer
+from pygments.lexers.r import RConsoleLexer, SLexer, RdLexer
+from pygments.lexers.modeling import BugsLexer, JagsLexer, StanLexer
+from pygments.lexers.idl import IDLLexer
+from pygments.lexers.algebra import MuPADLexer
+
+__all__ = []
diff --git a/wandb/vendor/pygments/lexers/matlab.py b/wandb/vendor/pygments/lexers/matlab.py
new file mode 100644
index 0000000000000000000000000000000000000000..56a0f6d6d1d6489b42eae1cebd46ba16b2e2b3ef
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/matlab.py
@@ -0,0 +1,663 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.matlab
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Matlab and related languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, bygroups, words, do_insertions
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Generic, Whitespace
+
+from pygments.lexers import _scilab_builtins
+
+__all__ = ['MatlabLexer', 'MatlabSessionLexer', 'OctaveLexer', 'ScilabLexer']
+
+
+class MatlabLexer(RegexLexer):
+ """
+ For Matlab source code.
+
+ .. versionadded:: 0.10
+ """
+ name = 'Matlab'
+ aliases = ['matlab']
+ filenames = ['*.m']
+ mimetypes = ['text/matlab']
+
+ #
+ # These lists are generated automatically.
+ # Run the following in bash shell:
+ #
+ # for f in elfun specfun elmat; do
+ # echo -n "$f = "
+ # matlab -nojvm -r "help $f;exit;" | perl -ne \
+ # 'push(@c,$1) if /^ (\w+)\s+-/; END {print q{["}.join(q{","},@c).qq{"]\n};}'
+ # done
+ #
+ # elfun: Elementary math functions
+ # specfun: Special Math functions
+ # elmat: Elementary matrices and matrix manipulation
+ #
+ # taken from Matlab version 7.4.0.336 (R2007a)
+ #
+ elfun = ("sin", "sind", "sinh", "asin", "asind", "asinh", "cos", "cosd", "cosh",
+ "acos", "acosd", "acosh", "tan", "tand", "tanh", "atan", "atand", "atan2",
+ "atanh", "sec", "secd", "sech", "asec", "asecd", "asech", "csc", "cscd",
+ "csch", "acsc", "acscd", "acsch", "cot", "cotd", "coth", "acot", "acotd",
+ "acoth", "hypot", "exp", "expm1", "log", "log1p", "log10", "log2", "pow2",
+ "realpow", "reallog", "realsqrt", "sqrt", "nthroot", "nextpow2", "abs",
+ "angle", "complex", "conj", "imag", "real", "unwrap", "isreal", "cplxpair",
+ "fix", "floor", "ceil", "round", "mod", "rem", "sign")
+ specfun = ("airy", "besselj", "bessely", "besselh", "besseli", "besselk", "beta",
+ "betainc", "betaln", "ellipj", "ellipke", "erf", "erfc", "erfcx",
+ "erfinv", "expint", "gamma", "gammainc", "gammaln", "psi", "legendre",
+ "cross", "dot", "factor", "isprime", "primes", "gcd", "lcm", "rat",
+ "rats", "perms", "nchoosek", "factorial", "cart2sph", "cart2pol",
+ "pol2cart", "sph2cart", "hsv2rgb", "rgb2hsv")
+ elmat = ("zeros", "ones", "eye", "repmat", "rand", "randn", "linspace", "logspace",
+ "freqspace", "meshgrid", "accumarray", "size", "length", "ndims", "numel",
+ "disp", "isempty", "isequal", "isequalwithequalnans", "cat", "reshape",
+ "diag", "blkdiag", "tril", "triu", "fliplr", "flipud", "flipdim", "rot90",
+ "find", "end", "sub2ind", "ind2sub", "bsxfun", "ndgrid", "permute",
+ "ipermute", "shiftdim", "circshift", "squeeze", "isscalar", "isvector",
+ "ans", "eps", "realmax", "realmin", "pi", "i", "inf", "nan", "isnan",
+ "isinf", "isfinite", "j", "why", "compan", "gallery", "hadamard", "hankel",
+ "hilb", "invhilb", "magic", "pascal", "rosser", "toeplitz", "vander",
+ "wilkinson")
+
+ tokens = {
+ 'root': [
+ # line starting with '!' is sent as a system command. not sure what
+ # label to use...
+ (r'^!.*', String.Other),
+ (r'%\{\s*\n', Comment.Multiline, 'blockcomment'),
+ (r'%.*$', Comment),
+ (r'^\s*function', Keyword, 'deffunc'),
+
+ # from 'iskeyword' on version 7.11 (R2010):
+ (words((
+ 'break', 'case', 'catch', 'classdef', 'continue', 'else', 'elseif',
+ 'end', 'enumerated', 'events', 'for', 'function', 'global', 'if',
+ 'methods', 'otherwise', 'parfor', 'persistent', 'properties',
+ 'return', 'spmd', 'switch', 'try', 'while'), suffix=r'\b'),
+ Keyword),
+
+ ("(" + "|".join(elfun + specfun + elmat) + r')\b', Name.Builtin),
+
+ # line continuation with following comment:
+ (r'\.\.\..*$', Comment),
+
+ # operators:
+ (r'-|==|~=|<|>|<=|>=|&&|&|~|\|\|?', Operator),
+ # operators requiring escape for re:
+ (r'\.\*|\*|\+|\.\^|\.\\|\.\/|\/|\\', Operator),
+
+ # punctuation:
+ (r'\[|\]|\(|\)|\{|\}|:|@|\.|,', Punctuation),
+ (r'=|:|;', Punctuation),
+
+ # quote can be transpose, instead of string:
+ # (not great, but handles common cases...)
+ (r'(?<=[\w)\].])\'+', Operator),
+
+ (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float),
+ (r'\d+[eEf][+-]?[0-9]+', Number.Float),
+ (r'\d+', Number.Integer),
+
+ (r'(?<![\w)\].])\'', String, 'string'),
+ (r'[a-zA-Z_]\w*', Name),
+ (r'.', Text),
+ ],
+ 'string': [
+ (r'[^\']*\'', String, '#pop')
+ ],
+ 'blockcomment': [
+ (r'^\s*%\}', Comment.Multiline, '#pop'),
+ (r'^.*\n', Comment.Multiline),
+ (r'.', Comment.Multiline),
+ ],
+ 'deffunc': [
+ (r'(\s*)(?:(.+)(\s*)(=)(\s*))?(.+)(\()(.*)(\))(\s*)',
+ bygroups(Whitespace, Text, Whitespace, Punctuation,
+ Whitespace, Name.Function, Punctuation, Text,
+ Punctuation, Whitespace), '#pop'),
+ # function with no args
+ (r'(\s*)([a-zA-Z_]\w*)', bygroups(Text, Name.Function), '#pop'),
+ ],
+ }
+
+ def analyse_text(text):
+ if re.match('^\s*%', text, re.M): # comment
+ return 0.2
+ elif re.match('^!\w+', text, re.M): # system cmd
+ return 0.2
+
+
+line_re = re.compile('.*?\n')
+
+
+class MatlabSessionLexer(Lexer):
+ """
+ For Matlab sessions. Modeled after PythonConsoleLexer.
+ Contributed by Ken Schutte <kschutte@csail.mit.edu>.
+
+ .. versionadded:: 0.10
+ """
+ name = 'Matlab session'
+ aliases = ['matlabsession']
+
+ def get_tokens_unprocessed(self, text):
+ mlexer = MatlabLexer(**self.options)
+
+ curcode = ''
+ insertions = []
+
+ for match in line_re.finditer(text):
+ line = match.group()
+
+ if line.startswith('>> '):
+ insertions.append((len(curcode),
+ [(0, Generic.Prompt, line[:3])]))
+ curcode += line[3:]
+
+ elif line.startswith('>>'):
+ insertions.append((len(curcode),
+ [(0, Generic.Prompt, line[:2])]))
+ curcode += line[2:]
+
+ elif line.startswith('???'):
+
+ idx = len(curcode)
+
+ # without is showing error on same line as before...?
+ # line = "\n" + line
+ token = (0, Generic.Traceback, line)
+ insertions.append((idx, [token]))
+
+ else:
+ if curcode:
+ for item in do_insertions(
+ insertions, mlexer.get_tokens_unprocessed(curcode)):
+ yield item
+ curcode = ''
+ insertions = []
+
+ yield match.start(), Generic.Output, line
+
+ if curcode: # or item:
+ for item in do_insertions(
+ insertions, mlexer.get_tokens_unprocessed(curcode)):
+ yield item
+
+
+class OctaveLexer(RegexLexer):
+ """
+ For GNU Octave source code.
+
+ .. versionadded:: 1.5
+ """
+ name = 'Octave'
+ aliases = ['octave']
+ filenames = ['*.m']
+ mimetypes = ['text/octave']
+
+ # These lists are generated automatically.
+ # Run the following in bash shell:
+ #
+ # First dump all of the Octave manual into a plain text file:
+ #
+ # $ info octave --subnodes -o octave-manual
+ #
+ # Now grep through it:
+
+ # for i in \
+ # "Built-in Function" "Command" "Function File" \
+ # "Loadable Function" "Mapping Function";
+ # do
+ # perl -e '@name = qw('"$i"');
+ # print lc($name[0]),"_kw = [\n"';
+ #
+ # perl -n -e 'print "\"$1\",\n" if /-- '"$i"': .* (\w*) \(/;' \
+ # octave-manual | sort | uniq ;
+ # echo "]" ;
+ # echo;
+ # done
+
+ # taken from Octave Mercurial changeset 8cc154f45e37 (30-jan-2011)
+
+ builtin_kw = (
+ "addlistener", "addpath", "addproperty", "all",
+ "and", "any", "argnames", "argv", "assignin",
+ "atexit", "autoload",
+ "available_graphics_toolkits", "beep_on_error",
+ "bitand", "bitmax", "bitor", "bitshift", "bitxor",
+ "cat", "cell", "cellstr", "char", "class", "clc",
+ "columns", "command_line_path",
+ "completion_append_char", "completion_matches",
+ "complex", "confirm_recursive_rmdir", "cputime",
+ "crash_dumps_octave_core", "ctranspose", "cumprod",
+ "cumsum", "debug_on_error", "debug_on_interrupt",
+ "debug_on_warning", "default_save_options",
+ "dellistener", "diag", "diff", "disp",
+ "doc_cache_file", "do_string_escapes", "double",
+ "drawnow", "e", "echo_executing_commands", "eps",
+ "eq", "errno", "errno_list", "error", "eval",
+ "evalin", "exec", "exist", "exit", "eye", "false",
+ "fclear", "fclose", "fcntl", "fdisp", "feof",
+ "ferror", "feval", "fflush", "fgetl", "fgets",
+ "fieldnames", "file_in_loadpath", "file_in_path",
+ "filemarker", "filesep", "find_dir_in_path",
+ "fixed_point_format", "fnmatch", "fopen", "fork",
+ "formula", "fprintf", "fputs", "fread", "freport",
+ "frewind", "fscanf", "fseek", "fskipl", "ftell",
+ "functions", "fwrite", "ge", "genpath", "get",
+ "getegid", "getenv", "geteuid", "getgid",
+ "getpgrp", "getpid", "getppid", "getuid", "glob",
+ "gt", "gui_mode", "history_control",
+ "history_file", "history_size",
+ "history_timestamp_format_string", "home",
+ "horzcat", "hypot", "ifelse",
+ "ignore_function_time_stamp", "inferiorto",
+ "info_file", "info_program", "inline", "input",
+ "intmax", "intmin", "ipermute",
+ "is_absolute_filename", "isargout", "isbool",
+ "iscell", "iscellstr", "ischar", "iscomplex",
+ "isempty", "isfield", "isfloat", "isglobal",
+ "ishandle", "isieee", "isindex", "isinteger",
+ "islogical", "ismatrix", "ismethod", "isnull",
+ "isnumeric", "isobject", "isreal",
+ "is_rooted_relative_filename", "issorted",
+ "isstruct", "isvarname", "kbhit", "keyboard",
+ "kill", "lasterr", "lasterror", "lastwarn",
+ "ldivide", "le", "length", "link", "linspace",
+ "logical", "lstat", "lt", "make_absolute_filename",
+ "makeinfo_program", "max_recursion_depth", "merge",
+ "methods", "mfilename", "minus", "mislocked",
+ "mkdir", "mkfifo", "mkstemp", "mldivide", "mlock",
+ "mouse_wheel_zoom", "mpower", "mrdivide", "mtimes",
+ "munlock", "nargin", "nargout",
+ "native_float_format", "ndims", "ne", "nfields",
+ "nnz", "norm", "not", "numel", "nzmax",
+ "octave_config_info", "octave_core_file_limit",
+ "octave_core_file_name",
+ "octave_core_file_options", "ones", "or",
+ "output_max_field_width", "output_precision",
+ "page_output_immediately", "page_screen_output",
+ "path", "pathsep", "pause", "pclose", "permute",
+ "pi", "pipe", "plus", "popen", "power",
+ "print_empty_dimensions", "printf",
+ "print_struct_array_contents", "prod",
+ "program_invocation_name", "program_name",
+ "putenv", "puts", "pwd", "quit", "rats", "rdivide",
+ "readdir", "readlink", "read_readline_init_file",
+ "realmax", "realmin", "rehash", "rename",
+ "repelems", "re_read_readline_init_file", "reset",
+ "reshape", "resize", "restoredefaultpath",
+ "rethrow", "rmdir", "rmfield", "rmpath", "rows",
+ "save_header_format_string", "save_precision",
+ "saving_history", "scanf", "set", "setenv",
+ "shell_cmd", "sighup_dumps_octave_core",
+ "sigterm_dumps_octave_core", "silent_functions",
+ "single", "size", "size_equal", "sizemax",
+ "sizeof", "sleep", "source", "sparse_auto_mutate",
+ "split_long_rows", "sprintf", "squeeze", "sscanf",
+ "stat", "stderr", "stdin", "stdout", "strcmp",
+ "strcmpi", "string_fill_char", "strncmp",
+ "strncmpi", "struct", "struct_levels_to_print",
+ "strvcat", "subsasgn", "subsref", "sum", "sumsq",
+ "superiorto", "suppress_verbose_help_message",
+ "symlink", "system", "tic", "tilde_expand",
+ "times", "tmpfile", "tmpnam", "toc", "toupper",
+ "transpose", "true", "typeinfo", "umask", "uminus",
+ "uname", "undo_string_escapes", "unlink", "uplus",
+ "upper", "usage", "usleep", "vec", "vectorize",
+ "vertcat", "waitpid", "warning", "warranty",
+ "whos_line_format", "yes_or_no", "zeros",
+ "inf", "Inf", "nan", "NaN")
+
+ command_kw = ("close", "load", "who", "whos")
+
+ function_kw = (
+ "accumarray", "accumdim", "acosd", "acotd",
+ "acscd", "addtodate", "allchild", "ancestor",
+ "anova", "arch_fit", "arch_rnd", "arch_test",
+ "area", "arma_rnd", "arrayfun", "ascii", "asctime",
+ "asecd", "asind", "assert", "atand",
+ "autoreg_matrix", "autumn", "axes", "axis", "bar",
+ "barh", "bartlett", "bartlett_test", "beep",
+ "betacdf", "betainv", "betapdf", "betarnd",
+ "bicgstab", "bicubic", "binary", "binocdf",
+ "binoinv", "binopdf", "binornd", "bitcmp",
+ "bitget", "bitset", "blackman", "blanks",
+ "blkdiag", "bone", "box", "brighten", "calendar",
+ "cast", "cauchy_cdf", "cauchy_inv", "cauchy_pdf",
+ "cauchy_rnd", "caxis", "celldisp", "center", "cgs",
+ "chisquare_test_homogeneity",
+ "chisquare_test_independence", "circshift", "cla",
+ "clabel", "clf", "clock", "cloglog", "closereq",
+ "colon", "colorbar", "colormap", "colperm",
+ "comet", "common_size", "commutation_matrix",
+ "compan", "compare_versions", "compass",
+ "computer", "cond", "condest", "contour",
+ "contourc", "contourf", "contrast", "conv",
+ "convhull", "cool", "copper", "copyfile", "cor",
+ "corrcoef", "cor_test", "cosd", "cotd", "cov",
+ "cplxpair", "cross", "cscd", "cstrcat", "csvread",
+ "csvwrite", "ctime", "cumtrapz", "curl", "cut",
+ "cylinder", "date", "datenum", "datestr",
+ "datetick", "datevec", "dblquad", "deal",
+ "deblank", "deconv", "delaunay", "delaunayn",
+ "delete", "demo", "detrend", "diffpara", "diffuse",
+ "dir", "discrete_cdf", "discrete_inv",
+ "discrete_pdf", "discrete_rnd", "display",
+ "divergence", "dlmwrite", "dos", "dsearch",
+ "dsearchn", "duplication_matrix", "durbinlevinson",
+ "ellipsoid", "empirical_cdf", "empirical_inv",
+ "empirical_pdf", "empirical_rnd", "eomday",
+ "errorbar", "etime", "etreeplot", "example",
+ "expcdf", "expinv", "expm", "exppdf", "exprnd",
+ "ezcontour", "ezcontourf", "ezmesh", "ezmeshc",
+ "ezplot", "ezpolar", "ezsurf", "ezsurfc", "factor",
+ "factorial", "fail", "fcdf", "feather", "fftconv",
+ "fftfilt", "fftshift", "figure", "fileattrib",
+ "fileparts", "fill", "findall", "findobj",
+ "findstr", "finv", "flag", "flipdim", "fliplr",
+ "flipud", "fpdf", "fplot", "fractdiff", "freqz",
+ "freqz_plot", "frnd", "fsolve",
+ "f_test_regression", "ftp", "fullfile", "fzero",
+ "gamcdf", "gaminv", "gampdf", "gamrnd", "gca",
+ "gcbf", "gcbo", "gcf", "genvarname", "geocdf",
+ "geoinv", "geopdf", "geornd", "getfield", "ginput",
+ "glpk", "gls", "gplot", "gradient",
+ "graphics_toolkit", "gray", "grid", "griddata",
+ "griddatan", "gtext", "gunzip", "gzip", "hadamard",
+ "hamming", "hankel", "hanning", "hggroup",
+ "hidden", "hilb", "hist", "histc", "hold", "hot",
+ "hotelling_test", "housh", "hsv", "hurst",
+ "hygecdf", "hygeinv", "hygepdf", "hygernd",
+ "idivide", "ifftshift", "image", "imagesc",
+ "imfinfo", "imread", "imshow", "imwrite", "index",
+ "info", "inpolygon", "inputname", "interpft",
+ "interpn", "intersect", "invhilb", "iqr", "isa",
+ "isdefinite", "isdir", "is_duplicate_entry",
+ "isequal", "isequalwithequalnans", "isfigure",
+ "ishermitian", "ishghandle", "is_leap_year",
+ "isletter", "ismac", "ismember", "ispc", "isprime",
+ "isprop", "isscalar", "issquare", "isstrprop",
+ "issymmetric", "isunix", "is_valid_file_id",
+ "isvector", "jet", "kendall",
+ "kolmogorov_smirnov_cdf",
+ "kolmogorov_smirnov_test", "kruskal_wallis_test",
+ "krylov", "kurtosis", "laplace_cdf", "laplace_inv",
+ "laplace_pdf", "laplace_rnd", "legend", "legendre",
+ "license", "line", "linkprop", "list_primes",
+ "loadaudio", "loadobj", "logistic_cdf",
+ "logistic_inv", "logistic_pdf", "logistic_rnd",
+ "logit", "loglog", "loglogerr", "logm", "logncdf",
+ "logninv", "lognpdf", "lognrnd", "logspace",
+ "lookfor", "ls_command", "lsqnonneg", "magic",
+ "mahalanobis", "manova", "matlabroot",
+ "mcnemar_test", "mean", "meansq", "median", "menu",
+ "mesh", "meshc", "meshgrid", "meshz", "mexext",
+ "mget", "mkpp", "mode", "moment", "movefile",
+ "mpoles", "mput", "namelengthmax", "nargchk",
+ "nargoutchk", "nbincdf", "nbininv", "nbinpdf",
+ "nbinrnd", "nchoosek", "ndgrid", "newplot", "news",
+ "nonzeros", "normcdf", "normest", "norminv",
+ "normpdf", "normrnd", "now", "nthroot", "null",
+ "ocean", "ols", "onenormest", "optimget",
+ "optimset", "orderfields", "orient", "orth",
+ "pack", "pareto", "parseparams", "pascal", "patch",
+ "pathdef", "pcg", "pchip", "pcolor", "pcr",
+ "peaks", "periodogram", "perl", "perms", "pie",
+ "pink", "planerot", "playaudio", "plot",
+ "plotmatrix", "plotyy", "poisscdf", "poissinv",
+ "poisspdf", "poissrnd", "polar", "poly",
+ "polyaffine", "polyarea", "polyderiv", "polyfit",
+ "polygcd", "polyint", "polyout", "polyreduce",
+ "polyval", "polyvalm", "postpad", "powerset",
+ "ppder", "ppint", "ppjumps", "ppplot", "ppval",
+ "pqpnonneg", "prepad", "primes", "print",
+ "print_usage", "prism", "probit", "qp", "qqplot",
+ "quadcc", "quadgk", "quadl", "quadv", "quiver",
+ "qzhess", "rainbow", "randi", "range", "rank",
+ "ranks", "rat", "reallog", "realpow", "realsqrt",
+ "record", "rectangle_lw", "rectangle_sw",
+ "rectint", "refresh", "refreshdata",
+ "regexptranslate", "repmat", "residue", "ribbon",
+ "rindex", "roots", "rose", "rosser", "rotdim",
+ "rref", "run", "run_count", "rundemos", "run_test",
+ "runtests", "saveas", "saveaudio", "saveobj",
+ "savepath", "scatter", "secd", "semilogx",
+ "semilogxerr", "semilogy", "semilogyerr",
+ "setaudio", "setdiff", "setfield", "setxor",
+ "shading", "shift", "shiftdim", "sign_test",
+ "sinc", "sind", "sinetone", "sinewave", "skewness",
+ "slice", "sombrero", "sortrows", "spaugment",
+ "spconvert", "spdiags", "spearman", "spectral_adf",
+ "spectral_xdf", "specular", "speed", "spencer",
+ "speye", "spfun", "sphere", "spinmap", "spline",
+ "spones", "sprand", "sprandn", "sprandsym",
+ "spring", "spstats", "spy", "sqp", "stairs",
+ "statistics", "std", "stdnormal_cdf",
+ "stdnormal_inv", "stdnormal_pdf", "stdnormal_rnd",
+ "stem", "stft", "strcat", "strchr", "strjust",
+ "strmatch", "strread", "strsplit", "strtok",
+ "strtrim", "strtrunc", "structfun", "studentize",
+ "subplot", "subsindex", "subspace", "substr",
+ "substruct", "summer", "surf", "surface", "surfc",
+ "surfl", "surfnorm", "svds", "swapbytes",
+ "sylvester_matrix", "symvar", "synthesis", "table",
+ "tand", "tar", "tcdf", "tempdir", "tempname",
+ "test", "text", "textread", "textscan", "tinv",
+ "title", "toeplitz", "tpdf", "trace", "trapz",
+ "treelayout", "treeplot", "triangle_lw",
+ "triangle_sw", "tril", "trimesh", "triplequad",
+ "triplot", "trisurf", "triu", "trnd", "tsearchn",
+ "t_test", "t_test_regression", "type", "unidcdf",
+ "unidinv", "unidpdf", "unidrnd", "unifcdf",
+ "unifinv", "unifpdf", "unifrnd", "union", "unique",
+ "unix", "unmkpp", "unpack", "untabify", "untar",
+ "unwrap", "unzip", "u_test", "validatestring",
+ "vander", "var", "var_test", "vech", "ver",
+ "version", "view", "voronoi", "voronoin",
+ "waitforbuttonpress", "wavread", "wavwrite",
+ "wblcdf", "wblinv", "wblpdf", "wblrnd", "weekday",
+ "welch_test", "what", "white", "whitebg",
+ "wienrnd", "wilcoxon_test", "wilkinson", "winter",
+ "xlabel", "xlim", "ylabel", "yulewalker", "zip",
+ "zlabel", "z_test")
+
+ loadable_kw = (
+ "airy", "amd", "balance", "besselh", "besseli",
+ "besselj", "besselk", "bessely", "bitpack",
+ "bsxfun", "builtin", "ccolamd", "cellfun",
+ "cellslices", "chol", "choldelete", "cholinsert",
+ "cholinv", "cholshift", "cholupdate", "colamd",
+ "colloc", "convhulln", "convn", "csymamd",
+ "cummax", "cummin", "daspk", "daspk_options",
+ "dasrt", "dasrt_options", "dassl", "dassl_options",
+ "dbclear", "dbdown", "dbstack", "dbstatus",
+ "dbstop", "dbtype", "dbup", "dbwhere", "det",
+ "dlmread", "dmperm", "dot", "eig", "eigs",
+ "endgrent", "endpwent", "etree", "fft", "fftn",
+ "fftw", "filter", "find", "full", "gcd",
+ "getgrent", "getgrgid", "getgrnam", "getpwent",
+ "getpwnam", "getpwuid", "getrusage", "givens",
+ "gmtime", "gnuplot_binary", "hess", "ifft",
+ "ifftn", "inv", "isdebugmode", "issparse", "kron",
+ "localtime", "lookup", "lsode", "lsode_options",
+ "lu", "luinc", "luupdate", "matrix_type", "max",
+ "min", "mktime", "pinv", "qr", "qrdelete",
+ "qrinsert", "qrshift", "qrupdate", "quad",
+ "quad_options", "qz", "rand", "rande", "randg",
+ "randn", "randp", "randperm", "rcond", "regexp",
+ "regexpi", "regexprep", "schur", "setgrent",
+ "setpwent", "sort", "spalloc", "sparse", "spparms",
+ "sprank", "sqrtm", "strfind", "strftime",
+ "strptime", "strrep", "svd", "svd_driver", "syl",
+ "symamd", "symbfact", "symrcm", "time", "tsearch",
+ "typecast", "urlread", "urlwrite")
+
+ mapping_kw = (
+ "abs", "acos", "acosh", "acot", "acoth", "acsc",
+ "acsch", "angle", "arg", "asec", "asech", "asin",
+ "asinh", "atan", "atanh", "beta", "betainc",
+ "betaln", "bincoeff", "cbrt", "ceil", "conj", "cos",
+ "cosh", "cot", "coth", "csc", "csch", "erf", "erfc",
+ "erfcx", "erfinv", "exp", "finite", "fix", "floor",
+ "fmod", "gamma", "gammainc", "gammaln", "imag",
+ "isalnum", "isalpha", "isascii", "iscntrl",
+ "isdigit", "isfinite", "isgraph", "isinf",
+ "islower", "isna", "isnan", "isprint", "ispunct",
+ "isspace", "isupper", "isxdigit", "lcm", "lgamma",
+ "log", "lower", "mod", "real", "rem", "round",
+ "roundb", "sec", "sech", "sign", "sin", "sinh",
+ "sqrt", "tan", "tanh", "toascii", "tolower", "xor")
+
+ builtin_consts = (
+ "EDITOR", "EXEC_PATH", "I", "IMAGE_PATH", "NA",
+ "OCTAVE_HOME", "OCTAVE_VERSION", "PAGER",
+ "PAGER_FLAGS", "SEEK_CUR", "SEEK_END", "SEEK_SET",
+ "SIG", "S_ISBLK", "S_ISCHR", "S_ISDIR", "S_ISFIFO",
+ "S_ISLNK", "S_ISREG", "S_ISSOCK", "WCONTINUE",
+ "WCOREDUMP", "WEXITSTATUS", "WIFCONTINUED",
+ "WIFEXITED", "WIFSIGNALED", "WIFSTOPPED", "WNOHANG",
+ "WSTOPSIG", "WTERMSIG", "WUNTRACED")
+
+ tokens = {
+ 'root': [
+ # We should look into multiline comments
+ (r'[%#].*$', Comment),
+ (r'^\s*function', Keyword, 'deffunc'),
+
+ # from 'iskeyword' on hg changeset 8cc154f45e37
+ (words((
+ '__FILE__', '__LINE__', 'break', 'case', 'catch', 'classdef', 'continue', 'do', 'else',
+ 'elseif', 'end', 'end_try_catch', 'end_unwind_protect', 'endclassdef',
+ 'endevents', 'endfor', 'endfunction', 'endif', 'endmethods', 'endproperties',
+ 'endswitch', 'endwhile', 'events', 'for', 'function', 'get', 'global', 'if', 'methods',
+ 'otherwise', 'persistent', 'properties', 'return', 'set', 'static', 'switch', 'try',
+ 'until', 'unwind_protect', 'unwind_protect_cleanup', 'while'), suffix=r'\b'),
+ Keyword),
+
+ (words(builtin_kw + command_kw + function_kw + loadable_kw + mapping_kw,
+ suffix=r'\b'), Name.Builtin),
+
+ (words(builtin_consts, suffix=r'\b'), Name.Constant),
+
+ # operators in Octave but not Matlab:
+ (r'-=|!=|!|/=|--', Operator),
+ # operators:
+ (r'-|==|~=|<|>|<=|>=|&&|&|~|\|\|?', Operator),
+ # operators in Octave but not Matlab requiring escape for re:
+ (r'\*=|\+=|\^=|\/=|\\=|\*\*|\+\+|\.\*\*', Operator),
+ # operators requiring escape for re:
+ (r'\.\*|\*|\+|\.\^|\.\\|\.\/|\/|\\', Operator),
+
+
+ # punctuation:
+ (r'[\[\](){}:@.,]', Punctuation),
+ (r'=|:|;', Punctuation),
+
+ (r'"[^"]*"', String),
+
+ (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float),
+ (r'\d+[eEf][+-]?[0-9]+', Number.Float),
+ (r'\d+', Number.Integer),
+
+ # quote can be transpose, instead of string:
+ # (not great, but handles common cases...)
+ (r'(?<=[\w)\].])\'+', Operator),
+ (r'(?<![\w)\].])\'', String, 'string'),
+
+ (r'[a-zA-Z_]\w*', Name),
+ (r'.', Text),
+ ],
+ 'string': [
+ (r"[^']*'", String, '#pop'),
+ ],
+ 'deffunc': [
+ (r'(\s*)(?:(.+)(\s*)(=)(\s*))?(.+)(\()(.*)(\))(\s*)',
+ bygroups(Whitespace, Text, Whitespace, Punctuation,
+ Whitespace, Name.Function, Punctuation, Text,
+ Punctuation, Whitespace), '#pop'),
+ # function with no args
+ (r'(\s*)([a-zA-Z_]\w*)', bygroups(Text, Name.Function), '#pop'),
+ ],
+ }
+
+
+class ScilabLexer(RegexLexer):
+ """
+ For Scilab source code.
+
+ .. versionadded:: 1.5
+ """
+ name = 'Scilab'
+ aliases = ['scilab']
+ filenames = ['*.sci', '*.sce', '*.tst']
+ mimetypes = ['text/scilab']
+
+ tokens = {
+ 'root': [
+ (r'//.*?$', Comment.Single),
+ (r'^\s*function', Keyword, 'deffunc'),
+
+ (words((
+ '__FILE__', '__LINE__', 'break', 'case', 'catch', 'classdef', 'continue', 'do', 'else',
+ 'elseif', 'end', 'end_try_catch', 'end_unwind_protect', 'endclassdef',
+ 'endevents', 'endfor', 'endfunction', 'endif', 'endmethods', 'endproperties',
+ 'endswitch', 'endwhile', 'events', 'for', 'function', 'get', 'global', 'if', 'methods',
+ 'otherwise', 'persistent', 'properties', 'return', 'set', 'static', 'switch', 'try',
+ 'until', 'unwind_protect', 'unwind_protect_cleanup', 'while'), suffix=r'\b'),
+ Keyword),
+
+ (words(_scilab_builtins.functions_kw +
+ _scilab_builtins.commands_kw +
+ _scilab_builtins.macros_kw, suffix=r'\b'), Name.Builtin),
+
+ (words(_scilab_builtins.variables_kw, suffix=r'\b'), Name.Constant),
+
+ # operators:
+ (r'-|==|~=|<|>|<=|>=|&&|&|~|\|\|?', Operator),
+ # operators requiring escape for re:
+ (r'\.\*|\*|\+|\.\^|\.\\|\.\/|\/|\\', Operator),
+
+ # punctuation:
+ (r'[\[\](){}@.,=:;]', Punctuation),
+
+ (r'"[^"]*"', String),
+
+ # quote can be transpose, instead of string:
+ # (not great, but handles common cases...)
+ (r'(?<=[\w)\].])\'+', Operator),
+ (r'(?<![\w)\].])\'', String, 'string'),
+
+ (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float),
+ (r'\d+[eEf][+-]?[0-9]+', Number.Float),
+ (r'\d+', Number.Integer),
+
+ (r'[a-zA-Z_]\w*', Name),
+ (r'.', Text),
+ ],
+ 'string': [
+ (r"[^']*'", String, '#pop'),
+ (r'.', String, '#pop'),
+ ],
+ 'deffunc': [
+ (r'(\s*)(?:(.+)(\s*)(=)(\s*))?(.+)(\()(.*)(\))(\s*)',
+ bygroups(Whitespace, Text, Whitespace, Punctuation,
+ Whitespace, Name.Function, Punctuation, Text,
+ Punctuation, Whitespace), '#pop'),
+ # function with no args
+ (r'(\s*)([a-zA-Z_]\w*)', bygroups(Text, Name.Function), '#pop'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/ml.py b/wandb/vendor/pygments/lexers/ml.py
new file mode 100644
index 0000000000000000000000000000000000000000..f80d5bfa75955152476d819467cfef6d90541af0
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/ml.py
@@ -0,0 +1,769 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.ml
+ ~~~~~~~~~~~~~~~~~~
+
+ Lexers for ML family languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, default, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Error
+
+__all__ = ['SMLLexer', 'OcamlLexer', 'OpaLexer']
+
+
+class SMLLexer(RegexLexer):
+ """
+ For the Standard ML language.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'Standard ML'
+ aliases = ['sml']
+ filenames = ['*.sml', '*.sig', '*.fun']
+ mimetypes = ['text/x-standardml', 'application/x-standardml']
+
+ alphanumid_reserved = set((
+ # Core
+ 'abstype', 'and', 'andalso', 'as', 'case', 'datatype', 'do', 'else',
+ 'end', 'exception', 'fn', 'fun', 'handle', 'if', 'in', 'infix',
+ 'infixr', 'let', 'local', 'nonfix', 'of', 'op', 'open', 'orelse',
+ 'raise', 'rec', 'then', 'type', 'val', 'with', 'withtype', 'while',
+ # Modules
+ 'eqtype', 'functor', 'include', 'sharing', 'sig', 'signature',
+ 'struct', 'structure', 'where',
+ ))
+
+ symbolicid_reserved = set((
+ # Core
+ ':', '\|', '=', '=>', '->', '#',
+ # Modules
+ ':>',
+ ))
+
+ nonid_reserved = set(('(', ')', '[', ']', '{', '}', ',', ';', '...', '_'))
+
+ alphanumid_re = r"[a-zA-Z][\w']*"
+ symbolicid_re = r"[!%&$#+\-/:<=>?@\\~`^|*]+"
+
+ # A character constant is a sequence of the form #s, where s is a string
+ # constant denoting a string of size one character. This setup just parses
+ # the entire string as either a String.Double or a String.Char (depending
+ # on the argument), even if the String.Char is an erronous
+ # multiple-character string.
+ def stringy(whatkind):
+ return [
+ (r'[^"\\]', whatkind),
+ (r'\\[\\"abtnvfr]', String.Escape),
+ # Control-character notation is used for codes < 32,
+ # where \^@ == \000
+ (r'\\\^[\x40-\x5e]', String.Escape),
+ # Docs say 'decimal digits'
+ (r'\\[0-9]{3}', String.Escape),
+ (r'\\u[0-9a-fA-F]{4}', String.Escape),
+ (r'\\\s+\\', String.Interpol),
+ (r'"', whatkind, '#pop'),
+ ]
+
+ # Callbacks for distinguishing tokens and reserved words
+ def long_id_callback(self, match):
+ if match.group(1) in self.alphanumid_reserved:
+ token = Error
+ else:
+ token = Name.Namespace
+ yield match.start(1), token, match.group(1)
+ yield match.start(2), Punctuation, match.group(2)
+
+ def end_id_callback(self, match):
+ if match.group(1) in self.alphanumid_reserved:
+ token = Error
+ elif match.group(1) in self.symbolicid_reserved:
+ token = Error
+ else:
+ token = Name
+ yield match.start(1), token, match.group(1)
+
+ def id_callback(self, match):
+ str = match.group(1)
+ if str in self.alphanumid_reserved:
+ token = Keyword.Reserved
+ elif str in self.symbolicid_reserved:
+ token = Punctuation
+ else:
+ token = Name
+ yield match.start(1), token, str
+
+ tokens = {
+ # Whitespace and comments are (almost) everywhere
+ 'whitespace': [
+ (r'\s+', Text),
+ (r'\(\*', Comment.Multiline, 'comment'),
+ ],
+
+ 'delimiters': [
+ # This lexer treats these delimiters specially:
+ # Delimiters define scopes, and the scope is how the meaning of
+ # the `|' is resolved - is it a case/handle expression, or function
+ # definition by cases? (This is not how the Definition works, but
+ # it's how MLton behaves, see http://mlton.org/SMLNJDeviations)
+ (r'\(|\[|\{', Punctuation, 'main'),
+ (r'\)|\]|\}', Punctuation, '#pop'),
+ (r'\b(let|if|local)\b(?!\')', Keyword.Reserved, ('main', 'main')),
+ (r'\b(struct|sig|while)\b(?!\')', Keyword.Reserved, 'main'),
+ (r'\b(do|else|end|in|then)\b(?!\')', Keyword.Reserved, '#pop'),
+ ],
+
+ 'core': [
+ # Punctuation that doesn't overlap symbolic identifiers
+ (r'(%s)' % '|'.join(re.escape(z) for z in nonid_reserved),
+ Punctuation),
+
+ # Special constants: strings, floats, numbers in decimal and hex
+ (r'#"', String.Char, 'char'),
+ (r'"', String.Double, 'string'),
+ (r'~?0x[0-9a-fA-F]+', Number.Hex),
+ (r'0wx[0-9a-fA-F]+', Number.Hex),
+ (r'0w\d+', Number.Integer),
+ (r'~?\d+\.\d+[eE]~?\d+', Number.Float),
+ (r'~?\d+\.\d+', Number.Float),
+ (r'~?\d+[eE]~?\d+', Number.Float),
+ (r'~?\d+', Number.Integer),
+
+ # Labels
+ (r'#\s*[1-9][0-9]*', Name.Label),
+ (r'#\s*(%s)' % alphanumid_re, Name.Label),
+ (r'#\s+(%s)' % symbolicid_re, Name.Label),
+ # Some reserved words trigger a special, local lexer state change
+ (r'\b(datatype|abstype)\b(?!\')', Keyword.Reserved, 'dname'),
+ (r'(?=\b(exception)\b(?!\'))', Text, ('ename')),
+ (r'\b(functor|include|open|signature|structure)\b(?!\')',
+ Keyword.Reserved, 'sname'),
+ (r'\b(type|eqtype)\b(?!\')', Keyword.Reserved, 'tname'),
+
+ # Regular identifiers, long and otherwise
+ (r'\'[\w\']*', Name.Decorator),
+ (r'(%s)(\.)' % alphanumid_re, long_id_callback, "dotted"),
+ (r'(%s)' % alphanumid_re, id_callback),
+ (r'(%s)' % symbolicid_re, id_callback),
+ ],
+ 'dotted': [
+ (r'(%s)(\.)' % alphanumid_re, long_id_callback),
+ (r'(%s)' % alphanumid_re, end_id_callback, "#pop"),
+ (r'(%s)' % symbolicid_re, end_id_callback, "#pop"),
+ (r'\s+', Error),
+ (r'\S+', Error),
+ ],
+
+
+ # Main parser (prevents errors in files that have scoping errors)
+ 'root': [
+ default('main')
+ ],
+
+ # In this scope, I expect '|' to not be followed by a function name,
+ # and I expect 'and' to be followed by a binding site
+ 'main': [
+ include('whitespace'),
+
+ # Special behavior of val/and/fun
+ (r'\b(val|and)\b(?!\')', Keyword.Reserved, 'vname'),
+ (r'\b(fun)\b(?!\')', Keyword.Reserved,
+ ('#pop', 'main-fun', 'fname')),
+
+ include('delimiters'),
+ include('core'),
+ (r'\S+', Error),
+ ],
+
+ # In this scope, I expect '|' and 'and' to be followed by a function
+ 'main-fun': [
+ include('whitespace'),
+
+ (r'\s', Text),
+ (r'\(\*', Comment.Multiline, 'comment'),
+
+ # Special behavior of val/and/fun
+ (r'\b(fun|and)\b(?!\')', Keyword.Reserved, 'fname'),
+ (r'\b(val)\b(?!\')', Keyword.Reserved,
+ ('#pop', 'main', 'vname')),
+
+ # Special behavior of '|' and '|'-manipulating keywords
+ (r'\|', Punctuation, 'fname'),
+ (r'\b(case|handle)\b(?!\')', Keyword.Reserved,
+ ('#pop', 'main')),
+
+ include('delimiters'),
+ include('core'),
+ (r'\S+', Error),
+ ],
+
+ # Character and string parsers
+ 'char': stringy(String.Char),
+ 'string': stringy(String.Double),
+
+ 'breakout': [
+ (r'(?=\b(%s)\b(?!\'))' % '|'.join(alphanumid_reserved), Text, '#pop'),
+ ],
+
+ # Dealing with what comes after module system keywords
+ 'sname': [
+ include('whitespace'),
+ include('breakout'),
+
+ (r'(%s)' % alphanumid_re, Name.Namespace),
+ default('#pop'),
+ ],
+
+ # Dealing with what comes after the 'fun' (or 'and' or '|') keyword
+ 'fname': [
+ include('whitespace'),
+ (r'\'[\w\']*', Name.Decorator),
+ (r'\(', Punctuation, 'tyvarseq'),
+
+ (r'(%s)' % alphanumid_re, Name.Function, '#pop'),
+ (r'(%s)' % symbolicid_re, Name.Function, '#pop'),
+
+ # Ignore interesting function declarations like "fun (x + y) = ..."
+ default('#pop'),
+ ],
+
+ # Dealing with what comes after the 'val' (or 'and') keyword
+ 'vname': [
+ include('whitespace'),
+ (r'\'[\w\']*', Name.Decorator),
+ (r'\(', Punctuation, 'tyvarseq'),
+
+ (r'(%s)(\s*)(=(?!%s))' % (alphanumid_re, symbolicid_re),
+ bygroups(Name.Variable, Text, Punctuation), '#pop'),
+ (r'(%s)(\s*)(=(?!%s))' % (symbolicid_re, symbolicid_re),
+ bygroups(Name.Variable, Text, Punctuation), '#pop'),
+ (r'(%s)' % alphanumid_re, Name.Variable, '#pop'),
+ (r'(%s)' % symbolicid_re, Name.Variable, '#pop'),
+
+ # Ignore interesting patterns like 'val (x, y)'
+ default('#pop'),
+ ],
+
+ # Dealing with what comes after the 'type' (or 'and') keyword
+ 'tname': [
+ include('whitespace'),
+ include('breakout'),
+
+ (r'\'[\w\']*', Name.Decorator),
+ (r'\(', Punctuation, 'tyvarseq'),
+ (r'=(?!%s)' % symbolicid_re, Punctuation, ('#pop', 'typbind')),
+
+ (r'(%s)' % alphanumid_re, Keyword.Type),
+ (r'(%s)' % symbolicid_re, Keyword.Type),
+ (r'\S+', Error, '#pop'),
+ ],
+
+ # A type binding includes most identifiers
+ 'typbind': [
+ include('whitespace'),
+
+ (r'\b(and)\b(?!\')', Keyword.Reserved, ('#pop', 'tname')),
+
+ include('breakout'),
+ include('core'),
+ (r'\S+', Error, '#pop'),
+ ],
+
+ # Dealing with what comes after the 'datatype' (or 'and') keyword
+ 'dname': [
+ include('whitespace'),
+ include('breakout'),
+
+ (r'\'[\w\']*', Name.Decorator),
+ (r'\(', Punctuation, 'tyvarseq'),
+ (r'(=)(\s*)(datatype)',
+ bygroups(Punctuation, Text, Keyword.Reserved), '#pop'),
+ (r'=(?!%s)' % symbolicid_re, Punctuation,
+ ('#pop', 'datbind', 'datcon')),
+
+ (r'(%s)' % alphanumid_re, Keyword.Type),
+ (r'(%s)' % symbolicid_re, Keyword.Type),
+ (r'\S+', Error, '#pop'),
+ ],
+
+ # common case - A | B | C of int
+ 'datbind': [
+ include('whitespace'),
+
+ (r'\b(and)\b(?!\')', Keyword.Reserved, ('#pop', 'dname')),
+ (r'\b(withtype)\b(?!\')', Keyword.Reserved, ('#pop', 'tname')),
+ (r'\b(of)\b(?!\')', Keyword.Reserved),
+
+ (r'(\|)(\s*)(%s)' % alphanumid_re,
+ bygroups(Punctuation, Text, Name.Class)),
+ (r'(\|)(\s+)(%s)' % symbolicid_re,
+ bygroups(Punctuation, Text, Name.Class)),
+
+ include('breakout'),
+ include('core'),
+ (r'\S+', Error),
+ ],
+
+ # Dealing with what comes after an exception
+ 'ename': [
+ include('whitespace'),
+
+ (r'(exception|and)\b(\s+)(%s)' % alphanumid_re,
+ bygroups(Keyword.Reserved, Text, Name.Class)),
+ (r'(exception|and)\b(\s*)(%s)' % symbolicid_re,
+ bygroups(Keyword.Reserved, Text, Name.Class)),
+ (r'\b(of)\b(?!\')', Keyword.Reserved),
+
+ include('breakout'),
+ include('core'),
+ (r'\S+', Error),
+ ],
+
+ 'datcon': [
+ include('whitespace'),
+ (r'(%s)' % alphanumid_re, Name.Class, '#pop'),
+ (r'(%s)' % symbolicid_re, Name.Class, '#pop'),
+ (r'\S+', Error, '#pop'),
+ ],
+
+ # Series of type variables
+ 'tyvarseq': [
+ (r'\s', Text),
+ (r'\(\*', Comment.Multiline, 'comment'),
+
+ (r'\'[\w\']*', Name.Decorator),
+ (alphanumid_re, Name),
+ (r',', Punctuation),
+ (r'\)', Punctuation, '#pop'),
+ (symbolicid_re, Name),
+ ],
+
+ 'comment': [
+ (r'[^(*)]', Comment.Multiline),
+ (r'\(\*', Comment.Multiline, '#push'),
+ (r'\*\)', Comment.Multiline, '#pop'),
+ (r'[(*)]', Comment.Multiline),
+ ],
+ }
+
+
+class OcamlLexer(RegexLexer):
+ """
+ For the OCaml language.
+
+ .. versionadded:: 0.7
+ """
+
+ name = 'OCaml'
+ aliases = ['ocaml']
+ filenames = ['*.ml', '*.mli', '*.mll', '*.mly']
+ mimetypes = ['text/x-ocaml']
+
+ keywords = (
+ 'as', 'assert', 'begin', 'class', 'constraint', 'do', 'done',
+ 'downto', 'else', 'end', 'exception', 'external', 'false',
+ 'for', 'fun', 'function', 'functor', 'if', 'in', 'include',
+ 'inherit', 'initializer', 'lazy', 'let', 'match', 'method',
+ 'module', 'mutable', 'new', 'object', 'of', 'open', 'private',
+ 'raise', 'rec', 'sig', 'struct', 'then', 'to', 'true', 'try',
+ 'type', 'value', 'val', 'virtual', 'when', 'while', 'with',
+ )
+ keyopts = (
+ '!=', '#', '&', '&&', r'\(', r'\)', r'\*', r'\+', ',', '-',
+ r'-\.', '->', r'\.', r'\.\.', ':', '::', ':=', ':>', ';', ';;', '<',
+ '<-', '=', '>', '>]', r'>\}', r'\?', r'\?\?', r'\[', r'\[<', r'\[>',
+ r'\[\|', ']', '_', '`', r'\{', r'\{<', r'\|', r'\|]', r'\}', '~'
+ )
+
+ operators = r'[!$%&*+\./:<=>?@^|~-]'
+ word_operators = ('and', 'asr', 'land', 'lor', 'lsl', 'lxor', 'mod', 'or')
+ prefix_syms = r'[!?~]'
+ infix_syms = r'[=<>@^|&+\*/$%-]'
+ primitives = ('unit', 'int', 'float', 'bool', 'string', 'char', 'list', 'array')
+
+ tokens = {
+ 'escape-sequence': [
+ (r'\\[\\"\'ntbr]', String.Escape),
+ (r'\\[0-9]{3}', String.Escape),
+ (r'\\x[0-9a-fA-F]{2}', String.Escape),
+ ],
+ 'root': [
+ (r'\s+', Text),
+ (r'false|true|\(\)|\[\]', Name.Builtin.Pseudo),
+ (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'),
+ (r'\b([A-Z][\w\']*)', Name.Class),
+ (r'\(\*(?![)])', Comment, 'comment'),
+ (r'\b(%s)\b' % '|'.join(keywords), Keyword),
+ (r'(%s)' % '|'.join(keyopts[::-1]), Operator),
+ (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator),
+ (r'\b(%s)\b' % '|'.join(word_operators), Operator.Word),
+ (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type),
+
+ (r"[^\W\d][\w']*", Name),
+
+ (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float),
+ (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
+ (r'0[oO][0-7][0-7_]*', Number.Oct),
+ (r'0[bB][01][01_]*', Number.Bin),
+ (r'\d[\d_]*', Number.Integer),
+
+ (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
+ String.Char),
+ (r"'.'", String.Char),
+ (r"'", Keyword), # a stray quote is another syntax element
+
+ (r'"', String.Double, 'string'),
+
+ (r'[~?][a-z][\w\']*:', Name.Variable),
+ ],
+ 'comment': [
+ (r'[^(*)]+', Comment),
+ (r'\(\*', Comment, '#push'),
+ (r'\*\)', Comment, '#pop'),
+ (r'[(*)]', Comment),
+ ],
+ 'string': [
+ (r'[^\\"]+', String.Double),
+ include('escape-sequence'),
+ (r'\\\n', String.Double),
+ (r'"', String.Double, '#pop'),
+ ],
+ 'dotted': [
+ (r'\s+', Text),
+ (r'\.', Punctuation),
+ (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace),
+ (r'[A-Z][\w\']*', Name.Class, '#pop'),
+ (r'[a-z_][\w\']*', Name, '#pop'),
+ default('#pop'),
+ ],
+ }
+
+
+class OpaLexer(RegexLexer):
+ """
+ Lexer for the Opa language (http://opalang.org).
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'Opa'
+ aliases = ['opa']
+ filenames = ['*.opa']
+ mimetypes = ['text/x-opa']
+
+ # most of these aren't strictly keywords
+ # but if you color only real keywords, you might just
+ # as well not color anything
+ keywords = (
+ 'and', 'as', 'begin', 'case', 'client', 'css', 'database', 'db', 'do',
+ 'else', 'end', 'external', 'forall', 'function', 'if', 'import',
+ 'match', 'module', 'or', 'package', 'parser', 'rec', 'server', 'then',
+ 'type', 'val', 'with', 'xml_parser',
+ )
+
+ # matches both stuff and `stuff`
+ ident_re = r'(([a-zA-Z_]\w*)|(`[^`]*`))'
+
+ op_re = r'[.=\-<>,@~%/+?*&^!]'
+ punc_re = r'[()\[\],;|]' # '{' and '}' are treated elsewhere
+ # because they are also used for inserts
+
+ tokens = {
+ # copied from the caml lexer, should be adapted
+ 'escape-sequence': [
+ (r'\\[\\"\'ntr}]', String.Escape),
+ (r'\\[0-9]{3}', String.Escape),
+ (r'\\x[0-9a-fA-F]{2}', String.Escape),
+ ],
+
+ # factorizing these rules, because they are inserted many times
+ 'comments': [
+ (r'/\*', Comment, 'nested-comment'),
+ (r'//.*?$', Comment),
+ ],
+ 'comments-and-spaces': [
+ include('comments'),
+ (r'\s+', Text),
+ ],
+
+ 'root': [
+ include('comments-and-spaces'),
+ # keywords
+ (words(keywords, prefix=r'\b', suffix=r'\b'), Keyword),
+ # directives
+ # we could parse the actual set of directives instead of anything
+ # starting with @, but this is troublesome
+ # because it needs to be adjusted all the time
+ # and assuming we parse only sources that compile, it is useless
+ (r'@' + ident_re + r'\b', Name.Builtin.Pseudo),
+
+ # number literals
+ (r'-?.[\d]+([eE][+\-]?\d+)', Number.Float),
+ (r'-?\d+.\d*([eE][+\-]?\d+)', Number.Float),
+ (r'-?\d+[eE][+\-]?\d+', Number.Float),
+ (r'0[xX][\da-fA-F]+', Number.Hex),
+ (r'0[oO][0-7]+', Number.Oct),
+ (r'0[bB][01]+', Number.Bin),
+ (r'\d+', Number.Integer),
+ # color literals
+ (r'#[\da-fA-F]{3,6}', Number.Integer),
+
+ # string literals
+ (r'"', String.Double, 'string'),
+ # char literal, should be checked because this is the regexp from
+ # the caml lexer
+ (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2})|.)'",
+ String.Char),
+
+ # this is meant to deal with embedded exprs in strings
+ # every time we find a '}' we pop a state so that if we were
+ # inside a string, we are back in the string state
+ # as a consequence, we must also push a state every time we find a
+ # '{' or else we will have errors when parsing {} for instance
+ (r'\{', Operator, '#push'),
+ (r'\}', Operator, '#pop'),
+
+ # html literals
+ # this is a much more strict that the actual parser,
+ # since a<b would not be parsed as html
+ # but then again, the parser is way too lax, and we can't hope
+ # to have something as tolerant
+ (r'<(?=[a-zA-Z>])', String.Single, 'html-open-tag'),
+
+ # db path
+ # matching the '[_]' in '/a[_]' because it is a part
+ # of the syntax of the db path definition
+ # unfortunately, i don't know how to match the ']' in
+ # /a[1], so this is somewhat inconsistent
+ (r'[@?!]?(/\w+)+(\[_\])?', Name.Variable),
+ # putting the same color on <- as on db path, since
+ # it can be used only to mean Db.write
+ (r'<-(?!'+op_re+r')', Name.Variable),
+
+ # 'modules'
+ # although modules are not distinguished by their names as in caml
+ # the standard library seems to follow the convention that modules
+ # only area capitalized
+ (r'\b([A-Z]\w*)(?=\.)', Name.Namespace),
+
+ # operators
+ # = has a special role because this is the only
+ # way to syntactic distinguish binding constructions
+ # unfortunately, this colors the equal in {x=2} too
+ (r'=(?!'+op_re+r')', Keyword),
+ (r'(%s)+' % op_re, Operator),
+ (r'(%s)+' % punc_re, Operator),
+
+ # coercions
+ (r':', Operator, 'type'),
+ # type variables
+ # we need this rule because we don't parse specially type
+ # definitions so in "type t('a) = ...", "'a" is parsed by 'root'
+ ("'"+ident_re, Keyword.Type),
+
+ # id literal, #something, or #{expr}
+ (r'#'+ident_re, String.Single),
+ (r'#(?=\{)', String.Single),
+
+ # identifiers
+ # this avoids to color '2' in 'a2' as an integer
+ (ident_re, Text),
+
+ # default, not sure if that is needed or not
+ # (r'.', Text),
+ ],
+
+ # it is quite painful to have to parse types to know where they end
+ # this is the general rule for a type
+ # a type is either:
+ # * -> ty
+ # * type-with-slash
+ # * type-with-slash -> ty
+ # * type-with-slash (, type-with-slash)+ -> ty
+ #
+ # the code is pretty funky in here, but this code would roughly
+ # translate in caml to:
+ # let rec type stream =
+ # match stream with
+ # | [< "->"; stream >] -> type stream
+ # | [< ""; stream >] ->
+ # type_with_slash stream
+ # type_lhs_1 stream;
+ # and type_1 stream = ...
+ 'type': [
+ include('comments-and-spaces'),
+ (r'->', Keyword.Type),
+ default(('#pop', 'type-lhs-1', 'type-with-slash')),
+ ],
+
+ # parses all the atomic or closed constructions in the syntax of type
+ # expressions: record types, tuple types, type constructors, basic type
+ # and type variables
+ 'type-1': [
+ include('comments-and-spaces'),
+ (r'\(', Keyword.Type, ('#pop', 'type-tuple')),
+ (r'~?\{', Keyword.Type, ('#pop', 'type-record')),
+ (ident_re+r'\(', Keyword.Type, ('#pop', 'type-tuple')),
+ (ident_re, Keyword.Type, '#pop'),
+ ("'"+ident_re, Keyword.Type),
+ # this case is not in the syntax but sometimes
+ # we think we are parsing types when in fact we are parsing
+ # some css, so we just pop the states until we get back into
+ # the root state
+ default('#pop'),
+ ],
+
+ # type-with-slash is either:
+ # * type-1
+ # * type-1 (/ type-1)+
+ 'type-with-slash': [
+ include('comments-and-spaces'),
+ default(('#pop', 'slash-type-1', 'type-1')),
+ ],
+ 'slash-type-1': [
+ include('comments-and-spaces'),
+ ('/', Keyword.Type, ('#pop', 'type-1')),
+ # same remark as above
+ default('#pop'),
+ ],
+
+ # we go in this state after having parsed a type-with-slash
+ # while trying to parse a type
+ # and at this point we must determine if we are parsing an arrow
+ # type (in which case we must continue parsing) or not (in which
+ # case we stop)
+ 'type-lhs-1': [
+ include('comments-and-spaces'),
+ (r'->', Keyword.Type, ('#pop', 'type')),
+ (r'(?=,)', Keyword.Type, ('#pop', 'type-arrow')),
+ default('#pop'),
+ ],
+ 'type-arrow': [
+ include('comments-and-spaces'),
+ # the look ahead here allows to parse f(x : int, y : float -> truc)
+ # correctly
+ (r',(?=[^:]*?->)', Keyword.Type, 'type-with-slash'),
+ (r'->', Keyword.Type, ('#pop', 'type')),
+ # same remark as above
+ default('#pop'),
+ ],
+
+ # no need to do precise parsing for tuples and records
+ # because they are closed constructions, so we can simply
+ # find the closing delimiter
+ # note that this function would be not work if the source
+ # contained identifiers like `{)` (although it could be patched
+ # to support it)
+ 'type-tuple': [
+ include('comments-and-spaces'),
+ (r'[^()/*]+', Keyword.Type),
+ (r'[/*]', Keyword.Type),
+ (r'\(', Keyword.Type, '#push'),
+ (r'\)', Keyword.Type, '#pop'),
+ ],
+ 'type-record': [
+ include('comments-and-spaces'),
+ (r'[^{}/*]+', Keyword.Type),
+ (r'[/*]', Keyword.Type),
+ (r'\{', Keyword.Type, '#push'),
+ (r'\}', Keyword.Type, '#pop'),
+ ],
+
+ # 'type-tuple': [
+ # include('comments-and-spaces'),
+ # (r'\)', Keyword.Type, '#pop'),
+ # default(('#pop', 'type-tuple-1', 'type-1')),
+ # ],
+ # 'type-tuple-1': [
+ # include('comments-and-spaces'),
+ # (r',?\s*\)', Keyword.Type, '#pop'), # ,) is a valid end of tuple, in (1,)
+ # (r',', Keyword.Type, 'type-1'),
+ # ],
+ # 'type-record':[
+ # include('comments-and-spaces'),
+ # (r'\}', Keyword.Type, '#pop'),
+ # (r'~?(?:\w+|`[^`]*`)', Keyword.Type, 'type-record-field-expr'),
+ # ],
+ # 'type-record-field-expr': [
+ #
+ # ],
+
+ 'nested-comment': [
+ (r'[^/*]+', Comment),
+ (r'/\*', Comment, '#push'),
+ (r'\*/', Comment, '#pop'),
+ (r'[/*]', Comment),
+ ],
+
+ # the copy pasting between string and single-string
+ # is kinda sad. Is there a way to avoid that??
+ 'string': [
+ (r'[^\\"{]+', String.Double),
+ (r'"', String.Double, '#pop'),
+ (r'\{', Operator, 'root'),
+ include('escape-sequence'),
+ ],
+ 'single-string': [
+ (r'[^\\\'{]+', String.Double),
+ (r'\'', String.Double, '#pop'),
+ (r'\{', Operator, 'root'),
+ include('escape-sequence'),
+ ],
+
+ # all the html stuff
+ # can't really reuse some existing html parser
+ # because we must be able to parse embedded expressions
+
+ # we are in this state after someone parsed the '<' that
+ # started the html literal
+ 'html-open-tag': [
+ (r'[\w\-:]+', String.Single, ('#pop', 'html-attr')),
+ (r'>', String.Single, ('#pop', 'html-content')),
+ ],
+
+ # we are in this state after someone parsed the '</' that
+ # started the end of the closing tag
+ 'html-end-tag': [
+ # this is a star, because </> is allowed
+ (r'[\w\-:]*>', String.Single, '#pop'),
+ ],
+
+ # we are in this state after having parsed '<ident(:ident)?'
+ # we thus parse a possibly empty list of attributes
+ 'html-attr': [
+ (r'\s+', Text),
+ (r'[\w\-:]+=', String.Single, 'html-attr-value'),
+ (r'/>', String.Single, '#pop'),
+ (r'>', String.Single, ('#pop', 'html-content')),
+ ],
+
+ 'html-attr-value': [
+ (r"'", String.Single, ('#pop', 'single-string')),
+ (r'"', String.Single, ('#pop', 'string')),
+ (r'#'+ident_re, String.Single, '#pop'),
+ (r'#(?=\{)', String.Single, ('#pop', 'root')),
+ (r'[^"\'{`=<>]+', String.Single, '#pop'),
+ (r'\{', Operator, ('#pop', 'root')), # this is a tail call!
+ ],
+
+ # we should probably deal with '\' escapes here
+ 'html-content': [
+ (r'<!--', Comment, 'html-comment'),
+ (r'</', String.Single, ('#pop', 'html-end-tag')),
+ (r'<', String.Single, 'html-open-tag'),
+ (r'\{', Operator, 'root'),
+ (r'[^<{]+', String.Single),
+ ],
+
+ 'html-comment': [
+ (r'-->', Comment, '#pop'),
+ (r'[^\-]+|-', Comment),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/modeling.py b/wandb/vendor/pygments/lexers/modeling.py
new file mode 100644
index 0000000000000000000000000000000000000000..b354f1cf0d156618e68ad79c49a57f528c5e701b
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/modeling.py
@@ -0,0 +1,358 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.modeling
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for modeling languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+from pygments.lexers.html import HtmlLexer
+from pygments.lexers import _stan_builtins
+
+__all__ = ['ModelicaLexer', 'BugsLexer', 'JagsLexer', 'StanLexer']
+
+
+class ModelicaLexer(RegexLexer):
+ """
+ For `Modelica <http://www.modelica.org/>`_ source code.
+
+ .. versionadded:: 1.1
+ """
+ name = 'Modelica'
+ aliases = ['modelica']
+ filenames = ['*.mo']
+ mimetypes = ['text/x-modelica']
+
+ flags = re.DOTALL | re.MULTILINE
+
+ _name = r"(?:'(?:[^\\']|\\.)+'|[a-zA-Z_]\w*)"
+
+ tokens = {
+ 'whitespace': [
+ (u'[\\s\ufeff]+', Text),
+ (r'//[^\n]*\n?', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline)
+ ],
+ 'root': [
+ include('whitespace'),
+ (r'"', String.Double, 'string'),
+ (r'[()\[\]{},;]+', Punctuation),
+ (r'\.?[*^/+-]|\.|<>|[<>:=]=?', Operator),
+ (r'\d+(\.?\d*[eE][-+]?\d+|\.\d*)', Number.Float),
+ (r'\d+', Number.Integer),
+ (r'(abs|acos|actualStream|array|asin|assert|AssertionLevel|atan|'
+ r'atan2|backSample|Boolean|cardinality|cat|ceil|change|Clock|'
+ r'Connections|cos|cosh|cross|delay|diagonal|div|edge|exp|'
+ r'ExternalObject|fill|floor|getInstanceName|hold|homotopy|'
+ r'identity|inStream|integer|Integer|interval|inverse|isPresent|'
+ r'linspace|log|log10|matrix|max|min|mod|ndims|noClock|noEvent|'
+ r'ones|outerProduct|pre|previous|product|Real|reinit|rem|rooted|'
+ r'sample|scalar|semiLinear|shiftSample|sign|sin|sinh|size|skew|'
+ r'smooth|spatialDistribution|sqrt|StateSelect|String|subSample|'
+ r'sum|superSample|symmetric|tan|tanh|terminal|terminate|time|'
+ r'transpose|vector|zeros)\b', Name.Builtin),
+ (r'(algorithm|annotation|break|connect|constant|constrainedby|der|'
+ r'discrete|each|else|elseif|elsewhen|encapsulated|enumeration|'
+ r'equation|exit|expandable|extends|external|final|flow|for|if|'
+ r'import|impure|in|initial|inner|input|loop|nondiscrete|outer|'
+ r'output|parameter|partial|protected|public|pure|redeclare|'
+ r'replaceable|return|stream|then|when|while)\b',
+ Keyword.Reserved),
+ (r'(and|not|or)\b', Operator.Word),
+ (r'(block|class|connector|end|function|model|operator|package|'
+ r'record|type)\b', Keyword.Reserved, 'class'),
+ (r'(false|true)\b', Keyword.Constant),
+ (r'within\b', Keyword.Reserved, 'package-prefix'),
+ (_name, Name)
+ ],
+ 'class': [
+ include('whitespace'),
+ (r'(function|record)\b', Keyword.Reserved),
+ (r'(if|for|when|while)\b', Keyword.Reserved, '#pop'),
+ (_name, Name.Class, '#pop'),
+ default('#pop')
+ ],
+ 'package-prefix': [
+ include('whitespace'),
+ (_name, Name.Namespace, '#pop'),
+ default('#pop')
+ ],
+ 'string': [
+ (r'"', String.Double, '#pop'),
+ (r'\\[\'"?\\abfnrtv]', String.Escape),
+ (r'(?i)<\s*html\s*>([^\\"]|\\.)+?(<\s*/\s*html\s*>|(?="))',
+ using(HtmlLexer)),
+ (r'<|\\?[^"\\<]+', String.Double)
+ ]
+ }
+
+
+class BugsLexer(RegexLexer):
+ """
+ Pygments Lexer for `OpenBugs <http://www.openbugs.net/>`_ and WinBugs
+ models.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'BUGS'
+ aliases = ['bugs', 'winbugs', 'openbugs']
+ filenames = ['*.bug']
+
+ _FUNCTIONS = (
+ # Scalar functions
+ 'abs', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctanh',
+ 'cloglog', 'cos', 'cosh', 'cumulative', 'cut', 'density', 'deviance',
+ 'equals', 'expr', 'gammap', 'ilogit', 'icloglog', 'integral', 'log',
+ 'logfact', 'loggam', 'logit', 'max', 'min', 'phi', 'post.p.value',
+ 'pow', 'prior.p.value', 'probit', 'replicate.post', 'replicate.prior',
+ 'round', 'sin', 'sinh', 'solution', 'sqrt', 'step', 'tan', 'tanh',
+ 'trunc',
+ # Vector functions
+ 'inprod', 'interp.lin', 'inverse', 'logdet', 'mean', 'eigen.vals',
+ 'ode', 'prod', 'p.valueM', 'rank', 'ranked', 'replicate.postM',
+ 'sd', 'sort', 'sum',
+ # Special
+ 'D', 'I', 'F', 'T', 'C')
+ """ OpenBUGS built-in functions
+
+ From http://www.openbugs.info/Manuals/ModelSpecification.html#ContentsAII
+
+ This also includes
+
+ - T, C, I : Truncation and censoring.
+ ``T`` and ``C`` are in OpenBUGS. ``I`` in WinBUGS.
+ - D : ODE
+ - F : Functional http://www.openbugs.info/Examples/Functionals.html
+
+ """
+
+ _DISTRIBUTIONS = ('dbern', 'dbin', 'dcat', 'dnegbin', 'dpois',
+ 'dhyper', 'dbeta', 'dchisqr', 'ddexp', 'dexp',
+ 'dflat', 'dgamma', 'dgev', 'df', 'dggamma', 'dgpar',
+ 'dloglik', 'dlnorm', 'dlogis', 'dnorm', 'dpar',
+ 'dt', 'dunif', 'dweib', 'dmulti', 'ddirch', 'dmnorm',
+ 'dmt', 'dwish')
+ """ OpenBUGS built-in distributions
+
+ Functions from
+ http://www.openbugs.info/Manuals/ModelSpecification.html#ContentsAI
+ """
+
+ tokens = {
+ 'whitespace': [
+ (r"\s+", Text),
+ ],
+ 'comments': [
+ # Comments
+ (r'#.*$', Comment.Single),
+ ],
+ 'root': [
+ # Comments
+ include('comments'),
+ include('whitespace'),
+ # Block start
+ (r'(model)(\s+)(\{)',
+ bygroups(Keyword.Namespace, Text, Punctuation)),
+ # Reserved Words
+ (r'(for|in)(?![\w.])', Keyword.Reserved),
+ # Built-in Functions
+ (r'(%s)(?=\s*\()'
+ % r'|'.join(_FUNCTIONS + _DISTRIBUTIONS),
+ Name.Builtin),
+ # Regular variable names
+ (r'[A-Za-z][\w.]*', Name),
+ # Number Literals
+ (r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', Number),
+ # Punctuation
+ (r'\[|\]|\(|\)|:|,|;', Punctuation),
+ # Assignment operators
+ # SLexer makes these tokens Operators.
+ (r'<-|~', Operator),
+ # Infix and prefix operators
+ (r'\+|-|\*|/', Operator),
+ # Block
+ (r'[{}]', Punctuation),
+ ]
+ }
+
+ def analyse_text(text):
+ if re.search(r"^\s*model\s*{", text, re.M):
+ return 0.7
+ else:
+ return 0.0
+
+
+class JagsLexer(RegexLexer):
+ """
+ Pygments Lexer for JAGS.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'JAGS'
+ aliases = ['jags']
+ filenames = ['*.jag', '*.bug']
+
+ # JAGS
+ _FUNCTIONS = (
+ 'abs', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctanh',
+ 'cos', 'cosh', 'cloglog',
+ 'equals', 'exp', 'icloglog', 'ifelse', 'ilogit', 'log', 'logfact',
+ 'loggam', 'logit', 'phi', 'pow', 'probit', 'round', 'sin', 'sinh',
+ 'sqrt', 'step', 'tan', 'tanh', 'trunc', 'inprod', 'interp.lin',
+ 'logdet', 'max', 'mean', 'min', 'prod', 'sum', 'sd', 'inverse',
+ 'rank', 'sort', 't', 'acos', 'acosh', 'asin', 'asinh', 'atan',
+ # Truncation/Censoring (should I include)
+ 'T', 'I')
+ # Distributions with density, probability and quartile functions
+ _DISTRIBUTIONS = tuple('[dpq]%s' % x for x in
+ ('bern', 'beta', 'dchiqsqr', 'ddexp', 'dexp',
+ 'df', 'gamma', 'gen.gamma', 'logis', 'lnorm',
+ 'negbin', 'nchisqr', 'norm', 'par', 'pois', 'weib'))
+ # Other distributions without density and probability
+ _OTHER_DISTRIBUTIONS = (
+ 'dt', 'dunif', 'dbetabin', 'dbern', 'dbin', 'dcat', 'dhyper',
+ 'ddirch', 'dmnorm', 'dwish', 'dmt', 'dmulti', 'dbinom', 'dchisq',
+ 'dnbinom', 'dweibull', 'ddirich')
+
+ tokens = {
+ 'whitespace': [
+ (r"\s+", Text),
+ ],
+ 'names': [
+ # Regular variable names
+ (r'[a-zA-Z][\w.]*\b', Name),
+ ],
+ 'comments': [
+ # do not use stateful comments
+ (r'(?s)/\*.*?\*/', Comment.Multiline),
+ # Comments
+ (r'#.*$', Comment.Single),
+ ],
+ 'root': [
+ # Comments
+ include('comments'),
+ include('whitespace'),
+ # Block start
+ (r'(model|data)(\s+)(\{)',
+ bygroups(Keyword.Namespace, Text, Punctuation)),
+ (r'var(?![\w.])', Keyword.Declaration),
+ # Reserved Words
+ (r'(for|in)(?![\w.])', Keyword.Reserved),
+ # Builtins
+ # Need to use lookahead because . is a valid char
+ (r'(%s)(?=\s*\()' % r'|'.join(_FUNCTIONS
+ + _DISTRIBUTIONS
+ + _OTHER_DISTRIBUTIONS),
+ Name.Builtin),
+ # Names
+ include('names'),
+ # Number Literals
+ (r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', Number),
+ (r'\[|\]|\(|\)|:|,|;', Punctuation),
+ # Assignment operators
+ (r'<-|~', Operator),
+ # # JAGS includes many more than OpenBUGS
+ (r'\+|-|\*|\/|\|\|[&]{2}|[<>=]=?|\^|%.*?%', Operator),
+ (r'[{}]', Punctuation),
+ ]
+ }
+
+ def analyse_text(text):
+ if re.search(r'^\s*model\s*\{', text, re.M):
+ if re.search(r'^\s*data\s*\{', text, re.M):
+ return 0.9
+ elif re.search(r'^\s*var', text, re.M):
+ return 0.9
+ else:
+ return 0.3
+ else:
+ return 0
+
+
+class StanLexer(RegexLexer):
+ """Pygments Lexer for Stan models.
+
+ The Stan modeling language is specified in the *Stan Modeling Language
+ User's Guide and Reference Manual, v2.8.0*,
+ `pdf <https://github.com/stan-dev/stan/releases/download/v2.8.8/stan-reference-2.8.0.pdf>`__.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'Stan'
+ aliases = ['stan']
+ filenames = ['*.stan']
+
+ tokens = {
+ 'whitespace': [
+ (r"\s+", Text),
+ ],
+ 'comments': [
+ (r'(?s)/\*.*?\*/', Comment.Multiline),
+ # Comments
+ (r'(//|#).*$', Comment.Single),
+ ],
+ 'root': [
+ # Stan is more restrictive on strings than this regex
+ (r'"[^"]*"', String),
+ # Comments
+ include('comments'),
+ # block start
+ include('whitespace'),
+ # Block start
+ (r'(%s)(\s*)(\{)' %
+ r'|'.join(('functions', 'data', r'transformed\s+?data',
+ 'parameters', r'transformed\s+parameters',
+ 'model', r'generated\s+quantities')),
+ bygroups(Keyword.Namespace, Text, Punctuation)),
+ # Reserved Words
+ (r'(%s)\b' % r'|'.join(_stan_builtins.KEYWORDS), Keyword),
+ # Truncation
+ (r'T(?=\s*\[)', Keyword),
+ # Data types
+ (r'(%s)\b' % r'|'.join(_stan_builtins.TYPES), Keyword.Type),
+ # Punctuation
+ (r"[;:,\[\]()]", Punctuation),
+ # Builtin
+ (r'(%s)(?=\s*\()'
+ % r'|'.join(_stan_builtins.FUNCTIONS
+ + _stan_builtins.DISTRIBUTIONS),
+ Name.Builtin),
+ # Special names ending in __, like lp__
+ (r'[A-Za-z]\w*__\b', Name.Builtin.Pseudo),
+ (r'(%s)\b' % r'|'.join(_stan_builtins.RESERVED), Keyword.Reserved),
+ # user-defined functions
+ (r'[A-Za-z]\w*(?=\s*\()]', Name.Function),
+ # Regular variable names
+ (r'[A-Za-z]\w*\b', Name),
+ # Real Literals
+ (r'-?[0-9]+(\.[0-9]+)?[eE]-?[0-9]+', Number.Float),
+ (r'-?[0-9]*\.[0-9]*', Number.Float),
+ # Integer Literals
+ (r'-?[0-9]+', Number.Integer),
+ # Assignment operators
+ # SLexer makes these tokens Operators.
+ (r'<-|~', Operator),
+ # Infix, prefix and postfix operators (and = )
+ (r"\+|-|\.?\*|\.?/|\\|'|\^|==?|!=?|<=?|>=?|\|\||&&", Operator),
+ # Block delimiters
+ (r'[{}]', Punctuation),
+ ]
+ }
+
+ def analyse_text(text):
+ if re.search(r'^\s*parameters\s*\{', text, re.M):
+ return 1.0
+ else:
+ return 0.0
diff --git a/wandb/vendor/pygments/lexers/modula2.py b/wandb/vendor/pygments/lexers/modula2.py
new file mode 100644
index 0000000000000000000000000000000000000000..c0a69b404f8626562845a466ea18d95d69ba6a9c
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/modula2.py
@@ -0,0 +1,1561 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.modula2
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Multi-Dialect Lexer for Modula-2.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include
+from pygments.util import get_bool_opt, get_list_opt
+from pygments.token import Text, Comment, Operator, Keyword, Name, \
+ String, Number, Punctuation, Error
+
+__all__ = ['Modula2Lexer']
+
+
+# Multi-Dialect Modula-2 Lexer
+class Modula2Lexer(RegexLexer):
+ """
+ For `Modula-2 <http://www.modula2.org/>`_ source code.
+
+ The Modula-2 lexer supports several dialects. By default, it operates in
+ fallback mode, recognising the *combined* literals, punctuation symbols
+ and operators of all supported dialects, and the *combined* reserved words
+ and builtins of PIM Modula-2, ISO Modula-2 and Modula-2 R10, while not
+ differentiating between library defined identifiers.
+
+ To select a specific dialect, a dialect option may be passed
+ or a dialect tag may be embedded into a source file.
+
+ Dialect Options:
+
+ `m2pim`
+ Select PIM Modula-2 dialect.
+ `m2iso`
+ Select ISO Modula-2 dialect.
+ `m2r10`
+ Select Modula-2 R10 dialect.
+ `objm2`
+ Select Objective Modula-2 dialect.
+
+ The PIM and ISO dialect options may be qualified with a language extension.
+
+ Language Extensions:
+
+ `+aglet`
+ Select Aglet Modula-2 extensions, available with m2iso.
+ `+gm2`
+ Select GNU Modula-2 extensions, available with m2pim.
+ `+p1`
+ Select p1 Modula-2 extensions, available with m2iso.
+ `+xds`
+ Select XDS Modula-2 extensions, available with m2iso.
+
+
+ Passing a Dialect Option via Unix Commandline Interface
+
+ Dialect options may be passed to the lexer using the `dialect` key.
+ Only one such option should be passed. If multiple dialect options are
+ passed, the first valid option is used, any subsequent options are ignored.
+
+ Examples:
+
+ `$ pygmentize -O full,dialect=m2iso -f html -o /path/to/output /path/to/input`
+ Use ISO dialect to render input to HTML output
+ `$ pygmentize -O full,dialect=m2iso+p1 -f rtf -o /path/to/output /path/to/input`
+ Use ISO dialect with p1 extensions to render input to RTF output
+
+
+ Embedding a Dialect Option within a source file
+
+ A dialect option may be embedded in a source file in form of a dialect
+ tag, a specially formatted comment that specifies a dialect option.
+
+ Dialect Tag EBNF::
+
+ dialectTag :
+ OpeningCommentDelim Prefix dialectOption ClosingCommentDelim ;
+
+ dialectOption :
+ 'm2pim' | 'm2iso' | 'm2r10' | 'objm2' |
+ 'm2iso+aglet' | 'm2pim+gm2' | 'm2iso+p1' | 'm2iso+xds' ;
+
+ Prefix : '!' ;
+
+ OpeningCommentDelim : '(*' ;
+
+ ClosingCommentDelim : '*)' ;
+
+ No whitespace is permitted between the tokens of a dialect tag.
+
+ In the event that a source file contains multiple dialect tags, the first
+ tag that contains a valid dialect option will be used and any subsequent
+ dialect tags will be ignored. Ideally, a dialect tag should be placed
+ at the beginning of a source file.
+
+ An embedded dialect tag overrides a dialect option set via command line.
+
+ Examples:
+
+ ``(*!m2r10*) DEFINITION MODULE Foobar; ...``
+ Use Modula2 R10 dialect to render this source file.
+ ``(*!m2pim+gm2*) DEFINITION MODULE Bazbam; ...``
+ Use PIM dialect with GNU extensions to render this source file.
+
+
+ Algol Publication Mode:
+
+ In Algol publication mode, source text is rendered for publication of
+ algorithms in scientific papers and academic texts, following the format
+ of the Revised Algol-60 Language Report. It is activated by passing
+ one of two corresponding styles as an option:
+
+ `algol`
+ render reserved words lowercase underline boldface
+ and builtins lowercase boldface italic
+ `algol_nu`
+ render reserved words lowercase boldface (no underlining)
+ and builtins lowercase boldface italic
+
+ The lexer automatically performs the required lowercase conversion when
+ this mode is activated.
+
+ Example:
+
+ ``$ pygmentize -O full,style=algol -f latex -o /path/to/output /path/to/input``
+ Render input file in Algol publication mode to LaTeX output.
+
+
+ Rendering Mode of First Class ADT Identifiers:
+
+ The rendering of standard library first class ADT identifiers is controlled
+ by option flag "treat_stdlib_adts_as_builtins".
+
+ When this option is turned on, standard library ADT identifiers are rendered
+ as builtins. When it is turned off, they are rendered as ordinary library
+ identifiers.
+
+ `treat_stdlib_adts_as_builtins` (default: On)
+
+ The option is useful for dialects that support ADTs as first class objects
+ and provide ADTs in the standard library that would otherwise be built-in.
+
+ At present, only Modula-2 R10 supports library ADTs as first class objects
+ and therefore, no ADT identifiers are defined for any other dialects.
+
+ Example:
+
+ ``$ pygmentize -O full,dialect=m2r10,treat_stdlib_adts_as_builtins=Off ...``
+ Render standard library ADTs as ordinary library types.
+
+ .. versionadded:: 1.3
+
+ .. versionchanged:: 2.1
+ Added multi-dialect support.
+ """
+ name = 'Modula-2'
+ aliases = ['modula2', 'm2']
+ filenames = ['*.def', '*.mod']
+ mimetypes = ['text/x-modula2']
+
+ flags = re.MULTILINE | re.DOTALL
+
+ tokens = {
+ 'whitespace': [
+ (r'\n+', Text), # blank lines
+ (r'\s+', Text), # whitespace
+ ],
+ 'dialecttags': [
+ # PIM Dialect Tag
+ (r'\(\*!m2pim\*\)', Comment.Special),
+ # ISO Dialect Tag
+ (r'\(\*!m2iso\*\)', Comment.Special),
+ # M2R10 Dialect Tag
+ (r'\(\*!m2r10\*\)', Comment.Special),
+ # ObjM2 Dialect Tag
+ (r'\(\*!objm2\*\)', Comment.Special),
+ # Aglet Extensions Dialect Tag
+ (r'\(\*!m2iso\+aglet\*\)', Comment.Special),
+ # GNU Extensions Dialect Tag
+ (r'\(\*!m2pim\+gm2\*\)', Comment.Special),
+ # p1 Extensions Dialect Tag
+ (r'\(\*!m2iso\+p1\*\)', Comment.Special),
+ # XDS Extensions Dialect Tag
+ (r'\(\*!m2iso\+xds\*\)', Comment.Special),
+ ],
+ 'identifiers': [
+ (r'([a-zA-Z_$][\w$]*)', Name),
+ ],
+ 'prefixed_number_literals': [
+ #
+ # Base-2, whole number
+ (r'0b[01]+(\'[01]+)*', Number.Bin),
+ #
+ # Base-16, whole number
+ (r'0[ux][0-9A-F]+(\'[0-9A-F]+)*', Number.Hex),
+ ],
+ 'plain_number_literals': [
+ #
+ # Base-10, real number with exponent
+ (r'[0-9]+(\'[0-9]+)*' # integral part
+ r'\.[0-9]+(\'[0-9]+)*' # fractional part
+ r'[eE][+-]?[0-9]+(\'[0-9]+)*', # exponent
+ Number.Float),
+ #
+ # Base-10, real number without exponent
+ (r'[0-9]+(\'[0-9]+)*' # integral part
+ r'\.[0-9]+(\'[0-9]+)*', # fractional part
+ Number.Float),
+ #
+ # Base-10, whole number
+ (r'[0-9]+(\'[0-9]+)*', Number.Integer),
+ ],
+ 'suffixed_number_literals': [
+ #
+ # Base-8, whole number
+ (r'[0-7]+B', Number.Oct),
+ #
+ # Base-8, character code
+ (r'[0-7]+C', Number.Oct),
+ #
+ # Base-16, number
+ (r'[0-9A-F]+H', Number.Hex),
+ ],
+ 'string_literals': [
+ (r"'(\\\\|\\'|[^'])*'", String), # single quoted string
+ (r'"(\\\\|\\"|[^"])*"', String), # double quoted string
+ ],
+ 'digraph_operators': [
+ # Dot Product Operator
+ (r'\*\.', Operator),
+ # Array Concatenation Operator
+ (r'\+>', Operator), # M2R10 + ObjM2
+ # Inequality Operator
+ (r'<>', Operator), # ISO + PIM
+ # Less-Or-Equal, Subset
+ (r'<=', Operator),
+ # Greater-Or-Equal, Superset
+ (r'>=', Operator),
+ # Identity Operator
+ (r'==', Operator), # M2R10 + ObjM2
+ # Type Conversion Operator
+ (r'::', Operator), # M2R10 + ObjM2
+ # Assignment Symbol
+ (r':=', Operator),
+ # Postfix Increment Mutator
+ (r'\+\+', Operator), # M2R10 + ObjM2
+ # Postfix Decrement Mutator
+ (r'--', Operator), # M2R10 + ObjM2
+ ],
+ 'unigraph_operators': [
+ # Arithmetic Operators
+ (r'[+-]', Operator),
+ (r'[*/]', Operator),
+ # ISO 80000-2 compliant Set Difference Operator
+ (r'\\', Operator), # M2R10 + ObjM2
+ # Relational Operators
+ (r'[=#<>]', Operator),
+ # Dereferencing Operator
+ (r'\^', Operator),
+ # Dereferencing Operator Synonym
+ (r'@', Operator), # ISO
+ # Logical AND Operator Synonym
+ (r'&', Operator), # PIM + ISO
+ # Logical NOT Operator Synonym
+ (r'~', Operator), # PIM + ISO
+ # Smalltalk Message Prefix
+ (r'`', Operator), # ObjM2
+ ],
+ 'digraph_punctuation': [
+ # Range Constructor
+ (r'\.\.', Punctuation),
+ # Opening Chevron Bracket
+ (r'<<', Punctuation), # M2R10 + ISO
+ # Closing Chevron Bracket
+ (r'>>', Punctuation), # M2R10 + ISO
+ # Blueprint Punctuation
+ (r'->', Punctuation), # M2R10 + ISO
+ # Distinguish |# and # in M2 R10
+ (r'\|#', Punctuation),
+ # Distinguish ## and # in M2 R10
+ (r'##', Punctuation),
+ # Distinguish |* and * in M2 R10
+ (r'\|\*', Punctuation),
+ ],
+ 'unigraph_punctuation': [
+ # Common Punctuation
+ (r'[()\[\]{},.:;|]', Punctuation),
+ # Case Label Separator Synonym
+ (r'!', Punctuation), # ISO
+ # Blueprint Punctuation
+ (r'\?', Punctuation), # M2R10 + ObjM2
+ ],
+ 'comments': [
+ # Single Line Comment
+ (r'^//.*?\n', Comment.Single), # M2R10 + ObjM2
+ # Block Comment
+ (r'\(\*([^$].*?)\*\)', Comment.Multiline),
+ # Template Block Comment
+ (r'/\*(.*?)\*/', Comment.Multiline), # M2R10 + ObjM2
+ ],
+ 'pragmas': [
+ # ISO Style Pragmas
+ (r'<\*.*?\*>', Comment.Preproc), # ISO, M2R10 + ObjM2
+ # Pascal Style Pragmas
+ (r'\(\*\$.*?\*\)', Comment.Preproc), # PIM
+ ],
+ 'root': [
+ include('whitespace'),
+ include('dialecttags'),
+ include('pragmas'),
+ include('comments'),
+ include('identifiers'),
+ include('suffixed_number_literals'), # PIM + ISO
+ include('prefixed_number_literals'), # M2R10 + ObjM2
+ include('plain_number_literals'),
+ include('string_literals'),
+ include('digraph_punctuation'),
+ include('digraph_operators'),
+ include('unigraph_punctuation'),
+ include('unigraph_operators'),
+ ]
+ }
+
+# C o m m o n D a t a s e t s
+
+ # Common Reserved Words Dataset
+ common_reserved_words = (
+ # 37 common reserved words
+ 'AND', 'ARRAY', 'BEGIN', 'BY', 'CASE', 'CONST', 'DEFINITION', 'DIV',
+ 'DO', 'ELSE', 'ELSIF', 'END', 'EXIT', 'FOR', 'FROM', 'IF',
+ 'IMPLEMENTATION', 'IMPORT', 'IN', 'LOOP', 'MOD', 'MODULE', 'NOT',
+ 'OF', 'OR', 'POINTER', 'PROCEDURE', 'RECORD', 'REPEAT', 'RETURN',
+ 'SET', 'THEN', 'TO', 'TYPE', 'UNTIL', 'VAR', 'WHILE',
+ )
+
+ # Common Builtins Dataset
+ common_builtins = (
+ # 16 common builtins
+ 'ABS', 'BOOLEAN', 'CARDINAL', 'CHAR', 'CHR', 'FALSE', 'INTEGER',
+ 'LONGINT', 'LONGREAL', 'MAX', 'MIN', 'NIL', 'ODD', 'ORD', 'REAL',
+ 'TRUE',
+ )
+
+ # Common Pseudo-Module Builtins Dataset
+ common_pseudo_builtins = (
+ # 4 common pseudo builtins
+ 'ADDRESS', 'BYTE', 'WORD', 'ADR'
+ )
+
+# P I M M o d u l a - 2 D a t a s e t s
+
+ # Lexemes to Mark as Error Tokens for PIM Modula-2
+ pim_lexemes_to_reject = (
+ '!', '`', '@', '$', '%', '?', '\\', '==', '++', '--', '::', '*.',
+ '+>', '->', '<<', '>>', '|#', '##',
+ )
+
+ # PIM Modula-2 Additional Reserved Words Dataset
+ pim_additional_reserved_words = (
+ # 3 additional reserved words
+ 'EXPORT', 'QUALIFIED', 'WITH',
+ )
+
+ # PIM Modula-2 Additional Builtins Dataset
+ pim_additional_builtins = (
+ # 16 additional builtins
+ 'BITSET', 'CAP', 'DEC', 'DISPOSE', 'EXCL', 'FLOAT', 'HALT', 'HIGH',
+ 'INC', 'INCL', 'NEW', 'NIL', 'PROC', 'SIZE', 'TRUNC', 'VAL',
+ )
+
+ # PIM Modula-2 Additional Pseudo-Module Builtins Dataset
+ pim_additional_pseudo_builtins = (
+ # 5 additional pseudo builtins
+ 'SYSTEM', 'PROCESS', 'TSIZE', 'NEWPROCESS', 'TRANSFER',
+ )
+
+# I S O M o d u l a - 2 D a t a s e t s
+
+ # Lexemes to Mark as Error Tokens for ISO Modula-2
+ iso_lexemes_to_reject = (
+ '`', '$', '%', '?', '\\', '==', '++', '--', '::', '*.', '+>', '->',
+ '<<', '>>', '|#', '##',
+ )
+
+ # ISO Modula-2 Additional Reserved Words Dataset
+ iso_additional_reserved_words = (
+ # 9 additional reserved words (ISO 10514-1)
+ 'EXCEPT', 'EXPORT', 'FINALLY', 'FORWARD', 'PACKEDSET', 'QUALIFIED',
+ 'REM', 'RETRY', 'WITH',
+ # 10 additional reserved words (ISO 10514-2 & ISO 10514-3)
+ 'ABSTRACT', 'AS', 'CLASS', 'GUARD', 'INHERIT', 'OVERRIDE', 'READONLY',
+ 'REVEAL', 'TRACED', 'UNSAFEGUARDED',
+ )
+
+ # ISO Modula-2 Additional Builtins Dataset
+ iso_additional_builtins = (
+ # 26 additional builtins (ISO 10514-1)
+ 'BITSET', 'CAP', 'CMPLX', 'COMPLEX', 'DEC', 'DISPOSE', 'EXCL', 'FLOAT',
+ 'HALT', 'HIGH', 'IM', 'INC', 'INCL', 'INT', 'INTERRUPTIBLE', 'LENGTH',
+ 'LFLOAT', 'LONGCOMPLEX', 'NEW', 'PROC', 'PROTECTION', 'RE', 'SIZE',
+ 'TRUNC', 'UNINTERRUBTIBLE', 'VAL',
+ # 5 additional builtins (ISO 10514-2 & ISO 10514-3)
+ 'CREATE', 'DESTROY', 'EMPTY', 'ISMEMBER', 'SELF',
+ )
+
+ # ISO Modula-2 Additional Pseudo-Module Builtins Dataset
+ iso_additional_pseudo_builtins = (
+ # 14 additional builtins (SYSTEM)
+ 'SYSTEM', 'BITSPERLOC', 'LOCSPERBYTE', 'LOCSPERWORD', 'LOC',
+ 'ADDADR', 'SUBADR', 'DIFADR', 'MAKEADR', 'ADR',
+ 'ROTATE', 'SHIFT', 'CAST', 'TSIZE',
+ # 13 additional builtins (COROUTINES)
+ 'COROUTINES', 'ATTACH', 'COROUTINE', 'CURRENT', 'DETACH', 'HANDLER',
+ 'INTERRUPTSOURCE', 'IOTRANSFER', 'IsATTACHED', 'LISTEN',
+ 'NEWCOROUTINE', 'PROT', 'TRANSFER',
+ # 9 additional builtins (EXCEPTIONS)
+ 'EXCEPTIONS', 'AllocateSource', 'CurrentNumber', 'ExceptionNumber',
+ 'ExceptionSource', 'GetMessage', 'IsCurrentSource',
+ 'IsExceptionalExecution', 'RAISE',
+ # 3 additional builtins (TERMINATION)
+ 'TERMINATION', 'IsTerminating', 'HasHalted',
+ # 4 additional builtins (M2EXCEPTION)
+ 'M2EXCEPTION', 'M2Exceptions', 'M2Exception', 'IsM2Exception',
+ 'indexException', 'rangeException', 'caseSelectException',
+ 'invalidLocation', 'functionException', 'wholeValueException',
+ 'wholeDivException', 'realValueException', 'realDivException',
+ 'complexValueException', 'complexDivException', 'protException',
+ 'sysException', 'coException', 'exException',
+ )
+
+# M o d u l a - 2 R 1 0 D a t a s e t s
+
+ # Lexemes to Mark as Error Tokens for Modula-2 R10
+ m2r10_lexemes_to_reject = (
+ '!', '`', '@', '$', '%', '&', '<>',
+ )
+
+ # Modula-2 R10 reserved words in addition to the common set
+ m2r10_additional_reserved_words = (
+ # 12 additional reserved words
+ 'ALIAS', 'ARGLIST', 'BLUEPRINT', 'COPY', 'GENLIB', 'INDETERMINATE',
+ 'NEW', 'NONE', 'OPAQUE', 'REFERENTIAL', 'RELEASE', 'RETAIN',
+ # 2 additional reserved words with symbolic assembly option
+ 'ASM', 'REG',
+ )
+
+ # Modula-2 R10 builtins in addition to the common set
+ m2r10_additional_builtins = (
+ # 26 additional builtins
+ 'CARDINAL', 'COUNT', 'EMPTY', 'EXISTS', 'INSERT', 'LENGTH', 'LONGCARD',
+ 'OCTET', 'PTR', 'PRED', 'READ', 'READNEW', 'REMOVE', 'RETRIEVE', 'SORT',
+ 'STORE', 'SUBSET', 'SUCC', 'TLIMIT', 'TMAX', 'TMIN', 'TRUE', 'TSIZE',
+ 'UNICHAR', 'WRITE', 'WRITEF',
+ )
+
+ # Modula-2 R10 Additional Pseudo-Module Builtins Dataset
+ m2r10_additional_pseudo_builtins = (
+ # 13 additional builtins (TPROPERTIES)
+ 'TPROPERTIES', 'PROPERTY', 'LITERAL', 'TPROPERTY', 'TLITERAL',
+ 'TBUILTIN', 'TDYN', 'TREFC', 'TNIL', 'TBASE', 'TPRECISION',
+ 'TMAXEXP', 'TMINEXP',
+ # 4 additional builtins (CONVERSION)
+ 'CONVERSION', 'TSXFSIZE', 'SXF', 'VAL',
+ # 35 additional builtins (UNSAFE)
+ 'UNSAFE', 'CAST', 'INTRINSIC', 'AVAIL', 'ADD', 'SUB', 'ADDC', 'SUBC',
+ 'FETCHADD', 'FETCHSUB', 'SHL', 'SHR', 'ASHR', 'ROTL', 'ROTR', 'ROTLC',
+ 'ROTRC', 'BWNOT', 'BWAND', 'BWOR', 'BWXOR', 'BWNAND', 'BWNOR',
+ 'SETBIT', 'TESTBIT', 'LSBIT', 'MSBIT', 'CSBITS', 'BAIL', 'HALT',
+ 'TODO', 'FFI', 'ADDR', 'VARGLIST', 'VARGC',
+ # 11 additional builtins (ATOMIC)
+ 'ATOMIC', 'INTRINSIC', 'AVAIL', 'SWAP', 'CAS', 'INC', 'DEC', 'BWAND',
+ 'BWNAND', 'BWOR', 'BWXOR',
+ # 7 additional builtins (COMPILER)
+ 'COMPILER', 'DEBUG', 'MODNAME', 'PROCNAME', 'LINENUM', 'DEFAULT',
+ 'HASH',
+ # 5 additional builtins (ASSEMBLER)
+ 'ASSEMBLER', 'REGISTER', 'SETREG', 'GETREG', 'CODE',
+ )
+
+# O b j e c t i v e M o d u l a - 2 D a t a s e t s
+
+ # Lexemes to Mark as Error Tokens for Objective Modula-2
+ objm2_lexemes_to_reject = (
+ '!', '$', '%', '&', '<>',
+ )
+
+ # Objective Modula-2 Extensions
+ # reserved words in addition to Modula-2 R10
+ objm2_additional_reserved_words = (
+ # 16 additional reserved words
+ 'BYCOPY', 'BYREF', 'CLASS', 'CONTINUE', 'CRITICAL', 'INOUT', 'METHOD',
+ 'ON', 'OPTIONAL', 'OUT', 'PRIVATE', 'PROTECTED', 'PROTOCOL', 'PUBLIC',
+ 'SUPER', 'TRY',
+ )
+
+ # Objective Modula-2 Extensions
+ # builtins in addition to Modula-2 R10
+ objm2_additional_builtins = (
+ # 3 additional builtins
+ 'OBJECT', 'NO', 'YES',
+ )
+
+ # Objective Modula-2 Extensions
+ # pseudo-module builtins in addition to Modula-2 R10
+ objm2_additional_pseudo_builtins = (
+ # None
+ )
+
+# A g l e t M o d u l a - 2 D a t a s e t s
+
+ # Aglet Extensions
+ # reserved words in addition to ISO Modula-2
+ aglet_additional_reserved_words = (
+ # None
+ )
+
+ # Aglet Extensions
+ # builtins in addition to ISO Modula-2
+ aglet_additional_builtins = (
+ # 9 additional builtins
+ 'BITSET8', 'BITSET16', 'BITSET32', 'CARDINAL8', 'CARDINAL16',
+ 'CARDINAL32', 'INTEGER8', 'INTEGER16', 'INTEGER32',
+ )
+
+ # Aglet Modula-2 Extensions
+ # pseudo-module builtins in addition to ISO Modula-2
+ aglet_additional_pseudo_builtins = (
+ # None
+ )
+
+# G N U M o d u l a - 2 D a t a s e t s
+
+ # GNU Extensions
+ # reserved words in addition to PIM Modula-2
+ gm2_additional_reserved_words = (
+ # 10 additional reserved words
+ 'ASM', '__ATTRIBUTE__', '__BUILTIN__', '__COLUMN__', '__DATE__',
+ '__FILE__', '__FUNCTION__', '__LINE__', '__MODULE__', 'VOLATILE',
+ )
+
+ # GNU Extensions
+ # builtins in addition to PIM Modula-2
+ gm2_additional_builtins = (
+ # 21 additional builtins
+ 'BITSET8', 'BITSET16', 'BITSET32', 'CARDINAL8', 'CARDINAL16',
+ 'CARDINAL32', 'CARDINAL64', 'COMPLEX32', 'COMPLEX64', 'COMPLEX96',
+ 'COMPLEX128', 'INTEGER8', 'INTEGER16', 'INTEGER32', 'INTEGER64',
+ 'REAL8', 'REAL16', 'REAL32', 'REAL96', 'REAL128', 'THROW',
+ )
+
+ # GNU Extensions
+ # pseudo-module builtins in addition to PIM Modula-2
+ gm2_additional_pseudo_builtins = (
+ # None
+ )
+
+# p 1 M o d u l a - 2 D a t a s e t s
+
+ # p1 Extensions
+ # reserved words in addition to ISO Modula-2
+ p1_additional_reserved_words = (
+ # None
+ )
+
+ # p1 Extensions
+ # builtins in addition to ISO Modula-2
+ p1_additional_builtins = (
+ # None
+ )
+
+ # p1 Modula-2 Extensions
+ # pseudo-module builtins in addition to ISO Modula-2
+ p1_additional_pseudo_builtins = (
+ # 1 additional builtin
+ 'BCD',
+ )
+
+# X D S M o d u l a - 2 D a t a s e t s
+
+ # XDS Extensions
+ # reserved words in addition to ISO Modula-2
+ xds_additional_reserved_words = (
+ # 1 additional reserved word
+ 'SEQ',
+ )
+
+ # XDS Extensions
+ # builtins in addition to ISO Modula-2
+ xds_additional_builtins = (
+ # 9 additional builtins
+ 'ASH', 'ASSERT', 'DIFFADR_TYPE', 'ENTIER', 'INDEX', 'LEN',
+ 'LONGCARD', 'SHORTCARD', 'SHORTINT',
+ )
+
+ # XDS Modula-2 Extensions
+ # pseudo-module builtins in addition to ISO Modula-2
+ xds_additional_pseudo_builtins = (
+ # 22 additional builtins (SYSTEM)
+ 'PROCESS', 'NEWPROCESS', 'BOOL8', 'BOOL16', 'BOOL32', 'CARD8',
+ 'CARD16', 'CARD32', 'INT8', 'INT16', 'INT32', 'REF', 'MOVE',
+ 'FILL', 'GET', 'PUT', 'CC', 'int', 'unsigned', 'size_t', 'void'
+ # 3 additional builtins (COMPILER)
+ 'COMPILER', 'OPTION', 'EQUATION'
+ )
+
+# P I M S t a n d a r d L i b r a r y D a t a s e t s
+
+ # PIM Modula-2 Standard Library Modules Dataset
+ pim_stdlib_module_identifiers = (
+ 'Terminal', 'FileSystem', 'InOut', 'RealInOut', 'MathLib0', 'Storage',
+ )
+
+ # PIM Modula-2 Standard Library Types Dataset
+ pim_stdlib_type_identifiers = (
+ 'Flag', 'FlagSet', 'Response', 'Command', 'Lock', 'Permission',
+ 'MediumType', 'File', 'FileProc', 'DirectoryProc', 'FileCommand',
+ 'DirectoryCommand',
+ )
+
+ # PIM Modula-2 Standard Library Procedures Dataset
+ pim_stdlib_proc_identifiers = (
+ 'Read', 'BusyRead', 'ReadAgain', 'Write', 'WriteString', 'WriteLn',
+ 'Create', 'Lookup', 'Close', 'Delete', 'Rename', 'SetRead', 'SetWrite',
+ 'SetModify', 'SetOpen', 'Doio', 'SetPos', 'GetPos', 'Length', 'Reset',
+ 'Again', 'ReadWord', 'WriteWord', 'ReadChar', 'WriteChar',
+ 'CreateMedium', 'DeleteMedium', 'AssignName', 'DeassignName',
+ 'ReadMedium', 'LookupMedium', 'OpenInput', 'OpenOutput', 'CloseInput',
+ 'CloseOutput', 'ReadString', 'ReadInt', 'ReadCard', 'ReadWrd',
+ 'WriteInt', 'WriteCard', 'WriteOct', 'WriteHex', 'WriteWrd',
+ 'ReadReal', 'WriteReal', 'WriteFixPt', 'WriteRealOct', 'sqrt', 'exp',
+ 'ln', 'sin', 'cos', 'arctan', 'entier', 'ALLOCATE', 'DEALLOCATE',
+ )
+
+ # PIM Modula-2 Standard Library Variables Dataset
+ pim_stdlib_var_identifiers = (
+ 'Done', 'termCH', 'in', 'out'
+ )
+
+ # PIM Modula-2 Standard Library Constants Dataset
+ pim_stdlib_const_identifiers = (
+ 'EOL',
+ )
+
+# I S O S t a n d a r d L i b r a r y D a t a s e t s
+
+ # ISO Modula-2 Standard Library Modules Dataset
+ iso_stdlib_module_identifiers = (
+ # TO DO
+ )
+
+ # ISO Modula-2 Standard Library Types Dataset
+ iso_stdlib_type_identifiers = (
+ # TO DO
+ )
+
+ # ISO Modula-2 Standard Library Procedures Dataset
+ iso_stdlib_proc_identifiers = (
+ # TO DO
+ )
+
+ # ISO Modula-2 Standard Library Variables Dataset
+ iso_stdlib_var_identifiers = (
+ # TO DO
+ )
+
+ # ISO Modula-2 Standard Library Constants Dataset
+ iso_stdlib_const_identifiers = (
+ # TO DO
+ )
+
+# M 2 R 1 0 S t a n d a r d L i b r a r y D a t a s e t s
+
+ # Modula-2 R10 Standard Library ADTs Dataset
+ m2r10_stdlib_adt_identifiers = (
+ 'BCD', 'LONGBCD', 'BITSET', 'SHORTBITSET', 'LONGBITSET',
+ 'LONGLONGBITSET', 'COMPLEX', 'LONGCOMPLEX', 'SHORTCARD', 'LONGLONGCARD',
+ 'SHORTINT', 'LONGLONGINT', 'POSINT', 'SHORTPOSINT', 'LONGPOSINT',
+ 'LONGLONGPOSINT', 'BITSET8', 'BITSET16', 'BITSET32', 'BITSET64',
+ 'BITSET128', 'BS8', 'BS16', 'BS32', 'BS64', 'BS128', 'CARDINAL8',
+ 'CARDINAL16', 'CARDINAL32', 'CARDINAL64', 'CARDINAL128', 'CARD8',
+ 'CARD16', 'CARD32', 'CARD64', 'CARD128', 'INTEGER8', 'INTEGER16',
+ 'INTEGER32', 'INTEGER64', 'INTEGER128', 'INT8', 'INT16', 'INT32',
+ 'INT64', 'INT128', 'STRING', 'UNISTRING',
+ )
+
+ # Modula-2 R10 Standard Library Blueprints Dataset
+ m2r10_stdlib_blueprint_identifiers = (
+ 'ProtoRoot', 'ProtoComputational', 'ProtoNumeric', 'ProtoScalar',
+ 'ProtoNonScalar', 'ProtoCardinal', 'ProtoInteger', 'ProtoReal',
+ 'ProtoComplex', 'ProtoVector', 'ProtoTuple', 'ProtoCompArray',
+ 'ProtoCollection', 'ProtoStaticArray', 'ProtoStaticSet',
+ 'ProtoStaticString', 'ProtoArray', 'ProtoString', 'ProtoSet',
+ 'ProtoMultiSet', 'ProtoDictionary', 'ProtoMultiDict', 'ProtoExtension',
+ 'ProtoIO', 'ProtoCardMath', 'ProtoIntMath', 'ProtoRealMath',
+ )
+
+ # Modula-2 R10 Standard Library Modules Dataset
+ m2r10_stdlib_module_identifiers = (
+ 'ASCII', 'BooleanIO', 'CharIO', 'UnicharIO', 'OctetIO',
+ 'CardinalIO', 'LongCardIO', 'IntegerIO', 'LongIntIO', 'RealIO',
+ 'LongRealIO', 'BCDIO', 'LongBCDIO', 'CardMath', 'LongCardMath',
+ 'IntMath', 'LongIntMath', 'RealMath', 'LongRealMath', 'BCDMath',
+ 'LongBCDMath', 'FileIO', 'FileSystem', 'Storage', 'IOSupport',
+ )
+
+ # Modula-2 R10 Standard Library Types Dataset
+ m2r10_stdlib_type_identifiers = (
+ 'File', 'Status',
+ # TO BE COMPLETED
+ )
+
+ # Modula-2 R10 Standard Library Procedures Dataset
+ m2r10_stdlib_proc_identifiers = (
+ 'ALLOCATE', 'DEALLOCATE', 'SIZE',
+ # TO BE COMPLETED
+ )
+
+ # Modula-2 R10 Standard Library Variables Dataset
+ m2r10_stdlib_var_identifiers = (
+ 'stdIn', 'stdOut', 'stdErr',
+ )
+
+ # Modula-2 R10 Standard Library Constants Dataset
+ m2r10_stdlib_const_identifiers = (
+ 'pi', 'tau',
+ )
+
+# D i a l e c t s
+
+ # Dialect modes
+ dialects = (
+ 'unknown',
+ 'm2pim', 'm2iso', 'm2r10', 'objm2',
+ 'm2iso+aglet', 'm2pim+gm2', 'm2iso+p1', 'm2iso+xds',
+ )
+
+# D a t a b a s e s
+
+ # Lexemes to Mark as Errors Database
+ lexemes_to_reject_db = {
+ # Lexemes to reject for unknown dialect
+ 'unknown': (
+ # LEAVE THIS EMPTY
+ ),
+ # Lexemes to reject for PIM Modula-2
+ 'm2pim': (
+ pim_lexemes_to_reject,
+ ),
+ # Lexemes to reject for ISO Modula-2
+ 'm2iso': (
+ iso_lexemes_to_reject,
+ ),
+ # Lexemes to reject for Modula-2 R10
+ 'm2r10': (
+ m2r10_lexemes_to_reject,
+ ),
+ # Lexemes to reject for Objective Modula-2
+ 'objm2': (
+ objm2_lexemes_to_reject,
+ ),
+ # Lexemes to reject for Aglet Modula-2
+ 'm2iso+aglet': (
+ iso_lexemes_to_reject,
+ ),
+ # Lexemes to reject for GNU Modula-2
+ 'm2pim+gm2': (
+ pim_lexemes_to_reject,
+ ),
+ # Lexemes to reject for p1 Modula-2
+ 'm2iso+p1': (
+ iso_lexemes_to_reject,
+ ),
+ # Lexemes to reject for XDS Modula-2
+ 'm2iso+xds': (
+ iso_lexemes_to_reject,
+ ),
+ }
+
+ # Reserved Words Database
+ reserved_words_db = {
+ # Reserved words for unknown dialect
+ 'unknown': (
+ common_reserved_words,
+ pim_additional_reserved_words,
+ iso_additional_reserved_words,
+ m2r10_additional_reserved_words,
+ ),
+
+ # Reserved words for PIM Modula-2
+ 'm2pim': (
+ common_reserved_words,
+ pim_additional_reserved_words,
+ ),
+
+ # Reserved words for Modula-2 R10
+ 'm2iso': (
+ common_reserved_words,
+ iso_additional_reserved_words,
+ ),
+
+ # Reserved words for ISO Modula-2
+ 'm2r10': (
+ common_reserved_words,
+ m2r10_additional_reserved_words,
+ ),
+
+ # Reserved words for Objective Modula-2
+ 'objm2': (
+ common_reserved_words,
+ m2r10_additional_reserved_words,
+ objm2_additional_reserved_words,
+ ),
+
+ # Reserved words for Aglet Modula-2 Extensions
+ 'm2iso+aglet': (
+ common_reserved_words,
+ iso_additional_reserved_words,
+ aglet_additional_reserved_words,
+ ),
+
+ # Reserved words for GNU Modula-2 Extensions
+ 'm2pim+gm2': (
+ common_reserved_words,
+ pim_additional_reserved_words,
+ gm2_additional_reserved_words,
+ ),
+
+ # Reserved words for p1 Modula-2 Extensions
+ 'm2iso+p1': (
+ common_reserved_words,
+ iso_additional_reserved_words,
+ p1_additional_reserved_words,
+ ),
+
+ # Reserved words for XDS Modula-2 Extensions
+ 'm2iso+xds': (
+ common_reserved_words,
+ iso_additional_reserved_words,
+ xds_additional_reserved_words,
+ ),
+ }
+
+ # Builtins Database
+ builtins_db = {
+ # Builtins for unknown dialect
+ 'unknown': (
+ common_builtins,
+ pim_additional_builtins,
+ iso_additional_builtins,
+ m2r10_additional_builtins,
+ ),
+
+ # Builtins for PIM Modula-2
+ 'm2pim': (
+ common_builtins,
+ pim_additional_builtins,
+ ),
+
+ # Builtins for ISO Modula-2
+ 'm2iso': (
+ common_builtins,
+ iso_additional_builtins,
+ ),
+
+ # Builtins for ISO Modula-2
+ 'm2r10': (
+ common_builtins,
+ m2r10_additional_builtins,
+ ),
+
+ # Builtins for Objective Modula-2
+ 'objm2': (
+ common_builtins,
+ m2r10_additional_builtins,
+ objm2_additional_builtins,
+ ),
+
+ # Builtins for Aglet Modula-2 Extensions
+ 'm2iso+aglet': (
+ common_builtins,
+ iso_additional_builtins,
+ aglet_additional_builtins,
+ ),
+
+ # Builtins for GNU Modula-2 Extensions
+ 'm2pim+gm2': (
+ common_builtins,
+ pim_additional_builtins,
+ gm2_additional_builtins,
+ ),
+
+ # Builtins for p1 Modula-2 Extensions
+ 'm2iso+p1': (
+ common_builtins,
+ iso_additional_builtins,
+ p1_additional_builtins,
+ ),
+
+ # Builtins for XDS Modula-2 Extensions
+ 'm2iso+xds': (
+ common_builtins,
+ iso_additional_builtins,
+ xds_additional_builtins,
+ ),
+ }
+
+ # Pseudo-Module Builtins Database
+ pseudo_builtins_db = {
+ # Builtins for unknown dialect
+ 'unknown': (
+ common_pseudo_builtins,
+ pim_additional_pseudo_builtins,
+ iso_additional_pseudo_builtins,
+ m2r10_additional_pseudo_builtins,
+ ),
+
+ # Builtins for PIM Modula-2
+ 'm2pim': (
+ common_pseudo_builtins,
+ pim_additional_pseudo_builtins,
+ ),
+
+ # Builtins for ISO Modula-2
+ 'm2iso': (
+ common_pseudo_builtins,
+ iso_additional_pseudo_builtins,
+ ),
+
+ # Builtins for ISO Modula-2
+ 'm2r10': (
+ common_pseudo_builtins,
+ m2r10_additional_pseudo_builtins,
+ ),
+
+ # Builtins for Objective Modula-2
+ 'objm2': (
+ common_pseudo_builtins,
+ m2r10_additional_pseudo_builtins,
+ objm2_additional_pseudo_builtins,
+ ),
+
+ # Builtins for Aglet Modula-2 Extensions
+ 'm2iso+aglet': (
+ common_pseudo_builtins,
+ iso_additional_pseudo_builtins,
+ aglet_additional_pseudo_builtins,
+ ),
+
+ # Builtins for GNU Modula-2 Extensions
+ 'm2pim+gm2': (
+ common_pseudo_builtins,
+ pim_additional_pseudo_builtins,
+ gm2_additional_pseudo_builtins,
+ ),
+
+ # Builtins for p1 Modula-2 Extensions
+ 'm2iso+p1': (
+ common_pseudo_builtins,
+ iso_additional_pseudo_builtins,
+ p1_additional_pseudo_builtins,
+ ),
+
+ # Builtins for XDS Modula-2 Extensions
+ 'm2iso+xds': (
+ common_pseudo_builtins,
+ iso_additional_pseudo_builtins,
+ xds_additional_pseudo_builtins,
+ ),
+ }
+
+ # Standard Library ADTs Database
+ stdlib_adts_db = {
+ # Empty entry for unknown dialect
+ 'unknown': (
+ # LEAVE THIS EMPTY
+ ),
+ # Standard Library ADTs for PIM Modula-2
+ 'm2pim': (
+ # No first class library types
+ ),
+
+ # Standard Library ADTs for ISO Modula-2
+ 'm2iso': (
+ # No first class library types
+ ),
+
+ # Standard Library ADTs for Modula-2 R10
+ 'm2r10': (
+ m2r10_stdlib_adt_identifiers,
+ ),
+
+ # Standard Library ADTs for Objective Modula-2
+ 'objm2': (
+ m2r10_stdlib_adt_identifiers,
+ ),
+
+ # Standard Library ADTs for Aglet Modula-2
+ 'm2iso+aglet': (
+ # No first class library types
+ ),
+
+ # Standard Library ADTs for GNU Modula-2
+ 'm2pim+gm2': (
+ # No first class library types
+ ),
+
+ # Standard Library ADTs for p1 Modula-2
+ 'm2iso+p1': (
+ # No first class library types
+ ),
+
+ # Standard Library ADTs for XDS Modula-2
+ 'm2iso+xds': (
+ # No first class library types
+ ),
+ }
+
+ # Standard Library Modules Database
+ stdlib_modules_db = {
+ # Empty entry for unknown dialect
+ 'unknown': (
+ # LEAVE THIS EMPTY
+ ),
+ # Standard Library Modules for PIM Modula-2
+ 'm2pim': (
+ pim_stdlib_module_identifiers,
+ ),
+
+ # Standard Library Modules for ISO Modula-2
+ 'm2iso': (
+ iso_stdlib_module_identifiers,
+ ),
+
+ # Standard Library Modules for Modula-2 R10
+ 'm2r10': (
+ m2r10_stdlib_blueprint_identifiers,
+ m2r10_stdlib_module_identifiers,
+ m2r10_stdlib_adt_identifiers,
+ ),
+
+ # Standard Library Modules for Objective Modula-2
+ 'objm2': (
+ m2r10_stdlib_blueprint_identifiers,
+ m2r10_stdlib_module_identifiers,
+ ),
+
+ # Standard Library Modules for Aglet Modula-2
+ 'm2iso+aglet': (
+ iso_stdlib_module_identifiers,
+ ),
+
+ # Standard Library Modules for GNU Modula-2
+ 'm2pim+gm2': (
+ pim_stdlib_module_identifiers,
+ ),
+
+ # Standard Library Modules for p1 Modula-2
+ 'm2iso+p1': (
+ iso_stdlib_module_identifiers,
+ ),
+
+ # Standard Library Modules for XDS Modula-2
+ 'm2iso+xds': (
+ iso_stdlib_module_identifiers,
+ ),
+ }
+
+ # Standard Library Types Database
+ stdlib_types_db = {
+ # Empty entry for unknown dialect
+ 'unknown': (
+ # LEAVE THIS EMPTY
+ ),
+ # Standard Library Types for PIM Modula-2
+ 'm2pim': (
+ pim_stdlib_type_identifiers,
+ ),
+
+ # Standard Library Types for ISO Modula-2
+ 'm2iso': (
+ iso_stdlib_type_identifiers,
+ ),
+
+ # Standard Library Types for Modula-2 R10
+ 'm2r10': (
+ m2r10_stdlib_type_identifiers,
+ ),
+
+ # Standard Library Types for Objective Modula-2
+ 'objm2': (
+ m2r10_stdlib_type_identifiers,
+ ),
+
+ # Standard Library Types for Aglet Modula-2
+ 'm2iso+aglet': (
+ iso_stdlib_type_identifiers,
+ ),
+
+ # Standard Library Types for GNU Modula-2
+ 'm2pim+gm2': (
+ pim_stdlib_type_identifiers,
+ ),
+
+ # Standard Library Types for p1 Modula-2
+ 'm2iso+p1': (
+ iso_stdlib_type_identifiers,
+ ),
+
+ # Standard Library Types for XDS Modula-2
+ 'm2iso+xds': (
+ iso_stdlib_type_identifiers,
+ ),
+ }
+
+ # Standard Library Procedures Database
+ stdlib_procedures_db = {
+ # Empty entry for unknown dialect
+ 'unknown': (
+ # LEAVE THIS EMPTY
+ ),
+ # Standard Library Procedures for PIM Modula-2
+ 'm2pim': (
+ pim_stdlib_proc_identifiers,
+ ),
+
+ # Standard Library Procedures for ISO Modula-2
+ 'm2iso': (
+ iso_stdlib_proc_identifiers,
+ ),
+
+ # Standard Library Procedures for Modula-2 R10
+ 'm2r10': (
+ m2r10_stdlib_proc_identifiers,
+ ),
+
+ # Standard Library Procedures for Objective Modula-2
+ 'objm2': (
+ m2r10_stdlib_proc_identifiers,
+ ),
+
+ # Standard Library Procedures for Aglet Modula-2
+ 'm2iso+aglet': (
+ iso_stdlib_proc_identifiers,
+ ),
+
+ # Standard Library Procedures for GNU Modula-2
+ 'm2pim+gm2': (
+ pim_stdlib_proc_identifiers,
+ ),
+
+ # Standard Library Procedures for p1 Modula-2
+ 'm2iso+p1': (
+ iso_stdlib_proc_identifiers,
+ ),
+
+ # Standard Library Procedures for XDS Modula-2
+ 'm2iso+xds': (
+ iso_stdlib_proc_identifiers,
+ ),
+ }
+
+ # Standard Library Variables Database
+ stdlib_variables_db = {
+ # Empty entry for unknown dialect
+ 'unknown': (
+ # LEAVE THIS EMPTY
+ ),
+ # Standard Library Variables for PIM Modula-2
+ 'm2pim': (
+ pim_stdlib_var_identifiers,
+ ),
+
+ # Standard Library Variables for ISO Modula-2
+ 'm2iso': (
+ iso_stdlib_var_identifiers,
+ ),
+
+ # Standard Library Variables for Modula-2 R10
+ 'm2r10': (
+ m2r10_stdlib_var_identifiers,
+ ),
+
+ # Standard Library Variables for Objective Modula-2
+ 'objm2': (
+ m2r10_stdlib_var_identifiers,
+ ),
+
+ # Standard Library Variables for Aglet Modula-2
+ 'm2iso+aglet': (
+ iso_stdlib_var_identifiers,
+ ),
+
+ # Standard Library Variables for GNU Modula-2
+ 'm2pim+gm2': (
+ pim_stdlib_var_identifiers,
+ ),
+
+ # Standard Library Variables for p1 Modula-2
+ 'm2iso+p1': (
+ iso_stdlib_var_identifiers,
+ ),
+
+ # Standard Library Variables for XDS Modula-2
+ 'm2iso+xds': (
+ iso_stdlib_var_identifiers,
+ ),
+ }
+
+ # Standard Library Constants Database
+ stdlib_constants_db = {
+ # Empty entry for unknown dialect
+ 'unknown': (
+ # LEAVE THIS EMPTY
+ ),
+ # Standard Library Constants for PIM Modula-2
+ 'm2pim': (
+ pim_stdlib_const_identifiers,
+ ),
+
+ # Standard Library Constants for ISO Modula-2
+ 'm2iso': (
+ iso_stdlib_const_identifiers,
+ ),
+
+ # Standard Library Constants for Modula-2 R10
+ 'm2r10': (
+ m2r10_stdlib_const_identifiers,
+ ),
+
+ # Standard Library Constants for Objective Modula-2
+ 'objm2': (
+ m2r10_stdlib_const_identifiers,
+ ),
+
+ # Standard Library Constants for Aglet Modula-2
+ 'm2iso+aglet': (
+ iso_stdlib_const_identifiers,
+ ),
+
+ # Standard Library Constants for GNU Modula-2
+ 'm2pim+gm2': (
+ pim_stdlib_const_identifiers,
+ ),
+
+ # Standard Library Constants for p1 Modula-2
+ 'm2iso+p1': (
+ iso_stdlib_const_identifiers,
+ ),
+
+ # Standard Library Constants for XDS Modula-2
+ 'm2iso+xds': (
+ iso_stdlib_const_identifiers,
+ ),
+ }
+
+# M e t h o d s
+
+ # initialise a lexer instance
+ def __init__(self, **options):
+ #
+ # check dialect options
+ #
+ dialects = get_list_opt(options, 'dialect', [])
+ #
+ for dialect_option in dialects:
+ if dialect_option in self.dialects[1:-1]:
+ # valid dialect option found
+ self.set_dialect(dialect_option)
+ break
+ #
+ # Fallback Mode (DEFAULT)
+ else:
+ # no valid dialect option
+ self.set_dialect('unknown')
+ #
+ self.dialect_set_by_tag = False
+ #
+ # check style options
+ #
+ styles = get_list_opt(options, 'style', [])
+ #
+ # use lowercase mode for Algol style
+ if 'algol' in styles or 'algol_nu' in styles:
+ self.algol_publication_mode = True
+ else:
+ self.algol_publication_mode = False
+ #
+ # Check option flags
+ #
+ self.treat_stdlib_adts_as_builtins = get_bool_opt(
+ options, 'treat_stdlib_adts_as_builtins', True)
+ #
+ # call superclass initialiser
+ RegexLexer.__init__(self, **options)
+
+ # Set lexer to a specified dialect
+ def set_dialect(self, dialect_id):
+ #
+ # if __debug__:
+ # print 'entered set_dialect with arg: ', dialect_id
+ #
+ # check dialect name against known dialects
+ if dialect_id not in self.dialects:
+ dialect = 'unknown' # default
+ else:
+ dialect = dialect_id
+ #
+ # compose lexemes to reject set
+ lexemes_to_reject_set = set()
+ # add each list of reject lexemes for this dialect
+ for list in self.lexemes_to_reject_db[dialect]:
+ lexemes_to_reject_set.update(set(list))
+ #
+ # compose reserved words set
+ reswords_set = set()
+ # add each list of reserved words for this dialect
+ for list in self.reserved_words_db[dialect]:
+ reswords_set.update(set(list))
+ #
+ # compose builtins set
+ builtins_set = set()
+ # add each list of builtins for this dialect excluding reserved words
+ for list in self.builtins_db[dialect]:
+ builtins_set.update(set(list).difference(reswords_set))
+ #
+ # compose pseudo-builtins set
+ pseudo_builtins_set = set()
+ # add each list of builtins for this dialect excluding reserved words
+ for list in self.pseudo_builtins_db[dialect]:
+ pseudo_builtins_set.update(set(list).difference(reswords_set))
+ #
+ # compose ADTs set
+ adts_set = set()
+ # add each list of ADTs for this dialect excluding reserved words
+ for list in self.stdlib_adts_db[dialect]:
+ adts_set.update(set(list).difference(reswords_set))
+ #
+ # compose modules set
+ modules_set = set()
+ # add each list of builtins for this dialect excluding builtins
+ for list in self.stdlib_modules_db[dialect]:
+ modules_set.update(set(list).difference(builtins_set))
+ #
+ # compose types set
+ types_set = set()
+ # add each list of types for this dialect excluding builtins
+ for list in self.stdlib_types_db[dialect]:
+ types_set.update(set(list).difference(builtins_set))
+ #
+ # compose procedures set
+ procedures_set = set()
+ # add each list of procedures for this dialect excluding builtins
+ for list in self.stdlib_procedures_db[dialect]:
+ procedures_set.update(set(list).difference(builtins_set))
+ #
+ # compose variables set
+ variables_set = set()
+ # add each list of variables for this dialect excluding builtins
+ for list in self.stdlib_variables_db[dialect]:
+ variables_set.update(set(list).difference(builtins_set))
+ #
+ # compose constants set
+ constants_set = set()
+ # add each list of constants for this dialect excluding builtins
+ for list in self.stdlib_constants_db[dialect]:
+ constants_set.update(set(list).difference(builtins_set))
+ #
+ # update lexer state
+ self.dialect = dialect
+ self.lexemes_to_reject = lexemes_to_reject_set
+ self.reserved_words = reswords_set
+ self.builtins = builtins_set
+ self.pseudo_builtins = pseudo_builtins_set
+ self.adts = adts_set
+ self.modules = modules_set
+ self.types = types_set
+ self.procedures = procedures_set
+ self.variables = variables_set
+ self.constants = constants_set
+ #
+ # if __debug__:
+ # print 'exiting set_dialect'
+ # print ' self.dialect: ', self.dialect
+ # print ' self.lexemes_to_reject: ', self.lexemes_to_reject
+ # print ' self.reserved_words: ', self.reserved_words
+ # print ' self.builtins: ', self.builtins
+ # print ' self.pseudo_builtins: ', self.pseudo_builtins
+ # print ' self.adts: ', self.adts
+ # print ' self.modules: ', self.modules
+ # print ' self.types: ', self.types
+ # print ' self.procedures: ', self.procedures
+ # print ' self.variables: ', self.variables
+ # print ' self.types: ', self.types
+ # print ' self.constants: ', self.constants
+
+ # Extracts a dialect name from a dialect tag comment string and checks
+ # the extracted name against known dialects. If a match is found, the
+ # matching name is returned, otherwise dialect id 'unknown' is returned
+ def get_dialect_from_dialect_tag(self, dialect_tag):
+ #
+ # if __debug__:
+ # print 'entered get_dialect_from_dialect_tag with arg: ', dialect_tag
+ #
+ # constants
+ left_tag_delim = '(*!'
+ right_tag_delim = '*)'
+ left_tag_delim_len = len(left_tag_delim)
+ right_tag_delim_len = len(right_tag_delim)
+ indicator_start = left_tag_delim_len
+ indicator_end = -(right_tag_delim_len)
+ #
+ # check comment string for dialect indicator
+ if len(dialect_tag) > (left_tag_delim_len + right_tag_delim_len) \
+ and dialect_tag.startswith(left_tag_delim) \
+ and dialect_tag.endswith(right_tag_delim):
+ #
+ # if __debug__:
+ # print 'dialect tag found'
+ #
+ # extract dialect indicator
+ indicator = dialect_tag[indicator_start:indicator_end]
+ #
+ # if __debug__:
+ # print 'extracted: ', indicator
+ #
+ # check against known dialects
+ for index in range(1, len(self.dialects)):
+ #
+ # if __debug__:
+ # print 'dialects[', index, ']: ', self.dialects[index]
+ #
+ if indicator == self.dialects[index]:
+ #
+ # if __debug__:
+ # print 'matching dialect found'
+ #
+ # indicator matches known dialect
+ return indicator
+ else:
+ # indicator does not match any dialect
+ return 'unknown' # default
+ else:
+ # invalid indicator string
+ return 'unknown' # default
+
+ # intercept the token stream, modify token attributes and return them
+ def get_tokens_unprocessed(self, text):
+ for index, token, value in RegexLexer.get_tokens_unprocessed(self, text):
+ #
+ # check for dialect tag if dialect has not been set by tag
+ if not self.dialect_set_by_tag and token == Comment.Special:
+ indicated_dialect = self.get_dialect_from_dialect_tag(value)
+ if indicated_dialect != 'unknown':
+ # token is a dialect indicator
+ # reset reserved words and builtins
+ self.set_dialect(indicated_dialect)
+ self.dialect_set_by_tag = True
+ #
+ # check for reserved words, predefined and stdlib identifiers
+ if token is Name:
+ if value in self.reserved_words:
+ token = Keyword.Reserved
+ if self.algol_publication_mode:
+ value = value.lower()
+ #
+ elif value in self.builtins:
+ token = Name.Builtin
+ if self.algol_publication_mode:
+ value = value.lower()
+ #
+ elif value in self.pseudo_builtins:
+ token = Name.Builtin.Pseudo
+ if self.algol_publication_mode:
+ value = value.lower()
+ #
+ elif value in self.adts:
+ if not self.treat_stdlib_adts_as_builtins:
+ token = Name.Namespace
+ else:
+ token = Name.Builtin.Pseudo
+ if self.algol_publication_mode:
+ value = value.lower()
+ #
+ elif value in self.modules:
+ token = Name.Namespace
+ #
+ elif value in self.types:
+ token = Name.Class
+ #
+ elif value in self.procedures:
+ token = Name.Function
+ #
+ elif value in self.variables:
+ token = Name.Variable
+ #
+ elif value in self.constants:
+ token = Name.Constant
+ #
+ elif token in Number:
+ #
+ # mark prefix number literals as error for PIM and ISO dialects
+ if self.dialect not in ('unknown', 'm2r10', 'objm2'):
+ if "'" in value or value[0:2] in ('0b', '0x', '0u'):
+ token = Error
+ #
+ elif self.dialect in ('m2r10', 'objm2'):
+ # mark base-8 number literals as errors for M2 R10 and ObjM2
+ if token is Number.Oct:
+ token = Error
+ # mark suffix base-16 literals as errors for M2 R10 and ObjM2
+ elif token is Number.Hex and 'H' in value:
+ token = Error
+ # mark real numbers with E as errors for M2 R10 and ObjM2
+ elif token is Number.Float and 'E' in value:
+ token = Error
+ #
+ elif token in Comment:
+ #
+ # mark single line comment as error for PIM and ISO dialects
+ if token is Comment.Single:
+ if self.dialect not in ('unknown', 'm2r10', 'objm2'):
+ token = Error
+ #
+ if token is Comment.Preproc:
+ # mark ISO pragma as error for PIM dialects
+ if value.startswith('<*') and \
+ self.dialect.startswith('m2pim'):
+ token = Error
+ # mark PIM pragma as comment for other dialects
+ elif value.startswith('(*$') and \
+ self.dialect != 'unknown' and \
+ not self.dialect.startswith('m2pim'):
+ token = Comment.Multiline
+ #
+ else: # token is neither Name nor Comment
+ #
+ # mark lexemes matching the dialect's error token set as errors
+ if value in self.lexemes_to_reject:
+ token = Error
+ #
+ # substitute lexemes when in Algol mode
+ if self.algol_publication_mode:
+ if value == '#':
+ value = u'≠'
+ elif value == '<=':
+ value = u'≤'
+ elif value == '>=':
+ value = u'≥'
+ elif value == '==':
+ value = u'≡'
+ elif value == '*.':
+ value = u'•'
+
+ # return result
+ yield index, token, value
diff --git a/wandb/vendor/pygments/lexers/monte.py b/wandb/vendor/pygments/lexers/monte.py
new file mode 100644
index 0000000000000000000000000000000000000000..ed6e20f86fc36f564773d0908eef91bdbdfcd334
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/monte.py
@@ -0,0 +1,204 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.monte
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for the Monte programming language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.token import Comment, Error, Keyword, Name, Number, Operator, \
+ Punctuation, String, Whitespace
+from pygments.lexer import RegexLexer, include, words
+
+__all__ = ['MonteLexer']
+
+
+# `var` handled separately
+# `interface` handled separately
+_declarations = ['bind', 'def', 'fn', 'object']
+_methods = ['method', 'to']
+_keywords = [
+ 'as', 'break', 'catch', 'continue', 'else', 'escape', 'exit', 'exports',
+ 'extends', 'finally', 'for', 'guards', 'if', 'implements', 'import',
+ 'in', 'match', 'meta', 'pass', 'return', 'switch', 'try', 'via', 'when',
+ 'while',
+]
+_operators = [
+ # Unary
+ '~', '!',
+ # Binary
+ '+', '-', '*', '/', '%', '**', '&', '|', '^', '<<', '>>',
+ # Binary augmented
+ '+=', '-=', '*=', '/=', '%=', '**=', '&=', '|=', '^=', '<<=', '>>=',
+ # Comparison
+ '==', '!=', '<', '<=', '>', '>=', '<=>',
+ # Patterns and assignment
+ ':=', '?', '=~', '!~', '=>',
+ # Calls and sends
+ '.', '<-', '->',
+]
+_escape_pattern = (
+ r'(?:\\x[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|'
+ r'\\["\'\\bftnr])')
+# _char = _escape_chars + [('.', String.Char)]
+_identifier = r'[_a-zA-Z]\w*'
+
+_constants = [
+ # Void constants
+ 'null',
+ # Bool constants
+ 'false', 'true',
+ # Double constants
+ 'Infinity', 'NaN',
+ # Special objects
+ 'M', 'Ref', 'throw', 'traceln',
+]
+
+_guards = [
+ 'Any', 'Binding', 'Bool', 'Bytes', 'Char', 'DeepFrozen', 'Double',
+ 'Empty', 'Int', 'List', 'Map', 'Near', 'NullOk', 'Same', 'Selfless',
+ 'Set', 'Str', 'SubrangeGuard', 'Transparent', 'Void',
+]
+
+_safeScope = [
+ '_accumulateList', '_accumulateMap', '_auditedBy', '_bind',
+ '_booleanFlow', '_comparer', '_equalizer', '_iterForever', '_loop',
+ '_makeBytes', '_makeDouble', '_makeFinalSlot', '_makeInt', '_makeList',
+ '_makeMap', '_makeMessageDesc', '_makeOrderedSpace', '_makeParamDesc',
+ '_makeProtocolDesc', '_makeSourceSpan', '_makeString', '_makeVarSlot',
+ '_makeVerbFacet', '_mapExtract', '_matchSame', '_quasiMatcher',
+ '_slotToBinding', '_splitList', '_suchThat', '_switchFailed',
+ '_validateFor', 'b__quasiParser', 'eval', 'import', 'm__quasiParser',
+ 'makeBrandPair', 'makeLazySlot', 'safeScope', 'simple__quasiParser',
+]
+
+
+class MonteLexer(RegexLexer):
+ """
+ Lexer for the `Monte <https://monte.readthedocs.io/>`_ programming language.
+
+ .. versionadded:: 2.2
+ """
+ name = 'Monte'
+ aliases = ['monte']
+ filenames = ['*.mt']
+
+ tokens = {
+ 'root': [
+ # Comments
+ (r'#[^\n]*\n', Comment),
+
+ # Docstrings
+ # Apologies for the non-greedy matcher here.
+ (r'/\*\*.*?\*/', String.Doc),
+
+ # `var` declarations
+ (r'\bvar\b', Keyword.Declaration, 'var'),
+
+ # `interface` declarations
+ (r'\binterface\b', Keyword.Declaration, 'interface'),
+
+ # method declarations
+ (words(_methods, prefix='\\b', suffix='\\b'),
+ Keyword, 'method'),
+
+ # All other declarations
+ (words(_declarations, prefix='\\b', suffix='\\b'),
+ Keyword.Declaration),
+
+ # Keywords
+ (words(_keywords, prefix='\\b', suffix='\\b'), Keyword),
+
+ # Literals
+ ('[+-]?0x[_0-9a-fA-F]+', Number.Hex),
+ (r'[+-]?[_0-9]+\.[_0-9]*([eE][+-]?[_0-9]+)?', Number.Float),
+ ('[+-]?[_0-9]+', Number.Integer),
+ ("'", String.Double, 'char'),
+ ('"', String.Double, 'string'),
+
+ # Quasiliterals
+ ('`', String.Backtick, 'ql'),
+
+ # Operators
+ (words(_operators), Operator),
+
+ # Verb operators
+ (_identifier + '=', Operator.Word),
+
+ # Safe scope constants
+ (words(_constants, prefix='\\b', suffix='\\b'),
+ Keyword.Pseudo),
+
+ # Safe scope guards
+ (words(_guards, prefix='\\b', suffix='\\b'), Keyword.Type),
+
+ # All other safe scope names
+ (words(_safeScope, prefix='\\b', suffix='\\b'),
+ Name.Builtin),
+
+ # Identifiers
+ (_identifier, Name),
+
+ # Punctuation
+ (r'\(|\)|\{|\}|\[|\]|:|,', Punctuation),
+
+ # Whitespace
+ (' +', Whitespace),
+
+ # Definite lexer errors
+ ('=', Error),
+ ],
+ 'char': [
+ # It is definitely an error to have a char of width == 0.
+ ("'", Error, 'root'),
+ (_escape_pattern, String.Escape, 'charEnd'),
+ ('.', String.Char, 'charEnd'),
+ ],
+ 'charEnd': [
+ ("'", String.Char, '#pop:2'),
+ # It is definitely an error to have a char of width > 1.
+ ('.', Error),
+ ],
+ # The state of things coming into an interface.
+ 'interface': [
+ (' +', Whitespace),
+ (_identifier, Name.Class, '#pop'),
+ include('root'),
+ ],
+ # The state of things coming into a method.
+ 'method': [
+ (' +', Whitespace),
+ (_identifier, Name.Function, '#pop'),
+ include('root'),
+ ],
+ 'string': [
+ ('"', String.Double, 'root'),
+ (_escape_pattern, String.Escape),
+ (r'\n', String.Double),
+ ('.', String.Double),
+ ],
+ 'ql': [
+ ('`', String.Backtick, 'root'),
+ (r'\$' + _escape_pattern, String.Escape),
+ (r'\$\$', String.Escape),
+ (r'@@', String.Escape),
+ (r'\$\{', String.Interpol, 'qlNest'),
+ (r'@\{', String.Interpol, 'qlNest'),
+ (r'\$' + _identifier, Name),
+ ('@' + _identifier, Name),
+ ('.', String.Backtick),
+ ],
+ 'qlNest': [
+ (r'\}', String.Interpol, '#pop'),
+ include('root'),
+ ],
+ # The state of things immediately following `var`.
+ 'var': [
+ (' +', Whitespace),
+ (_identifier, Name.Variable, '#pop'),
+ include('root'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/ncl.py b/wandb/vendor/pygments/lexers/ncl.py
new file mode 100644
index 0000000000000000000000000000000000000000..3ca5135cc1bb1e375af8e05fb2fa1a31be68df9a
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/ncl.py
@@ -0,0 +1,894 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.ncl
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexers for NCAR Command Language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['NCLLexer']
+
+
+class NCLLexer(RegexLexer):
+ """
+ Lexer for NCL code.
+
+ .. versionadded:: 2.2
+ """
+ name = 'NCL'
+ aliases = ['ncl']
+ filenames = ['*.ncl']
+ mimetypes = ['text/ncl']
+ flags = re.MULTILINE
+
+ tokens = {
+ 'root': [
+ (r';.*\n', Comment),
+ include('strings'),
+ include('core'),
+ (r'[a-zA-Z_]\w*', Name),
+ include('nums'),
+ (r'[\s]+', Text),
+ ],
+ 'core': [
+ # Statements
+ (words((
+ 'begin', 'break', 'continue', 'create', 'defaultapp', 'do',
+ 'else', 'end', 'external', 'exit', 'True', 'False', 'file', 'function',
+ 'getvalues', 'graphic', 'group', 'if', 'list', 'load', 'local',
+ 'new', '_Missing', 'Missing', 'noparent', 'procedure',
+ 'quit', 'QUIT', 'Quit', 'record', 'return', 'setvalues', 'stop',
+ 'then', 'while'), prefix=r'\b', suffix=r'\s*\b'),
+ Keyword),
+
+ # Data Types
+ (words((
+ 'ubyte', 'uint', 'uint64', 'ulong', 'string', 'byte',
+ 'character', 'double', 'float', 'integer', 'int64', 'logical',
+ 'long', 'short', 'ushort', 'enumeric', 'numeric', 'snumeric'),
+ prefix=r'\b', suffix=r'\s*\b'),
+ Keyword.Type),
+
+ # Operators
+ (r'[\%^*+\-/<>]', Operator),
+
+ # punctuation:
+ (r'[\[\]():@$!&|.,\\{}]', Punctuation),
+ (r'[=:]', Punctuation),
+
+ # Intrinsics
+ (words((
+ 'abs', 'acos', 'addfile', 'addfiles', 'all', 'angmom_atm', 'any',
+ 'area_conserve_remap', 'area_hi2lores', 'area_poly_sphere',
+ 'asciiread', 'asciiwrite', 'asin', 'atan', 'atan2', 'attsetvalues',
+ 'avg', 'betainc', 'bin_avg', 'bin_sum', 'bw_bandpass_filter',
+ 'cancor', 'cbinread', 'cbinwrite', 'cd_calendar', 'cd_inv_calendar',
+ 'cdfbin_p', 'cdfbin_pr', 'cdfbin_s', 'cdfbin_xn', 'cdfchi_p',
+ 'cdfchi_x', 'cdfgam_p', 'cdfgam_x', 'cdfnor_p', 'cdfnor_x',
+ 'cdft_p', 'cdft_t', 'ceil', 'center_finite_diff',
+ 'center_finite_diff_n', 'cfftb', 'cfftf', 'cfftf_frq_reorder',
+ 'charactertodouble', 'charactertofloat', 'charactertointeger',
+ 'charactertolong', 'charactertoshort', 'charactertostring',
+ 'chartodouble', 'chartofloat', 'chartoint', 'chartointeger',
+ 'chartolong', 'chartoshort', 'chartostring', 'chiinv', 'clear',
+ 'color_index_to_rgba', 'conform', 'conform_dims', 'cos', 'cosh',
+ 'count_unique_values', 'covcorm', 'covcorm_xy', 'craybinnumrec',
+ 'craybinrecread', 'create_graphic', 'csa1', 'csa1d', 'csa1s',
+ 'csa1x', 'csa1xd', 'csa1xs', 'csa2', 'csa2d', 'csa2l', 'csa2ld',
+ 'csa2ls', 'csa2lx', 'csa2lxd', 'csa2lxs', 'csa2s', 'csa2x',
+ 'csa2xd', 'csa2xs', 'csa3', 'csa3d', 'csa3l', 'csa3ld', 'csa3ls',
+ 'csa3lx', 'csa3lxd', 'csa3lxs', 'csa3s', 'csa3x', 'csa3xd',
+ 'csa3xs', 'csc2s', 'csgetp', 'css2c', 'cssetp', 'cssgrid', 'csstri',
+ 'csvoro', 'cumsum', 'cz2ccm', 'datatondc', 'day_of_week',
+ 'day_of_year', 'days_in_month', 'default_fillvalue', 'delete',
+ 'depth_to_pres', 'destroy', 'determinant', 'dewtemp_trh',
+ 'dgeevx_lapack', 'dim_acumrun_n', 'dim_avg', 'dim_avg_n',
+ 'dim_avg_wgt', 'dim_avg_wgt_n', 'dim_cumsum', 'dim_cumsum_n',
+ 'dim_gamfit_n', 'dim_gbits', 'dim_max', 'dim_max_n', 'dim_median',
+ 'dim_median_n', 'dim_min', 'dim_min_n', 'dim_num', 'dim_num_n',
+ 'dim_numrun_n', 'dim_pqsort', 'dim_pqsort_n', 'dim_product',
+ 'dim_product_n', 'dim_rmsd', 'dim_rmsd_n', 'dim_rmvmean',
+ 'dim_rmvmean_n', 'dim_rmvmed', 'dim_rmvmed_n', 'dim_spi_n',
+ 'dim_standardize', 'dim_standardize_n', 'dim_stat4', 'dim_stat4_n',
+ 'dim_stddev', 'dim_stddev_n', 'dim_sum', 'dim_sum_n', 'dim_sum_wgt',
+ 'dim_sum_wgt_n', 'dim_variance', 'dim_variance_n', 'dimsizes',
+ 'doubletobyte', 'doubletochar', 'doubletocharacter',
+ 'doubletofloat', 'doubletoint', 'doubletointeger', 'doubletolong',
+ 'doubletoshort', 'dpres_hybrid_ccm', 'dpres_plevel', 'draw',
+ 'draw_color_palette', 'dsgetp', 'dsgrid2', 'dsgrid2d', 'dsgrid2s',
+ 'dsgrid3', 'dsgrid3d', 'dsgrid3s', 'dspnt2', 'dspnt2d', 'dspnt2s',
+ 'dspnt3', 'dspnt3d', 'dspnt3s', 'dssetp', 'dtrend', 'dtrend_msg',
+ 'dtrend_msg_n', 'dtrend_n', 'dtrend_quadratic',
+ 'dtrend_quadratic_msg_n', 'dv2uvf', 'dv2uvg', 'dz_height',
+ 'echo_off', 'echo_on', 'eof2data', 'eof_varimax', 'eofcor',
+ 'eofcor_pcmsg', 'eofcor_ts', 'eofcov', 'eofcov_pcmsg', 'eofcov_ts',
+ 'eofunc', 'eofunc_ts', 'eofunc_varimax', 'equiv_sample_size', 'erf',
+ 'erfc', 'esacr', 'esacv', 'esccr', 'esccv', 'escorc', 'escorc_n',
+ 'escovc', 'exit', 'exp', 'exp_tapersh', 'exp_tapersh_wgts',
+ 'exp_tapershC', 'ezfftb', 'ezfftb_n', 'ezfftf', 'ezfftf_n',
+ 'f2fosh', 'f2foshv', 'f2fsh', 'f2fshv', 'f2gsh', 'f2gshv', 'fabs',
+ 'fbindirread', 'fbindirwrite', 'fbinnumrec', 'fbinread',
+ 'fbinrecread', 'fbinrecwrite', 'fbinwrite', 'fft2db', 'fft2df',
+ 'fftshift', 'fileattdef', 'filechunkdimdef', 'filedimdef',
+ 'fileexists', 'filegrpdef', 'filevarattdef', 'filevarchunkdef',
+ 'filevarcompressleveldef', 'filevardef', 'filevardimsizes',
+ 'filwgts_lancos', 'filwgts_lanczos', 'filwgts_normal',
+ 'floattobyte', 'floattochar', 'floattocharacter', 'floattoint',
+ 'floattointeger', 'floattolong', 'floattoshort', 'floor',
+ 'fluxEddy', 'fo2fsh', 'fo2fshv', 'fourier_info', 'frame', 'fspan',
+ 'ftcurv', 'ftcurvd', 'ftcurvi', 'ftcurvp', 'ftcurvpi', 'ftcurvps',
+ 'ftcurvs', 'ftest', 'ftgetp', 'ftkurv', 'ftkurvd', 'ftkurvp',
+ 'ftkurvpd', 'ftsetp', 'ftsurf', 'g2fsh', 'g2fshv', 'g2gsh',
+ 'g2gshv', 'gamma', 'gammainc', 'gaus', 'gaus_lobat',
+ 'gaus_lobat_wgt', 'gc_aangle', 'gc_clkwise', 'gc_dangle',
+ 'gc_inout', 'gc_latlon', 'gc_onarc', 'gc_pnt2gc', 'gc_qarea',
+ 'gc_tarea', 'generate_2d_array', 'get_color_index',
+ 'get_color_rgba', 'get_cpu_time', 'get_isolines', 'get_ncl_version',
+ 'get_script_name', 'get_script_prefix_name', 'get_sphere_radius',
+ 'get_unique_values', 'getbitsone', 'getenv', 'getfiledimsizes',
+ 'getfilegrpnames', 'getfilepath', 'getfilevaratts',
+ 'getfilevarchunkdimsizes', 'getfilevardims', 'getfilevardimsizes',
+ 'getfilevarnames', 'getfilevartypes', 'getvaratts', 'getvardims',
+ 'gradsf', 'gradsg', 'greg2jul', 'grid2triple', 'hlsrgb', 'hsvrgb',
+ 'hydro', 'hyi2hyo', 'idsfft', 'igradsf', 'igradsg', 'ilapsf',
+ 'ilapsg', 'ilapvf', 'ilapvg', 'ind', 'ind_resolve', 'int2p',
+ 'int2p_n', 'integertobyte', 'integertochar', 'integertocharacter',
+ 'integertoshort', 'inttobyte', 'inttochar', 'inttoshort',
+ 'inverse_matrix', 'isatt', 'isbigendian', 'isbyte', 'ischar',
+ 'iscoord', 'isdefined', 'isdim', 'isdimnamed', 'isdouble',
+ 'isenumeric', 'isfile', 'isfilepresent', 'isfilevar',
+ 'isfilevaratt', 'isfilevarcoord', 'isfilevardim', 'isfloat',
+ 'isfunc', 'isgraphic', 'isint', 'isint64', 'isinteger',
+ 'isleapyear', 'islogical', 'islong', 'ismissing', 'isnan_ieee',
+ 'isnumeric', 'ispan', 'isproc', 'isshort', 'issnumeric', 'isstring',
+ 'isubyte', 'isuint', 'isuint64', 'isulong', 'isunlimited',
+ 'isunsigned', 'isushort', 'isvar', 'jul2greg', 'kmeans_as136',
+ 'kolsm2_n', 'kron_product', 'lapsf', 'lapsg', 'lapvf', 'lapvg',
+ 'latlon2utm', 'lclvl', 'lderuvf', 'lderuvg', 'linint1', 'linint1_n',
+ 'linint2', 'linint2_points', 'linmsg', 'linmsg_n', 'linrood_latwgt',
+ 'linrood_wgt', 'list_files', 'list_filevars', 'list_hlus',
+ 'list_procfuncs', 'list_vars', 'ListAppend', 'ListCount',
+ 'ListGetType', 'ListIndex', 'ListIndexFromName', 'ListPop',
+ 'ListPush', 'ListSetType', 'loadscript', 'local_max', 'local_min',
+ 'log', 'log10', 'longtobyte', 'longtochar', 'longtocharacter',
+ 'longtoint', 'longtointeger', 'longtoshort', 'lspoly', 'lspoly_n',
+ 'mask', 'max', 'maxind', 'min', 'minind', 'mixed_layer_depth',
+ 'mixhum_ptd', 'mixhum_ptrh', 'mjo_cross_coh2pha',
+ 'mjo_cross_segment', 'moc_globe_atl', 'monthday', 'natgrid',
+ 'natgridd', 'natgrids', 'ncargpath', 'ncargversion', 'ndctodata',
+ 'ndtooned', 'new', 'NewList', 'ngezlogo', 'nggcog', 'nggetp',
+ 'nglogo', 'ngsetp', 'NhlAddAnnotation', 'NhlAddData',
+ 'NhlAddOverlay', 'NhlAddPrimitive', 'NhlAppGetDefaultParentId',
+ 'NhlChangeWorkstation', 'NhlClassName', 'NhlClearWorkstation',
+ 'NhlDataPolygon', 'NhlDataPolyline', 'NhlDataPolymarker',
+ 'NhlDataToNDC', 'NhlDestroy', 'NhlDraw', 'NhlFrame', 'NhlFreeColor',
+ 'NhlGetBB', 'NhlGetClassResources', 'NhlGetErrorObjectId',
+ 'NhlGetNamedColorIndex', 'NhlGetParentId',
+ 'NhlGetParentWorkstation', 'NhlGetWorkspaceObjectId',
+ 'NhlIsAllocatedColor', 'NhlIsApp', 'NhlIsDataComm', 'NhlIsDataItem',
+ 'NhlIsDataSpec', 'NhlIsTransform', 'NhlIsView', 'NhlIsWorkstation',
+ 'NhlName', 'NhlNDCPolygon', 'NhlNDCPolyline', 'NhlNDCPolymarker',
+ 'NhlNDCToData', 'NhlNewColor', 'NhlNewDashPattern', 'NhlNewMarker',
+ 'NhlPalGetDefined', 'NhlRemoveAnnotation', 'NhlRemoveData',
+ 'NhlRemoveOverlay', 'NhlRemovePrimitive', 'NhlSetColor',
+ 'NhlSetDashPattern', 'NhlSetMarker', 'NhlUpdateData',
+ 'NhlUpdateWorkstation', 'nice_mnmxintvl', 'nngetaspectd',
+ 'nngetaspects', 'nngetp', 'nngetsloped', 'nngetslopes', 'nngetwts',
+ 'nngetwtsd', 'nnpnt', 'nnpntd', 'nnpntend', 'nnpntendd',
+ 'nnpntinit', 'nnpntinitd', 'nnpntinits', 'nnpnts', 'nnsetp', 'num',
+ 'obj_anal_ic', 'omega_ccm', 'onedtond', 'overlay', 'paleo_outline',
+ 'pdfxy_bin', 'poisson_grid_fill', 'pop_remap', 'potmp_insitu_ocn',
+ 'prcwater_dp', 'pres2hybrid', 'pres_hybrid_ccm', 'pres_sigma',
+ 'print', 'print_table', 'printFileVarSummary', 'printVarSummary',
+ 'product', 'pslec', 'pslhor', 'pslhyp', 'qsort', 'rand',
+ 'random_chi', 'random_gamma', 'random_normal', 'random_setallseed',
+ 'random_uniform', 'rcm2points', 'rcm2rgrid', 'rdsstoi',
+ 'read_colormap_file', 'reg_multlin', 'regcoef', 'regCoef_n',
+ 'regline', 'relhum', 'replace_ieeenan', 'reshape', 'reshape_ind',
+ 'rgba_to_color_index', 'rgbhls', 'rgbhsv', 'rgbyiq', 'rgrid2rcm',
+ 'rhomb_trunc', 'rip_cape_2d', 'rip_cape_3d', 'round', 'rtest',
+ 'runave', 'runave_n', 'set_default_fillvalue', 'set_sphere_radius',
+ 'setfileoption', 'sfvp2uvf', 'sfvp2uvg', 'shaec', 'shagc',
+ 'shgetnp', 'shgetp', 'shgrid', 'shorttobyte', 'shorttochar',
+ 'shorttocharacter', 'show_ascii', 'shsec', 'shsetp', 'shsgc',
+ 'shsgc_R42', 'sigma2hybrid', 'simpeq', 'simpne', 'sin',
+ 'sindex_yrmo', 'sinh', 'sizeof', 'sleep', 'smth9', 'snindex_yrmo',
+ 'solve_linsys', 'span_color_indexes', 'span_color_rgba',
+ 'sparse_matrix_mult', 'spcorr', 'spcorr_n', 'specx_anal',
+ 'specxy_anal', 'spei', 'sprintf', 'sprinti', 'sqrt', 'sqsort',
+ 'srand', 'stat2', 'stat4', 'stat_medrng', 'stat_trim',
+ 'status_exit', 'stdatmus_p2tdz', 'stdatmus_z2tdp', 'stddev',
+ 'str_capital', 'str_concat', 'str_fields_count', 'str_get_cols',
+ 'str_get_dq', 'str_get_field', 'str_get_nl', 'str_get_sq',
+ 'str_get_tab', 'str_index_of_substr', 'str_insert', 'str_is_blank',
+ 'str_join', 'str_left_strip', 'str_lower', 'str_match',
+ 'str_match_ic', 'str_match_ic_regex', 'str_match_ind',
+ 'str_match_ind_ic', 'str_match_ind_ic_regex', 'str_match_ind_regex',
+ 'str_match_regex', 'str_right_strip', 'str_split',
+ 'str_split_by_length', 'str_split_csv', 'str_squeeze', 'str_strip',
+ 'str_sub_str', 'str_switch', 'str_upper', 'stringtochar',
+ 'stringtocharacter', 'stringtodouble', 'stringtofloat',
+ 'stringtoint', 'stringtointeger', 'stringtolong', 'stringtoshort',
+ 'strlen', 'student_t', 'sum', 'svd_lapack', 'svdcov', 'svdcov_sv',
+ 'svdstd', 'svdstd_sv', 'system', 'systemfunc', 'tan', 'tanh',
+ 'taper', 'taper_n', 'tdclrs', 'tdctri', 'tdcudp', 'tdcurv',
+ 'tddtri', 'tdez2d', 'tdez3d', 'tdgetp', 'tdgrds', 'tdgrid',
+ 'tdgtrs', 'tdinit', 'tditri', 'tdlbla', 'tdlblp', 'tdlbls',
+ 'tdline', 'tdlndp', 'tdlnpa', 'tdlpdp', 'tdmtri', 'tdotri',
+ 'tdpara', 'tdplch', 'tdprpa', 'tdprpi', 'tdprpt', 'tdsetp',
+ 'tdsort', 'tdstri', 'tdstrs', 'tdttri', 'thornthwaite', 'tobyte',
+ 'tochar', 'todouble', 'tofloat', 'toint', 'toint64', 'tointeger',
+ 'tolong', 'toshort', 'tosigned', 'tostring', 'tostring_with_format',
+ 'totype', 'toubyte', 'touint', 'touint64', 'toulong', 'tounsigned',
+ 'toushort', 'trend_manken', 'tri_trunc', 'triple2grid',
+ 'triple2grid2d', 'trop_wmo', 'ttest', 'typeof', 'undef',
+ 'unique_string', 'update', 'ushorttoint', 'ut_calendar',
+ 'ut_inv_calendar', 'utm2latlon', 'uv2dv_cfd', 'uv2dvf', 'uv2dvg',
+ 'uv2sfvpf', 'uv2sfvpg', 'uv2vr_cfd', 'uv2vrdvf', 'uv2vrdvg',
+ 'uv2vrf', 'uv2vrg', 'v5d_close', 'v5d_create', 'v5d_setLowLev',
+ 'v5d_setUnits', 'v5d_write', 'v5d_write_var', 'variance', 'vhaec',
+ 'vhagc', 'vhsec', 'vhsgc', 'vibeta', 'vinth2p', 'vinth2p_ecmwf',
+ 'vinth2p_ecmwf_nodes', 'vinth2p_nodes', 'vintp2p_ecmwf', 'vr2uvf',
+ 'vr2uvg', 'vrdv2uvf', 'vrdv2uvg', 'wavelet', 'wavelet_default',
+ 'weibull', 'wgt_area_smooth', 'wgt_areaave', 'wgt_areaave2',
+ 'wgt_arearmse', 'wgt_arearmse2', 'wgt_areasum2', 'wgt_runave',
+ 'wgt_runave_n', 'wgt_vert_avg_beta', 'wgt_volave', 'wgt_volave_ccm',
+ 'wgt_volrmse', 'wgt_volrmse_ccm', 'where', 'wk_smooth121', 'wmbarb',
+ 'wmbarbmap', 'wmdrft', 'wmgetp', 'wmlabs', 'wmsetp', 'wmstnm',
+ 'wmvect', 'wmvectmap', 'wmvlbl', 'wrf_avo', 'wrf_cape_2d',
+ 'wrf_cape_3d', 'wrf_dbz', 'wrf_eth', 'wrf_helicity', 'wrf_ij_to_ll',
+ 'wrf_interp_1d', 'wrf_interp_2d_xy', 'wrf_interp_3d_z',
+ 'wrf_latlon_to_ij', 'wrf_ll_to_ij', 'wrf_omega', 'wrf_pvo',
+ 'wrf_rh', 'wrf_slp', 'wrf_smooth_2d', 'wrf_td', 'wrf_tk',
+ 'wrf_updraft_helicity', 'wrf_uvmet', 'wrf_virtual_temp',
+ 'wrf_wetbulb', 'wrf_wps_close_int', 'wrf_wps_open_int',
+ 'wrf_wps_rddata_int', 'wrf_wps_rdhead_int', 'wrf_wps_read_int',
+ 'wrf_wps_write_int', 'write_matrix', 'write_table', 'yiqrgb',
+ 'z2geouv', 'zonal_mpsi', 'addfiles_GetVar', 'advect_variable',
+ 'area_conserve_remap_Wrap', 'area_hi2lores_Wrap',
+ 'array_append_record', 'assignFillValue', 'byte2flt',
+ 'byte2flt_hdf', 'calcDayAnomTLL', 'calcMonAnomLLLT',
+ 'calcMonAnomLLT', 'calcMonAnomTLL', 'calcMonAnomTLLL',
+ 'calculate_monthly_values', 'cd_convert', 'changeCase',
+ 'changeCaseChar', 'clmDayTLL', 'clmDayTLLL', 'clmMon2clmDay',
+ 'clmMonLLLT', 'clmMonLLT', 'clmMonTLL', 'clmMonTLLL', 'closest_val',
+ 'copy_VarAtts', 'copy_VarCoords', 'copy_VarCoords_1',
+ 'copy_VarCoords_2', 'copy_VarMeta', 'copyatt', 'crossp3',
+ 'cshstringtolist', 'cssgrid_Wrap', 'dble2flt', 'decimalPlaces',
+ 'delete_VarAtts', 'dim_avg_n_Wrap', 'dim_avg_wgt_n_Wrap',
+ 'dim_avg_wgt_Wrap', 'dim_avg_Wrap', 'dim_cumsum_n_Wrap',
+ 'dim_cumsum_Wrap', 'dim_max_n_Wrap', 'dim_min_n_Wrap',
+ 'dim_rmsd_n_Wrap', 'dim_rmsd_Wrap', 'dim_rmvmean_n_Wrap',
+ 'dim_rmvmean_Wrap', 'dim_rmvmed_n_Wrap', 'dim_rmvmed_Wrap',
+ 'dim_standardize_n_Wrap', 'dim_standardize_Wrap',
+ 'dim_stddev_n_Wrap', 'dim_stddev_Wrap', 'dim_sum_n_Wrap',
+ 'dim_sum_wgt_n_Wrap', 'dim_sum_wgt_Wrap', 'dim_sum_Wrap',
+ 'dim_variance_n_Wrap', 'dim_variance_Wrap', 'dpres_plevel_Wrap',
+ 'dtrend_leftdim', 'dv2uvF_Wrap', 'dv2uvG_Wrap', 'eof_north',
+ 'eofcor_Wrap', 'eofcov_Wrap', 'eofunc_north', 'eofunc_ts_Wrap',
+ 'eofunc_varimax_reorder', 'eofunc_varimax_Wrap', 'eofunc_Wrap',
+ 'epsZero', 'f2fosh_Wrap', 'f2foshv_Wrap', 'f2fsh_Wrap',
+ 'f2fshv_Wrap', 'f2gsh_Wrap', 'f2gshv_Wrap', 'fbindirSwap',
+ 'fbinseqSwap1', 'fbinseqSwap2', 'flt2dble', 'flt2string',
+ 'fo2fsh_Wrap', 'fo2fshv_Wrap', 'g2fsh_Wrap', 'g2fshv_Wrap',
+ 'g2gsh_Wrap', 'g2gshv_Wrap', 'generate_resample_indices',
+ 'generate_sample_indices', 'generate_unique_indices',
+ 'genNormalDist', 'get1Dindex', 'get1Dindex_Collapse',
+ 'get1Dindex_Exclude', 'get_file_suffix', 'GetFillColor',
+ 'GetFillColorIndex', 'getFillValue', 'getind_latlon2d',
+ 'getVarDimNames', 'getVarFillValue', 'grib_stime2itime',
+ 'hyi2hyo_Wrap', 'ilapsF_Wrap', 'ilapsG_Wrap', 'ind_nearest_coord',
+ 'indStrSubset', 'int2dble', 'int2flt', 'int2p_n_Wrap', 'int2p_Wrap',
+ 'isMonotonic', 'isStrSubset', 'latGau', 'latGauWgt', 'latGlobeF',
+ 'latGlobeFo', 'latRegWgt', 'linint1_n_Wrap', 'linint1_Wrap',
+ 'linint2_points_Wrap', 'linint2_Wrap', 'local_max_1d',
+ 'local_min_1d', 'lonFlip', 'lonGlobeF', 'lonGlobeFo', 'lonPivot',
+ 'merge_levels_sfc', 'mod', 'month_to_annual',
+ 'month_to_annual_weighted', 'month_to_season', 'month_to_season12',
+ 'month_to_seasonN', 'monthly_total_to_daily_mean', 'nameDim',
+ 'natgrid_Wrap', 'NewCosWeight', 'niceLatLon2D', 'NormCosWgtGlobe',
+ 'numAsciiCol', 'numAsciiRow', 'numeric2int',
+ 'obj_anal_ic_deprecated', 'obj_anal_ic_Wrap', 'omega_ccm_driver',
+ 'omega_to_w', 'oneDtostring', 'pack_values', 'pattern_cor', 'pdfx',
+ 'pdfxy', 'pdfxy_conform', 'pot_temp', 'pot_vort_hybrid',
+ 'pot_vort_isobaric', 'pres2hybrid_Wrap', 'print_clock',
+ 'printMinMax', 'quadroots', 'rcm2points_Wrap', 'rcm2rgrid_Wrap',
+ 'readAsciiHead', 'readAsciiTable', 'reg_multlin_stats',
+ 'region_ind', 'regline_stats', 'relhum_ttd', 'replaceSingleChar',
+ 'RGBtoCmap', 'rgrid2rcm_Wrap', 'rho_mwjf', 'rm_single_dims',
+ 'rmAnnCycle1D', 'rmInsufData', 'rmMonAnnCycLLLT', 'rmMonAnnCycLLT',
+ 'rmMonAnnCycTLL', 'runave_n_Wrap', 'runave_Wrap', 'short2flt',
+ 'short2flt_hdf', 'shsgc_R42_Wrap', 'sign_f90', 'sign_matlab',
+ 'smth9_Wrap', 'smthClmDayTLL', 'smthClmDayTLLL', 'SqrtCosWeight',
+ 'stat_dispersion', 'static_stability', 'stdMonLLLT', 'stdMonLLT',
+ 'stdMonTLL', 'stdMonTLLL', 'symMinMaxPlt', 'table_attach_columns',
+ 'table_attach_rows', 'time_to_newtime', 'transpose',
+ 'triple2grid_Wrap', 'ut_convert', 'uv2dvF_Wrap', 'uv2dvG_Wrap',
+ 'uv2vrF_Wrap', 'uv2vrG_Wrap', 'vr2uvF_Wrap', 'vr2uvG_Wrap',
+ 'w_to_omega', 'wallClockElapseTime', 'wave_number_spc',
+ 'wgt_areaave_Wrap', 'wgt_runave_leftdim', 'wgt_runave_n_Wrap',
+ 'wgt_runave_Wrap', 'wgt_vertical_n', 'wind_component',
+ 'wind_direction', 'yyyyddd_to_yyyymmdd', 'yyyymm_time',
+ 'yyyymm_to_yyyyfrac', 'yyyymmdd_time', 'yyyymmdd_to_yyyyddd',
+ 'yyyymmdd_to_yyyyfrac', 'yyyymmddhh_time', 'yyyymmddhh_to_yyyyfrac',
+ 'zonal_mpsi_Wrap', 'zonalAve', 'calendar_decode2', 'cd_string',
+ 'kf_filter', 'run_cor', 'time_axis_labels', 'ut_string',
+ 'wrf_contour', 'wrf_map', 'wrf_map_overlay', 'wrf_map_overlays',
+ 'wrf_map_resources', 'wrf_map_zoom', 'wrf_overlay', 'wrf_overlays',
+ 'wrf_user_getvar', 'wrf_user_ij_to_ll', 'wrf_user_intrp2d',
+ 'wrf_user_intrp3d', 'wrf_user_latlon_to_ij', 'wrf_user_list_times',
+ 'wrf_user_ll_to_ij', 'wrf_user_unstagger', 'wrf_user_vert_interp',
+ 'wrf_vector', 'gsn_add_annotation', 'gsn_add_polygon',
+ 'gsn_add_polyline', 'gsn_add_polymarker',
+ 'gsn_add_shapefile_polygons', 'gsn_add_shapefile_polylines',
+ 'gsn_add_shapefile_polymarkers', 'gsn_add_text', 'gsn_attach_plots',
+ 'gsn_blank_plot', 'gsn_contour', 'gsn_contour_map',
+ 'gsn_contour_shade', 'gsn_coordinates', 'gsn_create_labelbar',
+ 'gsn_create_legend', 'gsn_create_text',
+ 'gsn_csm_attach_zonal_means', 'gsn_csm_blank_plot',
+ 'gsn_csm_contour', 'gsn_csm_contour_map', 'gsn_csm_contour_map_ce',
+ 'gsn_csm_contour_map_overlay', 'gsn_csm_contour_map_polar',
+ 'gsn_csm_hov', 'gsn_csm_lat_time', 'gsn_csm_map', 'gsn_csm_map_ce',
+ 'gsn_csm_map_polar', 'gsn_csm_pres_hgt',
+ 'gsn_csm_pres_hgt_streamline', 'gsn_csm_pres_hgt_vector',
+ 'gsn_csm_streamline', 'gsn_csm_streamline_contour_map',
+ 'gsn_csm_streamline_contour_map_ce',
+ 'gsn_csm_streamline_contour_map_polar', 'gsn_csm_streamline_map',
+ 'gsn_csm_streamline_map_ce', 'gsn_csm_streamline_map_polar',
+ 'gsn_csm_streamline_scalar', 'gsn_csm_streamline_scalar_map',
+ 'gsn_csm_streamline_scalar_map_ce',
+ 'gsn_csm_streamline_scalar_map_polar', 'gsn_csm_time_lat',
+ 'gsn_csm_vector', 'gsn_csm_vector_map', 'gsn_csm_vector_map_ce',
+ 'gsn_csm_vector_map_polar', 'gsn_csm_vector_scalar',
+ 'gsn_csm_vector_scalar_map', 'gsn_csm_vector_scalar_map_ce',
+ 'gsn_csm_vector_scalar_map_polar', 'gsn_csm_x2y', 'gsn_csm_x2y2',
+ 'gsn_csm_xy', 'gsn_csm_xy2', 'gsn_csm_xy3', 'gsn_csm_y',
+ 'gsn_define_colormap', 'gsn_draw_colormap', 'gsn_draw_named_colors',
+ 'gsn_histogram', 'gsn_labelbar_ndc', 'gsn_legend_ndc', 'gsn_map',
+ 'gsn_merge_colormaps', 'gsn_open_wks', 'gsn_panel', 'gsn_polygon',
+ 'gsn_polygon_ndc', 'gsn_polyline', 'gsn_polyline_ndc',
+ 'gsn_polymarker', 'gsn_polymarker_ndc', 'gsn_retrieve_colormap',
+ 'gsn_reverse_colormap', 'gsn_streamline', 'gsn_streamline_map',
+ 'gsn_streamline_scalar', 'gsn_streamline_scalar_map', 'gsn_table',
+ 'gsn_text', 'gsn_text_ndc', 'gsn_vector', 'gsn_vector_map',
+ 'gsn_vector_scalar', 'gsn_vector_scalar_map', 'gsn_xy', 'gsn_y',
+ 'hsv2rgb', 'maximize_output', 'namedcolor2rgb', 'namedcolor2rgba',
+ 'reset_device_coordinates', 'span_named_colors'), prefix=r'\b'),
+ Name.Builtin),
+
+ # Resources
+ (words((
+ 'amDataXF', 'amDataYF', 'amJust', 'amOn', 'amOrthogonalPosF',
+ 'amParallelPosF', 'amResizeNotify', 'amSide', 'amTrackData',
+ 'amViewId', 'amZone', 'appDefaultParent', 'appFileSuffix',
+ 'appResources', 'appSysDir', 'appUsrDir', 'caCopyArrays',
+ 'caXArray', 'caXCast', 'caXMaxV', 'caXMinV', 'caXMissingV',
+ 'caYArray', 'caYCast', 'caYMaxV', 'caYMinV', 'caYMissingV',
+ 'cnCellFillEdgeColor', 'cnCellFillMissingValEdgeColor',
+ 'cnConpackParams', 'cnConstFEnableFill', 'cnConstFLabelAngleF',
+ 'cnConstFLabelBackgroundColor', 'cnConstFLabelConstantSpacingF',
+ 'cnConstFLabelFont', 'cnConstFLabelFontAspectF',
+ 'cnConstFLabelFontColor', 'cnConstFLabelFontHeightF',
+ 'cnConstFLabelFontQuality', 'cnConstFLabelFontThicknessF',
+ 'cnConstFLabelFormat', 'cnConstFLabelFuncCode', 'cnConstFLabelJust',
+ 'cnConstFLabelOn', 'cnConstFLabelOrthogonalPosF',
+ 'cnConstFLabelParallelPosF', 'cnConstFLabelPerimColor',
+ 'cnConstFLabelPerimOn', 'cnConstFLabelPerimSpaceF',
+ 'cnConstFLabelPerimThicknessF', 'cnConstFLabelSide',
+ 'cnConstFLabelString', 'cnConstFLabelTextDirection',
+ 'cnConstFLabelZone', 'cnConstFUseInfoLabelRes',
+ 'cnExplicitLabelBarLabelsOn', 'cnExplicitLegendLabelsOn',
+ 'cnExplicitLineLabelsOn', 'cnFillBackgroundColor', 'cnFillColor',
+ 'cnFillColors', 'cnFillDotSizeF', 'cnFillDrawOrder', 'cnFillMode',
+ 'cnFillOn', 'cnFillOpacityF', 'cnFillPalette', 'cnFillPattern',
+ 'cnFillPatterns', 'cnFillScaleF', 'cnFillScales', 'cnFixFillBleed',
+ 'cnGridBoundFillColor', 'cnGridBoundFillPattern',
+ 'cnGridBoundFillScaleF', 'cnGridBoundPerimColor',
+ 'cnGridBoundPerimDashPattern', 'cnGridBoundPerimOn',
+ 'cnGridBoundPerimThicknessF', 'cnHighLabelAngleF',
+ 'cnHighLabelBackgroundColor', 'cnHighLabelConstantSpacingF',
+ 'cnHighLabelCount', 'cnHighLabelFont', 'cnHighLabelFontAspectF',
+ 'cnHighLabelFontColor', 'cnHighLabelFontHeightF',
+ 'cnHighLabelFontQuality', 'cnHighLabelFontThicknessF',
+ 'cnHighLabelFormat', 'cnHighLabelFuncCode', 'cnHighLabelPerimColor',
+ 'cnHighLabelPerimOn', 'cnHighLabelPerimSpaceF',
+ 'cnHighLabelPerimThicknessF', 'cnHighLabelString', 'cnHighLabelsOn',
+ 'cnHighLowLabelOverlapMode', 'cnHighUseLineLabelRes',
+ 'cnInfoLabelAngleF', 'cnInfoLabelBackgroundColor',
+ 'cnInfoLabelConstantSpacingF', 'cnInfoLabelFont',
+ 'cnInfoLabelFontAspectF', 'cnInfoLabelFontColor',
+ 'cnInfoLabelFontHeightF', 'cnInfoLabelFontQuality',
+ 'cnInfoLabelFontThicknessF', 'cnInfoLabelFormat',
+ 'cnInfoLabelFuncCode', 'cnInfoLabelJust', 'cnInfoLabelOn',
+ 'cnInfoLabelOrthogonalPosF', 'cnInfoLabelParallelPosF',
+ 'cnInfoLabelPerimColor', 'cnInfoLabelPerimOn',
+ 'cnInfoLabelPerimSpaceF', 'cnInfoLabelPerimThicknessF',
+ 'cnInfoLabelSide', 'cnInfoLabelString', 'cnInfoLabelTextDirection',
+ 'cnInfoLabelZone', 'cnLabelBarEndLabelsOn', 'cnLabelBarEndStyle',
+ 'cnLabelDrawOrder', 'cnLabelMasking', 'cnLabelScaleFactorF',
+ 'cnLabelScaleValueF', 'cnLabelScalingMode', 'cnLegendLevelFlags',
+ 'cnLevelCount', 'cnLevelFlag', 'cnLevelFlags', 'cnLevelSelectionMode',
+ 'cnLevelSpacingF', 'cnLevels', 'cnLineColor', 'cnLineColors',
+ 'cnLineDashPattern', 'cnLineDashPatterns', 'cnLineDashSegLenF',
+ 'cnLineDrawOrder', 'cnLineLabelAngleF', 'cnLineLabelBackgroundColor',
+ 'cnLineLabelConstantSpacingF', 'cnLineLabelCount',
+ 'cnLineLabelDensityF', 'cnLineLabelFont', 'cnLineLabelFontAspectF',
+ 'cnLineLabelFontColor', 'cnLineLabelFontColors',
+ 'cnLineLabelFontHeightF', 'cnLineLabelFontQuality',
+ 'cnLineLabelFontThicknessF', 'cnLineLabelFormat',
+ 'cnLineLabelFuncCode', 'cnLineLabelInterval', 'cnLineLabelPerimColor',
+ 'cnLineLabelPerimOn', 'cnLineLabelPerimSpaceF',
+ 'cnLineLabelPerimThicknessF', 'cnLineLabelPlacementMode',
+ 'cnLineLabelStrings', 'cnLineLabelsOn', 'cnLinePalette',
+ 'cnLineThicknessF', 'cnLineThicknesses', 'cnLinesOn',
+ 'cnLowLabelAngleF', 'cnLowLabelBackgroundColor',
+ 'cnLowLabelConstantSpacingF', 'cnLowLabelCount', 'cnLowLabelFont',
+ 'cnLowLabelFontAspectF', 'cnLowLabelFontColor',
+ 'cnLowLabelFontHeightF', 'cnLowLabelFontQuality',
+ 'cnLowLabelFontThicknessF', 'cnLowLabelFormat', 'cnLowLabelFuncCode',
+ 'cnLowLabelPerimColor', 'cnLowLabelPerimOn', 'cnLowLabelPerimSpaceF',
+ 'cnLowLabelPerimThicknessF', 'cnLowLabelString', 'cnLowLabelsOn',
+ 'cnLowUseHighLabelRes', 'cnMaxDataValueFormat', 'cnMaxLevelCount',
+ 'cnMaxLevelValF', 'cnMaxPointDistanceF', 'cnMinLevelValF',
+ 'cnMissingValFillColor', 'cnMissingValFillPattern',
+ 'cnMissingValFillScaleF', 'cnMissingValPerimColor',
+ 'cnMissingValPerimDashPattern', 'cnMissingValPerimGridBoundOn',
+ 'cnMissingValPerimOn', 'cnMissingValPerimThicknessF',
+ 'cnMonoFillColor', 'cnMonoFillPattern', 'cnMonoFillScale',
+ 'cnMonoLevelFlag', 'cnMonoLineColor', 'cnMonoLineDashPattern',
+ 'cnMonoLineLabelFontColor', 'cnMonoLineThickness', 'cnNoDataLabelOn',
+ 'cnNoDataLabelString', 'cnOutOfRangeFillColor',
+ 'cnOutOfRangeFillPattern', 'cnOutOfRangeFillScaleF',
+ 'cnOutOfRangePerimColor', 'cnOutOfRangePerimDashPattern',
+ 'cnOutOfRangePerimOn', 'cnOutOfRangePerimThicknessF',
+ 'cnRasterCellSizeF', 'cnRasterMinCellSizeF', 'cnRasterModeOn',
+ 'cnRasterSampleFactorF', 'cnRasterSmoothingOn', 'cnScalarFieldData',
+ 'cnSmoothingDistanceF', 'cnSmoothingOn', 'cnSmoothingTensionF',
+ 'cnSpanFillPalette', 'cnSpanLinePalette', 'ctCopyTables',
+ 'ctXElementSize', 'ctXMaxV', 'ctXMinV', 'ctXMissingV', 'ctXTable',
+ 'ctXTableLengths', 'ctXTableType', 'ctYElementSize', 'ctYMaxV',
+ 'ctYMinV', 'ctYMissingV', 'ctYTable', 'ctYTableLengths',
+ 'ctYTableType', 'dcDelayCompute', 'errBuffer',
+ 'errFileName', 'errFilePtr', 'errLevel', 'errPrint', 'errUnitNumber',
+ 'gsClipOn', 'gsColors', 'gsEdgeColor', 'gsEdgeDashPattern',
+ 'gsEdgeDashSegLenF', 'gsEdgeThicknessF', 'gsEdgesOn',
+ 'gsFillBackgroundColor', 'gsFillColor', 'gsFillDotSizeF',
+ 'gsFillIndex', 'gsFillLineThicknessF', 'gsFillOpacityF',
+ 'gsFillScaleF', 'gsFont', 'gsFontAspectF', 'gsFontColor',
+ 'gsFontHeightF', 'gsFontOpacityF', 'gsFontQuality',
+ 'gsFontThicknessF', 'gsLineColor', 'gsLineDashPattern',
+ 'gsLineDashSegLenF', 'gsLineLabelConstantSpacingF', 'gsLineLabelFont',
+ 'gsLineLabelFontAspectF', 'gsLineLabelFontColor',
+ 'gsLineLabelFontHeightF', 'gsLineLabelFontQuality',
+ 'gsLineLabelFontThicknessF', 'gsLineLabelFuncCode',
+ 'gsLineLabelString', 'gsLineOpacityF', 'gsLineThicknessF',
+ 'gsMarkerColor', 'gsMarkerIndex', 'gsMarkerOpacityF', 'gsMarkerSizeF',
+ 'gsMarkerThicknessF', 'gsSegments', 'gsTextAngleF',
+ 'gsTextConstantSpacingF', 'gsTextDirection', 'gsTextFuncCode',
+ 'gsTextJustification', 'gsnAboveYRefLineBarColors',
+ 'gsnAboveYRefLineBarFillScales', 'gsnAboveYRefLineBarPatterns',
+ 'gsnAboveYRefLineColor', 'gsnAddCyclic', 'gsnAttachBorderOn',
+ 'gsnAttachPlotsXAxis', 'gsnBelowYRefLineBarColors',
+ 'gsnBelowYRefLineBarFillScales', 'gsnBelowYRefLineBarPatterns',
+ 'gsnBelowYRefLineColor', 'gsnBoxMargin', 'gsnCenterString',
+ 'gsnCenterStringFontColor', 'gsnCenterStringFontHeightF',
+ 'gsnCenterStringFuncCode', 'gsnCenterStringOrthogonalPosF',
+ 'gsnCenterStringParallelPosF', 'gsnContourLineThicknessesScale',
+ 'gsnContourNegLineDashPattern', 'gsnContourPosLineDashPattern',
+ 'gsnContourZeroLineThicknessF', 'gsnDebugWriteFileName', 'gsnDraw',
+ 'gsnFrame', 'gsnHistogramBarWidthPercent', 'gsnHistogramBinIntervals',
+ 'gsnHistogramBinMissing', 'gsnHistogramBinWidth',
+ 'gsnHistogramClassIntervals', 'gsnHistogramCompare',
+ 'gsnHistogramComputePercentages',
+ 'gsnHistogramComputePercentagesNoMissing',
+ 'gsnHistogramDiscreteBinValues', 'gsnHistogramDiscreteClassValues',
+ 'gsnHistogramHorizontal', 'gsnHistogramMinMaxBinsOn',
+ 'gsnHistogramNumberOfBins', 'gsnHistogramPercentSign',
+ 'gsnHistogramSelectNiceIntervals', 'gsnLeftString',
+ 'gsnLeftStringFontColor', 'gsnLeftStringFontHeightF',
+ 'gsnLeftStringFuncCode', 'gsnLeftStringOrthogonalPosF',
+ 'gsnLeftStringParallelPosF', 'gsnMajorLatSpacing',
+ 'gsnMajorLonSpacing', 'gsnMaskLambertConformal',
+ 'gsnMaskLambertConformalOutlineOn', 'gsnMaximize',
+ 'gsnMinorLatSpacing', 'gsnMinorLonSpacing', 'gsnPanelBottom',
+ 'gsnPanelCenter', 'gsnPanelDebug', 'gsnPanelFigureStrings',
+ 'gsnPanelFigureStringsBackgroundFillColor',
+ 'gsnPanelFigureStringsFontHeightF', 'gsnPanelFigureStringsJust',
+ 'gsnPanelFigureStringsPerimOn', 'gsnPanelLabelBar', 'gsnPanelLeft',
+ 'gsnPanelMainFont', 'gsnPanelMainFontColor',
+ 'gsnPanelMainFontHeightF', 'gsnPanelMainString', 'gsnPanelRight',
+ 'gsnPanelRowSpec', 'gsnPanelScalePlotIndex', 'gsnPanelTop',
+ 'gsnPanelXF', 'gsnPanelXWhiteSpacePercent', 'gsnPanelYF',
+ 'gsnPanelYWhiteSpacePercent', 'gsnPaperHeight', 'gsnPaperMargin',
+ 'gsnPaperOrientation', 'gsnPaperWidth', 'gsnPolar',
+ 'gsnPolarLabelDistance', 'gsnPolarLabelFont',
+ 'gsnPolarLabelFontHeightF', 'gsnPolarLabelSpacing', 'gsnPolarTime',
+ 'gsnPolarUT', 'gsnRightString', 'gsnRightStringFontColor',
+ 'gsnRightStringFontHeightF', 'gsnRightStringFuncCode',
+ 'gsnRightStringOrthogonalPosF', 'gsnRightStringParallelPosF',
+ 'gsnScalarContour', 'gsnScale', 'gsnShape', 'gsnSpreadColorEnd',
+ 'gsnSpreadColorStart', 'gsnSpreadColors', 'gsnStringFont',
+ 'gsnStringFontColor', 'gsnStringFontHeightF', 'gsnStringFuncCode',
+ 'gsnTickMarksOn', 'gsnXAxisIrregular2Linear', 'gsnXAxisIrregular2Log',
+ 'gsnXRefLine', 'gsnXRefLineColor', 'gsnXRefLineDashPattern',
+ 'gsnXRefLineThicknessF', 'gsnXYAboveFillColors', 'gsnXYBarChart',
+ 'gsnXYBarChartBarWidth', 'gsnXYBarChartColors',
+ 'gsnXYBarChartColors2', 'gsnXYBarChartFillDotSizeF',
+ 'gsnXYBarChartFillLineThicknessF', 'gsnXYBarChartFillOpacityF',
+ 'gsnXYBarChartFillScaleF', 'gsnXYBarChartOutlineOnly',
+ 'gsnXYBarChartOutlineThicknessF', 'gsnXYBarChartPatterns',
+ 'gsnXYBarChartPatterns2', 'gsnXYBelowFillColors', 'gsnXYFillColors',
+ 'gsnXYFillOpacities', 'gsnXYLeftFillColors', 'gsnXYRightFillColors',
+ 'gsnYAxisIrregular2Linear', 'gsnYAxisIrregular2Log', 'gsnYRefLine',
+ 'gsnYRefLineColor', 'gsnYRefLineColors', 'gsnYRefLineDashPattern',
+ 'gsnYRefLineDashPatterns', 'gsnYRefLineThicknessF',
+ 'gsnYRefLineThicknesses', 'gsnZonalMean', 'gsnZonalMeanXMaxF',
+ 'gsnZonalMeanXMinF', 'gsnZonalMeanYRefLine', 'lbAutoManage',
+ 'lbBottomMarginF', 'lbBoxCount', 'lbBoxEndCapStyle', 'lbBoxFractions',
+ 'lbBoxLineColor', 'lbBoxLineDashPattern', 'lbBoxLineDashSegLenF',
+ 'lbBoxLineThicknessF', 'lbBoxLinesOn', 'lbBoxMajorExtentF',
+ 'lbBoxMinorExtentF', 'lbBoxSeparatorLinesOn', 'lbBoxSizing',
+ 'lbFillBackground', 'lbFillColor', 'lbFillColors', 'lbFillDotSizeF',
+ 'lbFillLineThicknessF', 'lbFillPattern', 'lbFillPatterns',
+ 'lbFillScaleF', 'lbFillScales', 'lbJustification', 'lbLabelAlignment',
+ 'lbLabelAngleF', 'lbLabelAutoStride', 'lbLabelBarOn',
+ 'lbLabelConstantSpacingF', 'lbLabelDirection', 'lbLabelFont',
+ 'lbLabelFontAspectF', 'lbLabelFontColor', 'lbLabelFontHeightF',
+ 'lbLabelFontQuality', 'lbLabelFontThicknessF', 'lbLabelFuncCode',
+ 'lbLabelJust', 'lbLabelOffsetF', 'lbLabelPosition', 'lbLabelStride',
+ 'lbLabelStrings', 'lbLabelsOn', 'lbLeftMarginF', 'lbMaxLabelLenF',
+ 'lbMinLabelSpacingF', 'lbMonoFillColor', 'lbMonoFillPattern',
+ 'lbMonoFillScale', 'lbOrientation', 'lbPerimColor',
+ 'lbPerimDashPattern', 'lbPerimDashSegLenF', 'lbPerimFill',
+ 'lbPerimFillColor', 'lbPerimOn', 'lbPerimThicknessF',
+ 'lbRasterFillOn', 'lbRightMarginF', 'lbTitleAngleF',
+ 'lbTitleConstantSpacingF', 'lbTitleDirection', 'lbTitleExtentF',
+ 'lbTitleFont', 'lbTitleFontAspectF', 'lbTitleFontColor',
+ 'lbTitleFontHeightF', 'lbTitleFontQuality', 'lbTitleFontThicknessF',
+ 'lbTitleFuncCode', 'lbTitleJust', 'lbTitleOffsetF', 'lbTitleOn',
+ 'lbTitlePosition', 'lbTitleString', 'lbTopMarginF', 'lgAutoManage',
+ 'lgBottomMarginF', 'lgBoxBackground', 'lgBoxLineColor',
+ 'lgBoxLineDashPattern', 'lgBoxLineDashSegLenF', 'lgBoxLineThicknessF',
+ 'lgBoxLinesOn', 'lgBoxMajorExtentF', 'lgBoxMinorExtentF',
+ 'lgDashIndex', 'lgDashIndexes', 'lgItemCount', 'lgItemOrder',
+ 'lgItemPlacement', 'lgItemPositions', 'lgItemType', 'lgItemTypes',
+ 'lgJustification', 'lgLabelAlignment', 'lgLabelAngleF',
+ 'lgLabelAutoStride', 'lgLabelConstantSpacingF', 'lgLabelDirection',
+ 'lgLabelFont', 'lgLabelFontAspectF', 'lgLabelFontColor',
+ 'lgLabelFontHeightF', 'lgLabelFontQuality', 'lgLabelFontThicknessF',
+ 'lgLabelFuncCode', 'lgLabelJust', 'lgLabelOffsetF', 'lgLabelPosition',
+ 'lgLabelStride', 'lgLabelStrings', 'lgLabelsOn', 'lgLeftMarginF',
+ 'lgLegendOn', 'lgLineColor', 'lgLineColors', 'lgLineDashSegLenF',
+ 'lgLineDashSegLens', 'lgLineLabelConstantSpacingF', 'lgLineLabelFont',
+ 'lgLineLabelFontAspectF', 'lgLineLabelFontColor',
+ 'lgLineLabelFontColors', 'lgLineLabelFontHeightF',
+ 'lgLineLabelFontHeights', 'lgLineLabelFontQuality',
+ 'lgLineLabelFontThicknessF', 'lgLineLabelFuncCode',
+ 'lgLineLabelStrings', 'lgLineLabelsOn', 'lgLineThicknessF',
+ 'lgLineThicknesses', 'lgMarkerColor', 'lgMarkerColors',
+ 'lgMarkerIndex', 'lgMarkerIndexes', 'lgMarkerSizeF', 'lgMarkerSizes',
+ 'lgMarkerThicknessF', 'lgMarkerThicknesses', 'lgMonoDashIndex',
+ 'lgMonoItemType', 'lgMonoLineColor', 'lgMonoLineDashSegLen',
+ 'lgMonoLineLabelFontColor', 'lgMonoLineLabelFontHeight',
+ 'lgMonoLineThickness', 'lgMonoMarkerColor', 'lgMonoMarkerIndex',
+ 'lgMonoMarkerSize', 'lgMonoMarkerThickness', 'lgOrientation',
+ 'lgPerimColor', 'lgPerimDashPattern', 'lgPerimDashSegLenF',
+ 'lgPerimFill', 'lgPerimFillColor', 'lgPerimOn', 'lgPerimThicknessF',
+ 'lgRightMarginF', 'lgTitleAngleF', 'lgTitleConstantSpacingF',
+ 'lgTitleDirection', 'lgTitleExtentF', 'lgTitleFont',
+ 'lgTitleFontAspectF', 'lgTitleFontColor', 'lgTitleFontHeightF',
+ 'lgTitleFontQuality', 'lgTitleFontThicknessF', 'lgTitleFuncCode',
+ 'lgTitleJust', 'lgTitleOffsetF', 'lgTitleOn', 'lgTitlePosition',
+ 'lgTitleString', 'lgTopMarginF', 'mpAreaGroupCount',
+ 'mpAreaMaskingOn', 'mpAreaNames', 'mpAreaTypes', 'mpBottomAngleF',
+ 'mpBottomMapPosF', 'mpBottomNDCF', 'mpBottomNPCF',
+ 'mpBottomPointLatF', 'mpBottomPointLonF', 'mpBottomWindowF',
+ 'mpCenterLatF', 'mpCenterLonF', 'mpCenterRotF', 'mpCountyLineColor',
+ 'mpCountyLineDashPattern', 'mpCountyLineDashSegLenF',
+ 'mpCountyLineThicknessF', 'mpDataBaseVersion', 'mpDataResolution',
+ 'mpDataSetName', 'mpDefaultFillColor', 'mpDefaultFillPattern',
+ 'mpDefaultFillScaleF', 'mpDynamicAreaGroups', 'mpEllipticalBoundary',
+ 'mpFillAreaSpecifiers', 'mpFillBoundarySets', 'mpFillColor',
+ 'mpFillColors', 'mpFillColors-default', 'mpFillDotSizeF',
+ 'mpFillDrawOrder', 'mpFillOn', 'mpFillPatternBackground',
+ 'mpFillPattern', 'mpFillPatterns', 'mpFillPatterns-default',
+ 'mpFillScaleF', 'mpFillScales', 'mpFillScales-default',
+ 'mpFixedAreaGroups', 'mpGeophysicalLineColor',
+ 'mpGeophysicalLineDashPattern', 'mpGeophysicalLineDashSegLenF',
+ 'mpGeophysicalLineThicknessF', 'mpGreatCircleLinesOn',
+ 'mpGridAndLimbDrawOrder', 'mpGridAndLimbOn', 'mpGridLatSpacingF',
+ 'mpGridLineColor', 'mpGridLineDashPattern', 'mpGridLineDashSegLenF',
+ 'mpGridLineThicknessF', 'mpGridLonSpacingF', 'mpGridMaskMode',
+ 'mpGridMaxLatF', 'mpGridPolarLonSpacingF', 'mpGridSpacingF',
+ 'mpInlandWaterFillColor', 'mpInlandWaterFillPattern',
+ 'mpInlandWaterFillScaleF', 'mpLabelDrawOrder', 'mpLabelFontColor',
+ 'mpLabelFontHeightF', 'mpLabelsOn', 'mpLambertMeridianF',
+ 'mpLambertParallel1F', 'mpLambertParallel2F', 'mpLandFillColor',
+ 'mpLandFillPattern', 'mpLandFillScaleF', 'mpLeftAngleF',
+ 'mpLeftCornerLatF', 'mpLeftCornerLonF', 'mpLeftMapPosF',
+ 'mpLeftNDCF', 'mpLeftNPCF', 'mpLeftPointLatF',
+ 'mpLeftPointLonF', 'mpLeftWindowF', 'mpLimbLineColor',
+ 'mpLimbLineDashPattern', 'mpLimbLineDashSegLenF',
+ 'mpLimbLineThicknessF', 'mpLimitMode', 'mpMaskAreaSpecifiers',
+ 'mpMaskOutlineSpecifiers', 'mpMaxLatF', 'mpMaxLonF',
+ 'mpMinLatF', 'mpMinLonF', 'mpMonoFillColor', 'mpMonoFillPattern',
+ 'mpMonoFillScale', 'mpNationalLineColor', 'mpNationalLineDashPattern',
+ 'mpNationalLineThicknessF', 'mpOceanFillColor', 'mpOceanFillPattern',
+ 'mpOceanFillScaleF', 'mpOutlineBoundarySets', 'mpOutlineDrawOrder',
+ 'mpOutlineMaskingOn', 'mpOutlineOn', 'mpOutlineSpecifiers',
+ 'mpPerimDrawOrder', 'mpPerimLineColor', 'mpPerimLineDashPattern',
+ 'mpPerimLineDashSegLenF', 'mpPerimLineThicknessF', 'mpPerimOn',
+ 'mpPolyMode', 'mpProjection', 'mpProvincialLineColor',
+ 'mpProvincialLineDashPattern', 'mpProvincialLineDashSegLenF',
+ 'mpProvincialLineThicknessF', 'mpRelativeCenterLat',
+ 'mpRelativeCenterLon', 'mpRightAngleF', 'mpRightCornerLatF',
+ 'mpRightCornerLonF', 'mpRightMapPosF', 'mpRightNDCF',
+ 'mpRightNPCF', 'mpRightPointLatF', 'mpRightPointLonF',
+ 'mpRightWindowF', 'mpSatelliteAngle1F', 'mpSatelliteAngle2F',
+ 'mpSatelliteDistF', 'mpShapeMode', 'mpSpecifiedFillColors',
+ 'mpSpecifiedFillDirectIndexing', 'mpSpecifiedFillPatterns',
+ 'mpSpecifiedFillPriority', 'mpSpecifiedFillScales',
+ 'mpTopAngleF', 'mpTopMapPosF', 'mpTopNDCF', 'mpTopNPCF',
+ 'mpTopPointLatF', 'mpTopPointLonF', 'mpTopWindowF',
+ 'mpUSStateLineColor', 'mpUSStateLineDashPattern',
+ 'mpUSStateLineDashSegLenF', 'mpUSStateLineThicknessF',
+ 'pmAnnoManagers', 'pmAnnoViews', 'pmLabelBarDisplayMode',
+ 'pmLabelBarHeightF', 'pmLabelBarKeepAspect', 'pmLabelBarOrthogonalPosF',
+ 'pmLabelBarParallelPosF', 'pmLabelBarSide', 'pmLabelBarWidthF',
+ 'pmLabelBarZone', 'pmLegendDisplayMode', 'pmLegendHeightF',
+ 'pmLegendKeepAspect', 'pmLegendOrthogonalPosF',
+ 'pmLegendParallelPosF', 'pmLegendSide', 'pmLegendWidthF',
+ 'pmLegendZone', 'pmOverlaySequenceIds', 'pmTickMarkDisplayMode',
+ 'pmTickMarkZone', 'pmTitleDisplayMode', 'pmTitleZone',
+ 'prGraphicStyle', 'prPolyType', 'prXArray', 'prYArray',
+ 'sfCopyData', 'sfDataArray', 'sfDataMaxV', 'sfDataMinV',
+ 'sfElementNodes', 'sfExchangeDimensions', 'sfFirstNodeIndex',
+ 'sfMissingValueV', 'sfXArray', 'sfXCActualEndF', 'sfXCActualStartF',
+ 'sfXCEndIndex', 'sfXCEndSubsetV', 'sfXCEndV', 'sfXCStartIndex',
+ 'sfXCStartSubsetV', 'sfXCStartV', 'sfXCStride', 'sfXCellBounds',
+ 'sfYArray', 'sfYCActualEndF', 'sfYCActualStartF', 'sfYCEndIndex',
+ 'sfYCEndSubsetV', 'sfYCEndV', 'sfYCStartIndex', 'sfYCStartSubsetV',
+ 'sfYCStartV', 'sfYCStride', 'sfYCellBounds', 'stArrowLengthF',
+ 'stArrowStride', 'stCrossoverCheckCount',
+ 'stExplicitLabelBarLabelsOn', 'stLabelBarEndLabelsOn',
+ 'stLabelFormat', 'stLengthCheckCount', 'stLevelColors',
+ 'stLevelCount', 'stLevelPalette', 'stLevelSelectionMode',
+ 'stLevelSpacingF', 'stLevels', 'stLineColor', 'stLineOpacityF',
+ 'stLineStartStride', 'stLineThicknessF', 'stMapDirection',
+ 'stMaxLevelCount', 'stMaxLevelValF', 'stMinArrowSpacingF',
+ 'stMinDistanceF', 'stMinLevelValF', 'stMinLineSpacingF',
+ 'stMinStepFactorF', 'stMonoLineColor', 'stNoDataLabelOn',
+ 'stNoDataLabelString', 'stScalarFieldData', 'stScalarMissingValColor',
+ 'stSpanLevelPalette', 'stStepSizeF', 'stStreamlineDrawOrder',
+ 'stUseScalarArray', 'stVectorFieldData', 'stZeroFLabelAngleF',
+ 'stZeroFLabelBackgroundColor', 'stZeroFLabelConstantSpacingF',
+ 'stZeroFLabelFont', 'stZeroFLabelFontAspectF',
+ 'stZeroFLabelFontColor', 'stZeroFLabelFontHeightF',
+ 'stZeroFLabelFontQuality', 'stZeroFLabelFontThicknessF',
+ 'stZeroFLabelFuncCode', 'stZeroFLabelJust', 'stZeroFLabelOn',
+ 'stZeroFLabelOrthogonalPosF', 'stZeroFLabelParallelPosF',
+ 'stZeroFLabelPerimColor', 'stZeroFLabelPerimOn',
+ 'stZeroFLabelPerimSpaceF', 'stZeroFLabelPerimThicknessF',
+ 'stZeroFLabelSide', 'stZeroFLabelString', 'stZeroFLabelTextDirection',
+ 'stZeroFLabelZone', 'tfDoNDCOverlay', 'tfPlotManagerOn',
+ 'tfPolyDrawList', 'tfPolyDrawOrder', 'tiDeltaF', 'tiMainAngleF',
+ 'tiMainConstantSpacingF', 'tiMainDirection', 'tiMainFont',
+ 'tiMainFontAspectF', 'tiMainFontColor', 'tiMainFontHeightF',
+ 'tiMainFontQuality', 'tiMainFontThicknessF', 'tiMainFuncCode',
+ 'tiMainJust', 'tiMainOffsetXF', 'tiMainOffsetYF', 'tiMainOn',
+ 'tiMainPosition', 'tiMainSide', 'tiMainString', 'tiUseMainAttributes',
+ 'tiXAxisAngleF', 'tiXAxisConstantSpacingF', 'tiXAxisDirection',
+ 'tiXAxisFont', 'tiXAxisFontAspectF', 'tiXAxisFontColor',
+ 'tiXAxisFontHeightF', 'tiXAxisFontQuality', 'tiXAxisFontThicknessF',
+ 'tiXAxisFuncCode', 'tiXAxisJust', 'tiXAxisOffsetXF',
+ 'tiXAxisOffsetYF', 'tiXAxisOn', 'tiXAxisPosition', 'tiXAxisSide',
+ 'tiXAxisString', 'tiYAxisAngleF', 'tiYAxisConstantSpacingF',
+ 'tiYAxisDirection', 'tiYAxisFont', 'tiYAxisFontAspectF',
+ 'tiYAxisFontColor', 'tiYAxisFontHeightF', 'tiYAxisFontQuality',
+ 'tiYAxisFontThicknessF', 'tiYAxisFuncCode', 'tiYAxisJust',
+ 'tiYAxisOffsetXF', 'tiYAxisOffsetYF', 'tiYAxisOn', 'tiYAxisPosition',
+ 'tiYAxisSide', 'tiYAxisString', 'tmBorderLineColor',
+ 'tmBorderThicknessF', 'tmEqualizeXYSizes', 'tmLabelAutoStride',
+ 'tmSciNoteCutoff', 'tmXBAutoPrecision', 'tmXBBorderOn',
+ 'tmXBDataLeftF', 'tmXBDataRightF', 'tmXBFormat', 'tmXBIrrTensionF',
+ 'tmXBIrregularPoints', 'tmXBLabelAngleF', 'tmXBLabelConstantSpacingF',
+ 'tmXBLabelDeltaF', 'tmXBLabelDirection', 'tmXBLabelFont',
+ 'tmXBLabelFontAspectF', 'tmXBLabelFontColor', 'tmXBLabelFontHeightF',
+ 'tmXBLabelFontQuality', 'tmXBLabelFontThicknessF',
+ 'tmXBLabelFuncCode', 'tmXBLabelJust', 'tmXBLabelStride', 'tmXBLabels',
+ 'tmXBLabelsOn', 'tmXBMajorLengthF', 'tmXBMajorLineColor',
+ 'tmXBMajorOutwardLengthF', 'tmXBMajorThicknessF', 'tmXBMaxLabelLenF',
+ 'tmXBMaxTicks', 'tmXBMinLabelSpacingF', 'tmXBMinorLengthF',
+ 'tmXBMinorLineColor', 'tmXBMinorOn', 'tmXBMinorOutwardLengthF',
+ 'tmXBMinorPerMajor', 'tmXBMinorThicknessF', 'tmXBMinorValues',
+ 'tmXBMode', 'tmXBOn', 'tmXBPrecision', 'tmXBStyle', 'tmXBTickEndF',
+ 'tmXBTickSpacingF', 'tmXBTickStartF', 'tmXBValues', 'tmXMajorGrid',
+ 'tmXMajorGridLineColor', 'tmXMajorGridLineDashPattern',
+ 'tmXMajorGridThicknessF', 'tmXMinorGrid', 'tmXMinorGridLineColor',
+ 'tmXMinorGridLineDashPattern', 'tmXMinorGridThicknessF',
+ 'tmXTAutoPrecision', 'tmXTBorderOn', 'tmXTDataLeftF',
+ 'tmXTDataRightF', 'tmXTFormat', 'tmXTIrrTensionF',
+ 'tmXTIrregularPoints', 'tmXTLabelAngleF', 'tmXTLabelConstantSpacingF',
+ 'tmXTLabelDeltaF', 'tmXTLabelDirection', 'tmXTLabelFont',
+ 'tmXTLabelFontAspectF', 'tmXTLabelFontColor', 'tmXTLabelFontHeightF',
+ 'tmXTLabelFontQuality', 'tmXTLabelFontThicknessF',
+ 'tmXTLabelFuncCode', 'tmXTLabelJust', 'tmXTLabelStride', 'tmXTLabels',
+ 'tmXTLabelsOn', 'tmXTMajorLengthF', 'tmXTMajorLineColor',
+ 'tmXTMajorOutwardLengthF', 'tmXTMajorThicknessF', 'tmXTMaxLabelLenF',
+ 'tmXTMaxTicks', 'tmXTMinLabelSpacingF', 'tmXTMinorLengthF',
+ 'tmXTMinorLineColor', 'tmXTMinorOn', 'tmXTMinorOutwardLengthF',
+ 'tmXTMinorPerMajor', 'tmXTMinorThicknessF', 'tmXTMinorValues',
+ 'tmXTMode', 'tmXTOn', 'tmXTPrecision', 'tmXTStyle', 'tmXTTickEndF',
+ 'tmXTTickSpacingF', 'tmXTTickStartF', 'tmXTValues', 'tmXUseBottom',
+ 'tmYLAutoPrecision', 'tmYLBorderOn', 'tmYLDataBottomF',
+ 'tmYLDataTopF', 'tmYLFormat', 'tmYLIrrTensionF',
+ 'tmYLIrregularPoints', 'tmYLLabelAngleF', 'tmYLLabelConstantSpacingF',
+ 'tmYLLabelDeltaF', 'tmYLLabelDirection', 'tmYLLabelFont',
+ 'tmYLLabelFontAspectF', 'tmYLLabelFontColor', 'tmYLLabelFontHeightF',
+ 'tmYLLabelFontQuality', 'tmYLLabelFontThicknessF',
+ 'tmYLLabelFuncCode', 'tmYLLabelJust', 'tmYLLabelStride', 'tmYLLabels',
+ 'tmYLLabelsOn', 'tmYLMajorLengthF', 'tmYLMajorLineColor',
+ 'tmYLMajorOutwardLengthF', 'tmYLMajorThicknessF', 'tmYLMaxLabelLenF',
+ 'tmYLMaxTicks', 'tmYLMinLabelSpacingF', 'tmYLMinorLengthF',
+ 'tmYLMinorLineColor', 'tmYLMinorOn', 'tmYLMinorOutwardLengthF',
+ 'tmYLMinorPerMajor', 'tmYLMinorThicknessF', 'tmYLMinorValues',
+ 'tmYLMode', 'tmYLOn', 'tmYLPrecision', 'tmYLStyle', 'tmYLTickEndF',
+ 'tmYLTickSpacingF', 'tmYLTickStartF', 'tmYLValues', 'tmYMajorGrid',
+ 'tmYMajorGridLineColor', 'tmYMajorGridLineDashPattern',
+ 'tmYMajorGridThicknessF', 'tmYMinorGrid', 'tmYMinorGridLineColor',
+ 'tmYMinorGridLineDashPattern', 'tmYMinorGridThicknessF',
+ 'tmYRAutoPrecision', 'tmYRBorderOn', 'tmYRDataBottomF',
+ 'tmYRDataTopF', 'tmYRFormat', 'tmYRIrrTensionF',
+ 'tmYRIrregularPoints', 'tmYRLabelAngleF', 'tmYRLabelConstantSpacingF',
+ 'tmYRLabelDeltaF', 'tmYRLabelDirection', 'tmYRLabelFont',
+ 'tmYRLabelFontAspectF', 'tmYRLabelFontColor', 'tmYRLabelFontHeightF',
+ 'tmYRLabelFontQuality', 'tmYRLabelFontThicknessF',
+ 'tmYRLabelFuncCode', 'tmYRLabelJust', 'tmYRLabelStride', 'tmYRLabels',
+ 'tmYRLabelsOn', 'tmYRMajorLengthF', 'tmYRMajorLineColor',
+ 'tmYRMajorOutwardLengthF', 'tmYRMajorThicknessF', 'tmYRMaxLabelLenF',
+ 'tmYRMaxTicks', 'tmYRMinLabelSpacingF', 'tmYRMinorLengthF',
+ 'tmYRMinorLineColor', 'tmYRMinorOn', 'tmYRMinorOutwardLengthF',
+ 'tmYRMinorPerMajor', 'tmYRMinorThicknessF', 'tmYRMinorValues',
+ 'tmYRMode', 'tmYROn', 'tmYRPrecision', 'tmYRStyle', 'tmYRTickEndF',
+ 'tmYRTickSpacingF', 'tmYRTickStartF', 'tmYRValues', 'tmYUseLeft',
+ 'trGridType', 'trLineInterpolationOn',
+ 'trXAxisType', 'trXCoordPoints', 'trXInterPoints', 'trXLog',
+ 'trXMaxF', 'trXMinF', 'trXReverse', 'trXSamples', 'trXTensionF',
+ 'trYAxisType', 'trYCoordPoints', 'trYInterPoints', 'trYLog',
+ 'trYMaxF', 'trYMinF', 'trYReverse', 'trYSamples', 'trYTensionF',
+ 'txAngleF', 'txBackgroundFillColor', 'txConstantSpacingF', 'txDirection',
+ 'txFont', 'HLU-Fonts', 'txFontAspectF', 'txFontColor',
+ 'txFontHeightF', 'txFontOpacityF', 'txFontQuality',
+ 'txFontThicknessF', 'txFuncCode', 'txJust', 'txPerimColor',
+ 'txPerimDashLengthF', 'txPerimDashPattern', 'txPerimOn',
+ 'txPerimSpaceF', 'txPerimThicknessF', 'txPosXF', 'txPosYF',
+ 'txString', 'vcExplicitLabelBarLabelsOn', 'vcFillArrowEdgeColor',
+ 'vcFillArrowEdgeThicknessF', 'vcFillArrowFillColor',
+ 'vcFillArrowHeadInteriorXF', 'vcFillArrowHeadMinFracXF',
+ 'vcFillArrowHeadMinFracYF', 'vcFillArrowHeadXF', 'vcFillArrowHeadYF',
+ 'vcFillArrowMinFracWidthF', 'vcFillArrowWidthF', 'vcFillArrowsOn',
+ 'vcFillOverEdge', 'vcGlyphOpacityF', 'vcGlyphStyle',
+ 'vcLabelBarEndLabelsOn', 'vcLabelFontColor', 'vcLabelFontHeightF',
+ 'vcLabelsOn', 'vcLabelsUseVectorColor', 'vcLevelColors',
+ 'vcLevelCount', 'vcLevelPalette', 'vcLevelSelectionMode',
+ 'vcLevelSpacingF', 'vcLevels', 'vcLineArrowColor',
+ 'vcLineArrowHeadMaxSizeF', 'vcLineArrowHeadMinSizeF',
+ 'vcLineArrowThicknessF', 'vcMagnitudeFormat',
+ 'vcMagnitudeScaleFactorF', 'vcMagnitudeScaleValueF',
+ 'vcMagnitudeScalingMode', 'vcMapDirection', 'vcMaxLevelCount',
+ 'vcMaxLevelValF', 'vcMaxMagnitudeF', 'vcMinAnnoAngleF',
+ 'vcMinAnnoArrowAngleF', 'vcMinAnnoArrowEdgeColor',
+ 'vcMinAnnoArrowFillColor', 'vcMinAnnoArrowLineColor',
+ 'vcMinAnnoArrowMinOffsetF', 'vcMinAnnoArrowSpaceF',
+ 'vcMinAnnoArrowUseVecColor', 'vcMinAnnoBackgroundColor',
+ 'vcMinAnnoConstantSpacingF', 'vcMinAnnoExplicitMagnitudeF',
+ 'vcMinAnnoFont', 'vcMinAnnoFontAspectF', 'vcMinAnnoFontColor',
+ 'vcMinAnnoFontHeightF', 'vcMinAnnoFontQuality',
+ 'vcMinAnnoFontThicknessF', 'vcMinAnnoFuncCode', 'vcMinAnnoJust',
+ 'vcMinAnnoOn', 'vcMinAnnoOrientation', 'vcMinAnnoOrthogonalPosF',
+ 'vcMinAnnoParallelPosF', 'vcMinAnnoPerimColor', 'vcMinAnnoPerimOn',
+ 'vcMinAnnoPerimSpaceF', 'vcMinAnnoPerimThicknessF', 'vcMinAnnoSide',
+ 'vcMinAnnoString1', 'vcMinAnnoString1On', 'vcMinAnnoString2',
+ 'vcMinAnnoString2On', 'vcMinAnnoTextDirection', 'vcMinAnnoZone',
+ 'vcMinDistanceF', 'vcMinFracLengthF', 'vcMinLevelValF',
+ 'vcMinMagnitudeF', 'vcMonoFillArrowEdgeColor',
+ 'vcMonoFillArrowFillColor', 'vcMonoLineArrowColor',
+ 'vcMonoWindBarbColor', 'vcNoDataLabelOn', 'vcNoDataLabelString',
+ 'vcPositionMode', 'vcRefAnnoAngleF', 'vcRefAnnoArrowAngleF',
+ 'vcRefAnnoArrowEdgeColor', 'vcRefAnnoArrowFillColor',
+ 'vcRefAnnoArrowLineColor', 'vcRefAnnoArrowMinOffsetF',
+ 'vcRefAnnoArrowSpaceF', 'vcRefAnnoArrowUseVecColor',
+ 'vcRefAnnoBackgroundColor', 'vcRefAnnoConstantSpacingF',
+ 'vcRefAnnoExplicitMagnitudeF', 'vcRefAnnoFont',
+ 'vcRefAnnoFontAspectF', 'vcRefAnnoFontColor', 'vcRefAnnoFontHeightF',
+ 'vcRefAnnoFontQuality', 'vcRefAnnoFontThicknessF',
+ 'vcRefAnnoFuncCode', 'vcRefAnnoJust', 'vcRefAnnoOn',
+ 'vcRefAnnoOrientation', 'vcRefAnnoOrthogonalPosF',
+ 'vcRefAnnoParallelPosF', 'vcRefAnnoPerimColor', 'vcRefAnnoPerimOn',
+ 'vcRefAnnoPerimSpaceF', 'vcRefAnnoPerimThicknessF', 'vcRefAnnoSide',
+ 'vcRefAnnoString1', 'vcRefAnnoString1On', 'vcRefAnnoString2',
+ 'vcRefAnnoString2On', 'vcRefAnnoTextDirection', 'vcRefAnnoZone',
+ 'vcRefLengthF', 'vcRefMagnitudeF', 'vcScalarFieldData',
+ 'vcScalarMissingValColor', 'vcScalarValueFormat',
+ 'vcScalarValueScaleFactorF', 'vcScalarValueScaleValueF',
+ 'vcScalarValueScalingMode', 'vcSpanLevelPalette', 'vcUseRefAnnoRes',
+ 'vcUseScalarArray', 'vcVectorDrawOrder', 'vcVectorFieldData',
+ 'vcWindBarbCalmCircleSizeF', 'vcWindBarbColor',
+ 'vcWindBarbLineThicknessF', 'vcWindBarbScaleFactorF',
+ 'vcWindBarbTickAngleF', 'vcWindBarbTickLengthF',
+ 'vcWindBarbTickSpacingF', 'vcZeroFLabelAngleF',
+ 'vcZeroFLabelBackgroundColor', 'vcZeroFLabelConstantSpacingF',
+ 'vcZeroFLabelFont', 'vcZeroFLabelFontAspectF',
+ 'vcZeroFLabelFontColor', 'vcZeroFLabelFontHeightF',
+ 'vcZeroFLabelFontQuality', 'vcZeroFLabelFontThicknessF',
+ 'vcZeroFLabelFuncCode', 'vcZeroFLabelJust', 'vcZeroFLabelOn',
+ 'vcZeroFLabelOrthogonalPosF', 'vcZeroFLabelParallelPosF',
+ 'vcZeroFLabelPerimColor', 'vcZeroFLabelPerimOn',
+ 'vcZeroFLabelPerimSpaceF', 'vcZeroFLabelPerimThicknessF',
+ 'vcZeroFLabelSide', 'vcZeroFLabelString', 'vcZeroFLabelTextDirection',
+ 'vcZeroFLabelZone', 'vfCopyData', 'vfDataArray',
+ 'vfExchangeDimensions', 'vfExchangeUVData', 'vfMagMaxV', 'vfMagMinV',
+ 'vfMissingUValueV', 'vfMissingVValueV', 'vfPolarData',
+ 'vfSingleMissingValue', 'vfUDataArray', 'vfUMaxV', 'vfUMinV',
+ 'vfVDataArray', 'vfVMaxV', 'vfVMinV', 'vfXArray', 'vfXCActualEndF',
+ 'vfXCActualStartF', 'vfXCEndIndex', 'vfXCEndSubsetV', 'vfXCEndV',
+ 'vfXCStartIndex', 'vfXCStartSubsetV', 'vfXCStartV', 'vfXCStride',
+ 'vfYArray', 'vfYCActualEndF', 'vfYCActualStartF', 'vfYCEndIndex',
+ 'vfYCEndSubsetV', 'vfYCEndV', 'vfYCStartIndex', 'vfYCStartSubsetV',
+ 'vfYCStartV', 'vfYCStride', 'vpAnnoManagerId', 'vpClipOn',
+ 'vpHeightF', 'vpKeepAspect', 'vpOn', 'vpUseSegments', 'vpWidthF',
+ 'vpXF', 'vpYF', 'wkAntiAlias', 'wkBackgroundColor', 'wkBackgroundOpacityF',
+ 'wkColorMapLen', 'wkColorMap', 'wkColorModel', 'wkDashTableLength',
+ 'wkDefGraphicStyleId', 'wkDeviceLowerX', 'wkDeviceLowerY',
+ 'wkDeviceUpperX', 'wkDeviceUpperY', 'wkFileName', 'wkFillTableLength',
+ 'wkForegroundColor', 'wkFormat', 'wkFullBackground', 'wkGksWorkId',
+ 'wkHeight', 'wkMarkerTableLength', 'wkMetaName', 'wkOrientation',
+ 'wkPDFFileName', 'wkPDFFormat', 'wkPDFResolution', 'wkPSFileName',
+ 'wkPSFormat', 'wkPSResolution', 'wkPaperHeightF', 'wkPaperSize',
+ 'wkPaperWidthF', 'wkPause', 'wkTopLevelViews', 'wkViews',
+ 'wkVisualType', 'wkWidth', 'wkWindowId', 'wkXColorMode', 'wsCurrentSize',
+ 'wsMaximumSize', 'wsThresholdSize', 'xyComputeXMax',
+ 'xyComputeXMin', 'xyComputeYMax', 'xyComputeYMin', 'xyCoordData',
+ 'xyCoordDataSpec', 'xyCurveDrawOrder', 'xyDashPattern',
+ 'xyDashPatterns', 'xyExplicitLabels', 'xyExplicitLegendLabels',
+ 'xyLabelMode', 'xyLineColor', 'xyLineColors', 'xyLineDashSegLenF',
+ 'xyLineLabelConstantSpacingF', 'xyLineLabelFont',
+ 'xyLineLabelFontAspectF', 'xyLineLabelFontColor',
+ 'xyLineLabelFontColors', 'xyLineLabelFontHeightF',
+ 'xyLineLabelFontQuality', 'xyLineLabelFontThicknessF',
+ 'xyLineLabelFuncCode', 'xyLineThicknessF', 'xyLineThicknesses',
+ 'xyMarkLineMode', 'xyMarkLineModes', 'xyMarker', 'xyMarkerColor',
+ 'xyMarkerColors', 'xyMarkerSizeF', 'xyMarkerSizes',
+ 'xyMarkerThicknessF', 'xyMarkerThicknesses', 'xyMarkers',
+ 'xyMonoDashPattern', 'xyMonoLineColor', 'xyMonoLineLabelFontColor',
+ 'xyMonoLineThickness', 'xyMonoMarkLineMode', 'xyMonoMarker',
+ 'xyMonoMarkerColor', 'xyMonoMarkerSize', 'xyMonoMarkerThickness',
+ 'xyXIrrTensionF', 'xyXIrregularPoints', 'xyXStyle', 'xyYIrrTensionF',
+ 'xyYIrregularPoints', 'xyYStyle'), prefix=r'\b'),
+ Name.Builtin),
+
+ # Booleans
+ (r'\.(True|False)\.', Name.Builtin),
+ # Comparing Operators
+ (r'\.(eq|ne|lt|le|gt|ge|not|and|or|xor)\.', Operator.Word),
+ ],
+
+ 'strings': [
+ (r'(?s)"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double),
+ ],
+
+ 'nums': [
+ (r'\d+(?![.e])(_[a-z]\w+)?', Number.Integer),
+ (r'[+-]?\d*\.\d+(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float),
+ (r'[+-]?\d+\.\d*(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/nimrod.py b/wandb/vendor/pygments/lexers/nimrod.py
new file mode 100644
index 0000000000000000000000000000000000000000..d438c1bfc1ca4f43d3c27a86df229e8642f3df8d
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/nimrod.py
@@ -0,0 +1,159 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.nimrod
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for the Nim language (formerly known as Nimrod).
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Error
+
+__all__ = ['NimrodLexer']
+
+
+class NimrodLexer(RegexLexer):
+ """
+ For `Nim <http://nim-lang.org/>`_ source code.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'Nimrod'
+ aliases = ['nim', 'nimrod']
+ filenames = ['*.nim', '*.nimrod']
+ mimetypes = ['text/x-nim']
+
+ flags = re.MULTILINE | re.IGNORECASE | re.UNICODE
+
+ def underscorize(words):
+ newWords = []
+ new = ""
+ for word in words:
+ for ch in word:
+ new += (ch + "_?")
+ newWords.append(new)
+ new = ""
+ return "|".join(newWords)
+
+ keywords = [
+ 'addr', 'and', 'as', 'asm', 'atomic', 'bind', 'block', 'break', 'case',
+ 'cast', 'concept', 'const', 'continue', 'converter', 'defer', 'discard',
+ 'distinct', 'div', 'do', 'elif', 'else', 'end', 'enum', 'except',
+ 'export', 'finally', 'for', 'func', 'if', 'in', 'yield', 'interface',
+ 'is', 'isnot', 'iterator', 'let', 'macro', 'method', 'mixin', 'mod',
+ 'not', 'notin', 'object', 'of', 'or', 'out', 'proc', 'ptr', 'raise',
+ 'ref', 'return', 'shared', 'shl', 'shr', 'static', 'template', 'try',
+ 'tuple', 'type', 'when', 'while', 'with', 'without', 'xor'
+ ]
+
+ keywordsPseudo = [
+ 'nil', 'true', 'false'
+ ]
+
+ opWords = [
+ 'and', 'or', 'not', 'xor', 'shl', 'shr', 'div', 'mod', 'in',
+ 'notin', 'is', 'isnot'
+ ]
+
+ types = [
+ 'int', 'int8', 'int16', 'int32', 'int64', 'float', 'float32', 'float64',
+ 'bool', 'char', 'range', 'array', 'seq', 'set', 'string'
+ ]
+
+ tokens = {
+ 'root': [
+ (r'##.*$', String.Doc),
+ (r'#.*$', Comment),
+ (r'[*=><+\-/@$~&%!?|\\\[\]]', Operator),
+ (r'\.\.|\.|,|\[\.|\.\]|\{\.|\.\}|\(\.|\.\)|\{|\}|\(|\)|:|\^|`|;',
+ Punctuation),
+
+ # Strings
+ (r'(?:[\w]+)"', String, 'rdqs'),
+ (r'"""', String, 'tdqs'),
+ ('"', String, 'dqs'),
+
+ # Char
+ ("'", String.Char, 'chars'),
+
+ # Keywords
+ (r'(%s)\b' % underscorize(opWords), Operator.Word),
+ (r'(p_?r_?o_?c_?\s)(?![(\[\]])', Keyword, 'funcname'),
+ (r'(%s)\b' % underscorize(keywords), Keyword),
+ (r'(%s)\b' % underscorize(['from', 'import', 'include']),
+ Keyword.Namespace),
+ (r'(v_?a_?r)\b', Keyword.Declaration),
+ (r'(%s)\b' % underscorize(types), Keyword.Type),
+ (r'(%s)\b' % underscorize(keywordsPseudo), Keyword.Pseudo),
+ # Identifiers
+ (r'\b((?![_\d])\w)(((?!_)\w)|(_(?!_)\w))*', Name),
+ # Numbers
+ (r'[0-9][0-9_]*(?=([e.]|\'f(32|64)))',
+ Number.Float, ('float-suffix', 'float-number')),
+ (r'0x[a-f0-9][a-f0-9_]*', Number.Hex, 'int-suffix'),
+ (r'0b[01][01_]*', Number.Bin, 'int-suffix'),
+ (r'0o[0-7][0-7_]*', Number.Oct, 'int-suffix'),
+ (r'[0-9][0-9_]*', Number.Integer, 'int-suffix'),
+ # Whitespace
+ (r'\s+', Text),
+ (r'.+$', Error),
+ ],
+ 'chars': [
+ (r'\\([\\abcefnrtvl"\']|x[a-f0-9]{2}|[0-9]{1,3})', String.Escape),
+ (r"'", String.Char, '#pop'),
+ (r".", String.Char)
+ ],
+ 'strings': [
+ (r'(?<!\$)\$(\d+|#|\w+)+', String.Interpol),
+ (r'[^\\\'"$\n]+', String),
+ # quotes, dollars and backslashes must be parsed one at a time
+ (r'[\'"\\]', String),
+ # unhandled string formatting sign
+ (r'\$', String)
+ # newlines are an error (use "nl" state)
+ ],
+ 'dqs': [
+ (r'\\([\\abcefnrtvl"\']|\n|x[a-f0-9]{2}|[0-9]{1,3})',
+ String.Escape),
+ (r'"', String, '#pop'),
+ include('strings')
+ ],
+ 'rdqs': [
+ (r'"(?!")', String, '#pop'),
+ (r'""', String.Escape),
+ include('strings')
+ ],
+ 'tdqs': [
+ (r'"""(?!")', String, '#pop'),
+ include('strings'),
+ include('nl')
+ ],
+ 'funcname': [
+ (r'((?![\d_])\w)(((?!_)\w)|(_(?!_)\w))*', Name.Function, '#pop'),
+ (r'`.+`', Name.Function, '#pop')
+ ],
+ 'nl': [
+ (r'\n', String)
+ ],
+ 'float-number': [
+ (r'\.(?!\.)[0-9_]*', Number.Float),
+ (r'e[+-]?[0-9][0-9_]*', Number.Float),
+ default('#pop')
+ ],
+ 'float-suffix': [
+ (r'\'f(32|64)', Number.Float),
+ default('#pop')
+ ],
+ 'int-suffix': [
+ (r'\'i(32|64)', Number.Integer.Long),
+ (r'\'i(8|16)', Number.Integer),
+ default('#pop')
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/nit.py b/wandb/vendor/pygments/lexers/nit.py
new file mode 100644
index 0000000000000000000000000000000000000000..21116499d309a6010a6d49e48f6fa6bc551f2eca
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/nit.py
@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.nit
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexer for the Nit language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['NitLexer']
+
+
+class NitLexer(RegexLexer):
+ """
+ For `nit <http://nitlanguage.org>`_ source.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Nit'
+ aliases = ['nit']
+ filenames = ['*.nit']
+ tokens = {
+ 'root': [
+ (r'#.*?$', Comment.Single),
+ (words((
+ 'package', 'module', 'import', 'class', 'abstract', 'interface',
+ 'universal', 'enum', 'end', 'fun', 'type', 'init', 'redef',
+ 'isa', 'do', 'readable', 'writable', 'var', 'intern', 'extern',
+ 'public', 'protected', 'private', 'intrude', 'if', 'then',
+ 'else', 'while', 'loop', 'for', 'in', 'and', 'or', 'not',
+ 'implies', 'return', 'continue', 'break', 'abort', 'assert',
+ 'new', 'is', 'once', 'super', 'self', 'true', 'false', 'nullable',
+ 'null', 'as', 'isset', 'label', '__debug__'), suffix=r'(?=[\r\n\t( ])'),
+ Keyword),
+ (r'[A-Z]\w*', Name.Class),
+ (r'"""(([^\'\\]|\\.)|\\r|\\n)*((\{\{?)?(""?\{\{?)*""""*)', String), # Simple long string
+ (r'\'\'\'(((\\.|[^\'\\])|\\r|\\n)|\'((\\.|[^\'\\])|\\r|\\n)|'
+ r'\'\'((\\.|[^\'\\])|\\r|\\n))*\'\'\'', String), # Simple long string alt
+ (r'"""(([^\'\\]|\\.)|\\r|\\n)*((""?)?(\{\{?""?)*\{\{\{\{*)', String), # Start long string
+ (r'\}\}\}(((\\.|[^\'\\])|\\r|\\n))*(""?)?(\{\{?""?)*\{\{\{\{*', String), # Mid long string
+ (r'\}\}\}(((\\.|[^\'\\])|\\r|\\n))*(\{\{?)?(""?\{\{?)*""""*', String), # End long string
+ (r'"(\\.|([^"}{\\]))*"', String), # Simple String
+ (r'"(\\.|([^"}{\\]))*\{', String), # Start string
+ (r'\}(\\.|([^"}{\\]))*\{', String), # Mid String
+ (r'\}(\\.|([^"}{\\]))*"', String), # End String
+ (r'(\'[^\'\\]\')|(\'\\.\')', String.Char),
+ (r'[0-9]+', Number.Integer),
+ (r'[0-9]*.[0-9]+', Number.Float),
+ (r'0(x|X)[0-9A-Fa-f]+', Number.Hex),
+ (r'[a-z]\w*', Name),
+ (r'_\w+', Name.Variable.Instance),
+ (r'==|!=|<==>|>=|>>|>|<=|<<|<|\+|-|=|/|\*|%|\+=|-=|!|@', Operator),
+ (r'\(|\)|\[|\]|,|\.\.\.|\.\.|\.|::|:', Punctuation),
+ (r'`\{[^`]*`\}', Text), # Extern blocks won't be Lexed by Nit
+ (r'[\r\n\t ]+', Text),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/nix.py b/wandb/vendor/pygments/lexers/nix.py
new file mode 100644
index 0000000000000000000000000000000000000000..e148c9193886e50d60b4fba27dfac5ecc05f1055
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/nix.py
@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.nix
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the NixOS Nix language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Literal
+
+__all__ = ['NixLexer']
+
+
+class NixLexer(RegexLexer):
+ """
+ For the `Nix language <http://nixos.org/nix/>`_.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Nix'
+ aliases = ['nixos', 'nix']
+ filenames = ['*.nix']
+ mimetypes = ['text/x-nix']
+
+ flags = re.MULTILINE | re.UNICODE
+
+ keywords = ['rec', 'with', 'let', 'in', 'inherit', 'assert', 'if',
+ 'else', 'then', '...']
+ builtins = ['import', 'abort', 'baseNameOf', 'dirOf', 'isNull', 'builtins',
+ 'map', 'removeAttrs', 'throw', 'toString', 'derivation']
+ operators = ['++', '+', '?', '.', '!', '//', '==',
+ '!=', '&&', '||', '->', '=']
+
+ punctuations = ["(", ")", "[", "]", ";", "{", "}", ":", ",", "@"]
+
+ tokens = {
+ 'root': [
+ # comments starting with #
+ (r'#.*$', Comment.Single),
+
+ # multiline comments
+ (r'/\*', Comment.Multiline, 'comment'),
+
+ # whitespace
+ (r'\s+', Text),
+
+ # keywords
+ ('(%s)' % '|'.join(re.escape(entry) + '\\b' for entry in keywords), Keyword),
+
+ # highlight the builtins
+ ('(%s)' % '|'.join(re.escape(entry) + '\\b' for entry in builtins),
+ Name.Builtin),
+
+ (r'\b(true|false|null)\b', Name.Constant),
+
+ # operators
+ ('(%s)' % '|'.join(re.escape(entry) for entry in operators),
+ Operator),
+
+ # word operators
+ (r'\b(or|and)\b', Operator.Word),
+
+ # punctuations
+ ('(%s)' % '|'.join(re.escape(entry) for entry in punctuations), Punctuation),
+
+ # integers
+ (r'[0-9]+', Number.Integer),
+
+ # strings
+ (r'"', String.Double, 'doublequote'),
+ (r"''", String.Single, 'singlequote'),
+
+ # paths
+ (r'[\w.+-]*(\/[\w.+-]+)+', Literal),
+ (r'\<[\w.+-]+(\/[\w.+-]+)*\>', Literal),
+
+ # urls
+ (r'[a-zA-Z][a-zA-Z0-9\+\-\.]*\:[\w%/?:@&=+$,\\.!~*\'-]+', Literal),
+
+ # names of variables
+ (r'[\w-]+\s*=', String.Symbol),
+ (r'[a-zA-Z_][\w\'-]*', Text),
+
+ ],
+ 'comment': [
+ (r'[^/*]+', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline),
+ ],
+ 'singlequote': [
+ (r"'''", String.Escape),
+ (r"''\$\{", String.Escape),
+ (r"''\n", String.Escape),
+ (r"''\r", String.Escape),
+ (r"''\t", String.Escape),
+ (r"''", String.Single, '#pop'),
+ (r'\$\{', String.Interpol, 'antiquote'),
+ (r"[^']", String.Single),
+ ],
+ 'doublequote': [
+ (r'\\', String.Escape),
+ (r'\\"', String.Escape),
+ (r'\\$\{', String.Escape),
+ (r'"', String.Double, '#pop'),
+ (r'\$\{', String.Interpol, 'antiquote'),
+ (r'[^"]', String.Double),
+ ],
+ 'antiquote': [
+ (r"\}", String.Interpol, '#pop'),
+ # TODO: we should probably escape also here ''${ \${
+ (r"\$\{", String.Interpol, '#push'),
+ include('root'),
+ ],
+ }
+
+ def analyse_text(text):
+ rv = 0.0
+ # TODO: let/in
+ if re.search(r'import.+?<[^>]+>', text):
+ rv += 0.4
+ if re.search(r'mkDerivation\s+(\(|\{|rec)', text):
+ rv += 0.4
+ if re.search(r'=\s+mkIf\s+', text):
+ rv += 0.4
+ if re.search(r'\{[a-zA-Z,\s]+\}:', text):
+ rv += 0.1
+ return rv
diff --git a/wandb/vendor/pygments/lexers/oberon.py b/wandb/vendor/pygments/lexers/oberon.py
new file mode 100644
index 0000000000000000000000000000000000000000..3b5fb3e4ba3e2dc354a34a7924ee787fe857deb7
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/oberon.py
@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.oberon
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Oberon family languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['ComponentPascalLexer']
+
+
+class ComponentPascalLexer(RegexLexer):
+ """
+ For `Component Pascal <http://www.oberon.ch/pdf/CP-Lang.pdf>`_ source code.
+
+ .. versionadded:: 2.1
+ """
+ name = 'Component Pascal'
+ aliases = ['componentpascal', 'cp']
+ filenames = ['*.cp', '*.cps']
+ mimetypes = ['text/x-component-pascal']
+
+ flags = re.MULTILINE | re.DOTALL
+
+ tokens = {
+ 'root': [
+ include('whitespace'),
+ include('comments'),
+ include('punctuation'),
+ include('numliterals'),
+ include('strings'),
+ include('operators'),
+ include('builtins'),
+ include('identifiers'),
+ ],
+ 'whitespace': [
+ (r'\n+', Text), # blank lines
+ (r'\s+', Text), # whitespace
+ ],
+ 'comments': [
+ (r'\(\*([^$].*?)\*\)', Comment.Multiline),
+ # TODO: nested comments (* (* ... *) ... (* ... *) *) not supported!
+ ],
+ 'punctuation': [
+ (r'[()\[\]{},.:;|]', Punctuation),
+ ],
+ 'numliterals': [
+ (r'[0-9A-F]+X\b', Number.Hex), # char code
+ (r'[0-9A-F]+[HL]\b', Number.Hex), # hexadecimal number
+ (r'[0-9]+\.[0-9]+E[+-][0-9]+', Number.Float), # real number
+ (r'[0-9]+\.[0-9]+', Number.Float), # real number
+ (r'[0-9]+', Number.Integer), # decimal whole number
+ ],
+ 'strings': [
+ (r"'[^\n']*'", String), # single quoted string
+ (r'"[^\n"]*"', String), # double quoted string
+ ],
+ 'operators': [
+ # Arithmetic Operators
+ (r'[+-]', Operator),
+ (r'[*/]', Operator),
+ # Relational Operators
+ (r'[=#<>]', Operator),
+ # Dereferencing Operator
+ (r'\^', Operator),
+ # Logical AND Operator
+ (r'&', Operator),
+ # Logical NOT Operator
+ (r'~', Operator),
+ # Assignment Symbol
+ (r':=', Operator),
+ # Range Constructor
+ (r'\.\.', Operator),
+ (r'\$', Operator),
+ ],
+ 'identifiers': [
+ (r'([a-zA-Z_$][\w$]*)', Name),
+ ],
+ 'builtins': [
+ (words((
+ 'ANYPTR', 'ANYREC', 'BOOLEAN', 'BYTE', 'CHAR', 'INTEGER', 'LONGINT',
+ 'REAL', 'SET', 'SHORTCHAR', 'SHORTINT', 'SHORTREAL'
+ ), suffix=r'\b'), Keyword.Type),
+ (words((
+ 'ABS', 'ABSTRACT', 'ARRAY', 'ASH', 'ASSERT', 'BEGIN', 'BITS', 'BY',
+ 'CAP', 'CASE', 'CHR', 'CLOSE', 'CONST', 'DEC', 'DIV', 'DO', 'ELSE',
+ 'ELSIF', 'EMPTY', 'END', 'ENTIER', 'EXCL', 'EXIT', 'EXTENSIBLE', 'FOR',
+ 'HALT', 'IF', 'IMPORT', 'IN', 'INC', 'INCL', 'IS', 'LEN', 'LIMITED',
+ 'LONG', 'LOOP', 'MAX', 'MIN', 'MOD', 'MODULE', 'NEW', 'ODD', 'OF',
+ 'OR', 'ORD', 'OUT', 'POINTER', 'PROCEDURE', 'RECORD', 'REPEAT', 'RETURN',
+ 'SHORT', 'SHORTCHAR', 'SHORTINT', 'SIZE', 'THEN', 'TYPE', 'TO', 'UNTIL',
+ 'VAR', 'WHILE', 'WITH'
+ ), suffix=r'\b'), Keyword.Reserved),
+ (r'(TRUE|FALSE|NIL|INF)\b', Keyword.Constant),
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/objective.py b/wandb/vendor/pygments/lexers/objective.py
new file mode 100644
index 0000000000000000000000000000000000000000..7807255e6696adf3f7bc8b1ed457774098fc8d70
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/objective.py
@@ -0,0 +1,504 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.objective
+ ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Objective-C family languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, this, words, \
+ inherit, default
+from pygments.token import Text, Keyword, Name, String, Operator, \
+ Number, Punctuation, Literal, Comment
+
+from pygments.lexers.c_cpp import CLexer, CppLexer
+
+__all__ = ['ObjectiveCLexer', 'ObjectiveCppLexer', 'LogosLexer', 'SwiftLexer']
+
+
+def objective(baselexer):
+ """
+ Generate a subclass of baselexer that accepts the Objective-C syntax
+ extensions.
+ """
+
+ # Have to be careful not to accidentally match JavaDoc/Doxygen syntax here,
+ # since that's quite common in ordinary C/C++ files. It's OK to match
+ # JavaDoc/Doxygen keywords that only apply to Objective-C, mind.
+ #
+ # The upshot of this is that we CANNOT match @class or @interface
+ _oc_keywords = re.compile(r'@(?:end|implementation|protocol)')
+
+ # Matches [ <ws>? identifier <ws> ( identifier <ws>? ] | identifier? : )
+ # (note the identifier is *optional* when there is a ':'!)
+ _oc_message = re.compile(r'\[\s*[a-zA-Z_]\w*\s+'
+ r'(?:[a-zA-Z_]\w*\s*\]|'
+ r'(?:[a-zA-Z_]\w*)?:)')
+
+ class GeneratedObjectiveCVariant(baselexer):
+ """
+ Implements Objective-C syntax on top of an existing C family lexer.
+ """
+
+ tokens = {
+ 'statements': [
+ (r'@"', String, 'string'),
+ (r'@(YES|NO)', Number),
+ (r"@'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+ (r'@(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float),
+ (r'@(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+ (r'@0x[0-9a-fA-F]+[Ll]?', Number.Hex),
+ (r'@0[0-7]+[Ll]?', Number.Oct),
+ (r'@\d+[Ll]?', Number.Integer),
+ (r'@\(', Literal, 'literal_number'),
+ (r'@\[', Literal, 'literal_array'),
+ (r'@\{', Literal, 'literal_dictionary'),
+ (words((
+ '@selector', '@private', '@protected', '@public', '@encode',
+ '@synchronized', '@try', '@throw', '@catch', '@finally',
+ '@end', '@property', '@synthesize', '__bridge', '__bridge_transfer',
+ '__autoreleasing', '__block', '__weak', '__strong', 'weak', 'strong',
+ 'copy', 'retain', 'assign', 'unsafe_unretained', 'atomic', 'nonatomic',
+ 'readonly', 'readwrite', 'setter', 'getter', 'typeof', 'in',
+ 'out', 'inout', 'release', 'class', '@dynamic', '@optional',
+ '@required', '@autoreleasepool'), suffix=r'\b'),
+ Keyword),
+ (words(('id', 'instancetype', 'Class', 'IMP', 'SEL', 'BOOL',
+ 'IBOutlet', 'IBAction', 'unichar'), suffix=r'\b'),
+ Keyword.Type),
+ (r'@(true|false|YES|NO)\n', Name.Builtin),
+ (r'(YES|NO|nil|self|super)\b', Name.Builtin),
+ # Carbon types
+ (r'(Boolean|UInt8|SInt8|UInt16|SInt16|UInt32|SInt32)\b', Keyword.Type),
+ # Carbon built-ins
+ (r'(TRUE|FALSE)\b', Name.Builtin),
+ (r'(@interface|@implementation)(\s+)', bygroups(Keyword, Text),
+ ('#pop', 'oc_classname')),
+ (r'(@class|@protocol)(\s+)', bygroups(Keyword, Text),
+ ('#pop', 'oc_forward_classname')),
+ # @ can also prefix other expressions like @{...} or @(...)
+ (r'@', Punctuation),
+ inherit,
+ ],
+ 'oc_classname': [
+ # interface definition that inherits
+ ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?(\s*)(\{)',
+ bygroups(Name.Class, Text, Name.Class, Text, Punctuation),
+ ('#pop', 'oc_ivars')),
+ ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?',
+ bygroups(Name.Class, Text, Name.Class), '#pop'),
+ # interface definition for a category
+ ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))(\s*)(\{)',
+ bygroups(Name.Class, Text, Name.Label, Text, Punctuation),
+ ('#pop', 'oc_ivars')),
+ ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))',
+ bygroups(Name.Class, Text, Name.Label), '#pop'),
+ # simple interface / implementation
+ ('([a-zA-Z$_][\w$]*)(\s*)(\{)',
+ bygroups(Name.Class, Text, Punctuation), ('#pop', 'oc_ivars')),
+ ('([a-zA-Z$_][\w$]*)', Name.Class, '#pop')
+ ],
+ 'oc_forward_classname': [
+ ('([a-zA-Z$_][\w$]*)(\s*,\s*)',
+ bygroups(Name.Class, Text), 'oc_forward_classname'),
+ ('([a-zA-Z$_][\w$]*)(\s*;?)',
+ bygroups(Name.Class, Text), '#pop')
+ ],
+ 'oc_ivars': [
+ include('whitespace'),
+ include('statements'),
+ (';', Punctuation),
+ (r'\{', Punctuation, '#push'),
+ (r'\}', Punctuation, '#pop'),
+ ],
+ 'root': [
+ # methods
+ (r'^([-+])(\s*)' # method marker
+ r'(\(.*?\))?(\s*)' # return type
+ r'([a-zA-Z$_][\w$]*:?)', # begin of method name
+ bygroups(Punctuation, Text, using(this),
+ Text, Name.Function),
+ 'method'),
+ inherit,
+ ],
+ 'method': [
+ include('whitespace'),
+ # TODO unsure if ellipses are allowed elsewhere, see
+ # discussion in Issue 789
+ (r',', Punctuation),
+ (r'\.\.\.', Punctuation),
+ (r'(\(.*?\))(\s*)([a-zA-Z$_][\w$]*)',
+ bygroups(using(this), Text, Name.Variable)),
+ (r'[a-zA-Z$_][\w$]*:', Name.Function),
+ (';', Punctuation, '#pop'),
+ (r'\{', Punctuation, 'function'),
+ default('#pop'),
+ ],
+ 'literal_number': [
+ (r'\(', Punctuation, 'literal_number_inner'),
+ (r'\)', Literal, '#pop'),
+ include('statement'),
+ ],
+ 'literal_number_inner': [
+ (r'\(', Punctuation, '#push'),
+ (r'\)', Punctuation, '#pop'),
+ include('statement'),
+ ],
+ 'literal_array': [
+ (r'\[', Punctuation, 'literal_array_inner'),
+ (r'\]', Literal, '#pop'),
+ include('statement'),
+ ],
+ 'literal_array_inner': [
+ (r'\[', Punctuation, '#push'),
+ (r'\]', Punctuation, '#pop'),
+ include('statement'),
+ ],
+ 'literal_dictionary': [
+ (r'\}', Literal, '#pop'),
+ include('statement'),
+ ],
+ }
+
+ def analyse_text(text):
+ if _oc_keywords.search(text):
+ return 1.0
+ elif '@"' in text: # strings
+ return 0.8
+ elif re.search('@[0-9]+', text):
+ return 0.7
+ elif _oc_message.search(text):
+ return 0.8
+ return 0
+
+ def get_tokens_unprocessed(self, text):
+ from pygments.lexers._cocoa_builtins import COCOA_INTERFACES, \
+ COCOA_PROTOCOLS, COCOA_PRIMITIVES
+
+ for index, token, value in \
+ baselexer.get_tokens_unprocessed(self, text):
+ if token is Name or token is Name.Class:
+ if value in COCOA_INTERFACES or value in COCOA_PROTOCOLS \
+ or value in COCOA_PRIMITIVES:
+ token = Name.Builtin.Pseudo
+
+ yield index, token, value
+
+ return GeneratedObjectiveCVariant
+
+
+class ObjectiveCLexer(objective(CLexer)):
+ """
+ For Objective-C source code with preprocessor directives.
+ """
+
+ name = 'Objective-C'
+ aliases = ['objective-c', 'objectivec', 'obj-c', 'objc']
+ filenames = ['*.m', '*.h']
+ mimetypes = ['text/x-objective-c']
+ priority = 0.05 # Lower than C
+
+
+class ObjectiveCppLexer(objective(CppLexer)):
+ """
+ For Objective-C++ source code with preprocessor directives.
+ """
+
+ name = 'Objective-C++'
+ aliases = ['objective-c++', 'objectivec++', 'obj-c++', 'objc++']
+ filenames = ['*.mm', '*.hh']
+ mimetypes = ['text/x-objective-c++']
+ priority = 0.05 # Lower than C++
+
+
+class LogosLexer(ObjectiveCppLexer):
+ """
+ For Logos + Objective-C source code with preprocessor directives.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'Logos'
+ aliases = ['logos']
+ filenames = ['*.x', '*.xi', '*.xm', '*.xmi']
+ mimetypes = ['text/x-logos']
+ priority = 0.25
+
+ tokens = {
+ 'statements': [
+ (r'(%orig|%log)\b', Keyword),
+ (r'(%c)\b(\()(\s*)([a-zA-Z$_][\w$]*)(\s*)(\))',
+ bygroups(Keyword, Punctuation, Text, Name.Class, Text, Punctuation)),
+ (r'(%init)\b(\()',
+ bygroups(Keyword, Punctuation), 'logos_init_directive'),
+ (r'(%init)(?=\s*;)', bygroups(Keyword)),
+ (r'(%hook|%group)(\s+)([a-zA-Z$_][\w$]+)',
+ bygroups(Keyword, Text, Name.Class), '#pop'),
+ (r'(%subclass)(\s+)', bygroups(Keyword, Text),
+ ('#pop', 'logos_classname')),
+ inherit,
+ ],
+ 'logos_init_directive': [
+ ('\s+', Text),
+ (',', Punctuation, ('logos_init_directive', '#pop')),
+ ('([a-zA-Z$_][\w$]*)(\s*)(=)(\s*)([^);]*)',
+ bygroups(Name.Class, Text, Punctuation, Text, Text)),
+ ('([a-zA-Z$_][\w$]*)', Name.Class),
+ ('\)', Punctuation, '#pop'),
+ ],
+ 'logos_classname': [
+ ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?',
+ bygroups(Name.Class, Text, Name.Class), '#pop'),
+ ('([a-zA-Z$_][\w$]*)', Name.Class, '#pop')
+ ],
+ 'root': [
+ (r'(%subclass)(\s+)', bygroups(Keyword, Text),
+ 'logos_classname'),
+ (r'(%hook|%group)(\s+)([a-zA-Z$_][\w$]+)',
+ bygroups(Keyword, Text, Name.Class)),
+ (r'(%config)(\s*\(\s*)(\w+)(\s*=\s*)(.*?)(\s*\)\s*)',
+ bygroups(Keyword, Text, Name.Variable, Text, String, Text)),
+ (r'(%ctor)(\s*)(\{)', bygroups(Keyword, Text, Punctuation),
+ 'function'),
+ (r'(%new)(\s*)(\()(\s*.*?\s*)(\))',
+ bygroups(Keyword, Text, Keyword, String, Keyword)),
+ (r'(\s*)(%end)(\s*)', bygroups(Text, Keyword, Text)),
+ inherit,
+ ],
+ }
+
+ _logos_keywords = re.compile(r'%(?:hook|ctor|init|c\()')
+
+ def analyse_text(text):
+ if LogosLexer._logos_keywords.search(text):
+ return 1.0
+ return 0
+
+
+class SwiftLexer(RegexLexer):
+ """
+ For `Swift <https://developer.apple.com/swift/>`_ source.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Swift'
+ filenames = ['*.swift']
+ aliases = ['swift']
+ mimetypes = ['text/x-swift']
+
+ tokens = {
+ 'root': [
+ # Whitespace and Comments
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'//', Comment.Single, 'comment-single'),
+ (r'/\*', Comment.Multiline, 'comment-multi'),
+ (r'#(if|elseif|else|endif|available)\b', Comment.Preproc, 'preproc'),
+
+ # Keywords
+ include('keywords'),
+
+ # Global Types
+ (words((
+ 'Array', 'AutoreleasingUnsafeMutablePointer', 'BidirectionalReverseView',
+ 'Bit', 'Bool', 'CFunctionPointer', 'COpaquePointer', 'CVaListPointer',
+ 'Character', 'ClosedInterval', 'CollectionOfOne', 'ContiguousArray',
+ 'Dictionary', 'DictionaryGenerator', 'DictionaryIndex', 'Double',
+ 'EmptyCollection', 'EmptyGenerator', 'EnumerateGenerator',
+ 'EnumerateSequence', 'FilterCollectionView',
+ 'FilterCollectionViewIndex', 'FilterGenerator', 'FilterSequenceView',
+ 'Float', 'Float80', 'FloatingPointClassification', 'GeneratorOf',
+ 'GeneratorOfOne', 'GeneratorSequence', 'HalfOpenInterval', 'HeapBuffer',
+ 'HeapBufferStorage', 'ImplicitlyUnwrappedOptional', 'IndexingGenerator',
+ 'Int', 'Int16', 'Int32', 'Int64', 'Int8', 'LazyBidirectionalCollection',
+ 'LazyForwardCollection', 'LazyRandomAccessCollection',
+ 'LazySequence', 'MapCollectionView', 'MapSequenceGenerator',
+ 'MapSequenceView', 'MirrorDisposition', 'ObjectIdentifier', 'OnHeap',
+ 'Optional', 'PermutationGenerator', 'QuickLookObject',
+ 'RandomAccessReverseView', 'Range', 'RangeGenerator', 'RawByte', 'Repeat',
+ 'ReverseBidirectionalIndex', 'ReverseRandomAccessIndex', 'SequenceOf',
+ 'SinkOf', 'Slice', 'StaticString', 'StrideThrough', 'StrideThroughGenerator',
+ 'StrideTo', 'StrideToGenerator', 'String', 'UInt', 'UInt16', 'UInt32',
+ 'UInt64', 'UInt8', 'UTF16', 'UTF32', 'UTF8', 'UnicodeDecodingResult',
+ 'UnicodeScalar', 'Unmanaged', 'UnsafeBufferPointer',
+ 'UnsafeBufferPointerGenerator', 'UnsafeMutableBufferPointer',
+ 'UnsafeMutablePointer', 'UnsafePointer', 'Zip2', 'ZipGenerator2',
+ # Protocols
+ 'AbsoluteValuable', 'AnyObject', 'ArrayLiteralConvertible',
+ 'BidirectionalIndexType', 'BitwiseOperationsType',
+ 'BooleanLiteralConvertible', 'BooleanType', 'CVarArgType',
+ 'CollectionType', 'Comparable', 'DebugPrintable',
+ 'DictionaryLiteralConvertible', 'Equatable',
+ 'ExtendedGraphemeClusterLiteralConvertible',
+ 'ExtensibleCollectionType', 'FloatLiteralConvertible',
+ 'FloatingPointType', 'ForwardIndexType', 'GeneratorType', 'Hashable',
+ 'IntegerArithmeticType', 'IntegerLiteralConvertible', 'IntegerType',
+ 'IntervalType', 'MirrorType', 'MutableCollectionType', 'MutableSliceable',
+ 'NilLiteralConvertible', 'OutputStreamType', 'Printable',
+ 'RandomAccessIndexType', 'RangeReplaceableCollectionType',
+ 'RawOptionSetType', 'RawRepresentable', 'Reflectable', 'SequenceType',
+ 'SignedIntegerType', 'SignedNumberType', 'SinkType', 'Sliceable',
+ 'Streamable', 'Strideable', 'StringInterpolationConvertible',
+ 'StringLiteralConvertible', 'UnicodeCodecType',
+ 'UnicodeScalarLiteralConvertible', 'UnsignedIntegerType',
+ '_ArrayBufferType', '_BidirectionalIndexType', '_CocoaStringType',
+ '_CollectionType', '_Comparable', '_ExtensibleCollectionType',
+ '_ForwardIndexType', '_Incrementable', '_IntegerArithmeticType',
+ '_IntegerType', '_ObjectiveCBridgeable', '_RandomAccessIndexType',
+ '_RawOptionSetType', '_SequenceType', '_Sequence_Type',
+ '_SignedIntegerType', '_SignedNumberType', '_Sliceable', '_Strideable',
+ '_SwiftNSArrayRequiredOverridesType', '_SwiftNSArrayType',
+ '_SwiftNSCopyingType', '_SwiftNSDictionaryRequiredOverridesType',
+ '_SwiftNSDictionaryType', '_SwiftNSEnumeratorType',
+ '_SwiftNSFastEnumerationType', '_SwiftNSStringRequiredOverridesType',
+ '_SwiftNSStringType', '_UnsignedIntegerType',
+ # Variables
+ 'C_ARGC', 'C_ARGV', 'Process',
+ # Typealiases
+ 'Any', 'AnyClass', 'BooleanLiteralType', 'CBool', 'CChar', 'CChar16',
+ 'CChar32', 'CDouble', 'CFloat', 'CInt', 'CLong', 'CLongLong', 'CShort',
+ 'CSignedChar', 'CUnsignedInt', 'CUnsignedLong', 'CUnsignedShort',
+ 'CWideChar', 'ExtendedGraphemeClusterType', 'Float32', 'Float64',
+ 'FloatLiteralType', 'IntMax', 'IntegerLiteralType', 'StringLiteralType',
+ 'UIntMax', 'UWord', 'UnicodeScalarType', 'Void', 'Word',
+ # Foundation/Cocoa
+ 'NSErrorPointer', 'NSObjectProtocol', 'Selector'), suffix=r'\b'),
+ Name.Builtin),
+ # Functions
+ (words((
+ 'abs', 'advance', 'alignof', 'alignofValue', 'assert', 'assertionFailure',
+ 'contains', 'count', 'countElements', 'debugPrint', 'debugPrintln',
+ 'distance', 'dropFirst', 'dropLast', 'dump', 'enumerate', 'equal',
+ 'extend', 'fatalError', 'filter', 'find', 'first', 'getVaList', 'indices',
+ 'insert', 'isEmpty', 'join', 'last', 'lazy', 'lexicographicalCompare',
+ 'map', 'max', 'maxElement', 'min', 'minElement', 'numericCast', 'overlaps',
+ 'partition', 'precondition', 'preconditionFailure', 'prefix', 'print',
+ 'println', 'reduce', 'reflect', 'removeAll', 'removeAtIndex', 'removeLast',
+ 'removeRange', 'reverse', 'sizeof', 'sizeofValue', 'sort', 'sorted',
+ 'splice', 'split', 'startsWith', 'stride', 'strideof', 'strideofValue',
+ 'suffix', 'swap', 'toDebugString', 'toString', 'transcode',
+ 'underestimateCount', 'unsafeAddressOf', 'unsafeBitCast', 'unsafeDowncast',
+ 'withExtendedLifetime', 'withUnsafeMutablePointer',
+ 'withUnsafeMutablePointers', 'withUnsafePointer', 'withUnsafePointers',
+ 'withVaList'), suffix=r'\b'),
+ Name.Builtin.Pseudo),
+
+ # Implicit Block Variables
+ (r'\$\d+', Name.Variable),
+
+ # Binary Literal
+ (r'0b[01_]+', Number.Bin),
+ # Octal Literal
+ (r'0o[0-7_]+', Number.Oct),
+ # Hexadecimal Literal
+ (r'0x[0-9a-fA-F_]+', Number.Hex),
+ # Decimal Literal
+ (r'[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|'
+ r'\.[0-9_]*|[eE][+\-]?[0-9_]+)', Number.Float),
+ (r'[0-9][0-9_]*', Number.Integer),
+ # String Literal
+ (r'"', String, 'string'),
+
+ # Operators and Punctuation
+ (r'[(){}\[\].,:;=@#`?]|->|[<&?](?=\w)|(?<=\w)[>!?]', Punctuation),
+ (r'[/=\-+!*%<>&|^?~]+', Operator),
+
+ # Identifier
+ (r'[a-zA-Z_]\w*', Name)
+ ],
+ 'keywords': [
+ (words((
+ 'as', 'break', 'case', 'catch', 'continue', 'default', 'defer',
+ 'do', 'else', 'fallthrough', 'for', 'guard', 'if', 'in', 'is',
+ 'repeat', 'return', '#selector', 'switch', 'throw', 'try',
+ 'where', 'while'), suffix=r'\b'),
+ Keyword),
+ (r'@availability\([^)]+\)', Keyword.Reserved),
+ (words((
+ 'associativity', 'convenience', 'dynamic', 'didSet', 'final',
+ 'get', 'indirect', 'infix', 'inout', 'lazy', 'left', 'mutating',
+ 'none', 'nonmutating', 'optional', 'override', 'postfix',
+ 'precedence', 'prefix', 'Protocol', 'required', 'rethrows',
+ 'right', 'set', 'throws', 'Type', 'unowned', 'weak', 'willSet',
+ '@availability', '@autoclosure', '@noreturn',
+ '@NSApplicationMain', '@NSCopying', '@NSManaged', '@objc',
+ '@UIApplicationMain', '@IBAction', '@IBDesignable',
+ '@IBInspectable', '@IBOutlet'), suffix=r'\b'),
+ Keyword.Reserved),
+ (r'(as|dynamicType|false|is|nil|self|Self|super|true|__COLUMN__'
+ r'|__FILE__|__FUNCTION__|__LINE__|_'
+ r'|#(?:file|line|column|function))\b', Keyword.Constant),
+ (r'import\b', Keyword.Declaration, 'module'),
+ (r'(class|enum|extension|struct|protocol)(\s+)([a-zA-Z_]\w*)',
+ bygroups(Keyword.Declaration, Text, Name.Class)),
+ (r'(func)(\s+)([a-zA-Z_]\w*)',
+ bygroups(Keyword.Declaration, Text, Name.Function)),
+ (r'(var|let)(\s+)([a-zA-Z_]\w*)', bygroups(Keyword.Declaration,
+ Text, Name.Variable)),
+ (words((
+ 'class', 'deinit', 'enum', 'extension', 'func', 'import', 'init',
+ 'internal', 'let', 'operator', 'private', 'protocol', 'public',
+ 'static', 'struct', 'subscript', 'typealias', 'var'), suffix=r'\b'),
+ Keyword.Declaration)
+ ],
+ 'comment': [
+ (r':param: [a-zA-Z_]\w*|:returns?:|(FIXME|MARK|TODO):',
+ Comment.Special)
+ ],
+
+ # Nested
+ 'comment-single': [
+ (r'\n', Text, '#pop'),
+ include('comment'),
+ (r'[^\n]', Comment.Single)
+ ],
+ 'comment-multi': [
+ include('comment'),
+ (r'[^*/]', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline)
+ ],
+ 'module': [
+ (r'\n', Text, '#pop'),
+ (r'[a-zA-Z_]\w*', Name.Class),
+ include('root')
+ ],
+ 'preproc': [
+ (r'\n', Text, '#pop'),
+ include('keywords'),
+ (r'[A-Za-z]\w*', Comment.Preproc),
+ include('root')
+ ],
+ 'string': [
+ (r'\\\(', String.Interpol, 'string-intp'),
+ (r'"', String, '#pop'),
+ (r"""\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}"""
+ r"""|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}""", String.Escape),
+ (r'[^\\"]+', String),
+ (r'\\', String)
+ ],
+ 'string-intp': [
+ (r'\(', String.Interpol, '#push'),
+ (r'\)', String.Interpol, '#pop'),
+ include('root')
+ ]
+ }
+
+ def get_tokens_unprocessed(self, text):
+ from pygments.lexers._cocoa_builtins import COCOA_INTERFACES, \
+ COCOA_PROTOCOLS, COCOA_PRIMITIVES
+
+ for index, token, value in \
+ RegexLexer.get_tokens_unprocessed(self, text):
+ if token is Name or token is Name.Class:
+ if value in COCOA_INTERFACES or value in COCOA_PROTOCOLS \
+ or value in COCOA_PRIMITIVES:
+ token = Name.Builtin.Pseudo
+
+ yield index, token, value
diff --git a/wandb/vendor/pygments/lexers/ooc.py b/wandb/vendor/pygments/lexers/ooc.py
new file mode 100644
index 0000000000000000000000000000000000000000..957b72f168a56f2c63b58000ac13442dcb95c4bf
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/ooc.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.ooc
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the Ooc language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['OocLexer']
+
+
+class OocLexer(RegexLexer):
+ """
+ For `Ooc <http://ooc-lang.org/>`_ source code
+
+ .. versionadded:: 1.2
+ """
+ name = 'Ooc'
+ aliases = ['ooc']
+ filenames = ['*.ooc']
+ mimetypes = ['text/x-ooc']
+
+ tokens = {
+ 'root': [
+ (words((
+ 'class', 'interface', 'implement', 'abstract', 'extends', 'from',
+ 'this', 'super', 'new', 'const', 'final', 'static', 'import',
+ 'use', 'extern', 'inline', 'proto', 'break', 'continue',
+ 'fallthrough', 'operator', 'if', 'else', 'for', 'while', 'do',
+ 'switch', 'case', 'as', 'in', 'version', 'return', 'true',
+ 'false', 'null'), prefix=r'\b', suffix=r'\b'),
+ Keyword),
+ (r'include\b', Keyword, 'include'),
+ (r'(cover)([ \t]+)(from)([ \t]+)(\w+[*@]?)',
+ bygroups(Keyword, Text, Keyword, Text, Name.Class)),
+ (r'(func)((?:[ \t]|\\\n)+)(~[a-z_]\w*)',
+ bygroups(Keyword, Text, Name.Function)),
+ (r'\bfunc\b', Keyword),
+ # Note: %= and ^= not listed on http://ooc-lang.org/syntax
+ (r'//.*', Comment),
+ (r'(?s)/\*.*?\*/', Comment.Multiline),
+ (r'(==?|\+=?|-[=>]?|\*=?|/=?|:=|!=?|%=?|\?|>{1,3}=?|<{1,3}=?|\.\.|'
+ r'&&?|\|\|?|\^=?)', Operator),
+ (r'(\.)([ \t]*)([a-z]\w*)', bygroups(Operator, Text,
+ Name.Function)),
+ (r'[A-Z][A-Z0-9_]+', Name.Constant),
+ (r'[A-Z]\w*([@*]|\[[ \t]*\])?', Name.Class),
+
+ (r'([a-z]\w*(?:~[a-z]\w*)?)((?:[ \t]|\\\n)*)(?=\()',
+ bygroups(Name.Function, Text)),
+ (r'[a-z]\w*', Name.Variable),
+
+ # : introduces types
+ (r'[:(){}\[\];,]', Punctuation),
+
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'0c[0-9]+', Number.Oct),
+ (r'0b[01]+', Number.Bin),
+ (r'[0-9_]\.[0-9_]*(?!\.)', Number.Float),
+ (r'[0-9_]+', Number.Decimal),
+
+ (r'"(?:\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\"])*"',
+ String.Double),
+ (r"'(?:\\.|\\[0-9]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'",
+ String.Char),
+ (r'@', Punctuation), # pointer dereference
+ (r'\.', Punctuation), # imports or chain operator
+
+ (r'\\[ \t\n]', Text),
+ (r'[ \t]+', Text),
+ ],
+ 'include': [
+ (r'[\w/]+', Name),
+ (r',', Punctuation),
+ (r'[ \t]', Text),
+ (r'[;\n]', Text, '#pop'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/other.py b/wandb/vendor/pygments/lexers/other.py
new file mode 100644
index 0000000000000000000000000000000000000000..bfce4c3c4281e142b25bc438f10dbcec610a6c45
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/other.py
@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.other
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Just export lexer classes previously contained in this module.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexers.sql import SqlLexer, MySqlLexer, SqliteConsoleLexer
+from pygments.lexers.shell import BashLexer, BashSessionLexer, BatchLexer, \
+ TcshLexer
+from pygments.lexers.robotframework import RobotFrameworkLexer
+from pygments.lexers.testing import GherkinLexer
+from pygments.lexers.esoteric import BrainfuckLexer, BefungeLexer, RedcodeLexer
+from pygments.lexers.prolog import LogtalkLexer
+from pygments.lexers.snobol import SnobolLexer
+from pygments.lexers.rebol import RebolLexer
+from pygments.lexers.configs import KconfigLexer, Cfengine3Lexer
+from pygments.lexers.modeling import ModelicaLexer
+from pygments.lexers.scripting import AppleScriptLexer, MOOCodeLexer, \
+ HybrisLexer
+from pygments.lexers.graphics import PostScriptLexer, GnuplotLexer, \
+ AsymptoteLexer, PovrayLexer
+from pygments.lexers.business import ABAPLexer, OpenEdgeLexer, \
+ GoodDataCLLexer, MaqlLexer
+from pygments.lexers.automation import AutoItLexer, AutohotkeyLexer
+from pygments.lexers.dsls import ProtoBufLexer, BroLexer, PuppetLexer, \
+ MscgenLexer, VGLLexer
+from pygments.lexers.basic import CbmBasicV2Lexer
+from pygments.lexers.pawn import SourcePawnLexer, PawnLexer
+from pygments.lexers.ecl import ECLLexer
+from pygments.lexers.urbi import UrbiscriptLexer
+from pygments.lexers.smalltalk import SmalltalkLexer, NewspeakLexer
+from pygments.lexers.installers import NSISLexer, RPMSpecLexer
+from pygments.lexers.textedit import AwkLexer
+from pygments.lexers.smv import NuSMVLexer
+
+__all__ = []
diff --git a/wandb/vendor/pygments/lexers/parasail.py b/wandb/vendor/pygments/lexers/parasail.py
new file mode 100644
index 0000000000000000000000000000000000000000..53088023d456f2a4a5c655e948ee42b4d094a604
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/parasail.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.parasail
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for ParaSail.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Literal
+
+__all__ = ['ParaSailLexer']
+
+
+class ParaSailLexer(RegexLexer):
+ """
+ For `ParaSail <http://www.parasail-lang.org>`_ source code.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'ParaSail'
+ aliases = ['parasail']
+ filenames = ['*.psi', '*.psl']
+ mimetypes = ['text/x-parasail']
+
+ flags = re.MULTILINE
+
+ tokens = {
+ 'root': [
+ (r'[^\S\n]+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'\b(and|or|xor)=', Operator.Word),
+ (r'\b(and(\s+then)?|or(\s+else)?|xor|rem|mod|'
+ r'(is|not)\s+null)\b',
+ Operator.Word),
+ # Keywords
+ (r'\b(abs|abstract|all|block|class|concurrent|const|continue|'
+ r'each|end|exit|extends|exports|forward|func|global|implements|'
+ r'import|in|interface|is|lambda|locked|new|not|null|of|op|'
+ r'optional|private|queued|ref|return|reverse|separate|some|'
+ r'type|until|var|with|'
+ # Control flow
+ r'if|then|else|elsif|case|for|while|loop)\b',
+ Keyword.Reserved),
+ (r'(abstract\s+)?(interface|class|op|func|type)',
+ Keyword.Declaration),
+ # Literals
+ (r'"[^"]*"', String),
+ (r'\\[\'ntrf"0]', String.Escape),
+ (r'#[a-zA-Z]\w*', Literal), # Enumeration
+ include('numbers'),
+ (r"'[^']'", String.Char),
+ (r'[a-zA-Z]\w*', Name),
+ # Operators and Punctuation
+ (r'(<==|==>|<=>|\*\*=|<\|=|<<=|>>=|==|!=|=\?|<=|>=|'
+ r'\*\*|<<|>>|=>|:=|\+=|-=|\*=|\|=|\||/=|\+|-|\*|/|'
+ r'\.\.|<\.\.|\.\.<|<\.\.<)',
+ Operator),
+ (r'(<|>|\[|\]|\(|\)|\||:|;|,|.|\{|\}|->)',
+ Punctuation),
+ (r'\n+', Text),
+ ],
+ 'numbers': [
+ (r'\d[0-9_]*#[0-9a-fA-F][0-9a-fA-F_]*#', Number.Hex), # any base
+ (r'0[xX][0-9a-fA-F][0-9a-fA-F_]*', Number.Hex), # C-like hex
+ (r'0[bB][01][01_]*', Number.Bin), # C-like bin
+ (r'\d[0-9_]*\.\d[0-9_]*[eE][+-]\d[0-9_]*', # float exp
+ Number.Float),
+ (r'\d[0-9_]*\.\d[0-9_]*', Number.Float), # float
+ (r'\d[0-9_]*', Number.Integer), # integer
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/parsers.py b/wandb/vendor/pygments/lexers/parsers.py
new file mode 100644
index 0000000000000000000000000000000000000000..1f3c9b4d9755b7387180a6bac1343a5369a9380b
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/parsers.py
@@ -0,0 +1,835 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.parsers
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for parser generators.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, DelegatingLexer, \
+ include, bygroups, using
+from pygments.token import Punctuation, Other, Text, Comment, Operator, \
+ Keyword, Name, String, Number, Whitespace
+from pygments.lexers.jvm import JavaLexer
+from pygments.lexers.c_cpp import CLexer, CppLexer
+from pygments.lexers.objective import ObjectiveCLexer
+from pygments.lexers.d import DLexer
+from pygments.lexers.dotnet import CSharpLexer
+from pygments.lexers.ruby import RubyLexer
+from pygments.lexers.python import PythonLexer
+from pygments.lexers.perl import PerlLexer
+
+__all__ = ['RagelLexer', 'RagelEmbeddedLexer', 'RagelCLexer', 'RagelDLexer',
+ 'RagelCppLexer', 'RagelObjectiveCLexer', 'RagelRubyLexer',
+ 'RagelJavaLexer', 'AntlrLexer', 'AntlrPythonLexer',
+ 'AntlrPerlLexer', 'AntlrRubyLexer', 'AntlrCppLexer',
+ # 'AntlrCLexer',
+ 'AntlrCSharpLexer', 'AntlrObjectiveCLexer',
+ 'AntlrJavaLexer', 'AntlrActionScriptLexer',
+ 'TreetopLexer', 'EbnfLexer']
+
+
+class RagelLexer(RegexLexer):
+ """
+ A pure `Ragel <http://www.complang.org/ragel/>`_ lexer. Use this for
+ fragments of Ragel. For ``.rl`` files, use RagelEmbeddedLexer instead
+ (or one of the language-specific subclasses).
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'Ragel'
+ aliases = ['ragel']
+ filenames = []
+
+ tokens = {
+ 'whitespace': [
+ (r'\s+', Whitespace)
+ ],
+ 'comments': [
+ (r'\#.*$', Comment),
+ ],
+ 'keywords': [
+ (r'(access|action|alphtype)\b', Keyword),
+ (r'(getkey|write|machine|include)\b', Keyword),
+ (r'(any|ascii|extend|alpha|digit|alnum|lower|upper)\b', Keyword),
+ (r'(xdigit|cntrl|graph|print|punct|space|zlen|empty)\b', Keyword)
+ ],
+ 'numbers': [
+ (r'0x[0-9A-Fa-f]+', Number.Hex),
+ (r'[+-]?[0-9]+', Number.Integer),
+ ],
+ 'literals': [
+ (r'"(\\\\|\\"|[^"])*"', String), # double quote string
+ (r"'(\\\\|\\'|[^'])*'", String), # single quote string
+ (r'\[(\\\\|\\\]|[^\]])*\]', String), # square bracket literals
+ (r'/(?!\*)(\\\\|\\/|[^/])*/', String.Regex), # regular expressions
+ ],
+ 'identifiers': [
+ (r'[a-zA-Z_]\w*', Name.Variable),
+ ],
+ 'operators': [
+ (r',', Operator), # Join
+ (r'\||&|--?', Operator), # Union, Intersection and Subtraction
+ (r'\.|<:|:>>?', Operator), # Concatention
+ (r':', Operator), # Label
+ (r'->', Operator), # Epsilon Transition
+ (r'(>|\$|%|<|@|<>)(/|eof\b)', Operator), # EOF Actions
+ (r'(>|\$|%|<|@|<>)(!|err\b)', Operator), # Global Error Actions
+ (r'(>|\$|%|<|@|<>)(\^|lerr\b)', Operator), # Local Error Actions
+ (r'(>|\$|%|<|@|<>)(~|to\b)', Operator), # To-State Actions
+ (r'(>|\$|%|<|@|<>)(\*|from\b)', Operator), # From-State Actions
+ (r'>|@|\$|%', Operator), # Transition Actions and Priorities
+ (r'\*|\?|\+|\{[0-9]*,[0-9]*\}', Operator), # Repetition
+ (r'!|\^', Operator), # Negation
+ (r'\(|\)', Operator), # Grouping
+ ],
+ 'root': [
+ include('literals'),
+ include('whitespace'),
+ include('comments'),
+ include('keywords'),
+ include('numbers'),
+ include('identifiers'),
+ include('operators'),
+ (r'\{', Punctuation, 'host'),
+ (r'=', Operator),
+ (r';', Punctuation),
+ ],
+ 'host': [
+ (r'(' + r'|'.join(( # keep host code in largest possible chunks
+ r'[^{}\'"/#]+', # exclude unsafe characters
+ r'[^\\]\\[{}]', # allow escaped { or }
+
+ # strings and comments may safely contain unsafe characters
+ r'"(\\\\|\\"|[^"])*"', # double quote string
+ r"'(\\\\|\\'|[^'])*'", # single quote string
+ r'//.*$\n?', # single line comment
+ r'/\*(.|\n)*?\*/', # multi-line javadoc-style comment
+ r'\#.*$\n?', # ruby comment
+
+ # regular expression: There's no reason for it to start
+ # with a * and this stops confusion with comments.
+ r'/(?!\*)(\\\\|\\/|[^/])*/',
+
+ # / is safe now that we've handled regex and javadoc comments
+ r'/',
+ )) + r')+', Other),
+
+ (r'\{', Punctuation, '#push'),
+ (r'\}', Punctuation, '#pop'),
+ ],
+ }
+
+
+class RagelEmbeddedLexer(RegexLexer):
+ """
+ A lexer for `Ragel`_ embedded in a host language file.
+
+ This will only highlight Ragel statements. If you want host language
+ highlighting then call the language-specific Ragel lexer.
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'Embedded Ragel'
+ aliases = ['ragel-em']
+ filenames = ['*.rl']
+
+ tokens = {
+ 'root': [
+ (r'(' + r'|'.join(( # keep host code in largest possible chunks
+ r'[^%\'"/#]+', # exclude unsafe characters
+ r'%(?=[^%]|$)', # a single % sign is okay, just not 2 of them
+
+ # strings and comments may safely contain unsafe characters
+ r'"(\\\\|\\"|[^"])*"', # double quote string
+ r"'(\\\\|\\'|[^'])*'", # single quote string
+ r'/\*(.|\n)*?\*/', # multi-line javadoc-style comment
+ r'//.*$\n?', # single line comment
+ r'\#.*$\n?', # ruby/ragel comment
+ r'/(?!\*)(\\\\|\\/|[^/])*/', # regular expression
+
+ # / is safe now that we've handled regex and javadoc comments
+ r'/',
+ )) + r')+', Other),
+
+ # Single Line FSM.
+ # Please don't put a quoted newline in a single line FSM.
+ # That's just mean. It will break this.
+ (r'(%%)(?![{%])(.*)($|;)(\n?)', bygroups(Punctuation,
+ using(RagelLexer),
+ Punctuation, Text)),
+
+ # Multi Line FSM.
+ (r'(%%%%|%%)\{', Punctuation, 'multi-line-fsm'),
+ ],
+ 'multi-line-fsm': [
+ (r'(' + r'|'.join(( # keep ragel code in largest possible chunks.
+ r'(' + r'|'.join((
+ r'[^}\'"\[/#]', # exclude unsafe characters
+ r'\}(?=[^%]|$)', # } is okay as long as it's not followed by %
+ r'\}%(?=[^%]|$)', # ...well, one %'s okay, just not two...
+ r'[^\\]\\[{}]', # ...and } is okay if it's escaped
+
+ # allow / if it's preceded with one of these symbols
+ # (ragel EOF actions)
+ r'(>|\$|%|<|@|<>)/',
+
+ # specifically allow regex followed immediately by *
+ # so it doesn't get mistaken for a comment
+ r'/(?!\*)(\\\\|\\/|[^/])*/\*',
+
+ # allow / as long as it's not followed by another / or by a *
+ r'/(?=[^/*]|$)',
+
+ # We want to match as many of these as we can in one block.
+ # Not sure if we need the + sign here,
+ # does it help performance?
+ )) + r')+',
+
+ # strings and comments may safely contain unsafe characters
+ r'"(\\\\|\\"|[^"])*"', # double quote string
+ r"'(\\\\|\\'|[^'])*'", # single quote string
+ r"\[(\\\\|\\\]|[^\]])*\]", # square bracket literal
+ r'/\*(.|\n)*?\*/', # multi-line javadoc-style comment
+ r'//.*$\n?', # single line comment
+ r'\#.*$\n?', # ruby/ragel comment
+ )) + r')+', using(RagelLexer)),
+
+ (r'\}%%', Punctuation, '#pop'),
+ ]
+ }
+
+ def analyse_text(text):
+ return '@LANG: indep' in text
+
+
+class RagelRubyLexer(DelegatingLexer):
+ """
+ A lexer for `Ragel`_ in a Ruby host file.
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'Ragel in Ruby Host'
+ aliases = ['ragel-ruby', 'ragel-rb']
+ filenames = ['*.rl']
+
+ def __init__(self, **options):
+ super(RagelRubyLexer, self).__init__(RubyLexer, RagelEmbeddedLexer,
+ **options)
+
+ def analyse_text(text):
+ return '@LANG: ruby' in text
+
+
+class RagelCLexer(DelegatingLexer):
+ """
+ A lexer for `Ragel`_ in a C host file.
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'Ragel in C Host'
+ aliases = ['ragel-c']
+ filenames = ['*.rl']
+
+ def __init__(self, **options):
+ super(RagelCLexer, self).__init__(CLexer, RagelEmbeddedLexer,
+ **options)
+
+ def analyse_text(text):
+ return '@LANG: c' in text
+
+
+class RagelDLexer(DelegatingLexer):
+ """
+ A lexer for `Ragel`_ in a D host file.
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'Ragel in D Host'
+ aliases = ['ragel-d']
+ filenames = ['*.rl']
+
+ def __init__(self, **options):
+ super(RagelDLexer, self).__init__(DLexer, RagelEmbeddedLexer, **options)
+
+ def analyse_text(text):
+ return '@LANG: d' in text
+
+
+class RagelCppLexer(DelegatingLexer):
+ """
+ A lexer for `Ragel`_ in a CPP host file.
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'Ragel in CPP Host'
+ aliases = ['ragel-cpp']
+ filenames = ['*.rl']
+
+ def __init__(self, **options):
+ super(RagelCppLexer, self).__init__(CppLexer, RagelEmbeddedLexer, **options)
+
+ def analyse_text(text):
+ return '@LANG: c++' in text
+
+
+class RagelObjectiveCLexer(DelegatingLexer):
+ """
+ A lexer for `Ragel`_ in an Objective C host file.
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'Ragel in Objective C Host'
+ aliases = ['ragel-objc']
+ filenames = ['*.rl']
+
+ def __init__(self, **options):
+ super(RagelObjectiveCLexer, self).__init__(ObjectiveCLexer,
+ RagelEmbeddedLexer,
+ **options)
+
+ def analyse_text(text):
+ return '@LANG: objc' in text
+
+
+class RagelJavaLexer(DelegatingLexer):
+ """
+ A lexer for `Ragel`_ in a Java host file.
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'Ragel in Java Host'
+ aliases = ['ragel-java']
+ filenames = ['*.rl']
+
+ def __init__(self, **options):
+ super(RagelJavaLexer, self).__init__(JavaLexer, RagelEmbeddedLexer,
+ **options)
+
+ def analyse_text(text):
+ return '@LANG: java' in text
+
+
+class AntlrLexer(RegexLexer):
+ """
+ Generic `ANTLR`_ Lexer.
+ Should not be called directly, instead
+ use DelegatingLexer for your target language.
+
+ .. versionadded:: 1.1
+
+ .. _ANTLR: http://www.antlr.org/
+ """
+
+ name = 'ANTLR'
+ aliases = ['antlr']
+ filenames = []
+
+ _id = r'[A-Za-z]\w*'
+ _TOKEN_REF = r'[A-Z]\w*'
+ _RULE_REF = r'[a-z]\w*'
+ _STRING_LITERAL = r'\'(?:\\\\|\\\'|[^\']*)\''
+ _INT = r'[0-9]+'
+
+ tokens = {
+ 'whitespace': [
+ (r'\s+', Whitespace),
+ ],
+ 'comments': [
+ (r'//.*$', Comment),
+ (r'/\*(.|\n)*?\*/', Comment),
+ ],
+ 'root': [
+ include('whitespace'),
+ include('comments'),
+
+ (r'(lexer|parser|tree)?(\s*)(grammar\b)(\s*)(' + _id + ')(;)',
+ bygroups(Keyword, Whitespace, Keyword, Whitespace, Name.Class,
+ Punctuation)),
+ # optionsSpec
+ (r'options\b', Keyword, 'options'),
+ # tokensSpec
+ (r'tokens\b', Keyword, 'tokens'),
+ # attrScope
+ (r'(scope)(\s*)(' + _id + ')(\s*)(\{)',
+ bygroups(Keyword, Whitespace, Name.Variable, Whitespace,
+ Punctuation), 'action'),
+ # exception
+ (r'(catch|finally)\b', Keyword, 'exception'),
+ # action
+ (r'(@' + _id + ')(\s*)(::)?(\s*)(' + _id + ')(\s*)(\{)',
+ bygroups(Name.Label, Whitespace, Punctuation, Whitespace,
+ Name.Label, Whitespace, Punctuation), 'action'),
+ # rule
+ (r'((?:protected|private|public|fragment)\b)?(\s*)(' + _id + ')(!)?',
+ bygroups(Keyword, Whitespace, Name.Label, Punctuation),
+ ('rule-alts', 'rule-prelims')),
+ ],
+ 'exception': [
+ (r'\n', Whitespace, '#pop'),
+ (r'\s', Whitespace),
+ include('comments'),
+
+ (r'\[', Punctuation, 'nested-arg-action'),
+ (r'\{', Punctuation, 'action'),
+ ],
+ 'rule-prelims': [
+ include('whitespace'),
+ include('comments'),
+
+ (r'returns\b', Keyword),
+ (r'\[', Punctuation, 'nested-arg-action'),
+ (r'\{', Punctuation, 'action'),
+ # throwsSpec
+ (r'(throws)(\s+)(' + _id + ')',
+ bygroups(Keyword, Whitespace, Name.Label)),
+ (r'(,)(\s*)(' + _id + ')',
+ bygroups(Punctuation, Whitespace, Name.Label)), # Additional throws
+ # optionsSpec
+ (r'options\b', Keyword, 'options'),
+ # ruleScopeSpec - scope followed by target language code or name of action
+ # TODO finish implementing other possibilities for scope
+ # L173 ANTLRv3.g from ANTLR book
+ (r'(scope)(\s+)(\{)', bygroups(Keyword, Whitespace, Punctuation),
+ 'action'),
+ (r'(scope)(\s+)(' + _id + ')(\s*)(;)',
+ bygroups(Keyword, Whitespace, Name.Label, Whitespace, Punctuation)),
+ # ruleAction
+ (r'(@' + _id + ')(\s*)(\{)',
+ bygroups(Name.Label, Whitespace, Punctuation), 'action'),
+ # finished prelims, go to rule alts!
+ (r':', Punctuation, '#pop')
+ ],
+ 'rule-alts': [
+ include('whitespace'),
+ include('comments'),
+
+ # These might need to go in a separate 'block' state triggered by (
+ (r'options\b', Keyword, 'options'),
+ (r':', Punctuation),
+
+ # literals
+ (r"'(\\\\|\\'|[^'])*'", String),
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r'<<([^>]|>[^>])>>', String),
+ # identifiers
+ # Tokens start with capital letter.
+ (r'\$?[A-Z_]\w*', Name.Constant),
+ # Rules start with small letter.
+ (r'\$?[a-z_]\w*', Name.Variable),
+ # operators
+ (r'(\+|\||->|=>|=|\(|\)|\.\.|\.|\?|\*|\^|!|\#|~)', Operator),
+ (r',', Punctuation),
+ (r'\[', Punctuation, 'nested-arg-action'),
+ (r'\{', Punctuation, 'action'),
+ (r';', Punctuation, '#pop')
+ ],
+ 'tokens': [
+ include('whitespace'),
+ include('comments'),
+ (r'\{', Punctuation),
+ (r'(' + _TOKEN_REF + r')(\s*)(=)?(\s*)(' + _STRING_LITERAL
+ + ')?(\s*)(;)',
+ bygroups(Name.Label, Whitespace, Punctuation, Whitespace,
+ String, Whitespace, Punctuation)),
+ (r'\}', Punctuation, '#pop'),
+ ],
+ 'options': [
+ include('whitespace'),
+ include('comments'),
+ (r'\{', Punctuation),
+ (r'(' + _id + r')(\s*)(=)(\s*)(' +
+ '|'.join((_id, _STRING_LITERAL, _INT, '\*')) + ')(\s*)(;)',
+ bygroups(Name.Variable, Whitespace, Punctuation, Whitespace,
+ Text, Whitespace, Punctuation)),
+ (r'\}', Punctuation, '#pop'),
+ ],
+ 'action': [
+ (r'(' + r'|'.join(( # keep host code in largest possible chunks
+ r'[^${}\'"/\\]+', # exclude unsafe characters
+
+ # strings and comments may safely contain unsafe characters
+ r'"(\\\\|\\"|[^"])*"', # double quote string
+ r"'(\\\\|\\'|[^'])*'", # single quote string
+ r'//.*$\n?', # single line comment
+ r'/\*(.|\n)*?\*/', # multi-line javadoc-style comment
+
+ # regular expression: There's no reason for it to start
+ # with a * and this stops confusion with comments.
+ r'/(?!\*)(\\\\|\\/|[^/])*/',
+
+ # backslashes are okay, as long as we are not backslashing a %
+ r'\\(?!%)',
+
+ # Now that we've handled regex and javadoc comments
+ # it's safe to let / through.
+ r'/',
+ )) + r')+', Other),
+ (r'(\\)(%)', bygroups(Punctuation, Other)),
+ (r'(\$[a-zA-Z]+)(\.?)(text|value)?',
+ bygroups(Name.Variable, Punctuation, Name.Property)),
+ (r'\{', Punctuation, '#push'),
+ (r'\}', Punctuation, '#pop'),
+ ],
+ 'nested-arg-action': [
+ (r'(' + r'|'.join(( # keep host code in largest possible chunks.
+ r'[^$\[\]\'"/]+', # exclude unsafe characters
+
+ # strings and comments may safely contain unsafe characters
+ r'"(\\\\|\\"|[^"])*"', # double quote string
+ r"'(\\\\|\\'|[^'])*'", # single quote string
+ r'//.*$\n?', # single line comment
+ r'/\*(.|\n)*?\*/', # multi-line javadoc-style comment
+
+ # regular expression: There's no reason for it to start
+ # with a * and this stops confusion with comments.
+ r'/(?!\*)(\\\\|\\/|[^/])*/',
+
+ # Now that we've handled regex and javadoc comments
+ # it's safe to let / through.
+ r'/',
+ )) + r')+', Other),
+
+
+ (r'\[', Punctuation, '#push'),
+ (r'\]', Punctuation, '#pop'),
+ (r'(\$[a-zA-Z]+)(\.?)(text|value)?',
+ bygroups(Name.Variable, Punctuation, Name.Property)),
+ (r'(\\\\|\\\]|\\\[|[^\[\]])+', Other),
+ ]
+ }
+
+ def analyse_text(text):
+ return re.search(r'^\s*grammar\s+[a-zA-Z0-9]+\s*;', text, re.M)
+
+# http://www.antlr.org/wiki/display/ANTLR3/Code+Generation+Targets
+
+# TH: I'm not aware of any language features of C++ that will cause
+# incorrect lexing of C files. Antlr doesn't appear to make a distinction,
+# so just assume they're C++. No idea how to make Objective C work in the
+# future.
+
+# class AntlrCLexer(DelegatingLexer):
+# """
+# ANTLR with C Target
+#
+# .. versionadded:: 1.1
+# """
+#
+# name = 'ANTLR With C Target'
+# aliases = ['antlr-c']
+# filenames = ['*.G', '*.g']
+#
+# def __init__(self, **options):
+# super(AntlrCLexer, self).__init__(CLexer, AntlrLexer, **options)
+#
+# def analyse_text(text):
+# return re.match(r'^\s*language\s*=\s*C\s*;', text)
+
+
+class AntlrCppLexer(DelegatingLexer):
+ """
+ `ANTLR`_ with CPP Target
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'ANTLR With CPP Target'
+ aliases = ['antlr-cpp']
+ filenames = ['*.G', '*.g']
+
+ def __init__(self, **options):
+ super(AntlrCppLexer, self).__init__(CppLexer, AntlrLexer, **options)
+
+ def analyse_text(text):
+ return AntlrLexer.analyse_text(text) and \
+ re.search(r'^\s*language\s*=\s*C\s*;', text, re.M)
+
+
+class AntlrObjectiveCLexer(DelegatingLexer):
+ """
+ `ANTLR`_ with Objective-C Target
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'ANTLR With ObjectiveC Target'
+ aliases = ['antlr-objc']
+ filenames = ['*.G', '*.g']
+
+ def __init__(self, **options):
+ super(AntlrObjectiveCLexer, self).__init__(ObjectiveCLexer,
+ AntlrLexer, **options)
+
+ def analyse_text(text):
+ return AntlrLexer.analyse_text(text) and \
+ re.search(r'^\s*language\s*=\s*ObjC\s*;', text)
+
+
+class AntlrCSharpLexer(DelegatingLexer):
+ """
+ `ANTLR`_ with C# Target
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'ANTLR With C# Target'
+ aliases = ['antlr-csharp', 'antlr-c#']
+ filenames = ['*.G', '*.g']
+
+ def __init__(self, **options):
+ super(AntlrCSharpLexer, self).__init__(CSharpLexer, AntlrLexer,
+ **options)
+
+ def analyse_text(text):
+ return AntlrLexer.analyse_text(text) and \
+ re.search(r'^\s*language\s*=\s*CSharp2\s*;', text, re.M)
+
+
+class AntlrPythonLexer(DelegatingLexer):
+ """
+ `ANTLR`_ with Python Target
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'ANTLR With Python Target'
+ aliases = ['antlr-python']
+ filenames = ['*.G', '*.g']
+
+ def __init__(self, **options):
+ super(AntlrPythonLexer, self).__init__(PythonLexer, AntlrLexer,
+ **options)
+
+ def analyse_text(text):
+ return AntlrLexer.analyse_text(text) and \
+ re.search(r'^\s*language\s*=\s*Python\s*;', text, re.M)
+
+
+class AntlrJavaLexer(DelegatingLexer):
+ """
+ `ANTLR`_ with Java Target
+
+ .. versionadded:: 1.
+ """
+
+ name = 'ANTLR With Java Target'
+ aliases = ['antlr-java']
+ filenames = ['*.G', '*.g']
+
+ def __init__(self, **options):
+ super(AntlrJavaLexer, self).__init__(JavaLexer, AntlrLexer,
+ **options)
+
+ def analyse_text(text):
+ # Antlr language is Java by default
+ return AntlrLexer.analyse_text(text) and 0.9
+
+
+class AntlrRubyLexer(DelegatingLexer):
+ """
+ `ANTLR`_ with Ruby Target
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'ANTLR With Ruby Target'
+ aliases = ['antlr-ruby', 'antlr-rb']
+ filenames = ['*.G', '*.g']
+
+ def __init__(self, **options):
+ super(AntlrRubyLexer, self).__init__(RubyLexer, AntlrLexer,
+ **options)
+
+ def analyse_text(text):
+ return AntlrLexer.analyse_text(text) and \
+ re.search(r'^\s*language\s*=\s*Ruby\s*;', text, re.M)
+
+
+class AntlrPerlLexer(DelegatingLexer):
+ """
+ `ANTLR`_ with Perl Target
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'ANTLR With Perl Target'
+ aliases = ['antlr-perl']
+ filenames = ['*.G', '*.g']
+
+ def __init__(self, **options):
+ super(AntlrPerlLexer, self).__init__(PerlLexer, AntlrLexer,
+ **options)
+
+ def analyse_text(text):
+ return AntlrLexer.analyse_text(text) and \
+ re.search(r'^\s*language\s*=\s*Perl5\s*;', text, re.M)
+
+
+class AntlrActionScriptLexer(DelegatingLexer):
+ """
+ `ANTLR`_ with ActionScript Target
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'ANTLR With ActionScript Target'
+ aliases = ['antlr-as', 'antlr-actionscript']
+ filenames = ['*.G', '*.g']
+
+ def __init__(self, **options):
+ from pygments.lexers.actionscript import ActionScriptLexer
+ super(AntlrActionScriptLexer, self).__init__(ActionScriptLexer,
+ AntlrLexer, **options)
+
+ def analyse_text(text):
+ return AntlrLexer.analyse_text(text) and \
+ re.search(r'^\s*language\s*=\s*ActionScript\s*;', text, re.M)
+
+
+class TreetopBaseLexer(RegexLexer):
+ """
+ A base lexer for `Treetop <http://treetop.rubyforge.org/>`_ grammars.
+ Not for direct use; use TreetopLexer instead.
+
+ .. versionadded:: 1.6
+ """
+
+ tokens = {
+ 'root': [
+ include('space'),
+ (r'require[ \t]+[^\n\r]+[\n\r]', Other),
+ (r'module\b', Keyword.Namespace, 'module'),
+ (r'grammar\b', Keyword, 'grammar'),
+ ],
+ 'module': [
+ include('space'),
+ include('end'),
+ (r'module\b', Keyword, '#push'),
+ (r'grammar\b', Keyword, 'grammar'),
+ (r'[A-Z]\w*(?:::[A-Z]\w*)*', Name.Namespace),
+ ],
+ 'grammar': [
+ include('space'),
+ include('end'),
+ (r'rule\b', Keyword, 'rule'),
+ (r'include\b', Keyword, 'include'),
+ (r'[A-Z]\w*', Name),
+ ],
+ 'include': [
+ include('space'),
+ (r'[A-Z]\w*(?:::[A-Z]\w*)*', Name.Class, '#pop'),
+ ],
+ 'rule': [
+ include('space'),
+ include('end'),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ (r'([A-Za-z_]\w*)(:)', bygroups(Name.Label, Punctuation)),
+ (r'[A-Za-z_]\w*', Name),
+ (r'[()]', Punctuation),
+ (r'[?+*/&!~]', Operator),
+ (r'\[(?:\\.|\[:\^?[a-z]+:\]|[^\\\]])+\]', String.Regex),
+ (r'([0-9]*)(\.\.)([0-9]*)',
+ bygroups(Number.Integer, Operator, Number.Integer)),
+ (r'(<)([^>]+)(>)', bygroups(Punctuation, Name.Class, Punctuation)),
+ (r'\{', Punctuation, 'inline_module'),
+ (r'\.', String.Regex),
+ ],
+ 'inline_module': [
+ (r'\{', Other, 'ruby'),
+ (r'\}', Punctuation, '#pop'),
+ (r'[^{}]+', Other),
+ ],
+ 'ruby': [
+ (r'\{', Other, '#push'),
+ (r'\}', Other, '#pop'),
+ (r'[^{}]+', Other),
+ ],
+ 'space': [
+ (r'[ \t\n\r]+', Whitespace),
+ (r'#[^\n]*', Comment.Single),
+ ],
+ 'end': [
+ (r'end\b', Keyword, '#pop'),
+ ],
+ }
+
+
+class TreetopLexer(DelegatingLexer):
+ """
+ A lexer for `Treetop <http://treetop.rubyforge.org/>`_ grammars.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'Treetop'
+ aliases = ['treetop']
+ filenames = ['*.treetop', '*.tt']
+
+ def __init__(self, **options):
+ super(TreetopLexer, self).__init__(RubyLexer, TreetopBaseLexer, **options)
+
+
+class EbnfLexer(RegexLexer):
+ """
+ Lexer for `ISO/IEC 14977 EBNF
+ <http://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_Form>`_
+ grammars.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'EBNF'
+ aliases = ['ebnf']
+ filenames = ['*.ebnf']
+ mimetypes = ['text/x-ebnf']
+
+ tokens = {
+ 'root': [
+ include('whitespace'),
+ include('comment_start'),
+ include('identifier'),
+ (r'=', Operator, 'production'),
+ ],
+ 'production': [
+ include('whitespace'),
+ include('comment_start'),
+ include('identifier'),
+ (r'"[^"]*"', String.Double),
+ (r"'[^']*'", String.Single),
+ (r'(\?[^?]*\?)', Name.Entity),
+ (r'[\[\]{}(),|]', Punctuation),
+ (r'-', Operator),
+ (r';', Punctuation, '#pop'),
+ (r'\.', Punctuation, '#pop'),
+ ],
+ 'whitespace': [
+ (r'\s+', Text),
+ ],
+ 'comment_start': [
+ (r'\(\*', Comment.Multiline, 'comment'),
+ ],
+ 'comment': [
+ (r'[^*)]', Comment.Multiline),
+ include('comment_start'),
+ (r'\*\)', Comment.Multiline, '#pop'),
+ (r'[*)]', Comment.Multiline),
+ ],
+ 'identifier': [
+ (r'([a-zA-Z][\w \-]*)', Keyword),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/pascal.py b/wandb/vendor/pygments/lexers/pascal.py
new file mode 100644
index 0000000000000000000000000000000000000000..9aa1ac8f8833c60416e1fa6fb6f8711b6d92aca0
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/pascal.py
@@ -0,0 +1,644 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.pascal
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Pascal family languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, include, bygroups, words, \
+ using, this, default
+from pygments.util import get_bool_opt, get_list_opt
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Error
+from pygments.scanner import Scanner
+
+# compatibility import
+from pygments.lexers.modula2 import Modula2Lexer
+
+__all__ = ['DelphiLexer', 'AdaLexer']
+
+
+class DelphiLexer(Lexer):
+ """
+ For `Delphi <http://www.borland.com/delphi/>`_ (Borland Object Pascal),
+ Turbo Pascal and Free Pascal source code.
+
+ Additional options accepted:
+
+ `turbopascal`
+ Highlight Turbo Pascal specific keywords (default: ``True``).
+ `delphi`
+ Highlight Borland Delphi specific keywords (default: ``True``).
+ `freepascal`
+ Highlight Free Pascal specific keywords (default: ``True``).
+ `units`
+ A list of units that should be considered builtin, supported are
+ ``System``, ``SysUtils``, ``Classes`` and ``Math``.
+ Default is to consider all of them builtin.
+ """
+ name = 'Delphi'
+ aliases = ['delphi', 'pas', 'pascal', 'objectpascal']
+ filenames = ['*.pas', '*.dpr']
+ mimetypes = ['text/x-pascal']
+
+ TURBO_PASCAL_KEYWORDS = (
+ 'absolute', 'and', 'array', 'asm', 'begin', 'break', 'case',
+ 'const', 'constructor', 'continue', 'destructor', 'div', 'do',
+ 'downto', 'else', 'end', 'file', 'for', 'function', 'goto',
+ 'if', 'implementation', 'in', 'inherited', 'inline', 'interface',
+ 'label', 'mod', 'nil', 'not', 'object', 'of', 'on', 'operator',
+ 'or', 'packed', 'procedure', 'program', 'record', 'reintroduce',
+ 'repeat', 'self', 'set', 'shl', 'shr', 'string', 'then', 'to',
+ 'type', 'unit', 'until', 'uses', 'var', 'while', 'with', 'xor'
+ )
+
+ DELPHI_KEYWORDS = (
+ 'as', 'class', 'except', 'exports', 'finalization', 'finally',
+ 'initialization', 'is', 'library', 'on', 'property', 'raise',
+ 'threadvar', 'try'
+ )
+
+ FREE_PASCAL_KEYWORDS = (
+ 'dispose', 'exit', 'false', 'new', 'true'
+ )
+
+ BLOCK_KEYWORDS = set((
+ 'begin', 'class', 'const', 'constructor', 'destructor', 'end',
+ 'finalization', 'function', 'implementation', 'initialization',
+ 'label', 'library', 'operator', 'procedure', 'program', 'property',
+ 'record', 'threadvar', 'type', 'unit', 'uses', 'var'
+ ))
+
+ FUNCTION_MODIFIERS = set((
+ 'alias', 'cdecl', 'export', 'inline', 'interrupt', 'nostackframe',
+ 'pascal', 'register', 'safecall', 'softfloat', 'stdcall',
+ 'varargs', 'name', 'dynamic', 'near', 'virtual', 'external',
+ 'override', 'assembler'
+ ))
+
+ # XXX: those aren't global. but currently we know no way for defining
+ # them just for the type context.
+ DIRECTIVES = set((
+ 'absolute', 'abstract', 'assembler', 'cppdecl', 'default', 'far',
+ 'far16', 'forward', 'index', 'oldfpccall', 'private', 'protected',
+ 'published', 'public'
+ ))
+
+ BUILTIN_TYPES = set((
+ 'ansichar', 'ansistring', 'bool', 'boolean', 'byte', 'bytebool',
+ 'cardinal', 'char', 'comp', 'currency', 'double', 'dword',
+ 'extended', 'int64', 'integer', 'iunknown', 'longbool', 'longint',
+ 'longword', 'pansichar', 'pansistring', 'pbool', 'pboolean',
+ 'pbyte', 'pbytearray', 'pcardinal', 'pchar', 'pcomp', 'pcurrency',
+ 'pdate', 'pdatetime', 'pdouble', 'pdword', 'pextended', 'phandle',
+ 'pint64', 'pinteger', 'plongint', 'plongword', 'pointer',
+ 'ppointer', 'pshortint', 'pshortstring', 'psingle', 'psmallint',
+ 'pstring', 'pvariant', 'pwidechar', 'pwidestring', 'pword',
+ 'pwordarray', 'pwordbool', 'real', 'real48', 'shortint',
+ 'shortstring', 'single', 'smallint', 'string', 'tclass', 'tdate',
+ 'tdatetime', 'textfile', 'thandle', 'tobject', 'ttime', 'variant',
+ 'widechar', 'widestring', 'word', 'wordbool'
+ ))
+
+ BUILTIN_UNITS = {
+ 'System': (
+ 'abs', 'acquireexceptionobject', 'addr', 'ansitoutf8',
+ 'append', 'arctan', 'assert', 'assigned', 'assignfile',
+ 'beginthread', 'blockread', 'blockwrite', 'break', 'chdir',
+ 'chr', 'close', 'closefile', 'comptocurrency', 'comptodouble',
+ 'concat', 'continue', 'copy', 'cos', 'dec', 'delete',
+ 'dispose', 'doubletocomp', 'endthread', 'enummodules',
+ 'enumresourcemodules', 'eof', 'eoln', 'erase', 'exceptaddr',
+ 'exceptobject', 'exclude', 'exit', 'exp', 'filepos', 'filesize',
+ 'fillchar', 'finalize', 'findclasshinstance', 'findhinstance',
+ 'findresourcehinstance', 'flush', 'frac', 'freemem',
+ 'get8087cw', 'getdir', 'getlasterror', 'getmem',
+ 'getmemorymanager', 'getmodulefilename', 'getvariantmanager',
+ 'halt', 'hi', 'high', 'inc', 'include', 'initialize', 'insert',
+ 'int', 'ioresult', 'ismemorymanagerset', 'isvariantmanagerset',
+ 'length', 'ln', 'lo', 'low', 'mkdir', 'move', 'new', 'odd',
+ 'olestrtostring', 'olestrtostrvar', 'ord', 'paramcount',
+ 'paramstr', 'pi', 'pos', 'pred', 'ptr', 'pucs4chars', 'random',
+ 'randomize', 'read', 'readln', 'reallocmem',
+ 'releaseexceptionobject', 'rename', 'reset', 'rewrite', 'rmdir',
+ 'round', 'runerror', 'seek', 'seekeof', 'seekeoln',
+ 'set8087cw', 'setlength', 'setlinebreakstyle',
+ 'setmemorymanager', 'setstring', 'settextbuf',
+ 'setvariantmanager', 'sin', 'sizeof', 'slice', 'sqr', 'sqrt',
+ 'str', 'stringofchar', 'stringtoolestr', 'stringtowidechar',
+ 'succ', 'swap', 'trunc', 'truncate', 'typeinfo',
+ 'ucs4stringtowidestring', 'unicodetoutf8', 'uniquestring',
+ 'upcase', 'utf8decode', 'utf8encode', 'utf8toansi',
+ 'utf8tounicode', 'val', 'vararrayredim', 'varclear',
+ 'widecharlentostring', 'widecharlentostrvar',
+ 'widechartostring', 'widechartostrvar',
+ 'widestringtoucs4string', 'write', 'writeln'
+ ),
+ 'SysUtils': (
+ 'abort', 'addexitproc', 'addterminateproc', 'adjustlinebreaks',
+ 'allocmem', 'ansicomparefilename', 'ansicomparestr',
+ 'ansicomparetext', 'ansidequotedstr', 'ansiextractquotedstr',
+ 'ansilastchar', 'ansilowercase', 'ansilowercasefilename',
+ 'ansipos', 'ansiquotedstr', 'ansisamestr', 'ansisametext',
+ 'ansistrcomp', 'ansistricomp', 'ansistrlastchar', 'ansistrlcomp',
+ 'ansistrlicomp', 'ansistrlower', 'ansistrpos', 'ansistrrscan',
+ 'ansistrscan', 'ansistrupper', 'ansiuppercase',
+ 'ansiuppercasefilename', 'appendstr', 'assignstr', 'beep',
+ 'booltostr', 'bytetocharindex', 'bytetocharlen', 'bytetype',
+ 'callterminateprocs', 'changefileext', 'charlength',
+ 'chartobyteindex', 'chartobytelen', 'comparemem', 'comparestr',
+ 'comparetext', 'createdir', 'createguid', 'currentyear',
+ 'currtostr', 'currtostrf', 'date', 'datetimetofiledate',
+ 'datetimetostr', 'datetimetostring', 'datetimetosystemtime',
+ 'datetimetotimestamp', 'datetostr', 'dayofweek', 'decodedate',
+ 'decodedatefully', 'decodetime', 'deletefile', 'directoryexists',
+ 'diskfree', 'disksize', 'disposestr', 'encodedate', 'encodetime',
+ 'exceptionerrormessage', 'excludetrailingbackslash',
+ 'excludetrailingpathdelimiter', 'expandfilename',
+ 'expandfilenamecase', 'expanduncfilename', 'extractfiledir',
+ 'extractfiledrive', 'extractfileext', 'extractfilename',
+ 'extractfilepath', 'extractrelativepath', 'extractshortpathname',
+ 'fileage', 'fileclose', 'filecreate', 'filedatetodatetime',
+ 'fileexists', 'filegetattr', 'filegetdate', 'fileisreadonly',
+ 'fileopen', 'fileread', 'filesearch', 'fileseek', 'filesetattr',
+ 'filesetdate', 'filesetreadonly', 'filewrite', 'finalizepackage',
+ 'findclose', 'findcmdlineswitch', 'findfirst', 'findnext',
+ 'floattocurr', 'floattodatetime', 'floattodecimal', 'floattostr',
+ 'floattostrf', 'floattotext', 'floattotextfmt', 'fmtloadstr',
+ 'fmtstr', 'forcedirectories', 'format', 'formatbuf', 'formatcurr',
+ 'formatdatetime', 'formatfloat', 'freeandnil', 'getcurrentdir',
+ 'getenvironmentvariable', 'getfileversion', 'getformatsettings',
+ 'getlocaleformatsettings', 'getmodulename', 'getpackagedescription',
+ 'getpackageinfo', 'gettime', 'guidtostring', 'incamonth',
+ 'includetrailingbackslash', 'includetrailingpathdelimiter',
+ 'incmonth', 'initializepackage', 'interlockeddecrement',
+ 'interlockedexchange', 'interlockedexchangeadd',
+ 'interlockedincrement', 'inttohex', 'inttostr', 'isdelimiter',
+ 'isequalguid', 'isleapyear', 'ispathdelimiter', 'isvalidident',
+ 'languages', 'lastdelimiter', 'loadpackage', 'loadstr',
+ 'lowercase', 'msecstotimestamp', 'newstr', 'nextcharindex', 'now',
+ 'outofmemoryerror', 'quotedstr', 'raiselastoserror',
+ 'raiselastwin32error', 'removedir', 'renamefile', 'replacedate',
+ 'replacetime', 'safeloadlibrary', 'samefilename', 'sametext',
+ 'setcurrentdir', 'showexception', 'sleep', 'stralloc', 'strbufsize',
+ 'strbytetype', 'strcat', 'strcharlength', 'strcomp', 'strcopy',
+ 'strdispose', 'strecopy', 'strend', 'strfmt', 'stricomp',
+ 'stringreplace', 'stringtoguid', 'strlcat', 'strlcomp', 'strlcopy',
+ 'strlen', 'strlfmt', 'strlicomp', 'strlower', 'strmove', 'strnew',
+ 'strnextchar', 'strpas', 'strpcopy', 'strplcopy', 'strpos',
+ 'strrscan', 'strscan', 'strtobool', 'strtobooldef', 'strtocurr',
+ 'strtocurrdef', 'strtodate', 'strtodatedef', 'strtodatetime',
+ 'strtodatetimedef', 'strtofloat', 'strtofloatdef', 'strtoint',
+ 'strtoint64', 'strtoint64def', 'strtointdef', 'strtotime',
+ 'strtotimedef', 'strupper', 'supports', 'syserrormessage',
+ 'systemtimetodatetime', 'texttofloat', 'time', 'timestamptodatetime',
+ 'timestamptomsecs', 'timetostr', 'trim', 'trimleft', 'trimright',
+ 'tryencodedate', 'tryencodetime', 'tryfloattocurr', 'tryfloattodatetime',
+ 'trystrtobool', 'trystrtocurr', 'trystrtodate', 'trystrtodatetime',
+ 'trystrtofloat', 'trystrtoint', 'trystrtoint64', 'trystrtotime',
+ 'unloadpackage', 'uppercase', 'widecomparestr', 'widecomparetext',
+ 'widefmtstr', 'wideformat', 'wideformatbuf', 'widelowercase',
+ 'widesamestr', 'widesametext', 'wideuppercase', 'win32check',
+ 'wraptext'
+ ),
+ 'Classes': (
+ 'activateclassgroup', 'allocatehwnd', 'bintohex', 'checksynchronize',
+ 'collectionsequal', 'countgenerations', 'deallocatehwnd', 'equalrect',
+ 'extractstrings', 'findclass', 'findglobalcomponent', 'getclass',
+ 'groupdescendantswith', 'hextobin', 'identtoint',
+ 'initinheritedcomponent', 'inttoident', 'invalidpoint',
+ 'isuniqueglobalcomponentname', 'linestart', 'objectbinarytotext',
+ 'objectresourcetotext', 'objecttexttobinary', 'objecttexttoresource',
+ 'pointsequal', 'readcomponentres', 'readcomponentresex',
+ 'readcomponentresfile', 'rect', 'registerclass', 'registerclassalias',
+ 'registerclasses', 'registercomponents', 'registerintegerconsts',
+ 'registernoicon', 'registernonactivex', 'smallpoint', 'startclassgroup',
+ 'teststreamformat', 'unregisterclass', 'unregisterclasses',
+ 'unregisterintegerconsts', 'unregistermoduleclasses',
+ 'writecomponentresfile'
+ ),
+ 'Math': (
+ 'arccos', 'arccosh', 'arccot', 'arccoth', 'arccsc', 'arccsch', 'arcsec',
+ 'arcsech', 'arcsin', 'arcsinh', 'arctan2', 'arctanh', 'ceil',
+ 'comparevalue', 'cosecant', 'cosh', 'cot', 'cotan', 'coth', 'csc',
+ 'csch', 'cycletodeg', 'cycletograd', 'cycletorad', 'degtocycle',
+ 'degtograd', 'degtorad', 'divmod', 'doubledecliningbalance',
+ 'ensurerange', 'floor', 'frexp', 'futurevalue', 'getexceptionmask',
+ 'getprecisionmode', 'getroundmode', 'gradtocycle', 'gradtodeg',
+ 'gradtorad', 'hypot', 'inrange', 'interestpayment', 'interestrate',
+ 'internalrateofreturn', 'intpower', 'isinfinite', 'isnan', 'iszero',
+ 'ldexp', 'lnxp1', 'log10', 'log2', 'logn', 'max', 'maxintvalue',
+ 'maxvalue', 'mean', 'meanandstddev', 'min', 'minintvalue', 'minvalue',
+ 'momentskewkurtosis', 'netpresentvalue', 'norm', 'numberofperiods',
+ 'payment', 'periodpayment', 'poly', 'popnstddev', 'popnvariance',
+ 'power', 'presentvalue', 'radtocycle', 'radtodeg', 'radtograd',
+ 'randg', 'randomrange', 'roundto', 'samevalue', 'sec', 'secant',
+ 'sech', 'setexceptionmask', 'setprecisionmode', 'setroundmode',
+ 'sign', 'simpleroundto', 'sincos', 'sinh', 'slndepreciation', 'stddev',
+ 'sum', 'sumint', 'sumofsquares', 'sumsandsquares', 'syddepreciation',
+ 'tan', 'tanh', 'totalvariance', 'variance'
+ )
+ }
+
+ ASM_REGISTERS = set((
+ 'ah', 'al', 'ax', 'bh', 'bl', 'bp', 'bx', 'ch', 'cl', 'cr0',
+ 'cr1', 'cr2', 'cr3', 'cr4', 'cs', 'cx', 'dh', 'di', 'dl', 'dr0',
+ 'dr1', 'dr2', 'dr3', 'dr4', 'dr5', 'dr6', 'dr7', 'ds', 'dx',
+ 'eax', 'ebp', 'ebx', 'ecx', 'edi', 'edx', 'es', 'esi', 'esp',
+ 'fs', 'gs', 'mm0', 'mm1', 'mm2', 'mm3', 'mm4', 'mm5', 'mm6',
+ 'mm7', 'si', 'sp', 'ss', 'st0', 'st1', 'st2', 'st3', 'st4', 'st5',
+ 'st6', 'st7', 'xmm0', 'xmm1', 'xmm2', 'xmm3', 'xmm4', 'xmm5',
+ 'xmm6', 'xmm7'
+ ))
+
+ ASM_INSTRUCTIONS = set((
+ 'aaa', 'aad', 'aam', 'aas', 'adc', 'add', 'and', 'arpl', 'bound',
+ 'bsf', 'bsr', 'bswap', 'bt', 'btc', 'btr', 'bts', 'call', 'cbw',
+ 'cdq', 'clc', 'cld', 'cli', 'clts', 'cmc', 'cmova', 'cmovae',
+ 'cmovb', 'cmovbe', 'cmovc', 'cmovcxz', 'cmove', 'cmovg',
+ 'cmovge', 'cmovl', 'cmovle', 'cmovna', 'cmovnae', 'cmovnb',
+ 'cmovnbe', 'cmovnc', 'cmovne', 'cmovng', 'cmovnge', 'cmovnl',
+ 'cmovnle', 'cmovno', 'cmovnp', 'cmovns', 'cmovnz', 'cmovo',
+ 'cmovp', 'cmovpe', 'cmovpo', 'cmovs', 'cmovz', 'cmp', 'cmpsb',
+ 'cmpsd', 'cmpsw', 'cmpxchg', 'cmpxchg486', 'cmpxchg8b', 'cpuid',
+ 'cwd', 'cwde', 'daa', 'das', 'dec', 'div', 'emms', 'enter', 'hlt',
+ 'ibts', 'icebp', 'idiv', 'imul', 'in', 'inc', 'insb', 'insd',
+ 'insw', 'int', 'int01', 'int03', 'int1', 'int3', 'into', 'invd',
+ 'invlpg', 'iret', 'iretd', 'iretw', 'ja', 'jae', 'jb', 'jbe',
+ 'jc', 'jcxz', 'jcxz', 'je', 'jecxz', 'jg', 'jge', 'jl', 'jle',
+ 'jmp', 'jna', 'jnae', 'jnb', 'jnbe', 'jnc', 'jne', 'jng', 'jnge',
+ 'jnl', 'jnle', 'jno', 'jnp', 'jns', 'jnz', 'jo', 'jp', 'jpe',
+ 'jpo', 'js', 'jz', 'lahf', 'lar', 'lcall', 'lds', 'lea', 'leave',
+ 'les', 'lfs', 'lgdt', 'lgs', 'lidt', 'ljmp', 'lldt', 'lmsw',
+ 'loadall', 'loadall286', 'lock', 'lodsb', 'lodsd', 'lodsw',
+ 'loop', 'loope', 'loopne', 'loopnz', 'loopz', 'lsl', 'lss', 'ltr',
+ 'mov', 'movd', 'movq', 'movsb', 'movsd', 'movsw', 'movsx',
+ 'movzx', 'mul', 'neg', 'nop', 'not', 'or', 'out', 'outsb', 'outsd',
+ 'outsw', 'pop', 'popa', 'popad', 'popaw', 'popf', 'popfd', 'popfw',
+ 'push', 'pusha', 'pushad', 'pushaw', 'pushf', 'pushfd', 'pushfw',
+ 'rcl', 'rcr', 'rdmsr', 'rdpmc', 'rdshr', 'rdtsc', 'rep', 'repe',
+ 'repne', 'repnz', 'repz', 'ret', 'retf', 'retn', 'rol', 'ror',
+ 'rsdc', 'rsldt', 'rsm', 'sahf', 'sal', 'salc', 'sar', 'sbb',
+ 'scasb', 'scasd', 'scasw', 'seta', 'setae', 'setb', 'setbe',
+ 'setc', 'setcxz', 'sete', 'setg', 'setge', 'setl', 'setle',
+ 'setna', 'setnae', 'setnb', 'setnbe', 'setnc', 'setne', 'setng',
+ 'setnge', 'setnl', 'setnle', 'setno', 'setnp', 'setns', 'setnz',
+ 'seto', 'setp', 'setpe', 'setpo', 'sets', 'setz', 'sgdt', 'shl',
+ 'shld', 'shr', 'shrd', 'sidt', 'sldt', 'smi', 'smint', 'smintold',
+ 'smsw', 'stc', 'std', 'sti', 'stosb', 'stosd', 'stosw', 'str',
+ 'sub', 'svdc', 'svldt', 'svts', 'syscall', 'sysenter', 'sysexit',
+ 'sysret', 'test', 'ud1', 'ud2', 'umov', 'verr', 'verw', 'wait',
+ 'wbinvd', 'wrmsr', 'wrshr', 'xadd', 'xbts', 'xchg', 'xlat',
+ 'xlatb', 'xor'
+ ))
+
+ def __init__(self, **options):
+ Lexer.__init__(self, **options)
+ self.keywords = set()
+ if get_bool_opt(options, 'turbopascal', True):
+ self.keywords.update(self.TURBO_PASCAL_KEYWORDS)
+ if get_bool_opt(options, 'delphi', True):
+ self.keywords.update(self.DELPHI_KEYWORDS)
+ if get_bool_opt(options, 'freepascal', True):
+ self.keywords.update(self.FREE_PASCAL_KEYWORDS)
+ self.builtins = set()
+ for unit in get_list_opt(options, 'units', list(self.BUILTIN_UNITS)):
+ self.builtins.update(self.BUILTIN_UNITS[unit])
+
+ def get_tokens_unprocessed(self, text):
+ scanner = Scanner(text, re.DOTALL | re.MULTILINE | re.IGNORECASE)
+ stack = ['initial']
+ in_function_block = False
+ in_property_block = False
+ was_dot = False
+ next_token_is_function = False
+ next_token_is_property = False
+ collect_labels = False
+ block_labels = set()
+ brace_balance = [0, 0]
+
+ while not scanner.eos:
+ token = Error
+
+ if stack[-1] == 'initial':
+ if scanner.scan(r'\s+'):
+ token = Text
+ elif scanner.scan(r'\{.*?\}|\(\*.*?\*\)'):
+ if scanner.match.startswith('$'):
+ token = Comment.Preproc
+ else:
+ token = Comment.Multiline
+ elif scanner.scan(r'//.*?$'):
+ token = Comment.Single
+ elif scanner.scan(r'[-+*\/=<>:;,.@\^]'):
+ token = Operator
+ # stop label highlighting on next ";"
+ if collect_labels and scanner.match == ';':
+ collect_labels = False
+ elif scanner.scan(r'[\(\)\[\]]+'):
+ token = Punctuation
+ # abort function naming ``foo = Function(...)``
+ next_token_is_function = False
+ # if we are in a function block we count the open
+ # braces because ootherwise it's impossible to
+ # determine the end of the modifier context
+ if in_function_block or in_property_block:
+ if scanner.match == '(':
+ brace_balance[0] += 1
+ elif scanner.match == ')':
+ brace_balance[0] -= 1
+ elif scanner.match == '[':
+ brace_balance[1] += 1
+ elif scanner.match == ']':
+ brace_balance[1] -= 1
+ elif scanner.scan(r'[A-Za-z_][A-Za-z_0-9]*'):
+ lowercase_name = scanner.match.lower()
+ if lowercase_name == 'result':
+ token = Name.Builtin.Pseudo
+ elif lowercase_name in self.keywords:
+ token = Keyword
+ # if we are in a special block and a
+ # block ending keyword occours (and the parenthesis
+ # is balanced) we end the current block context
+ if (in_function_block or in_property_block) and \
+ lowercase_name in self.BLOCK_KEYWORDS and \
+ brace_balance[0] <= 0 and \
+ brace_balance[1] <= 0:
+ in_function_block = False
+ in_property_block = False
+ brace_balance = [0, 0]
+ block_labels = set()
+ if lowercase_name in ('label', 'goto'):
+ collect_labels = True
+ elif lowercase_name == 'asm':
+ stack.append('asm')
+ elif lowercase_name == 'property':
+ in_property_block = True
+ next_token_is_property = True
+ elif lowercase_name in ('procedure', 'operator',
+ 'function', 'constructor',
+ 'destructor'):
+ in_function_block = True
+ next_token_is_function = True
+ # we are in a function block and the current name
+ # is in the set of registered modifiers. highlight
+ # it as pseudo keyword
+ elif in_function_block and \
+ lowercase_name in self.FUNCTION_MODIFIERS:
+ token = Keyword.Pseudo
+ # if we are in a property highlight some more
+ # modifiers
+ elif in_property_block and \
+ lowercase_name in ('read', 'write'):
+ token = Keyword.Pseudo
+ next_token_is_function = True
+ # if the last iteration set next_token_is_function
+ # to true we now want this name highlighted as
+ # function. so do that and reset the state
+ elif next_token_is_function:
+ # Look if the next token is a dot. If yes it's
+ # not a function, but a class name and the
+ # part after the dot a function name
+ if scanner.test(r'\s*\.\s*'):
+ token = Name.Class
+ # it's not a dot, our job is done
+ else:
+ token = Name.Function
+ next_token_is_function = False
+ # same for properties
+ elif next_token_is_property:
+ token = Name.Property
+ next_token_is_property = False
+ # Highlight this token as label and add it
+ # to the list of known labels
+ elif collect_labels:
+ token = Name.Label
+ block_labels.add(scanner.match.lower())
+ # name is in list of known labels
+ elif lowercase_name in block_labels:
+ token = Name.Label
+ elif lowercase_name in self.BUILTIN_TYPES:
+ token = Keyword.Type
+ elif lowercase_name in self.DIRECTIVES:
+ token = Keyword.Pseudo
+ # builtins are just builtins if the token
+ # before isn't a dot
+ elif not was_dot and lowercase_name in self.builtins:
+ token = Name.Builtin
+ else:
+ token = Name
+ elif scanner.scan(r"'"):
+ token = String
+ stack.append('string')
+ elif scanner.scan(r'\#(\d+|\$[0-9A-Fa-f]+)'):
+ token = String.Char
+ elif scanner.scan(r'\$[0-9A-Fa-f]+'):
+ token = Number.Hex
+ elif scanner.scan(r'\d+(?![eE]|\.[^.])'):
+ token = Number.Integer
+ elif scanner.scan(r'\d+(\.\d+([eE][+-]?\d+)?|[eE][+-]?\d+)'):
+ token = Number.Float
+ else:
+ # if the stack depth is deeper than once, pop
+ if len(stack) > 1:
+ stack.pop()
+ scanner.get_char()
+
+ elif stack[-1] == 'string':
+ if scanner.scan(r"''"):
+ token = String.Escape
+ elif scanner.scan(r"'"):
+ token = String
+ stack.pop()
+ elif scanner.scan(r"[^']*"):
+ token = String
+ else:
+ scanner.get_char()
+ stack.pop()
+
+ elif stack[-1] == 'asm':
+ if scanner.scan(r'\s+'):
+ token = Text
+ elif scanner.scan(r'end'):
+ token = Keyword
+ stack.pop()
+ elif scanner.scan(r'\{.*?\}|\(\*.*?\*\)'):
+ if scanner.match.startswith('$'):
+ token = Comment.Preproc
+ else:
+ token = Comment.Multiline
+ elif scanner.scan(r'//.*?$'):
+ token = Comment.Single
+ elif scanner.scan(r"'"):
+ token = String
+ stack.append('string')
+ elif scanner.scan(r'@@[A-Za-z_][A-Za-z_0-9]*'):
+ token = Name.Label
+ elif scanner.scan(r'[A-Za-z_][A-Za-z_0-9]*'):
+ lowercase_name = scanner.match.lower()
+ if lowercase_name in self.ASM_INSTRUCTIONS:
+ token = Keyword
+ elif lowercase_name in self.ASM_REGISTERS:
+ token = Name.Builtin
+ else:
+ token = Name
+ elif scanner.scan(r'[-+*\/=<>:;,.@\^]+'):
+ token = Operator
+ elif scanner.scan(r'[\(\)\[\]]+'):
+ token = Punctuation
+ elif scanner.scan(r'\$[0-9A-Fa-f]+'):
+ token = Number.Hex
+ elif scanner.scan(r'\d+(?![eE]|\.[^.])'):
+ token = Number.Integer
+ elif scanner.scan(r'\d+(\.\d+([eE][+-]?\d+)?|[eE][+-]?\d+)'):
+ token = Number.Float
+ else:
+ scanner.get_char()
+ stack.pop()
+
+ # save the dot!!!11
+ if scanner.match.strip():
+ was_dot = scanner.match == '.'
+ yield scanner.start_pos, token, scanner.match or ''
+
+
+class AdaLexer(RegexLexer):
+ """
+ For Ada source code.
+
+ .. versionadded:: 1.3
+ """
+
+ name = 'Ada'
+ aliases = ['ada', 'ada95', 'ada2005']
+ filenames = ['*.adb', '*.ads', '*.ada']
+ mimetypes = ['text/x-ada']
+
+ flags = re.MULTILINE | re.IGNORECASE
+
+ tokens = {
+ 'root': [
+ (r'[^\S\n]+', Text),
+ (r'--.*?\n', Comment.Single),
+ (r'[^\S\n]+', Text),
+ (r'function|procedure|entry', Keyword.Declaration, 'subprogram'),
+ (r'(subtype|type)(\s+)(\w+)',
+ bygroups(Keyword.Declaration, Text, Keyword.Type), 'type_def'),
+ (r'task|protected', Keyword.Declaration),
+ (r'(subtype)(\s+)', bygroups(Keyword.Declaration, Text)),
+ (r'(end)(\s+)', bygroups(Keyword.Reserved, Text), 'end'),
+ (r'(pragma)(\s+)(\w+)', bygroups(Keyword.Reserved, Text,
+ Comment.Preproc)),
+ (r'(true|false|null)\b', Keyword.Constant),
+ (words((
+ 'Address', 'Byte', 'Boolean', 'Character', 'Controlled', 'Count',
+ 'Cursor', 'Duration', 'File_Mode', 'File_Type', 'Float', 'Generator',
+ 'Integer', 'Long_Float', 'Long_Integer', 'Long_Long_Float',
+ 'Long_Long_Integer', 'Natural', 'Positive', 'Reference_Type',
+ 'Short_Float', 'Short_Integer', 'Short_Short_Float',
+ 'Short_Short_Integer', 'String', 'Wide_Character', 'Wide_String'),
+ suffix=r'\b'),
+ Keyword.Type),
+ (r'(and(\s+then)?|in|mod|not|or(\s+else)|rem)\b', Operator.Word),
+ (r'generic|private', Keyword.Declaration),
+ (r'package', Keyword.Declaration, 'package'),
+ (r'array\b', Keyword.Reserved, 'array_def'),
+ (r'(with|use)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
+ (r'(\w+)(\s*)(:)(\s*)(constant)',
+ bygroups(Name.Constant, Text, Punctuation, Text,
+ Keyword.Reserved)),
+ (r'<<\w+>>', Name.Label),
+ (r'(\w+)(\s*)(:)(\s*)(declare|begin|loop|for|while)',
+ bygroups(Name.Label, Text, Punctuation, Text, Keyword.Reserved)),
+ (words((
+ 'abort', 'abs', 'abstract', 'accept', 'access', 'aliased', 'all',
+ 'array', 'at', 'begin', 'body', 'case', 'constant', 'declare',
+ 'delay', 'delta', 'digits', 'do', 'else', 'elsif', 'end', 'entry',
+ 'exception', 'exit', 'interface', 'for', 'goto', 'if', 'is', 'limited',
+ 'loop', 'new', 'null', 'of', 'or', 'others', 'out', 'overriding',
+ 'pragma', 'protected', 'raise', 'range', 'record', 'renames', 'requeue',
+ 'return', 'reverse', 'select', 'separate', 'subtype', 'synchronized',
+ 'task', 'tagged', 'terminate', 'then', 'type', 'until', 'when',
+ 'while', 'xor'), prefix=r'\b', suffix=r'\b'),
+ Keyword.Reserved),
+ (r'"[^"]*"', String),
+ include('attribute'),
+ include('numbers'),
+ (r"'[^']'", String.Character),
+ (r'(\w+)(\s*|[(,])', bygroups(Name, using(this))),
+ (r"(<>|=>|:=|[()|:;,.'])", Punctuation),
+ (r'[*<>+=/&-]', Operator),
+ (r'\n+', Text),
+ ],
+ 'numbers': [
+ (r'[0-9_]+#[0-9a-f]+#', Number.Hex),
+ (r'[0-9_]+\.[0-9_]*', Number.Float),
+ (r'[0-9_]+', Number.Integer),
+ ],
+ 'attribute': [
+ (r"(')(\w+)", bygroups(Punctuation, Name.Attribute)),
+ ],
+ 'subprogram': [
+ (r'\(', Punctuation, ('#pop', 'formal_part')),
+ (r';', Punctuation, '#pop'),
+ (r'is\b', Keyword.Reserved, '#pop'),
+ (r'"[^"]+"|\w+', Name.Function),
+ include('root'),
+ ],
+ 'end': [
+ ('(if|case|record|loop|select)', Keyword.Reserved),
+ ('"[^"]+"|[\w.]+', Name.Function),
+ ('\s+', Text),
+ (';', Punctuation, '#pop'),
+ ],
+ 'type_def': [
+ (r';', Punctuation, '#pop'),
+ (r'\(', Punctuation, 'formal_part'),
+ (r'with|and|use', Keyword.Reserved),
+ (r'array\b', Keyword.Reserved, ('#pop', 'array_def')),
+ (r'record\b', Keyword.Reserved, ('record_def')),
+ (r'(null record)(;)', bygroups(Keyword.Reserved, Punctuation), '#pop'),
+ include('root'),
+ ],
+ 'array_def': [
+ (r';', Punctuation, '#pop'),
+ (r'(\w+)(\s+)(range)', bygroups(Keyword.Type, Text, Keyword.Reserved)),
+ include('root'),
+ ],
+ 'record_def': [
+ (r'end record', Keyword.Reserved, '#pop'),
+ include('root'),
+ ],
+ 'import': [
+ (r'[\w.]+', Name.Namespace, '#pop'),
+ default('#pop'),
+ ],
+ 'formal_part': [
+ (r'\)', Punctuation, '#pop'),
+ (r'\w+', Name.Variable),
+ (r',|:[^=]', Punctuation),
+ (r'(in|not|null|out|access)\b', Keyword.Reserved),
+ include('root'),
+ ],
+ 'package': [
+ ('body', Keyword.Declaration),
+ ('is\s+new|renames', Keyword.Reserved),
+ ('is', Keyword.Reserved, '#pop'),
+ (';', Punctuation, '#pop'),
+ ('\(', Punctuation, 'package_instantiation'),
+ ('([\w.]+)', Name.Class),
+ include('root'),
+ ],
+ 'package_instantiation': [
+ (r'("[^"]+"|\w+)(\s+)(=>)', bygroups(Name.Variable, Text, Punctuation)),
+ (r'[\w.\'"]', Text),
+ (r'\)', Punctuation, '#pop'),
+ include('root'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/pawn.py b/wandb/vendor/pygments/lexers/pawn.py
new file mode 100644
index 0000000000000000000000000000000000000000..f462a883c1c6d4425e3f5e95eb3a32977188c5f0
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/pawn.py
@@ -0,0 +1,199 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.pawn
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the Pawn languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Error
+from pygments.util import get_bool_opt
+
+__all__ = ['SourcePawnLexer', 'PawnLexer']
+
+
+class SourcePawnLexer(RegexLexer):
+ """
+ For SourcePawn source code with preprocessor directives.
+
+ .. versionadded:: 1.6
+ """
+ name = 'SourcePawn'
+ aliases = ['sp']
+ filenames = ['*.sp']
+ mimetypes = ['text/x-sourcepawn']
+
+ #: optional Comment or Whitespace
+ _ws = r'(?:\s|//.*?\n|/\*.*?\*/)+'
+ #: only one /* */ style comment
+ _ws1 = r'\s*(?:/[*].*?[*]/\s*)*'
+
+ tokens = {
+ 'root': [
+ # preprocessor directives: without whitespace
+ ('^#if\s+0', Comment.Preproc, 'if0'),
+ ('^#', Comment.Preproc, 'macro'),
+ # or with whitespace
+ ('^' + _ws1 + r'#if\s+0', Comment.Preproc, 'if0'),
+ ('^' + _ws1 + '#', Comment.Preproc, 'macro'),
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'\\\n', Text), # line continuation
+ (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single),
+ (r'/(\\\n)?\*(.|\n)*?\*(\\\n)?/', Comment.Multiline),
+ (r'[{}]', Punctuation),
+ (r'L?"', String, 'string'),
+ (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+ (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float),
+ (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+ (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex),
+ (r'0[0-7]+[LlUu]*', Number.Oct),
+ (r'\d+[LlUu]*', Number.Integer),
+ (r'\*/', Error),
+ (r'[~!%^&*+=|?:<>/-]', Operator),
+ (r'[()\[\],.;]', Punctuation),
+ (r'(case|const|continue|native|'
+ r'default|else|enum|for|if|new|operator|'
+ r'public|return|sizeof|static|decl|struct|switch)\b', Keyword),
+ (r'(bool|Float)\b', Keyword.Type),
+ (r'(true|false)\b', Keyword.Constant),
+ ('[a-zA-Z_]\w*', Name),
+ ],
+ 'string': [
+ (r'"', String, '#pop'),
+ (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+ (r'[^\\"\n]+', String), # all other characters
+ (r'\\\n', String), # line continuation
+ (r'\\', String), # stray backslash
+ ],
+ 'macro': [
+ (r'[^/\n]+', Comment.Preproc),
+ (r'/\*(.|\n)*?\*/', Comment.Multiline),
+ (r'//.*?\n', Comment.Single, '#pop'),
+ (r'/', Comment.Preproc),
+ (r'(?<=\\)\n', Comment.Preproc),
+ (r'\n', Comment.Preproc, '#pop'),
+ ],
+ 'if0': [
+ (r'^\s*#if.*?(?<!\\)\n', Comment.Preproc, '#push'),
+ (r'^\s*#endif.*?(?<!\\)\n', Comment.Preproc, '#pop'),
+ (r'.*?\n', Comment),
+ ]
+ }
+
+ SM_TYPES = set(('Action', 'bool', 'Float', 'Plugin', 'String', 'any',
+ 'AdminFlag', 'OverrideType', 'OverrideRule', 'ImmunityType',
+ 'GroupId', 'AdminId', 'AdmAccessMode', 'AdminCachePart',
+ 'CookieAccess', 'CookieMenu', 'CookieMenuAction', 'NetFlow',
+ 'ConVarBounds', 'QueryCookie', 'ReplySource',
+ 'ConVarQueryResult', 'ConVarQueryFinished', 'Function',
+ 'Action', 'Identity', 'PluginStatus', 'PluginInfo', 'DBResult',
+ 'DBBindType', 'DBPriority', 'PropType', 'PropFieldType',
+ 'MoveType', 'RenderMode', 'RenderFx', 'EventHookMode',
+ 'EventHook', 'FileType', 'FileTimeMode', 'PathType',
+ 'ParamType', 'ExecType', 'DialogType', 'Handle', 'KvDataTypes',
+ 'NominateResult', 'MapChange', 'MenuStyle', 'MenuAction',
+ 'MenuSource', 'RegexError', 'SDKCallType', 'SDKLibrary',
+ 'SDKFuncConfSource', 'SDKType', 'SDKPassMethod', 'RayType',
+ 'TraceEntityFilter', 'ListenOverride', 'SortOrder', 'SortType',
+ 'SortFunc2D', 'APLRes', 'FeatureType', 'FeatureStatus',
+ 'SMCResult', 'SMCError', 'TFClassType', 'TFTeam', 'TFCond',
+ 'TFResourceType', 'Timer', 'TopMenuAction', 'TopMenuObjectType',
+ 'TopMenuPosition', 'TopMenuObject', 'UserMsg'))
+
+ def __init__(self, **options):
+ self.smhighlighting = get_bool_opt(options,
+ 'sourcemod', True)
+
+ self._functions = set()
+ if self.smhighlighting:
+ from pygments.lexers._sourcemod_builtins import FUNCTIONS
+ self._functions.update(FUNCTIONS)
+ RegexLexer.__init__(self, **options)
+
+ def get_tokens_unprocessed(self, text):
+ for index, token, value in \
+ RegexLexer.get_tokens_unprocessed(self, text):
+ if token is Name:
+ if self.smhighlighting:
+ if value in self.SM_TYPES:
+ token = Keyword.Type
+ elif value in self._functions:
+ token = Name.Builtin
+ yield index, token, value
+
+
+class PawnLexer(RegexLexer):
+ """
+ For Pawn source code.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Pawn'
+ aliases = ['pawn']
+ filenames = ['*.p', '*.pwn', '*.inc']
+ mimetypes = ['text/x-pawn']
+
+ #: optional Comment or Whitespace
+ _ws = r'(?:\s|//.*?\n|/[*][\w\W]*?[*]/)+'
+ #: only one /* */ style comment
+ _ws1 = r'\s*(?:/[*].*?[*]/\s*)*'
+
+ tokens = {
+ 'root': [
+ # preprocessor directives: without whitespace
+ ('^#if\s+0', Comment.Preproc, 'if0'),
+ ('^#', Comment.Preproc, 'macro'),
+ # or with whitespace
+ ('^' + _ws1 + r'#if\s+0', Comment.Preproc, 'if0'),
+ ('^' + _ws1 + '#', Comment.Preproc, 'macro'),
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'\\\n', Text), # line continuation
+ (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single),
+ (r'/(\\\n)?\*[\w\W]*?\*(\\\n)?/', Comment.Multiline),
+ (r'[{}]', Punctuation),
+ (r'L?"', String, 'string'),
+ (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+ (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float),
+ (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+ (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex),
+ (r'0[0-7]+[LlUu]*', Number.Oct),
+ (r'\d+[LlUu]*', Number.Integer),
+ (r'\*/', Error),
+ (r'[~!%^&*+=|?:<>/-]', Operator),
+ (r'[()\[\],.;]', Punctuation),
+ (r'(switch|case|default|const|new|static|char|continue|break|'
+ r'if|else|for|while|do|operator|enum|'
+ r'public|return|sizeof|tagof|state|goto)\b', Keyword),
+ (r'(bool|Float)\b', Keyword.Type),
+ (r'(true|false)\b', Keyword.Constant),
+ ('[a-zA-Z_]\w*', Name),
+ ],
+ 'string': [
+ (r'"', String, '#pop'),
+ (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+ (r'[^\\"\n]+', String), # all other characters
+ (r'\\\n', String), # line continuation
+ (r'\\', String), # stray backslash
+ ],
+ 'macro': [
+ (r'[^/\n]+', Comment.Preproc),
+ (r'/\*(.|\n)*?\*/', Comment.Multiline),
+ (r'//.*?\n', Comment.Single, '#pop'),
+ (r'/', Comment.Preproc),
+ (r'(?<=\\)\n', Comment.Preproc),
+ (r'\n', Comment.Preproc, '#pop'),
+ ],
+ 'if0': [
+ (r'^\s*#if.*?(?<!\\)\n', Comment.Preproc, '#push'),
+ (r'^\s*#endif.*?(?<!\\)\n', Comment.Preproc, '#pop'),
+ (r'.*?\n', Comment),
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/perl.py b/wandb/vendor/pygments/lexers/perl.py
new file mode 100644
index 0000000000000000000000000000000000000000..4d5ab3b385ac0003dc0fc2d1524df51b62e0a599
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/perl.py
@@ -0,0 +1,620 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.perl
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Perl and related languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, ExtendedRegexLexer, include, bygroups, \
+ using, this, default, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+from pygments.util import shebang_matches
+
+__all__ = ['PerlLexer', 'Perl6Lexer']
+
+
+class PerlLexer(RegexLexer):
+ """
+ For `Perl <http://www.perl.org>`_ source code.
+ """
+
+ name = 'Perl'
+ aliases = ['perl', 'pl']
+ filenames = ['*.pl', '*.pm', '*.t']
+ mimetypes = ['text/x-perl', 'application/x-perl']
+
+ flags = re.DOTALL | re.MULTILINE
+ # TODO: give this to a perl guy who knows how to parse perl...
+ tokens = {
+ 'balanced-regex': [
+ (r'/(\\\\|\\[^\\]|[^\\/])*/[egimosx]*', String.Regex, '#pop'),
+ (r'!(\\\\|\\[^\\]|[^\\!])*![egimosx]*', String.Regex, '#pop'),
+ (r'\\(\\\\|[^\\])*\\[egimosx]*', String.Regex, '#pop'),
+ (r'\{(\\\\|\\[^\\]|[^\\}])*\}[egimosx]*', String.Regex, '#pop'),
+ (r'<(\\\\|\\[^\\]|[^\\>])*>[egimosx]*', String.Regex, '#pop'),
+ (r'\[(\\\\|\\[^\\]|[^\\\]])*\][egimosx]*', String.Regex, '#pop'),
+ (r'\((\\\\|\\[^\\]|[^\\)])*\)[egimosx]*', String.Regex, '#pop'),
+ (r'@(\\\\|\\[^\\]|[^\\@])*@[egimosx]*', String.Regex, '#pop'),
+ (r'%(\\\\|\\[^\\]|[^\\%])*%[egimosx]*', String.Regex, '#pop'),
+ (r'\$(\\\\|\\[^\\]|[^\\$])*\$[egimosx]*', String.Regex, '#pop'),
+ ],
+ 'root': [
+ (r'\A\#!.+?$', Comment.Hashbang),
+ (r'\#.*?$', Comment.Single),
+ (r'^=[a-zA-Z0-9]+\s+.*?\n=cut', Comment.Multiline),
+ (words((
+ 'case', 'continue', 'do', 'else', 'elsif', 'for', 'foreach',
+ 'if', 'last', 'my', 'next', 'our', 'redo', 'reset', 'then',
+ 'unless', 'until', 'while', 'print', 'new', 'BEGIN',
+ 'CHECK', 'INIT', 'END', 'return'), suffix=r'\b'),
+ Keyword),
+ (r'(format)(\s+)(\w+)(\s*)(=)(\s*\n)',
+ bygroups(Keyword, Text, Name, Text, Punctuation, Text), 'format'),
+ (r'(eq|lt|gt|le|ge|ne|not|and|or|cmp)\b', Operator.Word),
+ # common delimiters
+ (r's/(\\\\|\\[^\\]|[^\\/])*/(\\\\|\\[^\\]|[^\\/])*/[egimosx]*',
+ String.Regex),
+ (r's!(\\\\|\\!|[^!])*!(\\\\|\\!|[^!])*![egimosx]*', String.Regex),
+ (r's\\(\\\\|[^\\])*\\(\\\\|[^\\])*\\[egimosx]*', String.Regex),
+ (r's@(\\\\|\\[^\\]|[^\\@])*@(\\\\|\\[^\\]|[^\\@])*@[egimosx]*',
+ String.Regex),
+ (r's%(\\\\|\\[^\\]|[^\\%])*%(\\\\|\\[^\\]|[^\\%])*%[egimosx]*',
+ String.Regex),
+ # balanced delimiters
+ (r's\{(\\\\|\\[^\\]|[^\\}])*\}\s*', String.Regex, 'balanced-regex'),
+ (r's<(\\\\|\\[^\\]|[^\\>])*>\s*', String.Regex, 'balanced-regex'),
+ (r's\[(\\\\|\\[^\\]|[^\\\]])*\]\s*', String.Regex,
+ 'balanced-regex'),
+ (r's\((\\\\|\\[^\\]|[^\\)])*\)\s*', String.Regex,
+ 'balanced-regex'),
+
+ (r'm?/(\\\\|\\[^\\]|[^\\/\n])*/[gcimosx]*', String.Regex),
+ (r'm(?=[/!\\{<\[(@%$])', String.Regex, 'balanced-regex'),
+ (r'((?<==~)|(?<=\())\s*/(\\\\|\\[^\\]|[^\\/])*/[gcimosx]*',
+ String.Regex),
+ (r'\s+', Text),
+ (words((
+ 'abs', 'accept', 'alarm', 'atan2', 'bind', 'binmode', 'bless', 'caller', 'chdir',
+ 'chmod', 'chomp', 'chop', 'chown', 'chr', 'chroot', 'close', 'closedir', 'connect',
+ 'continue', 'cos', 'crypt', 'dbmclose', 'dbmopen', 'defined', 'delete', 'die',
+ 'dump', 'each', 'endgrent', 'endhostent', 'endnetent', 'endprotoent',
+ 'endpwent', 'endservent', 'eof', 'eval', 'exec', 'exists', 'exit', 'exp', 'fcntl',
+ 'fileno', 'flock', 'fork', 'format', 'formline', 'getc', 'getgrent', 'getgrgid',
+ 'getgrnam', 'gethostbyaddr', 'gethostbyname', 'gethostent', 'getlogin',
+ 'getnetbyaddr', 'getnetbyname', 'getnetent', 'getpeername', 'getpgrp',
+ 'getppid', 'getpriority', 'getprotobyname', 'getprotobynumber',
+ 'getprotoent', 'getpwent', 'getpwnam', 'getpwuid', 'getservbyname',
+ 'getservbyport', 'getservent', 'getsockname', 'getsockopt', 'glob', 'gmtime',
+ 'goto', 'grep', 'hex', 'import', 'index', 'int', 'ioctl', 'join', 'keys', 'kill', 'last',
+ 'lc', 'lcfirst', 'length', 'link', 'listen', 'local', 'localtime', 'log', 'lstat',
+ 'map', 'mkdir', 'msgctl', 'msgget', 'msgrcv', 'msgsnd', 'my', 'next', 'oct', 'open',
+ 'opendir', 'ord', 'our', 'pack', 'pipe', 'pop', 'pos', 'printf',
+ 'prototype', 'push', 'quotemeta', 'rand', 'read', 'readdir',
+ 'readline', 'readlink', 'readpipe', 'recv', 'redo', 'ref', 'rename',
+ 'reverse', 'rewinddir', 'rindex', 'rmdir', 'scalar', 'seek', 'seekdir',
+ 'select', 'semctl', 'semget', 'semop', 'send', 'setgrent', 'sethostent', 'setnetent',
+ 'setpgrp', 'setpriority', 'setprotoent', 'setpwent', 'setservent',
+ 'setsockopt', 'shift', 'shmctl', 'shmget', 'shmread', 'shmwrite', 'shutdown',
+ 'sin', 'sleep', 'socket', 'socketpair', 'sort', 'splice', 'split', 'sprintf', 'sqrt',
+ 'srand', 'stat', 'study', 'substr', 'symlink', 'syscall', 'sysopen', 'sysread',
+ 'sysseek', 'system', 'syswrite', 'tell', 'telldir', 'tie', 'tied', 'time', 'times', 'tr',
+ 'truncate', 'uc', 'ucfirst', 'umask', 'undef', 'unlink', 'unpack', 'unshift', 'untie',
+ 'utime', 'values', 'vec', 'wait', 'waitpid', 'wantarray', 'warn', 'write'), suffix=r'\b'),
+ Name.Builtin),
+ (r'((__(DATA|DIE|WARN)__)|(STD(IN|OUT|ERR)))\b', Name.Builtin.Pseudo),
+ (r'(<<)([\'"]?)([a-zA-Z_]\w*)(\2;?\n.*?\n)(\3)(\n)',
+ bygroups(String, String, String.Delimiter, String, String.Delimiter, Text)),
+ (r'__END__', Comment.Preproc, 'end-part'),
+ (r'\$\^[ADEFHILMOPSTWX]', Name.Variable.Global),
+ (r"\$[\\\"\[\]'&`+*.,;=%~?@$!<>(^|/-](?!\w)", Name.Variable.Global),
+ (r'[$@%#]+', Name.Variable, 'varname'),
+ (r'0_?[0-7]+(_[0-7]+)*', Number.Oct),
+ (r'0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*', Number.Hex),
+ (r'0b[01]+(_[01]+)*', Number.Bin),
+ (r'(?i)(\d*(_\d*)*\.\d+(_\d*)*|\d+(_\d*)*\.\d+(_\d*)*)(e[+-]?\d+)?',
+ Number.Float),
+ (r'(?i)\d+(_\d*)*e[+-]?\d+(_\d*)*', Number.Float),
+ (r'\d+(_\d+)*', Number.Integer),
+ (r"'(\\\\|\\[^\\]|[^'\\])*'", String),
+ (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
+ (r'`(\\\\|\\[^\\]|[^`\\])*`', String.Backtick),
+ (r'<([^\s>]+)>', String.Regex),
+ (r'(q|qq|qw|qr|qx)\{', String.Other, 'cb-string'),
+ (r'(q|qq|qw|qr|qx)\(', String.Other, 'rb-string'),
+ (r'(q|qq|qw|qr|qx)\[', String.Other, 'sb-string'),
+ (r'(q|qq|qw|qr|qx)\<', String.Other, 'lt-string'),
+ (r'(q|qq|qw|qr|qx)([\W_])(.|\n)*?\2', String.Other),
+ (r'(package)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)',
+ bygroups(Keyword, Text, Name.Namespace)),
+ (r'(use|require|no)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)',
+ bygroups(Keyword, Text, Name.Namespace)),
+ (r'(sub)(\s+)', bygroups(Keyword, Text), 'funcname'),
+ (words((
+ 'no', 'package', 'require', 'use'), suffix=r'\b'),
+ Keyword),
+ (r'(\[\]|\*\*|::|<<|>>|>=|<=>|<=|={3}|!=|=~|'
+ r'!~|&&?|\|\||\.{1,3})', Operator),
+ (r'[-+/*%=<>&^|!\\~]=?', Operator),
+ (r'[()\[\]:;,<>/?{}]', Punctuation), # yes, there's no shortage
+ # of punctuation in Perl!
+ (r'(?=\w)', Name, 'name'),
+ ],
+ 'format': [
+ (r'\.\n', String.Interpol, '#pop'),
+ (r'[^\n]*\n', String.Interpol),
+ ],
+ 'varname': [
+ (r'\s+', Text),
+ (r'\{', Punctuation, '#pop'), # hash syntax?
+ (r'\)|,', Punctuation, '#pop'), # argument specifier
+ (r'\w+::', Name.Namespace),
+ (r'[\w:]+', Name.Variable, '#pop'),
+ ],
+ 'name': [
+ (r'[a-zA-Z_]\w*(::[a-zA-Z_]\w*)*(::)?(?=\s*->)', Name.Namespace, '#pop'),
+ (r'[a-zA-Z_]\w*(::[a-zA-Z_]\w*)*::', Name.Namespace, '#pop'),
+ (r'[\w:]+', Name, '#pop'),
+ (r'[A-Z_]+(?=\W)', Name.Constant, '#pop'),
+ (r'(?=\W)', Text, '#pop'),
+ ],
+ 'funcname': [
+ (r'[a-zA-Z_]\w*[!?]?', Name.Function),
+ (r'\s+', Text),
+ # argument declaration
+ (r'(\([$@%]*\))(\s*)', bygroups(Punctuation, Text)),
+ (r';', Punctuation, '#pop'),
+ (r'.*?\{', Punctuation, '#pop'),
+ ],
+ 'cb-string': [
+ (r'\\[{}\\]', String.Other),
+ (r'\\', String.Other),
+ (r'\{', String.Other, 'cb-string'),
+ (r'\}', String.Other, '#pop'),
+ (r'[^{}\\]+', String.Other)
+ ],
+ 'rb-string': [
+ (r'\\[()\\]', String.Other),
+ (r'\\', String.Other),
+ (r'\(', String.Other, 'rb-string'),
+ (r'\)', String.Other, '#pop'),
+ (r'[^()]+', String.Other)
+ ],
+ 'sb-string': [
+ (r'\\[\[\]\\]', String.Other),
+ (r'\\', String.Other),
+ (r'\[', String.Other, 'sb-string'),
+ (r'\]', String.Other, '#pop'),
+ (r'[^\[\]]+', String.Other)
+ ],
+ 'lt-string': [
+ (r'\\[<>\\]', String.Other),
+ (r'\\', String.Other),
+ (r'\<', String.Other, 'lt-string'),
+ (r'\>', String.Other, '#pop'),
+ (r'[^<>]+', String.Other)
+ ],
+ 'end-part': [
+ (r'.+', Comment.Preproc, '#pop')
+ ]
+ }
+
+ def analyse_text(text):
+ if shebang_matches(text, r'perl'):
+ return True
+ if re.search('(?:my|our)\s+[$@%(]', text):
+ return 0.9
+
+
+class Perl6Lexer(ExtendedRegexLexer):
+ """
+ For `Perl 6 <http://www.perl6.org>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Perl6'
+ aliases = ['perl6', 'pl6']
+ filenames = ['*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6',
+ '*.6pm', '*.p6m', '*.pm6', '*.t']
+ mimetypes = ['text/x-perl6', 'application/x-perl6']
+ flags = re.MULTILINE | re.DOTALL | re.UNICODE
+
+ PERL6_IDENTIFIER_RANGE = "['\w:-]"
+
+ PERL6_KEYWORDS = (
+ 'BEGIN', 'CATCH', 'CHECK', 'CONTROL', 'END', 'ENTER', 'FIRST', 'INIT',
+ 'KEEP', 'LAST', 'LEAVE', 'NEXT', 'POST', 'PRE', 'START', 'TEMP',
+ 'UNDO', 'as', 'assoc', 'async', 'augment', 'binary', 'break', 'but',
+ 'cached', 'category', 'class', 'constant', 'contend', 'continue',
+ 'copy', 'deep', 'default', 'defequiv', 'defer', 'die', 'do', 'else',
+ 'elsif', 'enum', 'equiv', 'exit', 'export', 'fail', 'fatal', 'for',
+ 'gather', 'given', 'goto', 'grammar', 'handles', 'has', 'if', 'inline',
+ 'irs', 'is', 'last', 'leave', 'let', 'lift', 'loop', 'looser', 'macro',
+ 'make', 'maybe', 'method', 'module', 'multi', 'my', 'next', 'of',
+ 'ofs', 'only', 'oo', 'ors', 'our', 'package', 'parsed', 'prec',
+ 'proto', 'readonly', 'redo', 'ref', 'regex', 'reparsed', 'repeat',
+ 'require', 'required', 'return', 'returns', 'role', 'rule', 'rw',
+ 'self', 'slang', 'state', 'sub', 'submethod', 'subset', 'supersede',
+ 'take', 'temp', 'tighter', 'token', 'trusts', 'try', 'unary',
+ 'unless', 'until', 'use', 'warn', 'when', 'where', 'while', 'will',
+ )
+
+ PERL6_BUILTINS = (
+ 'ACCEPTS', 'HOW', 'REJECTS', 'VAR', 'WHAT', 'WHENCE', 'WHERE', 'WHICH',
+ 'WHO', 'abs', 'acos', 'acosec', 'acosech', 'acosh', 'acotan', 'acotanh',
+ 'all', 'any', 'approx', 'arity', 'asec', 'asech', 'asin', 'asinh',
+ 'assuming', 'atan', 'atan2', 'atanh', 'attr', 'bless', 'body', 'by',
+ 'bytes', 'caller', 'callsame', 'callwith', 'can', 'capitalize', 'cat',
+ 'ceiling', 'chars', 'chmod', 'chomp', 'chop', 'chr', 'chroot',
+ 'circumfix', 'cis', 'classify', 'clone', 'close', 'cmp_ok', 'codes',
+ 'comb', 'connect', 'contains', 'context', 'cos', 'cosec', 'cosech',
+ 'cosh', 'cotan', 'cotanh', 'count', 'defined', 'delete', 'diag',
+ 'dies_ok', 'does', 'e', 'each', 'eager', 'elems', 'end', 'eof', 'eval',
+ 'eval_dies_ok', 'eval_elsewhere', 'eval_lives_ok', 'evalfile', 'exists',
+ 'exp', 'first', 'flip', 'floor', 'flunk', 'flush', 'fmt', 'force_todo',
+ 'fork', 'from', 'getc', 'gethost', 'getlogin', 'getpeername', 'getpw',
+ 'gmtime', 'graphs', 'grep', 'hints', 'hyper', 'im', 'index', 'infix',
+ 'invert', 'is_approx', 'is_deeply', 'isa', 'isa_ok', 'isnt', 'iterator',
+ 'join', 'key', 'keys', 'kill', 'kv', 'lastcall', 'lazy', 'lc', 'lcfirst',
+ 'like', 'lines', 'link', 'lives_ok', 'localtime', 'log', 'log10', 'map',
+ 'max', 'min', 'minmax', 'name', 'new', 'nextsame', 'nextwith', 'nfc',
+ 'nfd', 'nfkc', 'nfkd', 'nok_error', 'nonce', 'none', 'normalize', 'not',
+ 'nothing', 'ok', 'once', 'one', 'open', 'opendir', 'operator', 'ord',
+ 'p5chomp', 'p5chop', 'pack', 'pair', 'pairs', 'pass', 'perl', 'pi',
+ 'pick', 'plan', 'plan_ok', 'polar', 'pop', 'pos', 'postcircumfix',
+ 'postfix', 'pred', 'prefix', 'print', 'printf', 'push', 'quasi',
+ 'quotemeta', 'rand', 're', 'read', 'readdir', 'readline', 'reduce',
+ 'reverse', 'rewind', 'rewinddir', 'rindex', 'roots', 'round',
+ 'roundrobin', 'run', 'runinstead', 'sameaccent', 'samecase', 'say',
+ 'sec', 'sech', 'sech', 'seek', 'shape', 'shift', 'sign', 'signature',
+ 'sin', 'sinh', 'skip', 'skip_rest', 'sleep', 'slurp', 'sort', 'splice',
+ 'split', 'sprintf', 'sqrt', 'srand', 'strand', 'subst', 'substr', 'succ',
+ 'sum', 'symlink', 'tan', 'tanh', 'throws_ok', 'time', 'times', 'to',
+ 'todo', 'trim', 'trim_end', 'trim_start', 'true', 'truncate', 'uc',
+ 'ucfirst', 'undef', 'undefine', 'uniq', 'unlike', 'unlink', 'unpack',
+ 'unpolar', 'unshift', 'unwrap', 'use_ok', 'value', 'values', 'vec',
+ 'version_lt', 'void', 'wait', 'want', 'wrap', 'write', 'zip',
+ )
+
+ PERL6_BUILTIN_CLASSES = (
+ 'Abstraction', 'Any', 'AnyChar', 'Array', 'Associative', 'Bag', 'Bit',
+ 'Blob', 'Block', 'Bool', 'Buf', 'Byte', 'Callable', 'Capture', 'Char', 'Class',
+ 'Code', 'Codepoint', 'Comparator', 'Complex', 'Decreasing', 'Exception',
+ 'Failure', 'False', 'Grammar', 'Grapheme', 'Hash', 'IO', 'Increasing',
+ 'Int', 'Junction', 'KeyBag', 'KeyExtractor', 'KeyHash', 'KeySet',
+ 'KitchenSink', 'List', 'Macro', 'Mapping', 'Match', 'Matcher', 'Method',
+ 'Module', 'Num', 'Object', 'Ordered', 'Ordering', 'OrderingPair',
+ 'Package', 'Pair', 'Positional', 'Proxy', 'Range', 'Rat', 'Regex',
+ 'Role', 'Routine', 'Scalar', 'Seq', 'Set', 'Signature', 'Str', 'StrLen',
+ 'StrPos', 'Sub', 'Submethod', 'True', 'UInt', 'Undef', 'Version', 'Void',
+ 'Whatever', 'bit', 'bool', 'buf', 'buf1', 'buf16', 'buf2', 'buf32',
+ 'buf4', 'buf64', 'buf8', 'complex', 'int', 'int1', 'int16', 'int2',
+ 'int32', 'int4', 'int64', 'int8', 'num', 'rat', 'rat1', 'rat16', 'rat2',
+ 'rat32', 'rat4', 'rat64', 'rat8', 'uint', 'uint1', 'uint16', 'uint2',
+ 'uint32', 'uint4', 'uint64', 'uint8', 'utf16', 'utf32', 'utf8',
+ )
+
+ PERL6_OPERATORS = (
+ 'X', 'Z', 'after', 'also', 'and', 'andthen', 'before', 'cmp', 'div',
+ 'eq', 'eqv', 'extra', 'ff', 'fff', 'ge', 'gt', 'le', 'leg', 'lt', 'm',
+ 'mm', 'mod', 'ne', 'or', 'orelse', 'rx', 's', 'tr', 'x', 'xor', 'xx',
+ '++', '--', '**', '!', '+', '-', '~', '?', '|', '||', '+^', '~^', '?^',
+ '^', '*', '/', '%', '%%', '+&', '+<', '+>', '~&', '~<', '~>', '?&',
+ 'gcd', 'lcm', '+', '-', '+|', '+^', '~|', '~^', '?|', '?^',
+ '~', '&', '^', 'but', 'does', '<=>', '..', '..^', '^..', '^..^',
+ '!=', '==', '<', '<=', '>', '>=', '~~', '===', '!eqv',
+ '&&', '||', '^^', '//', 'min', 'max', '??', '!!', 'ff', 'fff', 'so',
+ 'not', '<==', '==>', '<<==', '==>>',
+ )
+
+ # Perl 6 has a *lot* of possible bracketing characters
+ # this list was lifted from STD.pm6 (https://github.com/perl6/std)
+ PERL6_BRACKETS = {
+ u'\u0028': u'\u0029', u'\u003c': u'\u003e', u'\u005b': u'\u005d',
+ u'\u007b': u'\u007d', u'\u00ab': u'\u00bb', u'\u0f3a': u'\u0f3b',
+ u'\u0f3c': u'\u0f3d', u'\u169b': u'\u169c', u'\u2018': u'\u2019',
+ u'\u201a': u'\u2019', u'\u201b': u'\u2019', u'\u201c': u'\u201d',
+ u'\u201e': u'\u201d', u'\u201f': u'\u201d', u'\u2039': u'\u203a',
+ u'\u2045': u'\u2046', u'\u207d': u'\u207e', u'\u208d': u'\u208e',
+ u'\u2208': u'\u220b', u'\u2209': u'\u220c', u'\u220a': u'\u220d',
+ u'\u2215': u'\u29f5', u'\u223c': u'\u223d', u'\u2243': u'\u22cd',
+ u'\u2252': u'\u2253', u'\u2254': u'\u2255', u'\u2264': u'\u2265',
+ u'\u2266': u'\u2267', u'\u2268': u'\u2269', u'\u226a': u'\u226b',
+ u'\u226e': u'\u226f', u'\u2270': u'\u2271', u'\u2272': u'\u2273',
+ u'\u2274': u'\u2275', u'\u2276': u'\u2277', u'\u2278': u'\u2279',
+ u'\u227a': u'\u227b', u'\u227c': u'\u227d', u'\u227e': u'\u227f',
+ u'\u2280': u'\u2281', u'\u2282': u'\u2283', u'\u2284': u'\u2285',
+ u'\u2286': u'\u2287', u'\u2288': u'\u2289', u'\u228a': u'\u228b',
+ u'\u228f': u'\u2290', u'\u2291': u'\u2292', u'\u2298': u'\u29b8',
+ u'\u22a2': u'\u22a3', u'\u22a6': u'\u2ade', u'\u22a8': u'\u2ae4',
+ u'\u22a9': u'\u2ae3', u'\u22ab': u'\u2ae5', u'\u22b0': u'\u22b1',
+ u'\u22b2': u'\u22b3', u'\u22b4': u'\u22b5', u'\u22b6': u'\u22b7',
+ u'\u22c9': u'\u22ca', u'\u22cb': u'\u22cc', u'\u22d0': u'\u22d1',
+ u'\u22d6': u'\u22d7', u'\u22d8': u'\u22d9', u'\u22da': u'\u22db',
+ u'\u22dc': u'\u22dd', u'\u22de': u'\u22df', u'\u22e0': u'\u22e1',
+ u'\u22e2': u'\u22e3', u'\u22e4': u'\u22e5', u'\u22e6': u'\u22e7',
+ u'\u22e8': u'\u22e9', u'\u22ea': u'\u22eb', u'\u22ec': u'\u22ed',
+ u'\u22f0': u'\u22f1', u'\u22f2': u'\u22fa', u'\u22f3': u'\u22fb',
+ u'\u22f4': u'\u22fc', u'\u22f6': u'\u22fd', u'\u22f7': u'\u22fe',
+ u'\u2308': u'\u2309', u'\u230a': u'\u230b', u'\u2329': u'\u232a',
+ u'\u23b4': u'\u23b5', u'\u2768': u'\u2769', u'\u276a': u'\u276b',
+ u'\u276c': u'\u276d', u'\u276e': u'\u276f', u'\u2770': u'\u2771',
+ u'\u2772': u'\u2773', u'\u2774': u'\u2775', u'\u27c3': u'\u27c4',
+ u'\u27c5': u'\u27c6', u'\u27d5': u'\u27d6', u'\u27dd': u'\u27de',
+ u'\u27e2': u'\u27e3', u'\u27e4': u'\u27e5', u'\u27e6': u'\u27e7',
+ u'\u27e8': u'\u27e9', u'\u27ea': u'\u27eb', u'\u2983': u'\u2984',
+ u'\u2985': u'\u2986', u'\u2987': u'\u2988', u'\u2989': u'\u298a',
+ u'\u298b': u'\u298c', u'\u298d': u'\u298e', u'\u298f': u'\u2990',
+ u'\u2991': u'\u2992', u'\u2993': u'\u2994', u'\u2995': u'\u2996',
+ u'\u2997': u'\u2998', u'\u29c0': u'\u29c1', u'\u29c4': u'\u29c5',
+ u'\u29cf': u'\u29d0', u'\u29d1': u'\u29d2', u'\u29d4': u'\u29d5',
+ u'\u29d8': u'\u29d9', u'\u29da': u'\u29db', u'\u29f8': u'\u29f9',
+ u'\u29fc': u'\u29fd', u'\u2a2b': u'\u2a2c', u'\u2a2d': u'\u2a2e',
+ u'\u2a34': u'\u2a35', u'\u2a3c': u'\u2a3d', u'\u2a64': u'\u2a65',
+ u'\u2a79': u'\u2a7a', u'\u2a7d': u'\u2a7e', u'\u2a7f': u'\u2a80',
+ u'\u2a81': u'\u2a82', u'\u2a83': u'\u2a84', u'\u2a8b': u'\u2a8c',
+ u'\u2a91': u'\u2a92', u'\u2a93': u'\u2a94', u'\u2a95': u'\u2a96',
+ u'\u2a97': u'\u2a98', u'\u2a99': u'\u2a9a', u'\u2a9b': u'\u2a9c',
+ u'\u2aa1': u'\u2aa2', u'\u2aa6': u'\u2aa7', u'\u2aa8': u'\u2aa9',
+ u'\u2aaa': u'\u2aab', u'\u2aac': u'\u2aad', u'\u2aaf': u'\u2ab0',
+ u'\u2ab3': u'\u2ab4', u'\u2abb': u'\u2abc', u'\u2abd': u'\u2abe',
+ u'\u2abf': u'\u2ac0', u'\u2ac1': u'\u2ac2', u'\u2ac3': u'\u2ac4',
+ u'\u2ac5': u'\u2ac6', u'\u2acd': u'\u2ace', u'\u2acf': u'\u2ad0',
+ u'\u2ad1': u'\u2ad2', u'\u2ad3': u'\u2ad4', u'\u2ad5': u'\u2ad6',
+ u'\u2aec': u'\u2aed', u'\u2af7': u'\u2af8', u'\u2af9': u'\u2afa',
+ u'\u2e02': u'\u2e03', u'\u2e04': u'\u2e05', u'\u2e09': u'\u2e0a',
+ u'\u2e0c': u'\u2e0d', u'\u2e1c': u'\u2e1d', u'\u2e20': u'\u2e21',
+ u'\u3008': u'\u3009', u'\u300a': u'\u300b', u'\u300c': u'\u300d',
+ u'\u300e': u'\u300f', u'\u3010': u'\u3011', u'\u3014': u'\u3015',
+ u'\u3016': u'\u3017', u'\u3018': u'\u3019', u'\u301a': u'\u301b',
+ u'\u301d': u'\u301e', u'\ufd3e': u'\ufd3f', u'\ufe17': u'\ufe18',
+ u'\ufe35': u'\ufe36', u'\ufe37': u'\ufe38', u'\ufe39': u'\ufe3a',
+ u'\ufe3b': u'\ufe3c', u'\ufe3d': u'\ufe3e', u'\ufe3f': u'\ufe40',
+ u'\ufe41': u'\ufe42', u'\ufe43': u'\ufe44', u'\ufe47': u'\ufe48',
+ u'\ufe59': u'\ufe5a', u'\ufe5b': u'\ufe5c', u'\ufe5d': u'\ufe5e',
+ u'\uff08': u'\uff09', u'\uff1c': u'\uff1e', u'\uff3b': u'\uff3d',
+ u'\uff5b': u'\uff5d', u'\uff5f': u'\uff60', u'\uff62': u'\uff63',
+ }
+
+ def _build_word_match(words, boundary_regex_fragment=None, prefix='', suffix=''):
+ if boundary_regex_fragment is None:
+ return r'\b(' + prefix + r'|'.join(re.escape(x) for x in words) + \
+ suffix + r')\b'
+ else:
+ return r'(?<!' + boundary_regex_fragment + r')' + prefix + r'(' + \
+ r'|'.join(re.escape(x) for x in words) + r')' + suffix + r'(?!' + \
+ boundary_regex_fragment + r')'
+
+ def brackets_callback(token_class):
+ def callback(lexer, match, context):
+ groups = match.groupdict()
+ opening_chars = groups['delimiter']
+ n_chars = len(opening_chars)
+ adverbs = groups.get('adverbs')
+
+ closer = Perl6Lexer.PERL6_BRACKETS.get(opening_chars[0])
+ text = context.text
+
+ if closer is None: # it's not a mirrored character, which means we
+ # just need to look for the next occurrence
+
+ end_pos = text.find(opening_chars, match.start('delimiter') + n_chars)
+ else: # we need to look for the corresponding closing character,
+ # keep nesting in mind
+ closing_chars = closer * n_chars
+ nesting_level = 1
+
+ search_pos = match.start('delimiter')
+
+ while nesting_level > 0:
+ next_open_pos = text.find(opening_chars, search_pos + n_chars)
+ next_close_pos = text.find(closing_chars, search_pos + n_chars)
+
+ if next_close_pos == -1:
+ next_close_pos = len(text)
+ nesting_level = 0
+ elif next_open_pos != -1 and next_open_pos < next_close_pos:
+ nesting_level += 1
+ search_pos = next_open_pos
+ else: # next_close_pos < next_open_pos
+ nesting_level -= 1
+ search_pos = next_close_pos
+
+ end_pos = next_close_pos
+
+ if end_pos < 0: # if we didn't find a closer, just highlight the
+ # rest of the text in this class
+ end_pos = len(text)
+
+ if adverbs is not None and re.search(r':to\b', adverbs):
+ heredoc_terminator = text[match.start('delimiter') + n_chars:end_pos]
+ end_heredoc = re.search(r'^\s*' + re.escape(heredoc_terminator) +
+ r'\s*$', text[end_pos:], re.MULTILINE)
+
+ if end_heredoc:
+ end_pos += end_heredoc.end()
+ else:
+ end_pos = len(text)
+
+ yield match.start(), token_class, text[match.start():end_pos + n_chars]
+ context.pos = end_pos + n_chars
+
+ return callback
+
+ def opening_brace_callback(lexer, match, context):
+ stack = context.stack
+
+ yield match.start(), Text, context.text[match.start():match.end()]
+ context.pos = match.end()
+
+ # if we encounter an opening brace and we're one level
+ # below a token state, it means we need to increment
+ # the nesting level for braces so we know later when
+ # we should return to the token rules.
+ if len(stack) > 2 and stack[-2] == 'token':
+ context.perl6_token_nesting_level += 1
+
+ def closing_brace_callback(lexer, match, context):
+ stack = context.stack
+
+ yield match.start(), Text, context.text[match.start():match.end()]
+ context.pos = match.end()
+
+ # if we encounter a free closing brace and we're one level
+ # below a token state, it means we need to check the nesting
+ # level to see if we need to return to the token state.
+ if len(stack) > 2 and stack[-2] == 'token':
+ context.perl6_token_nesting_level -= 1
+ if context.perl6_token_nesting_level == 0:
+ stack.pop()
+
+ def embedded_perl6_callback(lexer, match, context):
+ context.perl6_token_nesting_level = 1
+ yield match.start(), Text, context.text[match.start():match.end()]
+ context.pos = match.end()
+ context.stack.append('root')
+
+ # If you're modifying these rules, be careful if you need to process '{' or '}'
+ # characters. We have special logic for processing these characters (due to the fact
+ # that you can nest Perl 6 code in regex blocks), so if you need to process one of
+ # them, make sure you also process the corresponding one!
+ tokens = {
+ 'common': [
+ (r'#[`|=](?P<delimiter>(?P<first_char>[' + ''.join(PERL6_BRACKETS) + r'])(?P=first_char)*)',
+ brackets_callback(Comment.Multiline)),
+ (r'#[^\n]*$', Comment.Singleline),
+ (r'^(\s*)=begin\s+(\w+)\b.*?^\1=end\s+\2', Comment.Multiline),
+ (r'^(\s*)=for.*?\n\s*?\n', Comment.Multiline),
+ (r'^=.*?\n\s*?\n', Comment.Multiline),
+ (r'(regex|token|rule)(\s*' + PERL6_IDENTIFIER_RANGE + '+:sym)',
+ bygroups(Keyword, Name), 'token-sym-brackets'),
+ (r'(regex|token|rule)(?!' + PERL6_IDENTIFIER_RANGE + ')(\s*' + PERL6_IDENTIFIER_RANGE + '+)?',
+ bygroups(Keyword, Name), 'pre-token'),
+ # deal with a special case in the Perl 6 grammar (role q { ... })
+ (r'(role)(\s+)(q)(\s*)', bygroups(Keyword, Text, Name, Text)),
+ (_build_word_match(PERL6_KEYWORDS, PERL6_IDENTIFIER_RANGE), Keyword),
+ (_build_word_match(PERL6_BUILTIN_CLASSES, PERL6_IDENTIFIER_RANGE, suffix='(?::[UD])?'),
+ Name.Builtin),
+ (_build_word_match(PERL6_BUILTINS, PERL6_IDENTIFIER_RANGE), Name.Builtin),
+ # copied from PerlLexer
+ (r'[$@%&][.^:?=!~]?' + PERL6_IDENTIFIER_RANGE + u'+(?:<<.*?>>|<.*?>|«.*?»)*',
+ Name.Variable),
+ (r'\$[!/](?:<<.*?>>|<.*?>|«.*?»)*', Name.Variable.Global),
+ (r'::\?\w+', Name.Variable.Global),
+ (r'[$@%&]\*' + PERL6_IDENTIFIER_RANGE + u'+(?:<<.*?>>|<.*?>|«.*?»)*',
+ Name.Variable.Global),
+ (r'\$(?:<.*?>)+', Name.Variable),
+ (r'(?:q|qq|Q)[a-zA-Z]?\s*(?P<adverbs>:[\w\s:]+)?\s*(?P<delimiter>(?P<first_char>[^0-9a-zA-Z:\s])'
+ r'(?P=first_char)*)', brackets_callback(String)),
+ # copied from PerlLexer
+ (r'0_?[0-7]+(_[0-7]+)*', Number.Oct),
+ (r'0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*', Number.Hex),
+ (r'0b[01]+(_[01]+)*', Number.Bin),
+ (r'(?i)(\d*(_\d*)*\.\d+(_\d*)*|\d+(_\d*)*\.\d+(_\d*)*)(e[+-]?\d+)?',
+ Number.Float),
+ (r'(?i)\d+(_\d*)*e[+-]?\d+(_\d*)*', Number.Float),
+ (r'\d+(_\d+)*', Number.Integer),
+ (r'(?<=~~)\s*/(?:\\\\|\\/|.)*?/', String.Regex),
+ (r'(?<=[=(,])\s*/(?:\\\\|\\/|.)*?/', String.Regex),
+ (r'm\w+(?=\()', Name),
+ (r'(?:m|ms|rx)\s*(?P<adverbs>:[\w\s:]+)?\s*(?P<delimiter>(?P<first_char>[^\w:\s])'
+ r'(?P=first_char)*)', brackets_callback(String.Regex)),
+ (r'(?:s|ss|tr)\s*(?::[\w\s:]+)?\s*/(?:\\\\|\\/|.)*?/(?:\\\\|\\/|.)*?/',
+ String.Regex),
+ (r'<[^\s=].*?\S>', String),
+ (_build_word_match(PERL6_OPERATORS), Operator),
+ (r'\w' + PERL6_IDENTIFIER_RANGE + '*', Name),
+ (r"'(\\\\|\\[^\\]|[^'\\])*'", String),
+ (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
+ ],
+ 'root': [
+ include('common'),
+ (r'\{', opening_brace_callback),
+ (r'\}', closing_brace_callback),
+ (r'.+?', Text),
+ ],
+ 'pre-token': [
+ include('common'),
+ (r'\{', Text, ('#pop', 'token')),
+ (r'.+?', Text),
+ ],
+ 'token-sym-brackets': [
+ (r'(?P<delimiter>(?P<first_char>[' + ''.join(PERL6_BRACKETS) + '])(?P=first_char)*)',
+ brackets_callback(Name), ('#pop', 'pre-token')),
+ default(('#pop', 'pre-token')),
+ ],
+ 'token': [
+ (r'\}', Text, '#pop'),
+ (r'(?<=:)(?:my|our|state|constant|temp|let).*?;', using(this)),
+ # make sure that quotes in character classes aren't treated as strings
+ (r'<(?:[-!?+.]\s*)?\[.*?\]>', String.Regex),
+ # make sure that '#' characters in quotes aren't treated as comments
+ (r"(?<!\\)'(\\\\|\\[^\\]|[^'\\])*'", String.Regex),
+ (r'(?<!\\)"(\\\\|\\[^\\]|[^"\\])*"', String.Regex),
+ (r'#.*?$', Comment.Singleline),
+ (r'\{', embedded_perl6_callback),
+ ('.+?', String.Regex),
+ ],
+ }
+
+ def analyse_text(text):
+ def strip_pod(lines):
+ in_pod = False
+ stripped_lines = []
+
+ for line in lines:
+ if re.match(r'^=(?:end|cut)', line):
+ in_pod = False
+ elif re.match(r'^=\w+', line):
+ in_pod = True
+ elif not in_pod:
+ stripped_lines.append(line)
+
+ return stripped_lines
+
+ # XXX handle block comments
+ lines = text.splitlines()
+ lines = strip_pod(lines)
+ text = '\n'.join(lines)
+
+ if shebang_matches(text, r'perl6|rakudo|niecza|pugs'):
+ return True
+
+ saw_perl_decl = False
+ rating = False
+
+ # check for my/our/has declarations
+ if re.search("(?:my|our|has)\s+(?:" + Perl6Lexer.PERL6_IDENTIFIER_RANGE +
+ "+\s+)?[$@%&(]", text):
+ rating = 0.8
+ saw_perl_decl = True
+
+ for line in lines:
+ line = re.sub('#.*', '', line)
+ if re.match('^\s*$', line):
+ continue
+
+ # match v6; use v6; use v6.0; use v6.0.0;
+ if re.match('^\s*(?:use\s+)?v6(?:\.\d(?:\.\d)?)?;', line):
+ return True
+ # match class, module, role, enum, grammar declarations
+ class_decl = re.match('^\s*(?:(?P<scope>my|our)\s+)?(?:module|class|role|enum|grammar)', line)
+ if class_decl:
+ if saw_perl_decl or class_decl.group('scope') is not None:
+ return True
+ rating = 0.05
+ continue
+ break
+
+ return rating
+
+ def __init__(self, **options):
+ super(Perl6Lexer, self).__init__(**options)
+ self.encoding = options.get('encoding', 'utf-8')
diff --git a/wandb/vendor/pygments/lexers/php.py b/wandb/vendor/pygments/lexers/php.py
new file mode 100644
index 0000000000000000000000000000000000000000..f618b5fde638fffa70a6d00a8e348c5ccdda5ab7
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/php.py
@@ -0,0 +1,267 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.php
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexers for PHP and related languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, default, using, \
+ this, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Other
+from pygments.util import get_bool_opt, get_list_opt, iteritems
+
+__all__ = ['ZephirLexer', 'PhpLexer']
+
+
+class ZephirLexer(RegexLexer):
+ """
+ For `Zephir language <http://zephir-lang.com/>`_ source code.
+
+ Zephir is a compiled high level language aimed
+ to the creation of C-extensions for PHP.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Zephir'
+ aliases = ['zephir']
+ filenames = ['*.zep']
+
+ zephir_keywords = ['fetch', 'echo', 'isset', 'empty']
+ zephir_type = ['bit', 'bits', 'string']
+
+ flags = re.DOTALL | re.MULTILINE
+
+ tokens = {
+ 'commentsandwhitespace': [
+ (r'\s+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline)
+ ],
+ 'slashstartsregex': [
+ include('commentsandwhitespace'),
+ (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+ r'([gim]+\b|\B)', String.Regex, '#pop'),
+ default('#pop')
+ ],
+ 'badregex': [
+ (r'\n', Text, '#pop')
+ ],
+ 'root': [
+ (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'),
+ include('commentsandwhitespace'),
+ (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|'
+ r'(<<|>>>?|==?|!=?|->|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'),
+ (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
+ (r'[})\].]', Punctuation),
+ (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|loop|'
+ r'require|inline|throw|try|catch|finally|new|delete|typeof|instanceof|void|'
+ r'namespace|use|extends|this|fetch|isset|unset|echo|fetch|likely|unlikely|'
+ r'empty)\b', Keyword, 'slashstartsregex'),
+ (r'(var|let|with|function)\b', Keyword.Declaration, 'slashstartsregex'),
+ (r'(abstract|boolean|bool|char|class|const|double|enum|export|extends|final|'
+ r'native|goto|implements|import|int|string|interface|long|ulong|char|uchar|'
+ r'float|unsigned|private|protected|public|short|static|self|throws|reverse|'
+ r'transient|volatile)\b', Keyword.Reserved),
+ (r'(true|false|null|undefined)\b', Keyword.Constant),
+ (r'(Array|Boolean|Date|_REQUEST|_COOKIE|_SESSION|'
+ r'_GET|_POST|_SERVER|this|stdClass|range|count|iterator|'
+ r'window)\b', Name.Builtin),
+ (r'[$a-zA-Z_][\w\\]*', Name.Other),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+', Number.Integer),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ ]
+ }
+
+
+class PhpLexer(RegexLexer):
+ """
+ For `PHP <http://www.php.net/>`_ source code.
+ For PHP embedded in HTML, use the `HtmlPhpLexer`.
+
+ Additional options accepted:
+
+ `startinline`
+ If given and ``True`` the lexer starts highlighting with
+ php code (i.e.: no starting ``<?php`` required). The default
+ is ``False``.
+ `funcnamehighlighting`
+ If given and ``True``, highlight builtin function names
+ (default: ``True``).
+ `disabledmodules`
+ If given, must be a list of module names whose function names
+ should not be highlighted. By default all modules are highlighted
+ except the special ``'unknown'`` module that includes functions
+ that are known to php but are undocumented.
+
+ To get a list of allowed modules have a look into the
+ `_php_builtins` module:
+
+ .. sourcecode:: pycon
+
+ >>> from pygments.lexers._php_builtins import MODULES
+ >>> MODULES.keys()
+ ['PHP Options/Info', 'Zip', 'dba', ...]
+
+ In fact the names of those modules match the module names from
+ the php documentation.
+ """
+
+ name = 'PHP'
+ aliases = ['php', 'php3', 'php4', 'php5']
+ filenames = ['*.php', '*.php[345]', '*.inc']
+ mimetypes = ['text/x-php']
+
+ # Note that a backslash is included in the following two patterns
+ # PHP uses a backslash as a namespace separator
+ _ident_char = r'[\\\w]|[^\x00-\x7f]'
+ _ident_begin = r'(?:[\\_a-z]|[^\x00-\x7f])'
+ _ident_end = r'(?:' + _ident_char + ')*'
+ _ident_inner = _ident_begin + _ident_end
+
+ flags = re.IGNORECASE | re.DOTALL | re.MULTILINE
+ tokens = {
+ 'root': [
+ (r'<\?(php)?', Comment.Preproc, 'php'),
+ (r'[^<]+', Other),
+ (r'<', Other)
+ ],
+ 'php': [
+ (r'\?>', Comment.Preproc, '#pop'),
+ (r'(<<<)([\'"]?)(' + _ident_inner + r')(\2\n.*?\n\s*)(\3)(;?)(\n)',
+ bygroups(String, String, String.Delimiter, String, String.Delimiter,
+ Punctuation, Text)),
+ (r'\s+', Text),
+ (r'#.*?\n', Comment.Single),
+ (r'//.*?\n', Comment.Single),
+ # put the empty comment here, it is otherwise seen as
+ # the start of a docstring
+ (r'/\*\*/', Comment.Multiline),
+ (r'/\*\*.*?\*/', String.Doc),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'(->|::)(\s*)(' + _ident_inner + ')',
+ bygroups(Operator, Text, Name.Attribute)),
+ (r'[~!%^&*+=|:.<>/@-]+', Operator),
+ (r'\?', Operator), # don't add to the charclass above!
+ (r'[\[\]{}();,]+', Punctuation),
+ (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
+ (r'(function)(\s*)(?=\()', bygroups(Keyword, Text)),
+ (r'(function)(\s+)(&?)(\s*)',
+ bygroups(Keyword, Text, Operator, Text), 'functionname'),
+ (r'(const)(\s+)(' + _ident_inner + ')',
+ bygroups(Keyword, Text, Name.Constant)),
+ (r'(and|E_PARSE|old_function|E_ERROR|or|as|E_WARNING|parent|'
+ r'eval|PHP_OS|break|exit|case|extends|PHP_VERSION|cfunction|'
+ r'FALSE|print|for|require|continue|foreach|require_once|'
+ r'declare|return|default|static|do|switch|die|stdClass|'
+ r'echo|else|TRUE|elseif|var|empty|if|xor|enddeclare|include|'
+ r'virtual|endfor|include_once|while|endforeach|global|'
+ r'endif|list|endswitch|new|endwhile|not|'
+ r'array|E_ALL|NULL|final|php_user_filter|interface|'
+ r'implements|public|private|protected|abstract|clone|try|'
+ r'catch|throw|this|use|namespace|trait|yield|'
+ r'finally)\b', Keyword),
+ (r'(true|false|null)\b', Keyword.Constant),
+ include('magicconstants'),
+ (r'\$\{\$+' + _ident_inner + '\}', Name.Variable),
+ (r'\$+' + _ident_inner, Name.Variable),
+ (_ident_inner, Name.Other),
+ (r'(\d+\.\d*|\d*\.\d+)(e[+-]?[0-9]+)?', Number.Float),
+ (r'\d+e[+-]?[0-9]+', Number.Float),
+ (r'0[0-7]+', Number.Oct),
+ (r'0x[a-f0-9]+', Number.Hex),
+ (r'\d+', Number.Integer),
+ (r'0b[01]+', Number.Bin),
+ (r"'([^'\\]*(?:\\.[^'\\]*)*)'", String.Single),
+ (r'`([^`\\]*(?:\\.[^`\\]*)*)`', String.Backtick),
+ (r'"', String.Double, 'string'),
+ ],
+ 'magicfuncs': [
+ # source: http://php.net/manual/en/language.oop5.magic.php
+ (words((
+ '__construct', '__destruct', '__call', '__callStatic', '__get', '__set',
+ '__isset', '__unset', '__sleep', '__wakeup', '__toString', '__invoke',
+ '__set_state', '__clone', '__debugInfo',), suffix=r'\b'),
+ Name.Function.Magic),
+ ],
+ 'magicconstants': [
+ # source: http://php.net/manual/en/language.constants.predefined.php
+ (words((
+ '__LINE__', '__FILE__', '__DIR__', '__FUNCTION__', '__CLASS__',
+ '__TRAIT__', '__METHOD__', '__NAMESPACE__',),
+ suffix=r'\b'),
+ Name.Constant),
+ ],
+ 'classname': [
+ (_ident_inner, Name.Class, '#pop')
+ ],
+ 'functionname': [
+ include('magicfuncs'),
+ (_ident_inner, Name.Function, '#pop'),
+ default('#pop')
+ ],
+ 'string': [
+ (r'"', String.Double, '#pop'),
+ (r'[^{$"\\]+', String.Double),
+ (r'\\([nrt"$\\]|[0-7]{1,3}|x[0-9a-f]{1,2})', String.Escape),
+ (r'\$' + _ident_inner + '(\[\S+?\]|->' + _ident_inner + ')?',
+ String.Interpol),
+ (r'(\{\$\{)(.*?)(\}\})',
+ bygroups(String.Interpol, using(this, _startinline=True),
+ String.Interpol)),
+ (r'(\{)(\$.*?)(\})',
+ bygroups(String.Interpol, using(this, _startinline=True),
+ String.Interpol)),
+ (r'(\$\{)(\S+)(\})',
+ bygroups(String.Interpol, Name.Variable, String.Interpol)),
+ (r'[${\\]', String.Double)
+ ],
+ }
+
+ def __init__(self, **options):
+ self.funcnamehighlighting = get_bool_opt(
+ options, 'funcnamehighlighting', True)
+ self.disabledmodules = get_list_opt(
+ options, 'disabledmodules', ['unknown'])
+ self.startinline = get_bool_opt(options, 'startinline', False)
+
+ # private option argument for the lexer itself
+ if '_startinline' in options:
+ self.startinline = options.pop('_startinline')
+
+ # collect activated functions in a set
+ self._functions = set()
+ if self.funcnamehighlighting:
+ from pygments.lexers._php_builtins import MODULES
+ for key, value in iteritems(MODULES):
+ if key not in self.disabledmodules:
+ self._functions.update(value)
+ RegexLexer.__init__(self, **options)
+
+ def get_tokens_unprocessed(self, text):
+ stack = ['root']
+ if self.startinline:
+ stack.append('php')
+ for index, token, value in \
+ RegexLexer.get_tokens_unprocessed(self, text, stack):
+ if token is Name.Other:
+ if value in self._functions:
+ yield index, Name.Builtin, value
+ continue
+ yield index, token, value
+
+ def analyse_text(text):
+ rv = 0.0
+ if re.search(r'<\?(?!xml)', text):
+ rv += 0.3
+ return rv
diff --git a/wandb/vendor/pygments/lexers/praat.py b/wandb/vendor/pygments/lexers/praat.py
new file mode 100644
index 0000000000000000000000000000000000000000..1a38a9e8f75abbd74946ae4fca630d7dcf3316df
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/praat.py
@@ -0,0 +1,294 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.praat
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for Praat
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words, bygroups, include
+from pygments.token import Name, Text, Comment, Keyword, String, Punctuation, Number, \
+ Operator
+
+__all__ = ['PraatLexer']
+
+
+class PraatLexer(RegexLexer):
+ """
+ For `Praat <http://www.praat.org>`_ scripts.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'Praat'
+ aliases = ['praat']
+ filenames = ['*.praat', '*.proc', '*.psc']
+
+ keywords = (
+ 'if', 'then', 'else', 'elsif', 'elif', 'endif', 'fi', 'for', 'from', 'to',
+ 'endfor', 'endproc', 'while', 'endwhile', 'repeat', 'until', 'select', 'plus',
+ 'minus', 'demo', 'assert', 'stopwatch', 'nocheck', 'nowarn', 'noprogress',
+ 'editor', 'endeditor', 'clearinfo',
+ )
+
+ functions_string = (
+ 'backslashTrigraphsToUnicode', 'chooseDirectory', 'chooseReadFile',
+ 'chooseWriteFile', 'date', 'demoKey', 'do', 'environment', 'extractLine',
+ 'extractWord', 'fixed', 'info', 'left', 'mid', 'percent', 'readFile', 'replace',
+ 'replace_regex', 'right', 'selected', 'string', 'unicodeToBackslashTrigraphs',
+ )
+
+ functions_numeric = (
+ 'abs', 'appendFile', 'appendFileLine', 'appendInfo', 'appendInfoLine', 'arccos',
+ 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'barkToHertz',
+ 'beginPause', 'beginSendPraat', 'besselI', 'besselK', 'beta', 'beta2',
+ 'binomialP', 'binomialQ', 'boolean', 'ceiling', 'chiSquareP', 'chiSquareQ',
+ 'choice', 'comment', 'cos', 'cosh', 'createDirectory', 'deleteFile',
+ 'demoClicked', 'demoClickedIn', 'demoCommandKeyPressed',
+ 'demoExtraControlKeyPressed', 'demoInput', 'demoKeyPressed',
+ 'demoOptionKeyPressed', 'demoShiftKeyPressed', 'demoShow', 'demoWaitForInput',
+ 'demoWindowTitle', 'demoX', 'demoY', 'differenceLimensToPhon', 'do', 'editor',
+ 'endPause', 'endSendPraat', 'endsWith', 'erb', 'erbToHertz', 'erf', 'erfc',
+ 'exitScript', 'exp', 'extractNumber', 'fileReadable', 'fisherP', 'fisherQ',
+ 'floor', 'gaussP', 'gaussQ', 'hertzToBark', 'hertzToErb', 'hertzToMel',
+ 'hertzToSemitones', 'imax', 'imin', 'incompleteBeta', 'incompleteGammaP', 'index',
+ 'index_regex', 'invBinomialP', 'invBinomialQ', 'invChiSquareQ', 'invFisherQ',
+ 'invGaussQ', 'invSigmoid', 'invStudentQ', 'length', 'ln', 'lnBeta', 'lnGamma',
+ 'log10', 'log2', 'max', 'melToHertz', 'min', 'minusObject', 'natural', 'number',
+ 'numberOfColumns', 'numberOfRows', 'numberOfSelected', 'objectsAreIdentical',
+ 'option', 'optionMenu', 'pauseScript', 'phonToDifferenceLimens', 'plusObject',
+ 'positive', 'randomBinomial', 'randomGauss', 'randomInteger', 'randomPoisson',
+ 'randomUniform', 'real', 'readFile', 'removeObject', 'rindex', 'rindex_regex',
+ 'round', 'runScript', 'runSystem', 'runSystem_nocheck', 'selectObject',
+ 'selected', 'semitonesToHertz', 'sentencetext', 'sigmoid', 'sin', 'sinc',
+ 'sincpi', 'sinh', 'soundPressureToPhon', 'sqrt', 'startsWith', 'studentP',
+ 'studentQ', 'tan', 'tanh', 'variableExists', 'word', 'writeFile', 'writeFileLine',
+ 'writeInfo', 'writeInfoLine',
+ )
+
+ functions_array = (
+ 'linear', 'randomGauss', 'randomInteger', 'randomUniform', 'zero',
+ )
+
+ objects = (
+ 'Activation', 'AffineTransform', 'AmplitudeTier', 'Art', 'Artword',
+ 'Autosegment', 'BarkFilter', 'BarkSpectrogram', 'CCA', 'Categories',
+ 'Cepstrogram', 'Cepstrum', 'Cepstrumc', 'ChebyshevSeries', 'ClassificationTable',
+ 'Cochleagram', 'Collection', 'ComplexSpectrogram', 'Configuration', 'Confusion',
+ 'ContingencyTable', 'Corpus', 'Correlation', 'Covariance',
+ 'CrossCorrelationTable', 'CrossCorrelationTables', 'DTW', 'DataModeler',
+ 'Diagonalizer', 'Discriminant', 'Dissimilarity', 'Distance', 'Distributions',
+ 'DurationTier', 'EEG', 'ERP', 'ERPTier', 'EditCostsTable', 'EditDistanceTable',
+ 'Eigen', 'Excitation', 'Excitations', 'ExperimentMFC', 'FFNet', 'FeatureWeights',
+ 'FileInMemory', 'FilesInMemory', 'Formant', 'FormantFilter', 'FormantGrid',
+ 'FormantModeler', 'FormantPoint', 'FormantTier', 'GaussianMixture', 'HMM',
+ 'HMM_Observation', 'HMM_ObservationSequence', 'HMM_State', 'HMM_StateSequence',
+ 'Harmonicity', 'ISpline', 'Index', 'Intensity', 'IntensityTier', 'IntervalTier',
+ 'KNN', 'KlattGrid', 'KlattTable', 'LFCC', 'LPC', 'Label', 'LegendreSeries',
+ 'LinearRegression', 'LogisticRegression', 'LongSound', 'Ltas', 'MFCC', 'MSpline',
+ 'ManPages', 'Manipulation', 'Matrix', 'MelFilter', 'MelSpectrogram',
+ 'MixingMatrix', 'Movie', 'Network', 'OTGrammar', 'OTHistory', 'OTMulti', 'PCA',
+ 'PairDistribution', 'ParamCurve', 'Pattern', 'Permutation', 'Photo', 'Pitch',
+ 'PitchModeler', 'PitchTier', 'PointProcess', 'Polygon', 'Polynomial',
+ 'PowerCepstrogram', 'PowerCepstrum', 'Procrustes', 'RealPoint', 'RealTier',
+ 'ResultsMFC', 'Roots', 'SPINET', 'SSCP', 'SVD', 'Salience', 'ScalarProduct',
+ 'Similarity', 'SimpleString', 'SortedSetOfString', 'Sound', 'Speaker',
+ 'Spectrogram', 'Spectrum', 'SpectrumTier', 'SpeechSynthesizer', 'SpellingChecker',
+ 'Strings', 'StringsIndex', 'Table', 'TableOfReal', 'TextGrid', 'TextInterval',
+ 'TextPoint', 'TextTier', 'Tier', 'Transition', 'VocalTract', 'VocalTractTier',
+ 'Weight', 'WordList',
+ )
+
+ variables_numeric = (
+ 'macintosh', 'windows', 'unix', 'praatVersion', 'pi', 'e', 'undefined',
+ )
+
+ variables_string = (
+ 'praatVersion', 'tab', 'shellDirectory', 'homeDirectory',
+ 'preferencesDirectory', 'newline', 'temporaryDirectory',
+ 'defaultDirectory',
+ )
+
+ tokens = {
+ 'root': [
+ (r'(\s+)(#.*?$)', bygroups(Text, Comment.Single)),
+ (r'^#.*?$', Comment.Single),
+ (r';[^\n]*', Comment.Single),
+ (r'\s+', Text),
+
+ (r'\bprocedure\b', Keyword, 'procedure_definition'),
+ (r'\bcall\b', Keyword, 'procedure_call'),
+ (r'@', Name.Function, 'procedure_call'),
+
+ include('function_call'),
+
+ (words(keywords, suffix=r'\b'), Keyword),
+
+ (r'(\bform\b)(\s+)([^\n]+)',
+ bygroups(Keyword, Text, String), 'old_form'),
+
+ (r'(print(?:line|tab)?|echo|exit|asserterror|pause|send(?:praat|socket)|'
+ r'include|execute|system(?:_nocheck)?)(\s+)',
+ bygroups(Keyword, Text), 'string_unquoted'),
+
+ (r'(goto|label)(\s+)(\w+)', bygroups(Keyword, Text, Name.Label)),
+
+ include('variable_name'),
+ include('number'),
+
+ (r'"', String, 'string'),
+
+ (words((objects), suffix=r'(?=\s+\S+\n)'), Name.Class, 'string_unquoted'),
+
+ (r'\b[A-Z]', Keyword, 'command'),
+ (r'(\.{3}|[)(,])', Punctuation),
+ ],
+ 'command': [
+ (r'( ?[\w()-]+ ?)', Keyword),
+ (r"'(?=.*')", String.Interpol, 'string_interpolated'),
+ (r'\.{3}', Keyword, ('#pop', 'old_arguments')),
+ (r':', Keyword, ('#pop', 'comma_list')),
+ (r'\s', Text, '#pop'),
+ ],
+ 'procedure_call': [
+ (r'\s+', Text),
+ (r'([\w.]+)(:|\s*\()',
+ bygroups(Name.Function, Text), '#pop'),
+ (r'([\w.]+)', Name.Function, ('#pop', 'old_arguments')),
+ ],
+ 'procedure_definition': [
+ (r'\s', Text),
+ (r'([\w.]+)(\s*?[(:])',
+ bygroups(Name.Function, Text), '#pop'),
+ (r'([\w.]+)([^\n]*)',
+ bygroups(Name.Function, Text), '#pop'),
+ ],
+ 'function_call': [
+ (words(functions_string, suffix=r'\$(?=\s*[:(])'), Name.Function, 'function'),
+ (words(functions_array, suffix=r'#(?=\s*[:(])'), Name.Function, 'function'),
+ (words(functions_numeric, suffix=r'(?=\s*[:(])'), Name.Function, 'function'),
+ ],
+ 'function': [
+ (r'\s+', Text),
+ (r':', Punctuation, ('#pop', 'comma_list')),
+ (r'\s*\(', Punctuation, ('#pop', 'comma_list')),
+ ],
+ 'comma_list': [
+ (r'(\s*\n\s*)(\.{3})', bygroups(Text, Punctuation)),
+
+ (r'(\s*[])\n])', Text, '#pop'),
+
+ (r'\s+', Text),
+ (r'"', String, 'string'),
+ (r'\b(if|then|else|fi|endif)\b', Keyword),
+
+ include('function_call'),
+ include('variable_name'),
+ include('operator'),
+ include('number'),
+
+ (r'[()]', Text),
+ (r',', Punctuation),
+ ],
+ 'old_arguments': [
+ (r'\n', Text, '#pop'),
+
+ include('variable_name'),
+ include('operator'),
+ include('number'),
+
+ (r'"', String, 'string'),
+ (r'[^\n]', Text),
+ ],
+ 'number': [
+ (r'\n', Text, '#pop'),
+ (r'\b\d+(\.\d*)?([eE][-+]?\d+)?%?', Number),
+ ],
+ 'object_attributes': [
+ (r'\.?(n(col|row)|[xy]min|[xy]max|[nd][xy])\b', Name.Builtin, '#pop'),
+ (r'(\.?(?:col|row)\$)(\[)',
+ bygroups(Name.Builtin, Text), 'variable_name'),
+ (r'(\$?)(\[)',
+ bygroups(Name.Builtin, Text), ('#pop', 'comma_list')),
+ ],
+ 'variable_name': [
+ include('operator'),
+ include('number'),
+
+ (words(variables_string, suffix=r'\$'), Name.Variable.Global),
+ (words(variables_numeric, suffix=r'\b'), Name.Variable.Global),
+
+ (r'\bObject_\w+', Name.Builtin, 'object_attributes'),
+ (words(objects, prefix=r'\b', suffix=r'_\w+'),
+ Name.Builtin, 'object_attributes'),
+
+ (r"\b(Object_)(')",
+ bygroups(Name.Builtin, String.Interpol),
+ ('object_attributes', 'string_interpolated')),
+ (words(objects, prefix=r'\b', suffix=r"(_)(')"),
+ bygroups(Name.Builtin, Name.Builtin, String.Interpol),
+ ('object_attributes', 'string_interpolated')),
+
+ (r'\.?_?[a-z][\w.]*(\$|#)?', Text),
+ (r'[\[\]]', Punctuation, 'comma_list'),
+ (r"'(?=.*')", String.Interpol, 'string_interpolated'),
+ ],
+ 'operator': [
+ (r'([+\/*<>=!-]=?|[&*|][&*|]?|\^|<>)', Operator),
+ (r'(?<![\w.])(and|or|not|div|mod)(?![\w.])', Operator.Word),
+ ],
+ 'string_interpolated': [
+ (r'\.?[_a-z][\w.]*[$#]?(?:\[[a-zA-Z0-9,]+\])?(:[0-9]+)?',
+ String.Interpol),
+ (r"'", String.Interpol, '#pop'),
+ ],
+ 'string_unquoted': [
+ (r'(\n\s*)(\.{3})', bygroups(Text, Punctuation)),
+
+ (r'\n', Text, '#pop'),
+ (r'\s', Text),
+ (r"'(?=.*')", String.Interpol, 'string_interpolated'),
+ (r"'", String),
+ (r"[^'\n]+", String),
+ ],
+ 'string': [
+ (r'(\n\s*)(\.{3})', bygroups(Text, Punctuation)),
+
+ (r'"', String, '#pop'),
+ (r"'(?=.*')", String.Interpol, 'string_interpolated'),
+ (r"'", String),
+ (r'[^\'"\n]+', String),
+ ],
+ 'old_form': [
+ (r'\s+', Text),
+
+ (r'(optionmenu|choice)([ \t]+\S+:[ \t]+)',
+ bygroups(Keyword, Text), 'number'),
+
+ (r'(option|button)([ \t]+)',
+ bygroups(Keyword, Text), 'string_unquoted'),
+
+ (r'(sentence|text)([ \t]+\S+)',
+ bygroups(Keyword, Text), 'string_unquoted'),
+
+ (r'(word)([ \t]+\S+[ \t]*)(\S+)?([ \t]+.*)?',
+ bygroups(Keyword, Text, String, Text)),
+
+ (r'(boolean)(\s+\S+\s*)(0|1|"?(?:yes|no)"?)',
+ bygroups(Keyword, Text, Name.Variable)),
+
+ # Ideally processing of the number would happend in the 'number'
+ # but that doesn't seem to work
+ (r'(real|natural|positive|integer)([ \t]+\S+[ \t]*)([+-]?)(\d+(?:\.\d*)?'
+ r'(?:[eE][-+]?\d+)?%?)',
+ bygroups(Keyword, Text, Operator, Number)),
+
+ (r'(comment)(\s+)',
+ bygroups(Keyword, Text), 'string_unquoted'),
+
+ (r'\bendform\b', Keyword, '#pop'),
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/prolog.py b/wandb/vendor/pygments/lexers/prolog.py
new file mode 100644
index 0000000000000000000000000000000000000000..90f9529cb393197920dff703b84d56f59ea7c8b0
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/prolog.py
@@ -0,0 +1,306 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.prolog
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Prolog and Prolog-like languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['PrologLexer', 'LogtalkLexer']
+
+
+class PrologLexer(RegexLexer):
+ """
+ Lexer for Prolog files.
+ """
+ name = 'Prolog'
+ aliases = ['prolog']
+ filenames = ['*.ecl', '*.prolog', '*.pro', '*.pl']
+ mimetypes = ['text/x-prolog']
+
+ flags = re.UNICODE | re.MULTILINE
+
+ tokens = {
+ 'root': [
+ (r'^#.*', Comment.Single),
+ (r'/\*', Comment.Multiline, 'nested-comment'),
+ (r'%.*', Comment.Single),
+ # character literal
+ (r'0\'.', String.Char),
+ (r'0b[01]+', Number.Bin),
+ (r'0o[0-7]+', Number.Oct),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ # literal with prepended base
+ (r'\d\d?\'[a-zA-Z0-9]+', Number.Integer),
+ (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+ (r'\d+', Number.Integer),
+ (r'[\[\](){}|.,;!]', Punctuation),
+ (r':-|-->', Punctuation),
+ (r'"(?:\\x[0-9a-fA-F]+\\|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|'
+ r'\\[0-7]+\\|\\["\nabcefnrstv]|[^\\"])*"', String.Double),
+ (r"'(?:''|[^'])*'", String.Atom), # quoted atom
+ # Needs to not be followed by an atom.
+ # (r'=(?=\s|[a-zA-Z\[])', Operator),
+ (r'is\b', Operator),
+ (r'(<|>|=<|>=|==|=:=|=|/|//|\*|\+|-)(?=\s|[a-zA-Z0-9\[])',
+ Operator),
+ (r'(mod|div|not)\b', Operator),
+ (r'_', Keyword), # The don't-care variable
+ (r'([a-z]+)(:)', bygroups(Name.Namespace, Punctuation)),
+ (u'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]'
+ u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)'
+ u'(\\s*)(:-|-->)',
+ bygroups(Name.Function, Text, Operator)), # function defn
+ (u'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]'
+ u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)'
+ u'(\\s*)(\\()',
+ bygroups(Name.Function, Text, Punctuation)),
+ (u'[a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]'
+ u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*',
+ String.Atom), # atom, characters
+ # This one includes !
+ (u'[#&*+\\-./:<=>?@\\\\^~\u00a1-\u00bf\u2010-\u303f]+',
+ String.Atom), # atom, graphics
+ (r'[A-Z_]\w*', Name.Variable),
+ (u'\\s+|[\u2000-\u200f\ufff0-\ufffe\uffef]', Text),
+ ],
+ 'nested-comment': [
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'[^*/]+', Comment.Multiline),
+ (r'[*/]', Comment.Multiline),
+ ],
+ }
+
+ def analyse_text(text):
+ return ':-' in text
+
+
+class LogtalkLexer(RegexLexer):
+ """
+ For `Logtalk <http://logtalk.org/>`_ source code.
+
+ .. versionadded:: 0.10
+ """
+
+ name = 'Logtalk'
+ aliases = ['logtalk']
+ filenames = ['*.lgt', '*.logtalk']
+ mimetypes = ['text/x-logtalk']
+
+ tokens = {
+ 'root': [
+ # Directives
+ (r'^\s*:-\s', Punctuation, 'directive'),
+ # Comments
+ (r'%.*?\n', Comment),
+ (r'/\*(.|\n)*?\*/', Comment),
+ # Whitespace
+ (r'\n', Text),
+ (r'\s+', Text),
+ # Numbers
+ (r"0'.", Number),
+ (r'0b[01]+', Number.Bin),
+ (r'0o[0-7]+', Number.Oct),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number),
+ # Variables
+ (r'([A-Z_]\w*)', Name.Variable),
+ # Event handlers
+ (r'(after|before)(?=[(])', Keyword),
+ # Message forwarding handler
+ (r'forward(?=[(])', Keyword),
+ # Execution-context methods
+ (r'(parameter|this|se(lf|nder))(?=[(])', Keyword),
+ # Reflection
+ (r'(current_predicate|predicate_property)(?=[(])', Keyword),
+ # DCGs and term expansion
+ (r'(expand_(goal|term)|(goal|term)_expansion|phrase)(?=[(])', Keyword),
+ # Entity
+ (r'(abolish|c(reate|urrent))_(object|protocol|category)(?=[(])', Keyword),
+ (r'(object|protocol|category)_property(?=[(])', Keyword),
+ # Entity relations
+ (r'co(mplements_object|nforms_to_protocol)(?=[(])', Keyword),
+ (r'extends_(object|protocol|category)(?=[(])', Keyword),
+ (r'imp(lements_protocol|orts_category)(?=[(])', Keyword),
+ (r'(instantiat|specializ)es_class(?=[(])', Keyword),
+ # Events
+ (r'(current_event|(abolish|define)_events)(?=[(])', Keyword),
+ # Flags
+ (r'(current|set)_logtalk_flag(?=[(])', Keyword),
+ # Compiling, loading, and library paths
+ (r'logtalk_(compile|l(ibrary_path|oad|oad_context)|make)(?=[(])', Keyword),
+ (r'\blogtalk_make\b', Keyword),
+ # Database
+ (r'(clause|retract(all)?)(?=[(])', Keyword),
+ (r'a(bolish|ssert(a|z))(?=[(])', Keyword),
+ # Control constructs
+ (r'(ca(ll|tch)|throw)(?=[(])', Keyword),
+ (r'(fa(il|lse)|true)\b', Keyword),
+ # All solutions
+ (r'((bag|set)of|f(ind|or)all)(?=[(])', Keyword),
+ # Multi-threading meta-predicates
+ (r'threaded(_(call|once|ignore|exit|peek|wait|notify))?(?=[(])', Keyword),
+ # Term unification
+ (r'(subsumes_term|unify_with_occurs_check)(?=[(])', Keyword),
+ # Term creation and decomposition
+ (r'(functor|arg|copy_term|numbervars|term_variables)(?=[(])', Keyword),
+ # Evaluable functors
+ (r'(div|rem|m(ax|in|od)|abs|sign)(?=[(])', Keyword),
+ (r'float(_(integer|fractional)_part)?(?=[(])', Keyword),
+ (r'(floor|t(an|runcate)|round|ceiling)(?=[(])', Keyword),
+ # Other arithmetic functors
+ (r'(cos|a(cos|sin|tan|tan2)|exp|log|s(in|qrt)|xor)(?=[(])', Keyword),
+ # Term testing
+ (r'(var|atom(ic)?|integer|float|c(allable|ompound)|n(onvar|umber)|'
+ r'ground|acyclic_term)(?=[(])', Keyword),
+ # Term comparison
+ (r'compare(?=[(])', Keyword),
+ # Stream selection and control
+ (r'(curren|se)t_(in|out)put(?=[(])', Keyword),
+ (r'(open|close)(?=[(])', Keyword),
+ (r'flush_output(?=[(])', Keyword),
+ (r'(at_end_of_stream|flush_output)\b', Keyword),
+ (r'(stream_property|at_end_of_stream|set_stream_position)(?=[(])', Keyword),
+ # Character and byte input/output
+ (r'(nl|(get|peek|put)_(byte|c(har|ode)))(?=[(])', Keyword),
+ (r'\bnl\b', Keyword),
+ # Term input/output
+ (r'read(_term)?(?=[(])', Keyword),
+ (r'write(q|_(canonical|term))?(?=[(])', Keyword),
+ (r'(current_)?op(?=[(])', Keyword),
+ (r'(current_)?char_conversion(?=[(])', Keyword),
+ # Atomic term processing
+ (r'atom_(length|c(hars|o(ncat|des)))(?=[(])', Keyword),
+ (r'(char_code|sub_atom)(?=[(])', Keyword),
+ (r'number_c(har|ode)s(?=[(])', Keyword),
+ # Implementation defined hooks functions
+ (r'(se|curren)t_prolog_flag(?=[(])', Keyword),
+ (r'\bhalt\b', Keyword),
+ (r'halt(?=[(])', Keyword),
+ # Message sending operators
+ (r'(::|:|\^\^)', Operator),
+ # External call
+ (r'[{}]', Keyword),
+ # Logic and control
+ (r'(ignore|once)(?=[(])', Keyword),
+ (r'\brepeat\b', Keyword),
+ # Sorting
+ (r'(key)?sort(?=[(])', Keyword),
+ # Bitwise functors
+ (r'(>>|<<|/\\|\\\\|\\)', Operator),
+ # Predicate aliases
+ (r'\bas\b', Operator),
+ # Arithemtic evaluation
+ (r'\bis\b', Keyword),
+ # Arithemtic comparison
+ (r'(=:=|=\\=|<|=<|>=|>)', Operator),
+ # Term creation and decomposition
+ (r'=\.\.', Operator),
+ # Term unification
+ (r'(=|\\=)', Operator),
+ # Term comparison
+ (r'(==|\\==|@=<|@<|@>=|@>)', Operator),
+ # Evaluable functors
+ (r'(//|[-+*/])', Operator),
+ (r'\b(e|pi|div|mod|rem)\b', Operator),
+ # Other arithemtic functors
+ (r'\b\*\*\b', Operator),
+ # DCG rules
+ (r'-->', Operator),
+ # Control constructs
+ (r'([!;]|->)', Operator),
+ # Logic and control
+ (r'\\+', Operator),
+ # Mode operators
+ (r'[?@]', Operator),
+ # Existential quantifier
+ (r'\^', Operator),
+ # Strings
+ (r'"(\\\\|\\"|[^"])*"', String),
+ # Ponctuation
+ (r'[()\[\],.|]', Text),
+ # Atoms
+ (r"[a-z]\w*", Text),
+ (r"'", String, 'quoted_atom'),
+ ],
+
+ 'quoted_atom': [
+ (r"''", String),
+ (r"'", String, '#pop'),
+ (r'\\([\\abfnrtv"\']|(x[a-fA-F0-9]+|[0-7]+)\\)', String.Escape),
+ (r"[^\\'\n]+", String),
+ (r'\\', String),
+ ],
+
+ 'directive': [
+ # Conditional compilation directives
+ (r'(el)?if(?=[(])', Keyword, 'root'),
+ (r'(e(lse|ndif))[.]', Keyword, 'root'),
+ # Entity directives
+ (r'(category|object|protocol)(?=[(])', Keyword, 'entityrelations'),
+ (r'(end_(category|object|protocol))[.]', Keyword, 'root'),
+ # Predicate scope directives
+ (r'(public|protected|private)(?=[(])', Keyword, 'root'),
+ # Other directives
+ (r'e(n(coding|sure_loaded)|xport)(?=[(])', Keyword, 'root'),
+ (r'in(clude|itialization|fo)(?=[(])', Keyword, 'root'),
+ (r'(built_in|dynamic|synchronized|threaded)[.]', Keyword, 'root'),
+ (r'(alias|d(ynamic|iscontiguous)|m(eta_(non_terminal|predicate)|ode|ultifile)|'
+ r's(et_(logtalk|prolog)_flag|ynchronized))(?=[(])', Keyword, 'root'),
+ (r'op(?=[(])', Keyword, 'root'),
+ (r'(c(alls|oinductive)|module|reexport|use(s|_module))(?=[(])', Keyword, 'root'),
+ (r'[a-z]\w*(?=[(])', Text, 'root'),
+ (r'[a-z]\w*[.]', Text, 'root'),
+ ],
+
+ 'entityrelations': [
+ (r'(complements|extends|i(nstantiates|mp(lements|orts))|specializes)(?=[(])', Keyword),
+ # Numbers
+ (r"0'.", Number),
+ (r'0b[01]+', Number.Bin),
+ (r'0o[0-7]+', Number.Oct),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number),
+ # Variables
+ (r'([A-Z_]\w*)', Name.Variable),
+ # Atoms
+ (r"[a-z]\w*", Text),
+ (r"'", String, 'quoted_atom'),
+ # Strings
+ (r'"(\\\\|\\"|[^"])*"', String),
+ # End of entity-opening directive
+ (r'([)]\.)', Text, 'root'),
+ # Scope operator
+ (r'(::)', Operator),
+ # Ponctuation
+ (r'[()\[\],.|]', Text),
+ # Comments
+ (r'%.*?\n', Comment),
+ (r'/\*(.|\n)*?\*/', Comment),
+ # Whitespace
+ (r'\n', Text),
+ (r'\s+', Text),
+ ]
+ }
+
+ def analyse_text(text):
+ if ':- object(' in text:
+ return 1.0
+ elif ':- protocol(' in text:
+ return 1.0
+ elif ':- category(' in text:
+ return 1.0
+ elif re.search('^:-\s[a-z]', text, re.M):
+ return 0.9
+ else:
+ return 0.0
diff --git a/wandb/vendor/pygments/lexers/python.py b/wandb/vendor/pygments/lexers/python.py
new file mode 100644
index 0000000000000000000000000000000000000000..390eafe853a398248b6b1c0da827ba0670c0daca
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/python.py
@@ -0,0 +1,939 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.python
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Python and related languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \
+ default, words, combined, do_insertions
+from pygments.util import get_bool_opt, shebang_matches
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Generic, Other, Error
+from pygments import unistring as uni
+
+__all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer',
+ 'Python3Lexer', 'Python3TracebackLexer', 'CythonLexer',
+ 'DgLexer', 'NumPyLexer']
+
+line_re = re.compile('.*?\n')
+
+
+class PythonLexer(RegexLexer):
+ """
+ For `Python <http://www.python.org>`_ source code.
+ """
+
+ name = 'Python'
+ aliases = ['python', 'py', 'sage']
+ filenames = ['*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac', '*.sage']
+ mimetypes = ['text/x-python', 'application/x-python']
+
+ def innerstring_rules(ttype):
+ return [
+ # the old style '%s' % (...) string formatting
+ (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
+ '[hlL]?[E-GXc-giorsux%]', String.Interpol),
+ # backslashes, quotes and formatting signs must be parsed one at a time
+ (r'[^\\\'"%\n]+', ttype),
+ (r'[\'"\\]', ttype),
+ # unhandled string formatting sign
+ (r'%', ttype),
+ # newlines are an error (use "nl" state)
+ ]
+
+ tokens = {
+ 'root': [
+ (r'\n', Text),
+ (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")',
+ bygroups(Text, String.Affix, String.Doc)),
+ (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')",
+ bygroups(Text, String.Affix, String.Doc)),
+ (r'[^\S\n]+', Text),
+ (r'\A#!.+$', Comment.Hashbang),
+ (r'#.*$', Comment.Single),
+ (r'[]{}:(),;[]', Punctuation),
+ (r'\\\n', Text),
+ (r'\\', Text),
+ (r'(in|is|and|or|not)\b', Operator.Word),
+ (r'!=|==|<<|>>|[-~+/*%=<>&^|.]', Operator),
+ include('keywords'),
+ (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'),
+ (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'),
+ (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
+ 'fromimport'),
+ (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
+ 'import'),
+ include('builtins'),
+ include('magicfuncs'),
+ include('magicvars'),
+ include('backtick'),
+ ('([rR]|[uUbB][rR]|[rR][uUbB])(""")',
+ bygroups(String.Affix, String.Double), 'tdqs'),
+ ("([rR]|[uUbB][rR]|[rR][uUbB])(''')",
+ bygroups(String.Affix, String.Single), 'tsqs'),
+ ('([rR]|[uUbB][rR]|[rR][uUbB])(")',
+ bygroups(String.Affix, String.Double), 'dqs'),
+ ("([rR]|[uUbB][rR]|[rR][uUbB])(')",
+ bygroups(String.Affix, String.Single), 'sqs'),
+ ('([uUbB]?)(""")', bygroups(String.Affix, String.Double),
+ combined('stringescape', 'tdqs')),
+ ("([uUbB]?)(''')", bygroups(String.Affix, String.Single),
+ combined('stringescape', 'tsqs')),
+ ('([uUbB]?)(")', bygroups(String.Affix, String.Double),
+ combined('stringescape', 'dqs')),
+ ("([uUbB]?)(')", bygroups(String.Affix, String.Single),
+ combined('stringescape', 'sqs')),
+ include('name'),
+ include('numbers'),
+ ],
+ 'keywords': [
+ (words((
+ 'assert', 'break', 'continue', 'del', 'elif', 'else', 'except',
+ 'exec', 'finally', 'for', 'global', 'if', 'lambda', 'pass',
+ 'print', 'raise', 'return', 'try', 'while', 'yield',
+ 'yield from', 'as', 'with'), suffix=r'\b'),
+ Keyword),
+ ],
+ 'builtins': [
+ (words((
+ '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin',
+ 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr', 'classmethod',
+ 'cmp', 'coerce', 'compile', 'complex', 'delattr', 'dict', 'dir', 'divmod',
+ 'enumerate', 'eval', 'execfile', 'exit', 'file', 'filter', 'float',
+ 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'hex', 'id',
+ 'input', 'int', 'intern', 'isinstance', 'issubclass', 'iter', 'len',
+ 'list', 'locals', 'long', 'map', 'max', 'min', 'next', 'object',
+ 'oct', 'open', 'ord', 'pow', 'property', 'range', 'raw_input', 'reduce',
+ 'reload', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice',
+ 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type',
+ 'unichr', 'unicode', 'vars', 'xrange', 'zip'),
+ prefix=r'(?<!\.)', suffix=r'\b'),
+ Name.Builtin),
+ (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|cls'
+ r')\b', Name.Builtin.Pseudo),
+ (words((
+ 'ArithmeticError', 'AssertionError', 'AttributeError',
+ 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError',
+ 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit',
+ 'IOError', 'ImportError', 'ImportWarning', 'IndentationError',
+ 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError',
+ 'MemoryError', 'NameError', 'NotImplemented', 'NotImplementedError',
+ 'OSError', 'OverflowError', 'OverflowWarning', 'PendingDeprecationWarning',
+ 'ReferenceError', 'RuntimeError', 'RuntimeWarning', 'StandardError',
+ 'StopIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError',
+ 'SystemExit', 'TabError', 'TypeError', 'UnboundLocalError',
+ 'UnicodeDecodeError', 'UnicodeEncodeError', 'UnicodeError',
+ 'UnicodeTranslateError', 'UnicodeWarning', 'UserWarning',
+ 'ValueError', 'VMSError', 'Warning', 'WindowsError',
+ 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'),
+ Name.Exception),
+ ],
+ 'magicfuncs': [
+ (words((
+ '__abs__', '__add__', '__and__', '__call__', '__cmp__', '__coerce__',
+ '__complex__', '__contains__', '__del__', '__delattr__', '__delete__',
+ '__delitem__', '__delslice__', '__div__', '__divmod__', '__enter__',
+ '__eq__', '__exit__', '__float__', '__floordiv__', '__ge__', '__get__',
+ '__getattr__', '__getattribute__', '__getitem__', '__getslice__', '__gt__',
+ '__hash__', '__hex__', '__iadd__', '__iand__', '__idiv__', '__ifloordiv__',
+ '__ilshift__', '__imod__', '__imul__', '__index__', '__init__',
+ '__instancecheck__', '__int__', '__invert__', '__iop__', '__ior__',
+ '__ipow__', '__irshift__', '__isub__', '__iter__', '__itruediv__',
+ '__ixor__', '__le__', '__len__', '__long__', '__lshift__', '__lt__',
+ '__missing__', '__mod__', '__mul__', '__ne__', '__neg__', '__new__',
+ '__nonzero__', '__oct__', '__op__', '__or__', '__pos__', '__pow__',
+ '__radd__', '__rand__', '__rcmp__', '__rdiv__', '__rdivmod__', '__repr__',
+ '__reversed__', '__rfloordiv__', '__rlshift__', '__rmod__', '__rmul__',
+ '__rop__', '__ror__', '__rpow__', '__rrshift__', '__rshift__', '__rsub__',
+ '__rtruediv__', '__rxor__', '__set__', '__setattr__', '__setitem__',
+ '__setslice__', '__str__', '__sub__', '__subclasscheck__', '__truediv__',
+ '__unicode__', '__xor__'), suffix=r'\b'),
+ Name.Function.Magic),
+ ],
+ 'magicvars': [
+ (words((
+ '__bases__', '__class__', '__closure__', '__code__', '__defaults__',
+ '__dict__', '__doc__', '__file__', '__func__', '__globals__',
+ '__metaclass__', '__module__', '__mro__', '__name__', '__self__',
+ '__slots__', '__weakref__'),
+ suffix=r'\b'),
+ Name.Variable.Magic),
+ ],
+ 'numbers': [
+ (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
+ (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
+ (r'0[0-7]+j?', Number.Oct),
+ (r'0[bB][01]+', Number.Bin),
+ (r'0[xX][a-fA-F0-9]+', Number.Hex),
+ (r'\d+L', Number.Integer.Long),
+ (r'\d+j?', Number.Integer)
+ ],
+ 'backtick': [
+ ('`.*?`', String.Backtick),
+ ],
+ 'name': [
+ (r'@[\w.]+', Name.Decorator),
+ ('[a-zA-Z_]\w*', Name),
+ ],
+ 'funcname': [
+ include('magicfuncs'),
+ ('[a-zA-Z_]\w*', Name.Function, '#pop'),
+ default('#pop'),
+ ],
+ 'classname': [
+ ('[a-zA-Z_]\w*', Name.Class, '#pop')
+ ],
+ 'import': [
+ (r'(?:[ \t]|\\\n)+', Text),
+ (r'as\b', Keyword.Namespace),
+ (r',', Operator),
+ (r'[a-zA-Z_][\w.]*', Name.Namespace),
+ default('#pop') # all else: go back
+ ],
+ 'fromimport': [
+ (r'(?:[ \t]|\\\n)+', Text),
+ (r'import\b', Keyword.Namespace, '#pop'),
+ # if None occurs here, it's "raise x from None", since None can
+ # never be a module name
+ (r'None\b', Name.Builtin.Pseudo, '#pop'),
+ # sadly, in "raise x from y" y will be highlighted as namespace too
+ (r'[a-zA-Z_.][\w.]*', Name.Namespace),
+ # anything else here also means "raise x from y" and is therefore
+ # not an error
+ default('#pop'),
+ ],
+ 'stringescape': [
+ (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
+ r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
+ ],
+ 'strings-single': innerstring_rules(String.Single),
+ 'strings-double': innerstring_rules(String.Double),
+ 'dqs': [
+ (r'"', String.Double, '#pop'),
+ (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings
+ include('strings-double')
+ ],
+ 'sqs': [
+ (r"'", String.Single, '#pop'),
+ (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings
+ include('strings-single')
+ ],
+ 'tdqs': [
+ (r'"""', String.Double, '#pop'),
+ include('strings-double'),
+ (r'\n', String.Double)
+ ],
+ 'tsqs': [
+ (r"'''", String.Single, '#pop'),
+ include('strings-single'),
+ (r'\n', String.Single)
+ ],
+ }
+
+ def analyse_text(text):
+ return shebang_matches(text, r'pythonw?(2(\.\d)?)?') or \
+ 'import ' in text[:1000]
+
+
+class Python3Lexer(RegexLexer):
+ """
+ For `Python <http://www.python.org>`_ source code (version 3.0).
+
+ .. versionadded:: 0.10
+ """
+
+ name = 'Python 3'
+ aliases = ['python3', 'py3']
+ filenames = [] # Nothing until Python 3 gets widespread
+ mimetypes = ['text/x-python3', 'application/x-python3']
+
+ flags = re.MULTILINE | re.UNICODE
+
+ uni_name = "[%s][%s]*" % (uni.xid_start, uni.xid_continue)
+
+ def innerstring_rules(ttype):
+ return [
+ # the old style '%s' % (...) string formatting (still valid in Py3)
+ (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
+ '[hlL]?[E-GXc-giorsux%]', String.Interpol),
+ # the new style '{}'.format(...) string formatting
+ (r'\{'
+ '((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name
+ '(\![sra])?' # conversion
+ '(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?'
+ '\}', String.Interpol),
+
+ # backslashes, quotes and formatting signs must be parsed one at a time
+ (r'[^\\\'"%{\n]+', ttype),
+ (r'[\'"\\]', ttype),
+ # unhandled string formatting sign
+ (r'%|(\{{1,2})', ttype)
+ # newlines are an error (use "nl" state)
+ ]
+
+ tokens = PythonLexer.tokens.copy()
+ tokens['keywords'] = [
+ (words((
+ 'assert', 'async', 'await', 'break', 'continue', 'del', 'elif',
+ 'else', 'except', 'finally', 'for', 'global', 'if', 'lambda', 'pass',
+ 'raise', 'nonlocal', 'return', 'try', 'while', 'yield', 'yield from',
+ 'as', 'with'), suffix=r'\b'),
+ Keyword),
+ (words((
+ 'True', 'False', 'None'), suffix=r'\b'),
+ Keyword.Constant),
+ ]
+ tokens['builtins'] = [
+ (words((
+ '__import__', 'abs', 'all', 'any', 'bin', 'bool', 'bytearray', 'bytes',
+ 'chr', 'classmethod', 'cmp', 'compile', 'complex', 'delattr', 'dict',
+ 'dir', 'divmod', 'enumerate', 'eval', 'filter', 'float', 'format',
+ 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'hex', 'id',
+ 'input', 'int', 'isinstance', 'issubclass', 'iter', 'len', 'list',
+ 'locals', 'map', 'max', 'memoryview', 'min', 'next', 'object', 'oct',
+ 'open', 'ord', 'pow', 'print', 'property', 'range', 'repr', 'reversed',
+ 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod', 'str',
+ 'sum', 'super', 'tuple', 'type', 'vars', 'zip'), prefix=r'(?<!\.)',
+ suffix=r'\b'),
+ Name.Builtin),
+ (r'(?<!\.)(self|Ellipsis|NotImplemented|cls)\b', Name.Builtin.Pseudo),
+ (words((
+ 'ArithmeticError', 'AssertionError', 'AttributeError',
+ 'BaseException', 'BufferError', 'BytesWarning', 'DeprecationWarning',
+ 'EOFError', 'EnvironmentError', 'Exception', 'FloatingPointError',
+ 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError',
+ 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError',
+ 'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError',
+ 'NotImplementedError', 'OSError', 'OverflowError',
+ 'PendingDeprecationWarning', 'ReferenceError', 'ResourceWarning',
+ 'RuntimeError', 'RuntimeWarning', 'StopIteration',
+ 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError',
+ 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
+ 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
+ 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError', 'Warning',
+ 'WindowsError', 'ZeroDivisionError',
+ # new builtin exceptions from PEP 3151
+ 'BlockingIOError', 'ChildProcessError', 'ConnectionError',
+ 'BrokenPipeError', 'ConnectionAbortedError', 'ConnectionRefusedError',
+ 'ConnectionResetError', 'FileExistsError', 'FileNotFoundError',
+ 'InterruptedError', 'IsADirectoryError', 'NotADirectoryError',
+ 'PermissionError', 'ProcessLookupError', 'TimeoutError'),
+ prefix=r'(?<!\.)', suffix=r'\b'),
+ Name.Exception),
+ ]
+ tokens['magicfuncs'] = [
+ (words((
+ '__abs__', '__add__', '__aenter__', '__aexit__', '__aiter__', '__and__',
+ '__anext__', '__await__', '__bool__', '__bytes__', '__call__',
+ '__complex__', '__contains__', '__del__', '__delattr__', '__delete__',
+ '__delitem__', '__dir__', '__divmod__', '__enter__', '__eq__', '__exit__',
+ '__float__', '__floordiv__', '__format__', '__ge__', '__get__',
+ '__getattr__', '__getattribute__', '__getitem__', '__gt__', '__hash__',
+ '__iadd__', '__iand__', '__ifloordiv__', '__ilshift__', '__imatmul__',
+ '__imod__', '__import__', '__imul__', '__index__', '__init__',
+ '__instancecheck__', '__int__', '__invert__', '__ior__', '__ipow__',
+ '__irshift__', '__isub__', '__iter__', '__itruediv__', '__ixor__',
+ '__le__', '__len__', '__length_hint__', '__lshift__', '__lt__',
+ '__matmul__', '__missing__', '__mod__', '__mul__', '__ne__', '__neg__',
+ '__new__', '__next__', '__or__', '__pos__', '__pow__', '__prepare__',
+ '__radd__', '__rand__', '__rdivmod__', '__repr__', '__reversed__',
+ '__rfloordiv__', '__rlshift__', '__rmatmul__', '__rmod__', '__rmul__',
+ '__ror__', '__round__', '__rpow__', '__rrshift__', '__rshift__',
+ '__rsub__', '__rtruediv__', '__rxor__', '__set__', '__setattr__',
+ '__setitem__', '__str__', '__sub__', '__subclasscheck__', '__truediv__',
+ '__xor__'), suffix=r'\b'),
+ Name.Function.Magic),
+ ]
+ tokens['magicvars'] = [
+ (words((
+ '__annotations__', '__bases__', '__class__', '__closure__', '__code__',
+ '__defaults__', '__dict__', '__doc__', '__file__', '__func__',
+ '__globals__', '__kwdefaults__', '__module__', '__mro__', '__name__',
+ '__objclass__', '__qualname__', '__self__', '__slots__', '__weakref__'),
+ suffix=r'\b'),
+ Name.Variable.Magic),
+ ]
+ tokens['numbers'] = [
+ (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+ (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
+ (r'0[oO][0-7]+', Number.Oct),
+ (r'0[bB][01]+', Number.Bin),
+ (r'0[xX][a-fA-F0-9]+', Number.Hex),
+ (r'\d+', Number.Integer)
+ ]
+ tokens['backtick'] = []
+ tokens['name'] = [
+ (r'@\w+', Name.Decorator),
+ (r'@', Operator), # new matrix multiplication operator
+ (uni_name, Name),
+ ]
+ tokens['funcname'] = [
+ (uni_name, Name.Function, '#pop')
+ ]
+ tokens['classname'] = [
+ (uni_name, Name.Class, '#pop')
+ ]
+ tokens['import'] = [
+ (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)),
+ (r'\.', Name.Namespace),
+ (uni_name, Name.Namespace),
+ (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)),
+ default('#pop') # all else: go back
+ ]
+ tokens['fromimport'] = [
+ (r'(\s+)(import)\b', bygroups(Text, Keyword), '#pop'),
+ (r'\.', Name.Namespace),
+ (uni_name, Name.Namespace),
+ default('#pop'),
+ ]
+ tokens['strings-single'] = innerstring_rules(String.Single)
+ tokens['strings-double'] = innerstring_rules(String.Double)
+
+ def analyse_text(text):
+ return shebang_matches(text, r'pythonw?3(\.\d)?')
+
+
+class PythonConsoleLexer(Lexer):
+ """
+ For Python console output or doctests, such as:
+
+ .. sourcecode:: pycon
+
+ >>> a = 'foo'
+ >>> print a
+ foo
+ >>> 1 / 0
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ ZeroDivisionError: integer division or modulo by zero
+
+ Additional options:
+
+ `python3`
+ Use Python 3 lexer for code. Default is ``False``.
+
+ .. versionadded:: 1.0
+ """
+ name = 'Python console session'
+ aliases = ['pycon']
+ mimetypes = ['text/x-python-doctest']
+
+ def __init__(self, **options):
+ self.python3 = get_bool_opt(options, 'python3', False)
+ Lexer.__init__(self, **options)
+
+ def get_tokens_unprocessed(self, text):
+ if self.python3:
+ pylexer = Python3Lexer(**self.options)
+ tblexer = Python3TracebackLexer(**self.options)
+ else:
+ pylexer = PythonLexer(**self.options)
+ tblexer = PythonTracebackLexer(**self.options)
+
+ curcode = ''
+ insertions = []
+ curtb = ''
+ tbindex = 0
+ tb = 0
+ for match in line_re.finditer(text):
+ line = match.group()
+ if line.startswith(u'>>> ') or line.startswith(u'... '):
+ tb = 0
+ insertions.append((len(curcode),
+ [(0, Generic.Prompt, line[:4])]))
+ curcode += line[4:]
+ elif line.rstrip() == u'...' and not tb:
+ # only a new >>> prompt can end an exception block
+ # otherwise an ellipsis in place of the traceback frames
+ # will be mishandled
+ insertions.append((len(curcode),
+ [(0, Generic.Prompt, u'...')]))
+ curcode += line[3:]
+ else:
+ if curcode:
+ for item in do_insertions(
+ insertions, pylexer.get_tokens_unprocessed(curcode)):
+ yield item
+ curcode = ''
+ insertions = []
+ if (line.startswith(u'Traceback (most recent call last):') or
+ re.match(u' File "[^"]+", line \\d+\\n$', line)):
+ tb = 1
+ curtb = line
+ tbindex = match.start()
+ elif line == 'KeyboardInterrupt\n':
+ yield match.start(), Name.Class, line
+ elif tb:
+ curtb += line
+ if not (line.startswith(' ') or line.strip() == u'...'):
+ tb = 0
+ for i, t, v in tblexer.get_tokens_unprocessed(curtb):
+ yield tbindex+i, t, v
+ curtb = ''
+ else:
+ yield match.start(), Generic.Output, line
+ if curcode:
+ for item in do_insertions(insertions,
+ pylexer.get_tokens_unprocessed(curcode)):
+ yield item
+ if curtb:
+ for i, t, v in tblexer.get_tokens_unprocessed(curtb):
+ yield tbindex+i, t, v
+
+
+class PythonTracebackLexer(RegexLexer):
+ """
+ For Python tracebacks.
+
+ .. versionadded:: 0.7
+ """
+
+ name = 'Python Traceback'
+ aliases = ['pytb']
+ filenames = ['*.pytb']
+ mimetypes = ['text/x-python-traceback']
+
+ tokens = {
+ 'root': [
+ # Cover both (most recent call last) and (innermost last)
+ # The optional ^C allows us to catch keyboard interrupt signals.
+ (r'^(\^C)?(Traceback.*\n)',
+ bygroups(Text, Generic.Traceback), 'intb'),
+ # SyntaxError starts with this.
+ (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
+ (r'^.*\n', Other),
+ ],
+ 'intb': [
+ (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
+ bygroups(Text, Name.Builtin, Text, Number, Text, Name, Text)),
+ (r'^( File )("[^"]+")(, line )(\d+)(\n)',
+ bygroups(Text, Name.Builtin, Text, Number, Text)),
+ (r'^( )(.+)(\n)',
+ bygroups(Text, using(PythonLexer), Text)),
+ (r'^([ \t]*)(\.\.\.)(\n)',
+ bygroups(Text, Comment, Text)), # for doctests...
+ (r'^([^:]+)(: )(.+)(\n)',
+ bygroups(Generic.Error, Text, Name, Text), '#pop'),
+ (r'^([a-zA-Z_]\w*)(:?\n)',
+ bygroups(Generic.Error, Text), '#pop')
+ ],
+ }
+
+
+class Python3TracebackLexer(RegexLexer):
+ """
+ For Python 3.0 tracebacks, with support for chained exceptions.
+
+ .. versionadded:: 1.0
+ """
+
+ name = 'Python 3.0 Traceback'
+ aliases = ['py3tb']
+ filenames = ['*.py3tb']
+ mimetypes = ['text/x-python3-traceback']
+
+ tokens = {
+ 'root': [
+ (r'\n', Text),
+ (r'^Traceback \(most recent call last\):\n', Generic.Traceback, 'intb'),
+ (r'^During handling of the above exception, another '
+ r'exception occurred:\n\n', Generic.Traceback),
+ (r'^The above exception was the direct cause of the '
+ r'following exception:\n\n', Generic.Traceback),
+ (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
+ ],
+ 'intb': [
+ (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
+ bygroups(Text, Name.Builtin, Text, Number, Text, Name, Text)),
+ (r'^( File )("[^"]+")(, line )(\d+)(\n)',
+ bygroups(Text, Name.Builtin, Text, Number, Text)),
+ (r'^( )(.+)(\n)',
+ bygroups(Text, using(Python3Lexer), Text)),
+ (r'^([ \t]*)(\.\.\.)(\n)',
+ bygroups(Text, Comment, Text)), # for doctests...
+ (r'^([^:]+)(: )(.+)(\n)',
+ bygroups(Generic.Error, Text, Name, Text), '#pop'),
+ (r'^([a-zA-Z_]\w*)(:?\n)',
+ bygroups(Generic.Error, Text), '#pop')
+ ],
+ }
+
+
+class CythonLexer(RegexLexer):
+ """
+ For Pyrex and `Cython <http://cython.org>`_ source code.
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'Cython'
+ aliases = ['cython', 'pyx', 'pyrex']
+ filenames = ['*.pyx', '*.pxd', '*.pxi']
+ mimetypes = ['text/x-cython', 'application/x-cython']
+
+ tokens = {
+ 'root': [
+ (r'\n', Text),
+ (r'^(\s*)("""(?:.|\n)*?""")', bygroups(Text, String.Doc)),
+ (r"^(\s*)('''(?:.|\n)*?''')", bygroups(Text, String.Doc)),
+ (r'[^\S\n]+', Text),
+ (r'#.*$', Comment),
+ (r'[]{}:(),;[]', Punctuation),
+ (r'\\\n', Text),
+ (r'\\', Text),
+ (r'(in|is|and|or|not)\b', Operator.Word),
+ (r'(<)([a-zA-Z0-9.?]+)(>)',
+ bygroups(Punctuation, Keyword.Type, Punctuation)),
+ (r'!=|==|<<|>>|[-~+/*%=<>&^|.?]', Operator),
+ (r'(from)(\d+)(<=)(\s+)(<)(\d+)(:)',
+ bygroups(Keyword, Number.Integer, Operator, Name, Operator,
+ Name, Punctuation)),
+ include('keywords'),
+ (r'(def|property)(\s+)', bygroups(Keyword, Text), 'funcname'),
+ (r'(cp?def)(\s+)', bygroups(Keyword, Text), 'cdef'),
+ # (should actually start a block with only cdefs)
+ (r'(cdef)(:)', bygroups(Keyword, Punctuation)),
+ (r'(class|struct)(\s+)', bygroups(Keyword, Text), 'classname'),
+ (r'(from)(\s+)', bygroups(Keyword, Text), 'fromimport'),
+ (r'(c?import)(\s+)', bygroups(Keyword, Text), 'import'),
+ include('builtins'),
+ include('backtick'),
+ ('(?:[rR]|[uU][rR]|[rR][uU])"""', String, 'tdqs'),
+ ("(?:[rR]|[uU][rR]|[rR][uU])'''", String, 'tsqs'),
+ ('(?:[rR]|[uU][rR]|[rR][uU])"', String, 'dqs'),
+ ("(?:[rR]|[uU][rR]|[rR][uU])'", String, 'sqs'),
+ ('[uU]?"""', String, combined('stringescape', 'tdqs')),
+ ("[uU]?'''", String, combined('stringescape', 'tsqs')),
+ ('[uU]?"', String, combined('stringescape', 'dqs')),
+ ("[uU]?'", String, combined('stringescape', 'sqs')),
+ include('name'),
+ include('numbers'),
+ ],
+ 'keywords': [
+ (words((
+ 'assert', 'break', 'by', 'continue', 'ctypedef', 'del', 'elif',
+ 'else', 'except', 'except?', 'exec', 'finally', 'for', 'fused', 'gil',
+ 'global', 'if', 'include', 'lambda', 'nogil', 'pass', 'print',
+ 'raise', 'return', 'try', 'while', 'yield', 'as', 'with'), suffix=r'\b'),
+ Keyword),
+ (r'(DEF|IF|ELIF|ELSE)\b', Comment.Preproc),
+ ],
+ 'builtins': [
+ (words((
+ '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin',
+ 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr',
+ 'classmethod', 'cmp', 'coerce', 'compile', 'complex', 'delattr',
+ 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'execfile', 'exit',
+ 'file', 'filter', 'float', 'frozenset', 'getattr', 'globals',
+ 'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'intern', 'isinstance',
+ 'issubclass', 'iter', 'len', 'list', 'locals', 'long', 'map', 'max',
+ 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'property',
+ 'range', 'raw_input', 'reduce', 'reload', 'repr', 'reversed',
+ 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod',
+ 'str', 'sum', 'super', 'tuple', 'type', 'unichr', 'unicode', 'unsigned',
+ 'vars', 'xrange', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'),
+ Name.Builtin),
+ (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|NULL'
+ r')\b', Name.Builtin.Pseudo),
+ (words((
+ 'ArithmeticError', 'AssertionError', 'AttributeError',
+ 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError',
+ 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit',
+ 'IOError', 'ImportError', 'ImportWarning', 'IndentationError',
+ 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError',
+ 'MemoryError', 'NameError', 'NotImplemented', 'NotImplementedError',
+ 'OSError', 'OverflowError', 'OverflowWarning',
+ 'PendingDeprecationWarning', 'ReferenceError', 'RuntimeError',
+ 'RuntimeWarning', 'StandardError', 'StopIteration', 'SyntaxError',
+ 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError',
+ 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
+ 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
+ 'UnicodeWarning', 'UserWarning', 'ValueError', 'Warning',
+ 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'),
+ Name.Exception),
+ ],
+ 'numbers': [
+ (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+ (r'0\d+', Number.Oct),
+ (r'0[xX][a-fA-F0-9]+', Number.Hex),
+ (r'\d+L', Number.Integer.Long),
+ (r'\d+', Number.Integer)
+ ],
+ 'backtick': [
+ ('`.*?`', String.Backtick),
+ ],
+ 'name': [
+ (r'@\w+', Name.Decorator),
+ ('[a-zA-Z_]\w*', Name),
+ ],
+ 'funcname': [
+ ('[a-zA-Z_]\w*', Name.Function, '#pop')
+ ],
+ 'cdef': [
+ (r'(public|readonly|extern|api|inline)\b', Keyword.Reserved),
+ (r'(struct|enum|union|class)\b', Keyword),
+ (r'([a-zA-Z_]\w*)(\s*)(?=[(:#=]|$)',
+ bygroups(Name.Function, Text), '#pop'),
+ (r'([a-zA-Z_]\w*)(\s*)(,)',
+ bygroups(Name.Function, Text, Punctuation)),
+ (r'from\b', Keyword, '#pop'),
+ (r'as\b', Keyword),
+ (r':', Punctuation, '#pop'),
+ (r'(?=["\'])', Text, '#pop'),
+ (r'[a-zA-Z_]\w*', Keyword.Type),
+ (r'.', Text),
+ ],
+ 'classname': [
+ ('[a-zA-Z_]\w*', Name.Class, '#pop')
+ ],
+ 'import': [
+ (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)),
+ (r'[a-zA-Z_][\w.]*', Name.Namespace),
+ (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)),
+ default('#pop') # all else: go back
+ ],
+ 'fromimport': [
+ (r'(\s+)(c?import)\b', bygroups(Text, Keyword), '#pop'),
+ (r'[a-zA-Z_.][\w.]*', Name.Namespace),
+ # ``cdef foo from "header"``, or ``for foo from 0 < i < 10``
+ default('#pop'),
+ ],
+ 'stringescape': [
+ (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
+ r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
+ ],
+ 'strings': [
+ (r'%(\([a-zA-Z0-9]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
+ '[hlL]?[E-GXc-giorsux%]', String.Interpol),
+ (r'[^\\\'"%\n]+', String),
+ # quotes, percents and backslashes must be parsed one at a time
+ (r'[\'"\\]', String),
+ # unhandled string formatting sign
+ (r'%', String)
+ # newlines are an error (use "nl" state)
+ ],
+ 'nl': [
+ (r'\n', String)
+ ],
+ 'dqs': [
+ (r'"', String, '#pop'),
+ (r'\\\\|\\"|\\\n', String.Escape), # included here again for raw strings
+ include('strings')
+ ],
+ 'sqs': [
+ (r"'", String, '#pop'),
+ (r"\\\\|\\'|\\\n", String.Escape), # included here again for raw strings
+ include('strings')
+ ],
+ 'tdqs': [
+ (r'"""', String, '#pop'),
+ include('strings'),
+ include('nl')
+ ],
+ 'tsqs': [
+ (r"'''", String, '#pop'),
+ include('strings'),
+ include('nl')
+ ],
+ }
+
+
+class DgLexer(RegexLexer):
+ """
+ Lexer for `dg <http://pyos.github.com/dg>`_,
+ a functional and object-oriented programming language
+ running on the CPython 3 VM.
+
+ .. versionadded:: 1.6
+ """
+ name = 'dg'
+ aliases = ['dg']
+ filenames = ['*.dg']
+ mimetypes = ['text/x-dg']
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'#.*?$', Comment.Single),
+
+ (r'(?i)0b[01]+', Number.Bin),
+ (r'(?i)0o[0-7]+', Number.Oct),
+ (r'(?i)0x[0-9a-f]+', Number.Hex),
+ (r'(?i)[+-]?[0-9]+\.[0-9]+(e[+-]?[0-9]+)?j?', Number.Float),
+ (r'(?i)[+-]?[0-9]+e[+-]?\d+j?', Number.Float),
+ (r'(?i)[+-]?[0-9]+j?', Number.Integer),
+
+ (r"(?i)(br|r?b?)'''", String, combined('stringescape', 'tsqs', 'string')),
+ (r'(?i)(br|r?b?)"""', String, combined('stringescape', 'tdqs', 'string')),
+ (r"(?i)(br|r?b?)'", String, combined('stringescape', 'sqs', 'string')),
+ (r'(?i)(br|r?b?)"', String, combined('stringescape', 'dqs', 'string')),
+
+ (r"`\w+'*`", Operator),
+ (r'\b(and|in|is|or|where)\b', Operator.Word),
+ (r'[!$%&*+\-./:<-@\\^|~;,]+', Operator),
+
+ (words((
+ 'bool', 'bytearray', 'bytes', 'classmethod', 'complex', 'dict', 'dict\'',
+ 'float', 'frozenset', 'int', 'list', 'list\'', 'memoryview', 'object',
+ 'property', 'range', 'set', 'set\'', 'slice', 'staticmethod', 'str',
+ 'super', 'tuple', 'tuple\'', 'type'),
+ prefix=r'(?<!\.)', suffix=r'(?![\'\w])'),
+ Name.Builtin),
+ (words((
+ '__import__', 'abs', 'all', 'any', 'bin', 'bind', 'chr', 'cmp', 'compile',
+ 'complex', 'delattr', 'dir', 'divmod', 'drop', 'dropwhile', 'enumerate',
+ 'eval', 'exhaust', 'filter', 'flip', 'foldl1?', 'format', 'fst',
+ 'getattr', 'globals', 'hasattr', 'hash', 'head', 'hex', 'id', 'init',
+ 'input', 'isinstance', 'issubclass', 'iter', 'iterate', 'last', 'len',
+ 'locals', 'map', 'max', 'min', 'next', 'oct', 'open', 'ord', 'pow',
+ 'print', 'repr', 'reversed', 'round', 'setattr', 'scanl1?', 'snd',
+ 'sorted', 'sum', 'tail', 'take', 'takewhile', 'vars', 'zip'),
+ prefix=r'(?<!\.)', suffix=r'(?![\'\w])'),
+ Name.Builtin),
+ (r"(?<!\.)(self|Ellipsis|NotImplemented|None|True|False)(?!['\w])",
+ Name.Builtin.Pseudo),
+
+ (r"(?<!\.)[A-Z]\w*(Error|Exception|Warning)'*(?!['\w])",
+ Name.Exception),
+ (r"(?<!\.)(Exception|GeneratorExit|KeyboardInterrupt|StopIteration|"
+ r"SystemExit)(?!['\w])", Name.Exception),
+
+ (r"(?<![\w.])(except|finally|for|if|import|not|otherwise|raise|"
+ r"subclass|while|with|yield)(?!['\w])", Keyword.Reserved),
+
+ (r"[A-Z_]+'*(?!['\w])", Name),
+ (r"[A-Z]\w+'*(?!['\w])", Keyword.Type),
+ (r"\w+'*", Name),
+
+ (r'[()]', Punctuation),
+ (r'.', Error),
+ ],
+ 'stringescape': [
+ (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
+ r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
+ ],
+ 'string': [
+ (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
+ '[hlL]?[E-GXc-giorsux%]', String.Interpol),
+ (r'[^\\\'"%\n]+', String),
+ # quotes, percents and backslashes must be parsed one at a time
+ (r'[\'"\\]', String),
+ # unhandled string formatting sign
+ (r'%', String),
+ (r'\n', String)
+ ],
+ 'dqs': [
+ (r'"', String, '#pop')
+ ],
+ 'sqs': [
+ (r"'", String, '#pop')
+ ],
+ 'tdqs': [
+ (r'"""', String, '#pop')
+ ],
+ 'tsqs': [
+ (r"'''", String, '#pop')
+ ],
+ }
+
+
+class NumPyLexer(PythonLexer):
+ """
+ A Python lexer recognizing Numerical Python builtins.
+
+ .. versionadded:: 0.10
+ """
+
+ name = 'NumPy'
+ aliases = ['numpy']
+
+ # override the mimetypes to not inherit them from python
+ mimetypes = []
+ filenames = []
+
+ EXTRA_KEYWORDS = set((
+ 'abs', 'absolute', 'accumulate', 'add', 'alen', 'all', 'allclose',
+ 'alltrue', 'alterdot', 'amax', 'amin', 'angle', 'any', 'append',
+ 'apply_along_axis', 'apply_over_axes', 'arange', 'arccos', 'arccosh',
+ 'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'argmax', 'argmin',
+ 'argsort', 'argwhere', 'around', 'array', 'array2string', 'array_equal',
+ 'array_equiv', 'array_repr', 'array_split', 'array_str', 'arrayrange',
+ 'asanyarray', 'asarray', 'asarray_chkfinite', 'ascontiguousarray',
+ 'asfarray', 'asfortranarray', 'asmatrix', 'asscalar', 'astype',
+ 'atleast_1d', 'atleast_2d', 'atleast_3d', 'average', 'bartlett',
+ 'base_repr', 'beta', 'binary_repr', 'bincount', 'binomial',
+ 'bitwise_and', 'bitwise_not', 'bitwise_or', 'bitwise_xor', 'blackman',
+ 'bmat', 'broadcast', 'byte_bounds', 'bytes', 'byteswap', 'c_',
+ 'can_cast', 'ceil', 'choose', 'clip', 'column_stack', 'common_type',
+ 'compare_chararrays', 'compress', 'concatenate', 'conj', 'conjugate',
+ 'convolve', 'copy', 'corrcoef', 'correlate', 'cos', 'cosh', 'cov',
+ 'cross', 'cumprod', 'cumproduct', 'cumsum', 'delete', 'deprecate',
+ 'diag', 'diagflat', 'diagonal', 'diff', 'digitize', 'disp', 'divide',
+ 'dot', 'dsplit', 'dstack', 'dtype', 'dump', 'dumps', 'ediff1d', 'empty',
+ 'empty_like', 'equal', 'exp', 'expand_dims', 'expm1', 'extract', 'eye',
+ 'fabs', 'fastCopyAndTranspose', 'fft', 'fftfreq', 'fftshift', 'fill',
+ 'finfo', 'fix', 'flat', 'flatnonzero', 'flatten', 'fliplr', 'flipud',
+ 'floor', 'floor_divide', 'fmod', 'frexp', 'fromarrays', 'frombuffer',
+ 'fromfile', 'fromfunction', 'fromiter', 'frompyfunc', 'fromstring',
+ 'generic', 'get_array_wrap', 'get_include', 'get_numarray_include',
+ 'get_numpy_include', 'get_printoptions', 'getbuffer', 'getbufsize',
+ 'geterr', 'geterrcall', 'geterrobj', 'getfield', 'gradient', 'greater',
+ 'greater_equal', 'gumbel', 'hamming', 'hanning', 'histogram',
+ 'histogram2d', 'histogramdd', 'hsplit', 'hstack', 'hypot', 'i0',
+ 'identity', 'ifft', 'imag', 'index_exp', 'indices', 'inf', 'info',
+ 'inner', 'insert', 'int_asbuffer', 'interp', 'intersect1d',
+ 'intersect1d_nu', 'inv', 'invert', 'iscomplex', 'iscomplexobj',
+ 'isfinite', 'isfortran', 'isinf', 'isnan', 'isneginf', 'isposinf',
+ 'isreal', 'isrealobj', 'isscalar', 'issctype', 'issubclass_',
+ 'issubdtype', 'issubsctype', 'item', 'itemset', 'iterable', 'ix_',
+ 'kaiser', 'kron', 'ldexp', 'left_shift', 'less', 'less_equal', 'lexsort',
+ 'linspace', 'load', 'loads', 'loadtxt', 'log', 'log10', 'log1p', 'log2',
+ 'logical_and', 'logical_not', 'logical_or', 'logical_xor', 'logspace',
+ 'lstsq', 'mat', 'matrix', 'max', 'maximum', 'maximum_sctype',
+ 'may_share_memory', 'mean', 'median', 'meshgrid', 'mgrid', 'min',
+ 'minimum', 'mintypecode', 'mod', 'modf', 'msort', 'multiply', 'nan',
+ 'nan_to_num', 'nanargmax', 'nanargmin', 'nanmax', 'nanmin', 'nansum',
+ 'ndenumerate', 'ndim', 'ndindex', 'negative', 'newaxis', 'newbuffer',
+ 'newbyteorder', 'nonzero', 'not_equal', 'obj2sctype', 'ogrid', 'ones',
+ 'ones_like', 'outer', 'permutation', 'piecewise', 'pinv', 'pkgload',
+ 'place', 'poisson', 'poly', 'poly1d', 'polyadd', 'polyder', 'polydiv',
+ 'polyfit', 'polyint', 'polymul', 'polysub', 'polyval', 'power', 'prod',
+ 'product', 'ptp', 'put', 'putmask', 'r_', 'randint', 'random_integers',
+ 'random_sample', 'ranf', 'rank', 'ravel', 'real', 'real_if_close',
+ 'recarray', 'reciprocal', 'reduce', 'remainder', 'repeat', 'require',
+ 'reshape', 'resize', 'restoredot', 'right_shift', 'rint', 'roll',
+ 'rollaxis', 'roots', 'rot90', 'round', 'round_', 'row_stack', 's_',
+ 'sample', 'savetxt', 'sctype2char', 'searchsorted', 'seed', 'select',
+ 'set_numeric_ops', 'set_printoptions', 'set_string_function',
+ 'setbufsize', 'setdiff1d', 'seterr', 'seterrcall', 'seterrobj',
+ 'setfield', 'setflags', 'setmember1d', 'setxor1d', 'shape',
+ 'show_config', 'shuffle', 'sign', 'signbit', 'sin', 'sinc', 'sinh',
+ 'size', 'slice', 'solve', 'sometrue', 'sort', 'sort_complex', 'source',
+ 'split', 'sqrt', 'square', 'squeeze', 'standard_normal', 'std',
+ 'subtract', 'sum', 'svd', 'swapaxes', 'take', 'tan', 'tanh', 'tensordot',
+ 'test', 'tile', 'tofile', 'tolist', 'tostring', 'trace', 'transpose',
+ 'trapz', 'tri', 'tril', 'trim_zeros', 'triu', 'true_divide', 'typeDict',
+ 'typename', 'uniform', 'union1d', 'unique', 'unique1d', 'unravel_index',
+ 'unwrap', 'vander', 'var', 'vdot', 'vectorize', 'view', 'vonmises',
+ 'vsplit', 'vstack', 'weibull', 'where', 'who', 'zeros', 'zeros_like'
+ ))
+
+ def get_tokens_unprocessed(self, text):
+ for index, token, value in \
+ PythonLexer.get_tokens_unprocessed(self, text):
+ if token is Name and value in self.EXTRA_KEYWORDS:
+ yield index, Keyword.Pseudo, value
+ else:
+ yield index, token, value
+
+ def analyse_text(text):
+ return (shebang_matches(text, r'pythonw?(2(\.\d)?)?') or
+ 'import ' in text[:1000]) \
+ and ('import numpy' in text or 'from numpy import' in text)
diff --git a/wandb/vendor/pygments/lexers/qvt.py b/wandb/vendor/pygments/lexers/qvt.py
new file mode 100644
index 0000000000000000000000000000000000000000..f496d600291c27e00f444a006cf9a56c60cdbf86
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/qvt.py
@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.qvt
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexer for QVT Operational language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, include, combined, default, \
+ words
+from pygments.token import Text, Comment, Operator, Keyword, Punctuation, \
+ Name, String, Number
+
+__all__ = ['QVToLexer']
+
+
+class QVToLexer(RegexLexer):
+ """
+ For the `QVT Operational Mapping language <http://www.omg.org/spec/QVT/1.1/>`_.
+
+ Reference for implementing this: «Meta Object Facility (MOF) 2.0
+ Query/View/Transformation Specification», Version 1.1 - January 2011
+ (http://www.omg.org/spec/QVT/1.1/), see §8.4, «Concrete Syntax» in
+ particular.
+
+ Notable tokens assignments:
+
+ - Name.Class is assigned to the identifier following any of the following
+ keywords: metamodel, class, exception, primitive, enum, transformation
+ or library
+
+ - Name.Function is assigned to the names of mappings and queries
+
+ - Name.Builtin.Pseudo is assigned to the pre-defined variables 'this',
+ 'self' and 'result'.
+ """
+ # With obvious borrowings & inspiration from the Java, Python and C lexers
+
+ name = 'QVTO'
+ aliases = ['qvto', 'qvt']
+ filenames = ['*.qvto']
+
+ tokens = {
+ 'root': [
+ (r'\n', Text),
+ (r'[^\S\n]+', Text),
+ (r'(--|//)(\s*)(directive:)?(.*)$',
+ bygroups(Comment, Comment, Comment.Preproc, Comment)),
+ # Uncomment the following if you want to distinguish between
+ # '/*' and '/**', Ã la javadoc
+ # (r'/[*]{2}(.|\n)*?[*]/', Comment.Multiline),
+ (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+ (r'\\\n', Text),
+ (r'(and|not|or|xor|##?)\b', Operator.Word),
+ (r'(:{1,2}=|[-+]=)\b', Operator.Word),
+ (r'(@|<<|>>)\b', Keyword), # stereotypes
+ (r'!=|<>|==|=|!->|->|>=|<=|[.]{3}|[+/*%=<>&|.~]', Operator),
+ (r'[]{}:(),;[]', Punctuation),
+ (r'(true|false|unlimited|null)\b', Keyword.Constant),
+ (r'(this|self|result)\b', Name.Builtin.Pseudo),
+ (r'(var)\b', Keyword.Declaration),
+ (r'(from|import)\b', Keyword.Namespace, 'fromimport'),
+ (r'(metamodel|class|exception|primitive|enum|transformation|'
+ r'library)(\s+)(\w+)',
+ bygroups(Keyword.Word, Text, Name.Class)),
+ (r'(exception)(\s+)(\w+)',
+ bygroups(Keyword.Word, Text, Name.Exception)),
+ (r'(main)\b', Name.Function),
+ (r'(mapping|helper|query)(\s+)',
+ bygroups(Keyword.Declaration, Text), 'operation'),
+ (r'(assert)(\s+)\b', bygroups(Keyword, Text), 'assert'),
+ (r'(Bag|Collection|Dict|OrderedSet|Sequence|Set|Tuple|List)\b',
+ Keyword.Type),
+ include('keywords'),
+ ('"', String, combined('stringescape', 'dqs')),
+ ("'", String, combined('stringescape', 'sqs')),
+ include('name'),
+ include('numbers'),
+ # (r'([a-zA-Z_]\w*)(::)([a-zA-Z_]\w*)',
+ # bygroups(Text, Text, Text)),
+ ],
+
+ 'fromimport': [
+ (r'(?:[ \t]|\\\n)+', Text),
+ (r'[a-zA-Z_][\w.]*', Name.Namespace),
+ default('#pop'),
+ ],
+
+ 'operation': [
+ (r'::', Text),
+ (r'(.*::)([a-zA-Z_]\w*)([ \t]*)(\()',
+ bygroups(Text, Name.Function, Text, Punctuation), '#pop')
+ ],
+
+ 'assert': [
+ (r'(warning|error|fatal)\b', Keyword, '#pop'),
+ default('#pop'), # all else: go back
+ ],
+
+ 'keywords': [
+ (words((
+ 'abstract', 'access', 'any', 'assert', 'blackbox', 'break',
+ 'case', 'collect', 'collectNested', 'collectOne', 'collectselect',
+ 'collectselectOne', 'composes', 'compute', 'configuration',
+ 'constructor', 'continue', 'datatype', 'default', 'derived',
+ 'disjuncts', 'do', 'elif', 'else', 'end', 'endif', 'except',
+ 'exists', 'extends', 'forAll', 'forEach', 'forOne', 'from', 'if',
+ 'implies', 'in', 'inherits', 'init', 'inout', 'intermediate',
+ 'invresolve', 'invresolveIn', 'invresolveone', 'invresolveoneIn',
+ 'isUnique', 'iterate', 'late', 'let', 'literal', 'log', 'map',
+ 'merges', 'modeltype', 'new', 'object', 'one', 'ordered', 'out',
+ 'package', 'population', 'property', 'raise', 'readonly',
+ 'references', 'refines', 'reject', 'resolve', 'resolveIn',
+ 'resolveone', 'resolveoneIn', 'return', 'select', 'selectOne',
+ 'sortedBy', 'static', 'switch', 'tag', 'then', 'try', 'typedef',
+ 'unlimited', 'uses', 'when', 'where', 'while', 'with', 'xcollect',
+ 'xmap', 'xselect'), suffix=r'\b'), Keyword),
+ ],
+
+ # There is no need to distinguish between String.Single and
+ # String.Double: 'strings' is factorised for 'dqs' and 'sqs'
+ 'strings': [
+ (r'[^\\\'"\n]+', String),
+ # quotes, percents and backslashes must be parsed one at a time
+ (r'[\'"\\]', String),
+ ],
+ 'stringescape': [
+ (r'\\([\\btnfr"\']|u[0-3][0-7]{2}|u[0-7]{1,2})', String.Escape)
+ ],
+ 'dqs': [ # double-quoted string
+ (r'"', String, '#pop'),
+ (r'\\\\|\\"', String.Escape),
+ include('strings')
+ ],
+ 'sqs': [ # single-quoted string
+ (r"'", String, '#pop'),
+ (r"\\\\|\\'", String.Escape),
+ include('strings')
+ ],
+ 'name': [
+ ('[a-zA-Z_]\w*', Name),
+ ],
+ # numbers: excerpt taken from the python lexer
+ 'numbers': [
+ (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+ (r'\d+[eE][+-]?[0-9]+', Number.Float),
+ (r'\d+', Number.Integer)
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/r.py b/wandb/vendor/pygments/lexers/r.py
new file mode 100644
index 0000000000000000000000000000000000000000..dce6196943ab17dac8ba92e0340ee1718d3b5f09
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/r.py
@@ -0,0 +1,453 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.r
+ ~~~~~~~~~~~~~~~~~
+
+ Lexers for the R/S languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, include, words, do_insertions
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Generic
+
+__all__ = ['RConsoleLexer', 'SLexer', 'RdLexer']
+
+
+line_re = re.compile('.*?\n')
+
+
+class RConsoleLexer(Lexer):
+ """
+ For R console transcripts or R CMD BATCH output files.
+ """
+
+ name = 'RConsole'
+ aliases = ['rconsole', 'rout']
+ filenames = ['*.Rout']
+
+ def get_tokens_unprocessed(self, text):
+ slexer = SLexer(**self.options)
+
+ current_code_block = ''
+ insertions = []
+
+ for match in line_re.finditer(text):
+ line = match.group()
+ if line.startswith('>') or line.startswith('+'):
+ # Colorize the prompt as such,
+ # then put rest of line into current_code_block
+ insertions.append((len(current_code_block),
+ [(0, Generic.Prompt, line[:2])]))
+ current_code_block += line[2:]
+ else:
+ # We have reached a non-prompt line!
+ # If we have stored prompt lines, need to process them first.
+ if current_code_block:
+ # Weave together the prompts and highlight code.
+ for item in do_insertions(
+ insertions, slexer.get_tokens_unprocessed(current_code_block)):
+ yield item
+ # Reset vars for next code block.
+ current_code_block = ''
+ insertions = []
+ # Now process the actual line itself, this is output from R.
+ yield match.start(), Generic.Output, line
+
+ # If we happen to end on a code block with nothing after it, need to
+ # process the last code block. This is neither elegant nor DRY so
+ # should be changed.
+ if current_code_block:
+ for item in do_insertions(
+ insertions, slexer.get_tokens_unprocessed(current_code_block)):
+ yield item
+
+
+class SLexer(RegexLexer):
+ """
+ For S, S-plus, and R source code.
+
+ .. versionadded:: 0.10
+ """
+
+ name = 'S'
+ aliases = ['splus', 's', 'r']
+ filenames = ['*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron']
+ mimetypes = ['text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r',
+ 'text/x-R', 'text/x-r-history', 'text/x-r-profile']
+
+ builtins_base = (
+ 'Arg', 'Conj', 'Cstack_info', 'Encoding', 'FALSE',
+ 'Filter', 'Find', 'I', 'ISOdate', 'ISOdatetime', 'Im', 'Inf',
+ 'La.svd', 'Map', 'Math.Date', 'Math.POSIXt', 'Math.data.frame',
+ 'Math.difftime', 'Math.factor', 'Mod', 'NA_character_',
+ 'NA_complex_', 'NA_real_', 'NCOL', 'NROW', 'NULLNA_integer_', 'NaN',
+ 'Negate', 'NextMethod', 'Ops.Date', 'Ops.POSIXt', 'Ops.data.frame',
+ 'Ops.difftime', 'Ops.factor', 'Ops.numeric_version', 'Ops.ordered',
+ 'Position', 'R.Version', 'R.home', 'R.version', 'R.version.string',
+ 'RNGkind', 'RNGversion', 'R_system_version', 'Re', 'Recall',
+ 'Reduce', 'Summary.Date', 'Summary.POSIXct', 'Summary.POSIXlt',
+ 'Summary.data.frame', 'Summary.difftime', 'Summary.factor',
+ 'Summary.numeric_version', 'Summary.ordered', 'Sys.Date',
+ 'Sys.chmod', 'Sys.getenv', 'Sys.getlocale', 'Sys.getpid',
+ 'Sys.glob', 'Sys.info', 'Sys.localeconv', 'Sys.readlink',
+ 'Sys.setFileTime', 'Sys.setenv', 'Sys.setlocale', 'Sys.sleep',
+ 'Sys.time', 'Sys.timezone', 'Sys.umask', 'Sys.unsetenv',
+ 'Sys.which', 'TRUE', 'UseMethod', 'Vectorize', 'abbreviate', 'abs',
+ 'acos', 'acosh', 'addNA', 'addTaskCallback', 'agrep', 'alist',
+ 'all', 'all.equal', 'all.equal.POSIXct', 'all.equal.character',
+ 'all.equal.default', 'all.equal.factor', 'all.equal.formula',
+ 'all.equal.language', 'all.equal.list', 'all.equal.numeric',
+ 'all.equal.raw', 'all.names', 'all.vars', 'any', 'anyDuplicated',
+ 'anyDuplicated.array', 'anyDuplicated.data.frame',
+ 'anyDuplicated.default', 'anyDuplicated.matrix', 'aperm',
+ 'aperm.default', 'aperm.table', 'append', 'apply', 'args',
+ 'arrayInd', 'as.Date', 'as.Date.POSIXct', 'as.Date.POSIXlt',
+ 'as.Date.character', 'as.Date.date', 'as.Date.dates',
+ 'as.Date.default', 'as.Date.factor', 'as.Date.numeric',
+ 'as.POSIXct', 'as.POSIXct.Date', 'as.POSIXct.POSIXlt',
+ 'as.POSIXct.date', 'as.POSIXct.dates', 'as.POSIXct.default',
+ 'as.POSIXct.numeric', 'as.POSIXlt', 'as.POSIXlt.Date',
+ 'as.POSIXlt.POSIXct', 'as.POSIXlt.character', 'as.POSIXlt.date',
+ 'as.POSIXlt.dates', 'as.POSIXlt.default', 'as.POSIXlt.factor',
+ 'as.POSIXlt.numeric', 'as.array', 'as.array.default', 'as.call',
+ 'as.character', 'as.character.Date', 'as.character.POSIXt',
+ 'as.character.condition', 'as.character.default',
+ 'as.character.error', 'as.character.factor', 'as.character.hexmode',
+ 'as.character.numeric_version', 'as.character.octmode',
+ 'as.character.srcref', 'as.complex', 'as.data.frame',
+ 'as.data.frame.AsIs', 'as.data.frame.Date', 'as.data.frame.POSIXct',
+ 'as.data.frame.POSIXlt', 'as.data.frame.array',
+ 'as.data.frame.character', 'as.data.frame.complex',
+ 'as.data.frame.data.frame', 'as.data.frame.default',
+ 'as.data.frame.difftime', 'as.data.frame.factor',
+ 'as.data.frame.integer', 'as.data.frame.list',
+ 'as.data.frame.logical', 'as.data.frame.matrix',
+ 'as.data.frame.model.matrix', 'as.data.frame.numeric',
+ 'as.data.frame.numeric_version', 'as.data.frame.ordered',
+ 'as.data.frame.raw', 'as.data.frame.table', 'as.data.frame.ts',
+ 'as.data.frame.vector', 'as.difftime', 'as.double',
+ 'as.double.POSIXlt', 'as.double.difftime', 'as.environment',
+ 'as.expression', 'as.expression.default', 'as.factor',
+ 'as.function', 'as.function.default', 'as.hexmode', 'as.integer',
+ 'as.list', 'as.list.Date', 'as.list.POSIXct', 'as.list.data.frame',
+ 'as.list.default', 'as.list.environment', 'as.list.factor',
+ 'as.list.function', 'as.list.numeric_version', 'as.logical',
+ 'as.logical.factor', 'as.matrix', 'as.matrix.POSIXlt',
+ 'as.matrix.data.frame', 'as.matrix.default', 'as.matrix.noquote',
+ 'as.name', 'as.null', 'as.null.default', 'as.numeric',
+ 'as.numeric_version', 'as.octmode', 'as.ordered',
+ 'as.package_version', 'as.pairlist', 'as.qr', 'as.raw', 'as.single',
+ 'as.single.default', 'as.symbol', 'as.table', 'as.table.default',
+ 'as.vector', 'as.vector.factor', 'asNamespace', 'asS3', 'asS4',
+ 'asin', 'asinh', 'assign', 'atan', 'atan2', 'atanh',
+ 'attachNamespace', 'attr', 'attr.all.equal', 'attributes',
+ 'autoload', 'autoloader', 'backsolve', 'baseenv', 'basename',
+ 'besselI', 'besselJ', 'besselK', 'besselY', 'beta',
+ 'bindingIsActive', 'bindingIsLocked', 'bindtextdomain', 'bitwAnd',
+ 'bitwNot', 'bitwOr', 'bitwShiftL', 'bitwShiftR', 'bitwXor', 'body',
+ 'bquote', 'browser', 'browserCondition', 'browserSetDebug',
+ 'browserText', 'builtins', 'by', 'by.data.frame', 'by.default',
+ 'bzfile', 'c.Date', 'c.POSIXct', 'c.POSIXlt', 'c.noquote',
+ 'c.numeric_version', 'call', 'callCC', 'capabilities', 'casefold',
+ 'cat', 'category', 'cbind', 'cbind.data.frame', 'ceiling',
+ 'char.expand', 'charToRaw', 'charmatch', 'chartr', 'check_tzones',
+ 'chol', 'chol.default', 'chol2inv', 'choose', 'class',
+ 'clearPushBack', 'close', 'close.connection', 'close.srcfile',
+ 'close.srcfilealias', 'closeAllConnections', 'col', 'colMeans',
+ 'colSums', 'colnames', 'commandArgs', 'comment', 'computeRestarts',
+ 'conditionCall', 'conditionCall.condition', 'conditionMessage',
+ 'conditionMessage.condition', 'conflicts', 'contributors', 'cos',
+ 'cosh', 'crossprod', 'cummax', 'cummin', 'cumprod', 'cumsum', 'cut',
+ 'cut.Date', 'cut.POSIXt', 'cut.default', 'dQuote', 'data.class',
+ 'data.matrix', 'date', 'debug', 'debugonce',
+ 'default.stringsAsFactors', 'delayedAssign', 'deparse', 'det',
+ 'determinant', 'determinant.matrix', 'dget', 'diag', 'diff',
+ 'diff.Date', 'diff.POSIXt', 'diff.default', 'difftime', 'digamma',
+ 'dim', 'dim.data.frame', 'dimnames', 'dimnames.data.frame', 'dir',
+ 'dir.create', 'dirname', 'do.call', 'dput', 'drop', 'droplevels',
+ 'droplevels.data.frame', 'droplevels.factor', 'dump', 'duplicated',
+ 'duplicated.POSIXlt', 'duplicated.array', 'duplicated.data.frame',
+ 'duplicated.default', 'duplicated.matrix',
+ 'duplicated.numeric_version', 'dyn.load', 'dyn.unload', 'eapply',
+ 'eigen', 'else', 'emptyenv', 'enc2native', 'enc2utf8',
+ 'encodeString', 'enquote', 'env.profile', 'environment',
+ 'environmentIsLocked', 'environmentName', 'eval', 'eval.parent',
+ 'evalq', 'exists', 'exp', 'expand.grid', 'expm1', 'expression',
+ 'factor', 'factorial', 'fifo', 'file', 'file.access', 'file.append',
+ 'file.choose', 'file.copy', 'file.create', 'file.exists',
+ 'file.info', 'file.link', 'file.path', 'file.remove', 'file.rename',
+ 'file.show', 'file.symlink', 'find.package', 'findInterval',
+ 'findPackageEnv', 'findRestart', 'floor', 'flush',
+ 'flush.connection', 'force', 'formals', 'format',
+ 'format.AsIs', 'format.Date', 'format.POSIXct', 'format.POSIXlt',
+ 'format.data.frame', 'format.default', 'format.difftime',
+ 'format.factor', 'format.hexmode', 'format.info',
+ 'format.libraryIQR', 'format.numeric_version', 'format.octmode',
+ 'format.packageInfo', 'format.pval', 'format.summaryDefault',
+ 'formatC', 'formatDL', 'forwardsolve', 'gamma', 'gc', 'gc.time',
+ 'gcinfo', 'gctorture', 'gctorture2', 'get', 'getAllConnections',
+ 'getCallingDLL', 'getCallingDLLe', 'getConnection',
+ 'getDLLRegisteredRoutines', 'getDLLRegisteredRoutines.DLLInfo',
+ 'getDLLRegisteredRoutines.character', 'getElement',
+ 'getExportedValue', 'getHook', 'getLoadedDLLs', 'getNamespace',
+ 'getNamespaceExports', 'getNamespaceImports', 'getNamespaceInfo',
+ 'getNamespaceName', 'getNamespaceUsers', 'getNamespaceVersion',
+ 'getNativeSymbolInfo', 'getOption', 'getRversion', 'getSrcLines',
+ 'getTaskCallbackNames', 'geterrmessage', 'gettext', 'gettextf',
+ 'getwd', 'gl', 'globalenv', 'gregexpr', 'grep', 'grepRaw', 'grepl',
+ 'gsub', 'gzcon', 'gzfile', 'head', 'iconv', 'iconvlist',
+ 'icuSetCollate', 'identical', 'identity', 'ifelse', 'importIntoEnv',
+ 'in', 'inherits', 'intToBits', 'intToUtf8', 'interaction', 'interactive',
+ 'intersect', 'inverse.rle', 'invisible', 'invokeRestart',
+ 'invokeRestartInteractively', 'is.R', 'is.array', 'is.atomic',
+ 'is.call', 'is.character', 'is.complex', 'is.data.frame',
+ 'is.double', 'is.element', 'is.environment', 'is.expression',
+ 'is.factor', 'is.finite', 'is.function', 'is.infinite',
+ 'is.integer', 'is.language', 'is.list', 'is.loaded', 'is.logical',
+ 'is.matrix', 'is.na', 'is.na.POSIXlt', 'is.na.data.frame',
+ 'is.na.numeric_version', 'is.name', 'is.nan', 'is.null',
+ 'is.numeric', 'is.numeric.Date', 'is.numeric.POSIXt',
+ 'is.numeric.difftime', 'is.numeric_version', 'is.object',
+ 'is.ordered', 'is.package_version', 'is.pairlist', 'is.primitive',
+ 'is.qr', 'is.raw', 'is.recursive', 'is.single', 'is.symbol',
+ 'is.table', 'is.unsorted', 'is.vector', 'isBaseNamespace',
+ 'isIncomplete', 'isNamespace', 'isOpen', 'isRestart', 'isS4',
+ 'isSeekable', 'isSymmetric', 'isSymmetric.matrix', 'isTRUE',
+ 'isatty', 'isdebugged', 'jitter', 'julian', 'julian.Date',
+ 'julian.POSIXt', 'kappa', 'kappa.default', 'kappa.lm', 'kappa.qr',
+ 'kronecker', 'l10n_info', 'labels', 'labels.default', 'lapply',
+ 'lazyLoad', 'lazyLoadDBexec', 'lazyLoadDBfetch', 'lbeta', 'lchoose',
+ 'length', 'length.POSIXlt', 'letters', 'levels', 'levels.default',
+ 'lfactorial', 'lgamma', 'library.dynam', 'library.dynam.unload',
+ 'licence', 'license', 'list.dirs', 'list.files', 'list2env', 'load',
+ 'loadNamespace', 'loadedNamespaces', 'loadingNamespaceInfo',
+ 'local', 'lockBinding', 'lockEnvironment', 'log', 'log10', 'log1p',
+ 'log2', 'logb', 'lower.tri', 'ls', 'make.names', 'make.unique',
+ 'makeActiveBinding', 'mapply', 'margin.table', 'mat.or.vec',
+ 'match', 'match.arg', 'match.call', 'match.fun', 'max', 'max.col',
+ 'mean', 'mean.Date', 'mean.POSIXct', 'mean.POSIXlt', 'mean.default',
+ 'mean.difftime', 'mem.limits', 'memCompress', 'memDecompress',
+ 'memory.profile', 'merge', 'merge.data.frame', 'merge.default',
+ 'message', 'mget', 'min', 'missing', 'mode', 'month.abb',
+ 'month.name', 'months', 'months.Date', 'months.POSIXt',
+ 'months.abb', 'months.nameletters', 'names', 'names.POSIXlt',
+ 'namespaceExport', 'namespaceImport', 'namespaceImportClasses',
+ 'namespaceImportFrom', 'namespaceImportMethods', 'nargs', 'nchar',
+ 'ncol', 'new.env', 'ngettext', 'nlevels', 'noquote', 'norm',
+ 'normalizePath', 'nrow', 'numeric_version', 'nzchar', 'objects',
+ 'oldClass', 'on.exit', 'open', 'open.connection', 'open.srcfile',
+ 'open.srcfilealias', 'open.srcfilecopy', 'options', 'order',
+ 'ordered', 'outer', 'packBits', 'packageEvent',
+ 'packageHasNamespace', 'packageStartupMessage', 'package_version',
+ 'pairlist', 'parent.env', 'parent.frame', 'parse',
+ 'parseNamespaceFile', 'paste', 'paste0', 'path.expand',
+ 'path.package', 'pipe', 'pmatch', 'pmax', 'pmax.int', 'pmin',
+ 'pmin.int', 'polyroot', 'pos.to.env', 'pretty', 'pretty.default',
+ 'prettyNum', 'print', 'print.AsIs', 'print.DLLInfo',
+ 'print.DLLInfoList', 'print.DLLRegisteredRoutines', 'print.Date',
+ 'print.NativeRoutineList', 'print.POSIXct', 'print.POSIXlt',
+ 'print.by', 'print.condition', 'print.connection',
+ 'print.data.frame', 'print.default', 'print.difftime',
+ 'print.factor', 'print.function', 'print.hexmode',
+ 'print.libraryIQR', 'print.listof', 'print.noquote',
+ 'print.numeric_version', 'print.octmode', 'print.packageInfo',
+ 'print.proc_time', 'print.restart', 'print.rle',
+ 'print.simple.list', 'print.srcfile', 'print.srcref',
+ 'print.summary.table', 'print.summaryDefault', 'print.table',
+ 'print.warnings', 'prmatrix', 'proc.time', 'prod', 'prop.table',
+ 'provideDimnames', 'psigamma', 'pushBack', 'pushBackLength', 'q',
+ 'qr', 'qr.Q', 'qr.R', 'qr.X', 'qr.coef', 'qr.default', 'qr.fitted',
+ 'qr.qty', 'qr.qy', 'qr.resid', 'qr.solve', 'quarters',
+ 'quarters.Date', 'quarters.POSIXt', 'quit', 'quote', 'range',
+ 'range.default', 'rank', 'rapply', 'raw', 'rawConnection',
+ 'rawConnectionValue', 'rawShift', 'rawToBits', 'rawToChar', 'rbind',
+ 'rbind.data.frame', 'rcond', 'read.dcf', 'readBin', 'readChar',
+ 'readLines', 'readRDS', 'readRenviron', 'readline', 'reg.finalizer',
+ 'regexec', 'regexpr', 'registerS3method', 'registerS3methods',
+ 'regmatches', 'remove', 'removeTaskCallback', 'rep', 'rep.Date',
+ 'rep.POSIXct', 'rep.POSIXlt', 'rep.factor', 'rep.int',
+ 'rep.numeric_version', 'rep_len', 'replace', 'replicate',
+ 'requireNamespace', 'restartDescription', 'restartFormals',
+ 'retracemem', 'rev', 'rev.default', 'rle', 'rm', 'round',
+ 'round.Date', 'round.POSIXt', 'row', 'row.names',
+ 'row.names.data.frame', 'row.names.default', 'rowMeans', 'rowSums',
+ 'rownames', 'rowsum', 'rowsum.data.frame', 'rowsum.default',
+ 'sQuote', 'sample', 'sample.int', 'sapply', 'save', 'save.image',
+ 'saveRDS', 'scale', 'scale.default', 'scan', 'search',
+ 'searchpaths', 'seek', 'seek.connection', 'seq', 'seq.Date',
+ 'seq.POSIXt', 'seq.default', 'seq.int', 'seq_along', 'seq_len',
+ 'sequence', 'serialize', 'set.seed', 'setHook', 'setNamespaceInfo',
+ 'setSessionTimeLimit', 'setTimeLimit', 'setdiff', 'setequal',
+ 'setwd', 'shQuote', 'showConnections', 'sign', 'signalCondition',
+ 'signif', 'simpleCondition', 'simpleError', 'simpleMessage',
+ 'simpleWarning', 'simplify2array', 'sin', 'single',
+ 'sinh', 'sink', 'sink.number', 'slice.index', 'socketConnection',
+ 'socketSelect', 'solve', 'solve.default', 'solve.qr', 'sort',
+ 'sort.POSIXlt', 'sort.default', 'sort.int', 'sort.list', 'split',
+ 'split.Date', 'split.POSIXct', 'split.data.frame', 'split.default',
+ 'sprintf', 'sqrt', 'srcfile', 'srcfilealias', 'srcfilecopy',
+ 'srcref', 'standardGeneric', 'stderr', 'stdin', 'stdout', 'stop',
+ 'stopifnot', 'storage.mode', 'strftime', 'strptime', 'strsplit',
+ 'strtoi', 'strtrim', 'structure', 'strwrap', 'sub', 'subset',
+ 'subset.data.frame', 'subset.default', 'subset.matrix',
+ 'substitute', 'substr', 'substring', 'sum', 'summary',
+ 'summary.Date', 'summary.POSIXct', 'summary.POSIXlt',
+ 'summary.connection', 'summary.data.frame', 'summary.default',
+ 'summary.factor', 'summary.matrix', 'summary.proc_time',
+ 'summary.srcfile', 'summary.srcref', 'summary.table',
+ 'suppressMessages', 'suppressPackageStartupMessages',
+ 'suppressWarnings', 'svd', 'sweep', 'sys.call', 'sys.calls',
+ 'sys.frame', 'sys.frames', 'sys.function', 'sys.load.image',
+ 'sys.nframe', 'sys.on.exit', 'sys.parent', 'sys.parents',
+ 'sys.save.image', 'sys.source', 'sys.status', 'system',
+ 'system.file', 'system.time', 'system2', 't', 't.data.frame',
+ 't.default', 'table', 'tabulate', 'tail', 'tan', 'tanh', 'tapply',
+ 'taskCallbackManager', 'tcrossprod', 'tempdir', 'tempfile',
+ 'testPlatformEquivalence', 'textConnection', 'textConnectionValue',
+ 'toString', 'toString.default', 'tolower', 'topenv', 'toupper',
+ 'trace', 'traceback', 'tracemem', 'tracingState', 'transform',
+ 'transform.data.frame', 'transform.default', 'trigamma', 'trunc',
+ 'trunc.Date', 'trunc.POSIXt', 'truncate', 'truncate.connection',
+ 'try', 'tryCatch', 'typeof', 'unclass', 'undebug', 'union',
+ 'unique', 'unique.POSIXlt', 'unique.array', 'unique.data.frame',
+ 'unique.default', 'unique.matrix', 'unique.numeric_version',
+ 'units', 'units.difftime', 'unix.time', 'unlink', 'unlist',
+ 'unloadNamespace', 'unlockBinding', 'unname', 'unserialize',
+ 'unsplit', 'untrace', 'untracemem', 'unz', 'upper.tri', 'url',
+ 'utf8ToInt', 'vapply', 'version', 'warning', 'warnings', 'weekdays',
+ 'weekdays.Date', 'weekdays.POSIXt', 'which', 'which.max',
+ 'which.min', 'with', 'with.default', 'withCallingHandlers',
+ 'withRestarts', 'withVisible', 'within', 'within.data.frame',
+ 'within.list', 'write', 'write.dcf', 'writeBin', 'writeChar',
+ 'writeLines', 'xor', 'xor.hexmode', 'xor.octmode',
+ 'xpdrows.data.frame', 'xtfrm', 'xtfrm.AsIs', 'xtfrm.Date',
+ 'xtfrm.POSIXct', 'xtfrm.POSIXlt', 'xtfrm.Surv', 'xtfrm.default',
+ 'xtfrm.difftime', 'xtfrm.factor', 'xtfrm.numeric_version', 'xzfile',
+ 'zapsmall'
+ )
+
+ tokens = {
+ 'comments': [
+ (r'#.*$', Comment.Single),
+ ],
+ 'valid_name': [
+ (r'[a-zA-Z][\w.]*', Text),
+ # can begin with ., but not if that is followed by a digit
+ (r'\.[a-zA-Z_][\w.]*', Text),
+ ],
+ 'punctuation': [
+ (r'\[{1,2}|\]{1,2}|\(|\)|;|,', Punctuation),
+ ],
+ 'keywords': [
+ (words(builtins_base, suffix=r'(?![\w. =])'),
+ Keyword.Pseudo),
+ (r'(if|else|for|while|repeat|in|next|break|return|switch|function)'
+ r'(?![\w.])',
+ Keyword.Reserved),
+ (r'(array|category|character|complex|double|function|integer|list|'
+ r'logical|matrix|numeric|vector|data.frame|c)'
+ r'(?![\w.])',
+ Keyword.Type),
+ (r'(library|require|attach|detach|source)'
+ r'(?![\w.])',
+ Keyword.Namespace)
+ ],
+ 'operators': [
+ (r'<<?-|->>?|-|==|<=|>=|<|>|&&?|!=|\|\|?|\?', Operator),
+ (r'\*|\+|\^|/|!|%[^%]*%|=|~|\$|@|:{1,3}', Operator)
+ ],
+ 'builtin_symbols': [
+ (r'(NULL|NA(_(integer|real|complex|character)_)?|'
+ r'letters|LETTERS|Inf|TRUE|FALSE|NaN|pi|\.\.(\.|[0-9]+))'
+ r'(?![\w.])',
+ Keyword.Constant),
+ (r'(T|F)\b', Name.Builtin.Pseudo),
+ ],
+ 'numbers': [
+ # hex number
+ (r'0[xX][a-fA-F0-9]+([pP][0-9]+)?[Li]?', Number.Hex),
+ # decimal number
+ (r'[+-]?([0-9]+(\.[0-9]+)?|\.[0-9]+|\.)([eE][+-]?[0-9]+)?[Li]?',
+ Number),
+ ],
+ 'statements': [
+ include('comments'),
+ # whitespaces
+ (r'\s+', Text),
+ (r'`.*?`', String.Backtick),
+ (r'\'', String, 'string_squote'),
+ (r'\"', String, 'string_dquote'),
+ include('builtin_symbols'),
+ include('numbers'),
+ include('keywords'),
+ include('punctuation'),
+ include('operators'),
+ include('valid_name'),
+ ],
+ 'root': [
+ include('statements'),
+ # blocks:
+ (r'\{|\}', Punctuation),
+ # (r'\{', Punctuation, 'block'),
+ (r'.', Text),
+ ],
+ # 'block': [
+ # include('statements'),
+ # ('\{', Punctuation, '#push'),
+ # ('\}', Punctuation, '#pop')
+ # ],
+ 'string_squote': [
+ (r'([^\'\\]|\\.)*\'', String, '#pop'),
+ ],
+ 'string_dquote': [
+ (r'([^"\\]|\\.)*"', String, '#pop'),
+ ],
+ }
+
+ def analyse_text(text):
+ if re.search(r'[a-z0-9_\])\s]<-(?!-)', text):
+ return 0.11
+
+
+class RdLexer(RegexLexer):
+ """
+ Pygments Lexer for R documentation (Rd) files
+
+ This is a very minimal implementation, highlighting little more
+ than the macros. A description of Rd syntax is found in `Writing R
+ Extensions <http://cran.r-project.org/doc/manuals/R-exts.html>`_
+ and `Parsing Rd files <developer.r-project.org/parseRd.pdf>`_.
+
+ .. versionadded:: 1.6
+ """
+ name = 'Rd'
+ aliases = ['rd']
+ filenames = ['*.Rd']
+ mimetypes = ['text/x-r-doc']
+
+ # To account for verbatim / LaTeX-like / and R-like areas
+ # would require parsing.
+ tokens = {
+ 'root': [
+ # catch escaped brackets and percent sign
+ (r'\\[\\{}%]', String.Escape),
+ # comments
+ (r'%.*$', Comment),
+ # special macros with no arguments
+ (r'\\(?:cr|l?dots|R|tab)\b', Keyword.Constant),
+ # macros
+ (r'\\[a-zA-Z]+\b', Keyword),
+ # special preprocessor macros
+ (r'^\s*#(?:ifn?def|endif).*\b', Comment.Preproc),
+ # non-escaped brackets
+ (r'[{}]', Name.Builtin),
+ # everything else
+ (r'[^\\%\n{}]+', Text),
+ (r'.', Text),
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/rdf.py b/wandb/vendor/pygments/lexers/rdf.py
new file mode 100644
index 0000000000000000000000000000000000000000..d0f8778adf2ae7d3b7e248e106865ee12158b309
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/rdf.py
@@ -0,0 +1,270 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.rdf
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexers for semantic web and RDF query languages and markup.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, default
+from pygments.token import Keyword, Punctuation, String, Number, Operator, Generic, \
+ Whitespace, Name, Literal, Comment, Text
+
+__all__ = ['SparqlLexer', 'TurtleLexer']
+
+
+class SparqlLexer(RegexLexer):
+ """
+ Lexer for `SPARQL <http://www.w3.org/TR/rdf-sparql-query/>`_ query language.
+
+ .. versionadded:: 2.0
+ """
+ name = 'SPARQL'
+ aliases = ['sparql']
+ filenames = ['*.rq', '*.sparql']
+ mimetypes = ['application/sparql-query']
+
+ # character group definitions ::
+
+ PN_CHARS_BASE_GRP = (u'a-zA-Z'
+ u'\u00c0-\u00d6'
+ u'\u00d8-\u00f6'
+ u'\u00f8-\u02ff'
+ u'\u0370-\u037d'
+ u'\u037f-\u1fff'
+ u'\u200c-\u200d'
+ u'\u2070-\u218f'
+ u'\u2c00-\u2fef'
+ u'\u3001-\ud7ff'
+ u'\uf900-\ufdcf'
+ u'\ufdf0-\ufffd')
+
+ PN_CHARS_U_GRP = (PN_CHARS_BASE_GRP + '_')
+
+ PN_CHARS_GRP = (PN_CHARS_U_GRP +
+ r'\-' +
+ r'0-9' +
+ u'\u00b7' +
+ u'\u0300-\u036f' +
+ u'\u203f-\u2040')
+
+ HEX_GRP = '0-9A-Fa-f'
+
+ PN_LOCAL_ESC_CHARS_GRP = r' _~.\-!$&"()*+,;=/?#@%'
+
+ # terminal productions ::
+
+ PN_CHARS_BASE = '[' + PN_CHARS_BASE_GRP + ']'
+
+ PN_CHARS_U = '[' + PN_CHARS_U_GRP + ']'
+
+ PN_CHARS = '[' + PN_CHARS_GRP + ']'
+
+ HEX = '[' + HEX_GRP + ']'
+
+ PN_LOCAL_ESC_CHARS = '[' + PN_LOCAL_ESC_CHARS_GRP + ']'
+
+ IRIREF = r'<(?:[^<>"{}|^`\\\x00-\x20])*>'
+
+ BLANK_NODE_LABEL = '_:[0-9' + PN_CHARS_U_GRP + '](?:[' + PN_CHARS_GRP + \
+ '.]*' + PN_CHARS + ')?'
+
+ PN_PREFIX = PN_CHARS_BASE + '(?:[' + PN_CHARS_GRP + '.]*' + PN_CHARS + ')?'
+
+ VARNAME = u'[0-9' + PN_CHARS_U_GRP + '][' + PN_CHARS_U_GRP + \
+ u'0-9\u00b7\u0300-\u036f\u203f-\u2040]*'
+
+ PERCENT = '%' + HEX + HEX
+
+ PN_LOCAL_ESC = r'\\' + PN_LOCAL_ESC_CHARS
+
+ PLX = '(?:' + PERCENT + ')|(?:' + PN_LOCAL_ESC + ')'
+
+ PN_LOCAL = ('(?:[' + PN_CHARS_U_GRP + ':0-9' + ']|' + PLX + ')' +
+ '(?:(?:[' + PN_CHARS_GRP + '.:]|' + PLX + ')*(?:[' +
+ PN_CHARS_GRP + ':]|' + PLX + '))?')
+
+ EXPONENT = r'[eE][+-]?\d+'
+
+ # Lexer token definitions ::
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ # keywords ::
+ (r'((?i)select|construct|describe|ask|where|filter|group\s+by|minus|'
+ r'distinct|reduced|from\s+named|from|order\s+by|desc|asc|limit|'
+ r'offset|bindings|load|clear|drop|create|add|move|copy|'
+ r'insert\s+data|delete\s+data|delete\s+where|delete|insert|'
+ r'using\s+named|using|graph|default|named|all|optional|service|'
+ r'silent|bind|union|not\s+in|in|as|having|to|prefix|base)\b', Keyword),
+ (r'(a)\b', Keyword),
+ # IRIs ::
+ ('(' + IRIREF + ')', Name.Label),
+ # blank nodes ::
+ ('(' + BLANK_NODE_LABEL + ')', Name.Label),
+ # # variables ::
+ ('[?$]' + VARNAME, Name.Variable),
+ # prefixed names ::
+ (r'(' + PN_PREFIX + ')?(\:)(' + PN_LOCAL + ')?',
+ bygroups(Name.Namespace, Punctuation, Name.Tag)),
+ # function names ::
+ (r'((?i)str|lang|langmatches|datatype|bound|iri|uri|bnode|rand|abs|'
+ r'ceil|floor|round|concat|strlen|ucase|lcase|encode_for_uri|'
+ r'contains|strstarts|strends|strbefore|strafter|year|month|day|'
+ r'hours|minutes|seconds|timezone|tz|now|md5|sha1|sha256|sha384|'
+ r'sha512|coalesce|if|strlang|strdt|sameterm|isiri|isuri|isblank|'
+ r'isliteral|isnumeric|regex|substr|replace|exists|not\s+exists|'
+ r'count|sum|min|max|avg|sample|group_concat|separator)\b',
+ Name.Function),
+ # boolean literals ::
+ (r'(true|false)', Keyword.Constant),
+ # double literals ::
+ (r'[+\-]?(\d+\.\d*' + EXPONENT + '|\.?\d+' + EXPONENT + ')', Number.Float),
+ # decimal literals ::
+ (r'[+\-]?(\d+\.\d*|\.\d+)', Number.Float),
+ # integer literals ::
+ (r'[+\-]?\d+', Number.Integer),
+ # operators ::
+ (r'(\|\||&&|=|\*|\-|\+|/|!=|<=|>=|!|<|>)', Operator),
+ # punctuation characters ::
+ (r'[(){}.;,:^\[\]]', Punctuation),
+ # line comments ::
+ (r'#[^\n]*', Comment),
+ # strings ::
+ (r'"""', String, 'triple-double-quoted-string'),
+ (r'"', String, 'single-double-quoted-string'),
+ (r"'''", String, 'triple-single-quoted-string'),
+ (r"'", String, 'single-single-quoted-string'),
+ ],
+ 'triple-double-quoted-string': [
+ (r'"""', String, 'end-of-string'),
+ (r'[^\\]+', String),
+ (r'\\', String, 'string-escape'),
+ ],
+ 'single-double-quoted-string': [
+ (r'"', String, 'end-of-string'),
+ (r'[^"\\\n]+', String),
+ (r'\\', String, 'string-escape'),
+ ],
+ 'triple-single-quoted-string': [
+ (r"'''", String, 'end-of-string'),
+ (r'[^\\]+', String),
+ (r'\\', String.Escape, 'string-escape'),
+ ],
+ 'single-single-quoted-string': [
+ (r"'", String, 'end-of-string'),
+ (r"[^'\\\n]+", String),
+ (r'\\', String, 'string-escape'),
+ ],
+ 'string-escape': [
+ (r'u' + HEX + '{4}', String.Escape, '#pop'),
+ (r'U' + HEX + '{8}', String.Escape, '#pop'),
+ (r'.', String.Escape, '#pop'),
+ ],
+ 'end-of-string': [
+ (r'(@)([a-zA-Z]+(?:-[a-zA-Z0-9]+)*)',
+ bygroups(Operator, Name.Function), '#pop:2'),
+ (r'\^\^', Operator, '#pop:2'),
+ default('#pop:2'),
+ ],
+ }
+
+
+class TurtleLexer(RegexLexer):
+ """
+ Lexer for `Turtle <http://www.w3.org/TR/turtle/>`_ data language.
+
+ .. versionadded:: 2.1
+ """
+ name = 'Turtle'
+ aliases = ['turtle']
+ filenames = ['*.ttl']
+ mimetypes = ['text/turtle', 'application/x-turtle']
+
+ flags = re.IGNORECASE
+
+ patterns = {
+ 'PNAME_NS': r'((?:[a-z][\w-]*)?\:)', # Simplified character range
+ 'IRIREF': r'(<[^<>"{}|^`\\\x00-\x20]*>)'
+ }
+
+ # PNAME_NS PN_LOCAL (with simplified character range)
+ patterns['PrefixedName'] = r'%(PNAME_NS)s([a-z][\w-]*)' % patterns
+
+ tokens = {
+ 'root': [
+ (r'\s+', Whitespace),
+
+ # Base / prefix
+ (r'(@base|BASE)(\s+)%(IRIREF)s(\s*)(\.?)' % patterns,
+ bygroups(Keyword, Whitespace, Name.Variable, Whitespace,
+ Punctuation)),
+ (r'(@prefix|PREFIX)(\s+)%(PNAME_NS)s(\s+)%(IRIREF)s(\s*)(\.?)' % patterns,
+ bygroups(Keyword, Whitespace, Name.Namespace, Whitespace,
+ Name.Variable, Whitespace, Punctuation)),
+
+ # The shorthand predicate 'a'
+ (r'(?<=\s)a(?=\s)', Keyword.Type),
+
+ # IRIREF
+ (r'%(IRIREF)s' % patterns, Name.Variable),
+
+ # PrefixedName
+ (r'%(PrefixedName)s' % patterns,
+ bygroups(Name.Namespace, Name.Tag)),
+
+ # Comment
+ (r'#[^\n]+', Comment),
+
+ (r'\b(true|false)\b', Literal),
+ (r'[+\-]?\d*\.\d+', Number.Float),
+ (r'[+\-]?\d*(:?\.\d+)?E[+\-]?\d+', Number.Float),
+ (r'[+\-]?\d+', Number.Integer),
+ (r'[\[\](){}.;,:^]', Punctuation),
+
+ (r'"""', String, 'triple-double-quoted-string'),
+ (r'"', String, 'single-double-quoted-string'),
+ (r"'''", String, 'triple-single-quoted-string'),
+ (r"'", String, 'single-single-quoted-string'),
+ ],
+ 'triple-double-quoted-string': [
+ (r'"""', String, 'end-of-string'),
+ (r'[^\\]+', String),
+ (r'\\', String, 'string-escape'),
+ ],
+ 'single-double-quoted-string': [
+ (r'"', String, 'end-of-string'),
+ (r'[^"\\\n]+', String),
+ (r'\\', String, 'string-escape'),
+ ],
+ 'triple-single-quoted-string': [
+ (r"'''", String, 'end-of-string'),
+ (r'[^\\]+', String),
+ (r'\\', String, 'string-escape'),
+ ],
+ 'single-single-quoted-string': [
+ (r"'", String, 'end-of-string'),
+ (r"[^'\\\n]+", String),
+ (r'\\', String, 'string-escape'),
+ ],
+ 'string-escape': [
+ (r'.', String, '#pop'),
+ ],
+ 'end-of-string': [
+ (r'(@)([a-z]+(:?-[a-z0-9]+)*)',
+ bygroups(Operator, Generic.Emph), '#pop:2'),
+
+ (r'(\^\^)%(IRIREF)s' % patterns, bygroups(Operator, Generic.Emph), '#pop:2'),
+ (r'(\^\^)%(PrefixedName)s' % patterns,
+ bygroups(Operator, Generic.Emph, Generic.Emph), '#pop:2'),
+
+ default('#pop:2'),
+
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/rebol.py b/wandb/vendor/pygments/lexers/rebol.py
new file mode 100644
index 0000000000000000000000000000000000000000..f3d00200d02bb8ae825497b243b93ba8da6f500d
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/rebol.py
@@ -0,0 +1,431 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.rebol
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the REBOL and related languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Generic, Whitespace
+
+__all__ = ['RebolLexer', 'RedLexer']
+
+
+class RebolLexer(RegexLexer):
+ """
+ A `REBOL <http://www.rebol.com/>`_ lexer.
+
+ .. versionadded:: 1.1
+ """
+ name = 'REBOL'
+ aliases = ['rebol']
+ filenames = ['*.r', '*.r3', '*.reb']
+ mimetypes = ['text/x-rebol']
+
+ flags = re.IGNORECASE | re.MULTILINE
+
+ escape_re = r'(?:\^\([0-9a-f]{1,4}\)*)'
+
+ def word_callback(lexer, match):
+ word = match.group()
+
+ if re.match(".*:$", word):
+ yield match.start(), Generic.Subheading, word
+ elif re.match(
+ r'(native|alias|all|any|as-string|as-binary|bind|bound\?|case|'
+ r'catch|checksum|comment|debase|dehex|exclude|difference|disarm|'
+ r'either|else|enbase|foreach|remove-each|form|free|get|get-env|if|'
+ r'in|intersect|loop|minimum-of|maximum-of|mold|new-line|'
+ r'new-line\?|not|now|prin|print|reduce|compose|construct|repeat|'
+ r'reverse|save|script\?|set|shift|switch|throw|to-hex|trace|try|'
+ r'type\?|union|unique|unless|unprotect|unset|until|use|value\?|'
+ r'while|compress|decompress|secure|open|close|read|read-io|'
+ r'write-io|write|update|query|wait|input\?|exp|log-10|log-2|'
+ r'log-e|square-root|cosine|sine|tangent|arccosine|arcsine|'
+ r'arctangent|protect|lowercase|uppercase|entab|detab|connected\?|'
+ r'browse|launch|stats|get-modes|set-modes|to-local-file|'
+ r'to-rebol-file|encloak|decloak|create-link|do-browser|bind\?|'
+ r'hide|draw|show|size-text|textinfo|offset-to-caret|'
+ r'caret-to-offset|local-request-file|rgb-to-hsv|hsv-to-rgb|'
+ r'crypt-strength\?|dh-make-key|dh-generate-key|dh-compute-key|'
+ r'dsa-make-key|dsa-generate-key|dsa-make-signature|'
+ r'dsa-verify-signature|rsa-make-key|rsa-generate-key|'
+ r'rsa-encrypt)$', word):
+ yield match.start(), Name.Builtin, word
+ elif re.match(
+ r'(add|subtract|multiply|divide|remainder|power|and~|or~|xor~|'
+ r'minimum|maximum|negate|complement|absolute|random|head|tail|'
+ r'next|back|skip|at|pick|first|second|third|fourth|fifth|sixth|'
+ r'seventh|eighth|ninth|tenth|last|path|find|select|make|to|copy\*|'
+ r'insert|remove|change|poke|clear|trim|sort|min|max|abs|cp|'
+ r'copy)$', word):
+ yield match.start(), Name.Function, word
+ elif re.match(
+ r'(error|source|input|license|help|install|echo|Usage|with|func|'
+ r'throw-on-error|function|does|has|context|probe|\?\?|as-pair|'
+ r'mod|modulo|round|repend|about|set-net|append|join|rejoin|reform|'
+ r'remold|charset|array|replace|move|extract|forskip|forall|alter|'
+ r'first+|also|take|for|forever|dispatch|attempt|what-dir|'
+ r'change-dir|clean-path|list-dir|dirize|rename|split-path|delete|'
+ r'make-dir|delete-dir|in-dir|confirm|dump-obj|upgrade|what|'
+ r'build-tag|process-source|build-markup|decode-cgi|read-cgi|'
+ r'write-user|save-user|set-user-name|protect-system|parse-xml|'
+ r'cvs-date|cvs-version|do-boot|get-net-info|desktop|layout|'
+ r'scroll-para|get-face|alert|set-face|uninstall|unfocus|'
+ r'request-dir|center-face|do-events|net-error|decode-url|'
+ r'parse-header|parse-header-date|parse-email-addrs|import-email|'
+ r'send|build-attach-body|resend|show-popup|hide-popup|open-events|'
+ r'find-key-face|do-face|viewtop|confine|find-window|'
+ r'insert-event-func|remove-event-func|inform|dump-pane|dump-face|'
+ r'flag-face|deflag-face|clear-fields|read-net|vbug|path-thru|'
+ r'read-thru|load-thru|do-thru|launch-thru|load-image|'
+ r'request-download|do-face-alt|set-font|set-para|get-style|'
+ r'set-style|make-face|stylize|choose|hilight-text|hilight-all|'
+ r'unlight-text|focus|scroll-drag|clear-face|reset-face|scroll-face|'
+ r'resize-face|load-stock|load-stock-block|notify|request|flash|'
+ r'request-color|request-pass|request-text|request-list|'
+ r'request-date|request-file|dbug|editor|link-relative-path|'
+ r'emailer|parse-error)$', word):
+ yield match.start(), Keyword.Namespace, word
+ elif re.match(
+ r'(halt|quit|do|load|q|recycle|call|run|ask|parse|view|unview|'
+ r'return|exit|break)$', word):
+ yield match.start(), Name.Exception, word
+ elif re.match('REBOL$', word):
+ yield match.start(), Generic.Heading, word
+ elif re.match("to-.*", word):
+ yield match.start(), Keyword, word
+ elif re.match('(\+|-|\*|/|//|\*\*|and|or|xor|=\?|=|==|<>|<|>|<=|>=)$',
+ word):
+ yield match.start(), Operator, word
+ elif re.match(".*\?$", word):
+ yield match.start(), Keyword, word
+ elif re.match(".*\!$", word):
+ yield match.start(), Keyword.Type, word
+ elif re.match("'.*", word):
+ yield match.start(), Name.Variable.Instance, word # lit-word
+ elif re.match("#.*", word):
+ yield match.start(), Name.Label, word # issue
+ elif re.match("%.*", word):
+ yield match.start(), Name.Decorator, word # file
+ else:
+ yield match.start(), Name.Variable, word
+
+ tokens = {
+ 'root': [
+ (r'[^R]+', Comment),
+ (r'REBOL\s+\[', Generic.Strong, 'script'),
+ (r'R', Comment)
+ ],
+ 'script': [
+ (r'\s+', Text),
+ (r'#"', String.Char, 'char'),
+ (r'#\{[0-9a-f]*\}', Number.Hex),
+ (r'2#\{', Number.Hex, 'bin2'),
+ (r'64#\{[0-9a-z+/=\s]*\}', Number.Hex),
+ (r'"', String, 'string'),
+ (r'\{', String, 'string2'),
+ (r';#+.*\n', Comment.Special),
+ (r';\*+.*\n', Comment.Preproc),
+ (r';.*\n', Comment),
+ (r'%"', Name.Decorator, 'stringFile'),
+ (r'%[^(^{")\s\[\]]+', Name.Decorator),
+ (r'[+-]?([a-z]{1,3})?\$\d+(\.\d+)?', Number.Float), # money
+ (r'[+-]?\d+\:\d+(\:\d+)?(\.\d+)?', String.Other), # time
+ (r'\d+[\-/][0-9a-z]+[\-/]\d+(\/\d+\:\d+((\:\d+)?'
+ r'([.\d+]?([+-]?\d+:\d+)?)?)?)?', String.Other), # date
+ (r'\d+(\.\d+)+\.\d+', Keyword.Constant), # tuple
+ (r'\d+X\d+', Keyword.Constant), # pair
+ (r'[+-]?\d+(\'\d+)?([.,]\d*)?E[+-]?\d+', Number.Float),
+ (r'[+-]?\d+(\'\d+)?[.,]\d*', Number.Float),
+ (r'[+-]?\d+(\'\d+)?', Number),
+ (r'[\[\]()]', Generic.Strong),
+ (r'[a-z]+[^(^{"\s:)]*://[^(^{"\s)]*', Name.Decorator), # url
+ (r'mailto:[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator), # url
+ (r'[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator), # email
+ (r'comment\s"', Comment, 'commentString1'),
+ (r'comment\s\{', Comment, 'commentString2'),
+ (r'comment\s\[', Comment, 'commentBlock'),
+ (r'comment\s[^(\s{"\[]+', Comment),
+ (r'/[^(^{")\s/[\]]*', Name.Attribute),
+ (r'([^(^{")\s/[\]]+)(?=[:({"\s/\[\]])', word_callback),
+ (r'<[\w:.-]*>', Name.Tag),
+ (r'<[^(<>\s")]+', Name.Tag, 'tag'),
+ (r'([^(^{")\s]+)', Text),
+ ],
+ 'string': [
+ (r'[^(^")]+', String),
+ (escape_re, String.Escape),
+ (r'[(|)]+', String),
+ (r'\^.', String.Escape),
+ (r'"', String, '#pop'),
+ ],
+ 'string2': [
+ (r'[^(^{})]+', String),
+ (escape_re, String.Escape),
+ (r'[(|)]+', String),
+ (r'\^.', String.Escape),
+ (r'\{', String, '#push'),
+ (r'\}', String, '#pop'),
+ ],
+ 'stringFile': [
+ (r'[^(^")]+', Name.Decorator),
+ (escape_re, Name.Decorator),
+ (r'\^.', Name.Decorator),
+ (r'"', Name.Decorator, '#pop'),
+ ],
+ 'char': [
+ (escape_re + '"', String.Char, '#pop'),
+ (r'\^."', String.Char, '#pop'),
+ (r'."', String.Char, '#pop'),
+ ],
+ 'tag': [
+ (escape_re, Name.Tag),
+ (r'"', Name.Tag, 'tagString'),
+ (r'[^(<>\r\n")]+', Name.Tag),
+ (r'>', Name.Tag, '#pop'),
+ ],
+ 'tagString': [
+ (r'[^(^")]+', Name.Tag),
+ (escape_re, Name.Tag),
+ (r'[(|)]+', Name.Tag),
+ (r'\^.', Name.Tag),
+ (r'"', Name.Tag, '#pop'),
+ ],
+ 'tuple': [
+ (r'(\d+\.)+', Keyword.Constant),
+ (r'\d+', Keyword.Constant, '#pop'),
+ ],
+ 'bin2': [
+ (r'\s+', Number.Hex),
+ (r'([01]\s*){8}', Number.Hex),
+ (r'\}', Number.Hex, '#pop'),
+ ],
+ 'commentString1': [
+ (r'[^(^")]+', Comment),
+ (escape_re, Comment),
+ (r'[(|)]+', Comment),
+ (r'\^.', Comment),
+ (r'"', Comment, '#pop'),
+ ],
+ 'commentString2': [
+ (r'[^(^{})]+', Comment),
+ (escape_re, Comment),
+ (r'[(|)]+', Comment),
+ (r'\^.', Comment),
+ (r'\{', Comment, '#push'),
+ (r'\}', Comment, '#pop'),
+ ],
+ 'commentBlock': [
+ (r'\[', Comment, '#push'),
+ (r'\]', Comment, '#pop'),
+ (r'"', Comment, "commentString1"),
+ (r'\{', Comment, "commentString2"),
+ (r'[^(\[\]"{)]+', Comment),
+ ],
+ }
+
+ def analyse_text(text):
+ """
+ Check if code contains REBOL header and so it probably not R code
+ """
+ if re.match(r'^\s*REBOL\s*\[', text, re.IGNORECASE):
+ # The code starts with REBOL header
+ return 1.0
+ elif re.search(r'\s*REBOL\s*[', text, re.IGNORECASE):
+ # The code contains REBOL header but also some text before it
+ return 0.5
+
+
+class RedLexer(RegexLexer):
+ """
+ A `Red-language <http://www.red-lang.org/>`_ lexer.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Red'
+ aliases = ['red', 'red/system']
+ filenames = ['*.red', '*.reds']
+ mimetypes = ['text/x-red', 'text/x-red-system']
+
+ flags = re.IGNORECASE | re.MULTILINE
+
+ escape_re = r'(?:\^\([0-9a-f]{1,4}\)*)'
+
+ def word_callback(lexer, match):
+ word = match.group()
+
+ if re.match(".*:$", word):
+ yield match.start(), Generic.Subheading, word
+ elif re.match(r'(if|unless|either|any|all|while|until|loop|repeat|'
+ r'foreach|forall|func|function|does|has|switch|'
+ r'case|reduce|compose|get|set|print|prin|equal\?|'
+ r'not-equal\?|strict-equal\?|lesser\?|greater\?|lesser-or-equal\?|'
+ r'greater-or-equal\?|same\?|not|type\?|stats|'
+ r'bind|union|replace|charset|routine)$', word):
+ yield match.start(), Name.Builtin, word
+ elif re.match(r'(make|random|reflect|to|form|mold|absolute|add|divide|multiply|negate|'
+ r'power|remainder|round|subtract|even\?|odd\?|and~|complement|or~|xor~|'
+ r'append|at|back|change|clear|copy|find|head|head\?|index\?|insert|'
+ r'length\?|next|pick|poke|remove|reverse|select|sort|skip|swap|tail|tail\?|'
+ r'take|trim|create|close|delete|modify|open|open\?|query|read|rename|'
+ r'update|write)$', word):
+ yield match.start(), Name.Function, word
+ elif re.match(r'(yes|on|no|off|true|false|tab|cr|lf|newline|escape|slash|sp|space|null|'
+ r'none|crlf|dot|null-byte)$', word):
+ yield match.start(), Name.Builtin.Pseudo, word
+ elif re.match(r'(#system-global|#include|#enum|#define|#either|#if|#import|#export|'
+ r'#switch|#default|#get-definition)$', word):
+ yield match.start(), Keyword.Namespace, word
+ elif re.match(r'(system|halt|quit|quit-return|do|load|q|recycle|call|run|ask|parse|'
+ r'raise-error|return|exit|break|alias|push|pop|probe|\?\?|spec-of|body-of|'
+ r'quote|forever)$', word):
+ yield match.start(), Name.Exception, word
+ elif re.match(r'(action\?|block\?|char\?|datatype\?|file\?|function\?|get-path\?|zero\?|'
+ r'get-word\?|integer\?|issue\?|lit-path\?|lit-word\?|logic\?|native\?|'
+ r'op\?|paren\?|path\?|refinement\?|set-path\?|set-word\?|string\?|unset\?|'
+ r'any-struct\?|none\?|word\?|any-series\?)$', word):
+ yield match.start(), Keyword, word
+ elif re.match(r'(JNICALL|stdcall|cdecl|infix)$', word):
+ yield match.start(), Keyword.Namespace, word
+ elif re.match("to-.*", word):
+ yield match.start(), Keyword, word
+ elif re.match('(\+|-\*\*|-|\*\*|//|/|\*|and|or|xor|=\?|===|==|=|<>|<=|>=|'
+ '<<<|>>>|<<|>>|<|>%)$', word):
+ yield match.start(), Operator, word
+ elif re.match(".*\!$", word):
+ yield match.start(), Keyword.Type, word
+ elif re.match("'.*", word):
+ yield match.start(), Name.Variable.Instance, word # lit-word
+ elif re.match("#.*", word):
+ yield match.start(), Name.Label, word # issue
+ elif re.match("%.*", word):
+ yield match.start(), Name.Decorator, word # file
+ elif re.match(":.*", word):
+ yield match.start(), Generic.Subheading, word # get-word
+ else:
+ yield match.start(), Name.Variable, word
+
+ tokens = {
+ 'root': [
+ (r'[^R]+', Comment),
+ (r'Red/System\s+\[', Generic.Strong, 'script'),
+ (r'Red\s+\[', Generic.Strong, 'script'),
+ (r'R', Comment)
+ ],
+ 'script': [
+ (r'\s+', Text),
+ (r'#"', String.Char, 'char'),
+ (r'#\{[0-9a-f\s]*\}', Number.Hex),
+ (r'2#\{', Number.Hex, 'bin2'),
+ (r'64#\{[0-9a-z+/=\s]*\}', Number.Hex),
+ (r'([0-9a-f]+)(h)((\s)|(?=[\[\]{}"()]))',
+ bygroups(Number.Hex, Name.Variable, Whitespace)),
+ (r'"', String, 'string'),
+ (r'\{', String, 'string2'),
+ (r';#+.*\n', Comment.Special),
+ (r';\*+.*\n', Comment.Preproc),
+ (r';.*\n', Comment),
+ (r'%"', Name.Decorator, 'stringFile'),
+ (r'%[^(^{")\s\[\]]+', Name.Decorator),
+ (r'[+-]?([a-z]{1,3})?\$\d+(\.\d+)?', Number.Float), # money
+ (r'[+-]?\d+\:\d+(\:\d+)?(\.\d+)?', String.Other), # time
+ (r'\d+[\-/][0-9a-z]+[\-/]\d+(/\d+:\d+((:\d+)?'
+ r'([\.\d+]?([+-]?\d+:\d+)?)?)?)?', String.Other), # date
+ (r'\d+(\.\d+)+\.\d+', Keyword.Constant), # tuple
+ (r'\d+X\d+', Keyword.Constant), # pair
+ (r'[+-]?\d+(\'\d+)?([.,]\d*)?E[+-]?\d+', Number.Float),
+ (r'[+-]?\d+(\'\d+)?[.,]\d*', Number.Float),
+ (r'[+-]?\d+(\'\d+)?', Number),
+ (r'[\[\]()]', Generic.Strong),
+ (r'[a-z]+[^(^{"\s:)]*://[^(^{"\s)]*', Name.Decorator), # url
+ (r'mailto:[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator), # url
+ (r'[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator), # email
+ (r'comment\s"', Comment, 'commentString1'),
+ (r'comment\s\{', Comment, 'commentString2'),
+ (r'comment\s\[', Comment, 'commentBlock'),
+ (r'comment\s[^(\s{"\[]+', Comment),
+ (r'/[^(^{^")\s/[\]]*', Name.Attribute),
+ (r'([^(^{^")\s/[\]]+)(?=[:({"\s/\[\]])', word_callback),
+ (r'<[\w:.-]*>', Name.Tag),
+ (r'<[^(<>\s")]+', Name.Tag, 'tag'),
+ (r'([^(^{")\s]+)', Text),
+ ],
+ 'string': [
+ (r'[^(^")]+', String),
+ (escape_re, String.Escape),
+ (r'[(|)]+', String),
+ (r'\^.', String.Escape),
+ (r'"', String, '#pop'),
+ ],
+ 'string2': [
+ (r'[^(^{})]+', String),
+ (escape_re, String.Escape),
+ (r'[(|)]+', String),
+ (r'\^.', String.Escape),
+ (r'\{', String, '#push'),
+ (r'\}', String, '#pop'),
+ ],
+ 'stringFile': [
+ (r'[^(^")]+', Name.Decorator),
+ (escape_re, Name.Decorator),
+ (r'\^.', Name.Decorator),
+ (r'"', Name.Decorator, '#pop'),
+ ],
+ 'char': [
+ (escape_re + '"', String.Char, '#pop'),
+ (r'\^."', String.Char, '#pop'),
+ (r'."', String.Char, '#pop'),
+ ],
+ 'tag': [
+ (escape_re, Name.Tag),
+ (r'"', Name.Tag, 'tagString'),
+ (r'[^(<>\r\n")]+', Name.Tag),
+ (r'>', Name.Tag, '#pop'),
+ ],
+ 'tagString': [
+ (r'[^(^")]+', Name.Tag),
+ (escape_re, Name.Tag),
+ (r'[(|)]+', Name.Tag),
+ (r'\^.', Name.Tag),
+ (r'"', Name.Tag, '#pop'),
+ ],
+ 'tuple': [
+ (r'(\d+\.)+', Keyword.Constant),
+ (r'\d+', Keyword.Constant, '#pop'),
+ ],
+ 'bin2': [
+ (r'\s+', Number.Hex),
+ (r'([01]\s*){8}', Number.Hex),
+ (r'\}', Number.Hex, '#pop'),
+ ],
+ 'commentString1': [
+ (r'[^(^")]+', Comment),
+ (escape_re, Comment),
+ (r'[(|)]+', Comment),
+ (r'\^.', Comment),
+ (r'"', Comment, '#pop'),
+ ],
+ 'commentString2': [
+ (r'[^(^{})]+', Comment),
+ (escape_re, Comment),
+ (r'[(|)]+', Comment),
+ (r'\^.', Comment),
+ (r'\{', Comment, '#push'),
+ (r'\}', Comment, '#pop'),
+ ],
+ 'commentBlock': [
+ (r'\[', Comment, '#push'),
+ (r'\]', Comment, '#pop'),
+ (r'"', Comment, "commentString1"),
+ (r'\{', Comment, "commentString2"),
+ (r'[^(\[\]"{)]+', Comment),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/resource.py b/wandb/vendor/pygments/lexers/resource.py
new file mode 100644
index 0000000000000000000000000000000000000000..f7494904769ee25ccb089ce8adcbe328d527300e
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/resource.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.resource
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for resource definition files.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Comment, String, Number, Operator, Text, \
+ Keyword, Name
+
+__all__ = ['ResourceLexer']
+
+
+class ResourceLexer(RegexLexer):
+ """Lexer for `ICU Resource bundles
+ <http://userguide.icu-project.org/locale/resources>`_.
+
+ .. versionadded:: 2.0
+ """
+ name = 'ResourceBundle'
+ aliases = ['resource', 'resourcebundle']
+ filenames = ['*.txt']
+
+ _types = (':table', ':array', ':string', ':bin', ':import', ':intvector',
+ ':int', ':alias')
+
+ flags = re.MULTILINE | re.IGNORECASE
+ tokens = {
+ 'root': [
+ (r'//.*?$', Comment),
+ (r'"', String, 'string'),
+ (r'-?\d+', Number.Integer),
+ (r'[,{}]', Operator),
+ (r'([^\s{:]+)(\s*)(%s?)' % '|'.join(_types),
+ bygroups(Name, Text, Keyword)),
+ (r'\s+', Text),
+ (words(_types), Keyword),
+ ],
+ 'string': [
+ (r'(\\x[0-9a-f]{2}|\\u[0-9a-f]{4}|\\U00[0-9a-f]{6}|'
+ r'\\[0-7]{1,3}|\\c.|\\[abtnvfre\'"?\\]|\\\{|[^"{\\])+', String),
+ (r'\{', String.Escape, 'msgname'),
+ (r'"', String, '#pop')
+ ],
+ 'msgname': [
+ (r'([^{},]+)(\s*)', bygroups(Name, String.Escape), ('#pop', 'message'))
+ ],
+ 'message': [
+ (r'\{', String.Escape, 'msgname'),
+ (r'\}', String.Escape, '#pop'),
+ (r'(,)(\s*)([a-z]+)(\s*\})',
+ bygroups(Operator, String.Escape, Keyword, String.Escape), '#pop'),
+ (r'(,)(\s*)([a-z]+)(\s*)(,)(\s*)(offset)(\s*)(:)(\s*)(-?\d+)(\s*)',
+ bygroups(Operator, String.Escape, Keyword, String.Escape, Operator,
+ String.Escape, Operator.Word, String.Escape, Operator,
+ String.Escape, Number.Integer, String.Escape), 'choice'),
+ (r'(,)(\s*)([a-z]+)(\s*)(,)(\s*)',
+ bygroups(Operator, String.Escape, Keyword, String.Escape, Operator,
+ String.Escape), 'choice'),
+ (r'\s+', String.Escape)
+ ],
+ 'choice': [
+ (r'(=|<|>|<=|>=|!=)(-?\d+)(\s*\{)',
+ bygroups(Operator, Number.Integer, String.Escape), 'message'),
+ (r'([a-z]+)(\s*\{)', bygroups(Keyword.Type, String.Escape), 'str'),
+ (r'\}', String.Escape, ('#pop', '#pop')),
+ (r'\s+', String.Escape)
+ ],
+ 'str': [
+ (r'\}', String.Escape, '#pop'),
+ (r'\{', String.Escape, 'msgname'),
+ (r'[^{}]+', String)
+ ]
+ }
+
+ def analyse_text(text):
+ if text.startswith('root:table'):
+ return 1.0
diff --git a/wandb/vendor/pygments/lexers/rnc.py b/wandb/vendor/pygments/lexers/rnc.py
new file mode 100644
index 0000000000000000000000000000000000000000..2f2aacdd25a6be42cfc00728496a57f870376fd5
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/rnc.py
@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.rnc
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexer for Relax-NG Compact syntax
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Punctuation
+
+__all__ = ['RNCCompactLexer']
+
+
+class RNCCompactLexer(RegexLexer):
+ """
+ For `RelaxNG-compact <http://relaxng.org>`_ syntax.
+
+ .. versionadded:: 2.2
+ """
+
+ name = 'Relax-NG Compact'
+ aliases = ['rnc', 'rng-compact']
+ filenames = ['*.rnc']
+
+ tokens = {
+ 'root': [
+ (r'namespace\b', Keyword.Namespace),
+ (r'(?:default|datatypes)\b', Keyword.Declaration),
+ (r'##.*$', Comment.Preproc),
+ (r'#.*$', Comment.Single),
+ (r'"[^"]*"', String.Double),
+ # TODO single quoted strings and escape sequences outside of
+ # double-quoted strings
+ (r'(?:element|attribute|mixed)\b', Keyword.Declaration, 'variable'),
+ (r'(text\b|xsd:[^ ]+)', Keyword.Type, 'maybe_xsdattributes'),
+ (r'[,?&*=|~]|>>', Operator),
+ (r'[(){}]', Punctuation),
+ (r'.', Text),
+ ],
+
+ # a variable has been declared using `element` or `attribute`
+ 'variable': [
+ (r'[^{]+', Name.Variable),
+ (r'\{', Punctuation, '#pop'),
+ ],
+
+ # after an xsd:<datatype> declaration there may be attributes
+ 'maybe_xsdattributes': [
+ (r'\{', Punctuation, 'xsdattributes'),
+ (r'\}', Punctuation, '#pop'),
+ (r'.', Text),
+ ],
+
+ # attributes take the form { key1 = value1 key2 = value2 ... }
+ 'xsdattributes': [
+ (r'[^ =}]', Name.Attribute),
+ (r'=', Operator),
+ (r'"[^"]*"', String.Double),
+ (r'\}', Punctuation, '#pop'),
+ (r'.', Text),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/roboconf.py b/wandb/vendor/pygments/lexers/roboconf.py
new file mode 100644
index 0000000000000000000000000000000000000000..8c7df83db30f1f2442de738dd59a96a45200b9a8
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/roboconf.py
@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.roboconf
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Roboconf DSL.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words, re
+from pygments.token import Text, Operator, Keyword, Name, Comment
+
+__all__ = ['RoboconfGraphLexer', 'RoboconfInstancesLexer']
+
+
+class RoboconfGraphLexer(RegexLexer):
+ """
+ Lexer for `Roboconf <http://roboconf.net/en/roboconf.html>`_ graph files.
+
+ .. versionadded:: 2.1
+ """
+ name = 'Roboconf Graph'
+ aliases = ['roboconf-graph']
+ filenames = ['*.graph']
+
+ flags = re.IGNORECASE | re.MULTILINE
+ tokens = {
+ 'root': [
+ # Skip white spaces
+ (r'\s+', Text),
+
+ # There is one operator
+ (r'=', Operator),
+
+ # Keywords
+ (words(('facet', 'import'), suffix=r'\s*\b', prefix=r'\b'), Keyword),
+ (words((
+ 'installer', 'extends', 'exports', 'imports', 'facets',
+ 'children'), suffix=r'\s*:?', prefix=r'\b'), Name),
+
+ # Comments
+ (r'#.*\n', Comment),
+
+ # Default
+ (r'[^#]', Text),
+ (r'.*\n', Text)
+ ]
+ }
+
+
+class RoboconfInstancesLexer(RegexLexer):
+ """
+ Lexer for `Roboconf <http://roboconf.net/en/roboconf.html>`_ instances files.
+
+ .. versionadded:: 2.1
+ """
+ name = 'Roboconf Instances'
+ aliases = ['roboconf-instances']
+ filenames = ['*.instances']
+
+ flags = re.IGNORECASE | re.MULTILINE
+ tokens = {
+ 'root': [
+
+ # Skip white spaces
+ (r'\s+', Text),
+
+ # Keywords
+ (words(('instance of', 'import'), suffix=r'\s*\b', prefix=r'\b'), Keyword),
+ (words(('name', 'count'), suffix=r's*:?', prefix=r'\b'), Name),
+ (r'\s*[\w.-]+\s*:', Name),
+
+ # Comments
+ (r'#.*\n', Comment),
+
+ # Default
+ (r'[^#]', Text),
+ (r'.*\n', Text)
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/robotframework.py b/wandb/vendor/pygments/lexers/robotframework.py
new file mode 100644
index 0000000000000000000000000000000000000000..e868127b68a5869d9e5b2eb1b6224f22c6ef797d
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/robotframework.py
@@ -0,0 +1,560 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.robotframework
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for Robot Framework.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+# Copyright 2012 Nokia Siemens Networks Oyj
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+from pygments.lexer import Lexer
+from pygments.token import Token
+from pygments.util import text_type
+
+__all__ = ['RobotFrameworkLexer']
+
+
+HEADING = Token.Generic.Heading
+SETTING = Token.Keyword.Namespace
+IMPORT = Token.Name.Namespace
+TC_KW_NAME = Token.Generic.Subheading
+KEYWORD = Token.Name.Function
+ARGUMENT = Token.String
+VARIABLE = Token.Name.Variable
+COMMENT = Token.Comment
+SEPARATOR = Token.Punctuation
+SYNTAX = Token.Punctuation
+GHERKIN = Token.Generic.Emph
+ERROR = Token.Error
+
+
+def normalize(string, remove=''):
+ string = string.lower()
+ for char in remove + ' ':
+ if char in string:
+ string = string.replace(char, '')
+ return string
+
+
+class RobotFrameworkLexer(Lexer):
+ """
+ For `Robot Framework <http://robotframework.org>`_ test data.
+
+ Supports both space and pipe separated plain text formats.
+
+ .. versionadded:: 1.6
+ """
+ name = 'RobotFramework'
+ aliases = ['robotframework']
+ filenames = ['*.txt', '*.robot']
+ mimetypes = ['text/x-robotframework']
+
+ def __init__(self, **options):
+ options['tabsize'] = 2
+ options['encoding'] = 'UTF-8'
+ Lexer.__init__(self, **options)
+
+ def get_tokens_unprocessed(self, text):
+ row_tokenizer = RowTokenizer()
+ var_tokenizer = VariableTokenizer()
+ index = 0
+ for row in text.splitlines():
+ for value, token in row_tokenizer.tokenize(row):
+ for value, token in var_tokenizer.tokenize(value, token):
+ if value:
+ yield index, token, text_type(value)
+ index += len(value)
+
+
+class VariableTokenizer(object):
+
+ def tokenize(self, string, token):
+ var = VariableSplitter(string, identifiers='$@%&')
+ if var.start < 0 or token in (COMMENT, ERROR):
+ yield string, token
+ return
+ for value, token in self._tokenize(var, string, token):
+ if value:
+ yield value, token
+
+ def _tokenize(self, var, string, orig_token):
+ before = string[:var.start]
+ yield before, orig_token
+ yield var.identifier + '{', SYNTAX
+ for value, token in self.tokenize(var.base, VARIABLE):
+ yield value, token
+ yield '}', SYNTAX
+ if var.index:
+ yield '[', SYNTAX
+ for value, token in self.tokenize(var.index, VARIABLE):
+ yield value, token
+ yield ']', SYNTAX
+ for value, token in self.tokenize(string[var.end:], orig_token):
+ yield value, token
+
+
+class RowTokenizer(object):
+
+ def __init__(self):
+ self._table = UnknownTable()
+ self._splitter = RowSplitter()
+ testcases = TestCaseTable()
+ settings = SettingTable(testcases.set_default_template)
+ variables = VariableTable()
+ keywords = KeywordTable()
+ self._tables = {'settings': settings, 'setting': settings,
+ 'metadata': settings,
+ 'variables': variables, 'variable': variables,
+ 'testcases': testcases, 'testcase': testcases,
+ 'keywords': keywords, 'keyword': keywords,
+ 'userkeywords': keywords, 'userkeyword': keywords}
+
+ def tokenize(self, row):
+ commented = False
+ heading = False
+ for index, value in enumerate(self._splitter.split(row)):
+ # First value, and every second after that, is a separator.
+ index, separator = divmod(index-1, 2)
+ if value.startswith('#'):
+ commented = True
+ elif index == 0 and value.startswith('*'):
+ self._table = self._start_table(value)
+ heading = True
+ for value, token in self._tokenize(value, index, commented,
+ separator, heading):
+ yield value, token
+ self._table.end_row()
+
+ def _start_table(self, header):
+ name = normalize(header, remove='*')
+ return self._tables.get(name, UnknownTable())
+
+ def _tokenize(self, value, index, commented, separator, heading):
+ if commented:
+ yield value, COMMENT
+ elif separator:
+ yield value, SEPARATOR
+ elif heading:
+ yield value, HEADING
+ else:
+ for value, token in self._table.tokenize(value, index):
+ yield value, token
+
+
+class RowSplitter(object):
+ _space_splitter = re.compile('( {2,})')
+ _pipe_splitter = re.compile('((?:^| +)\|(?: +|$))')
+
+ def split(self, row):
+ splitter = (row.startswith('| ') and self._split_from_pipes
+ or self._split_from_spaces)
+ for value in splitter(row):
+ yield value
+ yield '\n'
+
+ def _split_from_spaces(self, row):
+ yield '' # Start with (pseudo)separator similarly as with pipes
+ for value in self._space_splitter.split(row):
+ yield value
+
+ def _split_from_pipes(self, row):
+ _, separator, rest = self._pipe_splitter.split(row, 1)
+ yield separator
+ while self._pipe_splitter.search(rest):
+ cell, separator, rest = self._pipe_splitter.split(rest, 1)
+ yield cell
+ yield separator
+ yield rest
+
+
+class Tokenizer(object):
+ _tokens = None
+
+ def __init__(self):
+ self._index = 0
+
+ def tokenize(self, value):
+ values_and_tokens = self._tokenize(value, self._index)
+ self._index += 1
+ if isinstance(values_and_tokens, type(Token)):
+ values_and_tokens = [(value, values_and_tokens)]
+ return values_and_tokens
+
+ def _tokenize(self, value, index):
+ index = min(index, len(self._tokens) - 1)
+ return self._tokens[index]
+
+ def _is_assign(self, value):
+ if value.endswith('='):
+ value = value[:-1].strip()
+ var = VariableSplitter(value, identifiers='$@&')
+ return var.start == 0 and var.end == len(value)
+
+
+class Comment(Tokenizer):
+ _tokens = (COMMENT,)
+
+
+class Setting(Tokenizer):
+ _tokens = (SETTING, ARGUMENT)
+ _keyword_settings = ('suitesetup', 'suiteprecondition', 'suiteteardown',
+ 'suitepostcondition', 'testsetup', 'testprecondition',
+ 'testteardown', 'testpostcondition', 'testtemplate')
+ _import_settings = ('library', 'resource', 'variables')
+ _other_settings = ('documentation', 'metadata', 'forcetags', 'defaulttags',
+ 'testtimeout')
+ _custom_tokenizer = None
+
+ def __init__(self, template_setter=None):
+ Tokenizer.__init__(self)
+ self._template_setter = template_setter
+
+ def _tokenize(self, value, index):
+ if index == 1 and self._template_setter:
+ self._template_setter(value)
+ if index == 0:
+ normalized = normalize(value)
+ if normalized in self._keyword_settings:
+ self._custom_tokenizer = KeywordCall(support_assign=False)
+ elif normalized in self._import_settings:
+ self._custom_tokenizer = ImportSetting()
+ elif normalized not in self._other_settings:
+ return ERROR
+ elif self._custom_tokenizer:
+ return self._custom_tokenizer.tokenize(value)
+ return Tokenizer._tokenize(self, value, index)
+
+
+class ImportSetting(Tokenizer):
+ _tokens = (IMPORT, ARGUMENT)
+
+
+class TestCaseSetting(Setting):
+ _keyword_settings = ('setup', 'precondition', 'teardown', 'postcondition',
+ 'template')
+ _import_settings = ()
+ _other_settings = ('documentation', 'tags', 'timeout')
+
+ def _tokenize(self, value, index):
+ if index == 0:
+ type = Setting._tokenize(self, value[1:-1], index)
+ return [('[', SYNTAX), (value[1:-1], type), (']', SYNTAX)]
+ return Setting._tokenize(self, value, index)
+
+
+class KeywordSetting(TestCaseSetting):
+ _keyword_settings = ('teardown',)
+ _other_settings = ('documentation', 'arguments', 'return', 'timeout', 'tags')
+
+
+class Variable(Tokenizer):
+ _tokens = (SYNTAX, ARGUMENT)
+
+ def _tokenize(self, value, index):
+ if index == 0 and not self._is_assign(value):
+ return ERROR
+ return Tokenizer._tokenize(self, value, index)
+
+
+class KeywordCall(Tokenizer):
+ _tokens = (KEYWORD, ARGUMENT)
+
+ def __init__(self, support_assign=True):
+ Tokenizer.__init__(self)
+ self._keyword_found = not support_assign
+ self._assigns = 0
+
+ def _tokenize(self, value, index):
+ if not self._keyword_found and self._is_assign(value):
+ self._assigns += 1
+ return SYNTAX # VariableTokenizer tokenizes this later.
+ if self._keyword_found:
+ return Tokenizer._tokenize(self, value, index - self._assigns)
+ self._keyword_found = True
+ return GherkinTokenizer().tokenize(value, KEYWORD)
+
+
+class GherkinTokenizer(object):
+ _gherkin_prefix = re.compile('^(Given|When|Then|And) ', re.IGNORECASE)
+
+ def tokenize(self, value, token):
+ match = self._gherkin_prefix.match(value)
+ if not match:
+ return [(value, token)]
+ end = match.end()
+ return [(value[:end], GHERKIN), (value[end:], token)]
+
+
+class TemplatedKeywordCall(Tokenizer):
+ _tokens = (ARGUMENT,)
+
+
+class ForLoop(Tokenizer):
+
+ def __init__(self):
+ Tokenizer.__init__(self)
+ self._in_arguments = False
+
+ def _tokenize(self, value, index):
+ token = self._in_arguments and ARGUMENT or SYNTAX
+ if value.upper() in ('IN', 'IN RANGE'):
+ self._in_arguments = True
+ return token
+
+
+class _Table(object):
+ _tokenizer_class = None
+
+ def __init__(self, prev_tokenizer=None):
+ self._tokenizer = self._tokenizer_class()
+ self._prev_tokenizer = prev_tokenizer
+ self._prev_values_on_row = []
+
+ def tokenize(self, value, index):
+ if self._continues(value, index):
+ self._tokenizer = self._prev_tokenizer
+ yield value, SYNTAX
+ else:
+ for value_and_token in self._tokenize(value, index):
+ yield value_and_token
+ self._prev_values_on_row.append(value)
+
+ def _continues(self, value, index):
+ return value == '...' and all(self._is_empty(t)
+ for t in self._prev_values_on_row)
+
+ def _is_empty(self, value):
+ return value in ('', '\\')
+
+ def _tokenize(self, value, index):
+ return self._tokenizer.tokenize(value)
+
+ def end_row(self):
+ self.__init__(prev_tokenizer=self._tokenizer)
+
+
+class UnknownTable(_Table):
+ _tokenizer_class = Comment
+
+ def _continues(self, value, index):
+ return False
+
+
+class VariableTable(_Table):
+ _tokenizer_class = Variable
+
+
+class SettingTable(_Table):
+ _tokenizer_class = Setting
+
+ def __init__(self, template_setter, prev_tokenizer=None):
+ _Table.__init__(self, prev_tokenizer)
+ self._template_setter = template_setter
+
+ def _tokenize(self, value, index):
+ if index == 0 and normalize(value) == 'testtemplate':
+ self._tokenizer = Setting(self._template_setter)
+ return _Table._tokenize(self, value, index)
+
+ def end_row(self):
+ self.__init__(self._template_setter, prev_tokenizer=self._tokenizer)
+
+
+class TestCaseTable(_Table):
+ _setting_class = TestCaseSetting
+ _test_template = None
+ _default_template = None
+
+ @property
+ def _tokenizer_class(self):
+ if self._test_template or (self._default_template and
+ self._test_template is not False):
+ return TemplatedKeywordCall
+ return KeywordCall
+
+ def _continues(self, value, index):
+ return index > 0 and _Table._continues(self, value, index)
+
+ def _tokenize(self, value, index):
+ if index == 0:
+ if value:
+ self._test_template = None
+ return GherkinTokenizer().tokenize(value, TC_KW_NAME)
+ if index == 1 and self._is_setting(value):
+ if self._is_template(value):
+ self._test_template = False
+ self._tokenizer = self._setting_class(self.set_test_template)
+ else:
+ self._tokenizer = self._setting_class()
+ if index == 1 and self._is_for_loop(value):
+ self._tokenizer = ForLoop()
+ if index == 1 and self._is_empty(value):
+ return [(value, SYNTAX)]
+ return _Table._tokenize(self, value, index)
+
+ def _is_setting(self, value):
+ return value.startswith('[') and value.endswith(']')
+
+ def _is_template(self, value):
+ return normalize(value) == '[template]'
+
+ def _is_for_loop(self, value):
+ return value.startswith(':') and normalize(value, remove=':') == 'for'
+
+ def set_test_template(self, template):
+ self._test_template = self._is_template_set(template)
+
+ def set_default_template(self, template):
+ self._default_template = self._is_template_set(template)
+
+ def _is_template_set(self, template):
+ return normalize(template) not in ('', '\\', 'none', '${empty}')
+
+
+class KeywordTable(TestCaseTable):
+ _tokenizer_class = KeywordCall
+ _setting_class = KeywordSetting
+
+ def _is_template(self, value):
+ return False
+
+
+# Following code copied directly from Robot Framework 2.7.5.
+
+class VariableSplitter:
+
+ def __init__(self, string, identifiers):
+ self.identifier = None
+ self.base = None
+ self.index = None
+ self.start = -1
+ self.end = -1
+ self._identifiers = identifiers
+ self._may_have_internal_variables = False
+ try:
+ self._split(string)
+ except ValueError:
+ pass
+ else:
+ self._finalize()
+
+ def get_replaced_base(self, variables):
+ if self._may_have_internal_variables:
+ return variables.replace_string(self.base)
+ return self.base
+
+ def _finalize(self):
+ self.identifier = self._variable_chars[0]
+ self.base = ''.join(self._variable_chars[2:-1])
+ self.end = self.start + len(self._variable_chars)
+ if self._has_list_or_dict_variable_index():
+ self.index = ''.join(self._list_and_dict_variable_index_chars[1:-1])
+ self.end += len(self._list_and_dict_variable_index_chars)
+
+ def _has_list_or_dict_variable_index(self):
+ return self._list_and_dict_variable_index_chars\
+ and self._list_and_dict_variable_index_chars[-1] == ']'
+
+ def _split(self, string):
+ start_index, max_index = self._find_variable(string)
+ self.start = start_index
+ self._open_curly = 1
+ self._state = self._variable_state
+ self._variable_chars = [string[start_index], '{']
+ self._list_and_dict_variable_index_chars = []
+ self._string = string
+ start_index += 2
+ for index, char in enumerate(string[start_index:]):
+ index += start_index # Giving start to enumerate only in Py 2.6+
+ try:
+ self._state(char, index)
+ except StopIteration:
+ return
+ if index == max_index and not self._scanning_list_variable_index():
+ return
+
+ def _scanning_list_variable_index(self):
+ return self._state in [self._waiting_list_variable_index_state,
+ self._list_variable_index_state]
+
+ def _find_variable(self, string):
+ max_end_index = string.rfind('}')
+ if max_end_index == -1:
+ raise ValueError('No variable end found')
+ if self._is_escaped(string, max_end_index):
+ return self._find_variable(string[:max_end_index])
+ start_index = self._find_start_index(string, 1, max_end_index)
+ if start_index == -1:
+ raise ValueError('No variable start found')
+ return start_index, max_end_index
+
+ def _find_start_index(self, string, start, end):
+ index = string.find('{', start, end) - 1
+ if index < 0:
+ return -1
+ if self._start_index_is_ok(string, index):
+ return index
+ return self._find_start_index(string, index+2, end)
+
+ def _start_index_is_ok(self, string, index):
+ return string[index] in self._identifiers\
+ and not self._is_escaped(string, index)
+
+ def _is_escaped(self, string, index):
+ escaped = False
+ while index > 0 and string[index-1] == '\\':
+ index -= 1
+ escaped = not escaped
+ return escaped
+
+ def _variable_state(self, char, index):
+ self._variable_chars.append(char)
+ if char == '}' and not self._is_escaped(self._string, index):
+ self._open_curly -= 1
+ if self._open_curly == 0:
+ if not self._is_list_or_dict_variable():
+ raise StopIteration
+ self._state = self._waiting_list_variable_index_state
+ elif char in self._identifiers:
+ self._state = self._internal_variable_start_state
+
+ def _is_list_or_dict_variable(self):
+ return self._variable_chars[0] in ('@','&')
+
+ def _internal_variable_start_state(self, char, index):
+ self._state = self._variable_state
+ if char == '{':
+ self._variable_chars.append(char)
+ self._open_curly += 1
+ self._may_have_internal_variables = True
+ else:
+ self._variable_state(char, index)
+
+ def _waiting_list_variable_index_state(self, char, index):
+ if char != '[':
+ raise StopIteration
+ self._list_and_dict_variable_index_chars.append(char)
+ self._state = self._list_variable_index_state
+
+ def _list_variable_index_state(self, char, index):
+ self._list_and_dict_variable_index_chars.append(char)
+ if char == ']':
+ raise StopIteration
diff --git a/wandb/vendor/pygments/lexers/ruby.py b/wandb/vendor/pygments/lexers/ruby.py
new file mode 100644
index 0000000000000000000000000000000000000000..fe750f1a2fb9bd5cadbdf430d8fc548197cdd21f
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/ruby.py
@@ -0,0 +1,519 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.ruby
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Ruby and related languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, ExtendedRegexLexer, include, \
+ bygroups, default, LexerContext, do_insertions, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Error, Generic
+from pygments.util import shebang_matches
+
+__all__ = ['RubyLexer', 'RubyConsoleLexer', 'FancyLexer']
+
+line_re = re.compile('.*?\n')
+
+
+RUBY_OPERATORS = (
+ '*', '**', '-', '+', '-@', '+@', '/', '%', '&', '|', '^', '`', '~',
+ '[]', '[]=', '<<', '>>', '<', '<>', '<=>', '>', '>=', '==', '==='
+)
+
+
+class RubyLexer(ExtendedRegexLexer):
+ """
+ For `Ruby <http://www.ruby-lang.org>`_ source code.
+ """
+
+ name = 'Ruby'
+ aliases = ['rb', 'ruby', 'duby']
+ filenames = ['*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec',
+ '*.rbx', '*.duby', 'Gemfile']
+ mimetypes = ['text/x-ruby', 'application/x-ruby']
+
+ flags = re.DOTALL | re.MULTILINE
+
+ def heredoc_callback(self, match, ctx):
+ # okay, this is the hardest part of parsing Ruby...
+ # match: 1 = <<-?, 2 = quote? 3 = name 4 = quote? 5 = rest of line
+
+ start = match.start(1)
+ yield start, Operator, match.group(1) # <<-?
+ yield match.start(2), String.Heredoc, match.group(2) # quote ", ', `
+ yield match.start(3), String.Delimiter, match.group(3) # heredoc name
+ yield match.start(4), String.Heredoc, match.group(4) # quote again
+
+ heredocstack = ctx.__dict__.setdefault('heredocstack', [])
+ outermost = not bool(heredocstack)
+ heredocstack.append((match.group(1) == '<<-', match.group(3)))
+
+ ctx.pos = match.start(5)
+ ctx.end = match.end(5)
+ # this may find other heredocs
+ for i, t, v in self.get_tokens_unprocessed(context=ctx):
+ yield i, t, v
+ ctx.pos = match.end()
+
+ if outermost:
+ # this is the outer heredoc again, now we can process them all
+ for tolerant, hdname in heredocstack:
+ lines = []
+ for match in line_re.finditer(ctx.text, ctx.pos):
+ if tolerant:
+ check = match.group().strip()
+ else:
+ check = match.group().rstrip()
+ if check == hdname:
+ for amatch in lines:
+ yield amatch.start(), String.Heredoc, amatch.group()
+ yield match.start(), String.Delimiter, match.group()
+ ctx.pos = match.end()
+ break
+ else:
+ lines.append(match)
+ else:
+ # end of heredoc not found -- error!
+ for amatch in lines:
+ yield amatch.start(), Error, amatch.group()
+ ctx.end = len(ctx.text)
+ del heredocstack[:]
+
+ def gen_rubystrings_rules():
+ def intp_regex_callback(self, match, ctx):
+ yield match.start(1), String.Regex, match.group(1) # begin
+ nctx = LexerContext(match.group(3), 0, ['interpolated-regex'])
+ for i, t, v in self.get_tokens_unprocessed(context=nctx):
+ yield match.start(3)+i, t, v
+ yield match.start(4), String.Regex, match.group(4) # end[mixounse]*
+ ctx.pos = match.end()
+
+ def intp_string_callback(self, match, ctx):
+ yield match.start(1), String.Other, match.group(1)
+ nctx = LexerContext(match.group(3), 0, ['interpolated-string'])
+ for i, t, v in self.get_tokens_unprocessed(context=nctx):
+ yield match.start(3)+i, t, v
+ yield match.start(4), String.Other, match.group(4) # end
+ ctx.pos = match.end()
+
+ states = {}
+ states['strings'] = [
+ # easy ones
+ (r'\:@{0,2}[a-zA-Z_]\w*[!?]?', String.Symbol),
+ (words(RUBY_OPERATORS, prefix=r'\:@{0,2}'), String.Symbol),
+ (r":'(\\\\|\\'|[^'])*'", String.Symbol),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ (r':"', String.Symbol, 'simple-sym'),
+ (r'([a-zA-Z_]\w*)(:)(?!:)',
+ bygroups(String.Symbol, Punctuation)), # Since Ruby 1.9
+ (r'"', String.Double, 'simple-string'),
+ (r'(?<!\.)`', String.Backtick, 'simple-backtick'),
+ ]
+
+ # double-quoted string and symbol
+ for name, ttype, end in ('string', String.Double, '"'), \
+ ('sym', String.Symbol, '"'), \
+ ('backtick', String.Backtick, '`'):
+ states['simple-'+name] = [
+ include('string-intp-escaped'),
+ (r'[^\\%s#]+' % end, ttype),
+ (r'[\\#]', ttype),
+ (end, ttype, '#pop'),
+ ]
+
+ # braced quoted strings
+ for lbrace, rbrace, bracecc, name in \
+ ('\\{', '\\}', '{}', 'cb'), \
+ ('\\[', '\\]', '\\[\\]', 'sb'), \
+ ('\\(', '\\)', '()', 'pa'), \
+ ('<', '>', '<>', 'ab'):
+ states[name+'-intp-string'] = [
+ (r'\\[\\' + bracecc + ']', String.Other),
+ (lbrace, String.Other, '#push'),
+ (rbrace, String.Other, '#pop'),
+ include('string-intp-escaped'),
+ (r'[\\#' + bracecc + ']', String.Other),
+ (r'[^\\#' + bracecc + ']+', String.Other),
+ ]
+ states['strings'].append((r'%[QWx]?' + lbrace, String.Other,
+ name+'-intp-string'))
+ states[name+'-string'] = [
+ (r'\\[\\' + bracecc + ']', String.Other),
+ (lbrace, String.Other, '#push'),
+ (rbrace, String.Other, '#pop'),
+ (r'[\\#' + bracecc + ']', String.Other),
+ (r'[^\\#' + bracecc + ']+', String.Other),
+ ]
+ states['strings'].append((r'%[qsw]' + lbrace, String.Other,
+ name+'-string'))
+ states[name+'-regex'] = [
+ (r'\\[\\' + bracecc + ']', String.Regex),
+ (lbrace, String.Regex, '#push'),
+ (rbrace + '[mixounse]*', String.Regex, '#pop'),
+ include('string-intp'),
+ (r'[\\#' + bracecc + ']', String.Regex),
+ (r'[^\\#' + bracecc + ']+', String.Regex),
+ ]
+ states['strings'].append((r'%r' + lbrace, String.Regex,
+ name+'-regex'))
+
+ # these must come after %<brace>!
+ states['strings'] += [
+ # %r regex
+ (r'(%r([\W_]))((?:\\\2|(?!\2).)*)(\2[mixounse]*)',
+ intp_regex_callback),
+ # regular fancy strings with qsw
+ (r'%[qsw]([\W_])((?:\\\1|(?!\1).)*)\1', String.Other),
+ (r'(%[QWx]([\W_]))((?:\\\2|(?!\2).)*)(\2)',
+ intp_string_callback),
+ # special forms of fancy strings after operators or
+ # in method calls with braces
+ (r'(?<=[-+/*%=<>&!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
+ bygroups(Text, String.Other, None)),
+ # and because of fixed width lookbehinds the whole thing a
+ # second time for line startings...
+ (r'^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
+ bygroups(Text, String.Other, None)),
+ # all regular fancy strings without qsw
+ (r'(%([^a-zA-Z0-9\s]))((?:\\\2|(?!\2).)*)(\2)',
+ intp_string_callback),
+ ]
+
+ return states
+
+ tokens = {
+ 'root': [
+ (r'\A#!.+?$', Comment.Hashbang),
+ (r'#.*?$', Comment.Single),
+ (r'=begin\s.*?\n=end.*?$', Comment.Multiline),
+ # keywords
+ (words((
+ 'BEGIN', 'END', 'alias', 'begin', 'break', 'case', 'defined?',
+ 'do', 'else', 'elsif', 'end', 'ensure', 'for', 'if', 'in', 'next', 'redo',
+ 'rescue', 'raise', 'retry', 'return', 'super', 'then', 'undef',
+ 'unless', 'until', 'when', 'while', 'yield'), suffix=r'\b'),
+ Keyword),
+ # start of function, class and module names
+ (r'(module)(\s+)([a-zA-Z_]\w*'
+ r'(?:::[a-zA-Z_]\w*)*)',
+ bygroups(Keyword, Text, Name.Namespace)),
+ (r'(def)(\s+)', bygroups(Keyword, Text), 'funcname'),
+ (r'def(?=[*%&^`~+-/\[<>=])', Keyword, 'funcname'),
+ (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
+ # special methods
+ (words((
+ 'initialize', 'new', 'loop', 'include', 'extend', 'raise', 'attr_reader',
+ 'attr_writer', 'attr_accessor', 'attr', 'catch', 'throw', 'private',
+ 'module_function', 'public', 'protected', 'true', 'false', 'nil'),
+ suffix=r'\b'),
+ Keyword.Pseudo),
+ (r'(not|and|or)\b', Operator.Word),
+ (words((
+ 'autoload', 'block_given', 'const_defined', 'eql', 'equal', 'frozen', 'include',
+ 'instance_of', 'is_a', 'iterator', 'kind_of', 'method_defined', 'nil',
+ 'private_method_defined', 'protected_method_defined',
+ 'public_method_defined', 'respond_to', 'tainted'), suffix=r'\?'),
+ Name.Builtin),
+ (r'(chomp|chop|exit|gsub|sub)!', Name.Builtin),
+ (words((
+ 'Array', 'Float', 'Integer', 'String', '__id__', '__send__', 'abort',
+ 'ancestors', 'at_exit', 'autoload', 'binding', 'callcc', 'caller',
+ 'catch', 'chomp', 'chop', 'class_eval', 'class_variables',
+ 'clone', 'const_defined?', 'const_get', 'const_missing', 'const_set',
+ 'constants', 'display', 'dup', 'eval', 'exec', 'exit', 'extend', 'fail', 'fork',
+ 'format', 'freeze', 'getc', 'gets', 'global_variables', 'gsub',
+ 'hash', 'id', 'included_modules', 'inspect', 'instance_eval',
+ 'instance_method', 'instance_methods',
+ 'instance_variable_get', 'instance_variable_set', 'instance_variables',
+ 'lambda', 'load', 'local_variables', 'loop',
+ 'method', 'method_missing', 'methods', 'module_eval', 'name',
+ 'object_id', 'open', 'p', 'print', 'printf', 'private_class_method',
+ 'private_instance_methods',
+ 'private_methods', 'proc', 'protected_instance_methods',
+ 'protected_methods', 'public_class_method',
+ 'public_instance_methods', 'public_methods',
+ 'putc', 'puts', 'raise', 'rand', 'readline', 'readlines', 'require',
+ 'scan', 'select', 'self', 'send', 'set_trace_func', 'singleton_methods', 'sleep',
+ 'split', 'sprintf', 'srand', 'sub', 'syscall', 'system', 'taint',
+ 'test', 'throw', 'to_a', 'to_s', 'trace_var', 'trap', 'untaint',
+ 'untrace_var', 'warn'), prefix=r'(?<!\.)', suffix=r'\b'),
+ Name.Builtin),
+ (r'__(FILE|LINE)__\b', Name.Builtin.Pseudo),
+ # normal heredocs
+ (r'(?<!\w)(<<-?)(["`\']?)([a-zA-Z_]\w*)(\2)(.*?\n)',
+ heredoc_callback),
+ # empty string heredocs
+ (r'(<<-?)("|\')()(\2)(.*?\n)', heredoc_callback),
+ (r'__END__', Comment.Preproc, 'end-part'),
+ # multiline regex (after keywords or assignments)
+ (r'(?:^|(?<=[=<>~!:])|'
+ r'(?<=(?:\s|;)when\s)|'
+ r'(?<=(?:\s|;)or\s)|'
+ r'(?<=(?:\s|;)and\s)|'
+ r'(?<=\.index\s)|'
+ r'(?<=\.scan\s)|'
+ r'(?<=\.sub\s)|'
+ r'(?<=\.sub!\s)|'
+ r'(?<=\.gsub\s)|'
+ r'(?<=\.gsub!\s)|'
+ r'(?<=\.match\s)|'
+ r'(?<=(?:\s|;)if\s)|'
+ r'(?<=(?:\s|;)elsif\s)|'
+ r'(?<=^when\s)|'
+ r'(?<=^index\s)|'
+ r'(?<=^scan\s)|'
+ r'(?<=^sub\s)|'
+ r'(?<=^gsub\s)|'
+ r'(?<=^sub!\s)|'
+ r'(?<=^gsub!\s)|'
+ r'(?<=^match\s)|'
+ r'(?<=^if\s)|'
+ r'(?<=^elsif\s)'
+ r')(\s*)(/)', bygroups(Text, String.Regex), 'multiline-regex'),
+ # multiline regex (in method calls or subscripts)
+ (r'(?<=\(|,|\[)/', String.Regex, 'multiline-regex'),
+ # multiline regex (this time the funny no whitespace rule)
+ (r'(\s+)(/)(?![\s=])', bygroups(Text, String.Regex),
+ 'multiline-regex'),
+ # lex numbers and ignore following regular expressions which
+ # are division operators in fact (grrrr. i hate that. any
+ # better ideas?)
+ # since pygments 0.7 we also eat a "?" operator after numbers
+ # so that the char operator does not work. Chars are not allowed
+ # there so that you can use the ternary operator.
+ # stupid example:
+ # x>=0?n[x]:""
+ (r'(0_?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?',
+ bygroups(Number.Oct, Text, Operator)),
+ (r'(0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*)(\s*)([/?])?',
+ bygroups(Number.Hex, Text, Operator)),
+ (r'(0b[01]+(?:_[01]+)*)(\s*)([/?])?',
+ bygroups(Number.Bin, Text, Operator)),
+ (r'([\d]+(?:_\d+)*)(\s*)([/?])?',
+ bygroups(Number.Integer, Text, Operator)),
+ # Names
+ (r'@@[a-zA-Z_]\w*', Name.Variable.Class),
+ (r'@[a-zA-Z_]\w*', Name.Variable.Instance),
+ (r'\$\w+', Name.Variable.Global),
+ (r'\$[!@&`\'+~=/\\,;.<>_*$?:"^-]', Name.Variable.Global),
+ (r'\$-[0adFiIlpvw]', Name.Variable.Global),
+ (r'::', Operator),
+ include('strings'),
+ # chars
+ (r'\?(\\[MC]-)*' # modifiers
+ r'(\\([\\abefnrstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)'
+ r'(?!\w)',
+ String.Char),
+ (r'[A-Z]\w+', Name.Constant),
+ # this is needed because ruby attributes can look
+ # like keywords (class) or like this: ` ?!?
+ (words(RUBY_OPERATORS, prefix=r'(\.|::)'),
+ bygroups(Operator, Name.Operator)),
+ (r'(\.|::)([a-zA-Z_]\w*[!?]?|[*%&^`~+\-/\[<>=])',
+ bygroups(Operator, Name)),
+ (r'[a-zA-Z_]\w*[!?]?', Name),
+ (r'(\[|\]|\*\*|<<?|>>?|>=|<=|<=>|=~|={3}|'
+ r'!~|&&?|\|\||\.{1,3})', Operator),
+ (r'[-+/*%=<>&!^|~]=?', Operator),
+ (r'[(){};,/?:\\]', Punctuation),
+ (r'\s+', Text)
+ ],
+ 'funcname': [
+ (r'\(', Punctuation, 'defexpr'),
+ (r'(?:([a-zA-Z_]\w*)(\.))?'
+ r'([a-zA-Z_]\w*[!?]?|\*\*?|[-+]@?|'
+ r'[/%&|^`~]|\[\]=?|<<|>>|<=?>|>=?|===?)',
+ bygroups(Name.Class, Operator, Name.Function), '#pop'),
+ default('#pop')
+ ],
+ 'classname': [
+ (r'\(', Punctuation, 'defexpr'),
+ (r'<<', Operator, '#pop'),
+ (r'[A-Z_]\w*', Name.Class, '#pop'),
+ default('#pop')
+ ],
+ 'defexpr': [
+ (r'(\))(\.|::)?', bygroups(Punctuation, Operator), '#pop'),
+ (r'\(', Operator, '#push'),
+ include('root')
+ ],
+ 'in-intp': [
+ (r'\{', String.Interpol, '#push'),
+ (r'\}', String.Interpol, '#pop'),
+ include('root'),
+ ],
+ 'string-intp': [
+ (r'#\{', String.Interpol, 'in-intp'),
+ (r'#@@?[a-zA-Z_]\w*', String.Interpol),
+ (r'#\$[a-zA-Z_]\w*', String.Interpol)
+ ],
+ 'string-intp-escaped': [
+ include('string-intp'),
+ (r'\\([\\abefnrstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})',
+ String.Escape)
+ ],
+ 'interpolated-regex': [
+ include('string-intp'),
+ (r'[\\#]', String.Regex),
+ (r'[^\\#]+', String.Regex),
+ ],
+ 'interpolated-string': [
+ include('string-intp'),
+ (r'[\\#]', String.Other),
+ (r'[^\\#]+', String.Other),
+ ],
+ 'multiline-regex': [
+ include('string-intp'),
+ (r'\\\\', String.Regex),
+ (r'\\/', String.Regex),
+ (r'[\\#]', String.Regex),
+ (r'[^\\/#]+', String.Regex),
+ (r'/[mixounse]*', String.Regex, '#pop'),
+ ],
+ 'end-part': [
+ (r'.+', Comment.Preproc, '#pop')
+ ]
+ }
+ tokens.update(gen_rubystrings_rules())
+
+ def analyse_text(text):
+ return shebang_matches(text, r'ruby(1\.\d)?')
+
+
+class RubyConsoleLexer(Lexer):
+ """
+ For Ruby interactive console (**irb**) output like:
+
+ .. sourcecode:: rbcon
+
+ irb(main):001:0> a = 1
+ => 1
+ irb(main):002:0> puts a
+ 1
+ => nil
+ """
+ name = 'Ruby irb session'
+ aliases = ['rbcon', 'irb']
+ mimetypes = ['text/x-ruby-shellsession']
+
+ _prompt_re = re.compile('irb\([a-zA-Z_]\w*\):\d{3}:\d+[>*"\'] '
+ '|>> |\?> ')
+
+ def get_tokens_unprocessed(self, text):
+ rblexer = RubyLexer(**self.options)
+
+ curcode = ''
+ insertions = []
+ for match in line_re.finditer(text):
+ line = match.group()
+ m = self._prompt_re.match(line)
+ if m is not None:
+ end = m.end()
+ insertions.append((len(curcode),
+ [(0, Generic.Prompt, line[:end])]))
+ curcode += line[end:]
+ else:
+ if curcode:
+ for item in do_insertions(
+ insertions, rblexer.get_tokens_unprocessed(curcode)):
+ yield item
+ curcode = ''
+ insertions = []
+ yield match.start(), Generic.Output, line
+ if curcode:
+ for item in do_insertions(
+ insertions, rblexer.get_tokens_unprocessed(curcode)):
+ yield item
+
+
+class FancyLexer(RegexLexer):
+ """
+ Pygments Lexer For `Fancy <http://www.fancy-lang.org/>`_.
+
+ Fancy is a self-hosted, pure object-oriented, dynamic,
+ class-based, concurrent general-purpose programming language
+ running on Rubinius, the Ruby VM.
+
+ .. versionadded:: 1.5
+ """
+ name = 'Fancy'
+ filenames = ['*.fy', '*.fancypack']
+ aliases = ['fancy', 'fy']
+ mimetypes = ['text/x-fancysrc']
+
+ tokens = {
+ # copied from PerlLexer:
+ 'balanced-regex': [
+ (r'/(\\\\|\\/|[^/])*/[egimosx]*', String.Regex, '#pop'),
+ (r'!(\\\\|\\!|[^!])*![egimosx]*', String.Regex, '#pop'),
+ (r'\\(\\\\|[^\\])*\\[egimosx]*', String.Regex, '#pop'),
+ (r'\{(\\\\|\\\}|[^}])*\}[egimosx]*', String.Regex, '#pop'),
+ (r'<(\\\\|\\>|[^>])*>[egimosx]*', String.Regex, '#pop'),
+ (r'\[(\\\\|\\\]|[^\]])*\][egimosx]*', String.Regex, '#pop'),
+ (r'\((\\\\|\\\)|[^)])*\)[egimosx]*', String.Regex, '#pop'),
+ (r'@(\\\\|\\@|[^@])*@[egimosx]*', String.Regex, '#pop'),
+ (r'%(\\\\|\\%|[^%])*%[egimosx]*', String.Regex, '#pop'),
+ (r'\$(\\\\|\\\$|[^$])*\$[egimosx]*', String.Regex, '#pop'),
+ ],
+ 'root': [
+ (r'\s+', Text),
+
+ # balanced delimiters (copied from PerlLexer):
+ (r's\{(\\\\|\\\}|[^}])*\}\s*', String.Regex, 'balanced-regex'),
+ (r's<(\\\\|\\>|[^>])*>\s*', String.Regex, 'balanced-regex'),
+ (r's\[(\\\\|\\\]|[^\]])*\]\s*', String.Regex, 'balanced-regex'),
+ (r's\((\\\\|\\\)|[^)])*\)\s*', String.Regex, 'balanced-regex'),
+ (r'm?/(\\\\|\\/|[^/\n])*/[gcimosx]*', String.Regex),
+ (r'm(?=[/!\\{<\[(@%$])', String.Regex, 'balanced-regex'),
+
+ # Comments
+ (r'#(.*?)\n', Comment.Single),
+ # Symbols
+ (r'\'([^\'\s\[\](){}]+|\[\])', String.Symbol),
+ # Multi-line DoubleQuotedString
+ (r'"""(\\\\|\\"|[^"])*"""', String),
+ # DoubleQuotedString
+ (r'"(\\\\|\\"|[^"])*"', String),
+ # keywords
+ (r'(def|class|try|catch|finally|retry|return|return_local|match|'
+ r'case|->|=>)\b', Keyword),
+ # constants
+ (r'(self|super|nil|false|true)\b', Name.Constant),
+ (r'[(){};,/?|:\\]', Punctuation),
+ # names
+ (words((
+ 'Object', 'Array', 'Hash', 'Directory', 'File', 'Class', 'String',
+ 'Number', 'Enumerable', 'FancyEnumerable', 'Block', 'TrueClass',
+ 'NilClass', 'FalseClass', 'Tuple', 'Symbol', 'Stack', 'Set',
+ 'FancySpec', 'Method', 'Package', 'Range'), suffix=r'\b'),
+ Name.Builtin),
+ # functions
+ (r'[a-zA-Z](\w|[-+?!=*/^><%])*:', Name.Function),
+ # operators, must be below functions
+ (r'[-+*/~,<>=&!?%^\[\].$]+', Operator),
+ ('[A-Z]\w*', Name.Constant),
+ ('@[a-zA-Z_]\w*', Name.Variable.Instance),
+ ('@@[a-zA-Z_]\w*', Name.Variable.Class),
+ ('@@?', Operator),
+ ('[a-zA-Z_]\w*', Name),
+ # numbers - / checks are necessary to avoid mismarking regexes,
+ # see comment in RubyLexer
+ (r'(0[oO]?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?',
+ bygroups(Number.Oct, Text, Operator)),
+ (r'(0[xX][0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*)(\s*)([/?])?',
+ bygroups(Number.Hex, Text, Operator)),
+ (r'(0[bB][01]+(?:_[01]+)*)(\s*)([/?])?',
+ bygroups(Number.Bin, Text, Operator)),
+ (r'([\d]+(?:_\d+)*)(\s*)([/?])?',
+ bygroups(Number.Integer, Text, Operator)),
+ (r'\d+([eE][+-]?[0-9]+)|\d+\.\d+([eE][+-]?[0-9]+)?', Number.Float),
+ (r'\d+', Number.Integer)
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/rust.py b/wandb/vendor/pygments/lexers/rust.py
new file mode 100644
index 0000000000000000000000000000000000000000..6914f54d69b96cf009e228e53a3b46dba0bc2649
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/rust.py
@@ -0,0 +1,220 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.rust
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the Rust language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, words, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Whitespace
+
+__all__ = ['RustLexer']
+
+
+class RustLexer(RegexLexer):
+ """
+ Lexer for the Rust programming language (version 1.10).
+
+ .. versionadded:: 1.6
+ """
+ name = 'Rust'
+ filenames = ['*.rs', '*.rs.in']
+ aliases = ['rust']
+ mimetypes = ['text/rust']
+
+ keyword_types = (
+ words(('u8', 'u16', 'u32', 'u64', 'i8', 'i16', 'i32', 'i64',
+ 'usize', 'isize', 'f32', 'f64', 'str', 'bool'),
+ suffix=r'\b'),
+ Keyword.Type)
+
+ builtin_types = (words((
+ # Reexported core operators
+ 'Copy', 'Send', 'Sized', 'Sync',
+ 'Drop', 'Fn', 'FnMut', 'FnOnce',
+
+ # Reexported types and traits
+ 'Box',
+ 'ToOwned',
+ 'Clone',
+ 'PartialEq', 'PartialOrd', 'Eq', 'Ord',
+ 'AsRef', 'AsMut', 'Into', 'From',
+ 'Default',
+ 'Iterator', 'Extend', 'IntoIterator',
+ 'DoubleEndedIterator', 'ExactSizeIterator',
+ 'Option',
+ 'Some', 'None',
+ 'Result',
+ 'Ok', 'Err',
+ 'SliceConcatExt',
+ 'String', 'ToString',
+ 'Vec'), suffix=r'\b'),
+ Name.Builtin)
+
+ tokens = {
+ 'root': [
+ # rust allows a file to start with a shebang, but if the first line
+ # starts with #![ then it’s not a shebang but a crate attribute.
+ (r'#![^[\r\n].*$', Comment.Preproc),
+ default('base'),
+ ],
+ 'base': [
+ # Whitespace and Comments
+ (r'\n', Whitespace),
+ (r'\s+', Whitespace),
+ (r'//!.*?\n', String.Doc),
+ (r'///(\n|[^/].*?\n)', String.Doc),
+ (r'//(.*?)\n', Comment.Single),
+ (r'/\*\*(\n|[^/*])', String.Doc, 'doccomment'),
+ (r'/\*!', String.Doc, 'doccomment'),
+ (r'/\*', Comment.Multiline, 'comment'),
+
+ # Macro parameters
+ (r"""\$([a-zA-Z_]\w*|\(,?|\),?|,?)""", Comment.Preproc),
+ # Keywords
+ (words((
+ 'as', 'box', 'const', 'crate', 'else', 'extern',
+ 'for', 'if', 'impl', 'in', 'loop', 'match', 'move',
+ 'mut', 'pub', 'ref', 'return', 'static', 'super',
+ 'trait', 'unsafe', 'use', 'where', 'while'), suffix=r'\b'),
+ Keyword),
+ (words(('abstract', 'alignof', 'become', 'do', 'final', 'macro',
+ 'offsetof', 'override', 'priv', 'proc', 'pure', 'sizeof',
+ 'typeof', 'unsized', 'virtual', 'yield'), suffix=r'\b'),
+ Keyword.Reserved),
+ (r'(true|false)\b', Keyword.Constant),
+ (r'mod\b', Keyword, 'modname'),
+ (r'let\b', Keyword.Declaration),
+ (r'fn\b', Keyword, 'funcname'),
+ (r'(struct|enum|type|union)\b', Keyword, 'typename'),
+ (r'(default)(\s+)(type|fn)\b', bygroups(Keyword, Text, Keyword)),
+ keyword_types,
+ (r'self\b', Name.Builtin.Pseudo),
+ # Prelude (taken from Rust’s src/libstd/prelude.rs)
+ builtin_types,
+ # Path seperators, so types don't catch them.
+ (r'::\b', Text),
+ # Types in positions.
+ (r'(?::|->)', Text, 'typename'),
+ # Labels
+ (r'(break|continue)(\s*)(\'[A-Za-z_]\w*)?',
+ bygroups(Keyword, Text.Whitespace, Name.Label)),
+ # Character Literal
+ (r"""'(\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0"""
+ r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""",
+ String.Char),
+ (r"""b'(\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\0"""
+ r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""",
+ String.Char),
+ # Binary Literal
+ (r'0b[01_]+', Number.Bin, 'number_lit'),
+ # Octal Literal
+ (r'0o[0-7_]+', Number.Oct, 'number_lit'),
+ # Hexadecimal Literal
+ (r'0[xX][0-9a-fA-F_]+', Number.Hex, 'number_lit'),
+ # Decimal Literal
+ (r'[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|'
+ r'\.[0-9_]*(?!\.)|[eE][+\-]?[0-9_]+)', Number.Float,
+ 'number_lit'),
+ (r'[0-9][0-9_]*', Number.Integer, 'number_lit'),
+ # String Literal
+ (r'b"', String, 'bytestring'),
+ (r'"', String, 'string'),
+ (r'b?r(#*)".*?"\1', String),
+
+ # Lifetime
+ (r"""'static""", Name.Builtin),
+ (r"""'[a-zA-Z_]\w*""", Name.Attribute),
+
+ # Operators and Punctuation
+ (r'[{}()\[\],.;]', Punctuation),
+ (r'[+\-*/%&|<>^!~@=:?]', Operator),
+
+ # Identifier
+ (r'[a-zA-Z_]\w*', Name),
+
+ # Attributes
+ (r'#!?\[', Comment.Preproc, 'attribute['),
+ # Macros
+ (r'([A-Za-z_]\w*)(!)(\s*)([A-Za-z_]\w*)?(\s*)(\{)',
+ bygroups(Comment.Preproc, Punctuation, Whitespace, Name,
+ Whitespace, Punctuation), 'macro{'),
+ (r'([A-Za-z_]\w*)(!)(\s*)([A-Za-z_]\w*)?(\()',
+ bygroups(Comment.Preproc, Punctuation, Whitespace, Name,
+ Punctuation), 'macro('),
+ ],
+ 'comment': [
+ (r'[^*/]+', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline),
+ ],
+ 'doccomment': [
+ (r'[^*/]+', String.Doc),
+ (r'/\*', String.Doc, '#push'),
+ (r'\*/', String.Doc, '#pop'),
+ (r'[*/]', String.Doc),
+ ],
+ 'modname': [
+ (r'\s+', Text),
+ (r'[a-zA-Z_]\w*', Name.Namespace, '#pop'),
+ default('#pop'),
+ ],
+ 'funcname': [
+ (r'\s+', Text),
+ (r'[a-zA-Z_]\w*', Name.Function, '#pop'),
+ default('#pop'),
+ ],
+ 'typename': [
+ (r'\s+', Text),
+ (r'&', Keyword.Pseudo),
+ builtin_types,
+ keyword_types,
+ (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
+ default('#pop'),
+ ],
+ 'number_lit': [
+ (r'[ui](8|16|32|64|size)', Keyword, '#pop'),
+ (r'f(32|64)', Keyword, '#pop'),
+ default('#pop'),
+ ],
+ 'string': [
+ (r'"', String, '#pop'),
+ (r"""\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0"""
+ r"""|\\u\{[0-9a-fA-F]{1,6}\}""", String.Escape),
+ (r'[^\\"]+', String),
+ (r'\\', String),
+ ],
+ 'bytestring': [
+ (r"""\\x[89a-fA-F][0-9a-fA-F]""", String.Escape),
+ include('string'),
+ ],
+ 'macro{': [
+ (r'\{', Operator, '#push'),
+ (r'\}', Operator, '#pop'),
+ ],
+ 'macro(': [
+ (r'\(', Operator, '#push'),
+ (r'\)', Operator, '#pop'),
+ ],
+ 'attribute_common': [
+ (r'"', String, 'string'),
+ (r'\[', Comment.Preproc, 'attribute['),
+ (r'\(', Comment.Preproc, 'attribute('),
+ ],
+ 'attribute[': [
+ include('attribute_common'),
+ (r'\];?', Comment.Preproc, '#pop'),
+ (r'[^"\]]+', Comment.Preproc),
+ ],
+ 'attribute(': [
+ include('attribute_common'),
+ (r'\);?', Comment.Preproc, '#pop'),
+ (r'[^")]+', Comment.Preproc),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/sas.py b/wandb/vendor/pygments/lexers/sas.py
new file mode 100644
index 0000000000000000000000000000000000000000..3747ed9ab8af521565d698d2b59e204b48e596d7
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/sas.py
@@ -0,0 +1,228 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.sas
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexer for SAS.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Comment, Keyword, Name, Number, String, Text, \
+ Other, Generic
+
+__all__ = ['SASLexer']
+
+
+class SASLexer(RegexLexer):
+ """
+ For `SAS <http://www.sas.com/>`_ files.
+
+ .. versionadded:: 2.2
+ """
+ # Syntax from syntax/sas.vim by James Kidd <james.kidd@covance.com>
+
+ name = 'SAS'
+ aliases = ['sas']
+ filenames = ['*.SAS', '*.sas']
+ mimetypes = ['text/x-sas', 'text/sas', 'application/x-sas']
+ flags = re.IGNORECASE | re.MULTILINE
+
+ builtins_macros = (
+ "bquote", "nrbquote", "cmpres", "qcmpres", "compstor", "datatyp",
+ "display", "do", "else", "end", "eval", "global", "goto", "if",
+ "index", "input", "keydef", "label", "left", "length", "let",
+ "local", "lowcase", "macro", "mend", "nrquote",
+ "nrstr", "put", "qleft", "qlowcase", "qscan",
+ "qsubstr", "qsysfunc", "qtrim", "quote", "qupcase", "scan",
+ "str", "substr", "superq", "syscall", "sysevalf", "sysexec",
+ "sysfunc", "sysget", "syslput", "sysprod", "sysrc", "sysrput",
+ "then", "to", "trim", "unquote", "until", "upcase", "verify",
+ "while", "window"
+ )
+
+ builtins_conditionals = (
+ "do", "if", "then", "else", "end", "until", "while"
+ )
+
+ builtins_statements = (
+ "abort", "array", "attrib", "by", "call", "cards", "cards4",
+ "catname", "continue", "datalines", "datalines4", "delete", "delim",
+ "delimiter", "display", "dm", "drop", "endsas", "error", "file",
+ "filename", "footnote", "format", "goto", "in", "infile", "informat",
+ "input", "keep", "label", "leave", "length", "libname", "link",
+ "list", "lostcard", "merge", "missing", "modify", "options", "output",
+ "out", "page", "put", "redirect", "remove", "rename", "replace",
+ "retain", "return", "select", "set", "skip", "startsas", "stop",
+ "title", "update", "waitsas", "where", "window", "x", "systask"
+ )
+
+ builtins_sql = (
+ "add", "and", "alter", "as", "cascade", "check", "create",
+ "delete", "describe", "distinct", "drop", "foreign", "from",
+ "group", "having", "index", "insert", "into", "in", "key", "like",
+ "message", "modify", "msgtype", "not", "null", "on", "or",
+ "order", "primary", "references", "reset", "restrict", "select",
+ "set", "table", "unique", "update", "validate", "view", "where"
+ )
+
+ builtins_functions = (
+ "abs", "addr", "airy", "arcos", "arsin", "atan", "attrc",
+ "attrn", "band", "betainv", "blshift", "bnot", "bor",
+ "brshift", "bxor", "byte", "cdf", "ceil", "cexist", "cinv",
+ "close", "cnonct", "collate", "compbl", "compound",
+ "compress", "cos", "cosh", "css", "curobs", "cv", "daccdb",
+ "daccdbsl", "daccsl", "daccsyd", "dacctab", "dairy", "date",
+ "datejul", "datepart", "datetime", "day", "dclose", "depdb",
+ "depdbsl", "depsl", "depsyd",
+ "deptab", "dequote", "dhms", "dif", "digamma",
+ "dim", "dinfo", "dnum", "dopen", "doptname", "doptnum",
+ "dread", "dropnote", "dsname", "erf", "erfc", "exist", "exp",
+ "fappend", "fclose", "fcol", "fdelete", "fetch", "fetchobs",
+ "fexist", "fget", "fileexist", "filename", "fileref",
+ "finfo", "finv", "fipname", "fipnamel", "fipstate", "floor",
+ "fnonct", "fnote", "fopen", "foptname", "foptnum", "fpoint",
+ "fpos", "fput", "fread", "frewind", "frlen", "fsep", "fuzz",
+ "fwrite", "gaminv", "gamma", "getoption", "getvarc", "getvarn",
+ "hbound", "hms", "hosthelp", "hour", "ibessel", "index",
+ "indexc", "indexw", "input", "inputc", "inputn", "int",
+ "intck", "intnx", "intrr", "irr", "jbessel", "juldate",
+ "kurtosis", "lag", "lbound", "left", "length", "lgamma",
+ "libname", "libref", "log", "log10", "log2", "logpdf", "logpmf",
+ "logsdf", "lowcase", "max", "mdy", "mean", "min", "minute",
+ "mod", "month", "mopen", "mort", "n", "netpv", "nmiss",
+ "normal", "note", "npv", "open", "ordinal", "pathname",
+ "pdf", "peek", "peekc", "pmf", "point", "poisson", "poke",
+ "probbeta", "probbnml", "probchi", "probf", "probgam",
+ "probhypr", "probit", "probnegb", "probnorm", "probt",
+ "put", "putc", "putn", "qtr", "quote", "ranbin", "rancau",
+ "ranexp", "rangam", "range", "rank", "rannor", "ranpoi",
+ "rantbl", "rantri", "ranuni", "repeat", "resolve", "reverse",
+ "rewind", "right", "round", "saving", "scan", "sdf", "second",
+ "sign", "sin", "sinh", "skewness", "soundex", "spedis",
+ "sqrt", "std", "stderr", "stfips", "stname", "stnamel",
+ "substr", "sum", "symget", "sysget", "sysmsg", "sysprod",
+ "sysrc", "system", "tan", "tanh", "time", "timepart", "tinv",
+ "tnonct", "today", "translate", "tranwrd", "trigamma",
+ "trim", "trimn", "trunc", "uniform", "upcase", "uss", "var",
+ "varfmt", "varinfmt", "varlabel", "varlen", "varname",
+ "varnum", "varray", "varrayx", "vartype", "verify", "vformat",
+ "vformatd", "vformatdx", "vformatn", "vformatnx", "vformatw",
+ "vformatwx", "vformatx", "vinarray", "vinarrayx", "vinformat",
+ "vinformatd", "vinformatdx", "vinformatn", "vinformatnx",
+ "vinformatw", "vinformatwx", "vinformatx", "vlabel",
+ "vlabelx", "vlength", "vlengthx", "vname", "vnamex", "vtype",
+ "vtypex", "weekday", "year", "yyq", "zipfips", "zipname",
+ "zipnamel", "zipstate"
+ )
+
+ tokens = {
+ 'root': [
+ include('comments'),
+ include('proc-data'),
+ include('cards-datalines'),
+ include('logs'),
+ include('general'),
+ (r'.', Text),
+ ],
+ # SAS is multi-line regardless, but * is ended by ;
+ 'comments': [
+ (r'^\s*\*.*?;', Comment),
+ (r'/\*.*?\*/', Comment),
+ (r'^\s*\*(.|\n)*?;', Comment.Multiline),
+ (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+ ],
+ # Special highlight for proc, data, quit, run
+ 'proc-data': [
+ (r'(^|;)\s*(proc \w+|data|run|quit)[\s;]',
+ Keyword.Reserved),
+ ],
+ # Special highlight cards and datalines
+ 'cards-datalines': [
+ (r'^\s*(datalines|cards)\s*;\s*$', Keyword, 'data'),
+ ],
+ 'data': [
+ (r'(.|\n)*^\s*;\s*$', Other, '#pop'),
+ ],
+ # Special highlight for put NOTE|ERROR|WARNING (order matters)
+ 'logs': [
+ (r'\n?^\s*%?put ', Keyword, 'log-messages'),
+ ],
+ 'log-messages': [
+ (r'NOTE(:|-).*', Generic, '#pop'),
+ (r'WARNING(:|-).*', Generic.Emph, '#pop'),
+ (r'ERROR(:|-).*', Generic.Error, '#pop'),
+ include('general'),
+ ],
+ 'general': [
+ include('keywords'),
+ include('vars-strings'),
+ include('special'),
+ include('numbers'),
+ ],
+ # Keywords, statements, functions, macros
+ 'keywords': [
+ (words(builtins_statements,
+ prefix = r'\b',
+ suffix = r'\b'),
+ Keyword),
+ (words(builtins_sql,
+ prefix = r'\b',
+ suffix = r'\b'),
+ Keyword),
+ (words(builtins_conditionals,
+ prefix = r'\b',
+ suffix = r'\b'),
+ Keyword),
+ (words(builtins_macros,
+ prefix = r'%',
+ suffix = r'\b'),
+ Name.Builtin),
+ (words(builtins_functions,
+ prefix = r'\b',
+ suffix = r'\('),
+ Name.Builtin),
+ ],
+ # Strings and user-defined variables and macros (order matters)
+ 'vars-strings': [
+ (r'&[a-z_]\w{0,31}\.?', Name.Variable),
+ (r'%[a-z_]\w{0,31}', Name.Function),
+ (r'\'', String, 'string_squote'),
+ (r'"', String, 'string_dquote'),
+ ],
+ 'string_squote': [
+ ('\'', String, '#pop'),
+ (r'\\\\|\\"|\\\n', String.Escape),
+ # AFAIK, macro variables are not evaluated in single quotes
+ # (r'&', Name.Variable, 'validvar'),
+ (r'[^$\'\\]+', String),
+ (r'[$\'\\]', String),
+ ],
+ 'string_dquote': [
+ (r'"', String, '#pop'),
+ (r'\\\\|\\"|\\\n', String.Escape),
+ (r'&', Name.Variable, 'validvar'),
+ (r'[^$&"\\]+', String),
+ (r'[$"\\]', String),
+ ],
+ 'validvar': [
+ (r'[a-z_]\w{0,31}\.?', Name.Variable, '#pop'),
+ ],
+ # SAS numbers and special variables
+ 'numbers': [
+ (r'\b[+-]?([0-9]+(\.[0-9]+)?|\.[0-9]+|\.)(E[+-]?[0-9]+)?i?\b',
+ Number),
+ ],
+ 'special': [
+ (r'(null|missing|_all_|_automatic_|_character_|_n_|'
+ r'_infile_|_name_|_null_|_numeric_|_user_|_webout_)',
+ Keyword.Constant),
+ ],
+ # 'operators': [
+ # (r'(-|=|<=|>=|<|>|<>|&|!=|'
+ # r'\||\*|\+|\^|/|!|~|~=)', Operator)
+ # ],
+ }
diff --git a/wandb/vendor/pygments/lexers/scripting.py b/wandb/vendor/pygments/lexers/scripting.py
new file mode 100644
index 0000000000000000000000000000000000000000..b3af606e981b310e68dd6169bae1f159371d2126
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/scripting.py
@@ -0,0 +1,1222 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.scripting
+ ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for scripting and embedded languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, default, combined, \
+ words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Error, Whitespace, Other
+from pygments.util import get_bool_opt, get_list_opt, iteritems
+
+__all__ = ['LuaLexer', 'MoonScriptLexer', 'ChaiscriptLexer', 'LSLLexer',
+ 'AppleScriptLexer', 'RexxLexer', 'MOOCodeLexer', 'HybrisLexer',
+ 'EasytrieveLexer', 'JclLexer']
+
+
+class LuaLexer(RegexLexer):
+ """
+ For `Lua <http://www.lua.org>`_ source code.
+
+ Additional options accepted:
+
+ `func_name_highlighting`
+ If given and ``True``, highlight builtin function names
+ (default: ``True``).
+ `disabled_modules`
+ If given, must be a list of module names whose function names
+ should not be highlighted. By default all modules are highlighted.
+
+ To get a list of allowed modules have a look into the
+ `_lua_builtins` module:
+
+ .. sourcecode:: pycon
+
+ >>> from pygments.lexers._lua_builtins import MODULES
+ >>> MODULES.keys()
+ ['string', 'coroutine', 'modules', 'io', 'basic', ...]
+ """
+
+ name = 'Lua'
+ aliases = ['lua']
+ filenames = ['*.lua', '*.wlua']
+ mimetypes = ['text/x-lua', 'application/x-lua']
+
+ _comment_multiline = r'(?:--\[(?P<level>=*)\[[\w\W]*?\](?P=level)\])'
+ _comment_single = r'(?:--.*$)'
+ _space = r'(?:\s+)'
+ _s = r'(?:%s|%s|%s)' % (_comment_multiline, _comment_single, _space)
+ _name = r'(?:[^\W\d]\w*)'
+
+ tokens = {
+ 'root': [
+ # Lua allows a file to start with a shebang.
+ (r'#!.*', Comment.Preproc),
+ default('base'),
+ ],
+ 'ws': [
+ (_comment_multiline, Comment.Multiline),
+ (_comment_single, Comment.Single),
+ (_space, Text),
+ ],
+ 'base': [
+ include('ws'),
+
+ (r'(?i)0x[\da-f]*(\.[\da-f]*)?(p[+-]?\d+)?', Number.Hex),
+ (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number.Float),
+ (r'(?i)\d+e[+-]?\d+', Number.Float),
+ (r'\d+', Number.Integer),
+
+ # multiline strings
+ (r'(?s)\[(=*)\[.*?\]\1\]', String),
+
+ (r'::', Punctuation, 'label'),
+ (r'\.{3}', Punctuation),
+ (r'[=<>|~&+\-*/%#^]+|\.\.', Operator),
+ (r'[\[\]{}().,:;]', Punctuation),
+ (r'(and|or|not)\b', Operator.Word),
+
+ ('(break|do|else|elseif|end|for|if|in|repeat|return|then|until|'
+ r'while)\b', Keyword.Reserved),
+ (r'goto\b', Keyword.Reserved, 'goto'),
+ (r'(local)\b', Keyword.Declaration),
+ (r'(true|false|nil)\b', Keyword.Constant),
+
+ (r'(function)\b', Keyword.Reserved, 'funcname'),
+
+ (r'[A-Za-z_]\w*(\.[A-Za-z_]\w*)?', Name),
+
+ ("'", String.Single, combined('stringescape', 'sqs')),
+ ('"', String.Double, combined('stringescape', 'dqs'))
+ ],
+
+ 'funcname': [
+ include('ws'),
+ (r'[.:]', Punctuation),
+ (r'%s(?=%s*[.:])' % (_name, _s), Name.Class),
+ (_name, Name.Function, '#pop'),
+ # inline function
+ ('\(', Punctuation, '#pop'),
+ ],
+
+ 'goto': [
+ include('ws'),
+ (_name, Name.Label, '#pop'),
+ ],
+
+ 'label': [
+ include('ws'),
+ (r'::', Punctuation, '#pop'),
+ (_name, Name.Label),
+ ],
+
+ 'stringescape': [
+ (r'\\([abfnrtv\\"\']|[\r\n]{1,2}|z\s*|x[0-9a-fA-F]{2}|\d{1,3}|'
+ r'u\{[0-9a-fA-F]+\})', String.Escape),
+ ],
+
+ 'sqs': [
+ (r"'", String.Single, '#pop'),
+ (r"[^\\']+", String.Single),
+ ],
+
+ 'dqs': [
+ (r'"', String.Double, '#pop'),
+ (r'[^\\"]+', String.Double),
+ ]
+ }
+
+ def __init__(self, **options):
+ self.func_name_highlighting = get_bool_opt(
+ options, 'func_name_highlighting', True)
+ self.disabled_modules = get_list_opt(options, 'disabled_modules', [])
+
+ self._functions = set()
+ if self.func_name_highlighting:
+ from pygments.lexers._lua_builtins import MODULES
+ for mod, func in iteritems(MODULES):
+ if mod not in self.disabled_modules:
+ self._functions.update(func)
+ RegexLexer.__init__(self, **options)
+
+ def get_tokens_unprocessed(self, text):
+ for index, token, value in \
+ RegexLexer.get_tokens_unprocessed(self, text):
+ if token is Name:
+ if value in self._functions:
+ yield index, Name.Builtin, value
+ continue
+ elif '.' in value:
+ a, b = value.split('.')
+ yield index, Name, a
+ yield index + len(a), Punctuation, u'.'
+ yield index + len(a) + 1, Name, b
+ continue
+ yield index, token, value
+
+
+class MoonScriptLexer(LuaLexer):
+ """
+ For `MoonScript <http://moonscript.org>`_ source code.
+
+ .. versionadded:: 1.5
+ """
+
+ name = "MoonScript"
+ aliases = ["moon", "moonscript"]
+ filenames = ["*.moon"]
+ mimetypes = ['text/x-moonscript', 'application/x-moonscript']
+
+ tokens = {
+ 'root': [
+ (r'#!(.*?)$', Comment.Preproc),
+ default('base'),
+ ],
+ 'base': [
+ ('--.*$', Comment.Single),
+ (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number.Float),
+ (r'(?i)\d+e[+-]?\d+', Number.Float),
+ (r'(?i)0x[0-9a-f]*', Number.Hex),
+ (r'\d+', Number.Integer),
+ (r'\n', Text),
+ (r'[^\S\n]+', Text),
+ (r'(?s)\[(=*)\[.*?\]\1\]', String),
+ (r'(->|=>)', Name.Function),
+ (r':[a-zA-Z_]\w*', Name.Variable),
+ (r'(==|!=|~=|<=|>=|\.\.\.|\.\.|[=+\-*/%^<>#!.\\:])', Operator),
+ (r'[;,]', Punctuation),
+ (r'[\[\]{}()]', Keyword.Type),
+ (r'[a-zA-Z_]\w*:', Name.Variable),
+ (words((
+ 'class', 'extends', 'if', 'then', 'super', 'do', 'with',
+ 'import', 'export', 'while', 'elseif', 'return', 'for', 'in',
+ 'from', 'when', 'using', 'else', 'and', 'or', 'not', 'switch',
+ 'break'), suffix=r'\b'),
+ Keyword),
+ (r'(true|false|nil)\b', Keyword.Constant),
+ (r'(and|or|not)\b', Operator.Word),
+ (r'(self)\b', Name.Builtin.Pseudo),
+ (r'@@?([a-zA-Z_]\w*)?', Name.Variable.Class),
+ (r'[A-Z]\w*', Name.Class), # proper name
+ (r'[A-Za-z_]\w*(\.[A-Za-z_]\w*)?', Name),
+ ("'", String.Single, combined('stringescape', 'sqs')),
+ ('"', String.Double, combined('stringescape', 'dqs'))
+ ],
+ 'stringescape': [
+ (r'''\\([abfnrtv\\"']|\d{1,3})''', String.Escape)
+ ],
+ 'sqs': [
+ ("'", String.Single, '#pop'),
+ (".", String)
+ ],
+ 'dqs': [
+ ('"', String.Double, '#pop'),
+ (".", String)
+ ]
+ }
+
+ def get_tokens_unprocessed(self, text):
+ # set . as Operator instead of Punctuation
+ for index, token, value in LuaLexer.get_tokens_unprocessed(self, text):
+ if token == Punctuation and value == ".":
+ token = Operator
+ yield index, token, value
+
+
+class ChaiscriptLexer(RegexLexer):
+ """
+ For `ChaiScript <http://chaiscript.com/>`_ source code.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'ChaiScript'
+ aliases = ['chai', 'chaiscript']
+ filenames = ['*.chai']
+ mimetypes = ['text/x-chaiscript', 'application/x-chaiscript']
+
+ flags = re.DOTALL | re.MULTILINE
+
+ tokens = {
+ 'commentsandwhitespace': [
+ (r'\s+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'^\#.*?\n', Comment.Single)
+ ],
+ 'slashstartsregex': [
+ include('commentsandwhitespace'),
+ (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+ r'([gim]+\b|\B)', String.Regex, '#pop'),
+ (r'(?=/)', Text, ('#pop', 'badregex')),
+ default('#pop')
+ ],
+ 'badregex': [
+ (r'\n', Text, '#pop')
+ ],
+ 'root': [
+ include('commentsandwhitespace'),
+ (r'\n', Text),
+ (r'[^\S\n]+', Text),
+ (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|\.\.'
+ r'(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'),
+ (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
+ (r'[})\].]', Punctuation),
+ (r'[=+\-*/]', Operator),
+ (r'(for|in|while|do|break|return|continue|if|else|'
+ r'throw|try|catch'
+ r')\b', Keyword, 'slashstartsregex'),
+ (r'(var)\b', Keyword.Declaration, 'slashstartsregex'),
+ (r'(attr|def|fun)\b', Keyword.Reserved),
+ (r'(true|false)\b', Keyword.Constant),
+ (r'(eval|throw)\b', Name.Builtin),
+ (r'`\S+`', Name.Builtin),
+ (r'[$a-zA-Z_]\w*', Name.Other),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+', Number.Integer),
+ (r'"', String.Double, 'dqstring'),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ ],
+ 'dqstring': [
+ (r'\$\{[^"}]+?\}', String.Interpol),
+ (r'\$', String.Double),
+ (r'\\\\', String.Double),
+ (r'\\"', String.Double),
+ (r'[^\\"$]+', String.Double),
+ (r'"', String.Double, '#pop'),
+ ],
+ }
+
+
+class LSLLexer(RegexLexer):
+ """
+ For Second Life's Linden Scripting Language source code.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'LSL'
+ aliases = ['lsl']
+ filenames = ['*.lsl']
+ mimetypes = ['text/x-lsl']
+
+ flags = re.MULTILINE
+
+ lsl_keywords = r'\b(?:do|else|for|if|jump|return|while)\b'
+ lsl_types = r'\b(?:float|integer|key|list|quaternion|rotation|string|vector)\b'
+ lsl_states = r'\b(?:(?:state)\s+\w+|default)\b'
+ lsl_events = r'\b(?:state_(?:entry|exit)|touch(?:_(?:start|end))?|(?:land_)?collision(?:_(?:start|end))?|timer|listen|(?:no_)?sensor|control|(?:not_)?at_(?:rot_)?target|money|email|run_time_permissions|changed|attach|dataserver|moving_(?:start|end)|link_message|(?:on|object)_rez|remote_data|http_re(?:sponse|quest)|path_update|transaction_result)\b'
+ lsl_functions_builtin = r'\b(?:ll(?:ReturnObjectsBy(?:ID|Owner)|Json(?:2List|[GS]etValue|ValueType)|Sin|Cos|Tan|Atan2|Sqrt|Pow|Abs|Fabs|Frand|Floor|Ceil|Round|Vec(?:Mag|Norm|Dist)|Rot(?:Between|2(?:Euler|Fwd|Left|Up))|(?:Euler|Axes)2Rot|Whisper|(?:Region|Owner)?Say|Shout|Listen(?:Control|Remove)?|Sensor(?:Repeat|Remove)?|Detected(?:Name|Key|Owner|Type|Pos|Vel|Grab|Rot|Group|LinkNumber)|Die|Ground|Wind|(?:[GS]et)(?:AnimationOverride|MemoryLimit|PrimMediaParams|ParcelMusicURL|Object(?:Desc|Name)|PhysicsMaterial|Status|Scale|Color|Alpha|Texture|Pos|Rot|Force|Torque)|ResetAnimationOverride|(?:Scale|Offset|Rotate)Texture|(?:Rot)?Target(?:Remove)?|(?:Stop)?MoveToTarget|Apply(?:Rotational)?Impulse|Set(?:KeyframedMotion|ContentType|RegionPos|(?:Angular)?Velocity|Buoyancy|HoverHeight|ForceAndTorque|TimerEvent|ScriptState|Damage|TextureAnim|Sound(?:Queueing|Radius)|Vehicle(?:Type|(?:Float|Vector|Rotation)Param)|(?:Touch|Sit)?Text|Camera(?:Eye|At)Offset|PrimitiveParams|ClickAction|Link(?:Alpha|Color|PrimitiveParams(?:Fast)?|Texture(?:Anim)?|Camera|Media)|RemoteScriptAccessPin|PayPrice|LocalRot)|ScaleByFactor|Get(?:(?:Max|Min)ScaleFactor|ClosestNavPoint|StaticPath|SimStats|Env|PrimitiveParams|Link(?:PrimitiveParams|Number(?:OfSides)?|Key|Name|Media)|HTTPHeader|FreeURLs|Object(?:Details|PermMask|PrimCount)|Parcel(?:MaxPrims|Details|Prim(?:Count|Owners))|Attached|(?:SPMax|Free|Used)Memory|Region(?:Name|TimeDilation|FPS|Corner|AgentCount)|Root(?:Position|Rotation)|UnixTime|(?:Parcel|Region)Flags|(?:Wall|GMT)clock|SimulatorHostname|BoundingBox|GeometricCenter|Creator|NumberOf(?:Prims|NotecardLines|Sides)|Animation(?:List)?|(?:Camera|Local)(?:Pos|Rot)|Vel|Accel|Omega|Time(?:stamp|OfDay)|(?:Object|CenterOf)?Mass|MassMKS|Energy|Owner|(?:Owner)?Key|SunDirection|Texture(?:Offset|Scale|Rot)|Inventory(?:Number|Name|Key|Type|Creator|PermMask)|Permissions(?:Key)?|StartParameter|List(?:Length|EntryType)|Date|Agent(?:Size|Info|Language|List)|LandOwnerAt|NotecardLine|Script(?:Name|State))|(?:Get|Reset|GetAndReset)Time|PlaySound(?:Slave)?|LoopSound(?:Master|Slave)?|(?:Trigger|Stop|Preload)Sound|(?:(?:Get|Delete)Sub|Insert)String|To(?:Upper|Lower)|Give(?:InventoryList|Money)|RezObject|(?:Stop)?LookAt|Sleep|CollisionFilter|(?:Take|Release)Controls|DetachFromAvatar|AttachToAvatar(?:Temp)?|InstantMessage|(?:GetNext)?Email|StopHover|MinEventDelay|RotLookAt|String(?:Length|Trim)|(?:Start|Stop)Animation|TargetOmega|RequestPermissions|(?:Create|Break)Link|BreakAllLinks|(?:Give|Remove)Inventory|Water|PassTouches|Request(?:Agent|Inventory)Data|TeleportAgent(?:Home|GlobalCoords)?|ModifyLand|CollisionSound|ResetScript|MessageLinked|PushObject|PassCollisions|AxisAngle2Rot|Rot2(?:Axis|Angle)|A(?:cos|sin)|AngleBetween|AllowInventoryDrop|SubStringIndex|List2(?:CSV|Integer|Json|Float|String|Key|Vector|Rot|List(?:Strided)?)|DeleteSubList|List(?:Statistics|Sort|Randomize|(?:Insert|Find|Replace)List)|EdgeOfWorld|AdjustSoundVolume|Key2Name|TriggerSoundLimited|EjectFromLand|(?:CSV|ParseString)2List|OverMyLand|SameGroup|UnSit|Ground(?:Slope|Normal|Contour)|GroundRepel|(?:Set|Remove)VehicleFlags|(?:AvatarOn)?(?:Link)?SitTarget|Script(?:Danger|Profiler)|Dialog|VolumeDetect|ResetOtherScript|RemoteLoadScriptPin|(?:Open|Close)RemoteDataChannel|SendRemoteData|RemoteDataReply|(?:Integer|String)ToBase64|XorBase64|Log(?:10)?|Base64To(?:String|Integer)|ParseStringKeepNulls|RezAtRoot|RequestSimulatorData|ForceMouselook|(?:Load|Release|(?:E|Une)scape)URL|ParcelMedia(?:CommandList|Query)|ModPow|MapDestination|(?:RemoveFrom|AddTo|Reset)Land(?:Pass|Ban)List|(?:Set|Clear)CameraParams|HTTP(?:Request|Response)|TextBox|DetectedTouch(?:UV|Face|Pos|(?:N|Bin)ormal|ST)|(?:MD5|SHA1|DumpList2)String|Request(?:Secure)?URL|Clear(?:Prim|Link)Media|(?:Link)?ParticleSystem|(?:Get|Request)(?:Username|DisplayName)|RegionSayTo|CastRay|GenerateKey|TransferLindenDollars|ManageEstateAccess|(?:Create|Delete)Character|ExecCharacterCmd|Evade|FleeFrom|NavigateTo|PatrolPoints|Pursue|UpdateCharacter|WanderWithin))\b'
+ lsl_constants_float = r'\b(?:DEG_TO_RAD|PI(?:_BY_TWO)?|RAD_TO_DEG|SQRT2|TWO_PI)\b'
+ lsl_constants_integer = r'\b(?:JSON_APPEND|STATUS_(?:PHYSICS|ROTATE_[XYZ]|PHANTOM|SANDBOX|BLOCK_GRAB(?:_OBJECT)?|(?:DIE|RETURN)_AT_EDGE|CAST_SHADOWS|OK|MALFORMED_PARAMS|TYPE_MISMATCH|BOUNDS_ERROR|NOT_(?:FOUND|SUPPORTED)|INTERNAL_ERROR|WHITELIST_FAILED)|AGENT(?:_(?:BY_(?:LEGACY_|USER)NAME|FLYING|ATTACHMENTS|SCRIPTED|MOUSELOOK|SITTING|ON_OBJECT|AWAY|WALKING|IN_AIR|TYPING|CROUCHING|BUSY|ALWAYS_RUN|AUTOPILOT|LIST_(?:PARCEL(?:_OWNER)?|REGION)))?|CAMERA_(?:PITCH|DISTANCE|BEHINDNESS_(?:ANGLE|LAG)|(?:FOCUS|POSITION)(?:_(?:THRESHOLD|LOCKED|LAG))?|FOCUS_OFFSET|ACTIVE)|ANIM_ON|LOOP|REVERSE|PING_PONG|SMOOTH|ROTATE|SCALE|ALL_SIDES|LINK_(?:ROOT|SET|ALL_(?:OTHERS|CHILDREN)|THIS)|ACTIVE|PASSIVE|SCRIPTED|CONTROL_(?:FWD|BACK|(?:ROT_)?(?:LEFT|RIGHT)|UP|DOWN|(?:ML_)?LBUTTON)|PERMISSION_(?:RETURN_OBJECTS|DEBIT|OVERRIDE_ANIMATIONS|SILENT_ESTATE_MANAGEMENT|TAKE_CONTROLS|TRIGGER_ANIMATION|ATTACH|CHANGE_LINKS|(?:CONTROL|TRACK)_CAMERA|TELEPORT)|INVENTORY_(?:TEXTURE|SOUND|OBJECT|SCRIPT|LANDMARK|CLOTHING|NOTECARD|BODYPART|ANIMATION|GESTURE|ALL|NONE)|CHANGED_(?:INVENTORY|COLOR|SHAPE|SCALE|TEXTURE|LINK|ALLOWED_DROP|OWNER|REGION(?:_START)?|TELEPORT|MEDIA)|OBJECT_(?:(?:PHYSICS|SERVER|STREAMING)_COST|UNKNOWN_DETAIL|CHARACTER_TIME|PHANTOM|PHYSICS|TEMP_ON_REZ|NAME|DESC|POS|PRIM_EQUIVALENCE|RETURN_(?:PARCEL(?:_OWNER)?|REGION)|ROO?T|VELOCITY|OWNER|GROUP|CREATOR|ATTACHED_POINT|RENDER_WEIGHT|PATHFINDING_TYPE|(?:RUNNING|TOTAL)_SCRIPT_COUNT|SCRIPT_(?:MEMORY|TIME))|TYPE_(?:INTEGER|FLOAT|STRING|KEY|VECTOR|ROTATION|INVALID)|(?:DEBUG|PUBLIC)_CHANNEL|ATTACH_(?:AVATAR_CENTER|CHEST|HEAD|BACK|PELVIS|MOUTH|CHIN|NECK|NOSE|BELLY|[LR](?:SHOULDER|HAND|FOOT|EAR|EYE|[UL](?:ARM|LEG)|HIP)|(?:LEFT|RIGHT)_PEC|HUD_(?:CENTER_[12]|TOP_(?:RIGHT|CENTER|LEFT)|BOTTOM(?:_(?:RIGHT|LEFT))?))|LAND_(?:LEVEL|RAISE|LOWER|SMOOTH|NOISE|REVERT)|DATA_(?:ONLINE|NAME|BORN|SIM_(?:POS|STATUS|RATING)|PAYINFO)|PAYMENT_INFO_(?:ON_FILE|USED)|REMOTE_DATA_(?:CHANNEL|REQUEST|REPLY)|PSYS_(?:PART_(?:BF_(?:ZERO|ONE(?:_MINUS_(?:DEST_COLOR|SOURCE_(ALPHA|COLOR)))?|DEST_COLOR|SOURCE_(ALPHA|COLOR))|BLEND_FUNC_(DEST|SOURCE)|FLAGS|(?:START|END)_(?:COLOR|ALPHA|SCALE|GLOW)|MAX_AGE|(?:RIBBON|WIND|INTERP_(?:COLOR|SCALE)|BOUNCE|FOLLOW_(?:SRC|VELOCITY)|TARGET_(?:POS|LINEAR)|EMISSIVE)_MASK)|SRC_(?:MAX_AGE|PATTERN|ANGLE_(?:BEGIN|END)|BURST_(?:RATE|PART_COUNT|RADIUS|SPEED_(?:MIN|MAX))|ACCEL|TEXTURE|TARGET_KEY|OMEGA|PATTERN_(?:DROP|EXPLODE|ANGLE(?:_CONE(?:_EMPTY)?)?)))|VEHICLE_(?:REFERENCE_FRAME|TYPE_(?:NONE|SLED|CAR|BOAT|AIRPLANE|BALLOON)|(?:LINEAR|ANGULAR)_(?:FRICTION_TIMESCALE|MOTOR_DIRECTION)|LINEAR_MOTOR_OFFSET|HOVER_(?:HEIGHT|EFFICIENCY|TIMESCALE)|BUOYANCY|(?:LINEAR|ANGULAR)_(?:DEFLECTION_(?:EFFICIENCY|TIMESCALE)|MOTOR_(?:DECAY_)?TIMESCALE)|VERTICAL_ATTRACTION_(?:EFFICIENCY|TIMESCALE)|BANKING_(?:EFFICIENCY|MIX|TIMESCALE)|FLAG_(?:NO_DEFLECTION_UP|LIMIT_(?:ROLL_ONLY|MOTOR_UP)|HOVER_(?:(?:WATER|TERRAIN|UP)_ONLY|GLOBAL_HEIGHT)|MOUSELOOK_(?:STEER|BANK)|CAMERA_DECOUPLED))|PRIM_(?:TYPE(?:_(?:BOX|CYLINDER|PRISM|SPHERE|TORUS|TUBE|RING|SCULPT))?|HOLE_(?:DEFAULT|CIRCLE|SQUARE|TRIANGLE)|MATERIAL(?:_(?:STONE|METAL|GLASS|WOOD|FLESH|PLASTIC|RUBBER))?|SHINY_(?:NONE|LOW|MEDIUM|HIGH)|BUMP_(?:NONE|BRIGHT|DARK|WOOD|BARK|BRICKS|CHECKER|CONCRETE|TILE|STONE|DISKS|GRAVEL|BLOBS|SIDING|LARGETILE|STUCCO|SUCTION|WEAVE)|TEXGEN_(?:DEFAULT|PLANAR)|SCULPT_(?:TYPE_(?:SPHERE|TORUS|PLANE|CYLINDER|MASK)|FLAG_(?:MIRROR|INVERT))|PHYSICS(?:_(?:SHAPE_(?:CONVEX|NONE|PRIM|TYPE)))?|(?:POS|ROT)_LOCAL|SLICE|TEXT|FLEXIBLE|POINT_LIGHT|TEMP_ON_REZ|PHANTOM|POSITION|SIZE|ROTATION|TEXTURE|NAME|OMEGA|DESC|LINK_TARGET|COLOR|BUMP_SHINY|FULLBRIGHT|TEXGEN|GLOW|MEDIA_(?:ALT_IMAGE_ENABLE|CONTROLS|(?:CURRENT|HOME)_URL|AUTO_(?:LOOP|PLAY|SCALE|ZOOM)|FIRST_CLICK_INTERACT|(?:WIDTH|HEIGHT)_PIXELS|WHITELIST(?:_ENABLE)?|PERMS_(?:INTERACT|CONTROL)|PARAM_MAX|CONTROLS_(?:STANDARD|MINI)|PERM_(?:NONE|OWNER|GROUP|ANYONE)|MAX_(?:URL_LENGTH|WHITELIST_(?:SIZE|COUNT)|(?:WIDTH|HEIGHT)_PIXELS)))|MASK_(?:BASE|OWNER|GROUP|EVERYONE|NEXT)|PERM_(?:TRANSFER|MODIFY|COPY|MOVE|ALL)|PARCEL_(?:MEDIA_COMMAND_(?:STOP|PAUSE|PLAY|LOOP|TEXTURE|URL|TIME|AGENT|UNLOAD|AUTO_ALIGN|TYPE|SIZE|DESC|LOOP_SET)|FLAG_(?:ALLOW_(?:FLY|(?:GROUP_)?SCRIPTS|LANDMARK|TERRAFORM|DAMAGE|CREATE_(?:GROUP_)?OBJECTS)|USE_(?:ACCESS_(?:GROUP|LIST)|BAN_LIST|LAND_PASS_LIST)|LOCAL_SOUND_ONLY|RESTRICT_PUSHOBJECT|ALLOW_(?:GROUP|ALL)_OBJECT_ENTRY)|COUNT_(?:TOTAL|OWNER|GROUP|OTHER|SELECTED|TEMP)|DETAILS_(?:NAME|DESC|OWNER|GROUP|AREA|ID|SEE_AVATARS))|LIST_STAT_(?:MAX|MIN|MEAN|MEDIAN|STD_DEV|SUM(?:_SQUARES)?|NUM_COUNT|GEOMETRIC_MEAN|RANGE)|PAY_(?:HIDE|DEFAULT)|REGION_FLAG_(?:ALLOW_DAMAGE|FIXED_SUN|BLOCK_TERRAFORM|SANDBOX|DISABLE_(?:COLLISIONS|PHYSICS)|BLOCK_FLY|ALLOW_DIRECT_TELEPORT|RESTRICT_PUSHOBJECT)|HTTP_(?:METHOD|MIMETYPE|BODY_(?:MAXLENGTH|TRUNCATED)|CUSTOM_HEADER|PRAGMA_NO_CACHE|VERBOSE_THROTTLE|VERIFY_CERT)|STRING_(?:TRIM(?:_(?:HEAD|TAIL))?)|CLICK_ACTION_(?:NONE|TOUCH|SIT|BUY|PAY|OPEN(?:_MEDIA)?|PLAY|ZOOM)|TOUCH_INVALID_FACE|PROFILE_(?:NONE|SCRIPT_MEMORY)|RC_(?:DATA_FLAGS|DETECT_PHANTOM|GET_(?:LINK_NUM|NORMAL|ROOT_KEY)|MAX_HITS|REJECT_(?:TYPES|AGENTS|(?:NON)?PHYSICAL|LAND))|RCERR_(?:CAST_TIME_EXCEEDED|SIM_PERF_LOW|UNKNOWN)|ESTATE_ACCESS_(?:ALLOWED_(?:AGENT|GROUP)_(?:ADD|REMOVE)|BANNED_AGENT_(?:ADD|REMOVE))|DENSITY|FRICTION|RESTITUTION|GRAVITY_MULTIPLIER|KFM_(?:COMMAND|CMD_(?:PLAY|STOP|PAUSE|SET_MODE)|MODE|FORWARD|LOOP|PING_PONG|REVERSE|DATA|ROTATION|TRANSLATION)|ERR_(?:GENERIC|PARCEL_PERMISSIONS|MALFORMED_PARAMS|RUNTIME_PERMISSIONS|THROTTLED)|CHARACTER_(?:CMD_(?:(?:SMOOTH_)?STOP|JUMP)|DESIRED_(?:TURN_)?SPEED|RADIUS|STAY_WITHIN_PARCEL|LENGTH|ORIENTATION|ACCOUNT_FOR_SKIPPED_FRAMES|AVOIDANCE_MODE|TYPE(?:_(?:[A-D]|NONE))?|MAX_(?:DECEL|TURN_RADIUS|(?:ACCEL|SPEED)))|PURSUIT_(?:OFFSET|FUZZ_FACTOR|GOAL_TOLERANCE|INTERCEPT)|REQUIRE_LINE_OF_SIGHT|FORCE_DIRECT_PATH|VERTICAL|HORIZONTAL|AVOID_(?:CHARACTERS|DYNAMIC_OBSTACLES|NONE)|PU_(?:EVADE_(?:HIDDEN|SPOTTED)|FAILURE_(?:DYNAMIC_PATHFINDING_DISABLED|INVALID_(?:GOAL|START)|NO_(?:NAVMESH|VALID_DESTINATION)|OTHER|TARGET_GONE|(?:PARCEL_)?UNREACHABLE)|(?:GOAL|SLOWDOWN_DISTANCE)_REACHED)|TRAVERSAL_TYPE(?:_(?:FAST|NONE|SLOW))?|CONTENT_TYPE_(?:ATOM|FORM|HTML|JSON|LLSD|RSS|TEXT|XHTML|XML)|GCNP_(?:RADIUS|STATIC)|(?:PATROL|WANDER)_PAUSE_AT_WAYPOINTS|OPT_(?:AVATAR|CHARACTER|EXCLUSION_VOLUME|LEGACY_LINKSET|MATERIAL_VOLUME|OTHER|STATIC_OBSTACLE|WALKABLE)|SIM_STAT_PCT_CHARS_STEPPED)\b'
+ lsl_constants_integer_boolean = r'\b(?:FALSE|TRUE)\b'
+ lsl_constants_rotation = r'\b(?:ZERO_ROTATION)\b'
+ lsl_constants_string = r'\b(?:EOF|JSON_(?:ARRAY|DELETE|FALSE|INVALID|NULL|NUMBER|OBJECT|STRING|TRUE)|NULL_KEY|TEXTURE_(?:BLANK|DEFAULT|MEDIA|PLYWOOD|TRANSPARENT)|URL_REQUEST_(?:GRANTED|DENIED))\b'
+ lsl_constants_vector = r'\b(?:TOUCH_INVALID_(?:TEXCOORD|VECTOR)|ZERO_VECTOR)\b'
+ lsl_invalid_broken = r'\b(?:LAND_(?:LARGE|MEDIUM|SMALL)_BRUSH)\b'
+ lsl_invalid_deprecated = r'\b(?:ATTACH_[LR]PEC|DATA_RATING|OBJECT_ATTACHMENT_(?:GEOMETRY_BYTES|SURFACE_AREA)|PRIM_(?:CAST_SHADOWS|MATERIAL_LIGHT|TYPE_LEGACY)|PSYS_SRC_(?:INNER|OUTER)ANGLE|VEHICLE_FLAG_NO_FLY_UP|ll(?:Cloud|Make(?:Explosion|Fountain|Smoke|Fire)|RemoteDataSetRegion|Sound(?:Preload)?|XorBase64Strings(?:Correct)?))\b'
+ lsl_invalid_illegal = r'\b(?:event)\b'
+ lsl_invalid_unimplemented = r'\b(?:CHARACTER_(?:MAX_ANGULAR_(?:ACCEL|SPEED)|TURN_SPEED_MULTIPLIER)|PERMISSION_(?:CHANGE_(?:JOINTS|PERMISSIONS)|RELEASE_OWNERSHIP|REMAP_CONTROLS)|PRIM_PHYSICS_MATERIAL|PSYS_SRC_OBJ_REL_MASK|ll(?:CollisionSprite|(?:Stop)?PointAt|(?:(?:Refresh|Set)Prim)URL|(?:Take|Release)Camera|RemoteLoadScript))\b'
+ lsl_reserved_godmode = r'\b(?:ll(?:GodLikeRezObject|Set(?:Inventory|Object)PermMask))\b'
+ lsl_reserved_log = r'\b(?:print)\b'
+ lsl_operators = r'\+\+|\-\-|<<|>>|&&?|\|\|?|\^|~|[!%<>=*+\-/]=?'
+
+ tokens = {
+ 'root':
+ [
+ (r'//.*?\n', Comment.Single),
+ (r'/\*', Comment.Multiline, 'comment'),
+ (r'"', String.Double, 'string'),
+ (lsl_keywords, Keyword),
+ (lsl_types, Keyword.Type),
+ (lsl_states, Name.Class),
+ (lsl_events, Name.Builtin),
+ (lsl_functions_builtin, Name.Function),
+ (lsl_constants_float, Keyword.Constant),
+ (lsl_constants_integer, Keyword.Constant),
+ (lsl_constants_integer_boolean, Keyword.Constant),
+ (lsl_constants_rotation, Keyword.Constant),
+ (lsl_constants_string, Keyword.Constant),
+ (lsl_constants_vector, Keyword.Constant),
+ (lsl_invalid_broken, Error),
+ (lsl_invalid_deprecated, Error),
+ (lsl_invalid_illegal, Error),
+ (lsl_invalid_unimplemented, Error),
+ (lsl_reserved_godmode, Keyword.Reserved),
+ (lsl_reserved_log, Keyword.Reserved),
+ (r'\b([a-zA-Z_]\w*)\b', Name.Variable),
+ (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d*', Number.Float),
+ (r'(\d+\.\d*|\.\d+)', Number.Float),
+ (r'0[xX][0-9a-fA-F]+', Number.Hex),
+ (r'\d+', Number.Integer),
+ (lsl_operators, Operator),
+ (r':=?', Error),
+ (r'[,;{}()\[\]]', Punctuation),
+ (r'\n+', Whitespace),
+ (r'\s+', Whitespace)
+ ],
+ 'comment':
+ [
+ (r'[^*/]+', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline)
+ ],
+ 'string':
+ [
+ (r'\\([nt"\\])', String.Escape),
+ (r'"', String.Double, '#pop'),
+ (r'\\.', Error),
+ (r'[^"\\]+', String.Double),
+ ]
+ }
+
+
+class AppleScriptLexer(RegexLexer):
+ """
+ For `AppleScript source code
+ <http://developer.apple.com/documentation/AppleScript/
+ Conceptual/AppleScriptLangGuide>`_,
+ including `AppleScript Studio
+ <http://developer.apple.com/documentation/AppleScript/
+ Reference/StudioReference>`_.
+ Contributed by Andreas Amann <aamann@mac.com>.
+
+ .. versionadded:: 1.0
+ """
+
+ name = 'AppleScript'
+ aliases = ['applescript']
+ filenames = ['*.applescript']
+
+ flags = re.MULTILINE | re.DOTALL
+
+ Identifiers = r'[a-zA-Z]\w*'
+
+ # XXX: use words() for all of these
+ Literals = ('AppleScript', 'current application', 'false', 'linefeed',
+ 'missing value', 'pi', 'quote', 'result', 'return', 'space',
+ 'tab', 'text item delimiters', 'true', 'version')
+ Classes = ('alias ', 'application ', 'boolean ', 'class ', 'constant ',
+ 'date ', 'file ', 'integer ', 'list ', 'number ', 'POSIX file ',
+ 'real ', 'record ', 'reference ', 'RGB color ', 'script ',
+ 'text ', 'unit types', '(?:Unicode )?text', 'string')
+ BuiltIn = ('attachment', 'attribute run', 'character', 'day', 'month',
+ 'paragraph', 'word', 'year')
+ HandlerParams = ('about', 'above', 'against', 'apart from', 'around',
+ 'aside from', 'at', 'below', 'beneath', 'beside',
+ 'between', 'for', 'given', 'instead of', 'on', 'onto',
+ 'out of', 'over', 'since')
+ Commands = ('ASCII (character|number)', 'activate', 'beep', 'choose URL',
+ 'choose application', 'choose color', 'choose file( name)?',
+ 'choose folder', 'choose from list',
+ 'choose remote application', 'clipboard info',
+ 'close( access)?', 'copy', 'count', 'current date', 'delay',
+ 'delete', 'display (alert|dialog)', 'do shell script',
+ 'duplicate', 'exists', 'get eof', 'get volume settings',
+ 'info for', 'launch', 'list (disks|folder)', 'load script',
+ 'log', 'make', 'mount volume', 'new', 'offset',
+ 'open( (for access|location))?', 'path to', 'print', 'quit',
+ 'random number', 'read', 'round', 'run( script)?',
+ 'say', 'scripting components',
+ 'set (eof|the clipboard to|volume)', 'store script',
+ 'summarize', 'system attribute', 'system info',
+ 'the clipboard', 'time to GMT', 'write', 'quoted form')
+ References = ('(in )?back of', '(in )?front of', '[0-9]+(st|nd|rd|th)',
+ 'first', 'second', 'third', 'fourth', 'fifth', 'sixth',
+ 'seventh', 'eighth', 'ninth', 'tenth', 'after', 'back',
+ 'before', 'behind', 'every', 'front', 'index', 'last',
+ 'middle', 'some', 'that', 'through', 'thru', 'where', 'whose')
+ Operators = ("and", "or", "is equal", "equals", "(is )?equal to", "is not",
+ "isn't", "isn't equal( to)?", "is not equal( to)?",
+ "doesn't equal", "does not equal", "(is )?greater than",
+ "comes after", "is not less than or equal( to)?",
+ "isn't less than or equal( to)?", "(is )?less than",
+ "comes before", "is not greater than or equal( to)?",
+ "isn't greater than or equal( to)?",
+ "(is )?greater than or equal( to)?", "is not less than",
+ "isn't less than", "does not come before",
+ "doesn't come before", "(is )?less than or equal( to)?",
+ "is not greater than", "isn't greater than",
+ "does not come after", "doesn't come after", "starts? with",
+ "begins? with", "ends? with", "contains?", "does not contain",
+ "doesn't contain", "is in", "is contained by", "is not in",
+ "is not contained by", "isn't contained by", "div", "mod",
+ "not", "(a )?(ref( to)?|reference to)", "is", "does")
+ Control = ('considering', 'else', 'error', 'exit', 'from', 'if',
+ 'ignoring', 'in', 'repeat', 'tell', 'then', 'times', 'to',
+ 'try', 'until', 'using terms from', 'while', 'whith',
+ 'with timeout( of)?', 'with transaction', 'by', 'continue',
+ 'end', 'its?', 'me', 'my', 'return', 'of', 'as')
+ Declarations = ('global', 'local', 'prop(erty)?', 'set', 'get')
+ Reserved = ('but', 'put', 'returning', 'the')
+ StudioClasses = ('action cell', 'alert reply', 'application', 'box',
+ 'browser( cell)?', 'bundle', 'button( cell)?', 'cell',
+ 'clip view', 'color well', 'color-panel',
+ 'combo box( item)?', 'control',
+ 'data( (cell|column|item|row|source))?', 'default entry',
+ 'dialog reply', 'document', 'drag info', 'drawer',
+ 'event', 'font(-panel)?', 'formatter',
+ 'image( (cell|view))?', 'matrix', 'menu( item)?', 'item',
+ 'movie( view)?', 'open-panel', 'outline view', 'panel',
+ 'pasteboard', 'plugin', 'popup button',
+ 'progress indicator', 'responder', 'save-panel',
+ 'scroll view', 'secure text field( cell)?', 'slider',
+ 'sound', 'split view', 'stepper', 'tab view( item)?',
+ 'table( (column|header cell|header view|view))',
+ 'text( (field( cell)?|view))?', 'toolbar( item)?',
+ 'user-defaults', 'view', 'window')
+ StudioEvents = ('accept outline drop', 'accept table drop', 'action',
+ 'activated', 'alert ended', 'awake from nib', 'became key',
+ 'became main', 'begin editing', 'bounds changed',
+ 'cell value', 'cell value changed', 'change cell value',
+ 'change item value', 'changed', 'child of item',
+ 'choose menu item', 'clicked', 'clicked toolbar item',
+ 'closed', 'column clicked', 'column moved',
+ 'column resized', 'conclude drop', 'data representation',
+ 'deminiaturized', 'dialog ended', 'document nib name',
+ 'double clicked', 'drag( (entered|exited|updated))?',
+ 'drop', 'end editing', 'exposed', 'idle', 'item expandable',
+ 'item value', 'item value changed', 'items changed',
+ 'keyboard down', 'keyboard up', 'launched',
+ 'load data representation', 'miniaturized', 'mouse down',
+ 'mouse dragged', 'mouse entered', 'mouse exited',
+ 'mouse moved', 'mouse up', 'moved',
+ 'number of browser rows', 'number of items',
+ 'number of rows', 'open untitled', 'opened', 'panel ended',
+ 'parameters updated', 'plugin loaded', 'prepare drop',
+ 'prepare outline drag', 'prepare outline drop',
+ 'prepare table drag', 'prepare table drop',
+ 'read from file', 'resigned active', 'resigned key',
+ 'resigned main', 'resized( sub views)?',
+ 'right mouse down', 'right mouse dragged',
+ 'right mouse up', 'rows changed', 'scroll wheel',
+ 'selected tab view item', 'selection changed',
+ 'selection changing', 'should begin editing',
+ 'should close', 'should collapse item',
+ 'should end editing', 'should expand item',
+ 'should open( untitled)?',
+ 'should quit( after last window closed)?',
+ 'should select column', 'should select item',
+ 'should select row', 'should select tab view item',
+ 'should selection change', 'should zoom', 'shown',
+ 'update menu item', 'update parameters',
+ 'update toolbar item', 'was hidden', 'was miniaturized',
+ 'will become active', 'will close', 'will dismiss',
+ 'will display browser cell', 'will display cell',
+ 'will display item cell', 'will display outline cell',
+ 'will finish launching', 'will hide', 'will miniaturize',
+ 'will move', 'will open', 'will pop up', 'will quit',
+ 'will resign active', 'will resize( sub views)?',
+ 'will select tab view item', 'will show', 'will zoom',
+ 'write to file', 'zoomed')
+ StudioCommands = ('animate', 'append', 'call method', 'center',
+ 'close drawer', 'close panel', 'display',
+ 'display alert', 'display dialog', 'display panel', 'go',
+ 'hide', 'highlight', 'increment', 'item for',
+ 'load image', 'load movie', 'load nib', 'load panel',
+ 'load sound', 'localized string', 'lock focus', 'log',
+ 'open drawer', 'path for', 'pause', 'perform action',
+ 'play', 'register', 'resume', 'scroll', 'select( all)?',
+ 'show', 'size to fit', 'start', 'step back',
+ 'step forward', 'stop', 'synchronize', 'unlock focus',
+ 'update')
+ StudioProperties = ('accepts arrow key', 'action method', 'active',
+ 'alignment', 'allowed identifiers',
+ 'allows branch selection', 'allows column reordering',
+ 'allows column resizing', 'allows column selection',
+ 'allows customization',
+ 'allows editing text attributes',
+ 'allows empty selection', 'allows mixed state',
+ 'allows multiple selection', 'allows reordering',
+ 'allows undo', 'alpha( value)?', 'alternate image',
+ 'alternate increment value', 'alternate title',
+ 'animation delay', 'associated file name',
+ 'associated object', 'auto completes', 'auto display',
+ 'auto enables items', 'auto repeat',
+ 'auto resizes( outline column)?',
+ 'auto save expanded items', 'auto save name',
+ 'auto save table columns', 'auto saves configuration',
+ 'auto scroll', 'auto sizes all columns to fit',
+ 'auto sizes cells', 'background color', 'bezel state',
+ 'bezel style', 'bezeled', 'border rect', 'border type',
+ 'bordered', 'bounds( rotation)?', 'box type',
+ 'button returned', 'button type',
+ 'can choose directories', 'can choose files',
+ 'can draw', 'can hide',
+ 'cell( (background color|size|type))?', 'characters',
+ 'class', 'click count', 'clicked( data)? column',
+ 'clicked data item', 'clicked( data)? row',
+ 'closeable', 'collating', 'color( (mode|panel))',
+ 'command key down', 'configuration',
+ 'content(s| (size|view( margins)?))?', 'context',
+ 'continuous', 'control key down', 'control size',
+ 'control tint', 'control view',
+ 'controller visible', 'coordinate system',
+ 'copies( on scroll)?', 'corner view', 'current cell',
+ 'current column', 'current( field)? editor',
+ 'current( menu)? item', 'current row',
+ 'current tab view item', 'data source',
+ 'default identifiers', 'delta (x|y|z)',
+ 'destination window', 'directory', 'display mode',
+ 'displayed cell', 'document( (edited|rect|view))?',
+ 'double value', 'dragged column', 'dragged distance',
+ 'dragged items', 'draws( cell)? background',
+ 'draws grid', 'dynamically scrolls', 'echos bullets',
+ 'edge', 'editable', 'edited( data)? column',
+ 'edited data item', 'edited( data)? row', 'enabled',
+ 'enclosing scroll view', 'ending page',
+ 'error handling', 'event number', 'event type',
+ 'excluded from windows menu', 'executable path',
+ 'expanded', 'fax number', 'field editor', 'file kind',
+ 'file name', 'file type', 'first responder',
+ 'first visible column', 'flipped', 'floating',
+ 'font( panel)?', 'formatter', 'frameworks path',
+ 'frontmost', 'gave up', 'grid color', 'has data items',
+ 'has horizontal ruler', 'has horizontal scroller',
+ 'has parent data item', 'has resize indicator',
+ 'has shadow', 'has sub menu', 'has vertical ruler',
+ 'has vertical scroller', 'header cell', 'header view',
+ 'hidden', 'hides when deactivated', 'highlights by',
+ 'horizontal line scroll', 'horizontal page scroll',
+ 'horizontal ruler view', 'horizontally resizable',
+ 'icon image', 'id', 'identifier',
+ 'ignores multiple clicks',
+ 'image( (alignment|dims when disabled|frame style|scaling))?',
+ 'imports graphics', 'increment value',
+ 'indentation per level', 'indeterminate', 'index',
+ 'integer value', 'intercell spacing', 'item height',
+ 'key( (code|equivalent( modifier)?|window))?',
+ 'knob thickness', 'label', 'last( visible)? column',
+ 'leading offset', 'leaf', 'level', 'line scroll',
+ 'loaded', 'localized sort', 'location', 'loop mode',
+ 'main( (bunde|menu|window))?', 'marker follows cell',
+ 'matrix mode', 'maximum( content)? size',
+ 'maximum visible columns',
+ 'menu( form representation)?', 'miniaturizable',
+ 'miniaturized', 'minimized image', 'minimized title',
+ 'minimum column width', 'minimum( content)? size',
+ 'modal', 'modified', 'mouse down state',
+ 'movie( (controller|file|rect))?', 'muted', 'name',
+ 'needs display', 'next state', 'next text',
+ 'number of tick marks', 'only tick mark values',
+ 'opaque', 'open panel', 'option key down',
+ 'outline table column', 'page scroll', 'pages across',
+ 'pages down', 'palette label', 'pane splitter',
+ 'parent data item', 'parent window', 'pasteboard',
+ 'path( (names|separator))?', 'playing',
+ 'plays every frame', 'plays selection only', 'position',
+ 'preferred edge', 'preferred type', 'pressure',
+ 'previous text', 'prompt', 'properties',
+ 'prototype cell', 'pulls down', 'rate',
+ 'released when closed', 'repeated',
+ 'requested print time', 'required file type',
+ 'resizable', 'resized column', 'resource path',
+ 'returns records', 'reuses columns', 'rich text',
+ 'roll over', 'row height', 'rulers visible',
+ 'save panel', 'scripts path', 'scrollable',
+ 'selectable( identifiers)?', 'selected cell',
+ 'selected( data)? columns?', 'selected data items?',
+ 'selected( data)? rows?', 'selected item identifier',
+ 'selection by rect', 'send action on arrow key',
+ 'sends action when done editing', 'separates columns',
+ 'separator item', 'sequence number', 'services menu',
+ 'shared frameworks path', 'shared support path',
+ 'sheet', 'shift key down', 'shows alpha',
+ 'shows state by', 'size( mode)?',
+ 'smart insert delete enabled', 'sort case sensitivity',
+ 'sort column', 'sort order', 'sort type',
+ 'sorted( data rows)?', 'sound', 'source( mask)?',
+ 'spell checking enabled', 'starting page', 'state',
+ 'string value', 'sub menu', 'super menu', 'super view',
+ 'tab key traverses cells', 'tab state', 'tab type',
+ 'tab view', 'table view', 'tag', 'target( printer)?',
+ 'text color', 'text container insert',
+ 'text container origin', 'text returned',
+ 'tick mark position', 'time stamp',
+ 'title(d| (cell|font|height|position|rect))?',
+ 'tool tip', 'toolbar', 'trailing offset', 'transparent',
+ 'treat packages as directories', 'truncated labels',
+ 'types', 'unmodified characters', 'update views',
+ 'use sort indicator', 'user defaults',
+ 'uses data source', 'uses ruler',
+ 'uses threaded animation',
+ 'uses title from previous column', 'value wraps',
+ 'version',
+ 'vertical( (line scroll|page scroll|ruler view))?',
+ 'vertically resizable', 'view',
+ 'visible( document rect)?', 'volume', 'width', 'window',
+ 'windows menu', 'wraps', 'zoomable', 'zoomed')
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (u'¬\\n', String.Escape),
+ (r"'s\s+", Text), # This is a possessive, consider moving
+ (r'(--|#).*?$', Comment),
+ (r'\(\*', Comment.Multiline, 'comment'),
+ (r'[(){}!,.:]', Punctuation),
+ (u'(«)([^»]+)(»)',
+ bygroups(Text, Name.Builtin, Text)),
+ (r'\b((?:considering|ignoring)\s*)'
+ r'(application responses|case|diacriticals|hyphens|'
+ r'numeric strings|punctuation|white space)',
+ bygroups(Keyword, Name.Builtin)),
+ (u'(-|\\*|\\+|&|≠|>=?|<=?|=|≥|≤|/|÷|\\^)', Operator),
+ (r"\b(%s)\b" % '|'.join(Operators), Operator.Word),
+ (r'^(\s*(?:on|end)\s+)'
+ r'(%s)' % '|'.join(StudioEvents[::-1]),
+ bygroups(Keyword, Name.Function)),
+ (r'^(\s*)(in|on|script|to)(\s+)', bygroups(Text, Keyword, Text)),
+ (r'\b(as )(%s)\b' % '|'.join(Classes),
+ bygroups(Keyword, Name.Class)),
+ (r'\b(%s)\b' % '|'.join(Literals), Name.Constant),
+ (r'\b(%s)\b' % '|'.join(Commands), Name.Builtin),
+ (r'\b(%s)\b' % '|'.join(Control), Keyword),
+ (r'\b(%s)\b' % '|'.join(Declarations), Keyword),
+ (r'\b(%s)\b' % '|'.join(Reserved), Name.Builtin),
+ (r'\b(%s)s?\b' % '|'.join(BuiltIn), Name.Builtin),
+ (r'\b(%s)\b' % '|'.join(HandlerParams), Name.Builtin),
+ (r'\b(%s)\b' % '|'.join(StudioProperties), Name.Attribute),
+ (r'\b(%s)s?\b' % '|'.join(StudioClasses), Name.Builtin),
+ (r'\b(%s)\b' % '|'.join(StudioCommands), Name.Builtin),
+ (r'\b(%s)\b' % '|'.join(References), Name.Builtin),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r'\b(%s)\b' % Identifiers, Name.Variable),
+ (r'[-+]?(\d+\.\d*|\d*\.\d+)(E[-+][0-9]+)?', Number.Float),
+ (r'[-+]?\d+', Number.Integer),
+ ],
+ 'comment': [
+ ('\(\*', Comment.Multiline, '#push'),
+ ('\*\)', Comment.Multiline, '#pop'),
+ ('[^*(]+', Comment.Multiline),
+ ('[*(]', Comment.Multiline),
+ ],
+ }
+
+
+class RexxLexer(RegexLexer):
+ """
+ `Rexx <http://www.rexxinfo.org/>`_ is a scripting language available for
+ a wide range of different platforms with its roots found on mainframe
+ systems. It is popular for I/O- and data based tasks and can act as glue
+ language to bind different applications together.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Rexx'
+ aliases = ['rexx', 'arexx']
+ filenames = ['*.rexx', '*.rex', '*.rx', '*.arexx']
+ mimetypes = ['text/x-rexx']
+ flags = re.IGNORECASE
+
+ tokens = {
+ 'root': [
+ (r'\s', Whitespace),
+ (r'/\*', Comment.Multiline, 'comment'),
+ (r'"', String, 'string_double'),
+ (r"'", String, 'string_single'),
+ (r'[0-9]+(\.[0-9]+)?(e[+-]?[0-9])?', Number),
+ (r'([a-z_]\w*)(\s*)(:)(\s*)(procedure)\b',
+ bygroups(Name.Function, Whitespace, Operator, Whitespace,
+ Keyword.Declaration)),
+ (r'([a-z_]\w*)(\s*)(:)',
+ bygroups(Name.Label, Whitespace, Operator)),
+ include('function'),
+ include('keyword'),
+ include('operator'),
+ (r'[a-z_]\w*', Text),
+ ],
+ 'function': [
+ (words((
+ 'abbrev', 'abs', 'address', 'arg', 'b2x', 'bitand', 'bitor', 'bitxor',
+ 'c2d', 'c2x', 'center', 'charin', 'charout', 'chars', 'compare',
+ 'condition', 'copies', 'd2c', 'd2x', 'datatype', 'date', 'delstr',
+ 'delword', 'digits', 'errortext', 'form', 'format', 'fuzz', 'insert',
+ 'lastpos', 'left', 'length', 'linein', 'lineout', 'lines', 'max',
+ 'min', 'overlay', 'pos', 'queued', 'random', 'reverse', 'right', 'sign',
+ 'sourceline', 'space', 'stream', 'strip', 'substr', 'subword', 'symbol',
+ 'time', 'trace', 'translate', 'trunc', 'value', 'verify', 'word',
+ 'wordindex', 'wordlength', 'wordpos', 'words', 'x2b', 'x2c', 'x2d',
+ 'xrange'), suffix=r'(\s*)(\()'),
+ bygroups(Name.Builtin, Whitespace, Operator)),
+ ],
+ 'keyword': [
+ (r'(address|arg|by|call|do|drop|else|end|exit|for|forever|if|'
+ r'interpret|iterate|leave|nop|numeric|off|on|options|parse|'
+ r'pull|push|queue|return|say|select|signal|to|then|trace|until|'
+ r'while)\b', Keyword.Reserved),
+ ],
+ 'operator': [
+ (r'(-|//|/|\(|\)|\*\*|\*|\\<<|\\<|\\==|\\=|\\>>|\\>|\\|\|\||\||'
+ r'&&|&|%|\+|<<=|<<|<=|<>|<|==|=|><|>=|>>=|>>|>|¬<<|¬<|¬==|¬=|'
+ r'¬>>|¬>|¬|\.|,)', Operator),
+ ],
+ 'string_double': [
+ (r'[^"\n]+', String),
+ (r'""', String),
+ (r'"', String, '#pop'),
+ (r'\n', Text, '#pop'), # Stray linefeed also terminates strings.
+ ],
+ 'string_single': [
+ (r'[^\'\n]', String),
+ (r'\'\'', String),
+ (r'\'', String, '#pop'),
+ (r'\n', Text, '#pop'), # Stray linefeed also terminates strings.
+ ],
+ 'comment': [
+ (r'[^*]+', Comment.Multiline),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'\*', Comment.Multiline),
+ ]
+ }
+
+ _c = lambda s: re.compile(s, re.MULTILINE)
+ _ADDRESS_COMMAND_PATTERN = _c(r'^\s*address\s+command\b')
+ _ADDRESS_PATTERN = _c(r'^\s*address\s+')
+ _DO_WHILE_PATTERN = _c(r'^\s*do\s+while\b')
+ _IF_THEN_DO_PATTERN = _c(r'^\s*if\b.+\bthen\s+do\s*$')
+ _PROCEDURE_PATTERN = _c(r'^\s*([a-z_]\w*)(\s*)(:)(\s*)(procedure)\b')
+ _ELSE_DO_PATTERN = _c(r'\belse\s+do\s*$')
+ _PARSE_ARG_PATTERN = _c(r'^\s*parse\s+(upper\s+)?(arg|value)\b')
+ PATTERNS_AND_WEIGHTS = (
+ (_ADDRESS_COMMAND_PATTERN, 0.2),
+ (_ADDRESS_PATTERN, 0.05),
+ (_DO_WHILE_PATTERN, 0.1),
+ (_ELSE_DO_PATTERN, 0.1),
+ (_IF_THEN_DO_PATTERN, 0.1),
+ (_PROCEDURE_PATTERN, 0.5),
+ (_PARSE_ARG_PATTERN, 0.2),
+ )
+
+ def analyse_text(text):
+ """
+ Check for inital comment and patterns that distinguish Rexx from other
+ C-like languages.
+ """
+ if re.search(r'/\*\**\s*rexx', text, re.IGNORECASE):
+ # Header matches MVS Rexx requirements, this is certainly a Rexx
+ # script.
+ return 1.0
+ elif text.startswith('/*'):
+ # Header matches general Rexx requirements; the source code might
+ # still be any language using C comments such as C++, C# or Java.
+ lowerText = text.lower()
+ result = sum(weight
+ for (pattern, weight) in RexxLexer.PATTERNS_AND_WEIGHTS
+ if pattern.search(lowerText)) + 0.01
+ return min(result, 1.0)
+
+
+class MOOCodeLexer(RegexLexer):
+ """
+ For `MOOCode <http://www.moo.mud.org/>`_ (the MOO scripting
+ language).
+
+ .. versionadded:: 0.9
+ """
+ name = 'MOOCode'
+ filenames = ['*.moo']
+ aliases = ['moocode', 'moo']
+ mimetypes = ['text/x-moocode']
+
+ tokens = {
+ 'root': [
+ # Numbers
+ (r'(0|[1-9][0-9_]*)', Number.Integer),
+ # Strings
+ (r'"(\\\\|\\"|[^"])*"', String),
+ # exceptions
+ (r'(E_PERM|E_DIV)', Name.Exception),
+ # db-refs
+ (r'((#[-0-9]+)|(\$\w+))', Name.Entity),
+ # Keywords
+ (r'\b(if|else|elseif|endif|for|endfor|fork|endfork|while'
+ r'|endwhile|break|continue|return|try'
+ r'|except|endtry|finally|in)\b', Keyword),
+ # builtins
+ (r'(random|length)', Name.Builtin),
+ # special variables
+ (r'(player|caller|this|args)', Name.Variable.Instance),
+ # skip whitespace
+ (r'\s+', Text),
+ (r'\n', Text),
+ # other operators
+ (r'([!;=,{}&|:.\[\]@()<>?]+)', Operator),
+ # function call
+ (r'(\w+)(\()', bygroups(Name.Function, Operator)),
+ # variables
+ (r'(\w+)', Text),
+ ]
+ }
+
+
+class HybrisLexer(RegexLexer):
+ """
+ For `Hybris <http://www.hybris-lang.org>`_ source code.
+
+ .. versionadded:: 1.4
+ """
+
+ name = 'Hybris'
+ aliases = ['hybris', 'hy']
+ filenames = ['*.hy', '*.hyb']
+ mimetypes = ['text/x-hybris', 'application/x-hybris']
+
+ flags = re.MULTILINE | re.DOTALL
+
+ tokens = {
+ 'root': [
+ # method names
+ (r'^(\s*(?:function|method|operator\s+)+?)'
+ r'([a-zA-Z_]\w*)'
+ r'(\s*)(\()', bygroups(Keyword, Name.Function, Text, Operator)),
+ (r'[^\S\n]+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'@[a-zA-Z_][\w.]*', Name.Decorator),
+ (r'(break|case|catch|next|default|do|else|finally|for|foreach|of|'
+ r'unless|if|new|return|switch|me|throw|try|while)\b', Keyword),
+ (r'(extends|private|protected|public|static|throws|function|method|'
+ r'operator)\b', Keyword.Declaration),
+ (r'(true|false|null|__FILE__|__LINE__|__VERSION__|__LIB_PATH__|'
+ r'__INC_PATH__)\b', Keyword.Constant),
+ (r'(class|struct)(\s+)',
+ bygroups(Keyword.Declaration, Text), 'class'),
+ (r'(import|include)(\s+)',
+ bygroups(Keyword.Namespace, Text), 'import'),
+ (words((
+ 'gc_collect', 'gc_mm_items', 'gc_mm_usage', 'gc_collect_threshold',
+ 'urlencode', 'urldecode', 'base64encode', 'base64decode', 'sha1', 'crc32',
+ 'sha2', 'md5', 'md5_file', 'acos', 'asin', 'atan', 'atan2', 'ceil', 'cos',
+ 'cosh', 'exp', 'fabs', 'floor', 'fmod', 'log', 'log10', 'pow', 'sin',
+ 'sinh', 'sqrt', 'tan', 'tanh', 'isint', 'isfloat', 'ischar', 'isstring',
+ 'isarray', 'ismap', 'isalias', 'typeof', 'sizeof', 'toint', 'tostring',
+ 'fromxml', 'toxml', 'binary', 'pack', 'load', 'eval', 'var_names',
+ 'var_values', 'user_functions', 'dyn_functions', 'methods', 'call',
+ 'call_method', 'mknod', 'mkfifo', 'mount', 'umount2', 'umount', 'ticks',
+ 'usleep', 'sleep', 'time', 'strtime', 'strdate', 'dllopen', 'dlllink',
+ 'dllcall', 'dllcall_argv', 'dllclose', 'env', 'exec', 'fork', 'getpid',
+ 'wait', 'popen', 'pclose', 'exit', 'kill', 'pthread_create',
+ 'pthread_create_argv', 'pthread_exit', 'pthread_join', 'pthread_kill',
+ 'smtp_send', 'http_get', 'http_post', 'http_download', 'socket', 'bind',
+ 'listen', 'accept', 'getsockname', 'getpeername', 'settimeout', 'connect',
+ 'server', 'recv', 'send', 'close', 'print', 'println', 'printf', 'input',
+ 'readline', 'serial_open', 'serial_fcntl', 'serial_get_attr',
+ 'serial_get_ispeed', 'serial_get_ospeed', 'serial_set_attr',
+ 'serial_set_ispeed', 'serial_set_ospeed', 'serial_write', 'serial_read',
+ 'serial_close', 'xml_load', 'xml_parse', 'fopen', 'fseek', 'ftell',
+ 'fsize', 'fread', 'fwrite', 'fgets', 'fclose', 'file', 'readdir',
+ 'pcre_replace', 'size', 'pop', 'unmap', 'has', 'keys', 'values',
+ 'length', 'find', 'substr', 'replace', 'split', 'trim', 'remove',
+ 'contains', 'join'), suffix=r'\b'),
+ Name.Builtin),
+ (words((
+ 'MethodReference', 'Runner', 'Dll', 'Thread', 'Pipe', 'Process',
+ 'Runnable', 'CGI', 'ClientSocket', 'Socket', 'ServerSocket',
+ 'File', 'Console', 'Directory', 'Exception'), suffix=r'\b'),
+ Keyword.Type),
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r"'\\.'|'[^\\]'|'\\u[0-9a-f]{4}'", String.Char),
+ (r'(\.)([a-zA-Z_]\w*)',
+ bygroups(Operator, Name.Attribute)),
+ (r'[a-zA-Z_]\w*:', Name.Label),
+ (r'[a-zA-Z_$]\w*', Name),
+ (r'[~^*!%&\[\](){}<>|+=:;,./?\-@]+', Operator),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-f]+', Number.Hex),
+ (r'[0-9]+L?', Number.Integer),
+ (r'\n', Text),
+ ],
+ 'class': [
+ (r'[a-zA-Z_]\w*', Name.Class, '#pop')
+ ],
+ 'import': [
+ (r'[\w.]+\*?', Name.Namespace, '#pop')
+ ],
+ }
+
+
+class EasytrieveLexer(RegexLexer):
+ """
+ Easytrieve Plus is a programming language for extracting, filtering and
+ converting sequential data. Furthermore it can layout data for reports.
+ It is mainly used on mainframe platforms and can access several of the
+ mainframe's native file formats. It is somewhat comparable to awk.
+
+ .. versionadded:: 2.1
+ """
+ name = 'Easytrieve'
+ aliases = ['easytrieve']
+ filenames = ['*.ezt', '*.mac']
+ mimetypes = ['text/x-easytrieve']
+ flags = 0
+
+ # Note: We cannot use r'\b' at the start and end of keywords because
+ # Easytrieve Plus delimiter characters are:
+ #
+ # * space ( )
+ # * apostrophe (')
+ # * period (.)
+ # * comma (,)
+ # * paranthesis ( and )
+ # * colon (:)
+ #
+ # Additionally words end once a '*' appears, indicatins a comment.
+ _DELIMITERS = r' \'.,():\n'
+ _DELIMITERS_OR_COMENT = _DELIMITERS + '*'
+ _DELIMITER_PATTERN = '[' + _DELIMITERS + ']'
+ _DELIMITER_PATTERN_CAPTURE = '(' + _DELIMITER_PATTERN + ')'
+ _NON_DELIMITER_OR_COMMENT_PATTERN = '[^' + _DELIMITERS_OR_COMENT + ']'
+ _OPERATORS_PATTERN = u'[.+\\-/=\\[\\](){}<>;,&%¬]'
+ _KEYWORDS = [
+ 'AFTER-BREAK', 'AFTER-LINE', 'AFTER-SCREEN', 'AIM', 'AND', 'ATTR',
+ 'BEFORE', 'BEFORE-BREAK', 'BEFORE-LINE', 'BEFORE-SCREEN', 'BUSHU',
+ 'BY', 'CALL', 'CASE', 'CHECKPOINT', 'CHKP', 'CHKP-STATUS', 'CLEAR',
+ 'CLOSE', 'COL', 'COLOR', 'COMMIT', 'CONTROL', 'COPY', 'CURSOR', 'D',
+ 'DECLARE', 'DEFAULT', 'DEFINE', 'DELETE', 'DENWA', 'DISPLAY', 'DLI',
+ 'DO', 'DUPLICATE', 'E', 'ELSE', 'ELSE-IF', 'END', 'END-CASE',
+ 'END-DO', 'END-IF', 'END-PROC', 'ENDPAGE', 'ENDTABLE', 'ENTER', 'EOF',
+ 'EQ', 'ERROR', 'EXIT', 'EXTERNAL', 'EZLIB', 'F1', 'F10', 'F11', 'F12',
+ 'F13', 'F14', 'F15', 'F16', 'F17', 'F18', 'F19', 'F2', 'F20', 'F21',
+ 'F22', 'F23', 'F24', 'F25', 'F26', 'F27', 'F28', 'F29', 'F3', 'F30',
+ 'F31', 'F32', 'F33', 'F34', 'F35', 'F36', 'F4', 'F5', 'F6', 'F7',
+ 'F8', 'F9', 'FETCH', 'FILE-STATUS', 'FILL', 'FINAL', 'FIRST',
+ 'FIRST-DUP', 'FOR', 'GE', 'GET', 'GO', 'GOTO', 'GQ', 'GR', 'GT',
+ 'HEADING', 'HEX', 'HIGH-VALUES', 'IDD', 'IDMS', 'IF', 'IN', 'INSERT',
+ 'JUSTIFY', 'KANJI-DATE', 'KANJI-DATE-LONG', 'KANJI-TIME', 'KEY',
+ 'KEY-PRESSED', 'KOKUGO', 'KUN', 'LAST-DUP', 'LE', 'LEVEL', 'LIKE',
+ 'LINE', 'LINE-COUNT', 'LINE-NUMBER', 'LINK', 'LIST', 'LOW-VALUES',
+ 'LQ', 'LS', 'LT', 'MACRO', 'MASK', 'MATCHED', 'MEND', 'MESSAGE',
+ 'MOVE', 'MSTART', 'NE', 'NEWPAGE', 'NOMASK', 'NOPRINT', 'NOT',
+ 'NOTE', 'NOVERIFY', 'NQ', 'NULL', 'OF', 'OR', 'OTHERWISE', 'PA1',
+ 'PA2', 'PA3', 'PAGE-COUNT', 'PAGE-NUMBER', 'PARM-REGISTER',
+ 'PATH-ID', 'PATTERN', 'PERFORM', 'POINT', 'POS', 'PRIMARY', 'PRINT',
+ 'PROCEDURE', 'PROGRAM', 'PUT', 'READ', 'RECORD', 'RECORD-COUNT',
+ 'RECORD-LENGTH', 'REFRESH', 'RELEASE', 'RENUM', 'REPEAT', 'REPORT',
+ 'REPORT-INPUT', 'RESHOW', 'RESTART', 'RETRIEVE', 'RETURN-CODE',
+ 'ROLLBACK', 'ROW', 'S', 'SCREEN', 'SEARCH', 'SECONDARY', 'SELECT',
+ 'SEQUENCE', 'SIZE', 'SKIP', 'SOKAKU', 'SORT', 'SQL', 'STOP', 'SUM',
+ 'SYSDATE', 'SYSDATE-LONG', 'SYSIN', 'SYSIPT', 'SYSLST', 'SYSPRINT',
+ 'SYSSNAP', 'SYSTIME', 'TALLY', 'TERM-COLUMNS', 'TERM-NAME',
+ 'TERM-ROWS', 'TERMINATION', 'TITLE', 'TO', 'TRANSFER', 'TRC',
+ 'UNIQUE', 'UNTIL', 'UPDATE', 'UPPERCASE', 'USER', 'USERID', 'VALUE',
+ 'VERIFY', 'W', 'WHEN', 'WHILE', 'WORK', 'WRITE', 'X', 'XDM', 'XRST'
+ ]
+
+ tokens = {
+ 'root': [
+ (r'\*.*\n', Comment.Single),
+ (r'\n+', Whitespace),
+ # Macro argument
+ (r'&' + _NON_DELIMITER_OR_COMMENT_PATTERN + r'+\.', Name.Variable,
+ 'after_macro_argument'),
+ # Macro call
+ (r'%' + _NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name.Variable),
+ (r'(FILE|MACRO|REPORT)(\s+)',
+ bygroups(Keyword.Declaration, Whitespace), 'after_declaration'),
+ (r'(JOB|PARM)' + r'(' + _DELIMITER_PATTERN + r')',
+ bygroups(Keyword.Declaration, Operator)),
+ (words(_KEYWORDS, suffix=_DELIMITER_PATTERN_CAPTURE),
+ bygroups(Keyword.Reserved, Operator)),
+ (_OPERATORS_PATTERN, Operator),
+ # Procedure declaration
+ (r'(' + _NON_DELIMITER_OR_COMMENT_PATTERN + r'+)(\s*)(\.?)(\s*)(PROC)(\s*\n)',
+ bygroups(Name.Function, Whitespace, Operator, Whitespace,
+ Keyword.Declaration, Whitespace)),
+ (r'[0-9]+\.[0-9]*', Number.Float),
+ (r'[0-9]+', Number.Integer),
+ (r"'(''|[^'])*'", String),
+ (r'\s+', Whitespace),
+ # Everything else just belongs to a name
+ (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name),
+ ],
+ 'after_declaration': [
+ (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name.Function),
+ default('#pop'),
+ ],
+ 'after_macro_argument': [
+ (r'\*.*\n', Comment.Single, '#pop'),
+ (r'\s+', Whitespace, '#pop'),
+ (_OPERATORS_PATTERN, Operator, '#pop'),
+ (r"'(''|[^'])*'", String, '#pop'),
+ # Everything else just belongs to a name
+ (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name),
+ ],
+ }
+ _COMMENT_LINE_REGEX = re.compile(r'^\s*\*')
+ _MACRO_HEADER_REGEX = re.compile(r'^\s*MACRO')
+
+ def analyse_text(text):
+ """
+ Perform a structural analysis for basic Easytrieve constructs.
+ """
+ result = 0.0
+ lines = text.split('\n')
+ hasEndProc = False
+ hasHeaderComment = False
+ hasFile = False
+ hasJob = False
+ hasProc = False
+ hasParm = False
+ hasReport = False
+
+ def isCommentLine(line):
+ return EasytrieveLexer._COMMENT_LINE_REGEX.match(lines[0]) is not None
+
+ def isEmptyLine(line):
+ return not bool(line.strip())
+
+ # Remove possible empty lines and header comments.
+ while lines and (isEmptyLine(lines[0]) or isCommentLine(lines[0])):
+ if not isEmptyLine(lines[0]):
+ hasHeaderComment = True
+ del lines[0]
+
+ if EasytrieveLexer._MACRO_HEADER_REGEX.match(lines[0]):
+ # Looks like an Easytrieve macro.
+ result = 0.4
+ if hasHeaderComment:
+ result += 0.4
+ else:
+ # Scan the source for lines starting with indicators.
+ for line in lines:
+ words = line.split()
+ if (len(words) >= 2):
+ firstWord = words[0]
+ if not hasReport:
+ if not hasJob:
+ if not hasFile:
+ if not hasParm:
+ if firstWord == 'PARM':
+ hasParm = True
+ if firstWord == 'FILE':
+ hasFile = True
+ if firstWord == 'JOB':
+ hasJob = True
+ elif firstWord == 'PROC':
+ hasProc = True
+ elif firstWord == 'END-PROC':
+ hasEndProc = True
+ elif firstWord == 'REPORT':
+ hasReport = True
+
+ # Weight the findings.
+ if hasJob and (hasProc == hasEndProc):
+ if hasHeaderComment:
+ result += 0.1
+ if hasParm:
+ if hasProc:
+ # Found PARM, JOB and PROC/END-PROC:
+ # pretty sure this is Easytrieve.
+ result += 0.8
+ else:
+ # Found PARAM and JOB: probably this is Easytrieve
+ result += 0.5
+ else:
+ # Found JOB and possibly other keywords: might be Easytrieve
+ result += 0.11
+ if hasParm:
+ # Note: PARAM is not a proper English word, so this is
+ # regarded a much better indicator for Easytrieve than
+ # the other words.
+ result += 0.2
+ if hasFile:
+ result += 0.01
+ if hasReport:
+ result += 0.01
+ assert 0.0 <= result <= 1.0
+ return result
+
+
+class JclLexer(RegexLexer):
+ """
+ `Job Control Language (JCL)
+ <http://publibz.boulder.ibm.com/cgi-bin/bookmgr_OS390/BOOKS/IEA2B570/CCONTENTS>`_
+ is a scripting language used on mainframe platforms to instruct the system
+ on how to run a batch job or start a subsystem. It is somewhat
+ comparable to MS DOS batch and Unix shell scripts.
+
+ .. versionadded:: 2.1
+ """
+ name = 'JCL'
+ aliases = ['jcl']
+ filenames = ['*.jcl']
+ mimetypes = ['text/x-jcl']
+ flags = re.IGNORECASE
+
+ tokens = {
+ 'root': [
+ (r'//\*.*\n', Comment.Single),
+ (r'//', Keyword.Pseudo, 'statement'),
+ (r'/\*', Keyword.Pseudo, 'jes2_statement'),
+ # TODO: JES3 statement
+ (r'.*\n', Other) # Input text or inline code in any language.
+ ],
+ 'statement': [
+ (r'\s*\n', Whitespace, '#pop'),
+ (r'([a-z]\w*)(\s+)(exec|job)(\s*)',
+ bygroups(Name.Label, Whitespace, Keyword.Reserved, Whitespace),
+ 'option'),
+ (r'[a-z]\w*', Name.Variable, 'statement_command'),
+ (r'\s+', Whitespace, 'statement_command'),
+ ],
+ 'statement_command': [
+ (r'\s+(command|cntl|dd|endctl|endif|else|include|jcllib|'
+ r'output|pend|proc|set|then|xmit)\s+', Keyword.Reserved, 'option'),
+ include('option')
+ ],
+ 'jes2_statement': [
+ (r'\s*\n', Whitespace, '#pop'),
+ (r'\$', Keyword, 'option'),
+ (r'\b(jobparam|message|netacct|notify|output|priority|route|'
+ r'setup|signoff|xeq|xmit)\b', Keyword, 'option'),
+ ],
+ 'option': [
+ # (r'\n', Text, 'root'),
+ (r'\*', Name.Builtin),
+ (r'[\[\](){}<>;,]', Punctuation),
+ (r'[-+*/=&%]', Operator),
+ (r'[a-z_]\w*', Name),
+ (r'\d+\.\d*', Number.Float),
+ (r'\.\d+', Number.Float),
+ (r'\d+', Number.Integer),
+ (r"'", String, 'option_string'),
+ (r'[ \t]+', Whitespace, 'option_comment'),
+ (r'\.', Punctuation),
+ ],
+ 'option_string': [
+ (r"(\n)(//)", bygroups(Text, Keyword.Pseudo)),
+ (r"''", String),
+ (r"[^']", String),
+ (r"'", String, '#pop'),
+ ],
+ 'option_comment': [
+ # (r'\n', Text, 'root'),
+ (r'.+', Comment.Single),
+ ]
+ }
+
+ _JOB_HEADER_PATTERN = re.compile(r'^//[a-z#$@][a-z0-9#$@]{0,7}\s+job(\s+.*)?$',
+ re.IGNORECASE)
+
+ def analyse_text(text):
+ """
+ Recognize JCL job by header.
+ """
+ result = 0.0
+ lines = text.split('\n')
+ if len(lines) > 0:
+ if JclLexer._JOB_HEADER_PATTERN.match(lines[0]):
+ result = 1.0
+ assert 0.0 <= result <= 1.0
+ return result
diff --git a/wandb/vendor/pygments/lexers/shell.py b/wandb/vendor/pygments/lexers/shell.py
new file mode 100644
index 0000000000000000000000000000000000000000..ceb6f14df7a83de7b5bb46a88d1506fe73a49fea
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/shell.py
@@ -0,0 +1,794 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.shell
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for various shells.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, do_insertions, bygroups, \
+ include, default, this, using, words
+from pygments.token import Punctuation, \
+ Text, Comment, Operator, Keyword, Name, String, Number, Generic
+from pygments.util import shebang_matches
+
+
+__all__ = ['BashLexer', 'BashSessionLexer', 'TcshLexer', 'BatchLexer',
+ 'MSDOSSessionLexer', 'PowerShellLexer',
+ 'PowerShellSessionLexer', 'TcshSessionLexer', 'FishShellLexer']
+
+line_re = re.compile('.*?\n')
+
+
+class BashLexer(RegexLexer):
+ """
+ Lexer for (ba|k|z|)sh shell scripts.
+
+ .. versionadded:: 0.6
+ """
+
+ name = 'Bash'
+ aliases = ['bash', 'sh', 'ksh', 'zsh', 'shell']
+ filenames = ['*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass',
+ '*.exheres-0', '*.exlib', '*.zsh',
+ '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc',
+ 'PKGBUILD']
+ mimetypes = ['application/x-sh', 'application/x-shellscript']
+
+ tokens = {
+ 'root': [
+ include('basic'),
+ (r'`', String.Backtick, 'backticks'),
+ include('data'),
+ include('interp'),
+ ],
+ 'interp': [
+ (r'\$\(\(', Keyword, 'math'),
+ (r'\$\(', Keyword, 'paren'),
+ (r'\$\{#?', String.Interpol, 'curly'),
+ (r'\$[a-zA-Z_]\w*', Name.Variable), # user variable
+ (r'\$(?:\d+|[#$?!_*@-])', Name.Variable), # builtin
+ (r'\$', Text),
+ ],
+ 'basic': [
+ (r'\b(if|fi|else|while|do|done|for|then|return|function|case|'
+ r'select|continue|until|esac|elif)(\s*)\b',
+ bygroups(Keyword, Text)),
+ (r'\b(alias|bg|bind|break|builtin|caller|cd|command|compgen|'
+ r'complete|declare|dirs|disown|echo|enable|eval|exec|exit|'
+ r'export|false|fc|fg|getopts|hash|help|history|jobs|kill|let|'
+ r'local|logout|popd|printf|pushd|pwd|read|readonly|set|shift|'
+ r'shopt|source|suspend|test|time|times|trap|true|type|typeset|'
+ r'ulimit|umask|unalias|unset|wait)(?=[\s)`])',
+ Name.Builtin),
+ (r'\A#!.+\n', Comment.Hashbang),
+ (r'#.*\n', Comment.Single),
+ (r'\\[\w\W]', String.Escape),
+ (r'(\b\w+)(\s*)(\+?=)', bygroups(Name.Variable, Text, Operator)),
+ (r'[\[\]{}()=]', Operator),
+ (r'<<<', Operator), # here-string
+ (r'<<-?\s*(\'?)\\?(\w+)[\w\W]+?\2', String),
+ (r'&&|\|\|', Operator),
+ ],
+ 'data': [
+ (r'(?s)\$?"(\\\\|\\[0-7]+|\\.|[^"\\$])*"', String.Double),
+ (r'"', String.Double, 'string'),
+ (r"(?s)\$'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single),
+ (r"(?s)'.*?'", String.Single),
+ (r';', Punctuation),
+ (r'&', Punctuation),
+ (r'\|', Punctuation),
+ (r'\s+', Text),
+ (r'\d+\b', Number),
+ (r'[^=\s\[\]{}()$"\'`\\<&|;]+', Text),
+ (r'<', Text),
+ ],
+ 'string': [
+ (r'"', String.Double, '#pop'),
+ (r'(?s)(\\\\|\\[0-7]+|\\.|[^"\\$])+', String.Double),
+ include('interp'),
+ ],
+ 'curly': [
+ (r'\}', String.Interpol, '#pop'),
+ (r':-', Keyword),
+ (r'\w+', Name.Variable),
+ (r'[^}:"\'`$\\]+', Punctuation),
+ (r':', Punctuation),
+ include('root'),
+ ],
+ 'paren': [
+ (r'\)', Keyword, '#pop'),
+ include('root'),
+ ],
+ 'math': [
+ (r'\)\)', Keyword, '#pop'),
+ (r'[-+*/%^|&]|\*\*|\|\|', Operator),
+ (r'\d+#\d+', Number),
+ (r'\d+#(?! )', Number),
+ (r'\d+', Number),
+ include('root'),
+ ],
+ 'backticks': [
+ (r'`', String.Backtick, '#pop'),
+ include('root'),
+ ],
+ }
+
+ def analyse_text(text):
+ if shebang_matches(text, r'(ba|z|)sh'):
+ return 1
+ if text.startswith('$ '):
+ return 0.2
+
+
+class ShellSessionBaseLexer(Lexer):
+ """
+ Base lexer for simplistic shell sessions.
+
+ .. versionadded:: 2.1
+ """
+ def get_tokens_unprocessed(self, text):
+ innerlexer = self._innerLexerCls(**self.options)
+
+ pos = 0
+ curcode = ''
+ insertions = []
+ backslash_continuation = False
+
+ for match in line_re.finditer(text):
+ line = match.group()
+ m = re.match(self._ps1rgx, line)
+ if backslash_continuation:
+ curcode += line
+ backslash_continuation = curcode.endswith('\\\n')
+ elif m:
+ # To support output lexers (say diff output), the output
+ # needs to be broken by prompts whenever the output lexer
+ # changes.
+ if not insertions:
+ pos = match.start()
+
+ insertions.append((len(curcode),
+ [(0, Generic.Prompt, m.group(1))]))
+ curcode += m.group(2)
+ backslash_continuation = curcode.endswith('\\\n')
+ elif line.startswith(self._ps2):
+ insertions.append((len(curcode),
+ [(0, Generic.Prompt, line[:len(self._ps2)])]))
+ curcode += line[len(self._ps2):]
+ backslash_continuation = curcode.endswith('\\\n')
+ else:
+ if insertions:
+ toks = innerlexer.get_tokens_unprocessed(curcode)
+ for i, t, v in do_insertions(insertions, toks):
+ yield pos+i, t, v
+ yield match.start(), Generic.Output, line
+ insertions = []
+ curcode = ''
+ if insertions:
+ for i, t, v in do_insertions(insertions,
+ innerlexer.get_tokens_unprocessed(curcode)):
+ yield pos+i, t, v
+
+
+class BashSessionLexer(ShellSessionBaseLexer):
+ """
+ Lexer for simplistic shell sessions.
+
+ .. versionadded:: 1.1
+ """
+
+ name = 'Bash Session'
+ aliases = ['console', 'shell-session']
+ filenames = ['*.sh-session', '*.shell-session']
+ mimetypes = ['application/x-shell-session', 'application/x-sh-session']
+
+ _innerLexerCls = BashLexer
+ _ps1rgx = \
+ r'^((?:(?:\[.*?\])|(?:\(\S+\))?(?:| |sh\S*?|\w+\S+[@:]\S+(?:\s+\S+)' \
+ r'?|\[\S+[@:][^\n]+\].+))\s*[$#%])(.*\n?)'
+ _ps2 = '>'
+
+
+class BatchLexer(RegexLexer):
+ """
+ Lexer for the DOS/Windows Batch file format.
+
+ .. versionadded:: 0.7
+ """
+ name = 'Batchfile'
+ aliases = ['bat', 'batch', 'dosbatch', 'winbatch']
+ filenames = ['*.bat', '*.cmd']
+ mimetypes = ['application/x-dos-batch']
+
+ flags = re.MULTILINE | re.IGNORECASE
+
+ _nl = r'\n\x1a'
+ _punct = r'&<>|'
+ _ws = r'\t\v\f\r ,;=\xa0'
+ _space = r'(?:(?:(?:\^[%s])?[%s])+)' % (_nl, _ws)
+ _keyword_terminator = (r'(?=(?:\^[%s]?)?[%s+./:[\\\]]|[%s%s(])' %
+ (_nl, _ws, _nl, _punct))
+ _token_terminator = r'(?=\^?[%s]|[%s%s])' % (_ws, _punct, _nl)
+ _start_label = r'((?:(?<=^[^:])|^[^:]?)[%s]*)(:)' % _ws
+ _label = r'(?:(?:[^%s%s%s+:^]|\^[%s]?[\w\W])*)' % (_nl, _punct, _ws, _nl)
+ _label_compound = (r'(?:(?:[^%s%s%s+:^)]|\^[%s]?[^)])*)' %
+ (_nl, _punct, _ws, _nl))
+ _number = r'(?:-?(?:0[0-7]+|0x[\da-f]+|\d+)%s)' % _token_terminator
+ _opword = r'(?:equ|geq|gtr|leq|lss|neq)'
+ _string = r'(?:"[^%s"]*(?:"|(?=[%s])))' % (_nl, _nl)
+ _variable = (r'(?:(?:%%(?:\*|(?:~[a-z]*(?:\$[^:]+:)?)?\d|'
+ r'[^%%:%s]+(?::(?:~(?:-?\d+)?(?:,(?:-?\d+)?)?|(?:[^%%%s^]|'
+ r'\^[^%%%s])[^=%s]*=(?:[^%%%s^]|\^[^%%%s])*)?)?%%))|'
+ r'(?:\^?![^!:%s]+(?::(?:~(?:-?\d+)?(?:,(?:-?\d+)?)?|(?:'
+ r'[^!%s^]|\^[^!%s])[^=%s]*=(?:[^!%s^]|\^[^!%s])*)?)?\^?!))' %
+ (_nl, _nl, _nl, _nl, _nl, _nl, _nl, _nl, _nl, _nl, _nl, _nl))
+ _core_token = r'(?:(?:(?:\^[%s]?)?[^"%s%s%s])+)' % (_nl, _nl, _punct, _ws)
+ _core_token_compound = r'(?:(?:(?:\^[%s]?)?[^"%s%s%s)])+)' % (_nl, _nl,
+ _punct, _ws)
+ _token = r'(?:[%s]+|%s)' % (_punct, _core_token)
+ _token_compound = r'(?:[%s]+|%s)' % (_punct, _core_token_compound)
+ _stoken = (r'(?:[%s]+|(?:%s|%s|%s)+)' %
+ (_punct, _string, _variable, _core_token))
+
+ def _make_begin_state(compound, _core_token=_core_token,
+ _core_token_compound=_core_token_compound,
+ _keyword_terminator=_keyword_terminator,
+ _nl=_nl, _punct=_punct, _string=_string,
+ _space=_space, _start_label=_start_label,
+ _stoken=_stoken, _token_terminator=_token_terminator,
+ _variable=_variable, _ws=_ws):
+ rest = '(?:%s|%s|[^"%%%s%s%s])*' % (_string, _variable, _nl, _punct,
+ ')' if compound else '')
+ rest_of_line = r'(?:(?:[^%s^]|\^[%s]?[\w\W])*)' % (_nl, _nl)
+ rest_of_line_compound = r'(?:(?:[^%s^)]|\^[%s]?[^)])*)' % (_nl, _nl)
+ set_space = r'((?:(?:\^[%s]?)?[^\S\n])*)' % _nl
+ suffix = ''
+ if compound:
+ _keyword_terminator = r'(?:(?=\))|%s)' % _keyword_terminator
+ _token_terminator = r'(?:(?=\))|%s)' % _token_terminator
+ suffix = '/compound'
+ return [
+ ((r'\)', Punctuation, '#pop') if compound else
+ (r'\)((?=\()|%s)%s' % (_token_terminator, rest_of_line),
+ Comment.Single)),
+ (r'(?=%s)' % _start_label, Text, 'follow%s' % suffix),
+ (_space, using(this, state='text')),
+ include('redirect%s' % suffix),
+ (r'[%s]+' % _nl, Text),
+ (r'\(', Punctuation, 'root/compound'),
+ (r'@+', Punctuation),
+ (r'((?:for|if|rem)(?:(?=(?:\^[%s]?)?/)|(?:(?!\^)|'
+ r'(?<=m))(?:(?=\()|%s)))(%s?%s?(?:\^[%s]?)?/(?:\^[%s]?)?\?)' %
+ (_nl, _token_terminator, _space,
+ _core_token_compound if compound else _core_token, _nl, _nl),
+ bygroups(Keyword, using(this, state='text')),
+ 'follow%s' % suffix),
+ (r'(goto%s)(%s(?:\^[%s]?)?/(?:\^[%s]?)?\?%s)' %
+ (_keyword_terminator, rest, _nl, _nl, rest),
+ bygroups(Keyword, using(this, state='text')),
+ 'follow%s' % suffix),
+ (words(('assoc', 'break', 'cd', 'chdir', 'cls', 'color', 'copy',
+ 'date', 'del', 'dir', 'dpath', 'echo', 'endlocal', 'erase',
+ 'exit', 'ftype', 'keys', 'md', 'mkdir', 'mklink', 'move',
+ 'path', 'pause', 'popd', 'prompt', 'pushd', 'rd', 'ren',
+ 'rename', 'rmdir', 'setlocal', 'shift', 'start', 'time',
+ 'title', 'type', 'ver', 'verify', 'vol'),
+ suffix=_keyword_terminator), Keyword, 'follow%s' % suffix),
+ (r'(call)(%s?)(:)' % _space,
+ bygroups(Keyword, using(this, state='text'), Punctuation),
+ 'call%s' % suffix),
+ (r'call%s' % _keyword_terminator, Keyword),
+ (r'(for%s(?!\^))(%s)(/f%s)' %
+ (_token_terminator, _space, _token_terminator),
+ bygroups(Keyword, using(this, state='text'), Keyword),
+ ('for/f', 'for')),
+ (r'(for%s(?!\^))(%s)(/l%s)' %
+ (_token_terminator, _space, _token_terminator),
+ bygroups(Keyword, using(this, state='text'), Keyword),
+ ('for/l', 'for')),
+ (r'for%s(?!\^)' % _token_terminator, Keyword, ('for2', 'for')),
+ (r'(goto%s)(%s?)(:?)' % (_keyword_terminator, _space),
+ bygroups(Keyword, using(this, state='text'), Punctuation),
+ 'label%s' % suffix),
+ (r'(if(?:(?=\()|%s)(?!\^))(%s?)((?:/i%s)?)(%s?)((?:not%s)?)(%s?)' %
+ (_token_terminator, _space, _token_terminator, _space,
+ _token_terminator, _space),
+ bygroups(Keyword, using(this, state='text'), Keyword,
+ using(this, state='text'), Keyword,
+ using(this, state='text')), ('(?', 'if')),
+ (r'rem(((?=\()|%s)%s?%s?.*|%s%s)' %
+ (_token_terminator, _space, _stoken, _keyword_terminator,
+ rest_of_line_compound if compound else rest_of_line),
+ Comment.Single, 'follow%s' % suffix),
+ (r'(set%s)%s(/a)' % (_keyword_terminator, set_space),
+ bygroups(Keyword, using(this, state='text'), Keyword),
+ 'arithmetic%s' % suffix),
+ (r'(set%s)%s((?:/p)?)%s((?:(?:(?:\^[%s]?)?[^"%s%s^=%s]|'
+ r'\^[%s]?[^"=])+)?)((?:(?:\^[%s]?)?=)?)' %
+ (_keyword_terminator, set_space, set_space, _nl, _nl, _punct,
+ ')' if compound else '', _nl, _nl),
+ bygroups(Keyword, using(this, state='text'), Keyword,
+ using(this, state='text'), using(this, state='variable'),
+ Punctuation),
+ 'follow%s' % suffix),
+ default('follow%s' % suffix)
+ ]
+
+ def _make_follow_state(compound, _label=_label,
+ _label_compound=_label_compound, _nl=_nl,
+ _space=_space, _start_label=_start_label,
+ _token=_token, _token_compound=_token_compound,
+ _ws=_ws):
+ suffix = '/compound' if compound else ''
+ state = []
+ if compound:
+ state.append((r'(?=\))', Text, '#pop'))
+ state += [
+ (r'%s([%s]*)(%s)(.*)' %
+ (_start_label, _ws, _label_compound if compound else _label),
+ bygroups(Text, Punctuation, Text, Name.Label, Comment.Single)),
+ include('redirect%s' % suffix),
+ (r'(?=[%s])' % _nl, Text, '#pop'),
+ (r'\|\|?|&&?', Punctuation, '#pop'),
+ include('text')
+ ]
+ return state
+
+ def _make_arithmetic_state(compound, _nl=_nl, _punct=_punct,
+ _string=_string, _variable=_variable, _ws=_ws):
+ op = r'=+\-*/!~'
+ state = []
+ if compound:
+ state.append((r'(?=\))', Text, '#pop'))
+ state += [
+ (r'0[0-7]+', Number.Oct),
+ (r'0x[\da-f]+', Number.Hex),
+ (r'\d+', Number.Integer),
+ (r'[(),]+', Punctuation),
+ (r'([%s]|%%|\^\^)+' % op, Operator),
+ (r'(%s|%s|(\^[%s]?)?[^()%s%%^"%s%s%s]|\^[%s%s]?%s)+' %
+ (_string, _variable, _nl, op, _nl, _punct, _ws, _nl, _ws,
+ r'[^)]' if compound else r'[\w\W]'),
+ using(this, state='variable')),
+ (r'(?=[\x00|&])', Text, '#pop'),
+ include('follow')
+ ]
+ return state
+
+ def _make_call_state(compound, _label=_label,
+ _label_compound=_label_compound):
+ state = []
+ if compound:
+ state.append((r'(?=\))', Text, '#pop'))
+ state.append((r'(:?)(%s)' % (_label_compound if compound else _label),
+ bygroups(Punctuation, Name.Label), '#pop'))
+ return state
+
+ def _make_label_state(compound, _label=_label,
+ _label_compound=_label_compound, _nl=_nl,
+ _punct=_punct, _string=_string, _variable=_variable):
+ state = []
+ if compound:
+ state.append((r'(?=\))', Text, '#pop'))
+ state.append((r'(%s?)((?:%s|%s|\^[%s]?%s|[^"%%^%s%s%s])*)' %
+ (_label_compound if compound else _label, _string,
+ _variable, _nl, r'[^)]' if compound else r'[\w\W]', _nl,
+ _punct, r')' if compound else ''),
+ bygroups(Name.Label, Comment.Single), '#pop'))
+ return state
+
+ def _make_redirect_state(compound,
+ _core_token_compound=_core_token_compound,
+ _nl=_nl, _punct=_punct, _stoken=_stoken,
+ _string=_string, _space=_space,
+ _variable=_variable, _ws=_ws):
+ stoken_compound = (r'(?:[%s]+|(?:%s|%s|%s)+)' %
+ (_punct, _string, _variable, _core_token_compound))
+ return [
+ (r'((?:(?<=[%s%s])\d)?)(>>?&|<&)([%s%s]*)(\d)' %
+ (_nl, _ws, _nl, _ws),
+ bygroups(Number.Integer, Punctuation, Text, Number.Integer)),
+ (r'((?:(?<=[%s%s])(?<!\^[%s])\d)?)(>>?|<)(%s?%s)' %
+ (_nl, _ws, _nl, _space, stoken_compound if compound else _stoken),
+ bygroups(Number.Integer, Punctuation, using(this, state='text')))
+ ]
+
+ tokens = {
+ 'root': _make_begin_state(False),
+ 'follow': _make_follow_state(False),
+ 'arithmetic': _make_arithmetic_state(False),
+ 'call': _make_call_state(False),
+ 'label': _make_label_state(False),
+ 'redirect': _make_redirect_state(False),
+ 'root/compound': _make_begin_state(True),
+ 'follow/compound': _make_follow_state(True),
+ 'arithmetic/compound': _make_arithmetic_state(True),
+ 'call/compound': _make_call_state(True),
+ 'label/compound': _make_label_state(True),
+ 'redirect/compound': _make_redirect_state(True),
+ 'variable-or-escape': [
+ (_variable, Name.Variable),
+ (r'%%%%|\^[%s]?(\^!|[\w\W])' % _nl, String.Escape)
+ ],
+ 'string': [
+ (r'"', String.Double, '#pop'),
+ (_variable, Name.Variable),
+ (r'\^!|%%', String.Escape),
+ (r'[^"%%^%s]+|[%%^]' % _nl, String.Double),
+ default('#pop')
+ ],
+ 'sqstring': [
+ include('variable-or-escape'),
+ (r'[^%]+|%', String.Single)
+ ],
+ 'bqstring': [
+ include('variable-or-escape'),
+ (r'[^%]+|%', String.Backtick)
+ ],
+ 'text': [
+ (r'"', String.Double, 'string'),
+ include('variable-or-escape'),
+ (r'[^"%%^%s%s%s\d)]+|.' % (_nl, _punct, _ws), Text)
+ ],
+ 'variable': [
+ (r'"', String.Double, 'string'),
+ include('variable-or-escape'),
+ (r'[^"%%^%s]+|.' % _nl, Name.Variable)
+ ],
+ 'for': [
+ (r'(%s)(in)(%s)(\()' % (_space, _space),
+ bygroups(using(this, state='text'), Keyword,
+ using(this, state='text'), Punctuation), '#pop'),
+ include('follow')
+ ],
+ 'for2': [
+ (r'\)', Punctuation),
+ (r'(%s)(do%s)' % (_space, _token_terminator),
+ bygroups(using(this, state='text'), Keyword), '#pop'),
+ (r'[%s]+' % _nl, Text),
+ include('follow')
+ ],
+ 'for/f': [
+ (r'(")((?:%s|[^"])*?")([%s%s]*)(\))' % (_variable, _nl, _ws),
+ bygroups(String.Double, using(this, state='string'), Text,
+ Punctuation)),
+ (r'"', String.Double, ('#pop', 'for2', 'string')),
+ (r"('(?:%%%%|%s|[\w\W])*?')([%s%s]*)(\))" % (_variable, _nl, _ws),
+ bygroups(using(this, state='sqstring'), Text, Punctuation)),
+ (r'(`(?:%%%%|%s|[\w\W])*?`)([%s%s]*)(\))' % (_variable, _nl, _ws),
+ bygroups(using(this, state='bqstring'), Text, Punctuation)),
+ include('for2')
+ ],
+ 'for/l': [
+ (r'-?\d+', Number.Integer),
+ include('for2')
+ ],
+ 'if': [
+ (r'((?:cmdextversion|errorlevel)%s)(%s)(\d+)' %
+ (_token_terminator, _space),
+ bygroups(Keyword, using(this, state='text'),
+ Number.Integer), '#pop'),
+ (r'(defined%s)(%s)(%s)' % (_token_terminator, _space, _stoken),
+ bygroups(Keyword, using(this, state='text'),
+ using(this, state='variable')), '#pop'),
+ (r'(exist%s)(%s%s)' % (_token_terminator, _space, _stoken),
+ bygroups(Keyword, using(this, state='text')), '#pop'),
+ (r'(%s%s)(%s)(%s%s)' % (_number, _space, _opword, _space, _number),
+ bygroups(using(this, state='arithmetic'), Operator.Word,
+ using(this, state='arithmetic')), '#pop'),
+ (_stoken, using(this, state='text'), ('#pop', 'if2')),
+ ],
+ 'if2': [
+ (r'(%s?)(==)(%s?%s)' % (_space, _space, _stoken),
+ bygroups(using(this, state='text'), Operator,
+ using(this, state='text')), '#pop'),
+ (r'(%s)(%s)(%s%s)' % (_space, _opword, _space, _stoken),
+ bygroups(using(this, state='text'), Operator.Word,
+ using(this, state='text')), '#pop')
+ ],
+ '(?': [
+ (_space, using(this, state='text')),
+ (r'\(', Punctuation, ('#pop', 'else?', 'root/compound')),
+ default('#pop')
+ ],
+ 'else?': [
+ (_space, using(this, state='text')),
+ (r'else%s' % _token_terminator, Keyword, '#pop'),
+ default('#pop')
+ ]
+ }
+
+
+class MSDOSSessionLexer(ShellSessionBaseLexer):
+ """
+ Lexer for simplistic MSDOS sessions.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'MSDOS Session'
+ aliases = ['doscon']
+ filenames = []
+ mimetypes = []
+
+ _innerLexerCls = BatchLexer
+ _ps1rgx = r'^([^>]+>)(.*\n?)'
+ _ps2 = 'More? '
+
+
+class TcshLexer(RegexLexer):
+ """
+ Lexer for tcsh scripts.
+
+ .. versionadded:: 0.10
+ """
+
+ name = 'Tcsh'
+ aliases = ['tcsh', 'csh']
+ filenames = ['*.tcsh', '*.csh']
+ mimetypes = ['application/x-csh']
+
+ tokens = {
+ 'root': [
+ include('basic'),
+ (r'\$\(', Keyword, 'paren'),
+ (r'\$\{#?', Keyword, 'curly'),
+ (r'`', String.Backtick, 'backticks'),
+ include('data'),
+ ],
+ 'basic': [
+ (r'\b(if|endif|else|while|then|foreach|case|default|'
+ r'continue|goto|breaksw|end|switch|endsw)\s*\b',
+ Keyword),
+ (r'\b(alias|alloc|bg|bindkey|break|builtins|bye|caller|cd|chdir|'
+ r'complete|dirs|echo|echotc|eval|exec|exit|fg|filetest|getxvers|'
+ r'glob|getspath|hashstat|history|hup|inlib|jobs|kill|'
+ r'limit|log|login|logout|ls-F|migrate|newgrp|nice|nohup|notify|'
+ r'onintr|popd|printenv|pushd|rehash|repeat|rootnode|popd|pushd|'
+ r'set|shift|sched|setenv|setpath|settc|setty|setxvers|shift|'
+ r'source|stop|suspend|source|suspend|telltc|time|'
+ r'umask|unalias|uncomplete|unhash|universe|unlimit|unset|unsetenv|'
+ r'ver|wait|warp|watchlog|where|which)\s*\b',
+ Name.Builtin),
+ (r'#.*', Comment),
+ (r'\\[\w\W]', String.Escape),
+ (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Text, Operator)),
+ (r'[\[\]{}()=]+', Operator),
+ (r'<<\s*(\'?)\\?(\w+)[\w\W]+?\2', String),
+ (r';', Punctuation),
+ ],
+ 'data': [
+ (r'(?s)"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double),
+ (r"(?s)'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single),
+ (r'\s+', Text),
+ (r'[^=\s\[\]{}()$"\'`\\;#]+', Text),
+ (r'\d+(?= |\Z)', Number),
+ (r'\$#?(\w+|.)', Name.Variable),
+ ],
+ 'curly': [
+ (r'\}', Keyword, '#pop'),
+ (r':-', Keyword),
+ (r'\w+', Name.Variable),
+ (r'[^}:"\'`$]+', Punctuation),
+ (r':', Punctuation),
+ include('root'),
+ ],
+ 'paren': [
+ (r'\)', Keyword, '#pop'),
+ include('root'),
+ ],
+ 'backticks': [
+ (r'`', String.Backtick, '#pop'),
+ include('root'),
+ ],
+ }
+
+
+class TcshSessionLexer(ShellSessionBaseLexer):
+ """
+ Lexer for Tcsh sessions.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'Tcsh Session'
+ aliases = ['tcshcon']
+ filenames = []
+ mimetypes = []
+
+ _innerLexerCls = TcshLexer
+ _ps1rgx = r'^([^>]+>)(.*\n?)'
+ _ps2 = '? '
+
+
+class PowerShellLexer(RegexLexer):
+ """
+ For Windows PowerShell code.
+
+ .. versionadded:: 1.5
+ """
+ name = 'PowerShell'
+ aliases = ['powershell', 'posh', 'ps1', 'psm1']
+ filenames = ['*.ps1', '*.psm1']
+ mimetypes = ['text/x-powershell']
+
+ flags = re.DOTALL | re.IGNORECASE | re.MULTILINE
+
+ keywords = (
+ 'while validateset validaterange validatepattern validatelength '
+ 'validatecount until trap switch return ref process param parameter in '
+ 'if global: function foreach for finally filter end elseif else '
+ 'dynamicparam do default continue cmdletbinding break begin alias \\? '
+ '% #script #private #local #global mandatory parametersetname position '
+ 'valuefrompipeline valuefrompipelinebypropertyname '
+ 'valuefromremainingarguments helpmessage try catch throw').split()
+
+ operators = (
+ 'and as band bnot bor bxor casesensitive ccontains ceq cge cgt cle '
+ 'clike clt cmatch cne cnotcontains cnotlike cnotmatch contains '
+ 'creplace eq exact f file ge gt icontains ieq ige igt ile ilike ilt '
+ 'imatch ine inotcontains inotlike inotmatch ireplace is isnot le like '
+ 'lt match ne not notcontains notlike notmatch or regex replace '
+ 'wildcard').split()
+
+ verbs = (
+ 'write where wait use update unregister undo trace test tee take '
+ 'suspend stop start split sort skip show set send select scroll resume '
+ 'restore restart resolve resize reset rename remove register receive '
+ 'read push pop ping out new move measure limit join invoke import '
+ 'group get format foreach export expand exit enter enable disconnect '
+ 'disable debug cxnew copy convertto convertfrom convert connect '
+ 'complete compare clear checkpoint aggregate add').split()
+
+ commenthelp = (
+ 'component description example externalhelp forwardhelpcategory '
+ 'forwardhelptargetname functionality inputs link '
+ 'notes outputs parameter remotehelprunspace role synopsis').split()
+
+ tokens = {
+ 'root': [
+ # we need to count pairs of parentheses for correct highlight
+ # of '$(...)' blocks in strings
+ (r'\(', Punctuation, 'child'),
+ (r'\s+', Text),
+ (r'^(\s*#[#\s]*)(\.(?:%s))([^\n]*$)' % '|'.join(commenthelp),
+ bygroups(Comment, String.Doc, Comment)),
+ (r'#[^\n]*?$', Comment),
+ (r'(<|<)#', Comment.Multiline, 'multline'),
+ (r'@"\n', String.Heredoc, 'heredoc-double'),
+ (r"@'\n.*?\n'@", String.Heredoc),
+ # escaped syntax
+ (r'`[\'"$@-]', Punctuation),
+ (r'"', String.Double, 'string'),
+ (r"'([^']|'')*'", String.Single),
+ (r'(\$|@@|@)((global|script|private|env):)?\w+',
+ Name.Variable),
+ (r'(%s)\b' % '|'.join(keywords), Keyword),
+ (r'-(%s)\b' % '|'.join(operators), Operator),
+ (r'(%s)-[a-z_]\w*\b' % '|'.join(verbs), Name.Builtin),
+ (r'\[[a-z_\[][\w. `,\[\]]*\]', Name.Constant), # .net [type]s
+ (r'-[a-z_]\w*', Name),
+ (r'\w+', Name),
+ (r'[.,;@{}\[\]$()=+*/\\&%!~?^`|<>-]|::', Punctuation),
+ ],
+ 'child': [
+ (r'\)', Punctuation, '#pop'),
+ include('root'),
+ ],
+ 'multline': [
+ (r'[^#&.]+', Comment.Multiline),
+ (r'#(>|>)', Comment.Multiline, '#pop'),
+ (r'\.(%s)' % '|'.join(commenthelp), String.Doc),
+ (r'[#&.]', Comment.Multiline),
+ ],
+ 'string': [
+ (r"`[0abfnrtv'\"$`]", String.Escape),
+ (r'[^$`"]+', String.Double),
+ (r'\$\(', Punctuation, 'child'),
+ (r'""', String.Double),
+ (r'[`$]', String.Double),
+ (r'"', String.Double, '#pop'),
+ ],
+ 'heredoc-double': [
+ (r'\n"@', String.Heredoc, '#pop'),
+ (r'\$\(', Punctuation, 'child'),
+ (r'[^@\n]+"]', String.Heredoc),
+ (r".", String.Heredoc),
+ ]
+ }
+
+
+class PowerShellSessionLexer(ShellSessionBaseLexer):
+ """
+ Lexer for simplistic Windows PowerShell sessions.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'PowerShell Session'
+ aliases = ['ps1con']
+ filenames = []
+ mimetypes = []
+
+ _innerLexerCls = PowerShellLexer
+ _ps1rgx = r'^(PS [^>]+> )(.*\n?)'
+ _ps2 = '>> '
+
+
+class FishShellLexer(RegexLexer):
+ """
+ Lexer for Fish shell scripts.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'Fish'
+ aliases = ['fish', 'fishshell']
+ filenames = ['*.fish', '*.load']
+ mimetypes = ['application/x-fish']
+
+ tokens = {
+ 'root': [
+ include('basic'),
+ include('data'),
+ include('interp'),
+ ],
+ 'interp': [
+ (r'\$\(\(', Keyword, 'math'),
+ (r'\(', Keyword, 'paren'),
+ (r'\$#?(\w+|.)', Name.Variable),
+ ],
+ 'basic': [
+ (r'\b(begin|end|if|else|while|break|for|in|return|function|block|'
+ r'case|continue|switch|not|and|or|set|echo|exit|pwd|true|false|'
+ r'cd|count|test)(\s*)\b',
+ bygroups(Keyword, Text)),
+ (r'\b(alias|bg|bind|breakpoint|builtin|command|commandline|'
+ r'complete|contains|dirh|dirs|emit|eval|exec|fg|fish|fish_config|'
+ r'fish_indent|fish_pager|fish_prompt|fish_right_prompt|'
+ r'fish_update_completions|fishd|funced|funcsave|functions|help|'
+ r'history|isatty|jobs|math|mimedb|nextd|open|popd|prevd|psub|'
+ r'pushd|random|read|set_color|source|status|trap|type|ulimit|'
+ r'umask|vared|fc|getopts|hash|kill|printf|time|wait)\s*\b(?!\.)',
+ Name.Builtin),
+ (r'#.*\n', Comment),
+ (r'\\[\w\W]', String.Escape),
+ (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Text, Operator)),
+ (r'[\[\]()=]', Operator),
+ (r'<<-?\s*(\'?)\\?(\w+)[\w\W]+?\2', String),
+ ],
+ 'data': [
+ (r'(?s)\$?"(\\\\|\\[0-7]+|\\.|[^"\\$])*"', String.Double),
+ (r'"', String.Double, 'string'),
+ (r"(?s)\$'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single),
+ (r"(?s)'.*?'", String.Single),
+ (r';', Punctuation),
+ (r'&|\||\^|<|>', Operator),
+ (r'\s+', Text),
+ (r'\d+(?= |\Z)', Number),
+ (r'[^=\s\[\]{}()$"\'`\\<&|;]+', Text),
+ ],
+ 'string': [
+ (r'"', String.Double, '#pop'),
+ (r'(?s)(\\\\|\\[0-7]+|\\.|[^"\\$])+', String.Double),
+ include('interp'),
+ ],
+ 'paren': [
+ (r'\)', Keyword, '#pop'),
+ include('root'),
+ ],
+ 'math': [
+ (r'\)\)', Keyword, '#pop'),
+ (r'[-+*/%^|&]|\*\*|\|\|', Operator),
+ (r'\d+#\d+', Number),
+ (r'\d+#(?! )', Number),
+ (r'\d+', Number),
+ include('root'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/smalltalk.py b/wandb/vendor/pygments/lexers/smalltalk.py
new file mode 100644
index 0000000000000000000000000000000000000000..79078b662dee357e0c099b4fd95ccd17c1e54069
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/smalltalk.py
@@ -0,0 +1,195 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.smalltalk
+ ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Smalltalk and related languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['SmalltalkLexer', 'NewspeakLexer']
+
+
+class SmalltalkLexer(RegexLexer):
+ """
+ For `Smalltalk <http://www.smalltalk.org/>`_ syntax.
+ Contributed by Stefan Matthias Aust.
+ Rewritten by Nils Winter.
+
+ .. versionadded:: 0.10
+ """
+ name = 'Smalltalk'
+ filenames = ['*.st']
+ aliases = ['smalltalk', 'squeak', 'st']
+ mimetypes = ['text/x-smalltalk']
+
+ tokens = {
+ 'root': [
+ (r'(<)(\w+:)(.*?)(>)', bygroups(Text, Keyword, Text, Text)),
+ include('squeak fileout'),
+ include('whitespaces'),
+ include('method definition'),
+ (r'(\|)([\w\s]*)(\|)', bygroups(Operator, Name.Variable, Operator)),
+ include('objects'),
+ (r'\^|\:=|\_', Operator),
+ # temporaries
+ (r'[\]({}.;!]', Text),
+ ],
+ 'method definition': [
+ # Not perfect can't allow whitespaces at the beginning and the
+ # without breaking everything
+ (r'([a-zA-Z]+\w*:)(\s*)(\w+)',
+ bygroups(Name.Function, Text, Name.Variable)),
+ (r'^(\b[a-zA-Z]+\w*\b)(\s*)$', bygroups(Name.Function, Text)),
+ (r'^([-+*/\\~<>=|&!?,@%]+)(\s*)(\w+)(\s*)$',
+ bygroups(Name.Function, Text, Name.Variable, Text)),
+ ],
+ 'blockvariables': [
+ include('whitespaces'),
+ (r'(:)(\s*)(\w+)',
+ bygroups(Operator, Text, Name.Variable)),
+ (r'\|', Operator, '#pop'),
+ default('#pop'), # else pop
+ ],
+ 'literals': [
+ (r"'(''|[^'])*'", String, 'afterobject'),
+ (r'\$.', String.Char, 'afterobject'),
+ (r'#\(', String.Symbol, 'parenth'),
+ (r'\)', Text, 'afterobject'),
+ (r'(\d+r)?-?\d+(\.\d+)?(e-?\d+)?', Number, 'afterobject'),
+ ],
+ '_parenth_helper': [
+ include('whitespaces'),
+ (r'(\d+r)?-?\d+(\.\d+)?(e-?\d+)?', Number),
+ (r'[-+*/\\~<>=|&#!?,@%\w:]+', String.Symbol),
+ # literals
+ (r"'(''|[^'])*'", String),
+ (r'\$.', String.Char),
+ (r'#*\(', String.Symbol, 'inner_parenth'),
+ ],
+ 'parenth': [
+ # This state is a bit tricky since
+ # we can't just pop this state
+ (r'\)', String.Symbol, ('root', 'afterobject')),
+ include('_parenth_helper'),
+ ],
+ 'inner_parenth': [
+ (r'\)', String.Symbol, '#pop'),
+ include('_parenth_helper'),
+ ],
+ 'whitespaces': [
+ # skip whitespace and comments
+ (r'\s+', Text),
+ (r'"(""|[^"])*"', Comment),
+ ],
+ 'objects': [
+ (r'\[', Text, 'blockvariables'),
+ (r'\]', Text, 'afterobject'),
+ (r'\b(self|super|true|false|nil|thisContext)\b',
+ Name.Builtin.Pseudo, 'afterobject'),
+ (r'\b[A-Z]\w*(?!:)\b', Name.Class, 'afterobject'),
+ (r'\b[a-z]\w*(?!:)\b', Name.Variable, 'afterobject'),
+ (r'#("(""|[^"])*"|[-+*/\\~<>=|&!?,@%]+|[\w:]+)',
+ String.Symbol, 'afterobject'),
+ include('literals'),
+ ],
+ 'afterobject': [
+ (r'! !$', Keyword, '#pop'), # squeak chunk delimiter
+ include('whitespaces'),
+ (r'\b(ifTrue:|ifFalse:|whileTrue:|whileFalse:|timesRepeat:)',
+ Name.Builtin, '#pop'),
+ (r'\b(new\b(?!:))', Name.Builtin),
+ (r'\:=|\_', Operator, '#pop'),
+ (r'\b[a-zA-Z]+\w*:', Name.Function, '#pop'),
+ (r'\b[a-zA-Z]+\w*', Name.Function),
+ (r'\w+:?|[-+*/\\~<>=|&!?,@%]+', Name.Function, '#pop'),
+ (r'\.', Punctuation, '#pop'),
+ (r';', Punctuation),
+ (r'[\])}]', Text),
+ (r'[\[({]', Text, '#pop'),
+ ],
+ 'squeak fileout': [
+ # Squeak fileout format (optional)
+ (r'^"(""|[^"])*"!', Keyword),
+ (r"^'(''|[^'])*'!", Keyword),
+ (r'^(!)(\w+)( commentStamp: )(.*?)( prior: .*?!\n)(.*?)(!)',
+ bygroups(Keyword, Name.Class, Keyword, String, Keyword, Text, Keyword)),
+ (r"^(!)(\w+(?: class)?)( methodsFor: )('(?:''|[^'])*')(.*?!)",
+ bygroups(Keyword, Name.Class, Keyword, String, Keyword)),
+ (r'^(\w+)( subclass: )(#\w+)'
+ r'(\s+instanceVariableNames: )(.*?)'
+ r'(\s+classVariableNames: )(.*?)'
+ r'(\s+poolDictionaries: )(.*?)'
+ r'(\s+category: )(.*?)(!)',
+ bygroups(Name.Class, Keyword, String.Symbol, Keyword, String, Keyword,
+ String, Keyword, String, Keyword, String, Keyword)),
+ (r'^(\w+(?: class)?)(\s+instanceVariableNames: )(.*?)(!)',
+ bygroups(Name.Class, Keyword, String, Keyword)),
+ (r'(!\n)(\].*)(! !)$', bygroups(Keyword, Text, Keyword)),
+ (r'! !$', Keyword),
+ ],
+ }
+
+
+class NewspeakLexer(RegexLexer):
+ """
+ For `Newspeak <http://newspeaklanguage.org/>` syntax.
+
+ .. versionadded:: 1.1
+ """
+ name = 'Newspeak'
+ filenames = ['*.ns2']
+ aliases = ['newspeak', ]
+ mimetypes = ['text/x-newspeak']
+
+ tokens = {
+ 'root': [
+ (r'\b(Newsqueak2)\b', Keyword.Declaration),
+ (r"'[^']*'", String),
+ (r'\b(class)(\s+)(\w+)(\s*)',
+ bygroups(Keyword.Declaration, Text, Name.Class, Text)),
+ (r'\b(mixin|self|super|private|public|protected|nil|true|false)\b',
+ Keyword),
+ (r'(\w+\:)(\s*)([a-zA-Z_]\w+)',
+ bygroups(Name.Function, Text, Name.Variable)),
+ (r'(\w+)(\s*)(=)',
+ bygroups(Name.Attribute, Text, Operator)),
+ (r'<\w+>', Comment.Special),
+ include('expressionstat'),
+ include('whitespace')
+ ],
+
+ 'expressionstat': [
+ (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+ (r'\d+', Number.Integer),
+ (r':\w+', Name.Variable),
+ (r'(\w+)(::)', bygroups(Name.Variable, Operator)),
+ (r'\w+:', Name.Function),
+ (r'\w+', Name.Variable),
+ (r'\(|\)', Punctuation),
+ (r'\[|\]', Punctuation),
+ (r'\{|\}', Punctuation),
+
+ (r'(\^|\+|\/|~|\*|<|>|=|@|%|\||&|\?|!|,|-|:)', Operator),
+ (r'\.|;', Punctuation),
+ include('whitespace'),
+ include('literals'),
+ ],
+ 'literals': [
+ (r'\$.', String),
+ (r"'[^']*'", String),
+ (r"#'[^']*'", String.Symbol),
+ (r"#\w+:?", String.Symbol),
+ (r"#(\+|\/|~|\*|<|>|=|@|%|\||&|\?|!|,|-)+", String.Symbol)
+ ],
+ 'whitespace': [
+ (r'\s+', Text),
+ (r'"[^"]*"', Comment)
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/smv.py b/wandb/vendor/pygments/lexers/smv.py
new file mode 100644
index 0000000000000000000000000000000000000000..380a3b703b4968952b9b2d859c2c2e51ddd5097f
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/smv.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.smv
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the SMV languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Comment, Generic, Keyword, Name, Number, \
+ Operator, Punctuation, Text
+
+__all__ = ['NuSMVLexer']
+
+
+class NuSMVLexer(RegexLexer):
+ """
+ Lexer for the NuSMV language.
+
+ .. versionadded:: 2.2
+ """
+
+ name = 'NuSMV'
+ aliases = ['nusmv']
+ filenames = ['*.smv']
+ mimetypes = []
+
+ tokens = {
+ 'root': [
+ # Comments
+ (r'(?s)\/\-\-.*?\-\-/', Comment),
+ (r'--.*\n', Comment),
+
+ # Reserved
+ (words(('MODULE', 'DEFINE', 'MDEFINE', 'CONSTANTS', 'VAR', 'IVAR',
+ 'FROZENVAR', 'INIT', 'TRANS', 'INVAR', 'SPEC', 'CTLSPEC',
+ 'LTLSPEC', 'PSLSPEC', 'COMPUTE', 'NAME', 'INVARSPEC',
+ 'FAIRNESS', 'JUSTICE', 'COMPASSION', 'ISA', 'ASSIGN',
+ 'CONSTRAINT', 'SIMPWFF', 'CTLWFF', 'LTLWFF', 'PSLWFF',
+ 'COMPWFF', 'IN', 'MIN', 'MAX', 'MIRROR', 'PRED',
+ 'PREDICATES'), suffix=r'(?![\w$#-])'),
+ Keyword.Declaration),
+ (r'process(?![\w$#-])', Keyword),
+ (words(('array', 'of', 'boolean', 'integer', 'real', 'word'),
+ suffix=r'(?![\w$#-])'), Keyword.Type),
+ (words(('case', 'esac'), suffix=r'(?![\w$#-])'), Keyword),
+ (words(('word1', 'bool', 'signed', 'unsigned', 'extend', 'resize',
+ 'sizeof', 'uwconst', 'swconst', 'init', 'self', 'count',
+ 'abs', 'max', 'min'), suffix=r'(?![\w$#-])'),
+ Name.Builtin),
+ (words(('EX', 'AX', 'EF', 'AF', 'EG', 'AG', 'E', 'F', 'O', 'G',
+ 'H', 'X', 'Y', 'Z', 'A', 'U', 'S', 'V', 'T', 'BU', 'EBF',
+ 'ABF', 'EBG', 'ABG', 'next', 'mod', 'union', 'in', 'xor',
+ 'xnor'), suffix=r'(?![\w$#-])'),
+ Operator.Word),
+ (words(('TRUE', 'FALSE'), suffix=r'(?![\w$#-])'), Keyword.Constant),
+
+ # Names
+ (r'[a-zA-Z_][\w$#-]*', Name.Variable),
+
+ # Operators
+ (r':=', Operator),
+ (r'[-&|+*/<>!=]', Operator),
+
+ # Literals
+ (r'\-?\d+\b', Number.Integer),
+ (r'0[su][bB]\d*_[01_]+', Number.Bin),
+ (r'0[su][oO]\d*_[0-7_]+', Number.Oct),
+ (r'0[su][dD]\d*_[\d_]+', Number.Dec),
+ (r'0[su][hH]\d*_[\da-fA-F_]+', Number.Hex),
+
+ # Whitespace, punctuation and the rest
+ (r'\s+', Text.Whitespace),
+ (r'[()\[\]{};?:.,]', Punctuation),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/snobol.py b/wandb/vendor/pygments/lexers/snobol.py
new file mode 100644
index 0000000000000000000000000000000000000000..f6e12fd26761d41d16e71260f679a6a86263f7a6
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/snobol.py
@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.snobol
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the SNOBOL language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['SnobolLexer']
+
+
+class SnobolLexer(RegexLexer):
+ """
+ Lexer for the SNOBOL4 programming language.
+
+ Recognizes the common ASCII equivalents of the original SNOBOL4 operators.
+ Does not require spaces around binary operators.
+
+ .. versionadded:: 1.5
+ """
+
+ name = "Snobol"
+ aliases = ["snobol"]
+ filenames = ['*.snobol']
+ mimetypes = ['text/x-snobol']
+
+ tokens = {
+ # root state, start of line
+ # comments, continuation lines, and directives start in column 1
+ # as do labels
+ 'root': [
+ (r'\*.*\n', Comment),
+ (r'[+.] ', Punctuation, 'statement'),
+ (r'-.*\n', Comment),
+ (r'END\s*\n', Name.Label, 'heredoc'),
+ (r'[A-Za-z$][\w$]*', Name.Label, 'statement'),
+ (r'\s+', Text, 'statement'),
+ ],
+ # statement state, line after continuation or label
+ 'statement': [
+ (r'\s*\n', Text, '#pop'),
+ (r'\s+', Text),
+ (r'(?<=[^\w.])(LT|LE|EQ|NE|GE|GT|INTEGER|IDENT|DIFFER|LGT|SIZE|'
+ r'REPLACE|TRIM|DUPL|REMDR|DATE|TIME|EVAL|APPLY|OPSYN|LOAD|UNLOAD|'
+ r'LEN|SPAN|BREAK|ANY|NOTANY|TAB|RTAB|REM|POS|RPOS|FAIL|FENCE|'
+ r'ABORT|ARB|ARBNO|BAL|SUCCEED|INPUT|OUTPUT|TERMINAL)(?=[^\w.])',
+ Name.Builtin),
+ (r'[A-Za-z][\w.]*', Name),
+ # ASCII equivalents of original operators
+ # | for the EBCDIC equivalent, ! likewise
+ # \ for EBCDIC negation
+ (r'\*\*|[?$.!%*/#+\-@|&\\=]', Operator),
+ (r'"[^"]*"', String),
+ (r"'[^']*'", String),
+ # Accept SPITBOL syntax for real numbers
+ # as well as Macro SNOBOL4
+ (r'[0-9]+(?=[^.EeDd])', Number.Integer),
+ (r'[0-9]+(\.[0-9]*)?([EDed][-+]?[0-9]+)?', Number.Float),
+ # Goto
+ (r':', Punctuation, 'goto'),
+ (r'[()<>,;]', Punctuation),
+ ],
+ # Goto block
+ 'goto': [
+ (r'\s*\n', Text, "#pop:2"),
+ (r'\s+', Text),
+ (r'F|S', Keyword),
+ (r'(\()([A-Za-z][\w.]*)(\))',
+ bygroups(Punctuation, Name.Label, Punctuation))
+ ],
+ # everything after the END statement is basically one
+ # big heredoc.
+ 'heredoc': [
+ (r'.*\n', String.Heredoc)
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/special.py b/wandb/vendor/pygments/lexers/special.py
new file mode 100644
index 0000000000000000000000000000000000000000..6e076b0ce7b5c9c4b963b9995e19d5862b3f4b4b
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/special.py
@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.special
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Special lexers.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer
+from pygments.token import Token, Error, Text
+from pygments.util import get_choice_opt, text_type, BytesIO
+
+
+__all__ = ['TextLexer', 'RawTokenLexer']
+
+
+class TextLexer(Lexer):
+ """
+ "Null" lexer, doesn't highlight anything.
+ """
+ name = 'Text only'
+ aliases = ['text']
+ filenames = ['*.txt']
+ mimetypes = ['text/plain']
+ priority = 0.01
+
+ def get_tokens_unprocessed(self, text):
+ yield 0, Text, text
+
+ def analyse_text(text):
+ return TextLexer.priority
+
+_ttype_cache = {}
+
+line_re = re.compile(b'.*?\n')
+
+
+class RawTokenLexer(Lexer):
+ """
+ Recreate a token stream formatted with the `RawTokenFormatter`. This
+ lexer raises exceptions during parsing if the token stream in the
+ file is malformed.
+
+ Additional options accepted:
+
+ `compress`
+ If set to ``"gz"`` or ``"bz2"``, decompress the token stream with
+ the given compression algorithm before lexing (default: ``""``).
+ """
+ name = 'Raw token data'
+ aliases = ['raw']
+ filenames = []
+ mimetypes = ['application/x-pygments-tokens']
+
+ def __init__(self, **options):
+ self.compress = get_choice_opt(options, 'compress',
+ ['', 'none', 'gz', 'bz2'], '')
+ Lexer.__init__(self, **options)
+
+ def get_tokens(self, text):
+ if isinstance(text, text_type):
+ # raw token stream never has any non-ASCII characters
+ text = text.encode('ascii')
+ if self.compress == 'gz':
+ import gzip
+ gzipfile = gzip.GzipFile('', 'rb', 9, BytesIO(text))
+ text = gzipfile.read()
+ elif self.compress == 'bz2':
+ import bz2
+ text = bz2.decompress(text)
+
+ # do not call Lexer.get_tokens() because we do not want Unicode
+ # decoding to occur, and stripping is not optional.
+ text = text.strip(b'\n') + b'\n'
+ for i, t, v in self.get_tokens_unprocessed(text):
+ yield t, v
+
+ def get_tokens_unprocessed(self, text):
+ length = 0
+ for match in line_re.finditer(text):
+ try:
+ ttypestr, val = match.group().split(b'\t', 1)
+ except ValueError:
+ val = match.group().decode('ascii', 'replace')
+ ttype = Error
+ else:
+ ttype = _ttype_cache.get(ttypestr)
+ if not ttype:
+ ttype = Token
+ ttypes = ttypestr.split('.')[1:]
+ for ttype_ in ttypes:
+ if not ttype_ or not ttype_[0].isupper():
+ raise ValueError('malformed token name')
+ ttype = getattr(ttype, ttype_)
+ _ttype_cache[ttypestr] = ttype
+ val = val[2:-2].decode('unicode-escape')
+ yield length, ttype, val
+ length += len(val)
diff --git a/wandb/vendor/pygments/lexers/sql.py b/wandb/vendor/pygments/lexers/sql.py
new file mode 100644
index 0000000000000000000000000000000000000000..7507c0fc80f9891ff16378de93a9362f2b17b904
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/sql.py
@@ -0,0 +1,681 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.sql
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexers for various SQL dialects and related interactive sessions.
+
+ Postgres specific lexers:
+
+ `PostgresLexer`
+ A SQL lexer for the PostgreSQL dialect. Differences w.r.t. the SQL
+ lexer are:
+
+ - keywords and data types list parsed from the PG docs (run the
+ `_postgres_builtins` module to update them);
+ - Content of $-strings parsed using a specific lexer, e.g. the content
+ of a PL/Python function is parsed using the Python lexer;
+ - parse PG specific constructs: E-strings, $-strings, U&-strings,
+ different operators and punctuation.
+
+ `PlPgsqlLexer`
+ A lexer for the PL/pgSQL language. Adds a few specific construct on
+ top of the PG SQL lexer (such as <<label>>).
+
+ `PostgresConsoleLexer`
+ A lexer to highlight an interactive psql session:
+
+ - identifies the prompt and does its best to detect the end of command
+ in multiline statement where not all the lines are prefixed by a
+ prompt, telling them apart from the output;
+ - highlights errors in the output and notification levels;
+ - handles psql backslash commands.
+
+ The ``tests/examplefiles`` contains a few test files with data to be
+ parsed by these lexers.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, do_insertions, bygroups, words
+from pygments.token import Punctuation, Whitespace, Error, \
+ Text, Comment, Operator, Keyword, Name, String, Number, Generic
+from pygments.lexers import get_lexer_by_name, ClassNotFound
+from pygments.util import iteritems
+
+from pygments.lexers._postgres_builtins import KEYWORDS, DATATYPES, \
+ PSEUDO_TYPES, PLPGSQL_KEYWORDS
+from pygments.lexers import _tsql_builtins
+
+
+__all__ = ['PostgresLexer', 'PlPgsqlLexer', 'PostgresConsoleLexer',
+ 'SqlLexer', 'TransactSqlLexer', 'MySqlLexer',
+ 'SqliteConsoleLexer', 'RqlLexer']
+
+line_re = re.compile('.*?\n')
+
+language_re = re.compile(r"\s+LANGUAGE\s+'?(\w+)'?", re.IGNORECASE)
+
+do_re = re.compile(r'\bDO\b', re.IGNORECASE)
+
+
+def language_callback(lexer, match):
+ """Parse the content of a $-string using a lexer
+
+ The lexer is chosen looking for a nearby LANGUAGE or assumed as
+ plpgsql if inside a DO statement and no LANGUAGE has been found.
+ """
+ l = None
+ m = language_re.match(lexer.text[match.end():match.end()+100])
+ if m is not None:
+ l = lexer._get_lexer(m.group(1))
+ else:
+ m = list(language_re.finditer(
+ lexer.text[max(0, match.start()-100):match.start()]))
+ if m:
+ l = lexer._get_lexer(m[-1].group(1))
+ else:
+ m = list(do_re.finditer(
+ lexer.text[max(0, match.start()-25):match.start()]))
+ if m:
+ l = lexer._get_lexer('plpgsql')
+
+ # 1 = $, 2 = delimiter, 3 = $
+ yield (match.start(1), String, match.group(1))
+ yield (match.start(2), String.Delimiter, match.group(2))
+ yield (match.start(3), String, match.group(3))
+ # 4 = string contents
+ if l:
+ for x in l.get_tokens_unprocessed(match.group(4)):
+ yield x
+ else:
+ yield (match.start(4), String, match.group(4))
+ # 5 = $, 6 = delimiter, 7 = $
+ yield (match.start(5), String, match.group(5))
+ yield (match.start(6), String.Delimiter, match.group(6))
+ yield (match.start(7), String, match.group(7))
+
+
+class PostgresBase(object):
+ """Base class for Postgres-related lexers.
+
+ This is implemented as a mixin to avoid the Lexer metaclass kicking in.
+ this way the different lexer don't have a common Lexer ancestor. If they
+ had, _tokens could be created on this ancestor and not updated for the
+ other classes, resulting e.g. in PL/pgSQL parsed as SQL. This shortcoming
+ seem to suggest that regexp lexers are not really subclassable.
+ """
+ def get_tokens_unprocessed(self, text, *args):
+ # Have a copy of the entire text to be used by `language_callback`.
+ self.text = text
+ for x in super(PostgresBase, self).get_tokens_unprocessed(
+ text, *args):
+ yield x
+
+ def _get_lexer(self, lang):
+ if lang.lower() == 'sql':
+ return get_lexer_by_name('postgresql', **self.options)
+
+ tries = [lang]
+ if lang.startswith('pl'):
+ tries.append(lang[2:])
+ if lang.endswith('u'):
+ tries.append(lang[:-1])
+ if lang.startswith('pl') and lang.endswith('u'):
+ tries.append(lang[2:-1])
+
+ for l in tries:
+ try:
+ return get_lexer_by_name(l, **self.options)
+ except ClassNotFound:
+ pass
+ else:
+ # TODO: better logging
+ # print >>sys.stderr, "language not found:", lang
+ return None
+
+
+class PostgresLexer(PostgresBase, RegexLexer):
+ """
+ Lexer for the PostgreSQL dialect of SQL.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'PostgreSQL SQL dialect'
+ aliases = ['postgresql', 'postgres']
+ mimetypes = ['text/x-postgresql']
+
+ flags = re.IGNORECASE
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'--.*\n?', Comment.Single),
+ (r'/\*', Comment.Multiline, 'multiline-comments'),
+ (r'(' + '|'.join(s.replace(" ", "\s+")
+ for s in DATATYPES + PSEUDO_TYPES)
+ + r')\b', Name.Builtin),
+ (words(KEYWORDS, suffix=r'\b'), Keyword),
+ (r'[+*/<>=~!@#%^&|`?-]+', Operator),
+ (r'::', Operator), # cast
+ (r'\$\d+', Name.Variable),
+ (r'([0-9]*\.[0-9]*|[0-9]+)(e[+-]?[0-9]+)?', Number.Float),
+ (r'[0-9]+', Number.Integer),
+ (r"((?:E|U&)?)(')", bygroups(String.Affix, String.Single), 'string'),
+ # quoted identifier
+ (r'((?:U&)?)(")', bygroups(String.Affix, String.Name), 'quoted-ident'),
+ (r'(?s)(\$)([^$]*)(\$)(.*?)(\$)(\2)(\$)', language_callback),
+ (r'[a-z_]\w*', Name),
+
+ # psql variable in SQL
+ (r""":(['"]?)[a-z]\w*\b\1""", Name.Variable),
+
+ (r'[;:()\[\]{},.]', Punctuation),
+ ],
+ 'multiline-comments': [
+ (r'/\*', Comment.Multiline, 'multiline-comments'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[^/*]+', Comment.Multiline),
+ (r'[/*]', Comment.Multiline)
+ ],
+ 'string': [
+ (r"[^']+", String.Single),
+ (r"''", String.Single),
+ (r"'", String.Single, '#pop'),
+ ],
+ 'quoted-ident': [
+ (r'[^"]+', String.Name),
+ (r'""', String.Name),
+ (r'"', String.Name, '#pop'),
+ ],
+ }
+
+
+class PlPgsqlLexer(PostgresBase, RegexLexer):
+ """
+ Handle the extra syntax in Pl/pgSQL language.
+
+ .. versionadded:: 1.5
+ """
+ name = 'PL/pgSQL'
+ aliases = ['plpgsql']
+ mimetypes = ['text/x-plpgsql']
+
+ flags = re.IGNORECASE
+ tokens = dict((k, l[:]) for (k, l) in iteritems(PostgresLexer.tokens))
+
+ # extend the keywords list
+ for i, pattern in enumerate(tokens['root']):
+ if pattern[1] == Keyword:
+ tokens['root'][i] = (
+ words(KEYWORDS + PLPGSQL_KEYWORDS, suffix=r'\b'),
+ Keyword)
+ del i
+ break
+ else:
+ assert 0, "SQL keywords not found"
+
+ # Add specific PL/pgSQL rules (before the SQL ones)
+ tokens['root'][:0] = [
+ (r'\%[a-z]\w*\b', Name.Builtin), # actually, a datatype
+ (r':=', Operator),
+ (r'\<\<[a-z]\w*\>\>', Name.Label),
+ (r'\#[a-z]\w*\b', Keyword.Pseudo), # #variable_conflict
+ ]
+
+
+class PsqlRegexLexer(PostgresBase, RegexLexer):
+ """
+ Extend the PostgresLexer adding support specific for psql commands.
+
+ This is not a complete psql lexer yet as it lacks prompt support
+ and output rendering.
+ """
+
+ name = 'PostgreSQL console - regexp based lexer'
+ aliases = [] # not public
+
+ flags = re.IGNORECASE
+ tokens = dict((k, l[:]) for (k, l) in iteritems(PostgresLexer.tokens))
+
+ tokens['root'].append(
+ (r'\\[^\s]+', Keyword.Pseudo, 'psql-command'))
+ tokens['psql-command'] = [
+ (r'\n', Text, 'root'),
+ (r'\s+', Text),
+ (r'\\[^\s]+', Keyword.Pseudo),
+ (r""":(['"]?)[a-z]\w*\b\1""", Name.Variable),
+ (r"'(''|[^'])*'", String.Single),
+ (r"`([^`])*`", String.Backtick),
+ (r"[^\s]+", String.Symbol),
+ ]
+
+re_prompt = re.compile(r'^(\S.*?)??[=\-\(\$\'\"][#>]')
+re_psql_command = re.compile(r'\s*\\')
+re_end_command = re.compile(r';\s*(--.*?)?$')
+re_psql_command = re.compile(r'(\s*)(\\.+?)(\s+)$')
+re_error = re.compile(r'(ERROR|FATAL):')
+re_message = re.compile(
+ r'((?:DEBUG|INFO|NOTICE|WARNING|ERROR|'
+ r'FATAL|HINT|DETAIL|CONTEXT|LINE [0-9]+):)(.*?\n)')
+
+
+class lookahead(object):
+ """Wrap an iterator and allow pushing back an item."""
+ def __init__(self, x):
+ self.iter = iter(x)
+ self._nextitem = None
+
+ def __iter__(self):
+ return self
+
+ def send(self, i):
+ self._nextitem = i
+ return i
+
+ def __next__(self):
+ if self._nextitem is not None:
+ ni = self._nextitem
+ self._nextitem = None
+ return ni
+ return next(self.iter)
+ next = __next__
+
+
+class PostgresConsoleLexer(Lexer):
+ """
+ Lexer for psql sessions.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'PostgreSQL console (psql)'
+ aliases = ['psql', 'postgresql-console', 'postgres-console']
+ mimetypes = ['text/x-postgresql-psql']
+
+ def get_tokens_unprocessed(self, data):
+ sql = PsqlRegexLexer(**self.options)
+
+ lines = lookahead(line_re.findall(data))
+
+ # prompt-output cycle
+ while 1:
+
+ # consume the lines of the command: start with an optional prompt
+ # and continue until the end of command is detected
+ curcode = ''
+ insertions = []
+ while 1:
+ try:
+ line = next(lines)
+ except StopIteration:
+ # allow the emission of partially collected items
+ # the repl loop will be broken below
+ break
+
+ # Identify a shell prompt in case of psql commandline example
+ if line.startswith('$') and not curcode:
+ lexer = get_lexer_by_name('console', **self.options)
+ for x in lexer.get_tokens_unprocessed(line):
+ yield x
+ break
+
+ # Identify a psql prompt
+ mprompt = re_prompt.match(line)
+ if mprompt is not None:
+ insertions.append((len(curcode),
+ [(0, Generic.Prompt, mprompt.group())]))
+ curcode += line[len(mprompt.group()):]
+ else:
+ curcode += line
+
+ # Check if this is the end of the command
+ # TODO: better handle multiline comments at the end with
+ # a lexer with an external state?
+ if re_psql_command.match(curcode) \
+ or re_end_command.search(curcode):
+ break
+
+ # Emit the combined stream of command and prompt(s)
+ for item in do_insertions(insertions,
+ sql.get_tokens_unprocessed(curcode)):
+ yield item
+
+ # Emit the output lines
+ out_token = Generic.Output
+ while 1:
+ line = next(lines)
+ mprompt = re_prompt.match(line)
+ if mprompt is not None:
+ # push the line back to have it processed by the prompt
+ lines.send(line)
+ break
+
+ mmsg = re_message.match(line)
+ if mmsg is not None:
+ if mmsg.group(1).startswith("ERROR") \
+ or mmsg.group(1).startswith("FATAL"):
+ out_token = Generic.Error
+ yield (mmsg.start(1), Generic.Strong, mmsg.group(1))
+ yield (mmsg.start(2), out_token, mmsg.group(2))
+ else:
+ yield (0, out_token, line)
+
+
+class SqlLexer(RegexLexer):
+ """
+ Lexer for Structured Query Language. Currently, this lexer does
+ not recognize any special syntax except ANSI SQL.
+ """
+
+ name = 'SQL'
+ aliases = ['sql']
+ filenames = ['*.sql']
+ mimetypes = ['text/x-sql']
+
+ flags = re.IGNORECASE
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'--.*\n?', Comment.Single),
+ (r'/\*', Comment.Multiline, 'multiline-comments'),
+ (words((
+ 'ABORT', 'ABS', 'ABSOLUTE', 'ACCESS', 'ADA', 'ADD', 'ADMIN', 'AFTER', 'AGGREGATE',
+ 'ALIAS', 'ALL', 'ALLOCATE', 'ALTER', 'ANALYSE', 'ANALYZE', 'AND', 'ANY', 'ARE', 'AS',
+ 'ASC', 'ASENSITIVE', 'ASSERTION', 'ASSIGNMENT', 'ASYMMETRIC', 'AT', 'ATOMIC',
+ 'AUTHORIZATION', 'AVG', 'BACKWARD', 'BEFORE', 'BEGIN', 'BETWEEN', 'BITVAR',
+ 'BIT_LENGTH', 'BOTH', 'BREADTH', 'BY', 'C', 'CACHE', 'CALL', 'CALLED', 'CARDINALITY',
+ 'CASCADE', 'CASCADED', 'CASE', 'CAST', 'CATALOG', 'CATALOG_NAME', 'CHAIN',
+ 'CHARACTERISTICS', 'CHARACTER_LENGTH', 'CHARACTER_SET_CATALOG',
+ 'CHARACTER_SET_NAME', 'CHARACTER_SET_SCHEMA', 'CHAR_LENGTH', 'CHECK',
+ 'CHECKED', 'CHECKPOINT', 'CLASS', 'CLASS_ORIGIN', 'CLOB', 'CLOSE', 'CLUSTER',
+ 'COALSECE', 'COBOL', 'COLLATE', 'COLLATION', 'COLLATION_CATALOG',
+ 'COLLATION_NAME', 'COLLATION_SCHEMA', 'COLUMN', 'COLUMN_NAME',
+ 'COMMAND_FUNCTION', 'COMMAND_FUNCTION_CODE', 'COMMENT', 'COMMIT',
+ 'COMMITTED', 'COMPLETION', 'CONDITION_NUMBER', 'CONNECT', 'CONNECTION',
+ 'CONNECTION_NAME', 'CONSTRAINT', 'CONSTRAINTS', 'CONSTRAINT_CATALOG',
+ 'CONSTRAINT_NAME', 'CONSTRAINT_SCHEMA', 'CONSTRUCTOR', 'CONTAINS',
+ 'CONTINUE', 'CONVERSION', 'CONVERT', 'COPY', 'CORRESPONTING', 'COUNT',
+ 'CREATE', 'CREATEDB', 'CREATEUSER', 'CROSS', 'CUBE', 'CURRENT', 'CURRENT_DATE',
+ 'CURRENT_PATH', 'CURRENT_ROLE', 'CURRENT_TIME', 'CURRENT_TIMESTAMP',
+ 'CURRENT_USER', 'CURSOR', 'CURSOR_NAME', 'CYCLE', 'DATA', 'DATABASE',
+ 'DATETIME_INTERVAL_CODE', 'DATETIME_INTERVAL_PRECISION', 'DAY',
+ 'DEALLOCATE', 'DECLARE', 'DEFAULT', 'DEFAULTS', 'DEFERRABLE', 'DEFERRED',
+ 'DEFINED', 'DEFINER', 'DELETE', 'DELIMITER', 'DELIMITERS', 'DEREF', 'DESC',
+ 'DESCRIBE', 'DESCRIPTOR', 'DESTROY', 'DESTRUCTOR', 'DETERMINISTIC',
+ 'DIAGNOSTICS', 'DICTIONARY', 'DISCONNECT', 'DISPATCH', 'DISTINCT', 'DO',
+ 'DOMAIN', 'DROP', 'DYNAMIC', 'DYNAMIC_FUNCTION', 'DYNAMIC_FUNCTION_CODE', 'EACH',
+ 'ELSE', 'ELSIF', 'ENCODING', 'ENCRYPTED', 'END', 'END-EXEC', 'EQUALS', 'ESCAPE', 'EVERY',
+ 'EXCEPTION', 'EXCEPT', 'EXCLUDING', 'EXCLUSIVE', 'EXEC', 'EXECUTE', 'EXISTING',
+ 'EXISTS', 'EXPLAIN', 'EXTERNAL', 'EXTRACT', 'FALSE', 'FETCH', 'FINAL', 'FIRST', 'FOR',
+ 'FORCE', 'FOREIGN', 'FORTRAN', 'FORWARD', 'FOUND', 'FREE', 'FREEZE', 'FROM', 'FULL',
+ 'FUNCTION', 'G', 'GENERAL', 'GENERATED', 'GET', 'GLOBAL', 'GO', 'GOTO', 'GRANT', 'GRANTED',
+ 'GROUP', 'GROUPING', 'HANDLER', 'HAVING', 'HIERARCHY', 'HOLD', 'HOST', 'IDENTITY', 'IF',
+ 'IGNORE', 'ILIKE', 'IMMEDIATE', 'IMMUTABLE', 'IMPLEMENTATION', 'IMPLICIT', 'IN',
+ 'INCLUDING', 'INCREMENT', 'INDEX', 'INDITCATOR', 'INFIX', 'INHERITS', 'INITIALIZE',
+ 'INITIALLY', 'INNER', 'INOUT', 'INPUT', 'INSENSITIVE', 'INSERT', 'INSTANTIABLE',
+ 'INSTEAD', 'INTERSECT', 'INTO', 'INVOKER', 'IS', 'ISNULL', 'ISOLATION', 'ITERATE', 'JOIN',
+ 'KEY', 'KEY_MEMBER', 'KEY_TYPE', 'LANCOMPILER', 'LANGUAGE', 'LARGE', 'LAST',
+ 'LATERAL', 'LEADING', 'LEFT', 'LENGTH', 'LESS', 'LEVEL', 'LIKE', 'LIMIT', 'LISTEN', 'LOAD',
+ 'LOCAL', 'LOCALTIME', 'LOCALTIMESTAMP', 'LOCATION', 'LOCATOR', 'LOCK', 'LOWER',
+ 'MAP', 'MATCH', 'MAX', 'MAXVALUE', 'MESSAGE_LENGTH', 'MESSAGE_OCTET_LENGTH',
+ 'MESSAGE_TEXT', 'METHOD', 'MIN', 'MINUTE', 'MINVALUE', 'MOD', 'MODE', 'MODIFIES',
+ 'MODIFY', 'MONTH', 'MORE', 'MOVE', 'MUMPS', 'NAMES', 'NATIONAL', 'NATURAL', 'NCHAR',
+ 'NCLOB', 'NEW', 'NEXT', 'NO', 'NOCREATEDB', 'NOCREATEUSER', 'NONE', 'NOT', 'NOTHING',
+ 'NOTIFY', 'NOTNULL', 'NULL', 'NULLABLE', 'NULLIF', 'OBJECT', 'OCTET_LENGTH', 'OF', 'OFF',
+ 'OFFSET', 'OIDS', 'OLD', 'ON', 'ONLY', 'OPEN', 'OPERATION', 'OPERATOR', 'OPTION', 'OPTIONS',
+ 'OR', 'ORDER', 'ORDINALITY', 'OUT', 'OUTER', 'OUTPUT', 'OVERLAPS', 'OVERLAY', 'OVERRIDING',
+ 'OWNER', 'PAD', 'PARAMETER', 'PARAMETERS', 'PARAMETER_MODE', 'PARAMATER_NAME',
+ 'PARAMATER_ORDINAL_POSITION', 'PARAMETER_SPECIFIC_CATALOG',
+ 'PARAMETER_SPECIFIC_NAME', 'PARAMATER_SPECIFIC_SCHEMA', 'PARTIAL',
+ 'PASCAL', 'PENDANT', 'PLACING', 'PLI', 'POSITION', 'POSTFIX', 'PRECISION', 'PREFIX',
+ 'PREORDER', 'PREPARE', 'PRESERVE', 'PRIMARY', 'PRIOR', 'PRIVILEGES', 'PROCEDURAL',
+ 'PROCEDURE', 'PUBLIC', 'READ', 'READS', 'RECHECK', 'RECURSIVE', 'REF', 'REFERENCES',
+ 'REFERENCING', 'REINDEX', 'RELATIVE', 'RENAME', 'REPEATABLE', 'REPLACE', 'RESET',
+ 'RESTART', 'RESTRICT', 'RESULT', 'RETURN', 'RETURNED_LENGTH',
+ 'RETURNED_OCTET_LENGTH', 'RETURNED_SQLSTATE', 'RETURNS', 'REVOKE', 'RIGHT',
+ 'ROLE', 'ROLLBACK', 'ROLLUP', 'ROUTINE', 'ROUTINE_CATALOG', 'ROUTINE_NAME',
+ 'ROUTINE_SCHEMA', 'ROW', 'ROWS', 'ROW_COUNT', 'RULE', 'SAVE_POINT', 'SCALE', 'SCHEMA',
+ 'SCHEMA_NAME', 'SCOPE', 'SCROLL', 'SEARCH', 'SECOND', 'SECURITY', 'SELECT', 'SELF',
+ 'SENSITIVE', 'SERIALIZABLE', 'SERVER_NAME', 'SESSION', 'SESSION_USER', 'SET',
+ 'SETOF', 'SETS', 'SHARE', 'SHOW', 'SIMILAR', 'SIMPLE', 'SIZE', 'SOME', 'SOURCE', 'SPACE',
+ 'SPECIFIC', 'SPECIFICTYPE', 'SPECIFIC_NAME', 'SQL', 'SQLCODE', 'SQLERROR',
+ 'SQLEXCEPTION', 'SQLSTATE', 'SQLWARNINIG', 'STABLE', 'START', 'STATE', 'STATEMENT',
+ 'STATIC', 'STATISTICS', 'STDIN', 'STDOUT', 'STORAGE', 'STRICT', 'STRUCTURE', 'STYPE',
+ 'SUBCLASS_ORIGIN', 'SUBLIST', 'SUBSTRING', 'SUM', 'SYMMETRIC', 'SYSID', 'SYSTEM',
+ 'SYSTEM_USER', 'TABLE', 'TABLE_NAME', ' TEMP', 'TEMPLATE', 'TEMPORARY', 'TERMINATE',
+ 'THAN', 'THEN', 'TIMESTAMP', 'TIMEZONE_HOUR', 'TIMEZONE_MINUTE', 'TO', 'TOAST',
+ 'TRAILING', 'TRANSATION', 'TRANSACTIONS_COMMITTED',
+ 'TRANSACTIONS_ROLLED_BACK', 'TRANSATION_ACTIVE', 'TRANSFORM',
+ 'TRANSFORMS', 'TRANSLATE', 'TRANSLATION', 'TREAT', 'TRIGGER', 'TRIGGER_CATALOG',
+ 'TRIGGER_NAME', 'TRIGGER_SCHEMA', 'TRIM', 'TRUE', 'TRUNCATE', 'TRUSTED', 'TYPE',
+ 'UNCOMMITTED', 'UNDER', 'UNENCRYPTED', 'UNION', 'UNIQUE', 'UNKNOWN', 'UNLISTEN',
+ 'UNNAMED', 'UNNEST', 'UNTIL', 'UPDATE', 'UPPER', 'USAGE', 'USER',
+ 'USER_DEFINED_TYPE_CATALOG', 'USER_DEFINED_TYPE_NAME',
+ 'USER_DEFINED_TYPE_SCHEMA', 'USING', 'VACUUM', 'VALID', 'VALIDATOR', 'VALUES',
+ 'VARIABLE', 'VERBOSE', 'VERSION', 'VIEW', 'VOLATILE', 'WHEN', 'WHENEVER', 'WHERE',
+ 'WITH', 'WITHOUT', 'WORK', 'WRITE', 'YEAR', 'ZONE'), suffix=r'\b'),
+ Keyword),
+ (words((
+ 'ARRAY', 'BIGINT', 'BINARY', 'BIT', 'BLOB', 'BOOLEAN', 'CHAR', 'CHARACTER', 'DATE',
+ 'DEC', 'DECIMAL', 'FLOAT', 'INT', 'INTEGER', 'INTERVAL', 'NUMBER', 'NUMERIC', 'REAL',
+ 'SERIAL', 'SMALLINT', 'VARCHAR', 'VARYING', 'INT8', 'SERIAL8', 'TEXT'), suffix=r'\b'),
+ Name.Builtin),
+ (r'[+*/<>=~!@#%^&|`?-]', Operator),
+ (r'[0-9]+', Number.Integer),
+ # TODO: Backslash escapes?
+ (r"'(''|[^'])*'", String.Single),
+ (r'"(""|[^"])*"', String.Symbol), # not a real string literal in ANSI SQL
+ (r'[a-z_][\w$]*', Name), # allow $s in strings for Oracle
+ (r'[;:()\[\],.]', Punctuation)
+ ],
+ 'multiline-comments': [
+ (r'/\*', Comment.Multiline, 'multiline-comments'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[^/*]+', Comment.Multiline),
+ (r'[/*]', Comment.Multiline)
+ ]
+ }
+
+
+class TransactSqlLexer(RegexLexer):
+ """
+ Transact-SQL (T-SQL) is Microsoft's and Sybase's proprietary extension to
+ SQL.
+
+ The list of keywords includes ODBC and keywords reserved for future use..
+ """
+
+ name = 'Transact-SQL'
+ aliases = ['tsql', 't-sql']
+ filenames = ['*.sql']
+ mimetypes = ['text/x-tsql']
+
+ # Use re.UNICODE to allow non ASCII letters in names.
+ flags = re.IGNORECASE | re.UNICODE
+ tokens = {
+ 'root': [
+ (r'\s+', Whitespace),
+ (r'--(?m).*?$\n?', Comment.Single),
+ (r'/\*', Comment.Multiline, 'multiline-comments'),
+ (words(_tsql_builtins.OPERATORS), Operator),
+ (words(_tsql_builtins.OPERATOR_WORDS, suffix=r'\b'), Operator.Word),
+ (words(_tsql_builtins.TYPES, suffix=r'\b'), Name.Class),
+ (words(_tsql_builtins.FUNCTIONS, suffix=r'\b'), Name.Function),
+ (r'(goto)(\s+)(\w+\b)', bygroups(Keyword, Whitespace, Name.Label)),
+ (words(_tsql_builtins.KEYWORDS, suffix=r'\b'), Keyword),
+ (r'(\[)([^]]+)(\])', bygroups(Operator, Name, Operator)),
+ (r'0x[0-9a-f]+', Number.Hex),
+ # Float variant 1, for example: 1., 1.e2, 1.2e3
+ (r'[0-9]+\.[0-9]*(e[+-]?[0-9]+)?', Number.Float),
+ # Float variant 2, for example: .1, .1e2
+ (r'\.[0-9]+(e[+-]?[0-9]+)?', Number.Float),
+ # Float variant 3, for example: 123e45
+ (r'[0-9]+e[+-]?[0-9]+', Number.Float),
+ (r'[0-9]+', Number.Integer),
+ (r"'(''|[^'])*'", String.Single),
+ (r'"(""|[^"])*"', String.Symbol),
+ (r'[;(),.]', Punctuation),
+ # Below we use \w even for the first "real" character because
+ # tokens starting with a digit have already been recognized
+ # as Number above.
+ (r'@@\w+', Name.Builtin),
+ (r'@\w+', Name.Variable),
+ (r'(\w+)(:)', bygroups(Name.Label, Punctuation)),
+ (r'#?#?\w+', Name), # names for temp tables and anything else
+ (r'\?', Name.Variable.Magic), # parameter for prepared statements
+ ],
+ 'multiline-comments': [
+ (r'/\*', Comment.Multiline, 'multiline-comments'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[^/*]+', Comment.Multiline),
+ (r'[/*]', Comment.Multiline)
+ ]
+ }
+
+
+class MySqlLexer(RegexLexer):
+ """
+ Special lexer for MySQL.
+ """
+
+ name = 'MySQL'
+ aliases = ['mysql']
+ mimetypes = ['text/x-mysql']
+
+ flags = re.IGNORECASE
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'(#|--\s+).*\n?', Comment.Single),
+ (r'/\*', Comment.Multiline, 'multiline-comments'),
+ (r'[0-9]+', Number.Integer),
+ (r'[0-9]*\.[0-9]+(e[+-][0-9]+)', Number.Float),
+ (r"'(\\\\|\\'|''|[^'])*'", String.Single),
+ (r'"(\\\\|\\"|""|[^"])*"', String.Double),
+ (r"`(\\\\|\\`|``|[^`])*`", String.Symbol),
+ (r'[+*/<>=~!@#%^&|`?-]', Operator),
+ (r'\b(tinyint|smallint|mediumint|int|integer|bigint|date|'
+ r'datetime|time|bit|bool|tinytext|mediumtext|longtext|text|'
+ r'tinyblob|mediumblob|longblob|blob|float|double|double\s+'
+ r'precision|real|numeric|dec|decimal|timestamp|year|char|'
+ r'varchar|varbinary|varcharacter|enum|set)(\b\s*)(\()?',
+ bygroups(Keyword.Type, Text, Punctuation)),
+ (r'\b(add|all|alter|analyze|and|as|asc|asensitive|before|between|'
+ r'bigint|binary|blob|both|by|call|cascade|case|change|char|'
+ r'character|check|collate|column|condition|constraint|continue|'
+ r'convert|create|cross|current_date|current_time|'
+ r'current_timestamp|current_user|cursor|database|databases|'
+ r'day_hour|day_microsecond|day_minute|day_second|dec|decimal|'
+ r'declare|default|delayed|delete|desc|describe|deterministic|'
+ r'distinct|distinctrow|div|double|drop|dual|each|else|elseif|'
+ r'enclosed|escaped|exists|exit|explain|fetch|flush|float|float4|'
+ r'float8|for|force|foreign|from|fulltext|grant|group|having|'
+ r'high_priority|hour_microsecond|hour_minute|hour_second|if|'
+ r'ignore|in|index|infile|inner|inout|insensitive|insert|int|'
+ r'int1|int2|int3|int4|int8|integer|interval|into|is|iterate|'
+ r'join|key|keys|kill|leading|leave|left|like|limit|lines|load|'
+ r'localtime|localtimestamp|lock|long|loop|low_priority|match|'
+ r'minute_microsecond|minute_second|mod|modifies|natural|'
+ r'no_write_to_binlog|not|numeric|on|optimize|option|optionally|'
+ r'or|order|out|outer|outfile|precision|primary|procedure|purge|'
+ r'raid0|read|reads|real|references|regexp|release|rename|repeat|'
+ r'replace|require|restrict|return|revoke|right|rlike|schema|'
+ r'schemas|second_microsecond|select|sensitive|separator|set|'
+ r'show|smallint|soname|spatial|specific|sql|sql_big_result|'
+ r'sql_calc_found_rows|sql_small_result|sqlexception|sqlstate|'
+ r'sqlwarning|ssl|starting|straight_join|table|terminated|then|'
+ r'to|trailing|trigger|undo|union|unique|unlock|unsigned|update|'
+ r'usage|use|using|utc_date|utc_time|utc_timestamp|values|'
+ r'varying|when|where|while|with|write|x509|xor|year_month|'
+ r'zerofill)\b', Keyword),
+ # TODO: this list is not complete
+ (r'\b(auto_increment|engine|charset|tables)\b', Keyword.Pseudo),
+ (r'(true|false|null)', Name.Constant),
+ (r'([a-z_]\w*)(\s*)(\()',
+ bygroups(Name.Function, Text, Punctuation)),
+ (r'[a-z_]\w*', Name),
+ (r'@[a-z0-9]*[._]*[a-z0-9]*', Name.Variable),
+ (r'[;:()\[\],.]', Punctuation)
+ ],
+ 'multiline-comments': [
+ (r'/\*', Comment.Multiline, 'multiline-comments'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[^/*]+', Comment.Multiline),
+ (r'[/*]', Comment.Multiline)
+ ]
+ }
+
+
+class SqliteConsoleLexer(Lexer):
+ """
+ Lexer for example sessions using sqlite3.
+
+ .. versionadded:: 0.11
+ """
+
+ name = 'sqlite3con'
+ aliases = ['sqlite3']
+ filenames = ['*.sqlite3-console']
+ mimetypes = ['text/x-sqlite3-console']
+
+ def get_tokens_unprocessed(self, data):
+ sql = SqlLexer(**self.options)
+
+ curcode = ''
+ insertions = []
+ for match in line_re.finditer(data):
+ line = match.group()
+ if line.startswith('sqlite> ') or line.startswith(' ...> '):
+ insertions.append((len(curcode),
+ [(0, Generic.Prompt, line[:8])]))
+ curcode += line[8:]
+ else:
+ if curcode:
+ for item in do_insertions(insertions,
+ sql.get_tokens_unprocessed(curcode)):
+ yield item
+ curcode = ''
+ insertions = []
+ if line.startswith('SQL error: '):
+ yield (match.start(), Generic.Traceback, line)
+ else:
+ yield (match.start(), Generic.Output, line)
+ if curcode:
+ for item in do_insertions(insertions,
+ sql.get_tokens_unprocessed(curcode)):
+ yield item
+
+
+class RqlLexer(RegexLexer):
+ """
+ Lexer for Relation Query Language.
+
+ `RQL <http://www.logilab.org/project/rql>`_
+
+ .. versionadded:: 2.0
+ """
+ name = 'RQL'
+ aliases = ['rql']
+ filenames = ['*.rql']
+ mimetypes = ['text/x-rql']
+
+ flags = re.IGNORECASE
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'(DELETE|SET|INSERT|UNION|DISTINCT|WITH|WHERE|BEING|OR'
+ r'|AND|NOT|GROUPBY|HAVING|ORDERBY|ASC|DESC|LIMIT|OFFSET'
+ r'|TODAY|NOW|TRUE|FALSE|NULL|EXISTS)\b', Keyword),
+ (r'[+*/<>=%-]', Operator),
+ (r'(Any|is|instance_of|CWEType|CWRelation)\b', Name.Builtin),
+ (r'[0-9]+', Number.Integer),
+ (r'[A-Z_]\w*\??', Name),
+ (r"'(''|[^'])*'", String.Single),
+ (r'"(""|[^"])*"', String.Single),
+ (r'[;:()\[\],.]', Punctuation)
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/stata.py b/wandb/vendor/pygments/lexers/stata.py
new file mode 100644
index 0000000000000000000000000000000000000000..a015a23ec297bb53dae75ab102c9ffe0c160f20e
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/stata.py
@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.stata
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for Stata
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Comment, Keyword, Name, Number, \
+ String, Text, Operator
+
+from pygments.lexers._stata_builtins import builtins_base, builtins_functions
+
+__all__ = ['StataLexer']
+
+
+class StataLexer(RegexLexer):
+ """
+ For `Stata <http://www.stata.com/>`_ do files.
+
+ .. versionadded:: 2.2
+ """
+ # Syntax based on
+ # - http://fmwww.bc.edu/RePEc/bocode/s/synlightlist.ado
+ # - http://github.com/isagalaev/highlight.js/blob/master/src/languages/stata.js
+ # - http://github.com/jpitblado/vim-stata/blob/master/syntax/stata.vim
+
+ name = 'Stata'
+ aliases = ['stata', 'do']
+ filenames = ['*.do', '*.ado']
+ mimetypes = ['text/x-stata', 'text/stata', 'application/x-stata']
+
+ tokens = {
+ 'root': [
+ include('comments'),
+ include('vars-strings'),
+ include('numbers'),
+ include('keywords'),
+ (r'.', Text),
+ ],
+ # Global and local macros; regular and special strings
+ 'vars-strings': [
+ (r'\$[\w{]', Name.Variable.Global, 'var_validglobal'),
+ (r'`\w{0,31}\'', Name.Variable),
+ (r'"', String, 'string_dquote'),
+ (r'`"', String, 'string_mquote'),
+ ],
+ # For either string type, highlight macros as macros
+ 'string_dquote': [
+ (r'"', String, '#pop'),
+ (r'\\\\|\\"|\\\n', String.Escape),
+ (r'\$', Name.Variable.Global, 'var_validglobal'),
+ (r'`', Name.Variable, 'var_validlocal'),
+ (r'[^$`"\\]+', String),
+ (r'[$"\\]', String),
+ ],
+ 'string_mquote': [
+ (r'"\'', String, '#pop'),
+ (r'\\\\|\\"|\\\n', String.Escape),
+ (r'\$', Name.Variable.Global, 'var_validglobal'),
+ (r'`', Name.Variable, 'var_validlocal'),
+ (r'[^$`"\\]+', String),
+ (r'[$"\\]', String),
+ ],
+ 'var_validglobal': [
+ (r'\{\w{0,32}\}', Name.Variable.Global, '#pop'),
+ (r'\w{1,32}', Name.Variable.Global, '#pop'),
+ ],
+ 'var_validlocal': [
+ (r'\w{0,31}\'', Name.Variable, '#pop'),
+ ],
+ # * only OK at line start, // OK anywhere
+ 'comments': [
+ (r'^\s*\*.*$', Comment),
+ (r'//.*', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline),
+ (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+ ],
+ # Built in functions and statements
+ 'keywords': [
+ (words(builtins_functions, prefix = r'\b', suffix = r'\('),
+ Name.Function),
+ (words(builtins_base, prefix = r'(^\s*|\s)', suffix = r'\b'),
+ Keyword),
+ ],
+ # http://www.stata.com/help.cgi?operators
+ 'operators': [
+ (r'-|==|<=|>=|<|>|&|!=', Operator),
+ (r'\*|\+|\^|/|!|~|==|~=', Operator)
+ ],
+ # Stata numbers
+ 'numbers': [
+ # decimal number
+ (r'\b[+-]?([0-9]+(\.[0-9]+)?|\.[0-9]+|\.)([eE][+-]?[0-9]+)?[i]?\b',
+ Number),
+ ],
+ # Stata formats
+ 'format': [
+ (r'%-?\d{1,2}(\.\d{1,2})?[gfe]c?', Name.Variable),
+ (r'%(21x|16H|16L|8H|8L)', Name.Variable),
+ (r'%-?(tc|tC|td|tw|tm|tq|th|ty|tg).{0,32}', Name.Variable),
+ (r'%[-~]?\d{1,4}s', Name.Variable),
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/supercollider.py b/wandb/vendor/pygments/lexers/supercollider.py
new file mode 100644
index 0000000000000000000000000000000000000000..40ff0aeb133a9daa53c4be8238b6a39c50bf3b3d
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/supercollider.py
@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.supercollider
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for SuperCollider
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, words, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['SuperColliderLexer']
+
+
+class SuperColliderLexer(RegexLexer):
+ """
+ For `SuperCollider <http://supercollider.github.io/>`_ source code.
+
+ .. versionadded:: 2.1
+ """
+
+ name = 'SuperCollider'
+ aliases = ['sc', 'supercollider']
+ filenames = ['*.sc', '*.scd']
+ mimetypes = ['application/supercollider', 'text/supercollider', ]
+
+ flags = re.DOTALL | re.MULTILINE
+ tokens = {
+ 'commentsandwhitespace': [
+ (r'\s+', Text),
+ (r'<!--', Comment),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline)
+ ],
+ 'slashstartsregex': [
+ include('commentsandwhitespace'),
+ (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+ r'([gim]+\b|\B)', String.Regex, '#pop'),
+ (r'(?=/)', Text, ('#pop', 'badregex')),
+ default('#pop'),
+ ],
+ 'badregex': [
+ (r'\n', Text, '#pop')
+ ],
+ 'root': [
+ (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'),
+ include('commentsandwhitespace'),
+ (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|'
+ r'(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'),
+ (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
+ (r'[})\].]', Punctuation),
+ (words((
+ 'for', 'in', 'while', 'do', 'break', 'return', 'continue',
+ 'switch', 'case', 'default', 'if', 'else', 'throw', 'try',
+ 'catch', 'finally', 'new', 'delete', 'typeof', 'instanceof',
+ 'void'), suffix=r'\b'),
+ Keyword, 'slashstartsregex'),
+ (words(('var', 'let', 'with', 'function', 'arg'), suffix=r'\b'),
+ Keyword.Declaration, 'slashstartsregex'),
+ (words((
+ '(abstract', 'boolean', 'byte', 'char', 'class', 'const',
+ 'debugger', 'double', 'enum', 'export', 'extends', 'final',
+ 'float', 'goto', 'implements', 'import', 'int', 'interface',
+ 'long', 'native', 'package', 'private', 'protected', 'public',
+ 'short', 'static', 'super', 'synchronized', 'throws',
+ 'transient', 'volatile'), suffix=r'\b'),
+ Keyword.Reserved),
+ (words(('true', 'false', 'nil', 'inf'), suffix=r'\b'), Keyword.Constant),
+ (words((
+ 'Array', 'Boolean', 'Date', 'Error', 'Function', 'Number',
+ 'Object', 'Packages', 'RegExp', 'String',
+ 'isFinite', 'isNaN', 'parseFloat', 'parseInt', 'super',
+ 'thisFunctionDef', 'thisFunction', 'thisMethod', 'thisProcess',
+ 'thisThread', 'this'), suffix=r'\b'),
+ Name.Builtin),
+ (r'[$a-zA-Z_]\w*', Name.Other),
+ (r'\\?[$a-zA-Z_]\w*', String.Symbol),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+', Number.Integer),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/tcl.py b/wandb/vendor/pygments/lexers/tcl.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d1be033302adb9c5ba49b68779cf6947f483de6
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/tcl.py
@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.tcl
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Tcl and related languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number
+from pygments.util import shebang_matches
+
+__all__ = ['TclLexer']
+
+
+class TclLexer(RegexLexer):
+ """
+ For Tcl source code.
+
+ .. versionadded:: 0.10
+ """
+
+ keyword_cmds_re = words((
+ 'after', 'apply', 'array', 'break', 'catch', 'continue', 'elseif', 'else', 'error',
+ 'eval', 'expr', 'for', 'foreach', 'global', 'if', 'namespace', 'proc', 'rename', 'return',
+ 'set', 'switch', 'then', 'trace', 'unset', 'update', 'uplevel', 'upvar', 'variable',
+ 'vwait', 'while'), prefix=r'\b', suffix=r'\b')
+
+ builtin_cmds_re = words((
+ 'append', 'bgerror', 'binary', 'cd', 'chan', 'clock', 'close', 'concat', 'dde', 'dict',
+ 'encoding', 'eof', 'exec', 'exit', 'fblocked', 'fconfigure', 'fcopy', 'file',
+ 'fileevent', 'flush', 'format', 'gets', 'glob', 'history', 'http', 'incr', 'info', 'interp',
+ 'join', 'lappend', 'lassign', 'lindex', 'linsert', 'list', 'llength', 'load', 'loadTk',
+ 'lrange', 'lrepeat', 'lreplace', 'lreverse', 'lsearch', 'lset', 'lsort', 'mathfunc',
+ 'mathop', 'memory', 'msgcat', 'open', 'package', 'pid', 'pkg::create', 'pkg_mkIndex',
+ 'platform', 'platform::shell', 'puts', 'pwd', 're_syntax', 'read', 'refchan',
+ 'regexp', 'registry', 'regsub', 'scan', 'seek', 'socket', 'source', 'split', 'string',
+ 'subst', 'tell', 'time', 'tm', 'unknown', 'unload'), prefix=r'\b', suffix=r'\b')
+
+ name = 'Tcl'
+ aliases = ['tcl']
+ filenames = ['*.tcl', '*.rvt']
+ mimetypes = ['text/x-tcl', 'text/x-script.tcl', 'application/x-tcl']
+
+ def _gen_command_rules(keyword_cmds_re, builtin_cmds_re, context=""):
+ return [
+ (keyword_cmds_re, Keyword, 'params' + context),
+ (builtin_cmds_re, Name.Builtin, 'params' + context),
+ (r'([\w.-]+)', Name.Variable, 'params' + context),
+ (r'#', Comment, 'comment'),
+ ]
+
+ tokens = {
+ 'root': [
+ include('command'),
+ include('basic'),
+ include('data'),
+ (r'\}', Keyword), # HACK: somehow we miscounted our braces
+ ],
+ 'command': _gen_command_rules(keyword_cmds_re, builtin_cmds_re),
+ 'command-in-brace': _gen_command_rules(keyword_cmds_re,
+ builtin_cmds_re,
+ "-in-brace"),
+ 'command-in-bracket': _gen_command_rules(keyword_cmds_re,
+ builtin_cmds_re,
+ "-in-bracket"),
+ 'command-in-paren': _gen_command_rules(keyword_cmds_re,
+ builtin_cmds_re,
+ "-in-paren"),
+ 'basic': [
+ (r'\(', Keyword, 'paren'),
+ (r'\[', Keyword, 'bracket'),
+ (r'\{', Keyword, 'brace'),
+ (r'"', String.Double, 'string'),
+ (r'(eq|ne|in|ni)\b', Operator.Word),
+ (r'!=|==|<<|>>|<=|>=|&&|\|\||\*\*|[-+~!*/%<>&^|?:]', Operator),
+ ],
+ 'data': [
+ (r'\s+', Text),
+ (r'0x[a-fA-F0-9]+', Number.Hex),
+ (r'0[0-7]+', Number.Oct),
+ (r'\d+\.\d+', Number.Float),
+ (r'\d+', Number.Integer),
+ (r'\$([\w.:-]+)', Name.Variable),
+ (r'([\w.:-]+)', Text),
+ ],
+ 'params': [
+ (r';', Keyword, '#pop'),
+ (r'\n', Text, '#pop'),
+ (r'(else|elseif|then)\b', Keyword),
+ include('basic'),
+ include('data'),
+ ],
+ 'params-in-brace': [
+ (r'\}', Keyword, ('#pop', '#pop')),
+ include('params')
+ ],
+ 'params-in-paren': [
+ (r'\)', Keyword, ('#pop', '#pop')),
+ include('params')
+ ],
+ 'params-in-bracket': [
+ (r'\]', Keyword, ('#pop', '#pop')),
+ include('params')
+ ],
+ 'string': [
+ (r'\[', String.Double, 'string-square'),
+ (r'(?s)(\\\\|\\[0-7]+|\\.|[^"\\])', String.Double),
+ (r'"', String.Double, '#pop')
+ ],
+ 'string-square': [
+ (r'\[', String.Double, 'string-square'),
+ (r'(?s)(\\\\|\\[0-7]+|\\.|\\\n|[^\]\\])', String.Double),
+ (r'\]', String.Double, '#pop')
+ ],
+ 'brace': [
+ (r'\}', Keyword, '#pop'),
+ include('command-in-brace'),
+ include('basic'),
+ include('data'),
+ ],
+ 'paren': [
+ (r'\)', Keyword, '#pop'),
+ include('command-in-paren'),
+ include('basic'),
+ include('data'),
+ ],
+ 'bracket': [
+ (r'\]', Keyword, '#pop'),
+ include('command-in-bracket'),
+ include('basic'),
+ include('data'),
+ ],
+ 'comment': [
+ (r'.*[^\\]\n', Comment, '#pop'),
+ (r'.*\\\n', Comment),
+ ],
+ }
+
+ def analyse_text(text):
+ return shebang_matches(text, r'(tcl)')
diff --git a/wandb/vendor/pygments/lexers/templates.py b/wandb/vendor/pygments/lexers/templates.py
new file mode 100644
index 0000000000000000000000000000000000000000..83c57db80b4d349833ebe9e1c66c0d70dfe65870
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/templates.py
@@ -0,0 +1,2283 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.templates
+ ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for various template engines' markup.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexers.html import HtmlLexer, XmlLexer
+from pygments.lexers.javascript import JavascriptLexer, LassoLexer
+from pygments.lexers.css import CssLexer
+from pygments.lexers.php import PhpLexer
+from pygments.lexers.python import PythonLexer
+from pygments.lexers.perl import PerlLexer
+from pygments.lexers.jvm import JavaLexer, TeaLangLexer
+from pygments.lexers.data import YamlLexer
+from pygments.lexer import Lexer, DelegatingLexer, RegexLexer, bygroups, \
+ include, using, this, default, combined
+from pygments.token import Error, Punctuation, Whitespace, \
+ Text, Comment, Operator, Keyword, Name, String, Number, Other, Token
+from pygments.util import html_doctype_matches, looks_like_xml
+
+__all__ = ['HtmlPhpLexer', 'XmlPhpLexer', 'CssPhpLexer',
+ 'JavascriptPhpLexer', 'ErbLexer', 'RhtmlLexer',
+ 'XmlErbLexer', 'CssErbLexer', 'JavascriptErbLexer',
+ 'SmartyLexer', 'HtmlSmartyLexer', 'XmlSmartyLexer',
+ 'CssSmartyLexer', 'JavascriptSmartyLexer', 'DjangoLexer',
+ 'HtmlDjangoLexer', 'CssDjangoLexer', 'XmlDjangoLexer',
+ 'JavascriptDjangoLexer', 'GenshiLexer', 'HtmlGenshiLexer',
+ 'GenshiTextLexer', 'CssGenshiLexer', 'JavascriptGenshiLexer',
+ 'MyghtyLexer', 'MyghtyHtmlLexer', 'MyghtyXmlLexer',
+ 'MyghtyCssLexer', 'MyghtyJavascriptLexer', 'MasonLexer', 'MakoLexer',
+ 'MakoHtmlLexer', 'MakoXmlLexer', 'MakoJavascriptLexer',
+ 'MakoCssLexer', 'JspLexer', 'CheetahLexer', 'CheetahHtmlLexer',
+ 'CheetahXmlLexer', 'CheetahJavascriptLexer', 'EvoqueLexer',
+ 'EvoqueHtmlLexer', 'EvoqueXmlLexer', 'ColdfusionLexer',
+ 'ColdfusionHtmlLexer', 'ColdfusionCFCLexer', 'VelocityLexer',
+ 'VelocityHtmlLexer', 'VelocityXmlLexer', 'SspLexer',
+ 'TeaTemplateLexer', 'LassoHtmlLexer', 'LassoXmlLexer',
+ 'LassoCssLexer', 'LassoJavascriptLexer', 'HandlebarsLexer',
+ 'HandlebarsHtmlLexer', 'YamlJinjaLexer', 'LiquidLexer',
+ 'TwigLexer', 'TwigHtmlLexer', 'Angular2Lexer', 'Angular2HtmlLexer']
+
+
+class ErbLexer(Lexer):
+ """
+ Generic `ERB <http://ruby-doc.org/core/classes/ERB.html>`_ (Ruby Templating)
+ lexer.
+
+ Just highlights ruby code between the preprocessor directives, other data
+ is left untouched by the lexer.
+
+ All options are also forwarded to the `RubyLexer`.
+ """
+
+ name = 'ERB'
+ aliases = ['erb']
+ mimetypes = ['application/x-ruby-templating']
+
+ _block_re = re.compile(r'(<%%|%%>|<%=|<%#|<%-|<%|-%>|%>|^%[^%].*?$)', re.M)
+
+ def __init__(self, **options):
+ from pygments.lexers.ruby import RubyLexer
+ self.ruby_lexer = RubyLexer(**options)
+ Lexer.__init__(self, **options)
+
+ def get_tokens_unprocessed(self, text):
+ """
+ Since ERB doesn't allow "<%" and other tags inside of ruby
+ blocks we have to use a split approach here that fails for
+ that too.
+ """
+ tokens = self._block_re.split(text)
+ tokens.reverse()
+ state = idx = 0
+ try:
+ while True:
+ # text
+ if state == 0:
+ val = tokens.pop()
+ yield idx, Other, val
+ idx += len(val)
+ state = 1
+ # block starts
+ elif state == 1:
+ tag = tokens.pop()
+ # literals
+ if tag in ('<%%', '%%>'):
+ yield idx, Other, tag
+ idx += 3
+ state = 0
+ # comment
+ elif tag == '<%#':
+ yield idx, Comment.Preproc, tag
+ val = tokens.pop()
+ yield idx + 3, Comment, val
+ idx += 3 + len(val)
+ state = 2
+ # blocks or output
+ elif tag in ('<%', '<%=', '<%-'):
+ yield idx, Comment.Preproc, tag
+ idx += len(tag)
+ data = tokens.pop()
+ r_idx = 0
+ for r_idx, r_token, r_value in \
+ self.ruby_lexer.get_tokens_unprocessed(data):
+ yield r_idx + idx, r_token, r_value
+ idx += len(data)
+ state = 2
+ elif tag in ('%>', '-%>'):
+ yield idx, Error, tag
+ idx += len(tag)
+ state = 0
+ # % raw ruby statements
+ else:
+ yield idx, Comment.Preproc, tag[0]
+ r_idx = 0
+ for r_idx, r_token, r_value in \
+ self.ruby_lexer.get_tokens_unprocessed(tag[1:]):
+ yield idx + 1 + r_idx, r_token, r_value
+ idx += len(tag)
+ state = 0
+ # block ends
+ elif state == 2:
+ tag = tokens.pop()
+ if tag not in ('%>', '-%>'):
+ yield idx, Other, tag
+ else:
+ yield idx, Comment.Preproc, tag
+ idx += len(tag)
+ state = 0
+ except IndexError:
+ return
+
+ def analyse_text(text):
+ if '<%' in text and '%>' in text:
+ return 0.4
+
+
+class SmartyLexer(RegexLexer):
+ """
+ Generic `Smarty <http://smarty.php.net/>`_ template lexer.
+
+ Just highlights smarty code between the preprocessor directives, other
+ data is left untouched by the lexer.
+ """
+
+ name = 'Smarty'
+ aliases = ['smarty']
+ filenames = ['*.tpl']
+ mimetypes = ['application/x-smarty']
+
+ flags = re.MULTILINE | re.DOTALL
+
+ tokens = {
+ 'root': [
+ (r'[^{]+', Other),
+ (r'(\{)(\*.*?\*)(\})',
+ bygroups(Comment.Preproc, Comment, Comment.Preproc)),
+ (r'(\{php\})(.*?)(\{/php\})',
+ bygroups(Comment.Preproc, using(PhpLexer, startinline=True),
+ Comment.Preproc)),
+ (r'(\{)(/?[a-zA-Z_]\w*)(\s*)',
+ bygroups(Comment.Preproc, Name.Function, Text), 'smarty'),
+ (r'\{', Comment.Preproc, 'smarty')
+ ],
+ 'smarty': [
+ (r'\s+', Text),
+ (r'\{', Comment.Preproc, '#push'),
+ (r'\}', Comment.Preproc, '#pop'),
+ (r'#[a-zA-Z_]\w*#', Name.Variable),
+ (r'\$[a-zA-Z_]\w*(\.\w+)*', Name.Variable),
+ (r'[~!%^&*()+=|\[\]:;,.<>/?@-]', Operator),
+ (r'(true|false|null)\b', Keyword.Constant),
+ (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|"
+ r"0[xX][0-9a-fA-F]+[Ll]?", Number),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ (r'[a-zA-Z_]\w*', Name.Attribute)
+ ]
+ }
+
+ def analyse_text(text):
+ rv = 0.0
+ if re.search('\{if\s+.*?\}.*?\{/if\}', text):
+ rv += 0.15
+ if re.search('\{include\s+file=.*?\}', text):
+ rv += 0.15
+ if re.search('\{foreach\s+.*?\}.*?\{/foreach\}', text):
+ rv += 0.15
+ if re.search('\{\$.*?\}', text):
+ rv += 0.01
+ return rv
+
+
+class VelocityLexer(RegexLexer):
+ """
+ Generic `Velocity <http://velocity.apache.org/>`_ template lexer.
+
+ Just highlights velocity directives and variable references, other
+ data is left untouched by the lexer.
+ """
+
+ name = 'Velocity'
+ aliases = ['velocity']
+ filenames = ['*.vm', '*.fhtml']
+
+ flags = re.MULTILINE | re.DOTALL
+
+ identifier = r'[a-zA-Z_]\w*'
+
+ tokens = {
+ 'root': [
+ (r'[^{#$]+', Other),
+ (r'(#)(\*.*?\*)(#)',
+ bygroups(Comment.Preproc, Comment, Comment.Preproc)),
+ (r'(##)(.*?$)',
+ bygroups(Comment.Preproc, Comment)),
+ (r'(#\{?)(' + identifier + r')(\}?)(\s?\()',
+ bygroups(Comment.Preproc, Name.Function, Comment.Preproc, Punctuation),
+ 'directiveparams'),
+ (r'(#\{?)(' + identifier + r')(\}|\b)',
+ bygroups(Comment.Preproc, Name.Function, Comment.Preproc)),
+ (r'\$\{?', Punctuation, 'variable')
+ ],
+ 'variable': [
+ (identifier, Name.Variable),
+ (r'\(', Punctuation, 'funcparams'),
+ (r'(\.)(' + identifier + r')',
+ bygroups(Punctuation, Name.Variable), '#push'),
+ (r'\}', Punctuation, '#pop'),
+ default('#pop')
+ ],
+ 'directiveparams': [
+ (r'(&&|\|\||==?|!=?|[-<>+*%&|^/])|\b(eq|ne|gt|lt|ge|le|not|in)\b',
+ Operator),
+ (r'\[', Operator, 'rangeoperator'),
+ (r'\b' + identifier + r'\b', Name.Function),
+ include('funcparams')
+ ],
+ 'rangeoperator': [
+ (r'\.\.', Operator),
+ include('funcparams'),
+ (r'\]', Operator, '#pop')
+ ],
+ 'funcparams': [
+ (r'\$\{?', Punctuation, 'variable'),
+ (r'\s+', Text),
+ (r'[,:]', Punctuation),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ (r"0[xX][0-9a-fA-F]+[Ll]?", Number),
+ (r"\b[0-9]+\b", Number),
+ (r'(true|false|null)\b', Keyword.Constant),
+ (r'\(', Punctuation, '#push'),
+ (r'\)', Punctuation, '#pop'),
+ (r'\{', Punctuation, '#push'),
+ (r'\}', Punctuation, '#pop'),
+ (r'\[', Punctuation, '#push'),
+ (r'\]', Punctuation, '#pop'),
+ ]
+ }
+
+ def analyse_text(text):
+ rv = 0.0
+ if re.search(r'#\{?macro\}?\(.*?\).*?#\{?end\}?', text):
+ rv += 0.25
+ if re.search(r'#\{?if\}?\(.+?\).*?#\{?end\}?', text):
+ rv += 0.15
+ if re.search(r'#\{?foreach\}?\(.+?\).*?#\{?end\}?', text):
+ rv += 0.15
+ if re.search(r'\$\{?[a-zA-Z_]\w*(\([^)]*\))?'
+ r'(\.\w+(\([^)]*\))?)*\}?', text):
+ rv += 0.01
+ return rv
+
+
+class VelocityHtmlLexer(DelegatingLexer):
+ """
+ Subclass of the `VelocityLexer` that highlights unlexed data
+ with the `HtmlLexer`.
+
+ """
+
+ name = 'HTML+Velocity'
+ aliases = ['html+velocity']
+ alias_filenames = ['*.html', '*.fhtml']
+ mimetypes = ['text/html+velocity']
+
+ def __init__(self, **options):
+ super(VelocityHtmlLexer, self).__init__(HtmlLexer, VelocityLexer,
+ **options)
+
+
+class VelocityXmlLexer(DelegatingLexer):
+ """
+ Subclass of the `VelocityLexer` that highlights unlexed data
+ with the `XmlLexer`.
+
+ """
+
+ name = 'XML+Velocity'
+ aliases = ['xml+velocity']
+ alias_filenames = ['*.xml', '*.vm']
+ mimetypes = ['application/xml+velocity']
+
+ def __init__(self, **options):
+ super(VelocityXmlLexer, self).__init__(XmlLexer, VelocityLexer,
+ **options)
+
+ def analyse_text(text):
+ rv = VelocityLexer.analyse_text(text) - 0.01
+ if looks_like_xml(text):
+ rv += 0.4
+ return rv
+
+
+class DjangoLexer(RegexLexer):
+ """
+ Generic `django <http://www.djangoproject.com/documentation/templates/>`_
+ and `jinja <http://wsgiarea.pocoo.org/jinja/>`_ template lexer.
+
+ It just highlights django/jinja code between the preprocessor directives,
+ other data is left untouched by the lexer.
+ """
+
+ name = 'Django/Jinja'
+ aliases = ['django', 'jinja']
+ mimetypes = ['application/x-django-templating', 'application/x-jinja']
+
+ flags = re.M | re.S
+
+ tokens = {
+ 'root': [
+ (r'[^{]+', Other),
+ (r'\{\{', Comment.Preproc, 'var'),
+ # jinja/django comments
+ (r'\{[*#].*?[*#]\}', Comment),
+ # django comments
+ (r'(\{%)(-?\s*)(comment)(\s*-?)(%\})(.*?)'
+ r'(\{%)(-?\s*)(endcomment)(\s*-?)(%\})',
+ bygroups(Comment.Preproc, Text, Keyword, Text, Comment.Preproc,
+ Comment, Comment.Preproc, Text, Keyword, Text,
+ Comment.Preproc)),
+ # raw jinja blocks
+ (r'(\{%)(-?\s*)(raw)(\s*-?)(%\})(.*?)'
+ r'(\{%)(-?\s*)(endraw)(\s*-?)(%\})',
+ bygroups(Comment.Preproc, Text, Keyword, Text, Comment.Preproc,
+ Text, Comment.Preproc, Text, Keyword, Text,
+ Comment.Preproc)),
+ # filter blocks
+ (r'(\{%)(-?\s*)(filter)(\s+)([a-zA-Z_]\w*)',
+ bygroups(Comment.Preproc, Text, Keyword, Text, Name.Function),
+ 'block'),
+ (r'(\{%)(-?\s*)([a-zA-Z_]\w*)',
+ bygroups(Comment.Preproc, Text, Keyword), 'block'),
+ (r'\{', Other)
+ ],
+ 'varnames': [
+ (r'(\|)(\s*)([a-zA-Z_]\w*)',
+ bygroups(Operator, Text, Name.Function)),
+ (r'(is)(\s+)(not)?(\s+)?([a-zA-Z_]\w*)',
+ bygroups(Keyword, Text, Keyword, Text, Name.Function)),
+ (r'(_|true|false|none|True|False|None)\b', Keyword.Pseudo),
+ (r'(in|as|reversed|recursive|not|and|or|is|if|else|import|'
+ r'with(?:(?:out)?\s*context)?|scoped|ignore\s+missing)\b',
+ Keyword),
+ (r'(loop|block|super|forloop)\b', Name.Builtin),
+ (r'[a-zA-Z_][\w-]*', Name.Variable),
+ (r'\.\w+', Name.Variable),
+ (r':?"(\\\\|\\"|[^"])*"', String.Double),
+ (r":?'(\\\\|\\'|[^'])*'", String.Single),
+ (r'([{}()\[\]+\-*/,:~]|[><=]=?)', Operator),
+ (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|"
+ r"0[xX][0-9a-fA-F]+[Ll]?", Number),
+ ],
+ 'var': [
+ (r'\s+', Text),
+ (r'(-?)(\}\})', bygroups(Text, Comment.Preproc), '#pop'),
+ include('varnames')
+ ],
+ 'block': [
+ (r'\s+', Text),
+ (r'(-?)(%\})', bygroups(Text, Comment.Preproc), '#pop'),
+ include('varnames'),
+ (r'.', Punctuation)
+ ]
+ }
+
+ def analyse_text(text):
+ rv = 0.0
+ if re.search(r'\{%\s*(block|extends)', text) is not None:
+ rv += 0.4
+ if re.search(r'\{%\s*if\s*.*?%\}', text) is not None:
+ rv += 0.1
+ if re.search(r'\{\{.*?\}\}', text) is not None:
+ rv += 0.1
+ return rv
+
+
+class MyghtyLexer(RegexLexer):
+ """
+ Generic `myghty templates`_ lexer. Code that isn't Myghty
+ markup is yielded as `Token.Other`.
+
+ .. versionadded:: 0.6
+
+ .. _myghty templates: http://www.myghty.org/
+ """
+
+ name = 'Myghty'
+ aliases = ['myghty']
+ filenames = ['*.myt', 'autodelegate']
+ mimetypes = ['application/x-myghty']
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'(<%(?:def|method))(\s*)(.*?)(>)(.*?)(</%\2\s*>)(?s)',
+ bygroups(Name.Tag, Text, Name.Function, Name.Tag,
+ using(this), Name.Tag)),
+ (r'(<%\w+)(.*?)(>)(.*?)(</%\2\s*>)(?s)',
+ bygroups(Name.Tag, Name.Function, Name.Tag,
+ using(PythonLexer), Name.Tag)),
+ (r'(<&[^|])(.*?)(,.*?)?(&>)',
+ bygroups(Name.Tag, Name.Function, using(PythonLexer), Name.Tag)),
+ (r'(<&\|)(.*?)(,.*?)?(&>)(?s)',
+ bygroups(Name.Tag, Name.Function, using(PythonLexer), Name.Tag)),
+ (r'</&>', Name.Tag),
+ (r'(<%!?)(.*?)(%>)(?s)',
+ bygroups(Name.Tag, using(PythonLexer), Name.Tag)),
+ (r'(?<=^)#[^\n]*(\n|\Z)', Comment),
+ (r'(?<=^)(%)([^\n]*)(\n|\Z)',
+ bygroups(Name.Tag, using(PythonLexer), Other)),
+ (r"""(?sx)
+ (.+?) # anything, followed by:
+ (?:
+ (?<=\n)(?=[%#]) | # an eval or comment line
+ (?=</?[%&]) | # a substitution or block or
+ # call start or end
+ # - don't consume
+ (\\\n) | # an escaped newline
+ \Z # end of string
+ )""", bygroups(Other, Operator)),
+ ]
+ }
+
+
+class MyghtyHtmlLexer(DelegatingLexer):
+ """
+ Subclass of the `MyghtyLexer` that highlights unlexed data
+ with the `HtmlLexer`.
+
+ .. versionadded:: 0.6
+ """
+
+ name = 'HTML+Myghty'
+ aliases = ['html+myghty']
+ mimetypes = ['text/html+myghty']
+
+ def __init__(self, **options):
+ super(MyghtyHtmlLexer, self).__init__(HtmlLexer, MyghtyLexer,
+ **options)
+
+
+class MyghtyXmlLexer(DelegatingLexer):
+ """
+ Subclass of the `MyghtyLexer` that highlights unlexed data
+ with the `XmlLexer`.
+
+ .. versionadded:: 0.6
+ """
+
+ name = 'XML+Myghty'
+ aliases = ['xml+myghty']
+ mimetypes = ['application/xml+myghty']
+
+ def __init__(self, **options):
+ super(MyghtyXmlLexer, self).__init__(XmlLexer, MyghtyLexer,
+ **options)
+
+
+class MyghtyJavascriptLexer(DelegatingLexer):
+ """
+ Subclass of the `MyghtyLexer` that highlights unlexed data
+ with the `JavascriptLexer`.
+
+ .. versionadded:: 0.6
+ """
+
+ name = 'JavaScript+Myghty'
+ aliases = ['js+myghty', 'javascript+myghty']
+ mimetypes = ['application/x-javascript+myghty',
+ 'text/x-javascript+myghty',
+ 'text/javascript+mygthy']
+
+ def __init__(self, **options):
+ super(MyghtyJavascriptLexer, self).__init__(JavascriptLexer,
+ MyghtyLexer, **options)
+
+
+class MyghtyCssLexer(DelegatingLexer):
+ """
+ Subclass of the `MyghtyLexer` that highlights unlexed data
+ with the `CssLexer`.
+
+ .. versionadded:: 0.6
+ """
+
+ name = 'CSS+Myghty'
+ aliases = ['css+myghty']
+ mimetypes = ['text/css+myghty']
+
+ def __init__(self, **options):
+ super(MyghtyCssLexer, self).__init__(CssLexer, MyghtyLexer,
+ **options)
+
+
+class MasonLexer(RegexLexer):
+ """
+ Generic `mason templates`_ lexer. Stolen from Myghty lexer. Code that isn't
+ Mason markup is HTML.
+
+ .. _mason templates: http://www.masonhq.com/
+
+ .. versionadded:: 1.4
+ """
+ name = 'Mason'
+ aliases = ['mason']
+ filenames = ['*.m', '*.mhtml', '*.mc', '*.mi', 'autohandler', 'dhandler']
+ mimetypes = ['application/x-mason']
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'(<%doc>)(.*?)(</%doc>)(?s)',
+ bygroups(Name.Tag, Comment.Multiline, Name.Tag)),
+ (r'(<%(?:def|method))(\s*)(.*?)(>)(.*?)(</%\2\s*>)(?s)',
+ bygroups(Name.Tag, Text, Name.Function, Name.Tag,
+ using(this), Name.Tag)),
+ (r'(<%\w+)(.*?)(>)(.*?)(</%\2\s*>)(?s)',
+ bygroups(Name.Tag, Name.Function, Name.Tag,
+ using(PerlLexer), Name.Tag)),
+ (r'(<&[^|])(.*?)(,.*?)?(&>)(?s)',
+ bygroups(Name.Tag, Name.Function, using(PerlLexer), Name.Tag)),
+ (r'(<&\|)(.*?)(,.*?)?(&>)(?s)',
+ bygroups(Name.Tag, Name.Function, using(PerlLexer), Name.Tag)),
+ (r'</&>', Name.Tag),
+ (r'(<%!?)(.*?)(%>)(?s)',
+ bygroups(Name.Tag, using(PerlLexer), Name.Tag)),
+ (r'(?<=^)#[^\n]*(\n|\Z)', Comment),
+ (r'(?<=^)(%)([^\n]*)(\n|\Z)',
+ bygroups(Name.Tag, using(PerlLexer), Other)),
+ (r"""(?sx)
+ (.+?) # anything, followed by:
+ (?:
+ (?<=\n)(?=[%#]) | # an eval or comment line
+ (?=</?[%&]) | # a substitution or block or
+ # call start or end
+ # - don't consume
+ (\\\n) | # an escaped newline
+ \Z # end of string
+ )""", bygroups(using(HtmlLexer), Operator)),
+ ]
+ }
+
+ def analyse_text(text):
+ result = 0.0
+ if re.search(r'</%(class|doc|init)%>', text) is not None:
+ result = 1.0
+ elif re.search(r'<&.+&>', text, re.DOTALL) is not None:
+ result = 0.11
+ return result
+
+
+class MakoLexer(RegexLexer):
+ """
+ Generic `mako templates`_ lexer. Code that isn't Mako
+ markup is yielded as `Token.Other`.
+
+ .. versionadded:: 0.7
+
+ .. _mako templates: http://www.makotemplates.org/
+ """
+
+ name = 'Mako'
+ aliases = ['mako']
+ filenames = ['*.mao']
+ mimetypes = ['application/x-mako']
+
+ tokens = {
+ 'root': [
+ (r'(\s*)(%)(\s*end(?:\w+))(\n|\Z)',
+ bygroups(Text, Comment.Preproc, Keyword, Other)),
+ (r'(\s*)(%)([^\n]*)(\n|\Z)',
+ bygroups(Text, Comment.Preproc, using(PythonLexer), Other)),
+ (r'(\s*)(##[^\n]*)(\n|\Z)',
+ bygroups(Text, Comment.Preproc, Other)),
+ (r'(?s)<%doc>.*?</%doc>', Comment.Preproc),
+ (r'(<%)([\w.:]+)',
+ bygroups(Comment.Preproc, Name.Builtin), 'tag'),
+ (r'(</%)([\w.:]+)(>)',
+ bygroups(Comment.Preproc, Name.Builtin, Comment.Preproc)),
+ (r'<%(?=([\w.:]+))', Comment.Preproc, 'ondeftags'),
+ (r'(<%(?:!?))(.*?)(%>)(?s)',
+ bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)),
+ (r'(\$\{)(.*?)(\})',
+ bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)),
+ (r'''(?sx)
+ (.+?) # anything, followed by:
+ (?:
+ (?<=\n)(?=%|\#\#) | # an eval or comment line
+ (?=\#\*) | # multiline comment
+ (?=</?%) | # a python block
+ # call start or end
+ (?=\$\{) | # a substitution
+ (?<=\n)(?=\s*%) |
+ # - don't consume
+ (\\\n) | # an escaped newline
+ \Z # end of string
+ )
+ ''', bygroups(Other, Operator)),
+ (r'\s+', Text),
+ ],
+ 'ondeftags': [
+ (r'<%', Comment.Preproc),
+ (r'(?<=<%)(include|inherit|namespace|page)', Name.Builtin),
+ include('tag'),
+ ],
+ 'tag': [
+ (r'((?:\w+)\s*=)(\s*)(".*?")',
+ bygroups(Name.Attribute, Text, String)),
+ (r'/?\s*>', Comment.Preproc, '#pop'),
+ (r'\s+', Text),
+ ],
+ 'attr': [
+ ('".*?"', String, '#pop'),
+ ("'.*?'", String, '#pop'),
+ (r'[^\s>]+', String, '#pop'),
+ ],
+ }
+
+
+class MakoHtmlLexer(DelegatingLexer):
+ """
+ Subclass of the `MakoLexer` that highlights unlexed data
+ with the `HtmlLexer`.
+
+ .. versionadded:: 0.7
+ """
+
+ name = 'HTML+Mako'
+ aliases = ['html+mako']
+ mimetypes = ['text/html+mako']
+
+ def __init__(self, **options):
+ super(MakoHtmlLexer, self).__init__(HtmlLexer, MakoLexer,
+ **options)
+
+
+class MakoXmlLexer(DelegatingLexer):
+ """
+ Subclass of the `MakoLexer` that highlights unlexed data
+ with the `XmlLexer`.
+
+ .. versionadded:: 0.7
+ """
+
+ name = 'XML+Mako'
+ aliases = ['xml+mako']
+ mimetypes = ['application/xml+mako']
+
+ def __init__(self, **options):
+ super(MakoXmlLexer, self).__init__(XmlLexer, MakoLexer,
+ **options)
+
+
+class MakoJavascriptLexer(DelegatingLexer):
+ """
+ Subclass of the `MakoLexer` that highlights unlexed data
+ with the `JavascriptLexer`.
+
+ .. versionadded:: 0.7
+ """
+
+ name = 'JavaScript+Mako'
+ aliases = ['js+mako', 'javascript+mako']
+ mimetypes = ['application/x-javascript+mako',
+ 'text/x-javascript+mako',
+ 'text/javascript+mako']
+
+ def __init__(self, **options):
+ super(MakoJavascriptLexer, self).__init__(JavascriptLexer,
+ MakoLexer, **options)
+
+
+class MakoCssLexer(DelegatingLexer):
+ """
+ Subclass of the `MakoLexer` that highlights unlexed data
+ with the `CssLexer`.
+
+ .. versionadded:: 0.7
+ """
+
+ name = 'CSS+Mako'
+ aliases = ['css+mako']
+ mimetypes = ['text/css+mako']
+
+ def __init__(self, **options):
+ super(MakoCssLexer, self).__init__(CssLexer, MakoLexer,
+ **options)
+
+
+# Genshi and Cheetah lexers courtesy of Matt Good.
+
+class CheetahPythonLexer(Lexer):
+ """
+ Lexer for handling Cheetah's special $ tokens in Python syntax.
+ """
+
+ def get_tokens_unprocessed(self, text):
+ pylexer = PythonLexer(**self.options)
+ for pos, type_, value in pylexer.get_tokens_unprocessed(text):
+ if type_ == Token.Error and value == '$':
+ type_ = Comment.Preproc
+ yield pos, type_, value
+
+
+class CheetahLexer(RegexLexer):
+ """
+ Generic `cheetah templates`_ lexer. Code that isn't Cheetah
+ markup is yielded as `Token.Other`. This also works for
+ `spitfire templates`_ which use the same syntax.
+
+ .. _cheetah templates: http://www.cheetahtemplate.org/
+ .. _spitfire templates: http://code.google.com/p/spitfire/
+ """
+
+ name = 'Cheetah'
+ aliases = ['cheetah', 'spitfire']
+ filenames = ['*.tmpl', '*.spt']
+ mimetypes = ['application/x-cheetah', 'application/x-spitfire']
+
+ tokens = {
+ 'root': [
+ (r'(##[^\n]*)$',
+ (bygroups(Comment))),
+ (r'#[*](.|\n)*?[*]#', Comment),
+ (r'#end[^#\n]*(?:#|$)', Comment.Preproc),
+ (r'#slurp$', Comment.Preproc),
+ (r'(#[a-zA-Z]+)([^#\n]*)(#|$)',
+ (bygroups(Comment.Preproc, using(CheetahPythonLexer),
+ Comment.Preproc))),
+ # TODO support other Python syntax like $foo['bar']
+ (r'(\$)([a-zA-Z_][\w.]*\w)',
+ bygroups(Comment.Preproc, using(CheetahPythonLexer))),
+ (r'(\$\{!?)(.*?)(\})(?s)',
+ bygroups(Comment.Preproc, using(CheetahPythonLexer),
+ Comment.Preproc)),
+ (r'''(?sx)
+ (.+?) # anything, followed by:
+ (?:
+ (?=\#[#a-zA-Z]*) | # an eval comment
+ (?=\$[a-zA-Z_{]) | # a substitution
+ \Z # end of string
+ )
+ ''', Other),
+ (r'\s+', Text),
+ ],
+ }
+
+
+class CheetahHtmlLexer(DelegatingLexer):
+ """
+ Subclass of the `CheetahLexer` that highlights unlexed data
+ with the `HtmlLexer`.
+ """
+
+ name = 'HTML+Cheetah'
+ aliases = ['html+cheetah', 'html+spitfire', 'htmlcheetah']
+ mimetypes = ['text/html+cheetah', 'text/html+spitfire']
+
+ def __init__(self, **options):
+ super(CheetahHtmlLexer, self).__init__(HtmlLexer, CheetahLexer,
+ **options)
+
+
+class CheetahXmlLexer(DelegatingLexer):
+ """
+ Subclass of the `CheetahLexer` that highlights unlexed data
+ with the `XmlLexer`.
+ """
+
+ name = 'XML+Cheetah'
+ aliases = ['xml+cheetah', 'xml+spitfire']
+ mimetypes = ['application/xml+cheetah', 'application/xml+spitfire']
+
+ def __init__(self, **options):
+ super(CheetahXmlLexer, self).__init__(XmlLexer, CheetahLexer,
+ **options)
+
+
+class CheetahJavascriptLexer(DelegatingLexer):
+ """
+ Subclass of the `CheetahLexer` that highlights unlexed data
+ with the `JavascriptLexer`.
+ """
+
+ name = 'JavaScript+Cheetah'
+ aliases = ['js+cheetah', 'javascript+cheetah',
+ 'js+spitfire', 'javascript+spitfire']
+ mimetypes = ['application/x-javascript+cheetah',
+ 'text/x-javascript+cheetah',
+ 'text/javascript+cheetah',
+ 'application/x-javascript+spitfire',
+ 'text/x-javascript+spitfire',
+ 'text/javascript+spitfire']
+
+ def __init__(self, **options):
+ super(CheetahJavascriptLexer, self).__init__(JavascriptLexer,
+ CheetahLexer, **options)
+
+
+class GenshiTextLexer(RegexLexer):
+ """
+ A lexer that highlights `genshi <http://genshi.edgewall.org/>`_ text
+ templates.
+ """
+
+ name = 'Genshi Text'
+ aliases = ['genshitext']
+ mimetypes = ['application/x-genshi-text', 'text/x-genshi']
+
+ tokens = {
+ 'root': [
+ (r'[^#$\s]+', Other),
+ (r'^(\s*)(##.*)$', bygroups(Text, Comment)),
+ (r'^(\s*)(#)', bygroups(Text, Comment.Preproc), 'directive'),
+ include('variable'),
+ (r'[#$\s]', Other),
+ ],
+ 'directive': [
+ (r'\n', Text, '#pop'),
+ (r'(?:def|for|if)\s+.*', using(PythonLexer), '#pop'),
+ (r'(choose|when|with)([^\S\n]+)(.*)',
+ bygroups(Keyword, Text, using(PythonLexer)), '#pop'),
+ (r'(choose|otherwise)\b', Keyword, '#pop'),
+ (r'(end\w*)([^\S\n]*)(.*)', bygroups(Keyword, Text, Comment), '#pop'),
+ ],
+ 'variable': [
+ (r'(?<!\$)(\$\{)(.+?)(\})',
+ bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)),
+ (r'(?<!\$)(\$)([a-zA-Z_][\w.]*)',
+ Name.Variable),
+ ]
+ }
+
+
+class GenshiMarkupLexer(RegexLexer):
+ """
+ Base lexer for Genshi markup, used by `HtmlGenshiLexer` and
+ `GenshiLexer`.
+ """
+
+ flags = re.DOTALL
+
+ tokens = {
+ 'root': [
+ (r'[^<$]+', Other),
+ (r'(<\?python)(.*?)(\?>)',
+ bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)),
+ # yield style and script blocks as Other
+ (r'<\s*(script|style)\s*.*?>.*?<\s*/\1\s*>', Other),
+ (r'<\s*py:[a-zA-Z0-9]+', Name.Tag, 'pytag'),
+ (r'<\s*[a-zA-Z0-9:.]+', Name.Tag, 'tag'),
+ include('variable'),
+ (r'[<$]', Other),
+ ],
+ 'pytag': [
+ (r'\s+', Text),
+ (r'[\w:-]+\s*=', Name.Attribute, 'pyattr'),
+ (r'/?\s*>', Name.Tag, '#pop'),
+ ],
+ 'pyattr': [
+ ('(")(.*?)(")', bygroups(String, using(PythonLexer), String), '#pop'),
+ ("(')(.*?)(')", bygroups(String, using(PythonLexer), String), '#pop'),
+ (r'[^\s>]+', String, '#pop'),
+ ],
+ 'tag': [
+ (r'\s+', Text),
+ (r'py:[\w-]+\s*=', Name.Attribute, 'pyattr'),
+ (r'[\w:-]+\s*=', Name.Attribute, 'attr'),
+ (r'/?\s*>', Name.Tag, '#pop'),
+ ],
+ 'attr': [
+ ('"', String, 'attr-dstring'),
+ ("'", String, 'attr-sstring'),
+ (r'[^\s>]*', String, '#pop')
+ ],
+ 'attr-dstring': [
+ ('"', String, '#pop'),
+ include('strings'),
+ ("'", String)
+ ],
+ 'attr-sstring': [
+ ("'", String, '#pop'),
+ include('strings'),
+ ("'", String)
+ ],
+ 'strings': [
+ ('[^"\'$]+', String),
+ include('variable')
+ ],
+ 'variable': [
+ (r'(?<!\$)(\$\{)(.+?)(\})',
+ bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)),
+ (r'(?<!\$)(\$)([a-zA-Z_][\w\.]*)',
+ Name.Variable),
+ ]
+ }
+
+
+class HtmlGenshiLexer(DelegatingLexer):
+ """
+ A lexer that highlights `genshi <http://genshi.edgewall.org/>`_ and
+ `kid <http://kid-templating.org/>`_ kid HTML templates.
+ """
+
+ name = 'HTML+Genshi'
+ aliases = ['html+genshi', 'html+kid']
+ alias_filenames = ['*.html', '*.htm', '*.xhtml']
+ mimetypes = ['text/html+genshi']
+
+ def __init__(self, **options):
+ super(HtmlGenshiLexer, self).__init__(HtmlLexer, GenshiMarkupLexer,
+ **options)
+
+ def analyse_text(text):
+ rv = 0.0
+ if re.search('\$\{.*?\}', text) is not None:
+ rv += 0.2
+ if re.search('py:(.*?)=["\']', text) is not None:
+ rv += 0.2
+ return rv + HtmlLexer.analyse_text(text) - 0.01
+
+
+class GenshiLexer(DelegatingLexer):
+ """
+ A lexer that highlights `genshi <http://genshi.edgewall.org/>`_ and
+ `kid <http://kid-templating.org/>`_ kid XML templates.
+ """
+
+ name = 'Genshi'
+ aliases = ['genshi', 'kid', 'xml+genshi', 'xml+kid']
+ filenames = ['*.kid']
+ alias_filenames = ['*.xml']
+ mimetypes = ['application/x-genshi', 'application/x-kid']
+
+ def __init__(self, **options):
+ super(GenshiLexer, self).__init__(XmlLexer, GenshiMarkupLexer,
+ **options)
+
+ def analyse_text(text):
+ rv = 0.0
+ if re.search('\$\{.*?\}', text) is not None:
+ rv += 0.2
+ if re.search('py:(.*?)=["\']', text) is not None:
+ rv += 0.2
+ return rv + XmlLexer.analyse_text(text) - 0.01
+
+
+class JavascriptGenshiLexer(DelegatingLexer):
+ """
+ A lexer that highlights javascript code in genshi text templates.
+ """
+
+ name = 'JavaScript+Genshi Text'
+ aliases = ['js+genshitext', 'js+genshi', 'javascript+genshitext',
+ 'javascript+genshi']
+ alias_filenames = ['*.js']
+ mimetypes = ['application/x-javascript+genshi',
+ 'text/x-javascript+genshi',
+ 'text/javascript+genshi']
+
+ def __init__(self, **options):
+ super(JavascriptGenshiLexer, self).__init__(JavascriptLexer,
+ GenshiTextLexer,
+ **options)
+
+ def analyse_text(text):
+ return GenshiLexer.analyse_text(text) - 0.05
+
+
+class CssGenshiLexer(DelegatingLexer):
+ """
+ A lexer that highlights CSS definitions in genshi text templates.
+ """
+
+ name = 'CSS+Genshi Text'
+ aliases = ['css+genshitext', 'css+genshi']
+ alias_filenames = ['*.css']
+ mimetypes = ['text/css+genshi']
+
+ def __init__(self, **options):
+ super(CssGenshiLexer, self).__init__(CssLexer, GenshiTextLexer,
+ **options)
+
+ def analyse_text(text):
+ return GenshiLexer.analyse_text(text) - 0.05
+
+
+class RhtmlLexer(DelegatingLexer):
+ """
+ Subclass of the ERB lexer that highlights the unlexed data with the
+ html lexer.
+
+ Nested Javascript and CSS is highlighted too.
+ """
+
+ name = 'RHTML'
+ aliases = ['rhtml', 'html+erb', 'html+ruby']
+ filenames = ['*.rhtml']
+ alias_filenames = ['*.html', '*.htm', '*.xhtml']
+ mimetypes = ['text/html+ruby']
+
+ def __init__(self, **options):
+ super(RhtmlLexer, self).__init__(HtmlLexer, ErbLexer, **options)
+
+ def analyse_text(text):
+ rv = ErbLexer.analyse_text(text) - 0.01
+ if html_doctype_matches(text):
+ # one more than the XmlErbLexer returns
+ rv += 0.5
+ return rv
+
+
+class XmlErbLexer(DelegatingLexer):
+ """
+ Subclass of `ErbLexer` which highlights data outside preprocessor
+ directives with the `XmlLexer`.
+ """
+
+ name = 'XML+Ruby'
+ aliases = ['xml+erb', 'xml+ruby']
+ alias_filenames = ['*.xml']
+ mimetypes = ['application/xml+ruby']
+
+ def __init__(self, **options):
+ super(XmlErbLexer, self).__init__(XmlLexer, ErbLexer, **options)
+
+ def analyse_text(text):
+ rv = ErbLexer.analyse_text(text) - 0.01
+ if looks_like_xml(text):
+ rv += 0.4
+ return rv
+
+
+class CssErbLexer(DelegatingLexer):
+ """
+ Subclass of `ErbLexer` which highlights unlexed data with the `CssLexer`.
+ """
+
+ name = 'CSS+Ruby'
+ aliases = ['css+erb', 'css+ruby']
+ alias_filenames = ['*.css']
+ mimetypes = ['text/css+ruby']
+
+ def __init__(self, **options):
+ super(CssErbLexer, self).__init__(CssLexer, ErbLexer, **options)
+
+ def analyse_text(text):
+ return ErbLexer.analyse_text(text) - 0.05
+
+
+class JavascriptErbLexer(DelegatingLexer):
+ """
+ Subclass of `ErbLexer` which highlights unlexed data with the
+ `JavascriptLexer`.
+ """
+
+ name = 'JavaScript+Ruby'
+ aliases = ['js+erb', 'javascript+erb', 'js+ruby', 'javascript+ruby']
+ alias_filenames = ['*.js']
+ mimetypes = ['application/x-javascript+ruby',
+ 'text/x-javascript+ruby',
+ 'text/javascript+ruby']
+
+ def __init__(self, **options):
+ super(JavascriptErbLexer, self).__init__(JavascriptLexer, ErbLexer,
+ **options)
+
+ def analyse_text(text):
+ return ErbLexer.analyse_text(text) - 0.05
+
+
+class HtmlPhpLexer(DelegatingLexer):
+ """
+ Subclass of `PhpLexer` that highlights unhandled data with the `HtmlLexer`.
+
+ Nested Javascript and CSS is highlighted too.
+ """
+
+ name = 'HTML+PHP'
+ aliases = ['html+php']
+ filenames = ['*.phtml']
+ alias_filenames = ['*.php', '*.html', '*.htm', '*.xhtml',
+ '*.php[345]']
+ mimetypes = ['application/x-php',
+ 'application/x-httpd-php', 'application/x-httpd-php3',
+ 'application/x-httpd-php4', 'application/x-httpd-php5']
+
+ def __init__(self, **options):
+ super(HtmlPhpLexer, self).__init__(HtmlLexer, PhpLexer, **options)
+
+ def analyse_text(text):
+ rv = PhpLexer.analyse_text(text) - 0.01
+ if html_doctype_matches(text):
+ rv += 0.5
+ return rv
+
+
+class XmlPhpLexer(DelegatingLexer):
+ """
+ Subclass of `PhpLexer` that highlights unhandled data with the `XmlLexer`.
+ """
+
+ name = 'XML+PHP'
+ aliases = ['xml+php']
+ alias_filenames = ['*.xml', '*.php', '*.php[345]']
+ mimetypes = ['application/xml+php']
+
+ def __init__(self, **options):
+ super(XmlPhpLexer, self).__init__(XmlLexer, PhpLexer, **options)
+
+ def analyse_text(text):
+ rv = PhpLexer.analyse_text(text) - 0.01
+ if looks_like_xml(text):
+ rv += 0.4
+ return rv
+
+
+class CssPhpLexer(DelegatingLexer):
+ """
+ Subclass of `PhpLexer` which highlights unmatched data with the `CssLexer`.
+ """
+
+ name = 'CSS+PHP'
+ aliases = ['css+php']
+ alias_filenames = ['*.css']
+ mimetypes = ['text/css+php']
+
+ def __init__(self, **options):
+ super(CssPhpLexer, self).__init__(CssLexer, PhpLexer, **options)
+
+ def analyse_text(text):
+ return PhpLexer.analyse_text(text) - 0.05
+
+
+class JavascriptPhpLexer(DelegatingLexer):
+ """
+ Subclass of `PhpLexer` which highlights unmatched data with the
+ `JavascriptLexer`.
+ """
+
+ name = 'JavaScript+PHP'
+ aliases = ['js+php', 'javascript+php']
+ alias_filenames = ['*.js']
+ mimetypes = ['application/x-javascript+php',
+ 'text/x-javascript+php',
+ 'text/javascript+php']
+
+ def __init__(self, **options):
+ super(JavascriptPhpLexer, self).__init__(JavascriptLexer, PhpLexer,
+ **options)
+
+ def analyse_text(text):
+ return PhpLexer.analyse_text(text)
+
+
+class HtmlSmartyLexer(DelegatingLexer):
+ """
+ Subclass of the `SmartyLexer` that highlights unlexed data with the
+ `HtmlLexer`.
+
+ Nested Javascript and CSS is highlighted too.
+ """
+
+ name = 'HTML+Smarty'
+ aliases = ['html+smarty']
+ alias_filenames = ['*.html', '*.htm', '*.xhtml', '*.tpl']
+ mimetypes = ['text/html+smarty']
+
+ def __init__(self, **options):
+ super(HtmlSmartyLexer, self).__init__(HtmlLexer, SmartyLexer, **options)
+
+ def analyse_text(text):
+ rv = SmartyLexer.analyse_text(text) - 0.01
+ if html_doctype_matches(text):
+ rv += 0.5
+ return rv
+
+
+class XmlSmartyLexer(DelegatingLexer):
+ """
+ Subclass of the `SmartyLexer` that highlights unlexed data with the
+ `XmlLexer`.
+ """
+
+ name = 'XML+Smarty'
+ aliases = ['xml+smarty']
+ alias_filenames = ['*.xml', '*.tpl']
+ mimetypes = ['application/xml+smarty']
+
+ def __init__(self, **options):
+ super(XmlSmartyLexer, self).__init__(XmlLexer, SmartyLexer, **options)
+
+ def analyse_text(text):
+ rv = SmartyLexer.analyse_text(text) - 0.01
+ if looks_like_xml(text):
+ rv += 0.4
+ return rv
+
+
+class CssSmartyLexer(DelegatingLexer):
+ """
+ Subclass of the `SmartyLexer` that highlights unlexed data with the
+ `CssLexer`.
+ """
+
+ name = 'CSS+Smarty'
+ aliases = ['css+smarty']
+ alias_filenames = ['*.css', '*.tpl']
+ mimetypes = ['text/css+smarty']
+
+ def __init__(self, **options):
+ super(CssSmartyLexer, self).__init__(CssLexer, SmartyLexer, **options)
+
+ def analyse_text(text):
+ return SmartyLexer.analyse_text(text) - 0.05
+
+
+class JavascriptSmartyLexer(DelegatingLexer):
+ """
+ Subclass of the `SmartyLexer` that highlights unlexed data with the
+ `JavascriptLexer`.
+ """
+
+ name = 'JavaScript+Smarty'
+ aliases = ['js+smarty', 'javascript+smarty']
+ alias_filenames = ['*.js', '*.tpl']
+ mimetypes = ['application/x-javascript+smarty',
+ 'text/x-javascript+smarty',
+ 'text/javascript+smarty']
+
+ def __init__(self, **options):
+ super(JavascriptSmartyLexer, self).__init__(JavascriptLexer, SmartyLexer,
+ **options)
+
+ def analyse_text(text):
+ return SmartyLexer.analyse_text(text) - 0.05
+
+
+class HtmlDjangoLexer(DelegatingLexer):
+ """
+ Subclass of the `DjangoLexer` that highlights unlexed data with the
+ `HtmlLexer`.
+
+ Nested Javascript and CSS is highlighted too.
+ """
+
+ name = 'HTML+Django/Jinja'
+ aliases = ['html+django', 'html+jinja', 'htmldjango']
+ alias_filenames = ['*.html', '*.htm', '*.xhtml']
+ mimetypes = ['text/html+django', 'text/html+jinja']
+
+ def __init__(self, **options):
+ super(HtmlDjangoLexer, self).__init__(HtmlLexer, DjangoLexer, **options)
+
+ def analyse_text(text):
+ rv = DjangoLexer.analyse_text(text) - 0.01
+ if html_doctype_matches(text):
+ rv += 0.5
+ return rv
+
+
+class XmlDjangoLexer(DelegatingLexer):
+ """
+ Subclass of the `DjangoLexer` that highlights unlexed data with the
+ `XmlLexer`.
+ """
+
+ name = 'XML+Django/Jinja'
+ aliases = ['xml+django', 'xml+jinja']
+ alias_filenames = ['*.xml']
+ mimetypes = ['application/xml+django', 'application/xml+jinja']
+
+ def __init__(self, **options):
+ super(XmlDjangoLexer, self).__init__(XmlLexer, DjangoLexer, **options)
+
+ def analyse_text(text):
+ rv = DjangoLexer.analyse_text(text) - 0.01
+ if looks_like_xml(text):
+ rv += 0.4
+ return rv
+
+
+class CssDjangoLexer(DelegatingLexer):
+ """
+ Subclass of the `DjangoLexer` that highlights unlexed data with the
+ `CssLexer`.
+ """
+
+ name = 'CSS+Django/Jinja'
+ aliases = ['css+django', 'css+jinja']
+ alias_filenames = ['*.css']
+ mimetypes = ['text/css+django', 'text/css+jinja']
+
+ def __init__(self, **options):
+ super(CssDjangoLexer, self).__init__(CssLexer, DjangoLexer, **options)
+
+ def analyse_text(text):
+ return DjangoLexer.analyse_text(text) - 0.05
+
+
+class JavascriptDjangoLexer(DelegatingLexer):
+ """
+ Subclass of the `DjangoLexer` that highlights unlexed data with the
+ `JavascriptLexer`.
+ """
+
+ name = 'JavaScript+Django/Jinja'
+ aliases = ['js+django', 'javascript+django',
+ 'js+jinja', 'javascript+jinja']
+ alias_filenames = ['*.js']
+ mimetypes = ['application/x-javascript+django',
+ 'application/x-javascript+jinja',
+ 'text/x-javascript+django',
+ 'text/x-javascript+jinja',
+ 'text/javascript+django',
+ 'text/javascript+jinja']
+
+ def __init__(self, **options):
+ super(JavascriptDjangoLexer, self).__init__(JavascriptLexer, DjangoLexer,
+ **options)
+
+ def analyse_text(text):
+ return DjangoLexer.analyse_text(text) - 0.05
+
+
+class JspRootLexer(RegexLexer):
+ """
+ Base for the `JspLexer`. Yields `Token.Other` for area outside of
+ JSP tags.
+
+ .. versionadded:: 0.7
+ """
+
+ tokens = {
+ 'root': [
+ (r'<%\S?', Keyword, 'sec'),
+ # FIXME: I want to make these keywords but still parse attributes.
+ (r'</?jsp:(forward|getProperty|include|plugin|setProperty|useBean).*?>',
+ Keyword),
+ (r'[^<]+', Other),
+ (r'<', Other),
+ ],
+ 'sec': [
+ (r'%>', Keyword, '#pop'),
+ # note: '\w\W' != '.' without DOTALL.
+ (r'[\w\W]+?(?=%>|\Z)', using(JavaLexer)),
+ ],
+ }
+
+
+class JspLexer(DelegatingLexer):
+ """
+ Lexer for Java Server Pages.
+
+ .. versionadded:: 0.7
+ """
+ name = 'Java Server Page'
+ aliases = ['jsp']
+ filenames = ['*.jsp']
+ mimetypes = ['application/x-jsp']
+
+ def __init__(self, **options):
+ super(JspLexer, self).__init__(XmlLexer, JspRootLexer, **options)
+
+ def analyse_text(text):
+ rv = JavaLexer.analyse_text(text) - 0.01
+ if looks_like_xml(text):
+ rv += 0.4
+ if '<%' in text and '%>' in text:
+ rv += 0.1
+ return rv
+
+
+class EvoqueLexer(RegexLexer):
+ """
+ For files using the Evoque templating system.
+
+ .. versionadded:: 1.1
+ """
+ name = 'Evoque'
+ aliases = ['evoque']
+ filenames = ['*.evoque']
+ mimetypes = ['application/x-evoque']
+
+ flags = re.DOTALL
+
+ tokens = {
+ 'root': [
+ (r'[^#$]+', Other),
+ (r'#\[', Comment.Multiline, 'comment'),
+ (r'\$\$', Other),
+ # svn keywords
+ (r'\$\w+:[^$\n]*\$', Comment.Multiline),
+ # directives: begin, end
+ (r'(\$)(begin|end)(\{(%)?)(.*?)((?(4)%)\})',
+ bygroups(Punctuation, Name.Builtin, Punctuation, None,
+ String, Punctuation)),
+ # directives: evoque, overlay
+ # see doc for handling first name arg: /directives/evoque/
+ # + minor inconsistency: the "name" in e.g. $overlay{name=site_base}
+ # should be using(PythonLexer), not passed out as String
+ (r'(\$)(evoque|overlay)(\{(%)?)(\s*[#\w\-"\'.]+[^=,%}]+?)?'
+ r'(.*?)((?(4)%)\})',
+ bygroups(Punctuation, Name.Builtin, Punctuation, None,
+ String, using(PythonLexer), Punctuation)),
+ # directives: if, for, prefer, test
+ (r'(\$)(\w+)(\{(%)?)(.*?)((?(4)%)\})',
+ bygroups(Punctuation, Name.Builtin, Punctuation, None,
+ using(PythonLexer), Punctuation)),
+ # directive clauses (no {} expression)
+ (r'(\$)(else|rof|fi)', bygroups(Punctuation, Name.Builtin)),
+ # expressions
+ (r'(\$\{(%)?)(.*?)((!)(.*?))?((?(2)%)\})',
+ bygroups(Punctuation, None, using(PythonLexer),
+ Name.Builtin, None, None, Punctuation)),
+ (r'#', Other),
+ ],
+ 'comment': [
+ (r'[^\]#]', Comment.Multiline),
+ (r'#\[', Comment.Multiline, '#push'),
+ (r'\]#', Comment.Multiline, '#pop'),
+ (r'[\]#]', Comment.Multiline)
+ ],
+ }
+
+
+class EvoqueHtmlLexer(DelegatingLexer):
+ """
+ Subclass of the `EvoqueLexer` that highlights unlexed data with the
+ `HtmlLexer`.
+
+ .. versionadded:: 1.1
+ """
+ name = 'HTML+Evoque'
+ aliases = ['html+evoque']
+ filenames = ['*.html']
+ mimetypes = ['text/html+evoque']
+
+ def __init__(self, **options):
+ super(EvoqueHtmlLexer, self).__init__(HtmlLexer, EvoqueLexer,
+ **options)
+
+
+class EvoqueXmlLexer(DelegatingLexer):
+ """
+ Subclass of the `EvoqueLexer` that highlights unlexed data with the
+ `XmlLexer`.
+
+ .. versionadded:: 1.1
+ """
+ name = 'XML+Evoque'
+ aliases = ['xml+evoque']
+ filenames = ['*.xml']
+ mimetypes = ['application/xml+evoque']
+
+ def __init__(self, **options):
+ super(EvoqueXmlLexer, self).__init__(XmlLexer, EvoqueLexer,
+ **options)
+
+
+class ColdfusionLexer(RegexLexer):
+ """
+ Coldfusion statements
+ """
+ name = 'cfstatement'
+ aliases = ['cfs']
+ filenames = []
+ mimetypes = []
+ flags = re.IGNORECASE
+
+ tokens = {
+ 'root': [
+ (r'//.*?\n', Comment.Single),
+ (r'/\*(?:.|\n)*?\*/', Comment.Multiline),
+ (r'\+\+|--', Operator),
+ (r'[-+*/^&=!]', Operator),
+ (r'<=|>=|<|>|==', Operator),
+ (r'mod\b', Operator),
+ (r'(eq|lt|gt|lte|gte|not|is|and|or)\b', Operator),
+ (r'\|\||&&', Operator),
+ (r'\?', Operator),
+ (r'"', String.Double, 'string'),
+ # There is a special rule for allowing html in single quoted
+ # strings, evidently.
+ (r"'.*?'", String.Single),
+ (r'\d+', Number),
+ (r'(if|else|len|var|xml|default|break|switch|component|property|function|do|'
+ r'try|catch|in|continue|for|return|while|required|any|array|binary|boolean|'
+ r'component|date|guid|numeric|query|string|struct|uuid|case)\b', Keyword),
+ (r'(true|false|null)\b', Keyword.Constant),
+ (r'(application|session|client|cookie|super|this|variables|arguments)\b',
+ Name.Constant),
+ (r'([a-z_$][\w.]*)(\s*)(\()',
+ bygroups(Name.Function, Text, Punctuation)),
+ (r'[a-z_$][\w.]*', Name.Variable),
+ (r'[()\[\]{};:,.\\]', Punctuation),
+ (r'\s+', Text),
+ ],
+ 'string': [
+ (r'""', String.Double),
+ (r'#.+?#', String.Interp),
+ (r'[^"#]+', String.Double),
+ (r'#', String.Double),
+ (r'"', String.Double, '#pop'),
+ ],
+ }
+
+
+class ColdfusionMarkupLexer(RegexLexer):
+ """
+ Coldfusion markup only
+ """
+ name = 'Coldfusion'
+ aliases = ['cf']
+ filenames = []
+ mimetypes = []
+
+ tokens = {
+ 'root': [
+ (r'[^<]+', Other),
+ include('tags'),
+ (r'<[^<>]*', Other),
+ ],
+ 'tags': [
+ (r'<!---', Comment.Multiline, 'cfcomment'),
+ (r'(?s)<!--.*?-->', Comment),
+ (r'<cfoutput.*?>', Name.Builtin, 'cfoutput'),
+ (r'(?s)(<cfscript.*?>)(.+?)(</cfscript.*?>)',
+ bygroups(Name.Builtin, using(ColdfusionLexer), Name.Builtin)),
+ # negative lookbehind is for strings with embedded >
+ (r'(?s)(</?cf(?:component|include|if|else|elseif|loop|return|'
+ r'dbinfo|dump|abort|location|invoke|throw|file|savecontent|'
+ r'mailpart|mail|header|content|zip|image|lock|argument|try|'
+ r'catch|break|directory|http|set|function|param)\b)(.*?)((?<!\\)>)',
+ bygroups(Name.Builtin, using(ColdfusionLexer), Name.Builtin)),
+ ],
+ 'cfoutput': [
+ (r'[^#<]+', Other),
+ (r'(#)(.*?)(#)', bygroups(Punctuation, using(ColdfusionLexer),
+ Punctuation)),
+ # (r'<cfoutput.*?>', Name.Builtin, '#push'),
+ (r'</cfoutput.*?>', Name.Builtin, '#pop'),
+ include('tags'),
+ (r'(?s)<[^<>]*', Other),
+ (r'#', Other),
+ ],
+ 'cfcomment': [
+ (r'<!---', Comment.Multiline, '#push'),
+ (r'--->', Comment.Multiline, '#pop'),
+ (r'([^<-]|<(?!!---)|-(?!-->))+', Comment.Multiline),
+ ],
+ }
+
+
+class ColdfusionHtmlLexer(DelegatingLexer):
+ """
+ Coldfusion markup in html
+ """
+ name = 'Coldfusion HTML'
+ aliases = ['cfm']
+ filenames = ['*.cfm', '*.cfml']
+ mimetypes = ['application/x-coldfusion']
+
+ def __init__(self, **options):
+ super(ColdfusionHtmlLexer, self).__init__(HtmlLexer, ColdfusionMarkupLexer,
+ **options)
+
+
+class ColdfusionCFCLexer(DelegatingLexer):
+ """
+ Coldfusion markup/script components
+
+ .. versionadded:: 2.0
+ """
+ name = 'Coldfusion CFC'
+ aliases = ['cfc']
+ filenames = ['*.cfc']
+ mimetypes = []
+
+ def __init__(self, **options):
+ super(ColdfusionCFCLexer, self).__init__(ColdfusionHtmlLexer, ColdfusionLexer,
+ **options)
+
+
+class SspLexer(DelegatingLexer):
+ """
+ Lexer for Scalate Server Pages.
+
+ .. versionadded:: 1.4
+ """
+ name = 'Scalate Server Page'
+ aliases = ['ssp']
+ filenames = ['*.ssp']
+ mimetypes = ['application/x-ssp']
+
+ def __init__(self, **options):
+ super(SspLexer, self).__init__(XmlLexer, JspRootLexer, **options)
+
+ def analyse_text(text):
+ rv = 0.0
+ if re.search('val \w+\s*:', text):
+ rv += 0.6
+ if looks_like_xml(text):
+ rv += 0.2
+ if '<%' in text and '%>' in text:
+ rv += 0.1
+ return rv
+
+
+class TeaTemplateRootLexer(RegexLexer):
+ """
+ Base for the `TeaTemplateLexer`. Yields `Token.Other` for area outside of
+ code blocks.
+
+ .. versionadded:: 1.5
+ """
+
+ tokens = {
+ 'root': [
+ (r'<%\S?', Keyword, 'sec'),
+ (r'[^<]+', Other),
+ (r'<', Other),
+ ],
+ 'sec': [
+ (r'%>', Keyword, '#pop'),
+ # note: '\w\W' != '.' without DOTALL.
+ (r'[\w\W]+?(?=%>|\Z)', using(TeaLangLexer)),
+ ],
+ }
+
+
+class TeaTemplateLexer(DelegatingLexer):
+ """
+ Lexer for `Tea Templates <http://teatrove.org/>`_.
+
+ .. versionadded:: 1.5
+ """
+ name = 'Tea'
+ aliases = ['tea']
+ filenames = ['*.tea']
+ mimetypes = ['text/x-tea']
+
+ def __init__(self, **options):
+ super(TeaTemplateLexer, self).__init__(XmlLexer,
+ TeaTemplateRootLexer, **options)
+
+ def analyse_text(text):
+ rv = TeaLangLexer.analyse_text(text) - 0.01
+ if looks_like_xml(text):
+ rv += 0.4
+ if '<%' in text and '%>' in text:
+ rv += 0.1
+ return rv
+
+
+class LassoHtmlLexer(DelegatingLexer):
+ """
+ Subclass of the `LassoLexer` which highlights unhandled data with the
+ `HtmlLexer`.
+
+ Nested JavaScript and CSS is also highlighted.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'HTML+Lasso'
+ aliases = ['html+lasso']
+ alias_filenames = ['*.html', '*.htm', '*.xhtml', '*.lasso', '*.lasso[89]',
+ '*.incl', '*.inc', '*.las']
+ mimetypes = ['text/html+lasso',
+ 'application/x-httpd-lasso',
+ 'application/x-httpd-lasso[89]']
+
+ def __init__(self, **options):
+ super(LassoHtmlLexer, self).__init__(HtmlLexer, LassoLexer, **options)
+
+ def analyse_text(text):
+ rv = LassoLexer.analyse_text(text) - 0.01
+ if html_doctype_matches(text): # same as HTML lexer
+ rv += 0.5
+ return rv
+
+
+class LassoXmlLexer(DelegatingLexer):
+ """
+ Subclass of the `LassoLexer` which highlights unhandled data with the
+ `XmlLexer`.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'XML+Lasso'
+ aliases = ['xml+lasso']
+ alias_filenames = ['*.xml', '*.lasso', '*.lasso[89]',
+ '*.incl', '*.inc', '*.las']
+ mimetypes = ['application/xml+lasso']
+
+ def __init__(self, **options):
+ super(LassoXmlLexer, self).__init__(XmlLexer, LassoLexer, **options)
+
+ def analyse_text(text):
+ rv = LassoLexer.analyse_text(text) - 0.01
+ if looks_like_xml(text):
+ rv += 0.4
+ return rv
+
+
+class LassoCssLexer(DelegatingLexer):
+ """
+ Subclass of the `LassoLexer` which highlights unhandled data with the
+ `CssLexer`.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'CSS+Lasso'
+ aliases = ['css+lasso']
+ alias_filenames = ['*.css']
+ mimetypes = ['text/css+lasso']
+
+ def __init__(self, **options):
+ options['requiredelimiters'] = True
+ super(LassoCssLexer, self).__init__(CssLexer, LassoLexer, **options)
+
+ def analyse_text(text):
+ rv = LassoLexer.analyse_text(text) - 0.05
+ if re.search(r'\w+:.+?;', text):
+ rv += 0.1
+ if 'padding:' in text:
+ rv += 0.1
+ return rv
+
+
+class LassoJavascriptLexer(DelegatingLexer):
+ """
+ Subclass of the `LassoLexer` which highlights unhandled data with the
+ `JavascriptLexer`.
+
+ .. versionadded:: 1.6
+ """
+
+ name = 'JavaScript+Lasso'
+ aliases = ['js+lasso', 'javascript+lasso']
+ alias_filenames = ['*.js']
+ mimetypes = ['application/x-javascript+lasso',
+ 'text/x-javascript+lasso',
+ 'text/javascript+lasso']
+
+ def __init__(self, **options):
+ options['requiredelimiters'] = True
+ super(LassoJavascriptLexer, self).__init__(JavascriptLexer, LassoLexer,
+ **options)
+
+ def analyse_text(text):
+ rv = LassoLexer.analyse_text(text) - 0.05
+ return rv
+
+
+class HandlebarsLexer(RegexLexer):
+ """
+ Generic `handlebars <http://handlebarsjs.com/>` template lexer.
+
+ Highlights only the Handlebars template tags (stuff between `{{` and `}}`).
+ Everything else is left for a delegating lexer.
+
+ .. versionadded:: 2.0
+ """
+
+ name = "Handlebars"
+ aliases = ['handlebars']
+
+ tokens = {
+ 'root': [
+ (r'[^{]+', Other),
+
+ (r'\{\{!.*\}\}', Comment),
+
+ (r'(\{\{\{)(\s*)', bygroups(Comment.Special, Text), 'tag'),
+ (r'(\{\{)(\s*)', bygroups(Comment.Preproc, Text), 'tag'),
+ ],
+
+ 'tag': [
+ (r'\s+', Text),
+ (r'\}\}\}', Comment.Special, '#pop'),
+ (r'\}\}', Comment.Preproc, '#pop'),
+
+ # Handlebars
+ (r'([#/]*)(each|if|unless|else|with|log|in(line)?)', bygroups(Keyword,
+ Keyword)),
+ (r'#\*inline', Keyword),
+
+ # General {{#block}}
+ (r'([#/])([\w-]+)', bygroups(Name.Function, Name.Function)),
+
+ # {{opt=something}}
+ (r'([\w-]+)(=)', bygroups(Name.Attribute, Operator)),
+
+ # Partials {{> ...}}
+ (r'(>)(\s*)(@partial-block)', bygroups(Keyword, Text, Keyword)),
+ (r'(#?>)(\s*)([\w-]+)', bygroups(Keyword, Text, Name.Variable)),
+ (r'(>)(\s*)(\()', bygroups(Keyword, Text, Punctuation),
+ 'dynamic-partial'),
+
+ include('generic'),
+ ],
+ 'dynamic-partial': [
+ (r'\s+', Text),
+ (r'\)', Punctuation, '#pop'),
+
+ (r'(lookup)(\s+)(\.|this)(\s+)', bygroups(Keyword, Text,
+ Name.Variable, Text)),
+ (r'(lookup)(\s+)(\S+)', bygroups(Keyword, Text,
+ using(this, state='variable'))),
+ (r'[\w-]+', Name.Function),
+
+ include('generic'),
+ ],
+ 'variable': [
+ (r'[a-zA-Z][\w-]*', Name.Variable),
+ (r'\.[\w-]+', Name.Variable),
+ (r'(this\/|\.\/|(\.\.\/)+)[\w-]+', Name.Variable),
+ ],
+ 'generic': [
+ include('variable'),
+
+ # borrowed from DjangoLexer
+ (r':?"(\\\\|\\"|[^"])*"', String.Double),
+ (r":?'(\\\\|\\'|[^'])*'", String.Single),
+ (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|"
+ r"0[xX][0-9a-fA-F]+[Ll]?", Number),
+ ]
+ }
+
+
+class HandlebarsHtmlLexer(DelegatingLexer):
+ """
+ Subclass of the `HandlebarsLexer` that highlights unlexed data with the
+ `HtmlLexer`.
+
+ .. versionadded:: 2.0
+ """
+
+ name = "HTML+Handlebars"
+ aliases = ["html+handlebars"]
+ filenames = ['*.handlebars', '*.hbs']
+ mimetypes = ['text/html+handlebars', 'text/x-handlebars-template']
+
+ def __init__(self, **options):
+ super(HandlebarsHtmlLexer, self).__init__(HtmlLexer, HandlebarsLexer, **options)
+
+
+class YamlJinjaLexer(DelegatingLexer):
+ """
+ Subclass of the `DjangoLexer` that highlights unlexed data with the
+ `YamlLexer`.
+
+ Commonly used in Saltstack salt states.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'YAML+Jinja'
+ aliases = ['yaml+jinja', 'salt', 'sls']
+ filenames = ['*.sls']
+ mimetypes = ['text/x-yaml+jinja', 'text/x-sls']
+
+ def __init__(self, **options):
+ super(YamlJinjaLexer, self).__init__(YamlLexer, DjangoLexer, **options)
+
+
+class LiquidLexer(RegexLexer):
+ """
+ Lexer for `Liquid templates
+ <http://www.rubydoc.info/github/Shopify/liquid>`_.
+
+ .. versionadded:: 2.0
+ """
+ name = 'liquid'
+ aliases = ['liquid']
+ filenames = ['*.liquid']
+
+ tokens = {
+ 'root': [
+ (r'[^{]+', Text),
+ # tags and block tags
+ (r'(\{%)(\s*)', bygroups(Punctuation, Whitespace), 'tag-or-block'),
+ # output tags
+ (r'(\{\{)(\s*)([^\s}]+)',
+ bygroups(Punctuation, Whitespace, using(this, state = 'generic')),
+ 'output'),
+ (r'\{', Text)
+ ],
+
+ 'tag-or-block': [
+ # builtin logic blocks
+ (r'(if|unless|elsif|case)(?=\s+)', Keyword.Reserved, 'condition'),
+ (r'(when)(\s+)', bygroups(Keyword.Reserved, Whitespace),
+ combined('end-of-block', 'whitespace', 'generic')),
+ (r'(else)(\s*)(%\})',
+ bygroups(Keyword.Reserved, Whitespace, Punctuation), '#pop'),
+
+ # other builtin blocks
+ (r'(capture)(\s+)([^\s%]+)(\s*)(%\})',
+ bygroups(Name.Tag, Whitespace, using(this, state = 'variable'),
+ Whitespace, Punctuation), '#pop'),
+ (r'(comment)(\s*)(%\})',
+ bygroups(Name.Tag, Whitespace, Punctuation), 'comment'),
+ (r'(raw)(\s*)(%\})',
+ bygroups(Name.Tag, Whitespace, Punctuation), 'raw'),
+
+ # end of block
+ (r'(end(case|unless|if))(\s*)(%\})',
+ bygroups(Keyword.Reserved, None, Whitespace, Punctuation), '#pop'),
+ (r'(end([^\s%]+))(\s*)(%\})',
+ bygroups(Name.Tag, None, Whitespace, Punctuation), '#pop'),
+
+ # builtin tags (assign and include are handled together with usual tags)
+ (r'(cycle)(\s+)(?:([^\s:]*)(:))?(\s*)',
+ bygroups(Name.Tag, Whitespace,
+ using(this, state='generic'), Punctuation, Whitespace),
+ 'variable-tag-markup'),
+
+ # other tags or blocks
+ (r'([^\s%]+)(\s*)', bygroups(Name.Tag, Whitespace), 'tag-markup')
+ ],
+
+ 'output': [
+ include('whitespace'),
+ ('\}\}', Punctuation, '#pop'), # end of output
+
+ (r'\|', Punctuation, 'filters')
+ ],
+
+ 'filters': [
+ include('whitespace'),
+ (r'\}\}', Punctuation, ('#pop', '#pop')), # end of filters and output
+
+ (r'([^\s|:]+)(:?)(\s*)',
+ bygroups(Name.Function, Punctuation, Whitespace), 'filter-markup')
+ ],
+
+ 'filter-markup': [
+ (r'\|', Punctuation, '#pop'),
+ include('end-of-tag'),
+ include('default-param-markup')
+ ],
+
+ 'condition': [
+ include('end-of-block'),
+ include('whitespace'),
+
+ (r'([^\s=!><]+)(\s*)([=!><]=?)(\s*)(\S+)(\s*)(%\})',
+ bygroups(using(this, state = 'generic'), Whitespace, Operator,
+ Whitespace, using(this, state = 'generic'), Whitespace,
+ Punctuation)),
+ (r'\b!', Operator),
+ (r'\bnot\b', Operator.Word),
+ (r'([\w.\'"]+)(\s+)(contains)(\s+)([\w.\'"]+)',
+ bygroups(using(this, state = 'generic'), Whitespace, Operator.Word,
+ Whitespace, using(this, state = 'generic'))),
+
+ include('generic'),
+ include('whitespace')
+ ],
+
+ 'generic-value': [
+ include('generic'),
+ include('end-at-whitespace')
+ ],
+
+ 'operator': [
+ (r'(\s*)((=|!|>|<)=?)(\s*)',
+ bygroups(Whitespace, Operator, None, Whitespace), '#pop'),
+ (r'(\s*)(\bcontains\b)(\s*)',
+ bygroups(Whitespace, Operator.Word, Whitespace), '#pop'),
+ ],
+
+ 'end-of-tag': [
+ (r'\}\}', Punctuation, '#pop')
+ ],
+
+ 'end-of-block': [
+ (r'%\}', Punctuation, ('#pop', '#pop'))
+ ],
+
+ 'end-at-whitespace': [
+ (r'\s+', Whitespace, '#pop')
+ ],
+
+ # states for unknown markup
+ 'param-markup': [
+ include('whitespace'),
+ # params with colons or equals
+ (r'([^\s=:]+)(\s*)(=|:)',
+ bygroups(Name.Attribute, Whitespace, Operator)),
+ # explicit variables
+ (r'(\{\{)(\s*)([^\s}])(\s*)(\}\})',
+ bygroups(Punctuation, Whitespace, using(this, state = 'variable'),
+ Whitespace, Punctuation)),
+
+ include('string'),
+ include('number'),
+ include('keyword'),
+ (r',', Punctuation)
+ ],
+
+ 'default-param-markup': [
+ include('param-markup'),
+ (r'.', Text) # fallback for switches / variables / un-quoted strings / ...
+ ],
+
+ 'variable-param-markup': [
+ include('param-markup'),
+ include('variable'),
+ (r'.', Text) # fallback
+ ],
+
+ 'tag-markup': [
+ (r'%\}', Punctuation, ('#pop', '#pop')), # end of tag
+ include('default-param-markup')
+ ],
+
+ 'variable-tag-markup': [
+ (r'%\}', Punctuation, ('#pop', '#pop')), # end of tag
+ include('variable-param-markup')
+ ],
+
+ # states for different values types
+ 'keyword': [
+ (r'\b(false|true)\b', Keyword.Constant)
+ ],
+
+ 'variable': [
+ (r'[a-zA-Z_]\w*', Name.Variable),
+ (r'(?<=\w)\.(?=\w)', Punctuation)
+ ],
+
+ 'string': [
+ (r"'[^']*'", String.Single),
+ (r'"[^"]*"', String.Double)
+ ],
+
+ 'number': [
+ (r'\d+\.\d+', Number.Float),
+ (r'\d+', Number.Integer)
+ ],
+
+ 'generic': [ # decides for variable, string, keyword or number
+ include('keyword'),
+ include('string'),
+ include('number'),
+ include('variable')
+ ],
+
+ 'whitespace': [
+ (r'[ \t]+', Whitespace)
+ ],
+
+ # states for builtin blocks
+ 'comment': [
+ (r'(\{%)(\s*)(endcomment)(\s*)(%\})',
+ bygroups(Punctuation, Whitespace, Name.Tag, Whitespace,
+ Punctuation), ('#pop', '#pop')),
+ (r'.', Comment)
+ ],
+
+ 'raw': [
+ (r'[^{]+', Text),
+ (r'(\{%)(\s*)(endraw)(\s*)(%\})',
+ bygroups(Punctuation, Whitespace, Name.Tag, Whitespace,
+ Punctuation), '#pop'),
+ (r'\{', Text)
+ ],
+ }
+
+
+class TwigLexer(RegexLexer):
+ """
+ `Twig <http://twig.sensiolabs.org/>`_ template lexer.
+
+ It just highlights Twig code between the preprocessor directives,
+ other data is left untouched by the lexer.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Twig'
+ aliases = ['twig']
+ mimetypes = ['application/x-twig']
+
+ flags = re.M | re.S
+
+ # Note that a backslash is included in the following two patterns
+ # PHP uses a backslash as a namespace separator
+ _ident_char = r'[\\\w-]|[^\x00-\x7f]'
+ _ident_begin = r'(?:[\\_a-z]|[^\x00-\x7f])'
+ _ident_end = r'(?:' + _ident_char + ')*'
+ _ident_inner = _ident_begin + _ident_end
+
+ tokens = {
+ 'root': [
+ (r'[^{]+', Other),
+ (r'\{\{', Comment.Preproc, 'var'),
+ # twig comments
+ (r'\{\#.*?\#\}', Comment),
+ # raw twig blocks
+ (r'(\{%)(-?\s*)(raw)(\s*-?)(%\})(.*?)'
+ r'(\{%)(-?\s*)(endraw)(\s*-?)(%\})',
+ bygroups(Comment.Preproc, Text, Keyword, Text, Comment.Preproc,
+ Other, Comment.Preproc, Text, Keyword, Text,
+ Comment.Preproc)),
+ (r'(\{%)(-?\s*)(verbatim)(\s*-?)(%\})(.*?)'
+ r'(\{%)(-?\s*)(endverbatim)(\s*-?)(%\})',
+ bygroups(Comment.Preproc, Text, Keyword, Text, Comment.Preproc,
+ Other, Comment.Preproc, Text, Keyword, Text,
+ Comment.Preproc)),
+ # filter blocks
+ (r'(\{%%)(-?\s*)(filter)(\s+)(%s)' % _ident_inner,
+ bygroups(Comment.Preproc, Text, Keyword, Text, Name.Function),
+ 'tag'),
+ (r'(\{%)(-?\s*)([a-zA-Z_]\w*)',
+ bygroups(Comment.Preproc, Text, Keyword), 'tag'),
+ (r'\{', Other),
+ ],
+ 'varnames': [
+ (r'(\|)(\s*)(%s)' % _ident_inner,
+ bygroups(Operator, Text, Name.Function)),
+ (r'(is)(\s+)(not)?(\s*)(%s)' % _ident_inner,
+ bygroups(Keyword, Text, Keyword, Text, Name.Function)),
+ (r'(?i)(true|false|none|null)\b', Keyword.Pseudo),
+ (r'(in|not|and|b-and|or|b-or|b-xor|is'
+ r'if|elseif|else|import'
+ r'constant|defined|divisibleby|empty|even|iterable|odd|sameas'
+ r'matches|starts\s+with|ends\s+with)\b',
+ Keyword),
+ (r'(loop|block|parent)\b', Name.Builtin),
+ (_ident_inner, Name.Variable),
+ (r'\.' + _ident_inner, Name.Variable),
+ (r'\.[0-9]+', Number),
+ (r':?"(\\\\|\\"|[^"])*"', String.Double),
+ (r":?'(\\\\|\\'|[^'])*'", String.Single),
+ (r'([{}()\[\]+\-*/,:~%]|\.\.|\?|:|\*\*|\/\/|!=|[><=]=?)', Operator),
+ (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|"
+ r"0[xX][0-9a-fA-F]+[Ll]?", Number),
+ ],
+ 'var': [
+ (r'\s+', Text),
+ (r'(-?)(\}\})', bygroups(Text, Comment.Preproc), '#pop'),
+ include('varnames')
+ ],
+ 'tag': [
+ (r'\s+', Text),
+ (r'(-?)(%\})', bygroups(Text, Comment.Preproc), '#pop'),
+ include('varnames'),
+ (r'.', Punctuation),
+ ],
+ }
+
+
+class TwigHtmlLexer(DelegatingLexer):
+ """
+ Subclass of the `TwigLexer` that highlights unlexed data with the
+ `HtmlLexer`.
+
+ .. versionadded:: 2.0
+ """
+
+ name = "HTML+Twig"
+ aliases = ["html+twig"]
+ filenames = ['*.twig']
+ mimetypes = ['text/html+twig']
+
+ def __init__(self, **options):
+ super(TwigHtmlLexer, self).__init__(HtmlLexer, TwigLexer, **options)
+
+
+class Angular2Lexer(RegexLexer):
+ """
+ Generic
+ `angular2 <http://victorsavkin.com/post/119943127151/angular-2-template-syntax>`_
+ template lexer.
+
+ Highlights only the Angular template tags (stuff between `{{` and `}}` and
+ special attributes: '(event)=', '[property]=', '[(twoWayBinding)]=').
+ Everything else is left for a delegating lexer.
+
+ .. versionadded:: 2.1
+ """
+
+ name = "Angular2"
+ aliases = ['ng2']
+
+ tokens = {
+ 'root': [
+ (r'[^{([*#]+', Other),
+
+ # {{meal.name}}
+ (r'(\{\{)(\s*)', bygroups(Comment.Preproc, Text), 'ngExpression'),
+
+ # (click)="deleteOrder()"; [value]="test"; [(twoWayTest)]="foo.bar"
+ (r'([([]+)([\w:.-]+)([\])]+)(\s*)(=)(\s*)',
+ bygroups(Punctuation, Name.Attribute, Punctuation, Text, Operator, Text),
+ 'attr'),
+ (r'([([]+)([\w:.-]+)([\])]+)(\s*)',
+ bygroups(Punctuation, Name.Attribute, Punctuation, Text)),
+
+ # *ngIf="..."; #f="ngForm"
+ (r'([*#])([\w:.-]+)(\s*)(=)(\s*)',
+ bygroups(Punctuation, Name.Attribute, Punctuation, Operator), 'attr'),
+ (r'([*#])([\w:.-]+)(\s*)',
+ bygroups(Punctuation, Name.Attribute, Punctuation)),
+ ],
+
+ 'ngExpression': [
+ (r'\s+(\|\s+)?', Text),
+ (r'\}\}', Comment.Preproc, '#pop'),
+
+ # Literals
+ (r':?(true|false)', String.Boolean),
+ (r':?"(\\\\|\\"|[^"])*"', String.Double),
+ (r":?'(\\\\|\\'|[^'])*'", String.Single),
+ (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|"
+ r"0[xX][0-9a-fA-F]+[Ll]?", Number),
+
+ # Variabletext
+ (r'[a-zA-Z][\w-]*(\(.*\))?', Name.Variable),
+ (r'\.[\w-]+(\(.*\))?', Name.Variable),
+
+ # inline If
+ (r'(\?)(\s*)([^}\s]+)(\s*)(:)(\s*)([^}\s]+)(\s*)',
+ bygroups(Operator, Text, String, Text, Operator, Text, String, Text)),
+ ],
+ 'attr': [
+ ('".*?"', String, '#pop'),
+ ("'.*?'", String, '#pop'),
+ (r'[^\s>]+', String, '#pop'),
+ ],
+ }
+
+
+class Angular2HtmlLexer(DelegatingLexer):
+ """
+ Subclass of the `Angular2Lexer` that highlights unlexed data with the
+ `HtmlLexer`.
+
+ .. versionadded:: 2.0
+ """
+
+ name = "HTML + Angular2"
+ aliases = ["html+ng2"]
+ filenames = ['*.ng2']
+
+ def __init__(self, **options):
+ super(Angular2HtmlLexer, self).__init__(HtmlLexer, Angular2Lexer, **options)
diff --git a/wandb/vendor/pygments/lexers/testing.py b/wandb/vendor/pygments/lexers/testing.py
new file mode 100644
index 0000000000000000000000000000000000000000..1e0795b1010bbf3f583a1d121cc7eb620c1bf2e9
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/testing.py
@@ -0,0 +1,207 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.testing
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for testing languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups
+from pygments.token import Comment, Keyword, Name, String, Number, Generic, Text
+
+__all__ = ['GherkinLexer', 'TAPLexer']
+
+
+class GherkinLexer(RegexLexer):
+ """
+ For `Gherkin <http://github.com/aslakhellesoy/gherkin/>` syntax.
+
+ .. versionadded:: 1.2
+ """
+ name = 'Gherkin'
+ aliases = ['cucumber', 'gherkin']
+ filenames = ['*.feature']
+ mimetypes = ['text/x-gherkin']
+
+ feature_keywords = u'^(기능|機能|功能|フィーãƒãƒ£|خاصية|×ª×›×•× ×”|Функціонал|ФункционалноÑÑ‚|Функционал|Фича|ОÑобина|МогућноÑÑ‚|Özellik|WÅ‚aÅ›ciwość|TÃnh năng|Trajto|SavybÄ—|Požiadavka|Požadavek|Osobina|Ominaisuus|Omadus|OH HAI|Mogućnost|Mogucnost|JellemzÅ‘|FÄ«Äa|Funzionalità |Funktionalität|Funkcionalnost|FunkcionalitÄte|FuncÈ›ionalitate|Functionaliteit|Functionalitate|Funcionalitat|Funcionalidade|Fonctionnalité|Fitur|Feature|Egenskap|Egenskab|Crikey|CaracterÃstica|Arwedd)(:)(.*)$'
+ feature_element_keywords = u'^(\\s*)(시나리오 개요|시나리오|ë°°ê²½|背景|å ´æ™¯å¤§ç¶±|å ´æ™¯|场景大纲|场景|劇本大綱|劇本|剧本大纲|剧本|テンプレ|シナリオテンプレート|シナリオテンプレ|シナリオアウトライン|シナリオ|سيناريو مخطط|سيناريو|الخلÙية|תרחיש|×ª×‘× ×™×ª תרחיש|רקע|Тарих|Сценарій|Сценарио|Сценарий ÑтруктураÑи|Сценарий|Структура Ñценарію|Структура Ñценарија|Структура ÑценариÑ|Скица|Рамка на Ñценарий|Пример|ПредыÑториÑ|ПредиÑториÑ|Позадина|Передумова|ОÑнова|Концепт|КонтекÑÑ‚|ZaÅ‚ożenia|Wharrimean is|Tình huống|The thing of it is|Tausta|Taust|Tapausaihio|Tapaus|Szenariogrundriss|Szenario|Szablon scenariusza|Stsenaarium|Struktura scenarija|Skica|Skenario konsep|Skenario|SituÄcija|Senaryo taslağı|Senaryo|Scénář|Scénario|Schema dello scenario|ScenÄrijs pÄ“c parauga|ScenÄrijs|Scenár|Scenaro|Scenariusz|Scenariul de ÅŸablon|Scenariul de sablon|Scenariu|Scenario Outline|Scenario Amlinellol|Scenario|Scenarijus|Scenarijaus Å¡ablonas|Scenarij|Scenarie|Rerefons|Raamstsenaarium|Primer|PozadÃ|Pozadina|Pozadie|Plan du scénario|Plan du Scénario|Osnova scénáře|Osnova|NáÄrt Scénáře|NáÄrt Scenáru|Mate|MISHUN SRSLY|MISHUN|Kịch bản|Konturo de la scenaro|Kontext|Konteksts|Kontekstas|Kontekst|Koncept|Khung tình huống|Khung kịch bản|Háttér|Grundlage|GeçmiÅŸ|Forgatókönyv vázlat|Forgatókönyv|Fono|Esquema do Cenário|Esquema do Cenario|Esquema del escenario|Esquema de l\'escenari|Escenario|Escenari|Dis is what went down|Dasar|Contexto|Contexte|Contesto|CondiÅ£ii|Conditii|Cenário|Cenario|Cefndir|Bối cảnh|Blokes|Bakgrunn|Bakgrund|Baggrund|Background|B4|Antecedents|Antecedentes|All y\'all|Achtergrond|Abstrakt Scenario|Abstract Scenario)(:)(.*)$'
+ examples_keywords = u'^(\\s*)(예|例å|例|サンプル|امثلة|דוגמ×ות|Сценарији|Примери|Приклади|МиÑоллар|ЗначениÑ|Örnekler|Voorbeelden|Variantai|Tapaukset|Scenarios|Scenariji|Scenarijai|PÅ™Ãklady|Példák|PrÃklady|PrzykÅ‚ady|Primjeri|Primeri|PiemÄ“ri|Pavyzdžiai|Paraugs|Juhtumid|Exemplos|Exemples|Exemplele|Exempel|Examples|Esempi|Enghreifftiau|Ekzemploj|Eksempler|Ejemplos|EXAMPLZ|Dữ liệu|Contoh|Cobber|Beispiele)(:)(.*)$'
+ step_keywords = u'^(\\s*)(하지만|ì¡°ê±´|ë¨¼ì €|만ì¼|만약|단|ê·¸ë¦¬ê³ |그러면|那麼|那么|而且|當|当|å‰æ|å‡è¨|å‡è®¾|å‡å¦‚|å‡å®š|但是|但ã—|並且|并且|åŒæ™‚|åŒæ—¶|ã‚‚ã—|ãªã‚‰ã°|ãŸã ã—|ã—ã‹ã—|ã‹ã¤|Ùˆ |متى |لكن |عندما |ثم |بÙرض |اذاً |×›×שר |×•×’× |×‘×”×™× ×ª×Ÿ |××–×™ |××– |×בל |Якщо |Унда |То |ПрипуÑтимо, що |ПрипуÑтимо |Онда |Ðо |Ðехай |Лекин |Когато |Када |Кад |К тому же |И |Задато |Задати |Задате |ЕÑли |ДопуÑтим |Дадено |Ва |Бирок |Ðммо |Ðли |Ðле |Ðгар |Ð |І |Și |És |Zatati |ZakÅ‚adajÄ…c |Zadato |Zadate |Zadano |Zadani |Zadan |Youse know when youse got |Youse know like when |Yna |Ya know how |Ya gotta |Y |Wun |Wtedy |When y\'all |When |Wenn |WEN |Và |Ve |Und |Un |Thì |Then y\'all |Then |Tapi |Tak |Tada |Tad |SÃ¥ |Stel |Soit |Siis |Si |Sed |Se |Quando |Quand |Quan |Pryd |Pokud |Pokiaľ |Però |Pero |Pak |Oraz |Onda |Ond |Oletetaan |Og |Och |O zaman |NÃ¥r |När |Niin |NhÆ°ng |N |Mutta |Men |Mas |Maka |Majd |Mais |Maar |Ma |Lorsque |Lorsqu\'|Kun |Kuid |Kui |Khi |KeÄ |Ketika |Když |Kaj |Kai |Kada |Kad |Jeżeli |Ja |Ir |I CAN HAZ |I |Ha |Givun |Givet |Given y\'all |Given |Gitt |Gegeven |Gegeben sei |Fakat |EÄŸer ki |Etant donné |Et |Então |Entonces |Entao |En |Eeldades |E |Duota |Dun |Donitaĵo |Donat |Donada |Do |Diyelim ki |Dengan |Den youse gotta |De |Dato |Dar |Dann |Dan |Dado |Dacă |Daca |DEN |Când |Cuando |Cho |Cept |Cand |Cal |But y\'all |But |Buh |Biết |Bet |BUT |Atès |Atunci |Atesa |Anrhegedig a |Angenommen |And y\'all |And |An |Ama |Als |Alors |Allora |Ali |Aleshores |Ale |Akkor |Aber |AN |A také |A |\* )'
+
+ tokens = {
+ 'comments': [
+ (r'^\s*#.*$', Comment),
+ ],
+ 'feature_elements': [
+ (step_keywords, Keyword, "step_content_stack"),
+ include('comments'),
+ (r"(\s|.)", Name.Function),
+ ],
+ 'feature_elements_on_stack': [
+ (step_keywords, Keyword, "#pop:2"),
+ include('comments'),
+ (r"(\s|.)", Name.Function),
+ ],
+ 'examples_table': [
+ (r"\s+\|", Keyword, 'examples_table_header'),
+ include('comments'),
+ (r"(\s|.)", Name.Function),
+ ],
+ 'examples_table_header': [
+ (r"\s+\|\s*$", Keyword, "#pop:2"),
+ include('comments'),
+ (r"\\\|", Name.Variable),
+ (r"\s*\|", Keyword),
+ (r"[^|]", Name.Variable),
+ ],
+ 'scenario_sections_on_stack': [
+ (feature_element_keywords,
+ bygroups(Name.Function, Keyword, Keyword, Name.Function),
+ "feature_elements_on_stack"),
+ ],
+ 'narrative': [
+ include('scenario_sections_on_stack'),
+ include('comments'),
+ (r"(\s|.)", Name.Function),
+ ],
+ 'table_vars': [
+ (r'(<[^>]+>)', Name.Variable),
+ ],
+ 'numbers': [
+ (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', String),
+ ],
+ 'string': [
+ include('table_vars'),
+ (r'(\s|.)', String),
+ ],
+ 'py_string': [
+ (r'"""', Keyword, "#pop"),
+ include('string'),
+ ],
+ 'step_content_root': [
+ (r"$", Keyword, "#pop"),
+ include('step_content'),
+ ],
+ 'step_content_stack': [
+ (r"$", Keyword, "#pop:2"),
+ include('step_content'),
+ ],
+ 'step_content': [
+ (r'"', Name.Function, "double_string"),
+ include('table_vars'),
+ include('numbers'),
+ include('comments'),
+ (r'(\s|.)', Name.Function),
+ ],
+ 'table_content': [
+ (r"\s+\|\s*$", Keyword, "#pop"),
+ include('comments'),
+ (r"\\\|", String),
+ (r"\s*\|", Keyword),
+ include('string'),
+ ],
+ 'double_string': [
+ (r'"', Name.Function, "#pop"),
+ include('string'),
+ ],
+ 'root': [
+ (r'\n', Name.Function),
+ include('comments'),
+ (r'"""', Keyword, "py_string"),
+ (r'\s+\|', Keyword, 'table_content'),
+ (r'"', Name.Function, "double_string"),
+ include('table_vars'),
+ include('numbers'),
+ (r'(\s*)(@[^@\r\n\t ]+)', bygroups(Name.Function, Name.Tag)),
+ (step_keywords, bygroups(Name.Function, Keyword),
+ 'step_content_root'),
+ (feature_keywords, bygroups(Keyword, Keyword, Name.Function),
+ 'narrative'),
+ (feature_element_keywords,
+ bygroups(Name.Function, Keyword, Keyword, Name.Function),
+ 'feature_elements'),
+ (examples_keywords,
+ bygroups(Name.Function, Keyword, Keyword, Name.Function),
+ 'examples_table'),
+ (r'(\s|.)', Name.Function),
+ ]
+ }
+
+
+class TAPLexer(RegexLexer):
+ """
+ For Test Anything Protocol (TAP) output.
+
+ .. versionadded:: 2.1
+ """
+ name = 'TAP'
+ aliases = ['tap']
+ filenames = ['*.tap']
+
+ tokens = {
+ 'root': [
+ # A TAP version may be specified.
+ (r'^TAP version \d+\n', Name.Namespace),
+
+ # Specify a plan with a plan line.
+ (r'^1\.\.\d+', Keyword.Declaration, 'plan'),
+
+ # A test failure
+ (r'^(not ok)([^\S\n]*)(\d*)',
+ bygroups(Generic.Error, Text, Number.Integer), 'test'),
+
+ # A test success
+ (r'^(ok)([^\S\n]*)(\d*)',
+ bygroups(Keyword.Reserved, Text, Number.Integer), 'test'),
+
+ # Diagnostics start with a hash.
+ (r'^#.*\n', Comment),
+
+ # TAP's version of an abort statement.
+ (r'^Bail out!.*\n', Generic.Error),
+
+ # TAP ignores any unrecognized lines.
+ (r'^.*\n', Text),
+ ],
+ 'plan': [
+ # Consume whitespace (but not newline).
+ (r'[^\S\n]+', Text),
+
+ # A plan may have a directive with it.
+ (r'#', Comment, 'directive'),
+
+ # Or it could just end.
+ (r'\n', Comment, '#pop'),
+
+ # Anything else is wrong.
+ (r'.*\n', Generic.Error, '#pop'),
+ ],
+ 'test': [
+ # Consume whitespace (but not newline).
+ (r'[^\S\n]+', Text),
+
+ # A test may have a directive with it.
+ (r'#', Comment, 'directive'),
+
+ (r'\S+', Text),
+
+ (r'\n', Text, '#pop'),
+ ],
+ 'directive': [
+ # Consume whitespace (but not newline).
+ (r'[^\S\n]+', Comment),
+
+ # Extract todo items.
+ (r'(?i)\bTODO\b', Comment.Preproc),
+
+ # Extract skip items.
+ (r'(?i)\bSKIP\S*', Comment.Preproc),
+
+ (r'\S+', Comment),
+
+ (r'\n', Comment, '#pop:2'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/text.py b/wandb/vendor/pygments/lexers/text.py
new file mode 100644
index 0000000000000000000000000000000000000000..9b3b5feacb21e37732ffe78892c644b5ee02e7e6
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/text.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.text
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for non-source code file types.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexers.configs import ApacheConfLexer, NginxConfLexer, \
+ SquidConfLexer, LighttpdConfLexer, IniLexer, RegeditLexer, PropertiesLexer
+from pygments.lexers.console import PyPyLogLexer
+from pygments.lexers.textedit import VimLexer
+from pygments.lexers.markup import BBCodeLexer, MoinWikiLexer, RstLexer, \
+ TexLexer, GroffLexer
+from pygments.lexers.installers import DebianControlLexer, SourcesListLexer
+from pygments.lexers.make import MakefileLexer, BaseMakefileLexer, CMakeLexer
+from pygments.lexers.haxe import HxmlLexer
+from pygments.lexers.diff import DiffLexer, DarcsPatchLexer
+from pygments.lexers.data import YamlLexer
+from pygments.lexers.textfmts import IrcLogsLexer, GettextLexer, HttpLexer
+
+__all__ = []
diff --git a/wandb/vendor/pygments/lexers/textedit.py b/wandb/vendor/pygments/lexers/textedit.py
new file mode 100644
index 0000000000000000000000000000000000000000..e8856dbde756735c1ba959f63b5bd683c9386cd2
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/textedit.py
@@ -0,0 +1,169 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.textedit
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for languages related to text processing.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+from bisect import bisect
+
+from pygments.lexer import RegexLexer, include, default, bygroups, using, this
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+from pygments.lexers.python import PythonLexer
+
+__all__ = ['AwkLexer', 'VimLexer']
+
+
+class AwkLexer(RegexLexer):
+ """
+ For Awk scripts.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'Awk'
+ aliases = ['awk', 'gawk', 'mawk', 'nawk']
+ filenames = ['*.awk']
+ mimetypes = ['application/x-awk']
+
+ tokens = {
+ 'commentsandwhitespace': [
+ (r'\s+', Text),
+ (r'#.*$', Comment.Single)
+ ],
+ 'slashstartsregex': [
+ include('commentsandwhitespace'),
+ (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+ r'\B', String.Regex, '#pop'),
+ (r'(?=/)', Text, ('#pop', 'badregex')),
+ default('#pop')
+ ],
+ 'badregex': [
+ (r'\n', Text, '#pop')
+ ],
+ 'root': [
+ (r'^(?=\s|/)', Text, 'slashstartsregex'),
+ include('commentsandwhitespace'),
+ (r'\+\+|--|\|\||&&|in\b|\$|!?~|'
+ r'(\*\*|[-<>+*%\^/!=|])=?', Operator, 'slashstartsregex'),
+ (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
+ (r'[})\].]', Punctuation),
+ (r'(break|continue|do|while|exit|for|if|else|'
+ r'return)\b', Keyword, 'slashstartsregex'),
+ (r'function\b', Keyword.Declaration, 'slashstartsregex'),
+ (r'(atan2|cos|exp|int|log|rand|sin|sqrt|srand|gensub|gsub|index|'
+ r'length|match|split|sprintf|sub|substr|tolower|toupper|close|'
+ r'fflush|getline|next|nextfile|print|printf|strftime|systime|'
+ r'delete|system)\b', Keyword.Reserved),
+ (r'(ARGC|ARGIND|ARGV|BEGIN|CONVFMT|ENVIRON|END|ERRNO|FIELDWIDTHS|'
+ r'FILENAME|FNR|FS|IGNORECASE|NF|NR|OFMT|OFS|ORFS|RLENGTH|RS|'
+ r'RSTART|RT|SUBSEP)\b', Name.Builtin),
+ (r'[$a-zA-Z_]\w*', Name.Other),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+', Number.Integer),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ ]
+ }
+
+
+class VimLexer(RegexLexer):
+ """
+ Lexer for VimL script files.
+
+ .. versionadded:: 0.8
+ """
+ name = 'VimL'
+ aliases = ['vim']
+ filenames = ['*.vim', '.vimrc', '.exrc', '.gvimrc',
+ '_vimrc', '_exrc', '_gvimrc', 'vimrc', 'gvimrc']
+ mimetypes = ['text/x-vim']
+ flags = re.MULTILINE
+
+ _python = r'py(?:t(?:h(?:o(?:n)?)?)?)?'
+
+ tokens = {
+ 'root': [
+ (r'^([ \t:]*)(' + _python + r')([ \t]*)(<<)([ \t]*)(.*)((?:\n|.)*)(\6)',
+ bygroups(using(this), Keyword, Text, Operator, Text, Text,
+ using(PythonLexer), Text)),
+ (r'^([ \t:]*)(' + _python + r')([ \t])(.*)',
+ bygroups(using(this), Keyword, Text, using(PythonLexer))),
+
+ (r'^\s*".*', Comment),
+
+ (r'[ \t]+', Text),
+ # TODO: regexes can have other delims
+ (r'/(\\\\|\\/|[^\n/])*/', String.Regex),
+ (r'"(\\\\|\\"|[^\n"])*"', String.Double),
+ (r"'(''|[^\n'])*'", String.Single),
+
+ # Who decided that doublequote was a good comment character??
+ (r'(?<=\s)"[^\-:.%#=*].*', Comment),
+ (r'-?\d+', Number),
+ (r'#[0-9a-f]{6}', Number.Hex),
+ (r'^:', Punctuation),
+ (r'[()<>+=!|,~-]', Punctuation), # Inexact list. Looks decent.
+ (r'\b(let|if|else|endif|elseif|fun|function|endfunction)\b',
+ Keyword),
+ (r'\b(NONE|bold|italic|underline|dark|light)\b', Name.Builtin),
+ (r'\b\w+\b', Name.Other), # These are postprocessed below
+ (r'.', Text),
+ ],
+ }
+
+ def __init__(self, **options):
+ from pygments.lexers._vim_builtins import command, option, auto
+ self._cmd = command
+ self._opt = option
+ self._aut = auto
+
+ RegexLexer.__init__(self, **options)
+
+ def is_in(self, w, mapping):
+ r"""
+ It's kind of difficult to decide if something might be a keyword
+ in VimL because it allows you to abbreviate them. In fact,
+ 'ab[breviate]' is a good example. :ab, :abbre, or :abbreviate are
+ valid ways to call it so rather than making really awful regexps
+ like::
+
+ \bab(?:b(?:r(?:e(?:v(?:i(?:a(?:t(?:e)?)?)?)?)?)?)?)?\b
+
+ we match `\b\w+\b` and then call is_in() on those tokens. See
+ `scripts/get_vimkw.py` for how the lists are extracted.
+ """
+ p = bisect(mapping, (w,))
+ if p > 0:
+ if mapping[p-1][0] == w[:len(mapping[p-1][0])] and \
+ mapping[p-1][1][:len(w)] == w:
+ return True
+ if p < len(mapping):
+ return mapping[p][0] == w[:len(mapping[p][0])] and \
+ mapping[p][1][:len(w)] == w
+ return False
+
+ def get_tokens_unprocessed(self, text):
+ # TODO: builtins are only subsequent tokens on lines
+ # and 'keywords' only happen at the beginning except
+ # for :au ones
+ for index, token, value in \
+ RegexLexer.get_tokens_unprocessed(self, text):
+ if token is Name.Other:
+ if self.is_in(value, self._cmd):
+ yield index, Keyword, value
+ elif self.is_in(value, self._opt) or \
+ self.is_in(value, self._aut):
+ yield index, Name.Builtin, value
+ else:
+ yield index, Text, value
+ else:
+ yield index, token, value
diff --git a/wandb/vendor/pygments/lexers/textfmts.py b/wandb/vendor/pygments/lexers/textfmts.py
new file mode 100644
index 0000000000000000000000000000000000000000..bb8124ef953c8d4fdd7a59e9bf29aec804340f05
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/textfmts.py
@@ -0,0 +1,297 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.textfmts
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for various text formats.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Generic, Literal
+from pygments.util import ClassNotFound
+
+__all__ = ['IrcLogsLexer', 'TodotxtLexer', 'HttpLexer', 'GettextLexer']
+
+
+class IrcLogsLexer(RegexLexer):
+ """
+ Lexer for IRC logs in *irssi*, *xchat* or *weechat* style.
+ """
+
+ name = 'IRC logs'
+ aliases = ['irc']
+ filenames = ['*.weechatlog']
+ mimetypes = ['text/x-irclog']
+
+ flags = re.VERBOSE | re.MULTILINE
+ timestamp = r"""
+ (
+ # irssi / xchat and others
+ (?: \[|\()? # Opening bracket or paren for the timestamp
+ (?: # Timestamp
+ (?: (?:\d{1,4} [-/])* # Date as - or /-separated groups of digits
+ (?:\d{1,4})
+ [T ])? # Date/time separator: T or space
+ (?: \d?\d [:.])* # Time as :/.-separated groups of 1 or 2 digits
+ (?: \d?\d)
+ )
+ (?: \]|\))?\s+ # Closing bracket or paren for the timestamp
+ |
+ # weechat
+ \d{4}\s\w{3}\s\d{2}\s # Date
+ \d{2}:\d{2}:\d{2}\s+ # Time + Whitespace
+ |
+ # xchat
+ \w{3}\s\d{2}\s # Date
+ \d{2}:\d{2}:\d{2}\s+ # Time + Whitespace
+ )?
+ """
+ tokens = {
+ 'root': [
+ # log start/end
+ (r'^\*\*\*\*(.*)\*\*\*\*$', Comment),
+ # hack
+ ("^" + timestamp + r'(\s*<[^>]*>\s*)$', bygroups(Comment.Preproc, Name.Tag)),
+ # normal msgs
+ ("^" + timestamp + r"""
+ (\s*<.*?>\s*) # Nick """,
+ bygroups(Comment.Preproc, Name.Tag), 'msg'),
+ # /me msgs
+ ("^" + timestamp + r"""
+ (\s*[*]\s+) # Star
+ (\S+\s+.*?\n) # Nick + rest of message """,
+ bygroups(Comment.Preproc, Keyword, Generic.Inserted)),
+ # join/part msgs
+ ("^" + timestamp + r"""
+ (\s*(?:\*{3}|<?-[!@=P]?->?)\s*) # Star(s) or symbols
+ (\S+\s+) # Nick + Space
+ (.*?\n) # Rest of message """,
+ bygroups(Comment.Preproc, Keyword, String, Comment)),
+ (r"^.*?\n", Text),
+ ],
+ 'msg': [
+ (r"\S+:(?!//)", Name.Attribute), # Prefix
+ (r".*\n", Text, '#pop'),
+ ],
+ }
+
+
+class GettextLexer(RegexLexer):
+ """
+ Lexer for Gettext catalog files.
+
+ .. versionadded:: 0.9
+ """
+ name = 'Gettext Catalog'
+ aliases = ['pot', 'po']
+ filenames = ['*.pot', '*.po']
+ mimetypes = ['application/x-gettext', 'text/x-gettext', 'text/gettext']
+
+ tokens = {
+ 'root': [
+ (r'^#,\s.*?$', Keyword.Type),
+ (r'^#:\s.*?$', Keyword.Declaration),
+ # (r'^#$', Comment),
+ (r'^(#|#\.\s|#\|\s|#~\s|#\s).*$', Comment.Single),
+ (r'^(")([A-Za-z-]+:)(.*")$',
+ bygroups(String, Name.Property, String)),
+ (r'^".*"$', String),
+ (r'^(msgid|msgid_plural|msgstr|msgctxt)(\s+)(".*")$',
+ bygroups(Name.Variable, Text, String)),
+ (r'^(msgstr\[)(\d)(\])(\s+)(".*")$',
+ bygroups(Name.Variable, Number.Integer, Name.Variable, Text, String)),
+ ]
+ }
+
+
+class HttpLexer(RegexLexer):
+ """
+ Lexer for HTTP sessions.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'HTTP'
+ aliases = ['http']
+
+ flags = re.DOTALL
+
+ def get_tokens_unprocessed(self, text, stack=('root',)):
+ """Reset the content-type state."""
+ self.content_type = None
+ return RegexLexer.get_tokens_unprocessed(self, text, stack)
+
+ def header_callback(self, match):
+ if match.group(1).lower() == 'content-type':
+ content_type = match.group(5).strip()
+ if ';' in content_type:
+ content_type = content_type[:content_type.find(';')].strip()
+ self.content_type = content_type
+ yield match.start(1), Name.Attribute, match.group(1)
+ yield match.start(2), Text, match.group(2)
+ yield match.start(3), Operator, match.group(3)
+ yield match.start(4), Text, match.group(4)
+ yield match.start(5), Literal, match.group(5)
+ yield match.start(6), Text, match.group(6)
+
+ def continuous_header_callback(self, match):
+ yield match.start(1), Text, match.group(1)
+ yield match.start(2), Literal, match.group(2)
+ yield match.start(3), Text, match.group(3)
+
+ def content_callback(self, match):
+ content_type = getattr(self, 'content_type', None)
+ content = match.group()
+ offset = match.start()
+ if content_type:
+ from pygments.lexers import get_lexer_for_mimetype
+ possible_lexer_mimetypes = [content_type]
+ if '+' in content_type:
+ # application/calendar+xml can be treated as application/xml
+ # if there's not a better match.
+ general_type = re.sub(r'^(.*)/.*\+(.*)$', r'\1/\2',
+ content_type)
+ possible_lexer_mimetypes.append(general_type)
+
+ for i in possible_lexer_mimetypes:
+ try:
+ lexer = get_lexer_for_mimetype(i)
+ except ClassNotFound:
+ pass
+ else:
+ for idx, token, value in lexer.get_tokens_unprocessed(content):
+ yield offset + idx, token, value
+ return
+ yield offset, Text, content
+
+ tokens = {
+ 'root': [
+ (r'(GET|POST|PUT|DELETE|HEAD|OPTIONS|TRACE|PATCH)( +)([^ ]+)( +)'
+ r'(HTTP)(/)(1\.[01])(\r?\n|\Z)',
+ bygroups(Name.Function, Text, Name.Namespace, Text,
+ Keyword.Reserved, Operator, Number, Text),
+ 'headers'),
+ (r'(HTTP)(/)(1\.[01])( +)(\d{3})( +)([^\r\n]+)(\r?\n|\Z)',
+ bygroups(Keyword.Reserved, Operator, Number, Text, Number,
+ Text, Name.Exception, Text),
+ 'headers'),
+ ],
+ 'headers': [
+ (r'([^\s:]+)( *)(:)( *)([^\r\n]+)(\r?\n|\Z)', header_callback),
+ (r'([\t ]+)([^\r\n]+)(\r?\n|\Z)', continuous_header_callback),
+ (r'\r?\n', Text, 'content')
+ ],
+ 'content': [
+ (r'.+', content_callback)
+ ]
+ }
+
+ def analyse_text(text):
+ return text.startswith(('GET /', 'POST /', 'PUT /', 'DELETE /', 'HEAD /',
+ 'OPTIONS /', 'TRACE /', 'PATCH /'))
+
+
+class TodotxtLexer(RegexLexer):
+ """
+ Lexer for `Todo.txt <http://todotxt.com/>`_ todo list format.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Todotxt'
+ aliases = ['todotxt']
+ # *.todotxt is not a standard extension for Todo.txt files; including it
+ # makes testing easier, and also makes autodetecting file type easier.
+ filenames = ['todo.txt', '*.todotxt']
+ mimetypes = ['text/x-todo']
+
+ # Aliases mapping standard token types of Todo.txt format concepts
+ CompleteTaskText = Operator # Chosen to de-emphasize complete tasks
+ IncompleteTaskText = Text # Incomplete tasks should look like plain text
+
+ # Priority should have most emphasis to indicate importance of tasks
+ Priority = Generic.Heading
+ # Dates should have next most emphasis because time is important
+ Date = Generic.Subheading
+
+ # Project and context should have equal weight, and be in different colors
+ Project = Generic.Error
+ Context = String
+
+ # If tag functionality is added, it should have the same weight as Project
+ # and Context, and a different color. Generic.Traceback would work well.
+
+ # Regex patterns for building up rules; dates, priorities, projects, and
+ # contexts are all atomic
+ # TODO: Make date regex more ISO 8601 compliant
+ date_regex = r'\d{4,}-\d{2}-\d{2}'
+ priority_regex = r'\([A-Z]\)'
+ project_regex = r'\+\S+'
+ context_regex = r'@\S+'
+
+ # Compound regex expressions
+ complete_one_date_regex = r'(x )(' + date_regex + r')'
+ complete_two_date_regex = (complete_one_date_regex + r'( )(' +
+ date_regex + r')')
+ priority_date_regex = r'(' + priority_regex + r')( )(' + date_regex + r')'
+
+ tokens = {
+ # Should parse starting at beginning of line; each line is a task
+ 'root': [
+ # Complete task entry points: two total:
+ # 1. Complete task with two dates
+ (complete_two_date_regex, bygroups(CompleteTaskText, Date,
+ CompleteTaskText, Date),
+ 'complete'),
+ # 2. Complete task with one date
+ (complete_one_date_regex, bygroups(CompleteTaskText, Date),
+ 'complete'),
+
+ # Incomplete task entry points: six total:
+ # 1. Priority plus date
+ (priority_date_regex, bygroups(Priority, IncompleteTaskText, Date),
+ 'incomplete'),
+ # 2. Priority only
+ (priority_regex, Priority, 'incomplete'),
+ # 3. Leading date
+ (date_regex, Date, 'incomplete'),
+ # 4. Leading context
+ (context_regex, Context, 'incomplete'),
+ # 5. Leading project
+ (project_regex, Project, 'incomplete'),
+ # 6. Non-whitespace catch-all
+ ('\S+', IncompleteTaskText, 'incomplete'),
+ ],
+
+ # Parse a complete task
+ 'complete': [
+ # Newline indicates end of task, should return to root
+ (r'\s*\n', CompleteTaskText, '#pop'),
+ # Tokenize contexts and projects
+ (context_regex, Context),
+ (project_regex, Project),
+ # Tokenize non-whitespace text
+ ('\S+', CompleteTaskText),
+ # Tokenize whitespace not containing a newline
+ ('\s+', CompleteTaskText),
+ ],
+
+ # Parse an incomplete task
+ 'incomplete': [
+ # Newline indicates end of task, should return to root
+ (r'\s*\n', IncompleteTaskText, '#pop'),
+ # Tokenize contexts and projects
+ (context_regex, Context),
+ (project_regex, Project),
+ # Tokenize non-whitespace text
+ ('\S+', IncompleteTaskText),
+ # Tokenize whitespace not containing a newline
+ ('\s+', IncompleteTaskText),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/theorem.py b/wandb/vendor/pygments/lexers/theorem.py
new file mode 100644
index 0000000000000000000000000000000000000000..e84a398bba420334a9cd96d2acb0bbc9b51b69fa
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/theorem.py
@@ -0,0 +1,458 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.theorem
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for theorem-proving languages.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, default, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Generic
+
+__all__ = ['CoqLexer', 'IsabelleLexer', 'LeanLexer']
+
+
+class CoqLexer(RegexLexer):
+ """
+ For the `Coq <http://coq.inria.fr/>`_ theorem prover.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'Coq'
+ aliases = ['coq']
+ filenames = ['*.v']
+ mimetypes = ['text/x-coq']
+
+ keywords1 = (
+ # Vernacular commands
+ 'Section', 'Module', 'End', 'Require', 'Import', 'Export', 'Variable',
+ 'Variables', 'Parameter', 'Parameters', 'Axiom', 'Hypothesis',
+ 'Hypotheses', 'Notation', 'Local', 'Tactic', 'Reserved', 'Scope',
+ 'Open', 'Close', 'Bind', 'Delimit', 'Definition', 'Let', 'Ltac',
+ 'Fixpoint', 'CoFixpoint', 'Morphism', 'Relation', 'Implicit',
+ 'Arguments', 'Set', 'Unset', 'Contextual', 'Strict', 'Prenex',
+ 'Implicits', 'Inductive', 'CoInductive', 'Record', 'Structure',
+ 'Canonical', 'Coercion', 'Theorem', 'Lemma', 'Corollary',
+ 'Proposition', 'Fact', 'Remark', 'Example', 'Proof', 'Goal', 'Save',
+ 'Qed', 'Defined', 'Hint', 'Resolve', 'Rewrite', 'View', 'Search',
+ 'Show', 'Print', 'Printing', 'All', 'Graph', 'Projections', 'inside',
+ 'outside', 'Check', 'Global', 'Instance', 'Class', 'Existing',
+ 'Universe', 'Polymorphic', 'Monomorphic', 'Context'
+ )
+ keywords2 = (
+ # Gallina
+ 'forall', 'exists', 'exists2', 'fun', 'fix', 'cofix', 'struct',
+ 'match', 'end', 'in', 'return', 'let', 'if', 'is', 'then', 'else',
+ 'for', 'of', 'nosimpl', 'with', 'as',
+ )
+ keywords3 = (
+ # Sorts
+ 'Type', 'Prop',
+ )
+ keywords4 = (
+ # Tactics
+ 'pose', 'set', 'move', 'case', 'elim', 'apply', 'clear', 'hnf', 'intro',
+ 'intros', 'generalize', 'rename', 'pattern', 'after', 'destruct',
+ 'induction', 'using', 'refine', 'inversion', 'injection', 'rewrite',
+ 'congr', 'unlock', 'compute', 'ring', 'field', 'replace', 'fold',
+ 'unfold', 'change', 'cutrewrite', 'simpl', 'have', 'suff', 'wlog',
+ 'suffices', 'without', 'loss', 'nat_norm', 'assert', 'cut', 'trivial',
+ 'revert', 'bool_congr', 'nat_congr', 'symmetry', 'transitivity', 'auto',
+ 'split', 'left', 'right', 'autorewrite', 'tauto', 'setoid_rewrite',
+ 'intuition', 'eauto', 'eapply', 'econstructor', 'etransitivity',
+ 'constructor', 'erewrite', 'red', 'cbv', 'lazy', 'vm_compute',
+ 'native_compute', 'subst',
+ )
+ keywords5 = (
+ # Terminators
+ 'by', 'done', 'exact', 'reflexivity', 'tauto', 'romega', 'omega',
+ 'assumption', 'solve', 'contradiction', 'discriminate',
+ 'congruence',
+ )
+ keywords6 = (
+ # Control
+ 'do', 'last', 'first', 'try', 'idtac', 'repeat',
+ )
+ # 'as', 'assert', 'begin', 'class', 'constraint', 'do', 'done',
+ # 'downto', 'else', 'end', 'exception', 'external', 'false',
+ # 'for', 'fun', 'function', 'functor', 'if', 'in', 'include',
+ # 'inherit', 'initializer', 'lazy', 'let', 'match', 'method',
+ # 'module', 'mutable', 'new', 'object', 'of', 'open', 'private',
+ # 'raise', 'rec', 'sig', 'struct', 'then', 'to', 'true', 'try',
+ # 'type', 'val', 'virtual', 'when', 'while', 'with'
+ keyopts = (
+ '!=', '#', '&', '&&', r'\(', r'\)', r'\*', r'\+', ',', '-', r'-\.',
+ '->', r'\.', r'\.\.', ':', '::', ':=', ':>', ';', ';;', '<', '<-',
+ '<->', '=', '>', '>]', r'>\}', r'\?', r'\?\?', r'\[', r'\[<', r'\[>',
+ r'\[\|', ']', '_', '`', r'\{', r'\{<', r'\|', r'\|]', r'\}', '~', '=>',
+ r'/\\', r'\\/', r'\{\|', r'\|\}',
+ u'Π', u'λ',
+ )
+ operators = r'[!$%&*+\./:<=>?@^|~-]'
+ prefix_syms = r'[!?~]'
+ infix_syms = r'[=<>@^|&+\*/$%-]'
+ primitives = ('unit', 'nat', 'bool', 'string', 'ascii', 'list')
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'false|true|\(\)|\[\]', Name.Builtin.Pseudo),
+ (r'\(\*', Comment, 'comment'),
+ (words(keywords1, prefix=r'\b', suffix=r'\b'), Keyword.Namespace),
+ (words(keywords2, prefix=r'\b', suffix=r'\b'), Keyword),
+ (words(keywords3, prefix=r'\b', suffix=r'\b'), Keyword.Type),
+ (words(keywords4, prefix=r'\b', suffix=r'\b'), Keyword),
+ (words(keywords5, prefix=r'\b', suffix=r'\b'), Keyword.Pseudo),
+ (words(keywords6, prefix=r'\b', suffix=r'\b'), Keyword.Reserved),
+ # (r'\b([A-Z][\w\']*)(\.)', Name.Namespace, 'dotted'),
+ (r'\b([A-Z][\w\']*)', Name),
+ (r'(%s)' % '|'.join(keyopts[::-1]), Operator),
+ (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator),
+ (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type),
+
+ (r"[^\W\d][\w']*", Name),
+
+ (r'\d[\d_]*', Number.Integer),
+ (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
+ (r'0[oO][0-7][0-7_]*', Number.Oct),
+ (r'0[bB][01][01_]*', Number.Bin),
+ (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float),
+
+ (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
+ String.Char),
+ (r"'.'", String.Char),
+ (r"'", Keyword), # a stray quote is another syntax element
+
+ (r'"', String.Double, 'string'),
+
+ (r'[~?][a-z][\w\']*:', Name),
+ ],
+ 'comment': [
+ (r'[^(*)]+', Comment),
+ (r'\(\*', Comment, '#push'),
+ (r'\*\)', Comment, '#pop'),
+ (r'[(*)]', Comment),
+ ],
+ 'string': [
+ (r'[^"]+', String.Double),
+ (r'""', String.Double),
+ (r'"', String.Double, '#pop'),
+ ],
+ 'dotted': [
+ (r'\s+', Text),
+ (r'\.', Punctuation),
+ (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace),
+ (r'[A-Z][\w\']*', Name.Class, '#pop'),
+ (r'[a-z][a-z0-9_\']*', Name, '#pop'),
+ default('#pop')
+ ],
+ }
+
+ def analyse_text(text):
+ if text.startswith('(*'):
+ return True
+
+
+class IsabelleLexer(RegexLexer):
+ """
+ For the `Isabelle <http://isabelle.in.tum.de/>`_ proof assistant.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Isabelle'
+ aliases = ['isabelle']
+ filenames = ['*.thy']
+ mimetypes = ['text/x-isabelle']
+
+ keyword_minor = (
+ 'and', 'assumes', 'attach', 'avoids', 'binder', 'checking',
+ 'class_instance', 'class_relation', 'code_module', 'congs',
+ 'constant', 'constrains', 'datatypes', 'defines', 'file', 'fixes',
+ 'for', 'functions', 'hints', 'identifier', 'if', 'imports', 'in',
+ 'includes', 'infix', 'infixl', 'infixr', 'is', 'keywords', 'lazy',
+ 'module_name', 'monos', 'morphisms', 'no_discs_sels', 'notes',
+ 'obtains', 'open', 'output', 'overloaded', 'parametric', 'permissive',
+ 'pervasive', 'rep_compat', 'shows', 'structure', 'type_class',
+ 'type_constructor', 'unchecked', 'unsafe', 'where',
+ )
+
+ keyword_diag = (
+ 'ML_command', 'ML_val', 'class_deps', 'code_deps', 'code_thms',
+ 'display_drafts', 'find_consts', 'find_theorems', 'find_unused_assms',
+ 'full_prf', 'help', 'locale_deps', 'nitpick', 'pr', 'prf',
+ 'print_abbrevs', 'print_antiquotations', 'print_attributes',
+ 'print_binds', 'print_bnfs', 'print_bundles',
+ 'print_case_translations', 'print_cases', 'print_claset',
+ 'print_classes', 'print_codeproc', 'print_codesetup',
+ 'print_coercions', 'print_commands', 'print_context',
+ 'print_defn_rules', 'print_dependencies', 'print_facts',
+ 'print_induct_rules', 'print_inductives', 'print_interps',
+ 'print_locale', 'print_locales', 'print_methods', 'print_options',
+ 'print_orders', 'print_quot_maps', 'print_quotconsts',
+ 'print_quotients', 'print_quotientsQ3', 'print_quotmapsQ3',
+ 'print_rules', 'print_simpset', 'print_state', 'print_statement',
+ 'print_syntax', 'print_theorems', 'print_theory', 'print_trans_rules',
+ 'prop', 'pwd', 'quickcheck', 'refute', 'sledgehammer', 'smt_status',
+ 'solve_direct', 'spark_status', 'term', 'thm', 'thm_deps', 'thy_deps',
+ 'try', 'try0', 'typ', 'unused_thms', 'value', 'values', 'welcome',
+ 'print_ML_antiquotations', 'print_term_bindings', 'values_prolog',
+ )
+
+ keyword_thy = ('theory', 'begin', 'end')
+
+ keyword_section = ('header', 'chapter')
+
+ keyword_subsection = (
+ 'section', 'subsection', 'subsubsection', 'sect', 'subsect',
+ 'subsubsect',
+ )
+
+ keyword_theory_decl = (
+ 'ML', 'ML_file', 'abbreviation', 'adhoc_overloading', 'arities',
+ 'atom_decl', 'attribute_setup', 'axiomatization', 'bundle',
+ 'case_of_simps', 'class', 'classes', 'classrel', 'codatatype',
+ 'code_abort', 'code_class', 'code_const', 'code_datatype',
+ 'code_identifier', 'code_include', 'code_instance', 'code_modulename',
+ 'code_monad', 'code_printing', 'code_reflect', 'code_reserved',
+ 'code_type', 'coinductive', 'coinductive_set', 'consts', 'context',
+ 'datatype', 'datatype_new', 'datatype_new_compat', 'declaration',
+ 'declare', 'default_sort', 'defer_recdef', 'definition', 'defs',
+ 'domain', 'domain_isomorphism', 'domaindef', 'equivariance',
+ 'export_code', 'extract', 'extract_type', 'fixrec', 'fun',
+ 'fun_cases', 'hide_class', 'hide_const', 'hide_fact', 'hide_type',
+ 'import_const_map', 'import_file', 'import_tptp', 'import_type_map',
+ 'inductive', 'inductive_set', 'instantiation', 'judgment', 'lemmas',
+ 'lifting_forget', 'lifting_update', 'local_setup', 'locale',
+ 'method_setup', 'nitpick_params', 'no_adhoc_overloading',
+ 'no_notation', 'no_syntax', 'no_translations', 'no_type_notation',
+ 'nominal_datatype', 'nonterminal', 'notation', 'notepad', 'oracle',
+ 'overloading', 'parse_ast_translation', 'parse_translation',
+ 'partial_function', 'primcorec', 'primrec', 'primrec_new',
+ 'print_ast_translation', 'print_translation', 'quickcheck_generator',
+ 'quickcheck_params', 'realizability', 'realizers', 'recdef', 'record',
+ 'refute_params', 'setup', 'setup_lifting', 'simproc_setup',
+ 'simps_of_case', 'sledgehammer_params', 'spark_end', 'spark_open',
+ 'spark_open_siv', 'spark_open_vcg', 'spark_proof_functions',
+ 'spark_types', 'statespace', 'syntax', 'syntax_declaration', 'text',
+ 'text_raw', 'theorems', 'translations', 'type_notation',
+ 'type_synonym', 'typed_print_translation', 'typedecl', 'hoarestate',
+ 'install_C_file', 'install_C_types', 'wpc_setup', 'c_defs', 'c_types',
+ 'memsafe', 'SML_export', 'SML_file', 'SML_import', 'approximate',
+ 'bnf_axiomatization', 'cartouche', 'datatype_compat',
+ 'free_constructors', 'functor', 'nominal_function',
+ 'nominal_termination', 'permanent_interpretation',
+ 'binds', 'defining', 'smt2_status', 'term_cartouche',
+ 'boogie_file', 'text_cartouche',
+ )
+
+ keyword_theory_script = ('inductive_cases', 'inductive_simps')
+
+ keyword_theory_goal = (
+ 'ax_specification', 'bnf', 'code_pred', 'corollary', 'cpodef',
+ 'crunch', 'crunch_ignore',
+ 'enriched_type', 'function', 'instance', 'interpretation', 'lemma',
+ 'lift_definition', 'nominal_inductive', 'nominal_inductive2',
+ 'nominal_primrec', 'pcpodef', 'primcorecursive',
+ 'quotient_definition', 'quotient_type', 'recdef_tc', 'rep_datatype',
+ 'schematic_corollary', 'schematic_lemma', 'schematic_theorem',
+ 'spark_vc', 'specification', 'subclass', 'sublocale', 'termination',
+ 'theorem', 'typedef', 'wrap_free_constructors',
+ )
+
+ keyword_qed = ('by', 'done', 'qed')
+ keyword_abandon_proof = ('sorry', 'oops')
+
+ keyword_proof_goal = ('have', 'hence', 'interpret')
+
+ keyword_proof_block = ('next', 'proof')
+
+ keyword_proof_chain = (
+ 'finally', 'from', 'then', 'ultimately', 'with',
+ )
+
+ keyword_proof_decl = (
+ 'ML_prf', 'also', 'include', 'including', 'let', 'moreover', 'note',
+ 'txt', 'txt_raw', 'unfolding', 'using', 'write',
+ )
+
+ keyword_proof_asm = ('assume', 'case', 'def', 'fix', 'presume')
+
+ keyword_proof_asm_goal = ('guess', 'obtain', 'show', 'thus')
+
+ keyword_proof_script = (
+ 'apply', 'apply_end', 'apply_trace', 'back', 'defer', 'prefer',
+ )
+
+ operators = (
+ '::', ':', '(', ')', '[', ']', '_', '=', ',', '|',
+ '+', '-', '!', '?',
+ )
+
+ proof_operators = ('{', '}', '.', '..')
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'\(\*', Comment, 'comment'),
+ (r'\{\*', Comment, 'text'),
+
+ (words(operators), Operator),
+ (words(proof_operators), Operator.Word),
+
+ (words(keyword_minor, prefix=r'\b', suffix=r'\b'), Keyword.Pseudo),
+
+ (words(keyword_diag, prefix=r'\b', suffix=r'\b'), Keyword.Type),
+
+ (words(keyword_thy, prefix=r'\b', suffix=r'\b'), Keyword),
+ (words(keyword_theory_decl, prefix=r'\b', suffix=r'\b'), Keyword),
+
+ (words(keyword_section, prefix=r'\b', suffix=r'\b'), Generic.Heading),
+ (words(keyword_subsection, prefix=r'\b', suffix=r'\b'), Generic.Subheading),
+
+ (words(keyword_theory_goal, prefix=r'\b', suffix=r'\b'), Keyword.Namespace),
+ (words(keyword_theory_script, prefix=r'\b', suffix=r'\b'), Keyword.Namespace),
+
+ (words(keyword_abandon_proof, prefix=r'\b', suffix=r'\b'), Generic.Error),
+
+ (words(keyword_qed, prefix=r'\b', suffix=r'\b'), Keyword),
+ (words(keyword_proof_goal, prefix=r'\b', suffix=r'\b'), Keyword),
+ (words(keyword_proof_block, prefix=r'\b', suffix=r'\b'), Keyword),
+ (words(keyword_proof_decl, prefix=r'\b', suffix=r'\b'), Keyword),
+
+ (words(keyword_proof_chain, prefix=r'\b', suffix=r'\b'), Keyword),
+ (words(keyword_proof_asm, prefix=r'\b', suffix=r'\b'), Keyword),
+ (words(keyword_proof_asm_goal, prefix=r'\b', suffix=r'\b'), Keyword),
+
+ (words(keyword_proof_script, prefix=r'\b', suffix=r'\b'), Keyword.Pseudo),
+
+ (r'\\<\w*>', Text.Symbol),
+
+ (r"[^\W\d][.\w']*", Name),
+ (r"\?[^\W\d][.\w']*", Name),
+ (r"'[^\W\d][.\w']*", Name.Type),
+
+ (r'\d[\d_]*', Name), # display numbers as name
+ (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
+ (r'0[oO][0-7][0-7_]*', Number.Oct),
+ (r'0[bB][01][01_]*', Number.Bin),
+
+ (r'"', String, 'string'),
+ (r'`', String.Other, 'fact'),
+ ],
+ 'comment': [
+ (r'[^(*)]+', Comment),
+ (r'\(\*', Comment, '#push'),
+ (r'\*\)', Comment, '#pop'),
+ (r'[(*)]', Comment),
+ ],
+ 'text': [
+ (r'[^*}]+', Comment),
+ (r'\*\}', Comment, '#pop'),
+ (r'\*', Comment),
+ (r'\}', Comment),
+ ],
+ 'string': [
+ (r'[^"\\]+', String),
+ (r'\\<\w*>', String.Symbol),
+ (r'\\"', String),
+ (r'\\', String),
+ (r'"', String, '#pop'),
+ ],
+ 'fact': [
+ (r'[^`\\]+', String.Other),
+ (r'\\<\w*>', String.Symbol),
+ (r'\\`', String.Other),
+ (r'\\', String.Other),
+ (r'`', String.Other, '#pop'),
+ ],
+ }
+
+
+class LeanLexer(RegexLexer):
+ """
+ For the `Lean <https://github.com/leanprover/lean>`_
+ theorem prover.
+
+ .. versionadded:: 2.0
+ """
+ name = 'Lean'
+ aliases = ['lean']
+ filenames = ['*.lean']
+ mimetypes = ['text/x-lean']
+
+ flags = re.MULTILINE | re.UNICODE
+
+ keywords1 = (
+ 'import', 'abbreviation', 'opaque_hint', 'tactic_hint', 'definition',
+ 'renaming', 'inline', 'hiding', 'exposing', 'parameter', 'parameters',
+ 'conjecture', 'hypothesis', 'lemma', 'corollary', 'variable', 'variables',
+ 'theorem', 'axiom', 'inductive', 'structure', 'universe', 'alias',
+ 'help', 'options', 'precedence', 'postfix', 'prefix', 'calc_trans',
+ 'calc_subst', 'calc_refl', 'infix', 'infixl', 'infixr', 'notation', 'eval',
+ 'check', 'exit', 'coercion', 'end', 'private', 'using', 'namespace',
+ 'including', 'instance', 'section', 'context', 'protected', 'expose',
+ 'export', 'set_option', 'add_rewrite', 'extends', 'open', 'example',
+ 'constant', 'constants', 'print', 'opaque', 'reducible', 'irreducible',
+ )
+
+ keywords2 = (
+ 'forall', 'fun', 'Pi', 'obtain', 'from', 'have', 'show', 'assume',
+ 'take', 'let', 'if', 'else', 'then', 'by', 'in', 'with', 'begin',
+ 'proof', 'qed', 'calc', 'match',
+ )
+
+ keywords3 = (
+ # Sorts
+ 'Type', 'Prop',
+ )
+
+ operators = (
+ u'!=', u'#', u'&', u'&&', u'*', u'+', u'-', u'/', u'@', u'!', u'`',
+ u'-.', u'->', u'.', u'..', u'...', u'::', u':>', u';', u';;', u'<',
+ u'<-', u'=', u'==', u'>', u'_', u'|', u'||', u'~', u'=>', u'<=', u'>=',
+ u'/\\', u'\\/', u'∀', u'Π', u'λ', u'↔', u'∧', u'∨', u'≠', u'≤', u'≥',
+ u'¬', u'â»Â¹', u'â¬', u'â–¸', u'→', u'∃', u'â„•', u'ℤ', u'≈', u'×', u'⌞',
+ u'⌟', u'≡', u'⟨', u'⟩',
+ )
+
+ punctuation = (u'(', u')', u':', u'{', u'}', u'[', u']', u'⦃', u'⦄',
+ u':=', u',')
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ (r'/-', Comment, 'comment'),
+ (r'--.*?$', Comment.Single),
+ (words(keywords1, prefix=r'\b', suffix=r'\b'), Keyword.Namespace),
+ (words(keywords2, prefix=r'\b', suffix=r'\b'), Keyword),
+ (words(keywords3, prefix=r'\b', suffix=r'\b'), Keyword.Type),
+ (words(operators), Name.Builtin.Pseudo),
+ (words(punctuation), Operator),
+ (u"[A-Za-z_\u03b1-\u03ba\u03bc-\u03fb\u1f00-\u1ffe\u2100-\u214f]"
+ u"[A-Za-z_'\u03b1-\u03ba\u03bc-\u03fb\u1f00-\u1ffe\u2070-\u2079"
+ u"\u207f-\u2089\u2090-\u209c\u2100-\u214f0-9]*", Name),
+ (r'\d+', Number.Integer),
+ (r'"', String.Double, 'string'),
+ (r'[~?][a-z][\w\']*:', Name.Variable)
+ ],
+ 'comment': [
+ # Multiline Comments
+ (r'[^/-]', Comment.Multiline),
+ (r'/-', Comment.Multiline, '#push'),
+ (r'-/', Comment.Multiline, '#pop'),
+ (r'[/-]', Comment.Multiline)
+ ],
+ 'string': [
+ (r'[^\\"]+', String.Double),
+ (r'\\[n"\\]', String.Escape),
+ ('"', String.Double, '#pop'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/trafficscript.py b/wandb/vendor/pygments/lexers/trafficscript.py
new file mode 100644
index 0000000000000000000000000000000000000000..4254228030924b5358c2f3b40e14ed63cdf058b9
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/trafficscript.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.trafficscript
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for RiverBed's TrafficScript (RTS) language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer
+from pygments.token import String, Number, Name, Keyword, Operator, Text, Comment
+
+__all__ = ['RtsLexer']
+
+
+class RtsLexer(RegexLexer):
+ """
+ For `Riverbed Stingray Traffic Manager <http://www.riverbed.com/stingray>`_
+
+ .. versionadded:: 2.1
+ """
+ name = 'TrafficScript'
+ aliases = ['rts','trafficscript']
+ filenames = ['*.rts']
+
+ tokens = {
+ 'root' : [
+ (r"'(\\\\|\\[^\\]|[^'\\])*'", String),
+ (r'"', String, 'escapable-string'),
+ (r'(0x[0-9a-fA-F]+|\d+)', Number),
+ (r'\d+\.\d+', Number.Float),
+ (r'\$[a-zA-Z](\w|_)*', Name.Variable),
+ (r'(if|else|for(each)?|in|while|do|break|sub|return|import)', Keyword),
+ (r'[a-zA-Z][\w.]*', Name.Function),
+ (r'[-+*/%=,;(){}<>^.!~|&\[\]\?\:]', Operator),
+ (r'(>=|<=|==|!=|'
+ r'&&|\|\||'
+ r'\+=|.=|-=|\*=|/=|%=|<<=|>>=|&=|\|=|\^=|'
+ r'>>|<<|'
+ r'\+\+|--|=>)', Operator),
+ (r'[ \t\r]+', Text),
+ (r'#[^\n]*', Comment),
+ ],
+ 'escapable-string' : [
+ (r'\\[tsn]', String.Escape),
+ (r'[^"]', String),
+ (r'"', String, '#pop'),
+ ],
+
+ }
diff --git a/wandb/vendor/pygments/lexers/typoscript.py b/wandb/vendor/pygments/lexers/typoscript.py
new file mode 100644
index 0000000000000000000000000000000000000000..e358af079a15472557dd3c0141025f5b8752a50c
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/typoscript.py
@@ -0,0 +1,226 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.typoscript
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for TypoScript
+
+ `TypoScriptLexer`
+ A TypoScript lexer.
+
+ `TypoScriptCssDataLexer`
+ Lexer that highlights markers, constants and registers within css.
+
+ `TypoScriptHtmlDataLexer`
+ Lexer that highlights markers, constants and registers within html tags.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using
+from pygments.token import Text, Comment, Name, String, Number, \
+ Operator, Punctuation
+
+__all__ = ['TypoScriptLexer', 'TypoScriptCssDataLexer', 'TypoScriptHtmlDataLexer']
+
+
+class TypoScriptCssDataLexer(RegexLexer):
+ """
+ Lexer that highlights markers, constants and registers within css blocks.
+
+ .. versionadded:: 2.2
+ """
+
+ name = 'TypoScriptCssData'
+ aliases = ['typoscriptcssdata']
+
+ tokens = {
+ 'root': [
+ # marker: ###MARK###
+ (r'(.*)(###\w+###)(.*)', bygroups(String, Name.Constant, String)),
+ # constant: {$some.constant}
+ (r'(\{)(\$)((?:[\w\-]+\.)*)([\w\-]+)(\})',
+ bygroups(String.Symbol, Operator, Name.Constant,
+ Name.Constant, String.Symbol)), # constant
+ # constant: {register:somevalue}
+ (r'(.*)(\{)([\w\-]+)(\s*:\s*)([\w\-]+)(\})(.*)',
+ bygroups(String, String.Symbol, Name.Constant, Operator,
+ Name.Constant, String.Symbol, String)), # constant
+ # whitespace
+ (r'\s+', Text),
+ # comments
+ (r'/\*(?:(?!\*/).)*\*/', Comment),
+ (r'(?<!(#|\'|"))(?:#(?!(?:[a-fA-F0-9]{6}|[a-fA-F0-9]{3}))[^\n#]+|//[^\n]*)',
+ Comment),
+ # other
+ (r'[<>,:=.*%+|]', String),
+ (r'[\w"\-!/&;(){}]+', String),
+ ]
+ }
+
+
+class TypoScriptHtmlDataLexer(RegexLexer):
+ """
+ Lexer that highlights markers, constants and registers within html tags.
+
+ .. versionadded:: 2.2
+ """
+
+ name = 'TypoScriptHtmlData'
+ aliases = ['typoscripthtmldata']
+
+ tokens = {
+ 'root': [
+ # INCLUDE_TYPOSCRIPT
+ (r'(INCLUDE_TYPOSCRIPT)', Name.Class),
+ # Language label or extension resource FILE:... or LLL:... or EXT:...
+ (r'(EXT|FILE|LLL):[^}\n"]*', String),
+ # marker: ###MARK###
+ (r'(.*)(###\w+###)(.*)', bygroups(String, Name.Constant, String)),
+ # constant: {$some.constant}
+ (r'(\{)(\$)((?:[\w\-]+\.)*)([\w\-]+)(\})',
+ bygroups(String.Symbol, Operator, Name.Constant,
+ Name.Constant, String.Symbol)), # constant
+ # constant: {register:somevalue}
+ (r'(.*)(\{)([\w\-]+)(\s*:\s*)([\w\-]+)(\})(.*)',
+ bygroups(String, String.Symbol, Name.Constant, Operator,
+ Name.Constant, String.Symbol, String)), # constant
+ # whitespace
+ (r'\s+', Text),
+ # other
+ (r'[<>,:=.*%+|]', String),
+ (r'[\w"\-!/&;(){}#]+', String),
+ ]
+ }
+
+
+class TypoScriptLexer(RegexLexer):
+ """
+ Lexer for TypoScript code.
+
+ http://docs.typo3.org/typo3cms/TyposcriptReference/
+
+ .. versionadded:: 2.2
+ """
+
+ name = 'TypoScript'
+ aliases = ['typoscript']
+ filenames = ['*.ts', '*.txt']
+ mimetypes = ['text/x-typoscript']
+
+ flags = re.DOTALL | re.MULTILINE
+
+ # Slightly higher than TypeScript (which is 0).
+ priority = 0.1
+
+ tokens = {
+ 'root': [
+ include('comment'),
+ include('constant'),
+ include('html'),
+ include('label'),
+ include('whitespace'),
+ include('keywords'),
+ include('punctuation'),
+ include('operator'),
+ include('structure'),
+ include('literal'),
+ include('other'),
+ ],
+ 'keywords': [
+ # Conditions
+ (r'(\[)(?i)(browser|compatVersion|dayofmonth|dayofweek|dayofyear|'
+ r'device|ELSE|END|GLOBAL|globalString|globalVar|hostname|hour|IP|'
+ r'language|loginUser|loginuser|minute|month|page|PIDinRootline|'
+ r'PIDupinRootline|system|treeLevel|useragent|userFunc|usergroup|'
+ r'version)([^\]]*)(\])',
+ bygroups(String.Symbol, Name.Constant, Text, String.Symbol)),
+ # Functions
+ (r'(?=[\w\-])(HTMLparser|HTMLparser_tags|addParams|cache|encapsLines|'
+ r'filelink|if|imageLinkWrap|imgResource|makelinks|numRows|numberFormat|'
+ r'parseFunc|replacement|round|select|split|stdWrap|strPad|tableStyle|'
+ r'tags|textStyle|typolink)(?![\w\-])', Name.Function),
+ # Toplevel objects and _*
+ (r'(?:(=?\s*<?\s+|^\s*))(cObj|field|config|content|constants|FEData|'
+ r'file|frameset|includeLibs|lib|page|plugin|register|resources|sitemap|'
+ r'sitetitle|styles|temp|tt_[^:.\s]*|types|xmlnews|INCLUDE_TYPOSCRIPT|'
+ r'_CSS_DEFAULT_STYLE|_DEFAULT_PI_VARS|_LOCAL_LANG)(?![\w\-])',
+ bygroups(Operator, Name.Builtin)),
+ # Content objects
+ (r'(?=[\w\-])(CASE|CLEARGIF|COA|COA_INT|COBJ_ARRAY|COLUMNS|CONTENT|'
+ r'CTABLE|EDITPANEL|FILE|FILES|FLUIDTEMPLATE|FORM|HMENU|HRULER|HTML|'
+ r'IMAGE|IMGTEXT|IMG_RESOURCE|LOAD_REGISTER|MEDIA|MULTIMEDIA|OTABLE|'
+ r'PAGE|QTOBJECT|RECORDS|RESTORE_REGISTER|SEARCHRESULT|SVG|SWFOBJECT|'
+ r'TEMPLATE|TEXT|USER|USER_INT)(?![\w\-])', Name.Class),
+ # Menu states
+ (r'(?=[\w\-])(ACTIFSUBRO|ACTIFSUB|ACTRO|ACT|CURIFSUBRO|CURIFSUB|CURRO|'
+ r'CUR|IFSUBRO|IFSUB|NO|SPC|USERDEF1RO|USERDEF1|USERDEF2RO|USERDEF2|'
+ r'USRRO|USR)', Name.Class),
+ # Menu objects
+ (r'(?=[\w\-])(GMENU_FOLDOUT|GMENU_LAYERS|GMENU|IMGMENUITEM|IMGMENU|'
+ r'JSMENUITEM|JSMENU|TMENUITEM|TMENU_LAYERS|TMENU)', Name.Class),
+ # PHP objects
+ (r'(?=[\w\-])(PHP_SCRIPT(_EXT|_INT)?)', Name.Class),
+ (r'(?=[\w\-])(userFunc)(?![\w\-])', Name.Function),
+ ],
+ 'whitespace': [
+ (r'\s+', Text),
+ ],
+ 'html': [
+ (r'<\S[^\n>]*>', using(TypoScriptHtmlDataLexer)),
+ (r'&[^;\n]*;', String),
+ (r'(_CSS_DEFAULT_STYLE)(\s*)(\()(?s)(.*(?=\n\)))',
+ bygroups(Name.Class, Text, String.Symbol, using(TypoScriptCssDataLexer))),
+ ],
+ 'literal': [
+ (r'0x[0-9A-Fa-f]+t?', Number.Hex),
+ # (r'[0-9]*\.[0-9]+([eE][0-9]+)?[fd]?\s*(?:[^=])', Number.Float),
+ (r'[0-9]+', Number.Integer),
+ (r'(###\w+###)', Name.Constant),
+ ],
+ 'label': [
+ # Language label or extension resource FILE:... or LLL:... or EXT:...
+ (r'(EXT|FILE|LLL):[^}\n"]*', String),
+ # Path to a resource
+ (r'(?![^\w\-])([\w\-]+(?:/[\w\-]+)+/?)(\S*\n)',
+ bygroups(String, String)),
+ ],
+ 'punctuation': [
+ (r'[,.]', Punctuation),
+ ],
+ 'operator': [
+ (r'[<>,:=.*%+|]', Operator),
+ ],
+ 'structure': [
+ # Brackets and braces
+ (r'[{}()\[\]\\]', String.Symbol),
+ ],
+ 'constant': [
+ # Constant: {$some.constant}
+ (r'(\{)(\$)((?:[\w\-]+\.)*)([\w\-]+)(\})',
+ bygroups(String.Symbol, Operator, Name.Constant,
+ Name.Constant, String.Symbol)), # constant
+ # Constant: {register:somevalue}
+ (r'(\{)([\w\-]+)(\s*:\s*)([\w\-]+)(\})',
+ bygroups(String.Symbol, Name.Constant, Operator,
+ Name.Constant, String.Symbol)), # constant
+ # Hex color: #ff0077
+ (r'(#[a-fA-F0-9]{6}\b|#[a-fA-F0-9]{3}\b)', String.Char)
+ ],
+ 'comment': [
+ (r'(?<!(#|\'|"))(?:#(?!(?:[a-fA-F0-9]{6}|[a-fA-F0-9]{3}))[^\n#]+|//[^\n]*)',
+ Comment),
+ (r'/\*(?:(?!\*/).)*\*/', Comment),
+ (r'(\s*#\s*\n)', Comment),
+ ],
+ 'other': [
+ (r'[\w"\-!/&;]+', Text),
+ ],
+ }
+
+ def analyse_text(text):
+ if '<INCLUDE_TYPOSCRIPT:' in text:
+ return 1.0
diff --git a/wandb/vendor/pygments/lexers/urbi.py b/wandb/vendor/pygments/lexers/urbi.py
new file mode 100644
index 0000000000000000000000000000000000000000..7aaba90c5876ee93806644312d7349d576ccd4d2
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/urbi.py
@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.urbi
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for UrbiScript language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import ExtendedRegexLexer, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation
+
+__all__ = ['UrbiscriptLexer']
+
+
+class UrbiscriptLexer(ExtendedRegexLexer):
+ """
+ For UrbiScript source code.
+
+ .. versionadded:: 1.5
+ """
+
+ name = 'UrbiScript'
+ aliases = ['urbiscript']
+ filenames = ['*.u']
+ mimetypes = ['application/x-urbiscript']
+
+ flags = re.DOTALL
+
+ # TODO
+ # - handle Experimental and deprecated tags with specific tokens
+ # - handle Angles and Durations with specific tokens
+
+ def blob_callback(lexer, match, ctx):
+ text_before_blob = match.group(1)
+ blob_start = match.group(2)
+ blob_size_str = match.group(3)
+ blob_size = int(blob_size_str)
+ yield match.start(), String, text_before_blob
+ ctx.pos += len(text_before_blob)
+
+ # if blob size doesn't match blob format (example : "\B(2)(aaa)")
+ # yield blob as a string
+ if ctx.text[match.end() + blob_size] != ")":
+ result = "\\B(" + blob_size_str + ")("
+ yield match.start(), String, result
+ ctx.pos += len(result)
+ return
+
+ # if blob is well formated, yield as Escape
+ blob_text = blob_start + ctx.text[match.end():match.end()+blob_size] + ")"
+ yield match.start(), String.Escape, blob_text
+ ctx.pos = match.end() + blob_size + 1 # +1 is the ending ")"
+
+ tokens = {
+ 'root': [
+ (r'\s+', Text),
+ # comments
+ (r'//.*?\n', Comment),
+ (r'/\*', Comment.Multiline, 'comment'),
+ (r'(every|for|loop|while)(?:;|&|\||,)', Keyword),
+ (words((
+ 'assert', 'at', 'break', 'case', 'catch', 'closure', 'compl',
+ 'continue', 'default', 'else', 'enum', 'every', 'external',
+ 'finally', 'for', 'freezeif', 'if', 'new', 'onleave', 'return',
+ 'stopif', 'switch', 'this', 'throw', 'timeout', 'try',
+ 'waituntil', 'whenever', 'while'), suffix=r'\b'),
+ Keyword),
+ (words((
+ 'asm', 'auto', 'bool', 'char', 'const_cast', 'delete', 'double',
+ 'dynamic_cast', 'explicit', 'export', 'extern', 'float', 'friend',
+ 'goto', 'inline', 'int', 'long', 'mutable', 'namespace', 'register',
+ 'reinterpret_cast', 'short', 'signed', 'sizeof', 'static_cast',
+ 'struct', 'template', 'typedef', 'typeid', 'typename', 'union',
+ 'unsigned', 'using', 'virtual', 'volatile', 'wchar_t'), suffix=r'\b'),
+ Keyword.Reserved),
+ # deprecated keywords, use a meaningfull token when available
+ (r'(emit|foreach|internal|loopn|static)\b', Keyword),
+ # ignored keywords, use a meaningfull token when available
+ (r'(private|protected|public)\b', Keyword),
+ (r'(var|do|const|function|class)\b', Keyword.Declaration),
+ (r'(true|false|nil|void)\b', Keyword.Constant),
+ (words((
+ 'Barrier', 'Binary', 'Boolean', 'CallMessage', 'Channel', 'Code',
+ 'Comparable', 'Container', 'Control', 'Date', 'Dictionary', 'Directory',
+ 'Duration', 'Enumeration', 'Event', 'Exception', 'Executable', 'File',
+ 'Finalizable', 'Float', 'FormatInfo', 'Formatter', 'Global', 'Group',
+ 'Hash', 'InputStream', 'IoService', 'Job', 'Kernel', 'Lazy', 'List',
+ 'Loadable', 'Lobby', 'Location', 'Logger', 'Math', 'Mutex', 'nil',
+ 'Object', 'Orderable', 'OutputStream', 'Pair', 'Path', 'Pattern',
+ 'Position', 'Primitive', 'Process', 'Profile', 'PseudoLazy', 'PubSub',
+ 'RangeIterable', 'Regexp', 'Semaphore', 'Server', 'Singleton', 'Socket',
+ 'StackFrame', 'Stream', 'String', 'System', 'Tag', 'Timeout',
+ 'Traceable', 'TrajectoryGenerator', 'Triplet', 'Tuple', 'UObject',
+ 'UValue', 'UVar'), suffix=r'\b'),
+ Name.Builtin),
+ (r'(?:this)\b', Name.Builtin.Pseudo),
+ # don't match single | and &
+ (r'(?:[-=+*%/<>~^:]+|\.&?|\|\||&&)', Operator),
+ (r'(?:and_eq|and|bitand|bitor|in|not|not_eq|or_eq|or|xor_eq|xor)\b',
+ Operator.Word),
+ (r'[{}\[\]()]+', Punctuation),
+ (r'(?:;|\||,|&|\?|!)+', Punctuation),
+ (r'[$a-zA-Z_]\w*', Name.Other),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ # Float, Integer, Angle and Duration
+ (r'(?:[0-9]+(?:(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)?'
+ r'((?:rad|deg|grad)|(?:ms|s|min|h|d))?)\b', Number.Float),
+ # handle binary blob in strings
+ (r'"', String.Double, "string.double"),
+ (r"'", String.Single, "string.single"),
+ ],
+ 'string.double': [
+ (r'((?:\\\\|\\"|[^"])*?)(\\B\((\d+)\)\()', blob_callback),
+ (r'(\\\\|\\"|[^"])*?"', String.Double, '#pop'),
+ ],
+ 'string.single': [
+ (r"((?:\\\\|\\'|[^'])*?)(\\B\((\d+)\)\()", blob_callback),
+ (r"(\\\\|\\'|[^'])*?'", String.Single, '#pop'),
+ ],
+ # from http://pygments.org/docs/lexerdevelopment/#changing-states
+ 'comment': [
+ (r'[^*/]', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline),
+ ]
+ }
diff --git a/wandb/vendor/pygments/lexers/varnish.py b/wandb/vendor/pygments/lexers/varnish.py
new file mode 100644
index 0000000000000000000000000000000000000000..44521422e5b7a05a767a73c245b92c4513636815
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/varnish.py
@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.varnish
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for Varnish configuration
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, using, this, \
+ inherit, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Literal
+
+__all__ = ['VCLLexer', 'VCLSnippetLexer']
+
+
+class VCLLexer(RegexLexer):
+ """
+ For Varnish Configuration Language (VCL).
+
+ .. versionadded:: 2.2
+ """
+ name = 'VCL'
+ aliases = ['vcl']
+ filenames = ['*.vcl']
+ mimetypes = ['text/x-vclsrc']
+
+ def analyse_text(text):
+ # If the very first line is 'vcl 4.0;' it's pretty much guaranteed
+ # that this is VCL
+ if text.startswith('vcl 4.0;'):
+ return 1.0
+ # Skip over comments and blank lines
+ # This is accurate enough that returning 0.9 is reasonable.
+ # Almost no VCL files start without some comments.
+ elif '\nvcl 4\.0;' in text[:1000]:
+ return 0.9
+
+ tokens = {
+ 'probe': [
+ include('whitespace'),
+ include('comments'),
+ (r'(\.\w+)(\s*=\s*)([^;]*)(;)',
+ bygroups(Name.Attribute, Operator, using(this), Punctuation)),
+ (r'\}', Punctuation, '#pop'),
+ ],
+ 'acl': [
+ include('whitespace'),
+ include('comments'),
+ (r'[!/]+', Operator),
+ (r';', Punctuation),
+ (r'\d+', Number),
+ (r'\}', Punctuation, '#pop'),
+ ],
+ 'backend': [
+ include('whitespace'),
+ (r'(\.probe)(\s*=\s*)(\w+)(;)',
+ bygroups(Name.Attribute, Operator, Name.Variable.Global, Punctuation)),
+ (r'(\.probe)(\s*=\s*)(\{)',
+ bygroups(Name.Attribute, Operator, Punctuation), 'probe'),
+ (r'(\.\w+\b)(\s*=\s*)([^;]*)(\s*;)',
+ bygroups(Name.Attribute, Operator, using(this), Punctuation)),
+ (r'\{', Punctuation, '#push'),
+ (r'\}', Punctuation, '#pop'),
+ ],
+ 'statements': [
+ (r'(\d\.)?\d+[sdwhmy]', Literal.Date),
+ (r'(\d\.)?\d+ms', Literal.Date),
+ (r'(vcl_pass|vcl_hash|vcl_hit|vcl_init|vcl_backend_fetch|vcl_pipe|'
+ r'vcl_backend_response|vcl_synth|vcl_deliver|vcl_backend_error|'
+ r'vcl_fini|vcl_recv|vcl_purge|vcl_miss)\b', Name.Function),
+ (r'(pipe|retry|hash|synth|deliver|purge|abandon|lookup|pass|fail|ok|'
+ r'miss|fetch|restart)\b', Name.Constant),
+ (r'(beresp|obj|resp|req|req_top|bereq)\.http\.[a-zA-Z_-]+\b', Name.Variable),
+ (words((
+ 'obj.status', 'req.hash_always_miss', 'beresp.backend', 'req.esi_level',
+ 'req.can_gzip', 'beresp.ttl', 'obj.uncacheable', 'req.ttl', 'obj.hits',
+ 'client.identity', 'req.hash_ignore_busy', 'obj.reason', 'req.xid',
+ 'req_top.proto', 'beresp.age', 'obj.proto', 'obj.age', 'local.ip',
+ 'beresp.uncacheable', 'req.method', 'beresp.backend.ip', 'now',
+ 'obj.grace', 'req.restarts', 'beresp.keep', 'req.proto', 'resp.proto',
+ 'bereq.xid', 'bereq.between_bytes_timeout', 'req.esi',
+ 'bereq.first_byte_timeout', 'bereq.method', 'bereq.connect_timeout',
+ 'beresp.do_gzip', 'resp.status', 'beresp.do_gunzip',
+ 'beresp.storage_hint', 'resp.is_streaming', 'beresp.do_stream',
+ 'req_top.method', 'bereq.backend', 'beresp.backend.name', 'beresp.status',
+ 'req.url', 'obj.keep', 'obj.ttl', 'beresp.reason', 'bereq.retries',
+ 'resp.reason', 'bereq.url', 'beresp.do_esi', 'beresp.proto', 'client.ip',
+ 'bereq.proto', 'server.hostname', 'remote.ip', 'req.backend_hint',
+ 'server.identity', 'req_top.url', 'beresp.grace', 'beresp.was_304',
+ 'server.ip', 'bereq.uncacheable'), suffix=r'\b'),
+ Name.Variable),
+ (r'[!%&+*\-,/<.}{>=|~]+', Operator),
+ (r'[();]', Punctuation),
+
+ (r'[,]+', Punctuation),
+ (words(('hash_data', 'regsub', 'regsuball', 'if', 'else',
+ 'elsif', 'elif', 'synth', 'synthetic', 'ban',
+ 'return', 'set', 'unset', 'import', 'include', 'new',
+ 'rollback', 'call'), suffix=r'\b'),
+ Keyword),
+ (r'storage\.\w+\.\w+\b', Name.Variable),
+ (words(('true', 'false')), Name.Builtin),
+ (r'\d+\b', Number),
+ (r'(backend)(\s+\w+)(\s*\{)',
+ bygroups(Keyword, Name.Variable.Global, Punctuation), 'backend'),
+ (r'(probe\s)(\s*\w+\s)(\{)',
+ bygroups(Keyword, Name.Variable.Global, Punctuation), 'probe'),
+ (r'(acl\s)(\s*\w+\s)(\{)',
+ bygroups(Keyword, Name.Variable.Global, Punctuation), 'acl'),
+ (r'(vcl )(4.0)(;)$',
+ bygroups(Keyword.Reserved, Name.Constant, Punctuation)),
+ (r'(sub\s+)([a-zA-Z]\w*)(\s*\{)',
+ bygroups(Keyword, Name.Function, Punctuation)),
+ (r'([a-zA-Z_]\w*)'
+ r'(\.)'
+ r'([a-zA-Z_]\w*)'
+ r'(\s*\(.*\))',
+ bygroups(Name.Function, Punctuation, Name.Function, using(this))),
+ ('[a-zA-Z_]\w*', Name),
+ ],
+ 'comment': [
+ (r'[^*/]+', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline),
+ ],
+ 'comments': [
+ (r'#.*$', Comment),
+ (r'/\*', Comment.Multiline, 'comment'),
+ (r'//.*$', Comment),
+ ],
+ 'string': [
+ (r'"', String, '#pop'),
+ (r'[^"\n]+', String), # all other characters
+ ],
+ 'multistring': [
+ (r'[^"}]', String),
+ (r'"\}', String, '#pop'),
+ (r'["}]', String),
+ ],
+ 'whitespace': [
+ (r'L?"', String, 'string'),
+ (r'\{"', String, 'multistring'),
+ (r'\n', Text),
+ (r'\s+', Text),
+ (r'\\\n', Text), # line continuation
+ ],
+ 'root': [
+ include('whitespace'),
+ include('comments'),
+ include('statements'),
+ (r'\s+', Text),
+ ],
+ }
+
+
+class VCLSnippetLexer(VCLLexer):
+ """
+ For Varnish Configuration Language snippets.
+
+ .. versionadded:: 2.2
+ """
+ name = 'VCLSnippets'
+ aliases = ['vclsnippets', 'vclsnippet']
+ mimetypes = ['text/x-vclsnippet']
+ filenames = []
+
+ def analyse_text(text):
+ # override method inherited from VCLLexer
+ return 0
+
+ tokens = {
+ 'snippetspre': [
+ (r'\.\.\.+', Comment),
+ (r'(bereq|req|req_top|resp|beresp|obj|client|server|local|remote|'
+ r'storage)($|\.\*)', Name.Variable),
+ ],
+ 'snippetspost': [
+ (r'(backend)\b', Keyword.Reserved),
+ ],
+ 'root': [
+ include('snippetspre'),
+ inherit,
+ include('snippetspost'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/verification.py b/wandb/vendor/pygments/lexers/verification.py
new file mode 100644
index 0000000000000000000000000000000000000000..5322e17fdeef899fcdeb76448a6f59d77af5dd53
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/verification.py
@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.verification
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexer for Intermediate Verification Languages (IVLs).
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Comment, Operator, Keyword, Name, Number, \
+ Punctuation, Whitespace
+
+__all__ = ['BoogieLexer', 'SilverLexer']
+
+
+class BoogieLexer(RegexLexer):
+ """
+ For `Boogie <https://boogie.codeplex.com/>`_ source code.
+
+ .. versionadded:: 2.1
+ """
+ name = 'Boogie'
+ aliases = ['boogie']
+ filenames = ['*.bpl']
+
+ tokens = {
+ 'root': [
+ # Whitespace and Comments
+ (r'\n', Whitespace),
+ (r'\s+', Whitespace),
+ (r'//[/!](.*?)\n', Comment.Doc),
+ (r'//(.*?)\n', Comment.Single),
+ (r'/\*', Comment.Multiline, 'comment'),
+
+ (words((
+ 'axiom', 'break', 'call', 'ensures', 'else', 'exists', 'function',
+ 'forall', 'if', 'invariant', 'modifies', 'procedure', 'requires',
+ 'then', 'var', 'while'),
+ suffix=r'\b'), Keyword),
+ (words(('const',), suffix=r'\b'), Keyword.Reserved),
+
+ (words(('bool', 'int', 'ref'), suffix=r'\b'), Keyword.Type),
+ include('numbers'),
+ (r"(>=|<=|:=|!=|==>|&&|\|\||[+/\-=>*<\[\]])", Operator),
+ (r"([{}():;,.])", Punctuation),
+ # Identifier
+ (r'[a-zA-Z_]\w*', Name),
+ ],
+ 'comment': [
+ (r'[^*/]+', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline),
+ ],
+ 'numbers': [
+ (r'[0-9]+', Number.Integer),
+ ],
+ }
+
+
+class SilverLexer(RegexLexer):
+ """
+ For `Silver <https://bitbucket.org/viperproject/silver>`_ source code.
+
+ .. versionadded:: 2.2
+ """
+ name = 'Silver'
+ aliases = ['silver']
+ filenames = ['*.sil', '*.vpr']
+
+ tokens = {
+ 'root': [
+ # Whitespace and Comments
+ (r'\n', Whitespace),
+ (r'\s+', Whitespace),
+ (r'//[/!](.*?)\n', Comment.Doc),
+ (r'//(.*?)\n', Comment.Single),
+ (r'/\*', Comment.Multiline, 'comment'),
+
+ (words((
+ 'result', 'true', 'false', 'null', 'method', 'function',
+ 'predicate', 'program', 'domain', 'axiom', 'var', 'returns',
+ 'field', 'define', 'requires', 'ensures', 'invariant',
+ 'fold', 'unfold', 'inhale', 'exhale', 'new', 'assert',
+ 'assume', 'goto', 'while', 'if', 'elseif', 'else', 'fresh',
+ 'constraining', 'Seq', 'Set', 'Multiset', 'union', 'intersection',
+ 'setminus', 'subset', 'unfolding', 'in', 'old', 'forall', 'exists',
+ 'acc', 'wildcard', 'write', 'none', 'epsilon', 'perm', 'unique',
+ 'apply', 'package', 'folding', 'label', 'forperm'),
+ suffix=r'\b'), Keyword),
+ (words(('Int', 'Perm', 'Bool', 'Ref'), suffix=r'\b'), Keyword.Type),
+ include('numbers'),
+
+ (r'[!%&*+=|?:<>/\-\[\]]', Operator),
+ (r'([{}():;,.])', Punctuation),
+ # Identifier
+ (r'[\w$]\w*', Name),
+ ],
+ 'comment': [
+ (r'[^*/]+', Comment.Multiline),
+ (r'/\*', Comment.Multiline, '#push'),
+ (r'\*/', Comment.Multiline, '#pop'),
+ (r'[*/]', Comment.Multiline),
+ ],
+ 'numbers': [
+ (r'[0-9]+', Number.Integer),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/web.py b/wandb/vendor/pygments/lexers/web.py
new file mode 100644
index 0000000000000000000000000000000000000000..6e9c4f92bb8a696655f72458442fe2b335290c16
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/web.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.web
+ ~~~~~~~~~~~~~~~~~~~
+
+ Just export previously exported lexers.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexers.html import HtmlLexer, DtdLexer, XmlLexer, XsltLexer, \
+ HamlLexer, ScamlLexer, JadeLexer
+from pygments.lexers.css import CssLexer, SassLexer, ScssLexer
+from pygments.lexers.javascript import JavascriptLexer, LiveScriptLexer, \
+ DartLexer, TypeScriptLexer, LassoLexer, ObjectiveJLexer, CoffeeScriptLexer
+from pygments.lexers.actionscript import ActionScriptLexer, \
+ ActionScript3Lexer, MxmlLexer
+from pygments.lexers.php import PhpLexer
+from pygments.lexers.webmisc import DuelLexer, XQueryLexer, SlimLexer, QmlLexer
+from pygments.lexers.data import JsonLexer
+JSONLexer = JsonLexer # for backwards compatibility with Pygments 1.5
+
+__all__ = []
diff --git a/wandb/vendor/pygments/lexers/webmisc.py b/wandb/vendor/pygments/lexers/webmisc.py
new file mode 100644
index 0000000000000000000000000000000000000000..712c8246320a80b0d6c29b98ddb73314dcff190a
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/webmisc.py
@@ -0,0 +1,988 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.webmisc
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for misc. web stuff.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, ExtendedRegexLexer, include, bygroups, \
+ default, using
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Literal
+from pygments.util import unirange
+
+from pygments.lexers.css import _indentation, _starts_block
+from pygments.lexers.html import HtmlLexer
+from pygments.lexers.javascript import JavascriptLexer
+from pygments.lexers.ruby import RubyLexer
+
+__all__ = ['DuelLexer', 'SlimLexer', 'XQueryLexer', 'QmlLexer', 'CirruLexer']
+
+
+class DuelLexer(RegexLexer):
+ """
+ Lexer for Duel Views Engine (formerly JBST) markup with JavaScript code blocks.
+ See http://duelengine.org/.
+ See http://jsonml.org/jbst/.
+
+ .. versionadded:: 1.4
+ """
+
+ name = 'Duel'
+ aliases = ['duel', 'jbst', 'jsonml+bst']
+ filenames = ['*.duel', '*.jbst']
+ mimetypes = ['text/x-duel', 'text/x-jbst']
+
+ flags = re.DOTALL
+
+ tokens = {
+ 'root': [
+ (r'(<%[@=#!:]?)(.*?)(%>)',
+ bygroups(Name.Tag, using(JavascriptLexer), Name.Tag)),
+ (r'(<%\$)(.*?)(:)(.*?)(%>)',
+ bygroups(Name.Tag, Name.Function, Punctuation, String, Name.Tag)),
+ (r'(<%--)(.*?)(--%>)',
+ bygroups(Name.Tag, Comment.Multiline, Name.Tag)),
+ (r'(<script.*?>)(.*?)(</script>)',
+ bygroups(using(HtmlLexer),
+ using(JavascriptLexer), using(HtmlLexer))),
+ (r'(.+?)(?=<)', using(HtmlLexer)),
+ (r'.+', using(HtmlLexer)),
+ ],
+ }
+
+
+class XQueryLexer(ExtendedRegexLexer):
+ """
+ An XQuery lexer, parsing a stream and outputting the tokens needed to
+ highlight xquery code.
+
+ .. versionadded:: 1.4
+ """
+ name = 'XQuery'
+ aliases = ['xquery', 'xqy', 'xq', 'xql', 'xqm']
+ filenames = ['*.xqy', '*.xquery', '*.xq', '*.xql', '*.xqm']
+ mimetypes = ['text/xquery', 'application/xquery']
+
+ xquery_parse_state = []
+
+ # FIX UNICODE LATER
+ # ncnamestartchar = (
+ # ur"[A-Z]|_|[a-z]|[\u00C0-\u00D6]|[\u00D8-\u00F6]|[\u00F8-\u02FF]|"
+ # ur"[\u0370-\u037D]|[\u037F-\u1FFF]|[\u200C-\u200D]|[\u2070-\u218F]|"
+ # ur"[\u2C00-\u2FEF]|[\u3001-\uD7FF]|[\uF900-\uFDCF]|[\uFDF0-\uFFFD]|"
+ # ur"[\u10000-\uEFFFF]"
+ # )
+ ncnamestartchar = r"(?:[A-Z]|_|[a-z])"
+ # FIX UNICODE LATER
+ # ncnamechar = ncnamestartchar + (ur"|-|\.|[0-9]|\u00B7|[\u0300-\u036F]|"
+ # ur"[\u203F-\u2040]")
+ ncnamechar = r"(?:" + ncnamestartchar + r"|-|\.|[0-9])"
+ ncname = "(?:%s+%s*)" % (ncnamestartchar, ncnamechar)
+ pitarget_namestartchar = r"(?:[A-KN-WYZ]|_|:|[a-kn-wyz])"
+ pitarget_namechar = r"(?:" + pitarget_namestartchar + r"|-|\.|[0-9])"
+ pitarget = "%s+%s*" % (pitarget_namestartchar, pitarget_namechar)
+ prefixedname = "%s:%s" % (ncname, ncname)
+ unprefixedname = ncname
+ qname = "(?:%s|%s)" % (prefixedname, unprefixedname)
+
+ entityref = r'(?:&(?:lt|gt|amp|quot|apos|nbsp);)'
+ charref = r'(?:&#[0-9]+;|&#x[0-9a-fA-F]+;)'
+
+ stringdouble = r'(?:"(?:' + entityref + r'|' + charref + r'|""|[^&"])*")'
+ stringsingle = r"(?:'(?:" + entityref + r"|" + charref + r"|''|[^&'])*')"
+
+ # FIX UNICODE LATER
+ # elementcontentchar = (ur'\t|\r|\n|[\u0020-\u0025]|[\u0028-\u003b]|'
+ # ur'[\u003d-\u007a]|\u007c|[\u007e-\u007F]')
+ elementcontentchar = r'[A-Za-z]|\s|\d|[!"#$%()*+,\-./:;=?@\[\\\]^_\'`|~]'
+ # quotattrcontentchar = (ur'\t|\r|\n|[\u0020-\u0021]|[\u0023-\u0025]|'
+ # ur'[\u0027-\u003b]|[\u003d-\u007a]|\u007c|[\u007e-\u007F]')
+ quotattrcontentchar = r'[A-Za-z]|\s|\d|[!#$%()*+,\-./:;=?@\[\\\]^_\'`|~]'
+ # aposattrcontentchar = (ur'\t|\r|\n|[\u0020-\u0025]|[\u0028-\u003b]|'
+ # ur'[\u003d-\u007a]|\u007c|[\u007e-\u007F]')
+ aposattrcontentchar = r'[A-Za-z]|\s|\d|[!"#$%()*+,\-./:;=?@\[\\\]^_`|~]'
+
+ # CHAR elements - fix the above elementcontentchar, quotattrcontentchar,
+ # aposattrcontentchar
+ # x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
+
+ flags = re.DOTALL | re.MULTILINE | re.UNICODE
+
+ def punctuation_root_callback(lexer, match, ctx):
+ yield match.start(), Punctuation, match.group(1)
+ # transition to root always - don't pop off stack
+ ctx.stack = ['root']
+ ctx.pos = match.end()
+
+ def operator_root_callback(lexer, match, ctx):
+ yield match.start(), Operator, match.group(1)
+ # transition to root always - don't pop off stack
+ ctx.stack = ['root']
+ ctx.pos = match.end()
+
+ def popstate_tag_callback(lexer, match, ctx):
+ yield match.start(), Name.Tag, match.group(1)
+ ctx.stack.append(lexer.xquery_parse_state.pop())
+ ctx.pos = match.end()
+
+ def popstate_xmlcomment_callback(lexer, match, ctx):
+ yield match.start(), String.Doc, match.group(1)
+ ctx.stack.append(lexer.xquery_parse_state.pop())
+ ctx.pos = match.end()
+
+ def popstate_kindtest_callback(lexer, match, ctx):
+ yield match.start(), Punctuation, match.group(1)
+ next_state = lexer.xquery_parse_state.pop()
+ if next_state == 'occurrenceindicator':
+ if re.match("[?*+]+", match.group(2)):
+ yield match.start(), Punctuation, match.group(2)
+ ctx.stack.append('operator')
+ ctx.pos = match.end()
+ else:
+ ctx.stack.append('operator')
+ ctx.pos = match.end(1)
+ else:
+ ctx.stack.append(next_state)
+ ctx.pos = match.end(1)
+
+ def popstate_callback(lexer, match, ctx):
+ yield match.start(), Punctuation, match.group(1)
+ # if we have run out of our state stack, pop whatever is on the pygments
+ # state stack
+ if len(lexer.xquery_parse_state) == 0:
+ ctx.stack.pop()
+ elif len(ctx.stack) > 1:
+ ctx.stack.append(lexer.xquery_parse_state.pop())
+ else:
+ # i don't know if i'll need this, but in case, default back to root
+ ctx.stack = ['root']
+ ctx.pos = match.end()
+
+ def pushstate_element_content_starttag_callback(lexer, match, ctx):
+ yield match.start(), Name.Tag, match.group(1)
+ lexer.xquery_parse_state.append('element_content')
+ ctx.stack.append('start_tag')
+ ctx.pos = match.end()
+
+ def pushstate_cdata_section_callback(lexer, match, ctx):
+ yield match.start(), String.Doc, match.group(1)
+ ctx.stack.append('cdata_section')
+ lexer.xquery_parse_state.append(ctx.state.pop)
+ ctx.pos = match.end()
+
+ def pushstate_starttag_callback(lexer, match, ctx):
+ yield match.start(), Name.Tag, match.group(1)
+ lexer.xquery_parse_state.append(ctx.state.pop)
+ ctx.stack.append('start_tag')
+ ctx.pos = match.end()
+
+ def pushstate_operator_order_callback(lexer, match, ctx):
+ yield match.start(), Keyword, match.group(1)
+ yield match.start(), Text, match.group(2)
+ yield match.start(), Punctuation, match.group(3)
+ ctx.stack = ['root']
+ lexer.xquery_parse_state.append('operator')
+ ctx.pos = match.end()
+
+ def pushstate_operator_map_callback(lexer, match, ctx):
+ yield match.start(), Keyword, match.group(1)
+ yield match.start(), Text, match.group(2)
+ yield match.start(), Punctuation, match.group(3)
+ ctx.stack = ['root']
+ lexer.xquery_parse_state.append('operator')
+ ctx.pos = match.end()
+
+ def pushstate_operator_root_validate(lexer, match, ctx):
+ yield match.start(), Keyword, match.group(1)
+ yield match.start(), Text, match.group(2)
+ yield match.start(), Punctuation, match.group(3)
+ ctx.stack = ['root']
+ lexer.xquery_parse_state.append('operator')
+ ctx.pos = match.end()
+
+ def pushstate_operator_root_validate_withmode(lexer, match, ctx):
+ yield match.start(), Keyword, match.group(1)
+ yield match.start(), Text, match.group(2)
+ yield match.start(), Keyword, match.group(3)
+ ctx.stack = ['root']
+ lexer.xquery_parse_state.append('operator')
+ ctx.pos = match.end()
+
+ def pushstate_operator_processing_instruction_callback(lexer, match, ctx):
+ yield match.start(), String.Doc, match.group(1)
+ ctx.stack.append('processing_instruction')
+ lexer.xquery_parse_state.append('operator')
+ ctx.pos = match.end()
+
+ def pushstate_element_content_processing_instruction_callback(lexer, match, ctx):
+ yield match.start(), String.Doc, match.group(1)
+ ctx.stack.append('processing_instruction')
+ lexer.xquery_parse_state.append('element_content')
+ ctx.pos = match.end()
+
+ def pushstate_element_content_cdata_section_callback(lexer, match, ctx):
+ yield match.start(), String.Doc, match.group(1)
+ ctx.stack.append('cdata_section')
+ lexer.xquery_parse_state.append('element_content')
+ ctx.pos = match.end()
+
+ def pushstate_operator_cdata_section_callback(lexer, match, ctx):
+ yield match.start(), String.Doc, match.group(1)
+ ctx.stack.append('cdata_section')
+ lexer.xquery_parse_state.append('operator')
+ ctx.pos = match.end()
+
+ def pushstate_element_content_xmlcomment_callback(lexer, match, ctx):
+ yield match.start(), String.Doc, match.group(1)
+ ctx.stack.append('xml_comment')
+ lexer.xquery_parse_state.append('element_content')
+ ctx.pos = match.end()
+
+ def pushstate_operator_xmlcomment_callback(lexer, match, ctx):
+ yield match.start(), String.Doc, match.group(1)
+ ctx.stack.append('xml_comment')
+ lexer.xquery_parse_state.append('operator')
+ ctx.pos = match.end()
+
+ def pushstate_kindtest_callback(lexer, match, ctx):
+ yield match.start(), Keyword, match.group(1)
+ yield match.start(), Text, match.group(2)
+ yield match.start(), Punctuation, match.group(3)
+ lexer.xquery_parse_state.append('kindtest')
+ ctx.stack.append('kindtest')
+ ctx.pos = match.end()
+
+ def pushstate_operator_kindtestforpi_callback(lexer, match, ctx):
+ yield match.start(), Keyword, match.group(1)
+ yield match.start(), Text, match.group(2)
+ yield match.start(), Punctuation, match.group(3)
+ lexer.xquery_parse_state.append('operator')
+ ctx.stack.append('kindtestforpi')
+ ctx.pos = match.end()
+
+ def pushstate_operator_kindtest_callback(lexer, match, ctx):
+ yield match.start(), Keyword, match.group(1)
+ yield match.start(), Text, match.group(2)
+ yield match.start(), Punctuation, match.group(3)
+ lexer.xquery_parse_state.append('operator')
+ ctx.stack.append('kindtest')
+ ctx.pos = match.end()
+
+ def pushstate_occurrenceindicator_kindtest_callback(lexer, match, ctx):
+ yield match.start(), Name.Tag, match.group(1)
+ yield match.start(), Text, match.group(2)
+ yield match.start(), Punctuation, match.group(3)
+ lexer.xquery_parse_state.append('occurrenceindicator')
+ ctx.stack.append('kindtest')
+ ctx.pos = match.end()
+
+ def pushstate_operator_starttag_callback(lexer, match, ctx):
+ yield match.start(), Name.Tag, match.group(1)
+ lexer.xquery_parse_state.append('operator')
+ ctx.stack.append('start_tag')
+ ctx.pos = match.end()
+
+ def pushstate_operator_root_callback(lexer, match, ctx):
+ yield match.start(), Punctuation, match.group(1)
+ lexer.xquery_parse_state.append('operator')
+ ctx.stack = ['root']
+ ctx.pos = match.end()
+
+ def pushstate_operator_root_construct_callback(lexer, match, ctx):
+ yield match.start(), Keyword, match.group(1)
+ yield match.start(), Text, match.group(2)
+ yield match.start(), Punctuation, match.group(3)
+ lexer.xquery_parse_state.append('operator')
+ ctx.stack = ['root']
+ ctx.pos = match.end()
+
+ def pushstate_root_callback(lexer, match, ctx):
+ yield match.start(), Punctuation, match.group(1)
+ cur_state = ctx.stack.pop()
+ lexer.xquery_parse_state.append(cur_state)
+ ctx.stack = ['root']
+ ctx.pos = match.end()
+
+ def pushstate_operator_attribute_callback(lexer, match, ctx):
+ yield match.start(), Name.Attribute, match.group(1)
+ ctx.stack.append('operator')
+ ctx.pos = match.end()
+
+ def pushstate_operator_callback(lexer, match, ctx):
+ yield match.start(), Keyword, match.group(1)
+ yield match.start(), Text, match.group(2)
+ yield match.start(), Punctuation, match.group(3)
+ lexer.xquery_parse_state.append('operator')
+ ctx.pos = match.end()
+
+ tokens = {
+ 'comment': [
+ # xquery comments
+ (r'(:\))', Comment, '#pop'),
+ (r'(\(:)', Comment, '#push'),
+ (r'[^:)]', Comment),
+ (r'([^:)]|:|\))', Comment),
+ ],
+ 'whitespace': [
+ (r'\s+', Text),
+ ],
+ 'operator': [
+ include('whitespace'),
+ (r'(\})', popstate_callback),
+ (r'\(:', Comment, 'comment'),
+
+ (r'(\{)', pushstate_root_callback),
+ (r'then|else|external|at|div|except', Keyword, 'root'),
+ (r'order by', Keyword, 'root'),
+ (r'group by', Keyword, 'root'),
+ (r'is|mod|order\s+by|stable\s+order\s+by', Keyword, 'root'),
+ (r'and|or', Operator.Word, 'root'),
+ (r'(eq|ge|gt|le|lt|ne|idiv|intersect|in)(?=\b)',
+ Operator.Word, 'root'),
+ (r'return|satisfies|to|union|where|count|preserve\s+strip',
+ Keyword, 'root'),
+ (r'(>=|>>|>|<=|<<|<|-|\*|!=|\+|\|\||\||:=|=|!)',
+ operator_root_callback),
+ (r'(::|:|;|\[|//|/|,)',
+ punctuation_root_callback),
+ (r'(castable|cast)(\s+)(as)\b',
+ bygroups(Keyword, Text, Keyword), 'singletype'),
+ (r'(instance)(\s+)(of)\b',
+ bygroups(Keyword, Text, Keyword), 'itemtype'),
+ (r'(treat)(\s+)(as)\b',
+ bygroups(Keyword, Text, Keyword), 'itemtype'),
+ (r'(case)(\s+)(' + stringdouble + ')',
+ bygroups(Keyword, Text, String.Double), 'itemtype'),
+ (r'(case)(\s+)(' + stringsingle + ')',
+ bygroups(Keyword, Text, String.Single), 'itemtype'),
+ (r'(case|as)\b', Keyword, 'itemtype'),
+ (r'(\))(\s*)(as)',
+ bygroups(Punctuation, Text, Keyword), 'itemtype'),
+ (r'\$', Name.Variable, 'varname'),
+ (r'(for|let|previous|next)(\s+)(\$)',
+ bygroups(Keyword, Text, Name.Variable), 'varname'),
+ (r'(for)(\s+)(tumbling|sliding)(\s+)(window)(\s+)(\$)',
+ bygroups(Keyword, Text, Keyword, Text, Keyword, Text, Name.Variable),
+ 'varname'),
+ # (r'\)|\?|\]', Punctuation, '#push'),
+ (r'\)|\?|\]', Punctuation),
+ (r'(empty)(\s+)(greatest|least)', bygroups(Keyword, Text, Keyword)),
+ (r'ascending|descending|default', Keyword, '#push'),
+ (r'(allowing)(\s+)(empty)', bygroups(Keyword, Text, Keyword)),
+ (r'external', Keyword),
+ (r'(start|when|end)', Keyword, 'root'),
+ (r'(only)(\s+)(end)', bygroups(Keyword, Text, Keyword), 'root'),
+ (r'collation', Keyword, 'uritooperator'),
+
+ # eXist specific XQUF
+ (r'(into|following|preceding|with)', Keyword, 'root'),
+
+ # support for current context on rhs of Simple Map Operator
+ (r'\.', Operator),
+
+ # finally catch all string literals and stay in operator state
+ (stringdouble, String.Double),
+ (stringsingle, String.Single),
+
+ (r'(catch)(\s*)', bygroups(Keyword, Text), 'root'),
+ ],
+ 'uritooperator': [
+ (stringdouble, String.Double, '#pop'),
+ (stringsingle, String.Single, '#pop'),
+ ],
+ 'namespacedecl': [
+ include('whitespace'),
+ (r'\(:', Comment, 'comment'),
+ (r'(at)(\s+)('+stringdouble+')', bygroups(Keyword, Text, String.Double)),
+ (r"(at)(\s+)("+stringsingle+')', bygroups(Keyword, Text, String.Single)),
+ (stringdouble, String.Double),
+ (stringsingle, String.Single),
+ (r',', Punctuation),
+ (r'=', Operator),
+ (r';', Punctuation, 'root'),
+ (ncname, Name.Namespace),
+ ],
+ 'namespacekeyword': [
+ include('whitespace'),
+ (r'\(:', Comment, 'comment'),
+ (stringdouble, String.Double, 'namespacedecl'),
+ (stringsingle, String.Single, 'namespacedecl'),
+ (r'inherit|no-inherit', Keyword, 'root'),
+ (r'namespace', Keyword, 'namespacedecl'),
+ (r'(default)(\s+)(element)', bygroups(Keyword, Text, Keyword)),
+ (r'preserve|no-preserve', Keyword),
+ (r',', Punctuation),
+ ],
+ 'annotationname': [
+ (r'\(:', Comment, 'comment'),
+ (qname, Name.Decorator),
+ (r'(\()(' + stringdouble + ')', bygroups(Punctuation, String.Double)),
+ (r'(\()(' + stringsingle + ')', bygroups(Punctuation, String.Single)),
+ (r'(\,)(\s+)(' + stringdouble + ')',
+ bygroups(Punctuation, Text, String.Double)),
+ (r'(\,)(\s+)(' + stringsingle + ')',
+ bygroups(Punctuation, Text, String.Single)),
+ (r'\)', Punctuation),
+ (r'(\s+)(\%)', bygroups(Text, Name.Decorator), 'annotationname'),
+ (r'(\s+)(variable)(\s+)(\$)',
+ bygroups(Text, Keyword.Declaration, Text, Name.Variable), 'varname'),
+ (r'(\s+)(function)(\s+)',
+ bygroups(Text, Keyword.Declaration, Text), 'root')
+ ],
+ 'varname': [
+ (r'\(:', Comment, 'comment'),
+ (r'(' + qname + ')(\()?', bygroups(Name, Punctuation), 'operator'),
+ ],
+ 'singletype': [
+ include('whitespace'),
+ (r'\(:', Comment, 'comment'),
+ (ncname + r'(:\*)', Name.Variable, 'operator'),
+ (qname, Name.Variable, 'operator'),
+ ],
+ 'itemtype': [
+ include('whitespace'),
+ (r'\(:', Comment, 'comment'),
+ (r'\$', Name.Variable, 'varname'),
+ (r'(void)(\s*)(\()(\s*)(\))',
+ bygroups(Keyword, Text, Punctuation, Text, Punctuation), 'operator'),
+ (r'(element|attribute|schema-element|schema-attribute|comment|text|'
+ r'node|binary|document-node|empty-sequence)(\s*)(\()',
+ pushstate_occurrenceindicator_kindtest_callback),
+ # Marklogic specific type?
+ (r'(processing-instruction)(\s*)(\()',
+ bygroups(Keyword, Text, Punctuation),
+ ('occurrenceindicator', 'kindtestforpi')),
+ (r'(item)(\s*)(\()(\s*)(\))(?=[*+?])',
+ bygroups(Keyword, Text, Punctuation, Text, Punctuation),
+ 'occurrenceindicator'),
+ (r'(\(\#)(\s*)', bygroups(Punctuation, Text), 'pragma'),
+ (r';', Punctuation, '#pop'),
+ (r'then|else', Keyword, '#pop'),
+ (r'(at)(\s+)(' + stringdouble + ')',
+ bygroups(Keyword, Text, String.Double), 'namespacedecl'),
+ (r'(at)(\s+)(' + stringsingle + ')',
+ bygroups(Keyword, Text, String.Single), 'namespacedecl'),
+ (r'except|intersect|in|is|return|satisfies|to|union|where|count',
+ Keyword, 'root'),
+ (r'and|div|eq|ge|gt|le|lt|ne|idiv|mod|or', Operator.Word, 'root'),
+ (r':=|=|,|>=|>>|>|\[|\(|<=|<<|<|-|!=|\|\||\|', Operator, 'root'),
+ (r'external|at', Keyword, 'root'),
+ (r'(stable)(\s+)(order)(\s+)(by)',
+ bygroups(Keyword, Text, Keyword, Text, Keyword), 'root'),
+ (r'(castable|cast)(\s+)(as)',
+ bygroups(Keyword, Text, Keyword), 'singletype'),
+ (r'(treat)(\s+)(as)', bygroups(Keyword, Text, Keyword)),
+ (r'(instance)(\s+)(of)', bygroups(Keyword, Text, Keyword)),
+ (r'(case)(\s+)(' + stringdouble + ')',
+ bygroups(Keyword, Text, String.Double), 'itemtype'),
+ (r'(case)(\s+)(' + stringsingle + ')',
+ bygroups(Keyword, Text, String.Single), 'itemtype'),
+ (r'case|as', Keyword, 'itemtype'),
+ (r'(\))(\s*)(as)', bygroups(Operator, Text, Keyword), 'itemtype'),
+ (ncname + r':\*', Keyword.Type, 'operator'),
+ (r'(function|map|array)(\()', bygroups(Keyword.Type, Punctuation)),
+ (qname, Keyword.Type, 'occurrenceindicator'),
+ ],
+ 'kindtest': [
+ (r'\(:', Comment, 'comment'),
+ (r'\{', Punctuation, 'root'),
+ (r'(\))([*+?]?)', popstate_kindtest_callback),
+ (r'\*', Name, 'closekindtest'),
+ (qname, Name, 'closekindtest'),
+ (r'(element|schema-element)(\s*)(\()', pushstate_kindtest_callback),
+ ],
+ 'kindtestforpi': [
+ (r'\(:', Comment, 'comment'),
+ (r'\)', Punctuation, '#pop'),
+ (ncname, Name.Variable),
+ (stringdouble, String.Double),
+ (stringsingle, String.Single),
+ ],
+ 'closekindtest': [
+ (r'\(:', Comment, 'comment'),
+ (r'(\))', popstate_callback),
+ (r',', Punctuation),
+ (r'(\{)', pushstate_operator_root_callback),
+ (r'\?', Punctuation),
+ ],
+ 'xml_comment': [
+ (r'(-->)', popstate_xmlcomment_callback),
+ (r'[^-]{1,2}', Literal),
+ (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' +
+ unirange(0x10000, 0x10ffff), Literal),
+ ],
+ 'processing_instruction': [
+ (r'\s+', Text, 'processing_instruction_content'),
+ (r'\?>', String.Doc, '#pop'),
+ (pitarget, Name),
+ ],
+ 'processing_instruction_content': [
+ (r'\?>', String.Doc, '#pop'),
+ (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' +
+ unirange(0x10000, 0x10ffff), Literal),
+ ],
+ 'cdata_section': [
+ (r']]>', String.Doc, '#pop'),
+ (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' +
+ unirange(0x10000, 0x10ffff), Literal),
+ ],
+ 'start_tag': [
+ include('whitespace'),
+ (r'(/>)', popstate_tag_callback),
+ (r'>', Name.Tag, 'element_content'),
+ (r'"', Punctuation, 'quot_attribute_content'),
+ (r"'", Punctuation, 'apos_attribute_content'),
+ (r'=', Operator),
+ (qname, Name.Tag),
+ ],
+ 'quot_attribute_content': [
+ (r'"', Punctuation, 'start_tag'),
+ (r'(\{)', pushstate_root_callback),
+ (r'""', Name.Attribute),
+ (quotattrcontentchar, Name.Attribute),
+ (entityref, Name.Attribute),
+ (charref, Name.Attribute),
+ (r'\{\{|\}\}', Name.Attribute),
+ ],
+ 'apos_attribute_content': [
+ (r"'", Punctuation, 'start_tag'),
+ (r'\{', Punctuation, 'root'),
+ (r"''", Name.Attribute),
+ (aposattrcontentchar, Name.Attribute),
+ (entityref, Name.Attribute),
+ (charref, Name.Attribute),
+ (r'\{\{|\}\}', Name.Attribute),
+ ],
+ 'element_content': [
+ (r'</', Name.Tag, 'end_tag'),
+ (r'(\{)', pushstate_root_callback),
+ (r'(<!--)', pushstate_element_content_xmlcomment_callback),
+ (r'(<\?)', pushstate_element_content_processing_instruction_callback),
+ (r'(<!\[CDATA\[)', pushstate_element_content_cdata_section_callback),
+ (r'(<)', pushstate_element_content_starttag_callback),
+ (elementcontentchar, Literal),
+ (entityref, Literal),
+ (charref, Literal),
+ (r'\{\{|\}\}', Literal),
+ ],
+ 'end_tag': [
+ include('whitespace'),
+ (r'(>)', popstate_tag_callback),
+ (qname, Name.Tag),
+ ],
+ 'xmlspace_decl': [
+ include('whitespace'),
+ (r'\(:', Comment, 'comment'),
+ (r'preserve|strip', Keyword, '#pop'),
+ ],
+ 'declareordering': [
+ (r'\(:', Comment, 'comment'),
+ include('whitespace'),
+ (r'ordered|unordered', Keyword, '#pop'),
+ ],
+ 'xqueryversion': [
+ include('whitespace'),
+ (r'\(:', Comment, 'comment'),
+ (stringdouble, String.Double),
+ (stringsingle, String.Single),
+ (r'encoding', Keyword),
+ (r';', Punctuation, '#pop'),
+ ],
+ 'pragma': [
+ (qname, Name.Variable, 'pragmacontents'),
+ ],
+ 'pragmacontents': [
+ (r'#\)', Punctuation, 'operator'),
+ (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' +
+ unirange(0x10000, 0x10ffff), Literal),
+ (r'(\s+)', Text),
+ ],
+ 'occurrenceindicator': [
+ include('whitespace'),
+ (r'\(:', Comment, 'comment'),
+ (r'\*|\?|\+', Operator, 'operator'),
+ (r':=', Operator, 'root'),
+ default('operator'),
+ ],
+ 'option': [
+ include('whitespace'),
+ (qname, Name.Variable, '#pop'),
+ ],
+ 'qname_braren': [
+ include('whitespace'),
+ (r'(\{)', pushstate_operator_root_callback),
+ (r'(\()', Punctuation, 'root'),
+ ],
+ 'element_qname': [
+ (qname, Name.Variable, 'root'),
+ ],
+ 'attribute_qname': [
+ (qname, Name.Variable, 'root'),
+ ],
+ 'root': [
+ include('whitespace'),
+ (r'\(:', Comment, 'comment'),
+
+ # handle operator state
+ # order on numbers matters - handle most complex first
+ (r'\d+(\.\d*)?[eE][+-]?\d+', Number.Float, 'operator'),
+ (r'(\.\d+)[eE][+-]?\d+', Number.Float, 'operator'),
+ (r'(\.\d+|\d+\.\d*)', Number.Float, 'operator'),
+ (r'(\d+)', Number.Integer, 'operator'),
+ (r'(\.\.|\.|\))', Punctuation, 'operator'),
+ (r'(declare)(\s+)(construction)',
+ bygroups(Keyword.Declaration, Text, Keyword.Declaration), 'operator'),
+ (r'(declare)(\s+)(default)(\s+)(order)',
+ bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'operator'),
+ (r'(declare)(\s+)(context)(\s+)(item)',
+ bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'operator'),
+ (ncname + ':\*', Name, 'operator'),
+ ('\*:'+ncname, Name.Tag, 'operator'),
+ ('\*', Name.Tag, 'operator'),
+ (stringdouble, String.Double, 'operator'),
+ (stringsingle, String.Single, 'operator'),
+
+ (r'(\}|\])', popstate_callback),
+
+ # NAMESPACE DECL
+ (r'(declare)(\s+)(default)(\s+)(collation)',
+ bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration)),
+ (r'(module|declare)(\s+)(namespace)',
+ bygroups(Keyword.Declaration, Text, Keyword.Declaration), 'namespacedecl'),
+ (r'(declare)(\s+)(base-uri)',
+ bygroups(Keyword.Declaration, Text, Keyword.Declaration), 'namespacedecl'),
+
+ # NAMESPACE KEYWORD
+ (r'(declare)(\s+)(default)(\s+)(element|function)',
+ bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'namespacekeyword'),
+ (r'(import)(\s+)(schema|module)',
+ bygroups(Keyword.Pseudo, Text, Keyword.Pseudo), 'namespacekeyword'),
+ (r'(declare)(\s+)(copy-namespaces)',
+ bygroups(Keyword.Declaration, Text, Keyword.Declaration), 'namespacekeyword'),
+
+ # VARNAMEs
+ (r'(for|let|some|every)(\s+)(\$)',
+ bygroups(Keyword, Text, Name.Variable), 'varname'),
+ (r'(for)(\s+)(tumbling|sliding)(\s+)(window)(\s+)(\$)',
+ bygroups(Keyword, Text, Keyword, Text, Keyword, Text, Name.Variable), 'varname'),
+ (r'\$', Name.Variable, 'varname'),
+ (r'(declare)(\s+)(variable)(\s+)(\$)',
+ bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Name.Variable), 'varname'),
+
+ # ANNOTATED GLOBAL VARIABLES AND FUNCTIONS
+ (r'(declare)(\s+)(\%)', bygroups(Keyword.Declaration, Text, Name.Decorator), 'annotationname'),
+
+ # ITEMTYPE
+ (r'(\))(\s+)(as)', bygroups(Operator, Text, Keyword), 'itemtype'),
+
+ (r'(element|attribute|schema-element|schema-attribute|comment|'
+ r'text|node|document-node|empty-sequence)(\s+)(\()',
+ pushstate_operator_kindtest_callback),
+
+ (r'(processing-instruction)(\s+)(\()',
+ pushstate_operator_kindtestforpi_callback),
+
+ (r'(<!--)', pushstate_operator_xmlcomment_callback),
+
+ (r'(<\?)', pushstate_operator_processing_instruction_callback),
+
+ (r'(<!\[CDATA\[)', pushstate_operator_cdata_section_callback),
+
+ # (r'</', Name.Tag, 'end_tag'),
+ (r'(<)', pushstate_operator_starttag_callback),
+
+ (r'(declare)(\s+)(boundary-space)',
+ bygroups(Keyword.Declaration, Text, Keyword.Declaration), 'xmlspace_decl'),
+
+ (r'(validate)(\s+)(lax|strict)',
+ pushstate_operator_root_validate_withmode),
+ (r'(validate)(\s*)(\{)', pushstate_operator_root_validate),
+ (r'(typeswitch)(\s*)(\()', bygroups(Keyword, Text, Punctuation)),
+ (r'(switch)(\s*)(\()', bygroups(Keyword, Text, Punctuation)),
+ (r'(element|attribute|namespace)(\s*)(\{)',
+ pushstate_operator_root_construct_callback),
+
+ (r'(document|text|processing-instruction|comment)(\s*)(\{)',
+ pushstate_operator_root_construct_callback),
+ # ATTRIBUTE
+ (r'(attribute)(\s+)(?=' + qname + r')',
+ bygroups(Keyword, Text), 'attribute_qname'),
+ # ELEMENT
+ (r'(element)(\s+)(?=' + qname + r')',
+ bygroups(Keyword, Text), 'element_qname'),
+ # PROCESSING_INSTRUCTION
+ (r'(processing-instruction|namespace)(\s+)(' + ncname + r')(\s*)(\{)',
+ bygroups(Keyword, Text, Name.Variable, Text, Punctuation),
+ 'operator'),
+
+ (r'(declare|define)(\s+)(function)',
+ bygroups(Keyword.Declaration, Text, Keyword.Declaration)),
+
+ (r'(\{|\[)', pushstate_operator_root_callback),
+
+ (r'(unordered|ordered)(\s*)(\{)',
+ pushstate_operator_order_callback),
+
+ (r'(map|array)(\s*)(\{)',
+ pushstate_operator_map_callback),
+
+ (r'(declare)(\s+)(ordering)',
+ bygroups(Keyword.Declaration, Text, Keyword.Declaration), 'declareordering'),
+
+ (r'(xquery)(\s+)(version)',
+ bygroups(Keyword.Pseudo, Text, Keyword.Pseudo), 'xqueryversion'),
+
+ (r'(\(#)(\s*)', bygroups(Punctuation, Text), 'pragma'),
+
+ # sometimes return can occur in root state
+ (r'return', Keyword),
+
+ (r'(declare)(\s+)(option)', bygroups(Keyword.Declaration, Text, Keyword.Declaration),
+ 'option'),
+
+ # URI LITERALS - single and double quoted
+ (r'(at)(\s+)('+stringdouble+')', String.Double, 'namespacedecl'),
+ (r'(at)(\s+)('+stringsingle+')', String.Single, 'namespacedecl'),
+
+ (r'(ancestor-or-self|ancestor|attribute|child|descendant-or-self)(::)',
+ bygroups(Keyword, Punctuation)),
+ (r'(descendant|following-sibling|following|parent|preceding-sibling'
+ r'|preceding|self)(::)', bygroups(Keyword, Punctuation)),
+
+ (r'(if)(\s*)(\()', bygroups(Keyword, Text, Punctuation)),
+
+ (r'then|else', Keyword),
+
+ # eXist specific XQUF
+ (r'(update)(\s*)(insert|delete|replace|value|rename)', bygroups(Keyword, Text, Keyword)),
+ (r'(into|following|preceding|with)', Keyword),
+
+ # Marklogic specific
+ (r'(try)(\s*)', bygroups(Keyword, Text), 'root'),
+ (r'(catch)(\s*)(\()(\$)',
+ bygroups(Keyword, Text, Punctuation, Name.Variable), 'varname'),
+
+
+ (r'(@'+qname+')', Name.Attribute, 'operator'),
+ (r'(@'+ncname+')', Name.Attribute, 'operator'),
+ (r'@\*:'+ncname, Name.Attribute, 'operator'),
+ (r'@\*', Name.Attribute, 'operator'),
+ (r'(@)', Name.Attribute, 'operator'),
+
+ (r'//|/|\+|-|;|,|\(|\)', Punctuation),
+
+ # STANDALONE QNAMES
+ (qname + r'(?=\s*\{)', Name.Tag, 'qname_braren'),
+ (qname + r'(?=\s*\([^:])', Name.Function, 'qname_braren'),
+ (r'(' + qname + ')(#)([0-9]+)', bygroups(Name.Function, Keyword.Type, Number.Integer)),
+ (qname, Name.Tag, 'operator'),
+ ]
+ }
+
+
+class QmlLexer(RegexLexer):
+ """
+ For QML files. See http://doc.qt.digia.com/4.7/qdeclarativeintroduction.html.
+
+ .. versionadded:: 1.6
+ """
+
+ # QML is based on javascript, so much of this is taken from the
+ # JavascriptLexer above.
+
+ name = 'QML'
+ aliases = ['qml', 'qbs']
+ filenames = ['*.qml', '*.qbs']
+ mimetypes = ['application/x-qml', 'application/x-qt.qbs+qml']
+
+ # pasted from JavascriptLexer, with some additions
+ flags = re.DOTALL | re.MULTILINE
+
+ tokens = {
+ 'commentsandwhitespace': [
+ (r'\s+', Text),
+ (r'<!--', Comment),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*.*?\*/', Comment.Multiline)
+ ],
+ 'slashstartsregex': [
+ include('commentsandwhitespace'),
+ (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+ r'([gim]+\b|\B)', String.Regex, '#pop'),
+ (r'(?=/)', Text, ('#pop', 'badregex')),
+ default('#pop')
+ ],
+ 'badregex': [
+ (r'\n', Text, '#pop')
+ ],
+ 'root': [
+ (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'),
+ include('commentsandwhitespace'),
+ (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|'
+ r'(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'),
+ (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
+ (r'[})\].]', Punctuation),
+
+ # QML insertions
+ (r'\bid\s*:\s*[A-Za-z][\w.]*', Keyword.Declaration,
+ 'slashstartsregex'),
+ (r'\b[A-Za-z][\w.]*\s*:', Keyword, 'slashstartsregex'),
+
+ # the rest from JavascriptLexer
+ (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|'
+ r'throw|try|catch|finally|new|delete|typeof|instanceof|void|'
+ r'this)\b', Keyword, 'slashstartsregex'),
+ (r'(var|let|with|function)\b', Keyword.Declaration, 'slashstartsregex'),
+ (r'(abstract|boolean|byte|char|class|const|debugger|double|enum|export|'
+ r'extends|final|float|goto|implements|import|int|interface|long|native|'
+ r'package|private|protected|public|short|static|super|synchronized|throws|'
+ r'transient|volatile)\b', Keyword.Reserved),
+ (r'(true|false|null|NaN|Infinity|undefined)\b', Keyword.Constant),
+ (r'(Array|Boolean|Date|Error|Function|Math|netscape|'
+ r'Number|Object|Packages|RegExp|String|sun|decodeURI|'
+ r'decodeURIComponent|encodeURI|encodeURIComponent|'
+ r'Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|'
+ r'window)\b', Name.Builtin),
+ (r'[$a-zA-Z_]\w*', Name.Other),
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+', Number.Integer),
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
+ (r"'(\\\\|\\'|[^'])*'", String.Single),
+ ]
+ }
+
+
+class CirruLexer(RegexLexer):
+ """
+ Syntax rules of Cirru can be found at:
+ http://cirru.org/
+
+ * using ``()`` for expressions, but restricted in a same line
+ * using ``""`` for strings, with ``\`` for escaping chars
+ * using ``$`` as folding operator
+ * using ``,`` as unfolding operator
+ * using indentations for nested blocks
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Cirru'
+ aliases = ['cirru']
+ filenames = ['*.cirru']
+ mimetypes = ['text/x-cirru']
+ flags = re.MULTILINE
+
+ tokens = {
+ 'string': [
+ (r'[^"\\\n]', String),
+ (r'\\', String.Escape, 'escape'),
+ (r'"', String, '#pop'),
+ ],
+ 'escape': [
+ (r'.', String.Escape, '#pop'),
+ ],
+ 'function': [
+ (r'\,', Operator, '#pop'),
+ (r'[^\s"()]+', Name.Function, '#pop'),
+ (r'\)', Operator, '#pop'),
+ (r'(?=\n)', Text, '#pop'),
+ (r'\(', Operator, '#push'),
+ (r'"', String, ('#pop', 'string')),
+ (r'[ ]+', Text.Whitespace),
+ ],
+ 'line': [
+ (r'(?<!\w)\$(?!\w)', Operator, 'function'),
+ (r'\(', Operator, 'function'),
+ (r'\)', Operator),
+ (r'\n', Text, '#pop'),
+ (r'"', String, 'string'),
+ (r'[ ]+', Text.Whitespace),
+ (r'[+-]?[\d.]+\b', Number),
+ (r'[^\s"()]+', Name.Variable)
+ ],
+ 'root': [
+ (r'^\n+', Text.Whitespace),
+ default(('line', 'function')),
+ ]
+ }
+
+
+class SlimLexer(ExtendedRegexLexer):
+ """
+ For Slim markup.
+
+ .. versionadded:: 2.0
+ """
+
+ name = 'Slim'
+ aliases = ['slim']
+ filenames = ['*.slim']
+ mimetypes = ['text/x-slim']
+
+ flags = re.IGNORECASE
+ _dot = r'(?: \|\n(?=.* \|)|.)'
+ tokens = {
+ 'root': [
+ (r'[ \t]*\n', Text),
+ (r'[ \t]*', _indentation),
+ ],
+
+ 'css': [
+ (r'\.[\w:-]+', Name.Class, 'tag'),
+ (r'\#[\w:-]+', Name.Function, 'tag'),
+ ],
+
+ 'eval-or-plain': [
+ (r'([ \t]*==?)(.*\n)',
+ bygroups(Punctuation, using(RubyLexer)),
+ 'root'),
+ (r'[ \t]+[\w:-]+(?==)', Name.Attribute, 'html-attributes'),
+ default('plain'),
+ ],
+
+ 'content': [
+ include('css'),
+ (r'[\w:-]+:[ \t]*\n', Text, 'plain'),
+ (r'(-)(.*\n)',
+ bygroups(Punctuation, using(RubyLexer)),
+ '#pop'),
+ (r'\|' + _dot + r'*\n', _starts_block(Text, 'plain'), '#pop'),
+ (r'/' + _dot + r'*\n', _starts_block(Comment.Preproc, 'slim-comment-block'), '#pop'),
+ (r'[\w:-]+', Name.Tag, 'tag'),
+ include('eval-or-plain'),
+ ],
+
+ 'tag': [
+ include('css'),
+ (r'[<>]{1,2}(?=[ \t=])', Punctuation),
+ (r'[ \t]+\n', Punctuation, '#pop:2'),
+ include('eval-or-plain'),
+ ],
+
+ 'plain': [
+ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Text),
+ (r'(#\{)(.*?)(\})',
+ bygroups(String.Interpol, using(RubyLexer), String.Interpol)),
+ (r'\n', Text, 'root'),
+ ],
+
+ 'html-attributes': [
+ (r'=', Punctuation),
+ (r'"[^"]+"', using(RubyLexer), 'tag'),
+ (r'\'[^\']+\'', using(RubyLexer), 'tag'),
+ (r'\w+', Text, 'tag'),
+ ],
+
+ 'slim-comment-block': [
+ (_dot + '+', Comment.Preproc),
+ (r'\n', Text, 'root'),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/whiley.py b/wandb/vendor/pygments/lexers/whiley.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d0e8ab886337424a544727a74588a7c2db98475
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/whiley.py
@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.whiley
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the Whiley language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Comment, Keyword, Name, Number, Operator, \
+ Punctuation, String, Text
+
+__all__ = ['WhileyLexer']
+
+
+class WhileyLexer(RegexLexer):
+ """
+ Lexer for the Whiley programming language.
+
+ .. versionadded:: 2.2
+ """
+ name = 'Whiley'
+ filenames = ['*.whiley']
+ aliases = ['whiley']
+ mimetypes = ['text/x-whiley']
+
+ # See the language specification:
+ # http://whiley.org/download/WhileyLanguageSpec.pdf
+
+ tokens = {
+ 'root': [
+ # Whitespace
+ (r'\s+', Text),
+
+ # Comments
+ (r'//.*', Comment.Single),
+ # don't parse empty comment as doc comment
+ (r'/\*\*/', Comment.Multiline),
+ (r'(?s)/\*\*.*?\*/', String.Doc),
+ (r'(?s)/\*.*?\*/', Comment.Multiline),
+
+ # Keywords
+ (words((
+ 'if', 'else', 'while', 'for', 'do', 'return',
+ 'switch', 'case', 'default', 'break', 'continue',
+ 'requires', 'ensures', 'where', 'assert', 'assume',
+ 'all', 'no', 'some', 'in', 'is', 'new',
+ 'throw', 'try', 'catch', 'debug', 'skip', 'fail',
+ 'finite', 'total'), suffix=r'\b'), Keyword.Reserved),
+ (words((
+ 'function', 'method', 'public', 'private', 'protected',
+ 'export', 'native'), suffix=r'\b'), Keyword.Declaration),
+ # "constant" & "type" are not keywords unless used in declarations
+ (r'(constant|type)(\s+)([a-zA-Z_]\w*)(\s+)(is)\b',
+ bygroups(Keyword.Declaration, Text, Name, Text, Keyword.Reserved)),
+ (r'(true|false|null)\b', Keyword.Constant),
+ (r'(bool|byte|int|real|any|void)\b', Keyword.Type),
+ # "from" is not a keyword unless used with import
+ (r'(import)(\s+)(\*)([^\S\n]+)(from)\b',
+ bygroups(Keyword.Namespace, Text, Punctuation, Text, Keyword.Namespace)),
+ (r'(import)(\s+)([a-zA-Z_]\w*)([^\S\n]+)(from)\b',
+ bygroups(Keyword.Namespace, Text, Name, Text, Keyword.Namespace)),
+ (r'(package|import)\b', Keyword.Namespace),
+
+ # standard library: https://github.com/Whiley/WhileyLibs/
+ (words((
+ # types defined in whiley.lang.Int
+ 'i8', 'i16', 'i32', 'i64',
+ 'u8', 'u16', 'u32', 'u64',
+ 'uint', 'nat',
+
+ # whiley.lang.Any
+ 'toString'), suffix=r'\b'), Name.Builtin),
+
+ # byte literal
+ (r'[01]+b', Number.Bin),
+
+ # decimal literal
+ (r'[0-9]+\.[0-9]+', Number.Float),
+ # match "1." but not ranges like "3..5"
+ (r'[0-9]+\.(?!\.)', Number.Float),
+
+ # integer literal
+ (r'0x[0-9a-fA-F]+', Number.Hex),
+ (r'[0-9]+', Number.Integer),
+
+ # character literal
+ (r"""'[^\\]'""", String.Char),
+ (r"""(')(\\['"\\btnfr])(')""",
+ bygroups(String.Char, String.Escape, String.Char)),
+
+ # string literal
+ (r'"', String, 'string'),
+
+ # operators and punctuation
+ (r'[{}()\[\],.;]', Punctuation),
+ (u'[+\\-*/%&|<>^!~@=:?'
+ # unicode operators
+ u'\u2200\u2203\u2205\u2282\u2286\u2283\u2287'
+ u'\u222A\u2229\u2264\u2265\u2208\u2227\u2228'
+ u']', Operator),
+
+ # identifier
+ (r'[a-zA-Z_]\w*', Name),
+ ],
+ 'string': [
+ (r'"', String, '#pop'),
+ (r'\\[btnfr]', String.Escape),
+ (r'\\u[0-9a-fA-F]{4}', String.Escape),
+ (r'\\.', String),
+ (r'[^\\"]+', String),
+ ],
+ }
diff --git a/wandb/vendor/pygments/lexers/x10.py b/wandb/vendor/pygments/lexers/x10.py
new file mode 100644
index 0000000000000000000000000000000000000000..1c63326dc4e8b4d4d617c1321e66f2d8fe05f3db
--- /dev/null
+++ b/wandb/vendor/pygments/lexers/x10.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.lexers.x10
+ ~~~~~~~~~~~~~~~~~~~
+
+ Lexers for the X10 programming language.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+ Number, Punctuation, Error
+
+__all__ = ['X10Lexer']
+
+class X10Lexer(RegexLexer):
+ """
+ For the X10 language.
+
+ .. versionadded:: 0.1
+ """
+
+ name = 'X10'
+ aliases = ['x10', 'xten']
+ filenames = ['*.x10']
+ mimetypes = ['text/x-x10']
+
+ keywords = (
+ 'as', 'assert', 'async', 'at', 'athome', 'ateach', 'atomic',
+ 'break', 'case', 'catch', 'class', 'clocked', 'continue',
+ 'def', 'default', 'do', 'else', 'final', 'finally', 'finish',
+ 'for', 'goto', 'haszero', 'here', 'if', 'import', 'in',
+ 'instanceof', 'interface', 'isref', 'new', 'offer',
+ 'operator', 'package', 'return', 'struct', 'switch', 'throw',
+ 'try', 'type', 'val', 'var', 'when', 'while'
+ )
+
+ types = (
+ 'void'
+ )
+
+ values = (
+ 'false', 'null', 'self', 'super', 'this', 'true'
+ )
+
+ modifiers = (
+ 'abstract', 'extends', 'implements', 'native', 'offers',
+ 'private', 'property', 'protected', 'public', 'static',
+ 'throws', 'transient'
+ )
+
+ tokens = {
+ 'root': [
+ (r'[^\S\n]+', Text),
+ (r'//.*?\n', Comment.Single),
+ (r'/\*(.|\n)*?\*/', Comment.Multiline),
+ (r'\b(%s)\b' % '|'.join(keywords), Keyword),
+ (r'\b(%s)\b' % '|'.join(types), Keyword.Type),
+ (r'\b(%s)\b' % '|'.join(values), Keyword.Constant),
+ (r'\b(%s)\b' % '|'.join(modifiers), Keyword.Declaration),
+ (r'"(\\\\|\\"|[^"])*"', String),
+ (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Char),
+ (r'.', Text)
+ ],
+ }
diff --git a/wandb/vendor/pygments/modeline.py b/wandb/vendor/pygments/modeline.py
new file mode 100644
index 0000000000000000000000000000000000000000..9f8d5dab3d413cb8327ea91bb14a0bb49bb656a2
--- /dev/null
+++ b/wandb/vendor/pygments/modeline.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.modeline
+ ~~~~~~~~~~~~~~~~~
+
+ A simple modeline parser (based on pymodeline).
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+__all__ = ['get_filetype_from_buffer']
+
+
+modeline_re = re.compile(r'''
+ (?: vi | vim | ex ) (?: [<=>]? \d* )? :
+ .* (?: ft | filetype | syn | syntax ) = ( [^:\s]+ )
+''', re.VERBOSE)
+
+
+def get_filetype_from_line(l):
+ m = modeline_re.search(l)
+ if m:
+ return m.group(1)
+
+
+def get_filetype_from_buffer(buf, max_lines=5):
+ """
+ Scan the buffer for modelines and return filetype if one is found.
+ """
+ lines = buf.splitlines()
+ for l in lines[-1:-max_lines-1:-1]:
+ ret = get_filetype_from_line(l)
+ if ret:
+ return ret
+ for i in range(max_lines, -1, -1):
+ if i < len(lines):
+ ret = get_filetype_from_line(lines[i])
+ if ret:
+ return ret
+
+ return None
diff --git a/wandb/vendor/pygments/plugin.py b/wandb/vendor/pygments/plugin.py
new file mode 100644
index 0000000000000000000000000000000000000000..7987d646315f357df9f2de884e298475d927a2e8
--- /dev/null
+++ b/wandb/vendor/pygments/plugin.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.plugin
+ ~~~~~~~~~~~~~~~
+
+ Pygments setuptools plugin interface. The methods defined
+ here also work if setuptools isn't installed but they just
+ return nothing.
+
+ lexer plugins::
+
+ [pygments.lexers]
+ yourlexer = yourmodule:YourLexer
+
+ formatter plugins::
+
+ [pygments.formatters]
+ yourformatter = yourformatter:YourFormatter
+ /.ext = yourformatter:YourFormatter
+
+ As you can see, you can define extensions for the formatter
+ with a leading slash.
+
+ syntax plugins::
+
+ [pygments.styles]
+ yourstyle = yourstyle:YourStyle
+
+ filter plugin::
+
+ [pygments.filter]
+ yourfilter = yourfilter:YourFilter
+
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+LEXER_ENTRY_POINT = 'pygments.lexers'
+FORMATTER_ENTRY_POINT = 'pygments.formatters'
+STYLE_ENTRY_POINT = 'pygments.styles'
+FILTER_ENTRY_POINT = 'pygments.filters'
+
+def iter_entry_points(group_name):
+ try:
+ import pkg_resources
+ except ImportError:
+ return []
+
+ return pkg_resources.iter_entry_points(group_name)
+
+def find_plugin_lexers():
+ for entrypoint in iter_entry_points(LEXER_ENTRY_POINT):
+ yield entrypoint.load()
+
+
+def find_plugin_formatters():
+ for entrypoint in iter_entry_points(FORMATTER_ENTRY_POINT):
+ yield entrypoint.name, entrypoint.load()
+
+
+def find_plugin_styles():
+ for entrypoint in iter_entry_points(STYLE_ENTRY_POINT):
+ yield entrypoint.name, entrypoint.load()
+
+
+def find_plugin_filters():
+ for entrypoint in iter_entry_points(FILTER_ENTRY_POINT):
+ yield entrypoint.name, entrypoint.load()
diff --git a/wandb/vendor/pygments/regexopt.py b/wandb/vendor/pygments/regexopt.py
new file mode 100644
index 0000000000000000000000000000000000000000..dcfae2fdbb3da6b5b5c06a31726368c5e5ed67b2
--- /dev/null
+++ b/wandb/vendor/pygments/regexopt.py
@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.regexopt
+ ~~~~~~~~~~~~~~~~~
+
+ An algorithm that generates optimized regexes for matching long lists of
+ literal strings.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+from re import escape
+from os.path import commonprefix
+from itertools import groupby
+from operator import itemgetter
+
+CS_ESCAPE = re.compile(r'[\^\\\-\]]')
+FIRST_ELEMENT = itemgetter(0)
+
+
+def make_charset(letters):
+ return '[' + CS_ESCAPE.sub(lambda m: '\\' + m.group(), ''.join(letters)) + ']'
+
+
+def regex_opt_inner(strings, open_paren):
+ """Return a regex that matches any string in the sorted list of strings."""
+ close_paren = open_paren and ')' or ''
+ # print strings, repr(open_paren)
+ if not strings:
+ # print '-> nothing left'
+ return ''
+ first = strings[0]
+ if len(strings) == 1:
+ # print '-> only 1 string'
+ return open_paren + escape(first) + close_paren
+ if not first:
+ # print '-> first string empty'
+ return open_paren + regex_opt_inner(strings[1:], '(?:') \
+ + '?' + close_paren
+ if len(first) == 1:
+ # multiple one-char strings? make a charset
+ oneletter = []
+ rest = []
+ for s in strings:
+ if len(s) == 1:
+ oneletter.append(s)
+ else:
+ rest.append(s)
+ if len(oneletter) > 1: # do we have more than one oneletter string?
+ if rest:
+ # print '-> 1-character + rest'
+ return open_paren + regex_opt_inner(rest, '') + '|' \
+ + make_charset(oneletter) + close_paren
+ # print '-> only 1-character'
+ return open_paren + make_charset(oneletter) + close_paren
+ prefix = commonprefix(strings)
+ if prefix:
+ plen = len(prefix)
+ # we have a prefix for all strings
+ # print '-> prefix:', prefix
+ return open_paren + escape(prefix) \
+ + regex_opt_inner([s[plen:] for s in strings], '(?:') \
+ + close_paren
+ # is there a suffix?
+ strings_rev = [s[::-1] for s in strings]
+ suffix = commonprefix(strings_rev)
+ if suffix:
+ slen = len(suffix)
+ # print '-> suffix:', suffix[::-1]
+ return open_paren \
+ + regex_opt_inner(sorted(s[:-slen] for s in strings), '(?:') \
+ + escape(suffix[::-1]) + close_paren
+ # recurse on common 1-string prefixes
+ # print '-> last resort'
+ return open_paren + \
+ '|'.join(regex_opt_inner(list(group[1]), '')
+ for group in groupby(strings, lambda s: s[0] == first[0])) \
+ + close_paren
+
+
+def regex_opt(strings, prefix='', suffix=''):
+ """Return a compiled regex that matches any string in the given list.
+
+ The strings to match must be literal strings, not regexes. They will be
+ regex-escaped.
+
+ *prefix* and *suffix* are pre- and appended to the final regex.
+ """
+ strings = sorted(strings)
+ return prefix + regex_opt_inner(strings, '(') + suffix
diff --git a/wandb/vendor/pygments/scanner.py b/wandb/vendor/pygments/scanner.py
new file mode 100644
index 0000000000000000000000000000000000000000..3350ac8e3b56b811838948a0bd646b51ccfe8c37
--- /dev/null
+++ b/wandb/vendor/pygments/scanner.py
@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.scanner
+ ~~~~~~~~~~~~~~~~
+
+ This library implements a regex based scanner. Some languages
+ like Pascal are easy to parse but have some keywords that
+ depend on the context. Because of this it's impossible to lex
+ that just by using a regular expression lexer like the
+ `RegexLexer`.
+
+ Have a look at the `DelphiLexer` to get an idea of how to use
+ this scanner.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+import re
+
+
+class EndOfText(RuntimeError):
+ """
+ Raise if end of text is reached and the user
+ tried to call a match function.
+ """
+
+
+class Scanner(object):
+ """
+ Simple scanner
+
+ All method patterns are regular expression strings (not
+ compiled expressions!)
+ """
+
+ def __init__(self, text, flags=0):
+ """
+ :param text: The text which should be scanned
+ :param flags: default regular expression flags
+ """
+ self.data = text
+ self.data_length = len(text)
+ self.start_pos = 0
+ self.pos = 0
+ self.flags = flags
+ self.last = None
+ self.match = None
+ self._re_cache = {}
+
+ def eos(self):
+ """`True` if the scanner reached the end of text."""
+ return self.pos >= self.data_length
+ eos = property(eos, eos.__doc__)
+
+ def check(self, pattern):
+ """
+ Apply `pattern` on the current position and return
+ the match object. (Doesn't touch pos). Use this for
+ lookahead.
+ """
+ if self.eos:
+ raise EndOfText()
+ if pattern not in self._re_cache:
+ self._re_cache[pattern] = re.compile(pattern, self.flags)
+ return self._re_cache[pattern].match(self.data, self.pos)
+
+ def test(self, pattern):
+ """Apply a pattern on the current position and check
+ if it patches. Doesn't touch pos.
+ """
+ return self.check(pattern) is not None
+
+ def scan(self, pattern):
+ """
+ Scan the text for the given pattern and update pos/match
+ and related fields. The return value is a boolen that
+ indicates if the pattern matched. The matched value is
+ stored on the instance as ``match``, the last value is
+ stored as ``last``. ``start_pos`` is the position of the
+ pointer before the pattern was matched, ``pos`` is the
+ end position.
+ """
+ if self.eos:
+ raise EndOfText()
+ if pattern not in self._re_cache:
+ self._re_cache[pattern] = re.compile(pattern, self.flags)
+ self.last = self.match
+ m = self._re_cache[pattern].match(self.data, self.pos)
+ if m is None:
+ return False
+ self.start_pos = m.start()
+ self.pos = m.end()
+ self.match = m.group()
+ return True
+
+ def get_char(self):
+ """Scan exactly one char."""
+ self.scan('.')
+
+ def __repr__(self):
+ return '<%s %d/%d>' % (
+ self.__class__.__name__,
+ self.pos,
+ self.data_length
+ )
diff --git a/wandb/vendor/pygments/sphinxext.py b/wandb/vendor/pygments/sphinxext.py
new file mode 100644
index 0000000000000000000000000000000000000000..f962f8c6b3746b9d86eb50fc06cd1c280c143601
--- /dev/null
+++ b/wandb/vendor/pygments/sphinxext.py
@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.sphinxext
+ ~~~~~~~~~~~~~~~~~~
+
+ Sphinx extension to generate automatic documentation of lexers,
+ formatters and filters.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import print_function
+
+import sys
+
+from docutils import nodes
+from docutils.statemachine import ViewList
+from sphinx.util.compat import Directive
+from sphinx.util.nodes import nested_parse_with_titles
+
+
+MODULEDOC = '''
+.. module:: %s
+
+%s
+%s
+'''
+
+LEXERDOC = '''
+.. class:: %s
+
+ :Short names: %s
+ :Filenames: %s
+ :MIME types: %s
+
+ %s
+
+'''
+
+FMTERDOC = '''
+.. class:: %s
+
+ :Short names: %s
+ :Filenames: %s
+
+ %s
+
+'''
+
+FILTERDOC = '''
+.. class:: %s
+
+ :Name: %s
+
+ %s
+
+'''
+
+
+class PygmentsDoc(Directive):
+ """
+ A directive to collect all lexers/formatters/filters and generate
+ autoclass directives for them.
+ """
+ has_content = False
+ required_arguments = 1
+ optional_arguments = 0
+ final_argument_whitespace = False
+ option_spec = {}
+
+ def run(self):
+ self.filenames = set()
+ if self.arguments[0] == 'lexers':
+ out = self.document_lexers()
+ elif self.arguments[0] == 'formatters':
+ out = self.document_formatters()
+ elif self.arguments[0] == 'filters':
+ out = self.document_filters()
+ else:
+ raise Exception('invalid argument for "pygmentsdoc" directive')
+ node = nodes.compound()
+ vl = ViewList(out.split('\n'), source='')
+ nested_parse_with_titles(self.state, vl, node)
+ for fn in self.filenames:
+ self.state.document.settings.record_dependencies.add(fn)
+ return node.children
+
+ def document_lexers(self):
+ from pygments.lexers._mapping import LEXERS
+ out = []
+ modules = {}
+ moduledocstrings = {}
+ for classname, data in sorted(LEXERS.items(), key=lambda x: x[0]):
+ module = data[0]
+ mod = __import__(module, None, None, [classname])
+ self.filenames.add(mod.__file__)
+ cls = getattr(mod, classname)
+ if not cls.__doc__:
+ print("Warning: %s does not have a docstring." % classname)
+ docstring = cls.__doc__
+ if isinstance(docstring, bytes):
+ docstring = docstring.decode('utf8')
+ modules.setdefault(module, []).append((
+ classname,
+ ', '.join(data[2]) or 'None',
+ ', '.join(data[3]).replace('*', '\\*').replace('_', '\\') or 'None',
+ ', '.join(data[4]) or 'None',
+ docstring))
+ if module not in moduledocstrings:
+ moddoc = mod.__doc__
+ if isinstance(moddoc, bytes):
+ moddoc = moddoc.decode('utf8')
+ moduledocstrings[module] = moddoc
+
+ for module, lexers in sorted(modules.items(), key=lambda x: x[0]):
+ if moduledocstrings[module] is None:
+ raise Exception("Missing docstring for %s" % (module,))
+ heading = moduledocstrings[module].splitlines()[4].strip().rstrip('.')
+ out.append(MODULEDOC % (module, heading, '-'*len(heading)))
+ for data in lexers:
+ out.append(LEXERDOC % data)
+
+ return ''.join(out)
+
+ def document_formatters(self):
+ from pygments.formatters import FORMATTERS
+
+ out = []
+ for classname, data in sorted(FORMATTERS.items(), key=lambda x: x[0]):
+ module = data[0]
+ mod = __import__(module, None, None, [classname])
+ self.filenames.add(mod.__file__)
+ cls = getattr(mod, classname)
+ docstring = cls.__doc__
+ if isinstance(docstring, bytes):
+ docstring = docstring.decode('utf8')
+ heading = cls.__name__
+ out.append(FMTERDOC % (heading, ', '.join(data[2]) or 'None',
+ ', '.join(data[3]).replace('*', '\\*') or 'None',
+ docstring))
+ return ''.join(out)
+
+ def document_filters(self):
+ from pygments.filters import FILTERS
+
+ out = []
+ for name, cls in FILTERS.items():
+ self.filenames.add(sys.modules[cls.__module__].__file__)
+ docstring = cls.__doc__
+ if isinstance(docstring, bytes):
+ docstring = docstring.decode('utf8')
+ out.append(FILTERDOC % (cls.__name__, name, docstring))
+ return ''.join(out)
+
+
+def setup(app):
+ app.add_directive('pygmentsdoc', PygmentsDoc)
diff --git a/wandb/vendor/pygments/style.py b/wandb/vendor/pygments/style.py
new file mode 100644
index 0000000000000000000000000000000000000000..879c4e05181a81069e2ba55da53b3c4f0f3d1e17
--- /dev/null
+++ b/wandb/vendor/pygments/style.py
@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.style
+ ~~~~~~~~~~~~~~
+
+ Basic style object.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.token import Token, STANDARD_TYPES
+from pygments.util import add_metaclass
+
+# Default mapping of #ansixxx to RGB colors.
+_ansimap = {
+ # dark
+ '#ansiblack': '000000',
+ '#ansidarkred': '7f0000',
+ '#ansidarkgreen': '007f00',
+ '#ansibrown': '7f7fe0',
+ '#ansidarkblue': '00007f',
+ '#ansipurple': '7f007f',
+ '#ansiteal': '007f7f',
+ '#ansilightgray': 'e5e5e5',
+ # normal
+ '#ansidarkgray': '555555',
+ '#ansired': 'ff0000',
+ '#ansigreen': '00ff00',
+ '#ansiyellow': 'ffff00',
+ '#ansiblue': '0000ff',
+ '#ansifuchsia': 'ff00ff',
+ '#ansiturquoise': '00ffff',
+ '#ansiwhite': 'ffffff',
+}
+ansicolors = set(_ansimap)
+
+
+class StyleMeta(type):
+
+ def __new__(mcs, name, bases, dct):
+ obj = type.__new__(mcs, name, bases, dct)
+ for token in STANDARD_TYPES:
+ if token not in obj.styles:
+ obj.styles[token] = ''
+
+ def colorformat(text):
+ if text in ansicolors:
+ return text
+ if text[0:1] == '#':
+ col = text[1:]
+ if len(col) == 6:
+ return col
+ elif len(col) == 3:
+ return col[0]*2 + col[1]*2 + col[2]*2
+ elif text == '':
+ return ''
+ assert False, "wrong color format %r" % text
+
+ _styles = obj._styles = {}
+
+ for ttype in obj.styles:
+ for token in ttype.split():
+ if token in _styles:
+ continue
+ ndef = _styles.get(token.parent, None)
+ styledefs = obj.styles.get(token, '').split()
+ if not ndef or token is None:
+ ndef = ['', 0, 0, 0, '', '', 0, 0, 0]
+ elif 'noinherit' in styledefs and token is not Token:
+ ndef = _styles[Token][:]
+ else:
+ ndef = ndef[:]
+ _styles[token] = ndef
+ for styledef in obj.styles.get(token, '').split():
+ if styledef == 'noinherit':
+ pass
+ elif styledef == 'bold':
+ ndef[1] = 1
+ elif styledef == 'nobold':
+ ndef[1] = 0
+ elif styledef == 'italic':
+ ndef[2] = 1
+ elif styledef == 'noitalic':
+ ndef[2] = 0
+ elif styledef == 'underline':
+ ndef[3] = 1
+ elif styledef == 'nounderline':
+ ndef[3] = 0
+ elif styledef[:3] == 'bg:':
+ ndef[4] = colorformat(styledef[3:])
+ elif styledef[:7] == 'border:':
+ ndef[5] = colorformat(styledef[7:])
+ elif styledef == 'roman':
+ ndef[6] = 1
+ elif styledef == 'sans':
+ ndef[7] = 1
+ elif styledef == 'mono':
+ ndef[8] = 1
+ else:
+ ndef[0] = colorformat(styledef)
+
+ return obj
+
+ def style_for_token(cls, token):
+ t = cls._styles[token]
+ ansicolor = bgansicolor = None
+ color = t[0]
+ if color.startswith('#ansi'):
+ ansicolor = color
+ color = _ansimap[color]
+ bgcolor = t[4]
+ if bgcolor.startswith('#ansi'):
+ bgansicolor = bgcolor
+ bgcolor = _ansimap[bgcolor]
+
+ return {
+ 'color': color or None,
+ 'bold': bool(t[1]),
+ 'italic': bool(t[2]),
+ 'underline': bool(t[3]),
+ 'bgcolor': bgcolor or None,
+ 'border': t[5] or None,
+ 'roman': bool(t[6]) or None,
+ 'sans': bool(t[7]) or None,
+ 'mono': bool(t[8]) or None,
+ 'ansicolor': ansicolor,
+ 'bgansicolor': bgansicolor,
+ }
+
+ def list_styles(cls):
+ return list(cls)
+
+ def styles_token(cls, ttype):
+ return ttype in cls._styles
+
+ def __iter__(cls):
+ for token in cls._styles:
+ yield token, cls.style_for_token(token)
+
+ def __len__(cls):
+ return len(cls._styles)
+
+
+@add_metaclass(StyleMeta)
+class Style(object):
+
+ #: overall background color (``None`` means transparent)
+ background_color = '#ffffff'
+
+ #: highlight background color
+ highlight_color = '#ffffcc'
+
+ #: Style definitions for individual token types.
+ styles = {}
diff --git a/wandb/vendor/pygments/styles/__init__.py b/wandb/vendor/pygments/styles/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..839a9b78dc78cbdf92065573c3d3f1960662c1a1
--- /dev/null
+++ b/wandb/vendor/pygments/styles/__init__.py
@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles
+ ~~~~~~~~~~~~~~~
+
+ Contains built-in styles.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.plugin import find_plugin_styles
+from pygments.util import ClassNotFound
+
+
+#: Maps style names to 'submodule::classname'.
+STYLE_MAP = {
+ 'default': 'default::DefaultStyle',
+ 'emacs': 'emacs::EmacsStyle',
+ 'friendly': 'friendly::FriendlyStyle',
+ 'colorful': 'colorful::ColorfulStyle',
+ 'autumn': 'autumn::AutumnStyle',
+ 'murphy': 'murphy::MurphyStyle',
+ 'manni': 'manni::ManniStyle',
+ 'monokai': 'monokai::MonokaiStyle',
+ 'perldoc': 'perldoc::PerldocStyle',
+ 'pastie': 'pastie::PastieStyle',
+ 'borland': 'borland::BorlandStyle',
+ 'trac': 'trac::TracStyle',
+ 'native': 'native::NativeStyle',
+ 'fruity': 'fruity::FruityStyle',
+ 'bw': 'bw::BlackWhiteStyle',
+ 'vim': 'vim::VimStyle',
+ 'vs': 'vs::VisualStudioStyle',
+ 'tango': 'tango::TangoStyle',
+ 'rrt': 'rrt::RrtStyle',
+ 'xcode': 'xcode::XcodeStyle',
+ 'igor': 'igor::IgorStyle',
+ 'paraiso-light': 'paraiso_light::ParaisoLightStyle',
+ 'paraiso-dark': 'paraiso_dark::ParaisoDarkStyle',
+ 'lovelace': 'lovelace::LovelaceStyle',
+ 'algol': 'algol::AlgolStyle',
+ 'algol_nu': 'algol_nu::Algol_NuStyle',
+ 'arduino': 'arduino::ArduinoStyle',
+ 'rainbow_dash': 'rainbow_dash::RainbowDashStyle',
+ 'abap': 'abap::AbapStyle',
+}
+
+
+def get_style_by_name(name):
+ if name in STYLE_MAP:
+ mod, cls = STYLE_MAP[name].split('::')
+ builtin = "yes"
+ else:
+ for found_name, style in find_plugin_styles():
+ if name == found_name:
+ return style
+ # perhaps it got dropped into our styles package
+ builtin = ""
+ mod = name
+ cls = name.title() + "Style"
+
+ try:
+ mod = __import__('pygments.styles.' + mod, None, None, [cls])
+ except ImportError:
+ raise ClassNotFound("Could not find style module %r" % mod +
+ (builtin and ", though it should be builtin") + ".")
+ try:
+ return getattr(mod, cls)
+ except AttributeError:
+ raise ClassNotFound("Could not find style class %r in style module." % cls)
+
+
+def get_all_styles():
+ """Return an generator for all styles by name,
+ both builtin and plugin."""
+ for name in STYLE_MAP:
+ yield name
+ for name, _ in find_plugin_styles():
+ yield name
diff --git a/wandb/vendor/pygments/styles/abap.py b/wandb/vendor/pygments/styles/abap.py
new file mode 100644
index 0000000000000000000000000000000000000000..91286a3a2b9b35a0de1af6bec4c42027cc75f657
--- /dev/null
+++ b/wandb/vendor/pygments/styles/abap.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.abap
+ ~~~~~~~~~~~~~~~~~~~~
+
+ ABAP workbench like style.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator
+
+
+class AbapStyle(Style):
+ default_style = ""
+ styles = {
+ Comment: 'italic #888',
+ Comment.Special: '#888',
+ Keyword: '#00f',
+ Operator.Word: '#00f',
+ Name: '#000',
+ Number: '#3af',
+ String: '#5a2',
+
+ Error: '#F00',
+ }
diff --git a/wandb/vendor/pygments/styles/algol.py b/wandb/vendor/pygments/styles/algol.py
new file mode 100644
index 0000000000000000000000000000000000000000..16461e0b416e34d0de6a94a812fdc452b32abc95
--- /dev/null
+++ b/wandb/vendor/pygments/styles/algol.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.algol
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Algol publication style.
+
+ This style renders source code for publication of algorithms in
+ scientific papers and academic texts, where its format is frequently used.
+
+ It is based on the style of the revised Algol-60 language report[1].
+
+ o No colours, only black, white and shades of grey are used.
+ o Keywords are rendered in lowercase underline boldface.
+ o Builtins are rendered in lowercase boldface italic.
+ o Docstrings and pragmas are rendered in dark grey boldface.
+ o Library identifiers are rendered in dark grey boldface italic.
+ o Comments are rendered in grey italic.
+
+ To render keywords without underlining, refer to the `Algol_Nu` style.
+
+ For lowercase conversion of keywords and builtins in languages where
+ these are not or might not be lowercase, a supporting lexer is required.
+ The Algol and Modula-2 lexers automatically convert to lowercase whenever
+ this style is selected.
+
+ [1] `Revised Report on the Algorithmic Language Algol-60 <http://www.masswerk.at/algol60/report.htm>`
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, Operator
+
+
+class AlgolStyle(Style):
+
+ background_color = "#ffffff"
+ default_style = ""
+
+ styles = {
+ Comment: "italic #888",
+ Comment.Preproc: "bold noitalic #888",
+ Comment.Special: "bold noitalic #888",
+
+ Keyword: "underline bold",
+ Keyword.Declaration: "italic",
+
+ Name.Builtin: "bold italic",
+ Name.Builtin.Pseudo: "bold italic",
+ Name.Namespace: "bold italic #666",
+ Name.Class: "bold italic #666",
+ Name.Function: "bold italic #666",
+ Name.Variable: "bold italic #666",
+ Name.Constant: "bold italic #666",
+
+ Operator.Word: "bold",
+
+ String: "italic #666",
+
+ Error: "border:#FF0000"
+ }
diff --git a/wandb/vendor/pygments/styles/algol_nu.py b/wandb/vendor/pygments/styles/algol_nu.py
new file mode 100644
index 0000000000000000000000000000000000000000..366ae215a277beb96835b4fc90c4faca56183606
--- /dev/null
+++ b/wandb/vendor/pygments/styles/algol_nu.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.algol_nu
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Algol publication style without underlining of keywords.
+
+ This style renders source code for publication of algorithms in
+ scientific papers and academic texts, where its format is frequently used.
+
+ It is based on the style of the revised Algol-60 language report[1].
+
+ o No colours, only black, white and shades of grey are used.
+ o Keywords are rendered in lowercase boldface.
+ o Builtins are rendered in lowercase boldface italic.
+ o Docstrings and pragmas are rendered in dark grey boldface.
+ o Library identifiers are rendered in dark grey boldface italic.
+ o Comments are rendered in grey italic.
+
+ To render keywords with underlining, refer to the `Algol` style.
+
+ For lowercase conversion of keywords and builtins in languages where
+ these are not or might not be lowercase, a supporting lexer is required.
+ The Algol and Modula-2 lexers automatically convert to lowercase whenever
+ this style is selected.
+
+ [1] `Revised Report on the Algorithmic Language Algol-60 <http://www.masswerk.at/algol60/report.htm>`
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, Operator
+
+
+class Algol_NuStyle(Style):
+
+ background_color = "#ffffff"
+ default_style = ""
+
+ styles = {
+ Comment: "italic #888",
+ Comment.Preproc: "bold noitalic #888",
+ Comment.Special: "bold noitalic #888",
+
+ Keyword: "bold",
+ Keyword.Declaration: "italic",
+
+ Name.Builtin: "bold italic",
+ Name.Builtin.Pseudo: "bold italic",
+ Name.Namespace: "bold italic #666",
+ Name.Class: "bold italic #666",
+ Name.Function: "bold italic #666",
+ Name.Variable: "bold italic #666",
+ Name.Constant: "bold italic #666",
+
+ Operator.Word: "bold",
+
+ String: "italic #666",
+
+ Error: "border:#FF0000"
+ }
diff --git a/wandb/vendor/pygments/styles/arduino.py b/wandb/vendor/pygments/styles/arduino.py
new file mode 100644
index 0000000000000000000000000000000000000000..57e3809e9af764fe42ba3a48d95a01780a15b086
--- /dev/null
+++ b/wandb/vendor/pygments/styles/arduino.py
@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.arduino
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Arduino® Syntax highlighting style.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Whitespace
+
+
+class ArduinoStyle(Style):
+ """
+ The Arduino® language style. This style is designed to highlight the
+ Arduino source code, so exepect the best results with it.
+ """
+
+ background_color = "#ffffff"
+ default_style = ""
+
+ styles = {
+ Whitespace: "", # class: 'w'
+ Error: "#a61717", # class: 'err'
+
+ Comment: "#95a5a6", # class: 'c'
+ Comment.Multiline: "", # class: 'cm'
+ Comment.Preproc: "#728E00", # class: 'cp'
+ Comment.Single: "", # class: 'c1'
+ Comment.Special: "", # class: 'cs'
+
+ Keyword: "#728E00", # class: 'k'
+ Keyword.Constant: "#00979D", # class: 'kc'
+ Keyword.Declaration: "", # class: 'kd'
+ Keyword.Namespace: "", # class: 'kn'
+ Keyword.Pseudo: "#00979D", # class: 'kp'
+ Keyword.Reserved: "#00979D", # class: 'kr'
+ Keyword.Type: "#00979D", # class: 'kt'
+
+ Operator: "#728E00", # class: 'o'
+ Operator.Word: "", # class: 'ow'
+
+ Name: "#434f54", # class: 'n'
+ Name.Attribute: "", # class: 'na'
+ Name.Builtin: "#728E00", # class: 'nb'
+ Name.Builtin.Pseudo: "", # class: 'bp'
+ Name.Class: "", # class: 'nc'
+ Name.Constant: "", # class: 'no'
+ Name.Decorator: "", # class: 'nd'
+ Name.Entity: "", # class: 'ni'
+ Name.Exception: "", # class: 'ne'
+ Name.Function: "#D35400", # class: 'nf'
+ Name.Property: "", # class: 'py'
+ Name.Label: "", # class: 'nl'
+ Name.Namespace: "", # class: 'nn'
+ Name.Other: "#728E00", # class: 'nx'
+ Name.Tag: "", # class: 'nt'
+ Name.Variable: "", # class: 'nv'
+ Name.Variable.Class: "", # class: 'vc'
+ Name.Variable.Global: "", # class: 'vg'
+ Name.Variable.Instance: "", # class: 'vi'
+
+ Number: "#8A7B52", # class: 'm'
+ Number.Float: "", # class: 'mf'
+ Number.Hex: "", # class: 'mh'
+ Number.Integer: "", # class: 'mi'
+ Number.Integer.Long: "", # class: 'il'
+ Number.Oct: "", # class: 'mo'
+
+ String: "#7F8C8D", # class: 's'
+ String.Backtick: "", # class: 'sb'
+ String.Char: "", # class: 'sc'
+ String.Doc: "", # class: 'sd'
+ String.Double: "", # class: 's2'
+ String.Escape: "", # class: 'se'
+ String.Heredoc: "", # class: 'sh'
+ String.Interpol: "", # class: 'si'
+ String.Other: "", # class: 'sx'
+ String.Regex: "", # class: 'sr'
+ String.Single: "", # class: 's1'
+ String.Symbol: "", # class: 'ss'
+
+ Generic: "", # class: 'g'
+ Generic.Deleted: "", # class: 'gd',
+ Generic.Emph: "", # class: 'ge'
+ Generic.Error: "", # class: 'gr'
+ Generic.Heading: "", # class: 'gh'
+ Generic.Inserted: "", # class: 'gi'
+ Generic.Output: "", # class: 'go'
+ Generic.Prompt: "", # class: 'gp'
+ Generic.Strong: "", # class: 'gs'
+ Generic.Subheading: "", # class: 'gu'
+ Generic.Traceback: "", # class: 'gt'
+ }
diff --git a/wandb/vendor/pygments/styles/autumn.py b/wandb/vendor/pygments/styles/autumn.py
new file mode 100644
index 0000000000000000000000000000000000000000..71b93b1e600c424fa0f9c5ebebd62461cbb34ffc
--- /dev/null
+++ b/wandb/vendor/pygments/styles/autumn.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.autumn
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ A colorful style, inspired by the terminal highlighting style.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Whitespace
+
+
+class AutumnStyle(Style):
+ """
+ A colorful style, inspired by the terminal highlighting style.
+ """
+
+ default_style = ""
+
+ styles = {
+ Whitespace: '#bbbbbb',
+
+ Comment: 'italic #aaaaaa',
+ Comment.Preproc: 'noitalic #4c8317',
+ Comment.Special: 'italic #0000aa',
+
+ Keyword: '#0000aa',
+ Keyword.Type: '#00aaaa',
+
+ Operator.Word: '#0000aa',
+
+ Name.Builtin: '#00aaaa',
+ Name.Function: '#00aa00',
+ Name.Class: 'underline #00aa00',
+ Name.Namespace: 'underline #00aaaa',
+ Name.Variable: '#aa0000',
+ Name.Constant: '#aa0000',
+ Name.Entity: 'bold #800',
+ Name.Attribute: '#1e90ff',
+ Name.Tag: 'bold #1e90ff',
+ Name.Decorator: '#888888',
+
+ String: '#aa5500',
+ String.Symbol: '#0000aa',
+ String.Regex: '#009999',
+
+ Number: '#009999',
+
+ Generic.Heading: 'bold #000080',
+ Generic.Subheading: 'bold #800080',
+ Generic.Deleted: '#aa0000',
+ Generic.Inserted: '#00aa00',
+ Generic.Error: '#aa0000',
+ Generic.Emph: 'italic',
+ Generic.Strong: 'bold',
+ Generic.Prompt: '#555555',
+ Generic.Output: '#888888',
+ Generic.Traceback: '#aa0000',
+
+ Error: '#F00 bg:#FAA'
+ }
diff --git a/wandb/vendor/pygments/styles/borland.py b/wandb/vendor/pygments/styles/borland.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d13d1aa99d6e0c2bce3fc7da6a8bdc3d7329db8
--- /dev/null
+++ b/wandb/vendor/pygments/styles/borland.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.borland
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Style similar to the style used in the Borland IDEs.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Whitespace
+
+
+class BorlandStyle(Style):
+ """
+ Style similar to the style used in the borland IDEs.
+ """
+
+ default_style = ''
+
+ styles = {
+ Whitespace: '#bbbbbb',
+
+ Comment: 'italic #008800',
+ Comment.Preproc: 'noitalic #008080',
+ Comment.Special: 'noitalic bold',
+
+ String: '#0000FF',
+ String.Char: '#800080',
+ Number: '#0000FF',
+ Keyword: 'bold #000080',
+ Operator.Word: 'bold',
+ Name.Tag: 'bold #000080',
+ Name.Attribute: '#FF0000',
+
+ Generic.Heading: '#999999',
+ Generic.Subheading: '#aaaaaa',
+ Generic.Deleted: 'bg:#ffdddd #000000',
+ Generic.Inserted: 'bg:#ddffdd #000000',
+ Generic.Error: '#aa0000',
+ Generic.Emph: 'italic',
+ Generic.Strong: 'bold',
+ Generic.Prompt: '#555555',
+ Generic.Output: '#888888',
+ Generic.Traceback: '#aa0000',
+
+ Error: 'bg:#e3d2d2 #a61717'
+ }
diff --git a/wandb/vendor/pygments/styles/bw.py b/wandb/vendor/pygments/styles/bw.py
new file mode 100644
index 0000000000000000000000000000000000000000..f0a6b1486a8f9df7e4398659188fdc49c129300e
--- /dev/null
+++ b/wandb/vendor/pygments/styles/bw.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.bw
+ ~~~~~~~~~~~~~~~~~~
+
+ Simple black/white only style.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Operator, Generic
+
+
+class BlackWhiteStyle(Style):
+
+ background_color = "#ffffff"
+ default_style = ""
+
+ styles = {
+ Comment: "italic",
+ Comment.Preproc: "noitalic",
+
+ Keyword: "bold",
+ Keyword.Pseudo: "nobold",
+ Keyword.Type: "nobold",
+
+ Operator.Word: "bold",
+
+ Name.Class: "bold",
+ Name.Namespace: "bold",
+ Name.Exception: "bold",
+ Name.Entity: "bold",
+ Name.Tag: "bold",
+
+ String: "italic",
+ String.Interpol: "bold",
+ String.Escape: "bold",
+
+ Generic.Heading: "bold",
+ Generic.Subheading: "bold",
+ Generic.Emph: "italic",
+ Generic.Strong: "bold",
+ Generic.Prompt: "bold",
+
+ Error: "border:#FF0000"
+ }
diff --git a/wandb/vendor/pygments/styles/colorful.py b/wandb/vendor/pygments/styles/colorful.py
new file mode 100644
index 0000000000000000000000000000000000000000..bfc0b5026f31af82cd428e678a6725568149e154
--- /dev/null
+++ b/wandb/vendor/pygments/styles/colorful.py
@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.colorful
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ A colorful style, inspired by CodeRay.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Whitespace
+
+
+class ColorfulStyle(Style):
+ """
+ A colorful style, inspired by CodeRay.
+ """
+
+ default_style = ""
+
+ styles = {
+ Whitespace: "#bbbbbb",
+
+ Comment: "#888",
+ Comment.Preproc: "#579",
+ Comment.Special: "bold #cc0000",
+
+ Keyword: "bold #080",
+ Keyword.Pseudo: "#038",
+ Keyword.Type: "#339",
+
+ Operator: "#333",
+ Operator.Word: "bold #000",
+
+ Name.Builtin: "#007020",
+ Name.Function: "bold #06B",
+ Name.Class: "bold #B06",
+ Name.Namespace: "bold #0e84b5",
+ Name.Exception: "bold #F00",
+ Name.Variable: "#963",
+ Name.Variable.Instance: "#33B",
+ Name.Variable.Class: "#369",
+ Name.Variable.Global: "bold #d70",
+ Name.Constant: "bold #036",
+ Name.Label: "bold #970",
+ Name.Entity: "bold #800",
+ Name.Attribute: "#00C",
+ Name.Tag: "#070",
+ Name.Decorator: "bold #555",
+
+ String: "bg:#fff0f0",
+ String.Char: "#04D bg:",
+ String.Doc: "#D42 bg:",
+ String.Interpol: "bg:#eee",
+ String.Escape: "bold #666",
+ String.Regex: "bg:#fff0ff #000",
+ String.Symbol: "#A60 bg:",
+ String.Other: "#D20",
+
+ Number: "bold #60E",
+ Number.Integer: "bold #00D",
+ Number.Float: "bold #60E",
+ Number.Hex: "bold #058",
+ Number.Oct: "bold #40E",
+
+ Generic.Heading: "bold #000080",
+ Generic.Subheading: "bold #800080",
+ Generic.Deleted: "#A00000",
+ Generic.Inserted: "#00A000",
+ Generic.Error: "#FF0000",
+ Generic.Emph: "italic",
+ Generic.Strong: "bold",
+ Generic.Prompt: "bold #c65d09",
+ Generic.Output: "#888",
+ Generic.Traceback: "#04D",
+
+ Error: "#F00 bg:#FAA"
+ }
diff --git a/wandb/vendor/pygments/styles/default.py b/wandb/vendor/pygments/styles/default.py
new file mode 100644
index 0000000000000000000000000000000000000000..6b9bd4461619a32751f0a2f21b95b9000e736537
--- /dev/null
+++ b/wandb/vendor/pygments/styles/default.py
@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.default
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ The default highlighting style.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Whitespace
+
+
+class DefaultStyle(Style):
+ """
+ The default style (inspired by Emacs 22).
+ """
+
+ background_color = "#f8f8f8"
+ default_style = ""
+
+ styles = {
+ Whitespace: "#bbbbbb",
+ Comment: "italic #408080",
+ Comment.Preproc: "noitalic #BC7A00",
+
+ #Keyword: "bold #AA22FF",
+ Keyword: "bold #008000",
+ Keyword.Pseudo: "nobold",
+ Keyword.Type: "nobold #B00040",
+
+ Operator: "#666666",
+ Operator.Word: "bold #AA22FF",
+
+ Name.Builtin: "#008000",
+ Name.Function: "#0000FF",
+ Name.Class: "bold #0000FF",
+ Name.Namespace: "bold #0000FF",
+ Name.Exception: "bold #D2413A",
+ Name.Variable: "#19177C",
+ Name.Constant: "#880000",
+ Name.Label: "#A0A000",
+ Name.Entity: "bold #999999",
+ Name.Attribute: "#7D9029",
+ Name.Tag: "bold #008000",
+ Name.Decorator: "#AA22FF",
+
+ String: "#BA2121",
+ String.Doc: "italic",
+ String.Interpol: "bold #BB6688",
+ String.Escape: "bold #BB6622",
+ String.Regex: "#BB6688",
+ #String.Symbol: "#B8860B",
+ String.Symbol: "#19177C",
+ String.Other: "#008000",
+ Number: "#666666",
+
+ Generic.Heading: "bold #000080",
+ Generic.Subheading: "bold #800080",
+ Generic.Deleted: "#A00000",
+ Generic.Inserted: "#00A000",
+ Generic.Error: "#FF0000",
+ Generic.Emph: "italic",
+ Generic.Strong: "bold",
+ Generic.Prompt: "bold #000080",
+ Generic.Output: "#888",
+ Generic.Traceback: "#04D",
+
+ Error: "border:#FF0000"
+ }
diff --git a/wandb/vendor/pygments/styles/emacs.py b/wandb/vendor/pygments/styles/emacs.py
new file mode 100644
index 0000000000000000000000000000000000000000..af15f30d609a2f69753a16c1a08bfd4900d89451
--- /dev/null
+++ b/wandb/vendor/pygments/styles/emacs.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.emacs
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ A highlighting style for Pygments, inspired by Emacs.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Whitespace
+
+
+class EmacsStyle(Style):
+ """
+ The default style (inspired by Emacs 22).
+ """
+
+ background_color = "#f8f8f8"
+ default_style = ""
+
+ styles = {
+ Whitespace: "#bbbbbb",
+ Comment: "italic #008800",
+ Comment.Preproc: "noitalic",
+ Comment.Special: "noitalic bold",
+
+ Keyword: "bold #AA22FF",
+ Keyword.Pseudo: "nobold",
+ Keyword.Type: "bold #00BB00",
+
+ Operator: "#666666",
+ Operator.Word: "bold #AA22FF",
+
+ Name.Builtin: "#AA22FF",
+ Name.Function: "#00A000",
+ Name.Class: "#0000FF",
+ Name.Namespace: "bold #0000FF",
+ Name.Exception: "bold #D2413A",
+ Name.Variable: "#B8860B",
+ Name.Constant: "#880000",
+ Name.Label: "#A0A000",
+ Name.Entity: "bold #999999",
+ Name.Attribute: "#BB4444",
+ Name.Tag: "bold #008000",
+ Name.Decorator: "#AA22FF",
+
+ String: "#BB4444",
+ String.Doc: "italic",
+ String.Interpol: "bold #BB6688",
+ String.Escape: "bold #BB6622",
+ String.Regex: "#BB6688",
+ String.Symbol: "#B8860B",
+ String.Other: "#008000",
+ Number: "#666666",
+
+ Generic.Heading: "bold #000080",
+ Generic.Subheading: "bold #800080",
+ Generic.Deleted: "#A00000",
+ Generic.Inserted: "#00A000",
+ Generic.Error: "#FF0000",
+ Generic.Emph: "italic",
+ Generic.Strong: "bold",
+ Generic.Prompt: "bold #000080",
+ Generic.Output: "#888",
+ Generic.Traceback: "#04D",
+
+ Error: "border:#FF0000"
+ }
diff --git a/wandb/vendor/pygments/styles/friendly.py b/wandb/vendor/pygments/styles/friendly.py
new file mode 100644
index 0000000000000000000000000000000000000000..b2d1c0ce689d8cd09b3ba0d06072f24c47212e78
--- /dev/null
+++ b/wandb/vendor/pygments/styles/friendly.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.friendly
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ A modern style based on the VIM pyte theme.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Whitespace
+
+
+class FriendlyStyle(Style):
+ """
+ A modern style based on the VIM pyte theme.
+ """
+
+ background_color = "#f0f0f0"
+ default_style = ""
+
+ styles = {
+ Whitespace: "#bbbbbb",
+ Comment: "italic #60a0b0",
+ Comment.Preproc: "noitalic #007020",
+ Comment.Special: "noitalic bg:#fff0f0",
+
+ Keyword: "bold #007020",
+ Keyword.Pseudo: "nobold",
+ Keyword.Type: "nobold #902000",
+
+ Operator: "#666666",
+ Operator.Word: "bold #007020",
+
+ Name.Builtin: "#007020",
+ Name.Function: "#06287e",
+ Name.Class: "bold #0e84b5",
+ Name.Namespace: "bold #0e84b5",
+ Name.Exception: "#007020",
+ Name.Variable: "#bb60d5",
+ Name.Constant: "#60add5",
+ Name.Label: "bold #002070",
+ Name.Entity: "bold #d55537",
+ Name.Attribute: "#4070a0",
+ Name.Tag: "bold #062873",
+ Name.Decorator: "bold #555555",
+
+ String: "#4070a0",
+ String.Doc: "italic",
+ String.Interpol: "italic #70a0d0",
+ String.Escape: "bold #4070a0",
+ String.Regex: "#235388",
+ String.Symbol: "#517918",
+ String.Other: "#c65d09",
+ Number: "#40a070",
+
+ Generic.Heading: "bold #000080",
+ Generic.Subheading: "bold #800080",
+ Generic.Deleted: "#A00000",
+ Generic.Inserted: "#00A000",
+ Generic.Error: "#FF0000",
+ Generic.Emph: "italic",
+ Generic.Strong: "bold",
+ Generic.Prompt: "bold #c65d09",
+ Generic.Output: "#888",
+ Generic.Traceback: "#04D",
+
+ Error: "border:#FF0000"
+ }
diff --git a/wandb/vendor/pygments/styles/fruity.py b/wandb/vendor/pygments/styles/fruity.py
new file mode 100644
index 0000000000000000000000000000000000000000..1bbe0316a1f04ed2904e5b06f3934787d0bd025b
--- /dev/null
+++ b/wandb/vendor/pygments/styles/fruity.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.fruity
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ pygments version of my "fruity" vim theme.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Token, Comment, Name, Keyword, \
+ Generic, Number, String, Whitespace
+
+class FruityStyle(Style):
+ """
+ Pygments version of the "native" vim theme.
+ """
+
+ background_color = '#111111'
+ highlight_color = '#333333'
+
+ styles = {
+ Whitespace: '#888888',
+ Token: '#ffffff',
+ Generic.Output: '#444444 bg:#222222',
+ Keyword: '#fb660a bold',
+ Keyword.Pseudo: 'nobold',
+ Number: '#0086f7 bold',
+ Name.Tag: '#fb660a bold',
+ Name.Variable: '#fb660a',
+ Comment: '#008800 bg:#0f140f italic',
+ Name.Attribute: '#ff0086 bold',
+ String: '#0086d2',
+ Name.Function: '#ff0086 bold',
+ Generic.Heading: '#ffffff bold',
+ Keyword.Type: '#cdcaa9 bold',
+ Generic.Subheading: '#ffffff bold',
+ Name.Constant: '#0086d2',
+ Comment.Preproc: '#ff0007 bold'
+ }
diff --git a/wandb/vendor/pygments/styles/igor.py b/wandb/vendor/pygments/styles/igor.py
new file mode 100644
index 0000000000000000000000000000000000000000..d4620a42a48aecbfd79e81193fa0067963a2bfe2
--- /dev/null
+++ b/wandb/vendor/pygments/styles/igor.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.igor
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Igor Pro default style.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String
+
+
+class IgorStyle(Style):
+ """
+ Pygments version of the official colors for Igor Pro procedures.
+ """
+ default_style = ""
+
+ styles = {
+ Comment: 'italic #FF0000',
+ Keyword: '#0000FF',
+ Name.Function: '#C34E00',
+ Name.Decorator: '#CC00A3',
+ Name.Class: '#007575',
+ String: '#009C00'
+ }
diff --git a/wandb/vendor/pygments/styles/lovelace.py b/wandb/vendor/pygments/styles/lovelace.py
new file mode 100644
index 0000000000000000000000000000000000000000..861f778d3d5d2f44947dd040ac397a30591ffc8d
--- /dev/null
+++ b/wandb/vendor/pygments/styles/lovelace.py
@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.lovelace
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Lovelace by Miikka Salminen
+
+ Pygments style by Miikka Salminen (https://github.com/miikkas)
+ A desaturated, somewhat subdued style created for the Lovelace interactive
+ learning environment.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Punctuation, Generic, Whitespace
+
+
+class LovelaceStyle(Style):
+ """
+ The style used in Lovelace interactive learning environment. Tries to avoid
+ the "angry fruit salad" effect with desaturated and dim colours.
+ """
+ _KW_BLUE = '#2838b0'
+ _NAME_GREEN = '#388038'
+ _DOC_ORANGE = '#b85820'
+ _OW_PURPLE = '#a848a8'
+ _FUN_BROWN = '#785840'
+ _STR_RED = '#b83838'
+ _CLS_CYAN = '#287088'
+ _ESCAPE_LIME = '#709030'
+ _LABEL_CYAN = '#289870'
+ _EXCEPT_YELLOW = '#908828'
+
+ default_style = '#222222'
+
+ styles = {
+ Whitespace: '#a89028',
+ Comment: 'italic #888888',
+ Comment.Hashbang: _CLS_CYAN,
+ Comment.Multiline: '#888888',
+ Comment.Preproc: 'noitalic '+_LABEL_CYAN,
+
+ Keyword: _KW_BLUE,
+ Keyword.Constant: 'italic #444444',
+ Keyword.Declaration: 'italic',
+ Keyword.Type: 'italic',
+
+ Operator: '#666666',
+ Operator.Word: _OW_PURPLE,
+
+ Punctuation: '#888888',
+
+ Name.Attribute: _NAME_GREEN,
+ Name.Builtin: _NAME_GREEN,
+ Name.Builtin.Pseudo: 'italic',
+ Name.Class: _CLS_CYAN,
+ Name.Constant: _DOC_ORANGE,
+ Name.Decorator: _CLS_CYAN,
+ Name.Entity: _ESCAPE_LIME,
+ Name.Exception: _EXCEPT_YELLOW,
+ Name.Function: _FUN_BROWN,
+ Name.Function.Magic: _DOC_ORANGE,
+ Name.Label: _LABEL_CYAN,
+ Name.Namespace: _LABEL_CYAN,
+ Name.Tag: _KW_BLUE,
+ Name.Variable: '#b04040',
+ Name.Variable.Global:_EXCEPT_YELLOW,
+ Name.Variable.Magic: _DOC_ORANGE,
+
+ String: _STR_RED,
+ String.Affix: '#444444',
+ String.Char: _OW_PURPLE,
+ String.Delimiter: _DOC_ORANGE,
+ String.Doc: 'italic '+_DOC_ORANGE,
+ String.Escape: _ESCAPE_LIME,
+ String.Interpol: 'underline',
+ String.Other: _OW_PURPLE,
+ String.Regex: _OW_PURPLE,
+
+ Number: '#444444',
+
+ Generic.Deleted: '#c02828',
+ Generic.Emph: 'italic',
+ Generic.Error: '#c02828',
+ Generic.Heading: '#666666',
+ Generic.Subheading: '#444444',
+ Generic.Inserted: _NAME_GREEN,
+ Generic.Output: '#666666',
+ Generic.Prompt: '#444444',
+ Generic.Strong: 'bold',
+ Generic.Traceback: _KW_BLUE,
+
+ Error: 'bg:'+_OW_PURPLE,
+ }
diff --git a/wandb/vendor/pygments/styles/manni.py b/wandb/vendor/pygments/styles/manni.py
new file mode 100644
index 0000000000000000000000000000000000000000..f0a325af42f748ceda9f4e39b34694e2d3ab45b1
--- /dev/null
+++ b/wandb/vendor/pygments/styles/manni.py
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.manni
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ A colorful style, inspired by the terminal highlighting style.
+
+ This is a port of the style used in the `php port`_ of pygments
+ by Manni. The style is called 'default' there.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Whitespace
+
+
+class ManniStyle(Style):
+ """
+ A colorful style, inspired by the terminal highlighting style.
+ """
+
+ background_color = '#f0f3f3'
+
+ styles = {
+ Whitespace: '#bbbbbb',
+ Comment: 'italic #0099FF',
+ Comment.Preproc: 'noitalic #009999',
+ Comment.Special: 'bold',
+
+ Keyword: 'bold #006699',
+ Keyword.Pseudo: 'nobold',
+ Keyword.Type: '#007788',
+
+ Operator: '#555555',
+ Operator.Word: 'bold #000000',
+
+ Name.Builtin: '#336666',
+ Name.Function: '#CC00FF',
+ Name.Class: 'bold #00AA88',
+ Name.Namespace: 'bold #00CCFF',
+ Name.Exception: 'bold #CC0000',
+ Name.Variable: '#003333',
+ Name.Constant: '#336600',
+ Name.Label: '#9999FF',
+ Name.Entity: 'bold #999999',
+ Name.Attribute: '#330099',
+ Name.Tag: 'bold #330099',
+ Name.Decorator: '#9999FF',
+
+ String: '#CC3300',
+ String.Doc: 'italic',
+ String.Interpol: '#AA0000',
+ String.Escape: 'bold #CC3300',
+ String.Regex: '#33AAAA',
+ String.Symbol: '#FFCC33',
+ String.Other: '#CC3300',
+
+ Number: '#FF6600',
+
+ Generic.Heading: 'bold #003300',
+ Generic.Subheading: 'bold #003300',
+ Generic.Deleted: 'border:#CC0000 bg:#FFCCCC',
+ Generic.Inserted: 'border:#00CC00 bg:#CCFFCC',
+ Generic.Error: '#FF0000',
+ Generic.Emph: 'italic',
+ Generic.Strong: 'bold',
+ Generic.Prompt: 'bold #000099',
+ Generic.Output: '#AAAAAA',
+ Generic.Traceback: '#99CC66',
+
+ Error: 'bg:#FFAAAA #AA0000'
+ }
diff --git a/wandb/vendor/pygments/styles/monokai.py b/wandb/vendor/pygments/styles/monokai.py
new file mode 100644
index 0000000000000000000000000000000000000000..337e2f8922093b98b38c5568c8f32d5004041504
--- /dev/null
+++ b/wandb/vendor/pygments/styles/monokai.py
@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.monokai
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Mimic the Monokai color scheme. Based on tango.py.
+
+ http://www.monokai.nl/blog/2006/07/15/textmate-color-theme/
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, Text, \
+ Number, Operator, Generic, Whitespace, Punctuation, Other, Literal
+
+class MonokaiStyle(Style):
+ """
+ This style mimics the Monokai color scheme.
+ """
+
+ background_color = "#272822"
+ highlight_color = "#49483e"
+
+ styles = {
+ # No corresponding class for the following:
+ Text: "#f8f8f2", # class: ''
+ Whitespace: "", # class: 'w'
+ Error: "#960050 bg:#1e0010", # class: 'err'
+ Other: "", # class 'x'
+
+ Comment: "#75715e", # class: 'c'
+ Comment.Multiline: "", # class: 'cm'
+ Comment.Preproc: "", # class: 'cp'
+ Comment.Single: "", # class: 'c1'
+ Comment.Special: "", # class: 'cs'
+
+ Keyword: "#66d9ef", # class: 'k'
+ Keyword.Constant: "", # class: 'kc'
+ Keyword.Declaration: "", # class: 'kd'
+ Keyword.Namespace: "#f92672", # class: 'kn'
+ Keyword.Pseudo: "", # class: 'kp'
+ Keyword.Reserved: "", # class: 'kr'
+ Keyword.Type: "", # class: 'kt'
+
+ Operator: "#f92672", # class: 'o'
+ Operator.Word: "", # class: 'ow' - like keywords
+
+ Punctuation: "#f8f8f2", # class: 'p'
+
+ Name: "#f8f8f2", # class: 'n'
+ Name.Attribute: "#a6e22e", # class: 'na' - to be revised
+ Name.Builtin: "", # class: 'nb'
+ Name.Builtin.Pseudo: "", # class: 'bp'
+ Name.Class: "#a6e22e", # class: 'nc' - to be revised
+ Name.Constant: "#66d9ef", # class: 'no' - to be revised
+ Name.Decorator: "#a6e22e", # class: 'nd' - to be revised
+ Name.Entity: "", # class: 'ni'
+ Name.Exception: "#a6e22e", # class: 'ne'
+ Name.Function: "#a6e22e", # class: 'nf'
+ Name.Property: "", # class: 'py'
+ Name.Label: "", # class: 'nl'
+ Name.Namespace: "", # class: 'nn' - to be revised
+ Name.Other: "#a6e22e", # class: 'nx'
+ Name.Tag: "#f92672", # class: 'nt' - like a keyword
+ Name.Variable: "", # class: 'nv' - to be revised
+ Name.Variable.Class: "", # class: 'vc' - to be revised
+ Name.Variable.Global: "", # class: 'vg' - to be revised
+ Name.Variable.Instance: "", # class: 'vi' - to be revised
+
+ Number: "#ae81ff", # class: 'm'
+ Number.Float: "", # class: 'mf'
+ Number.Hex: "", # class: 'mh'
+ Number.Integer: "", # class: 'mi'
+ Number.Integer.Long: "", # class: 'il'
+ Number.Oct: "", # class: 'mo'
+
+ Literal: "#ae81ff", # class: 'l'
+ Literal.Date: "#e6db74", # class: 'ld'
+
+ String: "#e6db74", # class: 's'
+ String.Backtick: "", # class: 'sb'
+ String.Char: "", # class: 'sc'
+ String.Doc: "", # class: 'sd' - like a comment
+ String.Double: "", # class: 's2'
+ String.Escape: "#ae81ff", # class: 'se'
+ String.Heredoc: "", # class: 'sh'
+ String.Interpol: "", # class: 'si'
+ String.Other: "", # class: 'sx'
+ String.Regex: "", # class: 'sr'
+ String.Single: "", # class: 's1'
+ String.Symbol: "", # class: 'ss'
+
+ Generic: "", # class: 'g'
+ Generic.Deleted: "#f92672", # class: 'gd',
+ Generic.Emph: "italic", # class: 'ge'
+ Generic.Error: "", # class: 'gr'
+ Generic.Heading: "", # class: 'gh'
+ Generic.Inserted: "#a6e22e", # class: 'gi'
+ Generic.Output: "", # class: 'go'
+ Generic.Prompt: "", # class: 'gp'
+ Generic.Strong: "bold", # class: 'gs'
+ Generic.Subheading: "#75715e", # class: 'gu'
+ Generic.Traceback: "", # class: 'gt'
+ }
diff --git a/wandb/vendor/pygments/styles/murphy.py b/wandb/vendor/pygments/styles/murphy.py
new file mode 100644
index 0000000000000000000000000000000000000000..c82700654ebab2e9039388a841593986a454dcae
--- /dev/null
+++ b/wandb/vendor/pygments/styles/murphy.py
@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.murphy
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Murphy's style from CodeRay.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Whitespace
+
+
+class MurphyStyle(Style):
+ """
+ Murphy's style from CodeRay.
+ """
+
+ default_style = ""
+
+ styles = {
+ Whitespace: "#bbbbbb",
+ Comment: "#666 italic",
+ Comment.Preproc: "#579 noitalic",
+ Comment.Special: "#c00 bold",
+
+ Keyword: "bold #289",
+ Keyword.Pseudo: "#08f",
+ Keyword.Type: "#66f",
+
+ Operator: "#333",
+ Operator.Word: "bold #000",
+
+ Name.Builtin: "#072",
+ Name.Function: "bold #5ed",
+ Name.Class: "bold #e9e",
+ Name.Namespace: "bold #0e84b5",
+ Name.Exception: "bold #F00",
+ Name.Variable: "#036",
+ Name.Variable.Instance: "#aaf",
+ Name.Variable.Class: "#ccf",
+ Name.Variable.Global: "#f84",
+ Name.Constant: "bold #5ed",
+ Name.Label: "bold #970",
+ Name.Entity: "#800",
+ Name.Attribute: "#007",
+ Name.Tag: "#070",
+ Name.Decorator: "bold #555",
+
+ String: "bg:#e0e0ff",
+ String.Char: "#88F bg:",
+ String.Doc: "#D42 bg:",
+ String.Interpol: "bg:#eee",
+ String.Escape: "bold #666",
+ String.Regex: "bg:#e0e0ff #000",
+ String.Symbol: "#fc8 bg:",
+ String.Other: "#f88",
+
+ Number: "bold #60E",
+ Number.Integer: "bold #66f",
+ Number.Float: "bold #60E",
+ Number.Hex: "bold #058",
+ Number.Oct: "bold #40E",
+
+ Generic.Heading: "bold #000080",
+ Generic.Subheading: "bold #800080",
+ Generic.Deleted: "#A00000",
+ Generic.Inserted: "#00A000",
+ Generic.Error: "#FF0000",
+ Generic.Emph: "italic",
+ Generic.Strong: "bold",
+ Generic.Prompt: "bold #c65d09",
+ Generic.Output: "#888",
+ Generic.Traceback: "#04D",
+
+ Error: "#F00 bg:#FAA"
+ }
diff --git a/wandb/vendor/pygments/styles/native.py b/wandb/vendor/pygments/styles/native.py
new file mode 100644
index 0000000000000000000000000000000000000000..921a58d9acfcaf74fb897bce7ebfd1dbf4e2dcdc
--- /dev/null
+++ b/wandb/vendor/pygments/styles/native.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.native
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ pygments version of my "native" vim theme.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Token, Whitespace
+
+
+class NativeStyle(Style):
+ """
+ Pygments version of the "native" vim theme.
+ """
+
+ background_color = '#202020'
+ highlight_color = '#404040'
+
+ styles = {
+ Token: '#d0d0d0',
+ Whitespace: '#666666',
+
+ Comment: 'italic #999999',
+ Comment.Preproc: 'noitalic bold #cd2828',
+ Comment.Special: 'noitalic bold #e50808 bg:#520000',
+
+ Keyword: 'bold #6ab825',
+ Keyword.Pseudo: 'nobold',
+ Operator.Word: 'bold #6ab825',
+
+ String: '#ed9d13',
+ String.Other: '#ffa500',
+
+ Number: '#3677a9',
+
+ Name.Builtin: '#24909d',
+ Name.Variable: '#40ffff',
+ Name.Constant: '#40ffff',
+ Name.Class: 'underline #447fcf',
+ Name.Function: '#447fcf',
+ Name.Namespace: 'underline #447fcf',
+ Name.Exception: '#bbbbbb',
+ Name.Tag: 'bold #6ab825',
+ Name.Attribute: '#bbbbbb',
+ Name.Decorator: '#ffa500',
+
+ Generic.Heading: 'bold #ffffff',
+ Generic.Subheading: 'underline #ffffff',
+ Generic.Deleted: '#d22323',
+ Generic.Inserted: '#589819',
+ Generic.Error: '#d22323',
+ Generic.Emph: 'italic',
+ Generic.Strong: 'bold',
+ Generic.Prompt: '#aaaaaa',
+ Generic.Output: '#cccccc',
+ Generic.Traceback: '#d22323',
+
+ Error: 'bg:#e3d2d2 #a61717'
+ }
diff --git a/wandb/vendor/pygments/styles/paraiso_dark.py b/wandb/vendor/pygments/styles/paraiso_dark.py
new file mode 100644
index 0000000000000000000000000000000000000000..5f334bb9c8f11595b989e62c327f90ff516a8751
--- /dev/null
+++ b/wandb/vendor/pygments/styles/paraiso_dark.py
@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.paraiso_dark
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ ParaÃso (Dark) by Jan T. Sott
+
+ Pygments template by Jan T. Sott (https://github.com/idleberg)
+ Created with Base16 Builder by Chris Kempson
+ (https://github.com/chriskempson/base16-builder).
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, Text, \
+ Number, Operator, Generic, Whitespace, Punctuation, Other, Literal
+
+
+BACKGROUND = "#2f1e2e"
+CURRENT_LINE = "#41323f"
+SELECTION = "#4f424c"
+FOREGROUND = "#e7e9db"
+COMMENT = "#776e71"
+RED = "#ef6155"
+ORANGE = "#f99b15"
+YELLOW = "#fec418"
+GREEN = "#48b685"
+AQUA = "#5bc4bf"
+BLUE = "#06b6ef"
+PURPLE = "#815ba4"
+
+
+class ParaisoDarkStyle(Style):
+
+ default_style = ''
+
+ background_color = BACKGROUND
+ highlight_color = SELECTION
+
+ background_color = BACKGROUND
+ highlight_color = SELECTION
+
+ styles = {
+ # No corresponding class for the following:
+ Text: FOREGROUND, # class: ''
+ Whitespace: "", # class: 'w'
+ Error: RED, # class: 'err'
+ Other: "", # class 'x'
+
+ Comment: COMMENT, # class: 'c'
+ Comment.Multiline: "", # class: 'cm'
+ Comment.Preproc: "", # class: 'cp'
+ Comment.Single: "", # class: 'c1'
+ Comment.Special: "", # class: 'cs'
+
+ Keyword: PURPLE, # class: 'k'
+ Keyword.Constant: "", # class: 'kc'
+ Keyword.Declaration: "", # class: 'kd'
+ Keyword.Namespace: AQUA, # class: 'kn'
+ Keyword.Pseudo: "", # class: 'kp'
+ Keyword.Reserved: "", # class: 'kr'
+ Keyword.Type: YELLOW, # class: 'kt'
+
+ Operator: AQUA, # class: 'o'
+ Operator.Word: "", # class: 'ow' - like keywords
+
+ Punctuation: FOREGROUND, # class: 'p'
+
+ Name: FOREGROUND, # class: 'n'
+ Name.Attribute: BLUE, # class: 'na' - to be revised
+ Name.Builtin: "", # class: 'nb'
+ Name.Builtin.Pseudo: "", # class: 'bp'
+ Name.Class: YELLOW, # class: 'nc' - to be revised
+ Name.Constant: RED, # class: 'no' - to be revised
+ Name.Decorator: AQUA, # class: 'nd' - to be revised
+ Name.Entity: "", # class: 'ni'
+ Name.Exception: RED, # class: 'ne'
+ Name.Function: BLUE, # class: 'nf'
+ Name.Property: "", # class: 'py'
+ Name.Label: "", # class: 'nl'
+ Name.Namespace: YELLOW, # class: 'nn' - to be revised
+ Name.Other: BLUE, # class: 'nx'
+ Name.Tag: AQUA, # class: 'nt' - like a keyword
+ Name.Variable: RED, # class: 'nv' - to be revised
+ Name.Variable.Class: "", # class: 'vc' - to be revised
+ Name.Variable.Global: "", # class: 'vg' - to be revised
+ Name.Variable.Instance: "", # class: 'vi' - to be revised
+
+ Number: ORANGE, # class: 'm'
+ Number.Float: "", # class: 'mf'
+ Number.Hex: "", # class: 'mh'
+ Number.Integer: "", # class: 'mi'
+ Number.Integer.Long: "", # class: 'il'
+ Number.Oct: "", # class: 'mo'
+
+ Literal: ORANGE, # class: 'l'
+ Literal.Date: GREEN, # class: 'ld'
+
+ String: GREEN, # class: 's'
+ String.Backtick: "", # class: 'sb'
+ String.Char: FOREGROUND, # class: 'sc'
+ String.Doc: COMMENT, # class: 'sd' - like a comment
+ String.Double: "", # class: 's2'
+ String.Escape: ORANGE, # class: 'se'
+ String.Heredoc: "", # class: 'sh'
+ String.Interpol: ORANGE, # class: 'si'
+ String.Other: "", # class: 'sx'
+ String.Regex: "", # class: 'sr'
+ String.Single: "", # class: 's1'
+ String.Symbol: "", # class: 'ss'
+
+ Generic: "", # class: 'g'
+ Generic.Deleted: RED, # class: 'gd',
+ Generic.Emph: "italic", # class: 'ge'
+ Generic.Error: "", # class: 'gr'
+ Generic.Heading: "bold " + FOREGROUND, # class: 'gh'
+ Generic.Inserted: GREEN, # class: 'gi'
+ Generic.Output: "", # class: 'go'
+ Generic.Prompt: "bold " + COMMENT, # class: 'gp'
+ Generic.Strong: "bold", # class: 'gs'
+ Generic.Subheading: "bold " + AQUA, # class: 'gu'
+ Generic.Traceback: "", # class: 'gt'
+ }
diff --git a/wandb/vendor/pygments/styles/paraiso_light.py b/wandb/vendor/pygments/styles/paraiso_light.py
new file mode 100644
index 0000000000000000000000000000000000000000..a8112819ea0f03fd663e3802ee3a70c82a254428
--- /dev/null
+++ b/wandb/vendor/pygments/styles/paraiso_light.py
@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.paraiso_light
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ ParaÃso (Light) by Jan T. Sott
+
+ Pygments template by Jan T. Sott (https://github.com/idleberg)
+ Created with Base16 Builder by Chris Kempson
+ (https://github.com/chriskempson/base16-builder).
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, Text, \
+ Number, Operator, Generic, Whitespace, Punctuation, Other, Literal
+
+
+BACKGROUND = "#e7e9db"
+CURRENT_LINE = "#b9b6b0"
+SELECTION = "#a39e9b"
+FOREGROUND = "#2f1e2e"
+COMMENT = "#8d8687"
+RED = "#ef6155"
+ORANGE = "#f99b15"
+YELLOW = "#fec418"
+GREEN = "#48b685"
+AQUA = "#5bc4bf"
+BLUE = "#06b6ef"
+PURPLE = "#815ba4"
+
+
+class ParaisoLightStyle(Style):
+
+ default_style = ''
+
+ background_color = BACKGROUND
+ highlight_color = SELECTION
+
+ background_color = BACKGROUND
+ highlight_color = SELECTION
+
+ styles = {
+ # No corresponding class for the following:
+ Text: FOREGROUND, # class: ''
+ Whitespace: "", # class: 'w'
+ Error: RED, # class: 'err'
+ Other: "", # class 'x'
+
+ Comment: COMMENT, # class: 'c'
+ Comment.Multiline: "", # class: 'cm'
+ Comment.Preproc: "", # class: 'cp'
+ Comment.Single: "", # class: 'c1'
+ Comment.Special: "", # class: 'cs'
+
+ Keyword: PURPLE, # class: 'k'
+ Keyword.Constant: "", # class: 'kc'
+ Keyword.Declaration: "", # class: 'kd'
+ Keyword.Namespace: AQUA, # class: 'kn'
+ Keyword.Pseudo: "", # class: 'kp'
+ Keyword.Reserved: "", # class: 'kr'
+ Keyword.Type: YELLOW, # class: 'kt'
+
+ Operator: AQUA, # class: 'o'
+ Operator.Word: "", # class: 'ow' - like keywords
+
+ Punctuation: FOREGROUND, # class: 'p'
+
+ Name: FOREGROUND, # class: 'n'
+ Name.Attribute: BLUE, # class: 'na' - to be revised
+ Name.Builtin: "", # class: 'nb'
+ Name.Builtin.Pseudo: "", # class: 'bp'
+ Name.Class: YELLOW, # class: 'nc' - to be revised
+ Name.Constant: RED, # class: 'no' - to be revised
+ Name.Decorator: AQUA, # class: 'nd' - to be revised
+ Name.Entity: "", # class: 'ni'
+ Name.Exception: RED, # class: 'ne'
+ Name.Function: BLUE, # class: 'nf'
+ Name.Property: "", # class: 'py'
+ Name.Label: "", # class: 'nl'
+ Name.Namespace: YELLOW, # class: 'nn' - to be revised
+ Name.Other: BLUE, # class: 'nx'
+ Name.Tag: AQUA, # class: 'nt' - like a keyword
+ Name.Variable: RED, # class: 'nv' - to be revised
+ Name.Variable.Class: "", # class: 'vc' - to be revised
+ Name.Variable.Global: "", # class: 'vg' - to be revised
+ Name.Variable.Instance: "", # class: 'vi' - to be revised
+
+ Number: ORANGE, # class: 'm'
+ Number.Float: "", # class: 'mf'
+ Number.Hex: "", # class: 'mh'
+ Number.Integer: "", # class: 'mi'
+ Number.Integer.Long: "", # class: 'il'
+ Number.Oct: "", # class: 'mo'
+
+ Literal: ORANGE, # class: 'l'
+ Literal.Date: GREEN, # class: 'ld'
+
+ String: GREEN, # class: 's'
+ String.Backtick: "", # class: 'sb'
+ String.Char: FOREGROUND, # class: 'sc'
+ String.Doc: COMMENT, # class: 'sd' - like a comment
+ String.Double: "", # class: 's2'
+ String.Escape: ORANGE, # class: 'se'
+ String.Heredoc: "", # class: 'sh'
+ String.Interpol: ORANGE, # class: 'si'
+ String.Other: "", # class: 'sx'
+ String.Regex: "", # class: 'sr'
+ String.Single: "", # class: 's1'
+ String.Symbol: "", # class: 'ss'
+
+ Generic: "", # class: 'g'
+ Generic.Deleted: RED, # class: 'gd',
+ Generic.Emph: "italic", # class: 'ge'
+ Generic.Error: "", # class: 'gr'
+ Generic.Heading: "bold " + FOREGROUND, # class: 'gh'
+ Generic.Inserted: GREEN, # class: 'gi'
+ Generic.Output: "", # class: 'go'
+ Generic.Prompt: "bold " + COMMENT, # class: 'gp'
+ Generic.Strong: "bold", # class: 'gs'
+ Generic.Subheading: "bold " + AQUA, # class: 'gu'
+ Generic.Traceback: "", # class: 'gt'
+ }
diff --git a/wandb/vendor/pygments/styles/pastie.py b/wandb/vendor/pygments/styles/pastie.py
new file mode 100644
index 0000000000000000000000000000000000000000..d6142908d6fbae58bd6be942ffd5db9a8e294b27
--- /dev/null
+++ b/wandb/vendor/pygments/styles/pastie.py
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.pastie
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Style similar to the `pastie`_ default style.
+
+ .. _pastie: http://pastie.caboo.se/
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Whitespace
+
+
+class PastieStyle(Style):
+ """
+ Style similar to the pastie default style.
+ """
+
+ default_style = ''
+
+ styles = {
+ Whitespace: '#bbbbbb',
+ Comment: '#888888',
+ Comment.Preproc: 'bold #cc0000',
+ Comment.Special: 'bg:#fff0f0 bold #cc0000',
+
+ String: 'bg:#fff0f0 #dd2200',
+ String.Regex: 'bg:#fff0ff #008800',
+ String.Other: 'bg:#f0fff0 #22bb22',
+ String.Symbol: '#aa6600',
+ String.Interpol: '#3333bb',
+ String.Escape: '#0044dd',
+
+ Operator.Word: '#008800',
+
+ Keyword: 'bold #008800',
+ Keyword.Pseudo: 'nobold',
+ Keyword.Type: '#888888',
+
+ Name.Class: 'bold #bb0066',
+ Name.Exception: 'bold #bb0066',
+ Name.Function: 'bold #0066bb',
+ Name.Property: 'bold #336699',
+ Name.Namespace: 'bold #bb0066',
+ Name.Builtin: '#003388',
+ Name.Variable: '#336699',
+ Name.Variable.Class: '#336699',
+ Name.Variable.Instance: '#3333bb',
+ Name.Variable.Global: '#dd7700',
+ Name.Constant: 'bold #003366',
+ Name.Tag: 'bold #bb0066',
+ Name.Attribute: '#336699',
+ Name.Decorator: '#555555',
+ Name.Label: 'italic #336699',
+
+ Number: 'bold #0000DD',
+
+ Generic.Heading: '#333',
+ Generic.Subheading: '#666',
+ Generic.Deleted: 'bg:#ffdddd #000000',
+ Generic.Inserted: 'bg:#ddffdd #000000',
+ Generic.Error: '#aa0000',
+ Generic.Emph: 'italic',
+ Generic.Strong: 'bold',
+ Generic.Prompt: '#555555',
+ Generic.Output: '#888888',
+ Generic.Traceback: '#aa0000',
+
+ Error: 'bg:#e3d2d2 #a61717'
+ }
diff --git a/wandb/vendor/pygments/styles/perldoc.py b/wandb/vendor/pygments/styles/perldoc.py
new file mode 100644
index 0000000000000000000000000000000000000000..24af2df6c6e435cb7e973f14cb7bd9da8366fa25
--- /dev/null
+++ b/wandb/vendor/pygments/styles/perldoc.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.perldoc
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Style similar to the style used in the `perldoc`_ code blocks.
+
+ .. _perldoc: http://perldoc.perl.org/
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Whitespace
+
+
+class PerldocStyle(Style):
+ """
+ Style similar to the style used in the perldoc code blocks.
+ """
+
+ background_color = '#eeeedd'
+ default_style = ''
+
+ styles = {
+ Whitespace: '#bbbbbb',
+ Comment: '#228B22',
+ Comment.Preproc: '#1e889b',
+ Comment.Special: '#8B008B bold',
+
+ String: '#CD5555',
+ String.Heredoc: '#1c7e71 italic',
+ String.Regex: '#B452CD',
+ String.Other: '#cb6c20',
+ String.Regex: '#1c7e71',
+
+ Number: '#B452CD',
+
+ Operator.Word: '#8B008B',
+
+ Keyword: '#8B008B bold',
+ Keyword.Type: '#00688B',
+
+ Name.Class: '#008b45 bold',
+ Name.Exception: '#008b45 bold',
+ Name.Function: '#008b45',
+ Name.Namespace: '#008b45 underline',
+ Name.Variable: '#00688B',
+ Name.Constant: '#00688B',
+ Name.Decorator: '#707a7c',
+ Name.Tag: '#8B008B bold',
+ Name.Attribute: '#658b00',
+ Name.Builtin: '#658b00',
+
+ Generic.Heading: 'bold #000080',
+ Generic.Subheading: 'bold #800080',
+ Generic.Deleted: '#aa0000',
+ Generic.Inserted: '#00aa00',
+ Generic.Error: '#aa0000',
+ Generic.Emph: 'italic',
+ Generic.Strong: 'bold',
+ Generic.Prompt: '#555555',
+ Generic.Output: '#888888',
+ Generic.Traceback: '#aa0000',
+
+ Error: 'bg:#e3d2d2 #a61717'
+ }
diff --git a/wandb/vendor/pygments/styles/rainbow_dash.py b/wandb/vendor/pygments/styles/rainbow_dash.py
new file mode 100644
index 0000000000000000000000000000000000000000..7cf5c9d776e6abb2e8ca4058fb7dee96360cebdb
--- /dev/null
+++ b/wandb/vendor/pygments/styles/rainbow_dash.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.rainbow_dash
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ A bright and colorful syntax highlighting `theme`.
+
+ .. _theme: http://sanssecours.github.io/Rainbow-Dash.tmbundle
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import (Comment, Error, Generic, Name, Number, Operator,
+ String, Text, Whitespace, Keyword)
+
+BLUE_LIGHT = '#0080ff'
+BLUE = '#2c5dcd'
+GREEN = '#00cc66'
+GREEN_LIGHT = '#ccffcc'
+GREEN_NEON = '#00cc00'
+GREY = '#aaaaaa'
+GREY_LIGHT = '#cbcbcb'
+GREY_DARK = '#4d4d4d'
+PURPLE = '#5918bb'
+RED = '#cc0000'
+RED_DARK = '#c5060b'
+RED_LIGHT = '#ffcccc'
+RED_BRIGHT = '#ff0000'
+WHITE = '#ffffff'
+TURQUOISE = '#318495'
+ORANGE = '#ff8000'
+
+
+class RainbowDashStyle(Style):
+ """
+ A bright and colorful syntax highlighting theme.
+ """
+
+ background_color = WHITE
+
+ styles = {
+ Comment: 'italic {}'.format(BLUE_LIGHT),
+ Comment.Preproc: 'noitalic',
+ Comment.Special: 'bold',
+
+ Error: 'bg:{} {}'.format(RED, WHITE),
+
+ Generic.Deleted: 'border:{} bg:{}'.format(RED_DARK, RED_LIGHT),
+ Generic.Emph: 'italic',
+ Generic.Error: RED_BRIGHT,
+ Generic.Heading: 'bold {}'.format(BLUE),
+ Generic.Inserted: 'border:{} bg:{}'.format(GREEN_NEON, GREEN_LIGHT),
+ Generic.Output: GREY,
+ Generic.Prompt: 'bold {}'.format(BLUE),
+ Generic.Strong: 'bold',
+ Generic.Subheading: 'bold {}'.format(BLUE),
+ Generic.Traceback: RED_DARK,
+
+ Keyword: 'bold {}'.format(BLUE),
+ Keyword.Pseudo: 'nobold',
+ Keyword.Type: PURPLE,
+
+ Name.Attribute: 'italic {}'.format(BLUE),
+ Name.Builtin: 'bold {}'.format(PURPLE),
+ Name.Class: 'underline',
+ Name.Constant: TURQUOISE,
+ Name.Decorator: 'bold {}'.format(ORANGE),
+ Name.Entity: 'bold {}'.format(PURPLE),
+ Name.Exception: 'bold {}'.format(PURPLE),
+ Name.Function: 'bold {}'.format(ORANGE),
+ Name.Tag: 'bold {}'.format(BLUE),
+
+ Number: 'bold {}'.format(PURPLE),
+
+ Operator: BLUE,
+ Operator.Word: 'bold',
+
+ String: GREEN,
+ String.Doc: 'italic',
+ String.Escape: 'bold {}'.format(RED_DARK),
+ String.Other: TURQUOISE,
+ String.Symbol: 'bold {}'.format(RED_DARK),
+
+ Text: GREY_DARK,
+
+ Whitespace: GREY_LIGHT
+ }
diff --git a/wandb/vendor/pygments/styles/rrt.py b/wandb/vendor/pygments/styles/rrt.py
new file mode 100644
index 0000000000000000000000000000000000000000..96f9490cad531ee5646439cdde207ea67da77793
--- /dev/null
+++ b/wandb/vendor/pygments/styles/rrt.py
@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.rrt
+ ~~~~~~~~~~~~~~~~~~~
+
+ pygments "rrt" theme, based on Zap and Emacs defaults.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Comment, Name, Keyword, String
+
+
+class RrtStyle(Style):
+ """
+ Minimalistic "rrt" theme, based on Zap and Emacs defaults.
+ """
+
+ background_color = '#000000'
+ highlight_color = '#0000ff'
+
+ styles = {
+ Comment: '#00ff00',
+ Name.Function: '#ffff00',
+ Name.Variable: '#eedd82',
+ Name.Constant: '#7fffd4',
+ Keyword: '#ff0000',
+ Comment.Preproc: '#e5e5e5',
+ String: '#87ceeb',
+ Keyword.Type: '#ee82ee',
+ }
diff --git a/wandb/vendor/pygments/styles/sas.py b/wandb/vendor/pygments/styles/sas.py
new file mode 100644
index 0000000000000000000000000000000000000000..78686fc2080210623e63a4d7346233e4f255f785
--- /dev/null
+++ b/wandb/vendor/pygments/styles/sas.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.sas
+ ~~~~~~~~~~~~~~~~~~~
+
+ Style inspired by SAS' enhanced program editor. Note This is not
+ meant to be a complete style. It's merely meant to mimic SAS'
+ program editor syntax highlighting.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Other, Whitespace, Generic
+
+
+class SasStyle(Style):
+ """
+ Style inspired by SAS' enhanced program editor. Note This is not
+ meant to be a complete style. It's merely meant to mimic SAS'
+ program editor syntax highlighting.
+ """
+
+ default_style = ''
+
+ styles = {
+ Whitespace: '#bbbbbb',
+ Comment: 'italic #008800',
+ String: '#800080',
+ Number: 'bold #2e8b57',
+ Other: 'bg:#ffffe0',
+ Keyword: '#2c2cff',
+ Keyword.Reserved: 'bold #353580',
+ Keyword.Constant: 'bold',
+ Name.Builtin: '#2c2cff',
+ Name.Function: 'bold italic',
+ Name.Variable: 'bold #2c2cff',
+ Generic: '#2c2cff',
+ Generic.Emph: '#008800',
+ Generic.Error: '#d30202',
+ Error: 'bg:#e3d2d2 #a61717'
+ }
diff --git a/wandb/vendor/pygments/styles/stata.py b/wandb/vendor/pygments/styles/stata.py
new file mode 100644
index 0000000000000000000000000000000000000000..2b5f5eddccf84193a509322773e710c90bed3ec1
--- /dev/null
+++ b/wandb/vendor/pygments/styles/stata.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.stata
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Style inspired by Stata's do-file editor. Note this is not meant
+ to be a complete style. It's merely meant to mimic Stata's do file
+ editor syntax highlighting.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Whitespace
+
+
+class StataStyle(Style):
+ """
+ Style inspired by Stata's do-file editor. Note this is not meant
+ to be a complete style. It's merely meant to mimic Stata's do file
+ editor syntax highlighting.
+ """
+
+ default_style = ''
+
+ styles = {
+ Whitespace: '#bbbbbb',
+ Comment: 'italic #008800',
+ String: '#7a2424',
+ Number: '#2c2cff',
+ Operator: '',
+ Keyword: 'bold #353580',
+ Keyword.Constant: '',
+ Name.Function: '#2c2cff',
+ Name.Variable: 'bold #35baba',
+ Name.Variable.Global: 'bold #b5565e',
+ Error: 'bg:#e3d2d2 #a61717'
+ }
diff --git a/wandb/vendor/pygments/styles/tango.py b/wandb/vendor/pygments/styles/tango.py
new file mode 100644
index 0000000000000000000000000000000000000000..2abc8c6178f2cddd5141dc7c22e424b341cf8e26
--- /dev/null
+++ b/wandb/vendor/pygments/styles/tango.py
@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.tango
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ The Crunchy default Style inspired from the color palette from
+ the Tango Icon Theme Guidelines.
+
+ http://tango.freedesktop.org/Tango_Icon_Theme_Guidelines
+
+ Butter: #fce94f #edd400 #c4a000
+ Orange: #fcaf3e #f57900 #ce5c00
+ Chocolate: #e9b96e #c17d11 #8f5902
+ Chameleon: #8ae234 #73d216 #4e9a06
+ Sky Blue: #729fcf #3465a4 #204a87
+ Plum: #ad7fa8 #75507b #5c35cc
+ Scarlet Red:#ef2929 #cc0000 #a40000
+ Aluminium: #eeeeec #d3d7cf #babdb6
+ #888a85 #555753 #2e3436
+
+ Not all of the above colors are used; other colors added:
+ very light grey: #f8f8f8 (for background)
+
+ This style can be used as a template as it includes all the known
+ Token types, unlike most (if not all) of the styles included in the
+ Pygments distribution.
+
+ However, since Crunchy is intended to be used by beginners, we have strived
+ to create a style that gloss over subtle distinctions between different
+ categories.
+
+ Taking Python for example, comments (Comment.*) and docstrings (String.Doc)
+ have been chosen to have the same style. Similarly, keywords (Keyword.*),
+ and Operator.Word (and, or, in) have been assigned the same style.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Whitespace, Punctuation, Other, Literal
+
+
+class TangoStyle(Style):
+ """
+ The Crunchy default Style inspired from the color palette from
+ the Tango Icon Theme Guidelines.
+ """
+
+ # work in progress...
+
+ background_color = "#f8f8f8"
+ default_style = ""
+
+ styles = {
+ # No corresponding class for the following:
+ #Text: "", # class: ''
+ Whitespace: "underline #f8f8f8", # class: 'w'
+ Error: "#a40000 border:#ef2929", # class: 'err'
+ Other: "#000000", # class 'x'
+
+ Comment: "italic #8f5902", # class: 'c'
+ Comment.Multiline: "italic #8f5902", # class: 'cm'
+ Comment.Preproc: "italic #8f5902", # class: 'cp'
+ Comment.Single: "italic #8f5902", # class: 'c1'
+ Comment.Special: "italic #8f5902", # class: 'cs'
+
+ Keyword: "bold #204a87", # class: 'k'
+ Keyword.Constant: "bold #204a87", # class: 'kc'
+ Keyword.Declaration: "bold #204a87", # class: 'kd'
+ Keyword.Namespace: "bold #204a87", # class: 'kn'
+ Keyword.Pseudo: "bold #204a87", # class: 'kp'
+ Keyword.Reserved: "bold #204a87", # class: 'kr'
+ Keyword.Type: "bold #204a87", # class: 'kt'
+
+ Operator: "bold #ce5c00", # class: 'o'
+ Operator.Word: "bold #204a87", # class: 'ow' - like keywords
+
+ Punctuation: "bold #000000", # class: 'p'
+
+ # because special names such as Name.Class, Name.Function, etc.
+ # are not recognized as such later in the parsing, we choose them
+ # to look the same as ordinary variables.
+ Name: "#000000", # class: 'n'
+ Name.Attribute: "#c4a000", # class: 'na' - to be revised
+ Name.Builtin: "#204a87", # class: 'nb'
+ Name.Builtin.Pseudo: "#3465a4", # class: 'bp'
+ Name.Class: "#000000", # class: 'nc' - to be revised
+ Name.Constant: "#000000", # class: 'no' - to be revised
+ Name.Decorator: "bold #5c35cc", # class: 'nd' - to be revised
+ Name.Entity: "#ce5c00", # class: 'ni'
+ Name.Exception: "bold #cc0000", # class: 'ne'
+ Name.Function: "#000000", # class: 'nf'
+ Name.Property: "#000000", # class: 'py'
+ Name.Label: "#f57900", # class: 'nl'
+ Name.Namespace: "#000000", # class: 'nn' - to be revised
+ Name.Other: "#000000", # class: 'nx'
+ Name.Tag: "bold #204a87", # class: 'nt' - like a keyword
+ Name.Variable: "#000000", # class: 'nv' - to be revised
+ Name.Variable.Class: "#000000", # class: 'vc' - to be revised
+ Name.Variable.Global: "#000000", # class: 'vg' - to be revised
+ Name.Variable.Instance: "#000000", # class: 'vi' - to be revised
+
+ # since the tango light blue does not show up well in text, we choose
+ # a pure blue instead.
+ Number: "bold #0000cf", # class: 'm'
+ Number.Float: "bold #0000cf", # class: 'mf'
+ Number.Hex: "bold #0000cf", # class: 'mh'
+ Number.Integer: "bold #0000cf", # class: 'mi'
+ Number.Integer.Long: "bold #0000cf", # class: 'il'
+ Number.Oct: "bold #0000cf", # class: 'mo'
+
+ Literal: "#000000", # class: 'l'
+ Literal.Date: "#000000", # class: 'ld'
+
+ String: "#4e9a06", # class: 's'
+ String.Backtick: "#4e9a06", # class: 'sb'
+ String.Char: "#4e9a06", # class: 'sc'
+ String.Doc: "italic #8f5902", # class: 'sd' - like a comment
+ String.Double: "#4e9a06", # class: 's2'
+ String.Escape: "#4e9a06", # class: 'se'
+ String.Heredoc: "#4e9a06", # class: 'sh'
+ String.Interpol: "#4e9a06", # class: 'si'
+ String.Other: "#4e9a06", # class: 'sx'
+ String.Regex: "#4e9a06", # class: 'sr'
+ String.Single: "#4e9a06", # class: 's1'
+ String.Symbol: "#4e9a06", # class: 'ss'
+
+ Generic: "#000000", # class: 'g'
+ Generic.Deleted: "#a40000", # class: 'gd'
+ Generic.Emph: "italic #000000", # class: 'ge'
+ Generic.Error: "#ef2929", # class: 'gr'
+ Generic.Heading: "bold #000080", # class: 'gh'
+ Generic.Inserted: "#00A000", # class: 'gi'
+ Generic.Output: "italic #000000", # class: 'go'
+ Generic.Prompt: "#8f5902", # class: 'gp'
+ Generic.Strong: "bold #000000", # class: 'gs'
+ Generic.Subheading: "bold #800080", # class: 'gu'
+ Generic.Traceback: "bold #a40000", # class: 'gt'
+ }
diff --git a/wandb/vendor/pygments/styles/trac.py b/wandb/vendor/pygments/styles/trac.py
new file mode 100644
index 0000000000000000000000000000000000000000..aff39fd4367e0a706f8c0d674f17afdf49807601
--- /dev/null
+++ b/wandb/vendor/pygments/styles/trac.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.trac
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Port of the default trac highlighter design.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Whitespace
+
+
+class TracStyle(Style):
+ """
+ Port of the default trac highlighter design.
+ """
+
+ default_style = ''
+
+ styles = {
+ Whitespace: '#bbbbbb',
+ Comment: 'italic #999988',
+ Comment.Preproc: 'bold noitalic #999999',
+ Comment.Special: 'bold #999999',
+
+ Operator: 'bold',
+
+ String: '#bb8844',
+ String.Regex: '#808000',
+
+ Number: '#009999',
+
+ Keyword: 'bold',
+ Keyword.Type: '#445588',
+
+ Name.Builtin: '#999999',
+ Name.Function: 'bold #990000',
+ Name.Class: 'bold #445588',
+ Name.Exception: 'bold #990000',
+ Name.Namespace: '#555555',
+ Name.Variable: '#008080',
+ Name.Constant: '#008080',
+ Name.Tag: '#000080',
+ Name.Attribute: '#008080',
+ Name.Entity: '#800080',
+
+ Generic.Heading: '#999999',
+ Generic.Subheading: '#aaaaaa',
+ Generic.Deleted: 'bg:#ffdddd #000000',
+ Generic.Inserted: 'bg:#ddffdd #000000',
+ Generic.Error: '#aa0000',
+ Generic.Emph: 'italic',
+ Generic.Strong: 'bold',
+ Generic.Prompt: '#555555',
+ Generic.Output: '#888888',
+ Generic.Traceback: '#aa0000',
+
+ Error: 'bg:#e3d2d2 #a61717'
+ }
diff --git a/wandb/vendor/pygments/styles/vim.py b/wandb/vendor/pygments/styles/vim.py
new file mode 100644
index 0000000000000000000000000000000000000000..888088b14a8f3bc83aad703b613cf90c6ef5e4a2
--- /dev/null
+++ b/wandb/vendor/pygments/styles/vim.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.vim
+ ~~~~~~~~~~~~~~~~~~~
+
+ A highlighting style for Pygments, inspired by vim.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic, Whitespace, Token
+
+
+class VimStyle(Style):
+ """
+ Styles somewhat like vim 7.0
+ """
+
+ background_color = "#000000"
+ highlight_color = "#222222"
+ default_style = "#cccccc"
+
+ styles = {
+ Token: "#cccccc",
+ Whitespace: "",
+ Comment: "#000080",
+ Comment.Preproc: "",
+ Comment.Special: "bold #cd0000",
+
+ Keyword: "#cdcd00",
+ Keyword.Declaration: "#00cd00",
+ Keyword.Namespace: "#cd00cd",
+ Keyword.Pseudo: "",
+ Keyword.Type: "#00cd00",
+
+ Operator: "#3399cc",
+ Operator.Word: "#cdcd00",
+
+ Name: "",
+ Name.Class: "#00cdcd",
+ Name.Builtin: "#cd00cd",
+ Name.Exception: "bold #666699",
+ Name.Variable: "#00cdcd",
+
+ String: "#cd0000",
+ Number: "#cd00cd",
+
+ Generic.Heading: "bold #000080",
+ Generic.Subheading: "bold #800080",
+ Generic.Deleted: "#cd0000",
+ Generic.Inserted: "#00cd00",
+ Generic.Error: "#FF0000",
+ Generic.Emph: "italic",
+ Generic.Strong: "bold",
+ Generic.Prompt: "bold #000080",
+ Generic.Output: "#888",
+ Generic.Traceback: "#04D",
+
+ Error: "border:#FF0000"
+ }
diff --git a/wandb/vendor/pygments/styles/vs.py b/wandb/vendor/pygments/styles/vs.py
new file mode 100644
index 0000000000000000000000000000000000000000..bc3ed2b588a0c3aa2cde8a7a28a98d5bf5b7a7f4
--- /dev/null
+++ b/wandb/vendor/pygments/styles/vs.py
@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.vs
+ ~~~~~~~~~~~~~~~~~~
+
+ Simple style with MS Visual Studio colors.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Operator, Generic
+
+
+class VisualStudioStyle(Style):
+
+ background_color = "#ffffff"
+ default_style = ""
+
+ styles = {
+ Comment: "#008000",
+ Comment.Preproc: "#0000ff",
+ Keyword: "#0000ff",
+ Operator.Word: "#0000ff",
+ Keyword.Type: "#2b91af",
+ Name.Class: "#2b91af",
+ String: "#a31515",
+
+ Generic.Heading: "bold",
+ Generic.Subheading: "bold",
+ Generic.Emph: "italic",
+ Generic.Strong: "bold",
+ Generic.Prompt: "bold",
+
+ Error: "border:#FF0000"
+ }
diff --git a/wandb/vendor/pygments/styles/xcode.py b/wandb/vendor/pygments/styles/xcode.py
new file mode 100644
index 0000000000000000000000000000000000000000..64bfcf03ea00f1f758082ad8aad65dc73a720423
--- /dev/null
+++ b/wandb/vendor/pygments/styles/xcode.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.styles.xcode
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Style similar to the `Xcode` default theme.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Literal
+
+
+class XcodeStyle(Style):
+ """
+ Style similar to the Xcode default colouring theme.
+ """
+
+ default_style = ''
+
+ styles = {
+ Comment: '#177500',
+ Comment.Preproc: '#633820',
+
+ String: '#C41A16',
+ String.Char: '#2300CE',
+
+ Operator: '#000000',
+
+ Keyword: '#A90D91',
+
+ Name: '#000000',
+ Name.Attribute: '#836C28',
+ Name.Class: '#3F6E75',
+ Name.Function: '#000000',
+ Name.Builtin: '#A90D91',
+ # In Obj-C code this token is used to colour Cocoa types
+ Name.Builtin.Pseudo: '#5B269A',
+ Name.Variable: '#000000',
+ Name.Tag: '#000000',
+ Name.Decorator: '#000000',
+ # Workaround for a BUG here: lexer treats multiline method signatres as labels
+ Name.Label: '#000000',
+
+ Literal: '#1C01CE',
+ Number: '#1C01CE',
+ Error: '#000000',
+ }
diff --git a/wandb/vendor/pygments/token.py b/wandb/vendor/pygments/token.py
new file mode 100644
index 0000000000000000000000000000000000000000..43f73c855625721d88096cd0e2a83a6b747a6700
--- /dev/null
+++ b/wandb/vendor/pygments/token.py
@@ -0,0 +1,213 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.token
+ ~~~~~~~~~~~~~~
+
+ Basic token types and the standard tokens.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+
+class _TokenType(tuple):
+ parent = None
+
+ def split(self):
+ buf = []
+ node = self
+ while node is not None:
+ buf.append(node)
+ node = node.parent
+ buf.reverse()
+ return buf
+
+ def __init__(self, *args):
+ # no need to call super.__init__
+ self.subtypes = set()
+
+ def __contains__(self, val):
+ return self is val or (
+ type(val) is self.__class__ and
+ val[:len(self)] == self
+ )
+
+ def __getattr__(self, val):
+ if not val or not val[0].isupper():
+ return tuple.__getattribute__(self, val)
+ new = _TokenType(self + (val,))
+ setattr(self, val, new)
+ self.subtypes.add(new)
+ new.parent = self
+ return new
+
+ def __repr__(self):
+ return 'Token' + (self and '.' or '') + '.'.join(self)
+
+ def __copy__(self):
+ # These instances are supposed to be singletons
+ return self
+
+ def __deepcopy__(self, memo):
+ # These instances are supposed to be singletons
+ return self
+
+
+Token = _TokenType()
+
+# Special token types
+Text = Token.Text
+Whitespace = Text.Whitespace
+Escape = Token.Escape
+Error = Token.Error
+# Text that doesn't belong to this lexer (e.g. HTML in PHP)
+Other = Token.Other
+
+# Common token types for source code
+Keyword = Token.Keyword
+Name = Token.Name
+Literal = Token.Literal
+String = Literal.String
+Number = Literal.Number
+Punctuation = Token.Punctuation
+Operator = Token.Operator
+Comment = Token.Comment
+
+# Generic types for non-source code
+Generic = Token.Generic
+
+# String and some others are not direct children of Token.
+# alias them:
+Token.Token = Token
+Token.String = String
+Token.Number = Number
+
+
+def is_token_subtype(ttype, other):
+ """
+ Return True if ``ttype`` is a subtype of ``other``.
+
+ exists for backwards compatibility. use ``ttype in other`` now.
+ """
+ return ttype in other
+
+
+def string_to_tokentype(s):
+ """
+ Convert a string into a token type::
+
+ >>> string_to_token('String.Double')
+ Token.Literal.String.Double
+ >>> string_to_token('Token.Literal.Number')
+ Token.Literal.Number
+ >>> string_to_token('')
+ Token
+
+ Tokens that are already tokens are returned unchanged:
+
+ >>> string_to_token(String)
+ Token.Literal.String
+ """
+ if isinstance(s, _TokenType):
+ return s
+ if not s:
+ return Token
+ node = Token
+ for item in s.split('.'):
+ node = getattr(node, item)
+ return node
+
+
+# Map standard token types to short names, used in CSS class naming.
+# If you add a new item, please be sure to run this file to perform
+# a consistency check for duplicate values.
+STANDARD_TYPES = {
+ Token: '',
+
+ Text: '',
+ Whitespace: 'w',
+ Escape: 'esc',
+ Error: 'err',
+ Other: 'x',
+
+ Keyword: 'k',
+ Keyword.Constant: 'kc',
+ Keyword.Declaration: 'kd',
+ Keyword.Namespace: 'kn',
+ Keyword.Pseudo: 'kp',
+ Keyword.Reserved: 'kr',
+ Keyword.Type: 'kt',
+
+ Name: 'n',
+ Name.Attribute: 'na',
+ Name.Builtin: 'nb',
+ Name.Builtin.Pseudo: 'bp',
+ Name.Class: 'nc',
+ Name.Constant: 'no',
+ Name.Decorator: 'nd',
+ Name.Entity: 'ni',
+ Name.Exception: 'ne',
+ Name.Function: 'nf',
+ Name.Function.Magic: 'fm',
+ Name.Property: 'py',
+ Name.Label: 'nl',
+ Name.Namespace: 'nn',
+ Name.Other: 'nx',
+ Name.Tag: 'nt',
+ Name.Variable: 'nv',
+ Name.Variable.Class: 'vc',
+ Name.Variable.Global: 'vg',
+ Name.Variable.Instance: 'vi',
+ Name.Variable.Magic: 'vm',
+
+ Literal: 'l',
+ Literal.Date: 'ld',
+
+ String: 's',
+ String.Affix: 'sa',
+ String.Backtick: 'sb',
+ String.Char: 'sc',
+ String.Delimiter: 'dl',
+ String.Doc: 'sd',
+ String.Double: 's2',
+ String.Escape: 'se',
+ String.Heredoc: 'sh',
+ String.Interpol: 'si',
+ String.Other: 'sx',
+ String.Regex: 'sr',
+ String.Single: 's1',
+ String.Symbol: 'ss',
+
+ Number: 'm',
+ Number.Bin: 'mb',
+ Number.Float: 'mf',
+ Number.Hex: 'mh',
+ Number.Integer: 'mi',
+ Number.Integer.Long: 'il',
+ Number.Oct: 'mo',
+
+ Operator: 'o',
+ Operator.Word: 'ow',
+
+ Punctuation: 'p',
+
+ Comment: 'c',
+ Comment.Hashbang: 'ch',
+ Comment.Multiline: 'cm',
+ Comment.Preproc: 'cp',
+ Comment.PreprocFile: 'cpf',
+ Comment.Single: 'c1',
+ Comment.Special: 'cs',
+
+ Generic: 'g',
+ Generic.Deleted: 'gd',
+ Generic.Emph: 'ge',
+ Generic.Error: 'gr',
+ Generic.Heading: 'gh',
+ Generic.Inserted: 'gi',
+ Generic.Output: 'go',
+ Generic.Prompt: 'gp',
+ Generic.Strong: 'gs',
+ Generic.Subheading: 'gu',
+ Generic.Traceback: 'gt',
+}
diff --git a/wandb/vendor/pygments/unistring.py b/wandb/vendor/pygments/unistring.py
new file mode 100644
index 0000000000000000000000000000000000000000..6096d110ac21a85fbaf72914abf7de4a712bae70
--- /dev/null
+++ b/wandb/vendor/pygments/unistring.py
@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.unistring
+ ~~~~~~~~~~~~~~~~~~
+
+ Strings of all Unicode characters of a certain category.
+ Used for matching in Unicode-aware languages. Run to regenerate.
+
+ Inspired by chartypes_create.py from the MoinMoin project.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import sys
+
+Cc = u'\x00-\x1f\x7f-\x9f'
+
+Cf = u'\xad\u0600-\u0604\u061c\u06dd\u070f\u180e\u200b-\u200f\u202a-\u202e\u2060-\u2064\u2066-\u206f\ufeff\ufff9-\ufffb'
+
+Cn = u'\u0378-\u0379\u037f-\u0383\u038b\u038d\u03a2\u0528-\u0530\u0557-\u0558\u0560\u0588\u058b-\u058e\u0590\u05c8-\u05cf\u05eb-\u05ef\u05f5-\u05ff\u0605\u061d\u070e\u074b-\u074c\u07b2-\u07bf\u07fb-\u07ff\u082e-\u082f\u083f\u085c-\u085d\u085f-\u089f\u08a1\u08ad-\u08e3\u08ff\u0978\u0980\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09c5-\u09c6\u09c9-\u09ca\u09cf-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09fc-\u0a00\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a50\u0a52-\u0a58\u0a5d\u0a5f-\u0a65\u0a76-\u0a80\u0a84\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0acf\u0ad1-\u0adf\u0ae4-\u0ae5\u0af2-\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34\u0b3a-\u0b3b\u0b45-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b64-\u0b65\u0b78-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bcf\u0bd1-\u0bd6\u0bd8-\u0be5\u0bfb-\u0c00\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c3c\u0c45\u0c49\u0c4e-\u0c54\u0c57\u0c5a-\u0c5f\u0c64-\u0c65\u0c70-\u0c77\u0c80-\u0c81\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbb\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce4-\u0ce5\u0cf0\u0cf3-\u0d01\u0d04\u0d0d\u0d11\u0d3b-\u0d3c\u0d45\u0d49\u0d4f-\u0d56\u0d58-\u0d5f\u0d64-\u0d65\u0d76-\u0d78\u0d80-\u0d81\u0d84\u0d97-\u0d99\u0db2\u0dbc\u0dbe-\u0dbf\u0dc7-\u0dc9\u0dcb-\u0dce\u0dd5\u0dd7\u0de0-\u0df1\u0df5-\u0e00\u0e3b-\u0e3e\u0e5c-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0edb\u0ee0-\u0eff\u0f48\u0f6d-\u0f70\u0f98\u0fbd\u0fcd\u0fdb-\u0fff\u10c6\u10c8-\u10cc\u10ce-\u10cf\u1249\u124e-\u124f\u1257\u1259\u125e-\u125f\u1289\u128e-\u128f\u12b1\u12b6-\u12b7\u12bf\u12c1\u12c6-\u12c7\u12d7\u1311\u1316-\u1317\u135b-\u135c\u137d-\u137f\u139a-\u139f\u13f5-\u13ff\u169d-\u169f\u16f1-\u16ff\u170d\u1715-\u171f\u1737-\u173f\u1754-\u175f\u176d\u1771\u1774-\u177f\u17de-\u17df\u17ea-\u17ef\u17fa-\u17ff\u180f\u181a-\u181f\u1878-\u187f\u18ab-\u18af\u18f6-\u18ff\u191d-\u191f\u192c-\u192f\u193c-\u193f\u1941-\u1943\u196e-\u196f\u1975-\u197f\u19ac-\u19af\u19ca-\u19cf\u19db-\u19dd\u1a1c-\u1a1d\u1a5f\u1a7d-\u1a7e\u1a8a-\u1a8f\u1a9a-\u1a9f\u1aae-\u1aff\u1b4c-\u1b4f\u1b7d-\u1b7f\u1bf4-\u1bfb\u1c38-\u1c3a\u1c4a-\u1c4c\u1c80-\u1cbf\u1cc8-\u1ccf\u1cf7-\u1cff\u1de7-\u1dfb\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fc5\u1fd4-\u1fd5\u1fdc\u1ff0-\u1ff1\u1ff5\u1fff\u2065\u2072-\u2073\u208f\u209d-\u209f\u20bb-\u20cf\u20f1-\u20ff\u218a-\u218f\u23f4-\u23ff\u2427-\u243f\u244b-\u245f\u2700\u2b4d-\u2b4f\u2b5a-\u2bff\u2c2f\u2c5f\u2cf4-\u2cf8\u2d26\u2d28-\u2d2c\u2d2e-\u2d2f\u2d68-\u2d6e\u2d71-\u2d7e\u2d97-\u2d9f\u2da7\u2daf\u2db7\u2dbf\u2dc7\u2dcf\u2dd7\u2ddf\u2e3c-\u2e7f\u2e9a\u2ef4-\u2eff\u2fd6-\u2fef\u2ffc-\u2fff\u3040\u3097-\u3098\u3100-\u3104\u312e-\u3130\u318f\u31bb-\u31bf\u31e4-\u31ef\u321f\u32ff\u4db6-\u4dbf\u9fcd-\u9fff\ua48d-\ua48f\ua4c7-\ua4cf\ua62c-\ua63f\ua698-\ua69e\ua6f8-\ua6ff\ua78f\ua794-\ua79f\ua7ab-\ua7f7\ua82c-\ua82f\ua83a-\ua83f\ua878-\ua87f\ua8c5-\ua8cd\ua8da-\ua8df\ua8fc-\ua8ff\ua954-\ua95e\ua97d-\ua97f\ua9ce\ua9da-\ua9dd\ua9e0-\ua9ff\uaa37-\uaa3f\uaa4e-\uaa4f\uaa5a-\uaa5b\uaa7c-\uaa7f\uaac3-\uaada\uaaf7-\uab00\uab07-\uab08\uab0f-\uab10\uab17-\uab1f\uab27\uab2f-\uabbf\uabee-\uabef\uabfa-\uabff\ud7a4-\ud7af\ud7c7-\ud7ca\ud7fc-\ud7ff\ufa6e-\ufa6f\ufada-\ufaff\ufb07-\ufb12\ufb18-\ufb1c\ufb37\ufb3d\ufb3f\ufb42\ufb45\ufbc2-\ufbd2\ufd40-\ufd4f\ufd90-\ufd91\ufdc8-\ufdef\ufdfe-\ufdff\ufe1a-\ufe1f\ufe27-\ufe2f\ufe53\ufe67\ufe6c-\ufe6f\ufe75\ufefd-\ufefe\uff00\uffbf-\uffc1\uffc8-\uffc9\uffd0-\uffd1\uffd8-\uffd9\uffdd-\uffdf\uffe7\uffef-\ufff8\ufffe-\uffff'
+
+Co = u'\ue000-\uf8ff'
+
+try:
+ Cs = eval(r"u'\ud800-\udbff\\\udc00\udc01-\udfff'")
+except UnicodeDecodeError:
+ Cs = '' # Jython can't handle isolated surrogates
+
+Ll = u'a-z\xb5\xdf-\xf6\xf8-\xff\u0101\u0103\u0105\u0107\u0109\u010b\u010d\u010f\u0111\u0113\u0115\u0117\u0119\u011b\u011d\u011f\u0121\u0123\u0125\u0127\u0129\u012b\u012d\u012f\u0131\u0133\u0135\u0137-\u0138\u013a\u013c\u013e\u0140\u0142\u0144\u0146\u0148-\u0149\u014b\u014d\u014f\u0151\u0153\u0155\u0157\u0159\u015b\u015d\u015f\u0161\u0163\u0165\u0167\u0169\u016b\u016d\u016f\u0171\u0173\u0175\u0177\u017a\u017c\u017e-\u0180\u0183\u0185\u0188\u018c-\u018d\u0192\u0195\u0199-\u019b\u019e\u01a1\u01a3\u01a5\u01a8\u01aa-\u01ab\u01ad\u01b0\u01b4\u01b6\u01b9-\u01ba\u01bd-\u01bf\u01c6\u01c9\u01cc\u01ce\u01d0\u01d2\u01d4\u01d6\u01d8\u01da\u01dc-\u01dd\u01df\u01e1\u01e3\u01e5\u01e7\u01e9\u01eb\u01ed\u01ef-\u01f0\u01f3\u01f5\u01f9\u01fb\u01fd\u01ff\u0201\u0203\u0205\u0207\u0209\u020b\u020d\u020f\u0211\u0213\u0215\u0217\u0219\u021b\u021d\u021f\u0221\u0223\u0225\u0227\u0229\u022b\u022d\u022f\u0231\u0233-\u0239\u023c\u023f-\u0240\u0242\u0247\u0249\u024b\u024d\u024f-\u0293\u0295-\u02af\u0371\u0373\u0377\u037b-\u037d\u0390\u03ac-\u03ce\u03d0-\u03d1\u03d5-\u03d7\u03d9\u03db\u03dd\u03df\u03e1\u03e3\u03e5\u03e7\u03e9\u03eb\u03ed\u03ef-\u03f3\u03f5\u03f8\u03fb-\u03fc\u0430-\u045f\u0461\u0463\u0465\u0467\u0469\u046b\u046d\u046f\u0471\u0473\u0475\u0477\u0479\u047b\u047d\u047f\u0481\u048b\u048d\u048f\u0491\u0493\u0495\u0497\u0499\u049b\u049d\u049f\u04a1\u04a3\u04a5\u04a7\u04a9\u04ab\u04ad\u04af\u04b1\u04b3\u04b5\u04b7\u04b9\u04bb\u04bd\u04bf\u04c2\u04c4\u04c6\u04c8\u04ca\u04cc\u04ce-\u04cf\u04d1\u04d3\u04d5\u04d7\u04d9\u04db\u04dd\u04df\u04e1\u04e3\u04e5\u04e7\u04e9\u04eb\u04ed\u04ef\u04f1\u04f3\u04f5\u04f7\u04f9\u04fb\u04fd\u04ff\u0501\u0503\u0505\u0507\u0509\u050b\u050d\u050f\u0511\u0513\u0515\u0517\u0519\u051b\u051d\u051f\u0521\u0523\u0525\u0527\u0561-\u0587\u1d00-\u1d2b\u1d6b-\u1d77\u1d79-\u1d9a\u1e01\u1e03\u1e05\u1e07\u1e09\u1e0b\u1e0d\u1e0f\u1e11\u1e13\u1e15\u1e17\u1e19\u1e1b\u1e1d\u1e1f\u1e21\u1e23\u1e25\u1e27\u1e29\u1e2b\u1e2d\u1e2f\u1e31\u1e33\u1e35\u1e37\u1e39\u1e3b\u1e3d\u1e3f\u1e41\u1e43\u1e45\u1e47\u1e49\u1e4b\u1e4d\u1e4f\u1e51\u1e53\u1e55\u1e57\u1e59\u1e5b\u1e5d\u1e5f\u1e61\u1e63\u1e65\u1e67\u1e69\u1e6b\u1e6d\u1e6f\u1e71\u1e73\u1e75\u1e77\u1e79\u1e7b\u1e7d\u1e7f\u1e81\u1e83\u1e85\u1e87\u1e89\u1e8b\u1e8d\u1e8f\u1e91\u1e93\u1e95-\u1e9d\u1e9f\u1ea1\u1ea3\u1ea5\u1ea7\u1ea9\u1eab\u1ead\u1eaf\u1eb1\u1eb3\u1eb5\u1eb7\u1eb9\u1ebb\u1ebd\u1ebf\u1ec1\u1ec3\u1ec5\u1ec7\u1ec9\u1ecb\u1ecd\u1ecf\u1ed1\u1ed3\u1ed5\u1ed7\u1ed9\u1edb\u1edd\u1edf\u1ee1\u1ee3\u1ee5\u1ee7\u1ee9\u1eeb\u1eed\u1eef\u1ef1\u1ef3\u1ef5\u1ef7\u1ef9\u1efb\u1efd\u1eff-\u1f07\u1f10-\u1f15\u1f20-\u1f27\u1f30-\u1f37\u1f40-\u1f45\u1f50-\u1f57\u1f60-\u1f67\u1f70-\u1f7d\u1f80-\u1f87\u1f90-\u1f97\u1fa0-\u1fa7\u1fb0-\u1fb4\u1fb6-\u1fb7\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fc7\u1fd0-\u1fd3\u1fd6-\u1fd7\u1fe0-\u1fe7\u1ff2-\u1ff4\u1ff6-\u1ff7\u210a\u210e-\u210f\u2113\u212f\u2134\u2139\u213c-\u213d\u2146-\u2149\u214e\u2184\u2c30-\u2c5e\u2c61\u2c65-\u2c66\u2c68\u2c6a\u2c6c\u2c71\u2c73-\u2c74\u2c76-\u2c7b\u2c81\u2c83\u2c85\u2c87\u2c89\u2c8b\u2c8d\u2c8f\u2c91\u2c93\u2c95\u2c97\u2c99\u2c9b\u2c9d\u2c9f\u2ca1\u2ca3\u2ca5\u2ca7\u2ca9\u2cab\u2cad\u2caf\u2cb1\u2cb3\u2cb5\u2cb7\u2cb9\u2cbb\u2cbd\u2cbf\u2cc1\u2cc3\u2cc5\u2cc7\u2cc9\u2ccb\u2ccd\u2ccf\u2cd1\u2cd3\u2cd5\u2cd7\u2cd9\u2cdb\u2cdd\u2cdf\u2ce1\u2ce3-\u2ce4\u2cec\u2cee\u2cf3\u2d00-\u2d25\u2d27\u2d2d\ua641\ua643\ua645\ua647\ua649\ua64b\ua64d\ua64f\ua651\ua653\ua655\ua657\ua659\ua65b\ua65d\ua65f\ua661\ua663\ua665\ua667\ua669\ua66b\ua66d\ua681\ua683\ua685\ua687\ua689\ua68b\ua68d\ua68f\ua691\ua693\ua695\ua697\ua723\ua725\ua727\ua729\ua72b\ua72d\ua72f-\ua731\ua733\ua735\ua737\ua739\ua73b\ua73d\ua73f\ua741\ua743\ua745\ua747\ua749\ua74b\ua74d\ua74f\ua751\ua753\ua755\ua757\ua759\ua75b\ua75d\ua75f\ua761\ua763\ua765\ua767\ua769\ua76b\ua76d\ua76f\ua771-\ua778\ua77a\ua77c\ua77f\ua781\ua783\ua785\ua787\ua78c\ua78e\ua791\ua793\ua7a1\ua7a3\ua7a5\ua7a7\ua7a9\ua7fa\ufb00-\ufb06\ufb13-\ufb17\uff41-\uff5a'
+
+Lm = u'\u02b0-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0374\u037a\u0559\u0640\u06e5-\u06e6\u07f4-\u07f5\u07fa\u081a\u0824\u0828\u0971\u0e46\u0ec6\u10fc\u17d7\u1843\u1aa7\u1c78-\u1c7d\u1d2c-\u1d6a\u1d78\u1d9b-\u1dbf\u2071\u207f\u2090-\u209c\u2c7c-\u2c7d\u2d6f\u2e2f\u3005\u3031-\u3035\u303b\u309d-\u309e\u30fc-\u30fe\ua015\ua4f8-\ua4fd\ua60c\ua67f\ua717-\ua71f\ua770\ua788\ua7f8-\ua7f9\ua9cf\uaa70\uaadd\uaaf3-\uaaf4\uff70\uff9e-\uff9f'
+
+Lo = u'\xaa\xba\u01bb\u01c0-\u01c3\u0294\u05d0-\u05ea\u05f0-\u05f2\u0620-\u063f\u0641-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u0800-\u0815\u0840-\u0858\u08a0\u08a2-\u08ac\u0904-\u0939\u093d\u0950\u0958-\u0961\u0972-\u0977\u0979-\u097f\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d\u0c58-\u0c59\u0c60-\u0c61\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d60-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32-\u0e33\u0e40-\u0e45\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2-\u0eb3\u0ebd\u0ec0-\u0ec4\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u10d0-\u10fa\u10fd-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17dc\u1820-\u1842\u1844-\u1877\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191c\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19c1-\u19c7\u1a00-\u1a16\u1a20-\u1a54\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c77\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u2135-\u2138\u2d30-\u2d67\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3006\u303c\u3041-\u3096\u309f\u30a1-\u30fa\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua014\ua016-\ua48c\ua4d0-\ua4f7\ua500-\ua60b\ua610-\ua61f\ua62a-\ua62b\ua66e\ua6a0-\ua6e5\ua7fb-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa6f\uaa71-\uaa76\uaa7a\uaa80-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadc\uaae0-\uaaea\uaaf2\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdfb\ufe70-\ufe74\ufe76-\ufefc\uff66-\uff6f\uff71-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc'
+
+Lt = u'\u01c5\u01c8\u01cb\u01f2\u1f88-\u1f8f\u1f98-\u1f9f\u1fa8-\u1faf\u1fbc\u1fcc\u1ffc'
+
+Lu = u'A-Z\xc0-\xd6\xd8-\xde\u0100\u0102\u0104\u0106\u0108\u010a\u010c\u010e\u0110\u0112\u0114\u0116\u0118\u011a\u011c\u011e\u0120\u0122\u0124\u0126\u0128\u012a\u012c\u012e\u0130\u0132\u0134\u0136\u0139\u013b\u013d\u013f\u0141\u0143\u0145\u0147\u014a\u014c\u014e\u0150\u0152\u0154\u0156\u0158\u015a\u015c\u015e\u0160\u0162\u0164\u0166\u0168\u016a\u016c\u016e\u0170\u0172\u0174\u0176\u0178-\u0179\u017b\u017d\u0181-\u0182\u0184\u0186-\u0187\u0189-\u018b\u018e-\u0191\u0193-\u0194\u0196-\u0198\u019c-\u019d\u019f-\u01a0\u01a2\u01a4\u01a6-\u01a7\u01a9\u01ac\u01ae-\u01af\u01b1-\u01b3\u01b5\u01b7-\u01b8\u01bc\u01c4\u01c7\u01ca\u01cd\u01cf\u01d1\u01d3\u01d5\u01d7\u01d9\u01db\u01de\u01e0\u01e2\u01e4\u01e6\u01e8\u01ea\u01ec\u01ee\u01f1\u01f4\u01f6-\u01f8\u01fa\u01fc\u01fe\u0200\u0202\u0204\u0206\u0208\u020a\u020c\u020e\u0210\u0212\u0214\u0216\u0218\u021a\u021c\u021e\u0220\u0222\u0224\u0226\u0228\u022a\u022c\u022e\u0230\u0232\u023a-\u023b\u023d-\u023e\u0241\u0243-\u0246\u0248\u024a\u024c\u024e\u0370\u0372\u0376\u0386\u0388-\u038a\u038c\u038e-\u038f\u0391-\u03a1\u03a3-\u03ab\u03cf\u03d2-\u03d4\u03d8\u03da\u03dc\u03de\u03e0\u03e2\u03e4\u03e6\u03e8\u03ea\u03ec\u03ee\u03f4\u03f7\u03f9-\u03fa\u03fd-\u042f\u0460\u0462\u0464\u0466\u0468\u046a\u046c\u046e\u0470\u0472\u0474\u0476\u0478\u047a\u047c\u047e\u0480\u048a\u048c\u048e\u0490\u0492\u0494\u0496\u0498\u049a\u049c\u049e\u04a0\u04a2\u04a4\u04a6\u04a8\u04aa\u04ac\u04ae\u04b0\u04b2\u04b4\u04b6\u04b8\u04ba\u04bc\u04be\u04c0-\u04c1\u04c3\u04c5\u04c7\u04c9\u04cb\u04cd\u04d0\u04d2\u04d4\u04d6\u04d8\u04da\u04dc\u04de\u04e0\u04e2\u04e4\u04e6\u04e8\u04ea\u04ec\u04ee\u04f0\u04f2\u04f4\u04f6\u04f8\u04fa\u04fc\u04fe\u0500\u0502\u0504\u0506\u0508\u050a\u050c\u050e\u0510\u0512\u0514\u0516\u0518\u051a\u051c\u051e\u0520\u0522\u0524\u0526\u0531-\u0556\u10a0-\u10c5\u10c7\u10cd\u1e00\u1e02\u1e04\u1e06\u1e08\u1e0a\u1e0c\u1e0e\u1e10\u1e12\u1e14\u1e16\u1e18\u1e1a\u1e1c\u1e1e\u1e20\u1e22\u1e24\u1e26\u1e28\u1e2a\u1e2c\u1e2e\u1e30\u1e32\u1e34\u1e36\u1e38\u1e3a\u1e3c\u1e3e\u1e40\u1e42\u1e44\u1e46\u1e48\u1e4a\u1e4c\u1e4e\u1e50\u1e52\u1e54\u1e56\u1e58\u1e5a\u1e5c\u1e5e\u1e60\u1e62\u1e64\u1e66\u1e68\u1e6a\u1e6c\u1e6e\u1e70\u1e72\u1e74\u1e76\u1e78\u1e7a\u1e7c\u1e7e\u1e80\u1e82\u1e84\u1e86\u1e88\u1e8a\u1e8c\u1e8e\u1e90\u1e92\u1e94\u1e9e\u1ea0\u1ea2\u1ea4\u1ea6\u1ea8\u1eaa\u1eac\u1eae\u1eb0\u1eb2\u1eb4\u1eb6\u1eb8\u1eba\u1ebc\u1ebe\u1ec0\u1ec2\u1ec4\u1ec6\u1ec8\u1eca\u1ecc\u1ece\u1ed0\u1ed2\u1ed4\u1ed6\u1ed8\u1eda\u1edc\u1ede\u1ee0\u1ee2\u1ee4\u1ee6\u1ee8\u1eea\u1eec\u1eee\u1ef0\u1ef2\u1ef4\u1ef6\u1ef8\u1efa\u1efc\u1efe\u1f08-\u1f0f\u1f18-\u1f1d\u1f28-\u1f2f\u1f38-\u1f3f\u1f48-\u1f4d\u1f59\u1f5b\u1f5d\u1f5f\u1f68-\u1f6f\u1fb8-\u1fbb\u1fc8-\u1fcb\u1fd8-\u1fdb\u1fe8-\u1fec\u1ff8-\u1ffb\u2102\u2107\u210b-\u210d\u2110-\u2112\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u2130-\u2133\u213e-\u213f\u2145\u2183\u2c00-\u2c2e\u2c60\u2c62-\u2c64\u2c67\u2c69\u2c6b\u2c6d-\u2c70\u2c72\u2c75\u2c7e-\u2c80\u2c82\u2c84\u2c86\u2c88\u2c8a\u2c8c\u2c8e\u2c90\u2c92\u2c94\u2c96\u2c98\u2c9a\u2c9c\u2c9e\u2ca0\u2ca2\u2ca4\u2ca6\u2ca8\u2caa\u2cac\u2cae\u2cb0\u2cb2\u2cb4\u2cb6\u2cb8\u2cba\u2cbc\u2cbe\u2cc0\u2cc2\u2cc4\u2cc6\u2cc8\u2cca\u2ccc\u2cce\u2cd0\u2cd2\u2cd4\u2cd6\u2cd8\u2cda\u2cdc\u2cde\u2ce0\u2ce2\u2ceb\u2ced\u2cf2\ua640\ua642\ua644\ua646\ua648\ua64a\ua64c\ua64e\ua650\ua652\ua654\ua656\ua658\ua65a\ua65c\ua65e\ua660\ua662\ua664\ua666\ua668\ua66a\ua66c\ua680\ua682\ua684\ua686\ua688\ua68a\ua68c\ua68e\ua690\ua692\ua694\ua696\ua722\ua724\ua726\ua728\ua72a\ua72c\ua72e\ua732\ua734\ua736\ua738\ua73a\ua73c\ua73e\ua740\ua742\ua744\ua746\ua748\ua74a\ua74c\ua74e\ua750\ua752\ua754\ua756\ua758\ua75a\ua75c\ua75e\ua760\ua762\ua764\ua766\ua768\ua76a\ua76c\ua76e\ua779\ua77b\ua77d-\ua77e\ua780\ua782\ua784\ua786\ua78b\ua78d\ua790\ua792\ua7a0\ua7a2\ua7a4\ua7a6\ua7a8\ua7aa\uff21-\uff3a'
+
+Mc = u'\u0903\u093b\u093e-\u0940\u0949-\u094c\u094e-\u094f\u0982-\u0983\u09be-\u09c0\u09c7-\u09c8\u09cb-\u09cc\u09d7\u0a03\u0a3e-\u0a40\u0a83\u0abe-\u0ac0\u0ac9\u0acb-\u0acc\u0b02-\u0b03\u0b3e\u0b40\u0b47-\u0b48\u0b4b-\u0b4c\u0b57\u0bbe-\u0bbf\u0bc1-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcc\u0bd7\u0c01-\u0c03\u0c41-\u0c44\u0c82-\u0c83\u0cbe\u0cc0-\u0cc4\u0cc7-\u0cc8\u0cca-\u0ccb\u0cd5-\u0cd6\u0d02-\u0d03\u0d3e-\u0d40\u0d46-\u0d48\u0d4a-\u0d4c\u0d57\u0d82-\u0d83\u0dcf-\u0dd1\u0dd8-\u0ddf\u0df2-\u0df3\u0f3e-\u0f3f\u0f7f\u102b-\u102c\u1031\u1038\u103b-\u103c\u1056-\u1057\u1062-\u1064\u1067-\u106d\u1083-\u1084\u1087-\u108c\u108f\u109a-\u109c\u17b6\u17be-\u17c5\u17c7-\u17c8\u1923-\u1926\u1929-\u192b\u1930-\u1931\u1933-\u1938\u19b0-\u19c0\u19c8-\u19c9\u1a19-\u1a1a\u1a55\u1a57\u1a61\u1a63-\u1a64\u1a6d-\u1a72\u1b04\u1b35\u1b3b\u1b3d-\u1b41\u1b43-\u1b44\u1b82\u1ba1\u1ba6-\u1ba7\u1baa\u1bac-\u1bad\u1be7\u1bea-\u1bec\u1bee\u1bf2-\u1bf3\u1c24-\u1c2b\u1c34-\u1c35\u1ce1\u1cf2-\u1cf3\u302e-\u302f\ua823-\ua824\ua827\ua880-\ua881\ua8b4-\ua8c3\ua952-\ua953\ua983\ua9b4-\ua9b5\ua9ba-\ua9bb\ua9bd-\ua9c0\uaa2f-\uaa30\uaa33-\uaa34\uaa4d\uaa7b\uaaeb\uaaee-\uaaef\uaaf5\uabe3-\uabe4\uabe6-\uabe7\uabe9-\uabea\uabec'
+
+Me = u'\u0488-\u0489\u20dd-\u20e0\u20e2-\u20e4\ua670-\ua672'
+
+Mn = u'\u0300-\u036f\u0483-\u0487\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u0610-\u061a\u064b-\u065f\u0670\u06d6-\u06dc\u06df-\u06e4\u06e7-\u06e8\u06ea-\u06ed\u0711\u0730-\u074a\u07a6-\u07b0\u07eb-\u07f3\u0816-\u0819\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0859-\u085b\u08e4-\u08fe\u0900-\u0902\u093a\u093c\u0941-\u0948\u094d\u0951-\u0957\u0962-\u0963\u0981\u09bc\u09c1-\u09c4\u09cd\u09e2-\u09e3\u0a01-\u0a02\u0a3c\u0a41-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a70-\u0a71\u0a75\u0a81-\u0a82\u0abc\u0ac1-\u0ac5\u0ac7-\u0ac8\u0acd\u0ae2-\u0ae3\u0b01\u0b3c\u0b3f\u0b41-\u0b44\u0b4d\u0b56\u0b62-\u0b63\u0b82\u0bc0\u0bcd\u0c3e-\u0c40\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c62-\u0c63\u0cbc\u0cbf\u0cc6\u0ccc-\u0ccd\u0ce2-\u0ce3\u0d41-\u0d44\u0d4d\u0d62-\u0d63\u0dca\u0dd2-\u0dd4\u0dd6\u0e31\u0e34-\u0e3a\u0e47-\u0e4e\u0eb1\u0eb4-\u0eb9\u0ebb-\u0ebc\u0ec8-\u0ecd\u0f18-\u0f19\u0f35\u0f37\u0f39\u0f71-\u0f7e\u0f80-\u0f84\u0f86-\u0f87\u0f8d-\u0f97\u0f99-\u0fbc\u0fc6\u102d-\u1030\u1032-\u1037\u1039-\u103a\u103d-\u103e\u1058-\u1059\u105e-\u1060\u1071-\u1074\u1082\u1085-\u1086\u108d\u109d\u135d-\u135f\u1712-\u1714\u1732-\u1734\u1752-\u1753\u1772-\u1773\u17b4-\u17b5\u17b7-\u17bd\u17c6\u17c9-\u17d3\u17dd\u180b-\u180d\u18a9\u1920-\u1922\u1927-\u1928\u1932\u1939-\u193b\u1a17-\u1a18\u1a1b\u1a56\u1a58-\u1a5e\u1a60\u1a62\u1a65-\u1a6c\u1a73-\u1a7c\u1a7f\u1b00-\u1b03\u1b34\u1b36-\u1b3a\u1b3c\u1b42\u1b6b-\u1b73\u1b80-\u1b81\u1ba2-\u1ba5\u1ba8-\u1ba9\u1bab\u1be6\u1be8-\u1be9\u1bed\u1bef-\u1bf1\u1c2c-\u1c33\u1c36-\u1c37\u1cd0-\u1cd2\u1cd4-\u1ce0\u1ce2-\u1ce8\u1ced\u1cf4\u1dc0-\u1de6\u1dfc-\u1dff\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2cef-\u2cf1\u2d7f\u2de0-\u2dff\u302a-\u302d\u3099-\u309a\ua66f\ua674-\ua67d\ua69f\ua6f0-\ua6f1\ua802\ua806\ua80b\ua825-\ua826\ua8c4\ua8e0-\ua8f1\ua926-\ua92d\ua947-\ua951\ua980-\ua982\ua9b3\ua9b6-\ua9b9\ua9bc\uaa29-\uaa2e\uaa31-\uaa32\uaa35-\uaa36\uaa43\uaa4c\uaab0\uaab2-\uaab4\uaab7-\uaab8\uaabe-\uaabf\uaac1\uaaec-\uaaed\uaaf6\uabe5\uabe8\uabed\ufb1e\ufe00-\ufe0f\ufe20-\ufe26'
+
+Nd = u'0-9\u0660-\u0669\u06f0-\u06f9\u07c0-\u07c9\u0966-\u096f\u09e6-\u09ef\u0a66-\u0a6f\u0ae6-\u0aef\u0b66-\u0b6f\u0be6-\u0bef\u0c66-\u0c6f\u0ce6-\u0cef\u0d66-\u0d6f\u0e50-\u0e59\u0ed0-\u0ed9\u0f20-\u0f29\u1040-\u1049\u1090-\u1099\u17e0-\u17e9\u1810-\u1819\u1946-\u194f\u19d0-\u19d9\u1a80-\u1a89\u1a90-\u1a99\u1b50-\u1b59\u1bb0-\u1bb9\u1c40-\u1c49\u1c50-\u1c59\ua620-\ua629\ua8d0-\ua8d9\ua900-\ua909\ua9d0-\ua9d9\uaa50-\uaa59\uabf0-\uabf9\uff10-\uff19'
+
+Nl = u'\u16ee-\u16f0\u2160-\u2182\u2185-\u2188\u3007\u3021-\u3029\u3038-\u303a\ua6e6-\ua6ef'
+
+No = u'\xb2-\xb3\xb9\xbc-\xbe\u09f4-\u09f9\u0b72-\u0b77\u0bf0-\u0bf2\u0c78-\u0c7e\u0d70-\u0d75\u0f2a-\u0f33\u1369-\u137c\u17f0-\u17f9\u19da\u2070\u2074-\u2079\u2080-\u2089\u2150-\u215f\u2189\u2460-\u249b\u24ea-\u24ff\u2776-\u2793\u2cfd\u3192-\u3195\u3220-\u3229\u3248-\u324f\u3251-\u325f\u3280-\u3289\u32b1-\u32bf\ua830-\ua835'
+
+Pc = u'_\u203f-\u2040\u2054\ufe33-\ufe34\ufe4d-\ufe4f\uff3f'
+
+Pd = u'\\-\u058a\u05be\u1400\u1806\u2010-\u2015\u2e17\u2e1a\u2e3a-\u2e3b\u301c\u3030\u30a0\ufe31-\ufe32\ufe58\ufe63\uff0d'
+
+Pe = u')\\]}\u0f3b\u0f3d\u169c\u2046\u207e\u208e\u2309\u230b\u232a\u2769\u276b\u276d\u276f\u2771\u2773\u2775\u27c6\u27e7\u27e9\u27eb\u27ed\u27ef\u2984\u2986\u2988\u298a\u298c\u298e\u2990\u2992\u2994\u2996\u2998\u29d9\u29db\u29fd\u2e23\u2e25\u2e27\u2e29\u3009\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b\u301e-\u301f\ufd3f\ufe18\ufe36\ufe38\ufe3a\ufe3c\ufe3e\ufe40\ufe42\ufe44\ufe48\ufe5a\ufe5c\ufe5e\uff09\uff3d\uff5d\uff60\uff63'
+
+Pf = u'\xbb\u2019\u201d\u203a\u2e03\u2e05\u2e0a\u2e0d\u2e1d\u2e21'
+
+Pi = u'\xab\u2018\u201b-\u201c\u201f\u2039\u2e02\u2e04\u2e09\u2e0c\u2e1c\u2e20'
+
+Po = u"!-#%-'*,.-/:-;?-@\\\\\xa1\xa7\xb6-\xb7\xbf\u037e\u0387\u055a-\u055f\u0589\u05c0\u05c3\u05c6\u05f3-\u05f4\u0609-\u060a\u060c-\u060d\u061b\u061e-\u061f\u066a-\u066d\u06d4\u0700-\u070d\u07f7-\u07f9\u0830-\u083e\u085e\u0964-\u0965\u0970\u0af0\u0df4\u0e4f\u0e5a-\u0e5b\u0f04-\u0f12\u0f14\u0f85\u0fd0-\u0fd4\u0fd9-\u0fda\u104a-\u104f\u10fb\u1360-\u1368\u166d-\u166e\u16eb-\u16ed\u1735-\u1736\u17d4-\u17d6\u17d8-\u17da\u1800-\u1805\u1807-\u180a\u1944-\u1945\u1a1e-\u1a1f\u1aa0-\u1aa6\u1aa8-\u1aad\u1b5a-\u1b60\u1bfc-\u1bff\u1c3b-\u1c3f\u1c7e-\u1c7f\u1cc0-\u1cc7\u1cd3\u2016-\u2017\u2020-\u2027\u2030-\u2038\u203b-\u203e\u2041-\u2043\u2047-\u2051\u2053\u2055-\u205e\u2cf9-\u2cfc\u2cfe-\u2cff\u2d70\u2e00-\u2e01\u2e06-\u2e08\u2e0b\u2e0e-\u2e16\u2e18-\u2e19\u2e1b\u2e1e-\u2e1f\u2e2a-\u2e2e\u2e30-\u2e39\u3001-\u3003\u303d\u30fb\ua4fe-\ua4ff\ua60d-\ua60f\ua673\ua67e\ua6f2-\ua6f7\ua874-\ua877\ua8ce-\ua8cf\ua8f8-\ua8fa\ua92e-\ua92f\ua95f\ua9c1-\ua9cd\ua9de-\ua9df\uaa5c-\uaa5f\uaade-\uaadf\uaaf0-\uaaf1\uabeb\ufe10-\ufe16\ufe19\ufe30\ufe45-\ufe46\ufe49-\ufe4c\ufe50-\ufe52\ufe54-\ufe57\ufe5f-\ufe61\ufe68\ufe6a-\ufe6b\uff01-\uff03\uff05-\uff07\uff0a\uff0c\uff0e-\uff0f\uff1a-\uff1b\uff1f-\uff20\uff3c\uff61\uff64-\uff65"
+
+Ps = u'(\\[{\u0f3a\u0f3c\u169b\u201a\u201e\u2045\u207d\u208d\u2308\u230a\u2329\u2768\u276a\u276c\u276e\u2770\u2772\u2774\u27c5\u27e6\u27e8\u27ea\u27ec\u27ee\u2983\u2985\u2987\u2989\u298b\u298d\u298f\u2991\u2993\u2995\u2997\u29d8\u29da\u29fc\u2e22\u2e24\u2e26\u2e28\u3008\u300a\u300c\u300e\u3010\u3014\u3016\u3018\u301a\u301d\ufd3e\ufe17\ufe35\ufe37\ufe39\ufe3b\ufe3d\ufe3f\ufe41\ufe43\ufe47\ufe59\ufe5b\ufe5d\uff08\uff3b\uff5b\uff5f\uff62'
+
+Sc = u'$\xa2-\xa5\u058f\u060b\u09f2-\u09f3\u09fb\u0af1\u0bf9\u0e3f\u17db\u20a0-\u20ba\ua838\ufdfc\ufe69\uff04\uffe0-\uffe1\uffe5-\uffe6'
+
+Sk = u'\\^`\xa8\xaf\xb4\xb8\u02c2-\u02c5\u02d2-\u02df\u02e5-\u02eb\u02ed\u02ef-\u02ff\u0375\u0384-\u0385\u1fbd\u1fbf-\u1fc1\u1fcd-\u1fcf\u1fdd-\u1fdf\u1fed-\u1fef\u1ffd-\u1ffe\u309b-\u309c\ua700-\ua716\ua720-\ua721\ua789-\ua78a\ufbb2-\ufbc1\uff3e\uff40\uffe3'
+
+Sm = u'+<->|~\xac\xb1\xd7\xf7\u03f6\u0606-\u0608\u2044\u2052\u207a-\u207c\u208a-\u208c\u2118\u2140-\u2144\u214b\u2190-\u2194\u219a-\u219b\u21a0\u21a3\u21a6\u21ae\u21ce-\u21cf\u21d2\u21d4\u21f4-\u22ff\u2320-\u2321\u237c\u239b-\u23b3\u23dc-\u23e1\u25b7\u25c1\u25f8-\u25ff\u266f\u27c0-\u27c4\u27c7-\u27e5\u27f0-\u27ff\u2900-\u2982\u2999-\u29d7\u29dc-\u29fb\u29fe-\u2aff\u2b30-\u2b44\u2b47-\u2b4c\ufb29\ufe62\ufe64-\ufe66\uff0b\uff1c-\uff1e\uff5c\uff5e\uffe2\uffe9-\uffec'
+
+So = u'\xa6\xa9\xae\xb0\u0482\u060e-\u060f\u06de\u06e9\u06fd-\u06fe\u07f6\u09fa\u0b70\u0bf3-\u0bf8\u0bfa\u0c7f\u0d79\u0f01-\u0f03\u0f13\u0f15-\u0f17\u0f1a-\u0f1f\u0f34\u0f36\u0f38\u0fbe-\u0fc5\u0fc7-\u0fcc\u0fce-\u0fcf\u0fd5-\u0fd8\u109e-\u109f\u1390-\u1399\u1940\u19de-\u19ff\u1b61-\u1b6a\u1b74-\u1b7c\u2100-\u2101\u2103-\u2106\u2108-\u2109\u2114\u2116-\u2117\u211e-\u2123\u2125\u2127\u2129\u212e\u213a-\u213b\u214a\u214c-\u214d\u214f\u2195-\u2199\u219c-\u219f\u21a1-\u21a2\u21a4-\u21a5\u21a7-\u21ad\u21af-\u21cd\u21d0-\u21d1\u21d3\u21d5-\u21f3\u2300-\u2307\u230c-\u231f\u2322-\u2328\u232b-\u237b\u237d-\u239a\u23b4-\u23db\u23e2-\u23f3\u2400-\u2426\u2440-\u244a\u249c-\u24e9\u2500-\u25b6\u25b8-\u25c0\u25c2-\u25f7\u2600-\u266e\u2670-\u26ff\u2701-\u2767\u2794-\u27bf\u2800-\u28ff\u2b00-\u2b2f\u2b45-\u2b46\u2b50-\u2b59\u2ce5-\u2cea\u2e80-\u2e99\u2e9b-\u2ef3\u2f00-\u2fd5\u2ff0-\u2ffb\u3004\u3012-\u3013\u3020\u3036-\u3037\u303e-\u303f\u3190-\u3191\u3196-\u319f\u31c0-\u31e3\u3200-\u321e\u322a-\u3247\u3250\u3260-\u327f\u328a-\u32b0\u32c0-\u32fe\u3300-\u33ff\u4dc0-\u4dff\ua490-\ua4c6\ua828-\ua82b\ua836-\ua837\ua839\uaa77-\uaa79\ufdfd\uffe4\uffe8\uffed-\uffee\ufffc-\ufffd'
+
+Zl = u'\u2028'
+
+Zp = u'\u2029'
+
+Zs = u' \xa0\u1680\u2000-\u200a\u202f\u205f\u3000'
+
+xid_continue = u'0-9A-Z_a-z\xaa\xb5\xb7\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0300-\u0374\u0376-\u0377\u037b-\u037d\u0386-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u0483-\u0487\u048a-\u0527\u0531-\u0556\u0559\u0561-\u0587\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u05d0-\u05ea\u05f0-\u05f2\u0610-\u061a\u0620-\u0669\u066e-\u06d3\u06d5-\u06dc\u06df-\u06e8\u06ea-\u06fc\u06ff\u0710-\u074a\u074d-\u07b1\u07c0-\u07f5\u07fa\u0800-\u082d\u0840-\u085b\u08a0\u08a2-\u08ac\u08e4-\u08fe\u0900-\u0963\u0966-\u096f\u0971-\u0977\u0979-\u097f\u0981-\u0983\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bc-\u09c4\u09c7-\u09c8\u09cb-\u09ce\u09d7\u09dc-\u09dd\u09df-\u09e3\u09e6-\u09f1\u0a01-\u0a03\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a3c\u0a3e-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a59-\u0a5c\u0a5e\u0a66-\u0a75\u0a81-\u0a83\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abc-\u0ac5\u0ac7-\u0ac9\u0acb-\u0acd\u0ad0\u0ae0-\u0ae3\u0ae6-\u0aef\u0b01-\u0b03\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3c-\u0b44\u0b47-\u0b48\u0b4b-\u0b4d\u0b56-\u0b57\u0b5c-\u0b5d\u0b5f-\u0b63\u0b66-\u0b6f\u0b71\u0b82-\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bbe-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcd\u0bd0\u0bd7\u0be6-\u0bef\u0c01-\u0c03\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d-\u0c44\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c58-\u0c59\u0c60-\u0c63\u0c66-\u0c6f\u0c82-\u0c83\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbc-\u0cc4\u0cc6-\u0cc8\u0cca-\u0ccd\u0cd5-\u0cd6\u0cde\u0ce0-\u0ce3\u0ce6-\u0cef\u0cf1-\u0cf2\u0d02-\u0d03\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d-\u0d44\u0d46-\u0d48\u0d4a-\u0d4e\u0d57\u0d60-\u0d63\u0d66-\u0d6f\u0d7a-\u0d7f\u0d82-\u0d83\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0dca\u0dcf-\u0dd4\u0dd6\u0dd8-\u0ddf\u0df2-\u0df3\u0e01-\u0e3a\u0e40-\u0e4e\u0e50-\u0e59\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb9\u0ebb-\u0ebd\u0ec0-\u0ec4\u0ec6\u0ec8-\u0ecd\u0ed0-\u0ed9\u0edc-\u0edf\u0f00\u0f18-\u0f19\u0f20-\u0f29\u0f35\u0f37\u0f39\u0f3e-\u0f47\u0f49-\u0f6c\u0f71-\u0f84\u0f86-\u0f97\u0f99-\u0fbc\u0fc6\u1000-\u1049\u1050-\u109d\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u135d-\u135f\u1369-\u1371\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f0\u1700-\u170c\u170e-\u1714\u1720-\u1734\u1740-\u1753\u1760-\u176c\u176e-\u1770\u1772-\u1773\u1780-\u17d3\u17d7\u17dc-\u17dd\u17e0-\u17e9\u180b-\u180d\u1810-\u1819\u1820-\u1877\u1880-\u18aa\u18b0-\u18f5\u1900-\u191c\u1920-\u192b\u1930-\u193b\u1946-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u19d0-\u19da\u1a00-\u1a1b\u1a20-\u1a5e\u1a60-\u1a7c\u1a7f-\u1a89\u1a90-\u1a99\u1aa7\u1b00-\u1b4b\u1b50-\u1b59\u1b6b-\u1b73\u1b80-\u1bf3\u1c00-\u1c37\u1c40-\u1c49\u1c4d-\u1c7d\u1cd0-\u1cd2\u1cd4-\u1cf6\u1d00-\u1de6\u1dfc-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u203f-\u2040\u2054\u2071\u207f\u2090-\u209c\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d7f-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u2de0-\u2dff\u3005-\u3007\u3021-\u302f\u3031-\u3035\u3038-\u303c\u3041-\u3096\u3099-\u309a\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua62b\ua640-\ua66f\ua674-\ua67d\ua67f-\ua697\ua69f-\ua6f1\ua717-\ua71f\ua722-\ua788\ua78b-\ua78e\ua790-\ua793\ua7a0-\ua7aa\ua7f8-\ua827\ua840-\ua873\ua880-\ua8c4\ua8d0-\ua8d9\ua8e0-\ua8f7\ua8fb\ua900-\ua92d\ua930-\ua953\ua960-\ua97c\ua980-\ua9c0\ua9cf-\ua9d9\uaa00-\uaa36\uaa40-\uaa4d\uaa50-\uaa59\uaa60-\uaa76\uaa7a-\uaa7b\uaa80-\uaac2\uaadb-\uaadd\uaae0-\uaaef\uaaf2-\uaaf6\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabea\uabec-\uabed\uabf0-\uabf9\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe00-\ufe0f\ufe20-\ufe26\ufe33-\ufe34\ufe4d-\ufe4f\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff10-\uff19\uff21-\uff3a\uff3f\uff41-\uff5a\uff66-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc'
+
+xid_start = u'A-Z_a-z\xaa\xb5\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0370-\u0374\u0376-\u0377\u037b-\u037d\u0386\u0388-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u048a-\u0527\u0531-\u0556\u0559\u0561-\u0587\u05d0-\u05ea\u05f0-\u05f2\u0620-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06e5-\u06e6\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u07f4-\u07f5\u07fa\u0800-\u0815\u081a\u0824\u0828\u0840-\u0858\u08a0\u08a2-\u08ac\u0904-\u0939\u093d\u0950\u0958-\u0961\u0971-\u0977\u0979-\u097f\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d\u0c58-\u0c59\u0c60-\u0c61\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d60-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32\u0e40-\u0e46\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2\u0ebd\u0ec0-\u0ec4\u0ec6\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f0\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17d7\u17dc\u1820-\u1877\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191c\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19c1-\u19c7\u1a00-\u1a16\u1a20-\u1a54\u1aa7\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c7d\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u1d00-\u1dbf\u1e00-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u2071\u207f\u2090-\u209c\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cee\u2cf2-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303c\u3041-\u3096\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua61f\ua62a-\ua62b\ua640-\ua66e\ua67f-\ua697\ua6a0-\ua6ef\ua717-\ua71f\ua722-\ua788\ua78b-\ua78e\ua790-\ua793\ua7a0-\ua7aa\ua7f8-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9cf\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa76\uaa7a\uaa80-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadd\uaae0-\uaaea\uaaf2-\uaaf4\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc'
+
+if sys.maxunicode > 0xFFFF:
+ # non-BMP characters, use only on wide Unicode builds
+ Cf += u'\U000110bd\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f'
+
+ Cn += u'\U0001000c\U00010027\U0001003b\U0001003e\U0001004e-\U0001004f\U0001005e-\U0001007f\U000100fb-\U000100ff\U00010103-\U00010106\U00010134-\U00010136\U0001018b-\U0001018f\U0001019c-\U000101cf\U000101fe-\U0001027f\U0001029d-\U0001029f\U000102d1-\U000102ff\U0001031f\U00010324-\U0001032f\U0001034b-\U0001037f\U0001039e\U000103c4-\U000103c7\U000103d6-\U000103ff\U0001049e-\U0001049f\U000104aa-\U000107ff\U00010806-\U00010807\U00010809\U00010836\U00010839-\U0001083b\U0001083d-\U0001083e\U00010856\U00010860-\U000108ff\U0001091c-\U0001091e\U0001093a-\U0001093e\U00010940-\U0001097f\U000109b8-\U000109bd\U000109c0-\U000109ff\U00010a04\U00010a07-\U00010a0b\U00010a14\U00010a18\U00010a34-\U00010a37\U00010a3b-\U00010a3e\U00010a48-\U00010a4f\U00010a59-\U00010a5f\U00010a80-\U00010aff\U00010b36-\U00010b38\U00010b56-\U00010b57\U00010b73-\U00010b77\U00010b80-\U00010bff\U00010c49-\U00010e5f\U00010e7f-\U00010fff\U0001104e-\U00011051\U00011070-\U0001107f\U000110c2-\U000110cf\U000110e9-\U000110ef\U000110fa-\U000110ff\U00011135\U00011144-\U0001117f\U000111c9-\U000111cf\U000111da-\U0001167f\U000116b8-\U000116bf\U000116ca-\U00011fff\U0001236f-\U000123ff\U00012463-\U0001246f\U00012474-\U00012fff\U0001342f-\U000167ff\U00016a39-\U00016eff\U00016f45-\U00016f4f\U00016f7f-\U00016f8e\U00016fa0-\U0001afff\U0001b002-\U0001cfff\U0001d0f6-\U0001d0ff\U0001d127-\U0001d128\U0001d1de-\U0001d1ff\U0001d246-\U0001d2ff\U0001d357-\U0001d35f\U0001d372-\U0001d3ff\U0001d455\U0001d49d\U0001d4a0-\U0001d4a1\U0001d4a3-\U0001d4a4\U0001d4a7-\U0001d4a8\U0001d4ad\U0001d4ba\U0001d4bc\U0001d4c4\U0001d506\U0001d50b-\U0001d50c\U0001d515\U0001d51d\U0001d53a\U0001d53f\U0001d545\U0001d547-\U0001d549\U0001d551\U0001d6a6-\U0001d6a7\U0001d7cc-\U0001d7cd\U0001d800-\U0001edff\U0001ee04\U0001ee20\U0001ee23\U0001ee25-\U0001ee26\U0001ee28\U0001ee33\U0001ee38\U0001ee3a\U0001ee3c-\U0001ee41\U0001ee43-\U0001ee46\U0001ee48\U0001ee4a\U0001ee4c\U0001ee50\U0001ee53\U0001ee55-\U0001ee56\U0001ee58\U0001ee5a\U0001ee5c\U0001ee5e\U0001ee60\U0001ee63\U0001ee65-\U0001ee66\U0001ee6b\U0001ee73\U0001ee78\U0001ee7d\U0001ee7f\U0001ee8a\U0001ee9c-\U0001eea0\U0001eea4\U0001eeaa\U0001eebc-\U0001eeef\U0001eef2-\U0001efff\U0001f02c-\U0001f02f\U0001f094-\U0001f09f\U0001f0af-\U0001f0b0\U0001f0bf-\U0001f0c0\U0001f0d0\U0001f0e0-\U0001f0ff\U0001f10b-\U0001f10f\U0001f12f\U0001f16c-\U0001f16f\U0001f19b-\U0001f1e5\U0001f203-\U0001f20f\U0001f23b-\U0001f23f\U0001f249-\U0001f24f\U0001f252-\U0001f2ff\U0001f321-\U0001f32f\U0001f336\U0001f37d-\U0001f37f\U0001f394-\U0001f39f\U0001f3c5\U0001f3cb-\U0001f3df\U0001f3f1-\U0001f3ff\U0001f43f\U0001f441\U0001f4f8\U0001f4fd-\U0001f4ff\U0001f53e-\U0001f53f\U0001f544-\U0001f54f\U0001f568-\U0001f5fa\U0001f641-\U0001f644\U0001f650-\U0001f67f\U0001f6c6-\U0001f6ff\U0001f774-\U0001ffff\U0002a6d7-\U0002a6ff\U0002b735-\U0002b73f\U0002b81e-\U0002f7ff\U0002fa1e-\U000e0000\U000e0002-\U000e001f\U000e0080-\U000e00ff\U000e01f0-\U000effff\U000ffffe-\U000fffff\U0010fffe-\U0010ffff'
+
+ Co += u'\U000f0000-\U000ffffd\U00100000-\U0010fffd'
+
+ Ll += u'\U00010428-\U0001044f\U0001d41a-\U0001d433\U0001d44e-\U0001d454\U0001d456-\U0001d467\U0001d482-\U0001d49b\U0001d4b6-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d4cf\U0001d4ea-\U0001d503\U0001d51e-\U0001d537\U0001d552-\U0001d56b\U0001d586-\U0001d59f\U0001d5ba-\U0001d5d3\U0001d5ee-\U0001d607\U0001d622-\U0001d63b\U0001d656-\U0001d66f\U0001d68a-\U0001d6a5\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6e1\U0001d6fc-\U0001d714\U0001d716-\U0001d71b\U0001d736-\U0001d74e\U0001d750-\U0001d755\U0001d770-\U0001d788\U0001d78a-\U0001d78f\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7c9\U0001d7cb'
+
+ Lm += u'\U00016f93-\U00016f9f'
+
+ Lo += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031e\U00010330-\U00010340\U00010342-\U00010349\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U00010450-\U0001049d\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a33\U00010a60-\U00010a7c\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010c00-\U00010c48\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011183-\U000111b2\U000111c1-\U000111c4\U00011680-\U000116aa\U00012000-\U0001236e\U00013000-\U0001342e\U00016800-\U00016a38\U00016f00-\U00016f44\U00016f50\U0001b000-\U0001b001\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002f800-\U0002fa1d'
+
+ Lu += u'\U00010400-\U00010427\U0001d400-\U0001d419\U0001d434-\U0001d44d\U0001d468-\U0001d481\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b5\U0001d4d0-\U0001d4e9\U0001d504-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d538-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d56c-\U0001d585\U0001d5a0-\U0001d5b9\U0001d5d4-\U0001d5ed\U0001d608-\U0001d621\U0001d63c-\U0001d655\U0001d670-\U0001d689\U0001d6a8-\U0001d6c0\U0001d6e2-\U0001d6fa\U0001d71c-\U0001d734\U0001d756-\U0001d76e\U0001d790-\U0001d7a8\U0001d7ca'
+
+ Mc += u'\U00011000\U00011002\U00011082\U000110b0-\U000110b2\U000110b7-\U000110b8\U0001112c\U00011182\U000111b3-\U000111b5\U000111bf-\U000111c0\U000116ac\U000116ae-\U000116af\U000116b6\U00016f51-\U00016f7e\U0001d165-\U0001d166\U0001d16d-\U0001d172'
+
+ Mn += u'\U000101fd\U00010a01-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a0f\U00010a38-\U00010a3a\U00010a3f\U00011001\U00011038-\U00011046\U00011080-\U00011081\U000110b3-\U000110b6\U000110b9-\U000110ba\U00011100-\U00011102\U00011127-\U0001112b\U0001112d-\U00011134\U00011180-\U00011181\U000111b6-\U000111be\U000116ab\U000116ad\U000116b0-\U000116b5\U000116b7\U00016f8f-\U00016f92\U0001d167-\U0001d169\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U000e0100-\U000e01ef'
+
+ Nd += u'\U000104a0-\U000104a9\U00011066-\U0001106f\U000110f0-\U000110f9\U00011136-\U0001113f\U000111d0-\U000111d9\U000116c0-\U000116c9\U0001d7ce-\U0001d7ff'
+
+ Nl += u'\U00010140-\U00010174\U00010341\U0001034a\U000103d1-\U000103d5\U00012400-\U00012462'
+
+ No += u'\U00010107-\U00010133\U00010175-\U00010178\U0001018a\U00010320-\U00010323\U00010858-\U0001085f\U00010916-\U0001091b\U00010a40-\U00010a47\U00010a7d-\U00010a7e\U00010b58-\U00010b5f\U00010b78-\U00010b7f\U00010e60-\U00010e7e\U00011052-\U00011065\U0001d360-\U0001d371\U0001f100-\U0001f10a'
+
+ Po += u'\U00010100-\U00010102\U0001039f\U000103d0\U00010857\U0001091f\U0001093f\U00010a50-\U00010a58\U00010a7f\U00010b39-\U00010b3f\U00011047-\U0001104d\U000110bb-\U000110bc\U000110be-\U000110c1\U00011140-\U00011143\U000111c5-\U000111c8\U00012470-\U00012473'
+
+ Sm += u'\U0001d6c1\U0001d6db\U0001d6fb\U0001d715\U0001d735\U0001d74f\U0001d76f\U0001d789\U0001d7a9\U0001d7c3\U0001eef0-\U0001eef1'
+
+ So += u'\U00010137-\U0001013f\U00010179-\U00010189\U00010190-\U0001019b\U000101d0-\U000101fc\U0001d000-\U0001d0f5\U0001d100-\U0001d126\U0001d129-\U0001d164\U0001d16a-\U0001d16c\U0001d183-\U0001d184\U0001d18c-\U0001d1a9\U0001d1ae-\U0001d1dd\U0001d200-\U0001d241\U0001d245\U0001d300-\U0001d356\U0001f000-\U0001f02b\U0001f030-\U0001f093\U0001f0a0-\U0001f0ae\U0001f0b1-\U0001f0be\U0001f0c1-\U0001f0cf\U0001f0d1-\U0001f0df\U0001f110-\U0001f12e\U0001f130-\U0001f16b\U0001f170-\U0001f19a\U0001f1e6-\U0001f202\U0001f210-\U0001f23a\U0001f240-\U0001f248\U0001f250-\U0001f251\U0001f300-\U0001f320\U0001f330-\U0001f335\U0001f337-\U0001f37c\U0001f380-\U0001f393\U0001f3a0-\U0001f3c4\U0001f3c6-\U0001f3ca\U0001f3e0-\U0001f3f0\U0001f400-\U0001f43e\U0001f440\U0001f442-\U0001f4f7\U0001f4f9-\U0001f4fc\U0001f500-\U0001f53d\U0001f540-\U0001f543\U0001f550-\U0001f567\U0001f5fb-\U0001f640\U0001f645-\U0001f64f\U0001f680-\U0001f6c5\U0001f700-\U0001f773'
+
+ xid_continue += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U000101fd\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031e\U00010330-\U0001034a\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U000104a0-\U000104a9\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a33\U00010a38-\U00010a3a\U00010a3f\U00010a60-\U00010a7c\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010c00-\U00010c48\U00011000-\U00011046\U00011066-\U0001106f\U00011080-\U000110ba\U000110d0-\U000110e8\U000110f0-\U000110f9\U00011100-\U00011134\U00011136-\U0001113f\U00011180-\U000111c4\U000111d0-\U000111d9\U00011680-\U000116b7\U000116c0-\U000116c9\U00012000-\U0001236e\U00012400-\U00012462\U00013000-\U0001342e\U00016800-\U00016a38\U00016f00-\U00016f44\U00016f50-\U00016f7e\U00016f8f-\U00016f9f\U0001b000-\U0001b001\U0001d165-\U0001d169\U0001d16d-\U0001d172\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001d7ce-\U0001d7ff\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002f800-\U0002fa1d\U000e0100-\U000e01ef'
+
+ xid_start += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031e\U00010330-\U0001034a\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a33\U00010a60-\U00010a7c\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010c00-\U00010c48\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011183-\U000111b2\U000111c1-\U000111c4\U00011680-\U000116aa\U00012000-\U0001236e\U00012400-\U00012462\U00013000-\U0001342e\U00016800-\U00016a38\U00016f00-\U00016f44\U00016f50\U00016f93-\U00016f9f\U0001b000-\U0001b001\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002f800-\U0002fa1d'
+
+cats = ['Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu', 'Mc', 'Me', 'Mn', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Pe', 'Pf', 'Pi', 'Po', 'Ps', 'Sc', 'Sk', 'Sm', 'So', 'Zl', 'Zp', 'Zs']
+
+# Generated from unidata 6.3.0
+
+def combine(*args):
+ return u''.join(globals()[cat] for cat in args)
+
+
+def allexcept(*args):
+ newcats = cats[:]
+ for arg in args:
+ newcats.remove(arg)
+ return u''.join(globals()[cat] for cat in newcats)
+
+
+def _handle_runs(char_list): # pragma: no cover
+ buf = []
+ for c in char_list:
+ if len(c) == 1:
+ if buf and buf[-1][1] == chr(ord(c)-1):
+ buf[-1] = (buf[-1][0], c)
+ else:
+ buf.append((c, c))
+ else:
+ buf.append((c, c))
+ for a, b in buf:
+ if a == b:
+ yield a
+ else:
+ yield u'%s-%s' % (a, b)
+
+
+if __name__ == '__main__': # pragma: no cover
+ import unicodedata
+
+ # we need Py3 for the determination of the XID_* properties
+ if sys.version_info[:2] < (3, 3):
+ raise RuntimeError('this file must be regenerated with Python 3.3+')
+
+ categories_bmp = {'xid_start': [], 'xid_continue': []}
+ categories_nonbmp = {'xid_start': [], 'xid_continue': []}
+
+ with open(__file__) as fp:
+ content = fp.read()
+
+ header = content[:content.find('Cc =')]
+ footer = content[content.find("def combine("):]
+
+ for code in range(0x110000):
+ c = chr(code)
+ cat = unicodedata.category(c)
+ if ord(c) == 0xdc00:
+ # Hack to avoid combining this combining with the preceeding high
+ # surrogate, 0xdbff, when doing a repr.
+ c = u'\\' + c
+ elif ord(c) in (0x2d, 0x5b, 0x5c, 0x5d, 0x5e):
+ # Escape regex metachars.
+ c = u'\\' + c
+ cat_dic = categories_bmp if code < 0x10000 else categories_nonbmp
+ cat_dic.setdefault(cat, []).append(c)
+ # XID_START and XID_CONTINUE are special categories used for matching
+ # identifiers in Python 3.
+ if c.isidentifier():
+ cat_dic['xid_start'].append(c)
+ if ('a' + c).isidentifier():
+ cat_dic['xid_continue'].append(c)
+
+ with open(__file__, 'w') as fp:
+ fp.write(header)
+
+ for cat in sorted(categories_bmp):
+ val = u''.join(_handle_runs(categories_bmp[cat]))
+ if cat == 'Cs':
+ # Jython can't handle isolated surrogates
+ fp.write("""\
+try:
+ Cs = eval(r"u%s")
+except UnicodeDecodeError:
+ Cs = '' # Jython can't handle isolated surrogates\n\n""" % ascii(val))
+ else:
+ fp.write('%s = u%a\n\n' % (cat, val))
+
+ fp.write('if sys.maxunicode > 0xFFFF:\n')
+ fp.write(' # non-BMP characters, use only on wide Unicode builds\n')
+ for cat in sorted(categories_nonbmp):
+ # no special case for Cs needed, since there are no surrogates
+ # in the higher planes
+ val = u''.join(_handle_runs(categories_nonbmp[cat]))
+ fp.write(' %s += u%a\n\n' % (cat, val))
+
+ cats = sorted(categories_bmp)
+ cats.remove('xid_start')
+ cats.remove('xid_continue')
+ fp.write('cats = %r\n\n' % cats)
+
+ fp.write('# Generated from unidata %s\n\n' % (unicodedata.unidata_version,))
+
+ fp.write(footer)
diff --git a/wandb/vendor/pygments/util.py b/wandb/vendor/pygments/util.py
new file mode 100644
index 0000000000000000000000000000000000000000..45070063710cd77322ee60d09e5d47e298a351f4
--- /dev/null
+++ b/wandb/vendor/pygments/util.py
@@ -0,0 +1,388 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.util
+ ~~~~~~~~~~~~~
+
+ Utility functions.
+
+ :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import re
+import sys
+
+
+split_path_re = re.compile(r'[/\\ ]')
+doctype_lookup_re = re.compile(r'''
+ (<\?.*?\?>)?\s*
+ <!DOCTYPE\s+(
+ [a-zA-Z_][a-zA-Z0-9]*
+ (?: \s+ # optional in HTML5
+ [a-zA-Z_][a-zA-Z0-9]*\s+
+ "[^"]*")?
+ )
+ [^>]*>
+''', re.DOTALL | re.MULTILINE | re.VERBOSE)
+tag_re = re.compile(r'<(.+?)(\s.*?)?>.*?</.+?>',
+ re.UNICODE | re.IGNORECASE | re.DOTALL | re.MULTILINE)
+xml_decl_re = re.compile(r'\s*<\?xml[^>]*\?>', re.I)
+
+
+class ClassNotFound(ValueError):
+ """Raised if one of the lookup functions didn't find a matching class."""
+
+
+class OptionError(Exception):
+ pass
+
+
+def get_choice_opt(options, optname, allowed, default=None, normcase=False):
+ string = options.get(optname, default)
+ if normcase:
+ string = string.lower()
+ if string not in allowed:
+ raise OptionError('Value for option %s must be one of %s' %
+ (optname, ', '.join(map(str, allowed))))
+ return string
+
+
+def get_bool_opt(options, optname, default=None):
+ string = options.get(optname, default)
+ if isinstance(string, bool):
+ return string
+ elif isinstance(string, int):
+ return bool(string)
+ elif not isinstance(string, string_types):
+ raise OptionError('Invalid type %r for option %s; use '
+ '1/0, yes/no, true/false, on/off' % (
+ string, optname))
+ elif string.lower() in ('1', 'yes', 'true', 'on'):
+ return True
+ elif string.lower() in ('0', 'no', 'false', 'off'):
+ return False
+ else:
+ raise OptionError('Invalid value %r for option %s; use '
+ '1/0, yes/no, true/false, on/off' % (
+ string, optname))
+
+
+def get_int_opt(options, optname, default=None):
+ string = options.get(optname, default)
+ try:
+ return int(string)
+ except TypeError:
+ raise OptionError('Invalid type %r for option %s; you '
+ 'must give an integer value' % (
+ string, optname))
+ except ValueError:
+ raise OptionError('Invalid value %r for option %s; you '
+ 'must give an integer value' % (
+ string, optname))
+
+
+def get_list_opt(options, optname, default=None):
+ val = options.get(optname, default)
+ if isinstance(val, string_types):
+ return val.split()
+ elif isinstance(val, (list, tuple)):
+ return list(val)
+ else:
+ raise OptionError('Invalid type %r for option %s; you '
+ 'must give a list value' % (
+ val, optname))
+
+
+def docstring_headline(obj):
+ if not obj.__doc__:
+ return ''
+ res = []
+ for line in obj.__doc__.strip().splitlines():
+ if line.strip():
+ res.append(" " + line.strip())
+ else:
+ break
+ return ''.join(res).lstrip()
+
+
+def make_analysator(f):
+ """Return a static text analyser function that returns float values."""
+ def text_analyse(text):
+ try:
+ rv = f(text)
+ except Exception:
+ return 0.0
+ if not rv:
+ return 0.0
+ try:
+ return min(1.0, max(0.0, float(rv)))
+ except (ValueError, TypeError):
+ return 0.0
+ text_analyse.__doc__ = f.__doc__
+ return staticmethod(text_analyse)
+
+
+def shebang_matches(text, regex):
+ r"""Check if the given regular expression matches the last part of the
+ shebang if one exists.
+
+ >>> from pygments.util import shebang_matches
+ >>> shebang_matches('#!/usr/bin/env python', r'python(2\.\d)?')
+ True
+ >>> shebang_matches('#!/usr/bin/python2.4', r'python(2\.\d)?')
+ True
+ >>> shebang_matches('#!/usr/bin/python-ruby', r'python(2\.\d)?')
+ False
+ >>> shebang_matches('#!/usr/bin/python/ruby', r'python(2\.\d)?')
+ False
+ >>> shebang_matches('#!/usr/bin/startsomethingwith python',
+ ... r'python(2\.\d)?')
+ True
+
+ It also checks for common windows executable file extensions::
+
+ >>> shebang_matches('#!C:\\Python2.4\\Python.exe', r'python(2\.\d)?')
+ True
+
+ Parameters (``'-f'`` or ``'--foo'`` are ignored so ``'perl'`` does
+ the same as ``'perl -e'``)
+
+ Note that this method automatically searches the whole string (eg:
+ the regular expression is wrapped in ``'^$'``)
+ """
+ index = text.find('\n')
+ if index >= 0:
+ first_line = text[:index].lower()
+ else:
+ first_line = text.lower()
+ if first_line.startswith('#!'):
+ try:
+ found = [x for x in split_path_re.split(first_line[2:].strip())
+ if x and not x.startswith('-')][-1]
+ except IndexError:
+ return False
+ regex = re.compile(r'^%s(\.(exe|cmd|bat|bin))?$' % regex, re.IGNORECASE)
+ if regex.search(found) is not None:
+ return True
+ return False
+
+
+def doctype_matches(text, regex):
+ """Check if the doctype matches a regular expression (if present).
+
+ Note that this method only checks the first part of a DOCTYPE.
+ eg: 'html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"'
+ """
+ m = doctype_lookup_re.match(text)
+ if m is None:
+ return False
+ doctype = m.group(2)
+ return re.compile(regex, re.I).match(doctype.strip()) is not None
+
+
+def html_doctype_matches(text):
+ """Check if the file looks like it has a html doctype."""
+ return doctype_matches(text, r'html')
+
+
+_looks_like_xml_cache = {}
+
+
+def looks_like_xml(text):
+ """Check if a doctype exists or if we have some tags."""
+ if xml_decl_re.match(text):
+ return True
+ key = hash(text)
+ try:
+ return _looks_like_xml_cache[key]
+ except KeyError:
+ m = doctype_lookup_re.match(text)
+ if m is not None:
+ return True
+ rv = tag_re.search(text[:1000]) is not None
+ _looks_like_xml_cache[key] = rv
+ return rv
+
+
+# Python narrow build compatibility
+
+def _surrogatepair(c):
+ # Given a unicode character code
+ # with length greater than 16 bits,
+ # return the two 16 bit surrogate pair.
+ # From example D28 of:
+ # http://www.unicode.org/book/ch03.pdf
+ return (0xd7c0 + (c >> 10), (0xdc00 + (c & 0x3ff)))
+
+
+def unirange(a, b):
+ """Returns a regular expression string to match the given non-BMP range."""
+ if b < a:
+ raise ValueError("Bad character range")
+ if a < 0x10000 or b < 0x10000:
+ raise ValueError("unirange is only defined for non-BMP ranges")
+
+ if sys.maxunicode > 0xffff:
+ # wide build
+ return u'[%s-%s]' % (unichr(a), unichr(b))
+ else:
+ # narrow build stores surrogates, and the 're' module handles them
+ # (incorrectly) as characters. Since there is still ordering among
+ # these characters, expand the range to one that it understands. Some
+ # background in http://bugs.python.org/issue3665 and
+ # http://bugs.python.org/issue12749
+ #
+ # Additionally, the lower constants are using unichr rather than
+ # literals because jython [which uses the wide path] can't load this
+ # file if they are literals.
+ ah, al = _surrogatepair(a)
+ bh, bl = _surrogatepair(b)
+ if ah == bh:
+ return u'(?:%s[%s-%s])' % (unichr(ah), unichr(al), unichr(bl))
+ else:
+ buf = []
+ buf.append(u'%s[%s-%s]' %
+ (unichr(ah), unichr(al),
+ ah == bh and unichr(bl) or unichr(0xdfff)))
+ if ah - bh > 1:
+ buf.append(u'[%s-%s][%s-%s]' %
+ unichr(ah+1), unichr(bh-1), unichr(0xdc00), unichr(0xdfff))
+ if ah != bh:
+ buf.append(u'%s[%s-%s]' %
+ (unichr(bh), unichr(0xdc00), unichr(bl)))
+
+ return u'(?:' + u'|'.join(buf) + u')'
+
+
+def format_lines(var_name, seq, raw=False, indent_level=0):
+ """Formats a sequence of strings for output."""
+ lines = []
+ base_indent = ' ' * indent_level * 4
+ inner_indent = ' ' * (indent_level + 1) * 4
+ lines.append(base_indent + var_name + ' = (')
+ if raw:
+ # These should be preformatted reprs of, say, tuples.
+ for i in seq:
+ lines.append(inner_indent + i + ',')
+ else:
+ for i in seq:
+ # Force use of single quotes
+ r = repr(i + '"')
+ lines.append(inner_indent + r[:-2] + r[-1] + ',')
+ lines.append(base_indent + ')')
+ return '\n'.join(lines)
+
+
+def duplicates_removed(it, already_seen=()):
+ """
+ Returns a list with duplicates removed from the iterable `it`.
+
+ Order is preserved.
+ """
+ lst = []
+ seen = set()
+ for i in it:
+ if i in seen or i in already_seen:
+ continue
+ lst.append(i)
+ seen.add(i)
+ return lst
+
+
+class Future(object):
+ """Generic class to defer some work.
+
+ Handled specially in RegexLexerMeta, to support regex string construction at
+ first use.
+ """
+ def get(self):
+ raise NotImplementedError
+
+
+def guess_decode(text):
+ """Decode *text* with guessed encoding.
+
+ First try UTF-8; this should fail for non-UTF-8 encodings.
+ Then try the preferred locale encoding.
+ Fall back to latin-1, which always works.
+ """
+ try:
+ text = text.decode('utf-8')
+ return text, 'utf-8'
+ except UnicodeDecodeError:
+ try:
+ import locale
+ prefencoding = locale.getpreferredencoding()
+ text = text.decode()
+ return text, prefencoding
+ except (UnicodeDecodeError, LookupError):
+ text = text.decode('latin1')
+ return text, 'latin1'
+
+
+def guess_decode_from_terminal(text, term):
+ """Decode *text* coming from terminal *term*.
+
+ First try the terminal encoding, if given.
+ Then try UTF-8. Then try the preferred locale encoding.
+ Fall back to latin-1, which always works.
+ """
+ if getattr(term, 'encoding', None):
+ try:
+ text = text.decode(term.encoding)
+ except UnicodeDecodeError:
+ pass
+ else:
+ return text, term.encoding
+ return guess_decode(text)
+
+
+def terminal_encoding(term):
+ """Return our best guess of encoding for the given *term*."""
+ if getattr(term, 'encoding', None):
+ return term.encoding
+ import locale
+ return locale.getpreferredencoding()
+
+
+# Python 2/3 compatibility
+
+if sys.version_info < (3, 0):
+ unichr = unichr
+ xrange = xrange
+ string_types = (str, unicode)
+ text_type = unicode
+ u_prefix = 'u'
+ iteritems = dict.iteritems
+ itervalues = dict.itervalues
+ import StringIO
+ import cStringIO
+ # unfortunately, io.StringIO in Python 2 doesn't accept str at all
+ StringIO = StringIO.StringIO
+ BytesIO = cStringIO.StringIO
+else:
+ unichr = chr
+ xrange = range
+ string_types = (str,)
+ text_type = str
+ u_prefix = ''
+ iteritems = dict.items
+ itervalues = dict.values
+ from io import StringIO, BytesIO, TextIOWrapper
+
+ class UnclosingTextIOWrapper(TextIOWrapper):
+ # Don't close underlying buffer on destruction.
+ def close(self):
+ self.flush()
+
+
+def add_metaclass(metaclass):
+ """Class decorator for creating a class with a metaclass."""
+ def wrapper(cls):
+ orig_vars = cls.__dict__.copy()
+ orig_vars.pop('__dict__', None)
+ orig_vars.pop('__weakref__', None)
+ for slots_var in orig_vars.get('__slots__', ()):
+ orig_vars.pop(slots_var)
+ return metaclass(cls.__name__, cls.__bases__, orig_vars)
+ return wrapper
diff --git a/wandb/vendor/pynvml/__init__.py b/wandb/vendor/pynvml/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/wandb/vendor/pynvml/pynvml.py b/wandb/vendor/pynvml/pynvml.py
new file mode 100644
index 0000000000000000000000000000000000000000..b1a3e05164a99fc4258db6bb2a15a7d9a8e94b01
--- /dev/null
+++ b/wandb/vendor/pynvml/pynvml.py
@@ -0,0 +1,4779 @@
+#####
+#
+# Based on nvidia-ml-py version 11.515.48
+# Sourced from https://pypi.org/project/nvidia-ml-py/
+# Modifications:
+# - Added `NVML_DLL_PATH` env var check to the library initialization path
+# - Applied black formatting to improve readability
+# - Try different versions ("_v3", "_v2", "") of the functions:
+# nvmlDeviceGetComputeRunningProcesses,
+# nvmlDeviceGetGraphicsRunningProcesses, and
+# nvmlDeviceGetMPSComputeRunningProcesses
+#
+# Copyright (c) 2011-2022, NVIDIA Corporation. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of the NVIDIA Corporation nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+# THE POSSIBILITY OF SUCH DAMAGE.
+#####
+
+##
+# Python bindings for the NVML library
+##
+from ctypes import *
+from ctypes.util import find_library
+from functools import wraps
+import sys
+import os
+import threading
+import string
+
+## C Type mappings ##
+## Enums
+_nvmlEnableState_t = c_uint
+NVML_FEATURE_DISABLED = 0
+NVML_FEATURE_ENABLED = 1
+
+_nvmlBrandType_t = c_uint
+NVML_BRAND_UNKNOWN = 0
+NVML_BRAND_QUADRO = 1
+NVML_BRAND_TESLA = 2
+NVML_BRAND_NVS = 3
+NVML_BRAND_GRID = (
+ 4 # Deprecated from API reporting. Keeping definition for backward compatibility.
+)
+NVML_BRAND_GEFORCE = 5
+NVML_BRAND_TITAN = 6
+NVML_BRAND_NVIDIA_VAPPS = 7 # NVIDIA Virtual Applications
+NVML_BRAND_NVIDIA_VPC = 8 # NVIDIA Virtual PC
+NVML_BRAND_NVIDIA_VCS = 9 # NVIDIA Virtual Compute Server
+NVML_BRAND_NVIDIA_VWS = 10 # NVIDIA RTX Virtual Workstation
+NVML_BRAND_NVIDIA_CLOUD_GAMING = 11 # NVIDIA Cloud Gaming
+NVML_BRAND_NVIDIA_VGAMING = NVML_BRAND_NVIDIA_CLOUD_GAMING # Deprecated from API reporting. Keeping definition for backward compatibility.
+NVML_BRAND_QUADRO_RTX = 12
+NVML_BRAND_NVIDIA_RTX = 13
+NVML_BRAND_NVIDIA = 14
+NVML_BRAND_GEFORCE_RTX = 15 # Unused
+NVML_BRAND_TITAN_RTX = 16 # Unused
+NVML_BRAND_COUNT = 17
+
+_nvmlTemperatureThresholds_t = c_uint
+NVML_TEMPERATURE_THRESHOLD_SHUTDOWN = 0
+NVML_TEMPERATURE_THRESHOLD_SLOWDOWN = 1
+NVML_TEMPERATURE_THRESHOLD_MEM_MAX = 2
+NVML_TEMPERATURE_THRESHOLD_GPU_MAX = 3
+NVML_TEMPERATURE_THRESHOLD_ACOUSTIC_MIN = 4
+NVML_TEMPERATURE_THRESHOLD_ACOUSTIC_CURR = 5
+NVML_TEMPERATURE_THRESHOLD_ACOUSTIC_MAX = 6
+NVML_TEMPERATURE_THRESHOLD_COUNT = 7
+
+_nvmlTemperatureSensors_t = c_uint
+NVML_TEMPERATURE_GPU = 0
+NVML_TEMPERATURE_COUNT = 1
+
+_nvmlComputeMode_t = c_uint
+NVML_COMPUTEMODE_DEFAULT = 0
+NVML_COMPUTEMODE_EXCLUSIVE_THREAD = 1 ## Support Removed
+NVML_COMPUTEMODE_PROHIBITED = 2
+NVML_COMPUTEMODE_EXCLUSIVE_PROCESS = 3
+NVML_COMPUTEMODE_COUNT = 4
+
+_nvmlMemoryLocation_t = c_uint
+NVML_MEMORY_LOCATION_L1_CACHE = 0
+NVML_MEMORY_LOCATION_L2_CACHE = 1
+NVML_MEMORY_LOCATION_DEVICE_MEMORY = 2
+NVML_MEMORY_LOCATION_DRAM = 2
+NVML_MEMORY_LOCATION_REGISTER_FILE = 3
+NVML_MEMORY_LOCATION_TEXTURE_MEMORY = 4
+NVML_MEMORY_LOCATION_TEXTURE_SHM = 5
+NVML_MEMORY_LOCATION_CBU = 6
+NVML_MEMORY_LOCATION_SRAM = 7
+NVML_MEMORY_LOCATION_COUNT = 8
+
+NVML_NVLINK_MAX_LINKS = 12
+
+# For backwards compatibility, maintain the incorrectly-named "LANES" define
+NVML_NVLINK_MAX_LANES = NVML_NVLINK_MAX_LINKS
+
+_nvmlNvLinkErrorCounter_t = c_uint
+NVML_NVLINK_ERROR_DL_REPLAY = 0
+NVML_NVLINK_ERROR_DL_RECOVERY = 1
+NVML_NVLINK_ERROR_DL_CRC_FLIT = 2
+NVML_NVLINK_ERROR_DL_CRC_DATA = 3
+NVML_NVLINK_ERROR_DL_ECC_DATA = 4
+NVML_NVLINK_ERROR_COUNT = 5
+
+_nvmlNvLinkEccLaneErrorCounter_t = c_uint
+NVML_NVLINK_ERROR_DL_ECC_LANE0 = 0
+NVML_NVLINK_ERROR_DL_ECC_LANE1 = 1
+NVML_NVLINK_ERROR_DL_ECC_LANE2 = 2
+NVML_NVLINK_ERROR_DL_ECC_LANE3 = 3
+NVML_NVLINK_ERROR_DL_ECC_COUNT = 5
+
+_nvmlNvLinkCapability_t = c_uint
+NVML_NVLINK_CAP_P2P_SUPPORTED = 0
+NVML_NVLINK_CAP_SYSMEM_ACCESS = 1
+NVML_NVLINK_CAP_P2P_ATOMICS = 2
+NVML_NVLINK_CAP_SYSMEM_ATOMICS = 3
+NVML_NVLINK_CAP_SLI_BRIDGE = 4
+NVML_NVLINK_CAP_VALID = 5
+NVML_NVLINK_CAP_COUNT = 6
+
+_nvmlNvLinkUtilizationCountPktTypes_t = c_uint
+NVML_NVLINK_COUNTER_PKTFILTER_NOP = 0x1
+NVML_NVLINK_COUNTER_PKTFILTER_READ = 0x2
+NVML_NVLINK_COUNTER_PKTFILTER_WRITE = 0x4
+NVML_NVLINK_COUNTER_PKTFILTER_RATOM = 0x8
+NVML_NVLINK_COUNTER_PKTFILTER_NRATOM = 0x10
+NVML_NVLINK_COUNTER_PKTFILTER_FLUSH = 0x20
+NVML_NVLINK_COUNTER_PKTFILTER_RESPDATA = 0x40
+NVML_NVLINK_COUNTER_PKTFILTER_RESPNODATA = 0x80
+NVML_NVLINK_COUNTER_PKTFILTER_ALL = 0xFF
+
+_nvmlNvLinkUtilizationCountUnits_t = c_uint
+NVML_NVLINK_COUNTER_UNIT_CYCLES = 0
+NVML_NVLINK_COUNTER_UNIT_PACKETS = 1
+NVML_NVLINK_COUNTER_UNIT_BYTES = 2
+NVML_NVLINK_COUNTER_UNIT_RESERVED = 3
+NVML_NVLINK_COUNTER_UNIT_COUNT = 4
+
+_nvmlNvLinkDeviceType_t = c_uint
+NVML_NVLINK_DEVICE_TYPE_GPU = 0x00
+NVML_NVLINK_DEVICE_TYPE_IBMNPU = 0x01
+NVML_NVLINK_DEVICE_TYPE_SWITCH = 0x02
+NVML_NVLINK_DEVICE_TYPE_UNKNOWN = 0xFF
+
+# These are deprecated, instead use _nvmlMemoryErrorType_t
+_nvmlEccBitType_t = c_uint
+NVML_SINGLE_BIT_ECC = 0
+NVML_DOUBLE_BIT_ECC = 1
+NVML_ECC_ERROR_TYPE_COUNT = 2
+
+_nvmlEccCounterType_t = c_uint
+NVML_VOLATILE_ECC = 0
+NVML_AGGREGATE_ECC = 1
+NVML_ECC_COUNTER_TYPE_COUNT = 2
+
+_nvmlMemoryErrorType_t = c_uint
+NVML_MEMORY_ERROR_TYPE_CORRECTED = 0
+NVML_MEMORY_ERROR_TYPE_UNCORRECTED = 1
+NVML_MEMORY_ERROR_TYPE_COUNT = 2
+
+_nvmlClockType_t = c_uint
+NVML_CLOCK_GRAPHICS = 0
+NVML_CLOCK_SM = 1
+NVML_CLOCK_MEM = 2
+NVML_CLOCK_VIDEO = 3
+NVML_CLOCK_COUNT = 4
+
+_nvmlClockId_t = c_uint
+NVML_CLOCK_ID_CURRENT = 0
+NVML_CLOCK_ID_APP_CLOCK_TARGET = 1
+NVML_CLOCK_ID_APP_CLOCK_DEFAULT = 2
+NVML_CLOCK_ID_CUSTOMER_BOOST_MAX = 3
+NVML_CLOCK_ID_COUNT = 4
+
+_nvmlDriverModel_t = c_uint
+NVML_DRIVER_WDDM = 0
+NVML_DRIVER_WDM = 1
+
+NVML_MAX_GPU_PERF_PSTATES = 16
+
+_nvmlPstates_t = c_uint
+NVML_PSTATE_0 = 0
+NVML_PSTATE_1 = 1
+NVML_PSTATE_2 = 2
+NVML_PSTATE_3 = 3
+NVML_PSTATE_4 = 4
+NVML_PSTATE_5 = 5
+NVML_PSTATE_6 = 6
+NVML_PSTATE_7 = 7
+NVML_PSTATE_8 = 8
+NVML_PSTATE_9 = 9
+NVML_PSTATE_10 = 10
+NVML_PSTATE_11 = 11
+NVML_PSTATE_12 = 12
+NVML_PSTATE_13 = 13
+NVML_PSTATE_14 = 14
+NVML_PSTATE_15 = 15
+NVML_PSTATE_UNKNOWN = 32
+
+_nvmlInforomObject_t = c_uint
+NVML_INFOROM_OEM = 0
+NVML_INFOROM_ECC = 1
+NVML_INFOROM_POWER = 2
+NVML_INFOROM_COUNT = 3
+
+_nvmlReturn_t = c_uint
+NVML_SUCCESS = 0
+NVML_ERROR_UNINITIALIZED = 1
+NVML_ERROR_INVALID_ARGUMENT = 2
+NVML_ERROR_NOT_SUPPORTED = 3
+NVML_ERROR_NO_PERMISSION = 4
+NVML_ERROR_ALREADY_INITIALIZED = 5
+NVML_ERROR_NOT_FOUND = 6
+NVML_ERROR_INSUFFICIENT_SIZE = 7
+NVML_ERROR_INSUFFICIENT_POWER = 8
+NVML_ERROR_DRIVER_NOT_LOADED = 9
+NVML_ERROR_TIMEOUT = 10
+NVML_ERROR_IRQ_ISSUE = 11
+NVML_ERROR_LIBRARY_NOT_FOUND = 12
+NVML_ERROR_FUNCTION_NOT_FOUND = 13
+NVML_ERROR_CORRUPTED_INFOROM = 14
+NVML_ERROR_GPU_IS_LOST = 15
+NVML_ERROR_RESET_REQUIRED = 16
+NVML_ERROR_OPERATING_SYSTEM = 17
+NVML_ERROR_LIB_RM_VERSION_MISMATCH = 18
+NVML_ERROR_IN_USE = 19
+NVML_ERROR_MEMORY = 20
+NVML_ERROR_NO_DATA = 21
+NVML_ERROR_VGPU_ECC_NOT_SUPPORTED = 22
+NVML_ERROR_INSUFFICIENT_RESOURCES = 23
+NVML_ERROR_FREQ_NOT_SUPPORTED = 24
+NVML_ERROR_UNKNOWN = 999
+
+_nvmlFanState_t = c_uint
+NVML_FAN_NORMAL = 0
+NVML_FAN_FAILED = 1
+
+_nvmlLedColor_t = c_uint
+NVML_LED_COLOR_GREEN = 0
+NVML_LED_COLOR_AMBER = 1
+
+_nvmlGpuOperationMode_t = c_uint
+NVML_GOM_ALL_ON = 0
+NVML_GOM_COMPUTE = 1
+NVML_GOM_LOW_DP = 2
+
+_nvmlPageRetirementCause_t = c_uint
+NVML_PAGE_RETIREMENT_CAUSE_MULTIPLE_SINGLE_BIT_ECC_ERRORS = 0
+NVML_PAGE_RETIREMENT_CAUSE_DOUBLE_BIT_ECC_ERROR = 1
+NVML_PAGE_RETIREMENT_CAUSE_COUNT = 2
+
+_nvmlRestrictedAPI_t = c_uint
+NVML_RESTRICTED_API_SET_APPLICATION_CLOCKS = 0
+NVML_RESTRICTED_API_SET_AUTO_BOOSTED_CLOCKS = 1
+NVML_RESTRICTED_API_COUNT = 2
+
+_nvmlBridgeChipType_t = c_uint
+NVML_BRIDGE_CHIP_PLX = 0
+NVML_BRIDGE_CHIP_BRO4 = 1
+NVML_MAX_PHYSICAL_BRIDGE = 128
+
+_nvmlValueType_t = c_uint
+NVML_VALUE_TYPE_DOUBLE = 0
+NVML_VALUE_TYPE_UNSIGNED_INT = 1
+NVML_VALUE_TYPE_UNSIGNED_LONG = 2
+NVML_VALUE_TYPE_UNSIGNED_LONG_LONG = 3
+NVML_VALUE_TYPE_SIGNED_LONG_LONG = 4
+NVML_VALUE_TYPE_COUNT = 5
+
+_nvmlPerfPolicyType_t = c_uint
+NVML_PERF_POLICY_POWER = 0
+NVML_PERF_POLICY_THERMAL = 1
+NVML_PERF_POLICY_SYNC_BOOST = 2
+NVML_PERF_POLICY_BOARD_LIMIT = 3
+NVML_PERF_POLICY_LOW_UTILIZATION = 4
+NVML_PERF_POLICY_RELIABILITY = 5
+NVML_PERF_POLICY_TOTAL_APP_CLOCKS = 10
+NVML_PERF_POLICY_TOTAL_BASE_CLOCKS = 11
+NVML_PERF_POLICY_COUNT = 12
+
+_nvmlEncoderQueryType_t = c_uint
+NVML_ENCODER_QUERY_H264 = 0
+NVML_ENCODER_QUERY_HEVC = 1
+
+_nvmlFBCSessionType_t = c_uint
+NVML_FBC_SESSION_TYPE_UNKNOWN = 0
+NVML_FBC_SESSION_TYPE_TOSYS = 1
+NVML_FBC_SESSION_TYPE_CUDA = 2
+NVML_FBC_SESSION_TYPE_VID = 3
+NVML_FBC_SESSION_TYPE_HWENC = 4
+
+_nvmlDetachGpuState_t = c_uint
+NVML_DETACH_GPU_KEEP = 0
+NVML_DETACH_GPU_REMOVE = 1
+
+_nvmlPcieLinkState_t = c_uint
+NVML_PCIE_LINK_KEEP = 0
+NVML_PCIE_LINK_SHUT_DOWN = 1
+
+_nvmlSamplingType_t = c_uint
+NVML_TOTAL_POWER_SAMPLES = 0
+NVML_GPU_UTILIZATION_SAMPLES = 1
+NVML_MEMORY_UTILIZATION_SAMPLES = 2
+NVML_ENC_UTILIZATION_SAMPLES = 3
+NVML_DEC_UTILIZATION_SAMPLES = 4
+NVML_PROCESSOR_CLK_SAMPLES = 5
+NVML_MEMORY_CLK_SAMPLES = 6
+NVML_SAMPLINGTYPE_COUNT = 7
+
+_nvmlPcieUtilCounter_t = c_uint
+NVML_PCIE_UTIL_TX_BYTES = 0
+NVML_PCIE_UTIL_RX_BYTES = 1
+NVML_PCIE_UTIL_COUNT = 2
+
+_nvmlGpuTopologyLevel_t = c_uint
+NVML_TOPOLOGY_INTERNAL = 0
+NVML_TOPOLOGY_SINGLE = 10
+NVML_TOPOLOGY_MULTIPLE = 20
+NVML_TOPOLOGY_HOSTBRIDGE = 30
+NVML_TOPOLOGY_NODE = 40
+NVML_TOPOLOGY_CPU = NVML_TOPOLOGY_NODE
+NVML_TOPOLOGY_SYSTEM = 50
+
+_nvmlGpuP2PCapsIndex_t = c_uint
+NVML_P2P_CAPS_INDEX_READ = (0,)
+NVML_P2P_CAPS_INDEX_WRITE = 1
+NVML_P2P_CAPS_INDEX_NVLINK = 2
+NVML_P2P_CAPS_INDEX_ATOMICS = 3
+NVML_P2P_CAPS_INDEX_PROP = 4
+NVML_P2P_CAPS_INDEX_LOOPBACK = 5
+NVML_P2P_CAPS_INDEX_UNKNOWN = 6
+
+_nvmlGpuP2PStatus_t = c_uint
+NVML_P2P_STATUS_OK = 0
+NVML_P2P_STATUS_CHIPSET_NOT_SUPPORED = 1
+NVML_P2P_STATUS_GPU_NOT_SUPPORTED = 2
+NVML_P2P_STATUS_IOH_TOPOLOGY_NOT_SUPPORTED = 3
+NVML_P2P_STATUS_DISABLED_BY_REGKEY = 4
+NVML_P2P_STATUS_NOT_SUPPORTED = 5
+NVML_P2P_STATUS_UNKNOWN = 6
+
+_nvmlDeviceArchitecture_t = c_uint
+NVML_DEVICE_ARCH_KEPLER = 2
+NVML_DEVICE_ARCH_MAXWELL = 3
+NVML_DEVICE_ARCH_PASCAL = 4
+NVML_DEVICE_ARCH_VOLTA = 5
+NVML_DEVICE_ARCH_TURING = 6
+NVML_DEVICE_ARCH_AMPERE = 7
+NVML_DEVICE_ARCH_UNKNOWN = 0xFFFFFFFF
+
+# PCI bus Types
+_nvmlBusType_t = c_uint
+NVML_BUS_TYPE_UNKNOWN = 0
+NVML_BUS_TYPE_PCI = 1
+NVML_BUS_TYPE_PCIE = 2
+NVML_BUS_TYPE_FPCI = 3
+NVML_BUS_TYPE_AGP = 4
+
+_nvmlPowerSource_t = c_uint
+NVML_POWER_SOURCE_AC = 0x00000000
+NVML_POWER_SOURCE_BATTERY = 0x00000001
+
+_nvmlAdaptiveClockInfoStatus_t = c_uint
+NVML_ADAPTIVE_CLOCKING_INFO_STATUS_DISABLED = 0x00000000
+NVML_ADAPTIVE_CLOCKING_INFO_STATUS_ENABLED = 0x00000001
+
+_nvmlClockLimitId_t = c_uint
+NVML_CLOCK_LIMIT_ID_RANGE_START = 0xFFFFFF00
+NVML_CLOCK_LIMIT_ID_TDP = 0xFFFFFF01
+NVML_CLOCK_LIMIT_ID_UNLIMITED = 0xFFFFFF02
+
+_nvmlPcieLinkMaxSpeed_t = c_uint
+NVML_PCIE_LINK_MAX_SPEED_INVALID = 0x00000000
+NVML_PCIE_LINK_MAX_SPEED_2500MBPS = 0x00000001
+NVML_PCIE_LINK_MAX_SPEED_5000MBPS = 0x00000002
+NVML_PCIE_LINK_MAX_SPEED_8000MBPS = 0x00000003
+NVML_PCIE_LINK_MAX_SPEED_16000MBPS = 0x00000004
+NVML_PCIE_LINK_MAX_SPEED_32000MBPS = 0x00000005
+
+_nvmlAffinityScope_t = c_uint
+NVML_AFFINITY_SCOPE_NODE = 0
+NVML_AFFINITY_SCOPE_SOCKET = 1
+
+# C preprocessor defined values
+nvmlFlagDefault = 0
+nvmlFlagForce = 1
+NVML_INIT_FLAG_NO_GPUS = 1
+NVML_INIT_FLAG_NO_ATTACH = 2
+
+NVML_MAX_GPC_COUNT = 32
+
+# buffer size
+NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE = 16
+NVML_DEVICE_UUID_BUFFER_SIZE = 80
+NVML_DEVICE_UUID_V2_BUFFER_SIZE = 96
+NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE = 80
+NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE = 80
+NVML_DEVICE_NAME_BUFFER_SIZE = 64
+NVML_DEVICE_NAME_V2_BUFFER_SIZE = 96
+NVML_DEVICE_SERIAL_BUFFER_SIZE = 30
+NVML_DEVICE_PART_NUMBER_BUFFER_SIZE = 80
+NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE = 32
+NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE = 32
+NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE = 16
+NVML_GRID_LICENSE_BUFFER_SIZE = 128
+NVML_VGPU_NAME_BUFFER_SIZE = 64
+NVML_GRID_LICENSE_FEATURE_MAX_COUNT = 3
+NVML_VGPU_METADATA_OPAQUE_DATA_SIZE = sizeof(c_uint) + 256
+NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE = 256
+
+# Format strings
+NVML_DEVICE_PCI_BUS_ID_LEGACY_FMT = "%04X:%02X:%02X.0"
+NVML_DEVICE_PCI_BUS_ID_FMT = "%08X:%02X:%02X.0"
+
+NVML_VALUE_NOT_AVAILABLE_ulonglong = c_ulonglong(-1)
+NVML_VALUE_NOT_AVAILABLE_uint = c_uint(-1)
+
+"""
+ Field Identifiers.
+
+ All Identifiers pertain to a device. Each ID is only used once and is guaranteed never to change.
+"""
+NVML_FI_DEV_ECC_CURRENT = 1 # Current ECC mode. 1=Active. 0=Inactive
+NVML_FI_DEV_ECC_PENDING = 2 # Pending ECC mode. 1=Active. 0=Inactive
+
+# ECC Count Totals
+NVML_FI_DEV_ECC_SBE_VOL_TOTAL = 3 # Total single bit volatile ECC errors
+NVML_FI_DEV_ECC_DBE_VOL_TOTAL = 4 # Total double bit volatile ECC errors
+NVML_FI_DEV_ECC_SBE_AGG_TOTAL = 5 # Total single bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_DBE_AGG_TOTAL = 6 # Total double bit aggregate (persistent) ECC errors
+# Individual ECC locations
+NVML_FI_DEV_ECC_SBE_VOL_L1 = 7 # L1 cache single bit volatile ECC errors
+NVML_FI_DEV_ECC_DBE_VOL_L1 = 8 # L1 cache double bit volatile ECC errors
+NVML_FI_DEV_ECC_SBE_VOL_L2 = 9 # L2 cache single bit volatile ECC errors
+NVML_FI_DEV_ECC_DBE_VOL_L2 = 10 # L2 cache double bit volatile ECC errors
+NVML_FI_DEV_ECC_SBE_VOL_DEV = 11 # Device memory single bit volatile ECC errors
+NVML_FI_DEV_ECC_DBE_VOL_DEV = 12 # Device memory double bit volatile ECC errors
+NVML_FI_DEV_ECC_SBE_VOL_REG = 13 # Register file single bit volatile ECC errors
+NVML_FI_DEV_ECC_DBE_VOL_REG = 14 # Register file double bit volatile ECC errors
+NVML_FI_DEV_ECC_SBE_VOL_TEX = 15 # Texture memory single bit volatile ECC errors
+NVML_FI_DEV_ECC_DBE_VOL_TEX = 16 # Texture memory double bit volatile ECC errors
+NVML_FI_DEV_ECC_DBE_VOL_CBU = 17 # CBU double bit volatile ECC errors
+NVML_FI_DEV_ECC_SBE_AGG_L1 = 18 # L1 cache single bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_DBE_AGG_L1 = 19 # L1 cache double bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_SBE_AGG_L2 = 20 # L2 cache single bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_DBE_AGG_L2 = 21 # L2 cache double bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_SBE_AGG_DEV = (
+ 22 # Device memory single bit aggregate (persistent) ECC errors
+)
+NVML_FI_DEV_ECC_DBE_AGG_DEV = (
+ 23 # Device memory double bit aggregate (persistent) ECC errors
+)
+NVML_FI_DEV_ECC_SBE_AGG_REG = (
+ 24 # Register File single bit aggregate (persistent) ECC errors
+)
+NVML_FI_DEV_ECC_DBE_AGG_REG = (
+ 25 # Register File double bit aggregate (persistent) ECC errors
+)
+NVML_FI_DEV_ECC_SBE_AGG_TEX = (
+ 26 # Texture memory single bit aggregate (persistent) ECC errors
+)
+NVML_FI_DEV_ECC_DBE_AGG_TEX = (
+ 27 # Texture memory double bit aggregate (persistent) ECC errors
+)
+NVML_FI_DEV_ECC_DBE_AGG_CBU = 28 # CBU double bit aggregate ECC errors
+
+# Page Retirement
+NVML_FI_DEV_RETIRED_SBE = 29 # Number of retired pages because of single bit errors
+NVML_FI_DEV_RETIRED_DBE = 30 # Number of retired pages because of double bit errors
+NVML_FI_DEV_RETIRED_PENDING = 31 # If any pages are pending retirement. 1=yes. 0=no.
+
+# NvLink Flit Error Counters
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L0 = (
+ 32 # NVLink flow control CRC Error Counter for Lane 0
+)
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L1 = (
+ 33 # NVLink flow control CRC Error Counter for Lane 1
+)
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L2 = (
+ 34 # NVLink flow control CRC Error Counter for Lane 2
+)
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L3 = (
+ 35 # NVLink flow control CRC Error Counter for Lane 3
+)
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L4 = (
+ 36 # NVLink flow control CRC Error Counter for Lane 4
+)
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L5 = (
+ 37 # NVLink flow control CRC Error Counter for Lane 5
+)
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL = (
+ 38 # NVLink flow control CRC Error Counter total for all Lanes
+)
+
+# NvLink CRC Data Error Counters
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L0 = (
+ 39 # NVLink data CRC Error Counter for Lane 0
+)
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L1 = (
+ 40 # NVLink data CRC Error Counter for Lane 1
+)
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L2 = (
+ 41 # NVLink data CRC Error Counter for Lane 2
+)
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L3 = (
+ 42 # NVLink data CRC Error Counter for Lane 3
+)
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L4 = (
+ 43 # NVLink data CRC Error Counter for Lane 4
+)
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L5 = (
+ 44 # NVLink data CRC Error Counter for Lane 5
+)
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL = (
+ 45 # NvLink data CRC Error Counter total for all Lanes
+)
+
+# NvLink Replay Error Counters
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L0 = 46 # NVLink Replay Error Counter for Lane 0
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L1 = 47 # NVLink Replay Error Counter for Lane 1
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L2 = 48 # NVLink Replay Error Counter for Lane 2
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L3 = 49 # NVLink Replay Error Counter for Lane 3
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L4 = 50 # NVLink Replay Error Counter for Lane 4
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L5 = 51 # NVLink Replay Error Counter for Lane 5
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL = (
+ 52 # NVLink Replay Error Counter total for all Lanes
+)
+
+# NvLink Recovery Error Counters
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L0 = (
+ 53 # NVLink Recovery Error Counter for Lane 0
+)
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L1 = (
+ 54 # NVLink Recovery Error Counter for Lane 1
+)
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L2 = (
+ 55 # NVLink Recovery Error Counter for Lane 2
+)
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L3 = (
+ 56 # NVLink Recovery Error Counter for Lane 3
+)
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L4 = (
+ 57 # NVLink Recovery Error Counter for Lane 4
+)
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L5 = (
+ 58 # NVLink Recovery Error Counter for Lane 5
+)
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL = (
+ 59 # NVLink Recovery Error Counter total for all Lanes
+)
+
+# NvLink Bandwidth Counters
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L0 = (
+ 60 # NVLink Bandwidth Counter for Counter Set 0, Lane 0
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L1 = (
+ 61 # NVLink Bandwidth Counter for Counter Set 0, Lane 1
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L2 = (
+ 62 # NVLink Bandwidth Counter for Counter Set 0, Lane 2
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L3 = (
+ 63 # NVLink Bandwidth Counter for Counter Set 0, Lane 3
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L4 = (
+ 64 # NVLink Bandwidth Counter for Counter Set 0, Lane 4
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L5 = (
+ 65 # NVLink Bandwidth Counter for Counter Set 0, Lane 5
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_TOTAL = (
+ 66 # NVLink Bandwidth Counter Total for Counter Set 0, All Lanes
+)
+
+# NvLink Bandwidth Counters
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L0 = (
+ 67 # NVLink Bandwidth Counter for Counter Set 1, Lane 0
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L1 = (
+ 68 # NVLink Bandwidth Counter for Counter Set 1, Lane 1
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L2 = (
+ 69 # NVLink Bandwidth Counter for Counter Set 1, Lane 2
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L3 = (
+ 70 # NVLink Bandwidth Counter for Counter Set 1, Lane 3
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L4 = (
+ 71 # NVLink Bandwidth Counter for Counter Set 1, Lane 4
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L5 = (
+ 72 # NVLink Bandwidth Counter for Counter Set 1, Lane 5
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_TOTAL = (
+ 73 # NVLink Bandwidth Counter Total for Counter Set 1, All Lanes
+)
+
+# Perf Policy Counters
+NVML_FI_DEV_PERF_POLICY_POWER = 74 # Perf Policy Counter for Power Policy
+NVML_FI_DEV_PERF_POLICY_THERMAL = 75 # Perf Policy Counter for Thermal Policy
+NVML_FI_DEV_PERF_POLICY_SYNC_BOOST = 76 # Perf Policy Counter for Sync boost Policy
+NVML_FI_DEV_PERF_POLICY_BOARD_LIMIT = 77 # Perf Policy Counter for Board Limit
+NVML_FI_DEV_PERF_POLICY_LOW_UTILIZATION = (
+ 78 # Perf Policy Counter for Low GPU Utilization Policy
+)
+NVML_FI_DEV_PERF_POLICY_RELIABILITY = 79 # Perf Policy Counter for Reliability Policy
+NVML_FI_DEV_PERF_POLICY_TOTAL_APP_CLOCKS = (
+ 80 # Perf Policy Counter for Total App Clock Policy
+)
+NVML_FI_DEV_PERF_POLICY_TOTAL_BASE_CLOCKS = (
+ 81 # Perf Policy Counter for Total Base Clocks Policy
+)
+
+# Memory temperatures
+NVML_FI_DEV_MEMORY_TEMP = 82 # Memory temperature for the device
+
+# Energy Counter
+NVML_FI_DEV_TOTAL_ENERGY_CONSUMPTION = (
+ 83 # Total energy consumption for the GPU in mJ since the driver was last reloaded
+)
+
+# NVLink Speed
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L0 = 84
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L1 = 85
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L2 = 86
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L3 = 87
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L4 = 88
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L5 = 89
+NVML_FI_DEV_NVLINK_SPEED_MBPS_COMMON = 90
+
+# NVLink Link Count
+NVML_FI_DEV_NVLINK_LINK_COUNT = 91
+
+# Page Retirement pending fields
+NVML_FI_DEV_RETIRED_PENDING_SBE = 92
+NVML_FI_DEV_RETIRED_PENDING_DBE = 93
+
+# PCIe replay and replay rollover counters
+NVML_FI_DEV_PCIE_REPLAY_COUNTER = 94
+NVML_FI_DEV_PCIE_REPLAY_ROLLOVER_COUNTER = 95
+
+# NvLink Flit Error Counters
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L6 = (
+ 96 # NVLink flow control CRC Error Counter for Lane 6
+)
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L7 = (
+ 97 # NVLink flow control CRC Error Counter for Lane 7
+)
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L8 = (
+ 98 # NVLink flow control CRC Error Counter for Lane 8
+)
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L9 = (
+ 99 # NVLink flow control CRC Error Counter for Lane 9
+)
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L10 = (
+ 100 # NVLink flow control CRC Error Counter for Lane 10
+)
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L11 = (
+ 101 # NVLink flow control CRC Error Counter for Lane 11
+)
+
+# NvLink CRC Data Error Counters
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L6 = (
+ 102 # NVLink data CRC Error Counter for Lane 6
+)
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L7 = (
+ 103 # NVLink data CRC Error Counter for Lane 7
+)
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L8 = (
+ 104 # NVLink data CRC Error Counter for Lane 8
+)
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L9 = (
+ 105 # NVLink data CRC Error Counter for Lane 9
+)
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L10 = (
+ 106 # NVLink data CRC Error Counter for Lane 10
+)
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L11 = (
+ 107 # NVLink data CRC Error Counter for Lane 11
+)
+
+# NvLink Replay Error Counters
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L6 = 108 # NVLink Replay Error Counter for Lane 6
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L7 = 109 # NVLink Replay Error Counter for Lane 7
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L8 = 110 # NVLink Replay Error Counter for Lane 8
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L9 = 111 # NVLink Replay Error Counter for Lane 9
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L10 = (
+ 112 # NVLink Replay Error Counter for Lane 10
+)
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L11 = (
+ 113 # NVLink Replay Error Counter for Lane 11
+)
+
+# NvLink Recovery Error Counters
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L6 = (
+ 114 # NVLink Recovery Error Counter for Lane 6
+)
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L7 = (
+ 115 # NVLink Recovery Error Counter for Lane 7
+)
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L8 = (
+ 116 # NVLink Recovery Error Counter for Lane 8
+)
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L9 = (
+ 117 # NVLink Recovery Error Counter for Lane 9
+)
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L10 = (
+ 118 # NVLink Recovery Error Counter for Lane 10
+)
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L11 = (
+ 119 # NVLink Recovery Error Counter for Lane 11
+)
+
+# NvLink Bandwidth Counters
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L6 = (
+ 120 # NVLink Bandwidth Counter for Counter Set 0, Lane 6
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L7 = (
+ 121 # NVLink Bandwidth Counter for Counter Set 0, Lane 7
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L8 = (
+ 122 # NVLink Bandwidth Counter for Counter Set 0, Lane 8
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L9 = (
+ 123 # NVLink Bandwidth Counter for Counter Set 0, Lane 9
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L10 = (
+ 124 # NVLink Bandwidth Counter for Counter Set 0, Lane 10
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L11 = (
+ 125 # NVLink Bandwidth Counter for Counter Set 0, Lane 11
+)
+
+# NvLink Bandwidth Counters
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L6 = (
+ 126 # NVLink Bandwidth Counter for Counter Set 1, Lane 6
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L7 = (
+ 127 # NVLink Bandwidth Counter for Counter Set 1, Lane 7
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L8 = (
+ 128 # NVLink Bandwidth Counter for Counter Set 1, Lane 8
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L9 = (
+ 129 # NVLink Bandwidth Counter for Counter Set 1, Lane 9
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L10 = (
+ 130 # NVLink Bandwidth Counter for Counter Set 1, Lane 10
+)
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L11 = (
+ 131 # NVLink Bandwidth Counter for Counter Set 1, Lane 11
+)
+
+# NVLink Speed
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L6 = 132
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L7 = 133
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L8 = 134
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L9 = 135
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L10 = 136
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L11 = 137
+
+# NVLink Throughput Counters
+NVML_FI_DEV_NVLINK_THROUGHPUT_DATA_TX = 138 # NVLink TX Data throughput in KiB
+NVML_FI_DEV_NVLINK_THROUGHPUT_DATA_RX = 139 # NVLink RX Data throughput in KiB
+NVML_FI_DEV_NVLINK_THROUGHPUT_RAW_TX = 140 # NVLink TX Data + protocol overhead in KiB
+NVML_FI_DEV_NVLINK_THROUGHPUT_RAW_RX = 141 # NVLink RX Data + protocol overhead in KiB
+
+# Row Remapper
+NVML_FI_DEV_REMAPPED_COR = 142
+NVML_FI_DEV_REMAPPED_UNC = 143
+NVML_FI_DEV_REMAPPED_PENDING = 144
+NVML_FI_DEV_REMAPPED_FAILURE = 145
+
+# Remote device NVLink ID
+NVML_FI_DEV_NVLINK_REMOTE_NVLINK_ID = 146
+
+# Number of NVLinks connected to NVSwitch
+NVML_FI_DEV_NVSWITCH_CONNECTED_LINK_COUNT = 147
+
+# NvLink ECC Data Error Counters
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L0 = (
+ 148 # < NVLink data ECC Error Counter for Link 0
+)
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L1 = (
+ 149 # < NVLink data ECC Error Counter for Link 1
+)
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L2 = (
+ 150 # < NVLink data ECC Error Counter for Link 2
+)
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L3 = (
+ 151 # < NVLink data ECC Error Counter for Link 3
+)
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L4 = (
+ 152 # < NVLink data ECC Error Counter for Link 4
+)
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L5 = (
+ 153 # < NVLink data ECC Error Counter for Link 5
+)
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L6 = (
+ 154 # < NVLink data ECC Error Counter for Link 6
+)
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L7 = (
+ 155 # < NVLink data ECC Error Counter for Link 7
+)
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L8 = (
+ 156 # < NVLink data ECC Error Counter for Link 8
+)
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L9 = (
+ 157 # < NVLink data ECC Error Counter for Link 9
+)
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L10 = (
+ 158 # < NVLink data ECC Error Counter for Link 10
+)
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L11 = (
+ 159 # < NVLink data ECC Error Counter for Link 11
+)
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_TOTAL = (
+ 160 # < NvLink data ECC Error Counter total for all Links
+)
+
+NVML_FI_MAX = 161 # One greater than the largest field ID defined above
+
+## Enums needed for the method nvmlDeviceGetVirtualizationMode and nvmlDeviceSetVirtualizationMode
+NVML_GPU_VIRTUALIZATION_MODE_NONE = 0 # Represents Bare Metal GPU
+NVML_GPU_VIRTUALIZATION_MODE_PASSTHROUGH = (
+ 1 # Device is associated with GPU-Passthorugh
+)
+NVML_GPU_VIRTUALIZATION_MODE_VGPU = (
+ 2 # Device is associated with vGPU inside virtual machine.
+)
+NVML_GPU_VIRTUALIZATION_MODE_HOST_VGPU = (
+ 3 # Device is associated with VGX hypervisor in vGPU mode
+)
+NVML_GPU_VIRTUALIZATION_MODE_HOST_VSGA = (
+ 4 # Device is associated with VGX hypervisor in vSGA mode
+)
+
+## Lib loading ##
+nvmlLib = None
+libLoadLock = threading.Lock()
+_nvmlLib_refcount = 0 # Incremented on each nvmlInit and decremented on nvmlShutdown
+
+## vGPU Management
+_nvmlVgpuTypeId_t = c_uint
+_nvmlVgpuInstance_t = c_uint
+
+_nvmlVgpuVmIdType_t = c_uint
+NVML_VGPU_VM_ID_DOMAIN_ID = 0
+NVML_VGPU_VM_ID_UUID = 1
+
+_nvmlGridLicenseFeatureCode_t = c_uint
+NVML_GRID_LICENSE_FEATURE_CODE_UNKNOWN = 0
+NVML_GRID_LICENSE_FEATURE_CODE_VGPU = 1
+NVML_GRID_LICENSE_FEATURE_CODE_NVIDIA_RTX = 2
+NVML_GRID_LICENSE_FEATURE_CODE_VWORKSTATION = (
+ 2 # deprecated, use NVML_GRID_LICENSE_FEATURE_CODE_NVIDIA_RTX.
+)
+NVML_GRID_LICENSE_FEATURE_CODE_GAMING = 3
+NVML_GRID_LICENSE_FEATURE_CODE_COMPUTE = 4
+
+_nvmlGridLicenseExpiryStatus_t = c_uint8
+NVML_GRID_LICENSE_EXPIRY_NOT_AVAILABLE = (0,) # Expiry information not available
+NVML_GRID_LICENSE_EXPIRY_INVALID = (1,) # Invalid expiry or error fetching expiry
+NVML_GRID_LICENSE_EXPIRY_VALID = (2,) # Valid expiry
+NVML_GRID_LICENSE_EXPIRY_NOT_APPLICABLE = (3,) # Expiry not applicable
+NVML_GRID_LICENSE_EXPIRY_PERMANENT = (4,) # Permanent expiry
+
+_nvmlVgpuCapability_t = c_uint
+NVML_VGPU_CAP_NVLINK_P2P = 0 # vGPU P2P over NVLink is supported
+NVML_VGPU_CAP_GPUDIRECT = 1 # GPUDirect capability is supported
+NVML_VGPU_CAP_COUNT = 2
+
+_nvmlVgpuGuestInfoState_t = c_uint
+NVML_VGPU_INSTANCE_GUEST_INFO_STATE_UNINITIALIZED = 0
+NVML_VGPU_INSTANCE_GUEST_INFO_STATE_INITIALIZED = 1
+
+_nvmlVgpuVmCompatibility_t = c_uint
+NVML_VGPU_VM_COMPATIBILITY_NONE = 0x0
+NVML_VGPU_VM_COMPATIBILITY_COLD = 0x1
+NVML_VGPU_VM_COMPATIBILITY_HIBERNATE = 0x2
+NVML_VGPU_VM_COMPATIBILITY_SLEEP = 0x4
+NVML_VGPU_VM_COMPATIBILITY_LIVE = 0x8
+
+_nvmlVgpuPgpuCompatibilityLimitCode_t = c_uint
+NVML_VGPU_COMPATIBILITY_LIMIT_NONE = 0x0
+NVML_VGPU_COMPATIBILITY_LIMIT_HOST_DRIVER = 0x1
+NVML_VGPU_COMPATIBILITY_LIMIT_GUEST_DRIVER = 0x2
+NVML_VGPU_COMPATIBILITY_LIMIT_GPU = 0x4
+NVML_VGPU_COMPATIBILITY_LIMIT_OTHER = 0x80000000
+
+_nvmlHostVgpuMode_t = c_uint
+NVML_HOST_VGPU_MODE_NON_SRIOV = 0
+NVML_HOST_VGPU_MODE_SRIOV = 1
+
+# GSP firmware
+NVML_GSP_FIRMWARE_VERSION_BUF_SIZE = 0x40
+
+
+## Error Checking ##
+class NVMLError(Exception):
+ _valClassMapping = dict() # type: ignore
+ # List of currently known error codes
+ _errcode_to_string = {
+ NVML_ERROR_UNINITIALIZED: "Uninitialized",
+ NVML_ERROR_INVALID_ARGUMENT: "Invalid Argument",
+ NVML_ERROR_NOT_SUPPORTED: "Not Supported",
+ NVML_ERROR_NO_PERMISSION: "Insufficient Permissions",
+ NVML_ERROR_ALREADY_INITIALIZED: "Already Initialized",
+ NVML_ERROR_NOT_FOUND: "Not Found",
+ NVML_ERROR_INSUFFICIENT_SIZE: "Insufficient Size",
+ NVML_ERROR_INSUFFICIENT_POWER: "Insufficient External Power",
+ NVML_ERROR_DRIVER_NOT_LOADED: "Driver Not Loaded",
+ NVML_ERROR_TIMEOUT: "Timeout",
+ NVML_ERROR_IRQ_ISSUE: "Interrupt Request Issue",
+ NVML_ERROR_LIBRARY_NOT_FOUND: "NVML Shared Library Not Found",
+ NVML_ERROR_FUNCTION_NOT_FOUND: "Function Not Found",
+ NVML_ERROR_CORRUPTED_INFOROM: "Corrupted infoROM",
+ NVML_ERROR_GPU_IS_LOST: "GPU is lost",
+ NVML_ERROR_RESET_REQUIRED: "GPU requires restart",
+ NVML_ERROR_OPERATING_SYSTEM: "The operating system has blocked the request.",
+ NVML_ERROR_LIB_RM_VERSION_MISMATCH: "RM has detected an NVML/RM version mismatch.",
+ NVML_ERROR_MEMORY: "Insufficient Memory",
+ NVML_ERROR_UNKNOWN: "Unknown Error",
+ }
+
+ def __new__(typ, value):
+ """
+ Maps value to a proper subclass of NVMLError.
+ See _extractNVMLErrorsAsClasses function for more details
+ """
+ if typ == NVMLError:
+ typ = NVMLError._valClassMapping.get(value, typ)
+ obj = Exception.__new__(typ)
+ obj.value = value
+ return obj
+
+ def __str__(self):
+ try:
+ if self.value not in NVMLError._errcode_to_string:
+ NVMLError._errcode_to_string[self.value] = str(
+ nvmlErrorString(self.value)
+ )
+ return NVMLError._errcode_to_string[self.value]
+ except NVMLError:
+ return "NVML Error with code %d" % self.value
+
+ def __eq__(self, other):
+ return self.value == other.value
+
+
+def nvmlExceptionClass(nvmlErrorCode):
+ if nvmlErrorCode not in NVMLError._valClassMapping:
+ raise ValueError("nvmlErrorCode %s is not valid" % nvmlErrorCode)
+ return NVMLError._valClassMapping[nvmlErrorCode]
+
+
+def _extractNVMLErrorsAsClasses():
+ """
+ Generates a hierarchy of classes on top of NVMLError class.
+
+ Each NVML Error gets a new NVMLError subclass. This way try,except blocks can filter appropriate
+ exceptions more easily.
+
+ NVMLError is a parent class. Each NVML_ERROR_* gets it's own subclass.
+ e.g. NVML_ERROR_ALREADY_INITIALIZED will be turned into NVMLError_AlreadyInitialized
+ """
+ this_module = sys.modules[__name__]
+ nvmlErrorsNames = [x for x in dir(this_module) if x.startswith("NVML_ERROR_")]
+ for err_name in nvmlErrorsNames:
+ # e.g. Turn NVML_ERROR_ALREADY_INITIALIZED into NVMLError_AlreadyInitialized
+ class_name = "NVMLError_" + string.capwords(
+ err_name.replace("NVML_ERROR_", ""), "_"
+ ).replace("_", "")
+ err_val = getattr(this_module, err_name)
+
+ def gen_new(val):
+ def new(typ):
+ obj = NVMLError.__new__(typ, val)
+ return obj
+
+ return new
+
+ new_error_class = type(class_name, (NVMLError,), {"__new__": gen_new(err_val)})
+ new_error_class.__module__ = __name__
+ setattr(this_module, class_name, new_error_class)
+ NVMLError._valClassMapping[err_val] = new_error_class
+
+
+_extractNVMLErrorsAsClasses()
+
+
+def _nvmlCheckReturn(ret):
+ if ret != NVML_SUCCESS:
+ raise NVMLError(ret)
+ return ret
+
+
+## Function access ##
+_nvmlGetFunctionPointer_cache = ( # type: ignore
+ dict()
+) # function pointers are cached to prevent unnecessary libLoadLock locking
+
+
+def _nvmlGetFunctionPointer(name):
+ global nvmlLib
+
+ if name in _nvmlGetFunctionPointer_cache:
+ return _nvmlGetFunctionPointer_cache[name]
+
+ libLoadLock.acquire()
+ try:
+ # ensure library was loaded
+ if nvmlLib == None:
+ raise NVMLError(NVML_ERROR_UNINITIALIZED)
+ try:
+ _nvmlGetFunctionPointer_cache[name] = getattr(nvmlLib, name)
+ return _nvmlGetFunctionPointer_cache[name]
+ except AttributeError:
+ raise NVMLError(NVML_ERROR_FUNCTION_NOT_FOUND)
+ finally:
+ # lock is always freed
+ libLoadLock.release()
+
+
+## Alternative object
+# Allows the object to be printed
+# Allows mismatched types to be assigned
+# - like None when the Structure variant requires c_uint
+class nvmlFriendlyObject:
+ def __init__(self, dictionary):
+ for x in dictionary:
+ setattr(self, x, dictionary[x])
+
+ def __str__(self):
+ return self.__dict__.__str__()
+
+
+def nvmlStructToFriendlyObject(struct):
+ d = {}
+ for x in struct._fields_:
+ key = x[0]
+ value = getattr(struct, key)
+ # only need to convert from bytes if bytes, no need to check python version.
+ d[key] = value.decode() if isinstance(value, bytes) else value
+ obj = nvmlFriendlyObject(d)
+ return obj
+
+
+# pack the object so it can be passed to the NVML library
+def nvmlFriendlyObjectToStruct(obj, model):
+ for x in model._fields_:
+ key = x[0]
+ value = obj.__dict__[key]
+ # any c_char_p in python3 needs to be bytes, default encoding works fine.
+ setattr(model, key, value.encode())
+ return model
+
+
+## Unit structures
+class struct_c_nvmlUnit_t(Structure):
+ pass # opaque handle
+
+
+c_nvmlUnit_t = POINTER(struct_c_nvmlUnit_t)
+
+
+class _PrintableStructure(Structure):
+ """
+ Abstract class that produces nicer __str__ output than ctypes.Structure.
+ e.g. instead of:
+ >>> print str(obj)
+ <class_name object at 0x7fdf82fef9e0>
+ this class will print
+ class_name(field_name: formatted_value, field_name: formatted_value)
+
+ _fmt_ dictionary of <str _field_ name> -> <str format>
+ e.g. class that has _field_ 'hex_value', c_uint could be formatted with
+ _fmt_ = {"hex_value" : "%08X"}
+ to produce nicer output.
+ Default fomratting string for all fields can be set with key "<default>" like:
+ _fmt_ = {"<default>" : "%d MHz"} # e.g all values are numbers in MHz.
+ If not set it's assumed to be just "%s"
+
+ Exact format of returned str from this class is subject to change in the future.
+ """
+
+ _fmt_ = {} # type: ignore
+
+ def __str__(self):
+ result = []
+ for x in self._fields_:
+ key = x[0]
+ value = getattr(self, key)
+ fmt = "%s"
+ if key in self._fmt_:
+ fmt = self._fmt_[key]
+ elif "<default>" in self._fmt_:
+ fmt = self._fmt_["<default>"]
+ result.append(("%s: " + fmt) % (key, value))
+ return self.__class__.__name__ + "(" + ", ".join(result) + ")"
+
+ def __getattribute__(self, name):
+ res = super().__getattribute__(name)
+ # need to convert bytes to unicode for python3 don't need to for python2
+ # Python 2 strings are of both str and bytes
+ # Python 3 strings are not of type bytes
+ # ctypes should convert everything to the correct values otherwise
+ if isinstance(res, bytes):
+ if isinstance(res, str):
+ return res
+ return res.decode()
+ return res
+
+ def __setattr__(self, name, value):
+ if isinstance(value, str):
+ # encoding a python2 string returns the same value, since python2 strings are bytes already
+ # bytes passed in python3 will be ignored.
+ value = value.encode()
+ super().__setattr__(name, value)
+
+
+class c_nvmlUnitInfo_t(_PrintableStructure):
+ _fields_ = [
+ ("name", c_char * 96),
+ ("id", c_char * 96),
+ ("serial", c_char * 96),
+ ("firmwareVersion", c_char * 96),
+ ]
+
+
+class c_nvmlLedState_t(_PrintableStructure):
+ _fields_ = [
+ ("cause", c_char * 256),
+ ("color", _nvmlLedColor_t),
+ ]
+
+
+class c_nvmlPSUInfo_t(_PrintableStructure):
+ _fields_ = [
+ ("state", c_char * 256),
+ ("current", c_uint),
+ ("voltage", c_uint),
+ ("power", c_uint),
+ ]
+
+
+class c_nvmlUnitFanInfo_t(_PrintableStructure):
+ _fields_ = [
+ ("speed", c_uint),
+ ("state", _nvmlFanState_t),
+ ]
+
+
+class c_nvmlUnitFanSpeeds_t(_PrintableStructure):
+ _fields_ = [("fans", c_nvmlUnitFanInfo_t * 24), ("count", c_uint)]
+
+
+## Device structures
+class struct_c_nvmlDevice_t(Structure):
+ pass # opaque handle
+
+
+c_nvmlDevice_t = POINTER(struct_c_nvmlDevice_t)
+
+
+# Legacy pciInfo used for _v1 and _v2
+class nvmlPciInfo_v2_t(_PrintableStructure):
+ _fields_ = [
+ ("busId", c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE),
+ ("domain", c_uint),
+ ("bus", c_uint),
+ ("device", c_uint),
+ ("pciDeviceId", c_uint),
+ # Added in 2.285
+ ("pciSubSystemId", c_uint),
+ ("reserved0", c_uint),
+ ("reserved1", c_uint),
+ ("reserved2", c_uint),
+ ("reserved3", c_uint),
+ ]
+ _fmt_ = {
+ "domain": "0x%04X",
+ "bus": "0x%02X",
+ "device": "0x%02X",
+ "pciDeviceId": "0x%08X",
+ "pciSubSystemId": "0x%08X",
+ }
+
+
+class nvmlPciInfo_t(_PrintableStructure):
+ _fields_ = [
+ # Moved to the new busId location below
+ ("busIdLegacy", c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE),
+ ("domain", c_uint),
+ ("bus", c_uint),
+ ("device", c_uint),
+ ("pciDeviceId", c_uint),
+ # Added in 2.285
+ ("pciSubSystemId", c_uint),
+ # New busId replaced the long deprecated and reserved fields with a
+ # field of the same size in 9.0
+ ("busId", c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE),
+ ]
+ _fmt_ = {
+ "domain": "0x%08X",
+ "bus": "0x%02X",
+ "device": "0x%02X",
+ "pciDeviceId": "0x%08X",
+ "pciSubSystemId": "0x%08X",
+ }
+
+
+class c_nvmlExcludedDeviceInfo_t(_PrintableStructure):
+ _fields_ = [("pci", nvmlPciInfo_t), ("uuid", c_char * NVML_DEVICE_UUID_BUFFER_SIZE)]
+
+
+class nvmlNvLinkUtilizationControl_t(_PrintableStructure):
+ _fields_ = [
+ ("units", _nvmlNvLinkUtilizationCountUnits_t),
+ ("pktfilter", _nvmlNvLinkUtilizationCountPktTypes_t),
+ ]
+
+
+class c_nvmlMemory_t(_PrintableStructure):
+ _fields_ = [
+ ("total", c_ulonglong),
+ ("free", c_ulonglong),
+ ("used", c_ulonglong),
+ ]
+ _fmt_ = {"<default>": "%d B"}
+
+
+class c_nvmlMemory_v2_t(_PrintableStructure):
+ _fields_ = [
+ ("version", c_uint),
+ ("total", c_ulonglong),
+ ("reserved", c_ulonglong),
+ ("free", c_ulonglong),
+ ("used", c_ulonglong),
+ ]
+ _fmt_ = {"<default>": "%d B"}
+
+
+nvmlMemory_v2 = 0x02000028
+
+
+class c_nvmlBAR1Memory_t(_PrintableStructure):
+ _fields_ = [
+ ("bar1Total", c_ulonglong),
+ ("bar1Free", c_ulonglong),
+ ("bar1Used", c_ulonglong),
+ ]
+ _fmt_ = {"<default>": "%d B"}
+
+
+class nvmlClkMonFaultInfo_t(Structure):
+ _fields_ = [("clkApiDomain", c_uint), ("clkDomainFaultMask", c_uint)]
+
+
+class nvmlClkMonStatus_t(Structure):
+ _fields_ = [
+ ("bGlobalStatus", c_uint),
+ ("clkMonListSize", c_uint),
+ ("clkMonList", nvmlClkMonFaultInfo_t),
+ ]
+
+
+# On Windows with the WDDM driver, usedGpuMemory is reported as None
+# Code that processes this structure should check for None, I.E.
+#
+# if (info.usedGpuMemory == None):
+# # TODO handle the error
+# pass
+# else:
+# print("Using %d MiB of memory" % (info.usedGpuMemory / 1024 / 1024))
+# endif
+#
+# See NVML documentation for more information
+class c_nvmlProcessInfo_t(_PrintableStructure):
+ _fields_ = [
+ ("pid", c_uint),
+ ("usedGpuMemory", c_ulonglong),
+ ("gpuInstanceId", c_uint),
+ ("computeInstanceId", c_uint),
+ ]
+ _fmt_ = {
+ "usedGpuMemory": "%d B",
+ }
+
+
+class c_nvmlBridgeChipInfo_t(_PrintableStructure):
+ _fields_ = [
+ ("type", _nvmlBridgeChipType_t),
+ ("fwVersion", c_uint),
+ ]
+
+
+class c_nvmlBridgeChipHierarchy_t(_PrintableStructure):
+ _fields_ = [
+ ("bridgeCount", c_uint),
+ ("bridgeChipInfo", c_nvmlBridgeChipInfo_t * 128),
+ ]
+
+
+class c_nvmlEccErrorCounts_t(_PrintableStructure):
+ _fields_ = [
+ ("l1Cache", c_ulonglong),
+ ("l2Cache", c_ulonglong),
+ ("deviceMemory", c_ulonglong),
+ ("registerFile", c_ulonglong),
+ ]
+
+
+class c_nvmlUtilization_t(_PrintableStructure):
+ _fields_ = [
+ ("gpu", c_uint),
+ ("memory", c_uint),
+ ]
+ _fmt_ = {"<default>": "%d %%"}
+
+
+# Added in 2.285
+class c_nvmlHwbcEntry_t(_PrintableStructure):
+ _fields_ = [
+ ("hwbcId", c_uint),
+ ("firmwareVersion", c_char * 32),
+ ]
+
+
+class c_nvmlValue_t(Union):
+ _fields_ = [
+ ("dVal", c_double),
+ ("uiVal", c_uint),
+ ("ulVal", c_ulong),
+ ("ullVal", c_ulonglong),
+ ("sllVal", c_longlong),
+ ]
+
+
+class c_nvmlSample_t(_PrintableStructure):
+ _fields_ = [
+ ("timeStamp", c_ulonglong),
+ ("sampleValue", c_nvmlValue_t),
+ ]
+
+
+class c_nvmlViolationTime_t(_PrintableStructure):
+ _fields_ = [
+ ("referenceTime", c_ulonglong),
+ ("violationTime", c_ulonglong),
+ ]
+
+
+class c_nvmlFieldValue_t(_PrintableStructure):
+ _fields_ = [
+ ("fieldId", c_uint32),
+ ("scopeId", c_uint32),
+ ("timestamp", c_int64),
+ ("latencyUsec", c_int64),
+ ("valueType", _nvmlValueType_t),
+ ("nvmlReturn", _nvmlReturn_t),
+ ("value", c_nvmlValue_t),
+ ]
+
+
+class c_nvmlVgpuInstanceUtilizationSample_t(_PrintableStructure):
+ _fields_ = [
+ ("vgpuInstance", _nvmlVgpuInstance_t),
+ ("timeStamp", c_ulonglong),
+ ("smUtil", c_nvmlValue_t),
+ ("memUtil", c_nvmlValue_t),
+ ("encUtil", c_nvmlValue_t),
+ ("decUtil", c_nvmlValue_t),
+ ]
+
+
+class c_nvmlVgpuProcessUtilizationSample_t(_PrintableStructure):
+ _fields_ = [
+ ("vgpuInstance", _nvmlVgpuInstance_t),
+ ("pid", c_uint),
+ ("processName", c_char * NVML_VGPU_NAME_BUFFER_SIZE),
+ ("timeStamp", c_ulonglong),
+ ("smUtil", c_uint),
+ ("memUtil", c_uint),
+ ("encUtil", c_uint),
+ ("decUtil", c_uint),
+ ]
+
+
+class c_nvmlVgpuLicenseExpiry_t(_PrintableStructure):
+ _fields_ = [
+ ("year", c_uint32),
+ ("month", c_uint16),
+ ("day", c_uint16),
+ ("hour", c_uint16),
+ ("min", c_uint16),
+ ("sec", c_uint16),
+ ("status", c_uint8),
+ ]
+
+
+NVML_GRID_LICENSE_STATE_UNKNOWN = 0
+NVML_GRID_LICENSE_STATE_UNINITIALIZED = 1
+NVML_GRID_LICENSE_STATE_UNLICENSED_UNRESTRICTED = 2
+NVML_GRID_LICENSE_STATE_UNLICENSED_RESTRICTED = 3
+NVML_GRID_LICENSE_STATE_UNLICENSED = 4
+NVML_GRID_LICENSE_STATE_LICENSED = 5
+
+
+class c_nvmlVgpuLicenseInfo_t(_PrintableStructure):
+ _fields_ = [
+ ("isLicensed", c_uint8),
+ ("licenseExpiry", c_nvmlVgpuLicenseExpiry_t),
+ ("currentState", c_uint),
+ ]
+
+
+class c_nvmlEncoderSession_t(_PrintableStructure):
+ _fields_ = [
+ ("sessionId", c_uint),
+ ("pid", c_uint),
+ ("vgpuInstance", _nvmlVgpuInstance_t),
+ ("codecType", c_uint),
+ ("hResolution", c_uint),
+ ("vResolution", c_uint),
+ ("averageFps", c_uint),
+ ("encodeLatency", c_uint),
+ ]
+
+
+class c_nvmlProcessUtilizationSample_t(_PrintableStructure):
+ _fields_ = [
+ ("pid", c_uint),
+ ("timeStamp", c_ulonglong),
+ ("smUtil", c_uint),
+ ("memUtil", c_uint),
+ ("encUtil", c_uint),
+ ("decUtil", c_uint),
+ ]
+
+
+class c_nvmlGridLicenseExpiry_t(_PrintableStructure):
+ _fields_ = [
+ ("year", c_uint32),
+ ("month", c_uint16),
+ ("day", c_uint16),
+ ("hour", c_uint16),
+ ("min", c_uint16),
+ ("sec", c_uint16),
+ ("status", c_uint8),
+ ]
+
+
+class c_nvmlGridLicensableFeature_v4_t(_PrintableStructure):
+ _fields_ = [
+ ("featureCode", _nvmlGridLicenseFeatureCode_t),
+ ("featureState", c_uint),
+ ("licenseInfo", c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
+ ("productName", c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
+ ("featureEnabled", c_uint),
+ ("licenseExpiry", c_nvmlGridLicenseExpiry_t),
+ ]
+
+
+class c_nvmlGridLicensableFeatures_v4_t(_PrintableStructure):
+ _fields_ = [
+ ("isGridLicenseSupported", c_int),
+ ("licensableFeaturesCount", c_uint),
+ (
+ "gridLicensableFeatures",
+ c_nvmlGridLicensableFeature_v4_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT,
+ ),
+ ]
+
+
+class c_nvmlGridLicensableFeature_v3_t(_PrintableStructure):
+ _fields_ = [
+ ("featureCode", _nvmlGridLicenseFeatureCode_t),
+ ("featureState", c_uint),
+ ("licenseInfo", c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
+ ("productName", c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
+ ("featureEnabled", c_uint),
+ ]
+
+
+class c_nvmlGridLicensableFeatures_v3_t(_PrintableStructure):
+ _fields_ = [
+ ("isGridLicenseSupported", c_int),
+ ("licensableFeaturesCount", c_uint),
+ (
+ "gridLicensableFeatures",
+ c_nvmlGridLicensableFeature_v3_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT,
+ ),
+ ]
+
+
+class c_nvmlGridLicensableFeature_v2_t(_PrintableStructure):
+ _fields_ = [
+ ("featureCode", _nvmlGridLicenseFeatureCode_t),
+ ("featureState", c_uint),
+ ("licenseInfo", c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
+ ("productName", c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
+ ]
+
+
+class c_nvmlGridLicensableFeatures_v2_t(_PrintableStructure):
+ _fields_ = [
+ ("isGridLicenseSupported", c_int),
+ ("licensableFeaturesCount", c_uint),
+ (
+ "gridLicensableFeatures",
+ c_nvmlGridLicensableFeature_v2_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT,
+ ),
+ ]
+
+
+class c_nvmlGridLicensableFeature_t(_PrintableStructure):
+ _fields_ = [
+ ("featureCode", _nvmlGridLicenseFeatureCode_t),
+ ("featureState", c_uint),
+ ("licenseInfo", c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
+ ]
+
+
+class c_nvmlGridLicensableFeatures_t(_PrintableStructure):
+ _fields_ = [
+ ("isGridLicenseSupported", c_int),
+ ("licensableFeaturesCount", c_uint),
+ (
+ "gridLicensableFeatures",
+ c_nvmlGridLicensableFeature_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT,
+ ),
+ ]
+
+
+## Event structures
+class struct_c_nvmlEventSet_t(Structure):
+ pass # opaque handle
+
+
+c_nvmlEventSet_t = POINTER(struct_c_nvmlEventSet_t)
+
+nvmlEventTypeSingleBitEccError = 0x0000000000000001
+nvmlEventTypeDoubleBitEccError = 0x0000000000000002
+nvmlEventTypePState = 0x0000000000000004
+nvmlEventTypeXidCriticalError = 0x0000000000000008
+nvmlEventTypeClock = 0x0000000000000010
+nvmlEventTypePowerSourceChange = 0x0000000000000080
+nvmlEventMigConfigChange = 0x0000000000000100
+nvmlEventTypeNone = 0x0000000000000000
+nvmlEventTypeAll = (
+ nvmlEventTypeNone
+ | nvmlEventTypeSingleBitEccError
+ | nvmlEventTypeDoubleBitEccError
+ | nvmlEventTypePState
+ | nvmlEventTypeClock
+ | nvmlEventTypePowerSourceChange
+ | nvmlEventTypeXidCriticalError
+ | nvmlEventMigConfigChange
+)
+
+## Clock Throttle Reasons defines
+nvmlClocksThrottleReasonGpuIdle = 0x0000000000000001
+nvmlClocksThrottleReasonApplicationsClocksSetting = 0x0000000000000002
+nvmlClocksThrottleReasonUserDefinedClocks = nvmlClocksThrottleReasonApplicationsClocksSetting # deprecated, use nvmlClocksThrottleReasonApplicationsClocksSetting
+nvmlClocksThrottleReasonSwPowerCap = 0x0000000000000004
+nvmlClocksThrottleReasonHwSlowdown = 0x0000000000000008
+nvmlClocksThrottleReasonSyncBoost = 0x0000000000000010
+nvmlClocksThrottleReasonSwThermalSlowdown = 0x0000000000000020
+nvmlClocksThrottleReasonHwThermalSlowdown = 0x0000000000000040
+nvmlClocksThrottleReasonHwPowerBrakeSlowdown = 0x0000000000000080
+nvmlClocksThrottleReasonDisplayClockSetting = 0x0000000000000100
+nvmlClocksThrottleReasonNone = 0x0000000000000000
+nvmlClocksThrottleReasonAll = (
+ nvmlClocksThrottleReasonNone
+ | nvmlClocksThrottleReasonGpuIdle
+ | nvmlClocksThrottleReasonApplicationsClocksSetting
+ | nvmlClocksThrottleReasonSwPowerCap
+ | nvmlClocksThrottleReasonHwSlowdown
+ | nvmlClocksThrottleReasonSyncBoost
+ | nvmlClocksThrottleReasonSwThermalSlowdown
+ | nvmlClocksThrottleReasonHwThermalSlowdown
+ | nvmlClocksThrottleReasonHwPowerBrakeSlowdown
+ | nvmlClocksThrottleReasonDisplayClockSetting
+)
+
+
+class c_nvmlEventData_t(_PrintableStructure):
+ _fields_ = [
+ ("device", c_nvmlDevice_t),
+ ("eventType", c_ulonglong),
+ ("eventData", c_ulonglong),
+ ("gpuInstanceId", c_uint),
+ ("computeInstanceId", c_uint),
+ ]
+ _fmt_ = {"eventType": "0x%08X"}
+
+
+class c_nvmlAccountingStats_t(_PrintableStructure):
+ _fields_ = [
+ ("gpuUtilization", c_uint),
+ ("memoryUtilization", c_uint),
+ ("maxMemoryUsage", c_ulonglong),
+ ("time", c_ulonglong),
+ ("startTime", c_ulonglong),
+ ("isRunning", c_uint),
+ ("reserved", c_uint * 5),
+ ]
+
+
+class c_nvmlVgpuVersion_t(Structure):
+ _fields_ = [("minVersion", c_uint), ("maxVersion", c_uint)]
+
+
+class c_nvmlVgpuMetadata_t(Structure):
+ _fields_ = [
+ ("version", c_uint),
+ ("revision", c_uint),
+ ("guestInfoState", _nvmlVgpuGuestInfoState_t),
+ ("guestDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE),
+ ("hostDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE),
+ ("reserved", c_uint * 6),
+ ("vgpuVirtualizationCaps", c_uint),
+ ("guestVgpuVersion", c_uint),
+ ("opaqueDataSize", c_uint),
+ ("opaqueData", c_char * NVML_VGPU_METADATA_OPAQUE_DATA_SIZE),
+ ]
+
+
+class c_nvmlVgpuPgpuMetadata_t(Structure):
+ _fields_ = [
+ ("version", c_uint),
+ ("revision", c_uint),
+ ("hostDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE),
+ ("pgpuVirtualizationCaps", c_uint),
+ ("reserved", c_uint * 5),
+ ("hostSupportedVgpuRange", c_nvmlVgpuVersion_t),
+ ("opaqueDataSize", c_uint),
+ ("opaqueData", c_char * NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE),
+ ]
+
+
+class c_nvmlVgpuPgpuCompatibility_t(Structure):
+ _fields_ = [
+ ("vgpuVmCompatibility", _nvmlVgpuVmCompatibility_t),
+ ("compatibilityLimitCode", _nvmlVgpuPgpuCompatibilityLimitCode_t),
+ ]
+
+
+class c_nvmlFBCStats_t(Structure):
+ _fields_ = [
+ ("sessionsCount", c_uint),
+ ("averageFPS", c_uint),
+ ("averageLatency", c_uint),
+ ]
+
+
+class c_nvmlFBCSession_t(_PrintableStructure):
+ _fields_ = [
+ ("sessionId", c_uint),
+ ("pid", c_uint),
+ ("vgpuInstance", _nvmlVgpuInstance_t),
+ ("displayOrdinal", c_uint),
+ ("sessionType", c_uint),
+ ("sessionFlags", c_uint),
+ ("hMaxResolution", c_uint),
+ ("vMaxResolution", c_uint),
+ ("hResolution", c_uint),
+ ("vResolution", c_uint),
+ ("averageFPS", c_uint),
+ ("averageLatency", c_uint),
+ ]
+
+
+NVML_DEVICE_MIG_DISABLE = 0x0
+NVML_DEVICE_MIG_ENABLE = 0x1
+
+NVML_GPU_INSTANCE_PROFILE_1_SLICE = 0x0
+NVML_GPU_INSTANCE_PROFILE_2_SLICE = 0x1
+NVML_GPU_INSTANCE_PROFILE_3_SLICE = 0x2
+NVML_GPU_INSTANCE_PROFILE_4_SLICE = 0x3
+NVML_GPU_INSTANCE_PROFILE_7_SLICE = 0x4
+NVML_GPU_INSTANCE_PROFILE_8_SLICE = 0x5
+NVML_GPU_INSTANCE_PROFILE_6_SLICE = 0x6
+NVML_GPU_INSTANCE_PROFILE_1_SLICE_REV1 = 0x7
+NVML_GPU_INSTANCE_PROFILE_COUNT = 0x8
+
+
+class c_nvmlGpuInstancePlacement_t(Structure):
+ _fields_ = [("start", c_uint), ("size", c_uint)]
+
+
+class c_nvmlGpuInstanceProfileInfo_t(Structure):
+ _fields_ = [
+ ("id", c_uint),
+ ("isP2pSupported", c_uint),
+ ("sliceCount", c_uint),
+ ("instanceCount", c_uint),
+ ("multiprocessorCount", c_uint),
+ ("copyEngineCount", c_uint),
+ ("decoderCount", c_uint),
+ ("encoderCount", c_uint),
+ ("jpegCount", c_uint),
+ ("ofaCount", c_uint),
+ ("memorySizeMB", c_ulonglong),
+ ]
+
+
+nvmlGpuInstanceProfileInfo_v2 = 0x02000098
+
+
+class c_nvmlGpuInstanceProfileInfo_v2_t(Structure):
+ _fields_ = [
+ ("version", c_uint),
+ ("id", c_uint),
+ ("isP2pSupported", c_uint),
+ ("sliceCount", c_uint),
+ ("instanceCount", c_uint),
+ ("multiprocessorCount", c_uint),
+ ("copyEngineCount", c_uint),
+ ("decoderCount", c_uint),
+ ("encoderCount", c_uint),
+ ("jpegCount", c_uint),
+ ("ofaCount", c_uint),
+ ("memorySizeMB", c_ulonglong),
+ ("name", c_char * NVML_DEVICE_NAME_V2_BUFFER_SIZE),
+ ]
+
+ def __init__(self):
+ super().__init__(version=nvmlGpuInstanceProfileInfo_v2)
+
+
+class c_nvmlGpuInstanceInfo_t(Structure):
+ _fields_ = [
+ ("device", c_nvmlDevice_t),
+ ("id", c_uint),
+ ("profileId", c_uint),
+ ("placement", c_nvmlGpuInstancePlacement_t),
+ ]
+
+
+class struct_c_nvmlGpuInstance_t(Structure):
+ pass # opaque handle
+
+
+c_nvmlGpuInstance_t = POINTER(struct_c_nvmlGpuInstance_t)
+
+NVML_COMPUTE_INSTANCE_PROFILE_1_SLICE = 0x0
+NVML_COMPUTE_INSTANCE_PROFILE_2_SLICE = 0x1
+NVML_COMPUTE_INSTANCE_PROFILE_3_SLICE = 0x2
+NVML_COMPUTE_INSTANCE_PROFILE_4_SLICE = 0x3
+NVML_COMPUTE_INSTANCE_PROFILE_7_SLICE = 0x4
+NVML_COMPUTE_INSTANCE_PROFILE_8_SLICE = 0x5
+NVML_COMPUTE_INSTANCE_PROFILE_6_SLICE = 0x6
+NVML_COMPUTE_INSTANCE_PROFILE_COUNT = 0x7
+
+NVML_COMPUTE_INSTANCE_ENGINE_PROFILE_SHARED = 0x0
+NVML_COMPUTE_INSTANCE_ENGINE_PROFILE_COUNT = 0x1
+
+
+class c_nvmlComputeInstancePlacement_t(Structure):
+ _fields_ = [("start", c_uint), ("size", c_uint)]
+
+
+class c_nvmlComputeInstanceProfileInfo_t(Structure):
+ _fields_ = [
+ ("id", c_uint),
+ ("sliceCount", c_uint),
+ ("instanceCount", c_uint),
+ ("multiprocessorCount", c_uint),
+ ("sharedCopyEngineCount", c_uint),
+ ("sharedDecoderCount", c_uint),
+ ("sharedEncoderCount", c_uint),
+ ("sharedJpegCount", c_uint),
+ ("sharedOfaCount", c_uint),
+ ]
+
+
+nvmlComputeInstanceProfileInfo_v2 = 0x02000088
+
+
+class c_nvmlComputeInstanceProfileInfo_v2_t(Structure):
+ _fields_ = [
+ ("version", c_uint),
+ ("id", c_uint),
+ ("sliceCount", c_uint),
+ ("instanceCount", c_uint),
+ ("multiprocessorCount", c_uint),
+ ("sharedCopyEngineCount", c_uint),
+ ("sharedDecoderCount", c_uint),
+ ("sharedEncoderCount", c_uint),
+ ("sharedJpegCount", c_uint),
+ ("sharedOfaCount", c_uint),
+ ("name", c_char * NVML_DEVICE_NAME_V2_BUFFER_SIZE),
+ ]
+
+ def __init__(self):
+ super().__init__(version=nvmlComputeInstanceProfileInfo_v2)
+
+
+class c_nvmlComputeInstanceInfo_t(Structure):
+ _fields_ = [
+ ("device", c_nvmlDevice_t),
+ ("gpuInstance", c_nvmlGpuInstance_t),
+ ("id", c_uint),
+ ("profileId", c_uint),
+ ("placement", c_nvmlComputeInstancePlacement_t),
+ ]
+
+
+NVML_MAX_GPU_UTILIZATIONS = 8
+NVML_GPU_UTILIZATION_DOMAIN_GPU = 0
+NVML_GPU_UTILIZATION_DOMAIN_FB = 1
+NVML_GPU_UTILIZATION_DOMAIN_VID = 2
+NVML_GPU_UTILIZATION_DOMAIN_BUS = 3
+
+
+class c_nvmlGpuDynamicPstatesUtilization_t(Structure):
+ _fields_ = [
+ ("bIsPresent", c_uint, 1),
+ ("percentage", c_uint),
+ ("incThreshold", c_uint),
+ ("decThreshold", c_uint),
+ ]
+
+
+class c_nvmlGpuDynamicPstatesInfo_t(Structure):
+ _fields_ = [
+ ("flags", c_uint),
+ (
+ "utilization",
+ c_nvmlGpuDynamicPstatesUtilization_t * NVML_MAX_GPU_UTILIZATIONS,
+ ),
+ ]
+
+
+NVML_MAX_THERMAL_SENSORS_PER_GPU = 3
+
+NVML_THERMAL_TARGET_NONE = 0
+NVML_THERMAL_TARGET_GPU = 1
+NVML_THERMAL_TARGET_MEMORY = 2
+NVML_THERMAL_TARGET_POWER_SUPPLY = 4
+NVML_THERMAL_TARGET_BOARD = 8
+NVML_THERMAL_TARGET_VCD_BOARD = 9
+NVML_THERMAL_TARGET_VCD_INLET = 10
+NVML_THERMAL_TARGET_VCD_OUTLET = 11
+NVML_THERMAL_TARGET_ALL = 15
+NVML_THERMAL_TARGET_UNKNOWN = -1
+
+NVML_THERMAL_CONTROLLER_NONE = 0
+NVML_THERMAL_CONTROLLER_GPU_INTERNAL = 1
+NVML_THERMAL_CONTROLLER_ADM1032 = 2
+NVML_THERMAL_CONTROLLER_ADT7461 = 3
+NVML_THERMAL_CONTROLLER_MAX6649 = 4
+NVML_THERMAL_CONTROLLER_MAX1617 = 5
+NVML_THERMAL_CONTROLLER_LM99 = 6
+NVML_THERMAL_CONTROLLER_LM89 = 7
+NVML_THERMAL_CONTROLLER_LM64 = 8
+NVML_THERMAL_CONTROLLER_G781 = 9
+NVML_THERMAL_CONTROLLER_ADT7473 = 10
+NVML_THERMAL_CONTROLLER_SBMAX6649 = 11
+NVML_THERMAL_CONTROLLER_VBIOSEVT = 12
+NVML_THERMAL_CONTROLLER_OS = 13
+NVML_THERMAL_CONTROLLER_NVSYSCON_CANOAS = 14
+NVML_THERMAL_CONTROLLER_NVSYSCON_E551 = 15
+NVML_THERMAL_CONTROLLER_MAX6649R = 16
+NVML_THERMAL_CONTROLLER_ADT7473S = 17
+NVML_THERMAL_CONTROLLER_UNKNOWN = -1
+
+
+class c_nvmlGpuThermalSensor_t(Structure):
+ _fields_ = [
+ ("controller", c_int),
+ ("defaultMinTemp", c_uint),
+ ("defaultMaxTemp", c_uint),
+ ("currentTemp", c_uint),
+ ("target", c_int),
+ ]
+
+
+class c_nvmlGpuThermalSettings_t(Structure):
+ _fields_ = [
+ ("count", c_uint),
+ ("sensor", c_nvmlGpuThermalSensor_t * NVML_MAX_THERMAL_SENSORS_PER_GPU),
+ ]
+
+
+class struct_c_nvmlComputeInstance_t(Structure):
+ pass # opaque handle
+
+
+c_nvmlComputeInstance_t = POINTER(struct_c_nvmlComputeInstance_t)
+
+
+class c_nvmlDeviceAttributes(Structure):
+ _fields_ = [
+ ("multiprocessorCount", c_uint),
+ ("sharedCopyEngineCount", c_uint),
+ ("sharedDecoderCount", c_uint),
+ ("sharedEncoderCount", c_uint),
+ ("sharedJpegCount", c_uint),
+ ("sharedOfaCount", c_uint),
+ ("gpuInstanceSliceCount", c_uint),
+ ("computeInstanceSliceCount", c_uint),
+ ("memorySizeMB", c_ulonglong),
+ ]
+
+
+class c_nvmlRowRemapperHistogramValues(Structure):
+ _fields_ = [
+ ("max", c_uint),
+ ("high", c_uint),
+ ("partial", c_uint),
+ ("low", c_uint),
+ ("none", c_uint),
+ ]
+
+
+## string/bytes conversion for ease of use
+def convertStrBytes(func):
+ """
+ In python 3, strings are unicode instead of bytes, and need to be converted for ctypes
+ Args from caller: (1, 'string', <__main__.c_nvmlDevice_t at 0xFFFFFFFF>)
+ Args passed to function: (1, b'string', <__main__.c_nvmlDevice_t at 0xFFFFFFFF)>
+ ----
+ Returned from function: b'returned string'
+ Returned to caller: 'returned string'
+ """
+
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ # encoding a str returns bytes in python 2 and 3
+ args = [arg.encode() if isinstance(arg, str) else arg for arg in args]
+ res = func(*args, **kwargs)
+ # In python 2, str and bytes are the same
+ # In python 3, str is unicode and should be decoded.
+ # Ctypes handles most conversions, this only effects c_char and char arrays.
+ if isinstance(res, bytes):
+ if isinstance(res, str):
+ return res
+ return res.decode()
+ return res
+
+ return wrapper
+ return func
+
+
+## C function wrappers ##
+def nvmlInitWithFlags(flags):
+ _LoadNvmlLibrary()
+
+ #
+ # Initialize the library
+ #
+ fn = _nvmlGetFunctionPointer("nvmlInitWithFlags")
+ ret = fn(flags)
+ _nvmlCheckReturn(ret)
+
+ # Atomically update refcount
+ global _nvmlLib_refcount
+ libLoadLock.acquire()
+ _nvmlLib_refcount += 1
+ libLoadLock.release()
+ return None
+
+
+def nvmlInit():
+ nvmlInitWithFlags(0)
+ return None
+
+
+def _LoadNvmlLibrary():
+ """
+ Load the library if it isn't loaded already
+ """
+ global nvmlLib
+
+ if nvmlLib is None:
+ # lock to ensure only one caller loads the library
+ libLoadLock.acquire()
+
+ try:
+ # ensure the library still isn't loaded
+ if nvmlLib is None:
+ if sys.platform[:3] == "win":
+ # cdecl calling convention
+ # First, check for nvml.dll in System32 first for DCH drivers.
+ # If nvml.dll is not found in System32, it should be in ProgramFiles
+ # load nvml.dll from %ProgramFiles%/NVIDIA Corporation/NVSMI/nvml.dll
+ search_paths = [
+ os.path.join(
+ os.getenv("WINDIR", "C:/Windows"), "System32/nvml.dll"
+ ),
+ os.path.join(
+ os.getenv("ProgramFiles", "C:/Program Files"),
+ "NVIDIA Corporation/NVSMI/nvml.dll",
+ ),
+ ]
+ # Finally, it may be overridden by an environment variable
+ nvml_path = os.getenv("NVML_DLL_PATH")
+ if nvml_path is not None:
+ search_paths.append(nvml_path)
+ for dll_path in search_paths:
+ try:
+ nvmlLib = CDLL(dll_path)
+ except OSError as ose:
+ continue
+ break
+ else:
+ # assume linux
+ try:
+ nvmlLib = CDLL("libnvidia-ml.so.1")
+ except OSError as ose:
+ _nvmlCheckReturn(NVML_ERROR_LIBRARY_NOT_FOUND)
+ if nvmlLib is None:
+ _nvmlCheckReturn(NVML_ERROR_LIBRARY_NOT_FOUND)
+ finally:
+ # lock is always freed
+ libLoadLock.release()
+
+
+def nvmlShutdown():
+ #
+ # Leave the library loaded, but shutdown the interface
+ #
+ fn = _nvmlGetFunctionPointer("nvmlShutdown")
+ ret = fn()
+ _nvmlCheckReturn(ret)
+
+ # Atomically update refcount
+ global _nvmlLib_refcount
+ libLoadLock.acquire()
+ if 0 < _nvmlLib_refcount:
+ _nvmlLib_refcount -= 1
+ libLoadLock.release()
+ return None
+
+
+# Added in 2.285
+@convertStrBytes
+def nvmlErrorString(result):
+ fn = _nvmlGetFunctionPointer("nvmlErrorString")
+ fn.restype = c_char_p # otherwise return is an int
+ ret = fn(result)
+ return ret
+
+
+# Added in 2.285
+@convertStrBytes
+def nvmlSystemGetNVMLVersion():
+ c_version = create_string_buffer(NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlSystemGetNVMLVersion")
+ ret = fn(c_version, c_uint(NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE))
+ _nvmlCheckReturn(ret)
+ return c_version.value
+
+
+def nvmlSystemGetCudaDriverVersion():
+ c_cuda_version = c_int()
+ fn = _nvmlGetFunctionPointer("nvmlSystemGetCudaDriverVersion")
+ ret = fn(byref(c_cuda_version))
+ _nvmlCheckReturn(ret)
+ return c_cuda_version.value
+
+
+def nvmlSystemGetCudaDriverVersion_v2():
+ c_cuda_version = c_int()
+ fn = _nvmlGetFunctionPointer("nvmlSystemGetCudaDriverVersion_v2")
+ ret = fn(byref(c_cuda_version))
+ _nvmlCheckReturn(ret)
+ return c_cuda_version.value
+
+
+# Added in 2.285
+@convertStrBytes
+def nvmlSystemGetProcessName(pid):
+ c_name = create_string_buffer(1024)
+ fn = _nvmlGetFunctionPointer("nvmlSystemGetProcessName")
+ ret = fn(c_uint(pid), c_name, c_uint(1024))
+ _nvmlCheckReturn(ret)
+ return c_name.value
+
+
+@convertStrBytes
+def nvmlSystemGetDriverVersion():
+ c_version = create_string_buffer(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlSystemGetDriverVersion")
+ ret = fn(c_version, c_uint(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE))
+ _nvmlCheckReturn(ret)
+ return c_version.value
+
+
+# Added in 2.285
+def nvmlSystemGetHicVersion():
+ c_count = c_uint(0)
+ hics = None
+ fn = _nvmlGetFunctionPointer("nvmlSystemGetHicVersion")
+
+ # get the count
+ ret = fn(byref(c_count), None)
+
+ # this should only fail with insufficient size
+ if (ret != NVML_SUCCESS) and (ret != NVML_ERROR_INSUFFICIENT_SIZE):
+ raise NVMLError(ret)
+
+ # If there are no hics
+ if c_count.value == 0:
+ return []
+
+ hic_array = c_nvmlHwbcEntry_t * c_count.value
+ hics = hic_array()
+ ret = fn(byref(c_count), hics)
+ _nvmlCheckReturn(ret)
+ return hics
+
+
+## Unit get functions
+def nvmlUnitGetCount():
+ c_count = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlUnitGetCount")
+ ret = fn(byref(c_count))
+ _nvmlCheckReturn(ret)
+ return c_count.value
+
+
+def nvmlUnitGetHandleByIndex(index):
+ c_index = c_uint(index)
+ unit = c_nvmlUnit_t()
+ fn = _nvmlGetFunctionPointer("nvmlUnitGetHandleByIndex")
+ ret = fn(c_index, byref(unit))
+ _nvmlCheckReturn(ret)
+ return unit
+
+
+def nvmlUnitGetUnitInfo(unit):
+ c_info = c_nvmlUnitInfo_t()
+ fn = _nvmlGetFunctionPointer("nvmlUnitGetUnitInfo")
+ ret = fn(unit, byref(c_info))
+ _nvmlCheckReturn(ret)
+ return c_info
+
+
+def nvmlUnitGetLedState(unit):
+ c_state = c_nvmlLedState_t()
+ fn = _nvmlGetFunctionPointer("nvmlUnitGetLedState")
+ ret = fn(unit, byref(c_state))
+ _nvmlCheckReturn(ret)
+ return c_state
+
+
+def nvmlUnitGetPsuInfo(unit):
+ c_info = c_nvmlPSUInfo_t()
+ fn = _nvmlGetFunctionPointer("nvmlUnitGetPsuInfo")
+ ret = fn(unit, byref(c_info))
+ _nvmlCheckReturn(ret)
+ return c_info
+
+
+def nvmlUnitGetTemperature(unit, type):
+ c_temp = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlUnitGetTemperature")
+ ret = fn(unit, c_uint(type), byref(c_temp))
+ _nvmlCheckReturn(ret)
+ return c_temp.value
+
+
+def nvmlUnitGetFanSpeedInfo(unit):
+ c_speeds = c_nvmlUnitFanSpeeds_t()
+ fn = _nvmlGetFunctionPointer("nvmlUnitGetFanSpeedInfo")
+ ret = fn(unit, byref(c_speeds))
+ _nvmlCheckReturn(ret)
+ return c_speeds
+
+
+# added to API
+def nvmlUnitGetDeviceCount(unit):
+ c_count = c_uint(0)
+ # query the unit to determine device count
+ fn = _nvmlGetFunctionPointer("nvmlUnitGetDevices")
+ ret = fn(unit, byref(c_count), None)
+ if ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ ret = NVML_SUCCESS
+ _nvmlCheckReturn(ret)
+ return c_count.value
+
+
+def nvmlUnitGetDevices(unit):
+ c_count = c_uint(nvmlUnitGetDeviceCount(unit))
+ device_array = c_nvmlDevice_t * c_count.value
+ c_devices = device_array()
+ fn = _nvmlGetFunctionPointer("nvmlUnitGetDevices")
+ ret = fn(unit, byref(c_count), c_devices)
+ _nvmlCheckReturn(ret)
+ return c_devices
+
+
+## Device get functions
+def nvmlDeviceGetCount():
+ c_count = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetCount_v2")
+ ret = fn(byref(c_count))
+ _nvmlCheckReturn(ret)
+ return c_count.value
+
+
+def nvmlDeviceGetHandleByIndex(index):
+ c_index = c_uint(index)
+ device = c_nvmlDevice_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleByIndex_v2")
+ ret = fn(c_index, byref(device))
+ _nvmlCheckReturn(ret)
+ return device
+
+
+@convertStrBytes
+def nvmlDeviceGetHandleBySerial(serial):
+ c_serial = c_char_p(serial)
+ device = c_nvmlDevice_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleBySerial")
+ ret = fn(c_serial, byref(device))
+ _nvmlCheckReturn(ret)
+ return device
+
+
+@convertStrBytes
+def nvmlDeviceGetHandleByUUID(uuid):
+ c_uuid = c_char_p(uuid)
+ device = c_nvmlDevice_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleByUUID")
+ ret = fn(c_uuid, byref(device))
+ _nvmlCheckReturn(ret)
+ return device
+
+
+@convertStrBytes
+def nvmlDeviceGetHandleByPciBusId(pciBusId):
+ c_busId = c_char_p(pciBusId)
+ device = c_nvmlDevice_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleByPciBusId_v2")
+ ret = fn(c_busId, byref(device))
+ _nvmlCheckReturn(ret)
+ return device
+
+
+@convertStrBytes
+def nvmlDeviceGetName(handle):
+ c_name = create_string_buffer(NVML_DEVICE_NAME_V2_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetName")
+ ret = fn(handle, c_name, c_uint(NVML_DEVICE_NAME_V2_BUFFER_SIZE))
+ _nvmlCheckReturn(ret)
+ return c_name.value
+
+
+def nvmlDeviceGetBoardId(handle):
+ c_id = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetBoardId")
+ ret = fn(handle, byref(c_id))
+ _nvmlCheckReturn(ret)
+ return c_id.value
+
+
+def nvmlDeviceGetMultiGpuBoard(handle):
+ c_multiGpu = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMultiGpuBoard")
+ ret = fn(handle, byref(c_multiGpu))
+ _nvmlCheckReturn(ret)
+ return c_multiGpu.value
+
+
+def nvmlDeviceGetBrand(handle):
+ c_type = _nvmlBrandType_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetBrand")
+ ret = fn(handle, byref(c_type))
+ _nvmlCheckReturn(ret)
+ return c_type.value
+
+
+@convertStrBytes
+def nvmlDeviceGetBoardPartNumber(handle):
+ c_part_number = create_string_buffer(NVML_DEVICE_PART_NUMBER_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetBoardPartNumber")
+ ret = fn(handle, c_part_number, c_uint(NVML_DEVICE_PART_NUMBER_BUFFER_SIZE))
+ _nvmlCheckReturn(ret)
+ return c_part_number.value
+
+
+@convertStrBytes
+def nvmlDeviceGetSerial(handle):
+ c_serial = create_string_buffer(NVML_DEVICE_SERIAL_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetSerial")
+ ret = fn(handle, c_serial, c_uint(NVML_DEVICE_SERIAL_BUFFER_SIZE))
+ _nvmlCheckReturn(ret)
+ return c_serial.value
+
+
+def nvmlDeviceGetMemoryAffinity(handle, nodeSetSize, scope):
+ affinity_array = c_ulonglong * nodeSetSize
+ c_affinity = affinity_array()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryAffinity")
+ ret = fn(handle, nodeSetSize, byref(c_affinity), _nvmlAffinityScope_t(scope))
+ _nvmlCheckReturn(ret)
+ return c_affinity
+
+
+def nvmlDeviceGetCpuAffinityWithinScope(handle, cpuSetSize, scope):
+ affinity_array = c_ulonglong * cpuSetSize
+ c_affinity = affinity_array()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetCpuAffinityWithinScope")
+ ret = fn(handle, cpuSetSize, byref(c_affinity), _nvmlAffinityScope_t(scope))
+ _nvmlCheckReturn(ret)
+ return c_affinity
+
+
+def nvmlDeviceGetCpuAffinity(handle, cpuSetSize):
+ affinity_array = c_ulonglong * cpuSetSize
+ c_affinity = affinity_array()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetCpuAffinity")
+ ret = fn(handle, cpuSetSize, byref(c_affinity))
+ _nvmlCheckReturn(ret)
+ return c_affinity
+
+
+def nvmlDeviceSetCpuAffinity(handle):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetCpuAffinity")
+ ret = fn(handle)
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceClearCpuAffinity(handle):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceClearCpuAffinity")
+ ret = fn(handle)
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceGetMinorNumber(handle):
+ c_minor_number = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMinorNumber")
+ ret = fn(handle, byref(c_minor_number))
+ _nvmlCheckReturn(ret)
+ return c_minor_number.value
+
+
+@convertStrBytes
+def nvmlDeviceGetUUID(handle):
+ c_uuid = create_string_buffer(NVML_DEVICE_UUID_V2_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetUUID")
+ ret = fn(handle, c_uuid, c_uint(NVML_DEVICE_UUID_V2_BUFFER_SIZE))
+ _nvmlCheckReturn(ret)
+ return c_uuid.value
+
+
+@convertStrBytes
+def nvmlDeviceGetInforomVersion(handle, infoRomObject):
+ c_version = create_string_buffer(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetInforomVersion")
+ ret = fn(
+ handle,
+ _nvmlInforomObject_t(infoRomObject),
+ c_version,
+ c_uint(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE),
+ )
+ _nvmlCheckReturn(ret)
+ return c_version.value
+
+
+# Added in 4.304
+@convertStrBytes
+def nvmlDeviceGetInforomImageVersion(handle):
+ c_version = create_string_buffer(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetInforomImageVersion")
+ ret = fn(handle, c_version, c_uint(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE))
+ _nvmlCheckReturn(ret)
+ return c_version.value
+
+
+# Added in 4.304
+def nvmlDeviceGetInforomConfigurationChecksum(handle):
+ c_checksum = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetInforomConfigurationChecksum")
+ ret = fn(handle, byref(c_checksum))
+ _nvmlCheckReturn(ret)
+ return c_checksum.value
+
+
+# Added in 4.304
+def nvmlDeviceValidateInforom(handle):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceValidateInforom")
+ ret = fn(handle)
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceGetDisplayMode(handle):
+ c_mode = _nvmlEnableState_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetDisplayMode")
+ ret = fn(handle, byref(c_mode))
+ _nvmlCheckReturn(ret)
+ return c_mode.value
+
+
+def nvmlDeviceGetDisplayActive(handle):
+ c_mode = _nvmlEnableState_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetDisplayActive")
+ ret = fn(handle, byref(c_mode))
+ _nvmlCheckReturn(ret)
+ return c_mode.value
+
+
+def nvmlDeviceGetPersistenceMode(handle):
+ c_state = _nvmlEnableState_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetPersistenceMode")
+ ret = fn(handle, byref(c_state))
+ _nvmlCheckReturn(ret)
+ return c_state.value
+
+
+def nvmlDeviceGetPciInfo_v3(handle):
+ c_info = nvmlPciInfo_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetPciInfo_v3")
+ ret = fn(handle, byref(c_info))
+ _nvmlCheckReturn(ret)
+ return c_info
+
+
+def nvmlDeviceGetPciInfo(handle):
+ return nvmlDeviceGetPciInfo_v3(handle)
+
+
+def nvmlDeviceGetClockInfo(handle, type):
+ c_clock = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetClockInfo")
+ ret = fn(handle, _nvmlClockType_t(type), byref(c_clock))
+ _nvmlCheckReturn(ret)
+ return c_clock.value
+
+
+# Added in 2.285
+def nvmlDeviceGetMaxClockInfo(handle, type):
+ c_clock = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxClockInfo")
+ ret = fn(handle, _nvmlClockType_t(type), byref(c_clock))
+ _nvmlCheckReturn(ret)
+ return c_clock.value
+
+
+# Added in 4.304
+def nvmlDeviceGetApplicationsClock(handle, type):
+ c_clock = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetApplicationsClock")
+ ret = fn(handle, _nvmlClockType_t(type), byref(c_clock))
+ _nvmlCheckReturn(ret)
+ return c_clock.value
+
+
+def nvmlDeviceGetMaxCustomerBoostClock(handle, type):
+ c_clock = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxCustomerBoostClock")
+ ret = fn(handle, _nvmlClockType_t(type), byref(c_clock))
+ _nvmlCheckReturn(ret)
+ return c_clock.value
+
+
+def nvmlDeviceGetClock(handle, type, id):
+ c_clock = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetClock")
+ ret = fn(handle, _nvmlClockType_t(type), _nvmlClockId_t(id), byref(c_clock))
+ _nvmlCheckReturn(ret)
+ return c_clock.value
+
+
+# Added in 5.319
+def nvmlDeviceGetDefaultApplicationsClock(handle, type):
+ c_clock = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetDefaultApplicationsClock")
+ ret = fn(handle, _nvmlClockType_t(type), byref(c_clock))
+ _nvmlCheckReturn(ret)
+ return c_clock.value
+
+
+# Added in 4.304
+def nvmlDeviceGetSupportedMemoryClocks(handle):
+ # first call to get the size
+ c_count = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedMemoryClocks")
+ ret = fn(handle, byref(c_count), None)
+
+ if ret == NVML_SUCCESS:
+ # special case, no clocks
+ return []
+ elif ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ # typical case
+ clocks_array = c_uint * c_count.value
+ c_clocks = clocks_array()
+
+ # make the call again
+ ret = fn(handle, byref(c_count), c_clocks)
+ _nvmlCheckReturn(ret)
+
+ procs = []
+ for i in range(c_count.value):
+ procs.append(c_clocks[i])
+
+ return procs
+ else:
+ # error case
+ raise NVMLError(ret)
+
+
+# Added in 4.304
+def nvmlDeviceGetSupportedGraphicsClocks(handle, memoryClockMHz):
+ # first call to get the size
+ c_count = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedGraphicsClocks")
+ ret = fn(handle, c_uint(memoryClockMHz), byref(c_count), None)
+
+ if ret == NVML_SUCCESS:
+ # special case, no clocks
+ return []
+ elif ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ # typical case
+ clocks_array = c_uint * c_count.value
+ c_clocks = clocks_array()
+
+ # make the call again
+ ret = fn(handle, c_uint(memoryClockMHz), byref(c_count), c_clocks)
+ _nvmlCheckReturn(ret)
+
+ procs = []
+ for i in range(c_count.value):
+ procs.append(c_clocks[i])
+
+ return procs
+ else:
+ # error case
+ raise NVMLError(ret)
+
+
+def nvmlDeviceGetFanSpeed(handle):
+ c_speed = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetFanSpeed")
+ ret = fn(handle, byref(c_speed))
+ _nvmlCheckReturn(ret)
+ return c_speed.value
+
+
+def nvmlDeviceGetFanSpeed_v2(handle, fan):
+ c_speed = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetFanSpeed_v2")
+ ret = fn(handle, fan, byref(c_speed))
+ _nvmlCheckReturn(ret)
+ return c_speed.value
+
+
+def nvmlDeviceGetNumFans(device):
+ c_numFans = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetNumFans")
+ ret = fn(device, byref(c_numFans))
+ _nvmlCheckReturn(ret)
+ return c_numFans.value
+
+
+def nvmlDeviceSetDefaultFanSpeed_v2(handle, index):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetDefaultFanSpeed_v2")
+ ret = fn(handle, index)
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlDeviceGetMinMaxFanSpeed(handle, minSpeed, maxSpeed):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMinMaxFanSpeed")
+ ret = fn(handle, minSpeed, maxSpeed)
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlDeviceGetTemperature(handle, sensor):
+ c_temp = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetTemperature")
+ ret = fn(handle, _nvmlTemperatureSensors_t(sensor), byref(c_temp))
+ _nvmlCheckReturn(ret)
+ return c_temp.value
+
+
+def nvmlDeviceGetTemperatureThreshold(handle, threshold):
+ c_temp = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetTemperatureThreshold")
+ ret = fn(handle, _nvmlTemperatureThresholds_t(threshold), byref(c_temp))
+ _nvmlCheckReturn(ret)
+ return c_temp.value
+
+
+def nvmlDeviceSetTemperatureThreshold(handle, threshold):
+ c_temp = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetTemperatureThreshold")
+ ret = fn(handle, _nvmlTemperatureThresholds_t(threshold), byref(c_temp))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+# DEPRECATED use nvmlDeviceGetPerformanceState
+def nvmlDeviceGetPowerState(handle):
+ c_pstate = _nvmlPstates_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerState")
+ ret = fn(handle, byref(c_pstate))
+ _nvmlCheckReturn(ret)
+ return c_pstate.value
+
+
+def nvmlDeviceGetPerformanceState(handle):
+ c_pstate = _nvmlPstates_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetPerformanceState")
+ ret = fn(handle, byref(c_pstate))
+ _nvmlCheckReturn(ret)
+ return c_pstate.value
+
+
+def nvmlDeviceGetPowerManagementMode(handle):
+ c_pcapMode = _nvmlEnableState_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementMode")
+ ret = fn(handle, byref(c_pcapMode))
+ _nvmlCheckReturn(ret)
+ return c_pcapMode.value
+
+
+def nvmlDeviceGetPowerManagementLimit(handle):
+ c_limit = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementLimit")
+ ret = fn(handle, byref(c_limit))
+ _nvmlCheckReturn(ret)
+ return c_limit.value
+
+
+# Added in 4.304
+def nvmlDeviceGetPowerManagementLimitConstraints(handle):
+ c_minLimit = c_uint()
+ c_maxLimit = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementLimitConstraints")
+ ret = fn(handle, byref(c_minLimit), byref(c_maxLimit))
+ _nvmlCheckReturn(ret)
+ return [c_minLimit.value, c_maxLimit.value]
+
+
+# Added in 4.304
+def nvmlDeviceGetPowerManagementDefaultLimit(handle):
+ c_limit = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementDefaultLimit")
+ ret = fn(handle, byref(c_limit))
+ _nvmlCheckReturn(ret)
+ return c_limit.value
+
+
+# Added in 331
+def nvmlDeviceGetEnforcedPowerLimit(handle):
+ c_limit = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetEnforcedPowerLimit")
+ ret = fn(handle, byref(c_limit))
+ _nvmlCheckReturn(ret)
+ return c_limit.value
+
+
+def nvmlDeviceGetPowerUsage(handle):
+ c_watts = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerUsage")
+ ret = fn(handle, byref(c_watts))
+ _nvmlCheckReturn(ret)
+ return c_watts.value
+
+
+def nvmlDeviceGetTotalEnergyConsumption(handle):
+ c_millijoules = c_uint64()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetTotalEnergyConsumption")
+ ret = fn(handle, byref(c_millijoules))
+ _nvmlCheckReturn(ret)
+ return c_millijoules.value
+
+
+# Added in 4.304
+def nvmlDeviceGetGpuOperationMode(handle):
+ c_currState = _nvmlGpuOperationMode_t()
+ c_pendingState = _nvmlGpuOperationMode_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuOperationMode")
+ ret = fn(handle, byref(c_currState), byref(c_pendingState))
+ _nvmlCheckReturn(ret)
+ return [c_currState.value, c_pendingState.value]
+
+
+# Added in 4.304
+def nvmlDeviceGetCurrentGpuOperationMode(handle):
+ return nvmlDeviceGetGpuOperationMode(handle)[0]
+
+
+# Added in 4.304
+def nvmlDeviceGetPendingGpuOperationMode(handle):
+ return nvmlDeviceGetGpuOperationMode(handle)[1]
+
+
+def nvmlDeviceGetMemoryInfo(handle, version=None):
+ if not version:
+ c_memory = c_nvmlMemory_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryInfo")
+ else:
+ c_memory = c_nvmlMemory_v2_t()
+ c_memory.version = version
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryInfo_v2")
+ ret = fn(handle, byref(c_memory))
+ _nvmlCheckReturn(ret)
+ return c_memory
+
+
+def nvmlDeviceGetBAR1MemoryInfo(handle):
+ c_bar1_memory = c_nvmlBAR1Memory_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetBAR1MemoryInfo")
+ ret = fn(handle, byref(c_bar1_memory))
+ _nvmlCheckReturn(ret)
+ return c_bar1_memory
+
+
+def nvmlDeviceGetComputeMode(handle):
+ c_mode = _nvmlComputeMode_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeMode")
+ ret = fn(handle, byref(c_mode))
+ _nvmlCheckReturn(ret)
+ return c_mode.value
+
+
+def nvmlDeviceGetCudaComputeCapability(handle):
+ c_major = c_int()
+ c_minor = c_int()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetCudaComputeCapability")
+ ret = fn(handle, byref(c_major), byref(c_minor))
+ _nvmlCheckReturn(ret)
+ return (c_major.value, c_minor.value)
+
+
+def nvmlDeviceGetEccMode(handle):
+ c_currState = _nvmlEnableState_t()
+ c_pendingState = _nvmlEnableState_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetEccMode")
+ ret = fn(handle, byref(c_currState), byref(c_pendingState))
+ _nvmlCheckReturn(ret)
+ return [c_currState.value, c_pendingState.value]
+
+
+# added to API
+def nvmlDeviceGetCurrentEccMode(handle):
+ return nvmlDeviceGetEccMode(handle)[0]
+
+
+# added to API
+def nvmlDeviceGetPendingEccMode(handle):
+ return nvmlDeviceGetEccMode(handle)[1]
+
+
+def nvmlDeviceGetDefaultEccMode(handle):
+ c_defaultState = _nvmlEnableState_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetDefaultEccMode")
+ ret = fn(handle, byref(c_defaultState))
+ _nvmlCheckReturn(ret)
+ return [c_defaultState.value]
+
+
+def nvmlDeviceGetTotalEccErrors(handle, errorType, counterType):
+ c_count = c_ulonglong()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetTotalEccErrors")
+ ret = fn(
+ handle,
+ _nvmlMemoryErrorType_t(errorType),
+ _nvmlEccCounterType_t(counterType),
+ byref(c_count),
+ )
+ _nvmlCheckReturn(ret)
+ return c_count.value
+
+
+# This is deprecated, instead use nvmlDeviceGetMemoryErrorCounter
+def nvmlDeviceGetDetailedEccErrors(handle, errorType, counterType):
+ c_counts = c_nvmlEccErrorCounts_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetDetailedEccErrors")
+ ret = fn(
+ handle,
+ _nvmlMemoryErrorType_t(errorType),
+ _nvmlEccCounterType_t(counterType),
+ byref(c_counts),
+ )
+ _nvmlCheckReturn(ret)
+ return c_counts
+
+
+# Added in 4.304
+def nvmlDeviceGetMemoryErrorCounter(handle, errorType, counterType, locationType):
+ c_count = c_ulonglong()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryErrorCounter")
+ ret = fn(
+ handle,
+ _nvmlMemoryErrorType_t(errorType),
+ _nvmlEccCounterType_t(counterType),
+ _nvmlMemoryLocation_t(locationType),
+ byref(c_count),
+ )
+ _nvmlCheckReturn(ret)
+ return c_count.value
+
+
+def nvmlDeviceGetUtilizationRates(handle):
+ c_util = c_nvmlUtilization_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetUtilizationRates")
+ ret = fn(handle, byref(c_util))
+ _nvmlCheckReturn(ret)
+ return c_util
+
+
+def nvmlDeviceGetEncoderUtilization(handle):
+ c_util = c_uint()
+ c_samplingPeriod = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderUtilization")
+ ret = fn(handle, byref(c_util), byref(c_samplingPeriod))
+ _nvmlCheckReturn(ret)
+ return [c_util.value, c_samplingPeriod.value]
+
+
+def nvmlDeviceGetDecoderUtilization(handle):
+ c_util = c_uint()
+ c_samplingPeriod = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetDecoderUtilization")
+ ret = fn(handle, byref(c_util), byref(c_samplingPeriod))
+ _nvmlCheckReturn(ret)
+ return [c_util.value, c_samplingPeriod.value]
+
+
+def nvmlDeviceGetPcieReplayCounter(handle):
+ c_replay = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieReplayCounter")
+ ret = fn(handle, byref(c_replay))
+ _nvmlCheckReturn(ret)
+ return c_replay.value
+
+
+def nvmlDeviceGetDriverModel(handle):
+ c_currModel = _nvmlDriverModel_t()
+ c_pendingModel = _nvmlDriverModel_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetDriverModel")
+ ret = fn(handle, byref(c_currModel), byref(c_pendingModel))
+ _nvmlCheckReturn(ret)
+ return [c_currModel.value, c_pendingModel.value]
+
+
+# added to API
+def nvmlDeviceGetCurrentDriverModel(handle):
+ return nvmlDeviceGetDriverModel(handle)[0]
+
+
+# added to API
+def nvmlDeviceGetPendingDriverModel(handle):
+ return nvmlDeviceGetDriverModel(handle)[1]
+
+
+# Added in 2.285
+@convertStrBytes
+def nvmlDeviceGetVbiosVersion(handle):
+ c_version = create_string_buffer(NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetVbiosVersion")
+ ret = fn(handle, c_version, c_uint(NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE))
+ _nvmlCheckReturn(ret)
+ return c_version.value
+
+
+# Added in 2.285
+def nvmlDeviceGetComputeRunningProcesses_v3(handle):
+ # first call to get the size
+ c_count = c_uint(0)
+ fn = None
+ for suffix in ("_v3", "_v2", ""):
+ try:
+ fn = _nvmlGetFunctionPointer(
+ f"nvmlDeviceGetComputeRunningProcesses{suffix}"
+ )
+ break
+ except NVMLError:
+ pass
+ if fn is None:
+ raise NVMLError(NVML_ERROR_FUNCTION_NOT_FOUND)
+ ret = fn(handle, byref(c_count), None)
+
+ if ret == NVML_SUCCESS:
+ # special case, no running processes
+ return []
+ elif ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ # typical case
+ # oversize the array incase more processes are created
+ c_count.value = c_count.value * 2 + 5
+ proc_array = c_nvmlProcessInfo_t * c_count.value
+ c_procs = proc_array()
+
+ # make the call again
+ ret = fn(handle, byref(c_count), c_procs)
+ _nvmlCheckReturn(ret)
+
+ procs = []
+ for i in range(c_count.value):
+ # use an alternative struct for this object
+ obj = nvmlStructToFriendlyObject(c_procs[i])
+ if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value:
+ # special case for WDDM on Windows, see comment above
+ obj.usedGpuMemory = None
+ procs.append(obj)
+
+ return procs
+ else:
+ # error case
+ raise NVMLError(ret)
+
+
+def nvmlDeviceGetComputeRunningProcesses(handle):
+ return nvmlDeviceGetComputeRunningProcesses_v3(handle)
+
+
+def nvmlDeviceGetGraphicsRunningProcesses_v3(handle):
+ # first call to get the size
+ c_count = c_uint(0)
+ fn = None
+ for suffix in ("_v3", "_v2", ""):
+ try:
+ fn = _nvmlGetFunctionPointer(
+ f"nvmlDeviceGetGraphicsRunningProcesses{suffix}"
+ )
+ break
+ except NVMLError:
+ pass
+ if fn is None:
+ raise NVMLError(NVML_ERROR_FUNCTION_NOT_FOUND)
+ ret = fn(handle, byref(c_count), None)
+
+ if ret == NVML_SUCCESS:
+ # special case, no running processes
+ return []
+ elif ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ # typical case
+ # oversize the array incase more processes are created
+ c_count.value = c_count.value * 2 + 5
+ proc_array = c_nvmlProcessInfo_t * c_count.value
+ c_procs = proc_array()
+
+ # make the call again
+ ret = fn(handle, byref(c_count), c_procs)
+ _nvmlCheckReturn(ret)
+
+ procs = []
+ for i in range(c_count.value):
+ # use an alternative struct for this object
+ obj = nvmlStructToFriendlyObject(c_procs[i])
+ if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value:
+ # special case for WDDM on Windows, see comment above
+ obj.usedGpuMemory = None
+ procs.append(obj)
+
+ return procs
+ else:
+ # error case
+ raise NVMLError(ret)
+
+
+def nvmlDeviceGetGraphicsRunningProcesses(handle):
+ return nvmlDeviceGetGraphicsRunningProcesses_v3(handle)
+
+
+def nvmlDeviceGetMPSComputeRunningProcesses(handle):
+ return nvmlDeviceGetMPSComputeRunningProcesses_v3(handle)
+
+
+def nvmlDeviceGetMPSComputeRunningProcesses_v3(handle):
+ # first call to get the size
+ c_count = c_uint(0)
+ fn = None
+ for suffix in ("_v3", "_v2", ""):
+ try:
+ fn = _nvmlGetFunctionPointer(
+ f"nvmlDeviceGetMPSComputeRunningProcesses{suffix}"
+ )
+ break
+ except NVMLError:
+ pass
+ if fn is None:
+ raise NVMLError(NVML_ERROR_FUNCTION_NOT_FOUND)
+
+ ret = fn(handle, byref(c_count), None)
+
+ if ret == NVML_SUCCESS:
+ # special case, no running processes
+ return []
+ elif ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ # typical case
+ # oversize the array incase more processes are created
+ c_count.value = c_count.value * 2 + 5
+ proc_array = c_nvmlProcessInfo_t * c_count.value
+ c_procs = proc_array()
+
+ # make the call again
+ ret = fn(handle, byref(c_count), c_procs)
+ _nvmlCheckReturn(ret)
+
+ procs = []
+ for i in range(c_count.value):
+ # use an alternative struct for this object
+ obj = nvmlStructToFriendlyObject(c_procs[i])
+ if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value:
+ # special case for WDDM on Windows, see comment above
+ obj.usedGpuMemory = None
+ procs.append(obj)
+
+ return procs
+ else:
+ # error case
+ raise NVMLError(ret)
+
+
+def nvmlDeviceGetAutoBoostedClocksEnabled(handle):
+ c_isEnabled = _nvmlEnableState_t()
+ c_defaultIsEnabled = _nvmlEnableState_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetAutoBoostedClocksEnabled")
+ ret = fn(handle, byref(c_isEnabled), byref(c_defaultIsEnabled))
+ _nvmlCheckReturn(ret)
+ return [c_isEnabled.value, c_defaultIsEnabled.value]
+ # Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks
+
+
+## Set functions
+def nvmlUnitSetLedState(unit, color):
+ fn = _nvmlGetFunctionPointer("nvmlUnitSetLedState")
+ ret = fn(unit, _nvmlLedColor_t(color))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceSetPersistenceMode(handle, mode):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetPersistenceMode")
+ ret = fn(handle, _nvmlEnableState_t(mode))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceSetComputeMode(handle, mode):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetComputeMode")
+ ret = fn(handle, _nvmlComputeMode_t(mode))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceSetEccMode(handle, mode):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetEccMode")
+ ret = fn(handle, _nvmlEnableState_t(mode))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceClearEccErrorCounts(handle, counterType):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceClearEccErrorCounts")
+ ret = fn(handle, _nvmlEccCounterType_t(counterType))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceSetDriverModel(handle, model):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetDriverModel")
+ ret = fn(handle, _nvmlDriverModel_t(model))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceSetAutoBoostedClocksEnabled(handle, enabled):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetAutoBoostedClocksEnabled")
+ ret = fn(handle, _nvmlEnableState_t(enabled))
+ _nvmlCheckReturn(ret)
+ return None
+ # Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks
+
+
+def nvmlDeviceSetDefaultAutoBoostedClocksEnabled(handle, enabled, flags):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetDefaultAutoBoostedClocksEnabled")
+ ret = fn(handle, _nvmlEnableState_t(enabled), c_uint(flags))
+ _nvmlCheckReturn(ret)
+ return None
+ # Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks
+
+
+def nvmlDeviceSetGpuLockedClocks(handle, minGpuClockMHz, maxGpuClockMHz):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetGpuLockedClocks")
+ ret = fn(handle, c_uint(minGpuClockMHz), c_uint(maxGpuClockMHz))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceResetGpuLockedClocks(handle):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceResetGpuLockedClocks")
+ ret = fn(handle)
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceSetMemoryLockedClocks(handle, minMemClockMHz, maxMemClockMHz):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetMemoryLockedClocks")
+ ret = fn(handle, c_uint(minMemClockMHz), c_uint(maxMemClockMHz))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceResetMemoryLockedClocks(handle):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceResetMemoryLockedClocks")
+ ret = fn(handle)
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceGetClkMonStatus(handle, c_clkMonInfo):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetClkMonStatus")
+ ret = fn(handle, c_clkMonInfo)
+ return ret
+
+
+# Added in 4.304
+def nvmlDeviceSetApplicationsClocks(handle, maxMemClockMHz, maxGraphicsClockMHz):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetApplicationsClocks")
+ ret = fn(handle, c_uint(maxMemClockMHz), c_uint(maxGraphicsClockMHz))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+# Added in 4.304
+def nvmlDeviceResetApplicationsClocks(handle):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceResetApplicationsClocks")
+ ret = fn(handle)
+ _nvmlCheckReturn(ret)
+ return None
+
+
+# Added in 4.304
+def nvmlDeviceSetPowerManagementLimit(handle, limit):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetPowerManagementLimit")
+ ret = fn(handle, c_uint(limit))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+# Added in 4.304
+def nvmlDeviceSetGpuOperationMode(handle, mode):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetGpuOperationMode")
+ ret = fn(handle, _nvmlGpuOperationMode_t(mode))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+# Added in 2.285
+def nvmlEventSetCreate():
+ fn = _nvmlGetFunctionPointer("nvmlEventSetCreate")
+ eventSet = c_nvmlEventSet_t()
+ ret = fn(byref(eventSet))
+ _nvmlCheckReturn(ret)
+ return eventSet
+
+
+# Added in 2.285
+def nvmlDeviceRegisterEvents(handle, eventTypes, eventSet):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceRegisterEvents")
+ ret = fn(handle, c_ulonglong(eventTypes), eventSet)
+ _nvmlCheckReturn(ret)
+ return None
+
+
+# Added in 2.285
+def nvmlDeviceGetSupportedEventTypes(handle):
+ c_eventTypes = c_ulonglong()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedEventTypes")
+ ret = fn(handle, byref(c_eventTypes))
+ _nvmlCheckReturn(ret)
+ return c_eventTypes.value
+
+
+# raises NVML_ERROR_TIMEOUT exception on timeout
+def nvmlEventSetWait_v2(eventSet, timeoutms):
+ fn = _nvmlGetFunctionPointer("nvmlEventSetWait_v2")
+ data = c_nvmlEventData_t()
+ ret = fn(eventSet, byref(data), c_uint(timeoutms))
+ _nvmlCheckReturn(ret)
+ return data
+
+
+def nvmlEventSetWait(eventSet, timeoutms):
+ return nvmlEventSetWait_v2(eventSet, timeoutms)
+
+
+# Added in 2.285
+def nvmlEventSetFree(eventSet):
+ fn = _nvmlGetFunctionPointer("nvmlEventSetFree")
+ ret = fn(eventSet)
+ _nvmlCheckReturn(ret)
+ return None
+
+
+# Added in 3.295
+def nvmlDeviceOnSameBoard(handle1, handle2):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceOnSameBoard")
+ onSameBoard = c_int()
+ ret = fn(handle1, handle2, byref(onSameBoard))
+ _nvmlCheckReturn(ret)
+ return onSameBoard.value != 0
+
+
+# Added in 3.295
+def nvmlDeviceGetCurrPcieLinkGeneration(handle):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrPcieLinkGeneration")
+ gen = c_uint()
+ ret = fn(handle, byref(gen))
+ _nvmlCheckReturn(ret)
+ return gen.value
+
+
+# Added in 3.295
+def nvmlDeviceGetMaxPcieLinkGeneration(handle):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxPcieLinkGeneration")
+ gen = c_uint()
+ ret = fn(handle, byref(gen))
+ _nvmlCheckReturn(ret)
+ return gen.value
+
+
+# Added in 3.295
+def nvmlDeviceGetCurrPcieLinkWidth(handle):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrPcieLinkWidth")
+ width = c_uint()
+ ret = fn(handle, byref(width))
+ _nvmlCheckReturn(ret)
+ return width.value
+
+
+# Added in 3.295
+def nvmlDeviceGetMaxPcieLinkWidth(handle):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxPcieLinkWidth")
+ width = c_uint()
+ ret = fn(handle, byref(width))
+ _nvmlCheckReturn(ret)
+ return width.value
+
+
+# Added in 4.304
+def nvmlDeviceGetSupportedClocksThrottleReasons(handle):
+ c_reasons = c_ulonglong()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedClocksThrottleReasons")
+ ret = fn(handle, byref(c_reasons))
+ _nvmlCheckReturn(ret)
+ return c_reasons.value
+
+
+# Added in 4.304
+def nvmlDeviceGetCurrentClocksThrottleReasons(handle):
+ c_reasons = c_ulonglong()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrentClocksThrottleReasons")
+ ret = fn(handle, byref(c_reasons))
+ _nvmlCheckReturn(ret)
+ return c_reasons.value
+
+
+# Added in 5.319
+def nvmlDeviceGetIndex(handle):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetIndex")
+ c_index = c_uint()
+ ret = fn(handle, byref(c_index))
+ _nvmlCheckReturn(ret)
+ return c_index.value
+
+
+# Added in 5.319
+def nvmlDeviceGetAccountingMode(handle):
+ c_mode = _nvmlEnableState_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingMode")
+ ret = fn(handle, byref(c_mode))
+ _nvmlCheckReturn(ret)
+ return c_mode.value
+
+
+def nvmlDeviceSetAccountingMode(handle, mode):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetAccountingMode")
+ ret = fn(handle, _nvmlEnableState_t(mode))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceClearAccountingPids(handle):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceClearAccountingPids")
+ ret = fn(handle)
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceGetAccountingStats(handle, pid):
+ stats = c_nvmlAccountingStats_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingStats")
+ ret = fn(handle, c_uint(pid), byref(stats))
+ _nvmlCheckReturn(ret)
+ if stats.maxMemoryUsage == NVML_VALUE_NOT_AVAILABLE_ulonglong.value:
+ # special case for WDDM on Windows, see comment above
+ stats.maxMemoryUsage = None
+ return stats
+
+
+def nvmlDeviceGetAccountingPids(handle):
+ count = c_uint(nvmlDeviceGetAccountingBufferSize(handle))
+ pids = (c_uint * count.value)()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingPids")
+ ret = fn(handle, byref(count), pids)
+ _nvmlCheckReturn(ret)
+ return list(map(int, pids[0 : count.value]))
+
+
+def nvmlDeviceGetAccountingBufferSize(handle):
+ bufferSize = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingBufferSize")
+ ret = fn(handle, byref(bufferSize))
+ _nvmlCheckReturn(ret)
+ return int(bufferSize.value)
+
+
+def nvmlDeviceGetRetiredPages(device, sourceFilter):
+ c_source = _nvmlPageRetirementCause_t(sourceFilter)
+ c_count = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetRetiredPages")
+
+ # First call will get the size
+ ret = fn(device, c_source, byref(c_count), None)
+
+ # this should only fail with insufficient size
+ if (ret != NVML_SUCCESS) and (ret != NVML_ERROR_INSUFFICIENT_SIZE):
+ raise NVMLError(ret)
+
+ # call again with a buffer
+ # oversize the array for the rare cases where additional pages
+ # are retired between NVML calls
+ c_count.value = c_count.value * 2 + 5
+ page_array = c_ulonglong * c_count.value
+ c_pages = page_array()
+ ret = fn(device, c_source, byref(c_count), c_pages)
+ _nvmlCheckReturn(ret)
+ return list(map(int, c_pages[0 : c_count.value]))
+
+
+def nvmlDeviceGetRetiredPages_v2(device, sourceFilter):
+ c_source = _nvmlPageRetirementCause_t(sourceFilter)
+ c_count = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetRetiredPages_v2")
+
+ # First call will get the size
+ ret = fn(device, c_source, byref(c_count), None)
+
+ # this should only fail with insufficient size
+ if (ret != NVML_SUCCESS) and (ret != NVML_ERROR_INSUFFICIENT_SIZE):
+ raise NVMLError(ret)
+
+ # call again with a buffer
+ # oversize the array for the rare cases where additional pages
+ # are retired between NVML calls
+ c_count.value = c_count.value * 2 + 5
+ page_array = c_ulonglong * c_count.value
+ c_pages = page_array()
+ times_array = c_ulonglong * c_count.value
+ c_times = times_array()
+ ret = fn(device, c_source, byref(c_count), c_pages, c_times)
+ _nvmlCheckReturn(ret)
+ return [
+ {"address": int(c_pages[i]), "timestamp": int(c_times[i])}
+ for i in range(c_count.value)
+ ]
+
+
+def nvmlDeviceGetRetiredPagesPendingStatus(device):
+ c_pending = _nvmlEnableState_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetRetiredPagesPendingStatus")
+ ret = fn(device, byref(c_pending))
+ _nvmlCheckReturn(ret)
+ return int(c_pending.value)
+
+
+def nvmlDeviceGetAPIRestriction(device, apiType):
+ c_permission = _nvmlEnableState_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetAPIRestriction")
+ ret = fn(device, _nvmlRestrictedAPI_t(apiType), byref(c_permission))
+ _nvmlCheckReturn(ret)
+ return int(c_permission.value)
+
+
+def nvmlDeviceSetAPIRestriction(handle, apiType, isRestricted):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetAPIRestriction")
+ ret = fn(handle, _nvmlRestrictedAPI_t(apiType), _nvmlEnableState_t(isRestricted))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceGetBridgeChipInfo(handle):
+ bridgeHierarchy = c_nvmlBridgeChipHierarchy_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetBridgeChipInfo")
+ ret = fn(handle, byref(bridgeHierarchy))
+ _nvmlCheckReturn(ret)
+ return bridgeHierarchy
+
+
+def nvmlDeviceGetSamples(device, sampling_type, timeStamp):
+ c_sampling_type = _nvmlSamplingType_t(sampling_type)
+ c_time_stamp = c_ulonglong(timeStamp)
+ c_sample_count = c_uint(0)
+ c_sample_value_type = _nvmlValueType_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetSamples")
+
+ ## First Call gets the size
+ ret = fn(
+ device,
+ c_sampling_type,
+ c_time_stamp,
+ byref(c_sample_value_type),
+ byref(c_sample_count),
+ None,
+ )
+
+ # Stop if this fails
+ if ret != NVML_SUCCESS:
+ raise NVMLError(ret)
+
+ sampleArray = c_sample_count.value * c_nvmlSample_t
+ c_samples = sampleArray()
+ ret = fn(
+ device,
+ c_sampling_type,
+ c_time_stamp,
+ byref(c_sample_value_type),
+ byref(c_sample_count),
+ c_samples,
+ )
+ _nvmlCheckReturn(ret)
+ return (c_sample_value_type.value, c_samples[0 : c_sample_count.value])
+
+
+def nvmlDeviceGetViolationStatus(device, perfPolicyType):
+ c_perfPolicy_type = _nvmlPerfPolicyType_t(perfPolicyType)
+ c_violTime = c_nvmlViolationTime_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetViolationStatus")
+
+ ## Invoke the method to get violation time
+ ret = fn(device, c_perfPolicy_type, byref(c_violTime))
+ _nvmlCheckReturn(ret)
+ return c_violTime
+
+
+def nvmlDeviceGetPcieThroughput(device, counter):
+ c_util = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieThroughput")
+ ret = fn(device, _nvmlPcieUtilCounter_t(counter), byref(c_util))
+ _nvmlCheckReturn(ret)
+ return c_util.value
+
+
+def nvmlSystemGetTopologyGpuSet(cpuNumber):
+ c_count = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlSystemGetTopologyGpuSet")
+
+ # First call will get the size
+ ret = fn(cpuNumber, byref(c_count), None)
+
+ if ret != NVML_SUCCESS:
+ raise NVMLError(ret)
+ # call again with a buffer
+ device_array = c_nvmlDevice_t * c_count.value
+ c_devices = device_array()
+ ret = fn(cpuNumber, byref(c_count), c_devices)
+ _nvmlCheckReturn(ret)
+ return list(c_devices[0 : c_count.value])
+
+
+def nvmlDeviceGetTopologyNearestGpus(device, level):
+ c_count = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetTopologyNearestGpus")
+
+ # First call will get the size
+ ret = fn(device, level, byref(c_count), None)
+
+ if ret != NVML_SUCCESS:
+ raise NVMLError(ret)
+
+ # call again with a buffer
+ device_array = c_nvmlDevice_t * c_count.value
+ c_devices = device_array()
+ ret = fn(device, level, byref(c_count), c_devices)
+ _nvmlCheckReturn(ret)
+ return list(c_devices[0 : c_count.value])
+
+
+def nvmlDeviceGetTopologyCommonAncestor(device1, device2):
+ c_level = _nvmlGpuTopologyLevel_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetTopologyCommonAncestor")
+ ret = fn(device1, device2, byref(c_level))
+ _nvmlCheckReturn(ret)
+ return c_level.value
+
+
+def nvmlDeviceGetNvLinkUtilizationCounter(device, link, counter):
+ c_rxcounter = c_ulonglong()
+ c_txcounter = c_ulonglong()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkUtilizationCounter")
+ ret = fn(device, link, counter, byref(c_rxcounter), byref(c_txcounter))
+ _nvmlCheckReturn(ret)
+ return (c_rxcounter.value, c_txcounter.value)
+
+
+def nvmlDeviceFreezeNvLinkUtilizationCounter(device, link, counter, freeze):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceFreezeNvLinkUtilizationCounter")
+ ret = fn(device, link, counter, freeze)
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceResetNvLinkUtilizationCounter(device, link, counter):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceResetNvLinkUtilizationCounter")
+ ret = fn(device, link, counter)
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceSetNvLinkUtilizationControl(device, link, counter, control, reset):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetNvLinkUtilizationControl")
+ ret = fn(device, link, counter, byref(control), reset)
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceGetNvLinkUtilizationControl(device, link, counter):
+ c_control = nvmlNvLinkUtilizationControl_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkUtilizationControl")
+ ret = fn(device, link, counter, byref(c_control))
+ _nvmlCheckReturn(ret)
+ return c_control
+
+
+def nvmlDeviceGetNvLinkCapability(device, link, capability):
+ c_capResult = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkCapability")
+ ret = fn(device, link, capability, byref(c_capResult))
+ _nvmlCheckReturn(ret)
+ return c_capResult.value
+
+
+def nvmlDeviceGetNvLinkErrorCounter(device, link, counter):
+ c_result = c_ulonglong()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkErrorCounter")
+ ret = fn(device, link, counter, byref(c_result))
+ _nvmlCheckReturn(ret)
+ return c_result.value
+
+
+def nvmlDeviceResetNvLinkErrorCounters(device, link):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceResetNvLinkErrorCounters")
+ ret = fn(device, link)
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceGetNvLinkRemotePciInfo(device, link):
+ c_pci = nvmlPciInfo_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkRemotePciInfo_v2")
+ ret = fn(device, link, byref(c_pci))
+ _nvmlCheckReturn(ret)
+ return c_pci
+
+
+def nvmlDeviceGetNvLinkRemoteDeviceType(handle, link):
+ c_type = _nvmlNvLinkDeviceType_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkRemoteDeviceType")
+ ret = fn(handle, link, byref(c_type))
+ _nvmlCheckReturn(ret)
+ return c_type.value
+
+
+def nvmlDeviceGetNvLinkState(device, link):
+ c_isActive = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkState")
+ ret = fn(device, link, byref(c_isActive))
+ _nvmlCheckReturn(ret)
+ return c_isActive.value
+
+
+def nvmlDeviceGetNvLinkVersion(device, link):
+ c_version = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkVersion")
+ ret = fn(device, link, byref(c_version))
+ _nvmlCheckReturn(ret)
+ return c_version.value
+
+
+def nvmlDeviceModifyDrainState(pciInfo, newState):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceModifyDrainState")
+ ret = fn(pointer(pciInfo), newState)
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceQueryDrainState(pciInfo):
+ c_newState = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceQueryDrainState")
+ ret = fn(pointer(pciInfo), byref(c_newState))
+ _nvmlCheckReturn(ret)
+ return c_newState.value
+
+
+def nvmlDeviceRemoveGpu(pciInfo):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceRemoveGpu")
+ ret = fn(pointer(pciInfo))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceDiscoverGpus(pciInfo):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceDiscoverGpus")
+ ret = fn(pointer(pciInfo))
+ _nvmlCheckReturn(ret)
+ return None
+
+
+def nvmlDeviceGetFieldValues(handle, fieldIds):
+ values_arr = c_nvmlFieldValue_t * len(fieldIds)
+ values = values_arr()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetFieldValues")
+
+ for i, fieldId in enumerate(fieldIds):
+ try:
+ (values[i].fieldId, values[i].scopeId) = fieldId
+ except TypeError:
+ values[i].fieldId = fieldId
+
+ ret = fn(handle, c_int32(len(fieldIds)), byref(values))
+ _nvmlCheckReturn(ret)
+ return values
+
+
+def nvmlDeviceGetVirtualizationMode(handle):
+ c_virtualization_mode = c_ulonglong()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetVirtualizationMode")
+ ret = fn(handle, byref(c_virtualization_mode))
+ _nvmlCheckReturn(ret)
+ return c_virtualization_mode.value
+
+
+def nvmlDeviceSetVirtualizationMode(handle, virtualization_mode):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetVirtualizationMode")
+ return fn(handle, virtualization_mode)
+
+
+def nvmlDeviceGetSupportedVgpus(handle):
+ # first call to get the size
+ c_vgpu_count = c_uint(0)
+
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedVgpus")
+ ret = fn(handle, byref(c_vgpu_count), None)
+
+ if ret == NVML_SUCCESS:
+ # special case, no supported vGPUs
+ return []
+ elif ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ # typical case
+ vgpu_type_ids_array = _nvmlVgpuTypeId_t * c_vgpu_count.value
+ c_vgpu_type_ids = vgpu_type_ids_array()
+
+ # make the call again
+ ret = fn(handle, byref(c_vgpu_count), c_vgpu_type_ids)
+ _nvmlCheckReturn(ret)
+ vgpus = []
+ for i in range(c_vgpu_count.value):
+ vgpus.append(c_vgpu_type_ids[i])
+ return vgpus
+ else:
+ # error case
+ raise NVMLError(ret)
+
+
+def nvmlDeviceGetCreatableVgpus(handle):
+ # first call to get the size
+ c_vgpu_count = c_uint(0)
+
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetCreatableVgpus")
+ ret = fn(handle, byref(c_vgpu_count), None)
+
+ if ret == NVML_SUCCESS:
+ # special case, no supported vGPUs
+ return []
+ elif ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ # typical case
+ vgpu_type_ids_array = _nvmlVgpuTypeId_t * c_vgpu_count.value
+ c_vgpu_type_ids = vgpu_type_ids_array()
+
+ # make the call again
+ ret = fn(handle, byref(c_vgpu_count), c_vgpu_type_ids)
+ _nvmlCheckReturn(ret)
+ vgpus = []
+ for i in range(c_vgpu_count.value):
+ vgpus.append(c_vgpu_type_ids[i])
+ return vgpus
+ else:
+ # error case
+ raise NVMLError(ret)
+
+
+def nvmlVgpuTypeGetGpuInstanceProfileId(vgpuTypeId):
+ c_profile_id = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetGpuInstanceProfileId")
+ ret = fn(vgpuTypeId, byref(c_profile_id))
+ _nvmlCheckReturn(ret)
+ return c_profile_id.value
+
+
+@convertStrBytes
+def nvmlVgpuTypeGetClass(vgpuTypeId):
+ c_class = create_string_buffer(NVML_DEVICE_NAME_BUFFER_SIZE)
+ c_buffer_size = c_uint(NVML_DEVICE_NAME_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetClass")
+ ret = fn(vgpuTypeId, c_class, byref(c_buffer_size))
+ _nvmlCheckReturn(ret)
+ return c_class.value
+
+
+@convertStrBytes
+def nvmlVgpuTypeGetName(vgpuTypeId):
+ c_name = create_string_buffer(NVML_DEVICE_NAME_BUFFER_SIZE)
+ c_buffer_size = c_uint(NVML_DEVICE_NAME_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetName")
+ ret = fn(vgpuTypeId, c_name, byref(c_buffer_size))
+ _nvmlCheckReturn(ret)
+ return c_name.value
+
+
+def nvmlVgpuTypeGetDeviceID(vgpuTypeId):
+ c_device_id = c_ulonglong(0)
+ c_subsystem_id = c_ulonglong(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetDeviceID")
+ ret = fn(vgpuTypeId, byref(c_device_id), byref(c_subsystem_id))
+ _nvmlCheckReturn(ret)
+ return (c_device_id.value, c_subsystem_id.value)
+
+
+def nvmlVgpuTypeGetFramebufferSize(vgpuTypeId):
+ c_fb_size = c_ulonglong(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFramebufferSize")
+ ret = fn(vgpuTypeId, byref(c_fb_size))
+ _nvmlCheckReturn(ret)
+ return c_fb_size.value
+
+
+def nvmlVgpuTypeGetNumDisplayHeads(vgpuTypeId):
+ c_num_heads = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetNumDisplayHeads")
+ ret = fn(vgpuTypeId, byref(c_num_heads))
+ _nvmlCheckReturn(ret)
+ return c_num_heads.value
+
+
+def nvmlVgpuTypeGetResolution(vgpuTypeId):
+ c_xdim = c_uint(0)
+ c_ydim = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetResolution")
+ ret = fn(vgpuTypeId, 0, byref(c_xdim), byref(c_ydim))
+ _nvmlCheckReturn(ret)
+ return (c_xdim.value, c_ydim.value)
+
+
+@convertStrBytes
+def nvmlVgpuTypeGetLicense(vgpuTypeId):
+ c_license = create_string_buffer(NVML_GRID_LICENSE_BUFFER_SIZE)
+ c_buffer_size = c_uint(NVML_GRID_LICENSE_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetLicense")
+ ret = fn(vgpuTypeId, c_license, c_buffer_size)
+ _nvmlCheckReturn(ret)
+ return c_license.value
+
+
+def nvmlVgpuTypeGetFrameRateLimit(vgpuTypeId):
+ c_frl_config = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFrameRateLimit")
+ ret = fn(vgpuTypeId, byref(c_frl_config))
+ _nvmlCheckReturn(ret)
+ return c_frl_config.value
+
+
+def nvmlVgpuTypeGetMaxInstances(handle, vgpuTypeId):
+ c_max_instances = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstances")
+ ret = fn(handle, vgpuTypeId, byref(c_max_instances))
+ _nvmlCheckReturn(ret)
+ return c_max_instances.value
+
+
+def nvmlVgpuTypeGetMaxInstancesPerVm(vgpuTypeId):
+ c_max_instances_per_vm = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstancesPerVm")
+ ret = fn(vgpuTypeId, byref(c_max_instances_per_vm))
+ _nvmlCheckReturn(ret)
+ return c_max_instances_per_vm.value
+
+
+def nvmlDeviceGetActiveVgpus(handle):
+ # first call to get the size
+ c_vgpu_count = c_uint(0)
+
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetActiveVgpus")
+ ret = fn(handle, byref(c_vgpu_count), None)
+
+ if ret == NVML_SUCCESS:
+ # special case, no active vGPUs
+ return []
+ elif ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ # typical case
+ vgpu_instance_array = _nvmlVgpuInstance_t * c_vgpu_count.value
+ c_vgpu_instances = vgpu_instance_array()
+
+ # make the call again
+ ret = fn(handle, byref(c_vgpu_count), c_vgpu_instances)
+ _nvmlCheckReturn(ret)
+ vgpus = []
+ for i in range(c_vgpu_count.value):
+ vgpus.append(c_vgpu_instances[i])
+ return vgpus
+ else:
+ # error case
+ raise NVMLError(ret)
+
+
+@convertStrBytes
+def nvmlVgpuInstanceGetVmID(vgpuInstance):
+ c_vm_id = create_string_buffer(NVML_DEVICE_UUID_BUFFER_SIZE)
+ c_buffer_size = c_uint(NVML_GRID_LICENSE_BUFFER_SIZE)
+ c_vm_id_type = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetVmID")
+ ret = fn(vgpuInstance, byref(c_vm_id), c_buffer_size, byref(c_vm_id_type))
+ _nvmlCheckReturn(ret)
+ return (c_vm_id.value, c_vm_id_type.value)
+
+
+@convertStrBytes
+def nvmlVgpuInstanceGetUUID(vgpuInstance):
+ c_uuid = create_string_buffer(NVML_DEVICE_UUID_BUFFER_SIZE)
+ c_buffer_size = c_uint(NVML_DEVICE_UUID_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetUUID")
+ ret = fn(vgpuInstance, byref(c_uuid), c_buffer_size)
+ _nvmlCheckReturn(ret)
+ return c_uuid.value
+
+
+@convertStrBytes
+def nvmlVgpuInstanceGetMdevUUID(vgpuInstance):
+ c_uuid = create_string_buffer(NVML_DEVICE_UUID_BUFFER_SIZE)
+ c_buffer_size = c_uint(NVML_DEVICE_UUID_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetMdevUUID")
+ ret = fn(vgpuInstance, byref(c_uuid), c_buffer_size)
+ _nvmlCheckReturn(ret)
+ return c_uuid.value
+
+
+@convertStrBytes
+def nvmlVgpuInstanceGetVmDriverVersion(vgpuInstance):
+ c_driver_version = create_string_buffer(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE)
+ c_buffer_size = c_uint(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetVmDriverVersion")
+ ret = fn(vgpuInstance, byref(c_driver_version), c_buffer_size)
+ _nvmlCheckReturn(ret)
+ return c_driver_version.value
+
+
+def nvmlVgpuInstanceGetLicenseStatus(vgpuInstance):
+ c_license_status = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetLicenseStatus")
+ ret = fn(vgpuInstance, byref(c_license_status))
+ _nvmlCheckReturn(ret)
+ return c_license_status.value
+
+
+def nvmlVgpuInstanceGetLicenseInfo_v2(vgpuInstance):
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetLicenseInfo_v2")
+ c_license_info = c_nvmlVgpuLicenseInfo_t()
+ ret = fn(vgpuInstance, byref(c_license_info))
+ _nvmlCheckReturn(ret)
+ return c_license_info
+
+
+def nvmlVgpuInstanceGetLicenseInfo(vgpuInstance):
+ return nvmlVgpuInstanceGetLicenseInfo_v2(vgpuInstance)
+
+
+def nvmlVgpuInstanceGetFrameRateLimit(vgpuInstance):
+ c_frl = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFrameRateLimit")
+ ret = fn(vgpuInstance, byref(c_frl))
+ _nvmlCheckReturn(ret)
+ return c_frl.value
+
+
+def nvmlVgpuInstanceGetEccMode(vgpuInstance):
+ c_mode = _nvmlEnableState_t()
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEccMode")
+ ret = fn(vgpuInstance, byref(c_mode))
+ _nvmlCheckReturn(ret)
+ return c_mode.value
+
+
+def nvmlVgpuInstanceGetType(vgpuInstance):
+ c_vgpu_type = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetType")
+ ret = fn(vgpuInstance, byref(c_vgpu_type))
+ _nvmlCheckReturn(ret)
+ return c_vgpu_type.value
+
+
+def nvmlVgpuInstanceGetEncoderCapacity(vgpuInstance):
+ c_encoder_capacity = c_ulonglong(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderCapacity")
+ ret = fn(vgpuInstance, byref(c_encoder_capacity))
+ _nvmlCheckReturn(ret)
+ return c_encoder_capacity.value
+
+
+def nvmlVgpuInstanceSetEncoderCapacity(vgpuInstance, encoder_capacity):
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceSetEncoderCapacity")
+ return fn(vgpuInstance, encoder_capacity)
+
+
+def nvmlVgpuInstanceGetFbUsage(vgpuInstance):
+ c_fb_usage = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFbUsage")
+ ret = fn(vgpuInstance, byref(c_fb_usage))
+ _nvmlCheckReturn(ret)
+ return c_fb_usage.value
+
+
+def nvmlVgpuTypeGetCapabilities(vgpuTypeId, capability):
+ c_cap_result = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetCapabilities")
+ ret = fn(vgpuTypeId, _nvmlVgpuCapability_t(capability), byref(c_cap_result))
+ _nvmlCheckReturn(ret)
+ return c_cap_result.value
+
+
+def nvmlVgpuInstanceGetGpuInstanceId(vgpuInstance):
+ c_id = c_uint(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetGpuInstanceId")
+ ret = fn(vgpuInstance, byref(c_id))
+ _nvmlCheckReturn(ret)
+ return c_id.value
+
+
+def nvmlVgpuInstanceGetGpuPciId(vgpuInstance):
+ c_vgpuPciId = create_string_buffer(NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetGpuPciId")
+ ret = fn(
+ vgpuInstance, c_vgpuPciId, byref(c_uint(NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE))
+ )
+ _nvmlCheckReturn(ret)
+ return c_vgpuPciId.value
+
+
+def nvmlDeviceGetVgpuUtilization(handle, timeStamp):
+ # first call to get the size
+ c_vgpu_count = c_uint(0)
+ c_time_stamp = c_ulonglong(timeStamp)
+ c_sample_value_type = _nvmlValueType_t()
+
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuUtilization")
+ ret = fn(
+ handle, c_time_stamp, byref(c_sample_value_type), byref(c_vgpu_count), None
+ )
+
+ if ret == NVML_SUCCESS:
+ # special case, no active vGPUs
+ return []
+ elif ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ # typical case
+ sampleArray = c_vgpu_count.value * c_nvmlVgpuInstanceUtilizationSample_t
+ c_samples = sampleArray()
+
+ # make the call again
+ ret = fn(
+ handle,
+ c_time_stamp,
+ byref(c_sample_value_type),
+ byref(c_vgpu_count),
+ c_samples,
+ )
+ _nvmlCheckReturn(ret)
+
+ return c_samples[0 : c_vgpu_count.value]
+ else:
+ # error case
+ raise NVMLError(ret)
+
+
+def nvmlDeviceGetP2PStatus(device1, device2, p2pIndex):
+ c_p2pstatus = _nvmlGpuP2PStatus_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetP2PStatus")
+ ret = fn(device1, device2, p2pIndex, byref(c_p2pstatus))
+ _nvmlCheckReturn(ret)
+ return c_p2pstatus.value
+
+
+def nvmlDeviceGetGridLicensableFeatures_v4(handle):
+ c_get_grid_licensable_features = c_nvmlGridLicensableFeatures_v4_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetGridLicensableFeatures_v4")
+ ret = fn(handle, byref(c_get_grid_licensable_features))
+ _nvmlCheckReturn(ret)
+
+ return c_get_grid_licensable_features
+
+
+def nvmlDeviceGetGridLicensableFeatures(handle):
+ return nvmlDeviceGetGridLicensableFeatures_v4(handle)
+
+
+def nvmlDeviceGetGspFirmwareVersion(handle, version):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetGspFirmwareVersion")
+ ret = fn(handle, version)
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlDeviceGetGspFirmwareMode(handle, isEnabled, defaultMode):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetGspFirmwareMode")
+ ret = fn(handle, isEnabled, defaultMode)
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlDeviceGetEncoderCapacity(handle, encoderQueryType):
+ c_encoder_capacity = c_ulonglong(0)
+ c_encoderQuery_type = _nvmlEncoderQueryType_t(encoderQueryType)
+
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderCapacity")
+ ret = fn(handle, c_encoderQuery_type, byref(c_encoder_capacity))
+ _nvmlCheckReturn(ret)
+ return c_encoder_capacity.value
+
+
+def nvmlDeviceGetVgpuProcessUtilization(handle, timeStamp):
+ # first call to get the size
+ c_vgpu_count = c_uint(0)
+ c_time_stamp = c_ulonglong(timeStamp)
+
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuProcessUtilization")
+ ret = fn(handle, c_time_stamp, byref(c_vgpu_count), None)
+
+ if ret == NVML_SUCCESS:
+ # special case, no active vGPUs
+ return []
+ elif ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ # typical case
+ sampleArray = c_vgpu_count.value * c_nvmlVgpuProcessUtilizationSample_t
+ c_samples = sampleArray()
+
+ # make the call again
+ ret = fn(handle, c_time_stamp, byref(c_vgpu_count), c_samples)
+ _nvmlCheckReturn(ret)
+
+ return c_samples[0 : c_vgpu_count.value]
+ else:
+ # error case
+ raise NVMLError(ret)
+
+
+def nvmlDeviceGetEncoderStats(handle):
+ c_encoderCount = c_ulonglong(0)
+ c_encodeFps = c_ulonglong(0)
+ c_encoderLatency = c_ulonglong(0)
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderStats")
+ ret = fn(handle, byref(c_encoderCount), byref(c_encodeFps), byref(c_encoderLatency))
+ _nvmlCheckReturn(ret)
+ return (c_encoderCount.value, c_encodeFps.value, c_encoderLatency.value)
+
+
+def nvmlDeviceGetEncoderSessions(handle):
+ # first call to get the size
+ c_session_count = c_uint(0)
+
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderSessions")
+ ret = fn(handle, byref(c_session_count), None)
+
+ if ret == NVML_SUCCESS:
+ if c_session_count.value != 0:
+ # typical case
+ session_array = c_nvmlEncoderSession_t * c_session_count.value
+ c_sessions = session_array()
+
+ # make the call again
+ ret = fn(handle, byref(c_session_count), c_sessions)
+ _nvmlCheckReturn(ret)
+ sessions = []
+ for i in range(c_session_count.value):
+ sessions.append(c_sessions[i])
+ return sessions
+ else:
+ return [] # no active sessions
+ else:
+ # error case
+ raise NVMLError(ret)
+
+
+def nvmlDeviceGetFBCStats(handle):
+ c_fbcStats = c_nvmlFBCStats_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetFBCStats")
+ ret = fn(handle, byref(c_fbcStats))
+ _nvmlCheckReturn(ret)
+ return c_fbcStats
+
+
+def nvmlDeviceGetFBCSessions(handle):
+ # first call to get the size
+ c_session_count = c_uint(0)
+
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetFBCSessions")
+ ret = fn(handle, byref(c_session_count), None)
+
+ if ret == NVML_SUCCESS:
+ if c_session_count.value != 0:
+ # typical case
+ session_array = c_nvmlFBCSession_t * c_session_count.value
+ c_sessions = session_array()
+
+ # make the call again
+ ret = fn(handle, byref(c_session_count), c_sessions)
+ _nvmlCheckReturn(ret)
+ sessions = []
+ for i in range(c_session_count.value):
+ sessions.append(c_sessions[i])
+ return sessions
+ else:
+ return [] # no active sessions
+ else:
+ # error case
+ raise NVMLError(ret)
+
+
+def nvmlVgpuInstanceGetEncoderStats(vgpuInstance):
+ c_encoderCount = c_ulonglong(0)
+ c_encodeFps = c_ulonglong(0)
+ c_encoderLatency = c_ulonglong(0)
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderStats")
+ ret = fn(
+ vgpuInstance, byref(c_encoderCount), byref(c_encodeFps), byref(c_encoderLatency)
+ )
+ _nvmlCheckReturn(ret)
+ return (c_encoderCount.value, c_encodeFps.value, c_encoderLatency.value)
+
+
+def nvmlVgpuInstanceGetEncoderSessions(vgpuInstance):
+ # first call to get the size
+ c_session_count = c_uint(0)
+
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderSessions")
+ ret = fn(vgpuInstance, byref(c_session_count), None)
+
+ if ret == NVML_SUCCESS:
+ if c_session_count.value != 0:
+ # typical case
+ session_array = c_nvmlEncoderSession_t * c_session_count.value
+ c_sessions = session_array()
+
+ # make the call again
+ ret = fn(vgpuInstance, byref(c_session_count), c_sessions)
+ _nvmlCheckReturn(ret)
+ sessions = []
+ for i in range(c_session_count.value):
+ sessions.append(c_sessions[i])
+ return sessions
+ else:
+ return [] # no active sessions
+ else:
+ # error case
+ raise NVMLError(ret)
+
+
+def nvmlVgpuInstanceGetFBCStats(vgpuInstance):
+ c_fbcStats = c_nvmlFBCStats_t()
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFBCStats")
+ ret = fn(vgpuInstance, byref(c_fbcStats))
+ _nvmlCheckReturn(ret)
+ return c_fbcStats
+
+
+def nvmlVgpuInstanceGetFBCSessions(vgpuInstance):
+ # first call to get the size
+ c_session_count = c_uint(0)
+
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFBCSessions")
+ ret = fn(vgpuInstance, byref(c_session_count), None)
+
+ if ret == NVML_SUCCESS:
+ if c_session_count.value != 0:
+ # typical case
+ session_array = c_nvmlFBCSession_t * c_session_count.value
+ c_sessions = session_array()
+
+ # make the call again
+ ret = fn(vgpuInstance, byref(c_session_count), c_sessions)
+ _nvmlCheckReturn(ret)
+ sessions = []
+ for i in range(c_session_count.value):
+ sessions.append(c_sessions[i])
+ return sessions
+ else:
+ return [] # no active sessions
+ else:
+ # error case
+ raise NVMLError(ret)
+
+
+def nvmlDeviceGetProcessUtilization(handle, timeStamp):
+ # first call to get the size
+ c_count = c_uint(0)
+ c_time_stamp = c_ulonglong(timeStamp)
+
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetProcessUtilization")
+ ret = fn(handle, None, byref(c_count), c_time_stamp)
+
+ if ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ # typical case
+ sampleArray = c_count.value * c_nvmlProcessUtilizationSample_t
+ c_samples = sampleArray()
+
+ # make the call again
+ ret = fn(handle, c_samples, byref(c_count), c_time_stamp)
+ _nvmlCheckReturn(ret)
+
+ return c_samples[0 : c_count.value]
+ else:
+ # error case
+ raise NVMLError(ret)
+
+
+def nvmlVgpuInstanceGetMetadata(vgpuInstance):
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetMetadata")
+ c_vgpuMetadata = c_nvmlVgpuMetadata_t()
+ c_bufferSize = c_uint(0)
+ # Make the first NVML API call to get the c_bufferSize value.
+ # We have already allocated required buffer above.
+ ret = fn(vgpuInstance, byref(c_vgpuMetadata), byref(c_bufferSize))
+ if ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ ret = fn(vgpuInstance, byref(c_vgpuMetadata), byref(c_bufferSize))
+ _nvmlCheckReturn(ret)
+ else:
+ raise NVMLError(ret)
+ return c_vgpuMetadata
+
+
+def nvmlDeviceGetVgpuMetadata(handle):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuMetadata")
+ c_vgpuPgpuMetadata = c_nvmlVgpuPgpuMetadata_t()
+ c_bufferSize = c_uint(0)
+ # Make the first NVML API call to get the c_bufferSize value.
+ # We have already allocated required buffer above.
+ ret = fn(handle, byref(c_vgpuPgpuMetadata), byref(c_bufferSize))
+ if ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ ret = fn(handle, byref(c_vgpuPgpuMetadata), byref(c_bufferSize))
+ _nvmlCheckReturn(ret)
+ else:
+ raise NVMLError(ret)
+ return c_vgpuPgpuMetadata
+
+
+def nvmlGetVgpuCompatibility(vgpuMetadata, pgpuMetadata):
+ fn = _nvmlGetFunctionPointer("nvmlGetVgpuCompatibility")
+ c_vgpuPgpuCompatibility = c_nvmlVgpuPgpuCompatibility_t()
+ ret = fn(byref(vgpuMetadata), byref(pgpuMetadata), byref(c_vgpuPgpuCompatibility))
+ _nvmlCheckReturn(ret)
+ return c_vgpuPgpuCompatibility
+
+
+@convertStrBytes
+def nvmlDeviceGetPgpuMetadataString(handle):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetPgpuMetadataString")
+ c_pgpuMetadata = create_string_buffer(NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE)
+ c_bufferSize = c_uint(0)
+ # Make the first NVML API call to get the c_bufferSize value.
+ # We have already allocated required buffer above.
+ ret = fn(handle, byref(c_pgpuMetadata), byref(c_bufferSize))
+ if ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ ret = fn(handle, byref(c_pgpuMetadata), byref(c_bufferSize))
+ _nvmlCheckReturn(ret)
+ else:
+ raise NVMLError(ret)
+ return (c_pgpuMetadata.value, c_bufferSize.value)
+
+
+def nvmlSetVgpuVersion(vgpuVersion):
+ fn = _nvmlGetFunctionPointer("nvmlSetVgpuVersion")
+ ret = fn(byref(vgpuVersion))
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlGetVgpuVersion(supported, current):
+ fn = _nvmlGetFunctionPointer("nvmlGetVgpuVersion")
+ ret = fn(byref(supported), byref(current))
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlVgpuInstanceGetAccountingMode(vgpuInstance):
+ c_mode = _nvmlEnableState_t()
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetAccountingMode")
+ ret = fn(vgpuInstance, byref(c_mode))
+ _nvmlCheckReturn(ret)
+ return c_mode.value
+
+
+def nvmlVgpuInstanceGetAccountingPids(vgpuInstance):
+ c_pidCount = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetAccountingPids")
+ ret = fn(vgpuInstance, byref(c_pidCount), None)
+ if ret == NVML_ERROR_INSUFFICIENT_SIZE:
+ sampleArray = c_pidCount.value * c_uint
+ c_pidArray = sampleArray()
+ ret = fn(vgpuInstance, byref(c_pidCount), byref(c_pidArray))
+ _nvmlCheckReturn(ret)
+ else:
+ raise NVMLError(ret)
+ return (c_pidCount, c_pidArray)
+
+
+def nvmlVgpuInstanceGetAccountingStats(vgpuInstance, pid):
+ c_accountingStats = c_nvmlAccountingStats_t()
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetAccountingStats")
+ ret = fn(vgpuInstance, pid, byref(c_accountingStats))
+ _nvmlCheckReturn(ret)
+ return c_accountingStats
+
+
+def nvmlVgpuInstanceClearAccountingPids(vgpuInstance):
+ fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceClearAccountingPids")
+ ret = fn(vgpuInstance)
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlGetExcludedDeviceCount():
+ c_count = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlGetExcludedDeviceCount")
+ ret = fn(byref(c_count))
+ _nvmlCheckReturn(ret)
+ return c_count.value
+
+
+def nvmlGetExcludedDeviceInfoByIndex(index):
+ c_index = c_uint(index)
+ info = c_nvmlExcludedDeviceInfo_t()
+ fn = _nvmlGetFunctionPointer("nvmlGetExcludedDeviceInfoByIndex")
+ ret = fn(c_index, byref(info))
+ _nvmlCheckReturn(ret)
+ return info
+
+
+def nvmlDeviceGetHostVgpuMode(handle):
+ c_host_vgpu_mode = _nvmlHostVgpuMode_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetHostVgpuMode")
+ ret = fn(handle, byref(c_host_vgpu_mode))
+ _nvmlCheckReturn(ret)
+ return c_host_vgpu_mode.value
+
+
+def nvmlDeviceSetMigMode(device, mode):
+ c_activationStatus = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetMigMode")
+ ret = fn(device, mode, byref(c_activationStatus))
+ _nvmlCheckReturn(ret)
+ return c_activationStatus.value
+
+
+def nvmlDeviceGetMigMode(device):
+ c_currentMode = c_uint()
+ c_pendingMode = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMigMode")
+ ret = fn(device, byref(c_currentMode), byref(c_pendingMode))
+ _nvmlCheckReturn(ret)
+ return [c_currentMode.value, c_pendingMode.value]
+
+
+def nvmlDeviceGetGpuInstanceProfileInfo(device, profile, version=2):
+ if version == 2:
+ c_info = c_nvmlGpuInstanceProfileInfo_v2_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceProfileInfoV")
+ elif version == 1:
+ c_info = c_nvmlGpuInstanceProfileInfo_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceProfileInfo")
+ else:
+ raise NVMLError(NVML_ERROR_FUNCTION_NOT_FOUND)
+ ret = fn(device, profile, byref(c_info))
+ _nvmlCheckReturn(ret)
+ return c_info
+
+
+# Define function alias for the API exposed by NVML
+nvmlDeviceGetGpuInstanceProfileInfoV = nvmlDeviceGetGpuInstanceProfileInfo
+
+
+def nvmlDeviceGetGpuInstanceRemainingCapacity(device, profileId):
+ c_count = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceRemainingCapacity")
+ ret = fn(device, profileId, byref(c_count))
+ _nvmlCheckReturn(ret)
+ return c_count.value
+
+
+def nvmlDeviceGetGpuInstancePossiblePlacements(
+ device, profileId, placementsRef, countRef
+):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstancePossiblePlacements_v2")
+ ret = fn(device, profileId, placementsRef, countRef)
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlDeviceCreateGpuInstance(device, profileId):
+ c_instance = c_nvmlGpuInstance_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceCreateGpuInstance")
+ ret = fn(device, profileId, byref(c_instance))
+ _nvmlCheckReturn(ret)
+ return c_instance
+
+
+def nvmlDeviceCreateGpuInstanceWithPlacement(device, profileId, placement):
+ c_instance = c_nvmlGpuInstance_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceCreateGpuInstanceWithPlacement")
+ ret = fn(device, profileId, placement, byref(c_instance))
+ _nvmlCheckReturn(ret)
+ return c_instance
+
+
+def nvmlGpuInstanceDestroy(gpuInstance):
+ fn = _nvmlGetFunctionPointer("nvmlGpuInstanceDestroy")
+ ret = fn(gpuInstance)
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlDeviceGetGpuInstances(device, profileId, gpuInstancesRef, countRef):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstances")
+ ret = fn(device, profileId, gpuInstancesRef, countRef)
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlDeviceGetGpuInstanceById(device, gpuInstanceId):
+ c_instance = c_nvmlGpuInstance_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceById")
+ ret = fn(device, gpuInstanceId, byref(c_instance))
+ _nvmlCheckReturn(ret)
+ return c_instance
+
+
+def nvmlGpuInstanceGetInfo(gpuInstance):
+ c_info = c_nvmlGpuInstanceInfo_t()
+ fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetInfo")
+ ret = fn(gpuInstance, byref(c_info))
+ _nvmlCheckReturn(ret)
+ return c_info
+
+
+def nvmlGpuInstanceGetComputeInstanceProfileInfo(
+ device, profile, engProfile, version=2
+):
+ if version == 2:
+ c_info = c_nvmlComputeInstanceProfileInfo_v2_t()
+ fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceProfileInfoV")
+ elif version == 1:
+ c_info = c_nvmlComputeInstanceProfileInfo_t()
+ fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceProfileInfo")
+ else:
+ raise NVMLError(NVML_ERROR_FUNCTION_NOT_FOUND)
+ ret = fn(device, profile, engProfile, byref(c_info))
+ _nvmlCheckReturn(ret)
+ return c_info
+
+
+# Define function alias for the API exposed by NVML
+nvmlGpuInstanceGetComputeInstanceProfileInfoV = (
+ nvmlGpuInstanceGetComputeInstanceProfileInfo
+)
+
+
+def nvmlGpuInstanceGetComputeInstanceRemainingCapacity(gpuInstance, profileId):
+ c_count = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceRemainingCapacity")
+ ret = fn(gpuInstance, profileId, byref(c_count))
+ _nvmlCheckReturn(ret)
+ return c_count.value
+
+
+def nvmlGpuInstanceCreateComputeInstance(gpuInstance, profileId):
+ c_instance = c_nvmlComputeInstance_t()
+ fn = _nvmlGetFunctionPointer("nvmlGpuInstanceCreateComputeInstance")
+ ret = fn(gpuInstance, profileId, byref(c_instance))
+ _nvmlCheckReturn(ret)
+ return c_instance
+
+
+def nvmlComputeInstanceDestroy(computeInstance):
+ fn = _nvmlGetFunctionPointer("nvmlComputeInstanceDestroy")
+ ret = fn(computeInstance)
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlGpuInstanceGetComputeInstances(
+ gpuInstance, profileId, computeInstancesRef, countRef
+):
+ fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstances")
+ ret = fn(gpuInstance, profileId, computeInstancesRef, countRef)
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlGpuInstanceGetComputeInstanceById(gpuInstance, computeInstanceId):
+ c_instance = c_nvmlComputeInstance_t()
+ fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceById")
+ ret = fn(gpuInstance, computeInstanceId, byref(c_instance))
+ _nvmlCheckReturn(ret)
+ return c_instance
+
+
+def nvmlComputeInstanceGetInfo_v2(computeInstance):
+ c_info = c_nvmlComputeInstanceInfo_t()
+ fn = _nvmlGetFunctionPointer("nvmlComputeInstanceGetInfo_v2")
+ ret = fn(computeInstance, byref(c_info))
+ _nvmlCheckReturn(ret)
+ return c_info
+
+
+def nvmlComputeInstanceGetInfo(computeInstance):
+ return nvmlComputeInstanceGetInfo_v2(computeInstance)
+
+
+def nvmlDeviceIsMigDeviceHandle(device):
+ c_isMigDevice = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceIsMigDeviceHandle")
+ ret = fn(device, byref(c_isMigDevice))
+ _nvmlCheckReturn(ret)
+ return c_isMigDevice
+
+
+def nvmlDeviceGetGpuInstanceId(device):
+ c_gpuInstanceId = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceId")
+ ret = fn(device, byref(c_gpuInstanceId))
+ _nvmlCheckReturn(ret)
+ return c_gpuInstanceId.value
+
+
+def nvmlDeviceGetComputeInstanceId(device):
+ c_computeInstanceId = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeInstanceId")
+ ret = fn(device, byref(c_computeInstanceId))
+ _nvmlCheckReturn(ret)
+ return c_computeInstanceId.value
+
+
+def nvmlDeviceGetMaxMigDeviceCount(device):
+ c_count = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxMigDeviceCount")
+ ret = fn(device, byref(c_count))
+ _nvmlCheckReturn(ret)
+ return c_count.value
+
+
+def nvmlDeviceGetMigDeviceHandleByIndex(device, index):
+ c_index = c_uint(index)
+ migDevice = c_nvmlDevice_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMigDeviceHandleByIndex")
+ ret = fn(device, c_index, byref(migDevice))
+ _nvmlCheckReturn(ret)
+ return migDevice
+
+
+def nvmlDeviceGetDeviceHandleFromMigDeviceHandle(migDevice):
+ device = c_nvmlDevice_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetDeviceHandleFromMigDeviceHandle")
+ ret = fn(migDevice, byref(device))
+ _nvmlCheckReturn(ret)
+ return device
+
+
+def nvmlDeviceGetAttributes_v2(device):
+ c_attrs = c_nvmlDeviceAttributes()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetAttributes_v2")
+ ret = fn(device, byref(c_attrs))
+ _nvmlCheckReturn(ret)
+ return c_attrs
+
+
+def nvmlDeviceGetAttributes(device):
+ return nvmlDeviceGetAttributes_v2(device)
+
+
+def nvmlDeviceGetRemappedRows(device):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetRemappedRows")
+ c_corr = c_uint()
+ c_unc = c_uint()
+ c_bpending = c_uint()
+ c_bfailure = c_uint()
+ ret = fn(device, byref(c_corr), byref(c_unc), byref(c_bpending), byref(c_bfailure))
+ _nvmlCheckReturn(ret)
+ return (c_corr.value, c_unc.value, c_bpending.value, c_bfailure.value)
+
+
+def nvmlDeviceGetRowRemapperHistogram(device):
+ c_vals = c_nvmlRowRemapperHistogramValues()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetRowRemapperHistogram")
+ ret = fn(device, byref(c_vals))
+ _nvmlCheckReturn(ret)
+ return c_vals
+
+
+def nvmlDeviceGetArchitecture(device):
+ arch = _nvmlDeviceArchitecture_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetArchitecture")
+ ret = fn(device, byref(arch))
+ _nvmlCheckReturn(ret)
+ return arch.value
+
+
+def nvmlDeviceGetBusType(device):
+ c_busType = _nvmlBusType_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetBusType")
+ ret = fn(device, byref(c_busType))
+ _nvmlCheckReturn(ret)
+ return c_busType.value
+
+
+def nvmlDeviceGetIrqNum(device):
+ c_irqNum = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetIrqNum")
+ ret = fn(device, byref(c_irqNum))
+ _nvmlCheckReturn(ret)
+ return c_irqNum.value
+
+
+def nvmlDeviceGetNumGpuCores(device):
+ c_numCores = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetNumGpuCores")
+ ret = fn(device, byref(c_numCores))
+ _nvmlCheckReturn(ret)
+ return c_numCores.value
+
+
+def nvmlDeviceGetPowerSource(device):
+ c_powerSource = _nvmlPowerSource_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerSource")
+ ret = fn(device, byref(c_powerSource))
+ _nvmlCheckReturn(ret)
+ return c_powerSource.value
+
+
+def nvmlDeviceGetMemoryBusWidth(device):
+ c_memBusWidth = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryBusWidth")
+ ret = fn(device, byref(c_memBusWidth))
+ _nvmlCheckReturn(ret)
+ return c_memBusWidth.value
+
+
+def nvmlDeviceGetPcieLinkMaxSpeed(device):
+ c_speed = _nvmlPcieLinkMaxSpeed_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieLinkMaxSpeed")
+ ret = fn(device, byref(c_speed))
+ _nvmlCheckReturn(ret)
+ return c_speed.value
+
+
+def nvmlDeviceGetAdaptiveClockInfoStatus(device):
+ c_adaptiveClockInfoStatus = _nvmlAdaptiveClockInfoStatus_t()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetAdaptiveClockInfoStatus")
+ ret = fn(device, byref(c_adaptiveClockInfoStatus))
+ _nvmlCheckReturn(ret)
+ return c_adaptiveClockInfoStatus.value
+
+
+def nvmlDeviceGetPcieSpeed(device):
+ c_speed = c_uint()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieSpeed")
+ ret = fn(device, byref(c_speed))
+ _nvmlCheckReturn(ret)
+ return c_speed.value
+
+
+def nvmlDeviceGetDynamicPstatesInfo(device, c_dynamicpstatesinfo):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetDynamicPstatesInfo")
+ ret = fn(device, c_dynamicpstatesinfo)
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlDeviceSetFanSpeed_v2(handle, index, speed):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetFanSpeed_v2")
+ ret = fn(handle, index, speed)
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlDeviceGetThermalSettings(device, sensorindex, c_thermalsettings):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetThermalSettings")
+ ret = fn(device, sensorindex, c_thermalsettings)
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlDeviceGetMinMaxClockOfPState(device, type, pstate, minClockMHz, maxClockMHz):
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetMinMaxClockOfPState")
+ ret = fn(
+ device,
+ _nvmlClockType_t(type),
+ _nvmlClockType_t(pstate),
+ minClockMHz,
+ maxClockMHz,
+ )
+ _nvmlCheckReturn(ret)
+ return ret
+
+
+def nvmlDeviceGetSupportedPerformanceStates(device):
+ pstates = []
+ c_count = c_uint(NVML_MAX_GPU_PERF_PSTATES)
+ c_size = sizeof(c_uint) * c_count.value
+
+ # NOTE: use 'c_uint' to represent the size of the nvmlPstate_t enumeration.
+ pstates_array = _nvmlPstates_t * c_count.value
+ c_pstates = pstates_array()
+
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedPerformanceStates")
+ ret = fn(device, c_pstates, c_size)
+ _nvmlCheckReturn(ret)
+
+ for value in c_pstates:
+ if value != NVML_PSTATE_UNKNOWN:
+ pstates.append(value)
+
+ return pstates
+
+
+def nvmlDeviceGetGpcClkVfOffset(device):
+ offset = c_int32()
+ fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpcClkVfOffset")
+ ret = fn(device, byref(offset))
+ _nvmlCheckReturn(ret)
+ return offset.value
+
+
+def nvmlDeviceSetGpcClkVfOffset(device, offset):
+ c_offset = c_int32(offset)
+ fn = _nvmlGetFunctionPointer("nvmlDeviceSetGpcClkVfOffset")
+ ret = fn(device, c_offset)
+ _nvmlCheckReturn(ret)
+ return ret
diff --git a/wandb/vendor/watchdog_0_9_0/wandb-vendor.md b/wandb/vendor/watchdog_0_9_0/wandb-vendor.md
new file mode 100644
index 0000000000000000000000000000000000000000..1c95ebdd649f5c5bcdda84510644684ad05a8ab0
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb-vendor.md
@@ -0,0 +1,9 @@
+
+# Modification steps
+
+```shell
+find . -type f -name \*.py | \
+ xargs egrep "from watchdog[.]" | \
+ sed s'/:.*//' | \
+ xargs sed -i bak -e 's/^from watchdog[.]/from wandb_watchdog./'
+```
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/__init__.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..1a641ff98f09d64bd08a436521005cbf3a915070
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/__init__.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/events.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/events.py
new file mode 100644
index 0000000000000000000000000000000000000000..ee67656d951ee650e1f1f8dd42e2ce1df7cbe008
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/events.py
@@ -0,0 +1,615 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+:module: watchdog.events
+:synopsis: File system events and event handlers.
+:author: yesudeep@google.com (Yesudeep Mangalapilly)
+
+Event Classes
+-------------
+.. autoclass:: FileSystemEvent
+ :members:
+ :show-inheritance:
+ :inherited-members:
+
+.. autoclass:: FileSystemMovedEvent
+ :members:
+ :show-inheritance:
+
+.. autoclass:: FileMovedEvent
+ :members:
+ :show-inheritance:
+
+.. autoclass:: DirMovedEvent
+ :members:
+ :show-inheritance:
+
+.. autoclass:: FileModifiedEvent
+ :members:
+ :show-inheritance:
+
+.. autoclass:: DirModifiedEvent
+ :members:
+ :show-inheritance:
+
+.. autoclass:: FileCreatedEvent
+ :members:
+ :show-inheritance:
+
+.. autoclass:: DirCreatedEvent
+ :members:
+ :show-inheritance:
+
+.. autoclass:: FileDeletedEvent
+ :members:
+ :show-inheritance:
+
+.. autoclass:: DirDeletedEvent
+ :members:
+ :show-inheritance:
+
+
+Event Handler Classes
+---------------------
+.. autoclass:: FileSystemEventHandler
+ :members:
+ :show-inheritance:
+
+.. autoclass:: PatternMatchingEventHandler
+ :members:
+ :show-inheritance:
+
+.. autoclass:: RegexMatchingEventHandler
+ :members:
+ :show-inheritance:
+
+.. autoclass:: LoggingEventHandler
+ :members:
+ :show-inheritance:
+
+"""
+
+import os.path
+import logging
+import re
+from .patterns import match_any_paths
+from wandb_watchdog.utils import has_attribute
+from wandb_watchdog.utils import unicode_paths
+
+
+EVENT_TYPE_MOVED = 'moved'
+EVENT_TYPE_DELETED = 'deleted'
+EVENT_TYPE_CREATED = 'created'
+EVENT_TYPE_MODIFIED = 'modified'
+
+
+class FileSystemEvent(object):
+ """
+ Immutable type that represents a file system event that is triggered
+ when a change occurs on the monitored file system.
+
+ All FileSystemEvent objects are required to be immutable and hence
+ can be used as keys in dictionaries or be added to sets.
+ """
+
+ event_type = None
+ """The type of the event as a string."""
+
+ is_directory = False
+ """True if event was emitted for a directory; False otherwise."""
+
+ def __init__(self, src_path):
+ self._src_path = src_path
+
+ @property
+ def src_path(self):
+ """Source path of the file system object that triggered this event."""
+ return self._src_path
+
+ def __str__(self):
+ return self.__repr__()
+
+ def __repr__(self):
+ return ("<%(class_name)s: event_type=%(event_type)s, "
+ "src_path=%(src_path)r, "
+ "is_directory=%(is_directory)s>"
+ ) % (dict(
+ class_name=self.__class__.__name__,
+ event_type=self.event_type,
+ src_path=self.src_path,
+ is_directory=self.is_directory))
+
+ # Used for comparison of events.
+ @property
+ def key(self):
+ return (self.event_type, self.src_path, self.is_directory)
+
+ def __eq__(self, event):
+ return self.key == event.key
+
+ def __ne__(self, event):
+ return self.key != event.key
+
+ def __hash__(self):
+ return hash(self.key)
+
+
+class FileSystemMovedEvent(FileSystemEvent):
+ """
+ File system event representing any kind of file system movement.
+ """
+
+ event_type = EVENT_TYPE_MOVED
+
+ def __init__(self, src_path, dest_path):
+ super(FileSystemMovedEvent, self).__init__(src_path)
+ self._dest_path = dest_path
+
+ @property
+ def dest_path(self):
+ """The destination path of the move event."""
+ return self._dest_path
+
+ # Used for hashing this as an immutable object.
+ @property
+ def key(self):
+ return (self.event_type, self.src_path, self.dest_path, self.is_directory)
+
+ def __repr__(self):
+ return ("<%(class_name)s: src_path=%(src_path)r, "
+ "dest_path=%(dest_path)r, "
+ "is_directory=%(is_directory)s>"
+ ) % (dict(class_name=self.__class__.__name__,
+ src_path=self.src_path,
+ dest_path=self.dest_path,
+ is_directory=self.is_directory))
+
+
+# File events.
+
+
+class FileDeletedEvent(FileSystemEvent):
+ """File system event representing file deletion on the file system."""
+
+ event_type = EVENT_TYPE_DELETED
+
+ def __init__(self, src_path):
+ super(FileDeletedEvent, self).__init__(src_path)
+
+ def __repr__(self):
+ return "<%(class_name)s: src_path=%(src_path)r>" %\
+ dict(class_name=self.__class__.__name__,
+ src_path=self.src_path)
+
+
+class FileModifiedEvent(FileSystemEvent):
+ """File system event representing file modification on the file system."""
+
+ event_type = EVENT_TYPE_MODIFIED
+
+ def __init__(self, src_path):
+ super(FileModifiedEvent, self).__init__(src_path)
+
+ def __repr__(self):
+ return ("<%(class_name)s: src_path=%(src_path)r>"
+ ) % (dict(class_name=self.__class__.__name__,
+ src_path=self.src_path))
+
+
+class FileCreatedEvent(FileSystemEvent):
+ """File system event representing file creation on the file system."""
+
+ event_type = EVENT_TYPE_CREATED
+
+ def __init__(self, src_path):
+ super(FileCreatedEvent, self).__init__(src_path)
+
+ def __repr__(self):
+ return ("<%(class_name)s: src_path=%(src_path)r>"
+ ) % (dict(class_name=self.__class__.__name__,
+ src_path=self.src_path))
+
+
+class FileMovedEvent(FileSystemMovedEvent):
+ """File system event representing file movement on the file system."""
+
+ def __init__(self, src_path, dest_path):
+ super(FileMovedEvent, self).__init__(src_path, dest_path)
+
+ def __repr__(self):
+ return ("<%(class_name)s: src_path=%(src_path)r, "
+ "dest_path=%(dest_path)r>"
+ ) % (dict(class_name=self.__class__.__name__,
+ src_path=self.src_path,
+ dest_path=self.dest_path))
+
+
+# Directory events.
+
+
+class DirDeletedEvent(FileSystemEvent):
+ """File system event representing directory deletion on the file system."""
+
+ event_type = EVENT_TYPE_DELETED
+ is_directory = True
+
+ def __init__(self, src_path):
+ super(DirDeletedEvent, self).__init__(src_path)
+
+ def __repr__(self):
+ return ("<%(class_name)s: src_path=%(src_path)r>"
+ ) % (dict(class_name=self.__class__.__name__,
+ src_path=self.src_path))
+
+
+class DirModifiedEvent(FileSystemEvent):
+ """
+ File system event representing directory modification on the file system.
+ """
+
+ event_type = EVENT_TYPE_MODIFIED
+ is_directory = True
+
+ def __init__(self, src_path):
+ super(DirModifiedEvent, self).__init__(src_path)
+
+ def __repr__(self):
+ return ("<%(class_name)s: src_path=%(src_path)r>"
+ ) % (dict(class_name=self.__class__.__name__,
+ src_path=self.src_path))
+
+
+class DirCreatedEvent(FileSystemEvent):
+ """File system event representing directory creation on the file system."""
+
+ event_type = EVENT_TYPE_CREATED
+ is_directory = True
+
+ def __init__(self, src_path):
+ super(DirCreatedEvent, self).__init__(src_path)
+
+ def __repr__(self):
+ return ("<%(class_name)s: src_path=%(src_path)r>"
+ ) % (dict(class_name=self.__class__.__name__,
+ src_path=self.src_path))
+
+
+class DirMovedEvent(FileSystemMovedEvent):
+ """File system event representing directory movement on the file system."""
+
+ is_directory = True
+
+ def __init__(self, src_path, dest_path):
+ super(DirMovedEvent, self).__init__(src_path, dest_path)
+
+ def __repr__(self):
+ return ("<%(class_name)s: src_path=%(src_path)r, "
+ "dest_path=%(dest_path)r>"
+ ) % (dict(class_name=self.__class__.__name__,
+ src_path=self.src_path,
+ dest_path=self.dest_path))
+
+
+class FileSystemEventHandler(object):
+ """
+ Base file system event handler that you can override methods from.
+ """
+
+ def dispatch(self, event):
+ """Dispatches events to the appropriate methods.
+
+ :param event:
+ The event object representing the file system event.
+ :type event:
+ :class:`FileSystemEvent`
+ """
+ self.on_any_event(event)
+ _method_map = {
+ EVENT_TYPE_MODIFIED: self.on_modified,
+ EVENT_TYPE_MOVED: self.on_moved,
+ EVENT_TYPE_CREATED: self.on_created,
+ EVENT_TYPE_DELETED: self.on_deleted,
+ }
+ event_type = event.event_type
+ _method_map[event_type](event)
+
+ def on_any_event(self, event):
+ """Catch-all event handler.
+
+ :param event:
+ The event object representing the file system event.
+ :type event:
+ :class:`FileSystemEvent`
+ """
+
+ def on_moved(self, event):
+ """Called when a file or a directory is moved or renamed.
+
+ :param event:
+ Event representing file/directory movement.
+ :type event:
+ :class:`DirMovedEvent` or :class:`FileMovedEvent`
+ """
+
+ def on_created(self, event):
+ """Called when a file or directory is created.
+
+ :param event:
+ Event representing file/directory creation.
+ :type event:
+ :class:`DirCreatedEvent` or :class:`FileCreatedEvent`
+ """
+
+ def on_deleted(self, event):
+ """Called when a file or directory is deleted.
+
+ :param event:
+ Event representing file/directory deletion.
+ :type event:
+ :class:`DirDeletedEvent` or :class:`FileDeletedEvent`
+ """
+
+ def on_modified(self, event):
+ """Called when a file or directory is modified.
+
+ :param event:
+ Event representing file/directory modification.
+ :type event:
+ :class:`DirModifiedEvent` or :class:`FileModifiedEvent`
+ """
+
+
+class PatternMatchingEventHandler(FileSystemEventHandler):
+ """
+ Matches given patterns with file paths associated with occurring events.
+ """
+
+ def __init__(self, patterns=None, ignore_patterns=None,
+ ignore_directories=False, case_sensitive=False):
+ super(PatternMatchingEventHandler, self).__init__()
+
+ self._patterns = patterns
+ self._ignore_patterns = ignore_patterns
+ self._ignore_directories = ignore_directories
+ self._case_sensitive = case_sensitive
+
+ @property
+ def patterns(self):
+ """
+ (Read-only)
+ Patterns to allow matching event paths.
+ """
+ return self._patterns
+
+ @property
+ def ignore_patterns(self):
+ """
+ (Read-only)
+ Patterns to ignore matching event paths.
+ """
+ return self._ignore_patterns
+
+ @property
+ def ignore_directories(self):
+ """
+ (Read-only)
+ ``True`` if directories should be ignored; ``False`` otherwise.
+ """
+ return self._ignore_directories
+
+ @property
+ def case_sensitive(self):
+ """
+ (Read-only)
+ ``True`` if path names should be matched sensitive to case; ``False``
+ otherwise.
+ """
+ return self._case_sensitive
+
+ def dispatch(self, event):
+ """Dispatches events to the appropriate methods.
+
+ :param event:
+ The event object representing the file system event.
+ :type event:
+ :class:`FileSystemEvent`
+ """
+ if self.ignore_directories and event.is_directory:
+ return
+
+ paths = []
+ if has_attribute(event, 'dest_path'):
+ paths.append(unicode_paths.decode(event.dest_path))
+ if event.src_path:
+ paths.append(unicode_paths.decode(event.src_path))
+
+ if match_any_paths(paths,
+ included_patterns=self.patterns,
+ excluded_patterns=self.ignore_patterns,
+ case_sensitive=self.case_sensitive):
+ self.on_any_event(event)
+ _method_map = {
+ EVENT_TYPE_MODIFIED: self.on_modified,
+ EVENT_TYPE_MOVED: self.on_moved,
+ EVENT_TYPE_CREATED: self.on_created,
+ EVENT_TYPE_DELETED: self.on_deleted,
+ }
+ event_type = event.event_type
+ _method_map[event_type](event)
+
+
+class RegexMatchingEventHandler(FileSystemEventHandler):
+ """
+ Matches given regexes with file paths associated with occurring events.
+ """
+
+ def __init__(self, regexes=[r".*"], ignore_regexes=[],
+ ignore_directories=False, case_sensitive=False):
+ super(RegexMatchingEventHandler, self).__init__()
+
+ if case_sensitive:
+ self._regexes = [re.compile(r) for r in regexes]
+ self._ignore_regexes = [re.compile(r) for r in ignore_regexes]
+ else:
+ self._regexes = [re.compile(r, re.I) for r in regexes]
+ self._ignore_regexes = [re.compile(r, re.I) for r in ignore_regexes]
+ self._ignore_directories = ignore_directories
+ self._case_sensitive = case_sensitive
+
+ @property
+ def regexes(self):
+ """
+ (Read-only)
+ Regexes to allow matching event paths.
+ """
+ return self._regexes
+
+ @property
+ def ignore_regexes(self):
+ """
+ (Read-only)
+ Regexes to ignore matching event paths.
+ """
+ return self._ignore_regexes
+
+ @property
+ def ignore_directories(self):
+ """
+ (Read-only)
+ ``True`` if directories should be ignored; ``False`` otherwise.
+ """
+ return self._ignore_directories
+
+ @property
+ def case_sensitive(self):
+ """
+ (Read-only)
+ ``True`` if path names should be matched sensitive to case; ``False``
+ otherwise.
+ """
+ return self._case_sensitive
+
+ def dispatch(self, event):
+ """Dispatches events to the appropriate methods.
+
+ :param event:
+ The event object representing the file system event.
+ :type event:
+ :class:`FileSystemEvent`
+ """
+ if self.ignore_directories and event.is_directory:
+ return
+
+ paths = []
+ if has_attribute(event, 'dest_path'):
+ paths.append(unicode_paths.decode(event.dest_path))
+ if event.src_path:
+ paths.append(unicode_paths.decode(event.src_path))
+
+ if any(r.match(p) for r in self.ignore_regexes for p in paths):
+ return
+
+ if any(r.match(p) for r in self.regexes for p in paths):
+ self.on_any_event(event)
+ _method_map = {
+ EVENT_TYPE_MODIFIED: self.on_modified,
+ EVENT_TYPE_MOVED: self.on_moved,
+ EVENT_TYPE_CREATED: self.on_created,
+ EVENT_TYPE_DELETED: self.on_deleted,
+ }
+ event_type = event.event_type
+ _method_map[event_type](event)
+
+
+class LoggingEventHandler(FileSystemEventHandler):
+ """Logs all the events captured."""
+
+ def on_moved(self, event):
+ super(LoggingEventHandler, self).on_moved(event)
+
+ what = 'directory' if event.is_directory else 'file'
+ logging.info("Moved %s: from %s to %s", what, event.src_path,
+ event.dest_path)
+
+ def on_created(self, event):
+ super(LoggingEventHandler, self).on_created(event)
+
+ what = 'directory' if event.is_directory else 'file'
+ logging.info("Created %s: %s", what, event.src_path)
+
+ def on_deleted(self, event):
+ super(LoggingEventHandler, self).on_deleted(event)
+
+ what = 'directory' if event.is_directory else 'file'
+ logging.info("Deleted %s: %s", what, event.src_path)
+
+ def on_modified(self, event):
+ super(LoggingEventHandler, self).on_modified(event)
+
+ what = 'directory' if event.is_directory else 'file'
+ logging.info("Modified %s: %s", what, event.src_path)
+
+
+class LoggingFileSystemEventHandler(LoggingEventHandler):
+ """
+ For backwards-compatibility. Please use :class:`LoggingEventHandler`
+ instead.
+ """
+
+
+def generate_sub_moved_events(src_dir_path, dest_dir_path):
+ """Generates an event list of :class:`DirMovedEvent` and
+ :class:`FileMovedEvent` objects for all the files and directories within
+ the given moved directory that were moved along with the directory.
+
+ :param src_dir_path:
+ The source path of the moved directory.
+ :param dest_dir_path:
+ The destination path of the moved directory.
+ :returns:
+ An iterable of file system events of type :class:`DirMovedEvent` and
+ :class:`FileMovedEvent`.
+ """
+ for root, directories, filenames in os.walk(dest_dir_path):
+ for directory in directories:
+ full_path = os.path.join(root, directory)
+ renamed_path = full_path.replace(dest_dir_path, src_dir_path) if src_dir_path else None
+ yield DirMovedEvent(renamed_path, full_path)
+ for filename in filenames:
+ full_path = os.path.join(root, filename)
+ renamed_path = full_path.replace(dest_dir_path, src_dir_path) if src_dir_path else None
+ yield FileMovedEvent(renamed_path, full_path)
+
+
+def generate_sub_created_events(src_dir_path):
+ """Generates an event list of :class:`DirCreatedEvent` and
+ :class:`FileCreatedEvent` objects for all the files and directories within
+ the given moved directory that were moved along with the directory.
+
+ :param src_dir_path:
+ The source path of the created directory.
+ :returns:
+ An iterable of file system events of type :class:`DirCreatedEvent` and
+ :class:`FileCreatedEvent`.
+ """
+ for root, directories, filenames in os.walk(src_dir_path):
+ for directory in directories:
+ yield DirCreatedEvent(os.path.join(root, directory))
+ for filename in filenames:
+ yield FileCreatedEvent(os.path.join(root, filename))
\ No newline at end of file
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/__init__.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..96cbd3b9e5fae4d971313fcd651c843fb7c6d3fc
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/__init__.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+:module: watchdog.observers
+:synopsis: Observer that picks a native implementation if available.
+:author: yesudeep@google.com (Yesudeep Mangalapilly)
+
+
+Classes
+=======
+.. autoclass:: Observer
+ :members:
+ :show-inheritance:
+ :inherited-members:
+
+Observer thread that schedules watching directories and dispatches
+calls to event handlers.
+
+You can also import platform specific classes directly and use it instead
+of :class:`Observer`. Here is a list of implemented observer classes.:
+
+============== ================================ ==============================
+Class Platforms Note
+============== ================================ ==============================
+|Inotify| Linux 2.6.13+ ``inotify(7)`` based observer
+|FSEvents| Mac OS X FSEvents based observer
+|Kqueue| Mac OS X and BSD with kqueue(2) ``kqueue(2)`` based observer
+|WinApi| MS Windows Windows API-based observer
+|Polling| Any fallback implementation
+============== ================================ ==============================
+
+.. |Inotify| replace:: :class:`.inotify.InotifyObserver`
+.. |FSEvents| replace:: :class:`.fsevents.FSEventsObserver`
+.. |Kqueue| replace:: :class:`.kqueue.KqueueObserver`
+.. |WinApi| replace:: :class:`.read_directory_changes.WindowsApiObserver`
+.. |WinApiAsync| replace:: :class:`.read_directory_changes_async.WindowsApiAsyncObserver`
+.. |Polling| replace:: :class:`.polling.PollingObserver`
+
+"""
+
+import warnings
+from wandb_watchdog.utils import platform
+from wandb_watchdog.utils import UnsupportedLibc
+
+if platform.is_linux():
+ try:
+ from .inotify import InotifyObserver as Observer
+ except UnsupportedLibc:
+ from .polling import PollingObserver as Observer
+
+elif platform.is_darwin():
+ # FIXME: catching too broad. Error prone
+ try:
+ from .fsevents import FSEventsObserver as Observer
+ except:
+ try:
+ from .kqueue import KqueueObserver as Observer
+ # Note: (tim): Commenting out these warnings since the _watchdog_fsevents
+ # module is not available unless installed directly.
+ # warnings.warn("Failed to import fsevents. Fall back to kqueue")
+ except:
+ from .polling import PollingObserver as Observer
+ # Note: (tim): Commenting out these warnings since the _watchdog_fsevents
+ # module is not available unless installed directly.
+ # warnings.warn("Failed to import fsevents and kqueue. Fall back to polling.")
+
+elif platform.is_bsd():
+ from .kqueue import KqueueObserver as Observer
+
+elif platform.is_windows():
+ # TODO: find a reliable way of checking Windows version and import
+ # polling explicitly for Windows XP
+ try:
+ from .read_directory_changes import WindowsApiObserver as Observer
+ except:
+ from .polling import PollingObserver as Observer
+ warnings.warn("Failed to import read_directory_changes. Fall back to polling.")
+
+else:
+ from .polling import PollingObserver as Observer
+
+__all__ = ["Observer"]
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/api.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/api.py
new file mode 100644
index 0000000000000000000000000000000000000000..4b422e27c6055d2b8e9a1e573e7e99e7621dbe19
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/api.py
@@ -0,0 +1,369 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import with_statement
+import threading
+from wandb_watchdog.utils import BaseThread
+from wandb_watchdog.utils.compat import queue
+from wandb_watchdog.utils.bricks import SkipRepeatsQueue
+
+DEFAULT_EMITTER_TIMEOUT = 1 # in seconds.
+DEFAULT_OBSERVER_TIMEOUT = 1 # in seconds.
+
+
+# Collection classes
+class EventQueue(SkipRepeatsQueue):
+ """Thread-safe event queue based on a special queue that skips adding
+ the same event (:class:`FileSystemEvent`) multiple times consecutively.
+ Thus avoiding dispatching multiple event handling
+ calls when multiple identical events are produced quicker than an observer
+ can consume them.
+ """
+
+
+class ObservedWatch(object):
+ """An scheduled watch.
+
+ :param path:
+ Path string.
+ :param recursive:
+ ``True`` if watch is recursive; ``False`` otherwise.
+ """
+
+ def __init__(self, path, recursive):
+ self._path = path
+ self._is_recursive = recursive
+
+ @property
+ def path(self):
+ """The path that this watch monitors."""
+ return self._path
+
+ @property
+ def is_recursive(self):
+ """Determines whether subdirectories are watched for the path."""
+ return self._is_recursive
+
+ @property
+ def key(self):
+ return self.path, self.is_recursive
+
+ def __eq__(self, watch):
+ return self.key == watch.key
+
+ def __ne__(self, watch):
+ return self.key != watch.key
+
+ def __hash__(self):
+ return hash(self.key)
+
+ def __repr__(self):
+ return "<ObservedWatch: path=%s, is_recursive=%s>" % (
+ self.path, self.is_recursive)
+
+
+# Observer classes
+class EventEmitter(BaseThread):
+ """
+ Producer thread base class subclassed by event emitters
+ that generate events and populate a queue with them.
+
+ :param event_queue:
+ The event queue to populate with generated events.
+ :type event_queue:
+ :class:`watchdog.events.EventQueue`
+ :param watch:
+ The watch to observe and produce events for.
+ :type watch:
+ :class:`ObservedWatch`
+ :param timeout:
+ Timeout (in seconds) between successive attempts at reading events.
+ :type timeout:
+ ``float``
+ """
+
+ def __init__(self, event_queue, watch, timeout=DEFAULT_EMITTER_TIMEOUT):
+ BaseThread.__init__(self)
+ self._event_queue = event_queue
+ self._watch = watch
+ self._timeout = timeout
+
+ @property
+ def timeout(self):
+ """
+ Blocking timeout for reading events.
+ """
+ return self._timeout
+
+ @property
+ def watch(self):
+ """
+ The watch associated with this emitter.
+ """
+ return self._watch
+
+ def queue_event(self, event):
+ """
+ Queues a single event.
+
+ :param event:
+ Event to be queued.
+ :type event:
+ An instance of :class:`watchdog.events.FileSystemEvent`
+ or a subclass.
+ """
+ self._event_queue.put((event, self.watch))
+
+ def queue_events(self, timeout):
+ """Override this method to populate the event queue with events
+ per interval period.
+
+ :param timeout:
+ Timeout (in seconds) between successive attempts at
+ reading events.
+ :type timeout:
+ ``float``
+ """
+
+ def run(self):
+ try:
+ while self.should_keep_running():
+ self.queue_events(self.timeout)
+ finally:
+ pass
+
+
+class EventDispatcher(BaseThread):
+ """
+ Consumer thread base class subclassed by event observer threads
+ that dispatch events from an event queue to appropriate event handlers.
+
+ :param timeout:
+ Event queue blocking timeout (in seconds).
+ :type timeout:
+ ``float``
+ """
+
+ def __init__(self, timeout=DEFAULT_OBSERVER_TIMEOUT):
+ BaseThread.__init__(self)
+ self._event_queue = EventQueue()
+ self._timeout = timeout
+
+ @property
+ def timeout(self):
+ """Event queue block timeout."""
+ return self._timeout
+
+ @property
+ def event_queue(self):
+ """The event queue which is populated with file system events
+ by emitters and from which events are dispatched by a dispatcher
+ thread."""
+ return self._event_queue
+
+ def dispatch_events(self, event_queue, timeout):
+ """Override this method to consume events from an event queue, blocking
+ on the queue for the specified timeout before raising :class:`queue.Empty`.
+
+ :param event_queue:
+ Event queue to populate with one set of events.
+ :type event_queue:
+ :class:`EventQueue`
+ :param timeout:
+ Interval period (in seconds) to wait before timing out on the
+ event queue.
+ :type timeout:
+ ``float``
+ :raises:
+ :class:`queue.Empty`
+ """
+
+ def run(self):
+ while self.should_keep_running():
+ try:
+ self.dispatch_events(self.event_queue, self.timeout)
+ except queue.Empty:
+ continue
+
+
+class BaseObserver(EventDispatcher):
+ """Base observer."""
+
+ def __init__(self, emitter_class, timeout=DEFAULT_OBSERVER_TIMEOUT):
+ EventDispatcher.__init__(self, timeout)
+ self._emitter_class = emitter_class
+ self._lock = threading.RLock()
+ self._watches = set()
+ self._handlers = dict()
+ self._emitters = set()
+ self._emitter_for_watch = dict()
+
+ def _add_emitter(self, emitter):
+ self._emitter_for_watch[emitter.watch] = emitter
+ self._emitters.add(emitter)
+
+ def _remove_emitter(self, emitter):
+ del self._emitter_for_watch[emitter.watch]
+ self._emitters.remove(emitter)
+ emitter.stop()
+ try:
+ emitter.join()
+ except RuntimeError:
+ pass
+
+ def _clear_emitters(self):
+ for emitter in self._emitters:
+ emitter.stop()
+ for emitter in self._emitters:
+ try:
+ emitter.join()
+ except RuntimeError:
+ pass
+ self._emitters.clear()
+ self._emitter_for_watch.clear()
+
+ def _add_handler_for_watch(self, event_handler, watch):
+ if watch not in self._handlers:
+ self._handlers[watch] = set()
+ self._handlers[watch].add(event_handler)
+
+ def _remove_handlers_for_watch(self, watch):
+ del self._handlers[watch]
+
+ @property
+ def emitters(self):
+ """Returns event emitter created by this observer."""
+ return self._emitters
+
+ def start(self):
+ for emitter in self._emitters:
+ emitter.start()
+ super(BaseObserver, self).start()
+
+ def schedule(self, event_handler, path, recursive=False):
+ """
+ Schedules watching a path and calls appropriate methods specified
+ in the given event handler in response to file system events.
+
+ :param event_handler:
+ An event handler instance that has appropriate event handling
+ methods which will be called by the observer in response to
+ file system events.
+ :type event_handler:
+ :class:`watchdog.events.FileSystemEventHandler` or a subclass
+ :param path:
+ Directory path that will be monitored.
+ :type path:
+ ``str``
+ :param recursive:
+ ``True`` if events will be emitted for sub-directories
+ traversed recursively; ``False`` otherwise.
+ :type recursive:
+ ``bool``
+ :return:
+ An :class:`ObservedWatch` object instance representing
+ a watch.
+ """
+ with self._lock:
+ watch = ObservedWatch(path, recursive)
+ self._add_handler_for_watch(event_handler, watch)
+
+ # If we don't have an emitter for this watch already, create it.
+ if self._emitter_for_watch.get(watch) is None:
+ emitter = self._emitter_class(event_queue=self.event_queue,
+ watch=watch,
+ timeout=self.timeout)
+ self._add_emitter(emitter)
+ if self.is_alive():
+ emitter.start()
+ self._watches.add(watch)
+ return watch
+
+ def add_handler_for_watch(self, event_handler, watch):
+ """Adds a handler for the given watch.
+
+ :param event_handler:
+ An event handler instance that has appropriate event handling
+ methods which will be called by the observer in response to
+ file system events.
+ :type event_handler:
+ :class:`watchdog.events.FileSystemEventHandler` or a subclass
+ :param watch:
+ The watch to add a handler for.
+ :type watch:
+ An instance of :class:`ObservedWatch` or a subclass of
+ :class:`ObservedWatch`
+ """
+ with self._lock:
+ self._add_handler_for_watch(event_handler, watch)
+
+ def remove_handler_for_watch(self, event_handler, watch):
+ """Removes a handler for the given watch.
+
+ :param event_handler:
+ An event handler instance that has appropriate event handling
+ methods which will be called by the observer in response to
+ file system events.
+ :type event_handler:
+ :class:`watchdog.events.FileSystemEventHandler` or a subclass
+ :param watch:
+ The watch to remove a handler for.
+ :type watch:
+ An instance of :class:`ObservedWatch` or a subclass of
+ :class:`ObservedWatch`
+ """
+ with self._lock:
+ self._handlers[watch].remove(event_handler)
+
+ def unschedule(self, watch):
+ """Unschedules a watch.
+
+ :param watch:
+ The watch to unschedule.
+ :type watch:
+ An instance of :class:`ObservedWatch` or a subclass of
+ :class:`ObservedWatch`
+ """
+ with self._lock:
+ emitter = self._emitter_for_watch[watch]
+ del self._handlers[watch]
+ self._remove_emitter(emitter)
+ self._watches.remove(watch)
+
+ def unschedule_all(self):
+ """Unschedules all watches and detaches all associated event
+ handlers."""
+ with self._lock:
+ self._handlers.clear()
+ self._clear_emitters()
+ self._watches.clear()
+
+ def on_thread_stop(self):
+ self.unschedule_all()
+
+ def dispatch_events(self, event_queue, timeout):
+ event, watch = event_queue.get(block=True, timeout=timeout)
+
+ with self._lock:
+ # To allow unschedule/stop and safe removal of event handlers
+ # within event handlers itself, check if the handler is still
+ # registered after every dispatch.
+ for handler in list(self._handlers.get(watch, [])):
+ if handler in self._handlers.get(watch, []):
+ handler.dispatch(event)
+ event_queue.task_done()
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/fsevents.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/fsevents.py
new file mode 100644
index 0000000000000000000000000000000000000000..3b7f007ae1e70315853cbfb1473f40a7424a52bc
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/fsevents.py
@@ -0,0 +1,172 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+:module: watchdog.observers.fsevents
+:synopsis: FSEvents based emitter implementation.
+:author: yesudeep@google.com (Yesudeep Mangalapilly)
+:platforms: Mac OS X
+"""
+
+from __future__ import with_statement
+
+import sys
+import threading
+import unicodedata
+import _watchdog_fsevents as _fsevents
+
+from wandb_watchdog.events import (
+ FileDeletedEvent,
+ FileModifiedEvent,
+ FileCreatedEvent,
+ FileMovedEvent,
+ DirDeletedEvent,
+ DirModifiedEvent,
+ DirCreatedEvent,
+ DirMovedEvent
+)
+
+from wandb_watchdog.utils.dirsnapshot import DirectorySnapshot
+from wandb_watchdog.observers.api import (
+ BaseObserver,
+ EventEmitter,
+ DEFAULT_EMITTER_TIMEOUT,
+ DEFAULT_OBSERVER_TIMEOUT
+)
+
+
+class FSEventsEmitter(EventEmitter):
+
+ """
+ Mac OS X FSEvents Emitter class.
+
+ :param event_queue:
+ The event queue to fill with events.
+ :param watch:
+ A watch object representing the directory to monitor.
+ :type watch:
+ :class:`watchdog.observers.api.ObservedWatch`
+ :param timeout:
+ Read events blocking timeout (in seconds).
+ :type timeout:
+ ``float``
+ """
+
+ def __init__(self, event_queue, watch, timeout=DEFAULT_EMITTER_TIMEOUT):
+ EventEmitter.__init__(self, event_queue, watch, timeout)
+ self._lock = threading.Lock()
+ self.snapshot = DirectorySnapshot(watch.path, watch.is_recursive)
+
+ def on_thread_stop(self):
+ _fsevents.remove_watch(self.watch)
+ _fsevents.stop(self)
+
+ def queue_events(self, timeout):
+ with self._lock:
+ if not self.watch.is_recursive\
+ and self.watch.path not in self.pathnames:
+ return
+ new_snapshot = DirectorySnapshot(self.watch.path,
+ self.watch.is_recursive)
+ events = new_snapshot - self.snapshot
+ self.snapshot = new_snapshot
+
+ # Files.
+ for src_path in events.files_deleted:
+ self.queue_event(FileDeletedEvent(src_path))
+ for src_path in events.files_modified:
+ self.queue_event(FileModifiedEvent(src_path))
+ for src_path in events.files_created:
+ self.queue_event(FileCreatedEvent(src_path))
+ for src_path, dest_path in events.files_moved:
+ self.queue_event(FileMovedEvent(src_path, dest_path))
+
+ # Directories.
+ for src_path in events.dirs_deleted:
+ self.queue_event(DirDeletedEvent(src_path))
+ for src_path in events.dirs_modified:
+ self.queue_event(DirModifiedEvent(src_path))
+ for src_path in events.dirs_created:
+ self.queue_event(DirCreatedEvent(src_path))
+ for src_path, dest_path in events.dirs_moved:
+ self.queue_event(DirMovedEvent(src_path, dest_path))
+
+ def run(self):
+ try:
+ def callback(pathnames, flags, emitter=self):
+ emitter.queue_events(emitter.timeout)
+
+ # for pathname, flag in zip(pathnames, flags):
+ # if emitter.watch.is_recursive: # and pathname != emitter.watch.path:
+ # new_sub_snapshot = DirectorySnapshot(pathname, True)
+ # old_sub_snapshot = self.snapshot.copy(pathname)
+ # diff = new_sub_snapshot - old_sub_snapshot
+ # self.snapshot += new_subsnapshot
+ # else:
+ # new_snapshot = DirectorySnapshot(emitter.watch.path, False)
+ # diff = new_snapshot - emitter.snapshot
+ # emitter.snapshot = new_snapshot
+
+ # INFO: FSEvents reports directory notifications recursively
+ # by default, so we do not need to add subdirectory paths.
+ #pathnames = set([self.watch.path])
+ # if self.watch.is_recursive:
+ # for root, directory_names, _ in os.walk(self.watch.path):
+ # for directory_name in directory_names:
+ # full_path = absolute_path(
+ # os.path.join(root, directory_name))
+ # pathnames.add(full_path)
+ self.pathnames = [self.watch.path]
+ _fsevents.add_watch(self,
+ self.watch,
+ callback,
+ self.pathnames)
+ _fsevents.read_events(self)
+ except:
+ pass
+
+
+class FSEventsObserver(BaseObserver):
+
+ def __init__(self, timeout=DEFAULT_OBSERVER_TIMEOUT):
+ BaseObserver.__init__(self, emitter_class=FSEventsEmitter,
+ timeout=timeout)
+
+ def schedule(self, event_handler, path, recursive=False):
+ # Python 2/3 compat
+ try:
+ str_class = unicode
+ except NameError:
+ str_class = str
+
+ # Fix for issue #26: Trace/BPT error when given a unicode path
+ # string. https://github.com/gorakhargosh/watchdog/issues#issue/26
+ if isinstance(path, str_class):
+ #path = unicode(path, 'utf-8')
+ path = unicodedata.normalize('NFC', path)
+ # We only encode the path in Python 2 for backwards compatibility.
+ # On Python 3 we want the path to stay as unicode if possible for
+ # the sake of path matching not having to be rewritten to use the
+ # bytes API instead of strings. The _watchdog_fsevent.so code for
+ # Python 3 can handle both str and bytes paths, which is why we
+ # do not HAVE to encode it with Python 3. The Python 2 code in
+ # _watchdog_fsevents.so was not changed for the sake of backwards
+ # compatibility.
+ if sys.version_info < (3,):
+ path = path.encode('utf-8')
+ return BaseObserver.schedule(self, event_handler, path, recursive)
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/fsevents2.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/fsevents2.py
new file mode 100644
index 0000000000000000000000000000000000000000..bb9433c8a6ba18c7d7886a2b261fd6252c2732af
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/fsevents2.py
@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2014 Thomas Amland <thomas.amland@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+:module: watchdog.observers.fsevents2
+:synopsis: FSEvents based emitter implementation.
+:platforms: Mac OS X
+"""
+
+import os
+import logging
+import unicodedata
+from threading import Thread
+from wandb_watchdog.utils.compat import queue
+
+from wandb_watchdog.events import (
+ FileDeletedEvent,
+ FileModifiedEvent,
+ FileCreatedEvent,
+ FileMovedEvent,
+ DirDeletedEvent,
+ DirModifiedEvent,
+ DirCreatedEvent,
+ DirMovedEvent
+)
+from wandb_watchdog.observers.api import (
+ BaseObserver,
+ EventEmitter,
+ DEFAULT_EMITTER_TIMEOUT,
+ DEFAULT_OBSERVER_TIMEOUT,
+)
+
+# pyobjc
+import AppKit
+from FSEvents import (
+ FSEventStreamCreate,
+ CFRunLoopGetCurrent,
+ FSEventStreamScheduleWithRunLoop,
+ FSEventStreamStart,
+ CFRunLoopRun,
+ CFRunLoopStop,
+ FSEventStreamStop,
+ FSEventStreamInvalidate,
+ FSEventStreamRelease,
+)
+
+from FSEvents import (
+ kCFAllocatorDefault,
+ kCFRunLoopDefaultMode,
+ kFSEventStreamEventIdSinceNow,
+ kFSEventStreamCreateFlagNoDefer,
+ kFSEventStreamCreateFlagFileEvents,
+ kFSEventStreamEventFlagItemCreated,
+ kFSEventStreamEventFlagItemRemoved,
+ kFSEventStreamEventFlagItemInodeMetaMod,
+ kFSEventStreamEventFlagItemRenamed,
+ kFSEventStreamEventFlagItemModified,
+ kFSEventStreamEventFlagItemFinderInfoMod,
+ kFSEventStreamEventFlagItemChangeOwner,
+ kFSEventStreamEventFlagItemXattrMod,
+ kFSEventStreamEventFlagItemIsDir,
+ kFSEventStreamEventFlagItemIsSymlink,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class FSEventsQueue(Thread):
+ """ Low level FSEvents client. """
+
+ def __init__(self, path):
+ Thread.__init__(self)
+ self._queue = queue.Queue()
+ self._run_loop = None
+
+ if isinstance(path, bytes):
+ path = path.decode('utf-8')
+ self._path = unicodedata.normalize('NFC', path)
+
+ context = None
+ latency = 1.0
+ self._stream_ref = FSEventStreamCreate(
+ kCFAllocatorDefault, self._callback, context, [self._path],
+ kFSEventStreamEventIdSinceNow, latency,
+ kFSEventStreamCreateFlagNoDefer | kFSEventStreamCreateFlagFileEvents)
+ if self._stream_ref is None:
+ raise IOError("FSEvents. Could not create stream.")
+
+ def run(self):
+ pool = AppKit.NSAutoreleasePool.alloc().init()
+ self._run_loop = CFRunLoopGetCurrent()
+ FSEventStreamScheduleWithRunLoop(
+ self._stream_ref, self._run_loop, kCFRunLoopDefaultMode)
+ if not FSEventStreamStart(self._stream_ref):
+ FSEventStreamInvalidate(self._stream_ref)
+ FSEventStreamRelease(self._stream_ref)
+ raise IOError("FSEvents. Could not start stream.")
+
+ CFRunLoopRun()
+ FSEventStreamStop(self._stream_ref)
+ FSEventStreamInvalidate(self._stream_ref)
+ FSEventStreamRelease(self._stream_ref)
+ del pool
+ # Make sure waiting thread is notified
+ self._queue.put(None)
+
+ def stop(self):
+ if self._run_loop is not None:
+ CFRunLoopStop(self._run_loop)
+
+ def _callback(self, streamRef, clientCallBackInfo, numEvents, eventPaths, eventFlags, eventIDs):
+ events = [NativeEvent(path, flags, _id) for path, flags, _id in
+ zip(eventPaths, eventFlags, eventIDs)]
+ logger.debug("FSEvents callback. Got %d events:" % numEvents)
+ for e in events:
+ logger.debug(e)
+ self._queue.put(events)
+
+ def read_events(self):
+ """
+ Returns a list or one or more events, or None if there are no more
+ events to be read.
+ """
+ if not self.is_alive():
+ return None
+ return self._queue.get()
+
+
+class NativeEvent(object):
+ def __init__(self, path, flags, event_id):
+ self.path = path
+ self.flags = flags
+ self.event_id = event_id
+ self.is_created = bool(flags & kFSEventStreamEventFlagItemCreated)
+ self.is_removed = bool(flags & kFSEventStreamEventFlagItemRemoved)
+ self.is_renamed = bool(flags & kFSEventStreamEventFlagItemRenamed)
+ self.is_modified = bool(flags & kFSEventStreamEventFlagItemModified)
+ self.is_change_owner = bool(flags & kFSEventStreamEventFlagItemChangeOwner)
+ self.is_inode_meta_mod = bool(flags & kFSEventStreamEventFlagItemInodeMetaMod)
+ self.is_finder_info_mod = bool(flags & kFSEventStreamEventFlagItemFinderInfoMod)
+ self.is_xattr_mod = bool(flags & kFSEventStreamEventFlagItemXattrMod)
+ self.is_symlink = bool(flags & kFSEventStreamEventFlagItemIsSymlink)
+ self.is_directory = bool(flags & kFSEventStreamEventFlagItemIsDir)
+
+ @property
+ def _event_type(self):
+ if self.is_created: return "Created"
+ if self.is_removed: return "Removed"
+ if self.is_renamed: return "Renamed"
+ if self.is_modified: return "Modified"
+ if self.is_inode_meta_mod: return "InodeMetaMod"
+ if self.is_xattr_mod: return "XattrMod"
+ return "Unknown"
+
+ def __repr__(self):
+ s ="<NativeEvent: path=%s, type=%s, is_dir=%s, flags=%s, id=%s>"
+ return s % (repr(self.path), self._event_type, self.is_directory, hex(self.flags), self.event_id)
+
+
+class FSEventsEmitter(EventEmitter):
+ """
+ FSEvents based event emitter. Handles conversion of native events.
+ """
+
+ def __init__(self, event_queue, watch, timeout=DEFAULT_EMITTER_TIMEOUT):
+ EventEmitter.__init__(self, event_queue, watch, timeout)
+ self._fsevents = FSEventsQueue(watch.path)
+ self._fsevents.start()
+
+ def on_thread_stop(self):
+ self._fsevents.stop()
+
+ def queue_events(self, timeout):
+ events = self._fsevents.read_events()
+ if events is None:
+ return
+ i = 0
+ while i < len(events):
+ event = events[i]
+
+ # For some reason the create and remove flags are sometimes also
+ # set for rename and modify type events, so let those take
+ # precedence.
+ if event.is_renamed:
+ # Internal moves appears to always be consecutive in the same
+ # buffer and have IDs differ by exactly one (while others
+ # don't) making it possible to pair up the two events coming
+ # from a singe move operation. (None of this is documented!)
+ # Otherwise, guess whether file was moved in or out.
+ #TODO: handle id wrapping
+ if (i+1 < len(events) and events[i+1].is_renamed and
+ events[i+1].event_id == event.event_id + 1):
+ cls = DirMovedEvent if event.is_directory else FileMovedEvent
+ self.queue_event(cls(event.path, events[i+1].path))
+ self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
+ self.queue_event(DirModifiedEvent(os.path.dirname(events[i+1].path)))
+ i += 1
+ elif os.path.exists(event.path):
+ cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
+ self.queue_event(cls(event.path))
+ self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
+ else:
+ cls = DirDeletedEvent if event.is_directory else FileDeletedEvent
+ self.queue_event(cls(event.path))
+ self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
+ #TODO: generate events for tree
+
+ elif event.is_modified or event.is_inode_meta_mod or event.is_xattr_mod :
+ cls = DirModifiedEvent if event.is_directory else FileModifiedEvent
+ self.queue_event(cls(event.path))
+
+ elif event.is_created:
+ cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
+ self.queue_event(cls(event.path))
+ self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
+
+ elif event.is_removed:
+ cls = DirDeletedEvent if event.is_directory else FileDeletedEvent
+ self.queue_event(cls(event.path))
+ self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
+ i += 1
+
+
+class FSEventsObserver2(BaseObserver):
+ def __init__(self, timeout=DEFAULT_OBSERVER_TIMEOUT):
+ BaseObserver.__init__(self, emitter_class=FSEventsEmitter, timeout=timeout)
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify.py
new file mode 100644
index 0000000000000000000000000000000000000000..9785148f61e14c73f030159dc2227323b20281b3
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify.py
@@ -0,0 +1,218 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+:module: watchdog.observers.inotify
+:synopsis: ``inotify(7)`` based emitter implementation.
+:author: Sebastien Martini <seb@dbzteam.org>
+:author: Luke McCarthy <luke@iogopro.co.uk>
+:author: yesudeep@google.com (Yesudeep Mangalapilly)
+:author: Tim Cuthbertson <tim+github@gfxmonk.net>
+:platforms: Linux 2.6.13+.
+
+.. ADMONITION:: About system requirements
+
+ Recommended minimum kernel version: 2.6.25.
+
+ Quote from the inotify(7) man page:
+
+ "Inotify was merged into the 2.6.13 Linux kernel. The required library
+ interfaces were added to glibc in version 2.4. (IN_DONT_FOLLOW,
+ IN_MASK_ADD, and IN_ONLYDIR were only added in version 2.5.)"
+
+ Therefore, you must ensure the system is running at least these versions
+ appropriate libraries and the kernel.
+
+.. ADMONITION:: About recursiveness, event order, and event coalescing
+
+ Quote from the inotify(7) man page:
+
+ If successive output inotify events produced on the inotify file
+ descriptor are identical (same wd, mask, cookie, and name) then they
+ are coalesced into a single event if the older event has not yet been
+ read (but see BUGS).
+
+ The events returned by reading from an inotify file descriptor form
+ an ordered queue. Thus, for example, it is guaranteed that when
+ renaming from one directory to another, events will be produced in
+ the correct order on the inotify file descriptor.
+
+ ...
+
+ Inotify monitoring of directories is not recursive: to monitor
+ subdirectories under a directory, additional watches must be created.
+
+ This emitter implementation therefore automatically adds watches for
+ sub-directories if running in recursive mode.
+
+Some extremely useful articles and documentation:
+
+.. _inotify FAQ: http://inotify.aiken.cz/?section=inotify&page=faq&lang=en
+.. _intro to inotify: http://www.linuxjournal.com/article/8478
+
+"""
+
+from __future__ import with_statement
+
+import os
+import threading
+from .inotify_buffer import InotifyBuffer
+
+from wandb_watchdog.observers.api import (
+ EventEmitter,
+ BaseObserver,
+ DEFAULT_EMITTER_TIMEOUT,
+ DEFAULT_OBSERVER_TIMEOUT
+)
+
+from wandb_watchdog.events import (
+ DirDeletedEvent,
+ DirModifiedEvent,
+ DirMovedEvent,
+ DirCreatedEvent,
+ FileDeletedEvent,
+ FileModifiedEvent,
+ FileMovedEvent,
+ FileCreatedEvent,
+ generate_sub_moved_events,
+ generate_sub_created_events,
+)
+from wandb_watchdog.utils import unicode_paths
+
+
+class InotifyEmitter(EventEmitter):
+ """
+ inotify(7)-based event emitter.
+
+ :param event_queue:
+ The event queue to fill with events.
+ :param watch:
+ A watch object representing the directory to monitor.
+ :type watch:
+ :class:`watchdog.observers.api.ObservedWatch`
+ :param timeout:
+ Read events blocking timeout (in seconds).
+ :type timeout:
+ ``float``
+ """
+
+ def __init__(self, event_queue, watch, timeout=DEFAULT_EMITTER_TIMEOUT):
+ EventEmitter.__init__(self, event_queue, watch, timeout)
+ self._lock = threading.Lock()
+ self._inotify = None
+
+ def on_thread_start(self):
+ path = unicode_paths.encode(self.watch.path)
+ self._inotify = InotifyBuffer(path, self.watch.is_recursive)
+
+ def on_thread_stop(self):
+ if self._inotify:
+ self._inotify.close()
+
+ def queue_events(self, timeout, full_events=False):
+ #If "full_events" is true, then the method will report unmatched move events as seperate events
+ #This behavior is by default only called by a InotifyFullEmitter
+ with self._lock:
+ event = self._inotify.read_event()
+ if event is None:
+ return
+ if isinstance(event, tuple):
+ move_from, move_to = event
+ src_path = self._decode_path(move_from.src_path)
+ dest_path = self._decode_path(move_to.src_path)
+ cls = DirMovedEvent if move_from.is_directory else FileMovedEvent
+ self.queue_event(cls(src_path, dest_path))
+ self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
+ self.queue_event(DirModifiedEvent(os.path.dirname(dest_path)))
+ if move_from.is_directory and self.watch.is_recursive:
+ for sub_event in generate_sub_moved_events(src_path, dest_path):
+ self.queue_event(sub_event)
+ return
+
+ src_path = self._decode_path(event.src_path)
+ if event.is_moved_to:
+ if (full_events):
+ cls = DirMovedEvent if event.is_directory else FileMovedEvent
+ self.queue_event(cls(None, src_path))
+ else:
+ cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
+ self.queue_event(cls(src_path))
+ self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
+ if event.is_directory and self.watch.is_recursive:
+ for sub_event in generate_sub_created_events(src_path):
+ self.queue_event(sub_event)
+ elif event.is_attrib:
+ cls = DirModifiedEvent if event.is_directory else FileModifiedEvent
+ self.queue_event(cls(src_path))
+ elif event.is_modify:
+ cls = DirModifiedEvent if event.is_directory else FileModifiedEvent
+ self.queue_event(cls(src_path))
+ elif event.is_delete or (event.is_moved_from and not full_events):
+ cls = DirDeletedEvent if event.is_directory else FileDeletedEvent
+ self.queue_event(cls(src_path))
+ self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
+ elif event.is_moved_from and full_events:
+ cls = DirMovedEvent if event.is_directory else FileMovedEvent
+ self.queue_event(cls(src_path, None))
+ self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
+ elif event.is_create:
+ cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
+ self.queue_event(cls(src_path))
+ self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
+
+ def _decode_path(self, path):
+ """ Decode path only if unicode string was passed to this emitter. """
+ if isinstance(self.watch.path, bytes):
+ return path
+ return unicode_paths.decode(path)
+
+
+class InotifyFullEmitter(InotifyEmitter):
+ """
+ inotify(7)-based event emitter. By default this class produces move events even if they are not matched
+ Such move events will have a ``None`` value for the unmatched part.
+
+ :param event_queue:
+ The event queue to fill with events.
+ :param watch:
+ A watch object representing the directory to monitor.
+ :type watch:
+ :class:`watchdog.observers.api.ObservedWatch`
+ :param timeout:
+ Read events blocking timeout (in seconds).
+ :type timeout:
+ ``float``
+ """
+ def __init__(self, event_queue, watch, timeout=DEFAULT_EMITTER_TIMEOUT):
+ InotifyEmitter.__init__(self, event_queue, watch, timeout)
+
+ def queue_events(self, timeout, events=True):
+ InotifyEmitter.queue_events(self, timeout, full_events=events)
+
+class InotifyObserver(BaseObserver):
+ """
+ Observer thread that schedules watching directories and dispatches
+ calls to event handlers.
+ """
+
+ def __init__(self, timeout=DEFAULT_OBSERVER_TIMEOUT, generate_full_events=False):
+ if (generate_full_events):
+ BaseObserver.__init__(self, emitter_class=InotifyFullEmitter, timeout=timeout)
+ else:
+ BaseObserver.__init__(self, emitter_class=InotifyEmitter,
+ timeout=timeout)
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify_buffer.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify_buffer.py
new file mode 100644
index 0000000000000000000000000000000000000000..58394c5fa9dcd629ce0602fbe31104daf4cf4fd7
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify_buffer.py
@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2014 Thomas Amland <thomas.amland@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from wandb_watchdog.utils import BaseThread
+from wandb_watchdog.utils.delayed_queue import DelayedQueue
+from wandb_watchdog.observers.inotify_c import Inotify
+
+logger = logging.getLogger(__name__)
+
+
+class InotifyBuffer(BaseThread):
+ """A wrapper for `Inotify` that holds events for `delay` seconds. During
+ this time, IN_MOVED_FROM and IN_MOVED_TO events are paired.
+ """
+
+ delay = 0.5
+
+ def __init__(self, path, recursive=False):
+ BaseThread.__init__(self)
+ self._queue = DelayedQueue(self.delay)
+ self._inotify = Inotify(path, recursive)
+ self.start()
+
+ def read_event(self):
+ """Returns a single event or a tuple of from/to events in case of a
+ paired move event. If this buffer has been closed, immediately return
+ None.
+ """
+ return self._queue.get()
+
+ def on_thread_stop(self):
+ self._inotify.close()
+ self._queue.close()
+
+ def close(self):
+ self.stop()
+ self.join()
+
+ def run(self):
+ """Read event from `inotify` and add them to `queue`. When reading a
+ IN_MOVE_TO event, remove the previous added matching IN_MOVE_FROM event
+ and add them back to the queue as a tuple.
+ """
+ deleted_self = False
+ while self.should_keep_running() and not deleted_self:
+ inotify_events = self._inotify.read_events()
+ for inotify_event in inotify_events:
+ logger.debug("in-event %s", inotify_event)
+ if inotify_event.is_moved_to:
+
+ def matching_from_event(event):
+ return (not isinstance(event, tuple) and event.is_moved_from
+ and event.cookie == inotify_event.cookie)
+
+ from_event = self._queue.remove(matching_from_event)
+ if from_event is not None:
+ self._queue.put((from_event, inotify_event))
+ else:
+ logger.debug("could not find matching move_from event")
+ self._queue.put(inotify_event)
+ else:
+ self._queue.put(inotify_event)
+
+ if inotify_event.is_delete_self and \
+ inotify_event.src_path == self._inotify.path:
+ # Deleted the watched directory, stop watching for events
+ deleted_self = True
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify_c.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify_c.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d60c4856d7ffa854b4148b77d1b75c2d8f469e2
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify_c.py
@@ -0,0 +1,575 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import with_statement
+import os
+import errno
+import struct
+import threading
+import ctypes
+import ctypes.util
+from functools import reduce
+from ctypes import c_int, c_char_p, c_uint32
+from wandb_watchdog.utils import has_attribute
+from wandb_watchdog.utils import UnsupportedLibc
+
+
+def _load_libc():
+ libc_path = None
+ try:
+ libc_path = ctypes.util.find_library('c')
+ except (OSError, IOError, RuntimeError):
+ # Note: find_library will on some platforms raise these undocumented
+ # errors, e.g.on android IOError "No usable temporary directory found"
+ # will be raised.
+ pass
+
+ if libc_path is not None:
+ return ctypes.CDLL(libc_path)
+
+ # Fallbacks
+ try:
+ return ctypes.CDLL('libc.so')
+ except (OSError, IOError):
+ pass
+
+ try:
+ return ctypes.CDLL('libc.so.6')
+ except (OSError, IOError):
+ pass
+
+ # uClibc
+ try:
+ return ctypes.CDLL('libc.so.0')
+ except (OSError, IOError) as err:
+ raise err
+
+
+libc = _load_libc()
+
+if not has_attribute(libc, 'inotify_init') or \
+ not has_attribute(libc, 'inotify_add_watch') or \
+ not has_attribute(libc, 'inotify_rm_watch'):
+ raise UnsupportedLibc("Unsupported libc version found: %s" % libc._name)
+
+inotify_add_watch = ctypes.CFUNCTYPE(c_int, c_int, c_char_p, c_uint32, use_errno=True)(
+ ("inotify_add_watch", libc))
+
+inotify_rm_watch = ctypes.CFUNCTYPE(c_int, c_int, c_uint32, use_errno=True)(
+ ("inotify_rm_watch", libc))
+
+inotify_init = ctypes.CFUNCTYPE(c_int, use_errno=True)(
+ ("inotify_init", libc))
+
+
+class InotifyConstants(object):
+ # User-space events
+ IN_ACCESS = 0x00000001 # File was accessed.
+ IN_MODIFY = 0x00000002 # File was modified.
+ IN_ATTRIB = 0x00000004 # Meta-data changed.
+ IN_CLOSE_WRITE = 0x00000008 # Writable file was closed.
+ IN_CLOSE_NOWRITE = 0x00000010 # Unwritable file closed.
+ IN_OPEN = 0x00000020 # File was opened.
+ IN_MOVED_FROM = 0x00000040 # File was moved from X.
+ IN_MOVED_TO = 0x00000080 # File was moved to Y.
+ IN_CREATE = 0x00000100 # Subfile was created.
+ IN_DELETE = 0x00000200 # Subfile was deleted.
+ IN_DELETE_SELF = 0x00000400 # Self was deleted.
+ IN_MOVE_SELF = 0x00000800 # Self was moved.
+
+ # Helper user-space events.
+ IN_CLOSE = IN_CLOSE_WRITE | IN_CLOSE_NOWRITE # Close.
+ IN_MOVE = IN_MOVED_FROM | IN_MOVED_TO # Moves.
+
+ # Events sent by the kernel to a watch.
+ IN_UNMOUNT = 0x00002000 # Backing file system was unmounted.
+ IN_Q_OVERFLOW = 0x00004000 # Event queued overflowed.
+ IN_IGNORED = 0x00008000 # File was ignored.
+
+ # Special flags.
+ IN_ONLYDIR = 0x01000000 # Only watch the path if it's a directory.
+ IN_DONT_FOLLOW = 0x02000000 # Do not follow a symbolic link.
+ IN_EXCL_UNLINK = 0x04000000 # Exclude events on unlinked objects
+ IN_MASK_ADD = 0x20000000 # Add to the mask of an existing watch.
+ IN_ISDIR = 0x40000000 # Event occurred against directory.
+ IN_ONESHOT = 0x80000000 # Only send event once.
+
+ # All user-space events.
+ IN_ALL_EVENTS = reduce(
+ lambda x, y: x | y, [
+ IN_ACCESS,
+ IN_MODIFY,
+ IN_ATTRIB,
+ IN_CLOSE_WRITE,
+ IN_CLOSE_NOWRITE,
+ IN_OPEN,
+ IN_MOVED_FROM,
+ IN_MOVED_TO,
+ IN_DELETE,
+ IN_CREATE,
+ IN_DELETE_SELF,
+ IN_MOVE_SELF,
+ ])
+
+ # Flags for ``inotify_init1``
+ IN_CLOEXEC = 0x02000000
+ IN_NONBLOCK = 0x00004000
+
+
+# Watchdog's API cares only about these events.
+WATCHDOG_ALL_EVENTS = reduce(
+ lambda x, y: x | y, [
+ InotifyConstants.IN_MODIFY,
+ InotifyConstants.IN_ATTRIB,
+ InotifyConstants.IN_MOVED_FROM,
+ InotifyConstants.IN_MOVED_TO,
+ InotifyConstants.IN_CREATE,
+ InotifyConstants.IN_DELETE,
+ InotifyConstants.IN_DELETE_SELF,
+ InotifyConstants.IN_DONT_FOLLOW,
+ ])
+
+
+class inotify_event_struct(ctypes.Structure):
+ """
+ Structure representation of the inotify_event structure
+ (used in buffer size calculations)::
+
+ struct inotify_event {
+ __s32 wd; /* watch descriptor */
+ __u32 mask; /* watch mask */
+ __u32 cookie; /* cookie to synchronize two events */
+ __u32 len; /* length (including nulls) of name */
+ char name[0]; /* stub for possible name */
+ };
+ """
+ _fields_ = [('wd', c_int),
+ ('mask', c_uint32),
+ ('cookie', c_uint32),
+ ('len', c_uint32),
+ ('name', c_char_p)]
+
+
+EVENT_SIZE = ctypes.sizeof(inotify_event_struct)
+DEFAULT_NUM_EVENTS = 2048
+DEFAULT_EVENT_BUFFER_SIZE = DEFAULT_NUM_EVENTS * (EVENT_SIZE + 16)
+
+
+class Inotify(object):
+ """
+ Linux inotify(7) API wrapper class.
+
+ :param path:
+ The directory path for which we want an inotify object.
+ :type path:
+ :class:`bytes`
+ :param recursive:
+ ``True`` if subdirectories should be monitored; ``False`` otherwise.
+ """
+
+ def __init__(self, path, recursive=False, event_mask=WATCHDOG_ALL_EVENTS):
+ # The file descriptor associated with the inotify instance.
+ inotify_fd = inotify_init()
+ if inotify_fd == -1:
+ Inotify._raise_error()
+ self._inotify_fd = inotify_fd
+ self._lock = threading.Lock()
+
+ # Stores the watch descriptor for a given path.
+ self._wd_for_path = dict()
+ self._path_for_wd = dict()
+
+ self._path = path
+ self._event_mask = event_mask
+ self._is_recursive = recursive
+ self._add_dir_watch(path, recursive, event_mask)
+ self._moved_from_events = dict()
+
+ @property
+ def event_mask(self):
+ """The event mask for this inotify instance."""
+ return self._event_mask
+
+ @property
+ def path(self):
+ """The path associated with the inotify instance."""
+ return self._path
+
+ @property
+ def is_recursive(self):
+ """Whether we are watching directories recursively."""
+ return self._is_recursive
+
+ @property
+ def fd(self):
+ """The file descriptor associated with the inotify instance."""
+ return self._inotify_fd
+
+ def clear_move_records(self):
+ """Clear cached records of MOVED_FROM events"""
+ self._moved_from_events = dict()
+
+ def source_for_move(self, destination_event):
+ """
+ The source path corresponding to the given MOVED_TO event.
+
+ If the source path is outside the monitored directories, None
+ is returned instead.
+ """
+ if destination_event.cookie in self._moved_from_events:
+ return self._moved_from_events[destination_event.cookie].src_path
+ else:
+ return None
+
+ def remember_move_from_event(self, event):
+ """
+ Save this event as the source event for future MOVED_TO events to
+ reference.
+ """
+ self._moved_from_events[event.cookie] = event
+
+ def add_watch(self, path):
+ """
+ Adds a watch for the given path.
+
+ :param path:
+ Path to begin monitoring.
+ """
+ with self._lock:
+ self._add_watch(path, self._event_mask)
+
+ def remove_watch(self, path):
+ """
+ Removes a watch for the given path.
+
+ :param path:
+ Path string for which the watch will be removed.
+ """
+ with self._lock:
+ wd = self._wd_for_path.pop(path)
+ del self._path_for_wd[wd]
+ if inotify_rm_watch(self._inotify_fd, wd) == -1:
+ Inotify._raise_error()
+
+ def close(self):
+ """
+ Closes the inotify instance and removes all associated watches.
+ """
+ with self._lock:
+ if self._path in self._wd_for_path:
+ wd = self._wd_for_path[self._path]
+ inotify_rm_watch(self._inotify_fd, wd)
+ os.close(self._inotify_fd)
+
+ def read_events(self, event_buffer_size=DEFAULT_EVENT_BUFFER_SIZE):
+ """
+ Reads events from inotify and yields them.
+ """
+ # HACK: We need to traverse the directory path
+ # recursively and simulate events for newly
+ # created subdirectories/files. This will handle
+ # mkdir -p foobar/blah/bar; touch foobar/afile
+
+ def _recursive_simulate(src_path):
+ events = []
+ for root, dirnames, filenames in os.walk(src_path):
+ for dirname in dirnames:
+ try:
+ full_path = os.path.join(root, dirname)
+ wd_dir = self._add_watch(full_path, self._event_mask)
+ e = InotifyEvent(
+ wd_dir, InotifyConstants.IN_CREATE | InotifyConstants.IN_ISDIR, 0, dirname, full_path)
+ events.append(e)
+ except OSError:
+ pass
+ for filename in filenames:
+ full_path = os.path.join(root, filename)
+ wd_parent_dir = self._wd_for_path[os.path.dirname(full_path)]
+ e = InotifyEvent(
+ wd_parent_dir, InotifyConstants.IN_CREATE, 0, filename, full_path)
+ events.append(e)
+ return events
+
+ event_buffer = None
+ while True:
+ try:
+ event_buffer = os.read(self._inotify_fd, event_buffer_size)
+ except OSError as e:
+ if e.errno == errno.EINTR:
+ continue
+ break
+
+ with self._lock:
+ event_list = []
+ for wd, mask, cookie, name in Inotify._parse_event_buffer(event_buffer):
+ if wd == -1:
+ continue
+ wd_path = self._path_for_wd[wd]
+ src_path = os.path.join(wd_path, name) if name else wd_path #avoid trailing slash
+ inotify_event = InotifyEvent(wd, mask, cookie, name, src_path)
+
+ if inotify_event.is_moved_from:
+ self.remember_move_from_event(inotify_event)
+ elif inotify_event.is_moved_to:
+ move_src_path = self.source_for_move(inotify_event)
+ if move_src_path in self._wd_for_path:
+ moved_wd = self._wd_for_path[move_src_path]
+ del self._wd_for_path[move_src_path]
+ self._wd_for_path[inotify_event.src_path] = moved_wd
+ self._path_for_wd[moved_wd] = inotify_event.src_path
+ src_path = os.path.join(wd_path, name)
+ inotify_event = InotifyEvent(wd, mask, cookie, name, src_path)
+
+ if inotify_event.is_ignored:
+ # Clean up book-keeping for deleted watches.
+ path = self._path_for_wd.pop(wd)
+ if self._wd_for_path[path] == wd:
+ del self._wd_for_path[path]
+ continue
+
+ event_list.append(inotify_event)
+
+ if (self.is_recursive and inotify_event.is_directory and
+ inotify_event.is_create):
+
+ # TODO: When a directory from another part of the
+ # filesystem is moved into a watched directory, this
+ # will not generate events for the directory tree.
+ # We need to coalesce IN_MOVED_TO events and those
+ # IN_MOVED_TO events which don't pair up with
+ # IN_MOVED_FROM events should be marked IN_CREATE
+ # instead relative to this directory.
+ try:
+ self._add_watch(src_path, self._event_mask)
+ except OSError:
+ continue
+
+ event_list.extend(_recursive_simulate(src_path))
+
+ return event_list
+
+ # Non-synchronized methods.
+ def _add_dir_watch(self, path, recursive, mask):
+ """
+ Adds a watch (optionally recursively) for the given directory path
+ to monitor events specified by the mask.
+
+ :param path:
+ Path to monitor
+ :param recursive:
+ ``True`` to monitor recursively.
+ :param mask:
+ Event bit mask.
+ """
+ if not os.path.isdir(path):
+ raise OSError('Path is not a directory')
+ self._add_watch(path, mask)
+ if recursive:
+ for root, dirnames, _ in os.walk(path):
+ for dirname in dirnames:
+ full_path = os.path.join(root, dirname)
+ if os.path.islink(full_path):
+ continue
+ self._add_watch(full_path, mask)
+
+ def _add_watch(self, path, mask):
+ """
+ Adds a watch for the given path to monitor events specified by the
+ mask.
+
+ :param path:
+ Path to monitor
+ :param mask:
+ Event bit mask.
+ """
+ wd = inotify_add_watch(self._inotify_fd, path, mask)
+ if wd == -1:
+ Inotify._raise_error()
+ self._wd_for_path[path] = wd
+ self._path_for_wd[wd] = path
+ return wd
+
+ @staticmethod
+ def _raise_error():
+ """
+ Raises errors for inotify failures.
+ """
+ err = ctypes.get_errno()
+ if err == errno.ENOSPC:
+ raise OSError("inotify watch limit reached")
+ elif err == errno.EMFILE:
+ raise OSError("inotify instance limit reached")
+ else:
+ raise OSError(os.strerror(err))
+
+ @staticmethod
+ def _parse_event_buffer(event_buffer):
+ """
+ Parses an event buffer of ``inotify_event`` structs returned by
+ inotify::
+
+ struct inotify_event {
+ __s32 wd; /* watch descriptor */
+ __u32 mask; /* watch mask */
+ __u32 cookie; /* cookie to synchronize two events */
+ __u32 len; /* length (including nulls) of name */
+ char name[0]; /* stub for possible name */
+ };
+
+ The ``cookie`` member of this struct is used to pair two related
+ events, for example, it pairs an IN_MOVED_FROM event with an
+ IN_MOVED_TO event.
+ """
+ i = 0
+ while i + 16 <= len(event_buffer):
+ wd, mask, cookie, length = struct.unpack_from('iIII', event_buffer, i)
+ name = event_buffer[i + 16:i + 16 + length].rstrip(b'\0')
+ i += 16 + length
+ yield wd, mask, cookie, name
+
+
+class InotifyEvent(object):
+ """
+ Inotify event struct wrapper.
+
+ :param wd:
+ Watch descriptor
+ :param mask:
+ Event mask
+ :param cookie:
+ Event cookie
+ :param name:
+ Event name.
+ :param src_path:
+ Event source path
+ """
+
+ def __init__(self, wd, mask, cookie, name, src_path):
+ self._wd = wd
+ self._mask = mask
+ self._cookie = cookie
+ self._name = name
+ self._src_path = src_path
+
+ @property
+ def src_path(self):
+ return self._src_path
+
+ @property
+ def wd(self):
+ return self._wd
+
+ @property
+ def mask(self):
+ return self._mask
+
+ @property
+ def cookie(self):
+ return self._cookie
+
+ @property
+ def name(self):
+ return self._name
+
+ @property
+ def is_modify(self):
+ return self._mask & InotifyConstants.IN_MODIFY > 0
+
+ @property
+ def is_close_write(self):
+ return self._mask & InotifyConstants.IN_CLOSE_WRITE > 0
+
+ @property
+ def is_close_nowrite(self):
+ return self._mask & InotifyConstants.IN_CLOSE_NOWRITE > 0
+
+ @property
+ def is_access(self):
+ return self._mask & InotifyConstants.IN_ACCESS > 0
+
+ @property
+ def is_delete(self):
+ return self._mask & InotifyConstants.IN_DELETE > 0
+
+ @property
+ def is_delete_self(self):
+ return self._mask & InotifyConstants.IN_DELETE_SELF > 0
+
+ @property
+ def is_create(self):
+ return self._mask & InotifyConstants.IN_CREATE > 0
+
+ @property
+ def is_moved_from(self):
+ return self._mask & InotifyConstants.IN_MOVED_FROM > 0
+
+ @property
+ def is_moved_to(self):
+ return self._mask & InotifyConstants.IN_MOVED_TO > 0
+
+ @property
+ def is_move(self):
+ return self._mask & InotifyConstants.IN_MOVE > 0
+
+ @property
+ def is_move_self(self):
+ return self._mask & InotifyConstants.IN_MOVE_SELF > 0
+
+ @property
+ def is_attrib(self):
+ return self._mask & InotifyConstants.IN_ATTRIB > 0
+
+ @property
+ def is_ignored(self):
+ return self._mask & InotifyConstants.IN_IGNORED > 0
+
+ @property
+ def is_directory(self):
+ # It looks like the kernel does not provide this information for
+ # IN_DELETE_SELF and IN_MOVE_SELF. In this case, assume it's a dir.
+ # See also: https://github.com/seb-m/pyinotify/blob/2c7e8f8/python2/pyinotify.py#L897
+ return (self.is_delete_self or self.is_move_self or
+ self._mask & InotifyConstants.IN_ISDIR > 0)
+
+ @property
+ def key(self):
+ return self._src_path, self._wd, self._mask, self._cookie, self._name
+
+ def __eq__(self, inotify_event):
+ return self.key == inotify_event.key
+
+ def __ne__(self, inotify_event):
+ return self.key == inotify_event.key
+
+ def __hash__(self):
+ return hash(self.key)
+
+ @staticmethod
+ def _get_mask_string(mask):
+ masks = []
+ for c in dir(InotifyConstants):
+ if c.startswith('IN_') and c not in ['IN_ALL_EVENTS', 'IN_CLOSE', 'IN_MOVE']:
+ c_val = getattr(InotifyConstants, c)
+ if mask & c_val:
+ masks.append(c)
+ mask_string = '|'.join(masks)
+ return mask_string
+
+ def __repr__(self):
+ mask_string = self._get_mask_string(self.mask)
+ s = "<InotifyEvent: src_path=%s, wd=%d, mask=%s, cookie=%d, name=%s>"
+ return s % (self.src_path, self.wd, mask_string, self.cookie, self.name)
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/kqueue.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/kqueue.py
new file mode 100644
index 0000000000000000000000000000000000000000..faf10cd12b8bf231ca42ea145d7260ae68b05f41
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/kqueue.py
@@ -0,0 +1,730 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+:module: watchdog.observers.kqueue
+:synopsis: ``kqueue(2)`` based emitter implementation.
+:author: yesudeep@google.com (Yesudeep Mangalapilly)
+:platforms: Mac OS X and BSD with kqueue(2).
+
+.. WARNING:: kqueue is a very heavyweight way to monitor file systems.
+ Each kqueue-detected directory modification triggers
+ a full directory scan. Traversing the entire directory tree
+ and opening file descriptors for all files will create
+ performance problems. We need to find a way to re-scan
+ only those directories which report changes and do a diff
+ between two sub-DirectorySnapshots perhaps.
+
+.. ADMONITION:: About ``select.kqueue`` and Python versions
+
+ * Python 2.5 does not ship with ``select.kqueue``
+ * Python 2.6 ships with a broken ``select.kqueue`` that cannot take
+ multiple events in the event list passed to ``kqueue.control``.
+ * Python 2.7 ships with a working ``select.kqueue``
+ implementation.
+
+ I have backported the Python 2.7 implementation to Python 2.5 and 2.6
+ in the ``select_backport`` package available on PyPI.
+
+.. ADMONITION:: About OS X performance guidelines
+
+ Quote from the `Mac OS X File System Performance Guidelines`_:
+
+ "When you only want to track changes on a file or directory, be sure to
+ open it using the ``O_EVTONLY`` flag. This flag prevents the file or
+ directory from being marked as open or in use. This is important
+ if you are tracking files on a removable volume and the user tries to
+ unmount the volume. With this flag in place, the system knows it can
+ dismiss the volume. If you had opened the files or directories without
+ this flag, the volume would be marked as busy and would not be
+ unmounted."
+
+ ``O_EVTONLY`` is defined as ``0x8000`` in the OS X header files.
+ More information here: http://www.mlsite.net/blog/?p=2312
+
+Classes
+-------
+.. autoclass:: KqueueEmitter
+ :members:
+ :show-inheritance:
+
+Collections and Utility Classes
+-------------------------------
+.. autoclass:: KeventDescriptor
+ :members:
+ :show-inheritance:
+
+.. autoclass:: KeventDescriptorSet
+ :members:
+ :show-inheritance:
+
+.. _Mac OS X File System Performance Guidelines: http://developer.apple.com/library/ios/#documentation/Performance/Conceptual/FileSystem/Articles/TrackingChanges.html#//apple_ref/doc/uid/20001993-CJBJFIDD
+
+"""
+
+from __future__ import with_statement
+from wandb_watchdog.utils import platform
+
+import threading
+import errno
+import sys
+import stat
+import os
+
+# See the notes for this module in the documentation above ^.
+#import select
+# if not has_attribute(select, 'kqueue') or sys.version_info < (2, 7, 0):
+if sys.version_info < (2, 7, 0):
+ import select_backport as select
+else:
+ import select
+
+from wandb_watchdog.observers.api import (
+ BaseObserver,
+ EventEmitter,
+ DEFAULT_OBSERVER_TIMEOUT,
+ DEFAULT_EMITTER_TIMEOUT
+)
+
+from wandb_watchdog.utils.dirsnapshot import DirectorySnapshot
+
+from wandb_watchdog.events import (
+ DirMovedEvent,
+ DirDeletedEvent,
+ DirCreatedEvent,
+ DirModifiedEvent,
+ FileMovedEvent,
+ FileDeletedEvent,
+ FileCreatedEvent,
+ FileModifiedEvent,
+ EVENT_TYPE_MOVED,
+ EVENT_TYPE_DELETED,
+ EVENT_TYPE_CREATED
+)
+
+# Maximum number of events to process.
+MAX_EVENTS = 4096
+
+# O_EVTONLY value from the header files for OS X only.
+O_EVTONLY = 0x8000
+
+# Pre-calculated values for the kevent filter, flags, and fflags attributes.
+if platform.is_darwin():
+ WATCHDOG_OS_OPEN_FLAGS = O_EVTONLY
+else:
+ WATCHDOG_OS_OPEN_FLAGS = os.O_RDONLY | os.O_NONBLOCK
+WATCHDOG_KQ_FILTER = select.KQ_FILTER_VNODE
+WATCHDOG_KQ_EV_FLAGS = select.KQ_EV_ADD | select.KQ_EV_ENABLE | select.KQ_EV_CLEAR
+WATCHDOG_KQ_FFLAGS = (
+ select.KQ_NOTE_DELETE |
+ select.KQ_NOTE_WRITE |
+ select.KQ_NOTE_EXTEND |
+ select.KQ_NOTE_ATTRIB |
+ select.KQ_NOTE_LINK |
+ select.KQ_NOTE_RENAME |
+ select.KQ_NOTE_REVOKE
+)
+
+
+def absolute_path(path):
+ """Return absolute normalized path for given path."""
+ return os.path.abspath(os.path.normpath(path))
+
+
+# Flag tests.
+
+
+def is_deleted(kev):
+ """Determines whether the given kevent represents deletion."""
+ return kev.fflags & select.KQ_NOTE_DELETE
+
+
+def is_modified(kev):
+ """Determines whether the given kevent represents modification."""
+ fflags = kev.fflags
+ return (fflags & select.KQ_NOTE_EXTEND) or (fflags & select.KQ_NOTE_WRITE)
+
+
+def is_attrib_modified(kev):
+ """Determines whether the given kevent represents attribute modification."""
+ return kev.fflags & select.KQ_NOTE_ATTRIB
+
+
+def is_renamed(kev):
+ """Determines whether the given kevent represents movement."""
+ return kev.fflags & select.KQ_NOTE_RENAME
+
+
+class KeventDescriptorSet(object):
+
+ """
+ Thread-safe kevent descriptor collection.
+ """
+
+ def __init__(self):
+ # Set of KeventDescriptor
+ self._descriptors = set()
+
+ # Descriptor for a given path.
+ self._descriptor_for_path = dict()
+
+ # Descriptor for a given fd.
+ self._descriptor_for_fd = dict()
+
+ # List of kevent objects.
+ self._kevents = list()
+
+ self._lock = threading.Lock()
+
+ @property
+ def kevents(self):
+ """
+ List of kevents monitored.
+ """
+ with self._lock:
+ return self._kevents
+
+ @property
+ def paths(self):
+ """
+ List of paths for which kevents have been created.
+ """
+ with self._lock:
+ return list(self._descriptor_for_path.keys())
+
+ def get_for_fd(self, fd):
+ """
+ Given a file descriptor, returns the kevent descriptor object
+ for it.
+
+ :param fd:
+ OS file descriptor.
+ :type fd:
+ ``int``
+ :returns:
+ A :class:`KeventDescriptor` object.
+ """
+ with self._lock:
+ return self._descriptor_for_fd[fd]
+
+ def get(self, path):
+ """
+ Obtains a :class:`KeventDescriptor` object for the specified path.
+
+ :param path:
+ Path for which the descriptor will be obtained.
+ """
+ with self._lock:
+ path = absolute_path(path)
+ return self._get(path)
+
+ def __contains__(self, path):
+ """
+ Determines whether a :class:`KeventDescriptor has been registered
+ for the specified path.
+
+ :param path:
+ Path for which the descriptor will be obtained.
+ """
+ with self._lock:
+ path = absolute_path(path)
+ return self._has_path(path)
+
+ def add(self, path, is_directory):
+ """
+ Adds a :class:`KeventDescriptor` to the collection for the given
+ path.
+
+ :param path:
+ The path for which a :class:`KeventDescriptor` object will be
+ added.
+ :param is_directory:
+ ``True`` if the path refers to a directory; ``False`` otherwise.
+ :type is_directory:
+ ``bool``
+ """
+ with self._lock:
+ path = absolute_path(path)
+ if not self._has_path(path):
+ self._add_descriptor(KeventDescriptor(path, is_directory))
+
+ def remove(self, path):
+ """
+ Removes the :class:`KeventDescriptor` object for the given path
+ if it already exists.
+
+ :param path:
+ Path for which the :class:`KeventDescriptor` object will be
+ removed.
+ """
+ with self._lock:
+ path = absolute_path(path)
+ if self._has_path(path):
+ self._remove_descriptor(self._get(path))
+
+ def clear(self):
+ """
+ Clears the collection and closes all open descriptors.
+ """
+ with self._lock:
+ for descriptor in self._descriptors:
+ descriptor.close()
+ self._descriptors.clear()
+ self._descriptor_for_fd.clear()
+ self._descriptor_for_path.clear()
+ self._kevents = []
+
+ # Thread-unsafe methods. Locking is provided at a higher level.
+ def _get(self, path):
+ """Returns a kevent descriptor for a given path."""
+ return self._descriptor_for_path[path]
+
+ def _has_path(self, path):
+ """Determines whether a :class:`KeventDescriptor` for the specified
+ path exists already in the collection."""
+ return path in self._descriptor_for_path
+
+ def _add_descriptor(self, descriptor):
+ """
+ Adds a descriptor to the collection.
+
+ :param descriptor:
+ An instance of :class:`KeventDescriptor` to be added.
+ """
+ self._descriptors.add(descriptor)
+ self._kevents.append(descriptor.kevent)
+ self._descriptor_for_path[descriptor.path] = descriptor
+ self._descriptor_for_fd[descriptor.fd] = descriptor
+
+ def _remove_descriptor(self, descriptor):
+ """
+ Removes a descriptor from the collection.
+
+ :param descriptor:
+ An instance of :class:`KeventDescriptor` to be removed.
+ """
+ self._descriptors.remove(descriptor)
+ del self._descriptor_for_fd[descriptor.fd]
+ del self._descriptor_for_path[descriptor.path]
+ self._kevents.remove(descriptor.kevent)
+ descriptor.close()
+
+
+class KeventDescriptor(object):
+
+ """
+ A kevent descriptor convenience data structure to keep together:
+
+ * kevent
+ * directory status
+ * path
+ * file descriptor
+
+ :param path:
+ Path string for which a kevent descriptor will be created.
+ :param is_directory:
+ ``True`` if the path refers to a directory; ``False`` otherwise.
+ :type is_directory:
+ ``bool``
+ """
+
+ def __init__(self, path, is_directory):
+ self._path = absolute_path(path)
+ self._is_directory = is_directory
+ self._fd = os.open(path, WATCHDOG_OS_OPEN_FLAGS)
+ self._kev = select.kevent(self._fd,
+ filter=WATCHDOG_KQ_FILTER,
+ flags=WATCHDOG_KQ_EV_FLAGS,
+ fflags=WATCHDOG_KQ_FFLAGS)
+
+ @property
+ def fd(self):
+ """OS file descriptor for the kevent descriptor."""
+ return self._fd
+
+ @property
+ def path(self):
+ """The path associated with the kevent descriptor."""
+ return self._path
+
+ @property
+ def kevent(self):
+ """The kevent object associated with the kevent descriptor."""
+ return self._kev
+
+ @property
+ def is_directory(self):
+ """Determines whether the kevent descriptor refers to a directory.
+
+ :returns:
+ ``True`` or ``False``
+ """
+ return self._is_directory
+
+ def close(self):
+ """
+ Closes the file descriptor associated with a kevent descriptor.
+ """
+ try:
+ os.close(self.fd)
+ except OSError:
+ pass
+
+ @property
+ def key(self):
+ return (self.path, self.is_directory)
+
+ def __eq__(self, descriptor):
+ return self.key == descriptor.key
+
+ def __ne__(self, descriptor):
+ return self.key != descriptor.key
+
+ def __hash__(self):
+ return hash(self.key)
+
+ def __repr__(self):
+ return "<KeventDescriptor: path=%s, is_directory=%s>"\
+ % (self.path, self.is_directory)
+
+
+class KqueueEmitter(EventEmitter):
+
+ """
+ kqueue(2)-based event emitter.
+
+ .. ADMONITION:: About ``kqueue(2)`` behavior and this implementation
+
+ ``kqueue(2)`` monitors file system events only for
+ open descriptors, which means, this emitter does a lot of
+ book-keeping behind the scenes to keep track of open
+ descriptors for every entry in the monitored directory tree.
+
+ This also means the number of maximum open file descriptors
+ on your system must be increased **manually**.
+ Usually, issuing a call to ``ulimit`` should suffice::
+
+ ulimit -n 1024
+
+ Ensure that you pick a number that is larger than the
+ number of files you expect to be monitored.
+
+ ``kqueue(2)`` does not provide enough information about the
+ following things:
+
+ * The destination path of a file or directory that is renamed.
+ * Creation of a file or directory within a directory; in this
+ case, ``kqueue(2)`` only indicates a modified event on the
+ parent directory.
+
+ Therefore, this emitter takes a snapshot of the directory
+ tree when ``kqueue(2)`` detects a change on the file system
+ to be able to determine the above information.
+
+ :param event_queue:
+ The event queue to fill with events.
+ :param watch:
+ A watch object representing the directory to monitor.
+ :type watch:
+ :class:`watchdog.observers.api.ObservedWatch`
+ :param timeout:
+ Read events blocking timeout (in seconds).
+ :type timeout:
+ ``float``
+ """
+
+ def __init__(self, event_queue, watch, timeout=DEFAULT_EMITTER_TIMEOUT):
+ EventEmitter.__init__(self, event_queue, watch, timeout)
+
+ self._kq = select.kqueue()
+ self._lock = threading.RLock()
+
+ # A collection of KeventDescriptor.
+ self._descriptors = KeventDescriptorSet()
+
+ def walker_callback(path, stat_info, self=self):
+ self._register_kevent(path, stat.S_ISDIR(stat_info.st_mode))
+
+ self._snapshot = DirectorySnapshot(watch.path,
+ watch.is_recursive,
+ walker_callback)
+
+ def _register_kevent(self, path, is_directory):
+ """
+ Registers a kevent descriptor for the given path.
+
+ :param path:
+ Path for which a kevent descriptor will be created.
+ :param is_directory:
+ ``True`` if the path refers to a directory; ``False`` otherwise.
+ :type is_directory:
+ ``bool``
+ """
+ try:
+ self._descriptors.add(path, is_directory)
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ # Probably dealing with a temporary file that was created
+ # and then quickly deleted before we could open
+ # a descriptor for it. Therefore, simply queue a sequence
+ # of created and deleted events for the path.
+ #path = absolute_path(path)
+ # if is_directory:
+ # self.queue_event(DirCreatedEvent(path))
+ # self.queue_event(DirDeletedEvent(path))
+ # else:
+ # self.queue_event(FileCreatedEvent(path))
+ # self.queue_event(FileDeletedEvent(path))
+
+ # TODO: We could simply ignore these files.
+ # Locked files cause the python process to die with
+ # a bus error when we handle temporary files.
+ # eg. .git/index.lock when running tig operations.
+ # I don't fully understand this at the moment.
+ pass
+ else:
+ # All other errors are propagated.
+ raise
+
+ def _unregister_kevent(self, path):
+ """
+ Convenience function to close the kevent descriptor for a
+ specified kqueue-monitored path.
+
+ :param path:
+ Path for which the kevent descriptor will be closed.
+ """
+ self._descriptors.remove(path)
+
+ def queue_event(self, event):
+ """
+ Handles queueing a single event object.
+
+ :param event:
+ An instance of :class:`watchdog.events.FileSystemEvent`
+ or a subclass.
+ """
+ # Handles all the book keeping for queued events.
+ # We do not need to fire moved/deleted events for all subitems in
+ # a directory tree here, because this function is called by kqueue
+ # for all those events anyway.
+ EventEmitter.queue_event(self, event)
+ if event.event_type == EVENT_TYPE_CREATED:
+ self._register_kevent(event.src_path, event.is_directory)
+ elif event.event_type == EVENT_TYPE_MOVED:
+ self._unregister_kevent(event.src_path)
+ self._register_kevent(event.dest_path, event.is_directory)
+ elif event.event_type == EVENT_TYPE_DELETED:
+ self._unregister_kevent(event.src_path)
+
+ def _queue_dirs_modified(self,
+ dirs_modified,
+ ref_snapshot,
+ new_snapshot):
+ """
+ Queues events for directory modifications by scanning the directory
+ for changes.
+
+ A scan is a comparison between two snapshots of the same directory
+ taken at two different times. This also determines whether files
+ or directories were created, which updated the modified timestamp
+ for the directory.
+ """
+ if dirs_modified:
+ for dir_modified in dirs_modified:
+ self.queue_event(DirModifiedEvent(dir_modified))
+ diff_events = new_snapshot - ref_snapshot
+ for file_created in diff_events.files_created:
+ self.queue_event(FileCreatedEvent(file_created))
+ for directory_created in diff_events.dirs_created:
+ self.queue_event(DirCreatedEvent(directory_created))
+
+ def _queue_events_except_renames_and_dir_modifications(self, event_list):
+ """
+ Queues events from the kevent list returned from the call to
+ :meth:`select.kqueue.control`.
+
+ .. NOTE:: Queues only the deletions, file modifications,
+ attribute modifications. The other events, namely,
+ file creation, directory modification, file rename,
+ directory rename, directory creation, etc. are
+ determined by comparing directory snapshots.
+ """
+ files_renamed = set()
+ dirs_renamed = set()
+ dirs_modified = set()
+
+ for kev in event_list:
+ descriptor = self._descriptors.get_for_fd(kev.ident)
+ src_path = descriptor.path
+
+ if is_deleted(kev):
+ if descriptor.is_directory:
+ self.queue_event(DirDeletedEvent(src_path))
+ else:
+ self.queue_event(FileDeletedEvent(src_path))
+ elif is_attrib_modified(kev):
+ if descriptor.is_directory:
+ self.queue_event(DirModifiedEvent(src_path))
+ else:
+ self.queue_event(FileModifiedEvent(src_path))
+ elif is_modified(kev):
+ if descriptor.is_directory:
+ # When a directory is modified, it may be due to
+ # sub-file/directory renames or new file/directory
+ # creation. We determine all this by comparing
+ # snapshots later.
+ dirs_modified.add(src_path)
+ else:
+ self.queue_event(FileModifiedEvent(src_path))
+ elif is_renamed(kev):
+ # Kqueue does not specify the destination names for renames
+ # to, so we have to process these after taking a snapshot
+ # of the directory.
+ if descriptor.is_directory:
+ dirs_renamed.add(src_path)
+ else:
+ files_renamed.add(src_path)
+ return files_renamed, dirs_renamed, dirs_modified
+
+ def _queue_renamed(self,
+ src_path,
+ is_directory,
+ ref_snapshot,
+ new_snapshot):
+ """
+ Compares information from two directory snapshots (one taken before
+ the rename operation and another taken right after) to determine the
+ destination path of the file system object renamed, and adds
+ appropriate events to the event queue.
+ """
+ try:
+ ref_stat_info = ref_snapshot.stat_info(src_path)
+ except KeyError:
+ # Probably caught a temporary file/directory that was renamed
+ # and deleted. Fires a sequence of created and deleted events
+ # for the path.
+ if is_directory:
+ self.queue_event(DirCreatedEvent(src_path))
+ self.queue_event(DirDeletedEvent(src_path))
+ else:
+ self.queue_event(FileCreatedEvent(src_path))
+ self.queue_event(FileDeletedEvent(src_path))
+ # We don't process any further and bail out assuming
+ # the event represents deletion/creation instead of movement.
+ return
+
+ try:
+ dest_path = absolute_path(
+ new_snapshot.path_for_inode(ref_stat_info.st_ino))
+ if is_directory:
+ event = DirMovedEvent(src_path, dest_path)
+ # TODO: Do we need to fire moved events for the items
+ # inside the directory tree? Does kqueue does this
+ # all by itself? Check this and then enable this code
+ # only if it doesn't already.
+ # A: It doesn't. So I've enabled this block.
+ if self.watch.is_recursive:
+ for sub_event in event.sub_moved_events():
+ self.queue_event(sub_event)
+ self.queue_event(event)
+ else:
+ self.queue_event(FileMovedEvent(src_path, dest_path))
+ except KeyError:
+ # If the new snapshot does not have an inode for the
+ # old path, we haven't found the new name. Therefore,
+ # we mark it as deleted and remove unregister the path.
+ if is_directory:
+ self.queue_event(DirDeletedEvent(src_path))
+ else:
+ self.queue_event(FileDeletedEvent(src_path))
+
+ def _read_events(self, timeout=None):
+ """
+ Reads events from a call to the blocking
+ :meth:`select.kqueue.control()` method.
+
+ :param timeout:
+ Blocking timeout for reading events.
+ :type timeout:
+ ``float`` (seconds)
+ """
+ return self._kq.control(self._descriptors.kevents,
+ MAX_EVENTS,
+ timeout)
+
+ def queue_events(self, timeout):
+ """
+ Queues events by reading them from a call to the blocking
+ :meth:`select.kqueue.control()` method.
+
+ :param timeout:
+ Blocking timeout for reading events.
+ :type timeout:
+ ``float`` (seconds)
+ """
+ with self._lock:
+ try:
+ event_list = self._read_events(timeout)
+ files_renamed, dirs_renamed, dirs_modified = (
+ self._queue_events_except_renames_and_dir_modifications(event_list))
+
+ # Take a fresh snapshot of the directory and update the
+ # saved snapshot.
+ new_snapshot = DirectorySnapshot(self.watch.path,
+ self.watch.is_recursive)
+ ref_snapshot = self._snapshot
+ self._snapshot = new_snapshot
+
+ if files_renamed or dirs_renamed or dirs_modified:
+ for src_path in files_renamed:
+ self._queue_renamed(src_path,
+ False,
+ ref_snapshot,
+ new_snapshot)
+ for src_path in dirs_renamed:
+ self._queue_renamed(src_path,
+ True,
+ ref_snapshot,
+ new_snapshot)
+ self._queue_dirs_modified(dirs_modified,
+ ref_snapshot,
+ new_snapshot)
+ except OSError as e:
+ if e.errno == errno.EBADF:
+ # logging.debug(e)
+ pass
+ else:
+ raise
+
+ def on_thread_stop(self):
+ # Clean up.
+ with self._lock:
+ self._descriptors.clear()
+ self._kq.close()
+
+
+class KqueueObserver(BaseObserver):
+
+ """
+ Observer thread that schedules watching directories and dispatches
+ calls to event handlers.
+ """
+
+ def __init__(self, timeout=DEFAULT_OBSERVER_TIMEOUT):
+ BaseObserver.__init__(self, emitter_class=KqueueEmitter, timeout=timeout)
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/polling.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/polling.py
new file mode 100644
index 0000000000000000000000000000000000000000..f0019fdbf3f8bd85df90fb71dbdf6656189daf3b
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/polling.py
@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+:module: watchdog.observers.polling
+:synopsis: Polling emitter implementation.
+:author: yesudeep@google.com (Yesudeep Mangalapilly)
+
+Classes
+-------
+.. autoclass:: PollingObserver
+ :members:
+ :show-inheritance:
+
+.. autoclass:: PollingObserverVFS
+ :members:
+ :show-inheritance:
+ :special-members:
+"""
+
+from __future__ import with_statement
+import os
+import threading
+from functools import partial
+from wandb_watchdog.utils import stat as default_stat
+from wandb_watchdog.utils.dirsnapshot import DirectorySnapshot, DirectorySnapshotDiff
+from wandb_watchdog.observers.api import (
+ EventEmitter,
+ BaseObserver,
+ DEFAULT_OBSERVER_TIMEOUT,
+ DEFAULT_EMITTER_TIMEOUT
+)
+
+from wandb_watchdog.events import (
+ DirMovedEvent,
+ DirDeletedEvent,
+ DirCreatedEvent,
+ DirModifiedEvent,
+ FileMovedEvent,
+ FileDeletedEvent,
+ FileCreatedEvent,
+ FileModifiedEvent
+)
+
+
+class PollingEmitter(EventEmitter):
+ """
+ Platform-independent emitter that polls a directory to detect file
+ system changes.
+ """
+
+ def __init__(self, event_queue, watch, timeout=DEFAULT_EMITTER_TIMEOUT,
+ stat=default_stat, listdir=os.listdir):
+ EventEmitter.__init__(self, event_queue, watch, timeout)
+ self._snapshot = None
+ self._lock = threading.Lock()
+ self._take_snapshot = lambda: DirectorySnapshot(
+ self.watch.path, self.watch.is_recursive, stat=stat, listdir=listdir)
+
+ def on_thread_start(self):
+ self._snapshot = self._take_snapshot()
+
+ def queue_events(self, timeout):
+ # We don't want to hit the disk continuously.
+ # timeout behaves like an interval for polling emitters.
+ if self.stopped_event.wait(timeout):
+ return
+
+ with self._lock:
+ if not self.should_keep_running():
+ return
+
+ # Get event diff between fresh snapshot and previous snapshot.
+ # Update snapshot.
+ try:
+ new_snapshot = self._take_snapshot()
+ except OSError:
+ self.queue_event(DirDeletedEvent(self.watch.path))
+ self.stop()
+ return
+
+ events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
+ self._snapshot = new_snapshot
+
+ # Files.
+ for src_path in events.files_deleted:
+ self.queue_event(FileDeletedEvent(src_path))
+ for src_path in events.files_modified:
+ self.queue_event(FileModifiedEvent(src_path))
+ for src_path in events.files_created:
+ self.queue_event(FileCreatedEvent(src_path))
+ for src_path, dest_path in events.files_moved:
+ self.queue_event(FileMovedEvent(src_path, dest_path))
+
+ # Directories.
+ for src_path in events.dirs_deleted:
+ self.queue_event(DirDeletedEvent(src_path))
+ for src_path in events.dirs_modified:
+ self.queue_event(DirModifiedEvent(src_path))
+ for src_path in events.dirs_created:
+ self.queue_event(DirCreatedEvent(src_path))
+ for src_path, dest_path in events.dirs_moved:
+ self.queue_event(DirMovedEvent(src_path, dest_path))
+
+
+class PollingObserver(BaseObserver):
+ """
+ Platform-independent observer that polls a directory to detect file
+ system changes.
+ """
+
+ def __init__(self, timeout=DEFAULT_OBSERVER_TIMEOUT):
+ BaseObserver.__init__(self, emitter_class=PollingEmitter, timeout=timeout)
+
+
+class PollingObserverVFS(BaseObserver):
+ """
+ File system independent observer that polls a directory to detect changes.
+ """
+
+ def __init__(self, stat, listdir, polling_interval=1):
+ """
+ :param stat: stat function. See ``os.stat`` for details.
+ :param listdir: listdir function. See ``os.listdir`` for details.
+ :type polling_interval: float
+ :param polling_interval: interval in seconds between polling the file system.
+ """
+ emitter_cls = partial(PollingEmitter, stat=stat, listdir=listdir)
+ BaseObserver.__init__(self, emitter_class=emitter_cls, timeout=polling_interval)
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/read_directory_changes.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/read_directory_changes.py
new file mode 100644
index 0000000000000000000000000000000000000000..b2da03a3be75b05ef2336700474be36578bc08b3
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/read_directory_changes.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+# Copyright 2014 Thomas Amland
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import with_statement
+
+import threading
+import os.path
+import time
+
+from wandb_watchdog.events import (
+ DirCreatedEvent,
+ DirMovedEvent,
+ DirModifiedEvent,
+ FileCreatedEvent,
+ FileDeletedEvent,
+ FileMovedEvent,
+ FileModifiedEvent,
+ generate_sub_moved_events,
+ generate_sub_created_events,
+)
+
+from wandb_watchdog.observers.api import (
+ EventEmitter,
+ BaseObserver,
+ DEFAULT_OBSERVER_TIMEOUT,
+ DEFAULT_EMITTER_TIMEOUT
+)
+
+from wandb_watchdog.observers.winapi import (
+ read_events,
+ get_directory_handle,
+ close_directory_handle,
+)
+
+
+# HACK:
+WATCHDOG_TRAVERSE_MOVED_DIR_DELAY = 1 # seconds
+
+
+class WindowsApiEmitter(EventEmitter):
+ """
+ Windows API-based emitter that uses ReadDirectoryChangesW
+ to detect file system changes for a watch.
+ """
+
+ def __init__(self, event_queue, watch, timeout=DEFAULT_EMITTER_TIMEOUT):
+ EventEmitter.__init__(self, event_queue, watch, timeout)
+ self._lock = threading.Lock()
+ self._handle = None
+
+ def on_thread_start(self):
+ self._handle = get_directory_handle(self.watch.path)
+
+ def on_thread_stop(self):
+ if self._handle:
+ close_directory_handle(self._handle)
+
+ def queue_events(self, timeout):
+ winapi_events = read_events(self._handle, self.watch.is_recursive)
+ with self._lock:
+ last_renamed_src_path = ""
+ for winapi_event in winapi_events:
+ src_path = os.path.join(self.watch.path, winapi_event.src_path)
+
+ if winapi_event.is_renamed_old:
+ last_renamed_src_path = src_path
+ elif winapi_event.is_renamed_new:
+ dest_path = src_path
+ src_path = last_renamed_src_path
+ if os.path.isdir(dest_path):
+ event = DirMovedEvent(src_path, dest_path)
+ if self.watch.is_recursive:
+ # HACK: We introduce a forced delay before
+ # traversing the moved directory. This will read
+ # only file movement that finishes within this
+ # delay time.
+ time.sleep(WATCHDOG_TRAVERSE_MOVED_DIR_DELAY)
+ # The following block of code may not
+ # obtain moved events for the entire tree if
+ # the I/O is not completed within the above
+ # delay time. So, it's not guaranteed to work.
+ # TODO: Come up with a better solution, possibly
+ # a way to wait for I/O to complete before
+ # queuing events.
+ for sub_moved_event in generate_sub_moved_events(src_path, dest_path):
+ self.queue_event(sub_moved_event)
+ self.queue_event(event)
+ else:
+ self.queue_event(FileMovedEvent(src_path, dest_path))
+ elif winapi_event.is_modified:
+ cls = DirModifiedEvent if os.path.isdir(src_path) else FileModifiedEvent
+ self.queue_event(cls(src_path))
+ elif winapi_event.is_added:
+ isdir = os.path.isdir(src_path)
+ cls = DirCreatedEvent if isdir else FileCreatedEvent
+ self.queue_event(cls(src_path))
+ if isdir:
+ # If a directory is moved from outside the watched folder to inside it
+ # we only get a created directory event out of it, not any events for its children
+ # so use the same hack as for file moves to get the child events
+ time.sleep(WATCHDOG_TRAVERSE_MOVED_DIR_DELAY)
+ sub_events = generate_sub_created_events(src_path)
+ for sub_created_event in sub_events:
+ self.queue_event(sub_created_event)
+ elif winapi_event.is_removed:
+ self.queue_event(FileDeletedEvent(src_path))
+
+
+class WindowsApiObserver(BaseObserver):
+ """
+ Observer thread that schedules watching directories and dispatches
+ calls to event handlers.
+ """
+
+ def __init__(self, timeout=DEFAULT_OBSERVER_TIMEOUT):
+ BaseObserver.__init__(self, emitter_class=WindowsApiEmitter,
+ timeout=timeout)
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/winapi.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/winapi.py
new file mode 100644
index 0000000000000000000000000000000000000000..a939a69b9135043098f41675039afeb8d19e2e65
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/winapi.py
@@ -0,0 +1,348 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# winapi.py: Windows API-Python interface (removes dependency on pywin32)
+#
+# Copyright (C) 2007 Thomas Heller <theller@ctypes.org>
+# Copyright (C) 2010 Will McGugan <will@willmcgugan.com>
+# Copyright (C) 2010 Ryan Kelly <ryan@rfk.id.au>
+# Copyright (C) 2010 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright (C) 2014 Thomas Amland
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and / or other materials provided with the distribution.
+# * Neither the name of the organization nor the names of its contributors may
+# be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# Portions of this code were taken from pyfilesystem, which uses the above
+# new BSD license.
+
+from __future__ import with_statement
+
+import ctypes.wintypes
+from functools import reduce
+
+try:
+ LPVOID = ctypes.wintypes.LPVOID
+except AttributeError:
+ # LPVOID wasn't defined in Py2.5, guess it was introduced in Py2.6
+ LPVOID = ctypes.c_void_p
+
+# Invalid handle value.
+INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value
+
+# File notification contants.
+FILE_NOTIFY_CHANGE_FILE_NAME = 0x01
+FILE_NOTIFY_CHANGE_DIR_NAME = 0x02
+FILE_NOTIFY_CHANGE_ATTRIBUTES = 0x04
+FILE_NOTIFY_CHANGE_SIZE = 0x08
+FILE_NOTIFY_CHANGE_LAST_WRITE = 0x010
+FILE_NOTIFY_CHANGE_LAST_ACCESS = 0x020
+FILE_NOTIFY_CHANGE_CREATION = 0x040
+FILE_NOTIFY_CHANGE_SECURITY = 0x0100
+
+FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
+FILE_FLAG_OVERLAPPED = 0x40000000
+FILE_LIST_DIRECTORY = 1
+FILE_SHARE_READ = 0x01
+FILE_SHARE_WRITE = 0x02
+FILE_SHARE_DELETE = 0x04
+OPEN_EXISTING = 3
+
+# File action constants.
+FILE_ACTION_CREATED = 1
+FILE_ACTION_DELETED = 2
+FILE_ACTION_MODIFIED = 3
+FILE_ACTION_RENAMED_OLD_NAME = 4
+FILE_ACTION_RENAMED_NEW_NAME = 5
+FILE_ACTION_OVERFLOW = 0xFFFF
+
+# Aliases
+FILE_ACTION_ADDED = FILE_ACTION_CREATED
+FILE_ACTION_REMOVED = FILE_ACTION_DELETED
+
+THREAD_TERMINATE = 0x0001
+
+# IO waiting constants.
+WAIT_ABANDONED = 0x00000080
+WAIT_IO_COMPLETION = 0x000000C0
+WAIT_OBJECT_0 = 0x00000000
+WAIT_TIMEOUT = 0x00000102
+
+# Error codes
+ERROR_OPERATION_ABORTED = 995
+
+
+class OVERLAPPED(ctypes.Structure):
+ _fields_ = [('Internal', LPVOID),
+ ('InternalHigh', LPVOID),
+ ('Offset', ctypes.wintypes.DWORD),
+ ('OffsetHigh', ctypes.wintypes.DWORD),
+ ('Pointer', LPVOID),
+ ('hEvent', ctypes.wintypes.HANDLE),
+ ]
+
+
+def _errcheck_bool(value, func, args):
+ if not value:
+ raise ctypes.WinError
+ return args
+
+
+def _errcheck_handle(value, func, args):
+ if not value:
+ raise ctypes.WinError
+ if value == INVALID_HANDLE_VALUE:
+ raise ctypes.WinError
+ return args
+
+
+def _errcheck_dword(value, func, args):
+ if value == 0xFFFFFFFF:
+ raise ctypes.WinError
+ return args
+
+
+ReadDirectoryChangesW = ctypes.windll.kernel32.ReadDirectoryChangesW
+ReadDirectoryChangesW.restype = ctypes.wintypes.BOOL
+ReadDirectoryChangesW.errcheck = _errcheck_bool
+ReadDirectoryChangesW.argtypes = (
+ ctypes.wintypes.HANDLE, # hDirectory
+ LPVOID, # lpBuffer
+ ctypes.wintypes.DWORD, # nBufferLength
+ ctypes.wintypes.BOOL, # bWatchSubtree
+ ctypes.wintypes.DWORD, # dwNotifyFilter
+ ctypes.POINTER(ctypes.wintypes.DWORD), # lpBytesReturned
+ ctypes.POINTER(OVERLAPPED), # lpOverlapped
+ LPVOID # FileIOCompletionRoutine # lpCompletionRoutine
+)
+
+CreateFileW = ctypes.windll.kernel32.CreateFileW
+CreateFileW.restype = ctypes.wintypes.HANDLE
+CreateFileW.errcheck = _errcheck_handle
+CreateFileW.argtypes = (
+ ctypes.wintypes.LPCWSTR, # lpFileName
+ ctypes.wintypes.DWORD, # dwDesiredAccess
+ ctypes.wintypes.DWORD, # dwShareMode
+ LPVOID, # lpSecurityAttributes
+ ctypes.wintypes.DWORD, # dwCreationDisposition
+ ctypes.wintypes.DWORD, # dwFlagsAndAttributes
+ ctypes.wintypes.HANDLE # hTemplateFile
+)
+
+CloseHandle = ctypes.windll.kernel32.CloseHandle
+CloseHandle.restype = ctypes.wintypes.BOOL
+CloseHandle.argtypes = (
+ ctypes.wintypes.HANDLE, # hObject
+)
+
+CancelIoEx = ctypes.windll.kernel32.CancelIoEx
+CancelIoEx.restype = ctypes.wintypes.BOOL
+CancelIoEx.errcheck = _errcheck_bool
+CancelIoEx.argtypes = (
+ ctypes.wintypes.HANDLE, # hObject
+ ctypes.POINTER(OVERLAPPED) # lpOverlapped
+)
+
+CreateEvent = ctypes.windll.kernel32.CreateEventW
+CreateEvent.restype = ctypes.wintypes.HANDLE
+CreateEvent.errcheck = _errcheck_handle
+CreateEvent.argtypes = (
+ LPVOID, # lpEventAttributes
+ ctypes.wintypes.BOOL, # bManualReset
+ ctypes.wintypes.BOOL, # bInitialState
+ ctypes.wintypes.LPCWSTR, # lpName
+)
+
+SetEvent = ctypes.windll.kernel32.SetEvent
+SetEvent.restype = ctypes.wintypes.BOOL
+SetEvent.errcheck = _errcheck_bool
+SetEvent.argtypes = (
+ ctypes.wintypes.HANDLE, # hEvent
+)
+
+WaitForSingleObjectEx = ctypes.windll.kernel32.WaitForSingleObjectEx
+WaitForSingleObjectEx.restype = ctypes.wintypes.DWORD
+WaitForSingleObjectEx.errcheck = _errcheck_dword
+WaitForSingleObjectEx.argtypes = (
+ ctypes.wintypes.HANDLE, # hObject
+ ctypes.wintypes.DWORD, # dwMilliseconds
+ ctypes.wintypes.BOOL, # bAlertable
+)
+
+CreateIoCompletionPort = ctypes.windll.kernel32.CreateIoCompletionPort
+CreateIoCompletionPort.restype = ctypes.wintypes.HANDLE
+CreateIoCompletionPort.errcheck = _errcheck_handle
+CreateIoCompletionPort.argtypes = (
+ ctypes.wintypes.HANDLE, # FileHandle
+ ctypes.wintypes.HANDLE, # ExistingCompletionPort
+ LPVOID, # CompletionKey
+ ctypes.wintypes.DWORD, # NumberOfConcurrentThreads
+)
+
+GetQueuedCompletionStatus = ctypes.windll.kernel32.GetQueuedCompletionStatus
+GetQueuedCompletionStatus.restype = ctypes.wintypes.BOOL
+GetQueuedCompletionStatus.errcheck = _errcheck_bool
+GetQueuedCompletionStatus.argtypes = (
+ ctypes.wintypes.HANDLE, # CompletionPort
+ LPVOID, # lpNumberOfBytesTransferred
+ LPVOID, # lpCompletionKey
+ ctypes.POINTER(OVERLAPPED), # lpOverlapped
+ ctypes.wintypes.DWORD, # dwMilliseconds
+)
+
+PostQueuedCompletionStatus = ctypes.windll.kernel32.PostQueuedCompletionStatus
+PostQueuedCompletionStatus.restype = ctypes.wintypes.BOOL
+PostQueuedCompletionStatus.errcheck = _errcheck_bool
+PostQueuedCompletionStatus.argtypes = (
+ ctypes.wintypes.HANDLE, # CompletionPort
+ ctypes.wintypes.DWORD, # lpNumberOfBytesTransferred
+ ctypes.wintypes.DWORD, # lpCompletionKey
+ ctypes.POINTER(OVERLAPPED), # lpOverlapped
+)
+
+
+class FILE_NOTIFY_INFORMATION(ctypes.Structure):
+ _fields_ = [("NextEntryOffset", ctypes.wintypes.DWORD),
+ ("Action", ctypes.wintypes.DWORD),
+ ("FileNameLength", ctypes.wintypes.DWORD),
+ #("FileName", (ctypes.wintypes.WCHAR * 1))]
+ ("FileName", (ctypes.c_char * 1))]
+
+LPFNI = ctypes.POINTER(FILE_NOTIFY_INFORMATION)
+
+
+# We don't need to recalculate these flags every time a call is made to
+# the win32 API functions.
+WATCHDOG_FILE_FLAGS = FILE_FLAG_BACKUP_SEMANTICS
+WATCHDOG_FILE_SHARE_FLAGS = reduce(
+ lambda x, y: x | y, [
+ FILE_SHARE_READ,
+ FILE_SHARE_WRITE,
+ FILE_SHARE_DELETE,
+ ])
+WATCHDOG_FILE_NOTIFY_FLAGS = reduce(
+ lambda x, y: x | y, [
+ FILE_NOTIFY_CHANGE_FILE_NAME,
+ FILE_NOTIFY_CHANGE_DIR_NAME,
+ FILE_NOTIFY_CHANGE_ATTRIBUTES,
+ FILE_NOTIFY_CHANGE_SIZE,
+ FILE_NOTIFY_CHANGE_LAST_WRITE,
+ FILE_NOTIFY_CHANGE_SECURITY,
+ FILE_NOTIFY_CHANGE_LAST_ACCESS,
+ FILE_NOTIFY_CHANGE_CREATION,
+ ])
+
+BUFFER_SIZE = 2048
+
+
+def _parse_event_buffer(readBuffer, nBytes):
+ results = []
+ while nBytes > 0:
+ fni = ctypes.cast(readBuffer, LPFNI)[0]
+ ptr = ctypes.addressof(fni) + FILE_NOTIFY_INFORMATION.FileName.offset
+ #filename = ctypes.wstring_at(ptr, fni.FileNameLength)
+ filename = ctypes.string_at(ptr, fni.FileNameLength)
+ results.append((fni.Action, filename.decode('utf-16')))
+ numToSkip = fni.NextEntryOffset
+ if numToSkip <= 0:
+ break
+ readBuffer = readBuffer[numToSkip:]
+ nBytes -= numToSkip # numToSkip is long. nBytes should be long too.
+ return results
+
+
+def get_directory_handle(path):
+ """Returns a Windows handle to the specified directory path."""
+ return CreateFileW(path, FILE_LIST_DIRECTORY, WATCHDOG_FILE_SHARE_FLAGS,
+ None, OPEN_EXISTING, WATCHDOG_FILE_FLAGS, None)
+
+
+def close_directory_handle(handle):
+ try:
+ CancelIoEx(handle, None) # force ReadDirectoryChangesW to return
+ CloseHandle(handle) # close directory handle
+ except WindowsError:
+ try:
+ CloseHandle(handle) # close directory handle
+ except:
+ return
+
+
+def read_directory_changes(handle, recursive):
+ """Read changes to the directory using the specified directory handle.
+
+ http://timgolden.me.uk/pywin32-docs/win32file__ReadDirectoryChangesW_meth.html
+ """
+ event_buffer = ctypes.create_string_buffer(BUFFER_SIZE)
+ nbytes = ctypes.wintypes.DWORD()
+ try:
+ ReadDirectoryChangesW(handle, ctypes.byref(event_buffer),
+ len(event_buffer), recursive,
+ WATCHDOG_FILE_NOTIFY_FLAGS,
+ ctypes.byref(nbytes), None, None)
+ except WindowsError as e:
+ if e.winerror == ERROR_OPERATION_ABORTED:
+ return [], 0
+ raise e
+
+ # Python 2/3 compat
+ try:
+ int_class = long
+ except NameError:
+ int_class = int
+ return event_buffer.raw, int_class(nbytes.value)
+
+
+class WinAPINativeEvent(object):
+ def __init__(self, action, src_path):
+ self.action = action
+ self.src_path = src_path
+
+ @property
+ def is_added(self):
+ return self.action == FILE_ACTION_CREATED
+
+ @property
+ def is_removed(self):
+ return self.action == FILE_ACTION_REMOVED
+
+ @property
+ def is_modified(self):
+ return self.action == FILE_ACTION_MODIFIED
+
+ @property
+ def is_renamed_old(self):
+ return self.action == FILE_ACTION_RENAMED_OLD_NAME
+
+ @property
+ def is_renamed_new(self):
+ return self.action == FILE_ACTION_RENAMED_NEW_NAME
+
+ def __repr__(self):
+ return ("<WinAPINativeEvent: action=%d, src_path=%r>" % (self.action, self.src_path))
+
+
+def read_events(handle, recursive):
+ buf, nbytes = read_directory_changes(handle, recursive)
+ events = _parse_event_buffer(buf, nbytes)
+ return [WinAPINativeEvent(action, path) for action, path in events]
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/patterns.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/patterns.py
new file mode 100644
index 0000000000000000000000000000000000000000..4ecd853745e4092d3e12f211cd7ca5641771f8f7
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/patterns.py
@@ -0,0 +1,265 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# patterns.py: Common wildcard searching/filtering functionality for files.
+#
+# Copyright (C) 2010 Yesudeep Mangalapilly <yesudeep@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+"""
+:module: pathtools.patterns
+:synopsis: Wildcard pattern matching and filtering functions for paths.
+:author: Yesudeep Mangalapilly <yesudeep@gmail.com>
+
+Functions
+---------
+.. autofunction:: match_path
+.. autofunction:: match_path_against
+.. autofunction:: filter_paths
+"""
+
+from fnmatch import fnmatch, fnmatchcase
+
+__all__ = ['match_path',
+ 'match_path_against',
+ 'match_any_paths',
+ 'filter_paths']
+
+
+def _string_lower(s):
+ """
+ Convenience function to lowercase a string (the :mod:`string` module is
+ deprecated/removed in Python 3.0).
+
+ :param s:
+ The string which will be lowercased.
+ :returns:
+ Lowercased copy of string s.
+ """
+ return s.lower()
+
+
+def match_path_against(pathname, patterns, case_sensitive=True):
+ """
+ Determines whether the pathname matches any of the given wildcard patterns,
+ optionally ignoring the case of the pathname and patterns.
+
+ :param pathname:
+ A path name that will be matched against a wildcard pattern.
+ :param patterns:
+ A list of wildcard patterns to match_path the filename against.
+ :param case_sensitive:
+ ``True`` if the matching should be case-sensitive; ``False`` otherwise.
+ :returns:
+ ``True`` if the pattern matches; ``False`` otherwise.
+
+ Doctests::
+ >>> match_path_against("/home/username/foobar/blah.py", ["*.py", "*.txt"], False)
+ True
+ >>> match_path_against("/home/username/foobar/blah.py", ["*.PY", "*.txt"], True)
+ False
+ >>> match_path_against("/home/username/foobar/blah.py", ["*.PY", "*.txt"], False)
+ True
+ >>> match_path_against("C:\\windows\\blah\\BLAH.PY", ["*.py", "*.txt"], True)
+ False
+ >>> match_path_against("C:\\windows\\blah\\BLAH.PY", ["*.py", "*.txt"], False)
+ True
+ """
+ if case_sensitive:
+ match_func = fnmatchcase
+ pattern_transform_func = (lambda w: w)
+ else:
+ match_func = fnmatch
+ pathname = pathname.lower()
+ pattern_transform_func = _string_lower
+ for pattern in set(patterns):
+ pattern = pattern_transform_func(pattern)
+ if match_func(pathname, pattern):
+ return True
+ return False
+
+
+def _match_path(pathname,
+ included_patterns,
+ excluded_patterns,
+ case_sensitive=True):
+ """Internal function same as :func:`match_path` but does not check arguments.
+
+ Doctests::
+ >>> _match_path("/users/gorakhargosh/foobar.py", ["*.py"], ["*.PY"], True)
+ True
+ >>> _match_path("/users/gorakhargosh/FOOBAR.PY", ["*.py"], ["*.PY"], True)
+ False
+ >>> _match_path("/users/gorakhargosh/foobar/", ["*.py"], ["*.txt"], False)
+ False
+ >>> _match_path("/users/gorakhargosh/FOOBAR.PY", ["*.py"], ["*.PY"], False)
+ Traceback (most recent call last):
+ ...
+ ValueError: conflicting patterns `set(['*.py'])` included and excluded
+ """
+ if not case_sensitive:
+ included_patterns = set(map(_string_lower, included_patterns))
+ excluded_patterns = set(map(_string_lower, excluded_patterns))
+ else:
+ included_patterns = set(included_patterns)
+ excluded_patterns = set(excluded_patterns)
+ common_patterns = included_patterns & excluded_patterns
+ if common_patterns:
+ raise ValueError('conflicting patterns `%s` included and excluded'\
+ % common_patterns)
+ return (match_path_against(pathname, included_patterns, case_sensitive)\
+ and not match_path_against(pathname, excluded_patterns,
+ case_sensitive))
+
+
+def match_path(pathname,
+ included_patterns=None,
+ excluded_patterns=None,
+ case_sensitive=True):
+ """
+ Matches a pathname against a set of acceptable and ignored patterns.
+
+ :param pathname:
+ A pathname which will be matched against a pattern.
+ :param included_patterns:
+ Allow filenames matching wildcard patterns specified in this list.
+ If no pattern is specified, the function treats the pathname as
+ a match_path.
+ :param excluded_patterns:
+ Ignores filenames matching wildcard patterns specified in this list.
+ If no pattern is specified, the function treats the pathname as
+ a match_path.
+ :param case_sensitive:
+ ``True`` if matching should be case-sensitive; ``False`` otherwise.
+ :returns:
+ ``True`` if the pathname matches; ``False`` otherwise.
+ :raises:
+ ValueError if included patterns and excluded patterns contain the
+ same pattern.
+
+ Doctests::
+ >>> match_path("/Users/gorakhargosh/foobar.py")
+ True
+ >>> match_path("/Users/gorakhargosh/foobar.py", case_sensitive=False)
+ True
+ >>> match_path("/users/gorakhargosh/foobar.py", ["*.py"], ["*.PY"], True)
+ True
+ >>> match_path("/users/gorakhargosh/FOOBAR.PY", ["*.py"], ["*.PY"], True)
+ False
+ >>> match_path("/users/gorakhargosh/foobar/", ["*.py"], ["*.txt"], False)
+ False
+ >>> match_path("/users/gorakhargosh/FOOBAR.PY", ["*.py"], ["*.PY"], False)
+ Traceback (most recent call last):
+ ...
+ ValueError: conflicting patterns `set(['*.py'])` included and excluded
+ """
+ included = ["*"] if included_patterns is None else included_patterns
+ excluded = [] if excluded_patterns is None else excluded_patterns
+ return _match_path(pathname, included, excluded, case_sensitive)
+
+
+def filter_paths(pathnames,
+ included_patterns=None,
+ excluded_patterns=None,
+ case_sensitive=True):
+ """
+ Filters from a set of paths based on acceptable patterns and
+ ignorable patterns.
+
+ :param pathnames:
+ A list of path names that will be filtered based on matching and
+ ignored patterns.
+ :param included_patterns:
+ Allow filenames matching wildcard patterns specified in this list.
+ If no pattern list is specified, ["*"] is used as the default pattern,
+ which matches all files.
+ :param excluded_patterns:
+ Ignores filenames matching wildcard patterns specified in this list.
+ If no pattern list is specified, no files are ignored.
+ :param case_sensitive:
+ ``True`` if matching should be case-sensitive; ``False`` otherwise.
+ :returns:
+ A list of pathnames that matched the allowable patterns and passed
+ through the ignored patterns.
+
+ Doctests::
+ >>> pathnames = set(["/users/gorakhargosh/foobar.py", "/var/cache/pdnsd.status", "/etc/pdnsd.conf", "/usr/local/bin/python"])
+ >>> set(filter_paths(pathnames)) == pathnames
+ True
+ >>> set(filter_paths(pathnames, case_sensitive=False)) == pathnames
+ True
+ >>> set(filter_paths(pathnames, ["*.py", "*.conf"], ["*.status"], case_sensitive=True)) == set(["/users/gorakhargosh/foobar.py", "/etc/pdnsd.conf"])
+ True
+ """
+ included = ["*"] if included_patterns is None else included_patterns
+ excluded = [] if excluded_patterns is None else excluded_patterns
+
+ for pathname in pathnames:
+ # We don't call the public match_path because it checks arguments
+ # and sets default values if none are found. We're already doing that
+ # above.
+ if _match_path(pathname, included, excluded, case_sensitive):
+ yield pathname
+
+def match_any_paths(pathnames,
+ included_patterns=None,
+ excluded_patterns=None,
+ case_sensitive=True):
+ """
+ Matches from a set of paths based on acceptable patterns and
+ ignorable patterns.
+
+ :param pathnames:
+ A list of path names that will be filtered based on matching and
+ ignored patterns.
+ :param included_patterns:
+ Allow filenames matching wildcard patterns specified in this list.
+ If no pattern list is specified, ["*"] is used as the default pattern,
+ which matches all files.
+ :param excluded_patterns:
+ Ignores filenames matching wildcard patterns specified in this list.
+ If no pattern list is specified, no files are ignored.
+ :param case_sensitive:
+ ``True`` if matching should be case-sensitive; ``False`` otherwise.
+ :returns:
+ ``True`` if any of the paths matches; ``False`` otherwise.
+
+ Doctests::
+ >>> pathnames = set(["/users/gorakhargosh/foobar.py", "/var/cache/pdnsd.status", "/etc/pdnsd.conf", "/usr/local/bin/python"])
+ >>> match_any_paths(pathnames)
+ True
+ >>> match_any_paths(pathnames, case_sensitive=False)
+ True
+ >>> match_any_paths(pathnames, ["*.py", "*.conf"], ["*.status"], case_sensitive=True)
+ True
+ >>> match_any_paths(pathnames, ["*.txt"], case_sensitive=False)
+ False
+ >>> match_any_paths(pathnames, ["*.txt"], case_sensitive=True)
+ False
+ """
+ included = ["*"] if included_patterns is None else included_patterns
+ excluded = [] if excluded_patterns is None else excluded_patterns
+
+ for pathname in pathnames:
+ # We don't call the public match_path because it checks arguments
+ # and sets default values if none are found. We're already doing that
+ # above.
+ if _match_path(pathname, included, excluded, case_sensitive):
+ return True
+ return False
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/tricks/__init__.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/tricks/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..cdcc14b226ff8364f60fa6b2e3eae1f3834587db
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/tricks/__init__.py
@@ -0,0 +1,174 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import signal
+import subprocess
+import time
+
+from wandb_watchdog.utils import echo, has_attribute
+from wandb_watchdog.events import PatternMatchingEventHandler
+
+
+class Trick(PatternMatchingEventHandler):
+
+ """Your tricks should subclass this class."""
+
+ @classmethod
+ def generate_yaml(cls):
+ context = dict(module_name=cls.__module__,
+ klass_name=cls.__name__)
+ template_yaml = """- %(module_name)s.%(klass_name)s:
+ args:
+ - argument1
+ - argument2
+ kwargs:
+ patterns:
+ - "*.py"
+ - "*.js"
+ ignore_patterns:
+ - "version.py"
+ ignore_directories: false
+"""
+ return template_yaml % context
+
+
+class LoggerTrick(Trick):
+
+ """A simple trick that does only logs events."""
+
+ def on_any_event(self, event):
+ pass
+
+ @echo.echo
+ def on_modified(self, event):
+ pass
+
+ @echo.echo
+ def on_deleted(self, event):
+ pass
+
+ @echo.echo
+ def on_created(self, event):
+ pass
+
+ @echo.echo
+ def on_moved(self, event):
+ pass
+
+
+class ShellCommandTrick(Trick):
+
+ """Executes shell commands in response to matched events."""
+
+ def __init__(self, shell_command=None, patterns=None, ignore_patterns=None,
+ ignore_directories=False, wait_for_process=False,
+ drop_during_process=False):
+ super(ShellCommandTrick, self).__init__(patterns, ignore_patterns,
+ ignore_directories)
+ self.shell_command = shell_command
+ self.wait_for_process = wait_for_process
+ self.drop_during_process = drop_during_process
+ self.process = None
+
+ def on_any_event(self, event):
+ from string import Template
+
+ if self.drop_during_process and self.process and self.process.poll() is None:
+ return
+
+ if event.is_directory:
+ object_type = 'directory'
+ else:
+ object_type = 'file'
+
+ context = {
+ 'watch_src_path': event.src_path,
+ 'watch_dest_path': '',
+ 'watch_event_type': event.event_type,
+ 'watch_object': object_type,
+ }
+
+ if self.shell_command is None:
+ if has_attribute(event, 'dest_path'):
+ context.update({'dest_path': event.dest_path})
+ command = 'echo "${watch_event_type} ${watch_object} from ${watch_src_path} to ${watch_dest_path}"'
+ else:
+ command = 'echo "${watch_event_type} ${watch_object} ${watch_src_path}"'
+ else:
+ if has_attribute(event, 'dest_path'):
+ context.update({'watch_dest_path': event.dest_path})
+ command = self.shell_command
+
+ command = Template(command).safe_substitute(**context)
+ self.process = subprocess.Popen(command, shell=True)
+ if self.wait_for_process:
+ self.process.wait()
+
+
+class AutoRestartTrick(Trick):
+
+ """Starts a long-running subprocess and restarts it on matched events.
+
+ The command parameter is a list of command arguments, such as
+ ['bin/myserver', '-c', 'etc/myconfig.ini'].
+
+ Call start() after creating the Trick. Call stop() when stopping
+ the process.
+ """
+
+ def __init__(self, command, patterns=None, ignore_patterns=None,
+ ignore_directories=False, stop_signal=signal.SIGINT,
+ kill_after=10):
+ super(AutoRestartTrick, self).__init__(
+ patterns, ignore_patterns, ignore_directories)
+ self.command = command
+ self.stop_signal = stop_signal
+ self.kill_after = kill_after
+ self.process = None
+
+ def start(self):
+ self.process = subprocess.Popen(self.command, preexec_fn=os.setsid)
+
+ def stop(self):
+ if self.process is None:
+ return
+ try:
+ os.killpg(os.getpgid(self.process.pid), self.stop_signal)
+ except OSError:
+ # Process is already gone
+ pass
+ else:
+ kill_time = time.time() + self.kill_after
+ while time.time() < kill_time:
+ if self.process.poll() is not None:
+ break
+ time.sleep(0.25)
+ else:
+ try:
+ os.killpg(os.getpgid(self.process.pid), 9)
+ except OSError:
+ # Process is already gone
+ pass
+ self.process = None
+
+ @echo.echo
+ def on_any_event(self, event):
+ self.stop()
+ self.start()
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/__init__.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..0e1b25803132190a2338f1f95ae6910295d2e473
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/__init__.py
@@ -0,0 +1,151 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+:module: watchdog.utils
+:synopsis: Utility classes and functions.
+:author: yesudeep@google.com (Yesudeep Mangalapilly)
+
+Classes
+-------
+.. autoclass:: BaseThread
+ :members:
+ :show-inheritance:
+ :inherited-members:
+
+"""
+import os
+import sys
+import threading
+from wandb_watchdog.utils import platform
+from wandb_watchdog.utils.compat import Event
+
+
+if sys.version_info[0] == 2 and platform.is_windows():
+ # st_ino is not implemented in os.stat on this platform
+ import win32stat
+ stat = win32stat.stat
+else:
+ stat = os.stat
+
+
+def has_attribute(ob, attribute):
+ """
+ :func:`hasattr` swallows exceptions. :func:`has_attribute` tests a Python object for the
+ presence of an attribute.
+
+ :param ob:
+ object to inspect
+ :param attribute:
+ ``str`` for the name of the attribute.
+ """
+ return getattr(ob, attribute, None) is not None
+
+
+class UnsupportedLibc(Exception):
+ pass
+
+
+class BaseThread(threading.Thread):
+ """ Convenience class for creating stoppable threads. """
+
+ def __init__(self):
+ threading.Thread.__init__(self)
+ self.daemon = True
+ self._stopped_event = Event()
+
+ @property
+ def stopped_event(self):
+ return self._stopped_event
+
+ def should_keep_running(self):
+ """Determines whether the thread should continue running."""
+ return not self._stopped_event.is_set()
+
+ def on_thread_stop(self):
+ """Override this method instead of :meth:`stop()`.
+ :meth:`stop()` calls this method.
+
+ This method is called immediately after the thread is signaled to stop.
+ """
+ pass
+
+ def stop(self):
+ """Signals the thread to stop."""
+ self._stopped_event.set()
+ self.on_thread_stop()
+
+ def on_thread_start(self):
+ """Override this method instead of :meth:`start()`. :meth:`start()`
+ calls this method.
+
+ This method is called right before this thread is started and this
+ object’s run() method is invoked.
+ """
+ pass
+
+ def start(self):
+ self.on_thread_start()
+ threading.Thread.start(self)
+
+
+def load_module(module_name):
+ """Imports a module given its name and returns a handle to it."""
+ try:
+ __import__(module_name)
+ except ImportError:
+ raise ImportError('No module named %s' % module_name)
+ return sys.modules[module_name]
+
+
+def load_class(dotted_path):
+ """Loads and returns a class definition provided a dotted path
+ specification the last part of the dotted path is the class name
+ and there is at least one module name preceding the class name.
+
+ Notes:
+ You will need to ensure that the module you are trying to load
+ exists in the Python path.
+
+ Examples:
+ - module.name.ClassName # Provided module.name is in the Python path.
+ - module.ClassName # Provided module is in the Python path.
+
+ What won't work:
+ - ClassName
+ - modle.name.ClassName # Typo in module name.
+ - module.name.ClasNam # Typo in classname.
+ """
+ dotted_path_split = dotted_path.split('.')
+ if len(dotted_path_split) > 1:
+ klass_name = dotted_path_split[-1]
+ module_name = '.'.join(dotted_path_split[:-1])
+
+ module = load_module(module_name)
+ if has_attribute(module, klass_name):
+ klass = getattr(module, klass_name)
+ return klass
+ # Finally create and return an instance of the class
+ # return klass(*args, **kwargs)
+ else:
+ raise AttributeError('Module %s does not have class attribute %s' % (
+ module_name, klass_name))
+ else:
+ raise ValueError(
+ 'Dotted module path %s must contain a module name and a classname' % dotted_path)
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/bricks.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/bricks.py
new file mode 100644
index 0000000000000000000000000000000000000000..b1c2bedc5e0a74cb7c664c3f094666c371f68905
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/bricks.py
@@ -0,0 +1,249 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+Utility collections or "bricks".
+
+:module: watchdog.utils.bricks
+:author: yesudeep@google.com (Yesudeep Mangalapilly)
+:author: lalinsky@gmail.com (Lukáš Lalinský)
+:author: python@rcn.com (Raymond Hettinger)
+
+Classes
+=======
+.. autoclass:: OrderedSetQueue
+ :members:
+ :show-inheritance:
+ :inherited-members:
+
+.. autoclass:: OrderedSet
+
+"""
+
+import sys
+from collections.abc import MutableSet
+from .compat import queue
+
+class SkipRepeatsQueue(queue.Queue):
+
+ """Thread-safe implementation of an special queue where a
+ put of the last-item put'd will be dropped.
+
+ The implementation leverages locking already implemented in the base class
+ redefining only the primitives.
+
+ Queued items must be immutable and hashable so that they can be used
+ as dictionary keys. You must implement **only read-only properties** and
+ the :meth:`Item.__hash__()`, :meth:`Item.__eq__()`, and
+ :meth:`Item.__ne__()` methods for items to be hashable.
+
+ An example implementation follows::
+
+ class Item(object):
+ def __init__(self, a, b):
+ self._a = a
+ self._b = b
+
+ @property
+ def a(self):
+ return self._a
+
+ @property
+ def b(self):
+ return self._b
+
+ def _key(self):
+ return (self._a, self._b)
+
+ def __eq__(self, item):
+ return self._key() == item._key()
+
+ def __ne__(self, item):
+ return self._key() != item._key()
+
+ def __hash__(self):
+ return hash(self._key())
+
+ based on the OrderedSetQueue below
+ """
+
+ def _init(self, maxsize):
+ queue.Queue._init(self, maxsize)
+ self._last_item = None
+
+ def _put(self, item):
+ if item != self._last_item:
+ queue.Queue._put(self, item)
+ self._last_item = item
+ else:
+ # `put` increments `unfinished_tasks` even if we did not put
+ # anything into the queue here
+ self.unfinished_tasks -= 1
+
+ def _get(self):
+ item = queue.Queue._get(self)
+ if item is self._last_item:
+ self._last_item = None
+ return item
+
+
+class OrderedSetQueue(queue.Queue):
+
+ """Thread-safe implementation of an ordered set queue.
+
+ Disallows adding a duplicate item while maintaining the
+ order of items in the queue. The implementation leverages
+ locking already implemented in the base class
+ redefining only the primitives. Since the internal queue
+ is not replaced, the order is maintained. The set is used
+ merely to check for the existence of an item.
+
+ Queued items must be immutable and hashable so that they can be used
+ as dictionary keys. You must implement **only read-only properties** and
+ the :meth:`Item.__hash__()`, :meth:`Item.__eq__()`, and
+ :meth:`Item.__ne__()` methods for items to be hashable.
+
+ An example implementation follows::
+
+ class Item(object):
+ def __init__(self, a, b):
+ self._a = a
+ self._b = b
+
+ @property
+ def a(self):
+ return self._a
+
+ @property
+ def b(self):
+ return self._b
+
+ def _key(self):
+ return (self._a, self._b)
+
+ def __eq__(self, item):
+ return self._key() == item._key()
+
+ def __ne__(self, item):
+ return self._key() != item._key()
+
+ def __hash__(self):
+ return hash(self._key())
+
+ :author: lalinsky@gmail.com (Lukáš Lalinský)
+ :url: http://stackoverflow.com/questions/1581895/how-check-if-a-task-is-already-in-python-queue
+ """
+
+ def _init(self, maxsize):
+ queue.Queue._init(self, maxsize)
+ self._set_of_items = set()
+
+ def _put(self, item):
+ if item not in self._set_of_items:
+ queue.Queue._put(self, item)
+ self._set_of_items.add(item)
+ else:
+ # `put` increments `unfinished_tasks` even if we did not put
+ # anything into the queue here
+ self.unfinished_tasks -= 1
+
+ def _get(self):
+ item = queue.Queue._get(self)
+ self._set_of_items.remove(item)
+ return item
+
+
+if sys.version_info >= (2, 6, 0):
+ KEY, PREV, NEXT = list(range(3))
+
+ class OrderedSet(MutableSet):
+
+ """
+ Implementation based on a doubly-linked link and an internal dictionary.
+ This design gives :class:`OrderedSet` the same big-Oh running times as
+ regular sets including O(1) adds, removes, and lookups as well as
+ O(n) iteration.
+
+ .. ADMONITION:: Implementation notes
+
+ Runs on Python 2.6 or later (and runs on Python 3.0 or later
+ without any modifications).
+
+ :author: python@rcn.com (Raymond Hettinger)
+ :url: http://code.activestate.com/recipes/576694/
+ """
+
+ def __init__(self, iterable=None):
+ self.end = end = []
+ end += [None, end, end] # sentinel node for doubly linked list
+ self.map = {} # key --> [key, prev, next]
+ if iterable is not None:
+ self |= iterable
+
+ def __len__(self):
+ return len(self.map)
+
+ def __contains__(self, key):
+ return key in self.map
+
+ def add(self, key):
+ if key not in self.map:
+ end = self.end
+ curr = end[PREV]
+ curr[NEXT] = end[PREV] = self.map[key] = [key, curr, end]
+
+ def discard(self, key):
+ if key in self.map:
+ key, prev, _next = self.map.pop(key)
+ prev[NEXT] = _next
+ _next[PREV] = prev
+
+ def __iter__(self):
+ end = self.end
+ curr = end[NEXT]
+ while curr is not end:
+ yield curr[KEY]
+ curr = curr[NEXT]
+
+ def __reversed__(self):
+ end = self.end
+ curr = end[PREV]
+ while curr is not end:
+ yield curr[KEY]
+ curr = curr[PREV]
+
+ def pop(self, last=True):
+ if not self:
+ raise KeyError('set is empty')
+ key = next(reversed(self)) if last else next(iter(self))
+ self.discard(key)
+ return key
+
+ def __repr__(self):
+ if not self:
+ return '%s()' % (self.__class__.__name__,)
+ return '%s(%r)' % (self.__class__.__name__, list(self))
+
+ def __eq__(self, other):
+ if isinstance(other, OrderedSet):
+ return len(self) == len(other) and list(self) == list(other)
+ return set(self) == set(other)
+
+ def __del__(self):
+ self.clear() # remove circular references
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/compat.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/compat.py
new file mode 100644
index 0000000000000000000000000000000000000000..0f6e7947b924a0bd07a99b7e6523c8a161beac42
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/compat.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2014 Thomas Amland <thomas.amland@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+
+__all__ = ['queue', 'Event']
+
+try:
+ import queue
+except ImportError:
+ import Queue as queue
+
+
+if sys.version_info < (2, 7):
+ from watchdog.utils.event_backport import Event
+else:
+ from threading import Event
\ No newline at end of file
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/decorators.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/decorators.py
new file mode 100644
index 0000000000000000000000000000000000000000..abb325c1c1028746cf2a9a1a99bf81432bbae657
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/decorators.py
@@ -0,0 +1,198 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Most of this code was obtained from the Python documentation online.
+
+"""Decorator utility functions.
+
+decorators:
+- synchronized
+- propertyx
+- accepts
+- returns
+- singleton
+- attrs
+- deprecated
+"""
+
+import functools
+import warnings
+import threading
+import sys
+
+
+def synchronized(lock=None):
+ """Decorator that synchronizes a method or a function with a mutex lock.
+
+ Example usage:
+
+ @synchronized()
+ def operation(self, a, b):
+ ...
+ """
+ if lock is None:
+ lock = threading.Lock()
+
+ def wrapper(function):
+ def new_function(*args, **kwargs):
+ lock.acquire()
+ try:
+ return function(*args, **kwargs)
+ finally:
+ lock.release()
+
+ return new_function
+
+ return wrapper
+
+
+def propertyx(function):
+ """Decorator to easily create properties in classes.
+
+ Example:
+
+ class Angle(object):
+ def __init__(self, rad):
+ self._rad = rad
+
+ @property
+ def rad():
+ def fget(self):
+ return self._rad
+ def fset(self, angle):
+ if isinstance(angle, Angle):
+ angle = angle.rad
+ self._rad = float(angle)
+
+ Arguments:
+ - `function`: The function to be decorated.
+ """
+ keys = ('fget', 'fset', 'fdel')
+ func_locals = {'doc': function.__doc__}
+
+ def probe_func(frame, event, arg):
+ if event == 'return':
+ locals = frame.f_locals
+ func_locals.update(dict((k, locals.get(k)) for k in keys))
+ sys.settrace(None)
+ return probe_func
+
+ sys.settrace(probe_func)
+ function()
+ return property(**func_locals)
+
+
+def accepts(*types):
+ """Decorator to ensure that the decorated function accepts the given types as arguments.
+
+ Example:
+ @accepts(int, (int,float))
+ @returns((int,float))
+ def func(arg1, arg2):
+ return arg1 * arg2
+ """
+
+ def check_accepts(f):
+ assert len(types) == f.__code__.co_argcount
+
+ def new_f(*args, **kwds):
+ for (a, t) in zip(args, types):
+ assert isinstance(a, t),\
+ "arg %r does not match %s" % (a, t)
+ return f(*args, **kwds)
+
+ new_f.__name__ = f.__name__
+ return new_f
+
+ return check_accepts
+
+
+def returns(rtype):
+ """Decorator to ensure that the decorated function returns the given
+ type as argument.
+
+ Example:
+ @accepts(int, (int,float))
+ @returns((int,float))
+ def func(arg1, arg2):
+ return arg1 * arg2
+ """
+
+ def check_returns(f):
+ def new_f(*args, **kwds):
+ result = f(*args, **kwds)
+ assert isinstance(result, rtype),\
+ "return value %r does not match %s" % (result, rtype)
+ return result
+
+ new_f.__name__ = f.__name__
+ return new_f
+
+ return check_returns
+
+
+def singleton(cls):
+ """Decorator to ensures a class follows the singleton pattern.
+
+ Example:
+ @singleton
+ class MyClass:
+ ...
+ """
+ instances = {}
+
+ def getinstance():
+ if cls not in instances:
+ instances[cls] = cls()
+ return instances[cls]
+
+ return getinstance
+
+
+def attrs(**kwds):
+ """Decorator to add attributes to a function.
+
+ Example:
+
+ @attrs(versionadded="2.2",
+ author="Guido van Rossum")
+ def mymethod(f):
+ ...
+ """
+
+ def decorate(f):
+ for k in kwds:
+ setattr(f, k, kwds[k])
+ return f
+
+ return decorate
+
+
+def deprecated(func):
+ """This is a decorator which can be used to mark functions
+ as deprecated. It will result in a warning being emitted
+ when the function is used.
+
+ ## Usage examples ##
+ @deprecated
+ def my_func():
+ pass
+
+ @other_decorators_must_be_upper
+ @deprecated
+ def my_func():
+ pass
+ """
+
+ @functools.wraps(func)
+ def new_func(*args, **kwargs):
+ warnings.warn_explicit(
+ "Call to deprecated function %(funcname)s." % {
+ 'funcname': func.__name__,
+ },
+ category=DeprecationWarning,
+ filename=func.__code__.co_filename,
+ lineno=func.__code__.co_firstlineno + 1
+ )
+ return func(*args, **kwargs)
+
+ return new_func
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/delayed_queue.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/delayed_queue.py
new file mode 100644
index 0000000000000000000000000000000000000000..6d98a50469b4bbe89f20639005fe9ba1e3f8d3cc
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/delayed_queue.py
@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2014 Thomas Amland <thomas.amland@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+import threading
+from collections import deque
+
+
+class DelayedQueue(object):
+
+ def __init__(self, delay):
+ self.delay = delay
+ self._lock = threading.Lock()
+ self._not_empty = threading.Condition(self._lock)
+ self._queue = deque()
+ self._closed = False
+
+ def put(self, element):
+ """Add element to queue."""
+ self._lock.acquire()
+ self._queue.append((element, time.time()))
+ self._not_empty.notify()
+ self._lock.release()
+
+ def close(self):
+ """Close queue, indicating no more items will be added."""
+ self._closed = True
+ # Interrupt the blocking _not_empty.wait() call in get
+ self._not_empty.acquire()
+ self._not_empty.notify()
+ self._not_empty.release()
+
+ def get(self):
+ """Remove and return an element from the queue, or this queue has been
+ closed raise the Closed exception.
+ """
+ while True:
+ # wait for element to be added to queue
+ self._not_empty.acquire()
+ while len(self._queue) == 0 and not self._closed:
+ self._not_empty.wait()
+
+ if self._closed:
+ self._not_empty.release()
+ return None
+ head, insert_time = self._queue[0]
+ self._not_empty.release()
+
+ # wait for delay
+ time_left = insert_time + self.delay - time.time()
+ while time_left > 0:
+ time.sleep(time_left)
+ time_left = insert_time + self.delay - time.time()
+
+ # return element if it's still in the queue
+ self._lock.acquire()
+ try:
+ if len(self._queue) > 0 and self._queue[0][0] is head:
+ self._queue.popleft()
+ return head
+ finally:
+ self._lock.release()
+
+ def remove(self, predicate):
+ """Remove and return the first items for which predicate is True,
+ ignoring delay."""
+ try:
+ self._lock.acquire()
+ for i, (elem, t) in enumerate(self._queue):
+ if predicate(elem):
+ del self._queue[i]
+ return elem
+ finally:
+ self._lock.release()
+ return None
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/dirsnapshot.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/dirsnapshot.py
new file mode 100644
index 0000000000000000000000000000000000000000..c321d0ffe45c3b943e4649c1dbeec9931bf0db5d
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/dirsnapshot.py
@@ -0,0 +1,293 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+# Copyright 2014 Thomas Amland <thomas.amland@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+:module: watchdog.utils.dirsnapshot
+:synopsis: Directory snapshots and comparison.
+:author: yesudeep@google.com (Yesudeep Mangalapilly)
+
+.. ADMONITION:: Where are the moved events? They "disappeared"
+
+ This implementation does not take partition boundaries
+ into consideration. It will only work when the directory
+ tree is entirely on the same file system. More specifically,
+ any part of the code that depends on inode numbers can
+ break if partition boundaries are crossed. In these cases,
+ the snapshot diff will represent file/directory movement as
+ created and deleted events.
+
+Classes
+-------
+.. autoclass:: DirectorySnapshot
+ :members:
+ :show-inheritance:
+
+.. autoclass:: DirectorySnapshotDiff
+ :members:
+ :show-inheritance:
+
+"""
+
+import errno
+import os
+from stat import S_ISDIR
+from wandb_watchdog.utils import stat as default_stat
+
+
+class DirectorySnapshotDiff(object):
+ """
+ Compares two directory snapshots and creates an object that represents
+ the difference between the two snapshots.
+
+ :param ref:
+ The reference directory snapshot.
+ :type ref:
+ :class:`DirectorySnapshot`
+ :param snapshot:
+ The directory snapshot which will be compared
+ with the reference snapshot.
+ :type snapshot:
+ :class:`DirectorySnapshot`
+ """
+
+ def __init__(self, ref, snapshot):
+ created = snapshot.paths - ref.paths
+ deleted = ref.paths - snapshot.paths
+
+ # check that all unchanged paths have the same inode
+ for path in ref.paths & snapshot.paths:
+ if ref.inode(path) != snapshot.inode(path):
+ created.add(path)
+ deleted.add(path)
+
+ # find moved paths
+ moved = set()
+ for path in set(deleted):
+ inode = ref.inode(path)
+ new_path = snapshot.path(inode)
+ if new_path:
+ # file is not deleted but moved
+ deleted.remove(path)
+ moved.add((path, new_path))
+
+ for path in set(created):
+ inode = snapshot.inode(path)
+ old_path = ref.path(inode)
+ if old_path:
+ created.remove(path)
+ moved.add((old_path, path))
+
+ # find modified paths
+ # first check paths that have not moved
+ modified = set()
+ for path in ref.paths & snapshot.paths:
+ if ref.inode(path) == snapshot.inode(path):
+ if ref.mtime(path) != snapshot.mtime(path):
+ modified.add(path)
+
+ for (old_path, new_path) in moved:
+ if ref.mtime(old_path) != snapshot.mtime(new_path):
+ modified.add(old_path)
+
+ self._dirs_created = [path for path in created if snapshot.isdir(path)]
+ self._dirs_deleted = [path for path in deleted if ref.isdir(path)]
+ self._dirs_modified = [path for path in modified if ref.isdir(path)]
+ self._dirs_moved = [(frm, to) for (frm, to) in moved if ref.isdir(frm)]
+
+ self._files_created = list(created - set(self._dirs_created))
+ self._files_deleted = list(deleted - set(self._dirs_deleted))
+ self._files_modified = list(modified - set(self._dirs_modified))
+ self._files_moved = list(moved - set(self._dirs_moved))
+
+ @property
+ def files_created(self):
+ """List of files that were created."""
+ return self._files_created
+
+ @property
+ def files_deleted(self):
+ """List of files that were deleted."""
+ return self._files_deleted
+
+ @property
+ def files_modified(self):
+ """List of files that were modified."""
+ return self._files_modified
+
+ @property
+ def files_moved(self):
+ """
+ List of files that were moved.
+
+ Each event is a two-tuple the first item of which is the path
+ that has been renamed to the second item in the tuple.
+ """
+ return self._files_moved
+
+ @property
+ def dirs_modified(self):
+ """
+ List of directories that were modified.
+ """
+ return self._dirs_modified
+
+ @property
+ def dirs_moved(self):
+ """
+ List of directories that were moved.
+
+ Each event is a two-tuple the first item of which is the path
+ that has been renamed to the second item in the tuple.
+ """
+ return self._dirs_moved
+
+ @property
+ def dirs_deleted(self):
+ """
+ List of directories that were deleted.
+ """
+ return self._dirs_deleted
+
+ @property
+ def dirs_created(self):
+ """
+ List of directories that were created.
+ """
+ return self._dirs_created
+
+class DirectorySnapshot(object):
+ """
+ A snapshot of stat information of files in a directory.
+
+ :param path:
+ The directory path for which a snapshot should be taken.
+ :type path:
+ ``str``
+ :param recursive:
+ ``True`` if the entire directory tree should be included in the
+ snapshot; ``False`` otherwise.
+ :type recursive:
+ ``bool``
+ :param walker_callback:
+ .. deprecated:: 0.7.2
+ :param stat:
+ Use custom stat function that returns a stat structure for path.
+ Currently only st_dev, st_ino, st_mode and st_mtime are needed.
+
+ A function with the signature ``walker_callback(path, stat_info)``
+ which will be called for every entry in the directory tree.
+ :param listdir:
+ Use custom listdir function. See ``os.listdir`` for details.
+ """
+
+ def __init__(self, path, recursive=True,
+ walker_callback=(lambda p, s: None),
+ stat=default_stat,
+ listdir=os.listdir):
+ self._stat_info = {}
+ self._inode_to_path = {}
+
+ st = stat(path)
+ self._stat_info[path] = st
+ self._inode_to_path[(st.st_ino, st.st_dev)] = path
+
+ def walk(root):
+ try:
+ paths = [os.path.join(root, name) for name in listdir(root)]
+ except OSError as e:
+ # Directory may have been deleted between finding it in the directory
+ # list of its parent and trying to delete its contents. If this
+ # happens we treat it as empty.
+ if e.errno == errno.ENOENT:
+ return
+ else:
+ raise
+ entries = []
+ for p in paths:
+ try:
+ entries.append((p, stat(p)))
+ except OSError:
+ continue
+ for _ in entries:
+ yield _
+ if recursive:
+ for path, st in entries:
+ if S_ISDIR(st.st_mode):
+ for _ in walk(path):
+ yield _
+
+ for p, st in walk(path):
+ i = (st.st_ino, st.st_dev)
+ self._inode_to_path[i] = p
+ self._stat_info[p] = st
+ walker_callback(p, st)
+
+ @property
+ def paths(self):
+ """
+ Set of file/directory paths in the snapshot.
+ """
+ return set(self._stat_info.keys())
+
+ def path(self, id):
+ """
+ Returns path for id. None if id is unknown to this snapshot.
+ """
+ return self._inode_to_path.get(id)
+
+ def inode(self, path):
+ """ Returns an id for path. """
+ st = self._stat_info[path]
+ return (st.st_ino, st.st_dev)
+
+ def isdir(self, path):
+ return S_ISDIR(self._stat_info[path].st_mode)
+
+ def mtime(self, path):
+ return self._stat_info[path].st_mtime
+
+ def stat_info(self, path):
+ """
+ Returns a stat information object for the specified path from
+ the snapshot.
+
+ Attached information is subject to change. Do not use unless
+ you specify `stat` in constructor. Use :func:`inode`, :func:`mtime`,
+ :func:`isdir` instead.
+
+ :param path:
+ The path for which stat information should be obtained
+ from a snapshot.
+ """
+ return self._stat_info[path]
+
+ def __sub__(self, previous_dirsnap):
+ """Allow subtracting a DirectorySnapshot object instance from
+ another.
+
+ :returns:
+ A :class:`DirectorySnapshotDiff` object.
+ """
+ return DirectorySnapshotDiff(previous_dirsnap, self)
+
+ def __str__(self):
+ return self.__repr__()
+
+ def __repr__(self):
+ return str(self._stat_info)
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/echo.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/echo.py
new file mode 100644
index 0000000000000000000000000000000000000000..12803e030d5c75308657c5f6717f76059907774c
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/echo.py
@@ -0,0 +1,157 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# echo.py: Tracing function calls using Python decorators.
+#
+# Written by Thomas Guest <tag@wordaligned.org>
+# Please see http://wordaligned.org/articles/echo
+#
+# Place into the public domain.
+
+""" Echo calls made to functions and methods in a module.
+
+"Echoing" a function call means printing out the name of the function
+and the values of its arguments before making the call (which is more
+commonly referred to as "tracing", but Python already has a trace module).
+
+Example: to echo calls made to functions in "my_module" do:
+
+ import echo
+ import my_module
+ echo.echo_module(my_module)
+
+Example: to echo calls made to functions in "my_module.my_class" do:
+
+ echo.echo_class(my_module.my_class)
+
+Alternatively, echo.echo can be used to decorate functions. Calls to the
+decorated function will be echoed.
+
+Example:
+
+ @echo.echo
+ def my_function(args):
+ pass
+"""
+import inspect
+import sys
+
+
+def name(item):
+ " Return an item's name. "
+ return item.__name__
+
+
+def is_classmethod(instancemethod, klass):
+ " Determine if an instancemethod is a classmethod. "
+ return inspect.ismethod(instancemethod) and instancemethod.__self__ is klass
+
+def is_static_method(method, klass):
+ """Returns True if method is an instance method of klass."""
+ for c in klass.mro():
+ if name(method) in c.__dict__:
+ return isinstance(c.__dict__[name(method)], staticmethod)
+ else:
+ return False
+
+def is_class_private_name(name):
+ " Determine if a name is a class private name. "
+ # Exclude system defined names such as __init__, __add__ etc
+ return name.startswith("__") and not name.endswith("__")
+
+
+def method_name(method):
+ """ Return a method's name.
+
+ This function returns the name the method is accessed by from
+ outside the class (i.e. it prefixes "private" methods appropriately).
+ """
+ mname = name(method)
+ if is_class_private_name(mname):
+ mname = "_%s%s" % (name(method.__self__.__class__), mname)
+ return mname
+
+
+def format_arg_value(arg_val):
+ """ Return a string representing a (name, value) pair.
+
+ >>> format_arg_value(('x', (1, 2, 3)))
+ 'x=(1, 2, 3)'
+ """
+ arg, val = arg_val
+ return "%s=%r" % (arg, val)
+
+
+def echo(fn, write=sys.stdout.write):
+ """ Echo calls to a function.
+
+ Returns a decorated version of the input function which "echoes" calls
+ made to it by writing out the function's name and the arguments it was
+ called with.
+ """
+ import functools
+ # Unpack function's arg count, arg names, arg defaults
+ code = fn.__code__
+ argcount = code.co_argcount
+ argnames = code.co_varnames[:argcount]
+ fn_defaults = fn.__defaults__ or list()
+ argdefs = dict(list(zip(argnames[-len(fn_defaults):], fn_defaults)))
+
+ @functools.wraps(fn)
+ def wrapped(*v, **k):
+ # Collect function arguments by chaining together positional,
+ # defaulted, extra positional and keyword arguments.
+ positional = list(map(format_arg_value, list(zip(argnames, v))))
+ defaulted = [format_arg_value((a, argdefs[a]))
+ for a in argnames[len(v):] if a not in k]
+ nameless = list(map(repr, v[argcount:]))
+ keyword = list(map(format_arg_value, list(k.items())))
+ args = positional + defaulted + nameless + keyword
+ write("%s(%s)\n" % (name(fn), ", ".join(args)))
+ return fn(*v, **k)
+
+ return wrapped
+
+
+def echo_instancemethod(klass, method, write=sys.stdout.write):
+ """ Change an instancemethod so that calls to it are echoed.
+
+ Replacing a classmethod is a little more tricky.
+ See: http://www.python.org/doc/current/ref/types.html
+ """
+ mname = method_name(method)
+ never_echo = "__str__", "__repr__", # Avoid recursion printing method calls
+ if mname in never_echo:
+ pass
+ elif is_classmethod(method, klass):
+ setattr(klass, mname, classmethod(echo(method.__func__, write)))
+ else:
+ setattr(klass, mname, echo(method, write))
+
+def echo_class(klass, write=sys.stdout.write):
+ """ Echo calls to class methods and static functions
+ """
+ for _, method in inspect.getmembers(klass, inspect.ismethod):
+ #In python 3 only class methods are returned here, but in python2 instance methods are too.
+ echo_instancemethod(klass, method, write)
+ for _, fn in inspect.getmembers(klass, inspect.isfunction):
+ if is_static_method(fn, klass):
+ setattr(klass, name(fn), staticmethod(echo(fn, write)))
+ else:
+ #It's not a class or a static method, so it must be an instance method.
+ #This should only be called in python 3, because in python 3 instance methods are considered functions.
+ echo_instancemethod(klass, fn, write)
+
+def echo_module(mod, write=sys.stdout.write):
+ """ Echo calls to functions and methods in a module.
+ """
+ for fname, fn in inspect.getmembers(mod, inspect.isfunction):
+ setattr(mod, fname, echo(fn, write))
+ for _, klass in inspect.getmembers(mod, inspect.isclass):
+ echo_class(klass, write)
+
+if __name__ == "__main__":
+ import doctest
+
+ optionflags = doctest.ELLIPSIS
+ doctest.testfile('echoexample.txt', optionflags=optionflags)
+ doctest.testmod(optionflags=optionflags)
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/event_backport.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/event_backport.py
new file mode 100644
index 0000000000000000000000000000000000000000..5c136e46d54839347c36e7f3ff81ff64f51b0d5b
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/event_backport.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Backport of Event from py2.7 (method wait in py2.6 returns None)
+
+from threading import Condition, Lock
+
+
+class Event(object):
+
+ def __init__(self,):
+ self.__cond = Condition(Lock())
+ self.__flag = False
+
+ def isSet(self):
+ return self.__flag
+
+ is_set = isSet
+
+ def set(self):
+ self.__cond.acquire()
+ try:
+ self.__flag = True
+ self.__cond.notify_all()
+ finally:
+ self.__cond.release()
+
+ def clear(self):
+ self.__cond.acquire()
+ try:
+ self.__flag = False
+ finally:
+ self.__cond.release()
+
+ def wait(self, timeout=None):
+ self.__cond.acquire()
+ try:
+ if not self.__flag:
+ self.__cond.wait(timeout)
+ return self.__flag
+ finally:
+ self.__cond.release()
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/importlib2.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/importlib2.py
new file mode 100644
index 0000000000000000000000000000000000000000..5ad3ec57204c6b8a6db0589b631c849f0b145a00
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/importlib2.py
@@ -0,0 +1,40 @@
+# The MIT License (MIT)
+
+# Copyright (c) 2013 Peter M. Elias
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE
+
+
+def import_module(target, relative_to=None):
+ target_parts = target.split('.')
+ target_depth = target_parts.count('')
+ target_path = target_parts[target_depth:]
+ target = target[target_depth:]
+ fromlist = [target]
+ if target_depth and relative_to:
+ relative_parts = relative_to.split('.')
+ relative_to = '.'.join(relative_parts[:-(target_depth - 1) or None])
+ if len(target_path) > 1:
+ relative_to = '.'.join(filter(None, [relative_to]) + target_path[:-1])
+ fromlist = target_path[-1:]
+ target = fromlist[0]
+ elif not relative_to:
+ fromlist = []
+ mod = __import__(relative_to or target, globals(), locals(), fromlist)
+ return getattr(mod, target, mod)
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/platform.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/platform.py
new file mode 100644
index 0000000000000000000000000000000000000000..239c6a25829dde1b70cfab092678820ca91a326d
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/platform.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import sys
+
+PLATFORM_WINDOWS = 'windows'
+PLATFORM_LINUX = 'linux'
+PLATFORM_BSD = 'bsd'
+PLATFORM_DARWIN = 'darwin'
+PLATFORM_UNKNOWN = 'unknown'
+
+
+def get_platform_name():
+ if sys.platform.startswith("win"):
+ return PLATFORM_WINDOWS
+ elif sys.platform.startswith('darwin'):
+ return PLATFORM_DARWIN
+ elif sys.platform.startswith('linux'):
+ return PLATFORM_LINUX
+ elif sys.platform.startswith(('dragonfly', 'freebsd', 'netbsd', 'openbsd', )):
+ return PLATFORM_BSD
+ else:
+ return PLATFORM_UNKNOWN
+
+__platform__ = get_platform_name()
+
+
+def is_linux():
+ return __platform__ == PLATFORM_LINUX
+
+
+def is_bsd():
+ return __platform__ == PLATFORM_BSD
+
+
+def is_darwin():
+ return __platform__ == PLATFORM_DARWIN
+
+
+def is_windows():
+ return __platform__ == PLATFORM_WINDOWS
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/unicode_paths.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/unicode_paths.py
new file mode 100644
index 0000000000000000000000000000000000000000..9e4b4b425fd9c7420d2a01ff5ddf0c7eadc60b06
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/unicode_paths.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 Will Bond <will@wbond.net>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+import sys
+
+from wandb_watchdog.utils import platform
+
+try:
+ # Python 2
+ str_cls = unicode
+ bytes_cls = str
+except NameError:
+ # Python 3
+ str_cls = str
+ bytes_cls = bytes
+
+
+# This is used by Linux when the locale seems to be improperly set. UTF-8 tends
+# to be the encoding used by all distros, so this is a good fallback.
+fs_fallback_encoding = 'utf-8'
+fs_encoding = sys.getfilesystemencoding() or fs_fallback_encoding
+
+
+def encode(path):
+ if isinstance(path, str_cls):
+ try:
+ path = path.encode(fs_encoding, 'strict')
+ except UnicodeEncodeError:
+ if not platform.is_linux():
+ raise
+ path = path.encode(fs_fallback_encoding, 'strict')
+ return path
+
+
+def decode(path):
+ if isinstance(path, bytes_cls):
+ try:
+ path = path.decode(fs_encoding, 'strict')
+ except UnicodeDecodeError:
+ if not platform.is_linux():
+ raise
+ path = path.decode(fs_fallback_encoding, 'strict')
+ return path
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/win32stat.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/win32stat.py
new file mode 100644
index 0000000000000000000000000000000000000000..398d1067942ab987d230ec87eaa54a3f58fd67c5
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/win32stat.py
@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2014 Thomas Amland <thomas.amland@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+:module: watchdog.utils.win32stat
+:synopsis: Implementation of stat with st_ino and st_dev support.
+
+Functions
+---------
+
+.. autofunction:: stat
+
+"""
+
+import ctypes
+import ctypes.wintypes
+import stat as stdstat
+from collections import namedtuple
+
+
+INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value
+OPEN_EXISTING = 3
+FILE_READ_ATTRIBUTES = 0x80
+FILE_ATTRIBUTE_NORMAL = 0x80
+FILE_ATTRIBUTE_READONLY = 0x1
+FILE_ATTRIBUTE_DIRECTORY = 0x10
+FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
+FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000
+
+
+class FILETIME(ctypes.Structure):
+ _fields_ = [("dwLowDateTime", ctypes.wintypes.DWORD),
+ ("dwHighDateTime", ctypes.wintypes.DWORD)]
+
+
+class BY_HANDLE_FILE_INFORMATION(ctypes.Structure):
+ _fields_ = [('dwFileAttributes', ctypes.wintypes.DWORD),
+ ('ftCreationTime', FILETIME),
+ ('ftLastAccessTime', FILETIME),
+ ('ftLastWriteTime', FILETIME),
+ ('dwVolumeSerialNumber', ctypes.wintypes.DWORD),
+ ('nFileSizeHigh', ctypes.wintypes.DWORD),
+ ('nFileSizeLow', ctypes.wintypes.DWORD),
+ ('nNumberOfLinks', ctypes.wintypes.DWORD),
+ ('nFileIndexHigh', ctypes.wintypes.DWORD),
+ ('nFileIndexLow', ctypes.wintypes.DWORD)]
+
+
+CreateFile = ctypes.windll.kernel32.CreateFileW
+CreateFile.restype = ctypes.wintypes.HANDLE
+CreateFile.argtypes = (
+ ctypes.c_wchar_p,
+ ctypes.wintypes.DWORD,
+ ctypes.wintypes.DWORD,
+ ctypes.c_void_p,
+ ctypes.wintypes.DWORD,
+ ctypes.wintypes.DWORD,
+ ctypes.wintypes.HANDLE,
+)
+
+GetFileInformationByHandle = ctypes.windll.kernel32.GetFileInformationByHandle
+GetFileInformationByHandle.restype = ctypes.wintypes.BOOL
+GetFileInformationByHandle.argtypes = (
+ ctypes.wintypes.HANDLE,
+ ctypes.wintypes.POINTER(BY_HANDLE_FILE_INFORMATION),
+)
+
+CloseHandle = ctypes.windll.kernel32.CloseHandle
+CloseHandle.restype = ctypes.wintypes.BOOL
+CloseHandle.argtypes = (ctypes.wintypes.HANDLE,)
+
+
+StatResult = namedtuple('StatResult', 'st_dev st_ino st_mode st_mtime')
+
+def _to_mode(attr):
+ m = 0
+ if (attr & FILE_ATTRIBUTE_DIRECTORY):
+ m |= stdstat.S_IFDIR | 0o111
+ else:
+ m |= stdstat.S_IFREG
+ if (attr & FILE_ATTRIBUTE_READONLY):
+ m |= 0o444
+ else:
+ m |= 0o666
+ return m
+
+def _to_unix_time(ft):
+ t = (ft.dwHighDateTime) << 32 | ft.dwLowDateTime
+ return (t / 10000000) - 11644473600
+
+def stat(path):
+ hfile = CreateFile(path,
+ FILE_READ_ATTRIBUTES,
+ 0,
+ None,
+ OPEN_EXISTING,
+ FILE_ATTRIBUTE_NORMAL | FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT,
+ None)
+ if hfile == INVALID_HANDLE_VALUE:
+ raise ctypes.WinError
+ info = BY_HANDLE_FILE_INFORMATION()
+ r = GetFileInformationByHandle(hfile, info)
+ CloseHandle(hfile)
+ if not r:
+ raise ctypes.WinError
+ return StatResult(st_dev=info.dwVolumeSerialNumber,
+ st_ino=(info.nFileIndexHigh << 32) + info.nFileIndexLow,
+ st_mode=_to_mode(info.dwFileAttributes),
+ st_mtime=_to_unix_time(info.ftLastWriteTime)
+ )
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/version.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/version.py
new file mode 100644
index 0000000000000000000000000000000000000000..295497c89bcf6c0f25562d7f9b576b5fef717ea6
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/version.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# When updating this version number, please update the
+# ``docs/source/global.rst.inc`` file as well.
+VERSION_MAJOR = 0
+VERSION_MINOR = 9
+VERSION_BUILD = 0
+VERSION_INFO = (VERSION_MAJOR, VERSION_MINOR, VERSION_BUILD)
+VERSION_STRING = "%d.%d.%d" % VERSION_INFO
+
+__version__ = VERSION_INFO
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/wandb-vendor.md b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/wandb-vendor.md
new file mode 100644
index 0000000000000000000000000000000000000000..dff2e0d00f0152952e0edcd11e77722f2ec71240
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/wandb-vendor.md
@@ -0,0 +1,11 @@
+# Vendoring notes
+
+This directory contains vendored code from the [watchdog](https://github.com/gorakhargosh/watchdog) project.
+It is based on the [v0.9.0](https://github.com/gorakhargosh/watchdog/releases/tag/v0.9.0) release and contains
+the following changes:
+
+- Removed dependency on the [`pathtools`](https://github.com/gorakhargosh/pathtools) and instead vendored the
+ `patterns.py` file from that project.
+- Added the `absolute_path` function to `observers/kqueue.py` instead of importing it from `pathtools`.
+
+See https://github.com/wandb/wandb/pull/3443 for more details.
diff --git a/wandb/vendor/watchdog_0_9_0/wandb_watchdog/watchmedo.py b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/watchmedo.py
new file mode 100755
index 0000000000000000000000000000000000000000..c634baac77096c4775b16f3db4e4796281b703ca
--- /dev/null
+++ b/wandb/vendor/watchdog_0_9_0/wandb_watchdog/watchmedo.py
@@ -0,0 +1,577 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
+# Copyright 2012 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+:module: watchdog.watchmedo
+:author: yesudeep@google.com (Yesudeep Mangalapilly)
+:synopsis: ``watchmedo`` shell script utility.
+"""
+
+import os.path
+import sys
+import yaml
+import time
+import logging
+
+try:
+ from cStringIO import StringIO
+except ImportError:
+ try:
+ from StringIO import StringIO
+ except ImportError:
+ from io import StringIO
+
+from argh import arg, aliases, ArghParser, expects_obj
+from wandb_watchdog.version import VERSION_STRING
+from wandb_watchdog.utils import load_class
+
+
+logging.basicConfig(level=logging.INFO)
+
+CONFIG_KEY_TRICKS = 'tricks'
+CONFIG_KEY_PYTHON_PATH = 'python-path'
+
+
+def path_split(pathname_spec, separator=os.path.sep):
+ """
+ Splits a pathname specification separated by an OS-dependent separator.
+
+ :param pathname_spec:
+ The pathname specification.
+ :param separator:
+ (OS Dependent) `:` on Unix and `;` on Windows or user-specified.
+ """
+ return list(pathname_spec.split(separator))
+
+
+def add_to_sys_path(pathnames, index=0):
+ """
+ Adds specified paths at specified index into the sys.path list.
+
+ :param paths:
+ A list of paths to add to the sys.path
+ :param index:
+ (Default 0) The index in the sys.path list where the paths will be
+ added.
+ """
+ for pathname in pathnames[::-1]:
+ sys.path.insert(index, pathname)
+
+
+def load_config(tricks_file_pathname):
+ """
+ Loads the YAML configuration from the specified file.
+
+ :param tricks_file_path:
+ The path to the tricks configuration file.
+ :returns:
+ A dictionary of configuration information.
+ """
+ f = open(tricks_file_pathname, 'rb')
+ content = f.read()
+ f.close()
+ config = yaml.safe_load(content)
+ return config
+
+
+def parse_patterns(patterns_spec, ignore_patterns_spec, separator=';'):
+ """
+ Parses pattern argument specs and returns a two-tuple of
+ (patterns, ignore_patterns).
+ """
+ patterns = patterns_spec.split(separator)
+ ignore_patterns = ignore_patterns_spec.split(separator)
+ if ignore_patterns == ['']:
+ ignore_patterns = []
+ return (patterns, ignore_patterns)
+
+
+def observe_with(observer, event_handler, pathnames, recursive):
+ """
+ Single observer thread with a scheduled path and event handler.
+
+ :param observer:
+ The observer thread.
+ :param event_handler:
+ Event handler which will be called in response to file system events.
+ :param pathnames:
+ A list of pathnames to monitor.
+ :param recursive:
+ ``True`` if recursive; ``False`` otherwise.
+ """
+ for pathname in set(pathnames):
+ observer.schedule(event_handler, pathname, recursive)
+ observer.start()
+ try:
+ while True:
+ time.sleep(1)
+ except KeyboardInterrupt:
+ observer.stop()
+ observer.join()
+
+
+def schedule_tricks(observer, tricks, pathname, recursive):
+ """
+ Schedules tricks with the specified observer and for the given watch
+ path.
+
+ :param observer:
+ The observer thread into which to schedule the trick and watch.
+ :param tricks:
+ A list of tricks.
+ :param pathname:
+ A path name which should be watched.
+ :param recursive:
+ ``True`` if recursive; ``False`` otherwise.
+ """
+ for trick in tricks:
+ for name, value in list(trick.items()):
+ TrickClass = load_class(name)
+ handler = TrickClass(**value)
+ trick_pathname = getattr(handler, 'source_directory', None) or pathname
+ observer.schedule(handler, trick_pathname, recursive)
+
+
+@aliases('tricks')
+@arg('files',
+ nargs='*',
+ help='perform tricks from given file')
+@arg('--python-path',
+ default='.',
+ help='paths separated by %s to add to the python path' % os.path.sep)
+@arg('--interval',
+ '--timeout',
+ dest='timeout',
+ default=1.0,
+ help='use this as the polling interval/blocking timeout')
+@arg('--recursive',
+ default=True,
+ help='recursively monitor paths')
+@expects_obj
+def tricks_from(args):
+ """
+ Subcommand to execute tricks from a tricks configuration file.
+
+ :param args:
+ Command line argument options.
+ """
+ from watchdog.observers import Observer
+
+ add_to_sys_path(path_split(args.python_path))
+ observers = []
+ for tricks_file in args.files:
+ observer = Observer(timeout=args.timeout)
+
+ if not os.path.exists(tricks_file):
+ raise IOError("cannot find tricks file: %s" % tricks_file)
+
+ config = load_config(tricks_file)
+
+ try:
+ tricks = config[CONFIG_KEY_TRICKS]
+ except KeyError:
+ raise KeyError("No `%s' key specified in %s." % (
+ CONFIG_KEY_TRICKS, tricks_file))
+
+ if CONFIG_KEY_PYTHON_PATH in config:
+ add_to_sys_path(config[CONFIG_KEY_PYTHON_PATH])
+
+ dir_path = os.path.dirname(tricks_file)
+ if not dir_path:
+ dir_path = os.path.relpath(os.getcwd())
+ schedule_tricks(observer, tricks, dir_path, args.recursive)
+ observer.start()
+ observers.append(observer)
+
+ try:
+ while True:
+ time.sleep(1)
+ except KeyboardInterrupt:
+ for o in observers:
+ o.unschedule_all()
+ o.stop()
+ for o in observers:
+ o.join()
+
+
+@aliases('generate-tricks-yaml')
+@arg('trick_paths',
+ nargs='*',
+ help='Dotted paths for all the tricks you want to generate')
+@arg('--python-path',
+ default='.',
+ help='paths separated by %s to add to the python path' % os.path.sep)
+@arg('--append-to-file',
+ default=None,
+ help='appends the generated tricks YAML to a file; \
+if not specified, prints to standard output')
+@arg('-a',
+ '--append-only',
+ dest='append_only',
+ default=False,
+ help='if --append-to-file is not specified, produces output for \
+appending instead of a complete tricks yaml file.')
+@expects_obj
+def tricks_generate_yaml(args):
+ """
+ Subcommand to generate Yaml configuration for tricks named on the command
+ line.
+
+ :param args:
+ Command line argument options.
+ """
+ python_paths = path_split(args.python_path)
+ add_to_sys_path(python_paths)
+ output = StringIO()
+
+ for trick_path in args.trick_paths:
+ TrickClass = load_class(trick_path)
+ output.write(TrickClass.generate_yaml())
+
+ content = output.getvalue()
+ output.close()
+
+ header = yaml.dump({CONFIG_KEY_PYTHON_PATH: python_paths})
+ header += "%s:\n" % CONFIG_KEY_TRICKS
+ if args.append_to_file is None:
+ # Output to standard output.
+ if not args.append_only:
+ content = header + content
+ sys.stdout.write(content)
+ else:
+ if not os.path.exists(args.append_to_file):
+ content = header + content
+ output = open(args.append_to_file, 'ab')
+ output.write(content)
+ output.close()
+
+
+@arg('directories',
+ nargs='*',
+ default='.',
+ help='directories to watch.')
+@arg('-p',
+ '--pattern',
+ '--patterns',
+ dest='patterns',
+ default='*',
+ help='matches event paths with these patterns (separated by ;).')
+@arg('-i',
+ '--ignore-pattern',
+ '--ignore-patterns',
+ dest='ignore_patterns',
+ default='',
+ help='ignores event paths with these patterns (separated by ;).')
+@arg('-D',
+ '--ignore-directories',
+ dest='ignore_directories',
+ default=False,
+ help='ignores events for directories')
+@arg('-R',
+ '--recursive',
+ dest='recursive',
+ default=False,
+ help='monitors the directories recursively')
+@arg('--interval',
+ '--timeout',
+ dest='timeout',
+ default=1.0,
+ help='use this as the polling interval/blocking timeout')
+@arg('--trace',
+ default=False,
+ help='dumps complete dispatching trace')
+@arg('--debug-force-polling',
+ default=False,
+ help='[debug] forces polling')
+@arg('--debug-force-kqueue',
+ default=False,
+ help='[debug] forces BSD kqueue(2)')
+@arg('--debug-force-winapi',
+ default=False,
+ help='[debug] forces Windows API')
+@arg('--debug-force-winapi-async',
+ default=False,
+ help='[debug] forces Windows API + I/O completion')
+@arg('--debug-force-fsevents',
+ default=False,
+ help='[debug] forces Mac OS X FSEvents')
+@arg('--debug-force-inotify',
+ default=False,
+ help='[debug] forces Linux inotify(7)')
+@expects_obj
+def log(args):
+ """
+ Subcommand to log file system events to the console.
+
+ :param args:
+ Command line argument options.
+ """
+ from watchdog.utils import echo
+ from watchdog.tricks import LoggerTrick
+
+ if args.trace:
+ echo.echo_class(LoggerTrick)
+
+ patterns, ignore_patterns =\
+ parse_patterns(args.patterns, args.ignore_patterns)
+ handler = LoggerTrick(patterns=patterns,
+ ignore_patterns=ignore_patterns,
+ ignore_directories=args.ignore_directories)
+ if args.debug_force_polling:
+ from watchdog.observers.polling import PollingObserver as Observer
+ elif args.debug_force_kqueue:
+ from watchdog.observers.kqueue import KqueueObserver as Observer
+ elif args.debug_force_winapi_async:
+ from watchdog.observers.read_directory_changes_async import\
+ WindowsApiAsyncObserver as Observer
+ elif args.debug_force_winapi:
+ from watchdog.observers.read_directory_changes import\
+ WindowsApiObserver as Observer
+ elif args.debug_force_inotify:
+ from watchdog.observers.inotify import InotifyObserver as Observer
+ elif args.debug_force_fsevents:
+ from watchdog.observers.fsevents import FSEventsObserver as Observer
+ else:
+ # Automatically picks the most appropriate observer for the platform
+ # on which it is running.
+ from watchdog.observers import Observer
+ observer = Observer(timeout=args.timeout)
+ observe_with(observer, handler, args.directories, args.recursive)
+
+
+@arg('directories',
+ nargs='*',
+ default='.',
+ help='directories to watch')
+@arg('-c',
+ '--command',
+ dest='command',
+ default=None,
+ help='''shell command executed in response to matching events.
+These interpolation variables are available to your command string::
+
+ ${watch_src_path} - event source path;
+ ${watch_dest_path} - event destination path (for moved events);
+ ${watch_event_type} - event type;
+ ${watch_object} - ``file`` or ``directory``
+
+Note::
+ Please ensure you do not use double quotes (") to quote
+ your command string. That will force your shell to
+ interpolate before the command is processed by this
+ subcommand.
+
+Example option usage::
+
+ --command='echo "${watch_src_path}"'
+''')
+@arg('-p',
+ '--pattern',
+ '--patterns',
+ dest='patterns',
+ default='*',
+ help='matches event paths with these patterns (separated by ;).')
+@arg('-i',
+ '--ignore-pattern',
+ '--ignore-patterns',
+ dest='ignore_patterns',
+ default='',
+ help='ignores event paths with these patterns (separated by ;).')
+@arg('-D',
+ '--ignore-directories',
+ dest='ignore_directories',
+ default=False,
+ help='ignores events for directories')
+@arg('-R',
+ '--recursive',
+ dest='recursive',
+ default=False,
+ help='monitors the directories recursively')
+@arg('--interval',
+ '--timeout',
+ dest='timeout',
+ default=1.0,
+ help='use this as the polling interval/blocking timeout')
+@arg('-w', '--wait',
+ dest='wait_for_process',
+ action='store_true',
+ default=False,
+ help="wait for process to finish to avoid multiple simultaneous instances")
+@arg('-W', '--drop',
+ dest='drop_during_process',
+ action='store_true',
+ default=False,
+ help="Ignore events that occur while command is still being executed " \
+ "to avoid multiple simultaneous instances")
+@arg('--debug-force-polling',
+ default=False,
+ help='[debug] forces polling')
+@expects_obj
+def shell_command(args):
+ """
+ Subcommand to execute shell commands in response to file system events.
+
+ :param args:
+ Command line argument options.
+ """
+ from watchdog.tricks import ShellCommandTrick
+
+ if not args.command:
+ args.command = None
+
+ if args.debug_force_polling:
+ from watchdog.observers.polling import PollingObserver as Observer
+ else:
+ from watchdog.observers import Observer
+
+ patterns, ignore_patterns = parse_patterns(args.patterns,
+ args.ignore_patterns)
+ handler = ShellCommandTrick(shell_command=args.command,
+ patterns=patterns,
+ ignore_patterns=ignore_patterns,
+ ignore_directories=args.ignore_directories,
+ wait_for_process=args.wait_for_process,
+ drop_during_process=args.drop_during_process)
+ observer = Observer(timeout=args.timeout)
+ observe_with(observer, handler, args.directories, args.recursive)
+
+
+@arg('command',
+ help='''Long-running command to run in a subprocess.
+''')
+@arg('command_args',
+ metavar='arg',
+ nargs='*',
+ help='''Command arguments.
+
+Note: Use -- before the command arguments, otherwise watchmedo will
+try to interpret them.
+''')
+@arg('-d',
+ '--directory',
+ dest='directories',
+ metavar='directory',
+ action='append',
+ help='Directory to watch. Use another -d or --directory option '
+ 'for each directory.')
+@arg('-p',
+ '--pattern',
+ '--patterns',
+ dest='patterns',
+ default='*',
+ help='matches event paths with these patterns (separated by ;).')
+@arg('-i',
+ '--ignore-pattern',
+ '--ignore-patterns',
+ dest='ignore_patterns',
+ default='',
+ help='ignores event paths with these patterns (separated by ;).')
+@arg('-D',
+ '--ignore-directories',
+ dest='ignore_directories',
+ default=False,
+ help='ignores events for directories')
+@arg('-R',
+ '--recursive',
+ dest='recursive',
+ default=False,
+ help='monitors the directories recursively')
+@arg('--interval',
+ '--timeout',
+ dest='timeout',
+ default=1.0,
+ help='use this as the polling interval/blocking timeout')
+@arg('--signal',
+ dest='signal',
+ default='SIGINT',
+ help='stop the subprocess with this signal (default SIGINT)')
+@arg('--kill-after',
+ dest='kill_after',
+ default=10.0,
+ help='when stopping, kill the subprocess after the specified timeout '
+ '(default 10)')
+@expects_obj
+def auto_restart(args):
+ """
+ Subcommand to start a long-running subprocess and restart it
+ on matched events.
+
+ :param args:
+ Command line argument options.
+ """
+ from watchdog.observers import Observer
+ from watchdog.tricks import AutoRestartTrick
+ import signal
+ import re
+
+ if not args.directories:
+ args.directories = ['.']
+
+ # Allow either signal name or number.
+ if re.match('^SIG[A-Z]+$', args.signal):
+ stop_signal = getattr(signal, args.signal)
+ else:
+ stop_signal = int(args.signal)
+
+ # Handle SIGTERM in the same manner as SIGINT so that
+ # this program has a chance to stop the child process.
+ def handle_sigterm(_signum, _frame):
+ raise KeyboardInterrupt()
+
+ signal.signal(signal.SIGTERM, handle_sigterm)
+
+ patterns, ignore_patterns = parse_patterns(args.patterns,
+ args.ignore_patterns)
+ command = [args.command]
+ command.extend(args.command_args)
+ handler = AutoRestartTrick(command=command,
+ patterns=patterns,
+ ignore_patterns=ignore_patterns,
+ ignore_directories=args.ignore_directories,
+ stop_signal=stop_signal,
+ kill_after=args.kill_after)
+ handler.start()
+ observer = Observer(timeout=args.timeout)
+ observe_with(observer, handler, args.directories, args.recursive)
+ handler.stop()
+
+
+epilog = """Copyright 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>.
+Copyright 2012 Google, Inc.
+
+Licensed under the terms of the Apache license, version 2.0. Please see
+LICENSE in the source code for more information."""
+
+parser = ArghParser(epilog=epilog)
+parser.add_commands([tricks_from,
+ tricks_generate_yaml,
+ log,
+ shell_command,
+ auto_restart])
+parser.add_argument('--version',
+ action='version',
+ version='%(prog)s ' + VERSION_STRING)
+
+
+def main():
+ """Entry-point function."""
+ parser.dispatch()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/wandb/viz.py b/wandb/viz.py
new file mode 100644
index 0000000000000000000000000000000000000000..866666beac919d20fa17d0fd85ba48be56217618
--- /dev/null
+++ b/wandb/viz.py
@@ -0,0 +1,123 @@
+from typing import Any, Dict, Optional, Tuple
+
+from wandb.data_types import Table
+from wandb.errors import Error
+
+
+class Visualize:
+ def __init__(self, id: str, data: Table) -> None:
+ self._id = id
+ self._data = data
+
+ def get_config_value(self, key: str) -> Dict[str, Any]:
+ return {
+ "id": self._id,
+ "historyFieldSettings": {"x-axis": "_step", "key": key},
+ }
+
+ @staticmethod
+ def get_config_key(key: str) -> Tuple[str, str, str]:
+ return "_wandb", "viz", key
+
+ @property
+ def value(self) -> Table:
+ return self._data
+
+
+class CustomChart:
+ def __init__(
+ self,
+ id: str,
+ data: Table,
+ fields: Dict[str, Any],
+ string_fields: Dict[str, Any],
+ split_table: Optional[bool] = False,
+ ) -> None:
+ self._id = id
+ self._data = data
+ self._fields = fields
+ self._string_fields = string_fields
+ self._split_table = split_table
+
+ def get_config_value(
+ self,
+ panel_type: str,
+ query: Dict[str, Any],
+ ) -> Dict[str, Any]:
+ return {
+ "panel_type": panel_type,
+ "panel_config": {
+ "panelDefId": self._id,
+ "fieldSettings": self._fields,
+ "stringSettings": self._string_fields,
+ "transform": {"name": "tableWithLeafColNames"},
+ "userQuery": query,
+ },
+ }
+
+ @staticmethod
+ def get_config_key(key: str) -> Tuple[str, str, str]:
+ return "_wandb", "visualize", key
+
+ @staticmethod
+ def user_query(table_key: str) -> Dict[str, Any]:
+ return {
+ "queryFields": [
+ {
+ "name": "runSets",
+ "args": [{"name": "runSets", "value": "${runSets}"}],
+ "fields": [
+ {"name": "id", "fields": []},
+ {"name": "name", "fields": []},
+ {"name": "_defaultColorIndex", "fields": []},
+ {
+ "name": "summaryTable",
+ "args": [{"name": "tableKey", "value": table_key}],
+ "fields": [],
+ },
+ ],
+ }
+ ],
+ }
+
+ @property
+ def table(self) -> Table:
+ return self._data
+
+ @property
+ def fields(self) -> Dict[str, Any]:
+ return self._fields
+
+ @property
+ def string_fields(self) -> Dict[str, Any]:
+ return self._string_fields
+
+
+def custom_chart(
+ vega_spec_name: str,
+ data_table: Table,
+ fields: Dict[str, Any],
+ string_fields: Optional[Dict[str, Any]] = None,
+ split_table: Optional[bool] = False,
+) -> CustomChart:
+ if string_fields is None:
+ string_fields = {}
+ if not isinstance(data_table, Table):
+ raise Error(
+ f"Expected `data_table` to be `wandb.Table` type, instead got {type(data_table).__name__}"
+ )
+ return CustomChart(
+ id=vega_spec_name,
+ data=data_table,
+ fields=fields,
+ string_fields=string_fields,
+ split_table=split_table,
+ )
+
+
+def visualize(id: str, value: Table) -> Visualize:
+ if not isinstance(value, Table):
+ raise Error(
+ f"Expected `value` to be `wandb.Table` type, instead got {type(value).__name__}"
+ )
+ return Visualize(id=id, data=value)
diff --git a/wandb/wandb_agent.py b/wandb/wandb_agent.py
new file mode 100644
index 0000000000000000000000000000000000000000..ecb77708feb21376a66b11dd5b3e7a7fd63a6fb3
--- /dev/null
+++ b/wandb/wandb_agent.py
@@ -0,0 +1,586 @@
+import logging
+import multiprocessing
+import os
+import platform
+import queue
+import re
+import signal
+import socket
+import subprocess
+import sys
+import time
+import traceback
+from typing import Any, Callable, Dict, List, Optional
+
+import yaml
+
+import wandb
+from wandb import util, wandb_lib, wandb_sdk
+from wandb.agents.pyagent import pyagent
+from wandb.apis import InternalApi
+from wandb.sdk.launch.sweeps import utils as sweep_utils
+
+logger = logging.getLogger(__name__)
+
+
+class AgentError(Exception):
+ pass
+
+
+class AgentProcess:
+ """Launch and manage a process."""
+
+ def __init__(
+ self, env=None, command=None, function=None, run_id=None, in_jupyter=None
+ ):
+ self._popen = None
+ self._proc = None
+ self._finished_q = multiprocessing.Queue()
+ self._proc_killed = False
+
+ if command:
+ if platform.system() == "Windows":
+ kwargs = dict(creationflags=subprocess.CREATE_NEW_PROCESS_GROUP)
+ else:
+ kwargs = dict(preexec_fn=os.setpgrp)
+ self._popen = subprocess.Popen(command, env=env, **kwargs)
+ elif function:
+ self._proc = multiprocessing.Process(
+ target=self._start,
+ args=(self._finished_q, env, function, run_id, in_jupyter),
+ )
+ self._proc.start()
+ else:
+ raise AgentError("Agent Process requires command or function")
+
+ def _start(self, finished_q, env, function, run_id, in_jupyter):
+ if env:
+ for k, v in env.items():
+ os.environ[k] = v
+
+ # call user function
+ print("wandb: Agent Started Run:", run_id)
+ if function:
+ function()
+ print("wandb: Agent Finished Run:", run_id, "\n")
+
+ # complete the run
+ run = wandb.run
+ if run:
+ wandb.join()
+
+ # signal that the process is finished
+ finished_q.put(True)
+
+ def poll(self):
+ if self._popen:
+ return self._popen.poll()
+ if self._proc_killed:
+ # we need to join process to prevent zombies
+ self._proc.join()
+ return True
+ try:
+ finished = self._finished_q.get(False, 0)
+ if finished:
+ return True
+ except queue.Empty:
+ pass
+ return
+
+ def wait(self):
+ if self._popen:
+ # if on windows, wait() will block and we wont be able to interrupt
+ if platform.system() == "Windows":
+ try:
+ while True:
+ p = self._popen.poll()
+ if p is not None:
+ return p
+ time.sleep(1)
+ except KeyboardInterrupt:
+ raise
+ return self._popen.wait()
+ return self._proc.join()
+
+ def kill(self):
+ if self._popen:
+ return self._popen.kill()
+ pid = self._proc.pid
+ if pid:
+ ret = os.kill(pid, signal.SIGKILL)
+ self._proc_killed = True
+ return ret
+ return
+
+ def terminate(self):
+ if self._popen:
+ # windows terminate is too strong, send Ctrl-C instead
+ if platform.system() == "Windows":
+ return self._popen.send_signal(signal.CTRL_C_EVENT)
+ return self._popen.terminate()
+ return self._proc.terminate()
+
+
+class Agent:
+ POLL_INTERVAL = 5
+ REPORT_INTERVAL = 0
+ KILL_DELAY = 30
+ FLAPPING_MAX_SECONDS = 60
+ FLAPPING_MAX_FAILURES = 3
+ MAX_INITIAL_FAILURES = 5
+ DEFAULT_SWEEP_COMMAND: List[str] = [
+ "${env}",
+ "${interpreter}",
+ "${program}",
+ "${args}",
+ ]
+ SWEEP_COMMAND_ENV_VAR_REGEX = re.compile(r"\$\{envvar\:([A-Z0-9_]*)\}")
+
+ def __init__(
+ self, api, queue, sweep_id=None, function=None, in_jupyter=None, count=None
+ ):
+ self._api = api
+ self._queue = queue
+ self._run_processes = {} # keyed by run.id (GQL run name)
+ self._server_responses = []
+ self._sweep_id = sweep_id
+ self._in_jupyter = in_jupyter
+ self._log = []
+ self._running = True
+ self._last_report_time = None
+ self._function = function
+ self._report_interval = wandb.env.get_agent_report_interval(
+ self.REPORT_INTERVAL
+ )
+ self._kill_delay = wandb.env.get_agent_kill_delay(self.KILL_DELAY)
+ self._finished = 0
+ self._failed = 0
+ self._count = count
+ self._sweep_command = []
+ self._max_initial_failures = wandb.env.get_agent_max_initial_failures(
+ self.MAX_INITIAL_FAILURES
+ )
+ if self._report_interval is None:
+ raise AgentError("Invalid agent report interval")
+ if self._kill_delay is None:
+ raise AgentError("Invalid agent kill delay")
+ # if the directory to log to is not set, set it
+ if os.environ.get("WANDB_DIR") is None:
+ os.environ["WANDB_DIR"] = os.path.abspath(os.getcwd())
+
+ def is_flapping(self):
+ """Determine if the process is flapping.
+
+ Flapping occurs if the agents receives FLAPPING_MAX_FAILURES non-0 exit codes in
+ the first FLAPPING_MAX_SECONDS.
+ """
+ if os.getenv(wandb.env.AGENT_DISABLE_FLAPPING) == "true":
+ return False
+ if time.time() < wandb.START_TIME + self.FLAPPING_MAX_SECONDS:
+ return self._failed >= self.FLAPPING_MAX_FAILURES
+
+ def is_failing(self):
+ return (
+ self._failed >= self._finished
+ and self._max_initial_failures <= self._failed
+ )
+
+ def run(self): # noqa: C901
+ # TODO: catch exceptions, handle errors, show validation warnings, and make more generic
+ sweep_obj = self._api.sweep(self._sweep_id, "{}")
+ if sweep_obj:
+ sweep_yaml = sweep_obj.get("config")
+ if sweep_yaml:
+ sweep_config = yaml.safe_load(sweep_yaml)
+ if sweep_config:
+ sweep_command = sweep_config.get("command")
+ if sweep_command and isinstance(sweep_command, list):
+ self._sweep_command = sweep_command
+
+ # TODO: include sweep ID
+ agent = self._api.register_agent(socket.gethostname(), sweep_id=self._sweep_id)
+ agent_id = agent["id"]
+
+ try:
+ while self._running:
+ commands = util.read_many_from_queue(
+ self._queue, 100, self.POLL_INTERVAL
+ )
+ for command in commands:
+ command["resp_queue"].put(self._process_command(command))
+
+ now = util.stopwatch_now()
+ if self._last_report_time is None or (
+ self._report_interval != 0
+ and now > self._last_report_time + self._report_interval
+ ):
+ logger.info("Running runs: %s", list(self._run_processes.keys()))
+ self._last_report_time = now
+ run_status = {}
+ for run_id, run_process in list(self._run_processes.items()):
+ poll_result = run_process.poll()
+ if poll_result is None:
+ run_status[run_id] = True
+ continue
+ elif (
+ not isinstance(poll_result, bool)
+ and isinstance(poll_result, int)
+ and poll_result > 0
+ ):
+ self._failed += 1
+ if self.is_flapping():
+ logger.error(
+ "Detected %i failed runs in the first %i seconds, shutting down.",
+ self.FLAPPING_MAX_FAILURES,
+ self.FLAPPING_MAX_SECONDS,
+ )
+ logger.info(
+ "To disable this check set WANDB_AGENT_DISABLE_FLAPPING=true"
+ )
+ self._running = False
+ break
+ if self.is_failing():
+ logger.error(
+ "Detected %i failed runs in a row, shutting down.",
+ self._max_initial_failures,
+ )
+ logger.info(
+ "To change this value set WANDB_AGENT_MAX_INITIAL_FAILURES=val"
+ )
+ self._running = False
+ break
+ logger.info("Cleaning up finished run: %s", run_id)
+
+ # wandb.teardown() was added with wandb service and is a hammer to make
+ # sure that active runs are finished before moving on to another agent run
+ #
+ # In the future, a lighter weight way to implement this could be to keep a
+ # service process open for all the agent instances and inform_finish when
+ # the run should be marked complete. This however could require
+ # inform_finish on every run created by this process.
+ if hasattr(wandb, "teardown"):
+ exit_code = 0
+ if isinstance(poll_result, int):
+ exit_code = poll_result
+ elif isinstance(poll_result, bool):
+ exit_code = -1
+ wandb.teardown(exit_code)
+
+ del self._run_processes[run_id]
+ self._last_report_time = None
+ self._finished += 1
+
+ if self._count and self._finished >= self._count or not self._running:
+ self._running = False
+ continue
+
+ commands = self._api.agent_heartbeat(agent_id, {}, run_status)
+
+ # TODO: send _server_responses
+ self._server_responses = []
+ for command in commands:
+ self._server_responses.append(self._process_command(command))
+
+ except KeyboardInterrupt:
+ try:
+ wandb.termlog(
+ "Ctrl-c pressed. Waiting for runs to end. Press ctrl-c again to terminate them."
+ )
+ for _, run_process in self._run_processes.items():
+ run_process.wait()
+ except KeyboardInterrupt:
+ pass
+ finally:
+ try:
+ if not self._in_jupyter:
+ wandb.termlog("Terminating and syncing runs. Press ctrl-c to kill.")
+ for _, run_process in self._run_processes.items():
+ try:
+ run_process.terminate()
+ except OSError:
+ pass # if process is already dead
+ for _, run_process in self._run_processes.items():
+ run_process.wait()
+ except KeyboardInterrupt:
+ wandb.termlog("Killing runs and quitting.")
+ for _, run_process in self._run_processes.items():
+ try:
+ run_process.kill()
+ except OSError:
+ pass # if process is already dead
+
+ def _process_command(self, command):
+ logger.info(
+ "Agent received command: %s"
+ % (command["type"] if "type" in command else "Unknown")
+ )
+ response = {
+ "id": command.get("id"),
+ "result": None,
+ }
+ try:
+ command_type = command["type"]
+ if command_type == "run":
+ result = self._command_run(command)
+ elif command_type == "stop":
+ result = self._command_stop(command)
+ elif command_type == "exit":
+ result = self._command_exit(command)
+ elif command_type == "resume":
+ result = self._command_run(command)
+ else:
+ raise AgentError("No such command: %s" % command_type)
+ response["result"] = result
+ except Exception:
+ logger.exception("Exception while processing command: %s", command)
+ ex_type, ex, tb = sys.exc_info()
+ response["exception"] = f"{ex_type.__name__}: {str(ex)}"
+ response["traceback"] = traceback.format_tb(tb)
+ del tb
+
+ self._log.append((command, response))
+
+ return response
+
+ def _command_run(self, command):
+ logger.info(
+ "Agent starting run with config:\n"
+ + "\n".join(
+ ["\t{}: {}".format(k, v["value"]) for k, v in command["args"].items()]
+ )
+ )
+ if self._in_jupyter:
+ print(
+ "wandb: Agent Starting Run: {} with config:\n".format(
+ command.get("run_id")
+ )
+ + "\n".join(
+ [
+ "\t{}: {}".format(k, v["value"])
+ for k, v in command["args"].items()
+ ]
+ )
+ )
+
+ # Setup sweep command
+ sweep_command: List[str] = sweep_utils.create_sweep_command(self._sweep_command)
+
+ run_id = command.get("run_id")
+ sweep_id = os.environ.get(wandb.env.SWEEP_ID)
+ # TODO(jhr): move into settings
+ config_file = os.path.join(
+ "wandb", "sweep-" + sweep_id, "config-" + run_id + ".yaml"
+ )
+ json_file = os.path.join(
+ "wandb", "sweep-" + sweep_id, "config-" + run_id + ".json"
+ )
+
+ os.environ[wandb.env.RUN_ID] = run_id
+
+ base_dir = os.environ.get(wandb.env.DIR, "")
+ sweep_param_path = os.path.join(base_dir, config_file)
+ os.environ[wandb.env.SWEEP_PARAM_PATH] = sweep_param_path
+ wandb_lib.config_util.save_config_file_from_dict(
+ sweep_param_path, command["args"]
+ )
+
+ env = dict(os.environ)
+
+ sweep_vars: Dict[str, Any] = sweep_utils.create_sweep_command_args(command)
+
+ if "${args_json_file}" in sweep_command:
+ with open(json_file, "w") as fp:
+ fp.write(sweep_vars["args_json"][0])
+
+ if self._function:
+ # make sure that each run regenerates setup singleton
+ wandb_sdk.wandb_setup._setup(_reset=True)
+ proc = AgentProcess(
+ function=self._function,
+ env=env,
+ run_id=run_id,
+ in_jupyter=self._in_jupyter,
+ )
+ else:
+ sweep_vars["interpreter"] = ["python"]
+ sweep_vars["program"] = [command["program"]]
+ sweep_vars["args_json_file"] = [json_file]
+ if not platform.system() == "Windows":
+ sweep_vars["env"] = ["/usr/bin/env"]
+ command_list = []
+ for c in sweep_command:
+ c = str(c)
+ if c.startswith("${") and c.endswith("}"):
+ replace_list = sweep_vars.get(c[2:-1])
+ command_list += replace_list or []
+ else:
+ command_list += [c]
+ logger.info(
+ "About to run command: {}".format(
+ " ".join('"%s"' % c if " " in c else c for c in command_list)
+ )
+ )
+ proc = AgentProcess(command=command_list, env=env)
+ self._run_processes[run_id] = proc
+
+ # we keep track of when we sent the sigterm to give processes a chance
+ # to handle the signal before sending sigkill every heartbeat
+ self._run_processes[run_id].last_sigterm_time = None
+ self._last_report_time = None
+
+ def _command_stop(self, command):
+ run_id = command["run_id"]
+ if run_id in self._run_processes:
+ proc = self._run_processes[run_id]
+ now = util.stopwatch_now()
+ if proc.last_sigterm_time is None:
+ proc.last_sigterm_time = now
+ logger.info("Stop: %s", run_id)
+ try:
+ proc.terminate()
+ except OSError: # if process is already dead
+ pass
+ elif now > proc.last_sigterm_time + self._kill_delay:
+ logger.info("Kill: %s", run_id)
+ try:
+ proc.kill()
+ except OSError: # if process is already dead
+ pass
+ else:
+ logger.error("Run %s not running", run_id)
+
+ def _command_exit(self, command):
+ logger.info("Received exit command. Killing runs and quitting.")
+ for _, proc in self._run_processes.items():
+ try:
+ proc.kill()
+ except OSError:
+ # process is already dead
+ pass
+ self._running = False
+
+
+class AgentApi:
+ def __init__(self, queue):
+ self._queue = queue
+ self._command_id = 0
+ self._multiproc_manager = multiprocessing.Manager()
+
+ def command(self, command):
+ command["origin"] = "local"
+ command["id"] = "local-%s" % self._command_id
+ self._command_id += 1
+ resp_queue = self._multiproc_manager.Queue()
+ command["resp_queue"] = resp_queue
+ self._queue.put(command)
+ result = resp_queue.get()
+ print("result:", result)
+ if "exception" in result:
+ print("Exception occurred while running command")
+ for line in result["traceback"]:
+ print(line.strip())
+ print(result["exception"])
+ return result
+
+
+def run_agent(
+ sweep_id, function=None, in_jupyter=None, entity=None, project=None, count=None
+):
+ parts = dict(entity=entity, project=project, name=sweep_id)
+ err = sweep_utils.parse_sweep_id(parts)
+ if err:
+ wandb.termerror(err)
+ return
+ entity = parts.get("entity") or entity
+ project = parts.get("project") or project
+ sweep_id = parts.get("name") or sweep_id
+
+ if entity:
+ wandb.env.set_entity(entity)
+ if project:
+ wandb.env.set_project(project)
+ if sweep_id:
+ # TODO(jhr): remove when jobspec is merged
+ os.environ[wandb.env.SWEEP_ID] = sweep_id
+ logger.setLevel(logging.DEBUG)
+ ch = logging.StreamHandler()
+ log_level = logging.DEBUG
+ if in_jupyter:
+ log_level = logging.ERROR
+ ch.setLevel(log_level)
+ formatter = logging.Formatter(
+ "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+ )
+ ch.setFormatter(formatter)
+ try:
+ logger.addHandler(ch)
+
+ api = InternalApi()
+ queue = multiprocessing.Queue()
+ agent = Agent(
+ api,
+ queue,
+ sweep_id=sweep_id,
+ function=function,
+ in_jupyter=in_jupyter,
+ count=count,
+ )
+ agent.run()
+ finally:
+ # make sure we remove the logging handler (important for jupyter notebooks)
+ logger.removeHandler(ch)
+
+
+def agent(
+ sweep_id: str,
+ function: Optional[Callable] = None,
+ entity: Optional[str] = None,
+ project: Optional[str] = None,
+ count: Optional[int] = None,
+) -> None:
+ """Start one or more sweep agents.
+
+ The sweep agent uses the `sweep_id` to know which sweep it
+ is a part of, what function to execute, and (optionally) how
+ many agents to run.
+
+ Arguments:
+ sweep_id: The unique identifier for a sweep. A sweep ID
+ is generated by W&B CLI or Python SDK.
+ function: A function to call instead of the "program"
+ specified in the sweep config.
+ entity: The username or team name where you want to send W&B
+ runs created by the sweep to. Ensure that the entity you
+ specify already exists. If you don't specify an entity,
+ the run will be sent to your default entity,
+ which is usually your username.
+ project: The name of the project where W&B runs created from
+ the sweep are sent to. If the project is not specified, the
+ run is sent to a project labeled "Uncategorized".
+ count: The number of sweep config trials to try.
+ """
+ global _INSTANCES
+ _INSTANCES += 1
+ try:
+ # make sure we are logged in
+ wandb_sdk.wandb_login._login(_silent=True)
+ if function:
+ return pyagent(sweep_id, function, entity, project, count)
+ in_jupyter = wandb.wandb_sdk.lib.ipython._get_python_type() != "python"
+ return run_agent(
+ sweep_id,
+ function=function,
+ in_jupyter=in_jupyter,
+ entity=entity,
+ project=project,
+ count=count,
+ )
+ finally:
+ _INSTANCES -= 1
+
+
+_INSTANCES = 0
+
+
+def _is_running():
+ return bool(_INSTANCES)
diff --git a/wandb/wandb_controller.py b/wandb/wandb_controller.py
new file mode 100644
index 0000000000000000000000000000000000000000..4362d692e7cfd833957f3efc768b680cc0438f88
--- /dev/null
+++ b/wandb/wandb_controller.py
@@ -0,0 +1,721 @@
+"""Sweep controller.
+
+This module implements the sweep controller.
+
+On error an exception is raised:
+ ControllerError
+
+Example:
+ import wandb
+
+ #
+ # create a sweep controller
+ #
+ # There are three different ways sweeps can be created:
+ # (1) create with sweep id from `wandb sweep` command
+ sweep_id = 'xyzxyz2'
+ tuner = wandb.controller(sweep_id)
+ # (2) create with sweep config
+ sweep_config = {}
+ tuner = wandb.controller()
+ tuner.configure(sweep_config)
+ tuner.create()
+ # (3) create by constructing progamatic sweep configuration
+ tuner = wandb.controller()
+ tuner.configure_search('random')
+ tuner.configure_program('train-dummy.py')
+ tuner.configure_parameter('param1', values=[1,2,3])
+ tuner.configure_parameter('param2', values=[1,2,3])
+ tuner.configure_controller(type="local")
+ tuner.create()
+ #
+ # run the sweep controller
+ #
+ # There are three different ways sweeps can be executed:
+ # (1) run to completion
+ tuner.run()
+ # (2) run in a simple loop
+ while not tuner.done():
+ tuner.step()
+ tuner.print_status()
+ # (3) run in a more complex loop
+ while not tuner.done():
+ params = tuner.search()
+ tuner.schedule(params)
+ runs = tuner.stopping()
+ if runs:
+ tuner.stop_runs(runs)
+"""
+
+
+import json
+import os
+import random
+import string
+import time
+from typing import Callable, Dict, List, Optional, Tuple, Union
+
+import yaml
+
+from wandb import env
+from wandb.apis import InternalApi
+from wandb.sdk import wandb_sweep
+from wandb.sdk.launch.sweeps.utils import (
+ handle_sweep_config_violations,
+ sweep_config_err_text_from_jsonschema_violations,
+)
+from wandb.util import get_module
+
+# TODO(jhr): Add metric status
+# TODO(jhr): Add print_space
+# TODO(jhr): Add print_summary
+
+
+sweeps = get_module(
+ "sweeps",
+ required="wandb[sweeps] is required to use the local controller. "
+ "Please run `pip install wandb[sweeps]`.",
+)
+
+
+# This should be something like 'pending' (but we need to make sure everyone else is ok with that)
+SWEEP_INITIAL_RUN_STATE = sweeps.RunState.pending
+
+
+def _id_generator(size=10, chars=string.ascii_lowercase + string.digits):
+ return "".join(random.choice(chars) for _ in range(size))
+
+
+class ControllerError(Exception):
+ """Base class for sweep errors."""
+
+ pass
+
+
+class _WandbController:
+ """Sweep controller class.
+
+ Internal datastructures on the sweep object to coordinate local controller with
+ cloud controller.
+
+ Data structures:
+ controller: {
+ schedule: [
+ { id: SCHEDULE_ID
+ data: {param1: val1, param2: val2}},
+ ]
+ earlystop: [RUN_ID, ...]
+ scheduler:
+ scheduled: [
+ { id: SCHEDULE_ID
+ runid: RUN_ID},
+ ]
+
+ `controller` is only updated by the client
+ `scheduler` is only updated by the cloud backend
+
+ Protocols:
+ Scheduling a run:
+ - client controller adds a schedule entry on the controller.schedule list
+ - cloud backend notices the new entry and creates a run with the parameters
+ - cloud backend adds a scheduled entry on the scheduler.scheduled list
+ - client controller notices that the run has been scheduled and removes it from
+ controller.schedule list
+
+ Current implementation details:
+ - Runs are only schedule if there are no other runs scheduled.
+
+ """
+
+ def __init__(self, sweep_id_or_config=None, entity=None, project=None):
+ # sweep id configured in constuctor
+ self._sweep_id: Optional[str] = None
+
+ # configured parameters
+ # Configuration to be created
+ self._create: Dict = {}
+ # Custom search
+ self._custom_search: Optional[
+ Callable[
+ [Union[dict, sweeps.SweepConfig], List[sweeps.SweepRun]],
+ Optional[sweeps.SweepRun],
+ ]
+ ] = None
+ # Custom stopping
+ self._custom_stopping: Optional[
+ Callable[
+ [Union[dict, sweeps.SweepConfig], List[sweeps.SweepRun]],
+ List[sweeps.SweepRun],
+ ]
+ ] = None
+ # Program function (used for future jupyter support)
+ self._program_function = None
+
+ # The following are updated every sweep step
+ # raw sweep object (dict of strings)
+ self._sweep_obj = None
+ # parsed sweep config (dict)
+ self._sweep_config: Optional[Union[dict, sweeps.SweepConfig]] = None
+ # sweep metric used to optimize (str or None)
+ self._sweep_metric: Optional[str] = None
+ # list of _Run objects
+ self._sweep_runs: Optional[List[sweeps.SweepRun]] = None
+ # dictionary mapping name of run to run object
+ self._sweep_runs_map: Optional[Dict[str, sweeps.SweepRun]] = None
+ # scheduler dict (read only from controller) - used as feedback from the server
+ self._scheduler: Optional[Dict] = None
+ # controller dict (write only from controller) - used to send commands to server
+ self._controller: Optional[Dict] = None
+ # keep track of controller dict from previous step
+ self._controller_prev_step: Optional[Dict] = None
+
+ # Internal
+ # Keep track of whether the sweep has been started
+ self._started: bool = False
+ # indicate whether there is more to schedule
+ self._done_scheduling: bool = False
+ # indicate whether the sweep needs to be created
+ self._defer_sweep_creation: bool = False
+ # count of logged lines since last status
+ self._logged: int = 0
+ # last status line printed
+ self._laststatus: str = ""
+ # keep track of logged actions for print_actions()
+ self._log_actions: List[Tuple[str, str]] = []
+ # keep track of logged debug for print_debug()
+ self._log_debug: List[str] = []
+
+ # all backend commands use internal api
+ environ = os.environ
+ if entity:
+ env.set_entity(entity, env=environ)
+ if project:
+ env.set_project(project, env=environ)
+ self._api = InternalApi(environ=environ)
+
+ if isinstance(sweep_id_or_config, str):
+ self._sweep_id = sweep_id_or_config
+ elif isinstance(sweep_id_or_config, dict) or isinstance(
+ sweep_id_or_config, sweeps.SweepConfig
+ ):
+ self._create = sweeps.SweepConfig(sweep_id_or_config)
+
+ # check for custom search and or stopping functions
+ for config_key, controller_attr in zip(
+ ["method", "early_terminate"], ["_custom_search", "_custom_stopping"]
+ ):
+ if callable(config_key in self._create and self._create[config_key]):
+ setattr(self, controller_attr, self._create[config_key])
+ self._create[config_key] = "custom"
+
+ self._sweep_id = self.create(from_dict=True)
+ elif sweep_id_or_config is None:
+ self._defer_sweep_creation = True
+ return
+ else:
+ raise ControllerError("Unhandled sweep controller type")
+ sweep_obj = self._sweep_object_read_from_backend()
+ if sweep_obj is None:
+ raise ControllerError("Can not find sweep")
+ self._sweep_obj = sweep_obj
+
+ def configure_search(
+ self,
+ search: Union[
+ str,
+ Callable[
+ [Union[dict, sweeps.SweepConfig], List[sweeps.SweepRun]],
+ Optional[sweeps.SweepRun],
+ ],
+ ],
+ ):
+ self._configure_check()
+ if isinstance(search, str):
+ self._create["method"] = search
+ elif callable(search):
+ self._create["method"] = "custom"
+ self._custom_search = search
+ else:
+ raise ControllerError("Unhandled search type.")
+
+ def configure_stopping(
+ self,
+ stopping: Union[
+ str,
+ Callable[
+ [Union[dict, sweeps.SweepConfig], List[sweeps.SweepRun]],
+ List[sweeps.SweepRun],
+ ],
+ ],
+ **kwargs,
+ ):
+ self._configure_check()
+ if isinstance(stopping, str):
+ self._create.setdefault("early_terminate", {})
+ self._create["early_terminate"]["type"] = stopping
+ for k, v in kwargs.items():
+ self._create["early_terminate"][k] = v
+ elif callable(stopping):
+ self._custom_stopping = stopping(kwargs)
+ self._create.setdefault("early_terminate", {})
+ self._create["early_terminate"]["type"] = "custom"
+ else:
+ raise ControllerError("Unhandled stopping type.")
+
+ def configure_metric(self, metric, goal=None):
+ self._configure_check()
+ self._create.setdefault("metric", {})
+ self._create["metric"]["name"] = metric
+ if goal:
+ self._create["metric"]["goal"] = goal
+
+ def configure_program(self, program):
+ self._configure_check()
+ if isinstance(program, str):
+ self._create["program"] = program
+ elif callable(program):
+ self._create["program"] = "__callable__"
+ self._program_function = program
+ raise ControllerError("Program functions are not supported yet")
+ else:
+ raise ControllerError("Unhandled sweep program type")
+
+ def configure_name(self, name):
+ self._configure_check()
+ self._create["name"] = name
+
+ def configure_description(self, description):
+ self._configure_check()
+ self._create["description"] = description
+
+ def configure_parameter(
+ self,
+ name,
+ values=None,
+ value=None,
+ distribution=None,
+ min=None,
+ max=None,
+ mu=None,
+ sigma=None,
+ q=None,
+ a=None,
+ b=None,
+ ):
+ self._configure_check()
+ self._create.setdefault("parameters", {}).setdefault(name, {})
+ if value is not None or (
+ values is None and min is None and max is None and distribution is None
+ ):
+ self._create["parameters"][name]["value"] = value
+ if values is not None:
+ self._create["parameters"][name]["values"] = values
+ if min is not None:
+ self._create["parameters"][name]["min"] = min
+ if max is not None:
+ self._create["parameters"][name]["max"] = max
+ if mu is not None:
+ self._create["parameters"][name]["mu"] = mu
+ if sigma is not None:
+ self._create["parameters"][name]["sigma"] = sigma
+ if q is not None:
+ self._create["parameters"][name]["q"] = q
+ if a is not None:
+ self._create["parameters"][name]["a"] = a
+ if b is not None:
+ self._create["parameters"][name]["b"] = b
+
+ def configure_controller(self, type):
+ """Configure controller to local if type == 'local'."""
+ self._configure_check()
+ self._create.setdefault("controller", {})
+ self._create["controller"].setdefault("type", type)
+
+ def configure(self, sweep_dict_or_config):
+ self._configure_check()
+ if self._create:
+ raise ControllerError("Already configured.")
+ if isinstance(sweep_dict_or_config, dict):
+ self._create = sweep_dict_or_config
+ elif isinstance(sweep_dict_or_config, str):
+ self._create = yaml.safe_load(sweep_dict_or_config)
+ else:
+ raise ControllerError("Unhandled sweep controller type")
+
+ @property
+ def sweep_config(self) -> Union[dict, sweeps.SweepConfig]:
+ return self._sweep_config
+
+ @property
+ def sweep_id(self) -> str:
+ return self._sweep_id
+
+ def _log(self) -> None:
+ self._logged += 1
+
+ def _error(self, s: str) -> None:
+ print("ERROR:", s)
+ self._log()
+
+ def _warn(self, s: str) -> None:
+ print("WARN:", s)
+ self._log()
+
+ def _info(self, s: str) -> None:
+ print("INFO:", s)
+ self._log()
+
+ def _debug(self, s: str) -> None:
+ print("DEBUG:", s)
+ self._log()
+
+ def _configure_check(self) -> None:
+ if self._started:
+ raise ControllerError("Can not configure after sweep has been started.")
+
+ def _validate(self, config: Dict) -> str:
+ violations = sweeps.schema_violations_from_proposed_config(config)
+ msg = (
+ sweep_config_err_text_from_jsonschema_violations(violations)
+ if len(violations) > 0
+ else ""
+ )
+ return msg
+
+ def create(self, from_dict: bool = False) -> str:
+ if self._started:
+ raise ControllerError("Can not create after sweep has been started.")
+ if not self._defer_sweep_creation and not from_dict:
+ raise ControllerError("Can not use create on already created sweep.")
+ if not self._create:
+ raise ControllerError("Must configure sweep before create.")
+
+ # validate sweep config
+ self._create = sweeps.SweepConfig(self._create)
+
+ # Create sweep
+ sweep_id, warnings = self._api.upsert_sweep(self._create)
+ handle_sweep_config_violations(warnings)
+
+ print("Create sweep with ID:", sweep_id)
+ sweep_url = wandb_sweep._get_sweep_url(self._api, sweep_id)
+ if sweep_url:
+ print("Sweep URL:", sweep_url)
+ self._sweep_id = sweep_id
+ self._defer_sweep_creation = False
+ return sweep_id
+
+ def run(
+ self,
+ verbose: bool = False,
+ print_status: bool = True,
+ print_actions: bool = False,
+ print_debug: bool = False,
+ ) -> None:
+ if verbose:
+ print_status = True
+ print_actions = True
+ print_debug = True
+ self._start_if_not_started()
+ while not self.done():
+ if print_status:
+ self.print_status()
+ self.step()
+ if print_actions:
+ self.print_actions()
+ if print_debug:
+ self.print_debug()
+ time.sleep(5)
+
+ def _sweep_object_read_from_backend(self) -> Optional[dict]:
+ specs_json = {}
+ if self._sweep_metric:
+ k = ["_step"]
+ k.append(self._sweep_metric)
+ specs_json = {"keys": k, "samples": 100000}
+ specs = json.dumps(specs_json)
+ # TODO(jhr): catch exceptions?
+ sweep_obj = self._api.sweep(self._sweep_id, specs)
+ if not sweep_obj:
+ return
+ self._sweep_obj = sweep_obj
+ self._sweep_config = yaml.safe_load(sweep_obj["config"])
+ self._sweep_metric = self._sweep_config.get("metric", {}).get("name")
+
+ _sweep_runs: List[sweeps.SweepRun] = []
+ for r in sweep_obj["runs"]:
+ rr = r.copy()
+ if "summaryMetrics" in rr:
+ if rr["summaryMetrics"]:
+ rr["summaryMetrics"] = json.loads(rr["summaryMetrics"])
+ if "config" not in rr:
+ raise ValueError("sweep object is missing config")
+ rr["config"] = json.loads(rr["config"])
+ if "history" in rr:
+ if isinstance(rr["history"], list):
+ rr["history"] = [json.loads(d) for d in rr["history"]]
+ else:
+ raise ValueError(
+ "Invalid history value: expected list of json strings: %s"
+ % rr["history"]
+ )
+ if "sampledHistory" in rr:
+ sampled_history = []
+ for historyDictList in rr["sampledHistory"]:
+ sampled_history += historyDictList
+ rr["sampledHistory"] = sampled_history
+ _sweep_runs.append(sweeps.SweepRun(**rr))
+
+ self._sweep_runs = _sweep_runs
+ self._sweep_runs_map = {r.name: r for r in self._sweep_runs}
+
+ self._controller = json.loads(sweep_obj.get("controller") or "{}")
+ self._scheduler = json.loads(sweep_obj.get("scheduler") or "{}")
+ self._controller_prev_step = self._controller.copy()
+ return sweep_obj
+
+ def _sweep_object_sync_to_backend(self) -> None:
+ if self._controller == self._controller_prev_step:
+ return
+ sweep_obj_id = self._sweep_obj["id"]
+ controller = json.dumps(self._controller)
+ _, warnings = self._api.upsert_sweep(
+ self._sweep_config, controller=controller, obj_id=sweep_obj_id
+ )
+ handle_sweep_config_violations(warnings)
+ self._controller_prev_step = self._controller.copy()
+
+ def _start_if_not_started(self) -> None:
+ if self._started:
+ return
+ if self._defer_sweep_creation:
+ raise ControllerError(
+ "Must specify or create a sweep before running controller."
+ )
+ obj = self._sweep_object_read_from_backend()
+ if not obj:
+ return
+ is_local = self._sweep_config.get("controller", {}).get("type") == "local"
+ if not is_local:
+ raise ControllerError(
+ "Only sweeps with a local controller are currently supported."
+ )
+ self._started = True
+ # reset controller state, we might want to parse this and decide
+ # what we can continue and add a version key, but for now we can
+ # be safe and just reset things on start
+ self._controller = {}
+ self._sweep_object_sync_to_backend()
+
+ def _parse_scheduled(self):
+ scheduled_list = self._scheduler.get("scheduled") or []
+ started_ids = []
+ stopped_runs = []
+ done_runs = []
+ for s in scheduled_list:
+ runid = s.get("runid")
+ objid = s.get("id")
+ r = self._sweep_runs_map.get(runid)
+ if not r:
+ continue
+ if r.stopped:
+ stopped_runs.append(runid)
+ summary = r.summary_metrics
+ if r.state == SWEEP_INITIAL_RUN_STATE and not summary:
+ continue
+ started_ids.append(objid)
+ if r.state != "running":
+ done_runs.append(runid)
+ return started_ids, stopped_runs, done_runs
+
+ def _step(self) -> None:
+ self._start_if_not_started()
+ self._sweep_object_read_from_backend()
+
+ started_ids, stopped_runs, done_runs = self._parse_scheduled()
+
+ # Remove schedule entry from controller dict if already scheduled
+ schedule_list = self._controller.get("schedule", [])
+ new_schedule_list = [s for s in schedule_list if s.get("id") not in started_ids]
+ self._controller["schedule"] = new_schedule_list
+
+ # Remove earlystop entry from controller if already stopped
+ earlystop_list = self._controller.get("earlystop", [])
+ new_earlystop_list = [
+ r for r in earlystop_list if r not in stopped_runs and r not in done_runs
+ ]
+ self._controller["earlystop"] = new_earlystop_list
+
+ # Clear out step logs
+ self._log_actions = []
+ self._log_debug = []
+
+ def step(self) -> None:
+ self._step()
+ suggestion = self.search()
+ self.schedule(suggestion)
+ to_stop = self.stopping()
+ if len(to_stop) > 0:
+ self.stop_runs(to_stop)
+
+ def done(self) -> bool:
+ self._start_if_not_started()
+ state = self._sweep_obj.get("state")
+ if state in [
+ s.upper()
+ for s in (
+ sweeps.RunState.preempting.value,
+ SWEEP_INITIAL_RUN_STATE.value,
+ sweeps.RunState.running.value,
+ )
+ ]:
+ return False
+ return True
+
+ def _search(self) -> Optional[sweeps.SweepRun]:
+ search = self._custom_search or sweeps.next_run
+ next_run = search(self._sweep_config, self._sweep_runs or [])
+ if next_run is None:
+ self._done_scheduling = True
+ return next_run
+
+ def search(self) -> Optional[sweeps.SweepRun]:
+ self._start_if_not_started()
+ suggestion = self._search()
+ return suggestion
+
+ def _stopping(self) -> List[sweeps.SweepRun]:
+ if "early_terminate" not in self.sweep_config:
+ return []
+ stopper = self._custom_stopping or sweeps.stop_runs
+ stop_runs = stopper(self._sweep_config, self._sweep_runs or [])
+
+ debug_lines = "\n".join(
+ [
+ " ".join([f"{k}={v}" for k, v in run.early_terminate_info.items()])
+ for run in stop_runs
+ if run.early_terminate_info is not None
+ ]
+ )
+ if debug_lines:
+ self._log_debug += debug_lines
+
+ return stop_runs
+
+ def stopping(self) -> List[sweeps.SweepRun]:
+ self._start_if_not_started()
+ return self._stopping()
+
+ def schedule(self, run: Optional[sweeps.SweepRun]) -> None:
+ self._start_if_not_started()
+
+ # only schedule one run at a time (for now)
+ if self._controller and self._controller.get("schedule"):
+ return
+
+ schedule_id = _id_generator()
+
+ if run is None:
+ schedule_list = [{"id": schedule_id, "data": {"args": None}}]
+ else:
+ param_list = [
+ "{}={}".format(k, v.get("value")) for k, v in sorted(run.config.items())
+ ]
+ self._log_actions.append(("schedule", ",".join(param_list)))
+
+ # schedule one run
+ schedule_list = [{"id": schedule_id, "data": {"args": run.config}}]
+
+ self._controller["schedule"] = schedule_list
+ self._sweep_object_sync_to_backend()
+
+ def stop_runs(self, runs: List[sweeps.SweepRun]) -> None:
+ earlystop_list = list({run.name for run in runs})
+ self._log_actions.append(("stop", ",".join(earlystop_list)))
+ self._controller["earlystop"] = earlystop_list
+ self._sweep_object_sync_to_backend()
+
+ def print_status(self) -> None:
+ status = _sweep_status(self._sweep_obj, self._sweep_config, self._sweep_runs)
+ if self._laststatus != status or self._logged:
+ print(status)
+ self._laststatus = status
+ self._logged = 0
+
+ def print_actions(self) -> None:
+ for action, line in self._log_actions:
+ self._info(f"{action.capitalize()} ({line})")
+ self._log_actions = []
+
+ def print_debug(self) -> None:
+ for line in self._log_debug:
+ self._debug(line)
+ self._log_debug = []
+
+ def print_space(self) -> None:
+ self._warn("Method not implemented yet.")
+
+ def print_summary(self) -> None:
+ self._warn("Method not implemented yet.")
+
+
+def _get_run_counts(runs: List[sweeps.SweepRun]) -> Dict[str, int]:
+ metrics = {}
+ categories = [name for name, _ in sweeps.RunState.__members__.items()] + ["unknown"]
+ for r in runs:
+ state = r.state
+ found = "unknown"
+ for c in categories:
+ if state == c:
+ found = c
+ break
+ metrics.setdefault(found, 0)
+ metrics[found] += 1
+ return metrics
+
+
+def _get_runs_status(metrics):
+ categories = [name for name, _ in sweeps.RunState.__members__.items()] + ["unknown"]
+ mlist = []
+ for c in categories:
+ if not metrics.get(c):
+ continue
+ mlist.append("%s: %d" % (c.capitalize(), metrics[c]))
+ s = ", ".join(mlist)
+ return s
+
+
+def _sweep_status(
+ sweep_obj: dict,
+ sweep_conf: Union[dict, sweeps.SweepConfig],
+ sweep_runs: List[sweeps.SweepRun],
+) -> str:
+ sweep = sweep_obj["name"]
+ _ = sweep_obj["state"]
+ run_count = len(sweep_runs)
+ run_type_counts = _get_run_counts(sweep_runs)
+ stopped = len([r for r in sweep_runs if r.stopped])
+ stopping = len([r for r in sweep_runs if r.should_stop])
+ stopstr = ""
+ if stopped or stopping:
+ stopstr = "Stopped: %d" % stopped
+ if stopping:
+ stopstr += " (Stopping: %d)" % stopping
+ runs_status = _get_runs_status(run_type_counts)
+ method = sweep_conf.get("method", "unknown")
+ stopping = sweep_conf.get("early_terminate", None)
+ sweep_options = []
+ sweep_options.append(method)
+ if stopping:
+ sweep_options.append(stopping.get("type", "unknown"))
+ sweep_options = ",".join(sweep_options)
+ sections = []
+ sections.append(f"Sweep: {sweep} ({sweep_options})")
+ if runs_status:
+ sections.append("Runs: %d (%s)" % (run_count, runs_status))
+ else:
+ sections.append("Runs: %d" % (run_count))
+ if stopstr:
+ sections.append(stopstr)
+ sections = " | ".join(sections)
+ return sections
diff --git a/wandb/wandb_run.py b/wandb/wandb_run.py
new file mode 100644
index 0000000000000000000000000000000000000000..05ce62facf71350e042e63e61a5e64043f3c37ef
--- /dev/null
+++ b/wandb/wandb_run.py
@@ -0,0 +1,9 @@
+"""Compatibility wandb_run module.
+
+In the future use:
+ from wandb.sdk.wandb_run import Run
+"""
+
+from wandb.sdk.wandb_run import Run
+
+__all__ = ["Run"]
diff --git a/wandb/wandb_torch.py b/wandb/wandb_torch.py
new file mode 100644
index 0000000000000000000000000000000000000000..15e31e1918094b1dce9588be47d3da53be6dfd9f
--- /dev/null
+++ b/wandb/wandb_torch.py
@@ -0,0 +1,551 @@
+#!/usr/bin/env python
+
+"""PyTorch-specific functionality
+"""
+
+import itertools
+from functools import reduce
+from operator import mul
+from typing import List
+
+import wandb
+from wandb import util
+from wandb.data_types import Node
+
+torch = None
+
+
+def nested_shape(array_or_tuple, seen=None):
+ """Figure out the shape of tensors possibly embedded in tuples
+ i.e
+ [0,0] returns (2)
+ ([0,0], [0,0]) returns (2,2)
+ (([0,0], [0,0]),[0,0]) returns ((2,2),2)
+ """
+ if seen is None:
+ seen = set()
+ if hasattr(array_or_tuple, "size"):
+ # pytorch tensors use V.size() to get size of tensor
+ return list(array_or_tuple.size())
+ elif hasattr(array_or_tuple, "get_shape"):
+ # tensorflow uses V.get_shape() to get size of tensor
+ return array_or_tuple.get_shape().as_list()
+ elif hasattr(array_or_tuple, "shape"):
+ return array_or_tuple.shape
+
+ seen.add(id(array_or_tuple))
+ try:
+ # treat object as iterable
+ return [
+ nested_shape(item, seen) if id(item) not in seen else 0
+ for item in list(array_or_tuple)
+ ]
+ except TypeError:
+ # object is not actually iterable
+ # LB: Maybe we should throw an error?
+ return []
+
+
+LOG_TRACK_COUNT, LOG_TRACK_THRESHOLD = range(2)
+
+
+def log_track_init(log_freq: int) -> List[int]:
+ """create tracking structure used by log_track_update"""
+ l = [0] * 2
+ l[LOG_TRACK_THRESHOLD] = log_freq
+ return l
+
+
+def log_track_update(log_track: int) -> bool:
+ """count (log_track[0]) up to threshold (log_track[1]), reset count (log_track[0]) and return true when reached"""
+ log_track[LOG_TRACK_COUNT] += 1
+ if log_track[LOG_TRACK_COUNT] < log_track[LOG_TRACK_THRESHOLD]:
+ return False
+ log_track[LOG_TRACK_COUNT] = 0
+ return True
+
+
+class TorchHistory:
+ """History methods specific to PyTorch"""
+
+ def __init__(self):
+ global torch
+ torch = wandb.util.get_module("torch", "Could not import torch")
+ self._hook_handles = {}
+ self._num_bins = 64
+ self._is_cuda_histc_supported = None
+ self.hook_torch = TorchGraph.hook_torch
+
+ def add_log_parameters_hook(
+ self,
+ module: "torch.nn.Module",
+ name: str = "",
+ prefix: str = "",
+ log_freq: int = 0,
+ ) -> None:
+ """This instruments hooks into the pytorch module
+ log parameters after a forward pass
+ log_freq - log gradients/parameters every N batches
+ """
+ # if name is not None:
+ prefix = prefix + name
+
+ if not hasattr(module, "_wandb_hook_names"):
+ module._wandb_hook_names = []
+
+ def parameter_log_hook(module, input_, output, log_track):
+ if not log_track_update(log_track):
+ return
+ for name, parameter in module.named_parameters():
+ # for pytorch 0.3 Variables
+ if isinstance(parameter, torch.autograd.Variable):
+ data = parameter.data
+ else:
+ data = parameter
+ self.log_tensor_stats(data.cpu(), "parameters/" + prefix + name)
+
+ log_track_params = log_track_init(log_freq)
+ try:
+ hook = module.register_forward_hook(
+ lambda mod, inp, outp: parameter_log_hook(
+ mod, inp, outp, log_track_params
+ )
+ )
+ self._hook_handles["parameters/" + prefix] = hook
+ module._wandb_hook_names.append("parameters/" + prefix)
+ except RuntimeError as e:
+ wandb.termwarn(
+ f"Trying to register forward_hook failed ({e}) - skipping parameter tracking."
+ )
+
+ def add_log_gradients_hook(
+ self,
+ module: "torch.nn.Module",
+ name: str = "",
+ prefix: str = "",
+ log_freq: int = 0,
+ ) -> None:
+ """This instruments hooks into the pytorch module
+ log gradients after a backward pass
+ log_freq - log gradients/parameters every N batches
+ """
+
+ # if name is not None:
+ prefix = prefix + name
+
+ if not hasattr(module, "_wandb_hook_names"):
+ module._wandb_hook_names = []
+
+ for name, parameter in module.named_parameters():
+ if parameter.requires_grad:
+ log_track_grad = log_track_init(log_freq)
+ module._wandb_hook_names.append("gradients/" + prefix + name)
+ self._hook_variable_gradient_stats(
+ parameter, "gradients/" + prefix + name, log_track_grad
+ )
+
+ def log_tensor_stats(self, tensor, name):
+ """Add distribution statistics on a tensor's elements to the current History entry"""
+ # TODO Handle the case of duplicate names.
+ if isinstance(tensor, (tuple, list)):
+ while isinstance(tensor, (tuple, list)) and isinstance(
+ tensor[0], (tuple, list)
+ ):
+ tensor = [item for sublist in tensor for item in sublist]
+ tensor = torch.cat([t.detach().clone().reshape(-1) for t in tensor])
+
+ tensor = tensor.detach().clone()
+ # checking for inheritance from _TensorBase didn't work for some reason
+ if not hasattr(tensor, "shape"):
+ cls = type(tensor)
+ raise TypeError(f"Expected Tensor, not {cls.__module__}.{cls.__name__}")
+
+ # Sparse tensors have a bunch of implicit zeros. In order to histo them correctly,
+ # we have to count them up and add them to the histo ourselves.
+ sparse_zeros = None
+ if tensor.is_sparse:
+ # Have to call this on a sparse tensor before most other ops.
+ tensor = tensor.cpu().coalesce()
+
+ backing_values = tensor._values()
+ sparse_zeros = tensor.numel() - backing_values.numel()
+ tensor = backing_values
+
+ flat = tensor.reshape(-1)
+
+ if flat.is_cuda:
+ if self._is_cuda_histc_supported is None:
+ try:
+ flat.histc(bins=self._num_bins)
+ except RuntimeError:
+ self._is_cuda_histc_supported = False
+ else:
+ self._is_cuda_histc_supported = True
+
+ # As of torch 1.0.1.post2+nightly, float16 cuda summary ops are not supported (convert to float32)
+ if not self._is_cuda_histc_supported:
+ flat = flat.cpu()
+ elif not isinstance(
+ flat, (torch.cuda.FloatTensor, torch.cuda.DoubleTensor)
+ ):
+ flat = flat.type(torch.cuda.FloatTensor)
+
+ # Since we use histc, we need to make sure that torch supports the operation on CPU,
+ # otherwise we'll get a runtime error. Hence, we need to upcast to float32.
+ if not flat.is_cuda and not isinstance(
+ flat, (torch.FloatTensor, torch.DoubleTensor)
+ ):
+ flat = flat.type(torch.FloatTensor)
+
+ # Skip logging if all values are nan or inf or the tensor is empty.
+ if self._no_finite_values(flat):
+ return
+
+ # Remove nans and infs if present. There's no good way to represent that in histograms.
+ flat = self._remove_infs_nans(flat)
+
+ tmin = flat.min().item()
+ tmax = flat.max().item()
+ if sparse_zeros:
+ # If we've got zeros to add in, make sure zero is in the hist range.
+ tmin = 0 if tmin > 0 else tmin
+ tmax = 0 if tmax < 0 else tmax
+ # Anecdotally, this can somehow happen sometimes. Maybe a precision error
+ # in min()/max() above. Swap here to prevent a runtime error.
+ # If all values are equal, just return a single bin.
+ if tmin > tmax:
+ tmin, tmax = tmax, tmin
+ if tmin == tmax:
+ tensor = torch.Tensor([flat.numel()])
+ tensor = tensor.cpu().clone().detach()
+ bins = torch.Tensor([tmin, tmax])
+ else:
+ tensor = flat.histc(bins=self._num_bins, min=tmin, max=tmax)
+ tensor = tensor.cpu().detach().clone()
+ bins = torch.linspace(tmin, tmax, steps=self._num_bins + 1)
+
+ # Add back zeroes from a sparse tensor.
+ if sparse_zeros:
+ bins_np = bins.numpy()
+ tensor_np = tensor.numpy()
+ bin_idx = 0
+ num_buckets = len(bins_np) - 1
+ for i in range(num_buckets):
+ start = bins_np[i]
+ end = bins_np[i + 1]
+ # There are 3 cases to consider here, all of which mean we've found the right bucket
+ # 1. The bucket range contains zero.
+ # 2. The bucket range lower bound *is* zero.
+ # 3. This is the last bucket and the bucket range upper bound is zero.
+ if (start <= 0 and end > 0) or (i == num_buckets - 1 and end == 0):
+ bin_idx = i
+ break
+
+ tensor_np[bin_idx] += sparse_zeros
+ tensor = torch.Tensor(tensor_np)
+ bins = torch.Tensor(bins_np)
+
+ wandb.run._log(
+ {name: wandb.Histogram(np_histogram=(tensor.tolist(), bins.tolist()))},
+ commit=False,
+ )
+
+ def _hook_variable_gradient_stats(self, var, name, log_track):
+ """Logs a Variable's gradient's distribution statistics next time backward()
+ is called on it.
+ """
+ if not isinstance(var, torch.autograd.Variable):
+ cls = type(var)
+ raise TypeError(
+ f"Expected torch.Variable, not {cls.__module__}.{cls.__name__}"
+ )
+
+ handle = self._hook_handles.get(name)
+ if handle is not None and self._torch_hook_handle_is_valid(handle):
+ raise ValueError(f'A hook has already been set under name "{name}"')
+
+ def _callback(grad, log_track):
+ if not log_track_update(log_track):
+ return
+ self.log_tensor_stats(grad.data, name)
+
+ handle = var.register_hook(lambda grad: _callback(grad, log_track))
+ self._hook_handles[name] = handle
+ return handle
+
+ def unhook_all(self):
+ for handle in self._hook_handles.values():
+ handle.remove()
+ self._hook_handles = {}
+
+ def unhook(self, name):
+ handle = self._hook_handles.pop(name)
+ handle.remove()
+
+ def _torch_hook_handle_is_valid(self, handle):
+ d = handle.hooks_dict_ref()
+ if d is None:
+ return False
+ else:
+ return handle.id in d
+
+ def _no_finite_values(self, tensor: "torch.Tensor") -> bool:
+ return tensor.shape == torch.Size([0]) or (~torch.isfinite(tensor)).all().item()
+
+ def _remove_infs_nans(self, tensor: "torch.Tensor") -> "torch.Tensor":
+ if not torch.isfinite(tensor).all():
+ tensor = tensor[torch.isfinite(tensor)]
+
+ return tensor
+
+
+class TorchGraph(wandb.data_types.Graph):
+ def __init__(self):
+ super().__init__("torch")
+ self._graph_hooks = set()
+
+ @classmethod
+ def hook_torch(cls, model, criterion=None, graph_idx=0):
+ wandb.termlog("logging graph, to disable use `wandb.watch(log_graph=False)`")
+ graph = TorchGraph()
+ graph.hook_torch_modules(model, criterion, graph_idx=graph_idx)
+ return graph
+
+ def create_forward_hook(self, name, graph_idx):
+ graph = self
+
+ def after_forward_hook(module, input, output):
+ if id(module) not in self._graph_hooks:
+ # hook already processed -> noop
+ return
+ if not isinstance(output, tuple):
+ output = (output,)
+ parameters = [
+ (pname, list(param.size()))
+ for pname, param in module.named_parameters()
+ ]
+
+ node = Node(
+ id=id(module),
+ name=name,
+ class_name=str(module),
+ output_shape=nested_shape(output),
+ parameters=parameters,
+ num_parameters=[reduce(mul, size, 1) for (pname, size) in parameters],
+ )
+ graph.nodes_by_id[id(module)] = node
+ for param in module.parameters():
+ graph.nodes_by_id[id(param)] = node
+ graph.add_node(node)
+ if not graph.criterion_passed:
+ if hasattr(output[0], "grad_fn"):
+ graph.criterion = output[0].grad_fn
+ elif (
+ isinstance(output[0], list)
+ and output[0]
+ and hasattr(output[0][0], "grad_fn")
+ ):
+ graph.criterion = output[0][0].grad_fn
+
+ # hook has been processed
+ self._graph_hooks -= {id(module)}
+
+ if not self._graph_hooks:
+ # we went through the entire graph
+ wandb.run.summary["graph_%i" % graph_idx] = self
+
+ return after_forward_hook
+
+ def hook_torch_modules(
+ self, module, criterion=None, prefix=None, graph_idx=0, parent=None
+ ):
+ torch = util.get_module("torch", "Could not import torch")
+ layers = 0
+ graph = self
+ if hasattr(module, "_wandb_watch_called") and module._wandb_watch_called:
+ raise ValueError(
+ "You can only call `wandb.watch` once per model. Pass a new instance of the model if you need to call wandb.watch again in your code."
+ )
+ module._wandb_watch_called = True
+ if criterion:
+ graph.criterion = criterion
+ graph.criterion_passed = True
+
+ for name, sub_module in module.named_children():
+ name = name or str(layers)
+ if prefix:
+ name = prefix + "." + name
+ layers += 1
+ if not isinstance(sub_module, torch.nn.Module):
+ # TODO: Why does this happen?
+ break
+
+ # Trying to support torch >0.3 making this code complicated
+ # We want a list of types that we should recurse into
+ # Torch 0.3 uses containers
+ # 0.4 has ModuleList
+ # 0.4.1 has ModuleDict
+ module_types = [
+ getattr(torch.nn, module_classname)
+ for module_classname in (
+ "Container",
+ "Sequential",
+ "ModuleList",
+ "ModuleDict",
+ )
+ if hasattr(torch.nn, module_classname)
+ ]
+ if parent is None:
+ parent = module
+
+ if isinstance(sub_module, tuple(module_types)):
+ self.hook_torch_modules(sub_module, prefix=name, parent=parent)
+ else:
+ self._graph_hooks |= {id(sub_module)}
+ try:
+ graph_hook = sub_module.register_forward_hook(
+ self.create_forward_hook(name, graph_idx)
+ )
+ wandb.run._torch._hook_handles[
+ "topology/" + str(id(graph_hook))
+ ] = graph_hook
+ if not hasattr(parent, "_wandb_hook_names"):
+ # should never happen but let's be extra safe
+ parent._wandb_hook_names = []
+ parent._wandb_hook_names.append("topology/" + str(id(graph_hook)))
+ except RuntimeError as e:
+ wandb.termwarn(
+ f"Trying to register forward_hook failed ({e}) - skipping graph tracking.",
+ repeat=False,
+ )
+
+ @classmethod
+ def from_torch_layers(cls, module_graph, variable):
+ """Recover something like neural net layers from PyTorch Module's and the
+ compute graph from a Variable.
+
+ Example output for a multi-layer RNN. We confusingly assign shared embedding values
+ to the encoder, but ordered next to the decoder.
+
+ rnns.0.linear.module.weight_raw rnns.0
+ rnns.0.linear.module.bias rnns.0
+ rnns.1.linear.module.weight_raw rnns.1
+ rnns.1.linear.module.bias rnns.1
+ rnns.2.linear.module.weight_raw rnns.2
+ rnns.2.linear.module.bias rnns.2
+ rnns.3.linear.module.weight_raw rnns.3
+ rnns.3.linear.module.bias rnns.3
+ decoder.weight encoder
+ decoder.bias decoder
+ """
+ # TODO: We're currently not using this, but I left it here incase we want to resurrect! - CVP
+ torch = util.get_module("torch", "Could not import torch")
+
+ module_nodes_by_hash = {id(n): n for n in module_graph.nodes}
+ module_parameter_nodes = [
+ n for n in module_graph.nodes if isinstance(n.obj, torch.nn.Parameter)
+ ]
+
+ names_by_pid = {id(n.obj): n.name for n in module_parameter_nodes}
+
+ reachable_param_nodes = module_graph[0].reachable_descendents()
+ reachable_params = {}
+ module_reachable_params = {}
+ names = {}
+ for pid, reachable_nodes in reachable_param_nodes.items():
+ node = module_nodes_by_hash[pid]
+ if not isinstance(node.obj, torch.nn.Module):
+ continue
+ module = node.obj
+ reachable_params = {} # by object id
+ module_reachable_params[id(module)] = reachable_params
+ names[node.name] = set()
+ for reachable_hash in reachable_nodes:
+ reachable = module_nodes_by_hash[reachable_hash]
+ if isinstance(reachable.obj, torch.nn.Parameter):
+ param = reachable.obj
+ reachable_params[id(param)] = param
+ names[node.name].add(names_by_pid[id(param)])
+
+ # we look for correspondences between sets of parameters used in subtrees of the
+ # computation graph and sets of parameters contained in subtrees of the module
+ # graph
+ node_depths = {id(n): d for n, d in module_graph[0].descendent_bfs()}
+ parameter_module_names = {}
+ parameter_modules = {}
+ for param_node in (
+ n for n in module_graph.nodes if isinstance(n.obj, torch.nn.Parameter)
+ ):
+ pid = id(param_node.obj)
+ best_node = None
+ best_depth = None
+ best_reachable_params = None
+ for node in module_graph.nodes:
+ if not isinstance(node.obj, torch.nn.Module):
+ continue
+ module = node.obj
+ reachable_params = module_reachable_params[id(module)]
+ if pid in reachable_params:
+ depth = node_depths[id(node)]
+ if best_node is None or (len(reachable_params), depth) <= (
+ len(best_reachable_params),
+ best_depth,
+ ):
+ best_node = node
+ best_depth = depth
+ best_reachable_params = reachable_params
+
+ parameter_modules[pid] = best_node
+ parameter_module_names[param_node.name] = best_node.name
+
+ # contains all parameters but only a minimal set of modules necessary
+ # to contain them (and which ideally correspond to conceptual layers)
+ reduced_module_graph = cls()
+ rmg_ids = itertools.count()
+ rmg_root = Node(id=next(rmg_ids), node=module_graph[0])
+ reduced_module_graph.add_node(rmg_root)
+ reduced_module_graph.root = rmg_root
+ rmg_nodes_by_pid = {}
+
+ module_nodes_by_pid = {id(n.obj): n for n in module_graph.nodes}
+
+ compute_graph, compute_node_vars = cls.from_torch_compute_graph(variable)
+ for node, _ in reversed(list(compute_graph[0].ancestor_bfs())):
+ param = compute_node_vars.get(node.id)
+ pid = id(param)
+ if not isinstance(param, torch.nn.Parameter):
+ continue
+ if pid not in module_nodes_by_pid:
+ # not all Parameters that occur in the compute graph come from the Module graph
+ continue
+
+ # add the nodes in the order we want to display them on the frontend
+ mid = id(parameter_modules[pid].obj)
+ if mid in rmg_nodes_by_pid:
+ rmg_module = rmg_nodes_by_pid[mid]
+ else:
+ rmg_module = rmg_nodes_by_pid[mid] = Node(
+ id=next(rmg_ids), node=module_nodes_by_pid[mid]
+ )
+ reduced_module_graph.add_node(rmg_module)
+ reduced_module_graph.add_edge(rmg_root, rmg_module)
+
+ rmg_param = Node(id=next(rmg_ids), node=module_nodes_by_pid[pid])
+ rmg_nodes_by_pid[pid] = rmg_param
+ reduced_module_graph.add_node(rmg_param)
+
+ reduced_module_graph.add_edge(rmg_module, rmg_param)
+ return reduced_module_graph
+
+ @classmethod
+ def node_from_module(cls, nid, module):
+ numpy = util.get_module("numpy", "Could not import numpy")
+
+ node = wandb.Node()
+ node.id = nid
+ node.child_parameters = 0
+ for parameter in module.parameters():
+ node.child_parameters += numpy.prod(parameter.size())
+ node.class_name = type(module).__name__
+
+ return node
diff --git a/wandb/xgboost/__init__.py b/wandb/xgboost/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..5ca456e910db9088ba9825791ca9922185cc52f3
--- /dev/null
+++ b/wandb/xgboost/__init__.py
@@ -0,0 +1,9 @@
+"""Compatibility xgboost module.
+
+In the future use:
+ from wandb.integration.xgboost import wandb_callback
+"""
+
+from wandb.integration.xgboost import WandbCallback, wandb_callback
+
+__all__ = ["wandb_callback", "WandbCallback"]
diff --git a/wandb_run/latest-run b/wandb_run/latest-run
new file mode 120000
index 0000000000000000000000000000000000000000..fe1caa0850b8b407c97b6cb60b8717781fccb100
--- /dev/null
+++ b/wandb_run/latest-run
@@ -0,0 +1 @@
+offline-run-20240125_114226-1ycnmixd
\ No newline at end of file